Add live memory ceiling for macOS (vz) VMs

cmd/api/api/instances.go

@@ -89,6 +89,19 @@ func (s *ApiService) CreateInstance(ctx context.Context, request oapi.CreateInst
 		hotplugSize = int64(hotplugBytes)
 	}
 
+	// Parse memory_ceiling (optional; omitted/empty means no ceiling)
+	memoryCeiling := int64(0)
+	if request.Body.MemoryCeiling != nil && *request.Body.MemoryCeiling != "" {
+		var ceilingBytes datasize.ByteSize
+		if err := ceilingBytes.UnmarshalText([]byte(*request.Body.MemoryCeiling)); err != nil {
+			return oapi.CreateInstance400JSONResponse{
+				Code:    "invalid_memory_ceiling",
+				Message: fmt.Sprintf("invalid memory_ceiling format: %v", err),
+			}, nil
+		}
+		memoryCeiling = int64(ceilingBytes)
+	}
+
 	// Parse overlay_size (default: 10GB)
 	overlaySize := int64(0)
 	if request.Body.OverlaySize != nil && *request.Body.OverlaySize != "" {
@@ -308,6 +321,7 @@ func (s *ApiService) CreateInstance(ctx context.Context, request oapi.CreateInst
 		Size:                     size,
 		Platform:                 derefString(request.Body.Platform),
 		HotplugSize:              hotplugSize,
+		MemoryCeilingBytes:       memoryCeiling,
 		OverlaySize:              overlaySize,
 		Vcpus:                    vcpus,
 		DiskIOBps:                diskIOBps,
@@ -1158,6 +1172,10 @@ func instanceToOAPI(inst instances.Instance) oapi.Instance {
 		oapiInst.Platform = lo.ToPtr(inst.Platform)
 	}
 
+	if inst.MemoryCeilingBytes > 0 {
+		oapiInst.MemoryCeiling = lo.ToPtr(datasize.ByteSize(inst.MemoryCeilingBytes).HR())
+	}
+
 	if inst.ExitMessage != "" {
 		oapiInst.ExitMessage = lo.ToPtr(inst.ExitMessage)
 	}

cmd/api/config/config.go

@@ -222,6 +222,8 @@ type HypervisorActiveBallooningConfig struct {
 	MinAdjustmentBytes                    string `koanf:"min_adjustment_bytes"`
 	PerVmMaxStepBytes                     string `koanf:"per_vm_max_step_bytes"`
 	PerVmCooldown                         string `koanf:"per_vm_cooldown"`
+	GrowOnDemandEnabled                   bool   `koanf:"grow_on_demand_enabled"`
+	GrowUtilizationPercent                int    `koanf:"grow_utilization_percent"`
 }
 
 // SnapshotCompressionDefaultConfig holds default snapshot compression settings.
@@ -428,6 +430,8 @@ func defaultConfig() *Config {
 					MinAdjustmentBytes:                    "67108864",
 					PerVmMaxStepBytes:                     "268435456",
 					PerVmCooldown:                         "5s",
+					GrowOnDemandEnabled:                   false,
+					GrowUtilizationPercent:                85,
 				},
 			},
 		},

cmd/vz-shim/main.go

@@ -71,12 +71,26 @@ func main() {
 			fmt.Fprintf(os.Stderr, "failed to start VM: %v\n", err)
 			os.Exit(1)
 		}
-		slog.Info("VM started", "vcpus", config.VCPUs, "memory_mb", config.MemoryBytes/1024/1024)
+		bootBytes := config.MemoryBytes
+		if config.MemoryCeilingBytes > config.MemoryBytes {
+			bootBytes = config.MemoryCeilingBytes
+		}
+		slog.Info("VM started", "vcpus", config.VCPUs, "boot_memory_mb", bootBytes/1024/1024, "baseline_memory_mb", config.MemoryBytes/1024/1024)
 	}
 
 	// Create the shim server
 	server := NewShimServer(vm, vmConfig, config)
 
+	// When booted at a ceiling, balloon the guest down to its baseline. A restore
+	// resumes a guest that was already ballooned, so this only applies to cold boot.
+	if config.RestoreMachineStatePath == "" {
+		if err := server.applyInitialBalloonTarget(); err != nil {
+			slog.Error("failed to balloon guest to baseline", "error", err)
+			fmt.Fprintf(os.Stderr, "failed to balloon guest to baseline: %v\n", err)
+			os.Exit(1)
+		}
+	}
+
 	// Start control socket listener (remove stale socket from previous run)
 	os.Remove(config.ControlSocket)
 	controlListener, err := net.Listen("unix", config.ControlSocket)

cmd/vz-shim/server.go

@@ -265,6 +265,25 @@ func (s *ShimServer) handlePowerButton(w http.ResponseWriter, r *http.Request) {
 	w.WriteHeader(http.StatusNoContent)
 }
 
+// applyInitialBalloonTarget inflates the balloon to the baseline (MemoryBytes)
+// when the VM was booted at a higher ceiling, so the guest settles at its normal
+// running size after coming up at the ceiling. No-op when no ceiling is active.
+func (s *ShimServer) applyInitialBalloonTarget() error {
+	if s.shimConfig.MemoryCeilingBytes <= s.shimConfig.MemoryBytes {
+		return nil
+	}
+	device, err := s.getTraditionalBalloonDevice()
+	if err != nil {
+		return err
+	}
+	device.SetTargetVirtualMachineMemorySize(uint64(s.shimConfig.MemoryBytes))
+	slog.Info("ballooned guest to baseline",
+		"boot_bytes", s.shimConfig.MemoryCeilingBytes,
+		"baseline_bytes", s.shimConfig.MemoryBytes,
+	)
+	return nil
+}
+
 func (s *ShimServer) getTraditionalBalloonDevice() (*vz.VirtioTraditionalMemoryBalloonDevice, error) {
 	for _, device := range s.vm.MemoryBalloonDevices() {
 		if traditional := vz.AsVirtioTraditionalMemoryBalloonDevice(device); traditional != nil {

cmd/vz-shim/vm.go

@@ -35,7 +35,20 @@ func createVM(config *shimconfig.ShimConfig) (*vz.VirtualMachine, *vz.VirtualMac
 	}
 
 	vcpus := computeCPUCount(config.VCPUs)
-	memoryBytes := computeMemorySize(uint64(config.MemoryBytes))
+	ceilingActive := config.MemoryCeilingBytes > config.MemoryBytes
+	bootBytes := uint64(config.MemoryBytes)
+	if ceilingActive {
+		bootBytes = uint64(config.MemoryCeilingBytes)
+	}
+	memoryBytes := computeMemorySize(bootBytes)
+	// computeMemorySize clamps an over-large request down to the host maximum. For
+	// an ordinary VM that is the desired fallback, but a ceiling that gets clamped
+	// would boot smaller than requested while the controller still treats the full
+	// ceiling as the balloon's upper bound, letting it drive the target above the
+	// guest's real memory. Reject instead, per the boot-ceiling contract.
+	if ceilingActive && memoryBytes < uint64(config.MemoryCeilingBytes) {
+		return nil, nil, fmt.Errorf("memory ceiling %d exceeds host maximum %d", config.MemoryCeilingBytes, vz.VirtualMachineConfigurationMaximumAllowedMemorySize())
+	}
 
 	slog.Debug("VM config", "vcpus", vcpus, "memory_bytes", memoryBytes, "kernel", config.KernelPath, "initrd", config.InitrdPath)
 
@@ -76,10 +89,13 @@ func createVM(config *shimconfig.ShimConfig) (*vz.VirtualMachine, *vz.VirtualMac
 	}
 	vmConfig.SetSocketDevicesVirtualMachineConfiguration([]vz.SocketDeviceConfiguration{vsockConfig})
 
-	if config.EnableMemoryBalloon {
+	// A memory ceiling boots the VM larger than its baseline and relies on the
+	// balloon to hold the guest down, so the balloon is mandatory whenever a
+	// ceiling is active regardless of the EnableMemoryBalloon/RequireMemoryBalloon flags.
+	if config.EnableMemoryBalloon || ceilingActive {
 		balloonConfig, err := vz.NewVirtioTraditionalMemoryBalloonDeviceConfiguration()
 		if err != nil {
-			if config.RequireMemoryBalloon {
+			if config.RequireMemoryBalloon || ceilingActive {
 				return nil, nil, fmt.Errorf("create memory balloon device: %w", err)
 			}
 			slog.Warn("memory balloon unavailable, continuing without balloon", "error", err)

config.example.darwin.yaml

@@ -49,6 +49,16 @@ hypervisor:
       min_adjustment_bytes: 67108864
       per_vm_max_step_bytes: 268435456
       per_vm_cooldown: 5s
+      # Grow a guest's usable memory above its baseline toward its configured
+      # boot ceiling while the host is healthy. Requires the instance to have
+      # been created with a memory ceiling above its size. NOTE: automatic grow
+      # is not active yet — it needs a per-guest memory-demand signal that is
+      # not wired up, so enabling this currently has no effect (ceiling VMs are
+      # held at their baseline; grow live via the balloon API instead). Off by default.
+      grow_on_demand_enabled: false
+      # Utilization threshold for automatic grow once the demand signal lands.
+      # Unused today. Ignored unless grow_on_demand_enabled.
+      grow_utilization_percent: 85
 
 # =============================================================================
 # Network Configuration (DIFFERENT ON MACOS)
@@ -161,7 +171,11 @@ limits:
 #    - No tc/HTB equivalent on macOS
 #
 # 3. CPU/Memory Hotplug
-#    - Resize operations not supported
+#    - True memory hotplug (growing boot RAM at runtime) is not supported.
+#    - Usable guest memory is instead made elastic between a baseline and a boot
+#      ceiling via the memory balloon: a VM created with a memory ceiling boots
+#      at the ceiling and is ballooned down to its baseline, then the active
+#      ballooning controller moves the target within [floor, ceiling].
 #
 # 4. Disk I/O Limiting
 #    - capacity.disk_io, oversubscription.disk_io are ignored