feat: Phase 2 — suspend-on-idle for ImpVM

api/v1alpha1/impvm_types.go

@@ -127,6 +127,13 @@ type ImpVMSpec struct {
 	// owned by it (PVCs cannot follow the VM to a new node).
 	// +optional
 	RescheduleOnNodeLoss bool `json:"rescheduleOnNodeLoss,omitempty"`
+
+	// DesiredState is the requested run state. When set to "Suspended" the agent
+	// snapshots the VM to node-local storage and frees its memory; setting it back
+	// to "Running" resumes it from that snapshot on the same node.
+	// +optional
+	// +kubebuilder:default=Running
+	DesiredState VMDesiredState `json:"desiredState,omitempty"`
 }
 
 // UserDataSource references a ConfigMap containing cloud-init user-data.
@@ -170,6 +177,16 @@ type ImpVMStatus struct {
 	// +optional
 	StartedAt *metav1.Time `json:"startedAt,omitempty"`
 
+	// SuspendSnapshotPath is the node-local directory holding the VM's suspend
+	// snapshot (vm.state + vm.mem). Set when the VM is Suspended; the resume path
+	// restores from here. Empty when the VM is not suspended.
+	// +optional
+	SuspendSnapshotPath string `json:"suspendSnapshotPath,omitempty"`
+
+	// SuspendedAt is the time the VM last transitioned to phase Suspended.
+	// +optional
+	SuspendedAt *metav1.Time `json:"suspendedAt,omitempty"`
+
 	// RestartCount is the cumulative number of times this VM has been restarted.
 	// +optional
 	RestartCount int32 `json:"restartCount,omitempty"`

api/v1alpha1/shared_types.go

@@ -79,7 +79,7 @@ const (
 )
 
 // VMPhase is the current lifecycle phase of an ImpVM.
-// +kubebuilder:validation:Enum=Pending;Scheduled;Starting;Running;Terminating;Succeeded;Failed;RetryExhausted
+// +kubebuilder:validation:Enum=Pending;Scheduled;Starting;Running;Terminating;Succeeded;Failed;RetryExhausted;Suspending;Suspended;Resuming
 type VMPhase string
 
 const (
@@ -93,6 +93,27 @@ const (
 	// VMPhaseRetryExhausted means all restart attempts are exhausted with onExhaustion="manual-reset".
 	// The VM will not restart until the retry counter is cleared via the imp/reset-retries annotation.
 	VMPhaseRetryExhausted VMPhase = "RetryExhausted"
+
+	// VMPhaseSuspending means the VM is being snapshotted to node-local storage
+	// before its runtime process is stopped to free memory.
+	VMPhaseSuspending VMPhase = "Suspending"
+	// VMPhaseSuspended means the VM's state is captured on node-local disk and its
+	// runtime process is stopped (no resident memory). It can be resumed on the same node.
+	VMPhaseSuspended VMPhase = "Suspended"
+	// VMPhaseResuming means the VM is being restored from its node-local suspend snapshot.
+	VMPhaseResuming VMPhase = "Resuming"
+)
+
+// VMDesiredState is the operator/user-requested run state for an ImpVM.
+// The agent drives the observed Phase toward this target.
+// +kubebuilder:validation:Enum=Running;Suspended
+type VMDesiredState string
+
+const (
+	// VMDesiredStateRunning keeps the VM running (default).
+	VMDesiredStateRunning VMDesiredState = "Running"
+	// VMDesiredStateSuspended requests the VM be snapshotted and its memory freed.
+	VMDesiredStateSuspended VMDesiredState = "Suspended"
 )
 
 // Arch is the CPU architecture for a VM class.

config/crd/bases/imp.dev_impvms.yaml

@@ -74,6 +74,16 @@ spec:
                 required:
                 - name
                 type: object
+              desiredState:
+                default: Running
+                description: |-
+                  DesiredState is the requested run state. When set to "Suspended" the agent
+                  snapshots the VM to node-local storage and frees its memory; setting it back
+                  to "Running" resumes it from that snapshot on the same node.
+                enum:
+                - Running
+                - Suspended
+                type: string
               env:
                 description: Env sets environment variables inside the VM via the
                   guest agent.
@@ -748,6 +758,9 @@ spec:
                 - Succeeded
                 - Failed
                 - RetryExhausted
+                - Suspending
+                - Suspended
+                - Resuming
                 type: string
               restartCount:
                 description: RestartCount is the cumulative number of times this VM
@@ -774,6 +787,17 @@ spec:
                   Used to detect and time out stuck start attempts.
                 format: date-time
                 type: string
+              suspendSnapshotPath:
+                description: |-
+                  SuspendSnapshotPath is the node-local directory holding the VM's suspend
+                  snapshot (vm.state + vm.mem). Set when the VM is Suspended; the resume path
+                  restores from here. Empty when the VM is not suspended.
+                type: string
+              suspendedAt:
+                description: SuspendedAt is the time the VM last transitioned to phase
+                  Suspended.
+                format: date-time
+                type: string
             type: object
         type: object
     served: true

internal/agent/firecracker_driver.go

@@ -209,8 +209,14 @@ func (d *FirecrackerDriver) Start(ctx context.Context, vm *impdevv1alpha1.ImpVM)
 	// 5. Build Firecracker config.
 	cfg := d.buildConfig(&class, rootfsPath, sockPath, netInfo, gaEnabled)
 
-	// 5a. Apply snapshot-based boot if requested.
-	if vm.Spec.SnapshotRef != "" {
+	// 5a. Apply snapshot-based boot. Resuming from a node-local suspend snapshot
+	// (status.suspendSnapshotPath) takes priority over a cold snapshot boot from
+	// an ImpVMSnapshot object (spec.snapshotRef).
+	switch {
+	case vm.Status.SuspendSnapshotPath != "":
+		configureSnapshotBoot(&cfg, vm.Status.SuspendSnapshotPath)
+		logf.FromContext(ctx).Info("configured resume from suspend snapshot", "path", vm.Status.SuspendSnapshotPath)
+	case vm.Spec.SnapshotRef != "":
 		if err := d.applySnapshotBoot(ctx, vm, &cfg); err != nil {
 			return 0, fmt.Errorf("apply snapshot boot: %w", err)
 		}
@@ -711,13 +717,20 @@ func (d *FirecrackerDriver) applySnapshotBoot(ctx context.Context, vm *impdevv1a
 		log.Info("snapshot has no node-local path, skipping snapshot boot", "snapshot", snap.Name)
 		return nil
 	}
+	configureSnapshotBoot(cfg, snap.Status.SnapshotPath)
+	log.Info("configured snapshot-based boot", "snapshotPath", cfg.Snapshot.SnapshotPath)
+	return nil
+}
+
+// configureSnapshotBoot points cfg at the node-local snapshot files (vm.state +
+// vm.mem) in dir and enables resume-on-load. dir must be a directory previously
+// written by Driver.Snapshot (which uses the same filenames).
+func configureSnapshotBoot(cfg *firecracker.Config, dir string) {
 	cfg.Snapshot = firecracker.SnapshotConfig{
-		SnapshotPath: filepath.Join(snap.Status.SnapshotPath, "vm.state"),
-		MemFilePath:  filepath.Join(snap.Status.SnapshotPath, "vm.mem"),
+		SnapshotPath: filepath.Join(dir, "vm.state"),
+		MemFilePath:  filepath.Join(dir, "vm.mem"),
 		ResumeVM:     true,
 	}
-	log.Info("configured snapshot-based boot", "snapshotPath", cfg.Snapshot.SnapshotPath)
-	return nil
 }
 
 // IsAlive reports whether the process with the given PID is still running.

internal/agent/reconciler.go

@@ -6,6 +6,8 @@ import (
 	"context"
 	"errors"
 	"net/http"
+	"os"
+	"path/filepath"
 	"strconv"
 	"time"
 
@@ -51,6 +53,23 @@ type ImpVMReconciler struct {
 	RetryInterval time.Duration
 	// Recorder emits lifecycle events for VM completion/failure.
 	Recorder record.EventRecorder
+	// SuspendDir is the node-local base directory under which suspend snapshots
+	// are written (one subdir per VM). Defaults to /var/lib/imp/suspend when empty.
+	SuspendDir string
+}
+
+// suspendBaseDir returns the configured suspend snapshot base directory,
+// defaulting to /var/lib/imp/suspend.
+func (r *ImpVMReconciler) suspendBaseDir() string {
+	if r.SuspendDir != "" {
+		return r.SuspendDir
+	}
+	return "/var/lib/imp/suspend"
+}
+
+// suspendDirFor returns the node-local directory holding vm's suspend snapshot.
+func (r *ImpVMReconciler) suspendDirFor(vm *impdevv1alpha1.ImpVM) string {
+	return filepath.Join(r.suspendBaseDir(), vm.Namespace+"_"+vm.Name)
 }
 
 // +kubebuilder:rbac:groups=imp.dev,resources=impvms,verbs=get;list;watch;update;patch
@@ -94,6 +113,12 @@ func (r *ImpVMReconciler) Reconcile(ctx context.Context, req ctrl.Request) (resu
 		return r.handleRunning(ctx, vm)
 	case impdevv1alpha1.VMPhaseStarting:
 		return r.handleStarting(ctx, vm)
+	case impdevv1alpha1.VMPhaseSuspending:
+		return r.handleSuspending(ctx, vm)
+	case impdevv1alpha1.VMPhaseSuspended:
+		return r.handleSuspended(ctx, vm)
+	case impdevv1alpha1.VMPhaseResuming:
+		return r.handleResuming(ctx, vm)
 	default:
 		// Pending, Succeeded, Failed — not our concern.
 		return ctrl.Result{}, nil
@@ -211,6 +236,20 @@ func (r *ImpVMReconciler) handleScheduled(ctx context.Context, vm *impdevv1alpha
 func (r *ImpVMReconciler) handleRunning(ctx context.Context, vm *impdevv1alpha1.ImpVM) (ctrl.Result, error) {
 	log := logf.FromContext(ctx)
 
+	// Suspend requested: transition to Suspending so the VM is snapshotted and
+	// its memory freed. The Suspending handler does the actual work.
+	if vm.Spec.DesiredState == impdevv1alpha1.VMDesiredStateSuspended {
+		base := vm.DeepCopy()
+		vm.Status.Phase = impdevv1alpha1.VMPhaseSuspending
+		if err := r.Status().Patch(ctx, vm, client.MergeFrom(base)); err != nil {
+			if apierrors.IsConflict(err) {
+				return ctrl.Result{Requeue: true}, nil
+			}
+			return ctrl.Result{}, err
+		}
+		return ctrl.Result{}, nil
+	}
+
 	state, err := r.Driver.Inspect(ctx, vm)
 	if err != nil {
 		log.Error(err, "Driver Inspect failed")
@@ -318,9 +357,157 @@ func (r *ImpVMReconciler) handleTerminating(ctx context.Context, vm *impdevv1alp
 		}
 	}
 
+	// Remove any node-local suspend snapshot so it does not outlive the VM.
+	if err := os.RemoveAll(r.suspendDirFor(vm)); err != nil {
+		log.Error(err, "failed to remove suspend snapshot dir", "dir", r.suspendDirFor(vm))
+	}
+
 	return r.clearOwnership(ctx, vm)
 }
 
+// handleSuspending snapshots the running VM to node-local storage, then stops its
+// runtime process to free memory. The snapshot MUST succeed before the process is
+// stopped, otherwise the VM state would be lost.
+func (r *ImpVMReconciler) handleSuspending(ctx context.Context, vm *impdevv1alpha1.ImpVM) (result ctrl.Result, err error) {
+	log := logf.FromContext(ctx)
+
+	ctx, span := tracing.SpanFromVM(ctx, vm, "agent.impvm.suspend",
+		trace.WithAttributes(
+			attribute.String("vm.name", vm.Name),
+			attribute.String("vm.namespace", vm.Namespace),
+			attribute.String("vm.node", r.NodeName),
+		),
+	)
+	defer func() {
+		tracing.RecordError(span, err)
+		span.End()
+	}()
+
+	destDir := r.suspendDirFor(vm)
+	if err = os.MkdirAll(destDir, 0o750); err != nil {
+		return ctrl.Result{}, err
+	}
+
+	// Snapshot first — never stop the process before the snapshot is durable.
+	if _, err = r.Driver.Snapshot(ctx, vm, destDir); err != nil {
+		log.Error(err, "Snapshot failed during suspend — will retry")
+		return ctrl.Result{RequeueAfter: r.retryInterval()}, err
+	}
+
+	// Deregister VTEP so other nodes stop routing to the VM about to be stopped.
+	if vm.Spec.NetworkRef != nil && vm.Status.IP != "" {
+		if derr := r.deregisterVTEP(ctx, vm); derr != nil {
+			log.Error(derr, "deregisterVTEP failed during suspend")
+		}
+	}
+
+	// Stop the runtime process — frees memory and tears down the TAP. The
+	// snapshot files on disk persist for resume.
+	if err = r.Driver.Stop(ctx, vm); err != nil {
+		log.Error(err, "Driver Stop failed during suspend — will retry")
+		return ctrl.Result{RequeueAfter: r.retryInterval()}, err
+	}
+
+	base := vm.DeepCopy()
+	vm.Status.Phase = impdevv1alpha1.VMPhaseSuspended
+	vm.Status.SuspendSnapshotPath = destDir
+	now := metav1.Now()
+	vm.Status.SuspendedAt = &now
+	vm.Status.RuntimePID = 0
+	if err = r.Status().Patch(ctx, vm, client.MergeFrom(base)); err != nil {
+		if apierrors.IsConflict(err) {
+			return ctrl.Result{Requeue: true}, nil
+		}
+		return ctrl.Result{}, err
+	}
+
+	if r.Metrics != nil {
+		r.Metrics.SetVMState(vm.Namespace+"/"+vm.Name, "Suspended", r.NodeName)
+	}
+	log.Info("VM suspended", "snapshotPath", destDir)
+	return ctrl.Result{}, nil
+}
+
+// handleSuspended holds the VM in the Suspended state until desiredState flips
+// back to Running, at which point it advances to Resuming.
+func (r *ImpVMReconciler) handleSuspended(ctx context.Context, vm *impdevv1alpha1.ImpVM) (ctrl.Result, error) {
+	if vm.Spec.DesiredState == impdevv1alpha1.VMDesiredStateSuspended {
+		return ctrl.Result{}, nil // steady state
+	}
+	base := vm.DeepCopy()
+	vm.Status.Phase = impdevv1alpha1.VMPhaseResuming
+	if err := r.Status().Patch(ctx, vm, client.MergeFrom(base)); err != nil {
+		if apierrors.IsConflict(err) {
+			return ctrl.Result{Requeue: true}, nil
+		}
+		return ctrl.Result{}, err
+	}
+	return ctrl.Result{}, nil
+}
+
+// handleResuming restores the VM from its node-local suspend snapshot and brings
+// it back to Running, re-establishing VTEP/FDB network state.
+func (r *ImpVMReconciler) handleResuming(ctx context.Context, vm *impdevv1alpha1.ImpVM) (result ctrl.Result, err error) {
+	log := logf.FromContext(ctx)
+
+	ctx, span := tracing.SpanFromVM(ctx, vm, "agent.impvm.resume",
+		trace.WithAttributes(
+			attribute.String("vm.name", vm.Name),
+			attribute.String("vm.namespace", vm.Namespace),
+			attribute.String("vm.node", r.NodeName),
+		),
+	)
+	defer func() {
+		tracing.RecordError(span, err)
+		span.End()
+	}()
+
+	// Start restores from status.suspendSnapshotPath (set on suspend).
+	pid, err := r.Driver.Start(ctx, vm)
+	if err != nil {
+		log.Error(err, "Driver Start failed during resume")
+		return ctrl.Result{}, err
+	}
+
+	state, err := r.Driver.Inspect(ctx, vm)
+	if err != nil {
+		log.Error(err, "Driver Inspect after resume failed")
+		return ctrl.Result{}, err
+	}
+
+	base := vm.DeepCopy()
+	vm.Status.Phase = impdevv1alpha1.VMPhaseRunning
+	vm.Status.IP = state.IP
+	vm.Status.RuntimePID = pid
+	// Clear the suspend snapshot reference so a later crash-restart cold-boots
+	// rather than resuming from a now-stale snapshot.
+	vm.Status.SuspendSnapshotPath = ""
+	vm.Status.SuspendedAt = nil
+	if err = r.Status().Patch(ctx, vm, client.MergeFrom(base)); err != nil {
+		if apierrors.IsConflict(err) {
+			return ctrl.Result{Requeue: true}, nil
+		}
+		return ctrl.Result{}, err
+	}
+
+	if r.Metrics != nil {
+		r.Metrics.SetVMState(vm.Namespace+"/"+vm.Name, "Running", r.NodeName)
+	}
+
+	// Re-register VTEP + sync FDB so other nodes route to the resumed VM.
+	if vm.Spec.NetworkRef != nil && state.IP != "" && r.NodeIP != "" {
+		macAddr := network.MACAddr(vm.Namespace + "/" + vm.Name)
+		if vtepErr := r.registerVTEP(ctx, vm, state.IP, macAddr); vtepErr != nil {
+			log.Error(vtepErr, "registerVTEP after resume failed — FDB sync may be incomplete")
+		} else if fdbErr := r.syncFDB(ctx, vm); fdbErr != nil {
+			log.Error(fdbErr, "syncFDB after resume failed")
+		}
+	}
+
+	log.Info("VM resumed", "pid", pid, "ip", state.IP)
+	return ctrl.Result{}, nil
+}
+
 // finishSucceeded clears spec.nodeName (triggers operator finalizer) + sets phase=Succeeded.
 func (r *ImpVMReconciler) finishSucceeded(ctx context.Context, vm *impdevv1alpha1.ImpVM) (ctrl.Result, error) {
 	// Spec patch first — spec.nodeName is a spec field, not a status field.