From e2b040285766982a3616a7b08f0c09a5e5fd89d9 Mon Sep 17 00:00:00 2001 From: Giovanni Ferri Date: Sat, 4 Jul 2026 03:31:06 +0100 Subject: [PATCH 1/2] feat(api): add suspend/resume phases and desiredState for Phase 2 Additive API for suspend-on-idle: VMPhaseSuspending/Suspended/Resuming, spec.desiredState (Running|Suspended, default Running), and status fields suspendSnapshotPath + suspendedAt. No behavior change; agent state machine and driver restore path land in follow-up tasks. --- api/v1alpha1/impvm_types.go | 17 +++++++++++++++++ api/v1alpha1/shared_types.go | 23 ++++++++++++++++++++++- api/v1alpha1/zz_generated.deepcopy.go | 4 ++++ config/crd/bases/imp.dev_impvms.yaml | 24 ++++++++++++++++++++++++ 4 files changed, 67 insertions(+), 1 deletion(-) diff --git a/api/v1alpha1/impvm_types.go b/api/v1alpha1/impvm_types.go index 660e252..d3d01c6 100644 --- a/api/v1alpha1/impvm_types.go +++ b/api/v1alpha1/impvm_types.go @@ -127,6 +127,13 @@ type ImpVMSpec struct { // owned by it (PVCs cannot follow the VM to a new node). // +optional RescheduleOnNodeLoss bool `json:"rescheduleOnNodeLoss,omitempty"` + + // DesiredState is the requested run state. When set to "Suspended" the agent + // snapshots the VM to node-local storage and frees its memory; setting it back + // to "Running" resumes it from that snapshot on the same node. + // +optional + // +kubebuilder:default=Running + DesiredState VMDesiredState `json:"desiredState,omitempty"` } // UserDataSource references a ConfigMap containing cloud-init user-data. @@ -170,6 +177,16 @@ type ImpVMStatus struct { // +optional StartedAt *metav1.Time `json:"startedAt,omitempty"` + // SuspendSnapshotPath is the node-local directory holding the VM's suspend + // snapshot (vm.state + vm.mem). Set when the VM is Suspended; the resume path + // restores from here. Empty when the VM is not suspended. + // +optional + SuspendSnapshotPath string `json:"suspendSnapshotPath,omitempty"` + + // SuspendedAt is the time the VM last transitioned to phase Suspended. + // +optional + SuspendedAt *metav1.Time `json:"suspendedAt,omitempty"` + // RestartCount is the cumulative number of times this VM has been restarted. // +optional RestartCount int32 `json:"restartCount,omitempty"` diff --git a/api/v1alpha1/shared_types.go b/api/v1alpha1/shared_types.go index 42d5f1d..e6c955f 100644 --- a/api/v1alpha1/shared_types.go +++ b/api/v1alpha1/shared_types.go @@ -79,7 +79,7 @@ const ( ) // VMPhase is the current lifecycle phase of an ImpVM. -// +kubebuilder:validation:Enum=Pending;Scheduled;Starting;Running;Terminating;Succeeded;Failed;RetryExhausted +// +kubebuilder:validation:Enum=Pending;Scheduled;Starting;Running;Terminating;Succeeded;Failed;RetryExhausted;Suspending;Suspended;Resuming type VMPhase string const ( @@ -93,6 +93,27 @@ const ( // VMPhaseRetryExhausted means all restart attempts are exhausted with onExhaustion="manual-reset". // The VM will not restart until the retry counter is cleared via the imp/reset-retries annotation. VMPhaseRetryExhausted VMPhase = "RetryExhausted" + + // VMPhaseSuspending means the VM is being snapshotted to node-local storage + // before its runtime process is stopped to free memory. + VMPhaseSuspending VMPhase = "Suspending" + // VMPhaseSuspended means the VM's state is captured on node-local disk and its + // runtime process is stopped (no resident memory). It can be resumed on the same node. + VMPhaseSuspended VMPhase = "Suspended" + // VMPhaseResuming means the VM is being restored from its node-local suspend snapshot. + VMPhaseResuming VMPhase = "Resuming" +) + +// VMDesiredState is the operator/user-requested run state for an ImpVM. +// The agent drives the observed Phase toward this target. +// +kubebuilder:validation:Enum=Running;Suspended +type VMDesiredState string + +const ( + // VMDesiredStateRunning keeps the VM running (default). + VMDesiredStateRunning VMDesiredState = "Running" + // VMDesiredStateSuspended requests the VM be snapshotted and its memory freed. + VMDesiredStateSuspended VMDesiredState = "Suspended" ) // Arch is the CPU architecture for a VM class. diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 9104609..4fde85b 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -1094,6 +1094,10 @@ func (in *ImpVMStatus) DeepCopyInto(out *ImpVMStatus) { in, out := &in.StartedAt, &out.StartedAt *out = (*in).DeepCopy() } + if in.SuspendedAt != nil { + in, out := &in.SuspendedAt, &out.SuspendedAt + *out = (*in).DeepCopy() + } if in.NextRetryAfter != nil { in, out := &in.NextRetryAfter, &out.NextRetryAfter *out = (*in).DeepCopy() diff --git a/config/crd/bases/imp.dev_impvms.yaml b/config/crd/bases/imp.dev_impvms.yaml index 4810150..92870fb 100644 --- a/config/crd/bases/imp.dev_impvms.yaml +++ b/config/crd/bases/imp.dev_impvms.yaml @@ -74,6 +74,16 @@ spec: required: - name type: object + desiredState: + default: Running + description: |- + DesiredState is the requested run state. When set to "Suspended" the agent + snapshots the VM to node-local storage and frees its memory; setting it back + to "Running" resumes it from that snapshot on the same node. + enum: + - Running + - Suspended + type: string env: description: Env sets environment variables inside the VM via the guest agent. @@ -748,6 +758,9 @@ spec: - Succeeded - Failed - RetryExhausted + - Suspending + - Suspended + - Resuming type: string restartCount: description: RestartCount is the cumulative number of times this VM @@ -774,6 +787,17 @@ spec: Used to detect and time out stuck start attempts. format: date-time type: string + suspendSnapshotPath: + description: |- + SuspendSnapshotPath is the node-local directory holding the VM's suspend + snapshot (vm.state + vm.mem). Set when the VM is Suspended; the resume path + restores from here. Empty when the VM is not suspended. + type: string + suspendedAt: + description: SuspendedAt is the time the VM last transitioned to phase + Suspended. + format: date-time + type: string type: object type: object served: true From 021520c4293b211a153c6076c40b460b95e0db12 Mon Sep 17 00:00:00 2001 From: Giovanni Ferri Date: Sun, 5 Jul 2026 23:13:02 +0100 Subject: [PATCH 2/2] feat(agent): suspend-on-idle state machine (Phase 2 Task 2/3) Agent drives ImpVM through Suspending -> Suspended -> Resuming when spec.desiredState flips. handleSuspending snapshots to node-local storage then stops the runtime to free memory (snapshot MUST succeed before stop); handleResuming restores via Start and re-establishes VTEP/FDB. Driver Start now prefers status.suspendSnapshotPath (resume) over spec.snapshotRef, via a shared configureSnapshotBoot helper. Operator clears status.suspendSnapshotPath when rescheduling a VM off a lost node so it cold-boots instead of failing to resume from a vanished snapshot. Tests cover suspend, resume, and the no-free-without-snapshot guarantee. --- internal/agent/firecracker_driver.go | 25 +++- internal/agent/reconciler.go | 187 ++++++++++++++++++++++++ internal/agent/reconciler_test.go | 108 ++++++++++++++ internal/controller/impvm_controller.go | 8 + 4 files changed, 322 insertions(+), 6 deletions(-) diff --git a/internal/agent/firecracker_driver.go b/internal/agent/firecracker_driver.go index b52bcf5..54e96cc 100644 --- a/internal/agent/firecracker_driver.go +++ b/internal/agent/firecracker_driver.go @@ -209,8 +209,14 @@ func (d *FirecrackerDriver) Start(ctx context.Context, vm *impdevv1alpha1.ImpVM) // 5. Build Firecracker config. cfg := d.buildConfig(&class, rootfsPath, sockPath, netInfo, gaEnabled) - // 5a. Apply snapshot-based boot if requested. - if vm.Spec.SnapshotRef != "" { + // 5a. Apply snapshot-based boot. Resuming from a node-local suspend snapshot + // (status.suspendSnapshotPath) takes priority over a cold snapshot boot from + // an ImpVMSnapshot object (spec.snapshotRef). + switch { + case vm.Status.SuspendSnapshotPath != "": + configureSnapshotBoot(&cfg, vm.Status.SuspendSnapshotPath) + logf.FromContext(ctx).Info("configured resume from suspend snapshot", "path", vm.Status.SuspendSnapshotPath) + case vm.Spec.SnapshotRef != "": if err := d.applySnapshotBoot(ctx, vm, &cfg); err != nil { return 0, fmt.Errorf("apply snapshot boot: %w", err) } @@ -711,13 +717,20 @@ func (d *FirecrackerDriver) applySnapshotBoot(ctx context.Context, vm *impdevv1a log.Info("snapshot has no node-local path, skipping snapshot boot", "snapshot", snap.Name) return nil } + configureSnapshotBoot(cfg, snap.Status.SnapshotPath) + log.Info("configured snapshot-based boot", "snapshotPath", cfg.Snapshot.SnapshotPath) + return nil +} + +// configureSnapshotBoot points cfg at the node-local snapshot files (vm.state + +// vm.mem) in dir and enables resume-on-load. dir must be a directory previously +// written by Driver.Snapshot (which uses the same filenames). +func configureSnapshotBoot(cfg *firecracker.Config, dir string) { cfg.Snapshot = firecracker.SnapshotConfig{ - SnapshotPath: filepath.Join(snap.Status.SnapshotPath, "vm.state"), - MemFilePath: filepath.Join(snap.Status.SnapshotPath, "vm.mem"), + SnapshotPath: filepath.Join(dir, "vm.state"), + MemFilePath: filepath.Join(dir, "vm.mem"), ResumeVM: true, } - log.Info("configured snapshot-based boot", "snapshotPath", cfg.Snapshot.SnapshotPath) - return nil } // IsAlive reports whether the process with the given PID is still running. diff --git a/internal/agent/reconciler.go b/internal/agent/reconciler.go index 2a19199..cecac2d 100644 --- a/internal/agent/reconciler.go +++ b/internal/agent/reconciler.go @@ -6,6 +6,8 @@ import ( "context" "errors" "net/http" + "os" + "path/filepath" "strconv" "time" @@ -51,6 +53,23 @@ type ImpVMReconciler struct { RetryInterval time.Duration // Recorder emits lifecycle events for VM completion/failure. Recorder record.EventRecorder + // SuspendDir is the node-local base directory under which suspend snapshots + // are written (one subdir per VM). Defaults to /var/lib/imp/suspend when empty. + SuspendDir string +} + +// suspendBaseDir returns the configured suspend snapshot base directory, +// defaulting to /var/lib/imp/suspend. +func (r *ImpVMReconciler) suspendBaseDir() string { + if r.SuspendDir != "" { + return r.SuspendDir + } + return "/var/lib/imp/suspend" +} + +// suspendDirFor returns the node-local directory holding vm's suspend snapshot. +func (r *ImpVMReconciler) suspendDirFor(vm *impdevv1alpha1.ImpVM) string { + return filepath.Join(r.suspendBaseDir(), vm.Namespace+"_"+vm.Name) } // +kubebuilder:rbac:groups=imp.dev,resources=impvms,verbs=get;list;watch;update;patch @@ -94,6 +113,12 @@ func (r *ImpVMReconciler) Reconcile(ctx context.Context, req ctrl.Request) (resu return r.handleRunning(ctx, vm) case impdevv1alpha1.VMPhaseStarting: return r.handleStarting(ctx, vm) + case impdevv1alpha1.VMPhaseSuspending: + return r.handleSuspending(ctx, vm) + case impdevv1alpha1.VMPhaseSuspended: + return r.handleSuspended(ctx, vm) + case impdevv1alpha1.VMPhaseResuming: + return r.handleResuming(ctx, vm) default: // Pending, Succeeded, Failed — not our concern. return ctrl.Result{}, nil @@ -211,6 +236,20 @@ func (r *ImpVMReconciler) handleScheduled(ctx context.Context, vm *impdevv1alpha func (r *ImpVMReconciler) handleRunning(ctx context.Context, vm *impdevv1alpha1.ImpVM) (ctrl.Result, error) { log := logf.FromContext(ctx) + // Suspend requested: transition to Suspending so the VM is snapshotted and + // its memory freed. The Suspending handler does the actual work. + if vm.Spec.DesiredState == impdevv1alpha1.VMDesiredStateSuspended { + base := vm.DeepCopy() + vm.Status.Phase = impdevv1alpha1.VMPhaseSuspending + if err := r.Status().Patch(ctx, vm, client.MergeFrom(base)); err != nil { + if apierrors.IsConflict(err) { + return ctrl.Result{Requeue: true}, nil + } + return ctrl.Result{}, err + } + return ctrl.Result{}, nil + } + state, err := r.Driver.Inspect(ctx, vm) if err != nil { log.Error(err, "Driver Inspect failed") @@ -318,9 +357,157 @@ func (r *ImpVMReconciler) handleTerminating(ctx context.Context, vm *impdevv1alp } } + // Remove any node-local suspend snapshot so it does not outlive the VM. + if err := os.RemoveAll(r.suspendDirFor(vm)); err != nil { + log.Error(err, "failed to remove suspend snapshot dir", "dir", r.suspendDirFor(vm)) + } + return r.clearOwnership(ctx, vm) } +// handleSuspending snapshots the running VM to node-local storage, then stops its +// runtime process to free memory. The snapshot MUST succeed before the process is +// stopped, otherwise the VM state would be lost. +func (r *ImpVMReconciler) handleSuspending(ctx context.Context, vm *impdevv1alpha1.ImpVM) (result ctrl.Result, err error) { + log := logf.FromContext(ctx) + + ctx, span := tracing.SpanFromVM(ctx, vm, "agent.impvm.suspend", + trace.WithAttributes( + attribute.String("vm.name", vm.Name), + attribute.String("vm.namespace", vm.Namespace), + attribute.String("vm.node", r.NodeName), + ), + ) + defer func() { + tracing.RecordError(span, err) + span.End() + }() + + destDir := r.suspendDirFor(vm) + if err = os.MkdirAll(destDir, 0o750); err != nil { + return ctrl.Result{}, err + } + + // Snapshot first — never stop the process before the snapshot is durable. + if _, err = r.Driver.Snapshot(ctx, vm, destDir); err != nil { + log.Error(err, "Snapshot failed during suspend — will retry") + return ctrl.Result{RequeueAfter: r.retryInterval()}, err + } + + // Deregister VTEP so other nodes stop routing to the VM about to be stopped. + if vm.Spec.NetworkRef != nil && vm.Status.IP != "" { + if derr := r.deregisterVTEP(ctx, vm); derr != nil { + log.Error(derr, "deregisterVTEP failed during suspend") + } + } + + // Stop the runtime process — frees memory and tears down the TAP. The + // snapshot files on disk persist for resume. + if err = r.Driver.Stop(ctx, vm); err != nil { + log.Error(err, "Driver Stop failed during suspend — will retry") + return ctrl.Result{RequeueAfter: r.retryInterval()}, err + } + + base := vm.DeepCopy() + vm.Status.Phase = impdevv1alpha1.VMPhaseSuspended + vm.Status.SuspendSnapshotPath = destDir + now := metav1.Now() + vm.Status.SuspendedAt = &now + vm.Status.RuntimePID = 0 + if err = r.Status().Patch(ctx, vm, client.MergeFrom(base)); err != nil { + if apierrors.IsConflict(err) { + return ctrl.Result{Requeue: true}, nil + } + return ctrl.Result{}, err + } + + if r.Metrics != nil { + r.Metrics.SetVMState(vm.Namespace+"/"+vm.Name, "Suspended", r.NodeName) + } + log.Info("VM suspended", "snapshotPath", destDir) + return ctrl.Result{}, nil +} + +// handleSuspended holds the VM in the Suspended state until desiredState flips +// back to Running, at which point it advances to Resuming. +func (r *ImpVMReconciler) handleSuspended(ctx context.Context, vm *impdevv1alpha1.ImpVM) (ctrl.Result, error) { + if vm.Spec.DesiredState == impdevv1alpha1.VMDesiredStateSuspended { + return ctrl.Result{}, nil // steady state + } + base := vm.DeepCopy() + vm.Status.Phase = impdevv1alpha1.VMPhaseResuming + if err := r.Status().Patch(ctx, vm, client.MergeFrom(base)); err != nil { + if apierrors.IsConflict(err) { + return ctrl.Result{Requeue: true}, nil + } + return ctrl.Result{}, err + } + return ctrl.Result{}, nil +} + +// handleResuming restores the VM from its node-local suspend snapshot and brings +// it back to Running, re-establishing VTEP/FDB network state. +func (r *ImpVMReconciler) handleResuming(ctx context.Context, vm *impdevv1alpha1.ImpVM) (result ctrl.Result, err error) { + log := logf.FromContext(ctx) + + ctx, span := tracing.SpanFromVM(ctx, vm, "agent.impvm.resume", + trace.WithAttributes( + attribute.String("vm.name", vm.Name), + attribute.String("vm.namespace", vm.Namespace), + attribute.String("vm.node", r.NodeName), + ), + ) + defer func() { + tracing.RecordError(span, err) + span.End() + }() + + // Start restores from status.suspendSnapshotPath (set on suspend). + pid, err := r.Driver.Start(ctx, vm) + if err != nil { + log.Error(err, "Driver Start failed during resume") + return ctrl.Result{}, err + } + + state, err := r.Driver.Inspect(ctx, vm) + if err != nil { + log.Error(err, "Driver Inspect after resume failed") + return ctrl.Result{}, err + } + + base := vm.DeepCopy() + vm.Status.Phase = impdevv1alpha1.VMPhaseRunning + vm.Status.IP = state.IP + vm.Status.RuntimePID = pid + // Clear the suspend snapshot reference so a later crash-restart cold-boots + // rather than resuming from a now-stale snapshot. + vm.Status.SuspendSnapshotPath = "" + vm.Status.SuspendedAt = nil + if err = r.Status().Patch(ctx, vm, client.MergeFrom(base)); err != nil { + if apierrors.IsConflict(err) { + return ctrl.Result{Requeue: true}, nil + } + return ctrl.Result{}, err + } + + if r.Metrics != nil { + r.Metrics.SetVMState(vm.Namespace+"/"+vm.Name, "Running", r.NodeName) + } + + // Re-register VTEP + sync FDB so other nodes route to the resumed VM. + if vm.Spec.NetworkRef != nil && state.IP != "" && r.NodeIP != "" { + macAddr := network.MACAddr(vm.Namespace + "/" + vm.Name) + if vtepErr := r.registerVTEP(ctx, vm, state.IP, macAddr); vtepErr != nil { + log.Error(vtepErr, "registerVTEP after resume failed — FDB sync may be incomplete") + } else if fdbErr := r.syncFDB(ctx, vm); fdbErr != nil { + log.Error(fdbErr, "syncFDB after resume failed") + } + } + + log.Info("VM resumed", "pid", pid, "ip", state.IP) + return ctrl.Result{}, nil +} + // finishSucceeded clears spec.nodeName (triggers operator finalizer) + sets phase=Succeeded. func (r *ImpVMReconciler) finishSucceeded(ctx context.Context, vm *impdevv1alpha1.ImpVM) (ctrl.Result, error) { // Spec patch first — spec.nodeName is a spec field, not a status field. diff --git a/internal/agent/reconciler_test.go b/internal/agent/reconciler_test.go index a55972e..5e08f41 100644 --- a/internal/agent/reconciler_test.go +++ b/internal/agent/reconciler_test.go @@ -5,6 +5,7 @@ package agent import ( "context" "errors" + "path/filepath" "sync" "time" @@ -59,6 +60,113 @@ var _ = Describe("ImpVM Agent: Scheduled → Running", func() { }) }) +var _ = Describe("ImpVM Agent: suspend / resume", func() { + ctx := context.Background() + + It("transitions Running(desiredState=Suspended) through Suspending to Suspended", func() { + driver := NewStubDriver() + r := &ImpVMReconciler{Client: k8sClient, NodeName: testNode, Driver: driver, SuspendDir: GinkgoT().TempDir()} + + vm := &impdevv1alpha1.ImpVM{ + ObjectMeta: metav1.ObjectMeta{Name: "tc-suspend", Namespace: "default", Finalizers: []string{"imp/finalizer"}}, + Spec: impdevv1alpha1.ImpVMSpec{NodeName: testNode, DesiredState: impdevv1alpha1.VMDesiredStateSuspended}, + } + Expect(k8sClient.Create(ctx, vm)).To(Succeed()) + DeferCleanup(func() { k8sClient.Delete(ctx, vm) }) //nolint:errcheck + + // Prime: VM running in the driver + status. + pid, err := driver.Start(ctx, vm) + Expect(err).NotTo(HaveOccurred()) + base := vm.DeepCopy() + vm.Status.Phase = impdevv1alpha1.VMPhaseRunning + vm.Status.RuntimePID = pid + vm.Status.IP = "192.168.100.50" + Expect(k8sClient.Status().Patch(ctx, vm, client.MergeFrom(base))).To(Succeed()) + + req := reconcile.Request{NamespacedName: types.NamespacedName{Name: "tc-suspend", Namespace: "default"}} + + // #1: Running(desired=Suspended) → Suspending. + _, err = r.Reconcile(ctx, req) + Expect(err).NotTo(HaveOccurred()) + updated := &impdevv1alpha1.ImpVM{} + Expect(k8sClient.Get(ctx, req.NamespacedName, updated)).To(Succeed()) + Expect(updated.Status.Phase).To(Equal(impdevv1alpha1.VMPhaseSuspending)) + + // #2: Suspending → snapshot + stop → Suspended. + _, err = r.Reconcile(ctx, req) + Expect(err).NotTo(HaveOccurred()) + Expect(k8sClient.Get(ctx, req.NamespacedName, updated)).To(Succeed()) + Expect(updated.Status.Phase).To(Equal(impdevv1alpha1.VMPhaseSuspended)) + Expect(updated.Status.SuspendSnapshotPath).NotTo(BeEmpty()) + Expect(updated.Status.SuspendedAt).NotTo(BeNil()) + Expect(updated.Status.RuntimePID).To(BeZero()) + }) + + It("transitions Suspended(desiredState=Running) through Resuming to Running and clears the snapshot ref", func() { + driver := NewStubDriver() + suspendDir := GinkgoT().TempDir() + r := &ImpVMReconciler{Client: k8sClient, NodeName: testNode, Driver: driver, SuspendDir: suspendDir} + + vm := &impdevv1alpha1.ImpVM{ + ObjectMeta: metav1.ObjectMeta{Name: "tc-resume", Namespace: "default", Finalizers: []string{"imp/finalizer"}}, + Spec: impdevv1alpha1.ImpVMSpec{NodeName: testNode, DesiredState: impdevv1alpha1.VMDesiredStateRunning}, + } + Expect(k8sClient.Create(ctx, vm)).To(Succeed()) + DeferCleanup(func() { k8sClient.Delete(ctx, vm) }) //nolint:errcheck + + base := vm.DeepCopy() + vm.Status.Phase = impdevv1alpha1.VMPhaseSuspended + vm.Status.SuspendSnapshotPath = filepath.Join(suspendDir, "default_tc-resume") + now := metav1.Now() + vm.Status.SuspendedAt = &now + Expect(k8sClient.Status().Patch(ctx, vm, client.MergeFrom(base))).To(Succeed()) + + req := reconcile.Request{NamespacedName: types.NamespacedName{Name: "tc-resume", Namespace: "default"}} + + // #1: Suspended(desired=Running) → Resuming. + _, err := r.Reconcile(ctx, req) + Expect(err).NotTo(HaveOccurred()) + updated := &impdevv1alpha1.ImpVM{} + Expect(k8sClient.Get(ctx, req.NamespacedName, updated)).To(Succeed()) + Expect(updated.Status.Phase).To(Equal(impdevv1alpha1.VMPhaseResuming)) + + // #2: Resuming → start (restore) → Running, snapshot ref cleared. + _, err = r.Reconcile(ctx, req) + Expect(err).NotTo(HaveOccurred()) + Expect(k8sClient.Get(ctx, req.NamespacedName, updated)).To(Succeed()) + Expect(updated.Status.Phase).To(Equal(impdevv1alpha1.VMPhaseRunning)) + Expect(updated.Status.IP).NotTo(BeEmpty()) + Expect(updated.Status.RuntimePID).To(BeNumerically(">", 0)) + Expect(updated.Status.SuspendSnapshotPath).To(BeEmpty()) + Expect(updated.Status.SuspendedAt).To(BeNil()) + }) + + It("stays in Suspending when the snapshot fails — never frees memory without a durable snapshot", func() { + driver := NewStubDriver() // VM never Started in the driver → Snapshot fails + r := &ImpVMReconciler{Client: k8sClient, NodeName: testNode, Driver: driver, SuspendDir: GinkgoT().TempDir()} + + vm := &impdevv1alpha1.ImpVM{ + ObjectMeta: metav1.ObjectMeta{Name: "tc-suspend-fail", Namespace: "default", Finalizers: []string{"imp/finalizer"}}, + Spec: impdevv1alpha1.ImpVMSpec{NodeName: testNode, DesiredState: impdevv1alpha1.VMDesiredStateSuspended}, + } + Expect(k8sClient.Create(ctx, vm)).To(Succeed()) + DeferCleanup(func() { k8sClient.Delete(ctx, vm) }) //nolint:errcheck + + base := vm.DeepCopy() + vm.Status.Phase = impdevv1alpha1.VMPhaseSuspending + Expect(k8sClient.Status().Patch(ctx, vm, client.MergeFrom(base))).To(Succeed()) + + req := reconcile.Request{NamespacedName: types.NamespacedName{Name: "tc-suspend-fail", Namespace: "default"}} + _, err := r.Reconcile(ctx, req) + Expect(err).To(HaveOccurred()) + + updated := &impdevv1alpha1.ImpVM{} + Expect(k8sClient.Get(ctx, req.NamespacedName, updated)).To(Succeed()) + Expect(updated.Status.Phase).To(Equal(impdevv1alpha1.VMPhaseSuspending)) + Expect(updated.Status.SuspendSnapshotPath).To(BeEmpty()) + }) +}) + var _ = Describe("ImpVM Agent: ephemeral exit → Succeeded", func() { ctx := context.Background() diff --git a/internal/controller/impvm_controller.go b/internal/controller/impvm_controller.go index fd6ed99..1deac08 100644 --- a/internal/controller/impvm_controller.go +++ b/internal/controller/impvm_controller.go @@ -255,6 +255,10 @@ func (r *ImpVMReconciler) syncStatus(ctx context.Context, vm *impdevv1alpha1.Imp setNodeUnhealthy(vm, reason) vm.Status.Phase = impdevv1alpha1.VMPhasePending setUnscheduled(vm) + // The node-local suspend snapshot died with the node; clear it so the + // rescheduled VM cold-boots instead of failing to resume from a lost path. + vm.Status.SuspendSnapshotPath = "" + vm.Status.SuspendedAt = nil if err2 := r.Status().Patch(ctx, vm, client.MergeFrom(vmCopy)); err2 != nil { return ctrl.Result{}, err2 } @@ -278,6 +282,10 @@ func (r *ImpVMReconciler) syncStatus(ctx context.Context, vm *impdevv1alpha1.Imp setNodeUnhealthy(vm, reason) vm.Status.Phase = impdevv1alpha1.VMPhasePending setUnscheduled(vm) + // Node-local suspend snapshot is gone with the node; clear it so the + // rescheduled VM cold-boots instead of failing to resume. + vm.Status.SuspendSnapshotPath = "" + vm.Status.SuspendedAt = nil if err2 := r.Status().Patch(ctx, vm, client.MergeFrom(vmCopy)); err2 != nil { return ctrl.Result{}, err2 }