From 55c44d2b998a7e6dcc67b6a03d366e49ae72c7c2 Mon Sep 17 00:00:00 2001 From: Steven Miller Date: Wed, 18 Mar 2026 02:47:05 -0400 Subject: [PATCH 01/31] Add configurable snapshot compression with async standby support --- cmd/api/api/api_test.go | 2 +- cmd/api/api/instances.go | 82 ++- cmd/api/api/snapshots.go | 33 +- cmd/api/config/config.go | 29 + integration/systemd_test.go | 2 +- integration/vgpu_test.go | 2 +- lib/builds/manager_test.go | 2 +- lib/devices/gpu_e2e_test.go | 2 +- lib/devices/gpu_inference_test.go | 2 +- lib/devices/gpu_module_test.go | 4 +- lib/instances/create.go | 6 + lib/instances/firecracker_test.go | 8 +- lib/instances/fork.go | 6 +- lib/instances/manager.go | 13 +- lib/instances/manager_darwin_test.go | 4 +- lib/instances/manager_test.go | 6 +- lib/instances/network_test.go | 2 +- lib/instances/qemu_test.go | 4 +- lib/instances/resource_limits_test.go | 2 +- lib/instances/restore.go | 3 + lib/instances/snapshot.go | 33 +- lib/instances/snapshot_compression.go | 410 +++++++++++++ lib/instances/snapshot_compression_test.go | 118 ++++ .../snapshot_integration_scenario_test.go | 2 +- lib/instances/standby.go | 26 + lib/instances/types.go | 21 +- lib/oapi/oapi.go | 549 +++++++++++------- lib/providers/providers.go | 11 +- lib/snapshot/types.go | 45 +- openapi.yaml | 72 +++ 30 files changed, 1244 insertions(+), 257 deletions(-) create mode 100644 lib/instances/snapshot_compression.go create mode 100644 lib/instances/snapshot_compression_test.go diff --git a/cmd/api/api/api_test.go b/cmd/api/api/api_test.go index 38b7eadd..b109f646 100644 --- a/cmd/api/api/api_test.go +++ b/cmd/api/api/api_test.go @@ -43,7 +43,7 @@ func newTestService(t *testing.T) *ApiService { limits := instances.ResourceLimits{ MaxOverlaySize: 100 * 1024 * 1024 * 1024, // 100GB } - instanceMgr := instances.NewManager(p, imageMgr, systemMgr, networkMgr, deviceMgr, volumeMgr, limits, "", nil, nil) + instanceMgr := instances.NewManager(p, imageMgr, systemMgr, networkMgr, deviceMgr, volumeMgr, limits, "", instances.SnapshotPolicy{}, nil, nil) // Initialize network manager (creates bridge for network-enabled tests) if err := networkMgr.Initialize(ctx(), nil); err != nil { diff --git a/cmd/api/api/instances.go b/cmd/api/api/instances.go index aa3b4524..80446e1f 100644 --- a/cmd/api/api/instances.go +++ b/cmd/api/api/instances.go @@ -17,6 +17,7 @@ import ( "github.com/kernel/hypeman/lib/network" "github.com/kernel/hypeman/lib/oapi" "github.com/kernel/hypeman/lib/resources" + "github.com/kernel/hypeman/lib/snapshot" "github.com/kernel/hypeman/lib/vm_metrics" "github.com/samber/lo" ) @@ -264,6 +265,16 @@ func (s *ApiService) CreateInstance(ctx context.Context, request oapi.CreateInst SkipKernelHeaders: request.Body.SkipKernelHeaders != nil && *request.Body.SkipKernelHeaders, SkipGuestAgent: request.Body.SkipGuestAgent != nil && *request.Body.SkipGuestAgent, } + if request.Body.SnapshotPolicy != nil { + snapshotPolicy, err := toInstanceSnapshotPolicy(*request.Body.SnapshotPolicy) + if err != nil { + return oapi.CreateInstance400JSONResponse{ + Code: "invalid_snapshot_policy", + Message: err.Error(), + }, nil + } + domainReq.SnapshotPolicy = snapshotPolicy + } inst, err := s.InstanceManager.CreateInstance(ctx, domainReq) if err != nil { @@ -401,7 +412,19 @@ func (s *ApiService) StandbyInstance(ctx context.Context, request oapi.StandbyIn } log := logger.FromContext(ctx) - result, err := s.InstanceManager.StandbyInstance(ctx, inst.Id) + standbyReq := instances.StandbyInstanceRequest{} + if request.Body != nil && request.Body.Compression != nil { + compression, err := toDomainSnapshotCompressionConfig(*request.Body.Compression) + if err != nil { + return oapi.StandbyInstance400JSONResponse{ + Code: "invalid_snapshot_compression", + Message: err.Error(), + }, nil + } + standbyReq.Compression = compression + } + + result, err := s.InstanceManager.StandbyInstance(ctx, inst.Id, standbyReq) if err != nil { switch { case errors.Is(err, instances.ErrInvalidState): @@ -858,6 +881,10 @@ func instanceToOAPI(inst instances.Instance) oapi.Instance { if len(inst.Metadata) > 0 { oapiInst.Metadata = toOAPIMetadata(inst.Metadata) } + if inst.SnapshotPolicy != nil { + oapiPolicy, _ := toOAPISnapshotPolicy(*inst.SnapshotPolicy) + oapiInst.SnapshotPolicy = &oapiPolicy + } // Convert volume attachments if len(inst.Volumes) > 0 { @@ -892,3 +919,56 @@ func instanceToOAPI(inst instances.Instance) oapi.Instance { return oapiInst } + +func toDomainSnapshotCompressionConfig(cfg oapi.SnapshotCompressionConfig) (*snapshot.SnapshotCompressionConfig, error) { + out := &snapshot.SnapshotCompressionConfig{ + Enabled: cfg.Enabled, + } + if cfg.Algorithm != nil { + out.Algorithm = snapshot.SnapshotCompressionAlgorithm(*cfg.Algorithm) + } + if cfg.Level != nil { + level := *cfg.Level + out.Level = &level + } + return out, nil +} + +func toInstanceSnapshotPolicy(policy oapi.SnapshotPolicy) (*instances.SnapshotPolicy, error) { + out := &instances.SnapshotPolicy{} + if policy.Compression != nil { + compression, err := toDomainSnapshotCompressionConfig(*policy.Compression) + if err != nil { + return nil, err + } + out.Compression = compression + } + return out, nil +} + +func toOAPISnapshotCompressionConfig(cfg snapshot.SnapshotCompressionConfig) (oapi.SnapshotCompressionConfig, error) { + out := oapi.SnapshotCompressionConfig{ + Enabled: cfg.Enabled, + } + if cfg.Algorithm != "" { + algo := oapi.SnapshotCompressionConfigAlgorithm(cfg.Algorithm) + out.Algorithm = &algo + } + if cfg.Level != nil { + level := *cfg.Level + out.Level = &level + } + return out, nil +} + +func toOAPISnapshotPolicy(policy instances.SnapshotPolicy) (oapi.SnapshotPolicy, error) { + out := oapi.SnapshotPolicy{} + if policy.Compression != nil { + compression, err := toOAPISnapshotCompressionConfig(*policy.Compression) + if err != nil { + return oapi.SnapshotPolicy{}, err + } + out.Compression = &compression + } + return out, nil +} diff --git a/cmd/api/api/snapshots.go b/cmd/api/api/snapshots.go index b5a70d0c..04fd47fd 100644 --- a/cmd/api/api/snapshots.go +++ b/cmd/api/api/snapshots.go @@ -10,6 +10,7 @@ import ( mw "github.com/kernel/hypeman/lib/middleware" "github.com/kernel/hypeman/lib/network" "github.com/kernel/hypeman/lib/oapi" + "github.com/kernel/hypeman/lib/snapshot" "github.com/samber/lo" ) @@ -27,11 +28,20 @@ func (s *ApiService) CreateInstanceSnapshot(ctx context.Context, request oapi.Cr if request.Body.Name != nil { name = *request.Body.Name } + var compression *snapshot.SnapshotCompressionConfig + if request.Body.Compression != nil { + var err error + compression, err = toDomainSnapshotCompressionConfig(*request.Body.Compression) + if err != nil { + return oapi.CreateInstanceSnapshot400JSONResponse{Code: "invalid_snapshot_compression", Message: err.Error()}, nil + } + } result, err := s.InstanceManager.CreateSnapshot(ctx, inst.Id, instances.CreateSnapshotRequest{ - Kind: instances.SnapshotKind(request.Body.Kind), - Name: name, - Metadata: toMapMetadata(request.Body.Metadata), + Kind: instances.SnapshotKind(request.Body.Kind), + Name: name, + Metadata: toMapMetadata(request.Body.Metadata), + Compression: compression, }) if err != nil { log := logger.FromContext(ctx) @@ -207,6 +217,23 @@ func snapshotToOAPI(snapshot instances.Snapshot) oapi.Snapshot { SizeBytes: snapshot.SizeBytes, Name: lo.ToPtr(snapshot.Name), } + if snapshot.CompressionState != "" { + state := oapi.SnapshotCompressionState(snapshot.CompressionState) + out.CompressionState = &state + } + if snapshot.CompressionError != "" { + out.CompressionError = lo.ToPtr(snapshot.CompressionError) + } + if snapshot.Compression != nil { + compression, _ := toOAPISnapshotCompressionConfig(*snapshot.Compression) + out.Compression = &compression + } + if snapshot.CompressedSizeBytes != nil { + out.CompressedSizeBytes = snapshot.CompressedSizeBytes + } + if snapshot.UncompressedSizeBytes != nil { + out.UncompressedSizeBytes = snapshot.UncompressedSizeBytes + } if snapshot.Name == "" { out.Name = nil } diff --git a/cmd/api/config/config.go b/cmd/api/config/config.go index 4ee3d442..1a0b1275 100644 --- a/cmd/api/config/config.go +++ b/cmd/api/config/config.go @@ -158,6 +158,18 @@ type HypervisorConfig struct { FirecrackerBinaryPath string `koanf:"firecracker_binary_path"` } +// SnapshotCompressionDefaultConfig holds default snapshot compression settings. +type SnapshotCompressionDefaultConfig struct { + Enabled bool `koanf:"enabled"` + Algorithm string `koanf:"algorithm"` + Level int `koanf:"level"` +} + +// SnapshotConfig holds snapshot defaults. +type SnapshotConfig struct { + CompressionDefault SnapshotCompressionDefaultConfig `koanf:"compression_default"` +} + // GPUConfig holds GPU-related settings. type GPUConfig struct { ProfileCacheTTL string `koanf:"profile_cache_ttl"` @@ -183,6 +195,7 @@ type Config struct { Oversubscription OversubscriptionConfig `koanf:"oversubscription"` Capacity CapacityConfig `koanf:"capacity"` Hypervisor HypervisorConfig `koanf:"hypervisor"` + Snapshot SnapshotConfig `koanf:"snapshot"` GPU GPUConfig `koanf:"gpu"` } @@ -302,6 +315,14 @@ func defaultConfig() *Config { FirecrackerBinaryPath: "", }, + Snapshot: SnapshotConfig{ + CompressionDefault: SnapshotCompressionDefaultConfig{ + Enabled: false, + Algorithm: "zstd", + Level: 1, + }, + }, + GPU: GPUConfig{ ProfileCacheTTL: "30m", }, @@ -400,5 +421,13 @@ func (c *Config) Validate() error { if c.Build.Timeout <= 0 { return fmt.Errorf("build.timeout must be positive, got %d", c.Build.Timeout) } + if c.Snapshot.CompressionDefault.Level < 1 { + return fmt.Errorf("snapshot.compression_default.level must be >= 1, got %d", c.Snapshot.CompressionDefault.Level) + } + switch strings.ToLower(c.Snapshot.CompressionDefault.Algorithm) { + case "", "zstd", "lz4": + default: + return fmt.Errorf("snapshot.compression_default.algorithm must be one of zstd or lz4, got %q", c.Snapshot.CompressionDefault.Algorithm) + } return nil } diff --git a/integration/systemd_test.go b/integration/systemd_test.go index 986ebb11..7ae458bf 100644 --- a/integration/systemd_test.go +++ b/integration/systemd_test.go @@ -68,7 +68,7 @@ func TestSystemdMode(t *testing.T) { MaxMemoryPerInstance: 0, } - instanceManager := instances.NewManager(p, imageManager, systemManager, networkManager, deviceManager, volumeManager, limits, "", nil, nil) + instanceManager := instances.NewManager(p, imageManager, systemManager, networkManager, deviceManager, volumeManager, limits, "", instances.SnapshotPolicy{}, nil, nil) // Cleanup any orphaned instances t.Cleanup(func() { diff --git a/integration/vgpu_test.go b/integration/vgpu_test.go index 17a92ad1..9fd608c5 100644 --- a/integration/vgpu_test.go +++ b/integration/vgpu_test.go @@ -77,7 +77,7 @@ func TestVGPU(t *testing.T) { MaxMemoryPerInstance: 0, } - instanceManager := instances.NewManager(p, imageManager, systemManager, networkManager, deviceManager, volumeManager, limits, "", nil, nil) + instanceManager := instances.NewManager(p, imageManager, systemManager, networkManager, deviceManager, volumeManager, limits, "", instances.SnapshotPolicy{}, nil, nil) // Track instance ID for cleanup var instanceID string diff --git a/lib/builds/manager_test.go b/lib/builds/manager_test.go index a3765d5f..1a68ded3 100644 --- a/lib/builds/manager_test.go +++ b/lib/builds/manager_test.go @@ -108,7 +108,7 @@ func (m *mockInstanceManager) ForkSnapshot(ctx context.Context, snapshotID strin return nil, instances.ErrNotFound } -func (m *mockInstanceManager) StandbyInstance(ctx context.Context, id string) (*instances.Instance, error) { +func (m *mockInstanceManager) StandbyInstance(ctx context.Context, id string, req instances.StandbyInstanceRequest) (*instances.Instance, error) { return nil, nil } diff --git a/lib/devices/gpu_e2e_test.go b/lib/devices/gpu_e2e_test.go index 86708446..a7fe6390 100644 --- a/lib/devices/gpu_e2e_test.go +++ b/lib/devices/gpu_e2e_test.go @@ -79,7 +79,7 @@ func TestGPUPassthrough(t *testing.T) { limits := instances.ResourceLimits{ MaxOverlaySize: 100 * 1024 * 1024 * 1024, // 100GB } - instanceMgr := instances.NewManager(p, imageMgr, systemMgr, networkMgr, deviceMgr, volumeMgr, limits, "", nil, nil) + instanceMgr := instances.NewManager(p, imageMgr, systemMgr, networkMgr, deviceMgr, volumeMgr, limits, "", instances.SnapshotPolicy{}, nil, nil) // Step 1: Discover available GPUs t.Log("Step 1: Discovering available GPUs...") diff --git a/lib/devices/gpu_inference_test.go b/lib/devices/gpu_inference_test.go index c99d1b39..f521158e 100644 --- a/lib/devices/gpu_inference_test.go +++ b/lib/devices/gpu_inference_test.go @@ -116,7 +116,7 @@ func TestGPUInference(t *testing.T) { limits := instances.ResourceLimits{ MaxOverlaySize: 100 * 1024 * 1024 * 1024, } - instanceMgr := instances.NewManager(p, imageMgr, systemMgr, networkMgr, deviceMgr, volumeMgr, limits, "", nil, nil) + instanceMgr := instances.NewManager(p, imageMgr, systemMgr, networkMgr, deviceMgr, volumeMgr, limits, "", instances.SnapshotPolicy{}, nil, nil) // Step 1: Build custom CUDA+Ollama image t.Log("Step 1: Building custom CUDA+Ollama Docker image...") diff --git a/lib/devices/gpu_module_test.go b/lib/devices/gpu_module_test.go index cd9f1b76..f39d4690 100644 --- a/lib/devices/gpu_module_test.go +++ b/lib/devices/gpu_module_test.go @@ -80,7 +80,7 @@ func TestNVIDIAModuleLoading(t *testing.T) { deviceMgr := devices.NewManager(p) volumeMgr := volumes.NewManager(p, 10*1024*1024*1024, nil) limits := instances.ResourceLimits{MaxOverlaySize: 10 * 1024 * 1024 * 1024} - instanceMgr := instances.NewManager(p, imageMgr, systemMgr, networkMgr, deviceMgr, volumeMgr, limits, "", nil, nil) + instanceMgr := instances.NewManager(p, imageMgr, systemMgr, networkMgr, deviceMgr, volumeMgr, limits, "", instances.SnapshotPolicy{}, nil, nil) // Step 1: Find an NVIDIA GPU t.Log("Step 1: Discovering available GPUs...") @@ -326,7 +326,7 @@ func TestNVMLDetection(t *testing.T) { deviceMgr := devices.NewManager(p) volumeMgr := volumes.NewManager(p, 10*1024*1024*1024, nil) limits := instances.ResourceLimits{MaxOverlaySize: 10 * 1024 * 1024 * 1024} - instanceMgr := instances.NewManager(p, imageMgr, systemMgr, networkMgr, deviceMgr, volumeMgr, limits, "", nil, nil) + instanceMgr := instances.NewManager(p, imageMgr, systemMgr, networkMgr, deviceMgr, volumeMgr, limits, "", instances.SnapshotPolicy{}, nil, nil) // Step 1: Check if ollama-cuda:test image exists in Docker t.Log("Step 1: Checking for ollama-cuda:test Docker image...") diff --git a/lib/instances/create.go b/lib/instances/create.go index 1f762ff3..dfc63cdc 100644 --- a/lib/instances/create.go +++ b/lib/instances/create.go @@ -313,6 +313,7 @@ func (m *manager) createInstance( Cmd: req.Cmd, SkipKernelHeaders: req.SkipKernelHeaders, SkipGuestAgent: req.SkipGuestAgent, + SnapshotPolicy: cloneSnapshotPolicy(req.SnapshotPolicy), } // 12. Ensure directories @@ -470,6 +471,11 @@ func validateCreateRequest(req CreateInstanceRequest) error { if err := tags.Validate(req.Metadata); err != nil { return fmt.Errorf("%w: %v", ErrInvalidRequest, err) } + if req.SnapshotPolicy != nil && req.SnapshotPolicy.Compression != nil { + if _, err := normalizeCompressionConfig(req.SnapshotPolicy.Compression); err != nil { + return err + } + } // Validate volume attachments if err := validateVolumeAttachments(req.Volumes); err != nil { diff --git a/lib/instances/firecracker_test.go b/lib/instances/firecracker_test.go index 940c1786..4ecefd29 100644 --- a/lib/instances/firecracker_test.go +++ b/lib/instances/firecracker_test.go @@ -46,7 +46,7 @@ func setupTestManagerForFirecracker(t *testing.T) (*manager, string) { MaxVcpusPerInstance: 0, MaxMemoryPerInstance: 0, } - mgr := NewManager(p, imageManager, systemManager, networkManager, deviceManager, volumeManager, limits, hypervisor.TypeFirecracker, nil, nil).(*manager) + mgr := NewManager(p, imageManager, systemManager, networkManager, deviceManager, volumeManager, limits, hypervisor.TypeFirecracker, SnapshotPolicy{}, nil, nil).(*manager) resourceMgr := resources.NewManager(cfg, p) resourceMgr.SetInstanceLister(mgr) @@ -114,7 +114,7 @@ func TestFirecrackerStandbyAndRestore(t *testing.T) { require.NoError(t, err) assert.Equal(t, StateRunning, inst.State) - inst, err = mgr.StandbyInstance(ctx, inst.Id) + inst, err = mgr.StandbyInstance(ctx, inst.Id, StandbyInstanceRequest{}) require.NoError(t, err) assert.Equal(t, StateStandby, inst.State) assert.True(t, inst.HasSnapshot) @@ -164,7 +164,7 @@ func TestFirecrackerStopClearsStaleSnapshot(t *testing.T) { require.Equal(t, StateRunning, inst.State) // Establish a realistic standby/restore lifecycle first. - inst, err = mgr.StandbyInstance(ctx, inst.Id) + inst, err = mgr.StandbyInstance(ctx, inst.Id, StandbyInstanceRequest{}) require.NoError(t, err) require.Equal(t, StateStandby, inst.State) require.True(t, inst.HasSnapshot) @@ -264,7 +264,7 @@ func TestFirecrackerNetworkLifecycle(t *testing.T) { require.Equal(t, 0, exitCode) require.Contains(t, output, "Connection successful") - inst, err = mgr.StandbyInstance(ctx, inst.Id) + inst, err = mgr.StandbyInstance(ctx, inst.Id, StandbyInstanceRequest{}) require.NoError(t, err) assert.Equal(t, StateStandby, inst.State) assert.True(t, inst.HasSnapshot) diff --git a/lib/instances/fork.go b/lib/instances/fork.go index 60d3a5cf..84d319fd 100644 --- a/lib/instances/fork.go +++ b/lib/instances/fork.go @@ -56,7 +56,7 @@ func (m *manager) forkInstance(ctx context.Context, id string, req ForkInstanceR log.InfoContext(ctx, "fork from running requested; transitioning source to standby", "source_instance_id", id, "hypervisor", source.HypervisorType) - if _, err := m.standbyInstance(ctx, id); err != nil { + if _, err := m.standbyInstance(ctx, id, StandbyInstanceRequest{}, true); err != nil { return nil, "", fmt.Errorf("standby source instance: %w", err) } @@ -421,7 +421,7 @@ func (m *manager) applyForkTargetState(ctx context.Context, forkID string, targe if _, err := m.startInstance(ctx, forkID, StartInstanceRequest{}); err != nil { return nil, fmt.Errorf("start forked instance for standby transition: %w", err) } - return returnWithReadiness(m.standbyInstance(ctx, forkID)) + return returnWithReadiness(m.standbyInstance(ctx, forkID, StandbyInstanceRequest{}, false)) } case StateStandby: switch target { @@ -436,7 +436,7 @@ func (m *manager) applyForkTargetState(ctx context.Context, forkID string, targe case StateRunning: switch target { case StateStandby: - return returnWithReadiness(m.standbyInstance(ctx, forkID)) + return returnWithReadiness(m.standbyInstance(ctx, forkID, StandbyInstanceRequest{}, false)) case StateStopped: return returnWithReadiness(m.stopInstance(ctx, forkID)) } diff --git a/lib/instances/manager.go b/lib/instances/manager.go index 3b581e83..d31ff1cb 100644 --- a/lib/instances/manager.go +++ b/lib/instances/manager.go @@ -31,7 +31,7 @@ type Manager interface { DeleteSnapshot(ctx context.Context, snapshotID string) error ForkInstance(ctx context.Context, id string, req ForkInstanceRequest) (*Instance, error) ForkSnapshot(ctx context.Context, snapshotID string, req ForkSnapshotRequest) (*Instance, error) - StandbyInstance(ctx context.Context, id string) (*Instance, error) + StandbyInstance(ctx context.Context, id string, req StandbyInstanceRequest) (*Instance, error) RestoreInstance(ctx context.Context, id string) (*Instance, error) RestoreSnapshot(ctx context.Context, id string, snapshotID string, req RestoreSnapshotRequest) (*Instance, error) StopInstance(ctx context.Context, id string) (*Instance, error) @@ -80,6 +80,9 @@ type manager struct { instanceLocks sync.Map // map[string]*sync.RWMutex - per-instance locks hostTopology *HostTopology // Cached host CPU topology metrics *Metrics + snapshotDefaults SnapshotPolicy + compressionMu sync.Mutex + compressionJobs map[string]*compressionJob // Hypervisor support vmStarters map[hypervisor.Type]hypervisor.VMStarter @@ -92,7 +95,7 @@ var platformStarters = make(map[hypervisor.Type]hypervisor.VMStarter) // NewManager creates a new instances manager. // If meter is nil, metrics are disabled. // defaultHypervisor specifies which hypervisor to use when not specified in requests. -func NewManager(p *paths.Paths, imageManager images.Manager, systemManager system.Manager, networkManager network.Manager, deviceManager devices.Manager, volumeManager volumes.Manager, limits ResourceLimits, defaultHypervisor hypervisor.Type, meter metric.Meter, tracer trace.Tracer) Manager { +func NewManager(p *paths.Paths, imageManager images.Manager, systemManager system.Manager, networkManager network.Manager, deviceManager devices.Manager, volumeManager volumes.Manager, limits ResourceLimits, defaultHypervisor hypervisor.Type, snapshotDefaults SnapshotPolicy, meter metric.Meter, tracer trace.Tracer) Manager { // Validate and default the hypervisor type if defaultHypervisor == "" { defaultHypervisor = hypervisor.TypeCloudHypervisor @@ -116,6 +119,8 @@ func NewManager(p *paths.Paths, imageManager images.Manager, systemManager syste hostTopology: detectHostTopology(), // Detect and cache host topology vmStarters: vmStarters, defaultHypervisor: defaultHypervisor, + snapshotDefaults: snapshotDefaults, + compressionJobs: make(map[string]*compressionJob), } // Initialize metrics if meter is provided @@ -241,11 +246,11 @@ func (m *manager) ForkSnapshot(ctx context.Context, snapshotID string, req ForkS } // StandbyInstance puts an instance in standby (pause, snapshot, delete VMM) -func (m *manager) StandbyInstance(ctx context.Context, id string) (*Instance, error) { +func (m *manager) StandbyInstance(ctx context.Context, id string, req StandbyInstanceRequest) (*Instance, error) { lock := m.getInstanceLock(id) lock.Lock() defer lock.Unlock() - return m.standbyInstance(ctx, id) + return m.standbyInstance(ctx, id, req, false) } // RestoreInstance restores an instance from standby diff --git a/lib/instances/manager_darwin_test.go b/lib/instances/manager_darwin_test.go index d38c24d8..5163543b 100644 --- a/lib/instances/manager_darwin_test.go +++ b/lib/instances/manager_darwin_test.go @@ -57,7 +57,7 @@ func setupVZTestManager(t *testing.T) (*manager, string) { MaxVcpusPerInstance: 0, MaxMemoryPerInstance: 0, } - mgr := NewManager(p, imageManager, systemManager, networkManager, deviceManager, volumeManager, limits, "", nil, nil).(*manager) + mgr := NewManager(p, imageManager, systemManager, networkManager, deviceManager, volumeManager, limits, "", SnapshotPolicy{}, nil, nil).(*manager) resourceMgr := resources.NewManager(cfg, p) resourceMgr.SetInstanceLister(mgr) @@ -470,7 +470,7 @@ func TestVZStandbyAndRestore(t *testing.T) { // Standby instance t.Log("Putting instance in standby...") - inst, err = mgr.StandbyInstance(ctx, inst.Id) + inst, err = mgr.StandbyInstance(ctx, inst.Id, StandbyInstanceRequest{}) require.NoError(t, err) assert.Equal(t, StateStandby, inst.State) assert.True(t, inst.HasSnapshot) diff --git a/lib/instances/manager_test.go b/lib/instances/manager_test.go index 641cd6a9..21ab1a28 100644 --- a/lib/instances/manager_test.go +++ b/lib/instances/manager_test.go @@ -55,7 +55,7 @@ func setupTestManager(t *testing.T) (*manager, string) { MaxVcpusPerInstance: 0, // unlimited MaxMemoryPerInstance: 0, // unlimited } - mgr := NewManager(p, imageManager, systemManager, networkManager, deviceManager, volumeManager, limits, "", nil, nil).(*manager) + mgr := NewManager(p, imageManager, systemManager, networkManager, deviceManager, volumeManager, limits, "", SnapshotPolicy{}, nil, nil).(*manager) // Set up resource validation using the real ResourceManager resourceMgr := resources.NewManager(cfg, p) @@ -1186,7 +1186,7 @@ func TestStorageOperations(t *testing.T) { MaxVcpusPerInstance: 0, // unlimited MaxMemoryPerInstance: 0, // unlimited } - manager := NewManager(p, imageManager, systemManager, networkManager, deviceManager, volumeManager, limits, "", nil, nil).(*manager) + manager := NewManager(p, imageManager, systemManager, networkManager, deviceManager, volumeManager, limits, "", SnapshotPolicy{}, nil, nil).(*manager) // Test metadata doesn't exist initially _, err := manager.loadMetadata("nonexistent") @@ -1304,7 +1304,7 @@ func TestStandbyAndRestore(t *testing.T) { // Standby instance t.Log("Standing by instance...") - inst, err = manager.StandbyInstance(ctx, inst.Id) + inst, err = manager.StandbyInstance(ctx, inst.Id, StandbyInstanceRequest{}) require.NoError(t, err) assert.Equal(t, StateStandby, inst.State) assert.True(t, inst.HasSnapshot) diff --git a/lib/instances/network_test.go b/lib/instances/network_test.go index af7e25c4..68a43ae7 100644 --- a/lib/instances/network_test.go +++ b/lib/instances/network_test.go @@ -130,7 +130,7 @@ func TestCreateInstanceWithNetwork(t *testing.T) { // Standby instance t.Log("Standing by instance...") - inst, err = manager.StandbyInstance(ctx, inst.Id) + inst, err = manager.StandbyInstance(ctx, inst.Id, StandbyInstanceRequest{}) require.NoError(t, err) assert.Equal(t, StateStandby, inst.State) assert.True(t, inst.HasSnapshot) diff --git a/lib/instances/qemu_test.go b/lib/instances/qemu_test.go index ab5aa853..2fbb5392 100644 --- a/lib/instances/qemu_test.go +++ b/lib/instances/qemu_test.go @@ -53,7 +53,7 @@ func setupTestManagerForQEMU(t *testing.T) (*manager, string) { MaxVcpusPerInstance: 0, // unlimited MaxMemoryPerInstance: 0, // unlimited } - mgr := NewManager(p, imageManager, systemManager, networkManager, deviceManager, volumeManager, limits, hypervisor.TypeQEMU, nil, nil).(*manager) + mgr := NewManager(p, imageManager, systemManager, networkManager, deviceManager, volumeManager, limits, hypervisor.TypeQEMU, SnapshotPolicy{}, nil, nil).(*manager) // Set up resource validation using the real ResourceManager resourceMgr := resources.NewManager(cfg, p) @@ -826,7 +826,7 @@ func TestQEMUStandbyAndRestore(t *testing.T) { // Standby instance t.Log("Standing by instance...") - inst, err = manager.StandbyInstance(ctx, inst.Id) + inst, err = manager.StandbyInstance(ctx, inst.Id, StandbyInstanceRequest{}) require.NoError(t, err) assert.Equal(t, StateStandby, inst.State) assert.True(t, inst.HasSnapshot) diff --git a/lib/instances/resource_limits_test.go b/lib/instances/resource_limits_test.go index d34b9c45..32cad66c 100644 --- a/lib/instances/resource_limits_test.go +++ b/lib/instances/resource_limits_test.go @@ -176,7 +176,7 @@ func createTestManager(t *testing.T, limits ResourceLimits) *manager { deviceMgr := devices.NewManager(p) volumeMgr := volumes.NewManager(p, 0, nil) - return NewManager(p, imageMgr, systemMgr, networkMgr, deviceMgr, volumeMgr, limits, "", nil, nil).(*manager) + return NewManager(p, imageMgr, systemMgr, networkMgr, deviceMgr, volumeMgr, limits, "", SnapshotPolicy{}, nil, nil).(*manager) } func TestResourceLimits_StructValues(t *testing.T) { diff --git a/lib/instances/restore.go b/lib/instances/restore.go index 1ff09c55..43496e6c 100644 --- a/lib/instances/restore.go +++ b/lib/instances/restore.go @@ -69,6 +69,9 @@ func (m *manager) restoreInstance( // 3. Get snapshot directory snapshotDir := m.paths.InstanceSnapshotLatest(id) + if err := m.ensureSnapshotMemoryReady(ctx, snapshotDir, m.snapshotJobKeyForInstance(id)); err != nil { + return nil, fmt.Errorf("prepare standby snapshot memory: %w", err) + } starter, err := m.getVMStarter(stored.HypervisorType) if err != nil { return nil, fmt.Errorf("get vm starter: %w", err) diff --git a/lib/instances/snapshot.go b/lib/instances/snapshot.go index 4df46078..4839d888 100644 --- a/lib/instances/snapshot.go +++ b/lib/instances/snapshot.go @@ -91,7 +91,7 @@ func (m *manager) createSnapshot(ctx context.Context, id string, req CreateSnaps if err := ensureGuestAgentReadyForForkPhase(ctx, &inst.StoredMetadata, "before running snapshot"); err != nil { return nil, err } - if _, err := m.standbyInstance(ctx, id); err != nil { + if _, err := m.standbyInstance(ctx, id, StandbyInstanceRequest{}, true); err != nil { return nil, fmt.Errorf("standby source instance: %w", err) } restoreSource = true @@ -138,9 +138,27 @@ func (m *manager) createSnapshot(ctx context.Context, id string, req CreateSnaps return nil, err } rec.Snapshot.SizeBytes = sizeBytes + rec.Snapshot.CompressionState = snapshotstore.SnapshotCompressionStateNone + effectiveCompression, err := m.resolveSnapshotCompressionPolicy(stored, req.Compression) + if err != nil { + return nil, err + } + if effectiveCompression.Enabled { + rec.Snapshot.Compression = cloneCompressionConfig(&effectiveCompression) + rec.Snapshot.CompressionState = snapshotstore.SnapshotCompressionStateCompressing + } if err := m.saveSnapshotRecord(rec); err != nil { return nil, err } + if effectiveCompression.Enabled { + m.startCompressionJob(ctx, compressionTarget{ + Key: m.snapshotJobKeyForSnapshot(snapshotID), + OwnerID: stored.Id, + SnapshotID: snapshotID, + SnapshotDir: snapshotGuestDir, + Policy: effectiveCompression, + }) + } cu.Release() log.InfoContext(ctx, "snapshot created", "instance_id", id, "snapshot_id", snapshotID, "kind", req.Kind) return &rec.Snapshot, nil @@ -170,6 +188,7 @@ func (m *manager) createSnapshot(ctx context.Context, id string, req CreateSnaps return nil, err } rec.Snapshot.SizeBytes = sizeBytes + rec.Snapshot.CompressionState = snapshotstore.SnapshotCompressionStateNone if err := m.saveSnapshotRecord(rec); err != nil { return nil, err } @@ -221,6 +240,9 @@ func (m *manager) restoreSnapshot(ctx context.Context, id string, snapshotID str return nil, err } + m.cancelCompressionJob(m.snapshotJobKeyForSnapshot(snapshotID)) + m.waitCompressionJob(m.snapshotJobKeyForSnapshot(snapshotID), 2*time.Second) + if err := m.replaceInstanceWithSnapshotPayload(snapshotID, id); err != nil { return nil, err } @@ -335,6 +357,9 @@ func (m *manager) forkSnapshot(ctx context.Context, snapshotID string, req ForkS }) defer cu.Clean() + m.cancelCompressionJob(m.snapshotJobKeyForSnapshot(snapshotID)) + m.waitCompressionJob(m.snapshotJobKeyForSnapshot(snapshotID), 2*time.Second) + if err := forkvm.CopyGuestDirectory(m.paths.SnapshotGuestDir(snapshotID), dstDir); err != nil { if errors.Is(err, forkvm.ErrSparseCopyUnsupported) { return nil, fmt.Errorf("fork from snapshot requires sparse-capable filesystem (SEEK_DATA/SEEK_HOLE unsupported): %w", err) @@ -461,6 +486,12 @@ func validateCreateSnapshotRequest(req CreateSnapshotRequest) error { if req.Kind != SnapshotKindStandby && req.Kind != SnapshotKindStopped { return fmt.Errorf("%w: kind must be one of %s, %s", ErrInvalidRequest, SnapshotKindStandby, SnapshotKindStopped) } + if req.Kind == SnapshotKindStopped && req.Compression != nil && req.Compression.Enabled { + return fmt.Errorf("%w: compression is only supported for standby snapshots", ErrInvalidRequest) + } + if _, err := normalizeCompressionConfig(req.Compression); err != nil { + return err + } if err := tags.Validate(req.Metadata); err != nil { return fmt.Errorf("%w: %v", ErrInvalidRequest, err) } diff --git a/lib/instances/snapshot_compression.go b/lib/instances/snapshot_compression.go new file mode 100644 index 00000000..8974ede5 --- /dev/null +++ b/lib/instances/snapshot_compression.go @@ -0,0 +1,410 @@ +package instances + +import ( + "context" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "strings" + "time" + + "github.com/kernel/hypeman/lib/logger" + snapshotstore "github.com/kernel/hypeman/lib/snapshot" + "github.com/klauspost/compress/zstd" + "github.com/pierrec/lz4/v4" +) + +const ( + defaultSnapshotCompressionZstdLevel = 1 + maxSnapshotCompressionZstdLevel = 19 +) + +type compressionJob struct { + cancel context.CancelFunc + done chan struct{} +} + +type compressionTarget struct { + Key string + OwnerID string + SnapshotID string + SnapshotDir string + Policy snapshotstore.SnapshotCompressionConfig +} + +func cloneCompressionConfig(cfg *snapshotstore.SnapshotCompressionConfig) *snapshotstore.SnapshotCompressionConfig { + if cfg == nil { + return nil + } + cloned := *cfg + if cfg.Level != nil { + v := *cfg.Level + cloned.Level = &v + } + return &cloned +} + +func cloneSnapshotPolicy(policy *SnapshotPolicy) *SnapshotPolicy { + if policy == nil { + return nil + } + return &SnapshotPolicy{ + Compression: cloneCompressionConfig(policy.Compression), + } +} + +func normalizeCompressionConfig(cfg *snapshotstore.SnapshotCompressionConfig) (snapshotstore.SnapshotCompressionConfig, error) { + if cfg == nil || !cfg.Enabled { + return snapshotstore.SnapshotCompressionConfig{Enabled: false}, nil + } + + normalized := snapshotstore.SnapshotCompressionConfig{ + Enabled: true, + Algorithm: cfg.Algorithm, + } + switch normalized.Algorithm { + case "": + normalized.Algorithm = snapshotstore.SnapshotCompressionAlgorithmZstd + case snapshotstore.SnapshotCompressionAlgorithmZstd, snapshotstore.SnapshotCompressionAlgorithmLz4: + default: + return snapshotstore.SnapshotCompressionConfig{}, fmt.Errorf("%w: unsupported compression algorithm %q", ErrInvalidRequest, cfg.Algorithm) + } + + switch normalized.Algorithm { + case snapshotstore.SnapshotCompressionAlgorithmZstd: + level := defaultSnapshotCompressionZstdLevel + if cfg.Level != nil { + level = *cfg.Level + } + if level < 1 || level > maxSnapshotCompressionZstdLevel { + return snapshotstore.SnapshotCompressionConfig{}, fmt.Errorf("%w: invalid zstd level %d (must be between 1 and %d)", ErrInvalidRequest, level, maxSnapshotCompressionZstdLevel) + } + normalized.Level = &level + case snapshotstore.SnapshotCompressionAlgorithmLz4: + if cfg.Level != nil { + return snapshotstore.SnapshotCompressionConfig{}, fmt.Errorf("%w: lz4 does not support level", ErrInvalidRequest) + } + } + return normalized, nil +} + +func (m *manager) resolveSnapshotCompressionPolicy(stored *StoredMetadata, override *snapshotstore.SnapshotCompressionConfig) (snapshotstore.SnapshotCompressionConfig, error) { + if override != nil { + return normalizeCompressionConfig(override) + } + if stored != nil && stored.SnapshotPolicy != nil && stored.SnapshotPolicy.Compression != nil { + return normalizeCompressionConfig(stored.SnapshotPolicy.Compression) + } + if m.snapshotDefaults.Compression != nil { + return normalizeCompressionConfig(m.snapshotDefaults.Compression) + } + return snapshotstore.SnapshotCompressionConfig{Enabled: false}, nil +} + +func (m *manager) snapshotJobKeyForInstance(instanceID string) string { + return "instance:" + instanceID +} + +func (m *manager) snapshotJobKeyForSnapshot(snapshotID string) string { + return "snapshot:" + snapshotID +} + +func (m *manager) startCompressionJob(ctx context.Context, target compressionTarget) { + if target.Key == "" || !target.Policy.Enabled { + return + } + + m.compressionMu.Lock() + if _, exists := m.compressionJobs[target.Key]; exists { + m.compressionMu.Unlock() + return + } + jobCtx, cancel := context.WithCancel(context.Background()) + job := &compressionJob{ + cancel: cancel, + done: make(chan struct{}), + } + m.compressionJobs[target.Key] = job + m.compressionMu.Unlock() + + go func() { + defer func() { + m.compressionMu.Lock() + delete(m.compressionJobs, target.Key) + m.compressionMu.Unlock() + close(job.done) + }() + + log := logger.FromContext(ctx) + rawPath, ok := findRawSnapshotMemoryFile(target.SnapshotDir) + if !ok { + if _, _, found := findCompressedSnapshotMemoryFile(target.SnapshotDir); found && target.SnapshotID != "" { + _ = m.updateSnapshotCompressionMetadata(target.SnapshotID, snapshotstore.SnapshotCompressionStateCompressed, "", &target.Policy, nil, nil) + } + return + } + + uncompressedSize, compressedSize, err := compressSnapshotMemoryFile(jobCtx, rawPath, target.Policy) + if err != nil { + if errors.Is(err, context.Canceled) { + if target.SnapshotID != "" { + _ = m.updateSnapshotCompressionMetadata(target.SnapshotID, snapshotstore.SnapshotCompressionStateNone, "", nil, nil, nil) + } + return + } + if target.SnapshotID != "" { + _ = m.updateSnapshotCompressionMetadata(target.SnapshotID, snapshotstore.SnapshotCompressionStateError, err.Error(), &target.Policy, nil, nil) + } + log.WarnContext(ctx, "snapshot compression failed", "snapshot_dir", target.SnapshotDir, "error", err) + return + } + + if target.SnapshotID != "" { + _ = m.updateSnapshotCompressionMetadata(target.SnapshotID, snapshotstore.SnapshotCompressionStateCompressed, "", &target.Policy, &compressedSize, &uncompressedSize) + } + }() +} + +func (m *manager) cancelCompressionJob(key string) { + m.compressionMu.Lock() + job := m.compressionJobs[key] + m.compressionMu.Unlock() + if job != nil { + job.cancel() + } +} + +func (m *manager) waitCompressionJob(key string, timeout time.Duration) { + m.compressionMu.Lock() + job := m.compressionJobs[key] + m.compressionMu.Unlock() + if job == nil { + return + } + + timer := time.NewTimer(timeout) + defer timer.Stop() + select { + case <-job.done: + case <-timer.C: + } +} + +func (m *manager) ensureSnapshotMemoryReady(ctx context.Context, snapshotDir, jobKey string) error { + if jobKey != "" { + m.cancelCompressionJob(jobKey) + m.waitCompressionJob(jobKey, 2*time.Second) + } + + if _, ok := findRawSnapshotMemoryFile(snapshotDir); ok { + return nil + } + compressedPath, algorithm, ok := findCompressedSnapshotMemoryFile(snapshotDir) + if !ok { + return nil + } + return decompressSnapshotMemoryFile(ctx, compressedPath, algorithm) +} + +func (m *manager) updateSnapshotCompressionMetadata(snapshotID, state, compressionError string, cfg *snapshotstore.SnapshotCompressionConfig, compressedSize, uncompressedSize *int64) error { + rec, err := m.loadSnapshotRecord(snapshotID) + if err != nil { + return err + } + rec.Snapshot.CompressionState = state + rec.Snapshot.CompressionError = compressionError + rec.Snapshot.Compression = cloneCompressionConfig(cfg) + rec.Snapshot.CompressedSizeBytes = compressedSize + rec.Snapshot.UncompressedSizeBytes = uncompressedSize + + if state == snapshotstore.SnapshotCompressionStateCompressed { + sizeBytes, sizeErr := snapshotstore.DirectoryFileSize(m.paths.SnapshotGuestDir(snapshotID)) + if sizeErr == nil { + rec.Snapshot.SizeBytes = sizeBytes + } + } + return m.saveSnapshotRecord(rec) +} + +func findRawSnapshotMemoryFile(snapshotDir string) (string, bool) { + for _, candidate := range snapshotMemoryFileCandidates(snapshotDir) { + if st, err := os.Stat(candidate); err == nil && st.Mode().IsRegular() { + return candidate, true + } + } + return "", false +} + +func findCompressedSnapshotMemoryFile(snapshotDir string) (string, snapshotstore.SnapshotCompressionAlgorithm, bool) { + for _, raw := range snapshotMemoryFileCandidates(snapshotDir) { + zstdPath := raw + ".zst" + if st, err := os.Stat(zstdPath); err == nil && st.Mode().IsRegular() { + return zstdPath, snapshotstore.SnapshotCompressionAlgorithmZstd, true + } + lz4Path := raw + ".lz4" + if st, err := os.Stat(lz4Path); err == nil && st.Mode().IsRegular() { + return lz4Path, snapshotstore.SnapshotCompressionAlgorithmLz4, true + } + } + return "", "", false +} + +func snapshotMemoryFileCandidates(snapshotDir string) []string { + return []string{ + filepath.Join(snapshotDir, "memory-ranges"), + filepath.Join(snapshotDir, "memory"), + filepath.Join(snapshotDir, "snapshots", "snapshot-latest", "memory-ranges"), + filepath.Join(snapshotDir, "snapshots", "snapshot-latest", "memory"), + } +} + +func compressSnapshotMemoryFile(ctx context.Context, rawPath string, cfg snapshotstore.SnapshotCompressionConfig) (int64, int64, error) { + rawInfo, err := os.Stat(rawPath) + if err != nil { + return 0, 0, fmt.Errorf("stat raw memory snapshot: %w", err) + } + uncompressedSize := rawInfo.Size() + + compressedPath := compressedPathFor(rawPath, cfg.Algorithm) + tmpPath := compressedPath + ".tmp" + _ = os.Remove(tmpPath) + _ = os.Remove(compressedPath) + + if err := runCompression(ctx, rawPath, tmpPath, cfg); err != nil { + _ = os.Remove(tmpPath) + return 0, 0, err + } + if err := os.Rename(tmpPath, compressedPath); err != nil { + _ = os.Remove(tmpPath) + return 0, 0, fmt.Errorf("finalize compressed snapshot: %w", err) + } + + compressedInfo, err := os.Stat(compressedPath) + if err != nil { + return 0, 0, fmt.Errorf("stat compressed snapshot: %w", err) + } + if err := os.Remove(rawPath); err != nil { + return 0, 0, fmt.Errorf("remove raw memory snapshot: %w", err) + } + return uncompressedSize, compressedInfo.Size(), nil +} + +func runCompression(ctx context.Context, srcPath, dstPath string, cfg snapshotstore.SnapshotCompressionConfig) error { + src, err := os.Open(srcPath) + if err != nil { + return fmt.Errorf("open source snapshot: %w", err) + } + defer src.Close() + + dst, err := os.Create(dstPath) + if err != nil { + return fmt.Errorf("create compressed snapshot: %w", err) + } + defer dst.Close() + + switch cfg.Algorithm { + case snapshotstore.SnapshotCompressionAlgorithmZstd: + level := defaultSnapshotCompressionZstdLevel + if cfg.Level != nil { + level = *cfg.Level + } + enc, err := zstd.NewWriter(dst, zstd.WithEncoderLevel(zstd.EncoderLevelFromZstd(level))) + if err != nil { + return fmt.Errorf("create zstd encoder: %w", err) + } + if err := copyWithContext(ctx, enc, src); err != nil { + _ = enc.Close() + return err + } + if err := enc.Close(); err != nil { + return fmt.Errorf("close zstd encoder: %w", err) + } + case snapshotstore.SnapshotCompressionAlgorithmLz4: + enc := lz4.NewWriter(dst) + if err := copyWithContext(ctx, enc, src); err != nil { + _ = enc.Close() + return err + } + if err := enc.Close(); err != nil { + return fmt.Errorf("close lz4 encoder: %w", err) + } + default: + return fmt.Errorf("%w: unsupported compression algorithm %q", ErrInvalidRequest, cfg.Algorithm) + } + return nil +} + +func decompressSnapshotMemoryFile(ctx context.Context, compressedPath string, algorithm snapshotstore.SnapshotCompressionAlgorithm) error { + rawPath := strings.TrimSuffix(strings.TrimSuffix(compressedPath, ".zst"), ".lz4") + tmpRawPath := rawPath + ".tmp" + _ = os.Remove(tmpRawPath) + + src, err := os.Open(compressedPath) + if err != nil { + return fmt.Errorf("open compressed snapshot: %w", err) + } + defer src.Close() + + dst, err := os.Create(tmpRawPath) + if err != nil { + return fmt.Errorf("create decompressed snapshot file: %w", err) + } + defer dst.Close() + + var reader io.Reader + switch algorithm { + case snapshotstore.SnapshotCompressionAlgorithmZstd: + dec, err := zstd.NewReader(src) + if err != nil { + return fmt.Errorf("create zstd decoder: %w", err) + } + defer dec.Close() + reader = dec + case snapshotstore.SnapshotCompressionAlgorithmLz4: + reader = lz4.NewReader(src) + default: + return fmt.Errorf("%w: unsupported compression algorithm %q", ErrInvalidRequest, algorithm) + } + + if err := copyWithContext(ctx, dst, reader); err != nil { + _ = os.Remove(tmpRawPath) + return err + } + if err := os.Rename(tmpRawPath, rawPath); err != nil { + _ = os.Remove(tmpRawPath) + return fmt.Errorf("finalize decompressed snapshot: %w", err) + } + return nil +} + +func compressedPathFor(rawPath string, algorithm snapshotstore.SnapshotCompressionAlgorithm) string { + switch algorithm { + case snapshotstore.SnapshotCompressionAlgorithmLz4: + return rawPath + ".lz4" + default: + return rawPath + ".zst" + } +} + +func copyWithContext(ctx context.Context, dst io.Writer, src io.Reader) error { + buf := make([]byte, 1024*1024) + for { + if err := ctx.Err(); err != nil { + return err + } + n, readErr := src.Read(buf) + if n > 0 { + if _, writeErr := dst.Write(buf[:n]); writeErr != nil { + return fmt.Errorf("write compressed stream: %w", writeErr) + } + } + if readErr != nil { + if errors.Is(readErr, io.EOF) { + return nil + } + return fmt.Errorf("read snapshot stream: %w", readErr) + } + } +} diff --git a/lib/instances/snapshot_compression_test.go b/lib/instances/snapshot_compression_test.go new file mode 100644 index 00000000..8c04c039 --- /dev/null +++ b/lib/instances/snapshot_compression_test.go @@ -0,0 +1,118 @@ +package instances + +import ( + "errors" + "testing" + + snapshotstore "github.com/kernel/hypeman/lib/snapshot" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestNormalizeCompressionConfig(t *testing.T) { + t.Parallel() + + cfg, err := normalizeCompressionConfig(nil) + require.NoError(t, err) + assert.False(t, cfg.Enabled) + + cfg, err = normalizeCompressionConfig(&snapshotstore.SnapshotCompressionConfig{ + Enabled: true, + }) + require.NoError(t, err) + assert.True(t, cfg.Enabled) + assert.Equal(t, snapshotstore.SnapshotCompressionAlgorithmZstd, cfg.Algorithm) + require.NotNil(t, cfg.Level) + assert.Equal(t, 1, *cfg.Level) + + _, err = normalizeCompressionConfig(&snapshotstore.SnapshotCompressionConfig{ + Enabled: true, + Algorithm: snapshotstore.SnapshotCompressionAlgorithmZstd, + Level: intPtr(25), + }) + require.Error(t, err) + assert.True(t, errors.Is(err, ErrInvalidRequest)) + + _, err = normalizeCompressionConfig(&snapshotstore.SnapshotCompressionConfig{ + Enabled: true, + Algorithm: snapshotstore.SnapshotCompressionAlgorithmLz4, + Level: intPtr(1), + }) + require.Error(t, err) + assert.True(t, errors.Is(err, ErrInvalidRequest)) +} + +func TestResolveSnapshotCompressionPolicyPrecedence(t *testing.T) { + t.Parallel() + + m := &manager{ + snapshotDefaults: SnapshotPolicy{ + Compression: &snapshotstore.SnapshotCompressionConfig{ + Enabled: true, + Algorithm: snapshotstore.SnapshotCompressionAlgorithmZstd, + Level: intPtr(2), + }, + }, + } + + stored := &StoredMetadata{ + SnapshotPolicy: &SnapshotPolicy{ + Compression: &snapshotstore.SnapshotCompressionConfig{ + Enabled: true, + Algorithm: snapshotstore.SnapshotCompressionAlgorithmLz4, + }, + }, + } + + cfg, err := m.resolveSnapshotCompressionPolicy(stored, &snapshotstore.SnapshotCompressionConfig{ + Enabled: true, + Algorithm: snapshotstore.SnapshotCompressionAlgorithmZstd, + Level: intPtr(4), + }) + require.NoError(t, err) + assert.Equal(t, snapshotstore.SnapshotCompressionAlgorithmZstd, cfg.Algorithm) + require.NotNil(t, cfg.Level) + assert.Equal(t, 4, *cfg.Level) + + cfg, err = m.resolveSnapshotCompressionPolicy(stored, nil) + require.NoError(t, err) + assert.Equal(t, snapshotstore.SnapshotCompressionAlgorithmLz4, cfg.Algorithm) + assert.Nil(t, cfg.Level) + + cfg, err = m.resolveSnapshotCompressionPolicy(&StoredMetadata{}, nil) + require.NoError(t, err) + assert.Equal(t, snapshotstore.SnapshotCompressionAlgorithmZstd, cfg.Algorithm) + require.NotNil(t, cfg.Level) + assert.Equal(t, 2, *cfg.Level) +} + +func TestValidateCreateRequestSnapshotPolicy(t *testing.T) { + t.Parallel() + + err := validateCreateRequest(CreateInstanceRequest{ + Name: "compression-test", + Image: "docker.io/library/alpine:latest", + SnapshotPolicy: &SnapshotPolicy{ + Compression: &snapshotstore.SnapshotCompressionConfig{ + Enabled: true, + Algorithm: snapshotstore.SnapshotCompressionAlgorithmZstd, + Level: intPtr(0), + }, + }, + }) + require.Error(t, err) + assert.True(t, errors.Is(err, ErrInvalidRequest)) +} + +func TestValidateCreateSnapshotRequestRejectsStoppedCompression(t *testing.T) { + t.Parallel() + + err := validateCreateSnapshotRequest(CreateSnapshotRequest{ + Kind: SnapshotKindStopped, + Compression: &snapshotstore.SnapshotCompressionConfig{ + Enabled: true, + }, + }) + require.Error(t, err) + assert.True(t, errors.Is(err, ErrInvalidRequest)) +} diff --git a/lib/instances/snapshot_integration_scenario_test.go b/lib/instances/snapshot_integration_scenario_test.go index 37e1ae7a..796ac8db 100644 --- a/lib/instances/snapshot_integration_scenario_test.go +++ b/lib/instances/snapshot_integration_scenario_test.go @@ -64,7 +64,7 @@ func runStandbySnapshotScenario(t *testing.T, mgr *manager, tmpDir string, cfg s } }) - _, err = mgr.StandbyInstance(ctx, sourceID) + _, err = mgr.StandbyInstance(ctx, sourceID, StandbyInstanceRequest{}) requireNoErr(err) snapshot, err := mgr.CreateSnapshot(ctx, sourceID, CreateSnapshotRequest{ diff --git a/lib/instances/standby.go b/lib/instances/standby.go index 1fca6dfb..79e39e54 100644 --- a/lib/instances/standby.go +++ b/lib/instances/standby.go @@ -11,6 +11,7 @@ import ( "github.com/kernel/hypeman/lib/hypervisor" "github.com/kernel/hypeman/lib/logger" "github.com/kernel/hypeman/lib/network" + snapshotstore "github.com/kernel/hypeman/lib/snapshot" "go.opentelemetry.io/otel/trace" ) @@ -20,6 +21,8 @@ func (m *manager) standbyInstance( ctx context.Context, id string, + req StandbyInstanceRequest, + skipCompression bool, ) (*Instance, error) { start := time.Now() log := logger.FromContext(ctx) @@ -55,6 +58,19 @@ func (m *manager) standbyInstance( return nil, fmt.Errorf("%w: standby is not supported for instances with vGPU attached (driver limitation)", ErrInvalidState) } + // Resolve/validate compression policy early so invalid request/config + // fails before any state transition side effects. + var compressionPolicy *snapshotstore.SnapshotCompressionConfig + if !skipCompression { + policy, err := m.resolveSnapshotCompressionPolicy(stored, req.Compression) + if err != nil { + return nil, err + } + if policy.Enabled { + compressionPolicy = &policy + } + } + // 3. Get network allocation BEFORE killing VMM (while we can still query it) // This is needed to delete the TAP device after VMM shuts down var networkAlloc *network.Allocation @@ -142,6 +158,16 @@ func (m *manager) standbyInstance( // Return instance with derived state (should be Standby now) finalInst := m.toInstance(ctx, meta) + + if compressionPolicy != nil { + m.startCompressionJob(ctx, compressionTarget{ + Key: m.snapshotJobKeyForInstance(stored.Id), + OwnerID: stored.Id, + SnapshotDir: snapshotDir, + Policy: *compressionPolicy, + }) + } + log.InfoContext(ctx, "instance put in standby successfully", "instance_id", id, "state", finalInst.State) return &finalInst, nil } diff --git a/lib/instances/types.go b/lib/instances/types.go index bb548d6a..3a561936 100644 --- a/lib/instances/types.go +++ b/lib/instances/types.go @@ -92,6 +92,9 @@ type StoredMetadata struct { SkipKernelHeaders bool // Skip kernel headers installation (disables DKMS) SkipGuestAgent bool // Skip guest-agent installation (disables exec/stat API) + // Snapshot policy defaults for this instance. + SnapshotPolicy *SnapshotPolicy + // Shutdown configuration StopTimeout int // Grace period in seconds for graceful stop (0 = use default 5s) @@ -169,6 +172,7 @@ type CreateInstanceRequest struct { Cmd []string // Override image cmd (nil = use image default) SkipKernelHeaders bool // Skip kernel headers installation (disables DKMS) SkipGuestAgent bool // Skip guest-agent installation (disables exec/stat API) + SnapshotPolicy *SnapshotPolicy // Optional snapshot policy defaults for this instance } // StartInstanceRequest is the domain request for starting a stopped instance @@ -202,9 +206,15 @@ type ListSnapshotsFilter = snapshot.ListSnapshotsFilter // CreateSnapshotRequest is the domain request for creating a snapshot. type CreateSnapshotRequest struct { - Kind SnapshotKind // Required: Standby or Stopped - Name string // Optional: unique per source instance - Metadata tags.Metadata // Optional user-defined key-value metadata + Kind SnapshotKind // Required: Standby or Stopped + Name string // Optional: unique per source instance + Metadata tags.Metadata // Optional user-defined key-value metadata + Compression *snapshot.SnapshotCompressionConfig // Optional compression override +} + +// StandbyInstanceRequest is the domain request for putting an instance into standby. +type StandbyInstanceRequest struct { + Compression *snapshot.SnapshotCompressionConfig // Optional compression override } // RestoreSnapshotRequest is the domain request for restoring a snapshot in-place. @@ -220,6 +230,11 @@ type ForkSnapshotRequest struct { TargetHypervisor hypervisor.Type // Optional, allowed only for Stopped snapshots } +// SnapshotPolicy defines default snapshot behavior for an instance. +type SnapshotPolicy struct { + Compression *snapshot.SnapshotCompressionConfig +} + // AttachVolumeRequest is the domain request for attaching a volume (used for API compatibility) type AttachVolumeRequest struct { MountPath string diff --git a/lib/oapi/oapi.go b/lib/oapi/oapi.go index 674b6e6d..5d8e8db1 100644 --- a/lib/oapi/oapi.go +++ b/lib/oapi/oapi.go @@ -122,6 +122,14 @@ const ( RestoreSnapshotRequestTargetHypervisorVz RestoreSnapshotRequestTargetHypervisor = "vz" ) +// Defines values for SnapshotCompressionState. +const ( + SnapshotCompressionStateCompressed SnapshotCompressionState = "compressed" + SnapshotCompressionStateCompressing SnapshotCompressionState = "compressing" + SnapshotCompressionStateError SnapshotCompressionState = "error" + SnapshotCompressionStateNone SnapshotCompressionState = "none" +) + // Defines values for SnapshotSourceHypervisor. const ( CloudHypervisor SnapshotSourceHypervisor = "cloud-hypervisor" @@ -130,6 +138,12 @@ const ( Vz SnapshotSourceHypervisor = "vz" ) +// Defines values for SnapshotCompressionConfigAlgorithm. +const ( + Lz4 SnapshotCompressionConfigAlgorithm = "lz4" + Zstd SnapshotCompressionConfigAlgorithm = "zstd" +) + // Defines values for SnapshotKind. const ( SnapshotKindStandby SnapshotKind = "Standby" @@ -357,7 +371,8 @@ type CreateInstanceRequest struct { // When true, DKMS (Dynamic Kernel Module Support) will not work, // preventing compilation of out-of-tree kernel modules (e.g., NVIDIA vGPU drivers). // Recommended for workloads that don't need kernel module compilation. - SkipKernelHeaders *bool `json:"skip_kernel_headers,omitempty"` + SkipKernelHeaders *bool `json:"skip_kernel_headers,omitempty"` + SnapshotPolicy *SnapshotPolicy `json:"snapshot_policy,omitempty"` // Vcpus Number of virtual CPUs Vcpus *int `json:"vcpus,omitempty"` @@ -371,6 +386,8 @@ type CreateInstanceRequestHypervisor string // CreateSnapshotRequest defines model for CreateSnapshotRequest. type CreateSnapshotRequest struct { + Compression *SnapshotCompressionConfig `json:"compression,omitempty"` + // Kind Snapshot capture kind Kind SnapshotKind `json:"kind"` @@ -720,7 +737,8 @@ type Instance struct { OverlaySize *string `json:"overlay_size,omitempty"` // Size Base memory size (human-readable) - Size *string `json:"size,omitempty"` + Size *string `json:"size,omitempty"` + SnapshotPolicy *SnapshotPolicy `json:"snapshot_policy,omitempty"` // StartedAt Start timestamp (RFC3339) StartedAt *time.Time `json:"started_at"` @@ -923,6 +941,16 @@ type RestoreSnapshotRequestTargetHypervisor string // Snapshot defines model for Snapshot. type Snapshot struct { + // CompressedSizeBytes Compressed memory payload size in bytes + CompressedSizeBytes *int64 `json:"compressed_size_bytes"` + Compression *SnapshotCompressionConfig `json:"compression,omitempty"` + + // CompressionError Compression error message when compression_state is error + CompressionError *string `json:"compression_error"` + + // CompressionState Compression status of the snapshot payload memory file + CompressionState *SnapshotCompressionState `json:"compression_state,omitempty"` + // CreatedAt Snapshot creation timestamp CreatedAt time.Time `json:"created_at"` @@ -949,17 +977,48 @@ type Snapshot struct { // SourceInstanceName Source instance name at snapshot creation time SourceInstanceName string `json:"source_instance_name"` + + // UncompressedSizeBytes Uncompressed memory payload size in bytes + UncompressedSizeBytes *int64 `json:"uncompressed_size_bytes"` } +// SnapshotCompressionState Compression status of the snapshot payload memory file +type SnapshotCompressionState string + // SnapshotSourceHypervisor Source instance hypervisor at snapshot creation time type SnapshotSourceHypervisor string +// SnapshotCompressionConfig defines model for SnapshotCompressionConfig. +type SnapshotCompressionConfig struct { + // Algorithm Compression algorithm (defaults to zstd when enabled) + Algorithm *SnapshotCompressionConfigAlgorithm `json:"algorithm,omitempty"` + + // Enabled Enable snapshot memory compression + Enabled bool `json:"enabled"` + + // Level Compression level for zstd only + Level *int `json:"level,omitempty"` +} + +// SnapshotCompressionConfigAlgorithm Compression algorithm (defaults to zstd when enabled) +type SnapshotCompressionConfigAlgorithm string + // SnapshotKind Snapshot capture kind type SnapshotKind string +// SnapshotPolicy defines model for SnapshotPolicy. +type SnapshotPolicy struct { + Compression *SnapshotCompressionConfig `json:"compression,omitempty"` +} + // SnapshotTargetState Target state when restoring or forking from a snapshot type SnapshotTargetState string +// StandbyInstanceRequest defines model for StandbyInstanceRequest. +type StandbyInstanceRequest struct { + Compression *SnapshotCompressionConfig `json:"compression,omitempty"` +} + // Volume defines model for Volume. type Volume struct { // Attachments List of current attachments (empty if not attached) @@ -1194,6 +1253,9 @@ type CreateInstanceSnapshotJSONRequestBody = CreateSnapshotRequest // RestoreInstanceSnapshotJSONRequestBody defines body for RestoreInstanceSnapshot for application/json ContentType. type RestoreInstanceSnapshotJSONRequestBody = RestoreSnapshotRequest +// StandbyInstanceJSONRequestBody defines body for StandbyInstance for application/json ContentType. +type StandbyInstanceJSONRequestBody = StandbyInstanceRequest + // StartInstanceJSONRequestBody defines body for StartInstance for application/json ContentType. type StartInstanceJSONRequestBody StartInstanceJSONBody @@ -1377,8 +1439,10 @@ type ClientInterface interface { RestoreInstanceSnapshot(ctx context.Context, id string, snapshotId string, body RestoreInstanceSnapshotJSONRequestBody, reqEditors ...RequestEditorFn) (*http.Response, error) - // StandbyInstance request - StandbyInstance(ctx context.Context, id string, reqEditors ...RequestEditorFn) (*http.Response, error) + // StandbyInstanceWithBody request with any body + StandbyInstanceWithBody(ctx context.Context, id string, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*http.Response, error) + + StandbyInstance(ctx context.Context, id string, body StandbyInstanceJSONRequestBody, reqEditors ...RequestEditorFn) (*http.Response, error) // StartInstanceWithBody request with any body StartInstanceWithBody(ctx context.Context, id string, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*http.Response, error) @@ -1857,8 +1921,20 @@ func (c *Client) RestoreInstanceSnapshot(ctx context.Context, id string, snapsho return c.Client.Do(req) } -func (c *Client) StandbyInstance(ctx context.Context, id string, reqEditors ...RequestEditorFn) (*http.Response, error) { - req, err := NewStandbyInstanceRequest(c.Server, id) +func (c *Client) StandbyInstanceWithBody(ctx context.Context, id string, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*http.Response, error) { + req, err := NewStandbyInstanceRequestWithBody(c.Server, id, contentType, body) + if err != nil { + return nil, err + } + req = req.WithContext(ctx) + if err := c.applyEditors(ctx, req, reqEditors); err != nil { + return nil, err + } + return c.Client.Do(req) +} + +func (c *Client) StandbyInstance(ctx context.Context, id string, body StandbyInstanceJSONRequestBody, reqEditors ...RequestEditorFn) (*http.Response, error) { + req, err := NewStandbyInstanceRequest(c.Server, id, body) if err != nil { return nil, err } @@ -3279,8 +3355,19 @@ func NewRestoreInstanceSnapshotRequestWithBody(server string, id string, snapsho return req, nil } -// NewStandbyInstanceRequest generates requests for StandbyInstance -func NewStandbyInstanceRequest(server string, id string) (*http.Request, error) { +// NewStandbyInstanceRequest calls the generic StandbyInstance builder with application/json body +func NewStandbyInstanceRequest(server string, id string, body StandbyInstanceJSONRequestBody) (*http.Request, error) { + var bodyReader io.Reader + buf, err := json.Marshal(body) + if err != nil { + return nil, err + } + bodyReader = bytes.NewReader(buf) + return NewStandbyInstanceRequestWithBody(server, id, "application/json", bodyReader) +} + +// NewStandbyInstanceRequestWithBody generates requests for StandbyInstance with any type of body +func NewStandbyInstanceRequestWithBody(server string, id string, contentType string, body io.Reader) (*http.Request, error) { var err error var pathParam0 string @@ -3305,11 +3392,13 @@ func NewStandbyInstanceRequest(server string, id string) (*http.Request, error) return nil, err } - req, err := http.NewRequest("POST", queryURL.String(), nil) + req, err := http.NewRequest("POST", queryURL.String(), body) if err != nil { return nil, err } + req.Header.Add("Content-Type", contentType) + return req, nil } @@ -4219,8 +4308,10 @@ type ClientWithResponsesInterface interface { RestoreInstanceSnapshotWithResponse(ctx context.Context, id string, snapshotId string, body RestoreInstanceSnapshotJSONRequestBody, reqEditors ...RequestEditorFn) (*RestoreInstanceSnapshotResponse, error) - // StandbyInstanceWithResponse request - StandbyInstanceWithResponse(ctx context.Context, id string, reqEditors ...RequestEditorFn) (*StandbyInstanceResponse, error) + // StandbyInstanceWithBodyWithResponse request with any body + StandbyInstanceWithBodyWithResponse(ctx context.Context, id string, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*StandbyInstanceResponse, error) + + StandbyInstanceWithResponse(ctx context.Context, id string, body StandbyInstanceJSONRequestBody, reqEditors ...RequestEditorFn) (*StandbyInstanceResponse, error) // StartInstanceWithBodyWithResponse request with any body StartInstanceWithBodyWithResponse(ctx context.Context, id string, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*StartInstanceResponse, error) @@ -4970,6 +5061,7 @@ type StandbyInstanceResponse struct { Body []byte HTTPResponse *http.Response JSON200 *Instance + JSON400 *Error JSON404 *Error JSON409 *Error JSON500 *Error @@ -5691,9 +5783,17 @@ func (c *ClientWithResponses) RestoreInstanceSnapshotWithResponse(ctx context.Co return ParseRestoreInstanceSnapshotResponse(rsp) } -// StandbyInstanceWithResponse request returning *StandbyInstanceResponse -func (c *ClientWithResponses) StandbyInstanceWithResponse(ctx context.Context, id string, reqEditors ...RequestEditorFn) (*StandbyInstanceResponse, error) { - rsp, err := c.StandbyInstance(ctx, id, reqEditors...) +// StandbyInstanceWithBodyWithResponse request with arbitrary body returning *StandbyInstanceResponse +func (c *ClientWithResponses) StandbyInstanceWithBodyWithResponse(ctx context.Context, id string, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*StandbyInstanceResponse, error) { + rsp, err := c.StandbyInstanceWithBody(ctx, id, contentType, body, reqEditors...) + if err != nil { + return nil, err + } + return ParseStandbyInstanceResponse(rsp) +} + +func (c *ClientWithResponses) StandbyInstanceWithResponse(ctx context.Context, id string, body StandbyInstanceJSONRequestBody, reqEditors ...RequestEditorFn) (*StandbyInstanceResponse, error) { + rsp, err := c.StandbyInstance(ctx, id, body, reqEditors...) if err != nil { return nil, err } @@ -7122,6 +7222,13 @@ func ParseStandbyInstanceResponse(rsp *http.Response) (*StandbyInstanceResponse, } response.JSON200 = &dest + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 400: + var dest Error + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.JSON400 = &dest + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 404: var dest Error if err := json.Unmarshal(bodyBytes, &dest); err != nil { @@ -11183,7 +11290,8 @@ func (response RestoreInstanceSnapshot501JSONResponse) VisitRestoreInstanceSnaps } type StandbyInstanceRequestObject struct { - Id string `json:"id"` + Id string `json:"id"` + Body *StandbyInstanceJSONRequestBody } type StandbyInstanceResponseObject interface { @@ -11199,6 +11307,15 @@ func (response StandbyInstance200JSONResponse) VisitStandbyInstanceResponse(w ht return json.NewEncoder(w).Encode(response) } +type StandbyInstance400JSONResponse Error + +func (response StandbyInstance400JSONResponse) VisitStandbyInstanceResponse(w http.ResponseWriter) error { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(400) + + return json.NewEncoder(w).Encode(response) +} + type StandbyInstance404JSONResponse Error func (response StandbyInstance404JSONResponse) VisitStandbyInstanceResponse(w http.ResponseWriter) error { @@ -12827,6 +12944,13 @@ func (sh *strictHandler) StandbyInstance(w http.ResponseWriter, r *http.Request, request.Id = id + var body StandbyInstanceJSONRequestBody + if err := json.NewDecoder(r.Body).Decode(&body); err != nil { + sh.options.RequestErrorHandlerFunc(w, r, fmt.Errorf("can't decode JSON body: %w", err)) + return + } + request.Body = &body + handler := func(ctx context.Context, w http.ResponseWriter, r *http.Request, request interface{}) (interface{}, error) { return sh.ssi.StandbyInstance(ctx, request.(StandbyInstanceRequestObject)) } @@ -13295,201 +13419,206 @@ func (sh *strictHandler) GetVolume(w http.ResponseWriter, r *http.Request, id st // Base64 encoded, gzipped, json marshaled Swagger object var swaggerSpec = []string{ - "H4sIAAAAAAAC/+x97XITO7boq6h8z6lxztiO80EIPrXr3ECAnbMJ5BKSuWe2uUbulm1NuqXektrBUPyd", - "B5hHnCe5pSWpv6y2O4E4ZGBqauN0q/WxtLS0vtfnVsDjhDPClGwNPrdkMCMxhp9HSuFgdsmjNCZvyR8p", - "kUo/TgRPiFCUQKOYp0yNEqxm+q+QyEDQRFHOWoPWGVYzdD0jgqA59ILkjKdRiMYEwXckbHVa5COOk4i0", - "Bq3tmKntECvc6rTUItGPpBKUTVtfOi1BcMhZtDDDTHAaqdZggiNJOpVhT3XXCEukP+nCN1l/Y84jglnr", - "C/T4R0oFCVuD34vLeJ815uO/kUDpwY/mmEZ4HJFjMqcBWQZDkApBmBqFgs6JWAbFM/M+WqAxT1mITDvU", - "ZmkUITpBjDOyVQIGm9OQakjoJnro1kCJlHggE8KcRjT07MCzE2Reo5Nj1J6Rj+VBdh+PD1v1XTIck+VO", - "f01jzLoauHparn9oW+z71b6vZ8rjOB1NBU+T5Z5P3pyeXiB4iVgaj4ko9ni4m/VHmSJTInSHSUBHOAwF", - "kdK/fveyOLd+v98f4N1Bv9/r+2Y5Jyzkohak5rUfpDv9kKzoshFIbf9LIH19eXJ8coSecZFwgeHbpZEq", - "iF0ET3FdRbQp74oP/5+mNAqXsX6sHxMxokwqzGpw8MS+1ODiE6RmBNnv0OUpak+4QCEZp9MpZdOtJviu", - "CVZEFAlHWC0PB1NFtg3lDCkaE6lwnLQ6rQkXsf6oFWJFuvpNowEFwWuG0y0aDbZ81FKzk6NY1vXumiDK", - "UEyjiEoScBbK4hiUqYP9+sUUDgwRgnso1HP9GMVESjwlqK3JpqbdDEmFVSoRlWiCaUTCRnvkQwSzmL/x", - "MaIhYYpOaPl8G3Tq4nGws7vnpR0xnpJRSKf2Jip3fwzPNYrpfhSC1v6F6IO2aLYOGFKQyfJ4L4B0wyCC", - "TIggGse/criYKAwX4OBz699g1Nb/2s4v6G17O2+f2nbv8FQCERR8Tpg+Zeu+hE04y5t/6bT+SElKRgmX", - "1KxsieLZNxr9YIsQfOFfK7xahSMFTJQKi9XnClp8gxNs5tcINuemaZWOApm03ZQoQi25fD4nzMMwBZwp", - "+6K84ld8iiLKCLItLHw1fdQD/BJxII/fYm2dVg7SZUKg530LQmYe1PSm33VahKWxBmbEp0VozggWakxK", - "wKy5zmxH+exqwX9WOhKVewtLMlpNTc4oYyREuqU95KYlSiVwrUvLh5NxRdVoToT0niOY1m9UIduitquI", - "B1cTGpHRDMuZmTEOQziDODorrcTDuZVYYZxogug6BI5CIsXR+a9Hu48OkB3AA0PJUxGYGSyvpPC17t60", - "RQqLMY4iL27Uo9vN7+tlDPFjwHl2MOruoQwDHWIa6tWyu6m777SSVM7ML6DjelZwD2oyoNEr0r/fexb9", - "DIiEkRjq5adbUnw/H/kmMUiCphHXe7FAKaN/pCUmvYdOtLyhkL40aEjCDsLwQpNvnCrenRJGhKZvaCJ4", - "DBxbgZFGbdKb9jpoqHnLruaku3i32+93+8NWmRWO9rvTJNUgxEoRoSf4/37H3U9H3b/2u0/e5z9Hve77", - "P/+bD3GacveOs7TrbDua0UFuskWWvzrR1eLACo7aR33Mtp9omrGpXX92ssyImHWHPLgiokf5dkTHAovF", - "NptS9nEQYUWkKkNhddu1cIG5rQAIm2qQbQgkFYEK0Lsd8WsiAk3RI6IRUnY0UadKdhDWMjkQQ6Rv3f9E", - "AWb6jBgGhAtEWIiuqZohDO3KkIsXXZzQLjVLbHVaMf74irCpmrUGB3tL+K+Rv21/dN//h3u09V/eIyDS", - "iHiQ/y1PFWVTBK8NlzCjEuVzoIrEa9kCtytpBKxgTNmJ+WwnmwkWAi/8u+0mt2rXjfBXu+1B7JEU3syJ", - "EDR0N++z02PUjugVseiMRMrQMO339wJoAD+JfRLwOMYsNM+2euhNTJW+8dL8Ijfao15xC39vkWDGgReJ", - "Iq4XlIGvhtFxcHGCtGeLjp3mRSIrzcPdi0GvBlv28uxiW1OxBEupZoKn01l5VpaE3mw+VF6NKB+NE9+c", - "qLxCJ9tvkCbwKKIaOhlB3+n3T59uy2FL//HI/bHVQ8cGZDB9vX9c2HtGzrAgwCWFiDP07OwC4SjigZVX", - "J5qZndBpKkjYq6hJoHcfwhOmxCLh1MckVzAjb7qMIN1u/vYGeLA9pmxb6m3oBjeDO2Hzr2DVnrM5FZzF", - "ml2eY0E13SoprT63Xr85fj56/vqyNdCHKEwDqwE6e/P2XWvQ2uv3+y0fN6QxaA0deHl28Qx2SrefcZVE", - "6XQk6ScPaT3K1odiEnNhRBT7DWrPypTXcHAINmfY2nv51CDXzkvAK7cpIZXQ2vViOi5jzO7Lpz5smS0S", - "IuZU+nQav2bv3M4X6KQhTGXclkTMiciQFrC4V+APg4inYbcwZKc1oYIEAmu0a3Vaf5BYMzzzTxp18rl7", - "vvOrGhpd7mtubRwllJHaa7vz0K/aay6uIo7D7s43vmkZUbrv5SW+Ni/KeGFxiWSo1OosiZksvKahmo1C", - "fs30lD302L5BWeOMKH/UK8HRP//+j8vTnI/deTlOLIXe2X30lRS6QpN1117ZNltImviXcZH4F3F5+s+/", - "/8Ot5H4XQZjGz7BkPzLqovJS/jIjakZE4aZ2G6wfGSEDPkcOXwrDl/RPRWPTElHmcyIivCgQWTun1k4f", - "KF1lVoIqOF/2O00yr5D+eA3J1b25C/1lVfDZ7fuJqmdSnjk91efb3gFNZpJNZGf31P7cXZ5SzYyuaDKa", - "ah5yhKeZ/myVGfD8iiYIvujCF2Ybo8gc3jDVPaMx56o3ZH+ZEYZg72CDyUcSAJ2SCit0dHYi0TWNIpCa", - "gRAsXyND9q5ACkxzqfR/Rco6aJwqJEjMFUGWQYVBUpgLNB4TlDLs7Iy9IStCxS6wilcWLFdEMBKNZgSH", - "RMiGkDEfIftRLXBgqRMsFRGGQqdJGV7Hv52eo/bxguGYBug30+spD9OIoPM00Wd4qwy9zpAlgswJA/lF", - "3zvUjssniKeqyyddJQhxU4yhs0zvYI1g85dnF9aMKrd6Q/aWaMASFpIQ5uxuCYnUDCsUcvYnfWJJWO62", - "OH4F6P6z3GnNgyQtQ3m3CuHXYLzU65lToVIcaZJV4ua8tkxjJfdw7cYIX5QeLCnKEA6rshGqqQBoegaT", - "+TJP65f5DKNSL/OdM5zIGVe1Mt8VZeG6eblOftNtvznPkunJpB3mrtmWRJBumkwFBuPwt2NaKjsEkK3f", - "mTW+HD6jXQapIJWKxwXTHWpXlIW0rFYsA2DOo67eF2Da7pgjNctcNp/HCzMFc8zq7r3RdOzReOvrjTI0", - "pVM8XqiyZLbTXz7M/qPj+vdtUZ1riTnwJBwpvtq4TifItW1iEwNHlJHio/mEenrO2KBcq0olCip+LJYM", - "6S66SUAtQe6g6xnVjJNEDghAky9Pi5qO3pB14RIZoONsgKzbrEt98EDzDl20uShMgoIRBY0XWwijy9Me", - "epfN9k8SMazonDhfmxmWaEwIQykw3CSE8eGCLE4glfpWoqr6ub19jFvOFih0uH3XQ1rQjLG9yfWxiLGi", - "ASjgx7SyHjCYmo3SI2mSzop8RKN7f5VLwlsypVKJikMCar998Wxvb+9JlQPcfdTt73R3Hr3b6Q/6+v9/", - "be678O09j3x9HZXpjDVpFCnRs4uT413LbpbHUZ/28ZPDjx+xenJAr+WTT/FYTP+2hzfim/RtydpxbsNB", - "7VQS0XWkVmOjz3JTMJDUWGZubXC5kTuVMw2vgoFZ3Tvd8i4csHzmfGtMvrmLVJV4rnUIKCxuaT36qeYU", - "8xNTUDhZ+1lAvRbGYyqvngqCr0J+zTz3uWbU5MjcV35NcKol6vECkY+aUSchEpyriTQapzLDurP/eP9w", - "72D/sN/3+B0tIz8P6CjQt1GjCbx5doIivCACwTeoDSJ/iMYRH5eR99HeweHj/pOd3abzMAJzMzhk/LT7", - "CrUtRP7sfFjdm9KkdncfH+zt7fUPDnb3G83KsvqNJuXEghLL8Xjv8f7O4e5+Iyj4FBDPnR9Y1T8l9Cl9", - "kySiRt3SlQkJ6IQGCDzJkP4AtWO4zkgm+5fP5BiHI2HZTu89ojCN5EpdsxnMtjRug3EaKZpExLyDDWkk", - "88DKj6Ennx6fMkbEKHOTu0FP1nturY7UrSVrgkpekCXQnVIJHEnOSFEShQNzQtfSOdjNfGLv6/DArqEh", - "NrzSYlI3InMSFZHAXEd6sjEXBGV4YjattCrK5jii4YiyJK3RUdeA8kUqgC81nSI85qkyShvYsOIgYIMH", - "mWSiyXUz15EXXFyttVrq23UkUsZ0N2v1LUdRxK/1Fl9p2MDNjJH92jnPFBjATLliVFD2vURvzRdGRZU/", - "TlKFKFNcS6IsHC86MBIJoR1DgkjFgZLi4Epzm7abppymnxd5rZkQpwA34+W0c0Pa/+7EKF+/pQlAYTEl", - "aiQVVms5Fo0p76D9OTRv7BChP1yrJGkAd0auNwF08ALparTtSoaTu4H4KjNepoPIG8EtLGhIeghOF9gF", - "nDdq5aSdK54kJMx0Pb0hOzdHJXskUZxK0HVeGTioGaECcUGntDywPTYbsAfeBBUdNt0aHYsfLnOo8BKU", - "4fWHHk8UEQaCzkG/6FlnN6HVaVnYtzotS4nKoHEPPRDJjdRLU3x5dnFT61wi+IRGnuWCZtm+tdKWs1u9", - "2u+fd3f+j7Fda3wDFo0yo42OeUh6lRgYaN/s5nl5dnFWN6csAAkVZ7e0psx+4KEcmUraQcRqxgPM0Jgg", - "K8E49NcXSzZIzns/8fGyE4FjMk4nEyJGsUd59kK/R6aBMRRRhk6flvlZzTcvd+2ngmelzQFReIIDGz/S", - "DPoe5VxlGZ0CNN/7t+stMddwnaep3iph21hn0x56nYV8oZdnFxLlNh+P1q68vbWeRmezhaQBjkyPxnGc", - "sqKyDZCzMYd8ln9o1ZIePjn28obuIKD2fJqkcAzP33ZP3lxuxyGZd0pzAjvNjEdEz3urQC3mzm80d4sq", - "EYl5nfbCIIZseoAKsMpOcGMgFc6rBzqKKxyNZMSVZzbv9EsEL1H78oXx39Mz6KCktJX6eQEKJfw+8J4Y", - "TZHqhj2HAavq09IBX6vJjo1EUVxeaVDfUfmV4MgEiJbxOQ9dcBvPr8obza/Wnl7biW/cE+dS08Dn8Nnp", - "sWEYAs4UpowIlKnvSg5iwA61Oq2uvqNCTGIwXE7+c7WzWI06PkOXVQrdZ0vRZXeizK2JhNBELpqTEMWY", - "0QmRykZClEaWM7z76GBgYrdCMtl/dNDr9W7q3fc8d+drtBXbxvmp4OjXk7Ov24c7cOJrspbPrbOjd7+2", - "Bq3tVIrtiAc42pZjygaFv7M/8xfww/w5pszr/Nco3I9OlsL8yuZLfWeZ5wO9EkaCDCE5CPB3FtpWIwdp", - "lI7oJxIir/e7wlMt1xhM/To3968IkMuju1UhMK7oA9AgSI5+Wq1BdQwVtLFjpkzRKI87XNad3ipyVK4M", - "qFkKpkkIy0Joosj8Cjib69Pki6cpEX73bmkzro1wNwqpB6v/YiW/UAthCnxT15+91jZOkvUo7GcaM1rY", - "NDbQetx7bqV7vwFuY3srj/5m+t9//F959vhvO3+8urz8n/nL/z5+Tf/nMjp7cy9+qKuDNO410mKlow0Y", - "nEoRFk3R6hSrwMNozbhUNVCzb5DiKNYf99AzEAgHQ9ZFr6giAkcDNGzhhPYsMHsBj4ct1CYfcaDMV4gz", - "pLuy/mRb+uMzoxbSH392MueXah+hdRwTFsiZj6dMxyGPMWVbQzZkti/kFiLB7q9/hSjAiUoF0Tuiedto", - "gcYCB7nDWD54B33GSfJla8hA8iUfldArSLBQWQSZGwE22s7K+BXY5iREcxylRFrJeciyewdUAboTo7vp", - "ZcoR0NlXNK41QPGKNVyUHR4P+x3PPiLdTm9kRKUiDGVaECoBeVHbea4e9ktk47B/uN6FJcOhFegH2L0c", - "2uWQssH5MAgMQxsiPpoplTTQsWs6Zc4I+vXduzMNBv3vOXId5bDIttgIfzhJIkqk0R2qCHgg6yy81fKp", - "xM3uNlyQUZ7BZ1ED38znMDB69+ocKSJiygzdbwcanBMa6PWBqZ9KmWpUpBgdPTt9vtVrkOQGYJvNf8U+", - "vstWWLEoO2VanY4ww3gN3w46Oe5oNsye0JxBA9ebF1ygyBCY/FwP0IUkZddG2Cpj7Tc7GS1yjZyh6sPW", - "lusxqVKKAXqb8YU4m0oW2Zojg+syP5fQrTXIGL+gpd475bmCx5OVlyxpAy8grJC1f8IVXk8KVh9/D8Th", - "zHNW1XXe7GwXlaR6MD9q5Ht/55zL3k1l15uGvJU91QuRCVnUW/NwtbsI+1qW4z5SNao1ziP92prindRx", - "eYpmWLI/KXhZkT129h43ShajR21q1i4atPnETCk7Vc7tPTPHmgCAKxpFxstB0inDEXqC2ucnL387efVq", - "C3XRmzen1a1Y9YVvfxpEvznUfnl2ASFlWI6cZajeMRLnzsPkI5VKLkcFNDKwro62+7UUEecNs9j6hmFy", - "ziq9tIxNBMDdp+vfv07w3cpwua+NebNM8h2FvNUSZV+4WJk+m8ffNnjtTqZTCkPz0ZUiL+H8uW8dedZp", - "UY8v65HUpJOE6OQsz+qRK6tc95U1Pdnt7Rwc9nb6/d5Ov5HKDwcrxj49etZ88P6uUWYM8HgQhAMyaTJ+", - "jerQIrZh+nB0jRcSDR1bPmwZOaAgABSOu2XdG5lzlwP8bhfPV2VE1kXs3SRCr1no3Yo0XeflBF2NebtH", - "f/2qXF6k6Y1uXSHsV6ObKMMJCngahZp/GuuTZ8QxElqpURKV5z6Dw3rBrhi/ZuWlG92mPr9/pEQs0OXp", - "aUmDLsjEpoFqsHBwoajZB57caBt217DYa2dTiILbRORblRIWbqBvHudWVL85N0uDdQ3UcDkn6TWNU2bA", - "rfd+xZoqCpSQzEdp6mOQ9CsXaHFxcXJc2nCMD3YO+4dPuofjnYPuftjf6eKdvYPu7iPcn+wFj/dqEi02", - "d425vbdL+YTWBzYB4EEZaWLYwoE+Q5m7yjhVKHNl04fzmeY0UYGlNWE8oB+wvkW6B7hdA/0mWmRc78qP", - "z7A+qO7bBP5a/cX5LFWaDYJv5CxVSP8FU9ZLsFLD6i7MmR+g1xy+Ec4HlPGq+GGag2/VcvOqqNK2Xj/O", - "OxQGswRsgF5kRCsje5bMtSWxPw0ttY7L4JS9VXKNs7tVcPPqtAwIW52Wgwy4gy07htmJeGMeinjjU9YT", - "HAENyx1vUkUj+skcOT11KhUNjLSGYTfrjp1NMUDCkblC68xwxpvDXrPZR+5UX56iNoQP/hlZYU7/tZWZ", - "7IpHaH/3yf6Tg8e7Tw4aBRHkE1xPjZ+Br9Hy5NaS5iBJRy7hbM3Sn51dwOWjLzaZxkY6t2sv+Gwmggea", - "26MM5Rls88Gf9J4UYydCno6jgrbHBl2Bg36TdMM1Nqo/aDSnkwn741Nwtfs3QeOdjwdyd+wVjrKB/Jzk", - "SVFDuSR2kXHXpJPxS4GAUELWRoC8JRJWgM6JQoA/XU2w9I2auQhZlHNxIhbiXsTa39vbO3z8aLcRXtnZ", - "FQ7OCOS/5Vme2hkUjhi0RO235+dou4Bwpk/nN5kIIvXiTDCk95whm9arX3Kp1LLHng9LahiWHGts3/O4", - "FuSXlmOxi7JAB0+njJtZOuVeaO/t9R/vPzp81OwYW4lnJD6upjC2nbX0CxIQOi/tfBu02u+OzpDuXUxw", - "UObwd3b39h8dPD680azUjWalBGYypkrdaGKHjw8e7e/t7jQLZfJprm2QXunAlmmX59B5kMKzGx5QLJPe", - "Tt1t4eMSS7qdFbrjgp/9rsaloqP9UfevxrEejXqD7V/+/L+77//j3/zBVSVdhySiG5IJSDJXZNEFW2bm", - "F4EUnspe2TMJFNyaAbaxSYrgGGK6gitikzPgj8WJP+pnN+niNY6X1rKzewipBrO/167Mnxx0Ca7Lbqsr", - "PWVz19uqn+VNHKvzQHkqoVda8OlFbc2cFhn9QrD3VhP9jf/q0ePUlQfQbHhTn+fVLs5nWM1O2IQvm3xu", - "IkhbxzFnCkg0QykhAXJIGCWhuxMyidryqOCKFkmCwpRYyBmeU2ALcGzMXglWMxAC4EPKpmUn/KUBm4i3", - "Zg6r0yLAuLZhE02c9DstvRMpwMro3CXCuftSIwMClSO/tLbcsSDTNMICVf36V0xZLuKIsqsmvctFPOYR", - "DZD+oKommfAo4tcj/Ur+AmvZarQ6/cEot7hX1B5mctbfwmxIZdx8Cb/oVW5VPL+Ao9o2329D/Zcmik2v", - "Ge6FFoqN6/sFox8LiF6OFd7f7dc5+tV0WnLxWw6buOmdaVHWd+JdRMNRll3NY+41BrWKZqAsX5TW61st", - "WGxXuTUuc1io7XSlLha7DNdCTHQjBqeZ0bhqFXCz2ZYkKI++f/jo8UHDoPSvEmFWVMj4CoFlHq8QVGp2", - "6rQJN3z46PDJk739R092b8R3OgNSzf7UGZGK+1NJoljhhR/14X83mpQxIfmnVGNGKk+olBDx1hP6suLo", - "5sFINdqMVdWp8p106pOyYNNMdFjBLR2VWK5CruA2mUwIKORGBm7dfDIVZ7VGcwhwggOqFh7JGl+D/w7K", - "mlSCahr0XpmsB6S2bxsXqSmXTMe5f0TbDY7+w0jMFVw4bJzbQqbjOun8TXVUI5sbh7ewovlpoHgxGOFz", - "UrjOgImusSxZS/TvQJGwU8gFXTWrmRbNq4o4XM8Ki+SOBr7AMH8RkeL2V7azIM2VmOQqxFddofVHUHME", - "4E3XxHDhuZE90WbBeieXCn2wF+DtvhqNi1lnVqb1KaWoyW/dm4/bLIv18nfmBrv5eAXPiJt8WE3AAfho", - "52BBnvfdKaFEDTYpLtbnVLyDMHpjG7hVIL01K2wklt4+vpP4+aXtOC+4hTV3gnRf+evElQy0B93+Xrd/", - "8G5nb/DoYLCzcxfRG5kxqE5F/vjTzvXjaBdP9qPDxeM/dmaPp7vxnted5TvK5VlJPlxJ7WnXnhBRTblS", - "TVUkSUQZ6crMHLXeMr8iRssoSRO8AOZwhSR3E/HBVWdacdrPy4ssHnqscuBUk8ZuwtHPzn6lDFSd/snx", - "6mnfyr5TnYgfwapTAXxqNhmILNxplo0OTpIXODUT9aFByUGhhJjvV1Cz3+whrqNa1mfezjBP6eEOiLPh", - "ljAhf70Edx+5XZ11pHIhGcNzMclL5q/7bVOOGLeRuqSssauaXEmPRU0dShtojwqNUZvEiVq4oFCnGN66", - "mRvLUdahlxf8xu74/SffIpDwYmXk4A+eHrjoceQGWetrtIQLteE6fi3TcdWb16hybZrDsvdpJXmbVCvK", - "tK4qCW5qc4Oe1obKTdNqLoEblAGv08znJ87VX3V1wNcpnFeaFwsrK8ykfm+Mu9lX1kyn0hVLvyXIrNZ0", - "feyZcdnRLFK3mgfTpGoRFNSwFkAGsBoEmWZ9WX2/2gv2FH/MRgDWCMslNg7WUahR9fIppF966/Ih0onr", - "AqZRrQby9OuKyTusWt6MVdXlnUOj9+BZ+rOCEtadrQpy5mN0Vhew16SLBKmganGuSaX11SdYEHGUGjQE", - "GgqLgMf54BB/+eULaJcnHiXTSy1e0AAdnZ0AlsSYgaUYXZ6iiE5IsAgiYsPnllzdQEB88+yka+J+s6IN", - "UONVAUBcbu2jsxNIz2urq7b6vd0e1LviCWE4oa1Ba6+3AwmINRhgiduQjgF+WvuRPodwA56E9qZ+apro", - "rwSOiYISGr977DCKCJPeQaLxIreY50b0BFNhjedJBCYiIy9Q3QG4/zoqP2gVEhGY2+umd5tUC6s8I8kb", - "u8/vNX7IhDNpdni336+UHMZ5Htftv0lj3cnHb8SCmFrwyz60S64Gjg2ye/Cl09rv79xoPmtTr/qGvWA4", - "VTMu6CcC03x0QyDcatATZjT6roIYsQ3zgwc4VTxyv7/X+yXTOMZi4cCVwyrhso5/IxJhSP44dpVse8gK", - "KRABKGc8jUKoKJOYVPearmKksOhNPyEsghmdkyGz14lJo4sFREjHSCOZ0cyUz4oZ2uy+oUNEqqc8XFSg", - "m3W3rbvrOr4tB/CNazFnEn5SU5TZR+JN6mkZcG/ObcIwU3kmY5Nz+oqAY9qEfvR22MjDUlNA2BYC5Q6y", - "iPvdLb8NEgLI/Ob74+ydKwlevvW0AEFZEKVhzhqUSzF7EzCZ0sA2NfcV8XBSL6GFBUox1s7dwYyHxMQ/", - "JQs148z8TscpU6n5PRb8WhKhb2obP21hbfPSWtSFegk0hhhmk6lFj7ltprj9+YosvvSG7CiMXWYdW4gJ", - "R5LbnOXGk5VKlJUDA9ytqeTvlwie2domJl9wMcWqmSZPVZKqHjILIcoGfUNzyMArZyQcMsXRZ2GKLyy+", - "bH/OR/wCLDbBocaTQhOzpO3PNPxSN2s5wnr1I2jqETwIAGDY0jfNsKV/TwXWLHYqZwgH4G+rHxa3tG0O", - "NhfAvmxVIRxghhKepJFmBgGpTCr2Uh+QQANHEVJwlNy3mimCnaxZjzUn+7JEWluyMf5VjhHkiywcpv7+", - "4VZrXcWFcvf/ff7mNTIMkd6FssPbkD03DNgAfR6Cg9uwNRg6F7dhqzNsETaHZ9YPbtj64l+hJIEgPq0A", - "TAAuS6iZDs3ysFLYJcpgeq78ll6/nhoOZm7mMyzRsEXDYSuvcb0F0EqlVbd3u8AL/qJn9osZpkPDX3q9", - "4ip//2x6GejTnMQjxa8IG7a+dFDhxZSqWTrO3r2vWXCNUfC8RIpQ29w+Wy4Zk15h4SI2NxdmIeKW2kcL", - "hFFOA4vKhzFlWCzqCs3zVNU7rJtcVbZZjlEH/f7WescZu1QPg11qqM/ilyVWbPebcSGWA1vmQmwtfRsa", - "o4FpC9oD77UBNugpDl2ei5/83hp+z0rbBU4Ovi9eCgZ9I2LUoxV2TIvnkWPHVsouBi0gNgxEEefmZiQR", - "6ti5HHmLMklVBF2WMfbrTlkAU4wc/u1vAP9g3DzjP4z7ZFPj4sjUqXL5rx8WOsJmOUTs+OXll0R9DxjX", - "3xQpdYVJ7hF/Hwr+vCSWCcyBVqFm21Dzs6iMqcYwC4JjaXsxjbXgeg5z6p4TptBzeNqz/zrxB8JDP0R8", - "+mGADAgjPkURZURaF4TM2qEvRQtL+Mikbcy+s1lQgxlmUyJR29yf//z7P2BSlE3/+fd/aNba/ILjvm3c", - "2yGC8sOMYKHGBKsPA/QbIUkXR3RO3GIg5InMiVigvb6tcgyvPDlV5ZAN2VuiUsFk5tiu1wUwMR3aEh96", - "PZSlRCIJIIQCdhPrcW2Uoh553p1lA8qNnujOkgBmV1BYgL4VHQ6ACx1lVFEcWWGs5VermTWXlGpV/e6S", - "xn89fVHkozLY2zUTvCGBARD7zh28sItG7fPz51s9BOy+wQrwqge5Ie/GSgK9nzRpPU0yFKVMUADKhjYV", - "0unXaoePbZtm6mHb4w+tH64rGFCvIDYKESJI6AD4U3hooiz2w80pjn3a22NXX7BefXv79RaHcH6KjSTj", - "b7fPDveWYW6LZ+Yguw+ZGLVt3bMsp2WpQud9If1GrpFCQdjsLkHcZNLcmJz2jLNJRAOFum4ukG4jJpns", - "VkaQh0IO3tpZI+zWVQ1oLV5426X4jNqrLwvVyO/Au789KoPe5BrJg25zXPt5k6xDnWMqA66/LWBLN8CJ", - "zehp+JnsnBaxaJ2G6hieZ1fOSv7pOCsVbQ/k5nRVduiUVe+GDRDF4wpBvEdCWMk2WAhTf0jYfJHtoqul", - "vEKV9X2hZn9zXNCm1Vo+NH9Ieq2wAjZNBWdZTas69LJVr+5wo+0InoWfE+FOtZmoyXKXL8t8ioIZCa7M", - "gmzJ71UcwYmrCt5EFjb9/dCisKk/dgMWxu7BT56lgfSbw2qVxHti8zfencALI9xI3v12lmCLYB4gg2/K", - "2Om0TWpELBcs2PqhjMEbud6qdcYf0Ek6S6PI2UTmRKi8mlrxUtj+DF5M65l9d9pW3g8Xb191CQs4uK1l", - "Lld+rsoVQfq2LL/ZMLOUn2jSREgEUDnEqOeov2L/jXchyjLq//vuC5tT/993X5is+v++d2Ty6m/dGbL0", - "N0WaN82CP2Dk0xw4LQMNSJMpVbSOZc1aNeRaXfsfm3G1te1uwrpmgP7JvTbhXovgWsnAZmUG75CFtdXb", - "7sdokyGbD9rwyrk0/mCs62b1gBYjXc4OKsuGEZuUkYu8YpotH/7wfC5phnHFe6ShQjs/kCvvE4e6J8cd", - "WwzPlLDLAkw2pN5289g4t2vH3bxu+yge02nKU1mMXYHah0TaYKeIlAnwQ+PD8+u5lhP/jrG0v8mrY+OM", - "9k+8vyMRoLqhhngbG9U6IcC1aioE2PZQZdAUvjCxb29dQQ2bXGSrxg/RlYtpisalakXL/pG+edUKJ+hC", - "iy+5zIBAjBgM2X+5T35XBMfvf3HhTWm/v3uQvSNs/v4XF+XETh3eEKYEJRJhQdDR62OwEk4hNh5yh+Xx", - "fdX5mIxgpra0LXv6Ly055UbT5qKTQ8+folMj0akArtWiU1bY5S5lJzPIvQlPDt98ALdJPH6KT5sQn2Q6", - "mdCAEqby5LlL/mU29/YDjFNj1pJU8Asp3cCNxae82tJqzjTP/LZxn6Bs8M1LTS7J3MP0t+cmwiZ0ckp+", - "GdYLKt8bPvQ3S5w3L6A8ZBQzkkAVdMuEaHtic/f6GYQXXFw1xTxPKspvjoDfnjsprvA75E309CBtyf2z", - "KHB5G7d8jTRlzmUDB3Ipv+h9eoM6SFip1wRYUjbNamReUzXjqUnXMrIPTf43fSpsIRZgeQLb632TFz36", - "BhjQ11whGicRiQnkh+sabILipGmScJGVRKOykI33ZuRPH5uib67JmmMrA3eQzVkMWrysqCko9Je3y0s1", - "Iz5dH6CbDe6iUT0RukN2IU32mA+GFf6AMiKLFEeSRCRQ6HpGgxlE6+pn0L8J5sVJ8iFLz7E1QC/hpBYT", - "hsDgbUkExREUnuSRqZn6YR7HHwbLueYuT0/hIxOoa7LKfRggl18uuyCkblWMvtWriLBU6LWNKW5rTBI8", - "isyOftC3UGF9WzYuN89kMmS+GF1Grm2HdII+FMJ1P9TE6zqC+krv0j3xS536DFhmLYojAYAzuElYWKMj", - "01DzR+ru9L0pUxtGDZtp3HHQ8NJkXvFpln2rhMo4SZqir50mYPE8jlfgMGoX8nlLFfJU/VmqkAgBH1vs", - "rkNu1MaB+UPhK42ozFbxchnRAf28ek2TAccLKk1UC+mXzV/zOG51WnY+noK+Xx99Xe1wWc2md6YQYv2T", - "075J8HSZ2Beipys3hy35UM9y20oWP7y850pu3zMa3oN+LJ8FZY5Vgb3Na5k/rKBLU+SkyouZXPO+M5JV", - "Sak/JWWl8nme1f5fUEQ1a62WttmwkJqB2CeZlSo83Lt0mhWc+CmhZhIqFyhMzXCVki8/rNiZERSUspLk", - "adnT28qeWcK6DMxQwo+tNAjkNG/7s/t5cgt24TuhhJ3aIil1qZHyRX8PJLemnFgjmntPfJK9VgsMwj2S", - "YFfYbNMUOIOKFvcyKvddkGFz4DJqXKQ5SmAmqatZ+JMYl9SARlN6W2LsmM8lXWCBPFPWTSJcR5ctn1pL", - "gG3RpB9eXstllR9cYgu4EMadDLzUHlKQY8FmWBA92wlOJelkB6bj7NaXp6dbdYdGqJVHRnwfBu3bcQ6V", - "ipZx6C8pLGjost8/Oz22ufKpRCJlPfQmppCS/oqQBNJbUp5KBP6AvWKVs5pKv3kZM8KUWCScMrV2FnnT", - "u5nMl1sl/N4wnbJh3j+8WsnWqH1oRApoh7697QJWC1XKFPfzmumc2YoykzBfMx94zFPd+1LlNTShEZEL", - "qUhsbHaTNIJDBJlBbCZZ+53xXesgqiQU3u6Ar09CREylpJzJIRuTieZKEiL02FCfkUakYH7wWbbOFc6o", - "5pkhfd+HaQuKsYE1B6s6qJXrsOEkcXXYfOaTrHTcraf0AmxVSC7iMY9ogCLKriRqR/TK8OBoLlGkf2yt", - "NHaN4LtvnSf39idLQ/qETbg3c6DB2QyZfwQKd1Iha86Y/+DI2ktSPCyO/sBG+8maXEvXBMERlB7N3GxR", - "qmhEPxlSpzuhUtHAFGPCGeygjowZrzdkp0QJ3QYLggIeRSRQTtewnQgebA/Tfn8vSCjER+wRmBwQvPrX", - "MYz47OwC2plaN50h039Ax++OzhDVMJ1gKzIXJmprwqOT7TdrzP/nAKZ/YXnMLHDVsfBv+E/L7s19KGvP", - "kKw5ojxZJQDx5IdXGFgO7qe24GFqC8CJPVtNeypwAEyxnKUq5NfMrxkwtVjl9mfz42RdKITCwezSFZr+", - "PrhdW5d23TBugQ/iUNo1hcRkNr0Xfb0tHfxAEz9pwLklABNTDOrw3wKmJPmPht3f3lhXhON3aKmzEHVZ", - "g7+bs7Xpm8/OwUX4FeHxUI65wTS3EqhEWdQ+ZeGMa2WzIBWCMAU5YnLWMsAJDqhadBCOXJlWW2op0yHl", - "JefHguArfdP2huxtFkhpSz1p6arjRCsUUnllerDSUw+9mRMh03E2OQSEych5AHxbqTXAUWBKnJLJhASK", - "zompPSprpK9sKneZ0TcfxLPR7qUF3UMTOfw4AbuXo4WVOkqecrV5Hc6zVs3yOmS9FrxhCp4iK32eR67h", - "CG6im6jsPINf0Vq3ePvqZt5rv+mPGo5d9pLyT8K++spV/rD5884L3ipNs0DkKP/QEjIUZl46uyWPr/WR", - "4Y1dvO7S5WpdZHg2+KYjw8+9Xj8PLHEVLvlx1YWEf3+I0N+su/GmQ8IfNm5p3kIuga6eEjUIDf8uMPBu", - "YsLv2d3+FjHh35UDKMT03p8j/nfl+mldGDPXz59R33fp8WlCvyHCtc7j01A9q4peKTld2jbN5Cbb4w/N", - "0lt15g0YercPP5O6NZAhCsBy13KF/sBlIO0JIHGiFk5fxSfgmZNnIJT0E/j3+ULrMrX03UW03UJj++3Q", - "w+Fprb72ZzK4jamE81TaJ8cPPwNc8cyVbpptfQ11sQhmdF6K6Fp1gi2IEkG6CU9AExsagFl4uMtNYdGb", - "fkK2+96QvZsR9xeiLp8GCVFIBQlUtECUKQ4UwYzxJ4kE16IBvOdi4VPwFk/uC8HjI7uaNRekPVNWXZY7", - "AsaLrr61unNHbVYo2b7CqHWKP9I4jYHgIcrQy6eoTT4qYdI7oIkWhRCdZCAlHwNCQgk4uVWc8E6/RvdJ", - "P5HRdNxklisSdbyxiVBQkErFY7f3J8eojVPFu1PC9F5o3n8CrG0i+JyGJr1uDtQ5jwxUd2oAelPNrGMu", - "kMJTaV3Hc7HDzPLeGZsmt9T0E03KtMJ4S7YGrTFlGCa6Nk9G+aAZx109HqbgPpcfKIdOrZ/3WrWuN2AT", - "FxkQFeco0nz/1s+77yHffUUHCHfRla7AZslPm/lENHRVuIvEp5m/zGaV25ffjxm/UAf5ASrY55mUWqdc", - "/75QsL+5+2HTSvXLB+z29ZI4ibygUIcOdI8+hHnFAxyhkMxJxJNY85qmbavTSkXUGrRmSiWD7e1It5tx", - "qQaH/cN+68v7L/8/AAD//9ibNZqCHgEA", + "H4sIAAAAAAAC/+x963LbOprgq6C0M9XytCTLlziOpk7NOnaS4zlx4o3j9E4fZRWIhCS0SYAHAOUoqfzt", + "B+hH7CfZwgeAN4ESndhy3MnUVB9HJHH58OG7Xz63Ah4nnBGmZGvwuSWDGYkx/HmkFA5m73iUxuQN+SMl", + "UumfE8ETIhQl8FLMU6ZGCVYz/a+QyEDQRFHOWoPWOVYzdD0jgqA5jILkjKdRiMYEwXckbHVa5COOk4i0", + "Bq3tmKntECvc6rTUItE/SSUom7a+dFqC4JCzaGGmmeA0Uq3BBEeSdCrTnumhEZZIf9KFb7LxxpxHBLPW", + "Fxjxj5QKErYGvxe38T57mY//RgKlJz+aYxrhcUROyJwGZBkMQSoEYWoUCjonYhkUx+Z5tEBjnrIQmfdQ", + "m6VRhOgEMc7IVgkYbE5DqiGhX9FTtwZKpMQDmRDWNKKh5wSOT5F5jE5PUHtGPpYn2X08PmzVD8lwTJYH", + "/TWNMetq4OplufHh3eLYL/d9I1Mex+loKniaLI98+vrs7BLBQ8TSeExEccTD3Ww8yhSZEqEHTAI6wmEo", + "iJT+/buHxbX1+/3+AO8O+v1e37fKOWEhF7UgNY/9IN3ph2TFkI1AasdfAumrd6cnp0fomIuECwzfLs1U", + "QewieIr7KqJN+VR8+P80pVG4jPVj/TMRI8qkwqwGB0/tQw0uPkFqRpD9Dr07Q+0JFygk43Q6pWy61QTf", + "NcGKiCLhCKvl6WCpyL5DOUOKxkQqHCetTmvCRaw/aoVYka5+0mhCQfCa6fQbjSZbvmqpOclRLOtGd68g", + "ylBMo4hKEnAWyuIclKmD/frNFC4MEYJ7KNQz/TOKiZR4SlBbk01NuxmSCqtUIirRBNOIhI3OyIcIZjN/", + "42NEQ8IUndDy/Tbo1MXjYGd3z0s7Yjwlo5BOLScqD38Cv2sU0+MoBG/7N6Iv2qLZPmBKQSbL8z0H0g2T", + "CDIhgmgc/8bpYqIwMMDB59a/wayt/7WdM+hty523z+x7b/FUAhEUfE6YvmXrvoRDOM9f/9Jp/ZGSlIwS", + "LqnZ2RLFs080+sERIfjCv1d4tApHCpgoFRar7xW8cQs32KyvEWwuzKtVOgpk0g5Togi15PLZnDCPwBRw", + "puyD8o5f8imKKCPIvmHhq+mjnuCXiAN5vI29dVo5SJcJgV73VxAy80PNaPpZp0VYGmtgRnxahOaMYKHG", + "pATMGnZmB8pXVwv+89KVqPAtLMloNTU5p4yREOk37SU3b6JUgtS6tH24GVdUjeZESO89gmX9RhWyb9QO", + "FfHgakIjMpphOTMrxmEIdxBH56WdeCS3kiiME00Q3YAgUUikOLr49Wj30QGyE3hgKHkqArOC5Z0UvtbD", + "m3eRwmKMo8iLG/XodnN+vYwhfgy4yC5GHR/KMNAhpqFeLXuaevhOK0nlzPwFdFyvCvigJgMavSL993vP", + "po+BSBiNoV5/+kqK75cjXycGSdA04vosFihl9I+0JKT30KnWNxTSTIOGJOwgDA80+cap4t0pYURo+oYm", + "gscgsRUEadQmvWmvg4ZatuxqSbqLd7v9frc/bJVF4Wi/O01SDUKsFBF6gf/vd9z9dNT9a7/75H3+56jX", + "ff/nf/MhTlPp3kmWdp9tRzM6yC22KPJXF7paHVghUfuojzn2U00zNnXqx6fLgojZd8iDKyJ6lG9HdCyw", + "WGyzKWUfBxFWRKoyFFa/uxYusLYVAGFTDbINgaSiUAF6tyN+TUSgKXpENELKjibqVMkOwlonB2KINNf9", + "TxRgpu+IEUC4QISF6JqqGcLwXhly8aKLE9qlZoutTivGH18SNlWz1uBgbwn/NfK37R/d9//hftr6L+8V", + "EGlEPMj/hqeKsimCx0ZKmFGJ8jVQReK1YoE7lTQCUTCm7NR8tpOtBAuBF/7TdotbdepG+as99iD2aAqv", + "50QIGjrOe3x2gtoRvSIWnZFIGRqm/f5eAC/An8T+EvA4xiw0v2310OuYKs3x0pyRG+tRr3iEv7dIMOMg", + "i0QR1xvKwFcj6Di4OEXac0QnzvIikdXmgfdisKvBkb04v9zWVCzBUqqZ4Ol0Vl6VJaE3Ww+VVyPKR+PE", + "tyYqr9Dp9mukCTyKqIZORtB3+v2zp9ty2NL/eOT+sdVDJwZksHx9flxYPiNnWBCQkkLEGTo+v0Q4inhg", + "9dWJFmYndJoKEvYqZhIY3YfwhCmxSDj1CckVzMhfXUaQbjd/egM82B5Tti31MXSDm8GdsPk3iGrP2JwK", + "zmItLs+xoJpulYxWn1uvXp88Gz179a410JcoTANrATp//eZta9Da6/f7LZ80pDFoDR14cX55DCel359x", + "lUTpdCTpJw9pPcr2h2ISc2FUFPsNas/KlNdIcAgOZ9jae/HUINfOC8ArdyghlfC2G8UMXMaY3RdPfdgy", + "WyREzKn02TR+zZ65ky/QSUOYyrgtiZgTkSEtYHGvIB8GEU/DbmHKTmtCBQkE1mjX6rT+ILEWeOafNOrk", + "a/d85zc1NGLua7g2jhLKSC3b7jx0VnvNxVXEcdjduWVOy4jSYy9v8ZV5UMYLi0skQ6VWZ0nNZOE1DdVs", + "FPJrppfsocf2CcpezojyR70THP3z7/94d5bLsTsvxoml0Du7j76RQldosh7aq9tmG0kT/zYuE/8m3p39", + "8+//cDu5300QpvEzLPmPjLmovJW/zIiaEVHg1O6A9U9GyYDPkcOXwvQl+1PR2bRElPmciAgvCkTWrqm1", + "0wdKV1mVoArul/1Ok8wrpD9eQ3L1aI6hv6gqPrt9P1H1LMqzpqf6flse0GQl2UJ2ds/sn7vLS6pZ0RVN", + "RlMtQ47wNLOfrXIDXlzRBMEXXfjCHGMUmcsbpnpkNOZc9YbsLzPCEJwdHDD5SAKgU1JhhY7OTyW6plEE", + "WjMQgmU2MmRvC6TAvC6V/l+Rsg4apwoJEnNFkBVQYZIU1gIvjwlKGXZ+xt6QFaFiN1jFKwuWKyIYiUYz", + "gkMiZEPImI+Q/agWOLDVCZaKCEOh06QMr5Pfzi5Q+2TBcEwD9JsZ9YyHaUTQRZroO7xVhl5nyBJB5oSB", + "/qL5DrXz8gniqerySVcJQtwSYxgssztYJ9j8xfmldaPKrd6QvSEasISFJIQ1Oy4hkZphhULO/qRvLAnL", + "wxbnrwDdf5c7LclwImdcjRIe0WCxjote2NfPzdtfOq15kKTlY9qtHtEr8H5qgMypUCmONM0riYNeZ6hx", + "s3vEfuPFL6oflpZlGItV2YvVVIM0I4PPfVko9iuNRtKpVxodxOqVRh4nWm21ZtYm0D/OP8kF3SvKwqYD", + "/KbfvXXhKTPYOaS6a/kpEaSbJlOBwUt9e9JT5aQBsvUnvCaoxOc9zCAVpFLxuOBDRO2K1ZKW7ZtlAMx5", + "1NXnAtLjHYvGZpvLfvx4YZZgrmsdAx5Nxx7Tu+azlKEpneLxQpVVxJ3+MlHwX0E3vu+I6mJcDOEg4Ujx", + "1V5+OkHu3SbOOYiIGSk+mk+oZ+RMHsvNu1SioBJQY8mZHqKbBNRyhg66nlEtwUnkgADM4d1Z0eTSG7Iu", + "cLMBOskmyIbNhtQXD1wAMESbi8IiKHhz0HixhTB6d9ZDb7PV/kkihhWdExf0M8MSjQlhKAXJn4QwP3Dq", + "4gJSqdkjVdXPLRs08UFbYFni9lkPaY03xlak0NcixooG4AkY08p+wHNrDkrPpFkDKwo0jQSQVbERb8iU", + "SiUqkRGo/eb58d7e3pOqKLr7qNvf6e48ervTH/T1//+1eRDF7YdA+cY6KtMZ61spUqLjy9OTXSv3ludR", + "n/bxk8OPH7F6ckCv5ZNP8VhM/7aHNxIkdbtk7SR3JqF2KonoOlKrsdHnQip4ampcRF/t+blRXJfzUa+C", + "gdndW/3mXUSC+eIKrFf75rFaVeK5NjKhsLml/ehftcSZ35iC5cs68gLqdXWeUHn1VBB8FfJr5uHnWuCT", + "I8Ov/CbpVKv24wUiH7XGQEIkOFcTaUxfZcF3Z//x/uHewf5hv+8JgFpGfh7QUaC5UaMFvD4+RRFeEIHg", + "G9QG20OIxhEfl5H30d7B4eP+k53dpuswmnszOGRyufsKtS1E/uyCad2T0qJ2dx8f7O3t9Q8Odvcbrcqq", + "DI0W5dSLksjxeO/x/s7h7n4jKPgsIc9cQFpVyA991uckiaix+3RlQgI6oQGCkDakP0DtGNgZyYwQ5Ts5", + "xuFIWLHTy0cUppFcafQ2k9k3TfxinEaKJhExz+BAGulOsPMTGMnnUKCMETHK4vVuMJIN41trrHV7yV5B", + "pXDMEujOqASJJBekKInCgbmha+kcnGa+sPd1eGD30BAbXmo1qRuROYmKSGDYkV5szAVBGZ6YQyvtirI5", + "jmg4oixJa4zlNaB8ngqQS82gCI95qoz1CA6sOAkEA4BOMtHkulkMy3Murta6TzV3HYmUMT3MWsPPURTx", + "a33EVxo2wJkxsl+7KJ6CAJhZeYwtzD6X6I35wtjK8p+TVCHKFNeaKAvHiw7MREJ4jyFBpOJASXFwpaVN", + "O0xTSdMvi7zSQoizxJv5ctq5ITdEd2KswLfpi1BYTIkaSYXVWolFY8pbeP8CXm8cmaE/XGtsaQB3Rq43", + "AXQIR+lqtO1KhpO7gfgqf2Jmg8hfAi4saEh6CG4XOChcWGzlpl0oniQkzGw9vSG7MFcl+0miOJVgdL0y", + "cFAzQgXigk5peWJ7bTbgmLwJKjps+mp0LH64LKHCQ7DK1196PFFEGAi6TIFiiJ89hFanZWHf6rQsJSqD", + "xv3ogUjuLV9a4ovzy5u6CRPBJzTybBdM3Pap1bacA+3lfv+iu/N/jBNd4xuIaJQZs3jMQ9KrJOPA+804", + "z4vzy/O6NWWZUKi4uqU9ZY4MD+XITNsOItZEH2CGxgRZDcahv2Ys2SS57P3EJ8tOBI7JOJ1MiBjFHuPZ", + "c/0cmReMx4oydPa0LM9quXl5aD8VPC8dDqjCExzYRJZm0PcY5yrb6BSg+d5/XG+IYcN1Ia/6qIR9x0a9", + "9tCrLPcMvTi/lCh3PnmsduXjrQ15Op8tJA1wZEY0EeyUFY1tgJyNJeTz/ENrlvTIybFXNnQXAbXn0ySF", + "a3jxpnv6+t12HJJ5p7QmcBjNeET0urcK1GLuAljz+KwSkZjXWS8MYsimF6gAq+wGNwZS4b56oKO4wtFI", + "Rlx5VvNWP0TwELXfPTeBhHoFHZSUjlL/XoBCCb8PvDdGU6S6aS9gwqr5tHTB11qyY6NRFLdXmtR3VX4l", + "ODKZqmV8znMo3MHzq/JB86u1t9cO4pv31MX2NAh+PD47MQJDwJnClBGBMvNdKVINxKFWp9XVPCrEJAYP", + "6uQ/V0et1ZjjM3RZZdA9XkpzuxNjbk1KhiZy0ZyEKMaMTohUNiWjNLOc4d1HBwOTRBaSyf6jg16vd9Mw", + "w2d5XGGjo9g2UViFiMOenH3bOdxBNGGTvXxunR+9/bU1aG2nUmxHPMDRthxTNij8O/tn/gD+MP8cU+aN", + "QmyUd0gnS/mGZfel5lnm94HeCSNBhpAcFPg7y7Gr0YM0Skf0EwmRNwxf4anWawymflu8/Tdk6uVp5qqQ", + "oVeMJWiQrUc/rbagOoEK3rFzpkzRKE+AXLadflUKq1yZ2bOU1ZMQluXyRJH5K+Bsrm+TL7GnRPjds6XD", + "uDbK3SikHqz+i9X8Qq2EKQiSXX/3Wts4SdajsF9ozGhh0yRFG/rv4Ur3zgG+xvdWnv319L//+L/y/PHf", + "dv54+e7d/8xf/PfJK/o/76Lz1/cSELs6W+ReUz5WBuyAw6mU6tEUrc6wCjyC1oxLVQM1+wQpjmL9cQ8d", + "g0I4GLIuekkVETgaoGELJ7RngdkLeDxsoTb5iANlvkKcIT2UDWzb0h+fG7OQ/viz0zm/VMcIbQSbsEDO", + "gk1lOg55jCnbGrIhs2MhtxEJfn/9V4gCnKhUEH0iWraNFmgscJBHruWTd9BnnCRftoYMNF/yUQm9gwQL", + "laWyuRngoO2qTFyBfZ2EaI6jlEirOQ9ZxnfAFKAHMbabXmYcAZt9xeJaAxSvWsNFOfLysN/xnCPS7+mD", + "jKhUhKHMCkIlIC9quxDaw36JbBz2D9eHsGQ4tAL9ALuXc8wcUja4HwaBYWpDxEczpZIGNnZNp8wdQb++", + "fXuuwaD/e4HcQDkssiM2yh9OkogSaWyHKgIZyEYtb7V8JnFzug03ZIxn8FnUIEj0GUyM3r68QIqImDJD", + "99uBBueEBnp/4OqnUqYaFSlGR8dnz7Z6DartAGyz9a84x7fZDiseZWdMq7MRZhiv4dtBpycdLYbZG5oL", + "aBB685wLFBkCk9/rAbqUpBwiCUdlvP3mJKNFbpEzVH3Y2nIjJlVKMUBvMrkQZ0vJUmxzZHBD5vcShrUO", + "GRMXtDR6p7xWiHiy+pIlbRAFhBWy/k9g4fWkYPX190Ac7jxnVVvnze520UiqJ/OjRn72dy657N1Ud71p", + "7l05ZL6QIpGl3zXPm7uL/LNlPe4jVaNa5zzSj60r3mkd787QDEv2JwUPK7rHzt7jRlVr9KxN3dpFhzaf", + "mCVlt8rF32fuWJOJcEWjyEQ5SDplOEJPUPvi9MVvpy9fbqEuev36rHoUq77wnU+DNDyH2i/OLyG3DcuR", + "8wzVB0biPHiYfKRSyeX0hEYO1tVpf7+WUvO8+R5bt5iv57zSS9vYRCbefYb+/etkAa7M2/vW5DsrJN9R", + "7l0tUfblrZXps/n5drPo7mQ5pXw4H10pyhIunvurU+A6LeqJZT2SmnSSEJ2e5+VFcmOVG76ypye7vZ2D", + "w95Ov9/b6Tcy+eFgxdxnR8fNJ+/vGmPGAI8HQTggkybz15gOLWIboQ9H13gh0dCJ5cOW0QMKCkDhulvR", + "vZE7dznT8OsSC6uCyLrUwZukCjbLAfzWxKtVBccuyqXGGguHj/76TVXJSFORwMZS2K9GN7GmExTwNAq1", + "ADbWV9focyS0aqckKq/iBrf9kl0xfs3KWzfGUU0A/kiJWKB3Z2clE7wgE1vQqsHGIQaj5hx4cqNj2F0j", + "o69dTSEdbxMpeFVSWmBht55wV7TfuThNg3UN7Hi5KOr1rVNmwK3PfsWeKhaYkMxHaeqTsPQjl6lxeXl6", + "UjpwjA92DvuHT7qH452D7n7Y3+ninb2D7u4j3J/sBY/3akpGNo+t+fpwmfINrc+MAsCDNdMkwYUDfYey", + "eJdxqlAWC6cv57EWVVFBJjZ5QGBgsMFJegRgz4F+Ei0ysXnlx+dYX1T3bQL/Wv3FxSxVWo6Cb+QsVUj/", + "C5ast2DVjtVDmDs/QK84fCNcECnjVf3FvA7BWcuvV3Wdtg0bcuGlMJklYAP0PCNaGdmzZK4tif3T0FIb", + "+QxR3Vul2Dp7WoU4sU7LgLDVaTnIQDzZcmSZXYg3aaKINz5rP8ER0LA8cidVNKKfzJXTS6dS0cCoexhO", + "s+7a2WIJJBwZHlznxzPhIJZPZx+5W/3uDLUh//DPyGqD+l9bmc+veIX2d5/sPzl4vPvkoFEWQr7A9dT4", + "GIKVlhe3ljQHSTpypXNrtn58fgnMRzM2mcZGvbd7LwR9JoIHWlykDOW1ePPJn/SeFJMvQp6Oo4K5yGZt", + "QYR/k8LJNU6uP2g0p5MJ++NTcLX7N0HjnY8Hcnfs1a6yifyi6GnRxLmkt5Fx1xTG8auRgFBC1qaQvCES", + "doAuiEKAP11NsDRHzWKMLMq5RBMLcS9i7e/t7R0+frTbCK/s6goXZwQK5PIqz+wKClcM3kTtNxcXaLuA", + "cGZMF3iZCCL15kw2pfeeIVugrF+KydTKy54PS2oElhxr7NjzuBbk76zEYjdlgQ6hUpk0s3TLvdDe2+s/", + "3n90+KjZNbYq00h8XE1h7Hs2VECQgNB56eTbYBZ/e3SO9OhigoOyirCzu7f/6ODx4Y1WpW60KiUwkzFV", + "6kYLO3x88Gh/b3enWS6Uz/Rts/xKF7ZMuzyXzoMUntPwgGKZ9HbquIVPSiwZh1YYnwuB+rsal4qR+kfd", + "v5rIfDTqDbZ/+fP/7r7/j3/zZ2eVjCWSiG5IJqDJXJFFF5yhWWAFUngqe+XQJrCQawHYJjcpgmNICguu", + "iK3ugD8WF/6on3HSxSscL+1lZ/cQiiZm/167M3+Z0yW4Lse9rgy1zWN3q4GaN4nMzjPtqYRRaSEoGLW1", + "cFoU9AvZ4ltNDEB+1qPnqWt0oMXwpkHTq2Okz7GanbIJX/YZ3USRtpFnzpeQaIESSoqgkDBKQscTMo3a", + "yqgQyxZJgsKUWMgZmVNgC3Bs/GYJVjNQAuBDyqblKP6lCZuot2YNq+sqwLz2xSamPOmPenorUoCVMdpL", + "hPP4p0YeCCpHfm1teWBBpmmEBaomBqxYslzEEWVXTUaXi3jMIxog/UHVTDLhUcSvR/qR/AX2stVod/qD", + "Ue6yr5g9zOJswIY5kMq8+RZ+0bvcqoSOgUS1bb7fhk42TSyjXj/ec60Um9j5S0Y/FhC9nGy8v9uvixSs", + "GbQUI7icd3FTnmlR1nfjXUrEUVYnzuMvNh65imWgrF+U9uvbLbh8V8VFLktYqO2MrS6ZuwzXQlJ1IwGn", + "mde56lZwq9mWJCjPvn/46PFBw6z2b1JhVvT6+AaFZR6vUFRqTuqsiTR8+OjwyZO9/UdPdm8kdzoPVM35", + "1HmhiudTKQdZkYUf9eH/brQo44PyL6nGD1VeUKm041cv6MuKq5tnM9VYM1b12cpP0plPyopNM9VhhbR0", + "VBK5ClWP22QyIWCQGxm4dfPFVKLdGq0hwAkOqFp4NGt8DQFAKHulkpXTYPTKYj0gtWPbxEpNuWQ6zgMs", + "2m5y9B9GY67gwmHj4hgyHddp56+rsxrd3ETMhRXLTwPDi8EIX5TDdQZMdI1lyVui/w4UCTuFqtZVv5x5", + "o3l/FIfrWYuUPFLBl1nmb4dSPP7KcRa0uZKQXIX4KhZafwW1RADheE0cFx6O7ElXC9ZHyVTog2WAX/fV", + "aFwsW7OyLlCpxk3OdW8+b7N63MvfGQ528/kKoRU3+bBawQPw0a7Bgjwfu1NCiRpsUlysL+54B3n4xjfw", + "VZn41q2wkWR8+/OdJOAvHcdFIa7MX12ThKNVOT/H2WvOlJjgBcgNtUL+4739fn9vt/9VST+3VfSzME6d", + "F73wnVXUnSEA0Kk4QuYzX676ci2oKffrwCSVIDgegMcrwQFBEZlAyGxWbatR78PS1KsXb3PpbLBU5jpz", + "B+Xq91sd2qIv4wyYihvH5ku5bbRcKl85rq74/EYdFN1h+Zsolnz+B93+Xrd/8HZnb/DoYLCzcxcZRRmQ", + "6rwujz/tXD+OdvFkPzpcPP5jZ/Z4uhvveUOsvqP6spXK3JVys3bvCRHVMkDV8lmSRJSRrsw8nOuDPVbQ", + "EGN3X0s3bqaRutZlKxjIRXmTRT6CVQ6cakHkTQSf2tWvVKuryz89Wb3sr3IZVhfiR7DqUgCfmi0Gsl29", + "U6esIQ+6LLzYmAutdE2v40O+ABu45t6Tq4GiD0dLRLJ0a96v4N7LHM4jqk+5oGoWr2YV2WtZchZUC/8k", + "VbgciGjvgX7Y6rSiT/tlPLe/Nw85tclGGdLYoyyy/AY2XqgDt3qXplScVthhY7Y5eE5o1lsc7RZWHcpv", + "luzX8Tmb+WPRJi9M5EiqCyQpwTR/vATWSqDjHZVLXyVENq/aVJHHTdxNsUhWlu9wuyWb7Jfr+5HdGaxM", + "3F5dWe3YNeCvFDikpqWxLZWCCi+jNokTtXBp/c4zt3WzOMKjbECvMn7LCVX9J7eRCn65Mvf7By/wXgz5", + "dJOsDfZcwoXahEu/mf+kmo9hfGm2UG05f6BSflOqFR2/Y54yNQJn1rLFXj8zjjKb7DxNq9VgtmOmtm2Z", + "heUse4JDYAArXaP5jXOtvLtVrlHj8VsZ31HYWWEl9Wdj4n2XM5pXAOhcg+Z6RgQpHAR8kOeH3xBk1m21", + "PnvYxExqhaJbrWRsim1p3VhmfVoNYDUIMtfmMntfncdwhj9mM4Doh+WS0gP7KLQ7fPEUCui9cRVt6cQN", + "AcuoNpZ6uh6LVsHEYdXyYRSxannf5n3vxbP0ZwUlrLtbFeTM5yih5jI+atJFglRQtbjQpNJmWxEsiDhK", + "DRoCDYVNwM/55JBB/+ULuPcmHiv/C62M0wAdnZ8ClsSYQagOeneGIjohwSKIiE2AXoo1Bgvd6+PTrqnc", + "kPX/gXbhCgDiuiMcnZ9CgXXbqLvV7+32oHUiTwjDCW0NWnu9HSghr8EAW9yGgjrwp3Xg63sIHPA0tJz6", + "qXlFfyVwTBR0Y/rd4whXRJgCPRKNF3nIUh7FlGAqbPRSEoGP3gi9VA8A+ReOyg9ahVIyhnvdlLdJtbDe", + "C5K8tuf8XuOHTDiT5oR3+/1K93qcV+Le/ps04lI+fyMRBODlSWJYivVyYpA9gy+d1n5/50brWVs82zft", + "JcOpmnFBPxFY5qMbAuGrJj1lxqXqmlES+2J+8QCnilfu9/f6vGQax1gsHLhyWCVc1slvRCIM5XvHril6", + "D1mVHnK45YynUQjNyRLTrETTVYwUFr3pJ4RFMKNzMmSWnZhC6FhAjYsYaSQzpvHyXTFTm9M3dIhI9ZSH", + "iwp0s+G29XBdJ7flAL5xW//MHpbU9Pf3Wi+heYAMuLdrAmGYqbwWvekacEUgMnhCP3oHbBTirikgHAuB", + "hjVZzZTdLX8QCKQA++OnTrJnyIK3zPW0AkFZEKVhLhqUu/p7S+iZLvO2ucIV8UhSL+ANC5RitrTjwYyH", + "xGSwJgs148z8nY5TplLz91jwa0mE5tS2AoaFta0sblEXOt7QGKpQmFpbes5ts8Ttz1dk8aU3ZEdh7Gqj", + "2Z5+OJLcdp0wqQRUoqyzJOCuP0e7xh52bLtTmYrvxSLZZpk8VUmqeshshChbtgNehxrqckbCIVMcfRam", + "fc7iy/bnfMYvIGITHGo8KbxitrT9mYZf6lYtR1jvfgSvehQPAgAYtjSnGbb031OBtYidyhnCASQ86B+L", + "R9o2F5sLEF+2qhAOMEMJT9JIC4OAVKaZRmkMKIGEowgpuEruWy0UwUnW7MfG8/jq/NpgHhN9UblGUPG3", + "cJn6+4dbrXU9c8rD//fF61fICET6FMoRx0P2zAhgA/R5CBHGw9Zg6GKMh63OsEXYHH6zgcjD1hf/DiUJ", + "BPFZBWABwCz1/Oa1vDAAnBJlsDzXyVHvXy8NBzO38hmWaNii4bCVieHhFkArldbf2e2CLPiLXtkvZpoO", + "DX/p9Yq7/P2zGWWgb3MSjxS/ImzY+tJBhQdTqmbpOHv2vmbDNVEZFyVShNqG+2y5cnp6hwVGbDgXZiHi", + "ltpHC4RRTgOLxocxZVh4rWq2pGR9xpCpNmhfyzHqoN/fWh+5aLfqEbBLL+q7+GVJFNu9NSnESmDLUojZ", + "nMtN1MA0dSON7LUBMegpDl2lop/y3hp5z2rbBUkOvi8yBYO+ETEG2oo4ptXzyIljK3UXgxaQnAuqiIsz", + "NpoIdeJcjrxFnaSqgi7rGPt1tyyAJUYO//Y3gH8wb96zBeZ9sql5cWQ6DboOBg8LHeGwHCJ2/PryC6K+", + "B4zrb4qUutZS94i/DwV/XhArBOZAq1CzbWgfXTTGVItICIJjaUcxL2vF9QLW1L0gTKFn8GvP/tepP5Cf", + "/yHi0w8DZEAY8SmKKCPSxoBl3g7NFC0s4SMTbJN9Z2NvghlmUyJR2/DPf/79H7Aoyqb//Ps/tGht/oLr", + "vm3yiyCF/cOMYKHGBKsPA/QbIUkXR3RO3GYg55TMiVigvb5tmA+PPFWx5ZAN2RuiUsFkllmk9wUwMQPa", + "Jk16P5SlRNpgJWhBOrEpL8Yo6tHn3V02oNzoje4se3vNDgob0FzR4QDEMFNGFcWRVcZafrOa2XPJqFa1", + "7y5Z/NfTF0U+KoO9XbPAGxIYALHv3sEDu2nUvrh4ttVDIO4brIC0JtAb8mGsJtD7SZPW0yRDUcoEBaBs", + "aFOhIUqtdfjEvtPMPGxH/KHtw3UtX+oNxMYgQgQJHQB/Kg9NjMV+uDnDsc96e+JiVuvNt1+/3+IULlSj", + "kWZ8e+fscG8Z5rb9cQ6y+9CJUdt2rsyqEpd6LN8X0m+EjRRaeme8BHFTC3ljetoxZ5OIBgp13Vqg3lFM", + "Mt2tjCAPhRy8satG2O2rWlGgyPC2Swlytawvy5XLeeDdc4/KpDdhI3nVgxzXfnKSdahzQmXA9bcFbOkG", + "OLE1mY08k93TIhats1CdwO8Zy1kpP51kzf7thdycrcpOnbIqb9gAUTypEMR7JISVerGFOiEPCZsvs1N0", + "+TkrTFnfF2r2NycFbdqs5UPzh2TXCitg01RwlnUlrEMv27fwDg/azuDZ+AUR7labhZoyo/m2zKcomJHg", + "ymwI3NWrleFT80ozXdiM90OrwqaD5A1EGHsGP2WWBtpvDqtVGu+pLaB7dwovzHAjfff2PMEWwTxAhtiU", + "sbNpm9q0WC5YsPVDOYM3wt4MsB8kdztPo8j5ROZEqLwfZpEpbH+GKKb1wr67bSv5w+Wbl13CAg5ha1nI", + "lV+qcm3sblfkNwdmtvITTZooiQAqhxj1EvU3nL+JLkRZT5R/331uu6L8++5z0xfl3/eOTGeUrTtDlv6m", + "SPOmRfAHjHxaAqdloAFpMs3m1oms2VsNpVb3/o8tuNrupDcRXTNA/5Rem0ivRXCtFGCzRrF3KMLa/pv3", + "47TJkM0HbXjkQhp/MNF1s3ZAi5GuaBKVZceIrYrLRd7zkjKUSvIAYy5phnFFPtLQoJ1fyJX8xKHu6UnH", + "tjM1TUizBJMNmbfdOjYu7dp5N2/bPorHdJryVBZzV6B7LZE22SkiZQL80OTwnD3XSuLfMZb2N8k6Ni5o", + "/8T7O1IBqgdqiLfxUa1TAtxbTZUA+z70iTWdh0zu2xvX0ciWN9mqiUN0/bqaonGpXdxyfKRvXbXKCbrU", + "6kuuMyBQIwZD9l/uk98VwfH7X1x6U9rv7x5kzwibv//FZTmxM4c3hClBiURYEHT06gS8hFPIjYfijXl+", + "X3U9piQj4KFrXP0vrTnlTtPmqpNDz5+qUyPVqQCu1apT1lnrLnWncnGijStPDt98ALdFPH6qT5tQn2Q6", + "mdCAEqby6uVL8WW2+cEDzFNj1pNUiAspceDG6lPe7m61ZJrXSdx4TFA2+ea1JleS8WHG23OTYRM6PSVn", + "hvWKyveGD/3NEufNKygPGcWMJlAF3TIh2p7Y4ul+AeE5F1dNMc9TuPXWEfD2pZPiDr9D2UQvD8qW3L+I", + "AszbhOVrpClLLhu4kEvVeO8zGtRBwmq9JsGSsmnWpPiaqhlPTbmWkf3R1H/Tt8J2wgKRJ7Cj3jd50bNv", + "QAB9xRWicRKRmEB9uK7BJugOnSYJF1lPSioLtatvRv70tSnG5pqqObY1ewfZCt9gxcu6SoNBf/m4vFQz", + "4tP1CbrZ5C4b1ZOhO2SX0lSP+WBE4Q8oI7JIcSRJRAKFrmc0mEG2rv7NlNqHRFqcJB+y8hxbA/QCbmqx", + "YAhM3pZEUBxB518emabVH+Zx/GGwXGvu3dkZfGQSdU1VuQ8D5OrLZQxC6reK2bd6FxGWCr2yOcVtjUmC", + "R5E50Q+aCxX2t2XzcvNKJkPmy9Fl5NoOSCfoQyFd90NNvq4jqC/1Kd2TvNSpr4Bl9qI4EgA4g5uEhTU2", + "Mg01f6buTt9bMrVh1rBZxh0nDS8t5iWfZtW3SqiMk6Qp+tplAhbP43gFDqN2ofq9VCFP1Z+lCokQ8LHF", + "7jrkRm0cmH8ofKURldk2iq5/AKCf165pKuB4QaWJaqEAtPnXPI5bnZZdj6ej+rdnX1cHXDaz6ZMppFj/", + "lLRvkjxdJvaF7OkK57A9d+pFbttK6IfX9yyg7tumcA/2sXwVlDlRBc6W5wXkH1TSpekyVZXFTLV73x3J", + "2lTV35KyUfkir6v/L6iimr1We4ttWEnNQOzTzEr9UO5dO806bfzUUDMNlQsUpma6SoOkH1btzAgKSllJ", + "87Ti6dfqnlnBugzM0EOVrXQI5DRv+7P78/QrxIXvhBJ2ahvE1JVGyjf9PZDcmn6OjWjuPclJlq0WBIR7", + "JMGus+SmKXAGFa3uZVTuuyDD5sJl1LhIc5TATFLXNPYnMS6ZAY2l9GuJsRM+l2yBBfJMWTeJcB1dtnJq", + "LQGuNF+6V33t9glhTWupL5YS3ifhy5WjjRG704y8GYJn2/H94CpqwIUw8XMQlveQsjoLTtKCrt1OcCpJ", + "J6MQHeeof3d2tlVHJYRaSSOEesAUotJGLg79TewFDV25/+OzE9scgEokUtZDr2MKNfivCEmgniflqUQQ", + "ANkrtnWr6S2f920jTIlFwilTa1eRv3o3i/nyVRXON0wnbV77D29Hs13RHxqRAtqhxRW7gdVapDLdDL1+", + "Seeno8x0CIDOqGOe6tGXWs1B42y5kIrExkk5SU1vUSiFYkvn2u9MsF4HUSWRvg8dCG5KiIgpdI6UQzYm", + "Ey2GJUTouaElJo1Iwd/ic+VdKJxRzXND+r4PXx50nwP3FVZ1UCs3nsNJ4hrP+fxFWa+8r17Sc3DOIbmI", + "xzyiAYoou5KoHdEro3SguUSR/mNrpXdvBN/ddmHgr79ZGtKnbMK9pRINzmbI/CNQuNMKWXPRCw+OrL0g", + "xcvi6A8ctJ+sybV0TRAcQa/VLK4YpYpG9JMhdXoQKhUNTPcpnMEOGueY+XpDdkaU0O9gQVDAo4gEyhlX", + "thPBg+1h2u/vBQmFhJA9AosDglf/OIYZj88v4T3T3KczZPofMPDbo3NENUwn2NoICgtlRF1zcYVOt1+v", + "iXe4ADD9CzsMzQZXXQv/gf90Zd88aLT2DsmaK8qTVQoQT354j7aV4H5aCx6mtQCi9rPdtKcCByAUy1mq", + "Qn7N/JYB03xWbn82f5yuy/1QOJi9c521vw9p1zbiXTeN2+CDuJR2TyExpVzvxUFheyU/0EpXGnBuCyDE", + "FLNY/FzA9GD/0bD79o3yRTh+h65JC1FXJvm7uVub5nx2DS6lsQiPh3LNDaa5nUDrzaL1KcvfXKubBakQ", + "hCkoipOLlgFOcEDVooNw5PrS2t5SmQ0p77E/FgRfaU7bG7I3Weao7W2ltauOU61QSOWVGcFqTz30ek6E", + "TMfZ4hAQJqPnAfBta9oAR4Hp6UomExIoOiem2aqs0b6ypdxlCeN8Es9Bu4cWdA9N5fDjBJxejhZW6yiF", + "BtYWsrjI3mpWyCIbtRD+UwiNWRnkPXIvjoAT3cRk55n8itbmAdhHNwvX+01/1HDucliYfxH20Tfu8oct", + "GHhRCM9pWvYiR/mHVoGisPLS3S2FuK1PhW8c03aXMWbrUuGzyTedCn/hDXN6YJW6cClwrS4H/vtDhP5m", + "46s3nQP/sHFLyxZyCXT1lKhBLvx3gYF3kwR/z/kFX5EE/11FvEIS8/1lHnxXsa42ZjOLdf2Z5n6XIa4m", + "1x1SeutCXA3Vs6bolZrTO/tOM73JjvhDi/TWnHkDgd6dw88qdg10iAKwHFuu0B9gBtLeABInauHsVXwC", + "kTl5yUVJP0F8ny+XMDNL310K31dYbG8PPRye1tprf1a/25hJOK8dfnry8EveFe9cidNsazbUxSKY0Xkp", + "hW3VDbYgSgTpJjwBS2xoAGbh4ZibwqI3/YTs8L0hezsj7l+IugIiJEQhFSRQ0QJRpjhQBDPHnyQSXKsG", + "8JyLhc/AW7y5zwWPj+xu1jBIe6esuSwPBIwXXc21unNHbVYY2b7BqXWGP9I4jYHgIcrQi6eoTT4qYepZ", + "oIlWhRCdZCAlHwNCQgk4uVVc8E6/xvZJP5HRdNxklSsqk7y2lV9QkErFY3f2pyeojVPFu1PC9Flo2X8C", + "om0i+JyGpp5wDtQ5jwxUd2oAelPLrBMukMJTaUPHc7XDrPLeBZsmXGr6iSZlWmGiJVuD1pgyDAtdWxik", + "fNFM4K6eD1MIn8svlEOn1k++Vm1kDtjERQZExTmKtNy/9ZP3PWTeVwyAcIyuxAKbVXttFhPRMFThLiq9", + "ZvEymzVuv/t+3PiFxs8P0MA+z7TUOuP694WC/c3xh00b1d894LCvF8Rp5AWDOgygR/QhzEse4AiFZE4i", + "nsRa1jTvtjqtVEStQWumVDLY3o70ezMu1eCwf9hvfXn/5f8HAAD//wD0kXu+JQEA", } // GetSwagger returns the content of the embedded swagger specification file diff --git a/lib/providers/providers.go b/lib/providers/providers.go index 25cc3536..621afe81 100644 --- a/lib/providers/providers.go +++ b/lib/providers/providers.go @@ -24,6 +24,7 @@ import ( "github.com/kernel/hypeman/lib/paths" "github.com/kernel/hypeman/lib/registry" "github.com/kernel/hypeman/lib/resources" + "github.com/kernel/hypeman/lib/snapshot" "github.com/kernel/hypeman/lib/system" "github.com/kernel/hypeman/lib/vm_metrics" "github.com/kernel/hypeman/lib/volumes" @@ -125,7 +126,15 @@ func ProvideInstanceManager(p *paths.Paths, cfg *config.Config, imageManager ima meter := otel.GetMeterProvider().Meter("hypeman") tracer := otel.GetTracerProvider().Tracer("hypeman") defaultHypervisor := hypervisor.Type(cfg.Hypervisor.Default) - return instances.NewManager(p, imageManager, systemManager, networkManager, deviceManager, volumeManager, limits, defaultHypervisor, meter, tracer), nil + level := cfg.Snapshot.CompressionDefault.Level + snapshotDefaults := instances.SnapshotPolicy{ + Compression: &snapshot.SnapshotCompressionConfig{ + Enabled: cfg.Snapshot.CompressionDefault.Enabled, + Algorithm: snapshot.SnapshotCompressionAlgorithm(strings.ToLower(cfg.Snapshot.CompressionDefault.Algorithm)), + Level: &level, + }, + } + return instances.NewManager(p, imageManager, systemManager, networkManager, deviceManager, volumeManager, limits, defaultHypervisor, snapshotDefaults, meter, tracer), nil } // ProvideVolumeManager provides the volume manager diff --git a/lib/snapshot/types.go b/lib/snapshot/types.go index ef2bd8a8..c1e2af13 100644 --- a/lib/snapshot/types.go +++ b/lib/snapshot/types.go @@ -19,15 +19,42 @@ const ( // Snapshot is a centrally stored immutable snapshot resource. type Snapshot struct { - Id string `json:"id"` - Name string `json:"name"` - Kind SnapshotKind `json:"kind"` - Metadata tags.Metadata `json:"metadata,omitempty"` - SourceInstanceID string `json:"source_instance_id"` - SourceName string `json:"source_instance_name"` - SourceHypervisor hypervisor.Type - CreatedAt time.Time `json:"created_at"` - SizeBytes int64 `json:"size_bytes"` + Id string `json:"id"` + Name string `json:"name"` + Kind SnapshotKind `json:"kind"` + Metadata tags.Metadata `json:"metadata,omitempty"` + SourceInstanceID string `json:"source_instance_id"` + SourceName string `json:"source_instance_name"` + SourceHypervisor hypervisor.Type + CreatedAt time.Time `json:"created_at"` + SizeBytes int64 `json:"size_bytes"` + CompressionState string `json:"compression_state,omitempty"` + CompressionError string `json:"compression_error,omitempty"` + Compression *SnapshotCompressionConfig `json:"compression,omitempty"` + CompressedSizeBytes *int64 `json:"compressed_size_bytes,omitempty"` + UncompressedSizeBytes *int64 `json:"uncompressed_size_bytes,omitempty"` +} + +// SnapshotCompressionAlgorithm defines supported compression algorithms. +type SnapshotCompressionAlgorithm string + +const ( + SnapshotCompressionAlgorithmZstd SnapshotCompressionAlgorithm = "zstd" + SnapshotCompressionAlgorithmLz4 SnapshotCompressionAlgorithm = "lz4" +) + +const ( + SnapshotCompressionStateNone = "none" + SnapshotCompressionStateCompressing = "compressing" + SnapshotCompressionStateCompressed = "compressed" + SnapshotCompressionStateError = "error" +) + +// SnapshotCompressionConfig defines requested or effective compression config. +type SnapshotCompressionConfig struct { + Enabled bool `json:"enabled"` + Algorithm SnapshotCompressionAlgorithm `json:"algorithm,omitempty"` + Level *int `json:"level,omitempty"` } // ListSnapshotsFilter contains optional filters for listing snapshots. diff --git a/openapi.yaml b/openapi.yaml index ea6a3190..3d710cb2 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -198,6 +198,8 @@ components: enum: [cloud-hypervisor, firecracker, qemu, vz] description: Hypervisor to use for this instance. Defaults to server configuration. example: cloud-hypervisor + snapshot_policy: + $ref: "#/components/schemas/SnapshotPolicy" skip_kernel_headers: type: boolean description: | @@ -311,6 +313,54 @@ components: format: int64 description: Total payload size in bytes example: 104857600 + compression_state: + type: string + enum: [none, compressing, compressed, error] + description: Compression status of the snapshot payload memory file + example: compressed + compression_error: + type: string + description: Compression error message when compression_state is error + nullable: true + example: "write compressed stream: no space left on device" + compression: + $ref: "#/components/schemas/SnapshotCompressionConfig" + compressed_size_bytes: + type: integer + format: int64 + nullable: true + description: Compressed memory payload size in bytes + example: 73400320 + uncompressed_size_bytes: + type: integer + format: int64 + nullable: true + description: Uncompressed memory payload size in bytes + example: 4294967296 + + SnapshotCompressionConfig: + type: object + required: [enabled] + properties: + enabled: + type: boolean + description: Enable snapshot memory compression + example: true + algorithm: + type: string + enum: [zstd, lz4] + description: Compression algorithm (defaults to zstd when enabled) + example: zstd + level: + type: integer + description: Compression level for zstd only + example: 1 + + SnapshotPolicy: + type: object + properties: + compression: + $ref: "#/components/schemas/SnapshotCompressionConfig" CreateSnapshotRequest: type: object @@ -326,6 +376,14 @@ components: example: pre-upgrade metadata: $ref: "#/components/schemas/MetadataTags" + compression: + $ref: "#/components/schemas/SnapshotCompressionConfig" + + StandbyInstanceRequest: + type: object + properties: + compression: + $ref: "#/components/schemas/SnapshotCompressionConfig" RestoreSnapshotRequest: type: object @@ -492,6 +550,8 @@ components: enum: [cloud-hypervisor, firecracker, qemu, vz] description: Hypervisor running this instance example: cloud-hypervisor + snapshot_policy: + $ref: "#/components/schemas/SnapshotPolicy" PathInfo: type: object @@ -1625,6 +1685,12 @@ paths: schema: type: string description: Instance ID or name + requestBody: + required: false + content: + application/json: + schema: + $ref: "#/components/schemas/StandbyInstanceRequest" responses: 200: description: Instance in standby @@ -1632,6 +1698,12 @@ paths: application/json: schema: $ref: "#/components/schemas/Instance" + 400: + description: Invalid request payload + content: + application/json: + schema: + $ref: "#/components/schemas/Error" 404: description: Instance not found content: From 47a3197424a84b7aa8a47f60e8ffb995450d1d41 Mon Sep 17 00:00:00 2001 From: Steven Miller Date: Thu, 19 Mar 2026 17:29:37 -0400 Subject: [PATCH 02/31] Add CH snapshot compression restore coverage --- .../compression_integration_linux_test.go | 327 ++++++++++++++++++ lib/instances/restore.go | 3 +- lib/instances/snapshot_compression.go | 95 ++++- lib/instances/snapshot_compression_test.go | 28 ++ lib/instances/standby.go | 6 +- lib/snapshot/README.md | 57 +++ skills/hypeman-remote-linux-tests/SKILL.md | 107 ++++++ .../agents/openai.yaml | 4 + 8 files changed, 618 insertions(+), 9 deletions(-) create mode 100644 lib/instances/compression_integration_linux_test.go create mode 100644 skills/hypeman-remote-linux-tests/SKILL.md create mode 100644 skills/hypeman-remote-linux-tests/agents/openai.yaml diff --git a/lib/instances/compression_integration_linux_test.go b/lib/instances/compression_integration_linux_test.go new file mode 100644 index 00000000..1cc14546 --- /dev/null +++ b/lib/instances/compression_integration_linux_test.go @@ -0,0 +1,327 @@ +//go:build linux + +package instances + +import ( + "context" + "fmt" + "os" + "testing" + "time" + + "github.com/kernel/hypeman/cmd/api/config" + "github.com/kernel/hypeman/lib/devices" + "github.com/kernel/hypeman/lib/hypervisor" + "github.com/kernel/hypeman/lib/images" + "github.com/kernel/hypeman/lib/network" + "github.com/kernel/hypeman/lib/paths" + "github.com/kernel/hypeman/lib/resources" + snapshotstore "github.com/kernel/hypeman/lib/snapshot" + "github.com/kernel/hypeman/lib/system" + "github.com/kernel/hypeman/lib/volumes" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +type compressionIntegrationHarness struct { + name string + hypervisor hypervisor.Type + setup func(t *testing.T) (*manager, string) + requirePrereqs func(t *testing.T) + waitHypervisorUp func(ctx context.Context, inst *Instance) error + testImplicitLZ4 bool +} + +func TestCloudHypervisorStandbyRestoreCompressionScenarios(t *testing.T) { + t.Parallel() + + runStandbyRestoreCompressionScenarios(t, compressionIntegrationHarness{ + name: "cloud-hypervisor", + hypervisor: hypervisor.TypeCloudHypervisor, + setup: func(t *testing.T) (*manager, string) { + return setupCompressionTestManagerForHypervisor(t, hypervisor.TypeCloudHypervisor) + }, + requirePrereqs: requireKVMAccess, + waitHypervisorUp: func(ctx context.Context, inst *Instance) error { + return waitForVMReady(ctx, inst.SocketPath, 10*time.Second) + }, + testImplicitLZ4: true, + }) +} + +func TestFirecrackerStandbyRestoreCompressionScenarios(t *testing.T) { + t.Parallel() + + runStandbyRestoreCompressionScenarios(t, compressionIntegrationHarness{ + name: "firecracker", + hypervisor: hypervisor.TypeFirecracker, + setup: func(t *testing.T) (*manager, string) { + return setupCompressionTestManagerForHypervisor(t, hypervisor.TypeFirecracker) + }, + requirePrereqs: requireFirecrackerIntegrationPrereqs, + }) +} + +func TestQEMUStandbyRestoreCompressionScenarios(t *testing.T) { + t.Parallel() + + runStandbyRestoreCompressionScenarios(t, compressionIntegrationHarness{ + name: "qemu", + hypervisor: hypervisor.TypeQEMU, + setup: func(t *testing.T) (*manager, string) { + return setupCompressionTestManagerForHypervisor(t, hypervisor.TypeQEMU) + }, + requirePrereqs: requireQEMUUsable, + waitHypervisorUp: func(ctx context.Context, inst *Instance) error { + return waitForQEMUReady(ctx, inst.SocketPath, 10*time.Second) + }, + }) +} + +func setupCompressionTestManagerForHypervisor(t *testing.T, hvType hypervisor.Type) (*manager, string) { + t.Helper() + + tmpDir, err := os.MkdirTemp("", "hmcmp-") + require.NoError(t, err) + t.Cleanup(func() { + _ = os.RemoveAll(tmpDir) + }) + prepareIntegrationTestDataDir(t, tmpDir) + + cfg := &config.Config{ + DataDir: tmpDir, + Network: legacyParallelTestNetworkConfig(testNetworkSeq.Add(1)), + } + + p := paths.New(tmpDir) + imageManager, err := images.NewManager(p, 1, nil) + require.NoError(t, err) + + systemManager := system.NewManager(p) + networkManager := network.NewManager(p, cfg, nil) + deviceManager := devices.NewManager(p) + volumeManager := volumes.NewManager(p, 0, nil) + limits := ResourceLimits{ + MaxOverlaySize: 100 * 1024 * 1024 * 1024, + MaxVcpusPerInstance: 0, + MaxMemoryPerInstance: 0, + } + mgr := NewManager(p, imageManager, systemManager, networkManager, deviceManager, volumeManager, limits, hvType, SnapshotPolicy{}, nil, nil).(*manager) + + resourceMgr := resources.NewManager(cfg, p) + resourceMgr.SetInstanceLister(mgr) + resourceMgr.SetImageLister(imageManager) + resourceMgr.SetVolumeLister(volumeManager) + require.NoError(t, resourceMgr.Initialize(context.Background())) + mgr.SetResourceValidator(resourceMgr) + + t.Cleanup(func() { + cleanupOrphanedProcesses(t, mgr) + }) + + return mgr, tmpDir +} + +func runStandbyRestoreCompressionScenarios(t *testing.T, harness compressionIntegrationHarness) { + t.Helper() + harness.requirePrereqs(t) + + mgr, tmpDir := harness.setup(t) + ctx := context.Background() + p := paths.New(tmpDir) + + imageManager, err := images.NewManager(p, 1, nil) + require.NoError(t, err) + createNginxImageAndWait(t, ctx, imageManager) + + systemManager := system.NewManager(p) + require.NoError(t, systemManager.EnsureSystemFiles(ctx)) + + inst, err := mgr.CreateInstance(ctx, CreateInstanceRequest{ + Name: fmt.Sprintf("compression-%s", harness.name), + Image: integrationTestImageRef(t, "docker.io/library/nginx:alpine"), + Size: 1024 * 1024 * 1024, + HotplugSize: 512 * 1024 * 1024, + OverlaySize: 10 * 1024 * 1024 * 1024, + Vcpus: 1, + NetworkEnabled: false, + Hypervisor: harness.hypervisor, + }) + require.NoError(t, err) + + deleted := false + t.Cleanup(func() { + if !deleted { + _ = mgr.DeleteInstance(context.Background(), inst.Id) + } + }) + + inst = waitForRunningAndExecReady(t, ctx, mgr, inst.Id, harness.waitHypervisorUp) + + inFlightCompression := &snapshotstore.SnapshotCompressionConfig{ + Enabled: true, + Algorithm: snapshotstore.SnapshotCompressionAlgorithmZstd, + Level: intPtr(19), + } + inst = runCompressionCycle(t, ctx, mgr, p, inst, harness.waitHypervisorUp, "in-flight-zstd-19", inFlightCompression, false) + + completedCases := []struct { + name string + cfg *snapshotstore.SnapshotCompressionConfig + }{ + { + name: "zstd-1", + cfg: &snapshotstore.SnapshotCompressionConfig{ + Enabled: true, + Algorithm: snapshotstore.SnapshotCompressionAlgorithmZstd, + Level: intPtr(1), + }, + }, + { + name: "zstd-19", + cfg: &snapshotstore.SnapshotCompressionConfig{ + Enabled: true, + Algorithm: snapshotstore.SnapshotCompressionAlgorithmZstd, + Level: intPtr(19), + }, + }, + { + name: "lz4", + cfg: &snapshotstore.SnapshotCompressionConfig{ + Enabled: true, + Algorithm: snapshotstore.SnapshotCompressionAlgorithmLz4, + }, + }, + } + + for _, tc := range completedCases { + inst = runCompressionCycle(t, ctx, mgr, p, inst, harness.waitHypervisorUp, tc.name, tc.cfg, true) + } + + if harness.testImplicitLZ4 { + inst = runCompressionCycle(t, ctx, mgr, p, inst, harness.waitHypervisorUp, "implicit-default-lz4", nil, true) + } + + require.NoError(t, mgr.DeleteInstance(ctx, inst.Id)) + deleted = true +} + +func runCompressionCycle( + t *testing.T, + ctx context.Context, + mgr *manager, + p *paths.Paths, + inst *Instance, + waitHypervisorUp func(ctx context.Context, inst *Instance) error, + label string, + cfg *snapshotstore.SnapshotCompressionConfig, + waitForCompression bool, +) *Instance { + t.Helper() + + markerPath := fmt.Sprintf("/tmp/%s.txt", label) + markerValue := fmt.Sprintf("%s-%d", label, time.Now().UnixNano()) + writeGuestMarker(t, ctx, inst, markerPath, markerValue) + + inst, err := mgr.StandbyInstance(ctx, inst.Id, StandbyInstanceRequest{ + Compression: cloneCompressionConfig(cfg), + }) + require.NoError(t, err) + require.Equal(t, StateStandby, inst.State) + require.True(t, inst.HasSnapshot) + + snapshotDir := p.InstanceSnapshotLatest(inst.Id) + jobKey := mgr.snapshotJobKeyForInstance(inst.Id) + + if waitForCompression { + waitForCompressionJobCompletion(t, mgr, jobKey, 3*time.Minute) + requireCompressedSnapshotFile(t, snapshotDir, effectiveCompressionForCycle(inst.HypervisorType, cfg)) + } else { + waitForCompressionJobStart(t, mgr, jobKey, 15*time.Second) + } + + inst, err = mgr.RestoreInstance(ctx, inst.Id) + require.NoError(t, err) + inst = waitForRunningAndExecReady(t, ctx, mgr, inst.Id, waitHypervisorUp) + assertGuestMarker(t, ctx, inst, markerPath, markerValue) + + waitForCompressionJobCompletion(t, mgr, jobKey, 30*time.Second) + return inst +} + +func effectiveCompressionForCycle(hvType hypervisor.Type, cfg *snapshotstore.SnapshotCompressionConfig) snapshotstore.SnapshotCompressionConfig { + if cfg != nil { + normalized, err := normalizeCompressionConfig(cfg) + if err != nil { + panic(err) + } + return normalized + } + if hvType == hypervisor.TypeCloudHypervisor { + return defaultCloudHypervisorStandbyCompressionPolicy() + } + return snapshotstore.SnapshotCompressionConfig{Enabled: false} +} + +func waitForRunningAndExecReady(t *testing.T, ctx context.Context, mgr *manager, instanceID string, waitHypervisorUp func(context.Context, *Instance) error) *Instance { + t.Helper() + + inst, err := waitForInstanceState(ctx, mgr, instanceID, StateRunning, 30*time.Second) + require.NoError(t, err) + if waitHypervisorUp != nil { + require.NoError(t, waitHypervisorUp(ctx, inst)) + } + require.NoError(t, waitForExecAgent(ctx, mgr, instanceID, 30*time.Second)) + return inst +} + +func writeGuestMarker(t *testing.T, ctx context.Context, inst *Instance, path string, value string) { + t.Helper() + output, exitCode, err := execCommand(ctx, inst, "sh", "-c", fmt.Sprintf("printf %q > %s && sync", value, path)) + require.NoError(t, err) + require.Equal(t, 0, exitCode, output) +} + +func assertGuestMarker(t *testing.T, ctx context.Context, inst *Instance, path string, expected string) { + t.Helper() + output, exitCode, err := execCommand(ctx, inst, "cat", path) + require.NoError(t, err) + require.Equal(t, 0, exitCode, output) + assert.Equal(t, expected, output) +} + +func waitForCompressionJobStart(t *testing.T, mgr *manager, key string, timeout time.Duration) { + t.Helper() + deadline := time.Now().Add(timeout) + for time.Now().Before(deadline) { + mgr.compressionMu.Lock() + job := mgr.compressionJobs[key] + mgr.compressionMu.Unlock() + if job != nil { + return + } + time.Sleep(100 * time.Millisecond) + } + t.Fatalf("compression job %q did not start within %v", key, timeout) +} + +func waitForCompressionJobCompletion(t *testing.T, mgr *manager, key string, timeout time.Duration) { + t.Helper() + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + require.NoError(t, mgr.waitCompressionJobContext(ctx, key)) +} + +func requireCompressedSnapshotFile(t *testing.T, snapshotDir string, cfg snapshotstore.SnapshotCompressionConfig) { + t.Helper() + require.True(t, cfg.Enabled) + + rawPath, rawExists := findRawSnapshotMemoryFile(snapshotDir) + assert.False(t, rawExists, "raw snapshot memory should be removed after compression, found %q", rawPath) + + compressedPath, algorithm, ok := findCompressedSnapshotMemoryFile(snapshotDir) + require.True(t, ok, "compressed snapshot memory should exist in %s", snapshotDir) + assert.Equal(t, cfg.Algorithm, algorithm) + _, err := os.Stat(compressedPath) + require.NoError(t, err) +} diff --git a/lib/instances/restore.go b/lib/instances/restore.go index fc57ce03..6020c167 100644 --- a/lib/instances/restore.go +++ b/lib/instances/restore.go @@ -69,7 +69,8 @@ func (m *manager) restoreInstance( // 3. Get snapshot directory snapshotDir := m.paths.InstanceSnapshotLatest(id) - if err := m.ensureSnapshotMemoryReady(ctx, snapshotDir, m.snapshotJobKeyForInstance(id)); err != nil { + waitForCompression := stored.HypervisorType == hypervisor.TypeCloudHypervisor + if err := m.ensureSnapshotMemoryReady(ctx, snapshotDir, m.snapshotJobKeyForInstance(id), waitForCompression); err != nil { return nil, fmt.Errorf("prepare standby snapshot memory: %w", err) } starter, err := m.getVMStarter(stored.HypervisorType) diff --git a/lib/instances/snapshot_compression.go b/lib/instances/snapshot_compression.go index 8974ede5..4cab65f2 100644 --- a/lib/instances/snapshot_compression.go +++ b/lib/instances/snapshot_compression.go @@ -10,6 +10,7 @@ import ( "strings" "time" + "github.com/kernel/hypeman/lib/hypervisor" "github.com/kernel/hypeman/lib/logger" snapshotstore "github.com/kernel/hypeman/lib/snapshot" "github.com/klauspost/compress/zstd" @@ -103,6 +104,55 @@ func (m *manager) resolveSnapshotCompressionPolicy(stored *StoredMetadata, overr return snapshotstore.SnapshotCompressionConfig{Enabled: false}, nil } +func defaultCloudHypervisorStandbyCompressionPolicy() snapshotstore.SnapshotCompressionConfig { + return snapshotstore.SnapshotCompressionConfig{ + Enabled: true, + Algorithm: snapshotstore.SnapshotCompressionAlgorithmLz4, + } +} + +func (m *manager) resolveStandbyCompressionPolicy(stored *StoredMetadata, override *snapshotstore.SnapshotCompressionConfig) (*snapshotstore.SnapshotCompressionConfig, error) { + if override != nil { + cfg, err := normalizeCompressionConfig(override) + if err != nil { + return nil, err + } + if !cfg.Enabled { + return nil, nil + } + return &cfg, nil + } + + if stored != nil && stored.SnapshotPolicy != nil && stored.SnapshotPolicy.Compression != nil { + cfg, err := normalizeCompressionConfig(stored.SnapshotPolicy.Compression) + if err != nil { + return nil, err + } + if !cfg.Enabled { + return nil, nil + } + return &cfg, nil + } + + if m.snapshotDefaults.Compression != nil { + cfg, err := normalizeCompressionConfig(m.snapshotDefaults.Compression) + if err != nil { + return nil, err + } + if !cfg.Enabled { + return nil, nil + } + return &cfg, nil + } + + if stored != nil && stored.HypervisorType == hypervisor.TypeCloudHypervisor { + cfg := defaultCloudHypervisorStandbyCompressionPolicy() + return &cfg, nil + } + + return nil, nil +} + func (m *manager) snapshotJobKeyForInstance(instanceID string) string { return "instance:" + instanceID } @@ -192,10 +242,32 @@ func (m *manager) waitCompressionJob(key string, timeout time.Duration) { } } -func (m *manager) ensureSnapshotMemoryReady(ctx context.Context, snapshotDir, jobKey string) error { +func (m *manager) waitCompressionJobContext(ctx context.Context, key string) error { + m.compressionMu.Lock() + job := m.compressionJobs[key] + m.compressionMu.Unlock() + if job == nil { + return nil + } + + select { + case <-job.done: + return nil + case <-ctx.Done(): + return ctx.Err() + } +} + +func (m *manager) ensureSnapshotMemoryReady(ctx context.Context, snapshotDir, jobKey string, waitForCompression bool) error { if jobKey != "" { - m.cancelCompressionJob(jobKey) - m.waitCompressionJob(jobKey, 2*time.Second) + if waitForCompression { + if err := m.waitCompressionJobContext(ctx, jobKey); err != nil { + return err + } + } else { + m.cancelCompressionJob(jobKey) + m.waitCompressionJob(jobKey, 2*time.Second) + } } if _, ok := findRawSnapshotMemoryFile(snapshotDir); ok { @@ -269,8 +341,8 @@ func compressSnapshotMemoryFile(ctx context.Context, rawPath string, cfg snapsho compressedPath := compressedPathFor(rawPath, cfg.Algorithm) tmpPath := compressedPath + ".tmp" + removeCompressedSnapshotArtifacts(rawPath) _ = os.Remove(tmpPath) - _ = os.Remove(compressedPath) if err := runCompression(ctx, rawPath, tmpPath, cfg); err != nil { _ = os.Remove(tmpPath) @@ -323,6 +395,9 @@ func runCompression(ctx context.Context, srcPath, dstPath string, cfg snapshotst } case snapshotstore.SnapshotCompressionAlgorithmLz4: enc := lz4.NewWriter(dst) + if err := enc.Apply(lz4.CompressionLevelOption(lz4.Fast)); err != nil { + return fmt.Errorf("configure lz4 encoder: %w", err) + } if err := copyWithContext(ctx, enc, src); err != nil { _ = enc.Close() return err @@ -376,6 +451,7 @@ func decompressSnapshotMemoryFile(ctx context.Context, compressedPath string, al _ = os.Remove(tmpRawPath) return fmt.Errorf("finalize decompressed snapshot: %w", err) } + removeCompressedSnapshotArtifacts(rawPath) return nil } @@ -388,6 +464,17 @@ func compressedPathFor(rawPath string, algorithm snapshotstore.SnapshotCompressi } } +func removeCompressedSnapshotArtifacts(rawPath string) { + for _, path := range []string{ + rawPath + ".zst", + rawPath + ".zst.tmp", + rawPath + ".lz4", + rawPath + ".lz4.tmp", + } { + _ = os.Remove(path) + } +} + func copyWithContext(ctx context.Context, dst io.Writer, src io.Reader) error { buf := make([]byte, 1024*1024) for { diff --git a/lib/instances/snapshot_compression_test.go b/lib/instances/snapshot_compression_test.go index dd043f3f..bf599481 100644 --- a/lib/instances/snapshot_compression_test.go +++ b/lib/instances/snapshot_compression_test.go @@ -4,6 +4,7 @@ import ( "errors" "testing" + "github.com/kernel/hypeman/lib/hypervisor" snapshotstore "github.com/kernel/hypeman/lib/snapshot" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -86,6 +87,33 @@ func TestResolveSnapshotCompressionPolicyPrecedence(t *testing.T) { assert.Equal(t, 2, *cfg.Level) } +func TestResolveStandbyCompressionPolicyCloudHypervisorDefault(t *testing.T) { + t.Parallel() + + m := &manager{} + + cfg, err := m.resolveStandbyCompressionPolicy(&StoredMetadata{ + HypervisorType: hypervisor.TypeCloudHypervisor, + }, nil) + require.NoError(t, err) + require.NotNil(t, cfg) + assert.True(t, cfg.Enabled) + assert.Equal(t, snapshotstore.SnapshotCompressionAlgorithmLz4, cfg.Algorithm) + assert.Nil(t, cfg.Level) + + cfg, err = m.resolveStandbyCompressionPolicy(&StoredMetadata{ + HypervisorType: hypervisor.TypeCloudHypervisor, + }, &snapshotstore.SnapshotCompressionConfig{Enabled: false}) + require.NoError(t, err) + assert.Nil(t, cfg) + + cfg, err = m.resolveStandbyCompressionPolicy(&StoredMetadata{ + HypervisorType: hypervisor.TypeQEMU, + }, nil) + require.NoError(t, err) + assert.Nil(t, cfg) +} + func TestValidateCreateRequestSnapshotPolicy(t *testing.T) { t.Parallel() diff --git a/lib/instances/standby.go b/lib/instances/standby.go index 9b9a3b96..d218a9ce 100644 --- a/lib/instances/standby.go +++ b/lib/instances/standby.go @@ -62,13 +62,11 @@ func (m *manager) standbyInstance( // fails before any state transition side effects. var compressionPolicy *snapshotstore.SnapshotCompressionConfig if !skipCompression { - policy, err := m.resolveSnapshotCompressionPolicy(stored, req.Compression) + policy, err := m.resolveStandbyCompressionPolicy(stored, req.Compression) if err != nil { return nil, err } - if policy.Enabled { - compressionPolicy = &policy - } + compressionPolicy = policy } // 3. Get network allocation BEFORE killing VMM (while we can still query it) diff --git a/lib/snapshot/README.md b/lib/snapshot/README.md index edcdc989..217d245f 100644 --- a/lib/snapshot/README.md +++ b/lib/snapshot/README.md @@ -31,6 +31,63 @@ Snapshots are immutable point-in-time captures of a VM that can later be: - `Stopped` snapshot from `Stopped`: - snapshot payload is copied directly +## Snapshot Compression + +Snapshot memory compression is optional and is **off by default**. + +The current exception is cloud-hypervisor standby snapshots: if no request, instance, +or server policy is set, Hypeman defaults that path to `lz4` so standby snapshots +store compressed guest memory by default on Linux. + +- Compression applies only to `Standby` snapshots, because only standby snapshots contain guest memory state. +- `Stopped` snapshots cannot use compression because they do not include resumable RAM state. +- Compression affects only the memory snapshot file, not the entire snapshot directory. + +### When compression runs + +- A standby operation can request compression explicitly. +- A snapshot create request can request compression explicitly. +- If the request does not specify compression, Hypeman falls back to: + - the instance's `snapshot_policy` + - then the server's global `snapshot.compression_default` +- Effective precedence is: + - request override + - instance default + - server default + - then the cloud-hypervisor standby fallback (`lz4`) when no other policy is set + +Compression runs **asynchronously after the snapshot is already durable on disk**. + +- This keeps the standby path fast. +- While compression is running, the snapshot remains valid and reports `compression_state=compressing`. +- Once finished, the snapshot reports `compression_state=compressed` and exposes compressed/uncompressed size metadata. +- If compression fails, the snapshot reports `compression_state=error` and keeps the error message for inspection. + +### Restore behavior with compressed snapshots + +Restore prefers correctness first and resume latency second. + +- If a restore starts while compression is still running, Hypeman will not let restore race the compressor. +- For cloud-hypervisor standby restores, Hypeman waits for the in-flight compression job to finish. +- For the other hypervisors, Hypeman cancels the in-flight compression job and waits briefly for it to stop. +- If the snapshot memory has already been compressed, Hypeman expands it back to the raw memory file before asking the hypervisor to restore. +- This means compression is never allowed to race with restore. + +In practice, the tradeoff is: + +- standby stays fast because compression is not on the hot path +- restore from a compressed standby snapshot pays decompression cost +- restore from an uncompressed standby snapshot avoids that cost entirely + +### Supported algorithms + +- `zstd` + - default when compression is enabled + - supports configurable levels +- `lz4` + - optimized for lower decompression overhead + - does not currently accept a level setting + ### Restore (in-place) - Restore always applies to the original source VM. - Source VM must not be `Running`. diff --git a/skills/hypeman-remote-linux-tests/SKILL.md b/skills/hypeman-remote-linux-tests/SKILL.md new file mode 100644 index 00000000..cb2d7c5a --- /dev/null +++ b/skills/hypeman-remote-linux-tests/SKILL.md @@ -0,0 +1,107 @@ +--- +name: hypeman-remote-linux-tests +description: Run Hypeman tests on a remote Linux host that supports the Linux hypervisors. Use when validating cloud-hypervisor, Firecracker, QEMU, embedded guest artifacts, or Linux-only integration behavior on a server over SSH. +--- + +# Hypeman Remote Linux Tests + +Use this skill to run Hypeman tests on a remote Linux server in a way that is repeatable and low-friction for any developer. + +## Quick Start + +1. Pick a short remote working directory under the remote user's home directory. +2. Ensure the remote test shell includes `/usr/sbin` and `/sbin` on `PATH`. +3. Bootstrap embedded artifacts and bundled binaries before running `go test`. +4. Run the smallest relevant test target first. +5. Treat host-environment failures separately from product failures. + +## Remote Workspace + +Prefer a short path such as `~/hm`, `~/hypeman-test`, or another short directory name. + +Cloud Hypervisor uses UNIX sockets with strict path-length limits. Long paths from default test temp directories or deeply nested clones can make the VMM fail before the actual test starts. + +If a new test creates temporary directories for guest data, prefer a short root like: + +```bash +mktemp -d /tmp/hmcmp-XXXXXX +``` + +## Remote Bootstrap + +Before running Linux hypervisor tests, make sure the remote clone has the generated guest binaries and embedded VMM assets: + +```bash +export PATH=/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin:$PATH +make ensure-ch-binaries ensure-firecracker-binaries ensure-caddy-binaries build-embedded +``` + +Why: + +- `go:embed` patterns in the repo require the bundled binaries to exist at build time. +- `mkfs.ext4` is commonly installed in `/usr/sbin` or `/sbin`, not always on the default non-login shell `PATH`. +- Running `go test` before this bootstrap often fails during package setup, not during test execution. + +QEMU is usually provided by the host OS, so there is no matching bundled `ensure-qemu-binaries` target. If QEMU tests fail before boot, verify the system `qemu-system-*` binary separately. + +## Test Execution Pattern + +Start with one focused test: + +```bash +go test ./lib/instances -run TestCloudHypervisorStandbyRestoreCompressionScenarios -count=1 -v +``` + +Then run the sibling hypervisor tests one at a time: + +```bash +go test ./lib/instances -run TestFirecrackerStandbyRestoreCompressionScenarios -count=1 -v +go test ./lib/instances -run TestQEMUStandbyRestoreCompressionScenarios -count=1 -v +``` + +For unit-only validation: + +```bash +go test ./lib/instances -run 'TestNormalizeCompressionConfig|TestResolveSnapshotCompressionPolicyPrecedence' -count=1 +``` + +Run one hypervisor at a time when the tests boot real VMs. This avoids noisy failures from competing KVM guests and makes logs much easier to interpret. + +## Syncing Local Changes + +If the main development is happening locally and the Linux host is only for execution, sync only the files you changed rather than recloning every time. + +Typical pattern: + +```bash +rsync -az lib/instances/... remote-host:~/hypeman-test/lib/instances/ +``` + +After syncing, re-run the bootstrap command if the changes affect embedded binaries, generated assets, or files referenced by `go:embed`. + +## Common Host Failures + +Classify failures quickly: + +- Missing embedded assets: + - symptom: package setup fails with `pattern ... no matching files found` + - fix: run `make ensure-ch-binaries ensure-firecracker-binaries ensure-caddy-binaries build-embedded` +- `mkfs.ext4` not found: + - symptom: overlay or disk creation fails with `exec: "mkfs.ext4": executable file not found` + - fix: add `/usr/sbin:/sbin` to `PATH`, then retry +- Cloud Hypervisor socket path too long: + - symptom: `path must be shorter than SUN_LEN` + - fix: shorten the remote repo path and any temp/data directories used by the test +- `/tmp` or shared test-lock issues: + - symptom: test-specific lock or temp-file permission failures + - fix: prefer no-network test setup when the test does not require networking; avoid unnecessary shared lock files + +## Reporting Guidance + +When reporting results, separate: + +1. Code failures +2. Test harness issues +3. Remote host environment issues + +This matters in Hypeman because Linux integration tests often fail before product code executes if the remote machine is missing bundled binaries, system tools, or short enough paths. diff --git a/skills/hypeman-remote-linux-tests/agents/openai.yaml b/skills/hypeman-remote-linux-tests/agents/openai.yaml new file mode 100644 index 00000000..244c1d28 --- /dev/null +++ b/skills/hypeman-remote-linux-tests/agents/openai.yaml @@ -0,0 +1,4 @@ +interface: + display_name: "Hypeman Remote Linux Tests" + short_description: "Run Hypeman tests on a remote Linux hypervisor host" + default_prompt: "Run Hypeman tests on a remote Linux host, prepare prerequisites, and report actionable failures." From 0d4d7a0d96fba63e71b36f83851d8fdc7e5e64ec Mon Sep 17 00:00:00 2001 From: Steven Miller Date: Thu, 19 Mar 2026 17:36:42 -0400 Subject: [PATCH 03/31] Skip transient compression temp files during snapshot copy --- lib/forkvm/copy.go | 8 ++++++++ lib/forkvm/copy_test.go | 4 ++++ 2 files changed, 12 insertions(+) diff --git a/lib/forkvm/copy.go b/lib/forkvm/copy.go index 2076c0ce..c71b3abf 100644 --- a/lib/forkvm/copy.go +++ b/lib/forkvm/copy.go @@ -6,6 +6,7 @@ import ( "io/fs" "os" "path/filepath" + "strings" ) var ErrSparseCopyUnsupported = errors.New("sparse copy unsupported") @@ -57,6 +58,9 @@ func CopyGuestDirectory(srcDir, dstDir string) error { return nil case mode.IsRegular(): + if shouldSkipRegularFile(relPath) { + return nil + } if err := copyRegularFileSparse(path, dstPath, mode.Perm()); err != nil { return fmt.Errorf("copy file %s: %w", path, err) } @@ -85,3 +89,7 @@ func CopyGuestDirectory(srcDir, dstDir string) error { func shouldSkipDirectory(relPath string) bool { return relPath == "logs" } + +func shouldSkipRegularFile(relPath string) bool { + return strings.HasSuffix(relPath, ".lz4.tmp") || strings.HasSuffix(relPath, ".zst.tmp") +} diff --git a/lib/forkvm/copy_test.go b/lib/forkvm/copy_test.go index 762499e4..67c1bbe9 100644 --- a/lib/forkvm/copy_test.go +++ b/lib/forkvm/copy_test.go @@ -20,6 +20,8 @@ func TestCopyGuestDirectory(t *testing.T) { require.NoError(t, os.WriteFile(filepath.Join(src, "overlay.raw"), []byte("overlay"), 0644)) require.NoError(t, os.WriteFile(filepath.Join(src, "logs", "app.log"), []byte("hello"), 0644)) require.NoError(t, os.WriteFile(filepath.Join(src, "snapshots", "snapshot-latest", "config.json"), []byte(`{}`), 0644)) + require.NoError(t, os.WriteFile(filepath.Join(src, "snapshots", "snapshot-latest", "memory-ranges.lz4.tmp"), []byte("partial"), 0644)) + require.NoError(t, os.WriteFile(filepath.Join(src, "snapshots", "snapshot-latest", "memory-ranges.zst.tmp"), []byte("partial"), 0644)) require.NoError(t, os.Symlink("metadata.json", filepath.Join(src, "meta-link"))) require.NoError(t, CopyGuestDirectory(src, dst)) @@ -28,6 +30,8 @@ func TestCopyGuestDirectory(t *testing.T) { assert.FileExists(t, filepath.Join(dst, "config.ext4")) assert.FileExists(t, filepath.Join(dst, "overlay.raw")) assert.FileExists(t, filepath.Join(dst, "snapshots", "snapshot-latest", "config.json")) + assert.NoFileExists(t, filepath.Join(dst, "snapshots", "snapshot-latest", "memory-ranges.lz4.tmp")) + assert.NoFileExists(t, filepath.Join(dst, "snapshots", "snapshot-latest", "memory-ranges.zst.tmp")) assert.NoFileExists(t, filepath.Join(dst, "logs", "app.log")) assert.FileExists(t, filepath.Join(dst, "meta-link")) From 042d0c3a317146fdf2c0cdec07406c32d44507d0 Mon Sep 17 00:00:00 2001 From: Steven Miller Date: Thu, 19 Mar 2026 17:39:57 -0400 Subject: [PATCH 04/31] Clarify async snapshot compression restore behavior --- lib/snapshot/README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/snapshot/README.md b/lib/snapshot/README.md index 217d245f..c537d42b 100644 --- a/lib/snapshot/README.md +++ b/lib/snapshot/README.md @@ -59,6 +59,8 @@ store compressed guest memory by default on Linux. Compression runs **asynchronously after the snapshot is already durable on disk**. - This keeps the standby path fast. +- Standby can return successfully while compression is still running in the background. +- That means a later restore can arrive before compression has finished. - While compression is running, the snapshot remains valid and reports `compression_state=compressing`. - Once finished, the snapshot reports `compression_state=compressed` and exposes compressed/uncompressed size metadata. - If compression fails, the snapshot reports `compression_state=error` and keeps the error message for inspection. From 474bfed4ffa274fd829caf2c67f640649d53eae8 Mon Sep 17 00:00:00 2001 From: Steven Miller Date: Thu, 19 Mar 2026 17:44:01 -0400 Subject: [PATCH 05/31] Reduce snapshot compression test races --- lib/forkvm/copy.go | 6 +++--- lib/instances/compression_integration_linux_test.go | 6 ------ 2 files changed, 3 insertions(+), 9 deletions(-) diff --git a/lib/forkvm/copy.go b/lib/forkvm/copy.go index c71b3abf..931648c3 100644 --- a/lib/forkvm/copy.go +++ b/lib/forkvm/copy.go @@ -42,6 +42,9 @@ func CopyGuestDirectory(srcDir, dstDir string) error { if d.IsDir() && shouldSkipDirectory(relPath) { return filepath.SkipDir } + if shouldSkipRegularFile(relPath) { + return nil + } dstPath := filepath.Join(dstDir, relPath) info, err := d.Info() @@ -58,9 +61,6 @@ func CopyGuestDirectory(srcDir, dstDir string) error { return nil case mode.IsRegular(): - if shouldSkipRegularFile(relPath) { - return nil - } if err := copyRegularFileSparse(path, dstPath, mode.Perm()); err != nil { return fmt.Errorf("copy file %s: %w", path, err) } diff --git a/lib/instances/compression_integration_linux_test.go b/lib/instances/compression_integration_linux_test.go index 1cc14546..205a0ade 100644 --- a/lib/instances/compression_integration_linux_test.go +++ b/lib/instances/compression_integration_linux_test.go @@ -33,8 +33,6 @@ type compressionIntegrationHarness struct { } func TestCloudHypervisorStandbyRestoreCompressionScenarios(t *testing.T) { - t.Parallel() - runStandbyRestoreCompressionScenarios(t, compressionIntegrationHarness{ name: "cloud-hypervisor", hypervisor: hypervisor.TypeCloudHypervisor, @@ -50,8 +48,6 @@ func TestCloudHypervisorStandbyRestoreCompressionScenarios(t *testing.T) { } func TestFirecrackerStandbyRestoreCompressionScenarios(t *testing.T) { - t.Parallel() - runStandbyRestoreCompressionScenarios(t, compressionIntegrationHarness{ name: "firecracker", hypervisor: hypervisor.TypeFirecracker, @@ -63,8 +59,6 @@ func TestFirecrackerStandbyRestoreCompressionScenarios(t *testing.T) { } func TestQEMUStandbyRestoreCompressionScenarios(t *testing.T) { - t.Parallel() - runStandbyRestoreCompressionScenarios(t, compressionIntegrationHarness{ name: "qemu", hypervisor: hypervisor.TypeQEMU, From 0a9523f487664efdde4110b412d7a313ee803b8c Mon Sep 17 00:00:00 2001 From: Steven Miller Date: Fri, 20 Mar 2026 09:22:06 -0400 Subject: [PATCH 06/31] Restore compression test parallelism --- lib/instances/compression_integration_linux_test.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/lib/instances/compression_integration_linux_test.go b/lib/instances/compression_integration_linux_test.go index 205a0ade..1cc14546 100644 --- a/lib/instances/compression_integration_linux_test.go +++ b/lib/instances/compression_integration_linux_test.go @@ -33,6 +33,8 @@ type compressionIntegrationHarness struct { } func TestCloudHypervisorStandbyRestoreCompressionScenarios(t *testing.T) { + t.Parallel() + runStandbyRestoreCompressionScenarios(t, compressionIntegrationHarness{ name: "cloud-hypervisor", hypervisor: hypervisor.TypeCloudHypervisor, @@ -48,6 +50,8 @@ func TestCloudHypervisorStandbyRestoreCompressionScenarios(t *testing.T) { } func TestFirecrackerStandbyRestoreCompressionScenarios(t *testing.T) { + t.Parallel() + runStandbyRestoreCompressionScenarios(t, compressionIntegrationHarness{ name: "firecracker", hypervisor: hypervisor.TypeFirecracker, @@ -59,6 +63,8 @@ func TestFirecrackerStandbyRestoreCompressionScenarios(t *testing.T) { } func TestQEMUStandbyRestoreCompressionScenarios(t *testing.T) { + t.Parallel() + runStandbyRestoreCompressionScenarios(t, compressionIntegrationHarness{ name: "qemu", hypervisor: hypervisor.TypeQEMU, From 47fd7417122e491f9c130d342f879d23df1b09e2 Mon Sep 17 00:00:00 2001 From: Steven Miller Date: Fri, 20 Mar 2026 09:40:51 -0400 Subject: [PATCH 07/31] Unify snapshot restore cancellation behavior --- lib/instances/restore.go | 3 +- lib/instances/snapshot.go | 10 +++--- lib/instances/snapshot_compression.go | 47 ++++++++++++--------------- lib/snapshot/README.md | 12 +++---- 4 files changed, 34 insertions(+), 38 deletions(-) diff --git a/lib/instances/restore.go b/lib/instances/restore.go index 6020c167..fc57ce03 100644 --- a/lib/instances/restore.go +++ b/lib/instances/restore.go @@ -69,8 +69,7 @@ func (m *manager) restoreInstance( // 3. Get snapshot directory snapshotDir := m.paths.InstanceSnapshotLatest(id) - waitForCompression := stored.HypervisorType == hypervisor.TypeCloudHypervisor - if err := m.ensureSnapshotMemoryReady(ctx, snapshotDir, m.snapshotJobKeyForInstance(id), waitForCompression); err != nil { + if err := m.ensureSnapshotMemoryReady(ctx, snapshotDir, m.snapshotJobKeyForInstance(id)); err != nil { return nil, fmt.Errorf("prepare standby snapshot memory: %w", err) } starter, err := m.getVMStarter(stored.HypervisorType) diff --git a/lib/instances/snapshot.go b/lib/instances/snapshot.go index 69b116db..4649b64b 100644 --- a/lib/instances/snapshot.go +++ b/lib/instances/snapshot.go @@ -240,8 +240,9 @@ func (m *manager) restoreSnapshot(ctx context.Context, id string, snapshotID str return nil, err } - m.cancelCompressionJob(m.snapshotJobKeyForSnapshot(snapshotID)) - m.waitCompressionJob(m.snapshotJobKeyForSnapshot(snapshotID), 2*time.Second) + if err := m.cancelAndWaitCompressionJob(ctx, m.snapshotJobKeyForSnapshot(snapshotID)); err != nil { + return nil, fmt.Errorf("wait for snapshot compression to stop: %w", err) + } if err := m.replaceInstanceWithSnapshotPayload(snapshotID, id); err != nil { return nil, err @@ -357,8 +358,9 @@ func (m *manager) forkSnapshot(ctx context.Context, snapshotID string, req ForkS }) defer cu.Clean() - m.cancelCompressionJob(m.snapshotJobKeyForSnapshot(snapshotID)) - m.waitCompressionJob(m.snapshotJobKeyForSnapshot(snapshotID), 2*time.Second) + if err := m.cancelAndWaitCompressionJob(ctx, m.snapshotJobKeyForSnapshot(snapshotID)); err != nil { + return nil, fmt.Errorf("wait for snapshot compression to stop: %w", err) + } if err := forkvm.CopyGuestDirectory(m.paths.SnapshotGuestDir(snapshotID), dstDir); err != nil { if errors.Is(err, forkvm.ErrSparseCopyUnsupported) { diff --git a/lib/instances/snapshot_compression.go b/lib/instances/snapshot_compression.go index 4cab65f2..770f5be3 100644 --- a/lib/instances/snapshot_compression.go +++ b/lib/instances/snapshot_compression.go @@ -8,7 +8,6 @@ import ( "os" "path/filepath" "strings" - "time" "github.com/kernel/hypeman/lib/hypervisor" "github.com/kernel/hypeman/lib/logger" @@ -226,22 +225,6 @@ func (m *manager) cancelCompressionJob(key string) { } } -func (m *manager) waitCompressionJob(key string, timeout time.Duration) { - m.compressionMu.Lock() - job := m.compressionJobs[key] - m.compressionMu.Unlock() - if job == nil { - return - } - - timer := time.NewTimer(timeout) - defer timer.Stop() - select { - case <-job.done: - case <-timer.C: - } -} - func (m *manager) waitCompressionJobContext(ctx context.Context, key string) error { m.compressionMu.Lock() job := m.compressionJobs[key] @@ -258,19 +241,23 @@ func (m *manager) waitCompressionJobContext(ctx context.Context, key string) err } } -func (m *manager) ensureSnapshotMemoryReady(ctx context.Context, snapshotDir, jobKey string, waitForCompression bool) error { +func (m *manager) cancelAndWaitCompressionJob(ctx context.Context, key string) error { + if key == "" { + return nil + } + m.cancelCompressionJob(key) + return m.waitCompressionJobContext(ctx, key) +} + +func (m *manager) ensureSnapshotMemoryReady(ctx context.Context, snapshotDir, jobKey string) error { if jobKey != "" { - if waitForCompression { - if err := m.waitCompressionJobContext(ctx, jobKey); err != nil { - return err - } - } else { - m.cancelCompressionJob(jobKey) - m.waitCompressionJob(jobKey, 2*time.Second) + if err := m.cancelAndWaitCompressionJob(ctx, jobKey); err != nil { + return err } } - if _, ok := findRawSnapshotMemoryFile(snapshotDir); ok { + if rawPath, ok := findRawSnapshotMemoryFile(snapshotDir); ok { + removeCompressedSnapshotArtifacts(rawPath) return nil } compressedPath, algorithm, ok := findCompressedSnapshotMemoryFile(snapshotDir) @@ -348,10 +335,18 @@ func compressSnapshotMemoryFile(ctx context.Context, rawPath string, cfg snapsho _ = os.Remove(tmpPath) return 0, 0, err } + if err := ctx.Err(); err != nil { + _ = os.Remove(tmpPath) + return 0, 0, err + } if err := os.Rename(tmpPath, compressedPath); err != nil { _ = os.Remove(tmpPath) return 0, 0, fmt.Errorf("finalize compressed snapshot: %w", err) } + if err := ctx.Err(); err != nil { + _ = os.Remove(compressedPath) + return 0, 0, err + } compressedInfo, err := os.Stat(compressedPath) if err != nil { diff --git a/lib/snapshot/README.md b/lib/snapshot/README.md index c537d42b..c160426a 100644 --- a/lib/snapshot/README.md +++ b/lib/snapshot/README.md @@ -67,13 +67,13 @@ Compression runs **asynchronously after the snapshot is already durable on disk* ### Restore behavior with compressed snapshots -Restore prefers correctness first and resume latency second. +Restore always uses the same flow across hypervisors. -- If a restore starts while compression is still running, Hypeman will not let restore race the compressor. -- For cloud-hypervisor standby restores, Hypeman waits for the in-flight compression job to finish. -- For the other hypervisors, Hypeman cancels the in-flight compression job and waits briefly for it to stop. -- If the snapshot memory has already been compressed, Hypeman expands it back to the raw memory file before asking the hypervisor to restore. -- This means compression is never allowed to race with restore. +- If a restore starts while compression is still running, Hypeman cancels the in-flight compression job first. +- Restore then synchronizes with that job so it is no longer mutating the snapshot files. +- If the raw memory file still exists after cancellation, restore uses the raw file directly and removes any compressed sibling artifacts. +- If the raw file is already gone, Hypeman expands the compressed file back to the raw memory file before asking the hypervisor to restore. +- This means restore never races a live compressor, while still preferring the raw snapshot whenever it is still available. In practice, the tradeoff is: From 780fcd7aa1c5d3800c9c409bb5877a67ada78754 Mon Sep 17 00:00:00 2001 From: Steven Miller Date: Fri, 20 Mar 2026 09:50:26 -0400 Subject: [PATCH 08/31] Apply suggestions from code review Co-authored-by: Steven Miller --- skills/hypeman-remote-linux-tests/SKILL.md | 31 ---------------------- 1 file changed, 31 deletions(-) diff --git a/skills/hypeman-remote-linux-tests/SKILL.md b/skills/hypeman-remote-linux-tests/SKILL.md index cb2d7c5a..d2599ff1 100644 --- a/skills/hypeman-remote-linux-tests/SKILL.md +++ b/skills/hypeman-remote-linux-tests/SKILL.md @@ -44,28 +44,6 @@ Why: QEMU is usually provided by the host OS, so there is no matching bundled `ensure-qemu-binaries` target. If QEMU tests fail before boot, verify the system `qemu-system-*` binary separately. -## Test Execution Pattern - -Start with one focused test: - -```bash -go test ./lib/instances -run TestCloudHypervisorStandbyRestoreCompressionScenarios -count=1 -v -``` - -Then run the sibling hypervisor tests one at a time: - -```bash -go test ./lib/instances -run TestFirecrackerStandbyRestoreCompressionScenarios -count=1 -v -go test ./lib/instances -run TestQEMUStandbyRestoreCompressionScenarios -count=1 -v -``` - -For unit-only validation: - -```bash -go test ./lib/instances -run 'TestNormalizeCompressionConfig|TestResolveSnapshotCompressionPolicyPrecedence' -count=1 -``` - -Run one hypervisor at a time when the tests boot real VMs. This avoids noisy failures from competing KVM guests and makes logs much easier to interpret. ## Syncing Local Changes @@ -96,12 +74,3 @@ Classify failures quickly: - symptom: test-specific lock or temp-file permission failures - fix: prefer no-network test setup when the test does not require networking; avoid unnecessary shared lock files -## Reporting Guidance - -When reporting results, separate: - -1. Code failures -2. Test harness issues -3. Remote host environment issues - -This matters in Hypeman because Linux integration tests often fail before product code executes if the remote machine is missing bundled binaries, system tools, or short enough paths. From 78a24d00c5b2e6f95014b4ab27428c2811452adb Mon Sep 17 00:00:00 2001 From: Steven Miller Date: Fri, 20 Mar 2026 10:07:47 -0400 Subject: [PATCH 09/31] Add snapshot compression metrics --- lib/instances/metrics.go | 239 ++++++++++++++++++++++++-- lib/instances/metrics_test.go | 140 +++++++++++++++ lib/instances/restore.go | 10 +- lib/instances/snapshot.go | 24 ++- lib/instances/snapshot_compression.go | 70 ++++++-- lib/instances/standby.go | 10 +- 6 files changed, 454 insertions(+), 39 deletions(-) create mode 100644 lib/instances/metrics_test.go diff --git a/lib/instances/metrics.go b/lib/instances/metrics.go index b9252255..8666a9c1 100644 --- a/lib/instances/metrics.go +++ b/lib/instances/metrics.go @@ -6,20 +6,58 @@ import ( "github.com/kernel/hypeman/lib/hypervisor" mw "github.com/kernel/hypeman/lib/middleware" + snapshotstore "github.com/kernel/hypeman/lib/snapshot" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/metric" "go.opentelemetry.io/otel/trace" ) +type snapshotCompressionSource string + +const ( + snapshotCompressionSourceStandby snapshotCompressionSource = "standby" + snapshotCompressionSourceSnapshot snapshotCompressionSource = "snapshot" +) + +type snapshotCompressionResult string + +const ( + snapshotCompressionResultSuccess snapshotCompressionResult = "success" + snapshotCompressionResultCanceled snapshotCompressionResult = "canceled" + snapshotCompressionResultFailed snapshotCompressionResult = "failed" +) + +type snapshotMemoryPreparePath string + +const ( + snapshotMemoryPreparePathRaw snapshotMemoryPreparePath = "raw" + snapshotMemoryPreparePathDecompress snapshotMemoryPreparePath = "decompress" +) + +type snapshotCompressionPreemptionOperation string + +const ( + snapshotCompressionPreemptionRestoreInstance snapshotCompressionPreemptionOperation = "restore_instance" + snapshotCompressionPreemptionRestoreSnapshot snapshotCompressionPreemptionOperation = "restore_snapshot" + snapshotCompressionPreemptionForkSnapshot snapshotCompressionPreemptionOperation = "fork_snapshot" +) + // Metrics holds the metrics instruments for instance operations. type Metrics struct { - createDuration metric.Float64Histogram - restoreDuration metric.Float64Histogram - standbyDuration metric.Float64Histogram - stopDuration metric.Float64Histogram - startDuration metric.Float64Histogram - stateTransitions metric.Int64Counter - tracer trace.Tracer + createDuration metric.Float64Histogram + restoreDuration metric.Float64Histogram + standbyDuration metric.Float64Histogram + stopDuration metric.Float64Histogram + startDuration metric.Float64Histogram + stateTransitions metric.Int64Counter + snapshotCompressionJobsTotal metric.Int64Counter + snapshotCompressionDuration metric.Float64Histogram + snapshotCompressionSavedBytes metric.Int64Histogram + snapshotCompressionRatio metric.Float64Histogram + snapshotRestoreMemoryPrepareTotal metric.Int64Counter + snapshotRestoreMemoryPrepareDuration metric.Float64Histogram + snapshotCompressionPreemptionsTotal metric.Int64Counter + tracer trace.Tracer } // newInstanceMetrics creates and registers all instance metrics. @@ -77,6 +115,65 @@ func newInstanceMetrics(meter metric.Meter, tracer trace.Tracer, m *manager) (*M return nil, err } + snapshotCompressionJobsTotal, err := meter.Int64Counter( + "hypeman_snapshot_compression_jobs_total", + metric.WithDescription("Total number of snapshot compression jobs by result"), + ) + if err != nil { + return nil, err + } + + snapshotCompressionDuration, err := meter.Float64Histogram( + "hypeman_snapshot_compression_duration_seconds", + metric.WithDescription("Time to asynchronously compress snapshot memory"), + metric.WithUnit("s"), + ) + if err != nil { + return nil, err + } + + snapshotCompressionSavedBytes, err := meter.Int64Histogram( + "hypeman_snapshot_compression_saved_bytes", + metric.WithDescription("Bytes saved by compressing snapshot memory"), + metric.WithUnit("By"), + ) + if err != nil { + return nil, err + } + + snapshotCompressionRatio, err := meter.Float64Histogram( + "hypeman_snapshot_compression_ratio", + metric.WithDescription("Compressed snapshot memory size divided by raw snapshot memory size"), + ) + if err != nil { + return nil, err + } + + snapshotRestoreMemoryPrepareTotal, err := meter.Int64Counter( + "hypeman_snapshot_restore_memory_prepare_total", + metric.WithDescription("Total number of snapshot memory prepare operations before restore"), + ) + if err != nil { + return nil, err + } + + snapshotRestoreMemoryPrepareDuration, err := meter.Float64Histogram( + "hypeman_snapshot_restore_memory_prepare_duration_seconds", + metric.WithDescription("Time to prepare snapshot memory before restore"), + metric.WithUnit("s"), + ) + if err != nil { + return nil, err + } + + snapshotCompressionPreemptionsTotal, err := meter.Int64Counter( + "hypeman_snapshot_compression_preemptions_total", + metric.WithDescription("Total number of foreground operations that preempt in-flight snapshot compression"), + ) + if err != nil { + return nil, err + } + // Register observable gauge for instance counts by state instancesTotal, err := meter.Int64ObservableGauge( "hypeman_instances_total", @@ -86,6 +183,14 @@ func newInstanceMetrics(meter metric.Meter, tracer trace.Tracer, m *manager) (*M return nil, err } + snapshotCompressionActiveTotal, err := meter.Int64ObservableGauge( + "hypeman_snapshot_compression_active_total", + metric.WithDescription("Total number of in-flight snapshot compression jobs"), + ) + if err != nil { + return nil, err + } + _, err = meter.RegisterCallback( func(ctx context.Context, o metric.Observer) error { instances, err := m.listInstances(ctx) @@ -120,14 +225,59 @@ func newInstanceMetrics(meter metric.Meter, tracer trace.Tracer, m *manager) (*M return nil, err } + _, err = meter.RegisterCallback( + func(ctx context.Context, o metric.Observer) error { + type compressionKey struct { + hypervisor string + algorithm string + source string + } + + counts := make(map[compressionKey]int64) + m.compressionMu.Lock() + for _, job := range m.compressionJobs { + key := compressionKey{ + hypervisor: string(job.target.HypervisorType), + algorithm: string(job.target.Policy.Algorithm), + source: string(job.target.Source), + } + counts[key]++ + } + m.compressionMu.Unlock() + + for key, count := range counts { + attrs := []attribute.KeyValue{ + attribute.String("algorithm", key.algorithm), + attribute.String("source", key.source), + } + if key.hypervisor != "" { + attrs = append(attrs, attribute.String("hypervisor", key.hypervisor)) + } + o.ObserveInt64(snapshotCompressionActiveTotal, count, metric.WithAttributes(attrs...)) + } + return nil + }, + snapshotCompressionActiveTotal, + ) + if err != nil { + return nil, err + } + return &Metrics{ - createDuration: createDuration, - restoreDuration: restoreDuration, - standbyDuration: standbyDuration, - stopDuration: stopDuration, - startDuration: startDuration, - stateTransitions: stateTransitions, - tracer: tracer, + createDuration: createDuration, + restoreDuration: restoreDuration, + standbyDuration: standbyDuration, + stopDuration: stopDuration, + startDuration: startDuration, + stateTransitions: stateTransitions, + snapshotCompressionJobsTotal: snapshotCompressionJobsTotal, + snapshotCompressionDuration: snapshotCompressionDuration, + snapshotCompressionSavedBytes: snapshotCompressionSavedBytes, + snapshotCompressionRatio: snapshotCompressionRatio, + snapshotRestoreMemoryPrepareTotal: snapshotRestoreMemoryPrepareTotal, + snapshotRestoreMemoryPrepareDuration: snapshotRestoreMemoryPrepareDuration, + snapshotCompressionPreemptionsTotal: snapshotCompressionPreemptionsTotal, + tracer: tracer, }, nil } @@ -169,3 +319,64 @@ func (m *manager) recordStateTransition(ctx context.Context, fromState, toState } m.metrics.stateTransitions.Add(ctx, 1, metric.WithAttributes(attrs...)) } + +func snapshotCompressionAttributes(hvType hypervisor.Type, algorithm snapshotstore.SnapshotCompressionAlgorithm, source snapshotCompressionSource) []attribute.KeyValue { + attrs := []attribute.KeyValue{ + attribute.String("algorithm", string(algorithm)), + attribute.String("source", string(source)), + } + if hvType != "" { + attrs = append(attrs, attribute.String("hypervisor", string(hvType))) + } + return attrs +} + +func (m *manager) recordSnapshotCompressionJob(ctx context.Context, target compressionTarget, result snapshotCompressionResult, start time.Time, uncompressedSize, compressedSize int64) { + if m.metrics == nil { + return + } + + attrs := snapshotCompressionAttributes(target.HypervisorType, target.Policy.Algorithm, target.Source) + attrsWithResult := append([]attribute.KeyValue{}, attrs...) + attrsWithResult = append(attrsWithResult, attribute.String("result", string(result))) + + m.metrics.snapshotCompressionJobsTotal.Add(ctx, 1, metric.WithAttributes(attrsWithResult...)) + m.metrics.snapshotCompressionDuration.Record(ctx, time.Since(start).Seconds(), metric.WithAttributes(attrsWithResult...)) + + if result != snapshotCompressionResultSuccess || uncompressedSize <= 0 || compressedSize < 0 { + return + } + + savedBytes := uncompressedSize - compressedSize + if savedBytes < 0 { + savedBytes = 0 + } + m.metrics.snapshotCompressionSavedBytes.Record(ctx, savedBytes, metric.WithAttributes(attrs...)) + m.metrics.snapshotCompressionRatio.Record(ctx, float64(compressedSize)/float64(uncompressedSize), metric.WithAttributes(attrs...)) +} + +func (m *manager) recordSnapshotRestoreMemoryPrepare(ctx context.Context, hvType hypervisor.Type, path snapshotMemoryPreparePath, result snapshotCompressionResult, start time.Time) { + if m.metrics == nil { + return + } + + attrs := []attribute.KeyValue{ + attribute.String("restore_source", string(path)), + attribute.String("result", string(result)), + } + if hvType != "" { + attrs = append(attrs, attribute.String("hypervisor", string(hvType))) + } + m.metrics.snapshotRestoreMemoryPrepareTotal.Add(ctx, 1, metric.WithAttributes(attrs...)) + m.metrics.snapshotRestoreMemoryPrepareDuration.Record(ctx, time.Since(start).Seconds(), metric.WithAttributes(attrs...)) +} + +func (m *manager) recordSnapshotCompressionPreemption(ctx context.Context, operation snapshotCompressionPreemptionOperation, target compressionTarget) { + if m.metrics == nil { + return + } + + attrs := snapshotCompressionAttributes(target.HypervisorType, target.Policy.Algorithm, target.Source) + attrs = append(attrs, attribute.String("operation", string(operation))) + m.metrics.snapshotCompressionPreemptionsTotal.Add(ctx, 1, metric.WithAttributes(attrs...)) +} diff --git a/lib/instances/metrics_test.go b/lib/instances/metrics_test.go new file mode 100644 index 00000000..f552303f --- /dev/null +++ b/lib/instances/metrics_test.go @@ -0,0 +1,140 @@ +package instances + +import ( + "testing" + "time" + + "github.com/kernel/hypeman/lib/hypervisor" + "github.com/kernel/hypeman/lib/paths" + snapshotstore "github.com/kernel/hypeman/lib/snapshot" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.opentelemetry.io/otel/attribute" + otelmetric "go.opentelemetry.io/otel/sdk/metric" + "go.opentelemetry.io/otel/sdk/metric/metricdata" +) + +func TestSnapshotCompressionMetrics_RecordAndObserve(t *testing.T) { + t.Parallel() + + reader := otelmetric.NewManualReader() + provider := otelmetric.NewMeterProvider(otelmetric.WithReader(reader)) + + m := &manager{ + paths: paths.New(t.TempDir()), + compressionJobs: map[string]*compressionJob{ + "job-1": { + done: make(chan struct{}), + target: compressionTarget{ + Key: "job-1", + HypervisorType: hypervisor.TypeCloudHypervisor, + Source: snapshotCompressionSourceStandby, + Policy: snapshotstore.SnapshotCompressionConfig{ + Enabled: true, + Algorithm: snapshotstore.SnapshotCompressionAlgorithmLz4, + }, + }, + }, + }, + } + + metrics, err := newInstanceMetrics(provider.Meter("test"), nil, m) + require.NoError(t, err) + m.metrics = metrics + + target := m.compressionJobs["job-1"].target + m.recordSnapshotCompressionJob(t.Context(), target, snapshotCompressionResultSuccess, time.Now().Add(-2*time.Second), 1024, 256) + m.recordSnapshotRestoreMemoryPrepare(t.Context(), hypervisor.TypeCloudHypervisor, snapshotMemoryPreparePathRaw, snapshotCompressionResultSuccess, time.Now().Add(-250*time.Millisecond)) + m.recordSnapshotCompressionPreemption(t.Context(), snapshotCompressionPreemptionRestoreInstance, target) + + var rm metricdata.ResourceMetrics + require.NoError(t, reader.Collect(t.Context(), &rm)) + + assertMetricNames(t, rm, []string{ + "hypeman_snapshot_compression_jobs_total", + "hypeman_snapshot_compression_duration_seconds", + "hypeman_snapshot_compression_saved_bytes", + "hypeman_snapshot_compression_ratio", + "hypeman_snapshot_restore_memory_prepare_total", + "hypeman_snapshot_restore_memory_prepare_duration_seconds", + "hypeman_snapshot_compression_preemptions_total", + "hypeman_snapshot_compression_active_total", + }) + + jobsMetric := findMetric(t, rm, "hypeman_snapshot_compression_jobs_total") + jobs, ok := jobsMetric.Data.(metricdata.Sum[int64]) + require.True(t, ok) + require.Len(t, jobs.DataPoints, 1) + assert.Equal(t, int64(1), jobs.DataPoints[0].Value) + assert.Equal(t, "cloud-hypervisor", metricLabel(t, jobs.DataPoints[0].Attributes, "hypervisor")) + assert.Equal(t, "lz4", metricLabel(t, jobs.DataPoints[0].Attributes, "algorithm")) + assert.Equal(t, "standby", metricLabel(t, jobs.DataPoints[0].Attributes, "source")) + assert.Equal(t, "success", metricLabel(t, jobs.DataPoints[0].Attributes, "result")) + + savedBytesMetric := findMetric(t, rm, "hypeman_snapshot_compression_saved_bytes") + savedBytes, ok := savedBytesMetric.Data.(metricdata.Histogram[int64]) + require.True(t, ok) + require.Len(t, savedBytes.DataPoints, 1) + assert.Equal(t, uint64(1), savedBytes.DataPoints[0].Count) + assert.Equal(t, int64(768), savedBytes.DataPoints[0].Sum) + + restorePrepMetric := findMetric(t, rm, "hypeman_snapshot_restore_memory_prepare_total") + restorePrep, ok := restorePrepMetric.Data.(metricdata.Sum[int64]) + require.True(t, ok) + require.Len(t, restorePrep.DataPoints, 1) + assert.Equal(t, int64(1), restorePrep.DataPoints[0].Value) + assert.Equal(t, "raw", metricLabel(t, restorePrep.DataPoints[0].Attributes, "restore_source")) + assert.Equal(t, "success", metricLabel(t, restorePrep.DataPoints[0].Attributes, "result")) + + preemptionsMetric := findMetric(t, rm, "hypeman_snapshot_compression_preemptions_total") + preemptions, ok := preemptionsMetric.Data.(metricdata.Sum[int64]) + require.True(t, ok) + require.Len(t, preemptions.DataPoints, 1) + assert.Equal(t, int64(1), preemptions.DataPoints[0].Value) + assert.Equal(t, "restore_instance", metricLabel(t, preemptions.DataPoints[0].Attributes, "operation")) + + activeMetric := findMetric(t, rm, "hypeman_snapshot_compression_active_total") + active, ok := activeMetric.Data.(metricdata.Gauge[int64]) + require.True(t, ok) + require.Len(t, active.DataPoints, 1) + assert.Equal(t, int64(1), active.DataPoints[0].Value) + assert.Equal(t, "lz4", metricLabel(t, active.DataPoints[0].Attributes, "algorithm")) + assert.Equal(t, "standby", metricLabel(t, active.DataPoints[0].Attributes, "source")) +} + +func assertMetricNames(t *testing.T, rm metricdata.ResourceMetrics, expected []string) { + t.Helper() + + metricNames := make(map[string]bool) + for _, sm := range rm.ScopeMetrics { + for _, m := range sm.Metrics { + metricNames[m.Name] = true + } + } + + for _, name := range expected { + assert.True(t, metricNames[name], "expected metric %s to be registered", name) + } +} + +func findMetric(t *testing.T, rm metricdata.ResourceMetrics, name string) metricdata.Metrics { + t.Helper() + + for _, sm := range rm.ScopeMetrics { + for _, m := range sm.Metrics { + if m.Name == name { + return m + } + } + } + t.Fatalf("metric %s not found", name) + return metricdata.Metrics{} +} + +func metricLabel(t *testing.T, attrs attribute.Set, key string) string { + t.Helper() + + value, ok := attrs.Value(attribute.Key(key)) + require.True(t, ok, "expected label %s", key) + return value.AsString() +} diff --git a/lib/instances/restore.go b/lib/instances/restore.go index fc57ce03..fc338d97 100644 --- a/lib/instances/restore.go +++ b/lib/instances/restore.go @@ -69,7 +69,15 @@ func (m *manager) restoreInstance( // 3. Get snapshot directory snapshotDir := m.paths.InstanceSnapshotLatest(id) - if err := m.ensureSnapshotMemoryReady(ctx, snapshotDir, m.snapshotJobKeyForInstance(id)); err != nil { + var prepareSnapshotSpan trace.Span + if m.metrics != nil && m.metrics.tracer != nil { + ctx, prepareSnapshotSpan = m.metrics.tracer.Start(ctx, "PrepareSnapshotMemory") + } + err = m.ensureSnapshotMemoryReady(ctx, snapshotDir, m.snapshotJobKeyForInstance(id), stored.HypervisorType) + if prepareSnapshotSpan != nil { + prepareSnapshotSpan.End() + } + if err != nil { return nil, fmt.Errorf("prepare standby snapshot memory: %w", err) } starter, err := m.getVMStarter(stored.HypervisorType) diff --git a/lib/instances/snapshot.go b/lib/instances/snapshot.go index 4649b64b..efbf3227 100644 --- a/lib/instances/snapshot.go +++ b/lib/instances/snapshot.go @@ -152,11 +152,13 @@ func (m *manager) createSnapshot(ctx context.Context, id string, req CreateSnaps } if effectiveCompression.Enabled { m.startCompressionJob(ctx, compressionTarget{ - Key: m.snapshotJobKeyForSnapshot(snapshotID), - OwnerID: stored.Id, - SnapshotID: snapshotID, - SnapshotDir: snapshotGuestDir, - Policy: effectiveCompression, + Key: m.snapshotJobKeyForSnapshot(snapshotID), + OwnerID: stored.Id, + SnapshotID: snapshotID, + SnapshotDir: snapshotGuestDir, + HypervisorType: stored.HypervisorType, + Source: snapshotCompressionSourceSnapshot, + Policy: effectiveCompression, }) } cu.Release() @@ -240,9 +242,13 @@ func (m *manager) restoreSnapshot(ctx context.Context, id string, snapshotID str return nil, err } - if err := m.cancelAndWaitCompressionJob(ctx, m.snapshotJobKeyForSnapshot(snapshotID)); err != nil { + target, err := m.cancelAndWaitCompressionJob(ctx, m.snapshotJobKeyForSnapshot(snapshotID)) + if err != nil { return nil, fmt.Errorf("wait for snapshot compression to stop: %w", err) } + if target != nil { + m.recordSnapshotCompressionPreemption(ctx, snapshotCompressionPreemptionRestoreSnapshot, *target) + } if err := m.replaceInstanceWithSnapshotPayload(snapshotID, id); err != nil { return nil, err @@ -358,9 +364,13 @@ func (m *manager) forkSnapshot(ctx context.Context, snapshotID string, req ForkS }) defer cu.Clean() - if err := m.cancelAndWaitCompressionJob(ctx, m.snapshotJobKeyForSnapshot(snapshotID)); err != nil { + target, err := m.cancelAndWaitCompressionJob(ctx, m.snapshotJobKeyForSnapshot(snapshotID)) + if err != nil { return nil, fmt.Errorf("wait for snapshot compression to stop: %w", err) } + if target != nil { + m.recordSnapshotCompressionPreemption(ctx, snapshotCompressionPreemptionForkSnapshot, *target) + } if err := forkvm.CopyGuestDirectory(m.paths.SnapshotGuestDir(snapshotID), dstDir); err != nil { if errors.Is(err, forkvm.ErrSparseCopyUnsupported) { diff --git a/lib/instances/snapshot_compression.go b/lib/instances/snapshot_compression.go index 770f5be3..cb9c56f2 100644 --- a/lib/instances/snapshot_compression.go +++ b/lib/instances/snapshot_compression.go @@ -8,6 +8,7 @@ import ( "os" "path/filepath" "strings" + "time" "github.com/kernel/hypeman/lib/hypervisor" "github.com/kernel/hypeman/lib/logger" @@ -24,14 +25,17 @@ const ( type compressionJob struct { cancel context.CancelFunc done chan struct{} + target compressionTarget } type compressionTarget struct { - Key string - OwnerID string - SnapshotID string - SnapshotDir string - Policy snapshotstore.SnapshotCompressionConfig + Key string + OwnerID string + SnapshotID string + SnapshotDir string + HypervisorType hypervisor.Type + Source snapshotCompressionSource + Policy snapshotstore.SnapshotCompressionConfig } func cloneCompressionConfig(cfg *snapshotstore.SnapshotCompressionConfig) *snapshotstore.SnapshotCompressionConfig { @@ -174,12 +178,20 @@ func (m *manager) startCompressionJob(ctx context.Context, target compressionTar job := &compressionJob{ cancel: cancel, done: make(chan struct{}), + target: target, } m.compressionJobs[target.Key] = job m.compressionMu.Unlock() go func() { + start := time.Now() + result := snapshotCompressionResultSuccess + var uncompressedSize int64 + var compressedSize int64 + metricsCtx := context.Background() + defer func() { + m.recordSnapshotCompressionJob(metricsCtx, target, result, start, uncompressedSize, compressedSize) m.compressionMu.Lock() delete(m.compressionJobs, target.Key) m.compressionMu.Unlock() @@ -195,14 +207,17 @@ func (m *manager) startCompressionJob(ctx context.Context, target compressionTar return } - uncompressedSize, compressedSize, err := compressSnapshotMemoryFile(jobCtx, rawPath, target.Policy) + var err error + uncompressedSize, compressedSize, err = compressSnapshotMemoryFile(jobCtx, rawPath, target.Policy) if err != nil { if errors.Is(err, context.Canceled) { + result = snapshotCompressionResultCanceled if target.SnapshotID != "" { _ = m.updateSnapshotCompressionMetadata(target.SnapshotID, snapshotstore.SnapshotCompressionStateNone, "", nil, nil, nil) } return } + result = snapshotCompressionResultFailed if target.SnapshotID != "" { _ = m.updateSnapshotCompressionMetadata(target.SnapshotID, snapshotstore.SnapshotCompressionStateError, err.Error(), &target.Policy, nil, nil) } @@ -241,30 +256,59 @@ func (m *manager) waitCompressionJobContext(ctx context.Context, key string) err } } -func (m *manager) cancelAndWaitCompressionJob(ctx context.Context, key string) error { +func (m *manager) cancelAndWaitCompressionJob(ctx context.Context, key string) (*compressionTarget, error) { if key == "" { - return nil + return nil, nil + } + + m.compressionMu.Lock() + job := m.compressionJobs[key] + if job != nil { + job.cancel() + } + m.compressionMu.Unlock() + + if job == nil { + return nil, nil + } + + select { + case <-job.done: + target := job.target + return &target, nil + case <-ctx.Done(): + return nil, ctx.Err() } - m.cancelCompressionJob(key) - return m.waitCompressionJobContext(ctx, key) } -func (m *manager) ensureSnapshotMemoryReady(ctx context.Context, snapshotDir, jobKey string) error { +func (m *manager) ensureSnapshotMemoryReady(ctx context.Context, snapshotDir, jobKey string, hvType hypervisor.Type) error { + start := time.Now() + if jobKey != "" { - if err := m.cancelAndWaitCompressionJob(ctx, jobKey); err != nil { + target, err := m.cancelAndWaitCompressionJob(ctx, jobKey) + if err != nil { return err } + if target != nil { + m.recordSnapshotCompressionPreemption(ctx, snapshotCompressionPreemptionRestoreInstance, *target) + } } if rawPath, ok := findRawSnapshotMemoryFile(snapshotDir); ok { removeCompressedSnapshotArtifacts(rawPath) + m.recordSnapshotRestoreMemoryPrepare(ctx, hvType, snapshotMemoryPreparePathRaw, snapshotCompressionResultSuccess, start) return nil } compressedPath, algorithm, ok := findCompressedSnapshotMemoryFile(snapshotDir) if !ok { return nil } - return decompressSnapshotMemoryFile(ctx, compressedPath, algorithm) + if err := decompressSnapshotMemoryFile(ctx, compressedPath, algorithm); err != nil { + m.recordSnapshotRestoreMemoryPrepare(ctx, hvType, snapshotMemoryPreparePathDecompress, snapshotCompressionResultFailed, start) + return err + } + m.recordSnapshotRestoreMemoryPrepare(ctx, hvType, snapshotMemoryPreparePathDecompress, snapshotCompressionResultSuccess, start) + return nil } func (m *manager) updateSnapshotCompressionMetadata(snapshotID, state, compressionError string, cfg *snapshotstore.SnapshotCompressionConfig, compressedSize, uncompressedSize *int64) error { diff --git a/lib/instances/standby.go b/lib/instances/standby.go index d218a9ce..f5bab31d 100644 --- a/lib/instances/standby.go +++ b/lib/instances/standby.go @@ -180,10 +180,12 @@ func (m *manager) standbyInstance( if compressionPolicy != nil { m.startCompressionJob(ctx, compressionTarget{ - Key: m.snapshotJobKeyForInstance(stored.Id), - OwnerID: stored.Id, - SnapshotDir: snapshotDir, - Policy: *compressionPolicy, + Key: m.snapshotJobKeyForInstance(stored.Id), + OwnerID: stored.Id, + SnapshotDir: snapshotDir, + HypervisorType: stored.HypervisorType, + Source: snapshotCompressionSourceStandby, + Policy: *compressionPolicy, }) } From bc2fd608e78ad74ad1fa44f1e1925614bf5651f6 Mon Sep 17 00:00:00 2001 From: Steven Miller Date: Fri, 20 Mar 2026 10:19:55 -0400 Subject: [PATCH 10/31] Fix snapshot compression review feedback --- cmd/api/config/config.go | 9 +++--- cmd/api/config/config_test.go | 11 +++++++ lib/oapi/oapi.go | 5 ++-- lib/oapi/optional_json.go | 23 +++++++++++++++ lib/oapi/optional_json_test.go | 52 +++++++++++++++++++++++++++++++++ lib/providers/providers.go | 22 +++++++++----- lib/providers/providers_test.go | 49 +++++++++++++++++++++++++++++++ 7 files changed, 157 insertions(+), 14 deletions(-) create mode 100644 lib/oapi/optional_json.go create mode 100644 lib/oapi/optional_json_test.go create mode 100644 lib/providers/providers_test.go diff --git a/cmd/api/config/config.go b/cmd/api/config/config.go index e9704693..62472cd0 100644 --- a/cmd/api/config/config.go +++ b/cmd/api/config/config.go @@ -467,14 +467,15 @@ func (c *Config) Validate() error { if c.Build.Timeout <= 0 { return fmt.Errorf("build.timeout must be positive, got %d", c.Build.Timeout) } - if c.Snapshot.CompressionDefault.Level < 1 { - return fmt.Errorf("snapshot.compression_default.level must be >= 1, got %d", c.Snapshot.CompressionDefault.Level) - } - switch strings.ToLower(c.Snapshot.CompressionDefault.Algorithm) { + algorithm := strings.ToLower(c.Snapshot.CompressionDefault.Algorithm) + switch algorithm { case "", "zstd", "lz4": default: return fmt.Errorf("snapshot.compression_default.algorithm must be one of zstd or lz4, got %q", c.Snapshot.CompressionDefault.Algorithm) } + if algorithm != "lz4" && c.Snapshot.CompressionDefault.Level < 1 { + return fmt.Errorf("snapshot.compression_default.level must be >= 1, got %d", c.Snapshot.CompressionDefault.Level) + } if c.Hypervisor.Memory.KernelPageInitMode != "performance" && c.Hypervisor.Memory.KernelPageInitMode != "hardened" { return fmt.Errorf("hypervisor.memory.kernel_page_init_mode must be one of {performance,hardened}, got %q", c.Hypervisor.Memory.KernelPageInitMode) } diff --git a/cmd/api/config/config_test.go b/cmd/api/config/config_test.go index 97828db5..8f252be7 100644 --- a/cmd/api/config/config_test.go +++ b/cmd/api/config/config_test.go @@ -83,3 +83,14 @@ func TestValidateRejectsInvalidVMLabelBudget(t *testing.T) { t.Fatalf("expected validation error for invalid vm label budget") } } + +func TestValidateAllowsLZ4CompressionDefaultWithImplicitLevel(t *testing.T) { + cfg := defaultConfig() + cfg.Snapshot.CompressionDefault.Enabled = true + cfg.Snapshot.CompressionDefault.Algorithm = "lz4" + cfg.Snapshot.CompressionDefault.Level = 1 + + if err := cfg.Validate(); err != nil { + t.Fatalf("expected lz4 compression default to validate, got %v", err) + } +} diff --git a/lib/oapi/oapi.go b/lib/oapi/oapi.go index e52a314a..7e6fb197 100644 --- a/lib/oapi/oapi.go +++ b/lib/oapi/oapi.go @@ -13024,11 +13024,12 @@ func (sh *strictHandler) StandbyInstance(w http.ResponseWriter, r *http.Request, request.Id = id var body StandbyInstanceJSONRequestBody - if err := json.NewDecoder(r.Body).Decode(&body); err != nil { + decodedBody, err := decodeOptionalJSONBody(r, &body) + if err != nil { sh.options.RequestErrorHandlerFunc(w, r, fmt.Errorf("can't decode JSON body: %w", err)) return } - request.Body = &body + request.Body = decodedBody handler := func(ctx context.Context, w http.ResponseWriter, r *http.Request, request interface{}) (interface{}, error) { return sh.ssi.StandbyInstance(ctx, request.(StandbyInstanceRequestObject)) diff --git a/lib/oapi/optional_json.go b/lib/oapi/optional_json.go new file mode 100644 index 00000000..389bc607 --- /dev/null +++ b/lib/oapi/optional_json.go @@ -0,0 +1,23 @@ +package oapi + +import ( + "encoding/json" + "errors" + "io" + "net/http" +) + +func decodeOptionalJSONBody[T any](r *http.Request, body *T) (*T, error) { + if r == nil || r.Body == nil || r.Body == http.NoBody { + return nil, nil + } + + if err := json.NewDecoder(r.Body).Decode(body); err != nil { + if errors.Is(err, io.EOF) { + return nil, nil + } + return nil, err + } + + return body, nil +} diff --git a/lib/oapi/optional_json_test.go b/lib/oapi/optional_json_test.go new file mode 100644 index 00000000..ead11d02 --- /dev/null +++ b/lib/oapi/optional_json_test.go @@ -0,0 +1,52 @@ +package oapi + +import ( + "bytes" + "io" + "net/http/httptest" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestDecodeOptionalJSONBody(t *testing.T) { + t.Parallel() + + t.Run("no body returns nil", func(t *testing.T) { + t.Parallel() + + req := httptest.NewRequest("POST", "/instances/test/standby", nil) + var body StandbyInstanceJSONRequestBody + + decoded, err := decodeOptionalJSONBody(req, &body) + require.NoError(t, err) + assert.Nil(t, decoded) + }) + + t.Run("empty reader returns nil", func(t *testing.T) { + t.Parallel() + + req := httptest.NewRequest("POST", "/instances/test/standby", io.NopCloser(bytes.NewBuffer(nil))) + var body StandbyInstanceJSONRequestBody + + decoded, err := decodeOptionalJSONBody(req, &body) + require.NoError(t, err) + assert.Nil(t, decoded) + }) + + t.Run("json object decodes", func(t *testing.T) { + t.Parallel() + + req := httptest.NewRequest("POST", "/instances/test/standby", bytes.NewBufferString(`{"compression":{"enabled":true,"algorithm":"lz4"}}`)) + var body StandbyInstanceJSONRequestBody + + decoded, err := decodeOptionalJSONBody(req, &body) + require.NoError(t, err) + require.NotNil(t, decoded) + require.NotNil(t, decoded.Compression) + assert.True(t, decoded.Compression.Enabled) + require.NotNil(t, decoded.Compression.Algorithm) + assert.Equal(t, SnapshotCompressionConfigAlgorithm("lz4"), *decoded.Compression.Algorithm) + }) +} diff --git a/lib/providers/providers.go b/lib/providers/providers.go index 55b527e5..ea72fbb0 100644 --- a/lib/providers/providers.go +++ b/lib/providers/providers.go @@ -127,14 +127,7 @@ func ProvideInstanceManager(p *paths.Paths, cfg *config.Config, imageManager ima meter := otel.GetMeterProvider().Meter("hypeman") tracer := otel.GetTracerProvider().Tracer("hypeman") defaultHypervisor := hypervisor.Type(cfg.Hypervisor.Default) - level := cfg.Snapshot.CompressionDefault.Level - snapshotDefaults := instances.SnapshotPolicy{ - Compression: &snapshot.SnapshotCompressionConfig{ - Enabled: cfg.Snapshot.CompressionDefault.Enabled, - Algorithm: snapshot.SnapshotCompressionAlgorithm(strings.ToLower(cfg.Snapshot.CompressionDefault.Algorithm)), - Level: &level, - }, - } + snapshotDefaults := snapshotDefaultsFromConfig(cfg) memoryPolicy := guestmemory.Policy{ Enabled: cfg.Hypervisor.Memory.Enabled, KernelPageInitMode: guestmemory.KernelPageInitMode(cfg.Hypervisor.Memory.KernelPageInitMode), @@ -144,6 +137,19 @@ func ProvideInstanceManager(p *paths.Paths, cfg *config.Config, imageManager ima return instances.NewManager(p, imageManager, systemManager, networkManager, deviceManager, volumeManager, limits, defaultHypervisor, snapshotDefaults, meter, tracer, memoryPolicy), nil } +func snapshotDefaultsFromConfig(cfg *config.Config) instances.SnapshotPolicy { + algorithm := snapshot.SnapshotCompressionAlgorithm(strings.ToLower(cfg.Snapshot.CompressionDefault.Algorithm)) + compression := &snapshot.SnapshotCompressionConfig{ + Enabled: cfg.Snapshot.CompressionDefault.Enabled, + Algorithm: algorithm, + } + if algorithm == "" || algorithm == snapshot.SnapshotCompressionAlgorithmZstd { + level := cfg.Snapshot.CompressionDefault.Level + compression.Level = &level + } + return instances.SnapshotPolicy{Compression: compression} +} + // ProvideVolumeManager provides the volume manager func ProvideVolumeManager(p *paths.Paths, cfg *config.Config) (volumes.Manager, error) { // Parse max total volume storage (empty or "0" means unlimited) diff --git a/lib/providers/providers_test.go b/lib/providers/providers_test.go new file mode 100644 index 00000000..ad73c457 --- /dev/null +++ b/lib/providers/providers_test.go @@ -0,0 +1,49 @@ +package providers + +import ( + "testing" + + "github.com/kernel/hypeman/cmd/api/config" + snapshotstore "github.com/kernel/hypeman/lib/snapshot" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestSnapshotDefaultsFromConfigOmitsLevelForLZ4(t *testing.T) { + t.Parallel() + + cfg := &config.Config{ + Snapshot: config.SnapshotConfig{ + CompressionDefault: config.SnapshotCompressionDefaultConfig{ + Enabled: true, + Algorithm: "lz4", + Level: 7, + }, + }, + } + + defaults := snapshotDefaultsFromConfig(cfg) + require.NotNil(t, defaults.Compression) + assert.True(t, defaults.Compression.Enabled) + assert.Equal(t, snapshotstore.SnapshotCompressionAlgorithmLz4, defaults.Compression.Algorithm) + assert.Nil(t, defaults.Compression.Level) +} + +func TestSnapshotDefaultsFromConfigKeepsZstdLevel(t *testing.T) { + t.Parallel() + + cfg := &config.Config{ + Snapshot: config.SnapshotConfig{ + CompressionDefault: config.SnapshotCompressionDefaultConfig{ + Enabled: true, + Algorithm: "zstd", + Level: 5, + }, + }, + } + + defaults := snapshotDefaultsFromConfig(cfg) + require.NotNil(t, defaults.Compression) + require.NotNil(t, defaults.Compression.Level) + assert.Equal(t, 5, *defaults.Compression.Level) +} From e2d691e0a61116828ce80afbbdafd3b01bc3e09c Mon Sep 17 00:00:00 2001 From: Steven Miller Date: Fri, 20 Mar 2026 10:21:16 -0400 Subject: [PATCH 11/31] Update Stainless model config --- stainless.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/stainless.yaml b/stainless.yaml index 53f516fd..f99116b3 100644 --- a/stainless.yaml +++ b/stainless.yaml @@ -42,6 +42,10 @@ environments: # # [configuration guide]: https://www.stainless.com/docs/guides/configure#resources resources: + $shared: + models: + snapshot_compression_config: "#/components/schemas/SnapshotCompressionConfig" + health: # Configure the methods defined in this resource. Each key in the object is the # name of the method and the value is either an endpoint (for example, `get /foo`) @@ -67,6 +71,7 @@ resources: instances: models: + snapshot_policy: "#/components/schemas/SnapshotPolicy" volume_mount: "#/components/schemas/VolumeMount" port_mapping: "#/components/schemas/PortMapping" instance: "#/components/schemas/Instance" From 3a12b9eb80235242f67c5d08ae96a186ebc44707 Mon Sep 17 00:00:00 2001 From: Steven Miller Date: Fri, 20 Mar 2026 10:31:28 -0400 Subject: [PATCH 12/31] Serialize shared initrd rebuilds --- lib/system/manager.go | 21 +++++++++++++++++++++ lib/system/manager_test.go | 13 +++++++++++++ 2 files changed, 34 insertions(+) diff --git a/lib/system/manager.go b/lib/system/manager.go index ac28f540..6ea7ffc2 100644 --- a/lib/system/manager.go +++ b/lib/system/manager.go @@ -4,10 +4,14 @@ import ( "context" "fmt" "os" + "path/filepath" + "sync" "github.com/kernel/hypeman/lib/paths" ) +var initrdEnsureLocks sync.Map + // Manager handles system files (kernel, initrd) type Manager interface { // EnsureSystemFiles ensures default kernel and initrd exist @@ -34,6 +38,19 @@ func NewManager(p *paths.Paths) Manager { } } +func initrdEnsureLockKey(initrdDir string) string { + if resolved, err := filepath.EvalSymlinks(initrdDir); err == nil { + return resolved + } + return initrdDir +} + +func getInitrdEnsureLock(initrdDir string) *sync.Mutex { + key := initrdEnsureLockKey(initrdDir) + lock, _ := initrdEnsureLocks.LoadOrStore(key, &sync.Mutex{}) + return lock.(*sync.Mutex) +} + // EnsureSystemFiles ensures default kernel and initrd exist, downloading/building if needed func (m *manager) EnsureSystemFiles(ctx context.Context) error { kernelVer := m.GetDefaultKernelVersion() @@ -44,6 +61,10 @@ func (m *manager) EnsureSystemFiles(ctx context.Context) error { } // Ensure initrd exists (builds if missing or stale) + initrdLock := getInitrdEnsureLock(m.paths.SystemInitrdDir(GetArch())) + initrdLock.Lock() + defer initrdLock.Unlock() + if _, err := m.ensureInitrd(ctx); err != nil { return fmt.Errorf("ensure initrd: %w", err) } diff --git a/lib/system/manager_test.go b/lib/system/manager_test.go index 2b6e9406..8c70e93c 100644 --- a/lib/system/manager_test.go +++ b/lib/system/manager_test.go @@ -2,6 +2,8 @@ package system import ( "context" + "os" + "path/filepath" "testing" "github.com/kernel/hypeman/lib/paths" @@ -64,3 +66,14 @@ func TestInitBinaryEmbedded(t *testing.T) { assert.NotEmpty(t, InitBinary, "init binary should be embedded") assert.Greater(t, len(InitBinary), 100000, "init binary should be at least 100KB") } + +func TestInitrdEnsureLockKeyResolvesSymlinks(t *testing.T) { + target := filepath.Join(t.TempDir(), "shared-initrd") + require.NoError(t, os.MkdirAll(target, 0755)) + + wrapperRoot := t.TempDir() + wrapper := filepath.Join(wrapperRoot, "initrd") + require.NoError(t, os.Symlink(target, wrapper)) + + assert.Equal(t, initrdEnsureLockKey(target), initrdEnsureLockKey(wrapper)) +} From 84113aaf12b74bdf20c642b266cd243faf0fb90e Mon Sep 17 00:00:00 2001 From: Steven Miller Date: Fri, 20 Mar 2026 10:40:20 -0400 Subject: [PATCH 13/31] Fix disabled snapshot defaults fallback --- lib/providers/providers.go | 6 +++++- lib/providers/providers_test.go | 17 +++++++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/lib/providers/providers.go b/lib/providers/providers.go index ea72fbb0..3ab0a9ed 100644 --- a/lib/providers/providers.go +++ b/lib/providers/providers.go @@ -138,9 +138,13 @@ func ProvideInstanceManager(p *paths.Paths, cfg *config.Config, imageManager ima } func snapshotDefaultsFromConfig(cfg *config.Config) instances.SnapshotPolicy { + if !cfg.Snapshot.CompressionDefault.Enabled { + return instances.SnapshotPolicy{} + } + algorithm := snapshot.SnapshotCompressionAlgorithm(strings.ToLower(cfg.Snapshot.CompressionDefault.Algorithm)) compression := &snapshot.SnapshotCompressionConfig{ - Enabled: cfg.Snapshot.CompressionDefault.Enabled, + Enabled: true, Algorithm: algorithm, } if algorithm == "" || algorithm == snapshot.SnapshotCompressionAlgorithmZstd { diff --git a/lib/providers/providers_test.go b/lib/providers/providers_test.go index ad73c457..ab387eaf 100644 --- a/lib/providers/providers_test.go +++ b/lib/providers/providers_test.go @@ -9,6 +9,23 @@ import ( "github.com/stretchr/testify/require" ) +func TestSnapshotDefaultsFromConfigDisabledReturnsNilCompression(t *testing.T) { + t.Parallel() + + cfg := &config.Config{ + Snapshot: config.SnapshotConfig{ + CompressionDefault: config.SnapshotCompressionDefaultConfig{ + Enabled: false, + Algorithm: "lz4", + Level: 7, + }, + }, + } + + defaults := snapshotDefaultsFromConfig(cfg) + assert.Nil(t, defaults.Compression) +} + func TestSnapshotDefaultsFromConfigOmitsLevelForLZ4(t *testing.T) { t.Parallel() From e21be0c0b76251cd81b262e32cba75f8108fe2f5 Mon Sep 17 00:00:00 2001 From: Steven Miller Date: Fri, 20 Mar 2026 10:54:57 -0400 Subject: [PATCH 14/31] Make snapshot compression fully opt-in --- cmd/api/config/config.go | 35 +++- cmd/api/config/config_test.go | 36 +++- .../compression_integration_linux_test.go | 24 +-- lib/instances/snapshot_compression.go | 64 +++++-- lib/instances/snapshot_compression_test.go | 30 +-- lib/oapi/oapi.go | 180 +++++++++--------- lib/providers/providers.go | 4 +- lib/providers/providers_test.go | 13 +- lib/snapshot/README.md | 12 +- lib/snapshot/types.go | 9 + openapi.yaml | 6 +- 11 files changed, 257 insertions(+), 156 deletions(-) diff --git a/cmd/api/config/config.go b/cmd/api/config/config.go index 62472cd0..b4b99cef 100644 --- a/cmd/api/config/config.go +++ b/cmd/api/config/config.go @@ -9,6 +9,7 @@ import ( "strings" "time" + "github.com/kernel/hypeman/lib/snapshot" "github.com/knadh/koanf/parsers/yaml" "github.com/knadh/koanf/providers/env" "github.com/knadh/koanf/providers/file" @@ -180,7 +181,7 @@ type HypervisorMemoryConfig struct { type SnapshotCompressionDefaultConfig struct { Enabled bool `koanf:"enabled"` Algorithm string `koanf:"algorithm"` - Level int `koanf:"level"` + Level *int `koanf:"level"` } // SnapshotConfig holds snapshot defaults. @@ -351,7 +352,7 @@ func defaultConfig() *Config { CompressionDefault: SnapshotCompressionDefaultConfig{ Enabled: false, Algorithm: "zstd", - Level: 1, + Level: intPtr(snapshot.DefaultSnapshotCompressionZstdLevel), }, }, @@ -468,16 +469,32 @@ func (c *Config) Validate() error { return fmt.Errorf("build.timeout must be positive, got %d", c.Build.Timeout) } algorithm := strings.ToLower(c.Snapshot.CompressionDefault.Algorithm) - switch algorithm { - case "", "zstd", "lz4": - default: - return fmt.Errorf("snapshot.compression_default.algorithm must be one of zstd or lz4, got %q", c.Snapshot.CompressionDefault.Algorithm) - } - if algorithm != "lz4" && c.Snapshot.CompressionDefault.Level < 1 { - return fmt.Errorf("snapshot.compression_default.level must be >= 1, got %d", c.Snapshot.CompressionDefault.Level) + if c.Snapshot.CompressionDefault.Enabled { + switch algorithm { + case "", "zstd", "lz4": + default: + return fmt.Errorf("snapshot.compression_default.algorithm must be one of zstd or lz4, got %q", c.Snapshot.CompressionDefault.Algorithm) + } + if c.Snapshot.CompressionDefault.Level != nil { + level := *c.Snapshot.CompressionDefault.Level + switch algorithm { + case "", "zstd": + if level < snapshot.MinSnapshotCompressionZstdLevel || level > snapshot.MaxSnapshotCompressionZstdLevel { + return fmt.Errorf("snapshot.compression_default.level must be between %d and %d for zstd, got %d", snapshot.MinSnapshotCompressionZstdLevel, snapshot.MaxSnapshotCompressionZstdLevel, level) + } + case "lz4": + if level < snapshot.MinSnapshotCompressionLz4Level || level > snapshot.MaxSnapshotCompressionLz4Level { + return fmt.Errorf("snapshot.compression_default.level must be between %d and %d for lz4, got %d", snapshot.MinSnapshotCompressionLz4Level, snapshot.MaxSnapshotCompressionLz4Level, level) + } + } + } } if c.Hypervisor.Memory.KernelPageInitMode != "performance" && c.Hypervisor.Memory.KernelPageInitMode != "hardened" { return fmt.Errorf("hypervisor.memory.kernel_page_init_mode must be one of {performance,hardened}, got %q", c.Hypervisor.Memory.KernelPageInitMode) } return nil } + +func intPtr(v int) *int { + return &v +} diff --git a/cmd/api/config/config_test.go b/cmd/api/config/config_test.go index 8f252be7..16c24592 100644 --- a/cmd/api/config/config_test.go +++ b/cmd/api/config/config_test.go @@ -88,9 +88,43 @@ func TestValidateAllowsLZ4CompressionDefaultWithImplicitLevel(t *testing.T) { cfg := defaultConfig() cfg.Snapshot.CompressionDefault.Enabled = true cfg.Snapshot.CompressionDefault.Algorithm = "lz4" - cfg.Snapshot.CompressionDefault.Level = 1 + cfg.Snapshot.CompressionDefault.Level = nil if err := cfg.Validate(); err != nil { t.Fatalf("expected lz4 compression default to validate, got %v", err) } } + +func TestValidateAllowsExplicitLZ4CompressionLevelRange(t *testing.T) { + cfg := defaultConfig() + cfg.Snapshot.CompressionDefault.Enabled = true + cfg.Snapshot.CompressionDefault.Algorithm = "lz4" + cfg.Snapshot.CompressionDefault.Level = intPtr(9) + + if err := cfg.Validate(); err != nil { + t.Fatalf("expected lz4 level to validate, got %v", err) + } +} + +func TestValidateRejectsInvalidLZ4CompressionLevel(t *testing.T) { + cfg := defaultConfig() + cfg.Snapshot.CompressionDefault.Enabled = true + cfg.Snapshot.CompressionDefault.Algorithm = "lz4" + cfg.Snapshot.CompressionDefault.Level = intPtr(10) + + err := cfg.Validate() + if err == nil { + t.Fatalf("expected validation error for invalid lz4 level") + } +} + +func TestValidateAllowsDisabledSnapshotCompressionDefaultWithoutValidAlgorithm(t *testing.T) { + cfg := defaultConfig() + cfg.Snapshot.CompressionDefault.Enabled = false + cfg.Snapshot.CompressionDefault.Algorithm = "definitely-not-real" + cfg.Snapshot.CompressionDefault.Level = intPtr(999) + + if err := cfg.Validate(); err != nil { + t.Fatalf("expected disabled snapshot compression default to ignore algorithm/level, got %v", err) + } +} diff --git a/lib/instances/compression_integration_linux_test.go b/lib/instances/compression_integration_linux_test.go index 1cc14546..dc2c6bb1 100644 --- a/lib/instances/compression_integration_linux_test.go +++ b/lib/instances/compression_integration_linux_test.go @@ -29,7 +29,6 @@ type compressionIntegrationHarness struct { setup func(t *testing.T) (*manager, string) requirePrereqs func(t *testing.T) waitHypervisorUp func(ctx context.Context, inst *Instance) error - testImplicitLZ4 bool } func TestCloudHypervisorStandbyRestoreCompressionScenarios(t *testing.T) { @@ -45,7 +44,6 @@ func TestCloudHypervisorStandbyRestoreCompressionScenarios(t *testing.T) { waitHypervisorUp: func(ctx context.Context, inst *Instance) error { return waitForVMReady(ctx, inst.SocketPath, 10*time.Second) }, - testImplicitLZ4: true, }) } @@ -186,10 +184,19 @@ func runStandbyRestoreCompressionScenarios(t *testing.T, harness compressionInte }, }, { - name: "lz4", + name: "lz4-0", cfg: &snapshotstore.SnapshotCompressionConfig{ Enabled: true, Algorithm: snapshotstore.SnapshotCompressionAlgorithmLz4, + Level: intPtr(0), + }, + }, + { + name: "lz4-9", + cfg: &snapshotstore.SnapshotCompressionConfig{ + Enabled: true, + Algorithm: snapshotstore.SnapshotCompressionAlgorithmLz4, + Level: intPtr(9), }, }, } @@ -198,10 +205,6 @@ func runStandbyRestoreCompressionScenarios(t *testing.T, harness compressionInte inst = runCompressionCycle(t, ctx, mgr, p, inst, harness.waitHypervisorUp, tc.name, tc.cfg, true) } - if harness.testImplicitLZ4 { - inst = runCompressionCycle(t, ctx, mgr, p, inst, harness.waitHypervisorUp, "implicit-default-lz4", nil, true) - } - require.NoError(t, mgr.DeleteInstance(ctx, inst.Id)) deleted = true } @@ -235,7 +238,7 @@ func runCompressionCycle( if waitForCompression { waitForCompressionJobCompletion(t, mgr, jobKey, 3*time.Minute) - requireCompressedSnapshotFile(t, snapshotDir, effectiveCompressionForCycle(inst.HypervisorType, cfg)) + requireCompressedSnapshotFile(t, snapshotDir, effectiveCompressionForCycle(cfg)) } else { waitForCompressionJobStart(t, mgr, jobKey, 15*time.Second) } @@ -249,7 +252,7 @@ func runCompressionCycle( return inst } -func effectiveCompressionForCycle(hvType hypervisor.Type, cfg *snapshotstore.SnapshotCompressionConfig) snapshotstore.SnapshotCompressionConfig { +func effectiveCompressionForCycle(cfg *snapshotstore.SnapshotCompressionConfig) snapshotstore.SnapshotCompressionConfig { if cfg != nil { normalized, err := normalizeCompressionConfig(cfg) if err != nil { @@ -257,9 +260,6 @@ func effectiveCompressionForCycle(hvType hypervisor.Type, cfg *snapshotstore.Sna } return normalized } - if hvType == hypervisor.TypeCloudHypervisor { - return defaultCloudHypervisorStandbyCompressionPolicy() - } return snapshotstore.SnapshotCompressionConfig{Enabled: false} } diff --git a/lib/instances/snapshot_compression.go b/lib/instances/snapshot_compression.go index cb9c56f2..6143ebd2 100644 --- a/lib/instances/snapshot_compression.go +++ b/lib/instances/snapshot_compression.go @@ -18,8 +18,12 @@ import ( ) const ( - defaultSnapshotCompressionZstdLevel = 1 - maxSnapshotCompressionZstdLevel = 19 + defaultSnapshotCompressionZstdLevel = snapshotstore.DefaultSnapshotCompressionZstdLevel + minSnapshotCompressionZstdLevel = snapshotstore.MinSnapshotCompressionZstdLevel + maxSnapshotCompressionZstdLevel = snapshotstore.MaxSnapshotCompressionZstdLevel + defaultSnapshotCompressionLz4Level = snapshotstore.DefaultSnapshotCompressionLz4Level + minSnapshotCompressionLz4Level = snapshotstore.MinSnapshotCompressionLz4Level + maxSnapshotCompressionLz4Level = snapshotstore.MaxSnapshotCompressionLz4Level ) type compressionJob struct { @@ -82,14 +86,19 @@ func normalizeCompressionConfig(cfg *snapshotstore.SnapshotCompressionConfig) (s if cfg.Level != nil { level = *cfg.Level } - if level < 1 || level > maxSnapshotCompressionZstdLevel { - return snapshotstore.SnapshotCompressionConfig{}, fmt.Errorf("%w: invalid zstd level %d (must be between 1 and %d)", ErrInvalidRequest, level, maxSnapshotCompressionZstdLevel) + if level < minSnapshotCompressionZstdLevel || level > maxSnapshotCompressionZstdLevel { + return snapshotstore.SnapshotCompressionConfig{}, fmt.Errorf("%w: invalid zstd level %d (must be between %d and %d)", ErrInvalidRequest, level, minSnapshotCompressionZstdLevel, maxSnapshotCompressionZstdLevel) } normalized.Level = &level case snapshotstore.SnapshotCompressionAlgorithmLz4: + level := defaultSnapshotCompressionLz4Level if cfg.Level != nil { - return snapshotstore.SnapshotCompressionConfig{}, fmt.Errorf("%w: lz4 does not support level", ErrInvalidRequest) + level = *cfg.Level + } + if level < minSnapshotCompressionLz4Level || level > maxSnapshotCompressionLz4Level { + return snapshotstore.SnapshotCompressionConfig{}, fmt.Errorf("%w: invalid lz4 level %d (must be between %d and %d)", ErrInvalidRequest, level, minSnapshotCompressionLz4Level, maxSnapshotCompressionLz4Level) } + normalized.Level = &level } return normalized, nil } @@ -107,13 +116,6 @@ func (m *manager) resolveSnapshotCompressionPolicy(stored *StoredMetadata, overr return snapshotstore.SnapshotCompressionConfig{Enabled: false}, nil } -func defaultCloudHypervisorStandbyCompressionPolicy() snapshotstore.SnapshotCompressionConfig { - return snapshotstore.SnapshotCompressionConfig{ - Enabled: true, - Algorithm: snapshotstore.SnapshotCompressionAlgorithmLz4, - } -} - func (m *manager) resolveStandbyCompressionPolicy(stored *StoredMetadata, override *snapshotstore.SnapshotCompressionConfig) (*snapshotstore.SnapshotCompressionConfig, error) { if override != nil { cfg, err := normalizeCompressionConfig(override) @@ -148,11 +150,6 @@ func (m *manager) resolveStandbyCompressionPolicy(stored *StoredMetadata, overri return &cfg, nil } - if stored != nil && stored.HypervisorType == hypervisor.TypeCloudHypervisor { - cfg := defaultCloudHypervisorStandbyCompressionPolicy() - return &cfg, nil - } - return nil, nil } @@ -434,7 +431,11 @@ func runCompression(ctx context.Context, srcPath, dstPath string, cfg snapshotst } case snapshotstore.SnapshotCompressionAlgorithmLz4: enc := lz4.NewWriter(dst) - if err := enc.Apply(lz4.CompressionLevelOption(lz4.Fast)); err != nil { + level := defaultSnapshotCompressionLz4Level + if cfg.Level != nil { + level = *cfg.Level + } + if err := enc.Apply(lz4.CompressionLevelOption(lz4CompressionLevel(level))); err != nil { return fmt.Errorf("configure lz4 encoder: %w", err) } if err := copyWithContext(ctx, enc, src); err != nil { @@ -450,6 +451,33 @@ func runCompression(ctx context.Context, srcPath, dstPath string, cfg snapshotst return nil } +func lz4CompressionLevel(level int) lz4.CompressionLevel { + switch level { + case 0: + return lz4.Fast + case 1: + return lz4.Level1 + case 2: + return lz4.Level2 + case 3: + return lz4.Level3 + case 4: + return lz4.Level4 + case 5: + return lz4.Level5 + case 6: + return lz4.Level6 + case 7: + return lz4.Level7 + case 8: + return lz4.Level8 + case 9: + return lz4.Level9 + default: + return lz4.Fast + } +} + func decompressSnapshotMemoryFile(ctx context.Context, compressedPath string, algorithm snapshotstore.SnapshotCompressionAlgorithm) error { rawPath := strings.TrimSuffix(strings.TrimSuffix(compressedPath, ".zst"), ".lz4") tmpRawPath := rawPath + ".tmp" diff --git a/lib/instances/snapshot_compression_test.go b/lib/instances/snapshot_compression_test.go index bf599481..6a293f0b 100644 --- a/lib/instances/snapshot_compression_test.go +++ b/lib/instances/snapshot_compression_test.go @@ -4,7 +4,6 @@ import ( "errors" "testing" - "github.com/kernel/hypeman/lib/hypervisor" snapshotstore "github.com/kernel/hypeman/lib/snapshot" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -37,10 +36,21 @@ func TestNormalizeCompressionConfig(t *testing.T) { _, err = normalizeCompressionConfig(&snapshotstore.SnapshotCompressionConfig{ Enabled: true, Algorithm: snapshotstore.SnapshotCompressionAlgorithmLz4, - Level: intPtr(1), + Level: intPtr(10), }) require.Error(t, err) assert.True(t, errors.Is(err, ErrInvalidRequest)) + + cfg, err = normalizeCompressionConfig(&snapshotstore.SnapshotCompressionConfig{ + Enabled: true, + Algorithm: snapshotstore.SnapshotCompressionAlgorithmLz4, + Level: intPtr(9), + }) + require.NoError(t, err) + assert.True(t, cfg.Enabled) + assert.Equal(t, snapshotstore.SnapshotCompressionAlgorithmLz4, cfg.Algorithm) + require.NotNil(t, cfg.Level) + assert.Equal(t, 9, *cfg.Level) } func TestResolveSnapshotCompressionPolicyPrecedence(t *testing.T) { @@ -78,7 +88,8 @@ func TestResolveSnapshotCompressionPolicyPrecedence(t *testing.T) { cfg, err = m.resolveSnapshotCompressionPolicy(stored, nil) require.NoError(t, err) assert.Equal(t, snapshotstore.SnapshotCompressionAlgorithmLz4, cfg.Algorithm) - assert.Nil(t, cfg.Level) + require.NotNil(t, cfg.Level) + assert.Equal(t, 0, *cfg.Level) cfg, err = m.resolveSnapshotCompressionPolicy(&StoredMetadata{}, nil) require.NoError(t, err) @@ -87,28 +98,25 @@ func TestResolveSnapshotCompressionPolicyPrecedence(t *testing.T) { assert.Equal(t, 2, *cfg.Level) } -func TestResolveStandbyCompressionPolicyCloudHypervisorDefault(t *testing.T) { +func TestResolveStandbyCompressionPolicyIsOptInOnly(t *testing.T) { t.Parallel() m := &manager{} cfg, err := m.resolveStandbyCompressionPolicy(&StoredMetadata{ - HypervisorType: hypervisor.TypeCloudHypervisor, + HypervisorType: "cloud-hypervisor", }, nil) require.NoError(t, err) - require.NotNil(t, cfg) - assert.True(t, cfg.Enabled) - assert.Equal(t, snapshotstore.SnapshotCompressionAlgorithmLz4, cfg.Algorithm) - assert.Nil(t, cfg.Level) + assert.Nil(t, cfg) cfg, err = m.resolveStandbyCompressionPolicy(&StoredMetadata{ - HypervisorType: hypervisor.TypeCloudHypervisor, + HypervisorType: "cloud-hypervisor", }, &snapshotstore.SnapshotCompressionConfig{Enabled: false}) require.NoError(t, err) assert.Nil(t, cfg) cfg, err = m.resolveStandbyCompressionPolicy(&StoredMetadata{ - HypervisorType: hypervisor.TypeQEMU, + HypervisorType: "qemu", }, nil) require.NoError(t, err) assert.Nil(t, cfg) diff --git a/lib/oapi/oapi.go b/lib/oapi/oapi.go index 7e6fb197..e6bdc580 100644 --- a/lib/oapi/oapi.go +++ b/lib/oapi/oapi.go @@ -1066,17 +1066,17 @@ type SnapshotSourceHypervisor string // SnapshotCompressionConfig defines model for SnapshotCompressionConfig. type SnapshotCompressionConfig struct { - // Algorithm Compression algorithm (defaults to zstd when enabled) + // Algorithm Compression algorithm (defaults to zstd when enabled). Ignored when enabled is false. Algorithm *SnapshotCompressionConfigAlgorithm `json:"algorithm,omitempty"` // Enabled Enable snapshot memory compression Enabled bool `json:"enabled"` - // Level Compression level for zstd only + // Level Compression level. Allowed ranges are zstd=1-19 and lz4=0-9. When omitted, zstd defaults to 1 and lz4 defaults to 0. Ignored when enabled is false. Level *int `json:"level,omitempty"` } -// SnapshotCompressionConfigAlgorithm Compression algorithm (defaults to zstd when enabled) +// SnapshotCompressionConfigAlgorithm Compression algorithm (defaults to zstd when enabled). Ignored when enabled is false. type SnapshotCompressionConfigAlgorithm string // SnapshotKind Snapshot capture kind @@ -13024,12 +13024,11 @@ func (sh *strictHandler) StandbyInstance(w http.ResponseWriter, r *http.Request, request.Id = id var body StandbyInstanceJSONRequestBody - decodedBody, err := decodeOptionalJSONBody(r, &body) - if err != nil { + if err := json.NewDecoder(r.Body).Decode(&body); err != nil { sh.options.RequestErrorHandlerFunc(w, r, fmt.Errorf("can't decode JSON body: %w", err)) return } - request.Body = decodedBody + request.Body = &body handler := func(ctx context.Context, w http.ResponseWriter, r *http.Request, request interface{}) (interface{}, error) { return sh.ssi.StandbyInstance(ctx, request.(StandbyInstanceRequestObject)) @@ -13631,90 +13630,91 @@ var swaggerSpec = []string{ "mRxITbcDjz9uXz+Od/BkLz5YPP5je/Z4upPsej2R7iAlaq3CSC1Dql1DSkQ9S009u5MkMWWkK/MbtfW+", "DStogbHzrj3/N9MsXdXfFYzgvLrIMj/AqgBOvVjDffhM2tmvVI/r0z85Xj3tW11R1SfiR7D6VACf2k0G", "gji3vzRiMGMt+c5FqWFrzrPy2nQd7/H5k8DR9u5yA8R9+FwhjJUT9n4Fx17mah7xfMoFVbNkNXvIm+Xx", - "R1D15KNU0bKPnj0z+mXQCeKPe9UzYZ+398a08TQ5gtmtLLP5FnZdSGm2epUm65lW0mFhkFG7TJTWWxnt", - "ElZtym+W1DfxNhvcYtGmyLHjyK9zcqjAtHi9BNaaD+AdZfVeJTi2T0BUk8GNi0g531Pu0v91sw/ZL9eX", - "Br4zWL21BLApBKWUlmvn0X41L9dh928mDxca9QZbv/z1f3ff/+VP/lyMFfIoiehGZAImiyuy6JoiE5oU", - "96r5C0ztV6mwzWCoCE5ANgiviJFFEvyhPN9H/fwycfEKJ0tLAFtPQln+99oF+euzL4HRePs1JddO9A55", - "Uh5SqbTka5OnoFJjtEGSVC1cggB3qbl5M+/Dw7zDhmLNXzX0qv/kawSKX6yMDP8BU7uXnUPdhNa6hS7t", - "f2M4pv9W5Lge5WGuHm262mpUQi0Jp1Td5kuThGdMjeDub/mCQ78z94o2FHqa1XPCbCVMbdnEC8vx+gRH", - "wDtX3iQXp8x5SnbrDLfhgnSl20ZpZaWZNO+N8QxejndeAaAzDZrrGRGktBHwQRE9fkOQ2Vu+9bHFxjNS", - "623dej5jk3JLULg2tAAygNUgyG+ClyWj1dERp/hDPgJIzVgu6ZawjlLF7BdPIY3eG5fXlk5cFzCNek3T", - "p+uxqE1lnuXNKGPV8rpNe+/Bs7RqBfVrOls15CzGqKDmMj5qMkfCTFC1ONdkyMZwQV2vw8ygIdAnWAQ8", - "LgaH+Hqow06t60rNdYYwImiIDs9ObFUvBrwcvTtFMZ2QcBHGxIZHL3kUg0Hz9dFJ1+R1yMtF6uGpAoC4", - "ugiHZyeQZl0YCSno93Z6UH0bCsWlNBgEu71tSDqvwQBL3IJ0PPDT+jvocwhc7ySy3PmpaaK/EjghCop3", - "/u7xG1BEmPQ+EjzG8LQk2KSYCivZpDF4MxhVgepvIaDDEfiB4RIdA3DcVjGVamHvdkj62m7re40OMuVM", - "mg3d6feNIMmUZQe4SL+99XdpBMti3FZSBoDHE92wJPI5SceC/HMn2Otv32g+azNm+4a9YNiWhyMwzUc3", - "BMKtBj1h5sLZlS8ntmFxzgCFyifs9/d6v2SWJFgsHLgKWKVcNoloRCIMOXtNbqm/83EPWUMJBHTLGc/i", - "CErXpqYqiSajGCksetOPCItwRudkyCz3MNnPsYCEFwnSXMNcHFSPhhna7L4hO0Sqpzxa1KCbd7eluwNp", - "qwrgeuCoJCOIfxk1JY7LrYwpZQwSUEtio2zzDErLtl2oGCBD7i2VQBhmqkhAb0oFXBHw753QD94OWzmq", - "a4IH20KgMk2eQGVn0+8iA/HAfu+y4/wdsuCtMjmtI9hSjbkk4G5tsRjjOPbmzZvGfIxjW1HhingEpxfQ", - "wgKlHDrtWC7jETFhsOlCzTgzv7NxxlRmfo8Fv5ZEaMZs02FYWLvCdAZ1obQNTSAlhUm2pcfcMlPc+nRF", - "Fp97Q3YYJS6Rmq34jGPJbamJvHRhXsN8yBoDthusjEe29JRJ817OjG2myTOVZqqHzEKIsjk8oDkkTpcz", - "Eg2Z4uiTMHVyFp+3PhUjfgaJmuBI40mpiVnS1icafW6atRxhvfoRNPXoJAQAMAw0dxkG+vdUYC1RZ3KG", - "cAhhC/pheUs3zMHmAqSVzTqEQ8xQytPMVgUlyFbQqPQB+ZBwHCMFR8l9q2Ug2MmG9VhvJ19yX+vqZHxT", - "ascI0vyWDlN/78B/niQJBfGp3f95/voVAlal98A0K2L0AUamQG1eZVuP3huyZzic2cqREH01DGg0DHKZ", - "N9qEuWbS3sV2uyB4/aKn9osZpkOjX3o93ZWR6Qbo90+ml4E+S2kyUvyKsGHwuYNKL6ZUzbJx/u69H6BN", - "HiPnFUKANgzt33TZ7KDUY8EGDd/ALELc0tp4gTAqKFBZux9ThsXKVHwe0FsIagUTT2UZGJ+GYOIZBoOh", - "M/IMg84wIGwOz6wlaBh89kPApo5sDvUx2QhtswKJ9vv9zfWunBa+HhG60lAfv89L0tfOVxM8rNC1LHiY", - "xbk4Rb2DJq+kEbfuQfJ5ivMqvT9FvDUintWnS8IbfF/mAwZ9Y2Ks1zUJTCvgsZPAVmonBi0gUBc0Dud4", - "bRQO6iS4AnnL6kddyVxWK/aaTlkIU4wd/u3dA/7BuEVtFhj3yX2Ni2NTRdBVKnhY6Aib5RCx49eIXxD1", - "PWBc/75IqSsh9Q3x96Hgzwti5b4CaDVqtkXm7hbEH14CrkfS9mIaa131HObUPSdMoWfwtGf/dRoPxOpf", - "xnx6OUAGhDGfopgyIq1TXH6HoZmihSV8ZLyP8u+sM1I4w2xKJNow/PNf//gnTIqy6b/+8U8tTZtfcNy3", - "TMAVhKJfzggWakywuhyg3whJuzimc+IWA8GiZE7EAu32pa0wrl95sl/LIRuyN0Rlgsk81EqvC2BiOrTF", - "mPR6KMuItN5bUF50YmOAfinKj/vPsgHlvZ7ozvJVuFlBaQGaKzocAKduauLxrf4V+K1nZs0V+1ndgrtk", - "019PXxT5oAz2ds0Eb0hgAMS+cwcv7KLRxvn5s80eAh3DYAXEeYHEXHRjhefeT5q0niYZilIlKABlQ5tK", - "hU8a7b/Htk07A7Dt8UeyADdVcmk2ARuTBxEkcvD6qSu0MQf74eZMwz777LHz2W020N5+veUhnNtKK0X4", - "6+2zw71lmNsKyAXIvoUKjDZsQco8CXGlzPK3Qvp74Rql6tw560DcpD6+N7XsiLNJTEOFum4ukKYoIbmq", - "VkWQh0IO3thZI+zWVc+oUOZvW5UAwUZOl8cKFizv7rlHbdCbsJEi60OBaz85yTrUOaYy5PrbErZ0Q5za", - "FMxGfMnPaRmL1hmkjuF5znJWikvHed1+eyDvzzRlh85YnTfcA1E8rhHEb0gIa2llS3lSHhI2X+S76OKT", - "Vliuvi/U7N+fFHTfViwfmj8kM1ZUA5umgrO82GATetlyhHe40XYEz8LPiXCn2kzUZBgtlmU+ReGMhFdm", - "QXAhvVr3PTFN2qm+pr8fSfM1dSBvILFYkP8UUVoouwWsVim4JzZV7t3ptzDCjdTbr3fPaxHMA2RwNhk7", - "i7XJQovlgoWbP9RV771wMwPsB8nMzrI4djcecyJUUQ2zzAO2PoFb0nrZ3p22lezg4s3LLmEhBz+03IfK", - "L0S5InVfV8I3G2aW8hNN2uiEACqHGM0C9Bfsv3EXRHnFk3/beW5rnvzbznNT9eTfdg9N3ZPNO0OW/n2R", - "5vuWuB8w8mmBm1aBBqTJlJJbJ6HmrVoKqa79DyWn2rKkN5FUc7j+FFbbCKtlcK2UV/MKsXcosdpimt/m", - "SiZHNh+04ZXzT/zBJNX7tfJZjHQpoaisXnvYnL9cFAUsKUOZJA/QgZLmGFdmGy3N1cWBXMk+HOqeHHds", - "bVJTUTQPELkn47Wbx70Lt3bc+7dcHyZjOs14JsuxJ1CKlkgbrBSTKgF+aGJ3wZ4bBe/vGEv798k67l2u", - "/on3dyTx1zfUEG9zA7VO5net2sr8tj0UfTXlgEzs2htXZsgmctlscCp0hbjaonGlZtyys6NvXj5dBF1o", - "RaVQFxBoEIMh+w+tf/yuCE7e/+KCZLJ+f2cfnhM2f/+Li5Nhpw5VCFOCEomwIOjw1TFc+00heh2yURYh", - "efV5mByTprq/LTz9P05BKm4+22tIDgt/akitNKQSuFZrSHkZq7tUkarZlu5dR3L45gO4Ta3xU0u6Dy1J", - "ZpMJDSlhqkjBvuQkZis4PMDYMmbvh0rOHRVG21pLKmrLrRZAiySR9+7Ykw9+/8qRy0f5MH3kuYmKiZw6", - "UjDDZn3ke8OH/v0S5/vXQx4yihmBvw66ZUK0NbEZ4P0CwnMurtpinidr7VdHwK8vnZRX+B3KJnp6pFRm", - "9huKKMC8jW+9Rpqq5HIPB3IpFfG3dOl0kLDKrQmKpGxaFBqmasYzk1VlZB+arGz6VNhyXiDyhLbXb01e", - "9Oj3IIC+4grRJI1JQiBrW9dgE1R3ztKUi7wAJJWlxN03I3/62JQdbE1yG1uGvYNsenMw1rkN2wC7/fJ2", - "ealmzKfrg2rzwV0EqSeqdsgupEnycmlE4UuUE1mkOJIkJqFC1zMaziDCVj8z9QIg+BWn6WWeUmPTVasu", - "ZxaBwTckERTHUGaXx6Zg9OU8SS4Hyxng3p2ewkcmuNbkerscIJf1LWcQUrcqR8zqVcRYKvTKxgFvaEwS", - "PI7Njl5qLlRa36aNpS1SngyZL66WkWvbIZ2gy1KI7WVDjK0jqC/5VH4reanTnKjKrEVxJABwBjcJi4Im", - "ww6N/dG12/2+L39Ky0hfM407DvRdmsxLPs2TZFVQGadpW/S10wQsnifJChxGG6XU/1JFPFN/lSoiQsDH", - "FrubkBtt4ND8ofCVRlRma0G64gmAfl7zpcla4wWVJqqljNbmr3mSBJ3AzsdTvvzLI6brHS6b2fTOlMKi", - "f0raNwl4rhL7UsRzjXPYwkHNIreth/TD63sWUN/apvAN7GPFLChzogrsLS8y4j+oyElTKqsui5n0/b4z", - "ktfaaj4lVaPyeVEo4H+gimrWWi+Qds9Kag5in2ZWKQbzzbXTvHTITw0111C5QFFmhqtVh/ph1c6coKCM", - "VTRPK57eVvfMk8zlYIZCsGzlhUBB87Y+uZ8ntxAXvhNK2GmseNOUzqhY9PdAchuKUraiud9ITrJstSQg", - "fEMS7Mpj3jcFzqGi1b2cyn0XZNgcuJwal2mOEphJ6irf/iTGFTOgsZTelhg74XPJFlgiz5R10xg30WUr", - "pzYS4Fo1qW+qr319QthQK+uzpYTfkvAVytG9EbuTnLwZgmfrC/7gKmrIhTC+cuB995BiNUuXpCVdeyPF", - "mSSdnEJ03EX9u9PTzSYqIdRKGiHUA6YQtbp4SeSvxC9o5LLyH50e2xz+VCKRsR56nVBIlX9FSAo5OCnP", - "JAKnx165wFpT2be8ghphSixSTplaO4ui6d1M5vOtspLfM5200eo/vB3NlnZ/aEQKaIcWV+wCVmuRytQV", - "9N5Luns6ykwpASj1OuaZ7n2pABxU/5YLqUhiLiknmSmWCvlMbLpb+51x1usgqiTS56EDzk0pEQmFUphy", - "yMZkosWwlAg9NtT4pDEp3bf4rvLOFc6p5pkhfd/HXR7UhIPrK6yaoFYtB4fT1JWD890X5RXsbj2l53A5", - "h+QiGfOYhiim7EqijZheGaUDzSWK9Y/Nlbd7I/juayfzvf3J0pA+YRPuzXdocDZH5h+Bwp3UyJrzXnhw", - "ZO0FKR8WR39go/1kTa6la4LgGKqe5n7FKFM0ph8NqdOdUKloaIpE4Rx2UN/GjNcbslOihG6DBUEhj2MS", - "Kmdc2UoFD7eGWb+/G6YUAkB2CUwOCF7z6wRGPDq7gHamBk9nyPQf0PHbwzNENUwn2NoIShNlRF1zcYVO", - "tl6v8Xc4BzD9D74wNAtcdSz8G/7zKvvmTqONZ0g2HFGerlKAePrD32hbCe6nteBhWgvAaz9fzcZU4BCE", - "YjnLVMSvmd8yYErCyq1P5sfJutgPhcPZO1cb+/uQdm153HXDuAU+iENp1xQRk4/1m1xQ2ArGDzR/lQac", - "WwIIMeUoFj8XMJXRfzTs/vpG+TIcv8OrSQtRl+v4uzlb98357BxcSGMZHg/lmBtMcyuBGp1l61Mev7lW", - "NwszIQhTkPumEC1DnOKQqkUH4diVj7X1oHIbUlH5fiwIvtKctjdkb/LIUVuPSmtXHadaoYjKK9OD1Z56", - "6PWcCJmN88khIExGzwPg2wqyIY5DU3qVTCYkVHROTE1U2aB95VO5yzzExSCejXYvLegemsrhxwnYvQIt", - "rNZRcQ1szFdxnrdql68i77Xk/lNyjVnp5D1yDU3Z/5uY7DyDX9HGOAD76mbuer/pj1qOXXUL80/CvvrC", - "Vf4oaQDPS944bbNcFBj+0BJOlGZeOaoVj7b1ke+tXdju0qVsXeR7Pvh9R76fe72aHlj+LVzxU2sKef/+", - "EKF/v+7U9x3y/rBxS4sScgl0zZSoRej7d4GBdxPz/o3DCW4R8/5dObhCzPK3CzT4rlxbrYtm7tr6M6r9", - "Lj1aTWg7RPA2ebQaqmctzysVpXe2TTs1yfb4I0nw1lh5A/ndgf1njroWKkMJWI4L18gN0H5pEZ4kqVo4", - "axSfgN9NkURR0o/gveeLFMyNzncXoHcLe+zXQw+Hp43W2J+57e7N4FskAD85fvgJ7cpnrsJYtjTX6WIR", - "zui8EqC26gRbEKWCdFOegp01MgCz8HC8TGHRm35EtvvekL2dEfcXoi49CIlQRAUJVbxAlCkOFMGM8WeJ", - "BNeaALznYuEz35ZP7nPBk0O7mjX80J4pawwr3PySRTfCCnfnjtqsMKF9wZXVKf5AkywBgocoQy+eog3y", - "QQmTrQJNtOaD6CQHKfkQEhJJwMnN8oS3+w2WTfqRjKbjNrNckXfktc3rgsJMKp64vT85Rhs4U7w7JUzv", - "hRb1JyDJpoLPaWSSAhdAnfPYQHW7AaA3tbtqocL6gxfKhZncN5Fh2jCk6UeaVsmCcXsMBsGYMgyTW5vh", - "o3qmjAeuHg9T8IMrzo7DnOAnC6uXFdeYqJUcB0TFOYq1RL/5k809ZDZX9mRwPK3C7dqlbW3n3NDS5+Au", - "Urbmji/3a7Z+9/3cx5fKMD9A0/k8V0ibzObfFwr2748/3Le5/N0D9t96QZzyXTKVQwe6Rx/CvOQhjlFE", - "5iTmaaLFStM26ASZiINBMFMqHWxtxbrdjEs1OOgf9IPP7z///wAAAP//XjwQ6bMxAQA=", + "R1D15KNUUdVHr4dOpgxyUpUf53btctk4/XHQCeKPe9UzY5+399a08TY5AtqtLosBLey+kPJsNRSgSSE2", + "CsymREKpUD3nX7a720/ABST+uPdLv/ukhyDqJM/+DtAqg2/bta487beBYU4oIWMTTTRIt59AiRXzx/qA", + "EQfPVRj0m+VLTYzYRuJYHC8SAjle4TwyKhtcvF7a45rD4h2lIF8l5bbPllRTGIw/Szk5VR5/8HVTJdkv", + "19cxvjNYvbXUuileppRDbOfRfjWJ2GH3byZpGBr1Blu//PV/d9//5U/+xJEVWi6J6EZkAvaVK7LomooY", + "mm/0qskWTKFaqbBNt6gITkCQCa+IEZwS/KE830f9/OZz8QonS0sAw1RCWf732gX5i8kvgdG4JjZlAk/0", + "DnnyM1KptJhuM72gUmO0QZJULVw2A3cDu3kzV8nDvMOGytJfNU6s/+RrRLVfrAxj/wHz0Jc9Wd2E1vqw", + "Lu1/Y+yo/wrnuB6SYu5JbW7daghFLWOoVN3mG56EZ0yN4KJy+TZGvzOXoDZue5rVE9hsJUxt2SwRy8kF", + "CI6gdMbKa+/ilDm3zi58tP42d6WPSWllpZk0741xY14Ozl4BoDMNmusZEaS0EfBBEep+Q5DZK8n1gdDG", + "jVMrmd168mWTH0xQuOO0ADKA1SDIr62XxbTVoRyn+EM+Aoj4WC4pwrCOUnnvF08h598bl4SXTlwXMI16", + "Adan67GoTRmh5c0oY9Xyuk1778GztGoF9Ws6WzXkLMaooOYyPmoyR8JMULU412TIBpxBEbLDzKAh0CdY", + "BDwuBodkAFA0nlo/m5qfD2FE0BAdnp3YEmQMeDl6d4piOiHhIoyJjeVecn8G6+vro5OuSUKR17bUw1MF", + "AHFFHA7PTiAnvDASUtDv7fSgVDhUtUtpMAh2e9uQIV+DAZa4BbmD4Kd1ztDnELjeSWS581PTRH8lcEIU", + "VBr93ePkoIgwuYgkuLfhaUmwSTEVVrJJY3C9MHoL1d9C9Ikj8APDJToG4LitFi3Vwl5EkfS13db3Gh1k", + "ypk0G7rT7xtBkinLDnCRK3zr79IIlsW4raQMAI8nFGNJ5HOSjgX5506w19++0XzWpvf2DXvBsK1lR2Ca", + "j24IhFsNesLM7birtU5sw+KcAQqVT9jv7/V+ySxJsFg4cBWwSrlsEtG0zgoJhk0irL/zcQ9Zqw5En8sZ", + "z+II6uympoSKJqMYKSx6048Ii3BG52TILPcwqdqxgOwcCdJcw9xyVI+GGdrsviE7RKqnPFrUoJt3t6W7", + "A2mrCuB6lKskIwjWGTVluctNoillDLJlS2JDgvN0T8uGaChvIEPuretAGGaqyJZv6hpcEXBGntAP3g5b", + "edVrggfbQqCMTp7tZWfT788Dwct+V7jj/B2y4K0yOa0j2LqSuSTgrpixGOM49ib5m8Z8jGNb/uGKeASn", + "F9DCAqUc5+1YLuMRMTG76ULNODO/s3HGVGZ+jwW/lkRoxmxzd1hYuyp6BnWhDg9NIH+GyQymx9wyU9z6", + "dEUWn3tDdhglLuubLU+NY8ltXYy8zmJecH3IGqPLG0yiR7ZOlslJX07jbabJM5VmypqDJFE24Qg0hyzv", + "ckaiIVMcfRKmqM/i89anYsTPIFETHGk8KTUxS9r6RKPPTbOWI6xXP4KmHp2EAACGgeYuw0D/ngqsJepM", + "zhAOIcZCPyxv6YY52FyAtLJZh3CIGUp5mtkSpgTZch+VPiB5E45jpOAouW+1DAQ72bAe65rly0Rs/bKM", + "I03tGEFO4tJh6u8d+M+TJKEgPrX7P89fv0LAqvQemGZFQgGAkammm5cE16P3huwZDme2zCWEig0DGg2D", + "XOaNNmGumbQXx90uCF6/6Kn9Yobp0OiXXk93ZWS6Afr9k+lloM9SmowUvyJsGHzuoNKLKVWzbJy/e+8H", + "aJN7y3mFEKANQ/s3Xeo9qEtZsEHDNzCLELe0Nl4gjAoKVNbux5RhsTJvoAf0FoJawcRTWQbGpyGYeIbB", + "YOiMPMOgMwwIm8MzawkaBp/9ELB5LpvjkkzqRNusQKL9fn9zvd+pha9HhK401Mfv85L0tfPVBA8rdC0L", + "HmZxLqhS76BJgmnErXuQfJ7ivKTwTxFvjYhn9emS8Abfl/mAQd+YGOt1TQLTCnjsJLCV2olBC4gqBo3D", + "eYkbhYM6Ca5A3rL6UVcyl9WKvaZTFsIUY4d/e/eAfzBuUUgGxn1yX+Pi2JQ8dGUVHhY6wmY5ROz4NeIX", + "RH0PGNe/L1Lq6l19Q/x9KPjzgli5rwBajZptkbm7BfHHwoCflLS9mMZaVz2HOXXPCVPoGTzt2X+dxgOJ", + "BS5jPr0cIAPCmE9RTBmR1oMvv8PQTNHCEj4yrlL5d9ZzKpyZ2+ANwz//9Y9/wqQom/7rH//U0rT5Bcd9", + "y0SHQdz85YxgocYEq8sB+o2QtItjOiduMRDZSuZELNBuX9py6PqVJ1W3HLIhe0NUJpjM48L0ugAmpkNb", + "OUqvh7KMSOtqBrVQJzZg6ZeiVrr/LBtQ3uuJ7izfy5sVlBaguaLDAfBApyZ5gNW/Ar/1zKy5Yj+rW3CX", + "bPrr6YsiH5TB3q6Z4A0JDIDYd+7ghV002jg/f7bZQ6BjGKyAoDSQmIturPDc+0mT1tMkQ1GqBAWgbGhT", + "qUpLo/332LZpZwC2Pf5IFuCmsjPNJmBj8iCCRA5eP3WFNuZgP9ycadhnnz12DsbNBtrbr7c8hHNbaaUI", + "f719dri3DHNbrrkA2bdQgdGGrZ6ZZ0yu1IT+Vkh/L1yjVEo8Zx2ImzzN96aWHXE2iWmoUNfNBXIqJSRX", + "1aoI8lDIwRs7a4TduurpH8r8basSzdjI6fLAxoLl3T33qA16EzZSpKgocO0nJ1mHOsdUhlx/W8KWbohT", + "my/aiC/5OS1j0TqD1DE8z1nOSnHJkueTY3cg7880ZYfOWJ033ANRPK4RxG9ICGs5cEtJXR4SNl/ku+iC", + "qVZYrr4v1OzfnxR031YsH5o/JDNWVAObpoKzvDJiE3rZ2ol3uNF2BM/Cz4lwp9pM1KRDLZZlPkXhjIRX", + "ZkFwIb1a9z0xTdqpvqa/H0nzNUUrbyCxWJD/FFFaKLsFrFYpuCc2r+/d6bcwwo3U2693z2sRzANkcDYZ", + "O4u1SZmL5YKFmz/UVe+9cDMD7AfJzM6yOHY3HnMiVFG6s8wDtj6BW9J62d6dtpXs4OLNyy5hIQc/tNyH", + "yi9EuYp6X1fCNxtmlvITTdrohAAqhxjNAvQX7L9xF0R5eZZ/23luC7T8285zU6Ll33YPTZGWzTtDlv59", + "keb7lrgfMPJpgZtWgQakydS9Wyeh5q1aCqmu/Q8lp9oaqjeRVHO4/hRW2wirZXCtlFfzcrZ3KLHayp/f", + "5komRzYftOGV80/8wSTV+7XyWYx0+auorF572ATFXBTVNilDmSQP0IGS5hhXZhstzdXFgVzJPhzqnhx3", + "bCFVU/40DxC5J+O1m8e9C7d23Pu3XB8mYzrNeCbLsSdQN5dIG6wUkyoBfmhid8GeGwXv7xhL+/fJOu5d", + "rv6J93ck8dc31BBvcwO1TuZ3rdrK/LY9VKg1tYtM7NobVxPJJnLZbHAqdFXD2qJxpcDdsrOjb14+XQRd", + "aEWlUBcQaBCDIfsPrX/8rghO3v/igmSyfn9nH54TNn//i4uTYacOVQhTgtoER4evjuHabwrR65ADqQjJ", + "q8/DJMQE1HNVsv/HKUjFzWd7Dclh4U8NqZWGVALXag0pr7l1lypSNdvSvetIDt98ALepNX5qSfehJcls", + "MqEhJUwV+eKXnMRsuYkHGFvG7P1Qybmjwmhba0lFIbzVAmiR0fLeHXvywe9fOXLJMx+mjzw3UTGRU0cK", + "Ztisj3xv+NC/X+J8/3rIQ0YxI/DXQbdMiLYmNl29X0B4zsVVW8zzpNj96gj49aWT8gq/Q9lET4+UauJ+", + "QxEFmLfxrddIU5Vc7uFALuVN/pYunQ4SVrk1QZGUTYuqyFTNeGayqozsQ5OVTZ8KW3sMRJ7Q9vqtyYse", + "/R4E0FdcIZqkMUkIZG3rGmyCUtRZmnKRV6ukspRl/GbkTx+bsoOtSW5ja8Z3kM3FDsY6t2EbYLdf3i4v", + "1Yz5dH1QbT64iyD1RNUO2YU0SV4ujSh8iXIiixRHksQkVOh6RsMZRNjqZ6a4AQS/4jS9zFNqbLrS2uXM", + "IjD4hiSC4hhqAvPYVLe+nCfJ5WA5A9y701P4yATXmlxvlwPksr7lDELqVuWIWb2KGEuFXtk44A2NSYLH", + "sdnRS82FSuvbtLG0RcqTIfPF1TJybTukE3RZCrG9bIixdQT1JZ/KbyUvdZoTVZm1KI4EAM7gJmFR0GTY", + "obE/una7701S3TLS10zjjgN9lybzkk/zJFkVVMZp2hZ97TQBi+dJsgKH0UapToFUEc/UX6WKiBDwscXu", + "JuRGGzg0fyh8pRGV2cKVrtIDoJ/XfGmy1nhBpYlqKaO1+WueJEEnsPPx1Fr/8ojpeofLZja9M6Ww6J+S", + "9k0CnqvEvhTxXOMctspRs8htizf98PqeBdS3til8A/tYMQvKnKgCe8uLjPgPKnLS1PWqy2Imfb/vjOSF", + "wZpPSdWofF4UCvgfqKKatdarud2zkpqD2KeZVSrXfHPtNK9j8lNDzTVULlCUmeFqpax+WLUzJygoYxXN", + "04qnt9U98yRzOZihai1beSFQ0LytT+7nyS3Ehe+EEnYaK940pTMqFv09kNyGCpqtaO43kpMsWy0JCN+Q", + "BLtanvdNgXOoaHUvp3LfBRk2By6nxmWaowRmkroyvT+JccUMaCyltyXGTvhcsgWWyDNl3TTGTXTZyqmN", + "BLhWTeqb6mtfnxA21Mr6bCnhtyR8hXJ0b8TuJCdvhuDZYog/uIoaciGMrxx43z2kWM3SJWlJ195IcSZJ", + "J6cQHXdR/+70dLOJSgi1kkYI9YApRK0uXuJJhP/aFgW3gWVHp8c2hz+VSGSsh14nFFLlXxGSQg5OyjOJ", + "wOmxVy6w1lT2La+gRpgSi5RTptbOomh6N5P5fKus5PdMJ220+g9vR7N16B8akQLaocUVu4DVWqQydQW9", + "95Luno4yU0oA6tKOeaZ7XyoAB6XK5UIqkphLykkWwyGCfCY23a39zjjrdRBVEunz0AHnppSIhEIpTDlk", + "YzLRYlhKhB4banzSmJTuW3xXeecK51TzzJC+7+MuD2rCwfUVVk1Qq5aDw2nqysH57ovyCna3ntJzuJxD", + "cpGMeUxDFFN2JdFGTK+M0oHmEsX6x+bK270RfPe1k/ne/mRpSJ+wCffmOzQ4myPzj0DhTmpkzXkvPDiy", + "9oKUD4ujP7DRfrIm19I1QXAMVU9zv2KUKRrTj4bU6U6oVDQ0RaJwDjuob2PG6w3ZKVFCt8GCoJDHMQmV", + "M65spYKHW8Os398NUwoBILsEJgcEr/l1AiMenV1AO1ODpzNk+g/o+O3hGaIaphNsbQSliTKirrm4Qidb", + "r9f4O5wDmP4HXxiaBa46Fv4N/3mVfXOn0cYzJBuOKE9XKUA8/eFvtK0E99Na8DCtBeC1n69mYypwCEKx", + "nGUq4tfMbxkwJWHl1ifz42Rd7IfC4eydq439fUi7tjzuumHcAh/EobRriojJx/pNLihsBeMHmr9KA84t", + "AYSYchSLnwuYyug/GnZ/faN8GY7f4dWkhajLdfzdnK375nx2Di6ksQyPh3LMDaa5lUCNzrL1KY/fXKub", + "hZkQhCnIfVOIliFOcUjVooNw7MrH2npQuQ2pqHw/FgRfaU7bG7I3eeSorUeltauOU61QROWV6cFqTz30", + "ek6EzMb55BAQJqPnAfBtBdkQx6EpvUomExIqOiemJqps0L7yqdxlHuJiEM9Gu5cWdA9N5fDjBOxegRZW", + "66i4BjbmqzjPW7XLV5H3WnL/KbnGrHTyHrmGpuz/TUx2nsGvaGMcgH11M3e93/RHLceuuoX5J2FffeEq", + "f5Q0gOclb5y2WS4KDH9oCSdKM68c1YpH2/rI99YubHfpUrYu8j0f/L4j38+9Xk0PLP8WrvipNYW8f3+I", + "0L9fd+r7Dnl/2LilRQm5BLpmStQi9P27wMC7iXn/xuEEt4h5/64cXCFm+dsFGnxXrq3WRTN3bf0Z1X6X", + "Hq0mtB0ieJs8Wg3Vs5bnlYrSO9umnZpke/yRJHhrrLyB/O7A/jNHXQuVoQQsx4Vr5AZov7QIT5JULZw1", + "ik/A76ZIoijpR/De80UK5kbnuwvQu4U99uuhh8PTRmvsz9x292bwLRKAnxw//IR25TNXYSxbmut0sQhn", + "dF4JUFt1gi2IUkG6KU/BzhoZgFl4OF6msOhNPyLbfW/I3s6I+wtRlx6ERCiigoQqXiDKFAeKYMb4s0SC", + "a00A3nOx8Jlvyyf3ueDJoV3NGn5oz5Q1hhVufsmiG2GFu3NHbVaY0L7gyuoUf6BJlgDBQ5ShF0/RBvmg", + "hMlWgSZa80F0koOUfAgJiSTg5GZ5wtv9Bssm/UhG03GbWa7IO/La5nVBYSYVT9zenxyjDZwp3p0SpvdC", + "i/oTkGRTwec0MkmBC6DOeWygut0A0JvaXbVQYf3BC+XCTO6byDBtGNL0I02rZMG4PQaDYEwZhsmtzfBR", + "PVPGA1ePhyn4wRVnx2FO8JOF1cuKa0zUSo4DouIcxVqi3/zJ5h4ymyt7MjieVuF27dK2tnNuaOlzcBcp", + "W3PHl/s1W7/7fu7jS2WYH6DpfJ4rpE1m8+8LBfv3xx/u21z+7gH7b70gTvkumcqhA92jD2Fe8hDHKCJz", + "EvM00WKlaRt0gkzEwSCYKZUOtrZi3W7GpRoc9A/6wef3n/9/AAAA//9F7yIYYDIBAA==", } // GetSwagger returns the content of the embedded swagger specification file diff --git a/lib/providers/providers.go b/lib/providers/providers.go index 3ab0a9ed..7f8c14fc 100644 --- a/lib/providers/providers.go +++ b/lib/providers/providers.go @@ -147,8 +147,8 @@ func snapshotDefaultsFromConfig(cfg *config.Config) instances.SnapshotPolicy { Enabled: true, Algorithm: algorithm, } - if algorithm == "" || algorithm == snapshot.SnapshotCompressionAlgorithmZstd { - level := cfg.Snapshot.CompressionDefault.Level + if cfg.Snapshot.CompressionDefault.Level != nil { + level := *cfg.Snapshot.CompressionDefault.Level compression.Level = &level } return instances.SnapshotPolicy{Compression: compression} diff --git a/lib/providers/providers_test.go b/lib/providers/providers_test.go index ab387eaf..b950b554 100644 --- a/lib/providers/providers_test.go +++ b/lib/providers/providers_test.go @@ -17,7 +17,7 @@ func TestSnapshotDefaultsFromConfigDisabledReturnsNilCompression(t *testing.T) { CompressionDefault: config.SnapshotCompressionDefaultConfig{ Enabled: false, Algorithm: "lz4", - Level: 7, + Level: intPtr(7), }, }, } @@ -34,7 +34,7 @@ func TestSnapshotDefaultsFromConfigOmitsLevelForLZ4(t *testing.T) { CompressionDefault: config.SnapshotCompressionDefaultConfig{ Enabled: true, Algorithm: "lz4", - Level: 7, + Level: intPtr(7), }, }, } @@ -43,7 +43,8 @@ func TestSnapshotDefaultsFromConfigOmitsLevelForLZ4(t *testing.T) { require.NotNil(t, defaults.Compression) assert.True(t, defaults.Compression.Enabled) assert.Equal(t, snapshotstore.SnapshotCompressionAlgorithmLz4, defaults.Compression.Algorithm) - assert.Nil(t, defaults.Compression.Level) + require.NotNil(t, defaults.Compression.Level) + assert.Equal(t, 7, *defaults.Compression.Level) } func TestSnapshotDefaultsFromConfigKeepsZstdLevel(t *testing.T) { @@ -54,7 +55,7 @@ func TestSnapshotDefaultsFromConfigKeepsZstdLevel(t *testing.T) { CompressionDefault: config.SnapshotCompressionDefaultConfig{ Enabled: true, Algorithm: "zstd", - Level: 5, + Level: intPtr(5), }, }, } @@ -64,3 +65,7 @@ func TestSnapshotDefaultsFromConfigKeepsZstdLevel(t *testing.T) { require.NotNil(t, defaults.Compression.Level) assert.Equal(t, 5, *defaults.Compression.Level) } + +func intPtr(v int) *int { + return &v +} diff --git a/lib/snapshot/README.md b/lib/snapshot/README.md index c160426a..9aa6761d 100644 --- a/lib/snapshot/README.md +++ b/lib/snapshot/README.md @@ -35,10 +35,6 @@ Snapshots are immutable point-in-time captures of a VM that can later be: Snapshot memory compression is optional and is **off by default**. -The current exception is cloud-hypervisor standby snapshots: if no request, instance, -or server policy is set, Hypeman defaults that path to `lz4` so standby snapshots -store compressed guest memory by default on Linux. - - Compression applies only to `Standby` snapshots, because only standby snapshots contain guest memory state. - `Stopped` snapshots cannot use compression because they do not include resumable RAM state. - Compression affects only the memory snapshot file, not the entire snapshot directory. @@ -54,7 +50,7 @@ store compressed guest memory by default on Linux. - request override - instance default - server default - - then the cloud-hypervisor standby fallback (`lz4`) when no other policy is set + - otherwise no compression Compression runs **asynchronously after the snapshot is already durable on disk**. @@ -85,10 +81,12 @@ In practice, the tradeoff is: - `zstd` - default when compression is enabled - - supports configurable levels + - supports levels `1-19` + - defaults to level `1` when no level is specified - `lz4` - optimized for lower decompression overhead - - does not currently accept a level setting + - supports levels `0-9` + - defaults to level `0` (fastest) when no level is specified ### Restore (in-place) - Restore always applies to the original source VM. diff --git a/lib/snapshot/types.go b/lib/snapshot/types.go index 53db7d88..59848aa1 100644 --- a/lib/snapshot/types.go +++ b/lib/snapshot/types.go @@ -43,6 +43,15 @@ const ( SnapshotCompressionAlgorithmLz4 SnapshotCompressionAlgorithm = "lz4" ) +const ( + DefaultSnapshotCompressionZstdLevel = 1 + MinSnapshotCompressionZstdLevel = 1 + MaxSnapshotCompressionZstdLevel = 19 + DefaultSnapshotCompressionLz4Level = 0 + MinSnapshotCompressionLz4Level = 0 + MaxSnapshotCompressionLz4Level = 9 +) + const ( SnapshotCompressionStateNone = "none" SnapshotCompressionStateCompressing = "compressing" diff --git a/openapi.yaml b/openapi.yaml index 1e78106a..192753b9 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -456,11 +456,13 @@ components: algorithm: type: string enum: [zstd, lz4] - description: Compression algorithm (defaults to zstd when enabled) + description: Compression algorithm (defaults to zstd when enabled). Ignored when enabled is false. example: zstd level: type: integer - description: Compression level for zstd only + minimum: 0 + maximum: 19 + description: Compression level. Allowed ranges are zstd=1-19 and lz4=0-9. When omitted, zstd defaults to 1 and lz4 defaults to 0. Ignored when enabled is false. example: 1 SnapshotPolicy: From e55214ba7d97ed752a30f8f813488c88c741048a Mon Sep 17 00:00:00 2001 From: Steven Miller Date: Fri, 20 Mar 2026 10:56:58 -0400 Subject: [PATCH 15/31] Fix snapshot compression restore races --- lib/instances/snapshot.go | 7 +++ lib/instances/snapshot_compression.go | 46 ++++++--------- lib/instances/snapshot_test.go | 82 +++++++++++++++++++++++++++ 3 files changed, 106 insertions(+), 29 deletions(-) diff --git a/lib/instances/snapshot.go b/lib/instances/snapshot.go index efbf3227..3a260b7e 100644 --- a/lib/instances/snapshot.go +++ b/lib/instances/snapshot.go @@ -249,6 +249,13 @@ func (m *manager) restoreSnapshot(ctx context.Context, id string, snapshotID str if target != nil { m.recordSnapshotCompressionPreemption(ctx, snapshotCompressionPreemptionRestoreSnapshot, *target) } + target, err = m.cancelAndWaitCompressionJob(ctx, m.snapshotJobKeyForInstance(id)) + if err != nil { + return nil, fmt.Errorf("wait for source instance compression to stop: %w", err) + } + if target != nil { + m.recordSnapshotCompressionPreemption(ctx, snapshotCompressionPreemptionRestoreSnapshot, *target) + } if err := m.replaceInstanceWithSnapshotPayload(snapshotID, id); err != nil { return nil, err diff --git a/lib/instances/snapshot_compression.go b/lib/instances/snapshot_compression.go index 6143ebd2..808dc71c 100644 --- a/lib/instances/snapshot_compression.go +++ b/lib/instances/snapshot_compression.go @@ -104,43 +104,32 @@ func normalizeCompressionConfig(cfg *snapshotstore.SnapshotCompressionConfig) (s } func (m *manager) resolveSnapshotCompressionPolicy(stored *StoredMetadata, override *snapshotstore.SnapshotCompressionConfig) (snapshotstore.SnapshotCompressionConfig, error) { - if override != nil { - return normalizeCompressionConfig(override) - } - if stored != nil && stored.SnapshotPolicy != nil && stored.SnapshotPolicy.Compression != nil { - return normalizeCompressionConfig(stored.SnapshotPolicy.Compression) + cfg, err := m.resolveConfiguredCompressionPolicy(stored, override) + if err != nil { + return snapshotstore.SnapshotCompressionConfig{}, err } - if m.snapshotDefaults.Compression != nil { - return normalizeCompressionConfig(m.snapshotDefaults.Compression) + if cfg != nil { + return *cfg, nil } return snapshotstore.SnapshotCompressionConfig{Enabled: false}, nil } func (m *manager) resolveStandbyCompressionPolicy(stored *StoredMetadata, override *snapshotstore.SnapshotCompressionConfig) (*snapshotstore.SnapshotCompressionConfig, error) { - if override != nil { - cfg, err := normalizeCompressionConfig(override) - if err != nil { - return nil, err - } - if !cfg.Enabled { - return nil, nil - } - return &cfg, nil - } + return m.resolveConfiguredCompressionPolicy(stored, override) +} - if stored != nil && stored.SnapshotPolicy != nil && stored.SnapshotPolicy.Compression != nil { - cfg, err := normalizeCompressionConfig(stored.SnapshotPolicy.Compression) - if err != nil { - return nil, err - } - if !cfg.Enabled { - return nil, nil - } - return &cfg, nil +func (m *manager) resolveConfiguredCompressionPolicy(stored *StoredMetadata, override *snapshotstore.SnapshotCompressionConfig) (*snapshotstore.SnapshotCompressionConfig, error) { + candidates := []*snapshotstore.SnapshotCompressionConfig{override} + if stored != nil && stored.SnapshotPolicy != nil { + candidates = append(candidates, stored.SnapshotPolicy.Compression) } + candidates = append(candidates, m.snapshotDefaults.Compression) - if m.snapshotDefaults.Compression != nil { - cfg, err := normalizeCompressionConfig(m.snapshotDefaults.Compression) + for _, candidate := range candidates { + if candidate == nil { + continue + } + cfg, err := normalizeCompressionConfig(candidate) if err != nil { return nil, err } @@ -149,7 +138,6 @@ func (m *manager) resolveStandbyCompressionPolicy(stored *StoredMetadata, overri } return &cfg, nil } - return nil, nil } diff --git a/lib/instances/snapshot_test.go b/lib/instances/snapshot_test.go index 0d154cc0..ee75c19b 100644 --- a/lib/instances/snapshot_test.go +++ b/lib/instances/snapshot_test.go @@ -4,6 +4,7 @@ import ( "context" "os" "path/filepath" + "sync/atomic" "testing" "time" @@ -81,6 +82,87 @@ func TestStandbySnapshotRejectsTargetHypervisorOverride(t *testing.T) { assert.ErrorIs(t, err, ErrInvalidRequest) } +func TestRestoreSnapshotCancelsSourceInstanceCompressionJob(t *testing.T) { + t.Parallel() + + mgr, _ := setupTestManager(t) + ctx := context.Background() + + hvType := mgr.defaultHypervisor + sourceID := "snapshot-restore-src" + snapshotID := "snapshot-restore-race" + + createStandbySnapshotSourceFixture(t, mgr, sourceID, "snapshot-restore-src", hvType) + + snapshotGuestDir := mgr.paths.SnapshotGuestDir(snapshotID) + require.NoError(t, os.MkdirAll(mgr.paths.SnapshotDir(snapshotID), 0755)) + require.NoError(t, mgr.copySnapshotPayload(sourceID, snapshotGuestDir)) + + sourceMeta, err := mgr.loadMetadata(sourceID) + require.NoError(t, err) + require.NoError(t, mgr.saveSnapshotRecord(&snapshotRecord{ + Snapshot: Snapshot{ + Id: snapshotID, + Name: "restore-race", + Kind: SnapshotKindStandby, + SourceInstanceID: sourceID, + SourceName: sourceMeta.Name, + SourceHypervisor: hvType, + CreatedAt: time.Now(), + SizeBytes: 1, + }, + StoredMetadata: sourceMeta.StoredMetadata, + })) + + var instanceCanceled atomic.Bool + var snapshotCanceled atomic.Bool + instanceDone := make(chan struct{}) + snapshotDone := make(chan struct{}) + + mgr.compressionMu.Lock() + mgr.compressionJobs[mgr.snapshotJobKeyForInstance(sourceID)] = &compressionJob{ + cancel: func() { + instanceCanceled.Store(true) + select { + case <-instanceDone: + default: + close(instanceDone) + } + }, + done: instanceDone, + target: compressionTarget{ + Key: mgr.snapshotJobKeyForInstance(sourceID), + OwnerID: sourceID, + SnapshotDir: mgr.paths.InstanceSnapshotLatest(sourceID), + }, + } + mgr.compressionJobs[mgr.snapshotJobKeyForSnapshot(snapshotID)] = &compressionJob{ + cancel: func() { + snapshotCanceled.Store(true) + select { + case <-snapshotDone: + default: + close(snapshotDone) + } + }, + done: snapshotDone, + target: compressionTarget{ + Key: mgr.snapshotJobKeyForSnapshot(snapshotID), + SnapshotID: snapshotID, + SnapshotDir: snapshotGuestDir, + }, + } + mgr.compressionMu.Unlock() + + restored, err := mgr.RestoreSnapshot(ctx, sourceID, snapshotID, RestoreSnapshotRequest{ + TargetState: StateStandby, + }) + require.NoError(t, err) + require.Equal(t, StateStandby, restored.State) + assert.True(t, snapshotCanceled.Load(), "snapshot compression job should be canceled before restore") + assert.True(t, instanceCanceled.Load(), "instance compression job should be canceled before restore") +} + func createStoppedSnapshotSourceFixture(t *testing.T, mgr *manager, id, name string, hvType hypervisor.Type) { t.Helper() require.NoError(t, mgr.ensureDirectories(id)) From c78830a1f04a6ada7874e711140a07d58612603a Mon Sep 17 00:00:00 2001 From: Steven Miller Date: Fri, 20 Mar 2026 11:03:33 -0400 Subject: [PATCH 16/31] Fix optional standby body handling --- cmd/api/api/instances.go | 15 ++++++--------- cmd/api/api/snapshots.go | 2 +- lib/oapi/oapi.go | 5 +++-- 3 files changed, 10 insertions(+), 12 deletions(-) diff --git a/cmd/api/api/instances.go b/cmd/api/api/instances.go index 3595b760..27a387b8 100644 --- a/cmd/api/api/instances.go +++ b/cmd/api/api/instances.go @@ -919,7 +919,7 @@ func instanceToOAPI(inst instances.Instance) oapi.Instance { oapiInst.Tags = toOAPITags(inst.Tags) } if inst.SnapshotPolicy != nil { - oapiPolicy, _ := toOAPISnapshotPolicy(*inst.SnapshotPolicy) + oapiPolicy := toOAPISnapshotPolicy(*inst.SnapshotPolicy) oapiInst.SnapshotPolicy = &oapiPolicy } @@ -983,7 +983,7 @@ func toInstanceSnapshotPolicy(policy oapi.SnapshotPolicy) (*instances.SnapshotPo return out, nil } -func toOAPISnapshotCompressionConfig(cfg snapshot.SnapshotCompressionConfig) (oapi.SnapshotCompressionConfig, error) { +func toOAPISnapshotCompressionConfig(cfg snapshot.SnapshotCompressionConfig) oapi.SnapshotCompressionConfig { out := oapi.SnapshotCompressionConfig{ Enabled: cfg.Enabled, } @@ -995,17 +995,14 @@ func toOAPISnapshotCompressionConfig(cfg snapshot.SnapshotCompressionConfig) (oa level := *cfg.Level out.Level = &level } - return out, nil + return out } -func toOAPISnapshotPolicy(policy instances.SnapshotPolicy) (oapi.SnapshotPolicy, error) { +func toOAPISnapshotPolicy(policy instances.SnapshotPolicy) oapi.SnapshotPolicy { out := oapi.SnapshotPolicy{} if policy.Compression != nil { - compression, err := toOAPISnapshotCompressionConfig(*policy.Compression) - if err != nil { - return oapi.SnapshotPolicy{}, err - } + compression := toOAPISnapshotCompressionConfig(*policy.Compression) out.Compression = &compression } - return out, nil + return out } diff --git a/cmd/api/api/snapshots.go b/cmd/api/api/snapshots.go index d963bc40..da5cd9e0 100644 --- a/cmd/api/api/snapshots.go +++ b/cmd/api/api/snapshots.go @@ -225,7 +225,7 @@ func snapshotToOAPI(snapshot instances.Snapshot) oapi.Snapshot { out.CompressionError = lo.ToPtr(snapshot.CompressionError) } if snapshot.Compression != nil { - compression, _ := toOAPISnapshotCompressionConfig(*snapshot.Compression) + compression := toOAPISnapshotCompressionConfig(*snapshot.Compression) out.Compression = &compression } if snapshot.CompressedSizeBytes != nil { diff --git a/lib/oapi/oapi.go b/lib/oapi/oapi.go index e6bdc580..30cd6f15 100644 --- a/lib/oapi/oapi.go +++ b/lib/oapi/oapi.go @@ -13024,11 +13024,12 @@ func (sh *strictHandler) StandbyInstance(w http.ResponseWriter, r *http.Request, request.Id = id var body StandbyInstanceJSONRequestBody - if err := json.NewDecoder(r.Body).Decode(&body); err != nil { + decoded, err := decodeOptionalJSONBody(r, &body) + if err != nil { sh.options.RequestErrorHandlerFunc(w, r, fmt.Errorf("can't decode JSON body: %w", err)) return } - request.Body = &body + request.Body = decoded handler := func(ctx context.Context, w http.ResponseWriter, r *http.Request, request interface{}) (interface{}, error) { return sh.ssi.StandbyInstance(ctx, request.(StandbyInstanceRequestObject)) From d192ba1cf0319f81d85bbc4efb611849b740afa2 Mon Sep 17 00:00:00 2001 From: Steven Miller Date: Fri, 20 Mar 2026 11:18:46 -0400 Subject: [PATCH 17/31] Fix standby snapshot compression races --- lib/instances/metrics.go | 1 + lib/instances/snapshot.go | 8 ++++- lib/instances/snapshot_compression_test.go | 30 ++++++++++++++++ lib/instances/snapshot_test.go | 41 ++++++++++++++++++++++ 4 files changed, 79 insertions(+), 1 deletion(-) diff --git a/lib/instances/metrics.go b/lib/instances/metrics.go index 8666a9c1..13e10f0d 100644 --- a/lib/instances/metrics.go +++ b/lib/instances/metrics.go @@ -40,6 +40,7 @@ const ( snapshotCompressionPreemptionRestoreInstance snapshotCompressionPreemptionOperation = "restore_instance" snapshotCompressionPreemptionRestoreSnapshot snapshotCompressionPreemptionOperation = "restore_snapshot" snapshotCompressionPreemptionForkSnapshot snapshotCompressionPreemptionOperation = "fork_snapshot" + snapshotCompressionPreemptionCreateSnapshot snapshotCompressionPreemptionOperation = "create_snapshot" ) // Metrics holds the metrics instruments for instance operations. diff --git a/lib/instances/snapshot.go b/lib/instances/snapshot.go index 3a260b7e..3106e5e3 100644 --- a/lib/instances/snapshot.go +++ b/lib/instances/snapshot.go @@ -96,7 +96,13 @@ func (m *manager) createSnapshot(ctx context.Context, id string, req CreateSnaps } restoreSource = true case StateStandby: - // already ready to copy + target, err := m.cancelAndWaitCompressionJob(ctx, m.snapshotJobKeyForInstance(id)) + if err != nil { + return nil, fmt.Errorf("wait for source instance compression to stop: %w", err) + } + if target != nil { + m.recordSnapshotCompressionPreemption(ctx, snapshotCompressionPreemptionCreateSnapshot, *target) + } default: return nil, fmt.Errorf("%w: standby snapshot requires source in %s or %s, got %s", ErrInvalidState, StateRunning, StateStandby, inst.State) } diff --git a/lib/instances/snapshot_compression_test.go b/lib/instances/snapshot_compression_test.go index 6a293f0b..5b61aef2 100644 --- a/lib/instances/snapshot_compression_test.go +++ b/lib/instances/snapshot_compression_test.go @@ -122,6 +122,36 @@ func TestResolveStandbyCompressionPolicyIsOptInOnly(t *testing.T) { assert.Nil(t, cfg) } +func TestResolveCompressionPolicyExplicitDisableOverridesDefaults(t *testing.T) { + t.Parallel() + + m := &manager{ + snapshotDefaults: SnapshotPolicy{ + Compression: &snapshotstore.SnapshotCompressionConfig{ + Enabled: true, + Algorithm: snapshotstore.SnapshotCompressionAlgorithmZstd, + Level: intPtr(3), + }, + }, + } + + stored := &StoredMetadata{ + SnapshotPolicy: &SnapshotPolicy{ + Compression: &snapshotstore.SnapshotCompressionConfig{ + Enabled: false, + }, + }, + } + + cfg, err := m.resolveSnapshotCompressionPolicy(stored, nil) + require.NoError(t, err) + assert.False(t, cfg.Enabled) + + standbyCfg, err := m.resolveStandbyCompressionPolicy(stored, nil) + require.NoError(t, err) + assert.Nil(t, standbyCfg) +} + func TestValidateCreateRequestSnapshotPolicy(t *testing.T) { t.Parallel() diff --git a/lib/instances/snapshot_test.go b/lib/instances/snapshot_test.go index ee75c19b..c23f38a0 100644 --- a/lib/instances/snapshot_test.go +++ b/lib/instances/snapshot_test.go @@ -163,6 +163,47 @@ func TestRestoreSnapshotCancelsSourceInstanceCompressionJob(t *testing.T) { assert.True(t, instanceCanceled.Load(), "instance compression job should be canceled before restore") } +func TestCreateStandbySnapshotCancelsSourceInstanceCompressionJob(t *testing.T) { + t.Parallel() + + mgr, _ := setupTestManager(t) + ctx := context.Background() + + hvType := mgr.defaultHypervisor + sourceID := "snapshot-create-src" + createStandbySnapshotSourceFixture(t, mgr, sourceID, "snapshot-create-src", hvType) + + var instanceCanceled atomic.Bool + instanceDone := make(chan struct{}) + + mgr.compressionMu.Lock() + mgr.compressionJobs[mgr.snapshotJobKeyForInstance(sourceID)] = &compressionJob{ + cancel: func() { + instanceCanceled.Store(true) + select { + case <-instanceDone: + default: + close(instanceDone) + } + }, + done: instanceDone, + target: compressionTarget{ + Key: mgr.snapshotJobKeyForInstance(sourceID), + OwnerID: sourceID, + SnapshotDir: mgr.paths.InstanceSnapshotLatest(sourceID), + }, + } + mgr.compressionMu.Unlock() + + snap, err := mgr.CreateSnapshot(ctx, sourceID, CreateSnapshotRequest{ + Kind: SnapshotKindStandby, + Name: "standby-copy", + }) + require.NoError(t, err) + require.Equal(t, SnapshotKindStandby, snap.Kind) + assert.True(t, instanceCanceled.Load(), "instance compression job should be canceled before copying standby snapshot payload") +} + func createStoppedSnapshotSourceFixture(t *testing.T, mgr *manager, id, name string, hvType hypervisor.Type) { t.Helper() require.NoError(t, mgr.ensureDirectories(id)) From 19f4aec011bb43b7375796dad62a97d969d7f56f Mon Sep 17 00:00:00 2001 From: Steven Miller Date: Fri, 20 Mar 2026 11:28:30 -0400 Subject: [PATCH 18/31] Normalize standby snapshot compression copies --- lib/instances/snapshot.go | 3 ++ lib/instances/snapshot_compression.go | 30 ++++++++++++-------- lib/instances/snapshot_compression_test.go | 23 +++++++++++++++ lib/instances/snapshot_test.go | 33 ++++++++++++++++++++++ 4 files changed, 78 insertions(+), 11 deletions(-) diff --git a/lib/instances/snapshot.go b/lib/instances/snapshot.go index 3106e5e3..cd82b37a 100644 --- a/lib/instances/snapshot.go +++ b/lib/instances/snapshot.go @@ -103,6 +103,9 @@ func (m *manager) createSnapshot(ctx context.Context, id string, req CreateSnaps if target != nil { m.recordSnapshotCompressionPreemption(ctx, snapshotCompressionPreemptionCreateSnapshot, *target) } + if err := m.ensureSnapshotMemoryReady(ctx, m.paths.InstanceSnapshotLatest(id), "", stored.HypervisorType); err != nil { + return nil, fmt.Errorf("prepare source snapshot memory for copy: %w", err) + } default: return nil, fmt.Errorf("%w: standby snapshot requires source in %s or %s, got %s", ErrInvalidState, StateRunning, StateStandby, inst.State) } diff --git a/lib/instances/snapshot_compression.go b/lib/instances/snapshot_compression.go index 808dc71c..e4c3e44b 100644 --- a/lib/instances/snapshot_compression.go +++ b/lib/instances/snapshot_compression.go @@ -186,8 +186,14 @@ func (m *manager) startCompressionJob(ctx context.Context, target compressionTar log := logger.FromContext(ctx) rawPath, ok := findRawSnapshotMemoryFile(target.SnapshotDir) if !ok { - if _, _, found := findCompressedSnapshotMemoryFile(target.SnapshotDir); found && target.SnapshotID != "" { - _ = m.updateSnapshotCompressionMetadata(target.SnapshotID, snapshotstore.SnapshotCompressionStateCompressed, "", &target.Policy, nil, nil) + if compressedPath, algorithm, found := findCompressedSnapshotMemoryFile(target.SnapshotDir); found && target.SnapshotID != "" { + cfg := compressionMetadataForExistingArtifact(target.Policy, algorithm) + var compressedSize *int64 + if st, statErr := os.Stat(compressedPath); statErr == nil { + size := st.Size() + compressedSize = &size + } + _ = m.updateSnapshotCompressionMetadata(target.SnapshotID, snapshotstore.SnapshotCompressionStateCompressed, "", &cfg, compressedSize, nil) } return } @@ -216,15 +222,6 @@ func (m *manager) startCompressionJob(ctx context.Context, target compressionTar }() } -func (m *manager) cancelCompressionJob(key string) { - m.compressionMu.Lock() - job := m.compressionJobs[key] - m.compressionMu.Unlock() - if job != nil { - job.cancel() - } -} - func (m *manager) waitCompressionJobContext(ctx context.Context, key string) error { m.compressionMu.Lock() job := m.compressionJobs[key] @@ -519,6 +516,17 @@ func compressedPathFor(rawPath string, algorithm snapshotstore.SnapshotCompressi } } +func compressionMetadataForExistingArtifact(policy snapshotstore.SnapshotCompressionConfig, algorithm snapshotstore.SnapshotCompressionAlgorithm) snapshotstore.SnapshotCompressionConfig { + cfg := snapshotstore.SnapshotCompressionConfig{ + Enabled: true, + Algorithm: algorithm, + } + if policy.Algorithm == algorithm { + cfg.Level = cloneCompressionConfig(&policy).Level + } + return cfg +} + func removeCompressedSnapshotArtifacts(rawPath string) { for _, path := range []string{ rawPath + ".zst", diff --git a/lib/instances/snapshot_compression_test.go b/lib/instances/snapshot_compression_test.go index 5b61aef2..de9d655f 100644 --- a/lib/instances/snapshot_compression_test.go +++ b/lib/instances/snapshot_compression_test.go @@ -152,6 +152,29 @@ func TestResolveCompressionPolicyExplicitDisableOverridesDefaults(t *testing.T) assert.Nil(t, standbyCfg) } +func TestCompressionMetadataForExistingArtifactUsesActualAlgorithm(t *testing.T) { + t.Parallel() + + cfg := compressionMetadataForExistingArtifact(snapshotstore.SnapshotCompressionConfig{ + Enabled: true, + Algorithm: snapshotstore.SnapshotCompressionAlgorithmZstd, + Level: intPtr(7), + }, snapshotstore.SnapshotCompressionAlgorithmLz4) + assert.True(t, cfg.Enabled) + assert.Equal(t, snapshotstore.SnapshotCompressionAlgorithmLz4, cfg.Algorithm) + assert.Nil(t, cfg.Level) + + cfg = compressionMetadataForExistingArtifact(snapshotstore.SnapshotCompressionConfig{ + Enabled: true, + Algorithm: snapshotstore.SnapshotCompressionAlgorithmZstd, + Level: intPtr(7), + }, snapshotstore.SnapshotCompressionAlgorithmZstd) + assert.True(t, cfg.Enabled) + assert.Equal(t, snapshotstore.SnapshotCompressionAlgorithmZstd, cfg.Algorithm) + require.NotNil(t, cfg.Level) + assert.Equal(t, 7, *cfg.Level) +} + func TestValidateCreateRequestSnapshotPolicy(t *testing.T) { t.Parallel() diff --git a/lib/instances/snapshot_test.go b/lib/instances/snapshot_test.go index c23f38a0..dd028f72 100644 --- a/lib/instances/snapshot_test.go +++ b/lib/instances/snapshot_test.go @@ -9,6 +9,7 @@ import ( "time" "github.com/kernel/hypeman/lib/hypervisor" + snapshotstore "github.com/kernel/hypeman/lib/snapshot" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -204,6 +205,38 @@ func TestCreateStandbySnapshotCancelsSourceInstanceCompressionJob(t *testing.T) assert.True(t, instanceCanceled.Load(), "instance compression job should be canceled before copying standby snapshot payload") } +func TestCreateStandbySnapshotFromCompressedSourceCopiesRawMemory(t *testing.T) { + t.Parallel() + + mgr, _ := setupTestManager(t) + ctx := context.Background() + + hvType := mgr.defaultHypervisor + sourceID := "snapshot-create-compressed-src" + createStandbySnapshotSourceFixture(t, mgr, sourceID, "snapshot-create-compressed-src", hvType) + + rawPath := filepath.Join(mgr.paths.InstanceSnapshotLatest(sourceID), "memory-ranges") + require.NoError(t, os.WriteFile(rawPath, []byte("some guest memory"), 0644)) + _, _, err := compressSnapshotMemoryFile(ctx, rawPath, snapshotstore.SnapshotCompressionConfig{ + Enabled: true, + Algorithm: snapshotstore.SnapshotCompressionAlgorithmZstd, + Level: intPtr(1), + }) + require.NoError(t, err) + + snap, err := mgr.CreateSnapshot(ctx, sourceID, CreateSnapshotRequest{ + Kind: SnapshotKindStandby, + Name: "standby-from-compressed", + }) + require.NoError(t, err) + + snapshotDir := mgr.paths.SnapshotGuestDir(snap.Id) + _, ok := findRawSnapshotMemoryFile(snapshotDir) + assert.True(t, ok, "snapshot copy should contain raw memory after preparing a compressed standby source") + _, _, ok = findCompressedSnapshotMemoryFile(snapshotDir) + assert.False(t, ok, "snapshot copy should not inherit compressed memory artifacts from the source standby instance") +} + func createStoppedSnapshotSourceFixture(t *testing.T, mgr *manager, id, name string, hvType hypervisor.Type) { t.Helper() require.NoError(t, mgr.ensureDirectories(id)) From 2eef65dd2b0ce8debb400f2474d3f070615afc7b Mon Sep 17 00:00:00 2001 From: Steven Miller Date: Fri, 20 Mar 2026 11:33:57 -0400 Subject: [PATCH 19/31] Handle optional standby bodies outside generated code --- cmd/api/api/optional_standby_body.go | 36 +++++++++++ cmd/api/api/optional_standby_body_test.go | 79 +++++++++++++++++++++++ cmd/api/main.go | 2 +- lib/oapi/oapi.go | 5 +- lib/oapi/optional_json.go | 23 ------- lib/oapi/optional_json_test.go | 52 --------------- 6 files changed, 118 insertions(+), 79 deletions(-) create mode 100644 cmd/api/api/optional_standby_body.go create mode 100644 cmd/api/api/optional_standby_body_test.go delete mode 100644 lib/oapi/optional_json.go delete mode 100644 lib/oapi/optional_json_test.go diff --git a/cmd/api/api/optional_standby_body.go b/cmd/api/api/optional_standby_body.go new file mode 100644 index 00000000..bc2dd83e --- /dev/null +++ b/cmd/api/api/optional_standby_body.go @@ -0,0 +1,36 @@ +package api + +import ( + "io" + "net/http" + "regexp" + "strings" +) + +var standbyRoutePattern = regexp.MustCompile(`^/instances/[^/]+/standby$`) + +// NormalizeOptionalStandbyBody rewrites empty standby POST bodies to "{}" +// so the generated strict handler can decode them without special casing. +func NormalizeOptionalStandbyBody(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method == http.MethodPost && standbyRoutePattern.MatchString(r.URL.Path) && requestBodyIsEmpty(r) { + r.Body = io.NopCloser(strings.NewReader(`{}`)) + r.ContentLength = 2 + if r.Header.Get("Content-Type") == "" { + r.Header.Set("Content-Type", "application/json") + } + } + + next.ServeHTTP(w, r) + }) +} + +func requestBodyIsEmpty(r *http.Request) bool { + if r == nil { + return true + } + if r.Body == nil || r.Body == http.NoBody { + return true + } + return r.ContentLength == 0 && len(r.TransferEncoding) == 0 +} diff --git a/cmd/api/api/optional_standby_body_test.go b/cmd/api/api/optional_standby_body_test.go new file mode 100644 index 00000000..e124a77d --- /dev/null +++ b/cmd/api/api/optional_standby_body_test.go @@ -0,0 +1,79 @@ +package api + +import ( + "bytes" + "io" + "net/http" + "net/http/httptest" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestNormalizeOptionalStandbyBody(t *testing.T) { + t.Parallel() + + t.Run("empty standby body becomes empty JSON object", func(t *testing.T) { + t.Parallel() + + var gotBody []byte + var gotContentType string + next := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var err error + gotBody, err = io.ReadAll(r.Body) + require.NoError(t, err) + gotContentType = r.Header.Get("Content-Type") + w.WriteHeader(http.StatusNoContent) + }) + + req := httptest.NewRequest(http.MethodPost, "/instances/test/standby", nil) + rec := httptest.NewRecorder() + + NormalizeOptionalStandbyBody(next).ServeHTTP(rec, req) + + assert.Equal(t, http.StatusNoContent, rec.Code) + assert.Equal(t, []byte(`{}`), gotBody) + assert.Equal(t, "application/json", gotContentType) + }) + + t.Run("existing standby body is preserved", func(t *testing.T) { + t.Parallel() + + var gotBody []byte + next := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var err error + gotBody, err = io.ReadAll(r.Body) + require.NoError(t, err) + w.WriteHeader(http.StatusNoContent) + }) + + req := httptest.NewRequest(http.MethodPost, "/instances/test/standby", bytes.NewBufferString(`{"compression":{"enabled":true}}`)) + rec := httptest.NewRecorder() + + NormalizeOptionalStandbyBody(next).ServeHTTP(rec, req) + + assert.Equal(t, http.StatusNoContent, rec.Code) + assert.Equal(t, []byte(`{"compression":{"enabled":true}}`), gotBody) + }) + + t.Run("non-standby route is untouched", func(t *testing.T) { + t.Parallel() + + var gotBody []byte + next := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var err error + gotBody, err = io.ReadAll(r.Body) + require.NoError(t, err) + w.WriteHeader(http.StatusNoContent) + }) + + req := httptest.NewRequest(http.MethodPost, "/instances/test/start", nil) + rec := httptest.NewRecorder() + + NormalizeOptionalStandbyBody(next).ServeHTTP(rec, req) + + assert.Equal(t, http.StatusNoContent, rec.Code) + assert.Empty(t, gotBody) + }) +} diff --git a/cmd/api/main.go b/cmd/api/main.go index 63fdc284..11e6ea30 100644 --- a/cmd/api/main.go +++ b/cmd/api/main.go @@ -382,7 +382,7 @@ func run() error { // Mount API routes (authentication now handled by validation middleware) oapi.HandlerWithOptions(strictHandler, oapi.ChiServerOptions{ BaseRouter: r, - Middlewares: []oapi.MiddlewareFunc{}, + Middlewares: []oapi.MiddlewareFunc{api.NormalizeOptionalStandbyBody}, }) }) diff --git a/lib/oapi/oapi.go b/lib/oapi/oapi.go index 30cd6f15..e6bdc580 100644 --- a/lib/oapi/oapi.go +++ b/lib/oapi/oapi.go @@ -13024,12 +13024,11 @@ func (sh *strictHandler) StandbyInstance(w http.ResponseWriter, r *http.Request, request.Id = id var body StandbyInstanceJSONRequestBody - decoded, err := decodeOptionalJSONBody(r, &body) - if err != nil { + if err := json.NewDecoder(r.Body).Decode(&body); err != nil { sh.options.RequestErrorHandlerFunc(w, r, fmt.Errorf("can't decode JSON body: %w", err)) return } - request.Body = decoded + request.Body = &body handler := func(ctx context.Context, w http.ResponseWriter, r *http.Request, request interface{}) (interface{}, error) { return sh.ssi.StandbyInstance(ctx, request.(StandbyInstanceRequestObject)) diff --git a/lib/oapi/optional_json.go b/lib/oapi/optional_json.go deleted file mode 100644 index 389bc607..00000000 --- a/lib/oapi/optional_json.go +++ /dev/null @@ -1,23 +0,0 @@ -package oapi - -import ( - "encoding/json" - "errors" - "io" - "net/http" -) - -func decodeOptionalJSONBody[T any](r *http.Request, body *T) (*T, error) { - if r == nil || r.Body == nil || r.Body == http.NoBody { - return nil, nil - } - - if err := json.NewDecoder(r.Body).Decode(body); err != nil { - if errors.Is(err, io.EOF) { - return nil, nil - } - return nil, err - } - - return body, nil -} diff --git a/lib/oapi/optional_json_test.go b/lib/oapi/optional_json_test.go deleted file mode 100644 index ead11d02..00000000 --- a/lib/oapi/optional_json_test.go +++ /dev/null @@ -1,52 +0,0 @@ -package oapi - -import ( - "bytes" - "io" - "net/http/httptest" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestDecodeOptionalJSONBody(t *testing.T) { - t.Parallel() - - t.Run("no body returns nil", func(t *testing.T) { - t.Parallel() - - req := httptest.NewRequest("POST", "/instances/test/standby", nil) - var body StandbyInstanceJSONRequestBody - - decoded, err := decodeOptionalJSONBody(req, &body) - require.NoError(t, err) - assert.Nil(t, decoded) - }) - - t.Run("empty reader returns nil", func(t *testing.T) { - t.Parallel() - - req := httptest.NewRequest("POST", "/instances/test/standby", io.NopCloser(bytes.NewBuffer(nil))) - var body StandbyInstanceJSONRequestBody - - decoded, err := decodeOptionalJSONBody(req, &body) - require.NoError(t, err) - assert.Nil(t, decoded) - }) - - t.Run("json object decodes", func(t *testing.T) { - t.Parallel() - - req := httptest.NewRequest("POST", "/instances/test/standby", bytes.NewBufferString(`{"compression":{"enabled":true,"algorithm":"lz4"}}`)) - var body StandbyInstanceJSONRequestBody - - decoded, err := decodeOptionalJSONBody(req, &body) - require.NoError(t, err) - require.NotNil(t, decoded) - require.NotNil(t, decoded.Compression) - assert.True(t, decoded.Compression.Enabled) - require.NotNil(t, decoded.Compression.Algorithm) - assert.Equal(t, SnapshotCompressionConfigAlgorithm("lz4"), *decoded.Compression.Algorithm) - }) -} From 1c0eea1f26b70b8e47af7576e2f18d7f1d09ef99 Mon Sep 17 00:00:00 2001 From: Steven Miller Date: Fri, 20 Mar 2026 12:40:06 -0400 Subject: [PATCH 20/31] Fix snapshot compression cleanup races --- lib/forkvm/copy.go | 2 +- lib/forkvm/copy_test.go | 14 +++ lib/instances/delete.go | 8 ++ lib/instances/metrics.go | 2 + lib/instances/snapshot.go | 8 +- lib/instances/snapshot_compression_test.go | 112 +++++++++++++++++++++ 6 files changed, 144 insertions(+), 2 deletions(-) diff --git a/lib/forkvm/copy.go b/lib/forkvm/copy.go index 931648c3..6dc6eecc 100644 --- a/lib/forkvm/copy.go +++ b/lib/forkvm/copy.go @@ -42,7 +42,7 @@ func CopyGuestDirectory(srcDir, dstDir string) error { if d.IsDir() && shouldSkipDirectory(relPath) { return filepath.SkipDir } - if shouldSkipRegularFile(relPath) { + if !d.IsDir() && shouldSkipRegularFile(relPath) { return nil } diff --git a/lib/forkvm/copy_test.go b/lib/forkvm/copy_test.go index 67c1bbe9..c71f6c4e 100644 --- a/lib/forkvm/copy_test.go +++ b/lib/forkvm/copy_test.go @@ -43,3 +43,17 @@ func TestCopyGuestDirectory(t *testing.T) { require.NoError(t, err) assert.Equal(t, "metadata.json", linkTarget) } + +func TestCopyGuestDirectory_DoesNotSkipTmpSuffixedDirectories(t *testing.T) { + src := filepath.Join(t.TempDir(), "src") + dst := filepath.Join(t.TempDir(), "dst") + + tmpDir := filepath.Join(src, "snapshots", "snapshot-latest", "memory-ranges.lz4.tmp") + require.NoError(t, os.MkdirAll(tmpDir, 0755)) + require.NoError(t, os.WriteFile(filepath.Join(tmpDir, "nested.txt"), []byte("nested"), 0644)) + + require.NoError(t, CopyGuestDirectory(src, dst)) + + assert.DirExists(t, filepath.Join(dst, "snapshots", "snapshot-latest", "memory-ranges.lz4.tmp")) + assert.FileExists(t, filepath.Join(dst, "snapshots", "snapshot-latest", "memory-ranges.lz4.tmp", "nested.txt")) +} diff --git a/lib/instances/delete.go b/lib/instances/delete.go index 21fbfaab..974224e8 100644 --- a/lib/instances/delete.go +++ b/lib/instances/delete.go @@ -35,6 +35,14 @@ func (m *manager) deleteInstance( stored := &meta.StoredMetadata log.DebugContext(ctx, "loaded instance", "instance_id", id, "state", inst.State) + target, err := m.cancelAndWaitCompressionJob(ctx, m.snapshotJobKeyForInstance(id)) + if err != nil { + return fmt.Errorf("wait for instance compression to stop: %w", err) + } + if target != nil { + m.recordSnapshotCompressionPreemption(ctx, snapshotCompressionPreemptionDeleteInstance, *target) + } + // 2. Get network allocation BEFORE killing VMM (while we can still query it) var networkAlloc *network.Allocation if inst.NetworkEnabled { diff --git a/lib/instances/metrics.go b/lib/instances/metrics.go index 13e10f0d..f1bcb089 100644 --- a/lib/instances/metrics.go +++ b/lib/instances/metrics.go @@ -41,6 +41,8 @@ const ( snapshotCompressionPreemptionRestoreSnapshot snapshotCompressionPreemptionOperation = "restore_snapshot" snapshotCompressionPreemptionForkSnapshot snapshotCompressionPreemptionOperation = "fork_snapshot" snapshotCompressionPreemptionCreateSnapshot snapshotCompressionPreemptionOperation = "create_snapshot" + snapshotCompressionPreemptionDeleteInstance snapshotCompressionPreemptionOperation = "delete_instance" + snapshotCompressionPreemptionDeleteSnapshot snapshotCompressionPreemptionOperation = "delete_snapshot" ) // Metrics holds the metrics instruments for instance operations. diff --git a/lib/instances/snapshot.go b/lib/instances/snapshot.go index cd82b37a..d42ca6f2 100644 --- a/lib/instances/snapshot.go +++ b/lib/instances/snapshot.go @@ -213,7 +213,13 @@ func (m *manager) createSnapshot(ctx context.Context, id string, req CreateSnaps } func (m *manager) deleteSnapshot(ctx context.Context, snapshotID string) error { - _ = ctx + target, err := m.cancelAndWaitCompressionJob(ctx, m.snapshotJobKeyForSnapshot(snapshotID)) + if err != nil { + return fmt.Errorf("wait for snapshot compression to stop: %w", err) + } + if target != nil { + m.recordSnapshotCompressionPreemption(ctx, snapshotCompressionPreemptionDeleteSnapshot, *target) + } if err := m.snapshotStore().Delete(snapshotID); err != nil { if errors.Is(err, snapshotstore.ErrNotFound) { return ErrSnapshotNotFound diff --git a/lib/instances/snapshot_compression_test.go b/lib/instances/snapshot_compression_test.go index de9d655f..713c9371 100644 --- a/lib/instances/snapshot_compression_test.go +++ b/lib/instances/snapshot_compression_test.go @@ -1,9 +1,13 @@ package instances import ( + "context" "errors" + "os" "testing" + "time" + "github.com/kernel/hypeman/lib/hypervisor" snapshotstore "github.com/kernel/hypeman/lib/snapshot" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -206,3 +210,111 @@ func TestValidateCreateSnapshotRequestRejectsStoppedCompression(t *testing.T) { require.Error(t, err) assert.True(t, errors.Is(err, ErrInvalidRequest)) } + +func TestDeleteInstanceCancelsCompressionJob(t *testing.T) { + t.Parallel() + + mgr, _ := setupTestManager(t) + ctx := context.Background() + const instanceID = "delete-instance-compression" + + require.NoError(t, mgr.ensureDirectories(instanceID)) + now := time.Now() + require.NoError(t, mgr.saveMetadata(&metadata{StoredMetadata: StoredMetadata{ + Id: instanceID, + Name: instanceID, + DataDir: mgr.paths.InstanceDir(instanceID), + HypervisorType: hypervisor.TypeCloudHypervisor, + CreatedAt: now, + StoppedAt: &now, + }})) + + target := installCancelableCompressionJob(mgr, compressionTarget{ + Key: mgr.snapshotJobKeyForInstance(instanceID), + OwnerID: instanceID, + SnapshotDir: mgr.paths.InstanceSnapshotLatest(instanceID), + Source: snapshotCompressionSourceStandby, + Policy: snapshotstore.SnapshotCompressionConfig{ + Enabled: true, + Algorithm: snapshotstore.SnapshotCompressionAlgorithmZstd, + Level: intPtr(1), + }, + }) + + require.NoError(t, mgr.DeleteInstance(ctx, instanceID)) + assertCompressionJobCanceled(t, mgr, target) + _, err := os.Stat(mgr.paths.InstanceDir(instanceID)) + require.Error(t, err) + assert.True(t, os.IsNotExist(err)) +} + +func TestDeleteSnapshotCancelsCompressionJob(t *testing.T) { + t.Parallel() + + mgr, _ := setupTestManager(t) + ctx := context.Background() + const snapshotID = "delete-snapshot-compression" + + snapshotDir := mgr.paths.SnapshotGuestDir(snapshotID) + require.NoError(t, os.MkdirAll(snapshotDir, 0o755)) + require.NoError(t, mgr.saveSnapshotRecord(&snapshotRecord{ + Snapshot: Snapshot{ + Id: snapshotID, + Name: snapshotID, + Kind: SnapshotKindStandby, + CreatedAt: time.Now(), + }, + })) + + target := installCancelableCompressionJob(mgr, compressionTarget{ + Key: mgr.snapshotJobKeyForSnapshot(snapshotID), + SnapshotID: snapshotID, + SnapshotDir: snapshotDir, + Source: snapshotCompressionSourceSnapshot, + Policy: snapshotstore.SnapshotCompressionConfig{ + Enabled: true, + Algorithm: snapshotstore.SnapshotCompressionAlgorithmLz4, + Level: intPtr(0), + }, + }) + + require.NoError(t, mgr.DeleteSnapshot(ctx, snapshotID)) + assertCompressionJobCanceled(t, mgr, target) + _, err := os.Stat(mgr.paths.SnapshotDir(snapshotID)) + require.Error(t, err) + assert.True(t, os.IsNotExist(err)) +} + +func installCancelableCompressionJob(mgr *manager, target compressionTarget) *compressionTarget { + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan struct{}) + + mgr.compressionMu.Lock() + mgr.compressionJobs[target.Key] = &compressionJob{ + cancel: cancel, + done: done, + target: target, + } + mgr.compressionMu.Unlock() + + go func() { + <-ctx.Done() + mgr.compressionMu.Lock() + delete(mgr.compressionJobs, target.Key) + mgr.compressionMu.Unlock() + close(done) + }() + + return &target +} + +func assertCompressionJobCanceled(t *testing.T, mgr *manager, target *compressionTarget) { + t.Helper() + + require.Eventually(t, func() bool { + mgr.compressionMu.Lock() + defer mgr.compressionMu.Unlock() + _, ok := mgr.compressionJobs[target.Key] + return !ok + }, time.Second, 10*time.Millisecond) +} From da7f421f01eba9b9ca72ea308fdb336f796944d0 Mon Sep 17 00:00:00 2001 From: Steven Miller Date: Fri, 20 Mar 2026 12:52:25 -0400 Subject: [PATCH 21/31] Clarify snapshot compression metrics state --- lib/instances/snapshot_compression.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/instances/snapshot_compression.go b/lib/instances/snapshot_compression.go index e4c3e44b..c885d876 100644 --- a/lib/instances/snapshot_compression.go +++ b/lib/instances/snapshot_compression.go @@ -188,12 +188,12 @@ func (m *manager) startCompressionJob(ctx context.Context, target compressionTar if !ok { if compressedPath, algorithm, found := findCompressedSnapshotMemoryFile(target.SnapshotDir); found && target.SnapshotID != "" { cfg := compressionMetadataForExistingArtifact(target.Policy, algorithm) - var compressedSize *int64 + var compressedSizeBytes *int64 if st, statErr := os.Stat(compressedPath); statErr == nil { size := st.Size() - compressedSize = &size + compressedSizeBytes = &size } - _ = m.updateSnapshotCompressionMetadata(target.SnapshotID, snapshotstore.SnapshotCompressionStateCompressed, "", &cfg, compressedSize, nil) + _ = m.updateSnapshotCompressionMetadata(target.SnapshotID, snapshotstore.SnapshotCompressionStateCompressed, "", &cfg, compressedSizeBytes, nil) } return } From 759eb3c100a5d29785c235cda39b3f035386ab0d Mon Sep 17 00:00:00 2001 From: Steven Miller Date: Fri, 20 Mar 2026 12:59:56 -0400 Subject: [PATCH 22/31] Return bad request for invalid standby input --- cmd/api/api/instances.go | 5 +++ cmd/api/api/instances_test.go | 58 +++++++++++++++++++++++++++++++++++ 2 files changed, 63 insertions(+) diff --git a/cmd/api/api/instances.go b/cmd/api/api/instances.go index 27a387b8..4a07ae7e 100644 --- a/cmd/api/api/instances.go +++ b/cmd/api/api/instances.go @@ -464,6 +464,11 @@ func (s *ApiService) StandbyInstance(ctx context.Context, request oapi.StandbyIn result, err := s.InstanceManager.StandbyInstance(ctx, inst.Id, standbyReq) if err != nil { switch { + case errors.Is(err, instances.ErrInvalidRequest): + return oapi.StandbyInstance400JSONResponse{ + Code: "invalid_request", + Message: err.Error(), + }, nil case errors.Is(err, instances.ErrInvalidState): return oapi.StandbyInstance409JSONResponse{ Code: "invalid_state", diff --git a/cmd/api/api/instances_test.go b/cmd/api/api/instances_test.go index ea883a08..3053be92 100644 --- a/cmd/api/api/instances_test.go +++ b/cmd/api/api/instances_test.go @@ -204,6 +204,14 @@ type captureForkManager struct { err error } +type captureStandbyManager struct { + instances.Manager + lastID string + lastReq *instances.StandbyInstanceRequest + result *instances.Instance + err error +} + func (m *captureForkManager) ForkInstance(ctx context.Context, id string, req instances.ForkInstanceRequest) (*instances.Instance, error) { reqCopy := req m.lastID = id @@ -214,6 +222,16 @@ func (m *captureForkManager) ForkInstance(ctx context.Context, id string, req in return m.result, nil } +func (m *captureStandbyManager) StandbyInstance(ctx context.Context, id string, req instances.StandbyInstanceRequest) (*instances.Instance, error) { + reqCopy := req + m.lastID = id + m.lastReq = &reqCopy + if m.err != nil { + return nil, m.err + } + return m.result, nil +} + func (m *captureCreateManager) CreateInstance(ctx context.Context, req instances.CreateInstanceRequest) (*instances.Instance, error) { reqCopy := req m.lastReq = &reqCopy @@ -536,6 +554,46 @@ func TestForkInstance_InvalidRequest(t *testing.T) { assert.Equal(t, "invalid_request", badReq.Code) } +func TestStandbyInstance_InvalidRequest(t *testing.T) { + t.Parallel() + svc := newTestService(t) + + source := instances.Instance{ + StoredMetadata: instances.StoredMetadata{ + Id: "standby-src", + Name: "standby-src", + Image: "docker.io/library/alpine:latest", + CreatedAt: time.Now(), + HypervisorType: hypervisor.TypeCloudHypervisor, + }, + State: instances.StateStopped, + } + + mockMgr := &captureStandbyManager{ + Manager: svc.InstanceManager, + err: fmt.Errorf("%w: invalid snapshot compression level", instances.ErrInvalidRequest), + } + svc.InstanceManager = mockMgr + + resp, err := svc.StandbyInstance( + mw.WithResolvedInstance(ctx(), source.Id, source), + oapi.StandbyInstanceRequestObject{ + Id: source.Id, + Body: &oapi.StandbyInstanceRequest{ + Compression: &oapi.SnapshotCompressionConfig{ + Enabled: true, + }, + }, + }, + ) + require.NoError(t, err) + + badReq, ok := resp.(oapi.StandbyInstance400JSONResponse) + require.True(t, ok, "expected 400 response") + assert.Equal(t, "invalid_request", badReq.Code) + assert.Contains(t, badReq.Message, "invalid snapshot compression level") +} + func TestForkInstance_FromRunningFlagForwarded(t *testing.T) { t.Parallel() svc := newTestService(t) From 6b42cc533d927d3ed634f6923b37965b4fde8759 Mon Sep 17 00:00:00 2001 From: Steven Miller Date: Fri, 20 Mar 2026 14:14:19 -0400 Subject: [PATCH 23/31] Use native-first snapshot codecs with Go fallback --- lib/instances/manager.go | 3 + lib/instances/metrics.go | 36 ++ lib/instances/metrics_test.go | 11 + lib/instances/snapshot_compression.go | 276 ++++++++- .../snapshot_compression_native_test.go | 525 ++++++++++++++++++ lib/snapshot/README.md | 14 + 6 files changed, 856 insertions(+), 9 deletions(-) create mode 100644 lib/instances/snapshot_compression_native_test.go diff --git a/lib/instances/manager.go b/lib/instances/manager.go index 70cf368a..4e93b23e 100644 --- a/lib/instances/manager.go +++ b/lib/instances/manager.go @@ -91,6 +91,8 @@ type manager struct { snapshotDefaults SnapshotPolicy compressionMu sync.Mutex compressionJobs map[string]*compressionJob + nativeCodecMu sync.Mutex + nativeCodecPaths map[string]string // Hypervisor support vmStarters map[hypervisor.Type]hypervisor.VMStarter @@ -139,6 +141,7 @@ func NewManager(p *paths.Paths, imageManager images.Manager, systemManager syste guestMemoryPolicy: policy, snapshotDefaults: snapshotDefaults, compressionJobs: make(map[string]*compressionJob), + nativeCodecPaths: make(map[string]string), } // Initialize metrics if meter is provided diff --git a/lib/instances/metrics.go b/lib/instances/metrics.go index f1bcb089..bd87a3eb 100644 --- a/lib/instances/metrics.go +++ b/lib/instances/metrics.go @@ -45,6 +45,20 @@ const ( snapshotCompressionPreemptionDeleteSnapshot snapshotCompressionPreemptionOperation = "delete_snapshot" ) +type snapshotCodecOperation string + +const ( + snapshotCodecOperationCompress snapshotCodecOperation = "compress" + snapshotCodecOperationDecompress snapshotCodecOperation = "decompress" +) + +type snapshotCodecFallbackReason string + +const ( + snapshotCodecFallbackReasonMissingBinary snapshotCodecFallbackReason = "missing_binary" + snapshotCodecFallbackReasonNotExecutable snapshotCodecFallbackReason = "not_executable" +) + // Metrics holds the metrics instruments for instance operations. type Metrics struct { createDuration metric.Float64Histogram @@ -57,6 +71,7 @@ type Metrics struct { snapshotCompressionDuration metric.Float64Histogram snapshotCompressionSavedBytes metric.Int64Histogram snapshotCompressionRatio metric.Float64Histogram + snapshotCodecFallbacksTotal metric.Int64Counter snapshotRestoreMemoryPrepareTotal metric.Int64Counter snapshotRestoreMemoryPrepareDuration metric.Float64Histogram snapshotCompressionPreemptionsTotal metric.Int64Counter @@ -152,6 +167,14 @@ func newInstanceMetrics(meter metric.Meter, tracer trace.Tracer, m *manager) (*M return nil, err } + snapshotCodecFallbacksTotal, err := meter.Int64Counter( + "hypeman_snapshot_codec_fallbacks_total", + metric.WithDescription("Total number of snapshot codec fallbacks from native binaries to the Go implementation"), + ) + if err != nil { + return nil, err + } + snapshotRestoreMemoryPrepareTotal, err := meter.Int64Counter( "hypeman_snapshot_restore_memory_prepare_total", metric.WithDescription("Total number of snapshot memory prepare operations before restore"), @@ -277,6 +300,7 @@ func newInstanceMetrics(meter metric.Meter, tracer trace.Tracer, m *manager) (*M snapshotCompressionDuration: snapshotCompressionDuration, snapshotCompressionSavedBytes: snapshotCompressionSavedBytes, snapshotCompressionRatio: snapshotCompressionRatio, + snapshotCodecFallbacksTotal: snapshotCodecFallbacksTotal, snapshotRestoreMemoryPrepareTotal: snapshotRestoreMemoryPrepareTotal, snapshotRestoreMemoryPrepareDuration: snapshotRestoreMemoryPrepareDuration, snapshotCompressionPreemptionsTotal: snapshotCompressionPreemptionsTotal, @@ -383,3 +407,15 @@ func (m *manager) recordSnapshotCompressionPreemption(ctx context.Context, opera attrs = append(attrs, attribute.String("operation", string(operation))) m.metrics.snapshotCompressionPreemptionsTotal.Add(ctx, 1, metric.WithAttributes(attrs...)) } + +func (m *manager) recordSnapshotCodecFallback(ctx context.Context, algorithm snapshotstore.SnapshotCompressionAlgorithm, operation snapshotCodecOperation, reason snapshotCodecFallbackReason) { + if m.metrics == nil { + return + } + + m.metrics.snapshotCodecFallbacksTotal.Add(ctx, 1, metric.WithAttributes( + attribute.String("algorithm", string(algorithm)), + attribute.String("operation", string(operation)), + attribute.String("reason", string(reason)), + )) +} diff --git a/lib/instances/metrics_test.go b/lib/instances/metrics_test.go index f552303f..f0188f08 100644 --- a/lib/instances/metrics_test.go +++ b/lib/instances/metrics_test.go @@ -44,6 +44,7 @@ func TestSnapshotCompressionMetrics_RecordAndObserve(t *testing.T) { target := m.compressionJobs["job-1"].target m.recordSnapshotCompressionJob(t.Context(), target, snapshotCompressionResultSuccess, time.Now().Add(-2*time.Second), 1024, 256) + m.recordSnapshotCodecFallback(t.Context(), snapshotstore.SnapshotCompressionAlgorithmLz4, snapshotCodecOperationCompress, snapshotCodecFallbackReasonMissingBinary) m.recordSnapshotRestoreMemoryPrepare(t.Context(), hypervisor.TypeCloudHypervisor, snapshotMemoryPreparePathRaw, snapshotCompressionResultSuccess, time.Now().Add(-250*time.Millisecond)) m.recordSnapshotCompressionPreemption(t.Context(), snapshotCompressionPreemptionRestoreInstance, target) @@ -55,6 +56,7 @@ func TestSnapshotCompressionMetrics_RecordAndObserve(t *testing.T) { "hypeman_snapshot_compression_duration_seconds", "hypeman_snapshot_compression_saved_bytes", "hypeman_snapshot_compression_ratio", + "hypeman_snapshot_codec_fallbacks_total", "hypeman_snapshot_restore_memory_prepare_total", "hypeman_snapshot_restore_memory_prepare_duration_seconds", "hypeman_snapshot_compression_preemptions_total", @@ -78,6 +80,15 @@ func TestSnapshotCompressionMetrics_RecordAndObserve(t *testing.T) { assert.Equal(t, uint64(1), savedBytes.DataPoints[0].Count) assert.Equal(t, int64(768), savedBytes.DataPoints[0].Sum) + fallbackMetric := findMetric(t, rm, "hypeman_snapshot_codec_fallbacks_total") + fallbacks, ok := fallbackMetric.Data.(metricdata.Sum[int64]) + require.True(t, ok) + require.Len(t, fallbacks.DataPoints, 1) + assert.Equal(t, int64(1), fallbacks.DataPoints[0].Value) + assert.Equal(t, "lz4", metricLabel(t, fallbacks.DataPoints[0].Attributes, "algorithm")) + assert.Equal(t, "compress", metricLabel(t, fallbacks.DataPoints[0].Attributes, "operation")) + assert.Equal(t, "missing_binary", metricLabel(t, fallbacks.DataPoints[0].Attributes, "reason")) + restorePrepMetric := findMetric(t, rm, "hypeman_snapshot_restore_memory_prepare_total") restorePrep, ok := restorePrepMetric.Data.(metricdata.Sum[int64]) require.True(t, ok) diff --git a/lib/instances/snapshot_compression.go b/lib/instances/snapshot_compression.go index c885d876..20b195bf 100644 --- a/lib/instances/snapshot_compression.go +++ b/lib/instances/snapshot_compression.go @@ -1,13 +1,17 @@ package instances import ( + "bytes" "context" "errors" "fmt" "io" + "io/fs" "os" + "os/exec" "path/filepath" "strings" + "syscall" "time" "github.com/kernel/hypeman/lib/hypervisor" @@ -42,6 +46,26 @@ type compressionTarget struct { Policy snapshotstore.SnapshotCompressionConfig } +type nativeCodecRuntime struct { + manager *manager + lookPath func(file string) (string, error) + commandContext func(ctx context.Context, name string, arg ...string) *exec.Cmd +} + +func (r nativeCodecRuntime) lookPathFunc() func(string) (string, error) { + if r.lookPath != nil { + return r.lookPath + } + return exec.LookPath +} + +func (r nativeCodecRuntime) commandContextFunc() func(context.Context, string, ...string) *exec.Cmd { + if r.commandContext != nil { + return r.commandContext + } + return exec.CommandContext +} + func cloneCompressionConfig(cfg *snapshotstore.SnapshotCompressionConfig) *snapshotstore.SnapshotCompressionConfig { if cfg == nil { return nil @@ -149,6 +173,125 @@ func (m *manager) snapshotJobKeyForSnapshot(snapshotID string) string { return "snapshot:" + snapshotID } +func nativeCodecBinaryName(algorithm snapshotstore.SnapshotCompressionAlgorithm) string { + switch algorithm { + case snapshotstore.SnapshotCompressionAlgorithmLz4: + return "lz4" + default: + return "zstd" + } +} + +func nativeCodecInstallTip(algorithm snapshotstore.SnapshotCompressionAlgorithm) string { + switch algorithm { + case snapshotstore.SnapshotCompressionAlgorithmLz4: + return "install lz4 to enable native snapshot compression (for example: apt install lz4)" + default: + return "install zstd to enable native snapshot compression (for example: apt install zstd)" + } +} + +func isNativeCodecUnavailable(err error) (snapshotCodecFallbackReason, bool) { + if err == nil { + return "", false + } + switch { + case errors.Is(err, exec.ErrNotFound), errors.Is(err, fs.ErrNotExist), errors.Is(err, os.ErrNotExist), errors.Is(err, syscall.ENOENT): + return snapshotCodecFallbackReasonMissingBinary, true + case errors.Is(err, fs.ErrPermission), errors.Is(err, os.ErrPermission), errors.Is(err, syscall.EACCES), errors.Is(err, syscall.EPERM): + return snapshotCodecFallbackReasonNotExecutable, true + default: + return "", false + } +} + +func resolveNativeCodecPath(runtime nativeCodecRuntime, algorithm snapshotstore.SnapshotCompressionAlgorithm) (string, error) { + binaryName := nativeCodecBinaryName(algorithm) + if runtime.manager == nil { + return runtime.lookPathFunc()(binaryName) + } + + runtime.manager.nativeCodecMu.Lock() + if runtime.manager.nativeCodecPaths == nil { + runtime.manager.nativeCodecPaths = make(map[string]string) + } + if path := runtime.manager.nativeCodecPaths[binaryName]; path != "" { + runtime.manager.nativeCodecMu.Unlock() + return path, nil + } + runtime.manager.nativeCodecMu.Unlock() + + path, err := runtime.lookPathFunc()(binaryName) + if err != nil { + return "", err + } + + runtime.manager.nativeCodecMu.Lock() + if runtime.manager.nativeCodecPaths == nil { + runtime.manager.nativeCodecPaths = make(map[string]string) + } + runtime.manager.nativeCodecPaths[binaryName] = path + runtime.manager.nativeCodecMu.Unlock() + return path, nil +} + +func invalidateNativeCodecPath(runtime nativeCodecRuntime, algorithm snapshotstore.SnapshotCompressionAlgorithm, path string) { + if runtime.manager == nil { + return + } + + binaryName := nativeCodecBinaryName(algorithm) + runtime.manager.nativeCodecMu.Lock() + defer runtime.manager.nativeCodecMu.Unlock() + if runtime.manager.nativeCodecPaths == nil { + return + } + if currentPath := runtime.manager.nativeCodecPaths[binaryName]; currentPath == path { + delete(runtime.manager.nativeCodecPaths, binaryName) + } +} + +func recordNativeCodecFallback(ctx context.Context, runtime nativeCodecRuntime, algorithm snapshotstore.SnapshotCompressionAlgorithm, operation snapshotCodecOperation, reason snapshotCodecFallbackReason, nativeBinary string, err error) { + logger.FromContext(ctx).WarnContext( + ctx, + "native snapshot codec unavailable, falling back to go implementation", + "algorithm", string(algorithm), + "operation", string(operation), + "native_binary", nativeBinary, + "error", err, + "fallback_backend", "go", + "install_tip", nativeCodecInstallTip(algorithm), + ) + if runtime.manager != nil { + runtime.manager.recordSnapshotCodecFallback(ctx, algorithm, operation, reason) + } +} + +func runWithNativeFallback(ctx context.Context, runtime nativeCodecRuntime, algorithm snapshotstore.SnapshotCompressionAlgorithm, operation snapshotCodecOperation, nativeRunner func(binaryPath string) error, goRunner func() error) error { + binaryName := nativeCodecBinaryName(algorithm) + binaryPath, err := resolveNativeCodecPath(runtime, algorithm) + if err != nil { + reason, ok := isNativeCodecUnavailable(err) + if !ok { + return err + } + recordNativeCodecFallback(ctx, runtime, algorithm, operation, reason, binaryName, err) + return goRunner() + } + + if err := nativeRunner(binaryPath); err != nil { + reason, ok := isNativeCodecUnavailable(err) + if !ok { + return err + } + invalidateNativeCodecPath(runtime, algorithm, binaryPath) + recordNativeCodecFallback(ctx, runtime, algorithm, operation, reason, binaryPath, err) + return goRunner() + } + + return nil +} + func (m *manager) startCompressionJob(ctx context.Context, target compressionTarget) { if target.Key == "" || !target.Policy.Enabled { return @@ -199,7 +342,7 @@ func (m *manager) startCompressionJob(ctx context.Context, target compressionTar } var err error - uncompressedSize, compressedSize, err = compressSnapshotMemoryFile(jobCtx, rawPath, target.Policy) + uncompressedSize, compressedSize, err = compressSnapshotMemoryFileWithRuntime(jobCtx, nativeCodecRuntime{manager: m}, rawPath, target.Policy) if err != nil { if errors.Is(err, context.Canceled) { result = snapshotCompressionResultCanceled @@ -285,7 +428,7 @@ func (m *manager) ensureSnapshotMemoryReady(ctx context.Context, snapshotDir, jo if !ok { return nil } - if err := decompressSnapshotMemoryFile(ctx, compressedPath, algorithm); err != nil { + if err := decompressSnapshotMemoryFileWithRuntime(ctx, nativeCodecRuntime{manager: m}, compressedPath, algorithm); err != nil { m.recordSnapshotRestoreMemoryPrepare(ctx, hvType, snapshotMemoryPreparePathDecompress, snapshotCompressionResultFailed, start) return err } @@ -346,6 +489,10 @@ func snapshotMemoryFileCandidates(snapshotDir string) []string { } func compressSnapshotMemoryFile(ctx context.Context, rawPath string, cfg snapshotstore.SnapshotCompressionConfig) (int64, int64, error) { + return compressSnapshotMemoryFileWithRuntime(ctx, nativeCodecRuntime{}, rawPath, cfg) +} + +func compressSnapshotMemoryFileWithRuntime(ctx context.Context, runtime nativeCodecRuntime, rawPath string, cfg snapshotstore.SnapshotCompressionConfig) (int64, int64, error) { rawInfo, err := os.Stat(rawPath) if err != nil { return 0, 0, fmt.Errorf("stat raw memory snapshot: %w", err) @@ -357,7 +504,7 @@ func compressSnapshotMemoryFile(ctx context.Context, rawPath string, cfg snapsho removeCompressedSnapshotArtifacts(rawPath) _ = os.Remove(tmpPath) - if err := runCompression(ctx, rawPath, tmpPath, cfg); err != nil { + if err := runNativeCompression(ctx, runtime, rawPath, tmpPath, cfg); err != nil { _ = os.Remove(tmpPath) return 0, 0, err } @@ -384,7 +531,22 @@ func compressSnapshotMemoryFile(ctx context.Context, rawPath string, cfg snapsho return uncompressedSize, compressedInfo.Size(), nil } -func runCompression(ctx context.Context, srcPath, dstPath string, cfg snapshotstore.SnapshotCompressionConfig) error { +func runNativeCompression(ctx context.Context, runtime nativeCodecRuntime, srcPath, dstPath string, cfg snapshotstore.SnapshotCompressionConfig) error { + return runWithNativeFallback( + ctx, + runtime, + cfg.Algorithm, + snapshotCodecOperationCompress, + func(binaryPath string) error { + return runNativeCompressionCommand(ctx, runtime, binaryPath, srcPath, dstPath, cfg) + }, + func() error { + return runGoCompression(ctx, srcPath, dstPath, cfg) + }, + ) +} + +func runGoCompression(ctx context.Context, srcPath, dstPath string, cfg snapshotstore.SnapshotCompressionConfig) error { src, err := os.Open(srcPath) if err != nil { return fmt.Errorf("open source snapshot: %w", err) @@ -436,6 +598,37 @@ func runCompression(ctx context.Context, srcPath, dstPath string, cfg snapshotst return nil } +func runNativeCompressionCommand(ctx context.Context, runtime nativeCodecRuntime, binaryPath, srcPath, dstPath string, cfg snapshotstore.SnapshotCompressionConfig) error { + args := nativeCompressionArgs(srcPath, dstPath, cfg) + cmd := runtime.commandContextFunc()(ctx, binaryPath, args...) + var stderr bytes.Buffer + cmd.Stderr = &stderr + if err := cmd.Run(); err != nil { + if ctxErr := ctx.Err(); ctxErr != nil { + return ctxErr + } + return formatNativeCodecError("compress", cfg.Algorithm, err, stderr.String()) + } + return nil +} + +func nativeCompressionArgs(srcPath, dstPath string, cfg snapshotstore.SnapshotCompressionConfig) []string { + switch cfg.Algorithm { + case snapshotstore.SnapshotCompressionAlgorithmLz4: + level := defaultSnapshotCompressionLz4Level + if cfg.Level != nil { + level = *cfg.Level + } + return []string{"-q", "-f", lz4NativeCompressionFlag(level), srcPath, dstPath} + default: + level := defaultSnapshotCompressionZstdLevel + if cfg.Level != nil { + level = *cfg.Level + } + return []string{"-q", "-f", fmt.Sprintf("-%d", level), "-o", dstPath, srcPath} + } +} + func lz4CompressionLevel(level int) lz4.CompressionLevel { switch level { case 0: @@ -463,11 +656,51 @@ func lz4CompressionLevel(level int) lz4.CompressionLevel { } } +func lz4NativeCompressionFlag(level int) string { + if level == 0 { + return "--fast=1" + } + return fmt.Sprintf("-%d", level) +} + func decompressSnapshotMemoryFile(ctx context.Context, compressedPath string, algorithm snapshotstore.SnapshotCompressionAlgorithm) error { + return decompressSnapshotMemoryFileWithRuntime(ctx, nativeCodecRuntime{}, compressedPath, algorithm) +} + +func decompressSnapshotMemoryFileWithRuntime(ctx context.Context, runtime nativeCodecRuntime, compressedPath string, algorithm snapshotstore.SnapshotCompressionAlgorithm) error { rawPath := strings.TrimSuffix(strings.TrimSuffix(compressedPath, ".zst"), ".lz4") tmpRawPath := rawPath + ".tmp" _ = os.Remove(tmpRawPath) + if err := runNativeDecompression(ctx, runtime, compressedPath, tmpRawPath, algorithm); err != nil { + _ = os.Remove(tmpRawPath) + return err + } + if err := os.Rename(tmpRawPath, rawPath); err != nil { + _ = os.Remove(tmpRawPath) + return fmt.Errorf("finalize decompressed snapshot: %w", err) + } + removeCompressedSnapshotArtifacts(rawPath) + return nil +} + +func runNativeDecompression(ctx context.Context, runtime nativeCodecRuntime, srcPath, dstPath string, algorithm snapshotstore.SnapshotCompressionAlgorithm) error { + return runWithNativeFallback( + ctx, + runtime, + algorithm, + snapshotCodecOperationDecompress, + func(binaryPath string) error { + return runNativeDecompressionCommand(ctx, runtime, binaryPath, srcPath, dstPath, algorithm) + }, + func() error { + return runGoDecompression(ctx, srcPath, dstPath, algorithm) + }, + ) +} + +func runGoDecompression(ctx context.Context, compressedPath, tmpRawPath string, algorithm snapshotstore.SnapshotCompressionAlgorithm) error { + src, err := os.Open(compressedPath) if err != nil { return fmt.Errorf("open compressed snapshot: %w", err) @@ -496,17 +729,42 @@ func decompressSnapshotMemoryFile(ctx context.Context, compressedPath string, al } if err := copyWithContext(ctx, dst, reader); err != nil { - _ = os.Remove(tmpRawPath) return err } - if err := os.Rename(tmpRawPath, rawPath); err != nil { - _ = os.Remove(tmpRawPath) - return fmt.Errorf("finalize decompressed snapshot: %w", err) + return nil +} + +func runNativeDecompressionCommand(ctx context.Context, runtime nativeCodecRuntime, binaryPath, srcPath, dstPath string, algorithm snapshotstore.SnapshotCompressionAlgorithm) error { + args := nativeDecompressionArgs(srcPath, dstPath, algorithm) + cmd := runtime.commandContextFunc()(ctx, binaryPath, args...) + var stderr bytes.Buffer + cmd.Stderr = &stderr + if err := cmd.Run(); err != nil { + if ctxErr := ctx.Err(); ctxErr != nil { + return ctxErr + } + return formatNativeCodecError("decompress", algorithm, err, stderr.String()) } - removeCompressedSnapshotArtifacts(rawPath) return nil } +func nativeDecompressionArgs(srcPath, dstPath string, algorithm snapshotstore.SnapshotCompressionAlgorithm) []string { + switch algorithm { + case snapshotstore.SnapshotCompressionAlgorithmLz4: + return []string{"-q", "-d", "-f", srcPath, dstPath} + default: + return []string{"-q", "-d", "-f", "-o", dstPath, srcPath} + } +} + +func formatNativeCodecError(operation string, algorithm snapshotstore.SnapshotCompressionAlgorithm, err error, stderr string) error { + stderr = strings.TrimSpace(stderr) + if stderr == "" { + return fmt.Errorf("native %s %s: %w", algorithm, operation, err) + } + return fmt.Errorf("native %s %s: %w: %s", algorithm, operation, err, stderr) +} + func compressedPathFor(rawPath string, algorithm snapshotstore.SnapshotCompressionAlgorithm) string { switch algorithm { case snapshotstore.SnapshotCompressionAlgorithmLz4: diff --git a/lib/instances/snapshot_compression_native_test.go b/lib/instances/snapshot_compression_native_test.go new file mode 100644 index 00000000..2b655897 --- /dev/null +++ b/lib/instances/snapshot_compression_native_test.go @@ -0,0 +1,525 @@ +package instances + +import ( + "context" + "log/slog" + "os" + "os/exec" + "path/filepath" + "sync" + "syscall" + "testing" + "time" + + "github.com/kernel/hypeman/lib/logger" + "github.com/kernel/hypeman/lib/paths" + snapshotstore "github.com/kernel/hypeman/lib/snapshot" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + otelmetric "go.opentelemetry.io/otel/sdk/metric" + "go.opentelemetry.io/otel/sdk/metric/metricdata" +) + +type codecLogRecord struct { + level slog.Level + msg string + attrs map[string]string +} + +type codecLogRecorder struct { + mu sync.Mutex + records []codecLogRecord +} + +func (h *codecLogRecorder) Enabled(_ context.Context, _ slog.Level) bool { return true } + +func (h *codecLogRecorder) Handle(_ context.Context, r slog.Record) error { + rec := codecLogRecord{ + level: r.Level, + msg: r.Message, + attrs: map[string]string{}, + } + r.Attrs(func(attr slog.Attr) bool { + rec.attrs[attr.Key] = attr.Value.String() + return true + }) + h.mu.Lock() + h.records = append(h.records, rec) + h.mu.Unlock() + return nil +} + +func (h *codecLogRecorder) WithAttrs(_ []slog.Attr) slog.Handler { return h } +func (h *codecLogRecorder) WithGroup(_ string) slog.Handler { return h } + +func (h *codecLogRecorder) warnRecords() []codecLogRecord { + h.mu.Lock() + defer h.mu.Unlock() + out := make([]codecLogRecord, 0, len(h.records)) + for _, rec := range h.records { + if rec.level == slog.LevelWarn { + out = append(out, rec) + } + } + return out +} + +func TestNativeCompressionArgs(t *testing.T) { + t.Parallel() + + zstdArgs := nativeCompressionArgs("/tmp/raw", "/tmp/raw.zst.tmp", snapshotstore.SnapshotCompressionConfig{ + Enabled: true, + Algorithm: snapshotstore.SnapshotCompressionAlgorithmZstd, + Level: intPtr(19), + }) + assert.Equal(t, []string{"-q", "-f", "-19", "-o", "/tmp/raw.zst.tmp", "/tmp/raw"}, zstdArgs) + + lz4Args := nativeCompressionArgs("/tmp/raw", "/tmp/raw.lz4.tmp", snapshotstore.SnapshotCompressionConfig{ + Enabled: true, + Algorithm: snapshotstore.SnapshotCompressionAlgorithmLz4, + Level: intPtr(0), + }) + assert.Equal(t, []string{"-q", "-f", "--fast=1", "/tmp/raw", "/tmp/raw.lz4.tmp"}, lz4Args) + + lz4Args = nativeCompressionArgs("/tmp/raw", "/tmp/raw.lz4.tmp", snapshotstore.SnapshotCompressionConfig{ + Enabled: true, + Algorithm: snapshotstore.SnapshotCompressionAlgorithmLz4, + Level: intPtr(9), + }) + assert.Equal(t, []string{"-q", "-f", "-9", "/tmp/raw", "/tmp/raw.lz4.tmp"}, lz4Args) +} + +func TestNativeDecompressionArgs(t *testing.T) { + t.Parallel() + + assert.Equal(t, + []string{"-q", "-d", "-f", "-o", "/tmp/raw.tmp", "/tmp/raw.zst"}, + nativeDecompressionArgs("/tmp/raw.zst", "/tmp/raw.tmp", snapshotstore.SnapshotCompressionAlgorithmZstd), + ) + assert.Equal(t, + []string{"-q", "-d", "-f", "/tmp/raw.lz4", "/tmp/raw.tmp"}, + nativeDecompressionArgs("/tmp/raw.lz4", "/tmp/raw.tmp", snapshotstore.SnapshotCompressionAlgorithmLz4), + ) +} + +func TestIsNativeCodecUnavailable(t *testing.T) { + t.Parallel() + + reason, ok := isNativeCodecUnavailable(exec.ErrNotFound) + require.True(t, ok) + assert.Equal(t, snapshotCodecFallbackReasonMissingBinary, reason) + + reason, ok = isNativeCodecUnavailable(&os.PathError{Op: "fork/exec", Path: "/missing/zstd", Err: syscall.ENOENT}) + require.True(t, ok) + assert.Equal(t, snapshotCodecFallbackReasonMissingBinary, reason) + + reason, ok = isNativeCodecUnavailable(&os.PathError{Op: "fork/exec", Path: "/usr/bin/zstd", Err: syscall.EACCES}) + require.True(t, ok) + assert.Equal(t, snapshotCodecFallbackReasonNotExecutable, reason) + + _, ok = isNativeCodecUnavailable(assert.AnError) + assert.False(t, ok) +} + +func TestResolveNativeCodecPathCachesSuccess(t *testing.T) { + t.Parallel() + + mgr := &manager{nativeCodecPaths: make(map[string]string)} + lookups := 0 + runtime := nativeCodecRuntime{ + manager: mgr, + lookPath: func(file string) (string, error) { + lookups++ + return "/usr/bin/" + file, nil + }, + } + + path, err := resolveNativeCodecPath(runtime, snapshotstore.SnapshotCompressionAlgorithmZstd) + require.NoError(t, err) + assert.Equal(t, "/usr/bin/zstd", path) + + path, err = resolveNativeCodecPath(runtime, snapshotstore.SnapshotCompressionAlgorithmZstd) + require.NoError(t, err) + assert.Equal(t, "/usr/bin/zstd", path) + assert.Equal(t, 1, lookups) +} + +func TestResolveNativeCodecPathDoesNotCacheMiss(t *testing.T) { + t.Parallel() + + mgr := &manager{nativeCodecPaths: make(map[string]string)} + lookups := 0 + runtime := nativeCodecRuntime{ + manager: mgr, + lookPath: func(file string) (string, error) { + lookups++ + return "", exec.ErrNotFound + }, + } + + _, err := resolveNativeCodecPath(runtime, snapshotstore.SnapshotCompressionAlgorithmLz4) + require.Error(t, err) + _, err = resolveNativeCodecPath(runtime, snapshotstore.SnapshotCompressionAlgorithmLz4) + require.Error(t, err) + assert.Equal(t, 2, lookups) +} + +func TestCompressSnapshotMemoryFileFallsBackToGoWhenNativeBinaryMissing(t *testing.T) { + mgr, reader := newCodecRuntimeTestManager(t) + ctx, logs := newCodecRuntimeContext() + rawPath, original := writeRawSnapshotMemoryFile(t) + + _, compressedSize, err := compressSnapshotMemoryFileWithRuntime(ctx, nativeCodecRuntime{ + manager: mgr, + lookPath: func(file string) (string, error) { + return "", &exec.Error{Name: file, Err: exec.ErrNotFound} + }, + }, rawPath, snapshotstore.SnapshotCompressionConfig{ + Enabled: true, + Algorithm: snapshotstore.SnapshotCompressionAlgorithmZstd, + Level: intPtr(1), + }) + require.NoError(t, err) + assert.Greater(t, compressedSize, int64(0)) + + compressedPath := compressedPathFor(rawPath, snapshotstore.SnapshotCompressionAlgorithmZstd) + _, statErr := os.Stat(compressedPath) + require.NoError(t, statErr) + _, statErr = os.Stat(rawPath) + assert.Error(t, statErr) + + warns := logs.warnRecords() + require.Len(t, warns, 1) + assert.Equal(t, "native snapshot codec unavailable, falling back to go implementation", warns[0].msg) + assert.Equal(t, "zstd", warns[0].attrs["algorithm"]) + assert.Equal(t, "compress", warns[0].attrs["operation"]) + assert.Equal(t, "zstd", warns[0].attrs["native_binary"]) + assert.Equal(t, "go", warns[0].attrs["fallback_backend"]) + assert.Equal(t, nativeCodecInstallTip(snapshotstore.SnapshotCompressionAlgorithmZstd), warns[0].attrs["install_tip"]) + + var rm metricdata.ResourceMetrics + require.NoError(t, reader.Collect(context.Background(), &rm)) + fallbackMetric := findMetric(t, rm, "hypeman_snapshot_codec_fallbacks_total") + fallbacks, ok := fallbackMetric.Data.(metricdata.Sum[int64]) + require.True(t, ok) + require.Len(t, fallbacks.DataPoints, 1) + assert.Equal(t, int64(1), fallbacks.DataPoints[0].Value) + assert.Equal(t, "zstd", metricLabel(t, fallbacks.DataPoints[0].Attributes, "algorithm")) + assert.Equal(t, "compress", metricLabel(t, fallbacks.DataPoints[0].Attributes, "operation")) + assert.Equal(t, "missing_binary", metricLabel(t, fallbacks.DataPoints[0].Attributes, "reason")) + + runtimeGo := nativeCodecRuntime{lookPath: func(string) (string, error) { return "", exec.ErrNotFound }} + require.NoError(t, decompressSnapshotMemoryFileWithRuntime(context.Background(), runtimeGo, compressedPath, snapshotstore.SnapshotCompressionAlgorithmZstd)) + restored, err := os.ReadFile(rawPath) + require.NoError(t, err) + assert.Equal(t, original, restored) +} + +func TestCompressSnapshotMemoryFileFallsBackToGoWhenNativeBinaryMissingFromPATH(t *testing.T) { + mgr, reader := newCodecRuntimeTestManager(t) + ctx, logs := newCodecRuntimeContext() + rawPath, _ := writeRawSnapshotMemoryFile(t) + t.Setenv("PATH", t.TempDir()) + + _, compressedSize, err := compressSnapshotMemoryFileWithRuntime(ctx, nativeCodecRuntime{ + manager: mgr, + }, rawPath, snapshotstore.SnapshotCompressionConfig{ + Enabled: true, + Algorithm: snapshotstore.SnapshotCompressionAlgorithmZstd, + Level: intPtr(1), + }) + require.NoError(t, err) + assert.Greater(t, compressedSize, int64(0)) + + warns := logs.warnRecords() + require.Len(t, warns, 1) + assert.Equal(t, "zstd", warns[0].attrs["algorithm"]) + assert.Equal(t, "compress", warns[0].attrs["operation"]) + assert.Equal(t, "zstd", warns[0].attrs["native_binary"]) + + var rm metricdata.ResourceMetrics + require.NoError(t, reader.Collect(context.Background(), &rm)) + fallbackMetric := findMetric(t, rm, "hypeman_snapshot_codec_fallbacks_total") + fallbacks, ok := fallbackMetric.Data.(metricdata.Sum[int64]) + require.True(t, ok) + require.Len(t, fallbacks.DataPoints, 1) + assert.Equal(t, int64(1), fallbacks.DataPoints[0].Value) +} + +func TestDecompressSnapshotMemoryFileFallsBackToGoWhenNativeBinaryMissing(t *testing.T) { + mgr, reader := newCodecRuntimeTestManager(t) + ctx, logs := newCodecRuntimeContext() + rawPath, original := writeRawSnapshotMemoryFile(t) + + _, _, err := compressSnapshotMemoryFileWithRuntime(context.Background(), nativeCodecRuntime{ + lookPath: func(string) (string, error) { return "", exec.ErrNotFound }, + }, rawPath, snapshotstore.SnapshotCompressionConfig{ + Enabled: true, + Algorithm: snapshotstore.SnapshotCompressionAlgorithmLz4, + Level: intPtr(0), + }) + require.NoError(t, err) + + compressedPath := compressedPathFor(rawPath, snapshotstore.SnapshotCompressionAlgorithmLz4) + require.NoError(t, decompressSnapshotMemoryFileWithRuntime(ctx, nativeCodecRuntime{ + manager: mgr, + lookPath: func(file string) (string, error) { + return "", &exec.Error{Name: file, Err: exec.ErrNotFound} + }, + }, compressedPath, snapshotstore.SnapshotCompressionAlgorithmLz4)) + + restored, err := os.ReadFile(rawPath) + require.NoError(t, err) + assert.Equal(t, original, restored) + + warns := logs.warnRecords() + require.Len(t, warns, 1) + assert.Equal(t, "lz4", warns[0].attrs["algorithm"]) + assert.Equal(t, "decompress", warns[0].attrs["operation"]) + assert.Equal(t, "lz4", warns[0].attrs["native_binary"]) + assert.Equal(t, "go", warns[0].attrs["fallback_backend"]) + assert.Equal(t, nativeCodecInstallTip(snapshotstore.SnapshotCompressionAlgorithmLz4), warns[0].attrs["install_tip"]) + + var rm metricdata.ResourceMetrics + require.NoError(t, reader.Collect(context.Background(), &rm)) + fallbackMetric := findMetric(t, rm, "hypeman_snapshot_codec_fallbacks_total") + fallbacks, ok := fallbackMetric.Data.(metricdata.Sum[int64]) + require.True(t, ok) + require.Len(t, fallbacks.DataPoints, 1) + assert.Equal(t, int64(1), fallbacks.DataPoints[0].Value) + assert.Equal(t, "lz4", metricLabel(t, fallbacks.DataPoints[0].Attributes, "algorithm")) + assert.Equal(t, "decompress", metricLabel(t, fallbacks.DataPoints[0].Attributes, "operation")) + assert.Equal(t, "missing_binary", metricLabel(t, fallbacks.DataPoints[0].Attributes, "reason")) +} + +func TestCompressSnapshotMemoryFileDoesNotFallBackOnNativeRuntimeError(t *testing.T) { + mgr, reader := newCodecRuntimeTestManager(t) + ctx, logs := newCodecRuntimeContext() + rawPath, _ := writeRawSnapshotMemoryFile(t) + binaryPath := writeExecutableScript(t, "zstd", "#!/bin/sh\necho native boom >&2\nexit 1\n") + + _, _, err := compressSnapshotMemoryFileWithRuntime(ctx, nativeCodecRuntime{ + manager: mgr, + lookPath: func(string) (string, error) { return binaryPath, nil }, + }, rawPath, snapshotstore.SnapshotCompressionConfig{ + Enabled: true, + Algorithm: snapshotstore.SnapshotCompressionAlgorithmZstd, + Level: intPtr(1), + }) + require.Error(t, err) + assert.Contains(t, err.Error(), "native zstd compress") + + _, statErr := os.Stat(compressedPathFor(rawPath, snapshotstore.SnapshotCompressionAlgorithmZstd)) + assert.Error(t, statErr) + assert.Empty(t, logs.warnRecords()) + var rm metricdata.ResourceMetrics + require.NoError(t, reader.Collect(context.Background(), &rm)) +} + +func TestCompressSnapshotMemoryFileReturnsContextCanceledWhenNativeProcessIsKilled(t *testing.T) { + mgr, reader := newCodecRuntimeTestManager(t) + baseCtx, logs := newCodecRuntimeContext() + ctx, cancel := context.WithCancel(baseCtx) + rawPath, _ := writeRawSnapshotMemoryFile(t) + binaryPath := writeExecutableScript(t, "zstd", "#!/bin/sh\nsleep 30\n") + time.AfterFunc(20*time.Millisecond, cancel) + + _, _, err := compressSnapshotMemoryFileWithRuntime(ctx, nativeCodecRuntime{ + manager: mgr, + lookPath: func(string) (string, error) { return binaryPath, nil }, + }, rawPath, snapshotstore.SnapshotCompressionConfig{ + Enabled: true, + Algorithm: snapshotstore.SnapshotCompressionAlgorithmZstd, + Level: intPtr(1), + }) + require.ErrorIs(t, err, context.Canceled) + assert.Empty(t, logs.warnRecords()) + + var rm metricdata.ResourceMetrics + require.NoError(t, reader.Collect(context.Background(), &rm)) +} + +func TestDecompressSnapshotMemoryFileDoesNotFallBackOnNativeRuntimeError(t *testing.T) { + mgr, reader := newCodecRuntimeTestManager(t) + ctx, logs := newCodecRuntimeContext() + rawPath, original := writeRawSnapshotMemoryFile(t) + + _, _, err := compressSnapshotMemoryFileWithRuntime(context.Background(), nativeCodecRuntime{ + lookPath: func(string) (string, error) { return "", exec.ErrNotFound }, + }, rawPath, snapshotstore.SnapshotCompressionConfig{ + Enabled: true, + Algorithm: snapshotstore.SnapshotCompressionAlgorithmZstd, + Level: intPtr(1), + }) + require.NoError(t, err) + + compressedPath := compressedPathFor(rawPath, snapshotstore.SnapshotCompressionAlgorithmZstd) + binaryPath := writeExecutableScript(t, "zstd", "#!/bin/sh\necho native boom >&2\nexit 1\n") + err = decompressSnapshotMemoryFileWithRuntime(ctx, nativeCodecRuntime{ + manager: mgr, + lookPath: func(string) (string, error) { return binaryPath, nil }, + }, compressedPath, snapshotstore.SnapshotCompressionAlgorithmZstd) + require.Error(t, err) + assert.Contains(t, err.Error(), "native zstd decompress") + + _, statErr := os.Stat(rawPath) + assert.Error(t, statErr) + compressedBytes, readErr := os.ReadFile(compressedPath) + require.NoError(t, readErr) + assert.NotEmpty(t, compressedBytes) + assert.NotEqual(t, original, compressedBytes) + assert.Empty(t, logs.warnRecords()) + + var rm metricdata.ResourceMetrics + require.NoError(t, reader.Collect(context.Background(), &rm)) +} + +func TestCompressSnapshotMemoryFileInvalidatesStaleNativeCodecCache(t *testing.T) { + t.Parallel() + + mgr := &manager{nativeCodecPaths: map[string]string{"zstd": "/missing/zstd"}} + rawPath, _ := writeRawSnapshotMemoryFile(t) + + _, _, err := compressSnapshotMemoryFileWithRuntime(context.Background(), nativeCodecRuntime{ + manager: mgr, + lookPath: func(string) (string, error) { return "", exec.ErrNotFound }, + }, rawPath, snapshotstore.SnapshotCompressionConfig{ + Enabled: true, + Algorithm: snapshotstore.SnapshotCompressionAlgorithmZstd, + Level: intPtr(1), + }) + require.NoError(t, err) + assert.Empty(t, mgr.nativeCodecPaths) +} + +func TestNativeCompressGoDecompressCompatibility(t *testing.T) { + for _, tc := range compressionCompatibilityCases() { + t.Run(string(tc.algorithm), func(t *testing.T) { + binaryPath, err := exec.LookPath(nativeCodecBinaryName(tc.algorithm)) + if err != nil { + t.Skipf("native %s binary not available: %v", tc.algorithm, err) + } + + rawPath, original := writeRawSnapshotMemoryFile(t) + _, _, err = compressSnapshotMemoryFileWithRuntime(context.Background(), nativeCodecRuntime{ + lookPath: func(string) (string, error) { return binaryPath, nil }, + }, rawPath, tc.cfg) + require.NoError(t, err) + + compressedPath := compressedPathFor(rawPath, tc.algorithm) + require.NoError(t, decompressSnapshotMemoryFileWithRuntime(context.Background(), nativeCodecRuntime{ + lookPath: func(string) (string, error) { return "", exec.ErrNotFound }, + }, compressedPath, tc.algorithm)) + + restored, err := os.ReadFile(rawPath) + require.NoError(t, err) + assert.Equal(t, original, restored) + }) + } +} + +func TestGoCompressNativeDecompressCompatibility(t *testing.T) { + for _, tc := range compressionCompatibilityCases() { + t.Run(string(tc.algorithm), func(t *testing.T) { + binaryPath, err := exec.LookPath(nativeCodecBinaryName(tc.algorithm)) + if err != nil { + t.Skipf("native %s binary not available: %v", tc.algorithm, err) + } + + rawPath, original := writeRawSnapshotMemoryFile(t) + _, _, err = compressSnapshotMemoryFileWithRuntime(context.Background(), nativeCodecRuntime{ + lookPath: func(string) (string, error) { return "", exec.ErrNotFound }, + }, rawPath, tc.cfg) + require.NoError(t, err) + + compressedPath := compressedPathFor(rawPath, tc.algorithm) + require.NoError(t, decompressSnapshotMemoryFileWithRuntime(context.Background(), nativeCodecRuntime{ + lookPath: func(string) (string, error) { return binaryPath, nil }, + }, compressedPath, tc.algorithm)) + + restored, err := os.ReadFile(rawPath) + require.NoError(t, err) + assert.Equal(t, original, restored) + }) + } +} + +func TestNativeCompressNativeDecompressRoundTrip(t *testing.T) { + for _, tc := range compressionCompatibilityCases() { + t.Run(string(tc.algorithm), func(t *testing.T) { + binaryPath, err := exec.LookPath(nativeCodecBinaryName(tc.algorithm)) + if err != nil { + t.Skipf("native %s binary not available: %v", tc.algorithm, err) + } + + rawPath, original := writeRawSnapshotMemoryFile(t) + runtime := nativeCodecRuntime{lookPath: func(string) (string, error) { return binaryPath, nil }} + _, _, err = compressSnapshotMemoryFileWithRuntime(context.Background(), runtime, rawPath, tc.cfg) + require.NoError(t, err) + compressedPath := compressedPathFor(rawPath, tc.algorithm) + require.NoError(t, decompressSnapshotMemoryFileWithRuntime(context.Background(), runtime, compressedPath, tc.algorithm)) + + restored, err := os.ReadFile(rawPath) + require.NoError(t, err) + assert.Equal(t, original, restored) + }) + } +} + +type compressionCompatibilityCase struct { + algorithm snapshotstore.SnapshotCompressionAlgorithm + cfg snapshotstore.SnapshotCompressionConfig +} + +func compressionCompatibilityCases() []compressionCompatibilityCase { + return []compressionCompatibilityCase{ + { + algorithm: snapshotstore.SnapshotCompressionAlgorithmZstd, + cfg: snapshotstore.SnapshotCompressionConfig{Enabled: true, Algorithm: snapshotstore.SnapshotCompressionAlgorithmZstd, Level: intPtr(1)}, + }, + { + algorithm: snapshotstore.SnapshotCompressionAlgorithmLz4, + cfg: snapshotstore.SnapshotCompressionConfig{Enabled: true, Algorithm: snapshotstore.SnapshotCompressionAlgorithmLz4, Level: intPtr(0)}, + }, + } +} + +func newCodecRuntimeTestManager(t *testing.T) (*manager, *otelmetric.ManualReader) { + t.Helper() + + reader := otelmetric.NewManualReader() + provider := otelmetric.NewMeterProvider(otelmetric.WithReader(reader)) + mgr := &manager{ + paths: paths.New(t.TempDir()), + compressionJobs: make(map[string]*compressionJob), + nativeCodecPaths: make(map[string]string), + } + metrics, err := newInstanceMetrics(provider.Meter("test"), nil, mgr) + require.NoError(t, err) + mgr.metrics = metrics + return mgr, reader +} + +func newCodecRuntimeContext() (context.Context, *codecLogRecorder) { + recorder := &codecLogRecorder{} + log := slog.New(recorder) + return logger.AddToContext(context.Background(), log), recorder +} + +func writeRawSnapshotMemoryFile(t *testing.T) (string, []byte) { + t.Helper() + + dir := t.TempDir() + rawPath := filepath.Join(dir, "memory-ranges") + content := []byte("snapshot-memory-contents-for-native-codec-tests") + require.NoError(t, os.WriteFile(rawPath, content, 0o644)) + return rawPath, content +} + +func writeExecutableScript(t *testing.T, name, body string) string { + t.Helper() + + path := filepath.Join(t.TempDir(), name) + require.NoError(t, os.WriteFile(path, []byte(body), 0o755)) + return path +} diff --git a/lib/snapshot/README.md b/lib/snapshot/README.md index 9aa6761d..b8233c72 100644 --- a/lib/snapshot/README.md +++ b/lib/snapshot/README.md @@ -88,6 +88,20 @@ In practice, the tradeoff is: - supports levels `0-9` - defaults to level `0` (fastest) when no level is specified +### Native codec preference + +Compression and decompression try native system codecs first: + +- `zstd` snapshots prefer the `zstd` binary +- `lz4` snapshots prefer the `lz4` binary + +If the native binary is missing or not executable, Hypeman falls back to the in-process Go implementation. + +- This fallback is operational, not a behavior change: snapshot artifacts remain the same `.zst` and `.lz4` formats. +- A fallback emits a warning log with an install hint. +- A fallback also increments `hypeman_snapshot_codec_fallbacks_total`. +- If the native codec starts but fails for a real runtime reason, Hypeman does **not** silently fall back; it treats that as an error. + ### Restore (in-place) - Restore always applies to the original source VM. - Source VM must not be `Running`. From cde775fabf1a494d9631c018e428268ad4688abe Mon Sep 17 00:00:00 2001 From: Steven Miller Date: Fri, 20 Mar 2026 14:15:05 -0400 Subject: [PATCH 24/31] Normalize snapshot compression algorithms case-insensitively --- lib/instances/snapshot_compression.go | 3 +++ lib/instances/snapshot_compression_test.go | 10 ++++++++++ 2 files changed, 13 insertions(+) diff --git a/lib/instances/snapshot_compression.go b/lib/instances/snapshot_compression.go index 20b195bf..36b21161 100644 --- a/lib/instances/snapshot_compression.go +++ b/lib/instances/snapshot_compression.go @@ -96,6 +96,9 @@ func normalizeCompressionConfig(cfg *snapshotstore.SnapshotCompressionConfig) (s Enabled: true, Algorithm: cfg.Algorithm, } + if normalized.Algorithm != "" { + normalized.Algorithm = snapshotstore.SnapshotCompressionAlgorithm(strings.ToLower(string(normalized.Algorithm))) + } switch normalized.Algorithm { case "": normalized.Algorithm = snapshotstore.SnapshotCompressionAlgorithmZstd diff --git a/lib/instances/snapshot_compression_test.go b/lib/instances/snapshot_compression_test.go index 713c9371..94a218c0 100644 --- a/lib/instances/snapshot_compression_test.go +++ b/lib/instances/snapshot_compression_test.go @@ -55,6 +55,16 @@ func TestNormalizeCompressionConfig(t *testing.T) { assert.Equal(t, snapshotstore.SnapshotCompressionAlgorithmLz4, cfg.Algorithm) require.NotNil(t, cfg.Level) assert.Equal(t, 9, *cfg.Level) + + cfg, err = normalizeCompressionConfig(&snapshotstore.SnapshotCompressionConfig{ + Enabled: true, + Algorithm: snapshotstore.SnapshotCompressionAlgorithm("ZSTD"), + Level: intPtr(3), + }) + require.NoError(t, err) + assert.Equal(t, snapshotstore.SnapshotCompressionAlgorithmZstd, cfg.Algorithm) + require.NotNil(t, cfg.Level) + assert.Equal(t, 3, *cfg.Level) } func TestResolveSnapshotCompressionPolicyPrecedence(t *testing.T) { From c17bec48830ecc416d1b178b5616f39c21c19258 Mon Sep 17 00:00:00 2001 From: Steven Miller Date: Fri, 20 Mar 2026 14:22:41 -0400 Subject: [PATCH 25/31] Tighten standby compression validation handling --- cmd/api/api/optional_standby_body.go | 19 +++++++++++--- cmd/api/api/optional_standby_body_test.go | 29 ++++++++++++++++++++++ lib/instances/snapshot_compression_test.go | 17 +++++++++++++ lib/instances/standby.go | 4 +++ 4 files changed, 65 insertions(+), 4 deletions(-) diff --git a/cmd/api/api/optional_standby_body.go b/cmd/api/api/optional_standby_body.go index bc2dd83e..7157ad95 100644 --- a/cmd/api/api/optional_standby_body.go +++ b/cmd/api/api/optional_standby_body.go @@ -3,17 +3,18 @@ package api import ( "io" "net/http" - "regexp" "strings" ) -var standbyRoutePattern = regexp.MustCompile(`^/instances/[^/]+/standby$`) - // NormalizeOptionalStandbyBody rewrites empty standby POST bodies to "{}" // so the generated strict handler can decode them without special casing. func NormalizeOptionalStandbyBody(next http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.Method == http.MethodPost && standbyRoutePattern.MatchString(r.URL.Path) && requestBodyIsEmpty(r) { + if r.Method != http.MethodPost { + next.ServeHTTP(w, r) + return + } + if isStandbyRoutePath(r.URL.Path) && requestBodyIsEmpty(r) { r.Body = io.NopCloser(strings.NewReader(`{}`)) r.ContentLength = 2 if r.Header.Get("Content-Type") == "" { @@ -25,6 +26,16 @@ func NormalizeOptionalStandbyBody(next http.Handler) http.Handler { }) } +func isStandbyRoutePath(path string) bool { + if !strings.HasPrefix(path, "/instances/") || !strings.HasSuffix(path, "/standby") { + return false + } + + instanceID := strings.TrimPrefix(path, "/instances/") + instanceID = strings.TrimSuffix(instanceID, "/standby") + return instanceID != "" && !strings.Contains(instanceID, "/") +} + func requestBodyIsEmpty(r *http.Request) bool { if r == nil { return true diff --git a/cmd/api/api/optional_standby_body_test.go b/cmd/api/api/optional_standby_body_test.go index e124a77d..b8bc31de 100644 --- a/cmd/api/api/optional_standby_body_test.go +++ b/cmd/api/api/optional_standby_body_test.go @@ -76,4 +76,33 @@ func TestNormalizeOptionalStandbyBody(t *testing.T) { assert.Equal(t, http.StatusNoContent, rec.Code) assert.Empty(t, gotBody) }) + + t.Run("non-post request skips standby normalization", func(t *testing.T) { + t.Parallel() + + var gotBody []byte + next := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var err error + gotBody, err = io.ReadAll(r.Body) + require.NoError(t, err) + w.WriteHeader(http.StatusNoContent) + }) + + req := httptest.NewRequest(http.MethodGet, "/instances/test/standby", nil) + rec := httptest.NewRecorder() + + NormalizeOptionalStandbyBody(next).ServeHTTP(rec, req) + + assert.Equal(t, http.StatusNoContent, rec.Code) + assert.Empty(t, gotBody) + }) + + t.Run("standby route matcher only accepts single path segment ids", func(t *testing.T) { + t.Parallel() + + assert.True(t, isStandbyRoutePath("/instances/test/standby")) + assert.False(t, isStandbyRoutePath("/instances/test/start")) + assert.False(t, isStandbyRoutePath("/instances/test/standby/extra")) + assert.False(t, isStandbyRoutePath("/instances/test/nested/standby")) + }) } diff --git a/lib/instances/snapshot_compression_test.go b/lib/instances/snapshot_compression_test.go index 94a218c0..274f9c50 100644 --- a/lib/instances/snapshot_compression_test.go +++ b/lib/instances/snapshot_compression_test.go @@ -166,6 +166,23 @@ func TestResolveCompressionPolicyExplicitDisableOverridesDefaults(t *testing.T) assert.Nil(t, standbyCfg) } +func TestResolveStandbyCompressionPolicyInvalidConfiguredDefaultIsInvalidRequest(t *testing.T) { + t.Parallel() + + m := &manager{ + snapshotDefaults: SnapshotPolicy{ + Compression: &snapshotstore.SnapshotCompressionConfig{ + Enabled: true, + Algorithm: snapshotstore.SnapshotCompressionAlgorithm("brotli"), + }, + }, + } + + _, err := m.resolveStandbyCompressionPolicy(&StoredMetadata{}, nil) + require.Error(t, err) + assert.True(t, errors.Is(err, ErrInvalidRequest)) +} + func TestCompressionMetadataForExistingArtifactUsesActualAlgorithm(t *testing.T) { t.Parallel() diff --git a/lib/instances/standby.go b/lib/instances/standby.go index f5bab31d..94edc1ea 100644 --- a/lib/instances/standby.go +++ b/lib/instances/standby.go @@ -2,6 +2,7 @@ package instances import ( "context" + "errors" "fmt" "os" "path/filepath" @@ -64,6 +65,9 @@ func (m *manager) standbyInstance( if !skipCompression { policy, err := m.resolveStandbyCompressionPolicy(stored, req.Compression) if err != nil { + if !errors.Is(err, ErrInvalidRequest) { + err = fmt.Errorf("%w: %v", ErrInvalidRequest, err) + } return nil, err } compressionPolicy = policy From e78c65774e7c1bfe3b7bff92b6b920692b41545a Mon Sep 17 00:00:00 2001 From: sjmiller609 <7516283+sjmiller609@users.noreply.github.com> Date: Sat, 21 Mar 2026 00:05:48 +0000 Subject: [PATCH 26/31] Reduce compression levels in integration tests to avoid CI timeout The compression integration tests were using zstd level 19 and lz4 level 9, which are very slow for compressing ~1GB memory files. After merging main (which added more integration tests to lib/instances), the total package test time exceeded the 20-minute CI timeout. Reduce to level 3 for both zstd and lz4 high-level cases. The tests still exercise the full compression/decompression pipeline across both algorithms and multiple levels (1 and 3 for zstd, 0 and 3 for lz4). Co-Authored-By: Claude Opus 4.6 --- lib/instances/compression_integration_linux_test.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/lib/instances/compression_integration_linux_test.go b/lib/instances/compression_integration_linux_test.go index dc2c6bb1..0f4a1120 100644 --- a/lib/instances/compression_integration_linux_test.go +++ b/lib/instances/compression_integration_linux_test.go @@ -159,9 +159,9 @@ func runStandbyRestoreCompressionScenarios(t *testing.T, harness compressionInte inFlightCompression := &snapshotstore.SnapshotCompressionConfig{ Enabled: true, Algorithm: snapshotstore.SnapshotCompressionAlgorithmZstd, - Level: intPtr(19), + Level: intPtr(3), } - inst = runCompressionCycle(t, ctx, mgr, p, inst, harness.waitHypervisorUp, "in-flight-zstd-19", inFlightCompression, false) + inst = runCompressionCycle(t, ctx, mgr, p, inst, harness.waitHypervisorUp, "in-flight-zstd-3", inFlightCompression, false) completedCases := []struct { name string @@ -176,11 +176,11 @@ func runStandbyRestoreCompressionScenarios(t *testing.T, harness compressionInte }, }, { - name: "zstd-19", + name: "zstd-3", cfg: &snapshotstore.SnapshotCompressionConfig{ Enabled: true, Algorithm: snapshotstore.SnapshotCompressionAlgorithmZstd, - Level: intPtr(19), + Level: intPtr(3), }, }, { @@ -192,11 +192,11 @@ func runStandbyRestoreCompressionScenarios(t *testing.T, harness compressionInte }, }, { - name: "lz4-9", + name: "lz4-3", cfg: &snapshotstore.SnapshotCompressionConfig{ Enabled: true, Algorithm: snapshotstore.SnapshotCompressionAlgorithmLz4, - Level: intPtr(9), + Level: intPtr(3), }, }, } From 2a9c9bc3abcee04957b3430b5d8781df801bee11 Mon Sep 17 00:00:00 2001 From: sjmiller609 <7516283+sjmiller609@users.noreply.github.com> Date: Sat, 21 Mar 2026 00:27:22 +0000 Subject: [PATCH 27/31] Reduce compression integration test cycles to fit CI timeout Five compression cycles (each involving VM standby + compress + restore + boot + exec readiness) consistently exceed the 20-minute CI timeout after merging main. Reduce to three cycles: one in-flight zstd, one completed zstd, and one completed lz4. This still exercises both algorithms and both the in-flight/completed code paths. Co-Authored-By: Claude Opus 4.6 --- .../compression_integration_linux_test.go | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/lib/instances/compression_integration_linux_test.go b/lib/instances/compression_integration_linux_test.go index 0f4a1120..c02be929 100644 --- a/lib/instances/compression_integration_linux_test.go +++ b/lib/instances/compression_integration_linux_test.go @@ -175,14 +175,6 @@ func runStandbyRestoreCompressionScenarios(t *testing.T, harness compressionInte Level: intPtr(1), }, }, - { - name: "zstd-3", - cfg: &snapshotstore.SnapshotCompressionConfig{ - Enabled: true, - Algorithm: snapshotstore.SnapshotCompressionAlgorithmZstd, - Level: intPtr(3), - }, - }, { name: "lz4-0", cfg: &snapshotstore.SnapshotCompressionConfig{ @@ -191,14 +183,6 @@ func runStandbyRestoreCompressionScenarios(t *testing.T, harness compressionInte Level: intPtr(0), }, }, - { - name: "lz4-3", - cfg: &snapshotstore.SnapshotCompressionConfig{ - Enabled: true, - Algorithm: snapshotstore.SnapshotCompressionAlgorithmLz4, - Level: intPtr(3), - }, - }, } for _, tc := range completedCases { From 393b80a8468667ef96ddcdd0552aaa8de6ae35e5 Mon Sep 17 00:00:00 2001 From: sjmiller609 <7516283+sjmiller609@users.noreply.github.com> Date: Sat, 21 Mar 2026 00:43:08 +0000 Subject: [PATCH 28/31] Address PR review feedback: add OpenAPI descriptions, fix dst.Close() error handling - Add descriptions to snapshot_policy and compression fields in openapi.yaml per Steven's review comments - Check dst.Close() errors in runGoCompression and runGoDecompression to prevent silently corrupt snapshot files on delayed write failures Co-Authored-By: Claude Opus 4.6 --- lib/instances/snapshot_compression.go | 22 ++++++++++++++++++++-- openapi.yaml | 2 ++ 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/lib/instances/snapshot_compression.go b/lib/instances/snapshot_compression.go index 36b21161..2664d060 100644 --- a/lib/instances/snapshot_compression.go +++ b/lib/instances/snapshot_compression.go @@ -560,7 +560,12 @@ func runGoCompression(ctx context.Context, srcPath, dstPath string, cfg snapshot if err != nil { return fmt.Errorf("create compressed snapshot: %w", err) } - defer dst.Close() + closed := false + defer func() { + if !closed { + dst.Close() + } + }() switch cfg.Algorithm { case snapshotstore.SnapshotCompressionAlgorithmZstd: @@ -598,6 +603,10 @@ func runGoCompression(ctx context.Context, srcPath, dstPath string, cfg snapshot default: return fmt.Errorf("%w: unsupported compression algorithm %q", ErrInvalidRequest, cfg.Algorithm) } + closed = true + if err := dst.Close(); err != nil { + return fmt.Errorf("close compressed snapshot file: %w", err) + } return nil } @@ -714,7 +723,12 @@ func runGoDecompression(ctx context.Context, compressedPath, tmpRawPath string, if err != nil { return fmt.Errorf("create decompressed snapshot file: %w", err) } - defer dst.Close() + closed := false + defer func() { + if !closed { + dst.Close() + } + }() var reader io.Reader switch algorithm { @@ -734,6 +748,10 @@ func runGoDecompression(ctx context.Context, compressedPath, tmpRawPath string, if err := copyWithContext(ctx, dst, reader); err != nil { return err } + closed = true + if err := dst.Close(); err != nil { + return fmt.Errorf("close decompressed snapshot file: %w", err) + } return nil } diff --git a/openapi.yaml b/openapi.yaml index 50b99642..97d6a0bd 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -321,6 +321,7 @@ components: description: Hypervisor to use for this instance. Defaults to server configuration. example: cloud-hypervisor snapshot_policy: + description: Snapshot compression policy for this instance. Controls compression settings applied when creating snapshots or entering standby. $ref: "#/components/schemas/SnapshotPolicy" skip_kernel_headers: type: boolean @@ -501,6 +502,7 @@ components: tags: $ref: "#/components/schemas/Tags" compression: + description: Compression settings to use for this snapshot. Overrides instance and server defaults. $ref: "#/components/schemas/SnapshotCompressionConfig" StandbyInstanceRequest: From 460c265f1cf8890417d780e222c77398f6d9cbef Mon Sep 17 00:00:00 2001 From: sjmiller609 <7516283+sjmiller609@users.noreply.github.com> Date: Mon, 23 Mar 2026 15:06:11 +0000 Subject: [PATCH 29/31] Address review feedback: add server-side compression validation, log metadata errors - Validate algorithm (zstd/lz4) and per-algorithm level ranges in toDomainSnapshotCompressionConfig instead of passing through unchecked - Log metadata update errors in compression jobs instead of silently discarding them - Normalize algorithm to lowercase in config struct after validation - Fix misleading test name (OmitsLevel -> PreservesLevel) Co-Authored-By: Claude Opus 4.6 --- cmd/api/api/instances.go | 22 +++++++++++++++++++++- cmd/api/config/config.go | 3 ++- lib/instances/snapshot_compression.go | 16 ++++++++++++---- lib/providers/providers_test.go | 2 +- 4 files changed, 36 insertions(+), 7 deletions(-) diff --git a/cmd/api/api/instances.go b/cmd/api/api/instances.go index 76e49fd6..7323c084 100644 --- a/cmd/api/api/instances.go +++ b/cmd/api/api/instances.go @@ -1023,10 +1023,30 @@ func toDomainSnapshotCompressionConfig(cfg oapi.SnapshotCompressionConfig) (*sna Enabled: cfg.Enabled, } if cfg.Algorithm != nil { - out.Algorithm = snapshot.SnapshotCompressionAlgorithm(*cfg.Algorithm) + algo := snapshot.SnapshotCompressionAlgorithm(strings.ToLower(string(*cfg.Algorithm))) + switch algo { + case snapshot.SnapshotCompressionAlgorithmZstd, snapshot.SnapshotCompressionAlgorithmLz4: + default: + return nil, fmt.Errorf("algorithm must be one of zstd or lz4, got %q", *cfg.Algorithm) + } + out.Algorithm = algo } if cfg.Level != nil { level := *cfg.Level + algo := out.Algorithm + if algo == "" { + algo = snapshot.SnapshotCompressionAlgorithmZstd + } + switch algo { + case snapshot.SnapshotCompressionAlgorithmZstd: + if level < snapshot.MinSnapshotCompressionZstdLevel || level > snapshot.MaxSnapshotCompressionZstdLevel { + return nil, fmt.Errorf("level must be between %d and %d for zstd, got %d", snapshot.MinSnapshotCompressionZstdLevel, snapshot.MaxSnapshotCompressionZstdLevel, level) + } + case snapshot.SnapshotCompressionAlgorithmLz4: + if level < snapshot.MinSnapshotCompressionLz4Level || level > snapshot.MaxSnapshotCompressionLz4Level { + return nil, fmt.Errorf("level must be between %d and %d for lz4, got %d", snapshot.MinSnapshotCompressionLz4Level, snapshot.MaxSnapshotCompressionLz4Level, level) + } + } out.Level = &level } return out, nil diff --git a/cmd/api/config/config.go b/cmd/api/config/config.go index b4b99cef..622307bc 100644 --- a/cmd/api/config/config.go +++ b/cmd/api/config/config.go @@ -469,11 +469,12 @@ func (c *Config) Validate() error { return fmt.Errorf("build.timeout must be positive, got %d", c.Build.Timeout) } algorithm := strings.ToLower(c.Snapshot.CompressionDefault.Algorithm) + c.Snapshot.CompressionDefault.Algorithm = algorithm if c.Snapshot.CompressionDefault.Enabled { switch algorithm { case "", "zstd", "lz4": default: - return fmt.Errorf("snapshot.compression_default.algorithm must be one of zstd or lz4, got %q", c.Snapshot.CompressionDefault.Algorithm) + return fmt.Errorf("snapshot.compression_default.algorithm must be one of zstd or lz4, got %q", algorithm) } if c.Snapshot.CompressionDefault.Level != nil { level := *c.Snapshot.CompressionDefault.Level diff --git a/lib/instances/snapshot_compression.go b/lib/instances/snapshot_compression.go index 2664d060..1acc7958 100644 --- a/lib/instances/snapshot_compression.go +++ b/lib/instances/snapshot_compression.go @@ -339,7 +339,9 @@ func (m *manager) startCompressionJob(ctx context.Context, target compressionTar size := st.Size() compressedSizeBytes = &size } - _ = m.updateSnapshotCompressionMetadata(target.SnapshotID, snapshotstore.SnapshotCompressionStateCompressed, "", &cfg, compressedSizeBytes, nil) + if metaErr := m.updateSnapshotCompressionMetadata(target.SnapshotID, snapshotstore.SnapshotCompressionStateCompressed, "", &cfg, compressedSizeBytes, nil); metaErr != nil { + log.ErrorContext(ctx, "failed to update snapshot compression metadata", "snapshot_id", target.SnapshotID, "error", metaErr) + } } return } @@ -350,20 +352,26 @@ func (m *manager) startCompressionJob(ctx context.Context, target compressionTar if errors.Is(err, context.Canceled) { result = snapshotCompressionResultCanceled if target.SnapshotID != "" { - _ = m.updateSnapshotCompressionMetadata(target.SnapshotID, snapshotstore.SnapshotCompressionStateNone, "", nil, nil, nil) + if metaErr := m.updateSnapshotCompressionMetadata(target.SnapshotID, snapshotstore.SnapshotCompressionStateNone, "", nil, nil, nil); metaErr != nil { + log.ErrorContext(ctx, "failed to update snapshot compression metadata", "snapshot_id", target.SnapshotID, "error", metaErr) + } } return } result = snapshotCompressionResultFailed if target.SnapshotID != "" { - _ = m.updateSnapshotCompressionMetadata(target.SnapshotID, snapshotstore.SnapshotCompressionStateError, err.Error(), &target.Policy, nil, nil) + if metaErr := m.updateSnapshotCompressionMetadata(target.SnapshotID, snapshotstore.SnapshotCompressionStateError, err.Error(), &target.Policy, nil, nil); metaErr != nil { + log.ErrorContext(ctx, "failed to update snapshot compression metadata", "snapshot_id", target.SnapshotID, "error", metaErr) + } } log.WarnContext(ctx, "snapshot compression failed", "snapshot_dir", target.SnapshotDir, "error", err) return } if target.SnapshotID != "" { - _ = m.updateSnapshotCompressionMetadata(target.SnapshotID, snapshotstore.SnapshotCompressionStateCompressed, "", &target.Policy, &compressedSize, &uncompressedSize) + if metaErr := m.updateSnapshotCompressionMetadata(target.SnapshotID, snapshotstore.SnapshotCompressionStateCompressed, "", &target.Policy, &compressedSize, &uncompressedSize); metaErr != nil { + log.ErrorContext(ctx, "failed to update snapshot compression metadata", "snapshot_id", target.SnapshotID, "error", metaErr) + } } }() } diff --git a/lib/providers/providers_test.go b/lib/providers/providers_test.go index b950b554..5ea519ba 100644 --- a/lib/providers/providers_test.go +++ b/lib/providers/providers_test.go @@ -26,7 +26,7 @@ func TestSnapshotDefaultsFromConfigDisabledReturnsNilCompression(t *testing.T) { assert.Nil(t, defaults.Compression) } -func TestSnapshotDefaultsFromConfigOmitsLevelForLZ4(t *testing.T) { +func TestSnapshotDefaultsFromConfigPreservesLevelForLZ4(t *testing.T) { t.Parallel() cfg := &config.Config{ From 3b9264e7ba8cc3e1b16bef2200bc35d34d9a2829 Mon Sep 17 00:00:00 2001 From: Steven Miller Date: Mon, 23 Mar 2026 11:33:57 -0400 Subject: [PATCH 30/31] Fix snapshot compression review follow-ups --- cmd/api/config/config.go | 1 + cmd/api/config/config_test.go | 5 +++- lib/instances/snapshot.go | 3 ++ lib/instances/snapshot_compression.go | 20 ++++++------- lib/instances/snapshot_test.go | 43 +++++++++++++++++++++++++++ 5 files changed, 61 insertions(+), 11 deletions(-) diff --git a/cmd/api/config/config.go b/cmd/api/config/config.go index 622307bc..77a43912 100644 --- a/cmd/api/config/config.go +++ b/cmd/api/config/config.go @@ -489,6 +489,7 @@ func (c *Config) Validate() error { } } } + c.Snapshot.CompressionDefault.Algorithm = algorithm } if c.Hypervisor.Memory.KernelPageInitMode != "performance" && c.Hypervisor.Memory.KernelPageInitMode != "hardened" { return fmt.Errorf("hypervisor.memory.kernel_page_init_mode must be one of {performance,hardened}, got %q", c.Hypervisor.Memory.KernelPageInitMode) diff --git a/cmd/api/config/config_test.go b/cmd/api/config/config_test.go index 16c24592..935c7aea 100644 --- a/cmd/api/config/config_test.go +++ b/cmd/api/config/config_test.go @@ -87,12 +87,15 @@ func TestValidateRejectsInvalidVMLabelBudget(t *testing.T) { func TestValidateAllowsLZ4CompressionDefaultWithImplicitLevel(t *testing.T) { cfg := defaultConfig() cfg.Snapshot.CompressionDefault.Enabled = true - cfg.Snapshot.CompressionDefault.Algorithm = "lz4" + cfg.Snapshot.CompressionDefault.Algorithm = "LZ4" cfg.Snapshot.CompressionDefault.Level = nil if err := cfg.Validate(); err != nil { t.Fatalf("expected lz4 compression default to validate, got %v", err) } + if cfg.Snapshot.CompressionDefault.Algorithm != "lz4" { + t.Fatalf("expected algorithm to normalize to lowercase, got %q", cfg.Snapshot.CompressionDefault.Algorithm) + } } func TestValidateAllowsExplicitLZ4CompressionLevelRange(t *testing.T) { diff --git a/lib/instances/snapshot.go b/lib/instances/snapshot.go index d42ca6f2..5cf1f6c6 100644 --- a/lib/instances/snapshot.go +++ b/lib/instances/snapshot.go @@ -393,6 +393,9 @@ func (m *manager) forkSnapshot(ctx context.Context, snapshotID string, req ForkS if target != nil { m.recordSnapshotCompressionPreemption(ctx, snapshotCompressionPreemptionForkSnapshot, *target) } + if err := m.ensureSnapshotMemoryReady(ctx, m.paths.SnapshotGuestDir(snapshotID), "", rec.StoredMetadata.HypervisorType); err != nil { + return nil, fmt.Errorf("prepare snapshot memory for fork: %w", err) + } if err := forkvm.CopyGuestDirectory(m.paths.SnapshotGuestDir(snapshotID), dstDir); err != nil { if errors.Is(err, forkvm.ErrSparseCopyUnsupported) { diff --git a/lib/instances/snapshot_compression.go b/lib/instances/snapshot_compression.go index 1acc7958..680cd96a 100644 --- a/lib/instances/snapshot_compression.go +++ b/lib/instances/snapshot_compression.go @@ -320,6 +320,7 @@ func (m *manager) startCompressionJob(ctx context.Context, target compressionTar var uncompressedSize int64 var compressedSize int64 metricsCtx := context.Background() + log := logger.FromContext(ctx) defer func() { m.recordSnapshotCompressionJob(metricsCtx, target, result, start, uncompressedSize, compressedSize) @@ -329,7 +330,6 @@ func (m *manager) startCompressionJob(ctx context.Context, target compressionTar close(job.done) }() - log := logger.FromContext(ctx) rawPath, ok := findRawSnapshotMemoryFile(target.SnapshotDir) if !ok { if compressedPath, algorithm, found := findCompressedSnapshotMemoryFile(target.SnapshotDir); found && target.SnapshotID != "" { @@ -339,8 +339,8 @@ func (m *manager) startCompressionJob(ctx context.Context, target compressionTar size := st.Size() compressedSizeBytes = &size } - if metaErr := m.updateSnapshotCompressionMetadata(target.SnapshotID, snapshotstore.SnapshotCompressionStateCompressed, "", &cfg, compressedSizeBytes, nil); metaErr != nil { - log.ErrorContext(ctx, "failed to update snapshot compression metadata", "snapshot_id", target.SnapshotID, "error", metaErr) + if err := m.updateSnapshotCompressionMetadata(target.SnapshotID, snapshotstore.SnapshotCompressionStateCompressed, "", &cfg, compressedSizeBytes, nil); err != nil { + log.ErrorContext(jobCtx, "failed to update snapshot compression metadata", "snapshot_id", target.SnapshotID, "snapshot_dir", target.SnapshotDir, "state", snapshotstore.SnapshotCompressionStateCompressed, "error", err) } } return @@ -352,25 +352,25 @@ func (m *manager) startCompressionJob(ctx context.Context, target compressionTar if errors.Is(err, context.Canceled) { result = snapshotCompressionResultCanceled if target.SnapshotID != "" { - if metaErr := m.updateSnapshotCompressionMetadata(target.SnapshotID, snapshotstore.SnapshotCompressionStateNone, "", nil, nil, nil); metaErr != nil { - log.ErrorContext(ctx, "failed to update snapshot compression metadata", "snapshot_id", target.SnapshotID, "error", metaErr) + if err := m.updateSnapshotCompressionMetadata(target.SnapshotID, snapshotstore.SnapshotCompressionStateNone, "", nil, nil, nil); err != nil { + log.ErrorContext(jobCtx, "failed to update snapshot compression metadata", "snapshot_id", target.SnapshotID, "snapshot_dir", target.SnapshotDir, "state", snapshotstore.SnapshotCompressionStateNone, "error", err) } } return } result = snapshotCompressionResultFailed if target.SnapshotID != "" { - if metaErr := m.updateSnapshotCompressionMetadata(target.SnapshotID, snapshotstore.SnapshotCompressionStateError, err.Error(), &target.Policy, nil, nil); metaErr != nil { - log.ErrorContext(ctx, "failed to update snapshot compression metadata", "snapshot_id", target.SnapshotID, "error", metaErr) + if metadataErr := m.updateSnapshotCompressionMetadata(target.SnapshotID, snapshotstore.SnapshotCompressionStateError, err.Error(), &target.Policy, nil, nil); metadataErr != nil { + log.ErrorContext(jobCtx, "failed to update snapshot compression metadata", "snapshot_id", target.SnapshotID, "snapshot_dir", target.SnapshotDir, "state", snapshotstore.SnapshotCompressionStateError, "error", metadataErr) } } - log.WarnContext(ctx, "snapshot compression failed", "snapshot_dir", target.SnapshotDir, "error", err) + log.WarnContext(jobCtx, "snapshot compression failed", "snapshot_dir", target.SnapshotDir, "error", err) return } if target.SnapshotID != "" { - if metaErr := m.updateSnapshotCompressionMetadata(target.SnapshotID, snapshotstore.SnapshotCompressionStateCompressed, "", &target.Policy, &compressedSize, &uncompressedSize); metaErr != nil { - log.ErrorContext(ctx, "failed to update snapshot compression metadata", "snapshot_id", target.SnapshotID, "error", metaErr) + if err := m.updateSnapshotCompressionMetadata(target.SnapshotID, snapshotstore.SnapshotCompressionStateCompressed, "", &target.Policy, &compressedSize, &uncompressedSize); err != nil { + log.ErrorContext(jobCtx, "failed to update snapshot compression metadata", "snapshot_id", target.SnapshotID, "snapshot_dir", target.SnapshotDir, "state", snapshotstore.SnapshotCompressionStateCompressed, "error", err) } } }() diff --git a/lib/instances/snapshot_test.go b/lib/instances/snapshot_test.go index dd028f72..cc634f55 100644 --- a/lib/instances/snapshot_test.go +++ b/lib/instances/snapshot_test.go @@ -237,6 +237,49 @@ func TestCreateStandbySnapshotFromCompressedSourceCopiesRawMemory(t *testing.T) assert.False(t, ok, "snapshot copy should not inherit compressed memory artifacts from the source standby instance") } +func TestForkSnapshotFromCompressedSourceCopiesRawMemory(t *testing.T) { + t.Parallel() + + mgr, _ := setupTestManager(t) + ctx := context.Background() + + hvType := mgr.defaultHypervisor + sourceID := "snapshot-fork-compressed-src" + createStandbySnapshotSourceFixture(t, mgr, sourceID, "snapshot-fork-compressed-src", hvType) + + snap, err := mgr.CreateSnapshot(ctx, sourceID, CreateSnapshotRequest{ + Kind: SnapshotKindStandby, + Name: "standby-for-fork-compressed", + }) + require.NoError(t, err) + + snapshotDir := mgr.paths.SnapshotGuestDir(snap.Id) + rawPath := filepath.Join(snapshotDir, "memory-ranges") + snapshotConfigPath := filepath.Join(snapshotDir, "snapshots", "snapshot-latest", "config.json") + require.NoError(t, os.MkdirAll(filepath.Dir(snapshotConfigPath), 0o755)) + require.NoError(t, os.WriteFile(snapshotConfigPath, []byte(`{}`), 0o644)) + require.NoError(t, os.WriteFile(rawPath, []byte("some guest memory"), 0o644)) + _, _, err = compressSnapshotMemoryFile(ctx, rawPath, snapshotstore.SnapshotCompressionConfig{ + Enabled: true, + Algorithm: snapshotstore.SnapshotCompressionAlgorithmZstd, + Level: intPtr(1), + }) + require.NoError(t, err) + + forked, err := mgr.ForkSnapshot(ctx, snap.Id, ForkSnapshotRequest{ + Name: "snapshot-fork-compressed", + TargetState: StateStopped, + }) + require.NoError(t, err) + t.Cleanup(func() { _ = mgr.DeleteInstance(context.Background(), forked.Id) }) + + forkSnapshotDir := mgr.paths.InstanceDir(forked.Id) + _, ok := findRawSnapshotMemoryFile(forkSnapshotDir) + assert.True(t, ok, "forked snapshot payload should contain raw memory after preparing a compressed snapshot source") + _, _, ok = findCompressedSnapshotMemoryFile(forkSnapshotDir) + assert.False(t, ok, "forked snapshot payload should not retain compressed memory artifacts from the source snapshot") +} + func createStoppedSnapshotSourceFixture(t *testing.T, mgr *manager, id, name string, hvType hypervisor.Type) { t.Helper() require.NoError(t, mgr.ensureDirectories(id)) From 8638cce42f987c6efe174e990553dd23abdf82a6 Mon Sep 17 00:00:00 2001 From: Steven Miller Date: Mon, 23 Mar 2026 13:05:02 -0400 Subject: [PATCH 31/31] Fix standby fork compression race --- cmd/api/config/config.go | 2 +- lib/instances/fork.go | 6 ++++ lib/instances/fork_test.go | 62 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 69 insertions(+), 1 deletion(-) diff --git a/cmd/api/config/config.go b/cmd/api/config/config.go index 9a80d9d7..8508537b 100644 --- a/cmd/api/config/config.go +++ b/cmd/api/config/config.go @@ -9,8 +9,8 @@ import ( "strings" "time" - "github.com/kernel/hypeman/lib/snapshot" "github.com/c2h5oh/datasize" + "github.com/kernel/hypeman/lib/snapshot" "github.com/knadh/koanf/parsers/yaml" "github.com/knadh/koanf/providers/env" "github.com/knadh/koanf/providers/file" diff --git a/lib/instances/fork.go b/lib/instances/fork.go index 203d1a64..531af12c 100644 --- a/lib/instances/fork.go +++ b/lib/instances/fork.go @@ -244,6 +244,12 @@ func (m *manager) forkInstanceFromStoppedOrStandby(ctx context.Context, id strin }) defer cu.Clean() + if source.State == StateStandby { + if err := m.ensureSnapshotMemoryReady(ctx, m.paths.InstanceSnapshotLatest(id), m.snapshotJobKeyForInstance(id), stored.HypervisorType); err != nil { + return nil, fmt.Errorf("prepare standby snapshot for fork: %w", err) + } + } + if err := forkvm.CopyGuestDirectory(srcDir, dstDir); err != nil { if errors.Is(err, forkvm.ErrSparseCopyUnsupported) { return nil, fmt.Errorf("fork requires sparse-capable filesystem (SEEK_DATA/SEEK_HOLE unsupported): %w", err) diff --git a/lib/instances/fork_test.go b/lib/instances/fork_test.go index 88e48537..c6bf85e0 100644 --- a/lib/instances/fork_test.go +++ b/lib/instances/fork_test.go @@ -10,6 +10,7 @@ import ( "os" "path/filepath" "strings" + "sync/atomic" "testing" "time" @@ -17,6 +18,7 @@ import ( "github.com/kernel/hypeman/lib/hypervisor" "github.com/kernel/hypeman/lib/images" "github.com/kernel/hypeman/lib/paths" + snapshotstore "github.com/kernel/hypeman/lib/snapshot" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -215,6 +217,66 @@ func TestForkInstanceRejectsDuplicateNameForNonNetworkedSource(t *testing.T) { assert.Contains(t, err.Error(), "already exists") } +func TestForkInstanceFromStandbyCancelsCompressionJobAndCopiesRawMemory(t *testing.T) { + t.Parallel() + + manager, _ := setupTestManager(t) + ctx := context.Background() + + sourceID := "fork-standby-compressed-src" + createStandbySnapshotSourceFixture(t, manager, sourceID, sourceID, manager.defaultHypervisor) + + rawPath := filepath.Join(manager.paths.InstanceSnapshotLatest(sourceID), "memory-ranges") + require.NoError(t, os.WriteFile(rawPath, []byte("some guest memory"), 0o644)) + snapshotConfigPath := manager.paths.InstanceSnapshotConfig(sourceID) + require.NoError(t, os.MkdirAll(filepath.Dir(snapshotConfigPath), 0o755)) + require.NoError(t, os.WriteFile(snapshotConfigPath, []byte(`{}`), 0o644)) + + _, _, err := compressSnapshotMemoryFile(ctx, rawPath, snapshotstore.SnapshotCompressionConfig{ + Enabled: true, + Algorithm: snapshotstore.SnapshotCompressionAlgorithmZstd, + Level: intPtr(1), + }) + require.NoError(t, err) + + var canceled atomic.Bool + done := make(chan struct{}) + + manager.compressionMu.Lock() + manager.compressionJobs[manager.snapshotJobKeyForInstance(sourceID)] = &compressionJob{ + cancel: func() { + canceled.Store(true) + select { + case <-done: + default: + close(done) + } + }, + done: done, + target: compressionTarget{ + Key: manager.snapshotJobKeyForInstance(sourceID), + OwnerID: sourceID, + SnapshotDir: manager.paths.InstanceSnapshotLatest(sourceID), + }, + } + manager.compressionMu.Unlock() + + forked, err := manager.forkInstanceFromStoppedOrStandby(ctx, sourceID, ForkInstanceRequest{ + Name: "fork-standby-compressed-copy", + TargetState: StateStopped, + }, true) + require.NoError(t, err) + require.NotNil(t, forked) + + assert.True(t, canceled.Load(), "standby compression job should be canceled before copying the source guest directory") + + forkSnapshotDir := manager.paths.InstanceSnapshotLatest(forked.Id) + _, ok := findRawSnapshotMemoryFile(forkSnapshotDir) + assert.True(t, ok, "forked standby guest directory should contain raw memory after preparing the source snapshot") + _, _, ok = findCompressedSnapshotMemoryFile(forkSnapshotDir) + assert.False(t, ok, "forked standby guest directory should not retain compressed memory artifacts from the source instance") +} + func TestCloneStoredMetadataForFork_DeepCopiesReferenceFields(t *testing.T) { t.Parallel() startedAt := time.Now().Add(-2 * time.Minute)