diff --git a/cmd/api/api/api_test.go b/cmd/api/api/api_test.go index a763b630..3c0f5a1b 100644 --- a/cmd/api/api/api_test.go +++ b/cmd/api/api/api_test.go @@ -43,7 +43,7 @@ func newTestService(t *testing.T) *ApiService { limits := instances.ResourceLimits{ MaxOverlaySize: 100 * 1024 * 1024 * 1024, // 100GB } - instanceMgr := instances.NewManager(p, imageMgr, systemMgr, networkMgr, deviceMgr, volumeMgr, limits, "", nil, nil) + instanceMgr := instances.NewManager(p, imageMgr, systemMgr, networkMgr, deviceMgr, volumeMgr, limits, "", instances.SnapshotPolicy{}, nil, nil) // Initialize network manager (creates bridge for network-enabled tests) if err := networkMgr.Initialize(ctx(), nil); err != nil { diff --git a/cmd/api/api/instances.go b/cmd/api/api/instances.go index 399aea20..7323c084 100644 --- a/cmd/api/api/instances.go +++ b/cmd/api/api/instances.go @@ -17,6 +17,7 @@ import ( "github.com/kernel/hypeman/lib/network" "github.com/kernel/hypeman/lib/oapi" "github.com/kernel/hypeman/lib/resources" + "github.com/kernel/hypeman/lib/snapshot" "github.com/kernel/hypeman/lib/vm_metrics" "github.com/samber/lo" ) @@ -301,6 +302,16 @@ func (s *ApiService) CreateInstance(ctx context.Context, request oapi.CreateInst SkipKernelHeaders: request.Body.SkipKernelHeaders != nil && *request.Body.SkipKernelHeaders, SkipGuestAgent: request.Body.SkipGuestAgent != nil && *request.Body.SkipGuestAgent, } + if request.Body.SnapshotPolicy != nil { + snapshotPolicy, err := toInstanceSnapshotPolicy(*request.Body.SnapshotPolicy) + if err != nil { + return oapi.CreateInstance400JSONResponse{ + Code: "invalid_snapshot_policy", + Message: err.Error(), + }, nil + } + domainReq.SnapshotPolicy = snapshotPolicy + } inst, err := s.InstanceManager.CreateInstance(ctx, domainReq) if err != nil { @@ -438,9 +449,26 @@ func (s *ApiService) StandbyInstance(ctx context.Context, request oapi.StandbyIn } log := logger.FromContext(ctx) - result, err := s.InstanceManager.StandbyInstance(ctx, inst.Id) + standbyReq := instances.StandbyInstanceRequest{} + if request.Body != nil && request.Body.Compression != nil { + compression, err := toDomainSnapshotCompressionConfig(*request.Body.Compression) + if err != nil { + return oapi.StandbyInstance400JSONResponse{ + Code: "invalid_snapshot_compression", + Message: err.Error(), + }, nil + } + standbyReq.Compression = compression + } + + result, err := s.InstanceManager.StandbyInstance(ctx, inst.Id, standbyReq) if err != nil { switch { + case errors.Is(err, instances.ErrInvalidRequest): + return oapi.StandbyInstance400JSONResponse{ + Code: "invalid_request", + Message: err.Error(), + }, nil case errors.Is(err, instances.ErrInvalidState): return oapi.StandbyInstance409JSONResponse{ Code: "invalid_state", @@ -951,6 +979,10 @@ func instanceToOAPI(inst instances.Instance) oapi.Instance { if len(inst.Tags) > 0 { oapiInst.Tags = toOAPITags(inst.Tags) } + if inst.SnapshotPolicy != nil { + oapiPolicy := toOAPISnapshotPolicy(*inst.SnapshotPolicy) + oapiInst.SnapshotPolicy = &oapiPolicy + } // Convert volume attachments if len(inst.Volumes) > 0 { @@ -985,3 +1017,73 @@ func instanceToOAPI(inst instances.Instance) oapi.Instance { return oapiInst } + +func toDomainSnapshotCompressionConfig(cfg oapi.SnapshotCompressionConfig) (*snapshot.SnapshotCompressionConfig, error) { + out := &snapshot.SnapshotCompressionConfig{ + Enabled: cfg.Enabled, + } + if cfg.Algorithm != nil { + algo := snapshot.SnapshotCompressionAlgorithm(strings.ToLower(string(*cfg.Algorithm))) + switch algo { + case snapshot.SnapshotCompressionAlgorithmZstd, snapshot.SnapshotCompressionAlgorithmLz4: + default: + return nil, fmt.Errorf("algorithm must be one of zstd or lz4, got %q", *cfg.Algorithm) + } + out.Algorithm = algo + } + if cfg.Level != nil { + level := *cfg.Level + algo := out.Algorithm + if algo == "" { + algo = snapshot.SnapshotCompressionAlgorithmZstd + } + switch algo { + case snapshot.SnapshotCompressionAlgorithmZstd: + if level < snapshot.MinSnapshotCompressionZstdLevel || level > snapshot.MaxSnapshotCompressionZstdLevel { + return nil, fmt.Errorf("level must be between %d and %d for zstd, got %d", snapshot.MinSnapshotCompressionZstdLevel, snapshot.MaxSnapshotCompressionZstdLevel, level) + } + case snapshot.SnapshotCompressionAlgorithmLz4: + if level < snapshot.MinSnapshotCompressionLz4Level || level > snapshot.MaxSnapshotCompressionLz4Level { + return nil, fmt.Errorf("level must be between %d and %d for lz4, got %d", snapshot.MinSnapshotCompressionLz4Level, snapshot.MaxSnapshotCompressionLz4Level, level) + } + } + out.Level = &level + } + return out, nil +} + +func toInstanceSnapshotPolicy(policy oapi.SnapshotPolicy) (*instances.SnapshotPolicy, error) { + out := &instances.SnapshotPolicy{} + if policy.Compression != nil { + compression, err := toDomainSnapshotCompressionConfig(*policy.Compression) + if err != nil { + return nil, err + } + out.Compression = compression + } + return out, nil +} + +func toOAPISnapshotCompressionConfig(cfg snapshot.SnapshotCompressionConfig) oapi.SnapshotCompressionConfig { + out := oapi.SnapshotCompressionConfig{ + Enabled: cfg.Enabled, + } + if cfg.Algorithm != "" { + algo := oapi.SnapshotCompressionConfigAlgorithm(cfg.Algorithm) + out.Algorithm = &algo + } + if cfg.Level != nil { + level := *cfg.Level + out.Level = &level + } + return out +} + +func toOAPISnapshotPolicy(policy instances.SnapshotPolicy) oapi.SnapshotPolicy { + out := oapi.SnapshotPolicy{} + if policy.Compression != nil { + compression := toOAPISnapshotCompressionConfig(*policy.Compression) + out.Compression = &compression + } + return out +} diff --git a/cmd/api/api/instances_test.go b/cmd/api/api/instances_test.go index d71d34cf..7a532936 100644 --- a/cmd/api/api/instances_test.go +++ b/cmd/api/api/instances_test.go @@ -204,6 +204,14 @@ type captureForkManager struct { err error } +type captureStandbyManager struct { + instances.Manager + lastID string + lastReq *instances.StandbyInstanceRequest + result *instances.Instance + err error +} + type captureUpdateManager struct { instances.Manager lastID string @@ -222,6 +230,16 @@ func (m *captureForkManager) ForkInstance(ctx context.Context, id string, req in return m.result, nil } +func (m *captureStandbyManager) StandbyInstance(ctx context.Context, id string, req instances.StandbyInstanceRequest) (*instances.Instance, error) { + reqCopy := req + m.lastID = id + m.lastReq = &reqCopy + if m.err != nil { + return nil, m.err + } + return m.result, nil +} + func (m *captureUpdateManager) UpdateInstance(ctx context.Context, id string, req instances.UpdateInstanceRequest) (*instances.Instance, error) { reqCopy := req m.lastID = id @@ -677,6 +695,46 @@ func TestForkInstance_InvalidRequest(t *testing.T) { assert.Equal(t, "invalid_request", badReq.Code) } +func TestStandbyInstance_InvalidRequest(t *testing.T) { + t.Parallel() + svc := newTestService(t) + + source := instances.Instance{ + StoredMetadata: instances.StoredMetadata{ + Id: "standby-src", + Name: "standby-src", + Image: "docker.io/library/alpine:latest", + CreatedAt: time.Now(), + HypervisorType: hypervisor.TypeCloudHypervisor, + }, + State: instances.StateStopped, + } + + mockMgr := &captureStandbyManager{ + Manager: svc.InstanceManager, + err: fmt.Errorf("%w: invalid snapshot compression level", instances.ErrInvalidRequest), + } + svc.InstanceManager = mockMgr + + resp, err := svc.StandbyInstance( + mw.WithResolvedInstance(ctx(), source.Id, source), + oapi.StandbyInstanceRequestObject{ + Id: source.Id, + Body: &oapi.StandbyInstanceRequest{ + Compression: &oapi.SnapshotCompressionConfig{ + Enabled: true, + }, + }, + }, + ) + require.NoError(t, err) + + badReq, ok := resp.(oapi.StandbyInstance400JSONResponse) + require.True(t, ok, "expected 400 response") + assert.Equal(t, "invalid_request", badReq.Code) + assert.Contains(t, badReq.Message, "invalid snapshot compression level") +} + func TestForkInstance_FromRunningFlagForwarded(t *testing.T) { t.Parallel() svc := newTestService(t) diff --git a/cmd/api/api/optional_standby_body.go b/cmd/api/api/optional_standby_body.go new file mode 100644 index 00000000..7157ad95 --- /dev/null +++ b/cmd/api/api/optional_standby_body.go @@ -0,0 +1,47 @@ +package api + +import ( + "io" + "net/http" + "strings" +) + +// NormalizeOptionalStandbyBody rewrites empty standby POST bodies to "{}" +// so the generated strict handler can decode them without special casing. +func NormalizeOptionalStandbyBody(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + next.ServeHTTP(w, r) + return + } + if isStandbyRoutePath(r.URL.Path) && requestBodyIsEmpty(r) { + r.Body = io.NopCloser(strings.NewReader(`{}`)) + r.ContentLength = 2 + if r.Header.Get("Content-Type") == "" { + r.Header.Set("Content-Type", "application/json") + } + } + + next.ServeHTTP(w, r) + }) +} + +func isStandbyRoutePath(path string) bool { + if !strings.HasPrefix(path, "/instances/") || !strings.HasSuffix(path, "/standby") { + return false + } + + instanceID := strings.TrimPrefix(path, "/instances/") + instanceID = strings.TrimSuffix(instanceID, "/standby") + return instanceID != "" && !strings.Contains(instanceID, "/") +} + +func requestBodyIsEmpty(r *http.Request) bool { + if r == nil { + return true + } + if r.Body == nil || r.Body == http.NoBody { + return true + } + return r.ContentLength == 0 && len(r.TransferEncoding) == 0 +} diff --git a/cmd/api/api/optional_standby_body_test.go b/cmd/api/api/optional_standby_body_test.go new file mode 100644 index 00000000..b8bc31de --- /dev/null +++ b/cmd/api/api/optional_standby_body_test.go @@ -0,0 +1,108 @@ +package api + +import ( + "bytes" + "io" + "net/http" + "net/http/httptest" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestNormalizeOptionalStandbyBody(t *testing.T) { + t.Parallel() + + t.Run("empty standby body becomes empty JSON object", func(t *testing.T) { + t.Parallel() + + var gotBody []byte + var gotContentType string + next := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var err error + gotBody, err = io.ReadAll(r.Body) + require.NoError(t, err) + gotContentType = r.Header.Get("Content-Type") + w.WriteHeader(http.StatusNoContent) + }) + + req := httptest.NewRequest(http.MethodPost, "/instances/test/standby", nil) + rec := httptest.NewRecorder() + + NormalizeOptionalStandbyBody(next).ServeHTTP(rec, req) + + assert.Equal(t, http.StatusNoContent, rec.Code) + assert.Equal(t, []byte(`{}`), gotBody) + assert.Equal(t, "application/json", gotContentType) + }) + + t.Run("existing standby body is preserved", func(t *testing.T) { + t.Parallel() + + var gotBody []byte + next := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var err error + gotBody, err = io.ReadAll(r.Body) + require.NoError(t, err) + w.WriteHeader(http.StatusNoContent) + }) + + req := httptest.NewRequest(http.MethodPost, "/instances/test/standby", bytes.NewBufferString(`{"compression":{"enabled":true}}`)) + rec := httptest.NewRecorder() + + NormalizeOptionalStandbyBody(next).ServeHTTP(rec, req) + + assert.Equal(t, http.StatusNoContent, rec.Code) + assert.Equal(t, []byte(`{"compression":{"enabled":true}}`), gotBody) + }) + + t.Run("non-standby route is untouched", func(t *testing.T) { + t.Parallel() + + var gotBody []byte + next := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var err error + gotBody, err = io.ReadAll(r.Body) + require.NoError(t, err) + w.WriteHeader(http.StatusNoContent) + }) + + req := httptest.NewRequest(http.MethodPost, "/instances/test/start", nil) + rec := httptest.NewRecorder() + + NormalizeOptionalStandbyBody(next).ServeHTTP(rec, req) + + assert.Equal(t, http.StatusNoContent, rec.Code) + assert.Empty(t, gotBody) + }) + + t.Run("non-post request skips standby normalization", func(t *testing.T) { + t.Parallel() + + var gotBody []byte + next := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var err error + gotBody, err = io.ReadAll(r.Body) + require.NoError(t, err) + w.WriteHeader(http.StatusNoContent) + }) + + req := httptest.NewRequest(http.MethodGet, "/instances/test/standby", nil) + rec := httptest.NewRecorder() + + NormalizeOptionalStandbyBody(next).ServeHTTP(rec, req) + + assert.Equal(t, http.StatusNoContent, rec.Code) + assert.Empty(t, gotBody) + }) + + t.Run("standby route matcher only accepts single path segment ids", func(t *testing.T) { + t.Parallel() + + assert.True(t, isStandbyRoutePath("/instances/test/standby")) + assert.False(t, isStandbyRoutePath("/instances/test/start")) + assert.False(t, isStandbyRoutePath("/instances/test/standby/extra")) + assert.False(t, isStandbyRoutePath("/instances/test/nested/standby")) + }) +} diff --git a/cmd/api/api/snapshots.go b/cmd/api/api/snapshots.go index 2609c8be..da5cd9e0 100644 --- a/cmd/api/api/snapshots.go +++ b/cmd/api/api/snapshots.go @@ -10,6 +10,7 @@ import ( mw "github.com/kernel/hypeman/lib/middleware" "github.com/kernel/hypeman/lib/network" "github.com/kernel/hypeman/lib/oapi" + "github.com/kernel/hypeman/lib/snapshot" "github.com/samber/lo" ) @@ -27,11 +28,20 @@ func (s *ApiService) CreateInstanceSnapshot(ctx context.Context, request oapi.Cr if request.Body.Name != nil { name = *request.Body.Name } + var compression *snapshot.SnapshotCompressionConfig + if request.Body.Compression != nil { + var err error + compression, err = toDomainSnapshotCompressionConfig(*request.Body.Compression) + if err != nil { + return oapi.CreateInstanceSnapshot400JSONResponse{Code: "invalid_snapshot_compression", Message: err.Error()}, nil + } + } result, err := s.InstanceManager.CreateSnapshot(ctx, inst.Id, instances.CreateSnapshotRequest{ - Kind: instances.SnapshotKind(request.Body.Kind), - Name: name, - Tags: toMapTags(request.Body.Tags), + Kind: instances.SnapshotKind(request.Body.Kind), + Name: name, + Tags: toMapTags(request.Body.Tags), + Compression: compression, }) if err != nil { log := logger.FromContext(ctx) @@ -207,6 +217,23 @@ func snapshotToOAPI(snapshot instances.Snapshot) oapi.Snapshot { SizeBytes: snapshot.SizeBytes, Name: lo.ToPtr(snapshot.Name), } + if snapshot.CompressionState != "" { + state := oapi.SnapshotCompressionState(snapshot.CompressionState) + out.CompressionState = &state + } + if snapshot.CompressionError != "" { + out.CompressionError = lo.ToPtr(snapshot.CompressionError) + } + if snapshot.Compression != nil { + compression := toOAPISnapshotCompressionConfig(*snapshot.Compression) + out.Compression = &compression + } + if snapshot.CompressedSizeBytes != nil { + out.CompressedSizeBytes = snapshot.CompressedSizeBytes + } + if snapshot.UncompressedSizeBytes != nil { + out.UncompressedSizeBytes = snapshot.UncompressedSizeBytes + } if snapshot.Name == "" { out.Name = nil } diff --git a/cmd/api/config/config.go b/cmd/api/config/config.go index 2179d97e..8508537b 100644 --- a/cmd/api/config/config.go +++ b/cmd/api/config/config.go @@ -10,6 +10,7 @@ import ( "time" "github.com/c2h5oh/datasize" + "github.com/kernel/hypeman/lib/snapshot" "github.com/knadh/koanf/parsers/yaml" "github.com/knadh/koanf/providers/env" "github.com/knadh/koanf/providers/file" @@ -191,6 +192,18 @@ type HypervisorActiveBallooningConfig struct { PerVmCooldown string `koanf:"per_vm_cooldown"` } +// SnapshotCompressionDefaultConfig holds default snapshot compression settings. +type SnapshotCompressionDefaultConfig struct { + Enabled bool `koanf:"enabled"` + Algorithm string `koanf:"algorithm"` + Level *int `koanf:"level"` +} + +// SnapshotConfig holds snapshot defaults. +type SnapshotConfig struct { + CompressionDefault SnapshotCompressionDefaultConfig `koanf:"compression_default"` +} + // GPUConfig holds GPU-related settings. type GPUConfig struct { ProfileCacheTTL string `koanf:"profile_cache_ttl"` @@ -217,6 +230,7 @@ type Config struct { Oversubscription OversubscriptionConfig `koanf:"oversubscription"` Capacity CapacityConfig `koanf:"capacity"` Hypervisor HypervisorConfig `koanf:"hypervisor"` + Snapshot SnapshotConfig `koanf:"snapshot"` GPU GPUConfig `koanf:"gpu"` } @@ -360,6 +374,14 @@ func defaultConfig() *Config { }, }, + Snapshot: SnapshotConfig{ + CompressionDefault: SnapshotCompressionDefaultConfig{ + Enabled: false, + Algorithm: "zstd", + Level: intPtr(snapshot.DefaultSnapshotCompressionZstdLevel), + }, + }, + GPU: GPUConfig{ ProfileCacheTTL: "30m", }, @@ -472,6 +494,29 @@ func (c *Config) Validate() error { if c.Build.Timeout <= 0 { return fmt.Errorf("build.timeout must be positive, got %d", c.Build.Timeout) } + algorithm := strings.ToLower(c.Snapshot.CompressionDefault.Algorithm) + c.Snapshot.CompressionDefault.Algorithm = algorithm + if c.Snapshot.CompressionDefault.Enabled { + switch algorithm { + case "", "zstd", "lz4": + default: + return fmt.Errorf("snapshot.compression_default.algorithm must be one of zstd or lz4, got %q", algorithm) + } + if c.Snapshot.CompressionDefault.Level != nil { + level := *c.Snapshot.CompressionDefault.Level + switch algorithm { + case "", "zstd": + if level < snapshot.MinSnapshotCompressionZstdLevel || level > snapshot.MaxSnapshotCompressionZstdLevel { + return fmt.Errorf("snapshot.compression_default.level must be between %d and %d for zstd, got %d", snapshot.MinSnapshotCompressionZstdLevel, snapshot.MaxSnapshotCompressionZstdLevel, level) + } + case "lz4": + if level < snapshot.MinSnapshotCompressionLz4Level || level > snapshot.MaxSnapshotCompressionLz4Level { + return fmt.Errorf("snapshot.compression_default.level must be between %d and %d for lz4, got %d", snapshot.MinSnapshotCompressionLz4Level, snapshot.MaxSnapshotCompressionLz4Level, level) + } + } + } + c.Snapshot.CompressionDefault.Algorithm = algorithm + } if c.Hypervisor.Memory.KernelPageInitMode != "performance" && c.Hypervisor.Memory.KernelPageInitMode != "hardened" { return fmt.Errorf("hypervisor.memory.kernel_page_init_mode must be one of {performance,hardened}, got %q", c.Hypervisor.Memory.KernelPageInitMode) } @@ -526,3 +571,7 @@ func validateDuration(field string, value string) error { } return nil } + +func intPtr(v int) *int { + return &v +} diff --git a/cmd/api/config/config_test.go b/cmd/api/config/config_test.go index 0429ac49..6fd9b594 100644 --- a/cmd/api/config/config_test.go +++ b/cmd/api/config/config_test.go @@ -130,3 +130,51 @@ func TestDefaultConfigActiveBallooningMatchesGoDefaults(t *testing.T) { t.Fatalf("per-vm max step default mismatch: got %d want %d", got, want.PerVMMaxStepBytes) } } + +func TestValidateAllowsLZ4CompressionDefaultWithImplicitLevel(t *testing.T) { + cfg := defaultConfig() + cfg.Snapshot.CompressionDefault.Enabled = true + cfg.Snapshot.CompressionDefault.Algorithm = "LZ4" + cfg.Snapshot.CompressionDefault.Level = nil + + if err := cfg.Validate(); err != nil { + t.Fatalf("expected lz4 compression default to validate, got %v", err) + } + if cfg.Snapshot.CompressionDefault.Algorithm != "lz4" { + t.Fatalf("expected algorithm to normalize to lowercase, got %q", cfg.Snapshot.CompressionDefault.Algorithm) + } +} + +func TestValidateAllowsExplicitLZ4CompressionLevelRange(t *testing.T) { + cfg := defaultConfig() + cfg.Snapshot.CompressionDefault.Enabled = true + cfg.Snapshot.CompressionDefault.Algorithm = "lz4" + cfg.Snapshot.CompressionDefault.Level = intPtr(9) + + if err := cfg.Validate(); err != nil { + t.Fatalf("expected lz4 level to validate, got %v", err) + } +} + +func TestValidateRejectsInvalidLZ4CompressionLevel(t *testing.T) { + cfg := defaultConfig() + cfg.Snapshot.CompressionDefault.Enabled = true + cfg.Snapshot.CompressionDefault.Algorithm = "lz4" + cfg.Snapshot.CompressionDefault.Level = intPtr(10) + + err := cfg.Validate() + if err == nil { + t.Fatalf("expected validation error for invalid lz4 level") + } +} + +func TestValidateAllowsDisabledSnapshotCompressionDefaultWithoutValidAlgorithm(t *testing.T) { + cfg := defaultConfig() + cfg.Snapshot.CompressionDefault.Enabled = false + cfg.Snapshot.CompressionDefault.Algorithm = "definitely-not-real" + cfg.Snapshot.CompressionDefault.Level = intPtr(999) + + if err := cfg.Validate(); err != nil { + t.Fatalf("expected disabled snapshot compression default to ignore algorithm/level, got %v", err) + } +} diff --git a/cmd/api/main.go b/cmd/api/main.go index 8057a36f..bfdeaf8e 100644 --- a/cmd/api/main.go +++ b/cmd/api/main.go @@ -382,7 +382,7 @@ func run() error { // Mount API routes (authentication now handled by validation middleware) oapi.HandlerWithOptions(strictHandler, oapi.ChiServerOptions{ BaseRouter: r, - Middlewares: []oapi.MiddlewareFunc{}, + Middlewares: []oapi.MiddlewareFunc{api.NormalizeOptionalStandbyBody}, }) }) diff --git a/integration/systemd_test.go b/integration/systemd_test.go index ad60510d..4c552db0 100644 --- a/integration/systemd_test.go +++ b/integration/systemd_test.go @@ -68,7 +68,7 @@ func TestSystemdMode(t *testing.T) { MaxMemoryPerInstance: 0, } - instanceManager := instances.NewManager(p, imageManager, systemManager, networkManager, deviceManager, volumeManager, limits, "", nil, nil) + instanceManager := instances.NewManager(p, imageManager, systemManager, networkManager, deviceManager, volumeManager, limits, "", instances.SnapshotPolicy{}, nil, nil) // Cleanup any orphaned instances t.Cleanup(func() { diff --git a/integration/vgpu_test.go b/integration/vgpu_test.go index cbf59d09..12861a6a 100644 --- a/integration/vgpu_test.go +++ b/integration/vgpu_test.go @@ -77,7 +77,7 @@ func TestVGPU(t *testing.T) { MaxMemoryPerInstance: 0, } - instanceManager := instances.NewManager(p, imageManager, systemManager, networkManager, deviceManager, volumeManager, limits, "", nil, nil) + instanceManager := instances.NewManager(p, imageManager, systemManager, networkManager, deviceManager, volumeManager, limits, "", instances.SnapshotPolicy{}, nil, nil) // Track instance ID for cleanup var instanceID string diff --git a/lib/builds/manager_test.go b/lib/builds/manager_test.go index 5e4d16f5..6809e227 100644 --- a/lib/builds/manager_test.go +++ b/lib/builds/manager_test.go @@ -108,7 +108,7 @@ func (m *mockInstanceManager) ForkSnapshot(ctx context.Context, snapshotID strin return nil, instances.ErrNotFound } -func (m *mockInstanceManager) StandbyInstance(ctx context.Context, id string) (*instances.Instance, error) { +func (m *mockInstanceManager) StandbyInstance(ctx context.Context, id string, req instances.StandbyInstanceRequest) (*instances.Instance, error) { return nil, nil } diff --git a/lib/devices/gpu_e2e_test.go b/lib/devices/gpu_e2e_test.go index 86708446..a7fe6390 100644 --- a/lib/devices/gpu_e2e_test.go +++ b/lib/devices/gpu_e2e_test.go @@ -79,7 +79,7 @@ func TestGPUPassthrough(t *testing.T) { limits := instances.ResourceLimits{ MaxOverlaySize: 100 * 1024 * 1024 * 1024, // 100GB } - instanceMgr := instances.NewManager(p, imageMgr, systemMgr, networkMgr, deviceMgr, volumeMgr, limits, "", nil, nil) + instanceMgr := instances.NewManager(p, imageMgr, systemMgr, networkMgr, deviceMgr, volumeMgr, limits, "", instances.SnapshotPolicy{}, nil, nil) // Step 1: Discover available GPUs t.Log("Step 1: Discovering available GPUs...") diff --git a/lib/devices/gpu_inference_test.go b/lib/devices/gpu_inference_test.go index c99d1b39..f521158e 100644 --- a/lib/devices/gpu_inference_test.go +++ b/lib/devices/gpu_inference_test.go @@ -116,7 +116,7 @@ func TestGPUInference(t *testing.T) { limits := instances.ResourceLimits{ MaxOverlaySize: 100 * 1024 * 1024 * 1024, } - instanceMgr := instances.NewManager(p, imageMgr, systemMgr, networkMgr, deviceMgr, volumeMgr, limits, "", nil, nil) + instanceMgr := instances.NewManager(p, imageMgr, systemMgr, networkMgr, deviceMgr, volumeMgr, limits, "", instances.SnapshotPolicy{}, nil, nil) // Step 1: Build custom CUDA+Ollama image t.Log("Step 1: Building custom CUDA+Ollama Docker image...") diff --git a/lib/devices/gpu_module_test.go b/lib/devices/gpu_module_test.go index cd9f1b76..f39d4690 100644 --- a/lib/devices/gpu_module_test.go +++ b/lib/devices/gpu_module_test.go @@ -80,7 +80,7 @@ func TestNVIDIAModuleLoading(t *testing.T) { deviceMgr := devices.NewManager(p) volumeMgr := volumes.NewManager(p, 10*1024*1024*1024, nil) limits := instances.ResourceLimits{MaxOverlaySize: 10 * 1024 * 1024 * 1024} - instanceMgr := instances.NewManager(p, imageMgr, systemMgr, networkMgr, deviceMgr, volumeMgr, limits, "", nil, nil) + instanceMgr := instances.NewManager(p, imageMgr, systemMgr, networkMgr, deviceMgr, volumeMgr, limits, "", instances.SnapshotPolicy{}, nil, nil) // Step 1: Find an NVIDIA GPU t.Log("Step 1: Discovering available GPUs...") @@ -326,7 +326,7 @@ func TestNVMLDetection(t *testing.T) { deviceMgr := devices.NewManager(p) volumeMgr := volumes.NewManager(p, 10*1024*1024*1024, nil) limits := instances.ResourceLimits{MaxOverlaySize: 10 * 1024 * 1024 * 1024} - instanceMgr := instances.NewManager(p, imageMgr, systemMgr, networkMgr, deviceMgr, volumeMgr, limits, "", nil, nil) + instanceMgr := instances.NewManager(p, imageMgr, systemMgr, networkMgr, deviceMgr, volumeMgr, limits, "", instances.SnapshotPolicy{}, nil, nil) // Step 1: Check if ollama-cuda:test image exists in Docker t.Log("Step 1: Checking for ollama-cuda:test Docker image...") diff --git a/lib/forkvm/copy.go b/lib/forkvm/copy.go index 2076c0ce..6dc6eecc 100644 --- a/lib/forkvm/copy.go +++ b/lib/forkvm/copy.go @@ -6,6 +6,7 @@ import ( "io/fs" "os" "path/filepath" + "strings" ) var ErrSparseCopyUnsupported = errors.New("sparse copy unsupported") @@ -41,6 +42,9 @@ func CopyGuestDirectory(srcDir, dstDir string) error { if d.IsDir() && shouldSkipDirectory(relPath) { return filepath.SkipDir } + if !d.IsDir() && shouldSkipRegularFile(relPath) { + return nil + } dstPath := filepath.Join(dstDir, relPath) info, err := d.Info() @@ -85,3 +89,7 @@ func CopyGuestDirectory(srcDir, dstDir string) error { func shouldSkipDirectory(relPath string) bool { return relPath == "logs" } + +func shouldSkipRegularFile(relPath string) bool { + return strings.HasSuffix(relPath, ".lz4.tmp") || strings.HasSuffix(relPath, ".zst.tmp") +} diff --git a/lib/forkvm/copy_test.go b/lib/forkvm/copy_test.go index 762499e4..c71f6c4e 100644 --- a/lib/forkvm/copy_test.go +++ b/lib/forkvm/copy_test.go @@ -20,6 +20,8 @@ func TestCopyGuestDirectory(t *testing.T) { require.NoError(t, os.WriteFile(filepath.Join(src, "overlay.raw"), []byte("overlay"), 0644)) require.NoError(t, os.WriteFile(filepath.Join(src, "logs", "app.log"), []byte("hello"), 0644)) require.NoError(t, os.WriteFile(filepath.Join(src, "snapshots", "snapshot-latest", "config.json"), []byte(`{}`), 0644)) + require.NoError(t, os.WriteFile(filepath.Join(src, "snapshots", "snapshot-latest", "memory-ranges.lz4.tmp"), []byte("partial"), 0644)) + require.NoError(t, os.WriteFile(filepath.Join(src, "snapshots", "snapshot-latest", "memory-ranges.zst.tmp"), []byte("partial"), 0644)) require.NoError(t, os.Symlink("metadata.json", filepath.Join(src, "meta-link"))) require.NoError(t, CopyGuestDirectory(src, dst)) @@ -28,6 +30,8 @@ func TestCopyGuestDirectory(t *testing.T) { assert.FileExists(t, filepath.Join(dst, "config.ext4")) assert.FileExists(t, filepath.Join(dst, "overlay.raw")) assert.FileExists(t, filepath.Join(dst, "snapshots", "snapshot-latest", "config.json")) + assert.NoFileExists(t, filepath.Join(dst, "snapshots", "snapshot-latest", "memory-ranges.lz4.tmp")) + assert.NoFileExists(t, filepath.Join(dst, "snapshots", "snapshot-latest", "memory-ranges.zst.tmp")) assert.NoFileExists(t, filepath.Join(dst, "logs", "app.log")) assert.FileExists(t, filepath.Join(dst, "meta-link")) @@ -39,3 +43,17 @@ func TestCopyGuestDirectory(t *testing.T) { require.NoError(t, err) assert.Equal(t, "metadata.json", linkTarget) } + +func TestCopyGuestDirectory_DoesNotSkipTmpSuffixedDirectories(t *testing.T) { + src := filepath.Join(t.TempDir(), "src") + dst := filepath.Join(t.TempDir(), "dst") + + tmpDir := filepath.Join(src, "snapshots", "snapshot-latest", "memory-ranges.lz4.tmp") + require.NoError(t, os.MkdirAll(tmpDir, 0755)) + require.NoError(t, os.WriteFile(filepath.Join(tmpDir, "nested.txt"), []byte("nested"), 0644)) + + require.NoError(t, CopyGuestDirectory(src, dst)) + + assert.DirExists(t, filepath.Join(dst, "snapshots", "snapshot-latest", "memory-ranges.lz4.tmp")) + assert.FileExists(t, filepath.Join(dst, "snapshots", "snapshot-latest", "memory-ranges.lz4.tmp", "nested.txt")) +} diff --git a/lib/instances/compression_integration_linux_test.go b/lib/instances/compression_integration_linux_test.go new file mode 100644 index 00000000..c02be929 --- /dev/null +++ b/lib/instances/compression_integration_linux_test.go @@ -0,0 +1,311 @@ +//go:build linux + +package instances + +import ( + "context" + "fmt" + "os" + "testing" + "time" + + "github.com/kernel/hypeman/cmd/api/config" + "github.com/kernel/hypeman/lib/devices" + "github.com/kernel/hypeman/lib/hypervisor" + "github.com/kernel/hypeman/lib/images" + "github.com/kernel/hypeman/lib/network" + "github.com/kernel/hypeman/lib/paths" + "github.com/kernel/hypeman/lib/resources" + snapshotstore "github.com/kernel/hypeman/lib/snapshot" + "github.com/kernel/hypeman/lib/system" + "github.com/kernel/hypeman/lib/volumes" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +type compressionIntegrationHarness struct { + name string + hypervisor hypervisor.Type + setup func(t *testing.T) (*manager, string) + requirePrereqs func(t *testing.T) + waitHypervisorUp func(ctx context.Context, inst *Instance) error +} + +func TestCloudHypervisorStandbyRestoreCompressionScenarios(t *testing.T) { + t.Parallel() + + runStandbyRestoreCompressionScenarios(t, compressionIntegrationHarness{ + name: "cloud-hypervisor", + hypervisor: hypervisor.TypeCloudHypervisor, + setup: func(t *testing.T) (*manager, string) { + return setupCompressionTestManagerForHypervisor(t, hypervisor.TypeCloudHypervisor) + }, + requirePrereqs: requireKVMAccess, + waitHypervisorUp: func(ctx context.Context, inst *Instance) error { + return waitForVMReady(ctx, inst.SocketPath, 10*time.Second) + }, + }) +} + +func TestFirecrackerStandbyRestoreCompressionScenarios(t *testing.T) { + t.Parallel() + + runStandbyRestoreCompressionScenarios(t, compressionIntegrationHarness{ + name: "firecracker", + hypervisor: hypervisor.TypeFirecracker, + setup: func(t *testing.T) (*manager, string) { + return setupCompressionTestManagerForHypervisor(t, hypervisor.TypeFirecracker) + }, + requirePrereqs: requireFirecrackerIntegrationPrereqs, + }) +} + +func TestQEMUStandbyRestoreCompressionScenarios(t *testing.T) { + t.Parallel() + + runStandbyRestoreCompressionScenarios(t, compressionIntegrationHarness{ + name: "qemu", + hypervisor: hypervisor.TypeQEMU, + setup: func(t *testing.T) (*manager, string) { + return setupCompressionTestManagerForHypervisor(t, hypervisor.TypeQEMU) + }, + requirePrereqs: requireQEMUUsable, + waitHypervisorUp: func(ctx context.Context, inst *Instance) error { + return waitForQEMUReady(ctx, inst.SocketPath, 10*time.Second) + }, + }) +} + +func setupCompressionTestManagerForHypervisor(t *testing.T, hvType hypervisor.Type) (*manager, string) { + t.Helper() + + tmpDir, err := os.MkdirTemp("", "hmcmp-") + require.NoError(t, err) + t.Cleanup(func() { + _ = os.RemoveAll(tmpDir) + }) + prepareIntegrationTestDataDir(t, tmpDir) + + cfg := &config.Config{ + DataDir: tmpDir, + Network: legacyParallelTestNetworkConfig(testNetworkSeq.Add(1)), + } + + p := paths.New(tmpDir) + imageManager, err := images.NewManager(p, 1, nil) + require.NoError(t, err) + + systemManager := system.NewManager(p) + networkManager := network.NewManager(p, cfg, nil) + deviceManager := devices.NewManager(p) + volumeManager := volumes.NewManager(p, 0, nil) + limits := ResourceLimits{ + MaxOverlaySize: 100 * 1024 * 1024 * 1024, + MaxVcpusPerInstance: 0, + MaxMemoryPerInstance: 0, + } + mgr := NewManager(p, imageManager, systemManager, networkManager, deviceManager, volumeManager, limits, hvType, SnapshotPolicy{}, nil, nil).(*manager) + + resourceMgr := resources.NewManager(cfg, p) + resourceMgr.SetInstanceLister(mgr) + resourceMgr.SetImageLister(imageManager) + resourceMgr.SetVolumeLister(volumeManager) + require.NoError(t, resourceMgr.Initialize(context.Background())) + mgr.SetResourceValidator(resourceMgr) + + t.Cleanup(func() { + cleanupOrphanedProcesses(t, mgr) + }) + + return mgr, tmpDir +} + +func runStandbyRestoreCompressionScenarios(t *testing.T, harness compressionIntegrationHarness) { + t.Helper() + harness.requirePrereqs(t) + + mgr, tmpDir := harness.setup(t) + ctx := context.Background() + p := paths.New(tmpDir) + + imageManager, err := images.NewManager(p, 1, nil) + require.NoError(t, err) + createNginxImageAndWait(t, ctx, imageManager) + + systemManager := system.NewManager(p) + require.NoError(t, systemManager.EnsureSystemFiles(ctx)) + + inst, err := mgr.CreateInstance(ctx, CreateInstanceRequest{ + Name: fmt.Sprintf("compression-%s", harness.name), + Image: integrationTestImageRef(t, "docker.io/library/nginx:alpine"), + Size: 1024 * 1024 * 1024, + HotplugSize: 512 * 1024 * 1024, + OverlaySize: 10 * 1024 * 1024 * 1024, + Vcpus: 1, + NetworkEnabled: false, + Hypervisor: harness.hypervisor, + }) + require.NoError(t, err) + + deleted := false + t.Cleanup(func() { + if !deleted { + _ = mgr.DeleteInstance(context.Background(), inst.Id) + } + }) + + inst = waitForRunningAndExecReady(t, ctx, mgr, inst.Id, harness.waitHypervisorUp) + + inFlightCompression := &snapshotstore.SnapshotCompressionConfig{ + Enabled: true, + Algorithm: snapshotstore.SnapshotCompressionAlgorithmZstd, + Level: intPtr(3), + } + inst = runCompressionCycle(t, ctx, mgr, p, inst, harness.waitHypervisorUp, "in-flight-zstd-3", inFlightCompression, false) + + completedCases := []struct { + name string + cfg *snapshotstore.SnapshotCompressionConfig + }{ + { + name: "zstd-1", + cfg: &snapshotstore.SnapshotCompressionConfig{ + Enabled: true, + Algorithm: snapshotstore.SnapshotCompressionAlgorithmZstd, + Level: intPtr(1), + }, + }, + { + name: "lz4-0", + cfg: &snapshotstore.SnapshotCompressionConfig{ + Enabled: true, + Algorithm: snapshotstore.SnapshotCompressionAlgorithmLz4, + Level: intPtr(0), + }, + }, + } + + for _, tc := range completedCases { + inst = runCompressionCycle(t, ctx, mgr, p, inst, harness.waitHypervisorUp, tc.name, tc.cfg, true) + } + + require.NoError(t, mgr.DeleteInstance(ctx, inst.Id)) + deleted = true +} + +func runCompressionCycle( + t *testing.T, + ctx context.Context, + mgr *manager, + p *paths.Paths, + inst *Instance, + waitHypervisorUp func(ctx context.Context, inst *Instance) error, + label string, + cfg *snapshotstore.SnapshotCompressionConfig, + waitForCompression bool, +) *Instance { + t.Helper() + + markerPath := fmt.Sprintf("/tmp/%s.txt", label) + markerValue := fmt.Sprintf("%s-%d", label, time.Now().UnixNano()) + writeGuestMarker(t, ctx, inst, markerPath, markerValue) + + inst, err := mgr.StandbyInstance(ctx, inst.Id, StandbyInstanceRequest{ + Compression: cloneCompressionConfig(cfg), + }) + require.NoError(t, err) + require.Equal(t, StateStandby, inst.State) + require.True(t, inst.HasSnapshot) + + snapshotDir := p.InstanceSnapshotLatest(inst.Id) + jobKey := mgr.snapshotJobKeyForInstance(inst.Id) + + if waitForCompression { + waitForCompressionJobCompletion(t, mgr, jobKey, 3*time.Minute) + requireCompressedSnapshotFile(t, snapshotDir, effectiveCompressionForCycle(cfg)) + } else { + waitForCompressionJobStart(t, mgr, jobKey, 15*time.Second) + } + + inst, err = mgr.RestoreInstance(ctx, inst.Id) + require.NoError(t, err) + inst = waitForRunningAndExecReady(t, ctx, mgr, inst.Id, waitHypervisorUp) + assertGuestMarker(t, ctx, inst, markerPath, markerValue) + + waitForCompressionJobCompletion(t, mgr, jobKey, 30*time.Second) + return inst +} + +func effectiveCompressionForCycle(cfg *snapshotstore.SnapshotCompressionConfig) snapshotstore.SnapshotCompressionConfig { + if cfg != nil { + normalized, err := normalizeCompressionConfig(cfg) + if err != nil { + panic(err) + } + return normalized + } + return snapshotstore.SnapshotCompressionConfig{Enabled: false} +} + +func waitForRunningAndExecReady(t *testing.T, ctx context.Context, mgr *manager, instanceID string, waitHypervisorUp func(context.Context, *Instance) error) *Instance { + t.Helper() + + inst, err := waitForInstanceState(ctx, mgr, instanceID, StateRunning, 30*time.Second) + require.NoError(t, err) + if waitHypervisorUp != nil { + require.NoError(t, waitHypervisorUp(ctx, inst)) + } + require.NoError(t, waitForExecAgent(ctx, mgr, instanceID, 30*time.Second)) + return inst +} + +func writeGuestMarker(t *testing.T, ctx context.Context, inst *Instance, path string, value string) { + t.Helper() + output, exitCode, err := execCommand(ctx, inst, "sh", "-c", fmt.Sprintf("printf %q > %s && sync", value, path)) + require.NoError(t, err) + require.Equal(t, 0, exitCode, output) +} + +func assertGuestMarker(t *testing.T, ctx context.Context, inst *Instance, path string, expected string) { + t.Helper() + output, exitCode, err := execCommand(ctx, inst, "cat", path) + require.NoError(t, err) + require.Equal(t, 0, exitCode, output) + assert.Equal(t, expected, output) +} + +func waitForCompressionJobStart(t *testing.T, mgr *manager, key string, timeout time.Duration) { + t.Helper() + deadline := time.Now().Add(timeout) + for time.Now().Before(deadline) { + mgr.compressionMu.Lock() + job := mgr.compressionJobs[key] + mgr.compressionMu.Unlock() + if job != nil { + return + } + time.Sleep(100 * time.Millisecond) + } + t.Fatalf("compression job %q did not start within %v", key, timeout) +} + +func waitForCompressionJobCompletion(t *testing.T, mgr *manager, key string, timeout time.Duration) { + t.Helper() + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + require.NoError(t, mgr.waitCompressionJobContext(ctx, key)) +} + +func requireCompressedSnapshotFile(t *testing.T, snapshotDir string, cfg snapshotstore.SnapshotCompressionConfig) { + t.Helper() + require.True(t, cfg.Enabled) + + rawPath, rawExists := findRawSnapshotMemoryFile(snapshotDir) + assert.False(t, rawExists, "raw snapshot memory should be removed after compression, found %q", rawPath) + + compressedPath, algorithm, ok := findCompressedSnapshotMemoryFile(snapshotDir) + require.True(t, ok, "compressed snapshot memory should exist in %s", snapshotDir) + assert.Equal(t, cfg.Algorithm, algorithm) + _, err := os.Stat(compressedPath) + require.NoError(t, err) +} diff --git a/lib/instances/create.go b/lib/instances/create.go index 9c9e85d3..e54a8898 100644 --- a/lib/instances/create.go +++ b/lib/instances/create.go @@ -342,6 +342,7 @@ func (m *manager) createInstance( Cmd: req.Cmd, SkipKernelHeaders: req.SkipKernelHeaders, SkipGuestAgent: req.SkipGuestAgent, + SnapshotPolicy: cloneSnapshotPolicy(req.SnapshotPolicy), } // 12. Ensure directories @@ -541,6 +542,11 @@ func validateCreateRequest(req *CreateInstanceRequest) error { if err := tags.Validate(req.Tags); err != nil { return fmt.Errorf("%w: %v", ErrInvalidRequest, err) } + if req.SnapshotPolicy != nil && req.SnapshotPolicy.Compression != nil { + if _, err := normalizeCompressionConfig(req.SnapshotPolicy.Compression); err != nil { + return err + } + } // Validate volume attachments if err := validateVolumeAttachments(req.Volumes); err != nil { diff --git a/lib/instances/delete.go b/lib/instances/delete.go index 21fbfaab..974224e8 100644 --- a/lib/instances/delete.go +++ b/lib/instances/delete.go @@ -35,6 +35,14 @@ func (m *manager) deleteInstance( stored := &meta.StoredMetadata log.DebugContext(ctx, "loaded instance", "instance_id", id, "state", inst.State) + target, err := m.cancelAndWaitCompressionJob(ctx, m.snapshotJobKeyForInstance(id)) + if err != nil { + return fmt.Errorf("wait for instance compression to stop: %w", err) + } + if target != nil { + m.recordSnapshotCompressionPreemption(ctx, snapshotCompressionPreemptionDeleteInstance, *target) + } + // 2. Get network allocation BEFORE killing VMM (while we can still query it) var networkAlloc *network.Allocation if inst.NetworkEnabled { diff --git a/lib/instances/firecracker_test.go b/lib/instances/firecracker_test.go index ebec9e68..0e3e8dc9 100644 --- a/lib/instances/firecracker_test.go +++ b/lib/instances/firecracker_test.go @@ -49,7 +49,7 @@ func setupTestManagerForFirecrackerWithNetworkConfig(t *testing.T, networkCfg co MaxVcpusPerInstance: 0, MaxMemoryPerInstance: 0, } - mgr := NewManager(p, imageManager, systemManager, networkManager, deviceManager, volumeManager, limits, hypervisor.TypeFirecracker, nil, nil).(*manager) + mgr := NewManager(p, imageManager, systemManager, networkManager, deviceManager, volumeManager, limits, hypervisor.TypeFirecracker, SnapshotPolicy{}, nil, nil).(*manager) resourceMgr := resources.NewManager(cfg, p) resourceMgr.SetInstanceLister(mgr) @@ -212,7 +212,7 @@ func TestFirecrackerStandbyAndRestore(t *testing.T) { writeGuestFile(firstFilePath, firstFileContents) firstStandbyStart := time.Now() - inst, err = mgr.StandbyInstance(ctx, inst.Id) + inst, err = mgr.StandbyInstance(ctx, inst.Id, StandbyInstanceRequest{}) require.NoError(t, err) firstStandbyDuration := time.Since(firstStandbyStart) t.Logf("first standby (full snapshot expected) took %v", firstStandbyDuration) @@ -231,7 +231,7 @@ func TestFirecrackerStandbyAndRestore(t *testing.T) { require.NoError(t, err, "restored instances should keep the retained snapshot base for the next diff snapshot") secondStandbyStart := time.Now() - inst, err = mgr.StandbyInstance(ctx, inst.Id) + inst, err = mgr.StandbyInstance(ctx, inst.Id, StandbyInstanceRequest{}) require.NoError(t, err) secondStandbyDuration := time.Since(secondStandbyStart) t.Logf("second standby (diff snapshot expected) took %v", secondStandbyDuration) @@ -281,7 +281,7 @@ func TestFirecrackerStopClearsStaleSnapshot(t *testing.T) { require.Equal(t, StateRunning, inst.State) // Establish a realistic standby/restore lifecycle first. - inst, err = mgr.StandbyInstance(ctx, inst.Id) + inst, err = mgr.StandbyInstance(ctx, inst.Id, StandbyInstanceRequest{}) require.NoError(t, err) require.Equal(t, StateStandby, inst.State) require.True(t, inst.HasSnapshot) @@ -397,7 +397,7 @@ func TestFirecrackerNetworkLifecycle(t *testing.T) { require.Equal(t, 0, exitCode) require.Contains(t, output, "Connection successful") - inst, err = mgr.StandbyInstance(ctx, inst.Id) + inst, err = mgr.StandbyInstance(ctx, inst.Id, StandbyInstanceRequest{}) require.NoError(t, err) assert.Equal(t, StateStandby, inst.State) assert.True(t, inst.HasSnapshot) diff --git a/lib/instances/fork.go b/lib/instances/fork.go index d68b8c22..531af12c 100644 --- a/lib/instances/fork.go +++ b/lib/instances/fork.go @@ -56,7 +56,7 @@ func (m *manager) forkInstance(ctx context.Context, id string, req ForkInstanceR log.InfoContext(ctx, "fork from running requested; transitioning source to standby", "source_instance_id", id, "hypervisor", source.HypervisorType) - if _, err := m.standbyInstance(ctx, id); err != nil { + if _, err := m.standbyInstance(ctx, id, StandbyInstanceRequest{}, true); err != nil { return nil, "", fmt.Errorf("standby source instance: %w", err) } @@ -244,6 +244,12 @@ func (m *manager) forkInstanceFromStoppedOrStandby(ctx context.Context, id strin }) defer cu.Clean() + if source.State == StateStandby { + if err := m.ensureSnapshotMemoryReady(ctx, m.paths.InstanceSnapshotLatest(id), m.snapshotJobKeyForInstance(id), stored.HypervisorType); err != nil { + return nil, fmt.Errorf("prepare standby snapshot for fork: %w", err) + } + } + if err := forkvm.CopyGuestDirectory(srcDir, dstDir); err != nil { if errors.Is(err, forkvm.ErrSparseCopyUnsupported) { return nil, fmt.Errorf("fork requires sparse-capable filesystem (SEEK_DATA/SEEK_HOLE unsupported): %w", err) @@ -421,7 +427,7 @@ func (m *manager) applyForkTargetState(ctx context.Context, forkID string, targe if _, err := m.startInstance(ctx, forkID, StartInstanceRequest{}); err != nil { return nil, fmt.Errorf("start forked instance for standby transition: %w", err) } - return returnWithReadiness(m.standbyInstance(ctx, forkID)) + return returnWithReadiness(m.standbyInstance(ctx, forkID, StandbyInstanceRequest{}, false)) } case StateStandby: switch target { @@ -436,7 +442,7 @@ func (m *manager) applyForkTargetState(ctx context.Context, forkID string, targe case StateRunning: switch target { case StateStandby: - return returnWithReadiness(m.standbyInstance(ctx, forkID)) + return returnWithReadiness(m.standbyInstance(ctx, forkID, StandbyInstanceRequest{}, false)) case StateStopped: return returnWithReadiness(m.stopInstance(ctx, forkID)) } diff --git a/lib/instances/fork_test.go b/lib/instances/fork_test.go index 88e48537..c6bf85e0 100644 --- a/lib/instances/fork_test.go +++ b/lib/instances/fork_test.go @@ -10,6 +10,7 @@ import ( "os" "path/filepath" "strings" + "sync/atomic" "testing" "time" @@ -17,6 +18,7 @@ import ( "github.com/kernel/hypeman/lib/hypervisor" "github.com/kernel/hypeman/lib/images" "github.com/kernel/hypeman/lib/paths" + snapshotstore "github.com/kernel/hypeman/lib/snapshot" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -215,6 +217,66 @@ func TestForkInstanceRejectsDuplicateNameForNonNetworkedSource(t *testing.T) { assert.Contains(t, err.Error(), "already exists") } +func TestForkInstanceFromStandbyCancelsCompressionJobAndCopiesRawMemory(t *testing.T) { + t.Parallel() + + manager, _ := setupTestManager(t) + ctx := context.Background() + + sourceID := "fork-standby-compressed-src" + createStandbySnapshotSourceFixture(t, manager, sourceID, sourceID, manager.defaultHypervisor) + + rawPath := filepath.Join(manager.paths.InstanceSnapshotLatest(sourceID), "memory-ranges") + require.NoError(t, os.WriteFile(rawPath, []byte("some guest memory"), 0o644)) + snapshotConfigPath := manager.paths.InstanceSnapshotConfig(sourceID) + require.NoError(t, os.MkdirAll(filepath.Dir(snapshotConfigPath), 0o755)) + require.NoError(t, os.WriteFile(snapshotConfigPath, []byte(`{}`), 0o644)) + + _, _, err := compressSnapshotMemoryFile(ctx, rawPath, snapshotstore.SnapshotCompressionConfig{ + Enabled: true, + Algorithm: snapshotstore.SnapshotCompressionAlgorithmZstd, + Level: intPtr(1), + }) + require.NoError(t, err) + + var canceled atomic.Bool + done := make(chan struct{}) + + manager.compressionMu.Lock() + manager.compressionJobs[manager.snapshotJobKeyForInstance(sourceID)] = &compressionJob{ + cancel: func() { + canceled.Store(true) + select { + case <-done: + default: + close(done) + } + }, + done: done, + target: compressionTarget{ + Key: manager.snapshotJobKeyForInstance(sourceID), + OwnerID: sourceID, + SnapshotDir: manager.paths.InstanceSnapshotLatest(sourceID), + }, + } + manager.compressionMu.Unlock() + + forked, err := manager.forkInstanceFromStoppedOrStandby(ctx, sourceID, ForkInstanceRequest{ + Name: "fork-standby-compressed-copy", + TargetState: StateStopped, + }, true) + require.NoError(t, err) + require.NotNil(t, forked) + + assert.True(t, canceled.Load(), "standby compression job should be canceled before copying the source guest directory") + + forkSnapshotDir := manager.paths.InstanceSnapshotLatest(forked.Id) + _, ok := findRawSnapshotMemoryFile(forkSnapshotDir) + assert.True(t, ok, "forked standby guest directory should contain raw memory after preparing the source snapshot") + _, _, ok = findCompressedSnapshotMemoryFile(forkSnapshotDir) + assert.False(t, ok, "forked standby guest directory should not retain compressed memory artifacts from the source instance") +} + func TestCloneStoredMetadataForFork_DeepCopiesReferenceFields(t *testing.T) { t.Parallel() startedAt := time.Now().Add(-2 * time.Minute) diff --git a/lib/instances/manager.go b/lib/instances/manager.go index 0f82f7c8..44d9b162 100644 --- a/lib/instances/manager.go +++ b/lib/instances/manager.go @@ -34,7 +34,7 @@ type Manager interface { DeleteSnapshot(ctx context.Context, snapshotID string) error ForkInstance(ctx context.Context, id string, req ForkInstanceRequest) (*Instance, error) ForkSnapshot(ctx context.Context, snapshotID string, req ForkSnapshotRequest) (*Instance, error) - StandbyInstance(ctx context.Context, id string) (*Instance, error) + StandbyInstance(ctx context.Context, id string, req StandbyInstanceRequest) (*Instance, error) RestoreInstance(ctx context.Context, id string) (*Instance, error) RestoreSnapshot(ctx context.Context, id string, snapshotID string, req RestoreSnapshotRequest) (*Instance, error) StopInstance(ctx context.Context, id string) (*Instance, error) @@ -91,6 +91,11 @@ type manager struct { egressProxy *egressproxy.Service egressProxyServiceOptions egressproxy.ServiceOptions egressProxyMu sync.Mutex + snapshotDefaults SnapshotPolicy + compressionMu sync.Mutex + compressionJobs map[string]*compressionJob + nativeCodecMu sync.Mutex + nativeCodecPaths map[string]string // Hypervisor support vmStarters map[hypervisor.Type]hypervisor.VMStarter @@ -104,7 +109,7 @@ var platformStarters = make(map[hypervisor.Type]hypervisor.VMStarter) // NewManager creates a new instances manager. // If meter is nil, metrics are disabled. // defaultHypervisor specifies which hypervisor to use when not specified in requests. -func NewManager(p *paths.Paths, imageManager images.Manager, systemManager system.Manager, networkManager network.Manager, deviceManager devices.Manager, volumeManager volumes.Manager, limits ResourceLimits, defaultHypervisor hypervisor.Type, meter metric.Meter, tracer trace.Tracer, memoryPolicy ...guestmemory.Policy) Manager { +func NewManager(p *paths.Paths, imageManager images.Manager, systemManager system.Manager, networkManager network.Manager, deviceManager devices.Manager, volumeManager volumes.Manager, limits ResourceLimits, defaultHypervisor hypervisor.Type, snapshotDefaults SnapshotPolicy, meter metric.Meter, tracer trace.Tracer, memoryPolicy ...guestmemory.Policy) Manager { // Validate and default the hypervisor type if defaultHypervisor == "" { defaultHypervisor = hypervisor.TypeCloudHypervisor @@ -139,6 +144,9 @@ func NewManager(p *paths.Paths, imageManager images.Manager, systemManager syste meter: meter, tracer: tracer, guestMemoryPolicy: policy, + snapshotDefaults: snapshotDefaults, + compressionJobs: make(map[string]*compressionJob), + nativeCodecPaths: make(map[string]string), } // Initialize metrics if meter is provided @@ -280,11 +288,11 @@ func (m *manager) ForkSnapshot(ctx context.Context, snapshotID string, req ForkS } // StandbyInstance puts an instance in standby (pause, snapshot, delete VMM) -func (m *manager) StandbyInstance(ctx context.Context, id string) (*Instance, error) { +func (m *manager) StandbyInstance(ctx context.Context, id string, req StandbyInstanceRequest) (*Instance, error) { lock := m.getInstanceLock(id) lock.Lock() defer lock.Unlock() - return m.standbyInstance(ctx, id) + return m.standbyInstance(ctx, id, req, false) } // RestoreInstance restores an instance from standby diff --git a/lib/instances/manager_darwin_test.go b/lib/instances/manager_darwin_test.go index a7557b20..7d966e02 100644 --- a/lib/instances/manager_darwin_test.go +++ b/lib/instances/manager_darwin_test.go @@ -57,7 +57,7 @@ func setupVZTestManager(t *testing.T) (*manager, string) { MaxVcpusPerInstance: 0, MaxMemoryPerInstance: 0, } - mgr := NewManager(p, imageManager, systemManager, networkManager, deviceManager, volumeManager, limits, "", nil, nil).(*manager) + mgr := NewManager(p, imageManager, systemManager, networkManager, deviceManager, volumeManager, limits, "", SnapshotPolicy{}, nil, nil).(*manager) resourceMgr := resources.NewManager(cfg, p) resourceMgr.SetInstanceLister(mgr) @@ -470,7 +470,7 @@ func TestVZStandbyAndRestore(t *testing.T) { // Standby instance t.Log("Putting instance in standby...") - inst, err = mgr.StandbyInstance(ctx, inst.Id) + inst, err = mgr.StandbyInstance(ctx, inst.Id, StandbyInstanceRequest{}) require.NoError(t, err) assert.Equal(t, StateStandby, inst.State) assert.True(t, inst.HasSnapshot) diff --git a/lib/instances/manager_test.go b/lib/instances/manager_test.go index 7fc7b5c3..0e87b8e3 100644 --- a/lib/instances/manager_test.go +++ b/lib/instances/manager_test.go @@ -55,7 +55,7 @@ func setupTestManager(t *testing.T) (*manager, string) { MaxVcpusPerInstance: 0, // unlimited MaxMemoryPerInstance: 0, // unlimited } - mgr := NewManager(p, imageManager, systemManager, networkManager, deviceManager, volumeManager, limits, "", nil, nil).(*manager) + mgr := NewManager(p, imageManager, systemManager, networkManager, deviceManager, volumeManager, limits, "", SnapshotPolicy{}, nil, nil).(*manager) // Set up resource validation using the real ResourceManager resourceMgr := resources.NewManager(cfg, p) @@ -1236,7 +1236,7 @@ func TestStorageOperations(t *testing.T) { MaxVcpusPerInstance: 0, // unlimited MaxMemoryPerInstance: 0, // unlimited } - manager := NewManager(p, imageManager, systemManager, networkManager, deviceManager, volumeManager, limits, "", nil, nil).(*manager) + manager := NewManager(p, imageManager, systemManager, networkManager, deviceManager, volumeManager, limits, "", SnapshotPolicy{}, nil, nil).(*manager) // Test metadata doesn't exist initially _, err := manager.loadMetadata("nonexistent") @@ -1356,7 +1356,7 @@ func TestStandbyAndRestore(t *testing.T) { // Standby instance t.Log("Standing by instance...") - inst, err = manager.StandbyInstance(ctx, inst.Id) + inst, err = manager.StandbyInstance(ctx, inst.Id, StandbyInstanceRequest{}) require.NoError(t, err) assert.Equal(t, StateStandby, inst.State) assert.True(t, inst.HasSnapshot) diff --git a/lib/instances/metrics.go b/lib/instances/metrics.go index b9252255..bd87a3eb 100644 --- a/lib/instances/metrics.go +++ b/lib/instances/metrics.go @@ -6,20 +6,76 @@ import ( "github.com/kernel/hypeman/lib/hypervisor" mw "github.com/kernel/hypeman/lib/middleware" + snapshotstore "github.com/kernel/hypeman/lib/snapshot" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/metric" "go.opentelemetry.io/otel/trace" ) +type snapshotCompressionSource string + +const ( + snapshotCompressionSourceStandby snapshotCompressionSource = "standby" + snapshotCompressionSourceSnapshot snapshotCompressionSource = "snapshot" +) + +type snapshotCompressionResult string + +const ( + snapshotCompressionResultSuccess snapshotCompressionResult = "success" + snapshotCompressionResultCanceled snapshotCompressionResult = "canceled" + snapshotCompressionResultFailed snapshotCompressionResult = "failed" +) + +type snapshotMemoryPreparePath string + +const ( + snapshotMemoryPreparePathRaw snapshotMemoryPreparePath = "raw" + snapshotMemoryPreparePathDecompress snapshotMemoryPreparePath = "decompress" +) + +type snapshotCompressionPreemptionOperation string + +const ( + snapshotCompressionPreemptionRestoreInstance snapshotCompressionPreemptionOperation = "restore_instance" + snapshotCompressionPreemptionRestoreSnapshot snapshotCompressionPreemptionOperation = "restore_snapshot" + snapshotCompressionPreemptionForkSnapshot snapshotCompressionPreemptionOperation = "fork_snapshot" + snapshotCompressionPreemptionCreateSnapshot snapshotCompressionPreemptionOperation = "create_snapshot" + snapshotCompressionPreemptionDeleteInstance snapshotCompressionPreemptionOperation = "delete_instance" + snapshotCompressionPreemptionDeleteSnapshot snapshotCompressionPreemptionOperation = "delete_snapshot" +) + +type snapshotCodecOperation string + +const ( + snapshotCodecOperationCompress snapshotCodecOperation = "compress" + snapshotCodecOperationDecompress snapshotCodecOperation = "decompress" +) + +type snapshotCodecFallbackReason string + +const ( + snapshotCodecFallbackReasonMissingBinary snapshotCodecFallbackReason = "missing_binary" + snapshotCodecFallbackReasonNotExecutable snapshotCodecFallbackReason = "not_executable" +) + // Metrics holds the metrics instruments for instance operations. type Metrics struct { - createDuration metric.Float64Histogram - restoreDuration metric.Float64Histogram - standbyDuration metric.Float64Histogram - stopDuration metric.Float64Histogram - startDuration metric.Float64Histogram - stateTransitions metric.Int64Counter - tracer trace.Tracer + createDuration metric.Float64Histogram + restoreDuration metric.Float64Histogram + standbyDuration metric.Float64Histogram + stopDuration metric.Float64Histogram + startDuration metric.Float64Histogram + stateTransitions metric.Int64Counter + snapshotCompressionJobsTotal metric.Int64Counter + snapshotCompressionDuration metric.Float64Histogram + snapshotCompressionSavedBytes metric.Int64Histogram + snapshotCompressionRatio metric.Float64Histogram + snapshotCodecFallbacksTotal metric.Int64Counter + snapshotRestoreMemoryPrepareTotal metric.Int64Counter + snapshotRestoreMemoryPrepareDuration metric.Float64Histogram + snapshotCompressionPreemptionsTotal metric.Int64Counter + tracer trace.Tracer } // newInstanceMetrics creates and registers all instance metrics. @@ -77,6 +133,73 @@ func newInstanceMetrics(meter metric.Meter, tracer trace.Tracer, m *manager) (*M return nil, err } + snapshotCompressionJobsTotal, err := meter.Int64Counter( + "hypeman_snapshot_compression_jobs_total", + metric.WithDescription("Total number of snapshot compression jobs by result"), + ) + if err != nil { + return nil, err + } + + snapshotCompressionDuration, err := meter.Float64Histogram( + "hypeman_snapshot_compression_duration_seconds", + metric.WithDescription("Time to asynchronously compress snapshot memory"), + metric.WithUnit("s"), + ) + if err != nil { + return nil, err + } + + snapshotCompressionSavedBytes, err := meter.Int64Histogram( + "hypeman_snapshot_compression_saved_bytes", + metric.WithDescription("Bytes saved by compressing snapshot memory"), + metric.WithUnit("By"), + ) + if err != nil { + return nil, err + } + + snapshotCompressionRatio, err := meter.Float64Histogram( + "hypeman_snapshot_compression_ratio", + metric.WithDescription("Compressed snapshot memory size divided by raw snapshot memory size"), + ) + if err != nil { + return nil, err + } + + snapshotCodecFallbacksTotal, err := meter.Int64Counter( + "hypeman_snapshot_codec_fallbacks_total", + metric.WithDescription("Total number of snapshot codec fallbacks from native binaries to the Go implementation"), + ) + if err != nil { + return nil, err + } + + snapshotRestoreMemoryPrepareTotal, err := meter.Int64Counter( + "hypeman_snapshot_restore_memory_prepare_total", + metric.WithDescription("Total number of snapshot memory prepare operations before restore"), + ) + if err != nil { + return nil, err + } + + snapshotRestoreMemoryPrepareDuration, err := meter.Float64Histogram( + "hypeman_snapshot_restore_memory_prepare_duration_seconds", + metric.WithDescription("Time to prepare snapshot memory before restore"), + metric.WithUnit("s"), + ) + if err != nil { + return nil, err + } + + snapshotCompressionPreemptionsTotal, err := meter.Int64Counter( + "hypeman_snapshot_compression_preemptions_total", + metric.WithDescription("Total number of foreground operations that preempt in-flight snapshot compression"), + ) + if err != nil { + return nil, err + } + // Register observable gauge for instance counts by state instancesTotal, err := meter.Int64ObservableGauge( "hypeman_instances_total", @@ -86,6 +209,14 @@ func newInstanceMetrics(meter metric.Meter, tracer trace.Tracer, m *manager) (*M return nil, err } + snapshotCompressionActiveTotal, err := meter.Int64ObservableGauge( + "hypeman_snapshot_compression_active_total", + metric.WithDescription("Total number of in-flight snapshot compression jobs"), + ) + if err != nil { + return nil, err + } + _, err = meter.RegisterCallback( func(ctx context.Context, o metric.Observer) error { instances, err := m.listInstances(ctx) @@ -120,14 +251,60 @@ func newInstanceMetrics(meter metric.Meter, tracer trace.Tracer, m *manager) (*M return nil, err } + _, err = meter.RegisterCallback( + func(ctx context.Context, o metric.Observer) error { + type compressionKey struct { + hypervisor string + algorithm string + source string + } + + counts := make(map[compressionKey]int64) + m.compressionMu.Lock() + for _, job := range m.compressionJobs { + key := compressionKey{ + hypervisor: string(job.target.HypervisorType), + algorithm: string(job.target.Policy.Algorithm), + source: string(job.target.Source), + } + counts[key]++ + } + m.compressionMu.Unlock() + + for key, count := range counts { + attrs := []attribute.KeyValue{ + attribute.String("algorithm", key.algorithm), + attribute.String("source", key.source), + } + if key.hypervisor != "" { + attrs = append(attrs, attribute.String("hypervisor", key.hypervisor)) + } + o.ObserveInt64(snapshotCompressionActiveTotal, count, metric.WithAttributes(attrs...)) + } + return nil + }, + snapshotCompressionActiveTotal, + ) + if err != nil { + return nil, err + } + return &Metrics{ - createDuration: createDuration, - restoreDuration: restoreDuration, - standbyDuration: standbyDuration, - stopDuration: stopDuration, - startDuration: startDuration, - stateTransitions: stateTransitions, - tracer: tracer, + createDuration: createDuration, + restoreDuration: restoreDuration, + standbyDuration: standbyDuration, + stopDuration: stopDuration, + startDuration: startDuration, + stateTransitions: stateTransitions, + snapshotCompressionJobsTotal: snapshotCompressionJobsTotal, + snapshotCompressionDuration: snapshotCompressionDuration, + snapshotCompressionSavedBytes: snapshotCompressionSavedBytes, + snapshotCompressionRatio: snapshotCompressionRatio, + snapshotCodecFallbacksTotal: snapshotCodecFallbacksTotal, + snapshotRestoreMemoryPrepareTotal: snapshotRestoreMemoryPrepareTotal, + snapshotRestoreMemoryPrepareDuration: snapshotRestoreMemoryPrepareDuration, + snapshotCompressionPreemptionsTotal: snapshotCompressionPreemptionsTotal, + tracer: tracer, }, nil } @@ -169,3 +346,76 @@ func (m *manager) recordStateTransition(ctx context.Context, fromState, toState } m.metrics.stateTransitions.Add(ctx, 1, metric.WithAttributes(attrs...)) } + +func snapshotCompressionAttributes(hvType hypervisor.Type, algorithm snapshotstore.SnapshotCompressionAlgorithm, source snapshotCompressionSource) []attribute.KeyValue { + attrs := []attribute.KeyValue{ + attribute.String("algorithm", string(algorithm)), + attribute.String("source", string(source)), + } + if hvType != "" { + attrs = append(attrs, attribute.String("hypervisor", string(hvType))) + } + return attrs +} + +func (m *manager) recordSnapshotCompressionJob(ctx context.Context, target compressionTarget, result snapshotCompressionResult, start time.Time, uncompressedSize, compressedSize int64) { + if m.metrics == nil { + return + } + + attrs := snapshotCompressionAttributes(target.HypervisorType, target.Policy.Algorithm, target.Source) + attrsWithResult := append([]attribute.KeyValue{}, attrs...) + attrsWithResult = append(attrsWithResult, attribute.String("result", string(result))) + + m.metrics.snapshotCompressionJobsTotal.Add(ctx, 1, metric.WithAttributes(attrsWithResult...)) + m.metrics.snapshotCompressionDuration.Record(ctx, time.Since(start).Seconds(), metric.WithAttributes(attrsWithResult...)) + + if result != snapshotCompressionResultSuccess || uncompressedSize <= 0 || compressedSize < 0 { + return + } + + savedBytes := uncompressedSize - compressedSize + if savedBytes < 0 { + savedBytes = 0 + } + m.metrics.snapshotCompressionSavedBytes.Record(ctx, savedBytes, metric.WithAttributes(attrs...)) + m.metrics.snapshotCompressionRatio.Record(ctx, float64(compressedSize)/float64(uncompressedSize), metric.WithAttributes(attrs...)) +} + +func (m *manager) recordSnapshotRestoreMemoryPrepare(ctx context.Context, hvType hypervisor.Type, path snapshotMemoryPreparePath, result snapshotCompressionResult, start time.Time) { + if m.metrics == nil { + return + } + + attrs := []attribute.KeyValue{ + attribute.String("restore_source", string(path)), + attribute.String("result", string(result)), + } + if hvType != "" { + attrs = append(attrs, attribute.String("hypervisor", string(hvType))) + } + m.metrics.snapshotRestoreMemoryPrepareTotal.Add(ctx, 1, metric.WithAttributes(attrs...)) + m.metrics.snapshotRestoreMemoryPrepareDuration.Record(ctx, time.Since(start).Seconds(), metric.WithAttributes(attrs...)) +} + +func (m *manager) recordSnapshotCompressionPreemption(ctx context.Context, operation snapshotCompressionPreemptionOperation, target compressionTarget) { + if m.metrics == nil { + return + } + + attrs := snapshotCompressionAttributes(target.HypervisorType, target.Policy.Algorithm, target.Source) + attrs = append(attrs, attribute.String("operation", string(operation))) + m.metrics.snapshotCompressionPreemptionsTotal.Add(ctx, 1, metric.WithAttributes(attrs...)) +} + +func (m *manager) recordSnapshotCodecFallback(ctx context.Context, algorithm snapshotstore.SnapshotCompressionAlgorithm, operation snapshotCodecOperation, reason snapshotCodecFallbackReason) { + if m.metrics == nil { + return + } + + m.metrics.snapshotCodecFallbacksTotal.Add(ctx, 1, metric.WithAttributes( + attribute.String("algorithm", string(algorithm)), + attribute.String("operation", string(operation)), + attribute.String("reason", string(reason)), + )) +} diff --git a/lib/instances/metrics_test.go b/lib/instances/metrics_test.go new file mode 100644 index 00000000..f0188f08 --- /dev/null +++ b/lib/instances/metrics_test.go @@ -0,0 +1,151 @@ +package instances + +import ( + "testing" + "time" + + "github.com/kernel/hypeman/lib/hypervisor" + "github.com/kernel/hypeman/lib/paths" + snapshotstore "github.com/kernel/hypeman/lib/snapshot" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.opentelemetry.io/otel/attribute" + otelmetric "go.opentelemetry.io/otel/sdk/metric" + "go.opentelemetry.io/otel/sdk/metric/metricdata" +) + +func TestSnapshotCompressionMetrics_RecordAndObserve(t *testing.T) { + t.Parallel() + + reader := otelmetric.NewManualReader() + provider := otelmetric.NewMeterProvider(otelmetric.WithReader(reader)) + + m := &manager{ + paths: paths.New(t.TempDir()), + compressionJobs: map[string]*compressionJob{ + "job-1": { + done: make(chan struct{}), + target: compressionTarget{ + Key: "job-1", + HypervisorType: hypervisor.TypeCloudHypervisor, + Source: snapshotCompressionSourceStandby, + Policy: snapshotstore.SnapshotCompressionConfig{ + Enabled: true, + Algorithm: snapshotstore.SnapshotCompressionAlgorithmLz4, + }, + }, + }, + }, + } + + metrics, err := newInstanceMetrics(provider.Meter("test"), nil, m) + require.NoError(t, err) + m.metrics = metrics + + target := m.compressionJobs["job-1"].target + m.recordSnapshotCompressionJob(t.Context(), target, snapshotCompressionResultSuccess, time.Now().Add(-2*time.Second), 1024, 256) + m.recordSnapshotCodecFallback(t.Context(), snapshotstore.SnapshotCompressionAlgorithmLz4, snapshotCodecOperationCompress, snapshotCodecFallbackReasonMissingBinary) + m.recordSnapshotRestoreMemoryPrepare(t.Context(), hypervisor.TypeCloudHypervisor, snapshotMemoryPreparePathRaw, snapshotCompressionResultSuccess, time.Now().Add(-250*time.Millisecond)) + m.recordSnapshotCompressionPreemption(t.Context(), snapshotCompressionPreemptionRestoreInstance, target) + + var rm metricdata.ResourceMetrics + require.NoError(t, reader.Collect(t.Context(), &rm)) + + assertMetricNames(t, rm, []string{ + "hypeman_snapshot_compression_jobs_total", + "hypeman_snapshot_compression_duration_seconds", + "hypeman_snapshot_compression_saved_bytes", + "hypeman_snapshot_compression_ratio", + "hypeman_snapshot_codec_fallbacks_total", + "hypeman_snapshot_restore_memory_prepare_total", + "hypeman_snapshot_restore_memory_prepare_duration_seconds", + "hypeman_snapshot_compression_preemptions_total", + "hypeman_snapshot_compression_active_total", + }) + + jobsMetric := findMetric(t, rm, "hypeman_snapshot_compression_jobs_total") + jobs, ok := jobsMetric.Data.(metricdata.Sum[int64]) + require.True(t, ok) + require.Len(t, jobs.DataPoints, 1) + assert.Equal(t, int64(1), jobs.DataPoints[0].Value) + assert.Equal(t, "cloud-hypervisor", metricLabel(t, jobs.DataPoints[0].Attributes, "hypervisor")) + assert.Equal(t, "lz4", metricLabel(t, jobs.DataPoints[0].Attributes, "algorithm")) + assert.Equal(t, "standby", metricLabel(t, jobs.DataPoints[0].Attributes, "source")) + assert.Equal(t, "success", metricLabel(t, jobs.DataPoints[0].Attributes, "result")) + + savedBytesMetric := findMetric(t, rm, "hypeman_snapshot_compression_saved_bytes") + savedBytes, ok := savedBytesMetric.Data.(metricdata.Histogram[int64]) + require.True(t, ok) + require.Len(t, savedBytes.DataPoints, 1) + assert.Equal(t, uint64(1), savedBytes.DataPoints[0].Count) + assert.Equal(t, int64(768), savedBytes.DataPoints[0].Sum) + + fallbackMetric := findMetric(t, rm, "hypeman_snapshot_codec_fallbacks_total") + fallbacks, ok := fallbackMetric.Data.(metricdata.Sum[int64]) + require.True(t, ok) + require.Len(t, fallbacks.DataPoints, 1) + assert.Equal(t, int64(1), fallbacks.DataPoints[0].Value) + assert.Equal(t, "lz4", metricLabel(t, fallbacks.DataPoints[0].Attributes, "algorithm")) + assert.Equal(t, "compress", metricLabel(t, fallbacks.DataPoints[0].Attributes, "operation")) + assert.Equal(t, "missing_binary", metricLabel(t, fallbacks.DataPoints[0].Attributes, "reason")) + + restorePrepMetric := findMetric(t, rm, "hypeman_snapshot_restore_memory_prepare_total") + restorePrep, ok := restorePrepMetric.Data.(metricdata.Sum[int64]) + require.True(t, ok) + require.Len(t, restorePrep.DataPoints, 1) + assert.Equal(t, int64(1), restorePrep.DataPoints[0].Value) + assert.Equal(t, "raw", metricLabel(t, restorePrep.DataPoints[0].Attributes, "restore_source")) + assert.Equal(t, "success", metricLabel(t, restorePrep.DataPoints[0].Attributes, "result")) + + preemptionsMetric := findMetric(t, rm, "hypeman_snapshot_compression_preemptions_total") + preemptions, ok := preemptionsMetric.Data.(metricdata.Sum[int64]) + require.True(t, ok) + require.Len(t, preemptions.DataPoints, 1) + assert.Equal(t, int64(1), preemptions.DataPoints[0].Value) + assert.Equal(t, "restore_instance", metricLabel(t, preemptions.DataPoints[0].Attributes, "operation")) + + activeMetric := findMetric(t, rm, "hypeman_snapshot_compression_active_total") + active, ok := activeMetric.Data.(metricdata.Gauge[int64]) + require.True(t, ok) + require.Len(t, active.DataPoints, 1) + assert.Equal(t, int64(1), active.DataPoints[0].Value) + assert.Equal(t, "lz4", metricLabel(t, active.DataPoints[0].Attributes, "algorithm")) + assert.Equal(t, "standby", metricLabel(t, active.DataPoints[0].Attributes, "source")) +} + +func assertMetricNames(t *testing.T, rm metricdata.ResourceMetrics, expected []string) { + t.Helper() + + metricNames := make(map[string]bool) + for _, sm := range rm.ScopeMetrics { + for _, m := range sm.Metrics { + metricNames[m.Name] = true + } + } + + for _, name := range expected { + assert.True(t, metricNames[name], "expected metric %s to be registered", name) + } +} + +func findMetric(t *testing.T, rm metricdata.ResourceMetrics, name string) metricdata.Metrics { + t.Helper() + + for _, sm := range rm.ScopeMetrics { + for _, m := range sm.Metrics { + if m.Name == name { + return m + } + } + } + t.Fatalf("metric %s not found", name) + return metricdata.Metrics{} +} + +func metricLabel(t *testing.T, attrs attribute.Set, key string) string { + t.Helper() + + value, ok := attrs.Value(attribute.Key(key)) + require.True(t, ok, "expected label %s", key) + return value.AsString() +} diff --git a/lib/instances/network_test.go b/lib/instances/network_test.go index d8537e80..a3febb0b 100644 --- a/lib/instances/network_test.go +++ b/lib/instances/network_test.go @@ -134,7 +134,7 @@ func TestCreateInstanceWithNetwork(t *testing.T) { // Standby instance t.Log("Standing by instance...") - inst, err = manager.StandbyInstance(ctx, inst.Id) + inst, err = manager.StandbyInstance(ctx, inst.Id, StandbyInstanceRequest{}) require.NoError(t, err) assert.Equal(t, StateStandby, inst.State) assert.True(t, inst.HasSnapshot) diff --git a/lib/instances/qemu_test.go b/lib/instances/qemu_test.go index 6a58abee..155435b8 100644 --- a/lib/instances/qemu_test.go +++ b/lib/instances/qemu_test.go @@ -54,7 +54,7 @@ func setupTestManagerForQEMU(t *testing.T) (*manager, string) { MaxVcpusPerInstance: 0, // unlimited MaxMemoryPerInstance: 0, // unlimited } - mgr := NewManager(p, imageManager, systemManager, networkManager, deviceManager, volumeManager, limits, hypervisor.TypeQEMU, nil, nil).(*manager) + mgr := NewManager(p, imageManager, systemManager, networkManager, deviceManager, volumeManager, limits, hypervisor.TypeQEMU, SnapshotPolicy{}, nil, nil).(*manager) // Set up resource validation using the real ResourceManager resourceMgr := resources.NewManager(cfg, p) @@ -841,7 +841,7 @@ func TestQEMUStandbyAndRestore(t *testing.T) { // Standby instance t.Log("Standing by instance...") - inst, err = manager.StandbyInstance(ctx, inst.Id) + inst, err = manager.StandbyInstance(ctx, inst.Id, StandbyInstanceRequest{}) require.NoError(t, err) assert.Equal(t, StateStandby, inst.State) assert.True(t, inst.HasSnapshot) diff --git a/lib/instances/resource_limits_test.go b/lib/instances/resource_limits_test.go index d34b9c45..32cad66c 100644 --- a/lib/instances/resource_limits_test.go +++ b/lib/instances/resource_limits_test.go @@ -176,7 +176,7 @@ func createTestManager(t *testing.T, limits ResourceLimits) *manager { deviceMgr := devices.NewManager(p) volumeMgr := volumes.NewManager(p, 0, nil) - return NewManager(p, imageMgr, systemMgr, networkMgr, deviceMgr, volumeMgr, limits, "", nil, nil).(*manager) + return NewManager(p, imageMgr, systemMgr, networkMgr, deviceMgr, volumeMgr, limits, "", SnapshotPolicy{}, nil, nil).(*manager) } func TestResourceLimits_StructValues(t *testing.T) { diff --git a/lib/instances/restore.go b/lib/instances/restore.go index bd6ef5aa..d9b97be6 100644 --- a/lib/instances/restore.go +++ b/lib/instances/restore.go @@ -69,6 +69,17 @@ func (m *manager) restoreInstance( // 3. Get snapshot directory snapshotDir := m.paths.InstanceSnapshotLatest(id) + var prepareSnapshotSpan trace.Span + if m.metrics != nil && m.metrics.tracer != nil { + ctx, prepareSnapshotSpan = m.metrics.tracer.Start(ctx, "PrepareSnapshotMemory") + } + err = m.ensureSnapshotMemoryReady(ctx, snapshotDir, m.snapshotJobKeyForInstance(id), stored.HypervisorType) + if prepareSnapshotSpan != nil { + prepareSnapshotSpan.End() + } + if err != nil { + return nil, fmt.Errorf("prepare standby snapshot memory: %w", err) + } starter, err := m.getVMStarter(stored.HypervisorType) if err != nil { return nil, fmt.Errorf("get vm starter: %w", err) diff --git a/lib/instances/snapshot.go b/lib/instances/snapshot.go index fcf65e79..5cf1f6c6 100644 --- a/lib/instances/snapshot.go +++ b/lib/instances/snapshot.go @@ -91,12 +91,21 @@ func (m *manager) createSnapshot(ctx context.Context, id string, req CreateSnaps if err := ensureGuestAgentReadyForForkPhase(ctx, &inst.StoredMetadata, "before running snapshot"); err != nil { return nil, err } - if _, err := m.standbyInstance(ctx, id); err != nil { + if _, err := m.standbyInstance(ctx, id, StandbyInstanceRequest{}, true); err != nil { return nil, fmt.Errorf("standby source instance: %w", err) } restoreSource = true case StateStandby: - // already ready to copy + target, err := m.cancelAndWaitCompressionJob(ctx, m.snapshotJobKeyForInstance(id)) + if err != nil { + return nil, fmt.Errorf("wait for source instance compression to stop: %w", err) + } + if target != nil { + m.recordSnapshotCompressionPreemption(ctx, snapshotCompressionPreemptionCreateSnapshot, *target) + } + if err := m.ensureSnapshotMemoryReady(ctx, m.paths.InstanceSnapshotLatest(id), "", stored.HypervisorType); err != nil { + return nil, fmt.Errorf("prepare source snapshot memory for copy: %w", err) + } default: return nil, fmt.Errorf("%w: standby snapshot requires source in %s or %s, got %s", ErrInvalidState, StateRunning, StateStandby, inst.State) } @@ -138,9 +147,29 @@ func (m *manager) createSnapshot(ctx context.Context, id string, req CreateSnaps return nil, err } rec.Snapshot.SizeBytes = sizeBytes + rec.Snapshot.CompressionState = snapshotstore.SnapshotCompressionStateNone + effectiveCompression, err := m.resolveSnapshotCompressionPolicy(stored, req.Compression) + if err != nil { + return nil, err + } + if effectiveCompression.Enabled { + rec.Snapshot.Compression = cloneCompressionConfig(&effectiveCompression) + rec.Snapshot.CompressionState = snapshotstore.SnapshotCompressionStateCompressing + } if err := m.saveSnapshotRecord(rec); err != nil { return nil, err } + if effectiveCompression.Enabled { + m.startCompressionJob(ctx, compressionTarget{ + Key: m.snapshotJobKeyForSnapshot(snapshotID), + OwnerID: stored.Id, + SnapshotID: snapshotID, + SnapshotDir: snapshotGuestDir, + HypervisorType: stored.HypervisorType, + Source: snapshotCompressionSourceSnapshot, + Policy: effectiveCompression, + }) + } cu.Release() log.InfoContext(ctx, "snapshot created", "instance_id", id, "snapshot_id", snapshotID, "kind", req.Kind) return &rec.Snapshot, nil @@ -170,6 +199,7 @@ func (m *manager) createSnapshot(ctx context.Context, id string, req CreateSnaps return nil, err } rec.Snapshot.SizeBytes = sizeBytes + rec.Snapshot.CompressionState = snapshotstore.SnapshotCompressionStateNone if err := m.saveSnapshotRecord(rec); err != nil { return nil, err } @@ -183,7 +213,13 @@ func (m *manager) createSnapshot(ctx context.Context, id string, req CreateSnaps } func (m *manager) deleteSnapshot(ctx context.Context, snapshotID string) error { - _ = ctx + target, err := m.cancelAndWaitCompressionJob(ctx, m.snapshotJobKeyForSnapshot(snapshotID)) + if err != nil { + return fmt.Errorf("wait for snapshot compression to stop: %w", err) + } + if target != nil { + m.recordSnapshotCompressionPreemption(ctx, snapshotCompressionPreemptionDeleteSnapshot, *target) + } if err := m.snapshotStore().Delete(snapshotID); err != nil { if errors.Is(err, snapshotstore.ErrNotFound) { return ErrSnapshotNotFound @@ -221,6 +257,21 @@ func (m *manager) restoreSnapshot(ctx context.Context, id string, snapshotID str return nil, err } + target, err := m.cancelAndWaitCompressionJob(ctx, m.snapshotJobKeyForSnapshot(snapshotID)) + if err != nil { + return nil, fmt.Errorf("wait for snapshot compression to stop: %w", err) + } + if target != nil { + m.recordSnapshotCompressionPreemption(ctx, snapshotCompressionPreemptionRestoreSnapshot, *target) + } + target, err = m.cancelAndWaitCompressionJob(ctx, m.snapshotJobKeyForInstance(id)) + if err != nil { + return nil, fmt.Errorf("wait for source instance compression to stop: %w", err) + } + if target != nil { + m.recordSnapshotCompressionPreemption(ctx, snapshotCompressionPreemptionRestoreSnapshot, *target) + } + if err := m.replaceInstanceWithSnapshotPayload(snapshotID, id); err != nil { return nil, err } @@ -335,6 +386,17 @@ func (m *manager) forkSnapshot(ctx context.Context, snapshotID string, req ForkS }) defer cu.Clean() + target, err := m.cancelAndWaitCompressionJob(ctx, m.snapshotJobKeyForSnapshot(snapshotID)) + if err != nil { + return nil, fmt.Errorf("wait for snapshot compression to stop: %w", err) + } + if target != nil { + m.recordSnapshotCompressionPreemption(ctx, snapshotCompressionPreemptionForkSnapshot, *target) + } + if err := m.ensureSnapshotMemoryReady(ctx, m.paths.SnapshotGuestDir(snapshotID), "", rec.StoredMetadata.HypervisorType); err != nil { + return nil, fmt.Errorf("prepare snapshot memory for fork: %w", err) + } + if err := forkvm.CopyGuestDirectory(m.paths.SnapshotGuestDir(snapshotID), dstDir); err != nil { if errors.Is(err, forkvm.ErrSparseCopyUnsupported) { return nil, fmt.Errorf("fork from snapshot requires sparse-capable filesystem (SEEK_DATA/SEEK_HOLE unsupported): %w", err) @@ -461,6 +523,12 @@ func validateCreateSnapshotRequest(req CreateSnapshotRequest) error { if req.Kind != SnapshotKindStandby && req.Kind != SnapshotKindStopped { return fmt.Errorf("%w: kind must be one of %s, %s", ErrInvalidRequest, SnapshotKindStandby, SnapshotKindStopped) } + if req.Kind == SnapshotKindStopped && req.Compression != nil && req.Compression.Enabled { + return fmt.Errorf("%w: compression is only supported for standby snapshots", ErrInvalidRequest) + } + if _, err := normalizeCompressionConfig(req.Compression); err != nil { + return err + } if err := tags.Validate(req.Tags); err != nil { return fmt.Errorf("%w: %v", ErrInvalidRequest, err) } diff --git a/lib/instances/snapshot_compression.go b/lib/instances/snapshot_compression.go new file mode 100644 index 00000000..680cd96a --- /dev/null +++ b/lib/instances/snapshot_compression.go @@ -0,0 +1,847 @@ +package instances + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "io/fs" + "os" + "os/exec" + "path/filepath" + "strings" + "syscall" + "time" + + "github.com/kernel/hypeman/lib/hypervisor" + "github.com/kernel/hypeman/lib/logger" + snapshotstore "github.com/kernel/hypeman/lib/snapshot" + "github.com/klauspost/compress/zstd" + "github.com/pierrec/lz4/v4" +) + +const ( + defaultSnapshotCompressionZstdLevel = snapshotstore.DefaultSnapshotCompressionZstdLevel + minSnapshotCompressionZstdLevel = snapshotstore.MinSnapshotCompressionZstdLevel + maxSnapshotCompressionZstdLevel = snapshotstore.MaxSnapshotCompressionZstdLevel + defaultSnapshotCompressionLz4Level = snapshotstore.DefaultSnapshotCompressionLz4Level + minSnapshotCompressionLz4Level = snapshotstore.MinSnapshotCompressionLz4Level + maxSnapshotCompressionLz4Level = snapshotstore.MaxSnapshotCompressionLz4Level +) + +type compressionJob struct { + cancel context.CancelFunc + done chan struct{} + target compressionTarget +} + +type compressionTarget struct { + Key string + OwnerID string + SnapshotID string + SnapshotDir string + HypervisorType hypervisor.Type + Source snapshotCompressionSource + Policy snapshotstore.SnapshotCompressionConfig +} + +type nativeCodecRuntime struct { + manager *manager + lookPath func(file string) (string, error) + commandContext func(ctx context.Context, name string, arg ...string) *exec.Cmd +} + +func (r nativeCodecRuntime) lookPathFunc() func(string) (string, error) { + if r.lookPath != nil { + return r.lookPath + } + return exec.LookPath +} + +func (r nativeCodecRuntime) commandContextFunc() func(context.Context, string, ...string) *exec.Cmd { + if r.commandContext != nil { + return r.commandContext + } + return exec.CommandContext +} + +func cloneCompressionConfig(cfg *snapshotstore.SnapshotCompressionConfig) *snapshotstore.SnapshotCompressionConfig { + if cfg == nil { + return nil + } + cloned := *cfg + if cfg.Level != nil { + v := *cfg.Level + cloned.Level = &v + } + return &cloned +} + +func cloneSnapshotPolicy(policy *SnapshotPolicy) *SnapshotPolicy { + if policy == nil { + return nil + } + return &SnapshotPolicy{ + Compression: cloneCompressionConfig(policy.Compression), + } +} + +func normalizeCompressionConfig(cfg *snapshotstore.SnapshotCompressionConfig) (snapshotstore.SnapshotCompressionConfig, error) { + if cfg == nil || !cfg.Enabled { + return snapshotstore.SnapshotCompressionConfig{Enabled: false}, nil + } + + normalized := snapshotstore.SnapshotCompressionConfig{ + Enabled: true, + Algorithm: cfg.Algorithm, + } + if normalized.Algorithm != "" { + normalized.Algorithm = snapshotstore.SnapshotCompressionAlgorithm(strings.ToLower(string(normalized.Algorithm))) + } + switch normalized.Algorithm { + case "": + normalized.Algorithm = snapshotstore.SnapshotCompressionAlgorithmZstd + case snapshotstore.SnapshotCompressionAlgorithmZstd, snapshotstore.SnapshotCompressionAlgorithmLz4: + default: + return snapshotstore.SnapshotCompressionConfig{}, fmt.Errorf("%w: unsupported compression algorithm %q", ErrInvalidRequest, cfg.Algorithm) + } + + switch normalized.Algorithm { + case snapshotstore.SnapshotCompressionAlgorithmZstd: + level := defaultSnapshotCompressionZstdLevel + if cfg.Level != nil { + level = *cfg.Level + } + if level < minSnapshotCompressionZstdLevel || level > maxSnapshotCompressionZstdLevel { + return snapshotstore.SnapshotCompressionConfig{}, fmt.Errorf("%w: invalid zstd level %d (must be between %d and %d)", ErrInvalidRequest, level, minSnapshotCompressionZstdLevel, maxSnapshotCompressionZstdLevel) + } + normalized.Level = &level + case snapshotstore.SnapshotCompressionAlgorithmLz4: + level := defaultSnapshotCompressionLz4Level + if cfg.Level != nil { + level = *cfg.Level + } + if level < minSnapshotCompressionLz4Level || level > maxSnapshotCompressionLz4Level { + return snapshotstore.SnapshotCompressionConfig{}, fmt.Errorf("%w: invalid lz4 level %d (must be between %d and %d)", ErrInvalidRequest, level, minSnapshotCompressionLz4Level, maxSnapshotCompressionLz4Level) + } + normalized.Level = &level + } + return normalized, nil +} + +func (m *manager) resolveSnapshotCompressionPolicy(stored *StoredMetadata, override *snapshotstore.SnapshotCompressionConfig) (snapshotstore.SnapshotCompressionConfig, error) { + cfg, err := m.resolveConfiguredCompressionPolicy(stored, override) + if err != nil { + return snapshotstore.SnapshotCompressionConfig{}, err + } + if cfg != nil { + return *cfg, nil + } + return snapshotstore.SnapshotCompressionConfig{Enabled: false}, nil +} + +func (m *manager) resolveStandbyCompressionPolicy(stored *StoredMetadata, override *snapshotstore.SnapshotCompressionConfig) (*snapshotstore.SnapshotCompressionConfig, error) { + return m.resolveConfiguredCompressionPolicy(stored, override) +} + +func (m *manager) resolveConfiguredCompressionPolicy(stored *StoredMetadata, override *snapshotstore.SnapshotCompressionConfig) (*snapshotstore.SnapshotCompressionConfig, error) { + candidates := []*snapshotstore.SnapshotCompressionConfig{override} + if stored != nil && stored.SnapshotPolicy != nil { + candidates = append(candidates, stored.SnapshotPolicy.Compression) + } + candidates = append(candidates, m.snapshotDefaults.Compression) + + for _, candidate := range candidates { + if candidate == nil { + continue + } + cfg, err := normalizeCompressionConfig(candidate) + if err != nil { + return nil, err + } + if !cfg.Enabled { + return nil, nil + } + return &cfg, nil + } + return nil, nil +} + +func (m *manager) snapshotJobKeyForInstance(instanceID string) string { + return "instance:" + instanceID +} + +func (m *manager) snapshotJobKeyForSnapshot(snapshotID string) string { + return "snapshot:" + snapshotID +} + +func nativeCodecBinaryName(algorithm snapshotstore.SnapshotCompressionAlgorithm) string { + switch algorithm { + case snapshotstore.SnapshotCompressionAlgorithmLz4: + return "lz4" + default: + return "zstd" + } +} + +func nativeCodecInstallTip(algorithm snapshotstore.SnapshotCompressionAlgorithm) string { + switch algorithm { + case snapshotstore.SnapshotCompressionAlgorithmLz4: + return "install lz4 to enable native snapshot compression (for example: apt install lz4)" + default: + return "install zstd to enable native snapshot compression (for example: apt install zstd)" + } +} + +func isNativeCodecUnavailable(err error) (snapshotCodecFallbackReason, bool) { + if err == nil { + return "", false + } + switch { + case errors.Is(err, exec.ErrNotFound), errors.Is(err, fs.ErrNotExist), errors.Is(err, os.ErrNotExist), errors.Is(err, syscall.ENOENT): + return snapshotCodecFallbackReasonMissingBinary, true + case errors.Is(err, fs.ErrPermission), errors.Is(err, os.ErrPermission), errors.Is(err, syscall.EACCES), errors.Is(err, syscall.EPERM): + return snapshotCodecFallbackReasonNotExecutable, true + default: + return "", false + } +} + +func resolveNativeCodecPath(runtime nativeCodecRuntime, algorithm snapshotstore.SnapshotCompressionAlgorithm) (string, error) { + binaryName := nativeCodecBinaryName(algorithm) + if runtime.manager == nil { + return runtime.lookPathFunc()(binaryName) + } + + runtime.manager.nativeCodecMu.Lock() + if runtime.manager.nativeCodecPaths == nil { + runtime.manager.nativeCodecPaths = make(map[string]string) + } + if path := runtime.manager.nativeCodecPaths[binaryName]; path != "" { + runtime.manager.nativeCodecMu.Unlock() + return path, nil + } + runtime.manager.nativeCodecMu.Unlock() + + path, err := runtime.lookPathFunc()(binaryName) + if err != nil { + return "", err + } + + runtime.manager.nativeCodecMu.Lock() + if runtime.manager.nativeCodecPaths == nil { + runtime.manager.nativeCodecPaths = make(map[string]string) + } + runtime.manager.nativeCodecPaths[binaryName] = path + runtime.manager.nativeCodecMu.Unlock() + return path, nil +} + +func invalidateNativeCodecPath(runtime nativeCodecRuntime, algorithm snapshotstore.SnapshotCompressionAlgorithm, path string) { + if runtime.manager == nil { + return + } + + binaryName := nativeCodecBinaryName(algorithm) + runtime.manager.nativeCodecMu.Lock() + defer runtime.manager.nativeCodecMu.Unlock() + if runtime.manager.nativeCodecPaths == nil { + return + } + if currentPath := runtime.manager.nativeCodecPaths[binaryName]; currentPath == path { + delete(runtime.manager.nativeCodecPaths, binaryName) + } +} + +func recordNativeCodecFallback(ctx context.Context, runtime nativeCodecRuntime, algorithm snapshotstore.SnapshotCompressionAlgorithm, operation snapshotCodecOperation, reason snapshotCodecFallbackReason, nativeBinary string, err error) { + logger.FromContext(ctx).WarnContext( + ctx, + "native snapshot codec unavailable, falling back to go implementation", + "algorithm", string(algorithm), + "operation", string(operation), + "native_binary", nativeBinary, + "error", err, + "fallback_backend", "go", + "install_tip", nativeCodecInstallTip(algorithm), + ) + if runtime.manager != nil { + runtime.manager.recordSnapshotCodecFallback(ctx, algorithm, operation, reason) + } +} + +func runWithNativeFallback(ctx context.Context, runtime nativeCodecRuntime, algorithm snapshotstore.SnapshotCompressionAlgorithm, operation snapshotCodecOperation, nativeRunner func(binaryPath string) error, goRunner func() error) error { + binaryName := nativeCodecBinaryName(algorithm) + binaryPath, err := resolveNativeCodecPath(runtime, algorithm) + if err != nil { + reason, ok := isNativeCodecUnavailable(err) + if !ok { + return err + } + recordNativeCodecFallback(ctx, runtime, algorithm, operation, reason, binaryName, err) + return goRunner() + } + + if err := nativeRunner(binaryPath); err != nil { + reason, ok := isNativeCodecUnavailable(err) + if !ok { + return err + } + invalidateNativeCodecPath(runtime, algorithm, binaryPath) + recordNativeCodecFallback(ctx, runtime, algorithm, operation, reason, binaryPath, err) + return goRunner() + } + + return nil +} + +func (m *manager) startCompressionJob(ctx context.Context, target compressionTarget) { + if target.Key == "" || !target.Policy.Enabled { + return + } + + m.compressionMu.Lock() + if _, exists := m.compressionJobs[target.Key]; exists { + m.compressionMu.Unlock() + return + } + jobCtx, cancel := context.WithCancel(context.Background()) + job := &compressionJob{ + cancel: cancel, + done: make(chan struct{}), + target: target, + } + m.compressionJobs[target.Key] = job + m.compressionMu.Unlock() + + go func() { + start := time.Now() + result := snapshotCompressionResultSuccess + var uncompressedSize int64 + var compressedSize int64 + metricsCtx := context.Background() + log := logger.FromContext(ctx) + + defer func() { + m.recordSnapshotCompressionJob(metricsCtx, target, result, start, uncompressedSize, compressedSize) + m.compressionMu.Lock() + delete(m.compressionJobs, target.Key) + m.compressionMu.Unlock() + close(job.done) + }() + + rawPath, ok := findRawSnapshotMemoryFile(target.SnapshotDir) + if !ok { + if compressedPath, algorithm, found := findCompressedSnapshotMemoryFile(target.SnapshotDir); found && target.SnapshotID != "" { + cfg := compressionMetadataForExistingArtifact(target.Policy, algorithm) + var compressedSizeBytes *int64 + if st, statErr := os.Stat(compressedPath); statErr == nil { + size := st.Size() + compressedSizeBytes = &size + } + if err := m.updateSnapshotCompressionMetadata(target.SnapshotID, snapshotstore.SnapshotCompressionStateCompressed, "", &cfg, compressedSizeBytes, nil); err != nil { + log.ErrorContext(jobCtx, "failed to update snapshot compression metadata", "snapshot_id", target.SnapshotID, "snapshot_dir", target.SnapshotDir, "state", snapshotstore.SnapshotCompressionStateCompressed, "error", err) + } + } + return + } + + var err error + uncompressedSize, compressedSize, err = compressSnapshotMemoryFileWithRuntime(jobCtx, nativeCodecRuntime{manager: m}, rawPath, target.Policy) + if err != nil { + if errors.Is(err, context.Canceled) { + result = snapshotCompressionResultCanceled + if target.SnapshotID != "" { + if err := m.updateSnapshotCompressionMetadata(target.SnapshotID, snapshotstore.SnapshotCompressionStateNone, "", nil, nil, nil); err != nil { + log.ErrorContext(jobCtx, "failed to update snapshot compression metadata", "snapshot_id", target.SnapshotID, "snapshot_dir", target.SnapshotDir, "state", snapshotstore.SnapshotCompressionStateNone, "error", err) + } + } + return + } + result = snapshotCompressionResultFailed + if target.SnapshotID != "" { + if metadataErr := m.updateSnapshotCompressionMetadata(target.SnapshotID, snapshotstore.SnapshotCompressionStateError, err.Error(), &target.Policy, nil, nil); metadataErr != nil { + log.ErrorContext(jobCtx, "failed to update snapshot compression metadata", "snapshot_id", target.SnapshotID, "snapshot_dir", target.SnapshotDir, "state", snapshotstore.SnapshotCompressionStateError, "error", metadataErr) + } + } + log.WarnContext(jobCtx, "snapshot compression failed", "snapshot_dir", target.SnapshotDir, "error", err) + return + } + + if target.SnapshotID != "" { + if err := m.updateSnapshotCompressionMetadata(target.SnapshotID, snapshotstore.SnapshotCompressionStateCompressed, "", &target.Policy, &compressedSize, &uncompressedSize); err != nil { + log.ErrorContext(jobCtx, "failed to update snapshot compression metadata", "snapshot_id", target.SnapshotID, "snapshot_dir", target.SnapshotDir, "state", snapshotstore.SnapshotCompressionStateCompressed, "error", err) + } + } + }() +} + +func (m *manager) waitCompressionJobContext(ctx context.Context, key string) error { + m.compressionMu.Lock() + job := m.compressionJobs[key] + m.compressionMu.Unlock() + if job == nil { + return nil + } + + select { + case <-job.done: + return nil + case <-ctx.Done(): + return ctx.Err() + } +} + +func (m *manager) cancelAndWaitCompressionJob(ctx context.Context, key string) (*compressionTarget, error) { + if key == "" { + return nil, nil + } + + m.compressionMu.Lock() + job := m.compressionJobs[key] + if job != nil { + job.cancel() + } + m.compressionMu.Unlock() + + if job == nil { + return nil, nil + } + + select { + case <-job.done: + target := job.target + return &target, nil + case <-ctx.Done(): + return nil, ctx.Err() + } +} + +func (m *manager) ensureSnapshotMemoryReady(ctx context.Context, snapshotDir, jobKey string, hvType hypervisor.Type) error { + start := time.Now() + + if jobKey != "" { + target, err := m.cancelAndWaitCompressionJob(ctx, jobKey) + if err != nil { + return err + } + if target != nil { + m.recordSnapshotCompressionPreemption(ctx, snapshotCompressionPreemptionRestoreInstance, *target) + } + } + + if rawPath, ok := findRawSnapshotMemoryFile(snapshotDir); ok { + removeCompressedSnapshotArtifacts(rawPath) + m.recordSnapshotRestoreMemoryPrepare(ctx, hvType, snapshotMemoryPreparePathRaw, snapshotCompressionResultSuccess, start) + return nil + } + compressedPath, algorithm, ok := findCompressedSnapshotMemoryFile(snapshotDir) + if !ok { + return nil + } + if err := decompressSnapshotMemoryFileWithRuntime(ctx, nativeCodecRuntime{manager: m}, compressedPath, algorithm); err != nil { + m.recordSnapshotRestoreMemoryPrepare(ctx, hvType, snapshotMemoryPreparePathDecompress, snapshotCompressionResultFailed, start) + return err + } + m.recordSnapshotRestoreMemoryPrepare(ctx, hvType, snapshotMemoryPreparePathDecompress, snapshotCompressionResultSuccess, start) + return nil +} + +func (m *manager) updateSnapshotCompressionMetadata(snapshotID, state, compressionError string, cfg *snapshotstore.SnapshotCompressionConfig, compressedSize, uncompressedSize *int64) error { + rec, err := m.loadSnapshotRecord(snapshotID) + if err != nil { + return err + } + rec.Snapshot.CompressionState = state + rec.Snapshot.CompressionError = compressionError + rec.Snapshot.Compression = cloneCompressionConfig(cfg) + rec.Snapshot.CompressedSizeBytes = compressedSize + rec.Snapshot.UncompressedSizeBytes = uncompressedSize + + if state == snapshotstore.SnapshotCompressionStateCompressed { + sizeBytes, sizeErr := snapshotstore.DirectoryFileSize(m.paths.SnapshotGuestDir(snapshotID)) + if sizeErr == nil { + rec.Snapshot.SizeBytes = sizeBytes + } + } + return m.saveSnapshotRecord(rec) +} + +func findRawSnapshotMemoryFile(snapshotDir string) (string, bool) { + for _, candidate := range snapshotMemoryFileCandidates(snapshotDir) { + if st, err := os.Stat(candidate); err == nil && st.Mode().IsRegular() { + return candidate, true + } + } + return "", false +} + +func findCompressedSnapshotMemoryFile(snapshotDir string) (string, snapshotstore.SnapshotCompressionAlgorithm, bool) { + for _, raw := range snapshotMemoryFileCandidates(snapshotDir) { + zstdPath := raw + ".zst" + if st, err := os.Stat(zstdPath); err == nil && st.Mode().IsRegular() { + return zstdPath, snapshotstore.SnapshotCompressionAlgorithmZstd, true + } + lz4Path := raw + ".lz4" + if st, err := os.Stat(lz4Path); err == nil && st.Mode().IsRegular() { + return lz4Path, snapshotstore.SnapshotCompressionAlgorithmLz4, true + } + } + return "", "", false +} + +func snapshotMemoryFileCandidates(snapshotDir string) []string { + return []string{ + filepath.Join(snapshotDir, "memory-ranges"), + filepath.Join(snapshotDir, "memory"), + filepath.Join(snapshotDir, "snapshots", "snapshot-latest", "memory-ranges"), + filepath.Join(snapshotDir, "snapshots", "snapshot-latest", "memory"), + } +} + +func compressSnapshotMemoryFile(ctx context.Context, rawPath string, cfg snapshotstore.SnapshotCompressionConfig) (int64, int64, error) { + return compressSnapshotMemoryFileWithRuntime(ctx, nativeCodecRuntime{}, rawPath, cfg) +} + +func compressSnapshotMemoryFileWithRuntime(ctx context.Context, runtime nativeCodecRuntime, rawPath string, cfg snapshotstore.SnapshotCompressionConfig) (int64, int64, error) { + rawInfo, err := os.Stat(rawPath) + if err != nil { + return 0, 0, fmt.Errorf("stat raw memory snapshot: %w", err) + } + uncompressedSize := rawInfo.Size() + + compressedPath := compressedPathFor(rawPath, cfg.Algorithm) + tmpPath := compressedPath + ".tmp" + removeCompressedSnapshotArtifacts(rawPath) + _ = os.Remove(tmpPath) + + if err := runNativeCompression(ctx, runtime, rawPath, tmpPath, cfg); err != nil { + _ = os.Remove(tmpPath) + return 0, 0, err + } + if err := ctx.Err(); err != nil { + _ = os.Remove(tmpPath) + return 0, 0, err + } + if err := os.Rename(tmpPath, compressedPath); err != nil { + _ = os.Remove(tmpPath) + return 0, 0, fmt.Errorf("finalize compressed snapshot: %w", err) + } + if err := ctx.Err(); err != nil { + _ = os.Remove(compressedPath) + return 0, 0, err + } + + compressedInfo, err := os.Stat(compressedPath) + if err != nil { + return 0, 0, fmt.Errorf("stat compressed snapshot: %w", err) + } + if err := os.Remove(rawPath); err != nil { + return 0, 0, fmt.Errorf("remove raw memory snapshot: %w", err) + } + return uncompressedSize, compressedInfo.Size(), nil +} + +func runNativeCompression(ctx context.Context, runtime nativeCodecRuntime, srcPath, dstPath string, cfg snapshotstore.SnapshotCompressionConfig) error { + return runWithNativeFallback( + ctx, + runtime, + cfg.Algorithm, + snapshotCodecOperationCompress, + func(binaryPath string) error { + return runNativeCompressionCommand(ctx, runtime, binaryPath, srcPath, dstPath, cfg) + }, + func() error { + return runGoCompression(ctx, srcPath, dstPath, cfg) + }, + ) +} + +func runGoCompression(ctx context.Context, srcPath, dstPath string, cfg snapshotstore.SnapshotCompressionConfig) error { + src, err := os.Open(srcPath) + if err != nil { + return fmt.Errorf("open source snapshot: %w", err) + } + defer src.Close() + + dst, err := os.Create(dstPath) + if err != nil { + return fmt.Errorf("create compressed snapshot: %w", err) + } + closed := false + defer func() { + if !closed { + dst.Close() + } + }() + + switch cfg.Algorithm { + case snapshotstore.SnapshotCompressionAlgorithmZstd: + level := defaultSnapshotCompressionZstdLevel + if cfg.Level != nil { + level = *cfg.Level + } + enc, err := zstd.NewWriter(dst, zstd.WithEncoderLevel(zstd.EncoderLevelFromZstd(level))) + if err != nil { + return fmt.Errorf("create zstd encoder: %w", err) + } + if err := copyWithContext(ctx, enc, src); err != nil { + _ = enc.Close() + return err + } + if err := enc.Close(); err != nil { + return fmt.Errorf("close zstd encoder: %w", err) + } + case snapshotstore.SnapshotCompressionAlgorithmLz4: + enc := lz4.NewWriter(dst) + level := defaultSnapshotCompressionLz4Level + if cfg.Level != nil { + level = *cfg.Level + } + if err := enc.Apply(lz4.CompressionLevelOption(lz4CompressionLevel(level))); err != nil { + return fmt.Errorf("configure lz4 encoder: %w", err) + } + if err := copyWithContext(ctx, enc, src); err != nil { + _ = enc.Close() + return err + } + if err := enc.Close(); err != nil { + return fmt.Errorf("close lz4 encoder: %w", err) + } + default: + return fmt.Errorf("%w: unsupported compression algorithm %q", ErrInvalidRequest, cfg.Algorithm) + } + closed = true + if err := dst.Close(); err != nil { + return fmt.Errorf("close compressed snapshot file: %w", err) + } + return nil +} + +func runNativeCompressionCommand(ctx context.Context, runtime nativeCodecRuntime, binaryPath, srcPath, dstPath string, cfg snapshotstore.SnapshotCompressionConfig) error { + args := nativeCompressionArgs(srcPath, dstPath, cfg) + cmd := runtime.commandContextFunc()(ctx, binaryPath, args...) + var stderr bytes.Buffer + cmd.Stderr = &stderr + if err := cmd.Run(); err != nil { + if ctxErr := ctx.Err(); ctxErr != nil { + return ctxErr + } + return formatNativeCodecError("compress", cfg.Algorithm, err, stderr.String()) + } + return nil +} + +func nativeCompressionArgs(srcPath, dstPath string, cfg snapshotstore.SnapshotCompressionConfig) []string { + switch cfg.Algorithm { + case snapshotstore.SnapshotCompressionAlgorithmLz4: + level := defaultSnapshotCompressionLz4Level + if cfg.Level != nil { + level = *cfg.Level + } + return []string{"-q", "-f", lz4NativeCompressionFlag(level), srcPath, dstPath} + default: + level := defaultSnapshotCompressionZstdLevel + if cfg.Level != nil { + level = *cfg.Level + } + return []string{"-q", "-f", fmt.Sprintf("-%d", level), "-o", dstPath, srcPath} + } +} + +func lz4CompressionLevel(level int) lz4.CompressionLevel { + switch level { + case 0: + return lz4.Fast + case 1: + return lz4.Level1 + case 2: + return lz4.Level2 + case 3: + return lz4.Level3 + case 4: + return lz4.Level4 + case 5: + return lz4.Level5 + case 6: + return lz4.Level6 + case 7: + return lz4.Level7 + case 8: + return lz4.Level8 + case 9: + return lz4.Level9 + default: + return lz4.Fast + } +} + +func lz4NativeCompressionFlag(level int) string { + if level == 0 { + return "--fast=1" + } + return fmt.Sprintf("-%d", level) +} + +func decompressSnapshotMemoryFile(ctx context.Context, compressedPath string, algorithm snapshotstore.SnapshotCompressionAlgorithm) error { + return decompressSnapshotMemoryFileWithRuntime(ctx, nativeCodecRuntime{}, compressedPath, algorithm) +} + +func decompressSnapshotMemoryFileWithRuntime(ctx context.Context, runtime nativeCodecRuntime, compressedPath string, algorithm snapshotstore.SnapshotCompressionAlgorithm) error { + rawPath := strings.TrimSuffix(strings.TrimSuffix(compressedPath, ".zst"), ".lz4") + tmpRawPath := rawPath + ".tmp" + _ = os.Remove(tmpRawPath) + + if err := runNativeDecompression(ctx, runtime, compressedPath, tmpRawPath, algorithm); err != nil { + _ = os.Remove(tmpRawPath) + return err + } + if err := os.Rename(tmpRawPath, rawPath); err != nil { + _ = os.Remove(tmpRawPath) + return fmt.Errorf("finalize decompressed snapshot: %w", err) + } + removeCompressedSnapshotArtifacts(rawPath) + return nil +} + +func runNativeDecompression(ctx context.Context, runtime nativeCodecRuntime, srcPath, dstPath string, algorithm snapshotstore.SnapshotCompressionAlgorithm) error { + return runWithNativeFallback( + ctx, + runtime, + algorithm, + snapshotCodecOperationDecompress, + func(binaryPath string) error { + return runNativeDecompressionCommand(ctx, runtime, binaryPath, srcPath, dstPath, algorithm) + }, + func() error { + return runGoDecompression(ctx, srcPath, dstPath, algorithm) + }, + ) +} + +func runGoDecompression(ctx context.Context, compressedPath, tmpRawPath string, algorithm snapshotstore.SnapshotCompressionAlgorithm) error { + + src, err := os.Open(compressedPath) + if err != nil { + return fmt.Errorf("open compressed snapshot: %w", err) + } + defer src.Close() + + dst, err := os.Create(tmpRawPath) + if err != nil { + return fmt.Errorf("create decompressed snapshot file: %w", err) + } + closed := false + defer func() { + if !closed { + dst.Close() + } + }() + + var reader io.Reader + switch algorithm { + case snapshotstore.SnapshotCompressionAlgorithmZstd: + dec, err := zstd.NewReader(src) + if err != nil { + return fmt.Errorf("create zstd decoder: %w", err) + } + defer dec.Close() + reader = dec + case snapshotstore.SnapshotCompressionAlgorithmLz4: + reader = lz4.NewReader(src) + default: + return fmt.Errorf("%w: unsupported compression algorithm %q", ErrInvalidRequest, algorithm) + } + + if err := copyWithContext(ctx, dst, reader); err != nil { + return err + } + closed = true + if err := dst.Close(); err != nil { + return fmt.Errorf("close decompressed snapshot file: %w", err) + } + return nil +} + +func runNativeDecompressionCommand(ctx context.Context, runtime nativeCodecRuntime, binaryPath, srcPath, dstPath string, algorithm snapshotstore.SnapshotCompressionAlgorithm) error { + args := nativeDecompressionArgs(srcPath, dstPath, algorithm) + cmd := runtime.commandContextFunc()(ctx, binaryPath, args...) + var stderr bytes.Buffer + cmd.Stderr = &stderr + if err := cmd.Run(); err != nil { + if ctxErr := ctx.Err(); ctxErr != nil { + return ctxErr + } + return formatNativeCodecError("decompress", algorithm, err, stderr.String()) + } + return nil +} + +func nativeDecompressionArgs(srcPath, dstPath string, algorithm snapshotstore.SnapshotCompressionAlgorithm) []string { + switch algorithm { + case snapshotstore.SnapshotCompressionAlgorithmLz4: + return []string{"-q", "-d", "-f", srcPath, dstPath} + default: + return []string{"-q", "-d", "-f", "-o", dstPath, srcPath} + } +} + +func formatNativeCodecError(operation string, algorithm snapshotstore.SnapshotCompressionAlgorithm, err error, stderr string) error { + stderr = strings.TrimSpace(stderr) + if stderr == "" { + return fmt.Errorf("native %s %s: %w", algorithm, operation, err) + } + return fmt.Errorf("native %s %s: %w: %s", algorithm, operation, err, stderr) +} + +func compressedPathFor(rawPath string, algorithm snapshotstore.SnapshotCompressionAlgorithm) string { + switch algorithm { + case snapshotstore.SnapshotCompressionAlgorithmLz4: + return rawPath + ".lz4" + default: + return rawPath + ".zst" + } +} + +func compressionMetadataForExistingArtifact(policy snapshotstore.SnapshotCompressionConfig, algorithm snapshotstore.SnapshotCompressionAlgorithm) snapshotstore.SnapshotCompressionConfig { + cfg := snapshotstore.SnapshotCompressionConfig{ + Enabled: true, + Algorithm: algorithm, + } + if policy.Algorithm == algorithm { + cfg.Level = cloneCompressionConfig(&policy).Level + } + return cfg +} + +func removeCompressedSnapshotArtifacts(rawPath string) { + for _, path := range []string{ + rawPath + ".zst", + rawPath + ".zst.tmp", + rawPath + ".lz4", + rawPath + ".lz4.tmp", + } { + _ = os.Remove(path) + } +} + +func copyWithContext(ctx context.Context, dst io.Writer, src io.Reader) error { + buf := make([]byte, 1024*1024) + for { + if err := ctx.Err(); err != nil { + return err + } + n, readErr := src.Read(buf) + if n > 0 { + if _, writeErr := dst.Write(buf[:n]); writeErr != nil { + return fmt.Errorf("write compressed stream: %w", writeErr) + } + } + if readErr != nil { + if errors.Is(readErr, io.EOF) { + return nil + } + return fmt.Errorf("read snapshot stream: %w", readErr) + } + } +} diff --git a/lib/instances/snapshot_compression_native_test.go b/lib/instances/snapshot_compression_native_test.go new file mode 100644 index 00000000..2b655897 --- /dev/null +++ b/lib/instances/snapshot_compression_native_test.go @@ -0,0 +1,525 @@ +package instances + +import ( + "context" + "log/slog" + "os" + "os/exec" + "path/filepath" + "sync" + "syscall" + "testing" + "time" + + "github.com/kernel/hypeman/lib/logger" + "github.com/kernel/hypeman/lib/paths" + snapshotstore "github.com/kernel/hypeman/lib/snapshot" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + otelmetric "go.opentelemetry.io/otel/sdk/metric" + "go.opentelemetry.io/otel/sdk/metric/metricdata" +) + +type codecLogRecord struct { + level slog.Level + msg string + attrs map[string]string +} + +type codecLogRecorder struct { + mu sync.Mutex + records []codecLogRecord +} + +func (h *codecLogRecorder) Enabled(_ context.Context, _ slog.Level) bool { return true } + +func (h *codecLogRecorder) Handle(_ context.Context, r slog.Record) error { + rec := codecLogRecord{ + level: r.Level, + msg: r.Message, + attrs: map[string]string{}, + } + r.Attrs(func(attr slog.Attr) bool { + rec.attrs[attr.Key] = attr.Value.String() + return true + }) + h.mu.Lock() + h.records = append(h.records, rec) + h.mu.Unlock() + return nil +} + +func (h *codecLogRecorder) WithAttrs(_ []slog.Attr) slog.Handler { return h } +func (h *codecLogRecorder) WithGroup(_ string) slog.Handler { return h } + +func (h *codecLogRecorder) warnRecords() []codecLogRecord { + h.mu.Lock() + defer h.mu.Unlock() + out := make([]codecLogRecord, 0, len(h.records)) + for _, rec := range h.records { + if rec.level == slog.LevelWarn { + out = append(out, rec) + } + } + return out +} + +func TestNativeCompressionArgs(t *testing.T) { + t.Parallel() + + zstdArgs := nativeCompressionArgs("/tmp/raw", "/tmp/raw.zst.tmp", snapshotstore.SnapshotCompressionConfig{ + Enabled: true, + Algorithm: snapshotstore.SnapshotCompressionAlgorithmZstd, + Level: intPtr(19), + }) + assert.Equal(t, []string{"-q", "-f", "-19", "-o", "/tmp/raw.zst.tmp", "/tmp/raw"}, zstdArgs) + + lz4Args := nativeCompressionArgs("/tmp/raw", "/tmp/raw.lz4.tmp", snapshotstore.SnapshotCompressionConfig{ + Enabled: true, + Algorithm: snapshotstore.SnapshotCompressionAlgorithmLz4, + Level: intPtr(0), + }) + assert.Equal(t, []string{"-q", "-f", "--fast=1", "/tmp/raw", "/tmp/raw.lz4.tmp"}, lz4Args) + + lz4Args = nativeCompressionArgs("/tmp/raw", "/tmp/raw.lz4.tmp", snapshotstore.SnapshotCompressionConfig{ + Enabled: true, + Algorithm: snapshotstore.SnapshotCompressionAlgorithmLz4, + Level: intPtr(9), + }) + assert.Equal(t, []string{"-q", "-f", "-9", "/tmp/raw", "/tmp/raw.lz4.tmp"}, lz4Args) +} + +func TestNativeDecompressionArgs(t *testing.T) { + t.Parallel() + + assert.Equal(t, + []string{"-q", "-d", "-f", "-o", "/tmp/raw.tmp", "/tmp/raw.zst"}, + nativeDecompressionArgs("/tmp/raw.zst", "/tmp/raw.tmp", snapshotstore.SnapshotCompressionAlgorithmZstd), + ) + assert.Equal(t, + []string{"-q", "-d", "-f", "/tmp/raw.lz4", "/tmp/raw.tmp"}, + nativeDecompressionArgs("/tmp/raw.lz4", "/tmp/raw.tmp", snapshotstore.SnapshotCompressionAlgorithmLz4), + ) +} + +func TestIsNativeCodecUnavailable(t *testing.T) { + t.Parallel() + + reason, ok := isNativeCodecUnavailable(exec.ErrNotFound) + require.True(t, ok) + assert.Equal(t, snapshotCodecFallbackReasonMissingBinary, reason) + + reason, ok = isNativeCodecUnavailable(&os.PathError{Op: "fork/exec", Path: "/missing/zstd", Err: syscall.ENOENT}) + require.True(t, ok) + assert.Equal(t, snapshotCodecFallbackReasonMissingBinary, reason) + + reason, ok = isNativeCodecUnavailable(&os.PathError{Op: "fork/exec", Path: "/usr/bin/zstd", Err: syscall.EACCES}) + require.True(t, ok) + assert.Equal(t, snapshotCodecFallbackReasonNotExecutable, reason) + + _, ok = isNativeCodecUnavailable(assert.AnError) + assert.False(t, ok) +} + +func TestResolveNativeCodecPathCachesSuccess(t *testing.T) { + t.Parallel() + + mgr := &manager{nativeCodecPaths: make(map[string]string)} + lookups := 0 + runtime := nativeCodecRuntime{ + manager: mgr, + lookPath: func(file string) (string, error) { + lookups++ + return "/usr/bin/" + file, nil + }, + } + + path, err := resolveNativeCodecPath(runtime, snapshotstore.SnapshotCompressionAlgorithmZstd) + require.NoError(t, err) + assert.Equal(t, "/usr/bin/zstd", path) + + path, err = resolveNativeCodecPath(runtime, snapshotstore.SnapshotCompressionAlgorithmZstd) + require.NoError(t, err) + assert.Equal(t, "/usr/bin/zstd", path) + assert.Equal(t, 1, lookups) +} + +func TestResolveNativeCodecPathDoesNotCacheMiss(t *testing.T) { + t.Parallel() + + mgr := &manager{nativeCodecPaths: make(map[string]string)} + lookups := 0 + runtime := nativeCodecRuntime{ + manager: mgr, + lookPath: func(file string) (string, error) { + lookups++ + return "", exec.ErrNotFound + }, + } + + _, err := resolveNativeCodecPath(runtime, snapshotstore.SnapshotCompressionAlgorithmLz4) + require.Error(t, err) + _, err = resolveNativeCodecPath(runtime, snapshotstore.SnapshotCompressionAlgorithmLz4) + require.Error(t, err) + assert.Equal(t, 2, lookups) +} + +func TestCompressSnapshotMemoryFileFallsBackToGoWhenNativeBinaryMissing(t *testing.T) { + mgr, reader := newCodecRuntimeTestManager(t) + ctx, logs := newCodecRuntimeContext() + rawPath, original := writeRawSnapshotMemoryFile(t) + + _, compressedSize, err := compressSnapshotMemoryFileWithRuntime(ctx, nativeCodecRuntime{ + manager: mgr, + lookPath: func(file string) (string, error) { + return "", &exec.Error{Name: file, Err: exec.ErrNotFound} + }, + }, rawPath, snapshotstore.SnapshotCompressionConfig{ + Enabled: true, + Algorithm: snapshotstore.SnapshotCompressionAlgorithmZstd, + Level: intPtr(1), + }) + require.NoError(t, err) + assert.Greater(t, compressedSize, int64(0)) + + compressedPath := compressedPathFor(rawPath, snapshotstore.SnapshotCompressionAlgorithmZstd) + _, statErr := os.Stat(compressedPath) + require.NoError(t, statErr) + _, statErr = os.Stat(rawPath) + assert.Error(t, statErr) + + warns := logs.warnRecords() + require.Len(t, warns, 1) + assert.Equal(t, "native snapshot codec unavailable, falling back to go implementation", warns[0].msg) + assert.Equal(t, "zstd", warns[0].attrs["algorithm"]) + assert.Equal(t, "compress", warns[0].attrs["operation"]) + assert.Equal(t, "zstd", warns[0].attrs["native_binary"]) + assert.Equal(t, "go", warns[0].attrs["fallback_backend"]) + assert.Equal(t, nativeCodecInstallTip(snapshotstore.SnapshotCompressionAlgorithmZstd), warns[0].attrs["install_tip"]) + + var rm metricdata.ResourceMetrics + require.NoError(t, reader.Collect(context.Background(), &rm)) + fallbackMetric := findMetric(t, rm, "hypeman_snapshot_codec_fallbacks_total") + fallbacks, ok := fallbackMetric.Data.(metricdata.Sum[int64]) + require.True(t, ok) + require.Len(t, fallbacks.DataPoints, 1) + assert.Equal(t, int64(1), fallbacks.DataPoints[0].Value) + assert.Equal(t, "zstd", metricLabel(t, fallbacks.DataPoints[0].Attributes, "algorithm")) + assert.Equal(t, "compress", metricLabel(t, fallbacks.DataPoints[0].Attributes, "operation")) + assert.Equal(t, "missing_binary", metricLabel(t, fallbacks.DataPoints[0].Attributes, "reason")) + + runtimeGo := nativeCodecRuntime{lookPath: func(string) (string, error) { return "", exec.ErrNotFound }} + require.NoError(t, decompressSnapshotMemoryFileWithRuntime(context.Background(), runtimeGo, compressedPath, snapshotstore.SnapshotCompressionAlgorithmZstd)) + restored, err := os.ReadFile(rawPath) + require.NoError(t, err) + assert.Equal(t, original, restored) +} + +func TestCompressSnapshotMemoryFileFallsBackToGoWhenNativeBinaryMissingFromPATH(t *testing.T) { + mgr, reader := newCodecRuntimeTestManager(t) + ctx, logs := newCodecRuntimeContext() + rawPath, _ := writeRawSnapshotMemoryFile(t) + t.Setenv("PATH", t.TempDir()) + + _, compressedSize, err := compressSnapshotMemoryFileWithRuntime(ctx, nativeCodecRuntime{ + manager: mgr, + }, rawPath, snapshotstore.SnapshotCompressionConfig{ + Enabled: true, + Algorithm: snapshotstore.SnapshotCompressionAlgorithmZstd, + Level: intPtr(1), + }) + require.NoError(t, err) + assert.Greater(t, compressedSize, int64(0)) + + warns := logs.warnRecords() + require.Len(t, warns, 1) + assert.Equal(t, "zstd", warns[0].attrs["algorithm"]) + assert.Equal(t, "compress", warns[0].attrs["operation"]) + assert.Equal(t, "zstd", warns[0].attrs["native_binary"]) + + var rm metricdata.ResourceMetrics + require.NoError(t, reader.Collect(context.Background(), &rm)) + fallbackMetric := findMetric(t, rm, "hypeman_snapshot_codec_fallbacks_total") + fallbacks, ok := fallbackMetric.Data.(metricdata.Sum[int64]) + require.True(t, ok) + require.Len(t, fallbacks.DataPoints, 1) + assert.Equal(t, int64(1), fallbacks.DataPoints[0].Value) +} + +func TestDecompressSnapshotMemoryFileFallsBackToGoWhenNativeBinaryMissing(t *testing.T) { + mgr, reader := newCodecRuntimeTestManager(t) + ctx, logs := newCodecRuntimeContext() + rawPath, original := writeRawSnapshotMemoryFile(t) + + _, _, err := compressSnapshotMemoryFileWithRuntime(context.Background(), nativeCodecRuntime{ + lookPath: func(string) (string, error) { return "", exec.ErrNotFound }, + }, rawPath, snapshotstore.SnapshotCompressionConfig{ + Enabled: true, + Algorithm: snapshotstore.SnapshotCompressionAlgorithmLz4, + Level: intPtr(0), + }) + require.NoError(t, err) + + compressedPath := compressedPathFor(rawPath, snapshotstore.SnapshotCompressionAlgorithmLz4) + require.NoError(t, decompressSnapshotMemoryFileWithRuntime(ctx, nativeCodecRuntime{ + manager: mgr, + lookPath: func(file string) (string, error) { + return "", &exec.Error{Name: file, Err: exec.ErrNotFound} + }, + }, compressedPath, snapshotstore.SnapshotCompressionAlgorithmLz4)) + + restored, err := os.ReadFile(rawPath) + require.NoError(t, err) + assert.Equal(t, original, restored) + + warns := logs.warnRecords() + require.Len(t, warns, 1) + assert.Equal(t, "lz4", warns[0].attrs["algorithm"]) + assert.Equal(t, "decompress", warns[0].attrs["operation"]) + assert.Equal(t, "lz4", warns[0].attrs["native_binary"]) + assert.Equal(t, "go", warns[0].attrs["fallback_backend"]) + assert.Equal(t, nativeCodecInstallTip(snapshotstore.SnapshotCompressionAlgorithmLz4), warns[0].attrs["install_tip"]) + + var rm metricdata.ResourceMetrics + require.NoError(t, reader.Collect(context.Background(), &rm)) + fallbackMetric := findMetric(t, rm, "hypeman_snapshot_codec_fallbacks_total") + fallbacks, ok := fallbackMetric.Data.(metricdata.Sum[int64]) + require.True(t, ok) + require.Len(t, fallbacks.DataPoints, 1) + assert.Equal(t, int64(1), fallbacks.DataPoints[0].Value) + assert.Equal(t, "lz4", metricLabel(t, fallbacks.DataPoints[0].Attributes, "algorithm")) + assert.Equal(t, "decompress", metricLabel(t, fallbacks.DataPoints[0].Attributes, "operation")) + assert.Equal(t, "missing_binary", metricLabel(t, fallbacks.DataPoints[0].Attributes, "reason")) +} + +func TestCompressSnapshotMemoryFileDoesNotFallBackOnNativeRuntimeError(t *testing.T) { + mgr, reader := newCodecRuntimeTestManager(t) + ctx, logs := newCodecRuntimeContext() + rawPath, _ := writeRawSnapshotMemoryFile(t) + binaryPath := writeExecutableScript(t, "zstd", "#!/bin/sh\necho native boom >&2\nexit 1\n") + + _, _, err := compressSnapshotMemoryFileWithRuntime(ctx, nativeCodecRuntime{ + manager: mgr, + lookPath: func(string) (string, error) { return binaryPath, nil }, + }, rawPath, snapshotstore.SnapshotCompressionConfig{ + Enabled: true, + Algorithm: snapshotstore.SnapshotCompressionAlgorithmZstd, + Level: intPtr(1), + }) + require.Error(t, err) + assert.Contains(t, err.Error(), "native zstd compress") + + _, statErr := os.Stat(compressedPathFor(rawPath, snapshotstore.SnapshotCompressionAlgorithmZstd)) + assert.Error(t, statErr) + assert.Empty(t, logs.warnRecords()) + var rm metricdata.ResourceMetrics + require.NoError(t, reader.Collect(context.Background(), &rm)) +} + +func TestCompressSnapshotMemoryFileReturnsContextCanceledWhenNativeProcessIsKilled(t *testing.T) { + mgr, reader := newCodecRuntimeTestManager(t) + baseCtx, logs := newCodecRuntimeContext() + ctx, cancel := context.WithCancel(baseCtx) + rawPath, _ := writeRawSnapshotMemoryFile(t) + binaryPath := writeExecutableScript(t, "zstd", "#!/bin/sh\nsleep 30\n") + time.AfterFunc(20*time.Millisecond, cancel) + + _, _, err := compressSnapshotMemoryFileWithRuntime(ctx, nativeCodecRuntime{ + manager: mgr, + lookPath: func(string) (string, error) { return binaryPath, nil }, + }, rawPath, snapshotstore.SnapshotCompressionConfig{ + Enabled: true, + Algorithm: snapshotstore.SnapshotCompressionAlgorithmZstd, + Level: intPtr(1), + }) + require.ErrorIs(t, err, context.Canceled) + assert.Empty(t, logs.warnRecords()) + + var rm metricdata.ResourceMetrics + require.NoError(t, reader.Collect(context.Background(), &rm)) +} + +func TestDecompressSnapshotMemoryFileDoesNotFallBackOnNativeRuntimeError(t *testing.T) { + mgr, reader := newCodecRuntimeTestManager(t) + ctx, logs := newCodecRuntimeContext() + rawPath, original := writeRawSnapshotMemoryFile(t) + + _, _, err := compressSnapshotMemoryFileWithRuntime(context.Background(), nativeCodecRuntime{ + lookPath: func(string) (string, error) { return "", exec.ErrNotFound }, + }, rawPath, snapshotstore.SnapshotCompressionConfig{ + Enabled: true, + Algorithm: snapshotstore.SnapshotCompressionAlgorithmZstd, + Level: intPtr(1), + }) + require.NoError(t, err) + + compressedPath := compressedPathFor(rawPath, snapshotstore.SnapshotCompressionAlgorithmZstd) + binaryPath := writeExecutableScript(t, "zstd", "#!/bin/sh\necho native boom >&2\nexit 1\n") + err = decompressSnapshotMemoryFileWithRuntime(ctx, nativeCodecRuntime{ + manager: mgr, + lookPath: func(string) (string, error) { return binaryPath, nil }, + }, compressedPath, snapshotstore.SnapshotCompressionAlgorithmZstd) + require.Error(t, err) + assert.Contains(t, err.Error(), "native zstd decompress") + + _, statErr := os.Stat(rawPath) + assert.Error(t, statErr) + compressedBytes, readErr := os.ReadFile(compressedPath) + require.NoError(t, readErr) + assert.NotEmpty(t, compressedBytes) + assert.NotEqual(t, original, compressedBytes) + assert.Empty(t, logs.warnRecords()) + + var rm metricdata.ResourceMetrics + require.NoError(t, reader.Collect(context.Background(), &rm)) +} + +func TestCompressSnapshotMemoryFileInvalidatesStaleNativeCodecCache(t *testing.T) { + t.Parallel() + + mgr := &manager{nativeCodecPaths: map[string]string{"zstd": "/missing/zstd"}} + rawPath, _ := writeRawSnapshotMemoryFile(t) + + _, _, err := compressSnapshotMemoryFileWithRuntime(context.Background(), nativeCodecRuntime{ + manager: mgr, + lookPath: func(string) (string, error) { return "", exec.ErrNotFound }, + }, rawPath, snapshotstore.SnapshotCompressionConfig{ + Enabled: true, + Algorithm: snapshotstore.SnapshotCompressionAlgorithmZstd, + Level: intPtr(1), + }) + require.NoError(t, err) + assert.Empty(t, mgr.nativeCodecPaths) +} + +func TestNativeCompressGoDecompressCompatibility(t *testing.T) { + for _, tc := range compressionCompatibilityCases() { + t.Run(string(tc.algorithm), func(t *testing.T) { + binaryPath, err := exec.LookPath(nativeCodecBinaryName(tc.algorithm)) + if err != nil { + t.Skipf("native %s binary not available: %v", tc.algorithm, err) + } + + rawPath, original := writeRawSnapshotMemoryFile(t) + _, _, err = compressSnapshotMemoryFileWithRuntime(context.Background(), nativeCodecRuntime{ + lookPath: func(string) (string, error) { return binaryPath, nil }, + }, rawPath, tc.cfg) + require.NoError(t, err) + + compressedPath := compressedPathFor(rawPath, tc.algorithm) + require.NoError(t, decompressSnapshotMemoryFileWithRuntime(context.Background(), nativeCodecRuntime{ + lookPath: func(string) (string, error) { return "", exec.ErrNotFound }, + }, compressedPath, tc.algorithm)) + + restored, err := os.ReadFile(rawPath) + require.NoError(t, err) + assert.Equal(t, original, restored) + }) + } +} + +func TestGoCompressNativeDecompressCompatibility(t *testing.T) { + for _, tc := range compressionCompatibilityCases() { + t.Run(string(tc.algorithm), func(t *testing.T) { + binaryPath, err := exec.LookPath(nativeCodecBinaryName(tc.algorithm)) + if err != nil { + t.Skipf("native %s binary not available: %v", tc.algorithm, err) + } + + rawPath, original := writeRawSnapshotMemoryFile(t) + _, _, err = compressSnapshotMemoryFileWithRuntime(context.Background(), nativeCodecRuntime{ + lookPath: func(string) (string, error) { return "", exec.ErrNotFound }, + }, rawPath, tc.cfg) + require.NoError(t, err) + + compressedPath := compressedPathFor(rawPath, tc.algorithm) + require.NoError(t, decompressSnapshotMemoryFileWithRuntime(context.Background(), nativeCodecRuntime{ + lookPath: func(string) (string, error) { return binaryPath, nil }, + }, compressedPath, tc.algorithm)) + + restored, err := os.ReadFile(rawPath) + require.NoError(t, err) + assert.Equal(t, original, restored) + }) + } +} + +func TestNativeCompressNativeDecompressRoundTrip(t *testing.T) { + for _, tc := range compressionCompatibilityCases() { + t.Run(string(tc.algorithm), func(t *testing.T) { + binaryPath, err := exec.LookPath(nativeCodecBinaryName(tc.algorithm)) + if err != nil { + t.Skipf("native %s binary not available: %v", tc.algorithm, err) + } + + rawPath, original := writeRawSnapshotMemoryFile(t) + runtime := nativeCodecRuntime{lookPath: func(string) (string, error) { return binaryPath, nil }} + _, _, err = compressSnapshotMemoryFileWithRuntime(context.Background(), runtime, rawPath, tc.cfg) + require.NoError(t, err) + compressedPath := compressedPathFor(rawPath, tc.algorithm) + require.NoError(t, decompressSnapshotMemoryFileWithRuntime(context.Background(), runtime, compressedPath, tc.algorithm)) + + restored, err := os.ReadFile(rawPath) + require.NoError(t, err) + assert.Equal(t, original, restored) + }) + } +} + +type compressionCompatibilityCase struct { + algorithm snapshotstore.SnapshotCompressionAlgorithm + cfg snapshotstore.SnapshotCompressionConfig +} + +func compressionCompatibilityCases() []compressionCompatibilityCase { + return []compressionCompatibilityCase{ + { + algorithm: snapshotstore.SnapshotCompressionAlgorithmZstd, + cfg: snapshotstore.SnapshotCompressionConfig{Enabled: true, Algorithm: snapshotstore.SnapshotCompressionAlgorithmZstd, Level: intPtr(1)}, + }, + { + algorithm: snapshotstore.SnapshotCompressionAlgorithmLz4, + cfg: snapshotstore.SnapshotCompressionConfig{Enabled: true, Algorithm: snapshotstore.SnapshotCompressionAlgorithmLz4, Level: intPtr(0)}, + }, + } +} + +func newCodecRuntimeTestManager(t *testing.T) (*manager, *otelmetric.ManualReader) { + t.Helper() + + reader := otelmetric.NewManualReader() + provider := otelmetric.NewMeterProvider(otelmetric.WithReader(reader)) + mgr := &manager{ + paths: paths.New(t.TempDir()), + compressionJobs: make(map[string]*compressionJob), + nativeCodecPaths: make(map[string]string), + } + metrics, err := newInstanceMetrics(provider.Meter("test"), nil, mgr) + require.NoError(t, err) + mgr.metrics = metrics + return mgr, reader +} + +func newCodecRuntimeContext() (context.Context, *codecLogRecorder) { + recorder := &codecLogRecorder{} + log := slog.New(recorder) + return logger.AddToContext(context.Background(), log), recorder +} + +func writeRawSnapshotMemoryFile(t *testing.T) (string, []byte) { + t.Helper() + + dir := t.TempDir() + rawPath := filepath.Join(dir, "memory-ranges") + content := []byte("snapshot-memory-contents-for-native-codec-tests") + require.NoError(t, os.WriteFile(rawPath, content, 0o644)) + return rawPath, content +} + +func writeExecutableScript(t *testing.T, name, body string) string { + t.Helper() + + path := filepath.Join(t.TempDir(), name) + require.NoError(t, os.WriteFile(path, []byte(body), 0o755)) + return path +} diff --git a/lib/instances/snapshot_compression_test.go b/lib/instances/snapshot_compression_test.go new file mode 100644 index 00000000..274f9c50 --- /dev/null +++ b/lib/instances/snapshot_compression_test.go @@ -0,0 +1,347 @@ +package instances + +import ( + "context" + "errors" + "os" + "testing" + "time" + + "github.com/kernel/hypeman/lib/hypervisor" + snapshotstore "github.com/kernel/hypeman/lib/snapshot" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestNormalizeCompressionConfig(t *testing.T) { + t.Parallel() + + cfg, err := normalizeCompressionConfig(nil) + require.NoError(t, err) + assert.False(t, cfg.Enabled) + + cfg, err = normalizeCompressionConfig(&snapshotstore.SnapshotCompressionConfig{ + Enabled: true, + }) + require.NoError(t, err) + assert.True(t, cfg.Enabled) + assert.Equal(t, snapshotstore.SnapshotCompressionAlgorithmZstd, cfg.Algorithm) + require.NotNil(t, cfg.Level) + assert.Equal(t, 1, *cfg.Level) + + _, err = normalizeCompressionConfig(&snapshotstore.SnapshotCompressionConfig{ + Enabled: true, + Algorithm: snapshotstore.SnapshotCompressionAlgorithmZstd, + Level: intPtr(25), + }) + require.Error(t, err) + assert.True(t, errors.Is(err, ErrInvalidRequest)) + + _, err = normalizeCompressionConfig(&snapshotstore.SnapshotCompressionConfig{ + Enabled: true, + Algorithm: snapshotstore.SnapshotCompressionAlgorithmLz4, + Level: intPtr(10), + }) + require.Error(t, err) + assert.True(t, errors.Is(err, ErrInvalidRequest)) + + cfg, err = normalizeCompressionConfig(&snapshotstore.SnapshotCompressionConfig{ + Enabled: true, + Algorithm: snapshotstore.SnapshotCompressionAlgorithmLz4, + Level: intPtr(9), + }) + require.NoError(t, err) + assert.True(t, cfg.Enabled) + assert.Equal(t, snapshotstore.SnapshotCompressionAlgorithmLz4, cfg.Algorithm) + require.NotNil(t, cfg.Level) + assert.Equal(t, 9, *cfg.Level) + + cfg, err = normalizeCompressionConfig(&snapshotstore.SnapshotCompressionConfig{ + Enabled: true, + Algorithm: snapshotstore.SnapshotCompressionAlgorithm("ZSTD"), + Level: intPtr(3), + }) + require.NoError(t, err) + assert.Equal(t, snapshotstore.SnapshotCompressionAlgorithmZstd, cfg.Algorithm) + require.NotNil(t, cfg.Level) + assert.Equal(t, 3, *cfg.Level) +} + +func TestResolveSnapshotCompressionPolicyPrecedence(t *testing.T) { + t.Parallel() + + m := &manager{ + snapshotDefaults: SnapshotPolicy{ + Compression: &snapshotstore.SnapshotCompressionConfig{ + Enabled: true, + Algorithm: snapshotstore.SnapshotCompressionAlgorithmZstd, + Level: intPtr(2), + }, + }, + } + + stored := &StoredMetadata{ + SnapshotPolicy: &SnapshotPolicy{ + Compression: &snapshotstore.SnapshotCompressionConfig{ + Enabled: true, + Algorithm: snapshotstore.SnapshotCompressionAlgorithmLz4, + }, + }, + } + + cfg, err := m.resolveSnapshotCompressionPolicy(stored, &snapshotstore.SnapshotCompressionConfig{ + Enabled: true, + Algorithm: snapshotstore.SnapshotCompressionAlgorithmZstd, + Level: intPtr(4), + }) + require.NoError(t, err) + assert.Equal(t, snapshotstore.SnapshotCompressionAlgorithmZstd, cfg.Algorithm) + require.NotNil(t, cfg.Level) + assert.Equal(t, 4, *cfg.Level) + + cfg, err = m.resolveSnapshotCompressionPolicy(stored, nil) + require.NoError(t, err) + assert.Equal(t, snapshotstore.SnapshotCompressionAlgorithmLz4, cfg.Algorithm) + require.NotNil(t, cfg.Level) + assert.Equal(t, 0, *cfg.Level) + + cfg, err = m.resolveSnapshotCompressionPolicy(&StoredMetadata{}, nil) + require.NoError(t, err) + assert.Equal(t, snapshotstore.SnapshotCompressionAlgorithmZstd, cfg.Algorithm) + require.NotNil(t, cfg.Level) + assert.Equal(t, 2, *cfg.Level) +} + +func TestResolveStandbyCompressionPolicyIsOptInOnly(t *testing.T) { + t.Parallel() + + m := &manager{} + + cfg, err := m.resolveStandbyCompressionPolicy(&StoredMetadata{ + HypervisorType: "cloud-hypervisor", + }, nil) + require.NoError(t, err) + assert.Nil(t, cfg) + + cfg, err = m.resolveStandbyCompressionPolicy(&StoredMetadata{ + HypervisorType: "cloud-hypervisor", + }, &snapshotstore.SnapshotCompressionConfig{Enabled: false}) + require.NoError(t, err) + assert.Nil(t, cfg) + + cfg, err = m.resolveStandbyCompressionPolicy(&StoredMetadata{ + HypervisorType: "qemu", + }, nil) + require.NoError(t, err) + assert.Nil(t, cfg) +} + +func TestResolveCompressionPolicyExplicitDisableOverridesDefaults(t *testing.T) { + t.Parallel() + + m := &manager{ + snapshotDefaults: SnapshotPolicy{ + Compression: &snapshotstore.SnapshotCompressionConfig{ + Enabled: true, + Algorithm: snapshotstore.SnapshotCompressionAlgorithmZstd, + Level: intPtr(3), + }, + }, + } + + stored := &StoredMetadata{ + SnapshotPolicy: &SnapshotPolicy{ + Compression: &snapshotstore.SnapshotCompressionConfig{ + Enabled: false, + }, + }, + } + + cfg, err := m.resolveSnapshotCompressionPolicy(stored, nil) + require.NoError(t, err) + assert.False(t, cfg.Enabled) + + standbyCfg, err := m.resolveStandbyCompressionPolicy(stored, nil) + require.NoError(t, err) + assert.Nil(t, standbyCfg) +} + +func TestResolveStandbyCompressionPolicyInvalidConfiguredDefaultIsInvalidRequest(t *testing.T) { + t.Parallel() + + m := &manager{ + snapshotDefaults: SnapshotPolicy{ + Compression: &snapshotstore.SnapshotCompressionConfig{ + Enabled: true, + Algorithm: snapshotstore.SnapshotCompressionAlgorithm("brotli"), + }, + }, + } + + _, err := m.resolveStandbyCompressionPolicy(&StoredMetadata{}, nil) + require.Error(t, err) + assert.True(t, errors.Is(err, ErrInvalidRequest)) +} + +func TestCompressionMetadataForExistingArtifactUsesActualAlgorithm(t *testing.T) { + t.Parallel() + + cfg := compressionMetadataForExistingArtifact(snapshotstore.SnapshotCompressionConfig{ + Enabled: true, + Algorithm: snapshotstore.SnapshotCompressionAlgorithmZstd, + Level: intPtr(7), + }, snapshotstore.SnapshotCompressionAlgorithmLz4) + assert.True(t, cfg.Enabled) + assert.Equal(t, snapshotstore.SnapshotCompressionAlgorithmLz4, cfg.Algorithm) + assert.Nil(t, cfg.Level) + + cfg = compressionMetadataForExistingArtifact(snapshotstore.SnapshotCompressionConfig{ + Enabled: true, + Algorithm: snapshotstore.SnapshotCompressionAlgorithmZstd, + Level: intPtr(7), + }, snapshotstore.SnapshotCompressionAlgorithmZstd) + assert.True(t, cfg.Enabled) + assert.Equal(t, snapshotstore.SnapshotCompressionAlgorithmZstd, cfg.Algorithm) + require.NotNil(t, cfg.Level) + assert.Equal(t, 7, *cfg.Level) +} + +func TestValidateCreateRequestSnapshotPolicy(t *testing.T) { + t.Parallel() + + req := &CreateInstanceRequest{ + Name: "compression-test", + Image: "docker.io/library/alpine:latest", + SnapshotPolicy: &SnapshotPolicy{ + Compression: &snapshotstore.SnapshotCompressionConfig{ + Enabled: true, + Algorithm: snapshotstore.SnapshotCompressionAlgorithmZstd, + Level: intPtr(0), + }, + }, + } + err := validateCreateRequest(req) + require.Error(t, err) + assert.True(t, errors.Is(err, ErrInvalidRequest)) +} + +func TestValidateCreateSnapshotRequestRejectsStoppedCompression(t *testing.T) { + t.Parallel() + + err := validateCreateSnapshotRequest(CreateSnapshotRequest{ + Kind: SnapshotKindStopped, + Compression: &snapshotstore.SnapshotCompressionConfig{ + Enabled: true, + }, + }) + require.Error(t, err) + assert.True(t, errors.Is(err, ErrInvalidRequest)) +} + +func TestDeleteInstanceCancelsCompressionJob(t *testing.T) { + t.Parallel() + + mgr, _ := setupTestManager(t) + ctx := context.Background() + const instanceID = "delete-instance-compression" + + require.NoError(t, mgr.ensureDirectories(instanceID)) + now := time.Now() + require.NoError(t, mgr.saveMetadata(&metadata{StoredMetadata: StoredMetadata{ + Id: instanceID, + Name: instanceID, + DataDir: mgr.paths.InstanceDir(instanceID), + HypervisorType: hypervisor.TypeCloudHypervisor, + CreatedAt: now, + StoppedAt: &now, + }})) + + target := installCancelableCompressionJob(mgr, compressionTarget{ + Key: mgr.snapshotJobKeyForInstance(instanceID), + OwnerID: instanceID, + SnapshotDir: mgr.paths.InstanceSnapshotLatest(instanceID), + Source: snapshotCompressionSourceStandby, + Policy: snapshotstore.SnapshotCompressionConfig{ + Enabled: true, + Algorithm: snapshotstore.SnapshotCompressionAlgorithmZstd, + Level: intPtr(1), + }, + }) + + require.NoError(t, mgr.DeleteInstance(ctx, instanceID)) + assertCompressionJobCanceled(t, mgr, target) + _, err := os.Stat(mgr.paths.InstanceDir(instanceID)) + require.Error(t, err) + assert.True(t, os.IsNotExist(err)) +} + +func TestDeleteSnapshotCancelsCompressionJob(t *testing.T) { + t.Parallel() + + mgr, _ := setupTestManager(t) + ctx := context.Background() + const snapshotID = "delete-snapshot-compression" + + snapshotDir := mgr.paths.SnapshotGuestDir(snapshotID) + require.NoError(t, os.MkdirAll(snapshotDir, 0o755)) + require.NoError(t, mgr.saveSnapshotRecord(&snapshotRecord{ + Snapshot: Snapshot{ + Id: snapshotID, + Name: snapshotID, + Kind: SnapshotKindStandby, + CreatedAt: time.Now(), + }, + })) + + target := installCancelableCompressionJob(mgr, compressionTarget{ + Key: mgr.snapshotJobKeyForSnapshot(snapshotID), + SnapshotID: snapshotID, + SnapshotDir: snapshotDir, + Source: snapshotCompressionSourceSnapshot, + Policy: snapshotstore.SnapshotCompressionConfig{ + Enabled: true, + Algorithm: snapshotstore.SnapshotCompressionAlgorithmLz4, + Level: intPtr(0), + }, + }) + + require.NoError(t, mgr.DeleteSnapshot(ctx, snapshotID)) + assertCompressionJobCanceled(t, mgr, target) + _, err := os.Stat(mgr.paths.SnapshotDir(snapshotID)) + require.Error(t, err) + assert.True(t, os.IsNotExist(err)) +} + +func installCancelableCompressionJob(mgr *manager, target compressionTarget) *compressionTarget { + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan struct{}) + + mgr.compressionMu.Lock() + mgr.compressionJobs[target.Key] = &compressionJob{ + cancel: cancel, + done: done, + target: target, + } + mgr.compressionMu.Unlock() + + go func() { + <-ctx.Done() + mgr.compressionMu.Lock() + delete(mgr.compressionJobs, target.Key) + mgr.compressionMu.Unlock() + close(done) + }() + + return &target +} + +func assertCompressionJobCanceled(t *testing.T, mgr *manager, target *compressionTarget) { + t.Helper() + + require.Eventually(t, func() bool { + mgr.compressionMu.Lock() + defer mgr.compressionMu.Unlock() + _, ok := mgr.compressionJobs[target.Key] + return !ok + }, time.Second, 10*time.Millisecond) +} diff --git a/lib/instances/snapshot_integration_scenario_test.go b/lib/instances/snapshot_integration_scenario_test.go index ab30cb56..fb2147e5 100644 --- a/lib/instances/snapshot_integration_scenario_test.go +++ b/lib/instances/snapshot_integration_scenario_test.go @@ -68,8 +68,7 @@ func runStandbySnapshotScenario(t *testing.T, mgr *manager, tmpDir string, cfg s source, err = waitForInstanceState(ctx, mgr, sourceID, StateRunning, 20*time.Second) requireNoErr(err) require.Equal(t, StateRunning, source.State) - - _, err = mgr.StandbyInstance(ctx, sourceID) + _, err = mgr.StandbyInstance(ctx, sourceID, StandbyInstanceRequest{}) requireNoErr(err) snapshot, err := mgr.CreateSnapshot(ctx, sourceID, CreateSnapshotRequest{ diff --git a/lib/instances/snapshot_test.go b/lib/instances/snapshot_test.go index 0d154cc0..cc634f55 100644 --- a/lib/instances/snapshot_test.go +++ b/lib/instances/snapshot_test.go @@ -4,10 +4,12 @@ import ( "context" "os" "path/filepath" + "sync/atomic" "testing" "time" "github.com/kernel/hypeman/lib/hypervisor" + snapshotstore "github.com/kernel/hypeman/lib/snapshot" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -81,6 +83,203 @@ func TestStandbySnapshotRejectsTargetHypervisorOverride(t *testing.T) { assert.ErrorIs(t, err, ErrInvalidRequest) } +func TestRestoreSnapshotCancelsSourceInstanceCompressionJob(t *testing.T) { + t.Parallel() + + mgr, _ := setupTestManager(t) + ctx := context.Background() + + hvType := mgr.defaultHypervisor + sourceID := "snapshot-restore-src" + snapshotID := "snapshot-restore-race" + + createStandbySnapshotSourceFixture(t, mgr, sourceID, "snapshot-restore-src", hvType) + + snapshotGuestDir := mgr.paths.SnapshotGuestDir(snapshotID) + require.NoError(t, os.MkdirAll(mgr.paths.SnapshotDir(snapshotID), 0755)) + require.NoError(t, mgr.copySnapshotPayload(sourceID, snapshotGuestDir)) + + sourceMeta, err := mgr.loadMetadata(sourceID) + require.NoError(t, err) + require.NoError(t, mgr.saveSnapshotRecord(&snapshotRecord{ + Snapshot: Snapshot{ + Id: snapshotID, + Name: "restore-race", + Kind: SnapshotKindStandby, + SourceInstanceID: sourceID, + SourceName: sourceMeta.Name, + SourceHypervisor: hvType, + CreatedAt: time.Now(), + SizeBytes: 1, + }, + StoredMetadata: sourceMeta.StoredMetadata, + })) + + var instanceCanceled atomic.Bool + var snapshotCanceled atomic.Bool + instanceDone := make(chan struct{}) + snapshotDone := make(chan struct{}) + + mgr.compressionMu.Lock() + mgr.compressionJobs[mgr.snapshotJobKeyForInstance(sourceID)] = &compressionJob{ + cancel: func() { + instanceCanceled.Store(true) + select { + case <-instanceDone: + default: + close(instanceDone) + } + }, + done: instanceDone, + target: compressionTarget{ + Key: mgr.snapshotJobKeyForInstance(sourceID), + OwnerID: sourceID, + SnapshotDir: mgr.paths.InstanceSnapshotLatest(sourceID), + }, + } + mgr.compressionJobs[mgr.snapshotJobKeyForSnapshot(snapshotID)] = &compressionJob{ + cancel: func() { + snapshotCanceled.Store(true) + select { + case <-snapshotDone: + default: + close(snapshotDone) + } + }, + done: snapshotDone, + target: compressionTarget{ + Key: mgr.snapshotJobKeyForSnapshot(snapshotID), + SnapshotID: snapshotID, + SnapshotDir: snapshotGuestDir, + }, + } + mgr.compressionMu.Unlock() + + restored, err := mgr.RestoreSnapshot(ctx, sourceID, snapshotID, RestoreSnapshotRequest{ + TargetState: StateStandby, + }) + require.NoError(t, err) + require.Equal(t, StateStandby, restored.State) + assert.True(t, snapshotCanceled.Load(), "snapshot compression job should be canceled before restore") + assert.True(t, instanceCanceled.Load(), "instance compression job should be canceled before restore") +} + +func TestCreateStandbySnapshotCancelsSourceInstanceCompressionJob(t *testing.T) { + t.Parallel() + + mgr, _ := setupTestManager(t) + ctx := context.Background() + + hvType := mgr.defaultHypervisor + sourceID := "snapshot-create-src" + createStandbySnapshotSourceFixture(t, mgr, sourceID, "snapshot-create-src", hvType) + + var instanceCanceled atomic.Bool + instanceDone := make(chan struct{}) + + mgr.compressionMu.Lock() + mgr.compressionJobs[mgr.snapshotJobKeyForInstance(sourceID)] = &compressionJob{ + cancel: func() { + instanceCanceled.Store(true) + select { + case <-instanceDone: + default: + close(instanceDone) + } + }, + done: instanceDone, + target: compressionTarget{ + Key: mgr.snapshotJobKeyForInstance(sourceID), + OwnerID: sourceID, + SnapshotDir: mgr.paths.InstanceSnapshotLatest(sourceID), + }, + } + mgr.compressionMu.Unlock() + + snap, err := mgr.CreateSnapshot(ctx, sourceID, CreateSnapshotRequest{ + Kind: SnapshotKindStandby, + Name: "standby-copy", + }) + require.NoError(t, err) + require.Equal(t, SnapshotKindStandby, snap.Kind) + assert.True(t, instanceCanceled.Load(), "instance compression job should be canceled before copying standby snapshot payload") +} + +func TestCreateStandbySnapshotFromCompressedSourceCopiesRawMemory(t *testing.T) { + t.Parallel() + + mgr, _ := setupTestManager(t) + ctx := context.Background() + + hvType := mgr.defaultHypervisor + sourceID := "snapshot-create-compressed-src" + createStandbySnapshotSourceFixture(t, mgr, sourceID, "snapshot-create-compressed-src", hvType) + + rawPath := filepath.Join(mgr.paths.InstanceSnapshotLatest(sourceID), "memory-ranges") + require.NoError(t, os.WriteFile(rawPath, []byte("some guest memory"), 0644)) + _, _, err := compressSnapshotMemoryFile(ctx, rawPath, snapshotstore.SnapshotCompressionConfig{ + Enabled: true, + Algorithm: snapshotstore.SnapshotCompressionAlgorithmZstd, + Level: intPtr(1), + }) + require.NoError(t, err) + + snap, err := mgr.CreateSnapshot(ctx, sourceID, CreateSnapshotRequest{ + Kind: SnapshotKindStandby, + Name: "standby-from-compressed", + }) + require.NoError(t, err) + + snapshotDir := mgr.paths.SnapshotGuestDir(snap.Id) + _, ok := findRawSnapshotMemoryFile(snapshotDir) + assert.True(t, ok, "snapshot copy should contain raw memory after preparing a compressed standby source") + _, _, ok = findCompressedSnapshotMemoryFile(snapshotDir) + assert.False(t, ok, "snapshot copy should not inherit compressed memory artifacts from the source standby instance") +} + +func TestForkSnapshotFromCompressedSourceCopiesRawMemory(t *testing.T) { + t.Parallel() + + mgr, _ := setupTestManager(t) + ctx := context.Background() + + hvType := mgr.defaultHypervisor + sourceID := "snapshot-fork-compressed-src" + createStandbySnapshotSourceFixture(t, mgr, sourceID, "snapshot-fork-compressed-src", hvType) + + snap, err := mgr.CreateSnapshot(ctx, sourceID, CreateSnapshotRequest{ + Kind: SnapshotKindStandby, + Name: "standby-for-fork-compressed", + }) + require.NoError(t, err) + + snapshotDir := mgr.paths.SnapshotGuestDir(snap.Id) + rawPath := filepath.Join(snapshotDir, "memory-ranges") + snapshotConfigPath := filepath.Join(snapshotDir, "snapshots", "snapshot-latest", "config.json") + require.NoError(t, os.MkdirAll(filepath.Dir(snapshotConfigPath), 0o755)) + require.NoError(t, os.WriteFile(snapshotConfigPath, []byte(`{}`), 0o644)) + require.NoError(t, os.WriteFile(rawPath, []byte("some guest memory"), 0o644)) + _, _, err = compressSnapshotMemoryFile(ctx, rawPath, snapshotstore.SnapshotCompressionConfig{ + Enabled: true, + Algorithm: snapshotstore.SnapshotCompressionAlgorithmZstd, + Level: intPtr(1), + }) + require.NoError(t, err) + + forked, err := mgr.ForkSnapshot(ctx, snap.Id, ForkSnapshotRequest{ + Name: "snapshot-fork-compressed", + TargetState: StateStopped, + }) + require.NoError(t, err) + t.Cleanup(func() { _ = mgr.DeleteInstance(context.Background(), forked.Id) }) + + forkSnapshotDir := mgr.paths.InstanceDir(forked.Id) + _, ok := findRawSnapshotMemoryFile(forkSnapshotDir) + assert.True(t, ok, "forked snapshot payload should contain raw memory after preparing a compressed snapshot source") + _, _, ok = findCompressedSnapshotMemoryFile(forkSnapshotDir) + assert.False(t, ok, "forked snapshot payload should not retain compressed memory artifacts from the source snapshot") +} + func createStoppedSnapshotSourceFixture(t *testing.T, mgr *manager, id, name string, hvType hypervisor.Type) { t.Helper() require.NoError(t, mgr.ensureDirectories(id)) diff --git a/lib/instances/standby.go b/lib/instances/standby.go index 51ef7b85..94edc1ea 100644 --- a/lib/instances/standby.go +++ b/lib/instances/standby.go @@ -2,6 +2,7 @@ package instances import ( "context" + "errors" "fmt" "os" "path/filepath" @@ -11,6 +12,7 @@ import ( "github.com/kernel/hypeman/lib/hypervisor" "github.com/kernel/hypeman/lib/logger" "github.com/kernel/hypeman/lib/network" + snapshotstore "github.com/kernel/hypeman/lib/snapshot" "go.opentelemetry.io/otel/trace" ) @@ -20,6 +22,8 @@ func (m *manager) standbyInstance( ctx context.Context, id string, + req StandbyInstanceRequest, + skipCompression bool, ) (*Instance, error) { start := time.Now() log := logger.FromContext(ctx) @@ -55,6 +59,20 @@ func (m *manager) standbyInstance( return nil, fmt.Errorf("%w: standby is not supported for instances with vGPU attached (driver limitation)", ErrInvalidState) } + // Resolve/validate compression policy early so invalid request/config + // fails before any state transition side effects. + var compressionPolicy *snapshotstore.SnapshotCompressionConfig + if !skipCompression { + policy, err := m.resolveStandbyCompressionPolicy(stored, req.Compression) + if err != nil { + if !errors.Is(err, ErrInvalidRequest) { + err = fmt.Errorf("%w: %v", ErrInvalidRequest, err) + } + return nil, err + } + compressionPolicy = policy + } + // 3. Get network allocation BEFORE killing VMM (while we can still query it) // This is needed to delete the TAP device after VMM shuts down var networkAlloc *network.Allocation @@ -163,6 +181,18 @@ func (m *manager) standbyInstance( // Return instance with derived state (should be Standby now) finalInst := m.toInstance(ctx, meta) + + if compressionPolicy != nil { + m.startCompressionJob(ctx, compressionTarget{ + Key: m.snapshotJobKeyForInstance(stored.Id), + OwnerID: stored.Id, + SnapshotDir: snapshotDir, + HypervisorType: stored.HypervisorType, + Source: snapshotCompressionSourceStandby, + Policy: *compressionPolicy, + }) + } + log.InfoContext(ctx, "instance put in standby successfully", "instance_id", id, "state", finalInst.State) return &finalInst, nil } diff --git a/lib/instances/types.go b/lib/instances/types.go index 66c0841f..fb8f355d 100644 --- a/lib/instances/types.go +++ b/lib/instances/types.go @@ -136,6 +136,9 @@ type StoredMetadata struct { SkipKernelHeaders bool // Skip kernel headers installation (disables DKMS) SkipGuestAgent bool // Skip guest-agent installation (disables exec/stat API) + // Snapshot policy defaults for this instance. + SnapshotPolicy *SnapshotPolicy + // Shutdown configuration StopTimeout int // Grace period in seconds for graceful stop (0 = use default 5s) @@ -216,6 +219,7 @@ type CreateInstanceRequest struct { Cmd []string // Override image cmd (nil = use image default) SkipKernelHeaders bool // Skip kernel headers installation (disables DKMS) SkipGuestAgent bool // Skip guest-agent installation (disables exec/stat API) + SnapshotPolicy *SnapshotPolicy // Optional snapshot policy defaults for this instance } // StartInstanceRequest is the domain request for starting a stopped instance @@ -256,9 +260,15 @@ type ListSnapshotsFilter = snapshot.ListSnapshotsFilter // CreateSnapshotRequest is the domain request for creating a snapshot. type CreateSnapshotRequest struct { - Kind SnapshotKind // Required: Standby or Stopped - Name string // Optional: unique per source instance - Tags tags.Tags // Optional user-defined key-value tags + Kind SnapshotKind // Required: Standby or Stopped + Name string // Optional: unique per source instance + Tags tags.Tags // Optional user-defined key-value tags + Compression *snapshot.SnapshotCompressionConfig // Optional compression override +} + +// StandbyInstanceRequest is the domain request for putting an instance into standby. +type StandbyInstanceRequest struct { + Compression *snapshot.SnapshotCompressionConfig // Optional compression override } // RestoreSnapshotRequest is the domain request for restoring a snapshot in-place. @@ -274,6 +284,11 @@ type ForkSnapshotRequest struct { TargetHypervisor hypervisor.Type // Optional, allowed only for Stopped snapshots } +// SnapshotPolicy defines default snapshot behavior for an instance. +type SnapshotPolicy struct { + Compression *snapshot.SnapshotCompressionConfig +} + // AttachVolumeRequest is the domain request for attaching a volume (used for API compatibility) type AttachVolumeRequest struct { MountPath string diff --git a/lib/oapi/oapi.go b/lib/oapi/oapi.go index 25a6f140..7a07110f 100644 --- a/lib/oapi/oapi.go +++ b/lib/oapi/oapi.go @@ -143,6 +143,14 @@ const ( RestoreSnapshotRequestTargetHypervisorVz RestoreSnapshotRequestTargetHypervisor = "vz" ) +// Defines values for SnapshotCompressionState. +const ( + SnapshotCompressionStateCompressed SnapshotCompressionState = "compressed" + SnapshotCompressionStateCompressing SnapshotCompressionState = "compressing" + SnapshotCompressionStateError SnapshotCompressionState = "error" + SnapshotCompressionStateNone SnapshotCompressionState = "none" +) + // Defines values for SnapshotSourceHypervisor. const ( SnapshotSourceHypervisorCloudHypervisor SnapshotSourceHypervisor = "cloud-hypervisor" @@ -151,6 +159,12 @@ const ( SnapshotSourceHypervisorVz SnapshotSourceHypervisor = "vz" ) +// Defines values for SnapshotCompressionConfigAlgorithm. +const ( + Lz4 SnapshotCompressionConfigAlgorithm = "lz4" + Zstd SnapshotCompressionConfigAlgorithm = "zstd" +) + // Defines values for SnapshotKind. const ( SnapshotKindStandby SnapshotKind = "Standby" @@ -386,7 +400,8 @@ type CreateInstanceRequest struct { // When true, DKMS (Dynamic Kernel Module Support) will not work, // preventing compilation of out-of-tree kernel modules (e.g., NVIDIA vGPU drivers). // Recommended for workloads that don't need kernel module compilation. - SkipKernelHeaders *bool `json:"skip_kernel_headers,omitempty"` + SkipKernelHeaders *bool `json:"skip_kernel_headers,omitempty"` + SnapshotPolicy *SnapshotPolicy `json:"snapshot_policy,omitempty"` // Tags User-defined key-value tags. Tags *Tags `json:"tags,omitempty"` @@ -462,6 +477,8 @@ type CreateInstanceRequestNetworkEgressEnforcementMode string // CreateSnapshotRequest defines model for CreateSnapshotRequest. type CreateSnapshotRequest struct { + Compression *SnapshotCompressionConfig `json:"compression,omitempty"` + // Kind Snapshot capture kind Kind SnapshotKind `json:"kind"` @@ -808,7 +825,8 @@ type Instance struct { OverlaySize *string `json:"overlay_size,omitempty"` // Size Base memory size (human-readable) - Size *string `json:"size,omitempty"` + Size *string `json:"size,omitempty"` + SnapshotPolicy *SnapshotPolicy `json:"snapshot_policy,omitempty"` // StartedAt Start timestamp (RFC3339) StartedAt *time.Time `json:"started_at"` @@ -1066,6 +1084,16 @@ type RestoreSnapshotRequestTargetHypervisor string // Snapshot defines model for Snapshot. type Snapshot struct { + // CompressedSizeBytes Compressed memory payload size in bytes + CompressedSizeBytes *int64 `json:"compressed_size_bytes"` + Compression *SnapshotCompressionConfig `json:"compression,omitempty"` + + // CompressionError Compression error message when compression_state is error + CompressionError *string `json:"compression_error"` + + // CompressionState Compression status of the snapshot payload memory file + CompressionState *SnapshotCompressionState `json:"compression_state,omitempty"` + // CreatedAt Snapshot creation timestamp CreatedAt time.Time `json:"created_at"` @@ -1092,17 +1120,48 @@ type Snapshot struct { // Tags User-defined key-value tags. Tags *Tags `json:"tags,omitempty"` + + // UncompressedSizeBytes Uncompressed memory payload size in bytes + UncompressedSizeBytes *int64 `json:"uncompressed_size_bytes"` } +// SnapshotCompressionState Compression status of the snapshot payload memory file +type SnapshotCompressionState string + // SnapshotSourceHypervisor Source instance hypervisor at snapshot creation time type SnapshotSourceHypervisor string +// SnapshotCompressionConfig defines model for SnapshotCompressionConfig. +type SnapshotCompressionConfig struct { + // Algorithm Compression algorithm (defaults to zstd when enabled). Ignored when enabled is false. + Algorithm *SnapshotCompressionConfigAlgorithm `json:"algorithm,omitempty"` + + // Enabled Enable snapshot memory compression + Enabled bool `json:"enabled"` + + // Level Compression level. Allowed ranges are zstd=1-19 and lz4=0-9. When omitted, zstd defaults to 1 and lz4 defaults to 0. Ignored when enabled is false. + Level *int `json:"level,omitempty"` +} + +// SnapshotCompressionConfigAlgorithm Compression algorithm (defaults to zstd when enabled). Ignored when enabled is false. +type SnapshotCompressionConfigAlgorithm string + // SnapshotKind Snapshot capture kind type SnapshotKind string +// SnapshotPolicy defines model for SnapshotPolicy. +type SnapshotPolicy struct { + Compression *SnapshotCompressionConfig `json:"compression,omitempty"` +} + // SnapshotTargetState Target state when restoring or forking from a snapshot type SnapshotTargetState string +// StandbyInstanceRequest defines model for StandbyInstanceRequest. +type StandbyInstanceRequest struct { + Compression *SnapshotCompressionConfig `json:"compression,omitempty"` +} + // Tags User-defined key-value tags. type Tags map[string]string @@ -1351,6 +1410,9 @@ type CreateInstanceSnapshotJSONRequestBody = CreateSnapshotRequest // RestoreInstanceSnapshotJSONRequestBody defines body for RestoreInstanceSnapshot for application/json ContentType. type RestoreInstanceSnapshotJSONRequestBody = RestoreSnapshotRequest +// StandbyInstanceJSONRequestBody defines body for StandbyInstance for application/json ContentType. +type StandbyInstanceJSONRequestBody = StandbyInstanceRequest + // StartInstanceJSONRequestBody defines body for StartInstance for application/json ContentType. type StartInstanceJSONRequestBody StartInstanceJSONBody @@ -1542,8 +1604,10 @@ type ClientInterface interface { RestoreInstanceSnapshot(ctx context.Context, id string, snapshotId string, body RestoreInstanceSnapshotJSONRequestBody, reqEditors ...RequestEditorFn) (*http.Response, error) - // StandbyInstance request - StandbyInstance(ctx context.Context, id string, reqEditors ...RequestEditorFn) (*http.Response, error) + // StandbyInstanceWithBody request with any body + StandbyInstanceWithBody(ctx context.Context, id string, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*http.Response, error) + + StandbyInstance(ctx context.Context, id string, body StandbyInstanceJSONRequestBody, reqEditors ...RequestEditorFn) (*http.Response, error) // StartInstanceWithBody request with any body StartInstanceWithBody(ctx context.Context, id string, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*http.Response, error) @@ -2051,8 +2115,20 @@ func (c *Client) RestoreInstanceSnapshot(ctx context.Context, id string, snapsho return c.Client.Do(req) } -func (c *Client) StandbyInstance(ctx context.Context, id string, reqEditors ...RequestEditorFn) (*http.Response, error) { - req, err := NewStandbyInstanceRequest(c.Server, id) +func (c *Client) StandbyInstanceWithBody(ctx context.Context, id string, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*http.Response, error) { + req, err := NewStandbyInstanceRequestWithBody(c.Server, id, contentType, body) + if err != nil { + return nil, err + } + req = req.WithContext(ctx) + if err := c.applyEditors(ctx, req, reqEditors); err != nil { + return nil, err + } + return c.Client.Do(req) +} + +func (c *Client) StandbyInstance(ctx context.Context, id string, body StandbyInstanceJSONRequestBody, reqEditors ...RequestEditorFn) (*http.Response, error) { + req, err := NewStandbyInstanceRequest(c.Server, id, body) if err != nil { return nil, err } @@ -3544,8 +3620,19 @@ func NewRestoreInstanceSnapshotRequestWithBody(server string, id string, snapsho return req, nil } -// NewStandbyInstanceRequest generates requests for StandbyInstance -func NewStandbyInstanceRequest(server string, id string) (*http.Request, error) { +// NewStandbyInstanceRequest calls the generic StandbyInstance builder with application/json body +func NewStandbyInstanceRequest(server string, id string, body StandbyInstanceJSONRequestBody) (*http.Request, error) { + var bodyReader io.Reader + buf, err := json.Marshal(body) + if err != nil { + return nil, err + } + bodyReader = bytes.NewReader(buf) + return NewStandbyInstanceRequestWithBody(server, id, "application/json", bodyReader) +} + +// NewStandbyInstanceRequestWithBody generates requests for StandbyInstance with any type of body +func NewStandbyInstanceRequestWithBody(server string, id string, contentType string, body io.Reader) (*http.Request, error) { var err error var pathParam0 string @@ -3570,11 +3657,13 @@ func NewStandbyInstanceRequest(server string, id string) (*http.Request, error) return nil, err } - req, err := http.NewRequest("POST", queryURL.String(), nil) + req, err := http.NewRequest("POST", queryURL.String(), body) if err != nil { return nil, err } + req.Header.Add("Content-Type", contentType) + return req, nil } @@ -4529,8 +4618,10 @@ type ClientWithResponsesInterface interface { RestoreInstanceSnapshotWithResponse(ctx context.Context, id string, snapshotId string, body RestoreInstanceSnapshotJSONRequestBody, reqEditors ...RequestEditorFn) (*RestoreInstanceSnapshotResponse, error) - // StandbyInstanceWithResponse request - StandbyInstanceWithResponse(ctx context.Context, id string, reqEditors ...RequestEditorFn) (*StandbyInstanceResponse, error) + // StandbyInstanceWithBodyWithResponse request with any body + StandbyInstanceWithBodyWithResponse(ctx context.Context, id string, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*StandbyInstanceResponse, error) + + StandbyInstanceWithResponse(ctx context.Context, id string, body StandbyInstanceJSONRequestBody, reqEditors ...RequestEditorFn) (*StandbyInstanceResponse, error) // StartInstanceWithBodyWithResponse request with any body StartInstanceWithBodyWithResponse(ctx context.Context, id string, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*StartInstanceResponse, error) @@ -5311,6 +5402,7 @@ type StandbyInstanceResponse struct { Body []byte HTTPResponse *http.Response JSON200 *Instance + JSON400 *Error JSON404 *Error JSON409 *Error JSON500 *Error @@ -6074,9 +6166,17 @@ func (c *ClientWithResponses) RestoreInstanceSnapshotWithResponse(ctx context.Co return ParseRestoreInstanceSnapshotResponse(rsp) } -// StandbyInstanceWithResponse request returning *StandbyInstanceResponse -func (c *ClientWithResponses) StandbyInstanceWithResponse(ctx context.Context, id string, reqEditors ...RequestEditorFn) (*StandbyInstanceResponse, error) { - rsp, err := c.StandbyInstance(ctx, id, reqEditors...) +// StandbyInstanceWithBodyWithResponse request with arbitrary body returning *StandbyInstanceResponse +func (c *ClientWithResponses) StandbyInstanceWithBodyWithResponse(ctx context.Context, id string, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*StandbyInstanceResponse, error) { + rsp, err := c.StandbyInstanceWithBody(ctx, id, contentType, body, reqEditors...) + if err != nil { + return nil, err + } + return ParseStandbyInstanceResponse(rsp) +} + +func (c *ClientWithResponses) StandbyInstanceWithResponse(ctx context.Context, id string, body StandbyInstanceJSONRequestBody, reqEditors ...RequestEditorFn) (*StandbyInstanceResponse, error) { + rsp, err := c.StandbyInstance(ctx, id, body, reqEditors...) if err != nil { return nil, err } @@ -7576,6 +7676,13 @@ func ParseStandbyInstanceResponse(rsp *http.Response) (*StandbyInstanceResponse, } response.JSON200 = &dest + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 400: + var dest Error + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.JSON400 = &dest + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 404: var dest Error if err := json.Unmarshal(bodyBytes, &dest); err != nil { @@ -11813,7 +11920,8 @@ func (response RestoreInstanceSnapshot501JSONResponse) VisitRestoreInstanceSnaps } type StandbyInstanceRequestObject struct { - Id string `json:"id"` + Id string `json:"id"` + Body *StandbyInstanceJSONRequestBody } type StandbyInstanceResponseObject interface { @@ -11829,6 +11937,15 @@ func (response StandbyInstance200JSONResponse) VisitStandbyInstanceResponse(w ht return json.NewEncoder(w).Encode(response) } +type StandbyInstance400JSONResponse Error + +func (response StandbyInstance400JSONResponse) VisitStandbyInstanceResponse(w http.ResponseWriter) error { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(400) + + return json.NewEncoder(w).Encode(response) +} + type StandbyInstance404JSONResponse Error func (response StandbyInstance404JSONResponse) VisitStandbyInstanceResponse(w http.ResponseWriter) error { @@ -13540,6 +13657,13 @@ func (sh *strictHandler) StandbyInstance(w http.ResponseWriter, r *http.Request, request.Id = id + var body StandbyInstanceJSONRequestBody + if err := json.NewDecoder(r.Body).Decode(&body); err != nil { + sh.options.RequestErrorHandlerFunc(w, r, fmt.Errorf("can't decode JSON body: %w", err)) + return + } + request.Body = &body + handler := func(ctx context.Context, w http.ResponseWriter, r *http.Request, request interface{}) (interface{}, error) { return sh.ssi.StandbyInstance(ctx, request.(StandbyInstanceRequestObject)) } @@ -14039,233 +14163,239 @@ func (sh *strictHandler) GetVolume(w http.ResponseWriter, r *http.Request, id st // Base64 encoded, gzipped, json marshaled Swagger object var swaggerSpec = []string{ - "H4sIAAAAAAAC/+x97XIbOZLgqyB4s9HUNElRH5ZlbXTsyZLt1rZl6yxLezNNHwVWgSRaVUA1gKJMO/x3", - "HmAecZ7kAgmgvogii7IlW2NvbEzLLHwmMhOZifz42Ap4nHBGmJKtg48tGUxJjOHPQ6VwML3kURqTN+TP", - "lEilf04ET4hQlECjmKdMDROspvpfIZGBoIminLUOWmdYTdHNlAiCZjAKklOeRiEaEQT9SNjqtMh7HCcR", - "aR20NmOmNkOscKvTUvNE/ySVoGzS+tRpCYJDzqK5mWaM00i1DsY4kqRTmfZUD42wRLpLF/pk4404jwhm", - "rU8w4p8pFSRsHfxe3Ma7rDEf/UECpSc/nGEa4VFEjsmMBmQRDEEqBGFqGAo6I2IRFEfmezRHI56yEJl2", - "qM3SKEJ0jBhnZKMEDDajIdWQ0E301K0DJVLigUwIaxrS0HMCRyfIfEYnx6g9Je/Lk2w/Hu236odkOCaL", - "g/6axph1NXD1stz40LY49std38iUx3E6nAieJosjn7w+Pb1A8BGxNB4RURxxfzsbjzJFJkToAZOADnEY", - "CiKlf//uY3Ft/X6/f4C3D/r9Xt+3yhlhIRe1IDWf/SDd6odkyZCNQGrHXwDpq8uT45NDdMRFwgWGvgsz", - "VRC7CJ7ivopoUz4VH/4/TWkULmL9SP9MxJAyqTCrwcET+1GDi4+RmhJk+6HLU9Qec4FCMkonE8omG03w", - "XTOsiCgSDrFanA6WimwbyhlSNCZS4ThpdVpjLmLdqRViRbr6S6MJBcErptMtGk22SGqpOclhLOtGd00Q", - "ZSimUUQlCTgLZXEOytTebv1mCgRDhOAeDvVM/4xiIiWeENTWbFPzboakwiqViEo0xjQiYaMz8iGC2cwf", - "fIRoSJiiY1qmb4NOXTwKtrZ3vLwjxhMyDOnE3kTl4Y/hd41iehyFoLV/I5rQ5s32AVMKMl6c7zmwbphE", - "kDERROP4Z06XCD4jTFOLnu8vMG/rf23mV/SmvZ83AZhnefNPndafKUnJMOGSmhUucC77RaMRgBpBD/+a", - "4dOysy5glFRYLKcPaPEFKNGsrxFszk3TT52WwpOVXd7qNlXeCazRTlniArUs8tmMMI+QFHCm7IcydF7y", - "CYooI8i2sGeheaKe4JeIA0v8QnDIwL9I/Hrdt2Be5oea0fS3TouwNNbAjPikCM0pwUKNSAmYNVeYHShf", - "XS34z0rkU7mrsCTD5RzkjDJGQqRbWsI2LVEqQVJd2D5Q0TVVwxkR0ktzsKzfqEK2Re1QEQ+uxzQiwymW", - "U7NiHIZArzg6K+3EI62VxF+caCboBgQpQiLF0fmvh9uP9pCdwANDyVMRmBUs7qTQWw9v2iKFxQhHkRc3", - "6tFt/Tt6EUP8GHCeEUbd3ZNhoENMw+la9jT18J1Wksqp+Qt4t14V3H2aDWj0ivTf7zybPgImYbSEWp3J", - "LwO+Tsxho0nENUznKGX0z7QkYPfQidYVFNIXBQ1J2EEYPmiWjVPFuxPCiNB8Co0Fj0HaKgjBqE16k14H", - "DbRc2NVScBdvd/v9bn/QKoux0W53kqQaFFgpIvQC/9/vuPvhsPv3fvfJu/zPYa/77ue/+BCgqWTupEK7", - "z7aj/Q5yiy2K69WFrhLlb839i8v3cRxz1CeaT6x70kcni4KD2WvIg2siepRvRnQksJhvsgll7w8irIhU", - "5Z0vb/tFYQH7WAIENtFgWhMMFaUH0Lgd8RsiAs2BI6IRT3Y0E6ZKdhDWejMwL6Rvyf9EAWaaFoxwwQUi", - "LEQ3VE0RhnZlaMXzLk5ol5qltjqtGL9/SdhETVsHezsLeK6RvG3/6L77q/tp47+8qC7SiHiQ/A1PFWUT", - "BJ/NrT6lEuVroIrEK0/EQTeNQMyLKTsx3baylWAh8PzzT9htZNlJG2Wu9qiD2CP5v54RIWjobtWj02PU", - "jug1seiORMrQIO33dwJoAH8S+0vA4xiz0Py20UOvY6r0bZbml7SxBvWKx/17iwRTDnJGFHG9oQzUNUJM", - "DsNAENBPcLT0Gl4GYi+wjrJxFy/tX7lU3RgzPCGgTdqGaCT4NdELRQmPaECJRNdkroWUOZroQbszKqkm", - "H8JmaIaN0aA3YG+nXBLTxH3SikhA6IygmAfXKIlwQKYcFPEZjlIiO+hmqiUGzYwFwZH9GQkSY8oGbKoX", - "KQOekFDrEKYZbA1dETa7QjFOgEqxIECiKMaKCIoj+oGEiJsuMQmpvqAGjABeowRrkg0CLvTtq8+W4GBa", - "gMJPEl0ZeeMKhr+iTGPllaGr3oAVT/5j6/XF26evL14dD1+fPXt1eDL87dnf9M+mU+vg948tY9/MBI2n", - "BAsi0F8+wn4/Gek0JKJ10DpM1ZQL+sEYWz51WhoGUuMXTmiPJ4Rh2gt43Oq0/lr857tP75w8pacibKbJ", - "wLOwT15ZxlyFHo5y7Ix5ElkDEYh2GEy1wGFenF1s6ss1wVKqqeDpZFomDHuzr0USIZXXQ8qHo8S3Jiqv", - "0cnma6TlDhRRTaCZnLHV758+3ZSDlv7HI/ePjR46NlQLy9cshAsr/sipRh8thAPKHJ1dIBxFPLAmkLHW", - "lcZ0kgoS9iqWNxjdx58JU2KecOrTwSrMKW+6yKO63fzrGqxoc0TZptTH0A3Wgzvgza01gWdsRgVnsdbG", - "ZlhQfc3KMq28en38bPjs1WXrQPPxMA2sUfHs9Zu3rYPWTr/fb/kQVGPQCh744uziCE7KkI1KonQylPSD", - "RxI4zPaHYhJzYTRg2we1p2VBwdAtgsMZtHZePDXItfUC8ModSkgltHajmIHLGLP94qkPW6bzhIgZlT4z", - "2a/ZN3fyhWvdsPsybksiZkRkSAtY3CuoH0HE07BbmLLTGlNBAoE12rU6rT9JrOXw2QeNOvnaPf381qtG", - "8ucKwRJHCWVkiWT5jUh4N1xcRxyH3a0vLOAxovTYi1t8ZT6Uz9fiBMlQotVZsEaw8IaGajoM+Q3TS/bw", - "VfsFZY0z5vpe7wRH//rHPy9PczVp68UosZx2a/vRZ3LaCm/VQ3tNINlG0sS/jYvEv4nL03/9459uJ193", - "E0YQuZVQZ8//mRkBWLbG9bD0TGmsmWWw/M+UqCkRhdvbIYv+yejD0B053CtspWQeLb5pLjBqPiMiwvMC", - "47Vram31gftVViWoAlq1/TQbvUa68wo2rEdzl/yLqo6+3fczWs+iPGt6qnmFvRearCRbyNb2qf1ze3FJ", - "NSu6pskQpOYhnmQm22WvzefXNLGiOPQwxxhFhhGEKQjvI85Vb8D+Z0oYgrODAybvSQA8Tyqs0OHZiUQ3", - "NIrAwANMZfFq0YJ9zlZMc6n0/4qUddAoVVpa54ogqzfBJCmsBRqPCEoZds/ZFdnZbrCKVxYs10QwEg2N", - "bCwbQsZ0QrZTLXBgq2MsFRGG26dJGV7Hv52eo/bxnOGYBug3M+opD9OIoPM00fxgowy9zoAlgsy0CsEm", - "YGykdl4+RjxVXT7uKkGIW2IMg2UmMvvWOntxdmFf6+VGb8DeEA1YwkISwprdjSORmmKFQs5+0hRLwvKw", - "xfkrQPfT8jqqfKc1C5K0fCLb1dN4Be/peu8zKlSKI80qS9Kg93ndOG54pH7jF1LUPizbypATq/K7aFN7", - "hxkZvDgWZWK/2cIIOo3NFgVNfMGA4dTEj80Wu2L8E+YWstRsk2uKnzHXuRmkCiI7dsft7BZQOslgUoYV", - "/jLgOZQFzbrWLB4SqSgz6KTbIivQSdS+0sq4xWOtfl910NVfSz9o0nWagRYPbpCBBrADpn8qjl+1KazU", - "9pvrdJXDwfL253Eoa/2M0GwLKYGZ1FejFpES0kO/Ag9GisSJZkRsgqhE0vBOEiLGb/4TcSOTuK4Dppcm", - "jZeGBUdm85F0wiibbGgpXd8rOAyNYWicqlTodjMqc2iWUccZX6obeGtWRww7jVOpL9QgSkOCrpyB5qos", - "1i2abxY1OmvPWVBQDEhAMQFdTW3GqdLT6w3HWAVTDSeeKuO2ZbcuywsoG4lWPWfatWQPXbc4//OMXZSB", - "as0FFcavN2efWMCqVzAv1lnxrJzhtzBekzkcubMm4gV7YtGQ6Df3CSJ5NCP21iyaIkc4uDZXifGcsFZI", - "Y0+0JkRN/hUS9RrXVh2Fhldj8Jcl/UVUAguu3WyOMVZ4N+bbecaF9ObMfB2t10oCwAfN4QCBNHXVMaoO", - "AQMCYhpZIhRSQQK1MDxlkwEDD44r+0vPjnaliVyLGD4i9OkqXlGuoKyYPqWjRYWTdVIbDKO3xmOqFAk7", - "ZdngmpBErt6Ulo6t3dljHBfkRlDHyKy9J2woXRE25iIgsZXxP0/ve1YYzKuFrTfEokOFgW9hzRafEE6S", - "iJLQeO+Y8wArqbTnBCbSqsduWFG6zAN+ecorHEVXqG0bbSBB9F6kOyvGWY7sb4/OHApkj86Xpx2NkZoL", - "XE2VSob6f+RQU/FVdTDb11G4Hk7fSRLt90E92t3dsadqbWZmwZVhy+Yxr1NC/dGcM5zIKVe171rXlIWr", - "EMUN8ptuW2sUywQaaZvftV0sEaSbJhOBwTH1S1rFbv3aCNCs57wrfM59zoUZVINUKh4XXAxRu+IYQcsu", - "FGVgzXjUDbHCYEFsaOY0y110143nZiijQ9UZQIaTkcfbhn7Q3BJN6ASP5qpstt/q+zS1z336dWvxHUud", - "27vR/Eg4VHy54y8dI9e2iZ8f3ANDxYezMfWMnF1HudcIlSio+NhbfVQP0U0CarV4kE2CqfHLNEAAYe/y", - "tPhk1huwLlybB+g4myAbNhsSg0yIQ/Ng0eaisAgKzl5oNN9AGF2e9tDbbLU/SaQVjRlxcQBTLNGIEIZS", - "sPjCLdY1d2hxAamEy05Vu1uThQkZ2ICXQW6/9dCv84TE2Jp/NCnEWNEAHIxGtLIfuEbMQdmnWMyKxqdG", - "xqJl7tJvyIRKJSrO0qj95vnRzs7Ok6rZcPtRt7/V3Xr0dqt/0Nf///fmftVfPirCN9ZhmbdYl60i9zm6", - "ODnetjbK8jzqwy5+sv/+PVZP9uiNfPIhHonJHzv4XuIm/KzsOPc1Q+1UEtF1bFJjlc/DrODIVeNBdmvH", - "sDvy88rdVpe1NZB4q1veRUCIz9XYOrquH7JRZZgrnZULm1vUwOcJ6Is5lRQkL+sTGFCv9+MxlddPBcHX", - "Ib9hnns7xhMih+Y+87sRpNL4tpD31iohOFdjaZ4ry9bKrd3Hu/s7e7v7/b4nDmIR4XlAh4G+gRot4PXR", - "CYrwnAgEfVAb3plCNIr4qIzoj3b29h/3n2xtN12HeVlpBodMYXK9UNtC5GcXU+e+lBa1vf14b2dnp7+3", - "t73baFXWzttoUc4mXBJJHu883t3a395tBAWfIP7MxaVUfedDn8eA1nvMG19XJiSgYxogiGxBugNqx3CF", - "keyRqEyTIxwOrdHDf3coTCO51FHBTGZbGgNZnEaKJhEx3+BAGtmQYefHMJLPCYQyRsQwC9tZYyQbzbPy", - "Yd7tJWuCSlFZJdCdUglSSC48URKFB4ZCV/I5OM18Ye/q8MDuoSE2vNSqUzciMxIVkcBcXXqxMRcEZXhi", - "Dq20K8pmOKLhkLIk9aJELSifpwJkUTMowiOeKvO6BwdWnAR8hUH3GGt23Uw/fc7F9UqvS30TD0XKmB5m", - "pTXnEAzgY2tigVscI9vbOfYXhL7sFc68VdrvEr0xPYxlJ/85SRWiTHGtnbJwNO/ATNYCxJAgUnHgpNbQ", - "Z4dpKl365RYwcjqvCzNfzjvvyeWkOzav9F9WwxYTooZSYbVSYtGY8hban0Pzxk7cuuNKA0gDuDNycx9A", - "By/3rkbbrmQ4uRuIL/MBy2wNeSO4hQUNSQ8BdYEziouqq1DaueJJQsLM/tMbsHNDKtlP0rx86I4GDmpK", - "qEBc0AktT1w2jN2lM9k6qOiw6dboWOy4KKHCR/CaqCd6PFZEGAi6gOFi1I89hFanZWHf6rQsJyqDxv3o", - "gUju4biwxBdnF+u6hCWCj2nk2S64INivVjNzzlIvd/vn3a3/YxwfNb6BiEaZcVuIeUh6lZh8aN/s5nlx", - "dnFWt6YsIQIqrm5hT5mjiYdzZP4IDiL2Mci+JloNxqG/vliySXLZ+4lPlh0LHJNROh4TMYw9xrXn+jsy", - "DYxHEWXo9GlZntVyc1Ot+ax0OKA2j3Fg49mbQd9jkKtso1OA5jv/cb0h5hqui4LTRyVsGxsI10OvshQU", - "6MXZhUS5c5DHUlc+3lo39bPpXNIAR2ZEE9RKWdHABsjZWEI+yztaU6RHTo69sqEjBNSeTZIUyPD8Tffk", - "9eVmHJJZp7QmcOiZ8ojodW8UuMXMxcLlPvUlJjGrs3QYxJBNCagAq4yCGwOpQK8e6CiucDSUEfc5WbzV", - "HxF8RO3L5yZWSa+gg5LSUerfC1Ao4feel2I0R6qb9hwmrJpMSwTu1R3LmVuMeaWwvdKkPlL5leDIJKwp", - "43MeVu0Onl+XD5pfr6ReO4hv3hPnj90gZuro9NgIDAFnClNGBIqJwjY9TsE1BcShVqfV1XdUiEkMHm7j", - "/1zulVJjgi8GQdUacY8Wsl3ciQG3Jkr7jXEdCFGMGR0TqWyUdmlmOcXbj/YOTC6JkIx3H+31er11Q0Oe", - "5bEgjY5i03jOF6JEenL6eedwBxEgTfbysXV2+PbX1kFrM5ViM+IBjjbliLKDwr+zf+Yf4A/zzxFl3siR", - "RulH6Hgh7Uj5SVPfWeb3A70TZl25NC5xUOBXPjHV6DPgkQDhat4oXYUnWj8xGPe54bi3TtiRZ41ShUQd", - "RUfOBkk76IflllAnGEEbO2fKFI3yfCaLNtBbZaSRS4P2FwL2E8KyMP0oMn8FnM00Vfhi9ksM3H37rPcD", - "650yDKkHk//HanvGuQGCmVbTW2sTJ8lqtPULihn/a5qrxEYUe26ir871b/PGVp799eS///y/8uzxH1t/", - "vry8/NvsxX8fv6J/u4zOXn9W4NLyYPKvGhH+xYLA4WGpFAneFJVOsQo8AtWUS1UDYfsFKW78LHvoCBS/", - "gwHropdUEYGjAzRoVVx7By3UJu9xoEwvxBnSQ9kAgw3d+cyYf3Tnj063/FQdI7SRBMIeSBZAJNNRyGNM", - "2caADZgdC7mNSHjT13+FKMCJSgXRp6dl2GiORgIHeQRBPnkHfcRJ8mljwEDDJe+V0DtIsFBZ9gs3AyCF", - "XZXxGbDNSejisY2GPGDZvZSFYxsbTS8zgoBtvuop6QeKV33hohwBs9/3Ba6Dt5Y+yIhKRcChOsNsjUaZ", - "Gxna75dYxX5/v79SwM9waAn6ASUs5qZ0SNmAlgwCw9SGcYNnWQNbuuZNhkbQr2/fnmkw6P+eIzdQDovs", - "iI2SZ3z3pLERqkgWvPY2Wv6IEH26DTdkjGTQLWoQrPPMuHW+fXmOFBGxc7RvBxqcYxro/cHzP5Uy1ahI", - "MTo8On220WuQXBNgm61/yTm+zXZYDcqwRrM6W2CG8Rq+HXRyDG61lkJzAQ7cap5zgSLDYHK6PkAXkpR9", - "VOGozKu+OclonlvezA0waG24EZMqpzhAbzK5EWdLyRwkc2RwQ+Z0CcPahxfj87MwesWfFryZrF5kWRt4", - "+GCVOXfrG7eeFSwnfw/EgeatP3bBprkebReNoXoyP2rkZ3/n0srOujrqunkRyqGLhbDXLDVC85wGd5Eb", - "YFFfe0/VsPYRHunP9sndaSWXp2iKJftJwceKbrK187hRkko9a9Pn6+LDNR+bJWVU5eIgs2dXExF6TaPI", - "eDNIOmE4Qk9Q+/zkxW8nL19uoC56/fq0ehTLevjOp0GKBIfaL84uIEoFy6F7Aap3esS54zB5T6WSi2Gi", - "jR5Sl6dk+LWUNsEbd7vxBXMpuNfnhW3cR5aEr+nW9+1laFiaU+FzEyNYYfeO8iLUMldfToEynzU/f9kM", - "B3eynFLMjo8/FGUC53N965QCnRb1+JseSs0CSYhOzvLMgrlRyg1f2dOT7d7W3n5vq9/vbfWbmOhiHCyZ", - "+/TwqPnk/W1jiDjAo4MgPCDjzzARWsQ2whuObvBcooETrwctI88XBPkC2VoRvNHz62LmhtslaqgKFKtS", - "MayTeqFZToUl6YHPy4mBG8toj/7+WTmESdOb2bou2F7DdYzXBAU8jUItB4005Rm1ioRW+5NE5TmXgVgv", - "2DXjN6y8dWPD1PT7Z0rEHF2enpYs3oKMbUrZBhsHl4eac+DJWsewvUJUXrmaW6Y3uI+UBlWuWbitvngC", - "g6LJzblQGgxtYHrLpUfvszdl5mg0nizZU8VoEpLZME19QpH+5AInLi5OjkvIgfHe1n5//0l3f7S1190N", - "+1tdvLWz191+hPvjneDxTk1S9+ZuL7f3ZClTc32gEgAeDJAmDi080PSWuaKMUoUyNzVNyEdaukQFMdaE", - "5YBN4IRRBZkPKZvoYUBFt1KuiYs0yRkpowoC8SGLC2V6y2AL0YNY56MD9ALawiccQ7iQW4TWbcpmABzO", - "jRlUMwY3dQL/Wr7k82mqtNgFfeQ0VUj/C7atwWC1jeVDGB5zgF5x6COcjyjjVbXFNAffq8XmVRWnbb2C", - "nPcoTGYZ5gF6njHJjM1attqWxP5peLd1bAan7Y2S65w98ZbGlvzkCl5hnZaBaKvTcoAC77FFPzK7Lm+I", - "RBEVfe8DBEfAQnM/nVTRyOYWgJ1QqWhglD4Mh1tHyTYNFgmH5gave+0zzh/2ls86OUZxeYraEI34M7I6", - "of7XRvYyWKTK3e0nu0/2Hm8/2WsUc5AvcDWDPwLXpMXFreT2QZIOXb2Mmq0fnV3A3afvVZnGRsm3ey+4", - "eCaCB1rYpAzlBTjyyZ/0nhRDLUKejqKC0cjGZYE/f5NqKTXPW3/SaEbHY/bnh+B6+w9B4633e3J75NXN", - "son8guxJ0dC5oPWRUdekLvR7wwNCCVkbMPKGSNgBOicKAf50EQ7gks48iizKubASC3EvYu3u7OzsP360", - "3Qiv7OoKhDME9XNxlad2BQUSg5ao/eb8HG0WEM6M6dwsIS0DswKcn86QzWLcL3lgatVnx4clNfJSjjV2", - "7FlcC/JLKwTZTVmgg2NUJiAtULkX2js7/ce7j/YfNSNjq3ANxfvlHMalwjDgsdlDiiffBuP428MzpEcX", - "YxyUFYyt7Z3dR3uP99dalVprVZD5xmSsWGNh+4/3Hu3ubG81i3zyGcBtTF+JYMu8y0N0HqTwnIYHFIus", - "t1N3W/gET4Ngb0gQYRofBs55pXL7mMwUQ2Ga5YfQ5GKwOv7CxdWgbyMVrVLsxogGXKCUZfmQequtmbcz", - "TtazaXMfrGbjizJ0hJkGl3XRN/kLbwG7RJAZ5an8AgNxRQKNTOOIc7FW3zpvoDdEppEyFkQq0eXpT8BE", - "NHIhqUhS9nS36LckkOGWm1uLgEs44cfqOmA1Oo0mR79sw50aMu0s82ItkX9tvFCoWVXKVr88H+EoSCHl", - "F87OU+8KPP95quCdfG58NKKIc4aCKWYTAhnQTYJBNkEYTXkU9lr+l44oHI69LxD8BkXcZDq4JiSx2bDM", - "InQ3LbPQGUHtFzwvw2ZQqZKU9lFsuIrNd1TGxkdxTUlL6fP7y+KLNDyx4oUgfNOlpM1HfCJBC1TgfdKr", - "5n5JsDBOJZiZ7G6z2CiP5cCpbX3be5ZY4d6+K9RcnXxsNVorYyieQRIHgkuJSEQnkEns8rS8zGXugzFl", - "NNZ8dvVrcnmxDVBXJpxJX1oTuNNk4ySQvgvR45f1OVci4DC4X3ot/+Yd3nrSoxizFPJjFRCZvE+oMOjR", - "7G17yqUaZsEgay5WqqGWkWUqSB4x5u7LKbjfzw2Lgzbee9GxttuAyzo93Kr3Alb5h6pbYD1P9ULUD61O", - "hoM+NF4Mh1kagZOH9FTjN9YJ2MqT7lAJo9JCrBBqM65KbKmQOGajyTuTX0fV89SVQX252z9vGku1PHTq", - "DKvpCRtzT0bFNQz+1iHduR4kRMRUQtG3kDBKQqc8ZpZ/a9sCF/dIEhSmxELOCKQCW4BjQ96QOZE5oxhl", - "kwqvr07YxAxv1rA8xRLMaxs2eTGUfsfotyIFWJk3folw7iLdyGGByqHfUrw4sCCTNMICVeMFlyxZzuOI", - "susmo8t5POIRDZDuUH3OGfMo4jdD/Un+AnvZaLQ73WGYe/hVnmfM4qx/pzmQyrz5Fn7Ru9yoeJeD6WXT", - "9N+EOtdNHmC9bj/PaURsSN0Fo+8LiF7OQbK73a8LPKgZtBRysBiOuS7ntijro3gXKXmYlQrwuJcZB57K", - "q0TZEFnar2+34CG2LMxi0RSD2u5N1+V4KcO1kGulkSWkmZNa1XvBrWZTkqA8++7+o8d7DZPdfJatc0kl", - "4M+wbM7iJRbNmpM6bWI223+0/+TJzu6jJ9trGaico0vN+dQ5uxTPp1IRpGI0e9SH/1trUcbVxb+kGneX", - "8oJK1T1uvaBPS0g3D3KuefZYVoU/P0n3zlK2gDazMS6Rlg5LIlehgFWbjMcElMqhgVs3X0zFOb7RGgKc", - "4ICqucdggm9MovSsSSVYt4k1rbxYD0jt2DbfguZcMh3l/phtNzn6qzGtV3Bhv3HOLJmO6sz4r6uzGiN+", - "bgMqPhE1eKHJ0/Evmguy/dxgWfLq0H8HkOk4L1BWdf8xLZpXUna4nhVTzh0bfQHn/sLJxeOvHGfB7FsS", - "kqsQX3aF1pPgWjq050b21WZc7VRb4Q/2Arxdr+GomM1uabrAUuq7/NZdf95mpdUW+5kbbP35Ch6c63Ss", - "JvYCfLRrsCDPx+6UUKIGmxQXq/Mw30F6HuNTcKsEPdYd4V5y9Nif7yQvz8JxnBfc0JsHXbhe5bovrrB3", - "yZFsr9vf6fb33m7tHDzaO9jauosI0cyJpO4t/fGHrZvH0TYe70b788d/bk0fT7bjHa/b7R3k/65Usaqk", - "A7d7SIiopmSrpjKUJKKMdGXmf7LaE3BJ7Lcx6SZ4DkLeEo1sHTXAVZZfQrXn5U0WiRerHDjVikL3ESBg", - "V79Ul6ku/+R4+bJv5dBRXYgfwapLAXxqthjIWLD1RVO9w7sYUI8XkDWb8qFMySGyhMTvlnCw3yzh1nEq", - "G5dnV5inB3PE5By8SliTf14AlI/FLs9gVrmEjJNaMWFcFhP0ZdOXvbVHWhdBVn0cKqbVO+z+3aTRQ8Pe", - "weYvP//v7ru//sWfSrWkOkoiuiEZg8R8TeZdU9tFI1evnH7EVEyWCtsEpIrgGLhdcE0Md43x++J6H/Uz", - "W/b8FY4XtgCqRkxZ9u+VG/r5L/WCegHRLpKwSWH0u0jkojhKYXbUjomYuGxpzjwMRa+0mHNN5hIVosWt", - "Z4sjuZ9k1qVYXsXW3O5BUaARhaQbcsCwIAgHAUkUCXs2apbCWgQHfK4WGbJR6+45V2M6hlQd1r2mWQHv", - "FiM3XTND2NWos/toz9ZVK0Jya+GEfGdmPLTr6hloKHtuxpdUgvuBe2UrNEZtEidq7nKyuHeQjfU8xg+z", - "AWvK0n/RaNf+ky+Rm+NiaTKO77CaRtGh3y1opSv/wvnXRsD7DanH1cA6Q5M2Q3g5EKyS91iqbr2dNeYp", - "U0N4Lli0iepv5inCZp+YpNU0XJsxU5s2143P/SGEwj1LH59yKnPe7V3otPpNZamnTmFnhZXUn42J5lhM", - "MbEEQGcaNDdTIkjhIKBDnrBjTZDZh4EGTjUmI0VCRLeaQt5kORQUXhosgAxgNQiyx6PFF6rlAWmn+H02", - "A2gNWC5oOLCPPDQbKvVu9NAbl0qcjt0QsIxq9eanq7GoSRGzxcMoYtXivk17L+FZXrWE+9XRVgU58zlK", - "qPnOd5tJEqSCqvm5ZkM2bBZKIB6mBg2BP8Em4Od8ckhp8ukTPKCMPXbUF1qDpgE6PDuxBRAZyF/o8hRF", - "dEyCeRARm5FiIQoERI/XRyddk0onK4yrp6cKAOJK0RyenUBlCyHNvP3edq8PKJYQhhPaOmjt9LagzocG", - "A2xxEzKgwZ/2idR4R1HOTkJ7Oz81TXQvgWOioEzx756nRkWEyagmQRbCk4IwmmAqrDSaRPAAarRoqvtC", - "EJ5j8AfmlugYgOOm0WxSza05mCSv7bG+0+hg3JFgi9v9vkl3z5S9DnBe8WDzD+s2ls/bSMoA8Hgi0hak", - "TSfpWJB/6rR2+1trrWdlkQLftBcM20qaBJb5aE0g3GrSE2beqJCJdrBp+4t0BihUpLDf3+nzkmkcYzF3", - "4MphlXBZJ6IRiTCkSTfp/P7gox6y6jrk0JBTnkYhFOlOTCEozUYxUlj0Jh8QFsGUzsiA2dvDFJzAAnIM", - "xUjfGkaYLpOGmdqcfuYa9ZSH8wp0s+E29XAgbZUBXI3Vl2QIMYvDulydma0roYxBzn9JbGKDLGndAkc3", - "RVpkwL3VaQjDTOU1P0x1lmsCMRlj+t47YKPgIs3w4FgIFAPLclZtb/hf1SEFg98h5Tj7hix4y5ec1hFs", - "VdtMEnAPPViMcBR5vc8nER/hyBaxuSYewekFtLBAKWarcFcu4yExmQeSuZpyZv5ORylTqfl7JPiNJEJf", - "zDYDkYW1q+FpUBeqidEYsgCZ/IZ6zk2zxM2P12T+qTdgh2Hsclfa2vY4ktxW98mqvLqXAYO7/hwZNbau", - "I1vtz1TWKBYjMMvkqUpS1UNmI0TZtEnQHGpVyCkJB0xx9FGY0mTzT5sf8xk/gURNcKjxpNDEbGnzIw0/", - "1a1aDrHe/RCaenQSAgAYtPTtMmjpvycCa4k6lVNQsCUo1ZPikbYzj2AtrWxUIRxghhKepLaAsuYnULSo", - "NAakoMNRhBSQkuurZSA4yZr9WAcJXz516x1hnrMrZASZ1QvE1N/d99OTJIEgPrX7v89fv0JwVekzMM1y", - "YwbAyNTyRmEK8iXM3huwZziY2iK7EDE7aNFw0Mpk3nAD1ppK+3zT7YLg9Yte2i9mmg4Nf+n19FBGpjtA", - "v380oxxoWkrioeLXhA1anzqo8GFC1TQdZd/e+QFa98h8XmIEqG14/4ZLIArO7vk1aO4NzELELa+N5gij", - "nAMVtfsRZVgszX7qAb2FoFYw8UQWgfFxAGa5Qetg4Axzg1Zn0CJsBr9Z692g9ckPAZuttz480ySAtc1y", - "JNrr9zdWe39Z+HpE6FJDTX6fFqSv7S8meFiha1HwMJtzseX6BE0qXyNu3YPk8xRnBc1/iHgrRDyrTxeE", - "N+hfvAcM+kbE2PsrEphWwCMngS3VTgxaQHIF0Dicr6ZROKiT4HLkLaofVSVzUa3YraOyAJYYOfzbvQf8", - "g3nzclgw75P7mhdHpnCrKw7zsNARDsshYsevEb8g6lvAuP59sVJXte8r4u9DwZ8XxMp9OdAq3GyTzNwr", - "iN8jXQmCY2lHMY21rnoOa+qeE6bQM/i1Z//rNB7Ir3IV8cnVATIgjPgERZTZN6LCG4a+FC0soZMJSsz6", - "2QBeFw7YNvfnv/7xT1gUZZN//eOfWpo2fwG5b5oYDUgfcjUlWKgRwerqAP1GSNLFEZ0RtxkI8CczIuZo", - "pw9iZiLgk6fggBywAXtDVCpY4S3NROZJO6Ctf6f3Q1lKJJIAQqjoPLZhA8bs6VHhHS0bUN4rRXcWdC67", - "g8IG9K3ocAD8QKnJoWL1r5bfemb2XLKfVS24Czb91fxFkffKYG/XLHBNBgMg9tEdfLCbRu3z82cbPQQ6", - "hsEKCA0BiTkfxgrPvR88aTVPMhylzFAAyoY3FWpN1dp/j22bZgZgO+L3ZAGuK55VbwI2Jg8iSOjg9UNX", - "aGIO9sPNmYZ99tljV2u73kB7+/0Wp3AuKo0U4S93zg73FmFui87nIPsaKjBq2xrAWd73UmX7r4X093Jr", - "5KXe86sDcZNt/t7UsiPOxhENFOq6tUBquZhkqloZQR4KO3hjV42w21c1CLt4v22WYopqb7osvCi/8u7+", - "9qhMus41kgeK57j24yZZhTrHVAZc9y1gSzfAic16b8SXjE6LWLTKIHUMv2dXzlJxybLnk2NHkPdnmrJT", - "p6x6N9wDUzyuMMSvyAgrmbwLqRUeEjZfZKdo97XMcvVtoWb//qSg+7Zi+dD8IZmxwgrYNBecZvVd69DL", - "VoC9w4O2M3g2fk6Eo2qXCQd2nW3LdEXBlATXZkPwIL1c9z0xTZqpvma870nzNaV315BYLMh/iCgNlN0c", - "VssU3BOb3vzu9FuYYS319su981oE8wAZnE1GzmJtModjOWfBxnf11Hsvt5kB9oO8zM7SKHIvHjMiVF6A", - "uHgHbH4Et6TVsr2jtqXXwcWbl13CAg5+aJkPlV+IcnVBv6yEbw7MbOUHmjTRCQFUDjHqBejPOH/jLoiy", - "IlP/sf3clpn6j+3nptDUf+wcmlJTG3eGLP37Ys33LXE/YOTTAjctAw1Yk6neuUpCzVo1FFJd++9KTrWV", - "oNeRVDO4/hBWmwirRXAtlVezotx3KLHa+sVf50kmQzYftOGT80/8ziTV+7XyWYx0WWSoLD972DShXOQ1", - "gylDqSQP0IGSZhhXvDYamqtzglx6fTjUPTnu2HLQpohzFiByT8Zrt457F27tvPdvuT6MR3SS8lQWY0+g", - "+jeRNlgpImUG/NDE7vx6rhW8v2Es7d/n1XHvcvUPvL8jib96oIZ5mxeoVTK/a9VU5rftoc62KeFmYtfe", - "uNJwNvXNRo1Tocuf3hSNS3U+F50dfevy6SLoQisqubqAQIM4GLD/0vrH74rg+N0vLkgm7fe39+B3wmbv", - "fnFxMuzUoQphSlAiERYEHb46hme/CUSvQwK7PCSvug6Tlg5Qz6VV+bdTkPKXz+YaksPCHxpSIw2pAK7l", - "GlJWevAuVaRyZqV715EcvvkAblNr/NCS7kNLkul4TANKmMqzNi84idmk7w8wtozZ96GCc0fpom2sJeX1", - "QJcLoHmqwnt37Mkmv3/lyGVFfJg+8txExYROHckvw3p95FvDh/79Muf710MeMooZgX8RdImWKX3FACD/", - "YJwqcErMM4SA12dW8dqN2EN5Dn5balGaHIYgAENSZjXVArAv32E5haEvZ2HCI31DyM6AEYZHkf5sYvk3", - "r8ncZCikPK8tl+3UZiX0xV6VEzx+VTL68jKWP3tlIxnrnsnYZLr8ijLWV2Md9yJpndjoB6t0Z4QBCuWI", - "ZJTMs+A+KJC+8aA8UA2zyvZWyGfkEbU2xzYtvl8Fes7FdVOm4MkO/QB4Q3GH36D2pZcH+ZO+vhIG6omh", - "H4009843FlJ+f02ndVrlJEGUhpp1OBbiLt+x4PHQ/mjyTmqqsDXOQKkL7Khfm8no2e9BxX7FFaJxEhEt", - "95AQdQ026dPM6lKb5M1UFhLkr8cENdkUQwhM+i5prKodZMsIwHOEO7A2vEwuHpeXa0Z8sjptQDa5i5H3", - "5A0YMJNcmrhM1FcoY7JIcSRJRAKFbqY0mEIOAf0bjG9SDOAkucqSBm0coBdAqcXcSTB5WxKhRceAM8kj", - "YlIDzOL46mAxx+Xl6Sl0MukDTDbLqwPk8lpmF4TUrYo5AfQuIiwVemUzHbQ1JgkeReZEr7ScXdjfhs0W", - "kCd1GjBf5gBGbuyAdIyuCkkErmqyCDiG+pJP5NcSZTv1qfjMXqB0sAacwU3Cwlad6ZpG/vwBW31vveCG", - "uQzMMu44lcHCYl7ySZYGsITKOEmaoq9dJmDxLI6X4DBqF0psSBXyVP0sVUiEgM4Wu+uQG7VtgVqk8LVG", - "VGYLZLoiJYB+3gcak5fLCyrNVAtVDsy/ZnFsS6nHmHkqBX9+TojqgIsPCfpkCokfftgS1knpUGb2hZwO", - "lZvDVlOqF7ltkajv3qJlARV+D3pp+QUgXwVlTlSBs+V5lZQHFRtu6odVZTFT0sVHI1kBsnoqKT+bnefF", - "Y/4NVVSz12rVuHtWUjMQ+zSzUtGlr66dZjWgfmiomYbKBQpTM12lCtt3q3ZmDAWlrKR5WvH0trpnlkYz", - "AzNUx2VLnzxznrf50f15cgtx4RvhhJ3aWmR1CdvyTX8LLLemUue3+GTg5CR7rRYEhK/Igl3N0K/2tqDV", - "vYzLfRNs2BBcxo2LPEcJzCR15YB/MOOSGdBYSm/LjJ3wuWALLLBnyrpJhOv4spVTaxmwrU343etrua7y", - "nWtsARfCOMeCu+1DCs4u+AoUVM92glNJOhnBdJxnzuXp6UYd0Qi1lGSEesC+BpVi0XHor9YvaOjKcByd", - "HtuiHVQikbIeeh1TqI1xTUgCSXcpT6UpI9krVlSsq82ZlUwkTIl5wilTK1eRN72bxXy6VRmCe+ZTNj3F", - "d29WsuXfHxqTAt6hb2+7geVKlTKFRL3PdO7ZijJTO0QLH3jEUz36QsVHNKYRkXOpSGze7MZpBEQECYxs", - "fmvbz3jndhBVEml66IA3Y0JETKWknMkBG5GxlkoSIvTcUAaZRqTw/OB72TpXOOOaZ4b1fRtPW1AEEl5z", - "sKqDWrn+I04SV//R93ySlay89ZKew1sVkvN4xCMaoIiya4naEb02MjiaSRTpPzaWPnYNod+Xzt59e8rS", - "kD5hY+5NcGpwNkPm78Ohq8zW3GP+g2NrL0iRWBz/gYP2szW5kq8JgiMoc5wFEqBU0Yh+MKxOD0KlooGp", - "Cpe7sUJBK+vJOmCnRAndBguCAh5FJFDO1rCZCB5sDtJ+fydIKER87RBYHDC8+s8xzHh0dgHtTNGtzoDp", - "f8DAbw/PENUwHWOrMhcWyoi64eIanWy+XvH8fw5g+jfWx8wGl/qTeg/8x8vu+l7itTQka0iUJ8sUIJ58", - "9wYDK8H9sBY8TGsBhOlku2lPBA5AKJbTVIX8hvktA6YGtNz8aP44WRXspXAwvXTF8L8NadfWw141jdvg", - "gyBKu6eQmATMX8Veb0uWP9CEdRpwbgsgxBTD1vy3wKH6HrH7yz/WFeH4Db7UWYi65ObfDG3d981n1+Bi", - "mIvweChkbjDN7QSK8hatT1nA9krdLDABgpDsKhctA5zggKp5B+HI1Yu2BeAyG1I3u3JHguBrfdP2BuxN", - "FipuC9Bp7arjVCsUUnltRrDaUw+9nhEh01G2OASMyeh5AHxbMjrAUWBqLZPxmASKzogpgixrtK9sKXeZ", - "eDyfxHPQ7qMF3UNTOfw4AaeXo4WsYNymOepNQYII07ioelSBAwwSDP9gHBjpQTlDlI0jawQNBJcS2aG6", - "JKITOoqsSU/20NspQRLHZMCSCDNGoL40PNbqpXcTQaRM4fEfBoDKDgajOigPlEkEV9aYEHEupNH/NYZf", - "niKpSLIEzd6YkU1N7jtKzGEGtzN9pWulsgYzix/lDaj1gRhMMQDXeJRGSt6bO8hJ5gBiFvSj9nIzwn8r", - "6GRChKYKbJisMaAbsnbgNERfco+tzUp1nrVqlpUqG7XgAldwD1sa6DB0DYcgfq5jp/dMfk1rY2Hsp/Vc", - "Vn/TnRrOXXaN9C/CfvrMXX4vyX7PCx5pTXNZ5Rj+0NJKFVZeItWSV+fq/DaN3Tjv0q1yVX6bbPL7zm9z", - "7vXse2BZNnHJV7Musc23hwj9+w0puO/ENg8bt7T+IBdAV8+JGqR/+CYw8G7yPnzlkJpb5H34ppy8IW7/", - "6wXbfFPu3dZNOXPv/pHZ4S69uk16B4hir/PqNlzPPjctVZQubZtmapId8XuS4O0LxRryuwP7D62/gcpQ", - "AJbfZGcCyqRFeBInau5M0HwMznZ5qmRJP4DLri9aNntpursg1Vs8wnw59HB4WvsE8yOD7b298uRlPk6O", - "H37a2iLNlS6WTX3rdLEIpnRG6o3uZQq2IEoE6SY8gceV0ADMwsPdZQqL3uQDssP3BuztlLh/IepS5JAQ", - "hVSQQEVzRJniwBHMHD9JJLjWBOA7F3OfMb1Iuc8Fjw/tblbch5amrDEs9+2N590QK9ydOW6zxIT2Ge/U", - "p/g9jdMYGB6iDL14itrkvRImYwsaa80H0XEGUvI+ICSUgJMbxQVv9Wssm/QDGU5GTVa5JPfOa5vbCAWp", - "VDx2Z39yjNo4Vbw7IUyfhRb1xyDJJoLPaGhS/+dAnfHIQHWrBqDr2l21UGGDQHLlwizuq8gwTS6kyQea", - "lNmC8XVuHbRGlGFY3MosN2WaMm73ej5Mwfk1px2HOa0fV5jV/NpO2dGYqJUcB0TFOYq0RL/x45p7yNdc", - "0X3J3Wml265ZcvZmHk0NHY3uIjF75u12v2bry2/HCYfKB+l/Y03ns0whrTObf1so2L+/++G+zeWXD9hp", - "8wVxynfBVA4D6BF9CPOSBzhCIZmRiCeQt920bXVaqYhaB62pUsnB5mak2025VAf7/f1+69O7T/8/AAD/", - "/0qTWDyDPgEA", + "H4sIAAAAAAAC/+x9a3MbuZXoX0HxZitUQlLUw7LMram9Gsn2aMeydS1LuZuhLwV2gyRG3UAPgKZMu/w1", + "PyA/Mb/kFg6AfhFNtmRLsmJvbWVkNp4HB+eF8/jUCniccEaYkq3Bp5YMZiTG8OeBUjiYXfAojclb8kdK", + "pNI/J4InRChKoFHMU6ZGCVYz/a+QyEDQRFHOWoPWKVYzdD0jgqA5jILkjKdRiMYEQT8Stjot8gHHSURa", + "g9ZmzNRmiBVudVpqkeifpBKUTVufOy1BcMhZtDDTTHAaqdZggiNJOpVpT/TQCEuku3ShTzbemPOIYNb6", + "DCP+kVJBwtbgt+I23meN+fh3Eig9+cEc0wiPI3JE5jQgy2AIUiEIU6NQ0DkRy6A4NN+jBRrzlIXItENt", + "lkYRohPEOCMbJWCwOQ2phoRuoqduDZRIiQcyIaxpREPPCRweI/MZHR+h9ox8KE+y/XS836ofkuGYLA/6", + "Sxpj1tXA1cty40Pb4tivdn0jUx7H6WgqeJosj3z85uTkHMFHxNJ4TERxxP3tbDzKFJkSoQdMAjrCYSiI", + "lP79u4/FtfX7/f4Abw/6/V7ft8o5YSEXtSA1n/0g3eqHZMWQjUBqx18C6euL46PjA3TIRcIFhr5LM1UQ", + "uwie4r6KaFM+FR/+/5zSKFzG+rH+mYgRZVJhVoODx/ajBhefIDUjyPZDFyeoPeEChWScTqeUTTea4Lsm", + "WBFRJBxhtTwdLBXZNpQzpGhMpMJx0uq0JlzEulMrxIp09ZdGEwqC10ynWzSabPmqpeYkR7GsG901QZSh", + "mEYRlSTgLJTFOShTe7v1mylcGCIE91Co5/pnFBMp8ZSgtiabmnYzJBVWqURUogmmEQkbnZEPEcxmfudj", + "REPCFJ3Q8v026NTF42Bre8dLO2I8JaOQTi0nKg9/BL9rFNPjKASt/RvRF23RbB8wpSCT5fleAOmGSQSZ", + "EEE0jn/hdIngc8L0bdHz/Qnmbf2vzZxFb1r+vAnAPM2bf+60/khJSkYJl9SscIly2S8ajQDUCHr41wyf", + "Vp11AaOkwmL1/YAWX+EmmvU1gs2Zafq501J4urbLO92mSjuBNNopS1SglkQ+nxPmEZICzpT9UIbOKz5F", + "EWUE2Rb2LDRN1BP8FHEgiV8JDhn4ly+/XvctiJf5oWY0/a3TIiyNNTAjPi1Cc0awUGNSAmYNC7MD5aur", + "Bf9p6fpUeBWWZLSagpxSxkiIdEt7sU1LlEqQVJe2D7foiqrRnAjpvXOwrF+pQrZF7VARD64mNCKjGZYz", + "s2IchnBfcXRa2olHWiuJvzjRRNANCFKERIqjs18Otp/sITuBB4aSpyIwK1jeSaG3Ht60RQqLMY4iL27U", + "o9vNefQyhvgx4Cy7GHW8J8NAh5iG0rXsaerhO60klTPzF9BuvSrgfZoMaPSK9N/vPZs+BCJhtIRanckv", + "A75JzGGjacQ1TBcoZfSPtCRg99Cx1hUU0oyChiTsIAwfNMnGqeLdKWFEaDqFJoLHIG0VhGDUJr1pr4OG", + "Wi7saim4i7e7/X63P2yVxdhotztNUg0KrBQReoH/7zfc/XjQ/Xu/++x9/ueo133/1z/5EKCpZO6kQrvP", + "trv7HeQWWxTXqwtdJ8rfmvoXl++jOOaojzWduOlJHx4vCw5mryEProjoUb4Z0bHAYrHJppR9GERYEanK", + "O1/d9qvCAvaxAghsqsF0QzBUlB5A43bEr4kINAWOiEY82dFEmCrZQVjrzUC8kOaS/4kCzPRdMMIFF4iw", + "EF1TNUMY2pWhFS+6OKFdapba6rRi/OEVYVM1aw32dpbwXCN52/7Rff8X99PGf3lRXaQR8SD5W54qyqYI", + "PhuuPqMS5WugisRrT8RBN41AzIspOzbdtrKVYCHw4stP2G1k1UkbZa72qIPYI/m/mRMhaOi46uHJEWpH", + "9IpYdEciZWiY9vs7ATSAP4n9JeBxjFloftvooTcxVZqbpTmTNtagXvG4f2uRYMZBzogirjeUgbpGiMlh", + "GAgC+gmOVrLhVSD2AuswG3eZaf/CperGmOEpAW3SNkRjwa+IXihKeEQDSiS6IgstpCzQVA/anVNJ9fUh", + "bI7m2BgNekP2bsYlMU3cJ62IBITOCYp5cIWSCAdkxkERn+MoJbKDrmdaYtDEWBAc2Z+RIDGmbMhmepEy", + "4AkJtQ5hmsHW0CVh80sU4wRuKRYEriiKsSKC4oh+JCHipktMQqoZ1JARwGuUYH1lg4ALzX312RIczApQ", + "+LNEl0beuIThLynTWHlp7lVvyIon/6n15vzdz2/OXx+N3pw+f31wPPr1+f/on02n1uC3Ty1j38wEjZ8J", + "FkSgP32C/X420mlIRGvQOkjVjAv60RhbPndaGgZS4xdOaI8nhGHaC3jc6rT+Uvzn+8/vnTylpyJsrq+B", + "Z2GfvbKMYYUeinLkjHkSWQMRiHYYTLVAYV6enm9q5ppgKdVM8HQ6K18My9lvdCVCKq9GlI/GiW9NVF6h", + "4803SMsdKKL6gmZyxla/f/Lzphy29D+euH9s9NCRubWwfE1CuLDij5xp9NFCOKDM4ek5wlHEA2sCmWhd", + "aUKnqSBhr2J5g9F99JkwJRYJpz4drEKc8qbLNKrbzb/egBRtjinblPoYusHN4A54c2tN4DmbU8FZrLWx", + "ORZUs1lZviuv3xw9Hz1/fdEaaDoepoE1Kp6+efuuNWjt9Pv9lg9BNQatoYEvT88P4aTMtVFJlE5Hkn70", + "SAIH2f5QTGIujAZs+6D2rCwomHuL4HCGrZ2XPxvk2noJeOUOJaQSWrtRzMBljNl++bMPW2aLhIg5lT4z", + "2S/ZN3fyBbZuyH0ZtyURcyIypAUs7hXUjyDiadgtTNlpTagggcAa7Vqd1h8k1nL4/KNGnXztnn5+61Uj", + "+XONYImjhDKyQrL8RiS8ay6uIo7D7tZXFvAYUXrs5S2+Nh/K52txgmQo0eosWSNYeE1DNRuF/JrpJXvo", + "qv2CssYZcf2gd4Kjf/3jnxcnuZq09XKcWEq7tf3kCylthbbqob0mkGwjaeLfxnni38TFyb/+8U+3k4fd", + "hBFEbiXU2fN/bkYAkq1xPSw9UxprZhksf5sRNSOiwL0dsuifjD4M3ZHDvcJWSubR4pvmEqHmcyIivCgQ", + "Xrum1lYfqF9lVYIquKu2nyajV0h3XkOG9WiOyb+s6ujbfT+h9SzKs6afNa2wfKHJSrKFbG2f2D+3l5dU", + "s6IrmoxAah7haWayXfXafHZFEyuKQw9zjFFkCEGYgvA+5lz1huxvM8IQnB0cMPlAAqB5UmGFDk6PJbqm", + "UQQGHiAqy6xFC/Y5WTHNpdL/K1LWQeNUaWmdK4Ks3gSTpLAWaDwmKGXYPWdXZGe7wSpeWbBcEcFINDKy", + "sWwIGdMJ2U61wIGtTrBURBhqnyZleB39enKG2kcLhmMaoF/NqCc8TCOCztJE04ONMvQ6Q5YIMtcqBJuC", + "sZHaefkE8VR1+aSrBCFuiTEMlpnI7Fvr/OXpuX2tlxu9IXtLNGAJC0kIa3YcRyI1wwqFnP1Z31gSloct", + "zl8Buv8ud1qS4UTOuBqB5rdYR53ObPNT0/pGtoBOax4kaflIt6vH+Roe5DXw5lSoFEea1pbESe/7vPH8", + "8KgNxrGkqL5YupdhN1blh9WmBhMzMriBLAvVfruHkZQa2z0KqvySBcTpmZ+aLXbN+MfMLWSl3SdXNb9g", + "rjMzSBVEduyO29ktoHScwaQMK/x1wHMgC6p5rV09JFJRZtBJt0VWIpSofam1eYvHWn+/7KDLv5R+0Hff", + "qRZavrhGBhpAT5j+qTh+1Six1lzQXCmsHA6Wtz+PA1nrqITmW0gJzKTmrVrGSkgP/QJEHCkSJ5qSsSmi", + "EklDfEmIGL/+T8SNUOO6DplemjRuHhYcmdFI0imjbLqhxXzNmHAYGsvSJFWp0O3mVObQLKOOs95UN/DO", + "rI4YehynUnPkIEpDgi6dheeyLBcu23+WVUJrEFrScAxIQLMBZU9txqnS0+sNx1gFMw0nnirj92W3LssL", + "KFuZ1r2H2rVkL2W3OP+zjFyUgWrtDRXCrzdn32jALFiwT9aZAa2g4jdRXpEFHLkzR+Ilg2TREum3Fwoi", + "eTQnlu0WbZljHFwZVmJcL6wZ0xgkrQ1SX//KFfVa59YdhYZXY/CXVYVlVAITsN1sjjFW+jf230VGhfTm", + "zHwdrRhLAsAH1WOAQBy77BhdiYAFAjGNLBEKqSCBWhqesumQgQvIpf2lZ0e71Jdcyyi+S+hTdryyYEHb", + "MX1KR4sKJ+vEPhhGb43HVCkSdsqywRUhiVy/KS1eW8O1x7ouyLWgjpBZg1HYUDwjbMJFQGKrJHyZ4vi8", + "MJhXjbvZEMseGQa+hTVbfEI4SSJKQuP+Y84DzKzSnhPYWKsuv2FFazMeAOUpL3EUXaK2bbSBBNF7ke6s", + "GGc5sr87PHUokL1aX5x0NEZqKnA5UyoZ6f+RI32LL6uD2b7uhuvhNE+SaL8P+tXu7o49VWt0MwuuDFu2", + "r3m9GuqPxonf9Q9jPNZ30bmJNBHlD/MuuSX1irKw6QC/6ra11rlMMHKaxl0b6BJBumkyFRg8ZL+mee7W", + "z54AzXoKvsb53eflmEE1SKXiccHXEbUrHhq07MtRBtacR90QKwymzIb2VrPcZb/heGGGMrpYnSVmNB17", + "3H7oR0110ZRO8Xihyu8HW32fxvelb9BuLb5jqfO/NxokCUeKr/ZAphPk2jZxOAR+MlJ8NJ9Qz8gZW8vd", + "V6hEQcXZ3+q1eohuElBrTgAZJ5gZB1EDBBAaL06Kb3e9IesC+x2go2yCbNhsSAyyJQ7Ny0mbi8IiKHid", + "ofFiA2F0cdJD77LV/lkirbDMiQtImGGJxoQwlILpGbhh1/Di4gJSCUxTVbtb24mJXdiAJ0puv/XQL4uE", + "xNjaofRViLGiAXg6jWllP8COzEHZN2HMilawRlarVX7bb8mUSiUqXtuo/fbF4c7OzrOq/XL7Sbe/1d16", + "8m6rP+jr//97cwfvrx+e4RvroExbrO9Ykfocnh8fbVtjaXke9XEXP9v/8AGrZ3v0Wj77GI/F9PcdfC8B", + "HH5SdpQ7vaF2KonoOjKpscrn6lbwKKtxZbu1h9odOZzl/rOr2hpIvNMt7yIyxefzbD1ubx47UiWYa72m", + "C5tb1uQXCeid+S0pSHDWOTGgXjfMIyqvfhYEX4X8mnn4doynRI4MP/P7M6TSONmQD9a6IThXE2neTctW", + "z63dp7v7O3u7+/2+JyBjGeF5QEeB5kCNFvDm8BhFeEEEgj6oDQ9eIRpHfFxG9Cc7e/tP+8+2tpuuwzzx", + "NINDpni5XqhtIfJXF9znvpQWtb39dG9nZ6e/t7e922hV1l7caFHOtlwSSZ7uPN3d2t/ebQQFn0D/3AXI", + "VAX40Oe6oPUn89jYlQkJ6IQGCEJskO6A2jGwMJK9VpXv5BiHI2s88fMOhWkkV3pMmMlsS2Noi9NI0SQi", + "5hscSCNbNOz8CEbyeaNQxogYZfFDNxjJhhWt9RBwe8maoFJ4WAl0J1SCFJILT5RE4cDc0LV0Dk4zX9j7", + "Ojywe2iIDa+06tSNyJxERSQwrEsvNuaCoAxPzKGVdkXZHEc0HFGWpF6UqAXli1SALGoGRXjMU2WeGeHA", + "ipOA0zLoHhNNrpvpuS+4uFrr/qk58UikjOlh1lqFDsCQPrGmGuDiGNneLsKgIPRlz4Hm0dR+l+it6WEs", + "RPnPSaoQZYpr7ZSF40UHZrKWJIYEkYoDJbUGQztMU+nSL7eAsdS5f5j5ctp5T74v3YlxF/i6GraYEjWS", + "Cqu1EovGlHfQ/gyaN/Ym1x3XGlIawJ2R6/sAOrjbdzXadiXDyd1AfJUzWmZryBsBFxY0JD0Etwu8Ylx4", + "X+WmnSmeJCTM7D+9ITszVyX7SZoXFN3RwEHNCBWICzql5YnLBra79Gq7CSo6bLo1OhY7Lkuo8BHcN+ov", + "PZ4oIgwEXeRyMfzIHkKr07Kwb3ValhKVQeN+9EAkd7VcWuLL0/Ob+qYlgk9o5Nku+ELYr1Yzc15br3b7", + "Z92t/2M8MDW+gYhGmfGfiHlIepXkANC+Ged5eXp+WremLDMDKq5uaU+Zx4uHcmR+DQ4i9lHJvkpaDcah", + "v2Ys2SS57P3MJ8tOBI7JOJ1MiBjFHuPaC/0dmQbGtYkydPJzWZ7VcnNTrfm0dDigNk9wYAPrm0HfY5Cr", + "bKNTgOZ7/3G9JYYN14Xj6aMSto2NyOuh11kuDPTy9Fyi3EvJY6krH2+tv/zpbCFpgCMzoomupaxoYAPk", + "bCwhn+YdrSnSIyfHXtnQXQTUnk+TFK7h2dvu8ZuLzTgk805pTeBZNOMR0eveKFCLuQvKy537S0RiXmfp", + "MIghm16gAqyyG9wYSIX76oGO4gpHIxlxn7PGO/0RwUfUvnhhgqb0CjooKR2l/r0AhRJ+73lvjKZIddOe", + "wYRVk2npgnt1x3IKGWNeKWyvNKnvqvxCcGQy55TxOY/vdgfPr8oHza/W3l47iG/eY+cY3iB46/DkyAgM", + "AWcKU0YEionCNk9PwcUFxKFWp9XVPCrEJAZXu8l/rvZuqTHBF6Oxao24h0tpN+7EgFsTLv7WuCCEKMaM", + "TohUNly8NLOc4e0newOT1CIkk90ne71e76YxKs/zoJRGR7FpXPgL4So9Ofuyc7iDUJQme/nUOj1490tr", + "0NpMpdiMeICjTTmmbFD4d/bP/AP8Yf45pswbwtIoDwqdLOU/KT9pap5lfh/onTDrEqZxiYMCv/aJqUaf", + "Ac8GiJvzhgsrPNX6icG4L40LvnXmkDx9lSpkDCk6hDbIHkI/rraEOsEI2tg5U6ZolCdWWbaB3io1jlyZ", + "PWApc0BCWJYvIIrMXwFnc30rfMkDSgTcffui9wPr5TIKqQeT/2a1PeMkAVFV6+9baxMnyXq09QuKGf1r", + "mjTFhjZ7ONGDU/3bvLGVZ38z/e8//q88ffr71h+vLi7+Z/7yv49e0/+5iE7ffFEE1eqo9gcNTf9q0ejw", + "sFQKSW+KSidYBR6BasalqoGw/YIUN/6aPXQIit9gyLroFVVE4GiAhq2Ki/CwhdrkAw6U6YU4Q3ooG+mw", + "oTufGvOP7vzJ6Zafq2OENqRB2APJIplkOg55jCnbGLIhs2MhtxEJb/r6rxAFOFGpIPr0tAwbLdBY4CAP", + "Zcgn76BPOEk+bwwZaLjkgxJ6BwkWKkvD4WYApLCrMj4DtjkJXWC40ZCHLONLWVy4sdH0MiMI2OarHpd+", + "oHjVFy7KoTj7fV8EPXh96YOMqFQEHLMzzNZolLmjof1+iVTs9/f7awX8DIdWoB/chOUkmQ4pG9wlg8Aw", + "tSHc4KHWwJauaZO5I+iXd+9ONRj0f8+QGyiHRXbERskzPoDS2AhVJAvefxstn+nbnG7DDRkjGXSLGkQN", + "PTfuoe9enSFFROwc9tuBBueEBnp/8PxPpUw1KlKMDg5Pnm/0GmT5BNhm619xju+yHVaDO6zRrM4WmGG8", + "hm8HHR+Be669obkAB241L7hAkSEw+b0eoHNJyr6ucFTmVd+cZLTILW+GAwxbG27EpEopBuhtJjfibCmZ", + "o2WODG7I/F7CsPbhxfj8LI1e8csFbyarF1nSBh4+WGVO4prj1pOC1dffA3G489avu2DTvNndLhpD9WR+", + "1MjP/s6llZ2b6qg3TdBQjqEsxN9mORqaJ1e4iyQFy/raB6pGtY/wSH+2T+5OK7k4QTMs2Z8VfKzoJls7", + "Txtly9SzNn2+Lj5c84lZUnarXEBm9uxqQlOvaBQZbwZJpwxH6Blqnx2//PX41asN1EVv3pxUj2JVD9/5", + "NMjV4FD75ek5RLtgOXIvQPVOjzh3HCYfqFRyOV610UPq6twQv5TyN3gDgDe+YlIH9/q8tI37SNfwkG59", + "316qiJXJHb40Q4MVdu8oQUMtcfUlNyjTWfPz1021cCfLKcX++OhDUSZwPte3zm3QaVGPv+mB1CSQhOj4", + "NE9xmBul3PCVPT3b7m3t7fe2+v3eVr+JiS7GwYq5Tw4Om0/e3zaGiAEeD4JwQCZfYCK0iG2ENxxd44VE", + "QydeD1tGni8I8oVra0XwRs+vyykkbpcxoipQrMsJcZMcEM2SO3xpRP2qRMdn5RTHjYW8J3//omzIpClr", + "t74PttfoJtZvggKeRqEWpMb66hq9jIRWfZRE5dmj4bafsyvGr1l568YIqgnAHykRC3RxclIymQsysclx", + "G2wcfCZqzoEnNzqG7TWy9trV3DLPwn3kVqiS3QK7++qZFIo2O+eDaTC0ge0uFz+97+aUmaPReLJiTxWr", + "S0jmozT1SVX6k4u8OD8/PiohB8Z7W/v9/Wfd/fHWXnc37G918dbOXnf7Ce5PdoKnOzXp6Zv7zdzeFaZ8", + "m+sjnQDwYME0gWzhQN+3zJdlnCqU+bnpi3yoxVNUkINNXA8YFY4ZVZDDkbKpHgZ0fCsmmwBNk2aSMqog", + "IwDko6FMbxmMKXoQ6700QC+hLXzCMcQbuUVo5ahsR8DhwthRNWFwUyfwr9VLPpulSstt0EfOUoX0v2Db", + "GgxWXVk9hKExA/SaQx/hnEwZr+o9pjk4by03r+pIbetW5NxPYTJLMAfoRUYkMzJryWpbEvunod3WMxq8", + "vjdKvnf2xFsaW/KTK7iVdVoGoq1OywEK3M+WHdHsurwxFkVU9D0wEBwBCc0dfVJFI5vkAHZCpaKB0Rox", + "HG7dTbYJvUg4MiJA3XOh8R6xYkLWyRGKixPUhnDGvyKrVOp/bWRPi8Vbubv9bPfZ3tPtZ3uNghbyBa4n", + "8Ifg27S8uLXUPkjSkav8UbP1w9Nz4H2ar8o0NlYCu/eCj2gieKClVcpQXkokn/xZ71kxViPk6TgqWJ1s", + "YBcEBDSp+1LzPvYHjeZ0MmF/fAyutn8XNN76sCe3x17lLpvILwkfFy2lS2ojGXdNEka/Oz0glJC1ESdv", + "iYQdoDOiEOBPF+EAmHTmkmRRzsWlWIh7EWt3Z2dn/+mT7UZ4ZVdXuDgj0F+XV3liV1C4YtAStd+enaHN", + "AsKZMZ2fJuSHYFaA898zZPMx90sunFp32vFhSY28lGONHXse14L8wgpBdlMW6OBZlQlIS7fcC+2dnf7T", + "3Sf7T5pdY6uxjcSH1RTG5eQw4LFpTIon3wbr+ruDU6RHFxMclDWUre2d3Sd7T/dvtCp1o1VBCh6TOuMG", + "C9t/uvdkd2d7q1nolM+CboMCSxe2TLs8l86DFJ7T8IBimfR26riFT/A0CPaWBBGm8UHgvF8q3MekyBgJ", + "0yw/hCaMwRoJlhhXg76NVLRK2R4jGnCBUpYlZuqtN4fezrpZT6YNP1hPxpdl6AgzDS7r428yMd4Cdokg", + "c8pT+RUG4ooEGpkmEefiRn3r3IneEplGypggqUQXJ38GIqKRC0lFkrKrvEW/FZEQt9zcjS5wCSf8WF0H", + "rEan0eToV224U3NNO6vcYEvXvzbgKNSkKmXrn64PcRSkkHsMZ+epdwWhAzxV8NC+ME4eUcQ5Q8EMsymB", + "XO4m0yGbIoxmPAp7Lf9TSRSOJt4nDH6NIm5SJVwRkti0XGYRupuWWeicoPZLnheUM6hUSa/7JDZUxSZe", + "KmPjk7imOKf0OQ5mAUoanljxQhS/6VLS5iM+laAFKnBf6VWTxyRYGK8UzEyauXlslMdy5NW25vaeJVao", + "t4+FGtbJJ1ajtTKG4hkkcSC4lIhEdAopzS5Oystc5X8YU0ZjTWfXP0eXF9sAdWXCmfTlRQGeJhtno/Qx", + "RI9j15ewRMBh8N/0Ph2Yh3zrio9izFJI1FVAZPIhocKgR7PH8RmXapRFk9xwsVKNIAlTKkgecub45Qz8", + "9xeGxEEbL190pO024LJeE7fqvYRV/qHqFlhPU70Q9UOrk+GgD42X42lWhvDkMUHVAJCbRHzlWXuohFFp", + "IdgItRlXJbJUyDyz0eShyq+j6nnqCrq+2u2fNQ3GWh17dYrV7JhNuCe14w0M/taj3fkuJETEFNKQoZAw", + "SkKnPGaWf2vbAh/5SBIUpsRCzgikAluAY3O9IYUjc0YxyqYVWl+dsIkZ3qxhdY4mmNc2bPLkKP2e1e9E", + "CrAyTgIS4dzHupHHA5Ujv6V4eWBBpmmEBaoGHK5YslzEEWVXTUaXi3jMIxog3aH6nDPhUcSvR/qT/An2", + "stFod7rDKHcRrDzPmMVZB1FzIJV58y38pHe5UXFPB9PLpum/CRW7m7zgev2GXtCI2Ji8c0Y/FBC9nMRk", + "d7tfF7lQM2gpZmE5nvOmlNuirO/Gu1DLg6zogcc/zXgAVV4lyobI0n59uwUXs1VxGsumGNR2j8IuSUwZ", + "roVkLY0sIc283KruD241m5IE5dl395883WuYLeeLbJ0rahp/gWVzHq+waNac1EkTs9n+k/1nz3Z2nzzb", + "vpGBynnK1JxPnbdM8XwqtU0qRrMnffi/Gy3K+Mr4l1TjL1NeUKlOya0X9HnF1c2jpGuePWrTdEfFk3Tv", + "LGULaDMb4wpp6aAkchVKcbXJZEJAqRwZuHXzxVS86xutIcAJDqhaeAwm+NpkbM+aVKJ9m1jTyov1gNSO", + "bRM2aMol03Hu0Nl2k6O/GNN6BRf2Gyfdkum4zoz/pjqrMeLnNqDiE1GDF5q8LsCyuSDbzzWWJa8O/XcA", + "KZfzUmtV/yHTonlNaIfrWVno3DPSF7HuLwFdPP7KcRbMviUhuQrxVSy0/greSIf2cGRflcn1XrkV+mAZ", + "4O16jcbFdHgr8w2WcuflXPfm8zYrErfcz3Cwm89XcAG9ScdqZjDAR7sGC/J87E4JJWqwSXGxPiH0HeT3", + "MT4Ft8rwY90R7iXJj/35ThL7LB3HWcGP3Z+Rm4SjVTHIh1kzZw9M8ALkhloh/+nObr+/s92/VRDy10oU", + "Xhinztuv0M8q6qVnpeIImW/fcja5a0FNvSkHJqkEwfEAPGUSHBAUkQmE6GRZPNfqa0tTr168fQCzTt2Z", + "y407KFdU0urQFn0ZZ8BU3Dg2fttto+Ve4Mp+/MXvy8teEQjkDqtc08hVvS/5Ju51+zvd/t67rZ3Bk73B", + "1tZdRC1nQKpzz3j6cev6abSNJ7vR/uLpH1uzp9PteMfrCn4HOekrJd4qKertHhIiqmkCq+k1JYkoI12Z", + "uTStdy5dQQvMK8Ha+38zzdLsYCUjOCtvssgPsMqBU62WdR9BK3b1K9Xj6vKPj1Yv+1Y+QtWF+BGsuhTA", + "p2aLgSwaW1+asiFlDfnOeaFhY86z0m9tHe/xOfTC1faecg3EffhcIoylG/Z+Bcde5moe8XzKBVWzeDV7", + "yJplAeDw0PlRqrAcJNFDx1MGSUGLP2d27WLdXt251WlFH3fLd8b+3jxcxgY8Zwhoj7ooBjSw+0LO2dVQ", + "gCa52CjM0zMWBADx01Z36xm8vkYfd3/qd5/10N8Kr8AdA60i+LZc69Kv/SYwzAglPNyaV9GtZzd6InXw", + "XIVBv1q+VMeIbSi0xfE8I6PjFc4ltnTA+eelM65EjNxRDZhVUm7zdJUVhcE4FBezg2YBoF83V6XtuTaT", + "8N3B6p2l1nUBy1VXgmIW14Pu303WVjTqDTZ/+uv/7r7/y5/8mbtLtFwS0Q3JBOwrV2TRNSXJNN/olbNd", + "QTB1Syps810rgmMQZIIrYgSnGH8orvdJP3v5XLzG8dIWwDAVU5b9e+2G/vqnerNOAYznSbhckqq2pNxX", + "zRumOEphdtSOiZi65JzuMRFqNWql+IosJCokJ7F+kI5h/VlmXYpVwS4NA+tBLbsxhRxPcsg0ocRBQBJF", + "wp5N0kBhLYLDjarWxrNJUpzzj75rGDJDWWfMSg6ET94KdIMWI9ddM0PY1aiz+2TPlgMtQnJr6YR8Z2bi", + "eerK52goewSRV1SCs5rzySg0Rm0SJ2rhUoC5V/ONm8UXHWQDeg1lXzm5Qv/Z10gFdb4y99N3WLypGP7l", + "FrQ28Gvp/GsTrvif3Y6qcdzmTtqCFOW440qafam69a9yMU+ZGsHj8vILmv5mHq5tsqNpWs36uBkztWlT", + "q/mc5UKoN7fSVSG/ZS4Wqgud1r/Ar/TrLOyssJL6szGxf8sZjVYA6FSD5npGBCkcBHTI80PdEGT2GbmB", + "C6ZJgJQQ0a1WLDFJdQWFd2kLIANYDYLM1WBZtF4d/3yCP2QzgFqG5ZLxAvaRZwKBCvUbPfTWVa6gEzcE", + "LKPsDLPlD2YuY1GT2pvLh1HEquV9m/bei2dp1QrqV3e3KsiZz1FCzfc+biZJkAqqFmeaDNksDVC59yA1", + "aAj0CTYBP+eTQwatz5/huX3ieXV7SRgRNEAHp8e2bi8D+QtdnKCITkiwCCJiEyAtxQyC6PHm8LhrMrdl", + "BeH19FQBQFzls4PTYyikJIxU2+r3tnt9QLGEMJzQ1qC109uCslIaDLDFTUi4CX9ahxrjS0s5Ow4td/7Z", + "NNG9BI6JgvL8v3kcUxQRJoGnBFkITwvCaIKpsNJoEoG7jNE1qe4LIduOwA8Ml+gYgOOmlg+pFvbxkCRv", + "7LG+1+hgnFdhi9v9vhH+mbLsAOcFdjZ/t07G+byNpAwAjyd+eUnadJKOBfnnTmu3v3Wj9aytieOb9pxh", + "WwCawDKf3BAIt5r0mBmPBmRi46xdv3jPAIWKN+y39/q8ZBrHWCwcuHJYJVzWiWhEIgxVOUz22N/5uIes", + "JQ5SNskZT6MQjQky7hok1GQUI4VFb/oRYRHM6JwMmeUepr4RFpDSLkaaaxhhunw1zNTm9DNH2p95uKhA", + "NxtuUw8H0lYZwNXUMJKMIMJ9VJcaOjNjJ5QxKDEjic2jk+VIXX48gJpgMuDeYmiEYabyElOmGNgVgQi+", + "Cf3gHbBRKKomeHAsBGpPZikStzf8PliQ8cfvvniUfUMWvGUmp3UEW4w9kwScWwAWYxxF3lilacTHOLI1", + "066IR3B6CS0sUIrJkRzLZTwkJtFNslAzzszf6ThlKjV/jwW/lkRoxmwT3llYu9LTBnWheCWNIemcSaer", + "59w0S9z8dEUWn3tDdhDGLlWytPUuI8ltMbmsOLl7Rza460/JVGPGPrTFZU0hp2LtG7NMnqokVdaEJ4my", + "WfqgOZRGkjMSDpni6JMwlTAXnzc/5TN+Boma4FDjSaGJ2dLmJxp+rlu1HGG9+xE09egkBAAwbGnuMmzp", + "v6cCa4k6lTNQsCUo1dPikbaz+BEtrWxUIRxghhKepLbuv6YnUCOvNAZkPMVRhBRcJddXy0BwkjX7se50", + "vvId1pfOOD9VrhEU8ihcpv7uvv8+SRII4lO7//vszWsErEqfgWmWGzMARpRpLorCFORLmL03ZM9xMLO1", + "4SG/wrBFw2Erk3nDDVhrKu1jf7cLgtdPemk/mWk6NPyp19NDGZlugH77ZEYZ6LuUxCPFrwgbtj53UOHD", + "lKpZOs6+vfcDtM4l6axECFDb0P4Nl68aQqNyNmj4BmYh4pbWRguEUU6Bitr9mDIsVibb9oDeQlArmHgq", + "i8D4NASz3LA1GDrD3LDVGbYIm8Nv1no3bH32Q8Amh68P5jf5xm2zHIn2+v2N9b7CFr4eEbrUUF+/z0vS", + "1/ZXEzys0LUseJjNuUwk+gRN5ngjbt2D5PMzDl0u0h8i3hoRz+rTBeEN+hf5gEHfiJgXh4oEphXwyElg", + "K7UTgxaQigc0DufZbxQO6iS4HHmL6kdVyVxWK3brblkAS4wc/u3eA/7BvHn1RZj32X3NiyNTJ9zVIntc", + "6AiH5RCx49eIXxL1LWBc/75IqSsS+4D4+1jw5yWxcl8OtAo12yRz9wrij18C3zZpRzGNta56BmvqnhGm", + "0HP4tWf/6zQeyMZ1GfHp5QAZEEZ8iiLK7BtR4Q1DM0ULS+hk3NuyftbbzQWPtw3//Nc//gmLomz6r3/8", + "U0vT5i+47psmog+STV3OCBZqTLC6HKBfCUm6OKJz4jYD6WDInIgF2umDmJkI+OSpbyOHbMjeEpUKVnhL", + "M3Hc0g5oy63q/VCWEmndA3VDOrFBZsbs6VHh3V02oLzXG91Z9qUwOyhsQHNFhwMQNUBNxi2rf7X81jOz", + "55L9rGrBXbLpr6cvinxQBnu7ZoE3JDAAYt+9gw9206h9dvZ8o4dAxzBYAYGEIDHnw1jhufeDJq2nSYai", + "lAkKQNnQpkJpw1r775Ft08wAbEf8nizAdbUa603AxuRBBAkdvH7oCk3MwX64OdOwzz575JzC6w20t99v", + "cQrnotJIEf565+xwbxnm5ksBZA+hAqO2LTmflRk5PTx2+aw3Hgzp74Vr6J3aLLAZ60DcFDe5N7XskLNJ", + "RAOFum4tkIg0JpmqVkaQx0IO3tpVI+z2VU3ZUeRvm6UI1FpOlwWj5izv7rlHZdKbsJE8rUiOaz84yTrU", + "OaIy4LpvAVu6AU5skRUjvmT3tIhF6wxSR/B7xnJWikuWPB8fuQt5f6YpO3XKqrzhHojiUYUgPiAhrBSO", + "KCTieUzYfJ6doguAW2G5+rZQs39/UtB9W7F8aP6YzFhhBWyaCs6ycuJ16GULjt/hQdsZPBs/I8Ldapc3", + "DXadbct0RcGMBFdmQ/AgvVr3PTZNmqm+ZrzvSfM1ld5vILFYkP8QURoouzmsVim4x7YYxt3ptzDDjdTb", + "r/fOaxHMA2RwNhk7i7WpM4HlggUb39VT771wMwPsR8nMTtMoci8ecyJUXu++yAM2P4Fb0nrZ3t22lezg", + "/O2rLmEBBz+0zIfKL0S5MtRfV8I3B2a28gNNmuiEACqHGPUC9Becv3EXRFlNw//YfmGrGv7H9gtT1/A/", + "dg5MZcONO0OW/n2R5vuWuB8x8mmBm5aBBqTJFIteJ6FmrRoKqa79dyWnmk3fSFLN4PpDWG0irBbBtVJe", + "tUdxpxKrLZf/ME8yGbL5oA2fnH/idyap3q+Vz2KkyzlGZfnZwyaV5iIvUU8ZSiV5hA6UNMO4IttoaK7O", + "L+RK9uFQ9/ioA4DsaNAdH+UBIvdkvHbruHfh1s57/5brg3hMpylPZTH2JMYqmBFpg5UiUibAj03sztlz", + "reD9DWNp/z5Zx73L1T/w/o4k/uqBGuJtXqDWyfyuVVOZ37bXMr8t+Gli1966QqI2+c5GjVOhq7bRFI1L", + "VaGXnR196/LpIuhcKyq5uoBAgxgM2X9p/eM3RXD8/icXJJP2+9t78Dth8/c/uTgZduJQhTAlqE1KdfD6", + "CJ79phC9Dnmr8pC86jpMElNAPZdW5d9OQcpfPptrSA4Lf2hIjTSkArhWa0hZodq7VJHKmZXuXUdy+OYD", + "uE2t8UNLug8tSaaTCQ0oYSrP8b/kJGZLhDzC2DJm34cKzh0lRttYS8qrR68WQPMspPfu2JNNfv/KkUt4", + "+jh95LmJigmdOpIzw3p95FvDh/79Euf710MeM4oZgX8ZdImWKX2lYyD/YJwqcErMM4SA16cr/5+N2EN5", + "xRZbmFeaHIYgAEMKfzXTArAv32E5haEvZ2HCI80hZGfIIDGq/mxi+TevyMJkKKQ8r0Sa7dRmJfTFXpUT", + "PD7oNfr6MpY/e2UjGeuer7HJdPmAMtaDkY57kbSObfSDVbqziwEK5ZhkN5lnwX30I2XTjUflgWqIVba3", + "Qj4jj6i1ObFFVPwq0AsurpoSBU/i90dAG4o7/Aa1L708yJ/08EoYqCfm/mikuXe6sZTN/yGd1mmVkgRR", + "GmrS4UiIY74TweOR/dHkndS3wlbEBKUusKM+NJHRs9+Div2aK0TjJCJa7iEh6hps0qdphSWXvJnKQu2L", + "mxFBfW2KIQQmfZc0VtUOshVC4DnCHVgbXiaXj8tLNSM+XZ82IJvcxch78gYMmUkuTVwm6kuUEVmkOJIk", + "IoFC1zMazCCHgP7NlNyB8H6cJJdZ0qCNAXoJN7WYOwkmb0sitOgYcCZ5RExqgHkcXw6Wc1xenJxAJ5M+", + "wGSzvBwgl9cyYxBStyrmBNC7iLBU6LXNdNDWmCR4FJkTvdRydmF/GzZbQJ7Uach8mQMYubYD0gm6LCQR", + "uKzJIuAI6is+lQ8lynbqU/GZvUCheQ04g5sEyhL4Tdc08ucP2Op7Syc0zGVglnHHqQyWFvOKT7M0gCVU", + "xknSFH3tMgGL53G8AodRu1A9R6qQp+qvUoVECOhssbsOuVHbljNHCl9pRGW2nLKrPwTo532gMXm5vKDS", + "RLVQZ8H8ax7HrU7LrsdTV/7Lc0JUB1x+SNAnU0j88MOWcJOUDmViX8jpUOEctvZevchtSwp+9xYtC6jw", + "e9BLyy8A+Sooc6IKnC3P67Q8qthwU22yKouZojK+O5KVq6y/JeVns7O8fM2/oYpq9lqtMXrPSmoGYp9m", + "Vqqn9uDaaVZd64eGmmmoXKAwNdNVCix+t2pnRlBQykqapxVPb6t7Zmk0MzBDLXW28skzp3mbn9yfx7cQ", + "F74RStiprcNWl7At3/S3QHJr6jp/i08GTk6ybLUgIDwgCXYVph/sbUGrexmV+ybIsLlwGTUu0hwlMJPU", + "FY//QYxLZkBjKb0tMXbC55ItsECeKesmEa6jy1ZOrSXAlRqH/2ZPpzUVHD9bSviQhC9Xju6N2B1n5M0Q", + "PFui9ztXUQMuhPEGBv/ixxSNXnCOKOja7QSnknQyCtFxrkgXJycbdVRCqJU0QqhHTCEq1VpjT6mPN3Mi", + "BA1d3ZHDkyNbpYRKJFLWQ29iCsVArghJIMsw5ak0dTN7xRKSdcVIsxqRhCmxSDhlau0q8qZ3s5jPt6q7", + "cM900ubj+O7taPD29/iIFNAOLa7YDazWIpWpnOp9l3TvdJSZYilQLX3MUz36UolLNKERkQupSGweKSdp", + "BJcIMjbZhN62n3FH7iCqJNL3oQPumwkRMYUCzXLIxmSixbCECD03VJ6mESm8t/ie8s4UzqjmqSF938Zb", + "HlS9hOcrrOqgVi54iZPEFbz0vRdlNTpvvaQX8DiH5CIe84gGKKLsSqJ2RK+M0oHmEkX6j42Vr3sj6Pe1", + "05Xf/mZpSB+zCfdmdDU4myHz9+HBViZrznvh0ZG1l6R4WRz9gYP2kzW5lq4JgiOo65xFTqBU0Yh+NKRO", + "D0KlooEpg5f77UIFL+u6O2QnRAndBguCAh5FJFDOuLKZCB5sDtN+fydIKIS47RBYHBC8+s8xzHh4eg7t", + "TJWxzpDpf8DA7w5OEdUwnWBrIygslBF1zcUVOt58s8bf4QzA9G/8YGg2uNKB1nvgP56yb+4WX3uHZM0V", + "5ckqBYgn3/2LtpXgflgLHqe1AOKSst20pwIHIBTLWapCfs38lgFT9FpufjJ/HK+LblM4mF246v/fhrRr", + "C4Cvm8Zt8FFcSrunkJiM0w/yQGFrtD/SDH0acG4LIMQU4/T8XOBAfY/Y/fWN8kU4foNPkxaiLpv7N3O3", + "7pvz2TW4oO0iPB7LNTeY5nYCVYiL1qcsQn2tbhaYiEjI7pWLlgFOcEDVooNw5Apk24p3mQ2pm7HcsSD4", + "SnPa3pC9zWLjbcU9rV11nGqFQiqvzAhWe+qhN3MiZDrOFoeAMBk9D4Bva2QHOApMcWkymZBA0TkxVZ9l", + "jfaVLeUuM63nk3gO2n20oHtsKocfJ+D0crSQFYzbNEe9KUgQYRoXVY8qcIBAguEfjANjPShniLJJZI2g", + "geBSIjtUl0R0SseRNenJHno3I0jimAxZEmHGCBTUhtdpvfRuIoiUKXg7wABQysJgVAflkUGJ4MoaEyLO", + "hTT6v8bwixMkFUlWoNlbM7IpQn5HmUjM4HamB2IrlTWYWfwob0CtD8RgigG4xqM0UvIBnoTNgn4Um252", + "8d8JOp0SoW8FNkTWGNDNtXbgNJe+5A9cm4brLGvVLA1XNmrB56/gD7cysmPkGo5A/LyJnd4z+RWtDf6x", + "n27mo/ur7tRw7rIvqH8R9tMX7vJ7yW58VnDBa5q8K8fwx5ZHq7Dy0lUtubGuT+jT2G/1Lv1I1yX0ySa/", + "74Q+Z15XxkeWVhSXnFPrMvl8e4jQv98YivvO5PO4cUvrD3IJdPWUqEG+i28CA+8m0cUDxxDdItHFN+XV", + "DokKHi666JvyZ7d+2Zk/+49UFnfpxm7yWUDYfp0bu6F69rlppaJ0Yds0U5PsiN+TBG9fKG4gvzuw/9D6", + "G6gMBWD5TXYmgk5ahCdxohbOBM0n4GyX54aW9CO47PrCg7OXpruLyr3FI8zXQw+Hp7VPMD9S9t7bK09e", + "1+T46PHn6S3euRJj2dRcp4tFMKNzUm90L99gC6JEkG7CE3hcCQ3ALDwcL1NY9KYfkR2+N2TvZsT9C1GX", + "E4iEKKSCBCpaIMoUB4pg5vizRIJrTQC+c7HwGdOLN/eF4PGB3c0afmjvlDWG5b698aIbYoW7c0dtVpjQ", + "vuCd+gR/oHEaA8FDlKGXP6M2+aCESVGDJlrzQXSSgZR8CAgJJeDkRnHBW/0ayyb9SEbTcZNVrkg29MYm", + "c0JBKhWP3dkfH6E2ThXvTgnTZ6FF/QlIsongcxqaWgc5UOc8MlDdqgHoTe2uWqiwQSC5cmEW9yAyTBOG", + "NP1IkzJZML7OrUFrTBmGxa1N61O+U8btXs+HKTi/5nfHYU7rBwuzml/bKTsaE7WS44CoOEeRlug3frC5", + "x8zmiu5LjqeVuF2zbPTNPJoaOhrdRSb6zNvtfs3WF9+OEw6Vj9L/xprO55lCWmc2/7ZQsH9//OG+zeUX", + "j9hp8yVxynfBVA4D6BF9CPOKBzhCIZmTiCeQqN60bXVaqYhag9ZMqWSwuRnpdjMu1WC/v99vfX7/+f8H", + "AAD//xqeUJdsRgEA", } // GetSwagger returns the content of the embedded swagger specification file diff --git a/lib/providers/providers.go b/lib/providers/providers.go index 501f738d..566e0077 100644 --- a/lib/providers/providers.go +++ b/lib/providers/providers.go @@ -25,6 +25,7 @@ import ( "github.com/kernel/hypeman/lib/paths" "github.com/kernel/hypeman/lib/registry" "github.com/kernel/hypeman/lib/resources" + "github.com/kernel/hypeman/lib/snapshot" "github.com/kernel/hypeman/lib/system" "github.com/kernel/hypeman/lib/vm_metrics" "github.com/kernel/hypeman/lib/volumes" @@ -126,13 +127,31 @@ func ProvideInstanceManager(p *paths.Paths, cfg *config.Config, imageManager ima meter := otel.GetMeterProvider().Meter("hypeman") tracer := otel.GetTracerProvider().Tracer("hypeman") defaultHypervisor := hypervisor.Type(cfg.Hypervisor.Default) + snapshotDefaults := snapshotDefaultsFromConfig(cfg) memoryPolicy := guestmemory.Policy{ Enabled: cfg.Hypervisor.Memory.Enabled, KernelPageInitMode: guestmemory.KernelPageInitMode(cfg.Hypervisor.Memory.KernelPageInitMode), ReclaimEnabled: cfg.Hypervisor.Memory.ReclaimEnabled, VZBalloonRequired: cfg.Hypervisor.Memory.VZBalloonRequired, } - return instances.NewManager(p, imageManager, systemManager, networkManager, deviceManager, volumeManager, limits, defaultHypervisor, meter, tracer, memoryPolicy), nil + return instances.NewManager(p, imageManager, systemManager, networkManager, deviceManager, volumeManager, limits, defaultHypervisor, snapshotDefaults, meter, tracer, memoryPolicy), nil +} + +func snapshotDefaultsFromConfig(cfg *config.Config) instances.SnapshotPolicy { + if !cfg.Snapshot.CompressionDefault.Enabled { + return instances.SnapshotPolicy{} + } + + algorithm := snapshot.SnapshotCompressionAlgorithm(strings.ToLower(cfg.Snapshot.CompressionDefault.Algorithm)) + compression := &snapshot.SnapshotCompressionConfig{ + Enabled: true, + Algorithm: algorithm, + } + if cfg.Snapshot.CompressionDefault.Level != nil { + level := *cfg.Snapshot.CompressionDefault.Level + compression.Level = &level + } + return instances.SnapshotPolicy{Compression: compression} } // ProvideGuestMemoryController provides the active ballooning controller. diff --git a/lib/providers/providers_test.go b/lib/providers/providers_test.go new file mode 100644 index 00000000..5ea519ba --- /dev/null +++ b/lib/providers/providers_test.go @@ -0,0 +1,71 @@ +package providers + +import ( + "testing" + + "github.com/kernel/hypeman/cmd/api/config" + snapshotstore "github.com/kernel/hypeman/lib/snapshot" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestSnapshotDefaultsFromConfigDisabledReturnsNilCompression(t *testing.T) { + t.Parallel() + + cfg := &config.Config{ + Snapshot: config.SnapshotConfig{ + CompressionDefault: config.SnapshotCompressionDefaultConfig{ + Enabled: false, + Algorithm: "lz4", + Level: intPtr(7), + }, + }, + } + + defaults := snapshotDefaultsFromConfig(cfg) + assert.Nil(t, defaults.Compression) +} + +func TestSnapshotDefaultsFromConfigPreservesLevelForLZ4(t *testing.T) { + t.Parallel() + + cfg := &config.Config{ + Snapshot: config.SnapshotConfig{ + CompressionDefault: config.SnapshotCompressionDefaultConfig{ + Enabled: true, + Algorithm: "lz4", + Level: intPtr(7), + }, + }, + } + + defaults := snapshotDefaultsFromConfig(cfg) + require.NotNil(t, defaults.Compression) + assert.True(t, defaults.Compression.Enabled) + assert.Equal(t, snapshotstore.SnapshotCompressionAlgorithmLz4, defaults.Compression.Algorithm) + require.NotNil(t, defaults.Compression.Level) + assert.Equal(t, 7, *defaults.Compression.Level) +} + +func TestSnapshotDefaultsFromConfigKeepsZstdLevel(t *testing.T) { + t.Parallel() + + cfg := &config.Config{ + Snapshot: config.SnapshotConfig{ + CompressionDefault: config.SnapshotCompressionDefaultConfig{ + Enabled: true, + Algorithm: "zstd", + Level: intPtr(5), + }, + }, + } + + defaults := snapshotDefaultsFromConfig(cfg) + require.NotNil(t, defaults.Compression) + require.NotNil(t, defaults.Compression.Level) + assert.Equal(t, 5, *defaults.Compression.Level) +} + +func intPtr(v int) *int { + return &v +} diff --git a/lib/snapshot/README.md b/lib/snapshot/README.md index edcdc989..b8233c72 100644 --- a/lib/snapshot/README.md +++ b/lib/snapshot/README.md @@ -31,6 +31,77 @@ Snapshots are immutable point-in-time captures of a VM that can later be: - `Stopped` snapshot from `Stopped`: - snapshot payload is copied directly +## Snapshot Compression + +Snapshot memory compression is optional and is **off by default**. + +- Compression applies only to `Standby` snapshots, because only standby snapshots contain guest memory state. +- `Stopped` snapshots cannot use compression because they do not include resumable RAM state. +- Compression affects only the memory snapshot file, not the entire snapshot directory. + +### When compression runs + +- A standby operation can request compression explicitly. +- A snapshot create request can request compression explicitly. +- If the request does not specify compression, Hypeman falls back to: + - the instance's `snapshot_policy` + - then the server's global `snapshot.compression_default` +- Effective precedence is: + - request override + - instance default + - server default + - otherwise no compression + +Compression runs **asynchronously after the snapshot is already durable on disk**. + +- This keeps the standby path fast. +- Standby can return successfully while compression is still running in the background. +- That means a later restore can arrive before compression has finished. +- While compression is running, the snapshot remains valid and reports `compression_state=compressing`. +- Once finished, the snapshot reports `compression_state=compressed` and exposes compressed/uncompressed size metadata. +- If compression fails, the snapshot reports `compression_state=error` and keeps the error message for inspection. + +### Restore behavior with compressed snapshots + +Restore always uses the same flow across hypervisors. + +- If a restore starts while compression is still running, Hypeman cancels the in-flight compression job first. +- Restore then synchronizes with that job so it is no longer mutating the snapshot files. +- If the raw memory file still exists after cancellation, restore uses the raw file directly and removes any compressed sibling artifacts. +- If the raw file is already gone, Hypeman expands the compressed file back to the raw memory file before asking the hypervisor to restore. +- This means restore never races a live compressor, while still preferring the raw snapshot whenever it is still available. + +In practice, the tradeoff is: + +- standby stays fast because compression is not on the hot path +- restore from a compressed standby snapshot pays decompression cost +- restore from an uncompressed standby snapshot avoids that cost entirely + +### Supported algorithms + +- `zstd` + - default when compression is enabled + - supports levels `1-19` + - defaults to level `1` when no level is specified +- `lz4` + - optimized for lower decompression overhead + - supports levels `0-9` + - defaults to level `0` (fastest) when no level is specified + +### Native codec preference + +Compression and decompression try native system codecs first: + +- `zstd` snapshots prefer the `zstd` binary +- `lz4` snapshots prefer the `lz4` binary + +If the native binary is missing or not executable, Hypeman falls back to the in-process Go implementation. + +- This fallback is operational, not a behavior change: snapshot artifacts remain the same `.zst` and `.lz4` formats. +- A fallback emits a warning log with an install hint. +- A fallback also increments `hypeman_snapshot_codec_fallbacks_total`. +- If the native codec starts but fails for a real runtime reason, Hypeman does **not** silently fall back; it treats that as an error. + ### Restore (in-place) - Restore always applies to the original source VM. - Source VM must not be `Running`. diff --git a/lib/snapshot/types.go b/lib/snapshot/types.go index 0c19596d..59848aa1 100644 --- a/lib/snapshot/types.go +++ b/lib/snapshot/types.go @@ -19,15 +19,51 @@ const ( // Snapshot is a centrally stored immutable snapshot resource. type Snapshot struct { - Id string `json:"id"` - Name string `json:"name"` - Kind SnapshotKind `json:"kind"` - Tags tags.Tags `json:"tags,omitempty"` - SourceInstanceID string `json:"source_instance_id"` - SourceName string `json:"source_instance_name"` - SourceHypervisor hypervisor.Type - CreatedAt time.Time `json:"created_at"` - SizeBytes int64 `json:"size_bytes"` + Id string `json:"id"` + Name string `json:"name"` + Kind SnapshotKind `json:"kind"` + Tags tags.Tags `json:"tags,omitempty"` + SourceInstanceID string `json:"source_instance_id"` + SourceName string `json:"source_instance_name"` + SourceHypervisor hypervisor.Type + CreatedAt time.Time `json:"created_at"` + SizeBytes int64 `json:"size_bytes"` + CompressionState string `json:"compression_state,omitempty"` + CompressionError string `json:"compression_error,omitempty"` + Compression *SnapshotCompressionConfig `json:"compression,omitempty"` + CompressedSizeBytes *int64 `json:"compressed_size_bytes,omitempty"` + UncompressedSizeBytes *int64 `json:"uncompressed_size_bytes,omitempty"` +} + +// SnapshotCompressionAlgorithm defines supported compression algorithms. +type SnapshotCompressionAlgorithm string + +const ( + SnapshotCompressionAlgorithmZstd SnapshotCompressionAlgorithm = "zstd" + SnapshotCompressionAlgorithmLz4 SnapshotCompressionAlgorithm = "lz4" +) + +const ( + DefaultSnapshotCompressionZstdLevel = 1 + MinSnapshotCompressionZstdLevel = 1 + MaxSnapshotCompressionZstdLevel = 19 + DefaultSnapshotCompressionLz4Level = 0 + MinSnapshotCompressionLz4Level = 0 + MaxSnapshotCompressionLz4Level = 9 +) + +const ( + SnapshotCompressionStateNone = "none" + SnapshotCompressionStateCompressing = "compressing" + SnapshotCompressionStateCompressed = "compressed" + SnapshotCompressionStateError = "error" +) + +// SnapshotCompressionConfig defines requested or effective compression config. +type SnapshotCompressionConfig struct { + Enabled bool `json:"enabled"` + Algorithm SnapshotCompressionAlgorithm `json:"algorithm,omitempty"` + Level *int `json:"level,omitempty"` } // ListSnapshotsFilter contains optional filters for listing snapshots. diff --git a/lib/system/manager.go b/lib/system/manager.go index ac28f540..6ea7ffc2 100644 --- a/lib/system/manager.go +++ b/lib/system/manager.go @@ -4,10 +4,14 @@ import ( "context" "fmt" "os" + "path/filepath" + "sync" "github.com/kernel/hypeman/lib/paths" ) +var initrdEnsureLocks sync.Map + // Manager handles system files (kernel, initrd) type Manager interface { // EnsureSystemFiles ensures default kernel and initrd exist @@ -34,6 +38,19 @@ func NewManager(p *paths.Paths) Manager { } } +func initrdEnsureLockKey(initrdDir string) string { + if resolved, err := filepath.EvalSymlinks(initrdDir); err == nil { + return resolved + } + return initrdDir +} + +func getInitrdEnsureLock(initrdDir string) *sync.Mutex { + key := initrdEnsureLockKey(initrdDir) + lock, _ := initrdEnsureLocks.LoadOrStore(key, &sync.Mutex{}) + return lock.(*sync.Mutex) +} + // EnsureSystemFiles ensures default kernel and initrd exist, downloading/building if needed func (m *manager) EnsureSystemFiles(ctx context.Context) error { kernelVer := m.GetDefaultKernelVersion() @@ -44,6 +61,10 @@ func (m *manager) EnsureSystemFiles(ctx context.Context) error { } // Ensure initrd exists (builds if missing or stale) + initrdLock := getInitrdEnsureLock(m.paths.SystemInitrdDir(GetArch())) + initrdLock.Lock() + defer initrdLock.Unlock() + if _, err := m.ensureInitrd(ctx); err != nil { return fmt.Errorf("ensure initrd: %w", err) } diff --git a/lib/system/manager_test.go b/lib/system/manager_test.go index 2b6e9406..8c70e93c 100644 --- a/lib/system/manager_test.go +++ b/lib/system/manager_test.go @@ -2,6 +2,8 @@ package system import ( "context" + "os" + "path/filepath" "testing" "github.com/kernel/hypeman/lib/paths" @@ -64,3 +66,14 @@ func TestInitBinaryEmbedded(t *testing.T) { assert.NotEmpty(t, InitBinary, "init binary should be embedded") assert.Greater(t, len(InitBinary), 100000, "init binary should be at least 100KB") } + +func TestInitrdEnsureLockKeyResolvesSymlinks(t *testing.T) { + target := filepath.Join(t.TempDir(), "shared-initrd") + require.NoError(t, os.MkdirAll(target, 0755)) + + wrapperRoot := t.TempDir() + wrapper := filepath.Join(wrapperRoot, "initrd") + require.NoError(t, os.Symlink(target, wrapper)) + + assert.Equal(t, initrdEnsureLockKey(target), initrdEnsureLockKey(wrapper)) +} diff --git a/openapi.yaml b/openapi.yaml index 4b0b59d4..0c41698d 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -320,6 +320,9 @@ components: enum: [cloud-hypervisor, firecracker, qemu, vz] description: Hypervisor to use for this instance. Defaults to server configuration. example: cloud-hypervisor + snapshot_policy: + description: Snapshot compression policy for this instance. Controls compression settings applied when creating snapshots or entering standby. + $ref: "#/components/schemas/SnapshotPolicy" skip_kernel_headers: type: boolean description: | @@ -433,6 +436,56 @@ components: format: int64 description: Total payload size in bytes example: 104857600 + compression_state: + type: string + enum: [none, compressing, compressed, error] + description: Compression status of the snapshot payload memory file + example: compressed + compression_error: + type: string + description: Compression error message when compression_state is error + nullable: true + example: "write compressed stream: no space left on device" + compression: + $ref: "#/components/schemas/SnapshotCompressionConfig" + compressed_size_bytes: + type: integer + format: int64 + nullable: true + description: Compressed memory payload size in bytes + example: 73400320 + uncompressed_size_bytes: + type: integer + format: int64 + nullable: true + description: Uncompressed memory payload size in bytes + example: 4294967296 + + SnapshotCompressionConfig: + type: object + required: [enabled] + properties: + enabled: + type: boolean + description: Enable snapshot memory compression + example: true + algorithm: + type: string + enum: [zstd, lz4] + description: Compression algorithm (defaults to zstd when enabled). Ignored when enabled is false. + example: zstd + level: + type: integer + minimum: 0 + maximum: 19 + description: Compression level. Allowed ranges are zstd=1-19 and lz4=0-9. When omitted, zstd defaults to 1 and lz4 defaults to 0. Ignored when enabled is false. + example: 1 + + SnapshotPolicy: + type: object + properties: + compression: + $ref: "#/components/schemas/SnapshotCompressionConfig" CreateSnapshotRequest: type: object @@ -448,6 +501,15 @@ components: example: pre-upgrade tags: $ref: "#/components/schemas/Tags" + compression: + description: Compression settings to use for this snapshot. Overrides instance and server defaults. + $ref: "#/components/schemas/SnapshotCompressionConfig" + + StandbyInstanceRequest: + type: object + properties: + compression: + $ref: "#/components/schemas/SnapshotCompressionConfig" RestoreSnapshotRequest: type: object @@ -614,6 +676,8 @@ components: enum: [cloud-hypervisor, firecracker, qemu, vz] description: Hypervisor running this instance example: cloud-hypervisor + snapshot_policy: + $ref: "#/components/schemas/SnapshotPolicy" PathInfo: type: object @@ -1947,6 +2011,12 @@ paths: schema: type: string description: Instance ID or name + requestBody: + required: false + content: + application/json: + schema: + $ref: "#/components/schemas/StandbyInstanceRequest" responses: 200: description: Instance in standby @@ -1954,6 +2024,12 @@ paths: application/json: schema: $ref: "#/components/schemas/Instance" + 400: + description: Invalid request payload + content: + application/json: + schema: + $ref: "#/components/schemas/Error" 404: description: Instance not found content: diff --git a/skills/hypeman-remote-linux-tests/SKILL.md b/skills/hypeman-remote-linux-tests/SKILL.md new file mode 100644 index 00000000..d2599ff1 --- /dev/null +++ b/skills/hypeman-remote-linux-tests/SKILL.md @@ -0,0 +1,76 @@ +--- +name: hypeman-remote-linux-tests +description: Run Hypeman tests on a remote Linux host that supports the Linux hypervisors. Use when validating cloud-hypervisor, Firecracker, QEMU, embedded guest artifacts, or Linux-only integration behavior on a server over SSH. +--- + +# Hypeman Remote Linux Tests + +Use this skill to run Hypeman tests on a remote Linux server in a way that is repeatable and low-friction for any developer. + +## Quick Start + +1. Pick a short remote working directory under the remote user's home directory. +2. Ensure the remote test shell includes `/usr/sbin` and `/sbin` on `PATH`. +3. Bootstrap embedded artifacts and bundled binaries before running `go test`. +4. Run the smallest relevant test target first. +5. Treat host-environment failures separately from product failures. + +## Remote Workspace + +Prefer a short path such as `~/hm`, `~/hypeman-test`, or another short directory name. + +Cloud Hypervisor uses UNIX sockets with strict path-length limits. Long paths from default test temp directories or deeply nested clones can make the VMM fail before the actual test starts. + +If a new test creates temporary directories for guest data, prefer a short root like: + +```bash +mktemp -d /tmp/hmcmp-XXXXXX +``` + +## Remote Bootstrap + +Before running Linux hypervisor tests, make sure the remote clone has the generated guest binaries and embedded VMM assets: + +```bash +export PATH=/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin:$PATH +make ensure-ch-binaries ensure-firecracker-binaries ensure-caddy-binaries build-embedded +``` + +Why: + +- `go:embed` patterns in the repo require the bundled binaries to exist at build time. +- `mkfs.ext4` is commonly installed in `/usr/sbin` or `/sbin`, not always on the default non-login shell `PATH`. +- Running `go test` before this bootstrap often fails during package setup, not during test execution. + +QEMU is usually provided by the host OS, so there is no matching bundled `ensure-qemu-binaries` target. If QEMU tests fail before boot, verify the system `qemu-system-*` binary separately. + + +## Syncing Local Changes + +If the main development is happening locally and the Linux host is only for execution, sync only the files you changed rather than recloning every time. + +Typical pattern: + +```bash +rsync -az lib/instances/... remote-host:~/hypeman-test/lib/instances/ +``` + +After syncing, re-run the bootstrap command if the changes affect embedded binaries, generated assets, or files referenced by `go:embed`. + +## Common Host Failures + +Classify failures quickly: + +- Missing embedded assets: + - symptom: package setup fails with `pattern ... no matching files found` + - fix: run `make ensure-ch-binaries ensure-firecracker-binaries ensure-caddy-binaries build-embedded` +- `mkfs.ext4` not found: + - symptom: overlay or disk creation fails with `exec: "mkfs.ext4": executable file not found` + - fix: add `/usr/sbin:/sbin` to `PATH`, then retry +- Cloud Hypervisor socket path too long: + - symptom: `path must be shorter than SUN_LEN` + - fix: shorten the remote repo path and any temp/data directories used by the test +- `/tmp` or shared test-lock issues: + - symptom: test-specific lock or temp-file permission failures + - fix: prefer no-network test setup when the test does not require networking; avoid unnecessary shared lock files + diff --git a/skills/hypeman-remote-linux-tests/agents/openai.yaml b/skills/hypeman-remote-linux-tests/agents/openai.yaml new file mode 100644 index 00000000..244c1d28 --- /dev/null +++ b/skills/hypeman-remote-linux-tests/agents/openai.yaml @@ -0,0 +1,4 @@ +interface: + display_name: "Hypeman Remote Linux Tests" + short_description: "Run Hypeman tests on a remote Linux hypervisor host" + default_prompt: "Run Hypeman tests on a remote Linux host, prepare prerequisites, and report actionable failures." diff --git a/stainless.yaml b/stainless.yaml index d1e746f4..dbd1ed2c 100644 --- a/stainless.yaml +++ b/stainless.yaml @@ -42,6 +42,10 @@ environments: # # [configuration guide]: https://www.stainless.com/docs/guides/configure#resources resources: + $shared: + models: + snapshot_compression_config: "#/components/schemas/SnapshotCompressionConfig" + health: # Configure the methods defined in this resource. Each key in the object is the # name of the method and the value is either an endpoint (for example, `get /foo`) @@ -67,6 +71,7 @@ resources: instances: models: + snapshot_policy: "#/components/schemas/SnapshotPolicy" volume_mount: "#/components/schemas/VolumeMount" port_mapping: "#/components/schemas/PortMapping" instance: "#/components/schemas/Instance"