diff --git a/cns/azure-cns-windows.yaml b/cns/azure-cns-windows.yaml index 442678698b4..7562c10582e 100644 --- a/cns/azure-cns-windows.yaml +++ b/cns/azure-cns-windows.yaml @@ -110,6 +110,7 @@ data: }, "ChannelMode": "CRD", "EnableBoltStateStore": false, + "EnablePersistentStateDebug": false, "InitializeFromCNI": true, "StateStoreBackend": "json", "StateStoreMode": "normal" diff --git a/cns/azure-cns.yaml b/cns/azure-cns.yaml index 97e2f5c1953..32f0c71742f 100644 --- a/cns/azure-cns.yaml +++ b/cns/azure-cns.yaml @@ -186,6 +186,7 @@ data: }, "ChannelMode": "CRD", "EnableBoltStateStore": false, + "EnablePersistentStateDebug": false, "InitializeFromCNI": true, "StateStoreBackend": "json", "StateStoreMode": "normal", diff --git a/cns/configuration/cns_config.json b/cns/configuration/cns_config.json index a32c8e74a48..25908d2ef86 100644 --- a/cns/configuration/cns_config.json +++ b/cns/configuration/cns_config.json @@ -18,6 +18,7 @@ }, "ChannelMode": "Direct", "EnableBoltStateStore": false, + "EnablePersistentStateDebug": false, "InitializeFromCNI": false, "StateStoreBackend": "json", "StateStoreMode": "normal", diff --git a/cns/configuration/configuration.go b/cns/configuration/configuration.go index 39304be7ee5..012cc24aaa5 100644 --- a/cns/configuration/configuration.go +++ b/cns/configuration/configuration.go @@ -56,6 +56,7 @@ type CNSConfig struct { EnableIPAMv2 bool EnableK8sDevicePlugin bool EnableLoggerV2 bool + EnablePersistentStateDebug bool EnablePprof bool EnableStateMigration bool EnableSubnetScarcity bool @@ -112,13 +113,38 @@ func (cnsconfig CNSConfig) ValidateStateStore() error { return fmt.Errorf("%w: mode %q", ErrInvalidStateStoreConfig, mode) } - boltEnabled := cnsconfig.EnableBoltStateStore - boltSelected := backend == StateStoreBackendBolt - rollbackSelected := mode == StateStoreModeRollbackToJSON - if boltEnabled || boltSelected || rollbackSelected { - return ErrStateStoreFeatureUnavailable + if !cnsconfig.EnableBoltStateStore { + if backend != StateStoreBackendJSON || mode != StateStoreModeNormal || cnsconfig.EnablePersistentStateDebug { + return fmt.Errorf("%w: Bolt state store master flag is disabled", ErrInvalidStateStoreConfig) + } + return nil + } + + switch { + case backend == StateStoreBackendBolt && mode == StateStoreModeNormal: + if !cnsconfig.ManageEndpointState { + return fmt.Errorf("%w: Bolt state store requires CNS-managed endpoint state", ErrInvalidStateStoreConfig) + } + if cnsconfig.EnableStateMigration && !cnsconfig.InitializeFromCNI { + return fmt.Errorf("%w: Bolt CNI ownership import requires CNI initialization", ErrInvalidStateStoreConfig) + } + return nil + case backend == StateStoreBackendJSON && mode == StateStoreModeRollbackToJSON: + if !cnsconfig.ManageEndpointState { + return fmt.Errorf("%w: Bolt rollback requires CNS-managed endpoint state", ErrInvalidStateStoreConfig) + } + if cnsconfig.EnablePersistentStateDebug { + return fmt.Errorf("%w: persistent state debug requires normal Bolt mode", ErrInvalidStateStoreConfig) + } + return nil + case backend == StateStoreBackendJSON && mode == StateStoreModeNormal: + if cnsconfig.EnablePersistentStateDebug { + return fmt.Errorf("%w: persistent state debug requires normal Bolt mode", ErrInvalidStateStoreConfig) + } + return nil + default: + return fmt.Errorf("%w: backend %q does not support mode %q", ErrInvalidStateStoreConfig, backend, mode) } - return nil } type TelemetrySettings struct { diff --git a/cns/configuration/configuration_test.go b/cns/configuration/configuration_test.go index d61208e1378..6c4f69428ae 100644 --- a/cns/configuration/configuration_test.go +++ b/cns/configuration/configuration_test.go @@ -1,8 +1,10 @@ package configuration import ( + "fmt" "os" "path/filepath" + "strings" "testing" "github.com/Azure/azure-container-networking/common" @@ -10,6 +12,49 @@ import ( "github.com/stretchr/testify/require" ) +func TestPersistentStateManifestDefaultsRemainDark(t *testing.T) { + repositoryRoot := filepath.Clean(filepath.Join("..", "..")) + root, err := os.OpenRoot(repositoryRoot) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, root.Close()) }) + required := []string{ + `"EnableBoltStateStore": false`, + `"EnablePersistentStateDebug": false`, + `"StateStoreBackend": "json"`, + `"StateStoreMode": "normal"`, + } + found := 0 + err = filepath.WalkDir(repositoryRoot, func(path string, entry os.DirEntry, walkErr error) error { + if walkErr != nil { + return fmt.Errorf("walking persistent state manifests: %w", walkErr) + } + if entry.IsDir() || (filepath.Ext(path) != ".json" && filepath.Ext(path) != ".yaml" && filepath.Ext(path) != ".yml") { + return nil + } + relativePath, relativeErr := filepath.Rel(repositoryRoot, path) + if relativeErr != nil { + return fmt.Errorf("resolving persistent state manifest path: %w", relativeErr) + } + contents, readErr := root.ReadFile(relativePath) + if readErr != nil { + return fmt.Errorf("reading persistent state manifest %q: %w", relativePath, readErr) + } + text := string(contents) + if !strings.Contains(text, `"EnableBoltStateStore"`) { + return nil + } + found++ + for _, value := range required { + assert.Contains(t, text, value, "%s must explicitly preserve dark persistent-state defaults", path) + } + assert.NotContains(t, text, `"EnableBoltStateStore": true`, path) + assert.NotContains(t, text, `"StateStoreBackend": "bolt"`, path) + return nil + }) + require.NoError(t, err) + require.Equal(t, 17, found) +} + func TestGetConfigFilePath(t *testing.T) { execpath, _ := common.GetExecutableDirectory() @@ -317,142 +362,83 @@ func TestSetCNSConfigDefaults(t *testing.T) { } func TestCNSConfigValidateStateStore(t *testing.T) { - tests := []struct { - name string - enableBoltStateStore bool - enableStateMigration bool - backend StateStoreBackend - mode StateStoreMode - wantErr error - }{ - { - name: "json normal", - backend: StateStoreBackendJSON, - mode: StateStoreModeNormal, - }, - { - name: "json rollback", - backend: StateStoreBackendJSON, - mode: StateStoreModeRollbackToJSON, - wantErr: ErrStateStoreFeatureUnavailable, - }, - { - name: "bolt normal", - backend: StateStoreBackendBolt, - mode: StateStoreModeNormal, - wantErr: ErrStateStoreFeatureUnavailable, - }, - { - name: "bolt rollback", - backend: StateStoreBackendBolt, - mode: StateStoreModeRollbackToJSON, - wantErr: ErrStateStoreFeatureUnavailable, - }, - { - name: "enabled json normal", - enableBoltStateStore: true, - backend: StateStoreBackendJSON, - mode: StateStoreModeNormal, - wantErr: ErrStateStoreFeatureUnavailable, - }, - { - name: "enabled json rollback", - enableBoltStateStore: true, - backend: StateStoreBackendJSON, - mode: StateStoreModeRollbackToJSON, - wantErr: ErrStateStoreFeatureUnavailable, - }, - { - name: "enabled bolt normal", - enableBoltStateStore: true, - backend: StateStoreBackendBolt, - mode: StateStoreModeNormal, - wantErr: ErrStateStoreFeatureUnavailable, - }, - { - name: "enabled bolt rollback", - enableBoltStateStore: true, - backend: StateStoreBackendBolt, - mode: StateStoreModeRollbackToJSON, - wantErr: ErrStateStoreFeatureUnavailable, - }, - { - name: "migration json normal", - enableStateMigration: true, - backend: StateStoreBackendJSON, - mode: StateStoreModeNormal, - }, - { - name: "migration json rollback", - enableStateMigration: true, - backend: StateStoreBackendJSON, - mode: StateStoreModeRollbackToJSON, - wantErr: ErrStateStoreFeatureUnavailable, - }, - { - name: "migration bolt normal", - enableStateMigration: true, - backend: StateStoreBackendBolt, - mode: StateStoreModeNormal, - wantErr: ErrStateStoreFeatureUnavailable, - }, - { - name: "migration bolt rollback", - enableStateMigration: true, - backend: StateStoreBackendBolt, - mode: StateStoreModeRollbackToJSON, - wantErr: ErrStateStoreFeatureUnavailable, - }, - { - name: "migration enabled json normal", - enableBoltStateStore: true, - enableStateMigration: true, - backend: StateStoreBackendJSON, - mode: StateStoreModeNormal, - wantErr: ErrStateStoreFeatureUnavailable, - }, - { - name: "migration enabled json rollback", - enableBoltStateStore: true, - enableStateMigration: true, - backend: StateStoreBackendJSON, - mode: StateStoreModeRollbackToJSON, - wantErr: ErrStateStoreFeatureUnavailable, - }, - { - name: "migration enabled bolt normal", - enableBoltStateStore: true, - enableStateMigration: true, - backend: StateStoreBackendBolt, - mode: StateStoreModeNormal, - wantErr: ErrStateStoreFeatureUnavailable, - }, - { - name: "migration enabled bolt rollback", - enableBoltStateStore: true, - enableStateMigration: true, - backend: StateStoreBackendBolt, - mode: StateStoreModeRollbackToJSON, - wantErr: ErrStateStoreFeatureUnavailable, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - config := CNSConfig{ - EnableBoltStateStore: tt.enableBoltStateStore, - EnableStateMigration: tt.enableStateMigration, - StateStoreBackend: tt.backend, - StateStoreMode: tt.mode, + boolValues := []bool{false, true} + backends := []StateStoreBackend{StateStoreBackendJSON, StateStoreBackendBolt} + modes := []StateStoreMode{StateStoreModeNormal, StateStoreModeRollbackToJSON} + for _, enable := range boolValues { + for _, backend := range backends { + for _, mode := range modes { + for _, manage := range boolValues { + for _, debug := range boolValues { + for _, migrate := range boolValues { + for _, initializeFromCNI := range boolValues { + config := CNSConfig{ + EnableBoltStateStore: enable, + EnablePersistentStateDebug: debug, + EnableStateMigration: migrate, + InitializeFromCNI: initializeFromCNI, + ManageEndpointState: manage, + StateStoreBackend: backend, + StateStoreMode: mode, + } + name := fmt.Sprintf( + "enable=%t/backend=%s/mode=%s/manage=%t/debug=%t/migrate=%t/from-cni=%t", + enable, + backend, + mode, + manage, + debug, + migrate, + initializeFromCNI, + ) + t.Run(name, func(t *testing.T) { + wantErr := expectedStateStoreValidationError(config) + err := config.ValidateStateStore() + if wantErr == nil { + require.NoError(t, err) + return + } + require.ErrorIs(t, err, wantErr) + }) + } + } + } + } } + } + } +} - err := config.ValidateStateStore() - if tt.wantErr == nil { - require.NoError(t, err) - return - } - require.ErrorIs(t, err, tt.wantErr) - }) +func expectedStateStoreValidationError(config CNSConfig) error { + if !config.EnableBoltStateStore { + if config.StateStoreBackend != StateStoreBackendJSON || + config.StateStoreMode != StateStoreModeNormal || + config.EnablePersistentStateDebug { + return ErrInvalidStateStoreConfig + } + return nil + } + switch { + case config.StateStoreBackend == StateStoreBackendBolt && config.StateStoreMode == StateStoreModeNormal: + if !config.ManageEndpointState { + return ErrInvalidStateStoreConfig + } + if config.EnableStateMigration && !config.InitializeFromCNI { + return ErrInvalidStateStoreConfig + } + return nil + case config.StateStoreBackend == StateStoreBackendJSON && config.StateStoreMode == StateStoreModeRollbackToJSON: + if !config.ManageEndpointState || config.EnablePersistentStateDebug { + return ErrInvalidStateStoreConfig + } + return nil + case config.StateStoreBackend == StateStoreBackendJSON && config.StateStoreMode == StateStoreModeNormal: + if config.EnablePersistentStateDebug { + return ErrInvalidStateStoreConfig + } + return nil + default: + return ErrInvalidStateStoreConfig } } @@ -471,21 +457,20 @@ func TestCNSConfigValidateStateStoreDefaultsAndInvalidEnums(t *testing.T) { config: CNSConfig{ EnableBoltStateStore: true, }, - wantErr: ErrStateStoreFeatureUnavailable, }, { name: "bolt with default mode", config: CNSConfig{ StateStoreBackend: StateStoreBackendBolt, }, - wantErr: ErrStateStoreFeatureUnavailable, + wantErr: ErrInvalidStateStoreConfig, }, { name: "rollback with default backend", config: CNSConfig{ StateStoreMode: StateStoreModeRollbackToJSON, }, - wantErr: ErrStateStoreFeatureUnavailable, + wantErr: ErrInvalidStateStoreConfig, }, { name: "invalid backend", diff --git a/cns/configuration/testdata/good.json b/cns/configuration/testdata/good.json index 285af515237..5bc904d0c78 100644 --- a/cns/configuration/testdata/good.json +++ b/cns/configuration/testdata/good.json @@ -1,6 +1,7 @@ { "ChannelMode": "Direct", "EnableBoltStateStore": false, + "EnablePersistentStateDebug": false, "InitializeFromCNI": true, "StateStoreBackend": "json", "StateStoreMode": "normal", diff --git a/cns/restserver/cni_import_adapter_test.go b/cns/restserver/cni_import_adapter_test.go index f958cefcf51..8a586840370 100644 --- a/cns/restserver/cni_import_adapter_test.go +++ b/cns/restserver/cni_import_adapter_test.go @@ -20,6 +20,7 @@ import ( var ( errCNIImportProjection = errors.New("projection failure") errCNIImportStatus = errors.New("status failure") + errCNIQuery = errors.New("CNI query failure") ) const ( @@ -44,6 +45,7 @@ func TestCNIEndpointImportLifecycleKeepsStatefulProviderAvailable(t *testing.T) if contextErr := ctx.Err(); contextErr != nil { return nil, fmt.Errorf("reading CNI endpoint state: %w", contextErr) } + return adapterImportRecords(t), nil } records, err := provider(context.Background()) @@ -74,6 +76,80 @@ func TestCNIEndpointImportLifecycleKeepsStatefulProviderAvailable(t *testing.T) assert.Equal(t, 2, providerCalls) } +func TestDurableStateLifecycleImportsCNIBeforeSelection(t *testing.T) { + t.Run("success selects unified state and leaves provider callable", func(t *testing.T) { + db := openAdapterImportDB(t) + seedAdapterImportInventory(t, db) + service := newAdapterTestService() + providerCalls := 0 + provider := cns.CNIEndpointStateProvider(func(ctx context.Context) ([]cns.CNIEndpointState, error) { + providerCalls++ + if contextErr := ctx.Err(); contextErr != nil { + return nil, fmt.Errorf("reading CNI endpoint state: %w", contextErr) + } + return adapterImportRecords(t), nil + }) + restore, closeState, err := NewDurableStateLifecycleWithCNIImport(service, db, true, provider) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, closeState()) }) + + require.Nil(t, service.selectedUnifiedStateAdapter()) + require.NoError(t, restore(context.Background())) + require.NotNil(t, service.selectedUnifiedStateAdapter()) + assert.Equal(t, 1, providerCalls) + assert.Contains(t, service.EndpointState, "container-a") + + _, err = provider(context.Background()) + require.NoError(t, err) + assert.Equal(t, 2, providerCalls) + }) + + t.Run("provider failure blocks unified selection", func(t *testing.T) { + db := openAdapterImportDB(t) + seedAdapterImportInventory(t, db) + service := newAdapterTestService() + before, err := cloneJSON(service.EndpointState) + require.NoError(t, err) + restore, closeState, err := NewDurableStateLifecycleWithCNIImport( + service, + db, + true, + func(context.Context) ([]cns.CNIEndpointState, error) { + return nil, errCNIQuery + }, + ) + require.NoError(t, err) + err = restore(context.Background()) + require.ErrorIs(t, err, errCNIQuery) + assert.Nil(t, service.selectedUnifiedStateAdapter()) + assert.Equal(t, before, service.EndpointState) + require.NoError(t, closeState()) + }) + + t.Run("preflight failure blocks unified selection without cache projection", func(t *testing.T) { + db := openAdapterImportDB(t) + seedAdapterImportInventory(t, db) + service := newAdapterTestService() + before, err := cloneJSON(service.EndpointState) + require.NoError(t, err) + records := adapterImportRecords(t) + records[1].InterfaceKey = records[0].InterfaceKey + restore, closeState, err := NewDurableStateLifecycleWithCNIImport( + service, + db, + true, + func(context.Context) ([]cns.CNIEndpointState, error) { + return records, nil + }, + ) + require.NoError(t, err) + require.Error(t, restore(context.Background())) + assert.Nil(t, service.selectedUnifiedStateAdapter()) + assert.Equal(t, before, service.EndpointState) + require.NoError(t, closeState()) + }) +} + func TestCNIEndpointImportAdapterProjectionFailures(t *testing.T) { t.Run("prebuild failure does not commit", func(t *testing.T) { db := openAdapterImportDB(t) diff --git a/cns/restserver/durable_state_adapter.go b/cns/restserver/durable_state_adapter.go index 3f041699228..3a82cdf5bda 100644 --- a/cns/restserver/durable_state_adapter.go +++ b/cns/restserver/durable_state_adapter.go @@ -147,6 +147,17 @@ func NewDurableStateLifecycle( service *HTTPRestService, db *state.DB, projectEndpointState bool, +) (restore func(context.Context) error, closeFn func() error, err error) { + return NewDurableStateLifecycleWithCNIImport(service, db, projectEndpointState, nil) +} + +// NewDurableStateLifecycleWithCNIImport binds unified state and optionally +// imports live stateful CNI ownership before making the adapter selectable. +func NewDurableStateLifecycleWithCNIImport( + service *HTTPRestService, + db *state.DB, + projectEndpointState bool, + cniState cns.CNIEndpointStateProvider, ) (restore func(context.Context) error, closeFn func() error, err error) { adapter, err := newDurableStateAdapter(service, db, projectEndpointState) if err != nil { @@ -154,14 +165,42 @@ func NewDurableStateLifecycle( } if projectEndpointState { return func(ctx context.Context) error { - if err := adapter.restore(ctx); err != nil { - return err + if cniState == nil { + if restoreErr := adapter.restore(ctx); restoreErr != nil { + return restoreErr + } + } else { + records, queryErr := cniState(ctx) + if queryErr != nil { + return fmt.Errorf("reading CNI endpoint state: %w", queryErr) + } + if prepareErr := adapter.prepare(ctx); prepareErr != nil { + return prepareErr + } + plan, preflightErr := adapter.preflightCNIEndpointImport(ctx, records) + if preflightErr != nil { + return preflightErr + } + if importErr := adapter.importCNIEndpointState(ctx, records, plan); importErr != nil { + return importErr + } + } + if _, metricsErr := db.RefreshMetrics(ctx); metricsErr != nil { + return fmt.Errorf("refreshing persistent state metrics: %w", metricsErr) } service.setUnifiedStateAdapter(adapter) return nil }, adapter.Close, nil } - return adapter.restore, adapter.Close, nil + return func(ctx context.Context) error { + if restoreErr := adapter.restore(ctx); restoreErr != nil { + return restoreErr + } + if _, metricsErr := db.RefreshMetrics(ctx); metricsErr != nil { + return fmt.Errorf("refreshing persistent state metrics: %w", metricsErr) + } + return nil + }, adapter.Close, nil } // NewCNIEndpointImportLifecycle creates an import-only adapter for transferring @@ -377,6 +416,26 @@ func (a *durableStateAdapter) restore(ctx context.Context) error { return nil } +func (a *durableStateAdapter) prepare(ctx context.Context) error { + a.mu.Lock() + defer a.mu.Unlock() + + snapshot, err := a.store.snapshot(ctx) + if err != nil { + return err + } + projection, err := a.buildProjection(snapshot) + if err != nil { + return err + } + if err := a.verifyStatus(ctx, projection.generation); err != nil { + return err + } + a.generation = projection.generation + a.projected = true + return nil +} + func (a *durableStateAdapter) applyNetworkContainer( ctx context.Context, record state.NetworkContainerRecord, diff --git a/cns/restserver/persistent_state.go b/cns/restserver/persistent_state.go index 2906e3d1e0d..69e5de66ca8 100644 --- a/cns/restserver/persistent_state.go +++ b/cns/restserver/persistent_state.go @@ -14,9 +14,17 @@ import ( "github.com/Azure/azure-container-networking/cns/state" ) +const ( + // PersistentStateStatusPath exposes bounded persistent-state metadata and counts. + PersistentStateStatusPath = "/debug/persistent-state/status" + // PersistentStateSnapshotPath exposes the token-redacted logical state when explicitly enabled. + PersistentStateSnapshotPath = "/debug/persistent-state/snapshot" +) + var ( errNilPersistentStateStatusProvider = errors.New("persistent state status provider is nil") errNilPersistentStateSnapshotProvider = errors.New("persistent state snapshot provider is nil") + errNilPersistentStateListener = errors.New("persistent state listener is nil") ) type statusProvider func(context.Context) (state.Status, error) @@ -66,6 +74,36 @@ func NewPersistentStateSnapshotHandler( }, nil } +// RegisterPersistentStateRoutes registers the safe status route and optionally the logical snapshot route. +func (service *HTTPRestService) RegisterPersistentStateRoutes( + status func(context.Context) (state.Status, error), + snapshot func(context.Context) (state.Snapshot, error), + enableSnapshot bool, +) error { + if service == nil || service.Service == nil || service.Listener == nil { + return errNilPersistentStateListener + } + service.persistentStateRoutesOnce.Do(func() { + statusHandler, err := NewPersistentStateStatusHandler(status) + if err != nil { + service.persistentStateRoutesErr = err + return + } + mux := service.Listener.GetMux() + mux.Handle(PersistentStateStatusPath, statusHandler) + if !enableSnapshot { + return + } + snapshotHandler, err := NewPersistentStateSnapshotHandler(snapshot, true) + if err != nil { + service.persistentStateRoutesErr = err + return + } + mux.Handle(PersistentStateSnapshotPath, snapshotHandler) + }) + return service.persistentStateRoutesErr +} + func (h *PersistentStateSnapshotHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { if !allowGET(w, r) { return diff --git a/cns/restserver/persistent_state_test.go b/cns/restserver/persistent_state_test.go index d5bef86018d..22db54df428 100644 --- a/cns/restserver/persistent_state_test.go +++ b/cns/restserver/persistent_state_test.go @@ -8,11 +8,13 @@ import ( "errors" "net/http" "net/http/httptest" + "net/url" "strings" "testing" "github.com/Azure/azure-container-networking/cns" "github.com/Azure/azure-container-networking/cns/state" + acn "github.com/Azure/azure-container-networking/common" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -25,6 +27,7 @@ var ( const ( persistentStateTestNetwork = "network" persistentStateTestNestedKey = "nested" + persistentStateTestScheme = "tcp" ) func TestPersistentStateHandlerConstructors(t *testing.T) { @@ -37,6 +40,59 @@ func TestPersistentStateHandlerConstructors(t *testing.T) { assert.Nil(t, snapshotHandler) } +func TestRegisterPersistentStateRoutes(t *testing.T) { + newService := func(t *testing.T) *HTTPRestService { + t.Helper() + listener, err := acn.NewListener(&url.URL{Scheme: persistentStateTestScheme, Host: "127.0.0.1:0"}) + require.NoError(t, err) + return &HTTPRestService{ + Service: &cns.Service{Listener: listener}, + } + } + status := func(context.Context) (state.Status, error) { + return state.Status{Backend: state.BackendBolt, InvariantStatus: state.InvariantHealthy}, nil + } + snapshot := func(context.Context) (state.Snapshot, error) { + return state.NewSnapshot(), nil + } + + t.Run("safe only", func(t *testing.T) { + service := newService(t) + require.NoError(t, service.RegisterPersistentStateRoutes(status, snapshot, false)) + require.NoError(t, service.RegisterPersistentStateRoutes(status, snapshot, false)) + + response := httptest.NewRecorder() + service.Listener.GetMux().ServeHTTP( + response, + httptest.NewRequestWithContext(t.Context(), http.MethodGet, PersistentStateStatusPath, http.NoBody), + ) + assert.Equal(t, http.StatusOK, response.Code) + + response = httptest.NewRecorder() + service.Listener.GetMux().ServeHTTP( + response, + httptest.NewRequestWithContext(t.Context(), http.MethodGet, PersistentStateSnapshotPath, http.NoBody), + ) + assert.Equal(t, http.StatusNotFound, response.Code) + }) + + t.Run("debug snapshot", func(t *testing.T) { + service := newService(t) + require.NoError(t, service.RegisterPersistentStateRoutes(status, snapshot, true)) + response := httptest.NewRecorder() + service.Listener.GetMux().ServeHTTP( + response, + httptest.NewRequestWithContext(t.Context(), http.MethodGet, PersistentStateSnapshotPath, http.NoBody), + ) + assert.Equal(t, http.StatusOK, response.Code) + }) + + t.Run("listener required", func(t *testing.T) { + service := &HTTPRestService{} + require.Error(t, service.RegisterPersistentStateRoutes(status, snapshot, false)) + }) +} + func TestPersistentStateStatusHandlerContract(t *testing.T) { safeStatus := state.Status{ Backend: state.BackendBolt, diff --git a/cns/restserver/pod_info_provider.go b/cns/restserver/pod_info_provider.go new file mode 100644 index 00000000000..e6165453e1b --- /dev/null +++ b/cns/restserver/pod_info_provider.go @@ -0,0 +1,67 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package restserver + +import ( + stderrors "errors" + "fmt" + "net" + + "github.com/Azure/azure-container-networking/cns" + pkgerrors "github.com/pkg/errors" +) + +var errUnifiedEndpointProviderInactive = stderrors.New("unified endpoint state provider is not active") + +// EndpointStatePodInfoByIP projects CNS-owned infra endpoint state by IP. +func EndpointStatePodInfoByIP(state map[string]*EndpointInfo) (map[string]cns.PodInfo, error) { + podInfoByIP := make(map[string]cns.PodInfo) + for containerID, endpointInfo := range state { + for _, ipInfo := range endpointInfo.IfnameToIPMap { + if !ipInfo.NICType.IsInfraOrLegacy() { + continue + } + addIP := func(ipConfig net.IPNet) error { + ip := ipConfig.IP.String() + if _, ok := podInfoByIP[ip]; ok { + return pkgerrors.Wrap(cns.ErrDuplicateIP, ip) + } + podInfoByIP[ip] = cns.NewPodInfo( + containerID, + containerID, + endpointInfo.PodName, + endpointInfo.PodNamespace, + ) + return nil + } + for _, ipConfig := range ipInfo.IPv4 { + if err := addIP(ipConfig); err != nil { + return nil, err + } + } + for _, ipConfig := range ipInfo.IPv6 { + if err := addIP(ipConfig); err != nil { + return nil, err + } + } + } + } + return podInfoByIP, nil +} + +// UnifiedPodInfoByIPProvider reads pod information from the active unified cache. +func (service *HTTPRestService) UnifiedPodInfoByIPProvider() cns.PodInfoByIPProvider { + return cns.PodInfoByIPProviderFunc(func() (map[string]cns.PodInfo, error) { + if service.selectedUnifiedStateAdapter() == nil { + return nil, errUnifiedEndpointProviderInactive + } + service.RLock() + defer service.RUnlock() + podInfo, err := EndpointStatePodInfoByIP(service.EndpointState) + if err != nil { + return nil, fmt.Errorf("projecting unified endpoint pod information: %w", err) + } + return podInfo, nil + }) +} diff --git a/cns/restserver/pod_info_provider_test.go b/cns/restserver/pod_info_provider_test.go new file mode 100644 index 00000000000..36af2a09f29 --- /dev/null +++ b/cns/restserver/pod_info_provider_test.go @@ -0,0 +1,42 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package restserver + +import ( + "net" + "testing" + + "github.com/Azure/azure-container-networking/cns" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestUnifiedPodInfoByIPProvider(t *testing.T) { + service := newAdapterTestService() + provider := service.UnifiedPodInfoByIPProvider() + _, err := provider.PodInfoByIP() + require.Error(t, err) + + service.EndpointState = map[string]*EndpointInfo{ + cniImportContainerA: { + PodName: cniImportPodA, + PodNamespace: cniImportNamespaceA, + IfnameToIPMap: map[string]*IPInfo{ + InfraInterfaceName: { + IPv4: []net.IPNet{{IP: net.IPv4(10, 0, 0, 4), Mask: net.CIDRMask(24, 32)}}, + NICType: cns.InfraNIC, + }, + }, + }, + } + service.setUnifiedStateAdapter(&durableStateAdapter{}) + pods, err := provider.PodInfoByIP() + require.NoError(t, err) + require.Contains(t, pods, adapterTestIPv4) + assert.Equal( + t, + cns.NewPodInfo(cniImportContainerA, cniImportContainerA, cniImportPodA, cniImportNamespaceA), + pods[adapterTestIPv4], + ) +} diff --git a/cns/restserver/restserver.go b/cns/restserver/restserver.go index 8ad0804ea5e..3fad50df088 100644 --- a/cns/restserver/restserver.go +++ b/cns/restserver/restserver.go @@ -132,6 +132,8 @@ type HTTPRestService struct { stateRestoreLogger stateRestoreLogger unifiedStateAdapter *durableStateAdapter faultInjector *faultInjector + persistentStateRoutesOnce sync.Once + persistentStateRoutesErr error } type CNIConflistGenerator interface { diff --git a/cns/service/bolt_startup.go b/cns/service/bolt_startup.go index 14affef8e39..a6640e6aeb8 100644 --- a/cns/service/bolt_startup.go +++ b/cns/service/bolt_startup.go @@ -7,12 +7,15 @@ import ( "context" "errors" "fmt" + "sync" + "github.com/Azure/azure-container-networking/cns" "github.com/Azure/azure-container-networking/cns/restserver" "github.com/Azure/azure-container-networking/cns/state" "github.com/Azure/azure-container-networking/platform" "github.com/Azure/azure-container-networking/processlock" "github.com/Azure/azure-container-networking/store" + "github.com/prometheus/client_golang/prometheus" ) var ( @@ -61,8 +64,25 @@ type boltPersistentStateDependencies struct { restoreJSON func(context.Context, store.KeyValueStore, store.KeyValueStore) error } +var ( + productionPersistentStateMetricsOnce sync.Once + productionPersistentStateMetrics *state.Metrics + productionPersistentStateMetricsErr error +) + +func getProductionPersistentStateMetrics() (*state.Metrics, error) { + productionPersistentStateMetricsOnce.Do(func() { + productionPersistentStateMetrics, productionPersistentStateMetricsErr = state.NewMetrics(prometheus.DefaultRegisterer) + }) + if productionPersistentStateMetricsErr != nil { + return nil, fmt.Errorf("registering production persistent state metrics: %w", productionPersistentStateMetricsErr) + } + return productionPersistentStateMetrics, nil +} + func productionBoltPersistentStateDependencies( service *restserver.HTTPRestService, + cniState cns.CNIEndpointStateProvider, restoreJSON func(context.Context, store.KeyValueStore, store.KeyValueStore) error, ) boltPersistentStateDependencies { return boltPersistentStateDependencies{ @@ -74,7 +94,12 @@ func productionBoltPersistentStateDependencies( openDB: state.OpenContext, currentBootID: platform.BootID, attachBolt: func(db *state.DB, projectEndpointState bool) (persistentStateAttachment, error) { - restore, closeFn, err := restserver.NewDurableStateLifecycle(service, db, projectEndpointState) + restore, closeFn, err := restserver.NewDurableStateLifecycleWithCNIImport( + service, + db, + projectEndpointState, + cniState, + ) if err != nil { return persistentStateAttachment{}, fmt.Errorf("creating durable state lifecycle: %w", err) } @@ -180,14 +205,13 @@ func newBoltPersistentStateStartup( if _, bootErr := db.ApplyBoot(ctx, bootID, config.bootPolicy); bootErr != nil { return nil, closeDBAfterError(bootErr) } - if _, metricsErr := db.RefreshMetrics(ctx); metricsErr != nil { - return nil, closeDBAfterError(metricsErr) - } attachment, attachmentErr := deps.attachBolt(db, config.manageEndpointState) if attachmentErr != nil { return nil, closeDBAfterError(fmt.Errorf("attaching Bolt persistent state: %w", attachmentErr)) } startup.attachments = append(startup.attachments, attachment) + startup.status = db.Status + startup.snapshot = db.Snapshot return startup, nil } diff --git a/cns/service/bolt_startup_test.go b/cns/service/bolt_startup_test.go index f126b24799a..63f5839e538 100644 --- a/cns/service/bolt_startup_test.go +++ b/cns/service/bolt_startup_test.go @@ -40,6 +40,7 @@ type startupTestLock struct { func TestProductionBoltPersistentStateDependencies(t *testing.T) { deps := productionBoltPersistentStateDependencies( + nil, nil, func(context.Context, store.KeyValueStore, store.KeyValueStore) error { return nil }, ) @@ -52,6 +53,14 @@ func TestProductionBoltPersistentStateDependencies(t *testing.T) { assert.NotNil(t, deps.restoreJSON) } +func TestProductionPersistentStateMetricsRegisterOnce(t *testing.T) { + first, err := getProductionPersistentStateMetrics() + require.NoError(t, err) + second, err := getProductionPersistentStateMetrics() + require.NoError(t, err) + assert.Same(t, first, second) +} + func (l *startupTestLock) Lock() error { l.lockCalls++ *l.events = append(*l.events, "lock:"+l.name) @@ -145,6 +154,44 @@ func TestBoltPersistentStateStartupFirstImportAndRestart(t *testing.T) { require.NoError(t, db.Close()) } +func TestBoltPersistentStateStartupCorruptDatabaseDoesNotFallBackToValidJSON(t *testing.T) { + for _, tt := range []struct { + name string + mode boltStartupMode + }{ + {name: "normal Bolt", mode: boltStartupNormal}, + {name: "rollback", mode: boltStartupRollback}, + } { + t.Run(tt.name, func(t *testing.T) { + paths := writeStartupLegacyState(t, "valid-json-node") + require.NoError(t, os.MkdirAll(filepath.Dir(paths.databaseFile), 0o700)) + require.NoError(t, os.WriteFile(paths.databaseFile, []byte("corrupt bolt database"), 0o600)) + var events []string + deps, _, _, _ := startupTestDependencies(t, &events) + listenerCalls := 0 + + startup, err := newBoltPersistentStateStartup( + context.Background(), + boltPersistentStateConfig{ + paths: paths, + mode: tt.mode, + manageEndpointState: true, + options: state.Options{Timeout: 50 * time.Millisecond}, + }, + func(context.Context) error { + listenerCalls++ + return nil + }, + deps, + ) + require.Error(t, err) + assert.Nil(t, startup) + assert.Zero(t, listenerCalls) + assert.NotContains(t, events, "restore") + }) + } +} + func TestBoltPersistentStateStartupRollbackAndReupgrade(t *testing.T) { paths := writeStartupLegacyState(t, "node-before-rollback") var events []string diff --git a/cns/service/endpoint_state_provider.go b/cns/service/endpoint_state_provider.go index af20ceffdb8..e0142f7c105 100644 --- a/cns/service/endpoint_state_provider.go +++ b/cns/service/endpoint_state_provider.go @@ -6,6 +6,8 @@ package main import ( "errors" "fmt" + + "github.com/Azure/azure-container-networking/cns/configuration" ) var ( @@ -20,7 +22,15 @@ const ( endpointStateProviderUnified endpointStateProvider = "unified" ) -const productionEndpointStateProvider = endpointStateProviderJSON +func selectEndpointStateProvider( + backend configuration.StateStoreBackend, + mode configuration.StateStoreMode, +) endpointStateProvider { + if backend == configuration.StateStoreBackendBolt && mode == configuration.StateStoreModeNormal { + return endpointStateProviderUnified + } + return endpointStateProviderJSON +} func (provider endpointStateProvider) restoresStateFromJSON() bool { return provider == endpointStateProviderJSON diff --git a/cns/service/endpoint_state_provider_test.go b/cns/service/endpoint_state_provider_test.go index 53fec5ba432..3de236bd4d4 100644 --- a/cns/service/endpoint_state_provider_test.go +++ b/cns/service/endpoint_state_provider_test.go @@ -7,6 +7,7 @@ import ( "errors" "testing" + "github.com/Azure/azure-container-networking/cns/configuration" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -17,14 +18,34 @@ var ( ) func TestEndpointStateProviderSelection(t *testing.T) { - assert.True(t, productionEndpointStateProvider.restoresStateFromJSON()) + t.Run("configuration matrix", func(t *testing.T) { + tests := []struct { + name string + backend configuration.StateStoreBackend + mode configuration.StateStoreMode + want endpointStateProvider + wantRestoreJSON bool + }{ + {name: "default JSON", backend: configuration.StateStoreBackendJSON, mode: configuration.StateStoreModeNormal, want: endpointStateProviderJSON, wantRestoreJSON: true}, + {name: "cooling JSON", backend: configuration.StateStoreBackendJSON, mode: configuration.StateStoreModeNormal, want: endpointStateProviderJSON, wantRestoreJSON: true}, + {name: "rollback JSON", backend: configuration.StateStoreBackendJSON, mode: configuration.StateStoreModeRollbackToJSON, want: endpointStateProviderJSON, wantRestoreJSON: true}, + {name: "normal Bolt", backend: configuration.StateStoreBackendBolt, mode: configuration.StateStoreModeNormal, want: endpointStateProviderUnified}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + provider := selectEndpointStateProvider(tt.backend, tt.mode) + assert.Equal(t, tt.want, provider) + assert.Equal(t, tt.wantRestoreJSON, provider.restoresStateFromJSON()) + }) + } + }) t.Run("production JSON does not open unified state", func(t *testing.T) { jsonStartup := &persistentStateStartup{} jsonCalls := 0 unifiedCalls := 0 startup, err := newEndpointStateStartup( - productionEndpointStateProvider, + selectEndpointStateProvider(configuration.StateStoreBackendJSON, configuration.StateStoreModeNormal), func() (*persistentStateStartup, error) { jsonCalls++ return jsonStartup, nil diff --git a/cns/service/main.go b/cns/service/main.go index 19a11e8360a..255acd55499 100644 --- a/cns/service/main.go +++ b/cns/service/main.go @@ -48,6 +48,7 @@ import ( "github.com/Azure/azure-container-networking/cns/multitenantcontroller/multitenantoperator" "github.com/Azure/azure-container-networking/cns/restserver" restserverv2 "github.com/Azure/azure-container-networking/cns/restserver/v2" + "github.com/Azure/azure-container-networking/cns/state" cnipodprovider "github.com/Azure/azure-container-networking/cns/stateprovider/cni" cnspodprovider "github.com/Azure/azure-container-networking/cns/stateprovider/cns" cnstypes "github.com/Azure/azure-container-networking/cns/types" @@ -527,6 +528,7 @@ func main() { config common.ServiceConfig endpointStateStore store.KeyValueStore httpRemoteRestService *restserver.HTTPRestService + persistentState *persistentStateStartup ) config.Version = version @@ -693,7 +695,15 @@ func main() { logger.Errorf("unable to initialize a healthz handler: %v", err) return } - go healthserver.Start(z, cnsconfig.MetricsBindAddress, healthzHandler, readyChecker) + gateHealthServerOnPersistentState := cnsconfig.EnableBoltStateStore && + (cnsconfig.StateStoreBackend == configuration.StateStoreBackendBolt || + cnsconfig.StateStoreMode == configuration.StateStoreModeRollbackToJSON) + startHealthServer := func() { + go healthserver.Start(z, cnsconfig.MetricsBindAddress, healthzHandler, readyChecker) + } + if !gateHealthServerOnPersistentState { + startHealthServer() + } nmaConfig, err := nmagent.NewConfig(cnsconfig.WireserverIP) if err != nil { @@ -739,28 +749,62 @@ func main() { logger.Printf("EndpointStoreState path is %s", endpointStorePath+endpointStoreName+".json") //nolint:staticcheck // main still uses the legacy global logger } - persistentState, err := newEndpointStateStartup( - productionEndpointStateProvider, - func() (*persistentStateStartup, error) { - return newJSONPersistentStateStartup( - resolvePersistentStatePaths(storeFileLocation, endpointStorePath), - cnsconfig.ManageEndpointState, - func(context.Context) error { - return httpRemoteRestService.Start(&config) - }, - ) - }, - nil, - ) - if err != nil { - logger.Errorf("Failed to initialize persistent state: %v", err) //nolint:staticcheck // main still uses the legacy global logger - return + statePaths := resolvePersistentStatePaths(storeFileLocation, endpointStorePath) + persistentStateProvider := selectEndpointStateProvider(cnsconfig.StateStoreBackend, cnsconfig.StateStoreMode) + var persistentStateOptions state.Options + if cnsconfig.EnableBoltStateStore && + (cnsconfig.StateStoreBackend == configuration.StateStoreBackendBolt || + cnsconfig.StateStoreMode == configuration.StateStoreModeRollbackToJSON) { + persistentStateMetrics, metricsErr := getProductionPersistentStateMetrics() + if metricsErr != nil { + config.Logger.Error("failed to register persistent state metrics", zap.Error(metricsErr)) + return + } + persistentStateOptions = state.Options{ + Metrics: persistentStateMetrics, + Logger: config.Logger, + } + } + if persistentStateProvider == endpointStateProviderJSON { + persistentState, err = newEndpointStateStartup( + persistentStateProvider, + func() (*persistentStateStartup, error) { + if cnsconfig.StateStoreMode == configuration.StateStoreModeRollbackToJSON { + return newBoltPersistentStateStartup( + rootCtx, + boltPersistentStateConfig{ + paths: statePaths, + mode: boltStartupRollback, + manageEndpointState: cnsconfig.ManageEndpointState, + options: persistentStateOptions, + }, + func(context.Context) error { + return httpRemoteRestService.Start(&config) + }, + productionBoltPersistentStateDependencies( + nil, + nil, + func(context.Context, store.KeyValueStore, store.KeyValueStore) error { return nil }, + ), + ) + } + return newJSONPersistentStateStartup( + statePaths, + cnsconfig.ManageEndpointState, + func(context.Context) error { + return httpRemoteRestService.Start(&config) + }, + ) + }, + nil, + ) + if err != nil { + config.Logger.Error("failed to initialize persistent state", zap.Error(err)) + return + } + config.Store = persistentState.stateStore + endpointStateStore = persistentState.endpointStateStore } - defer func() { - _ = persistentState.Close() - }() - config.Store = persistentState.stateStore - endpointStateStore = persistentState.endpointStateStore wsProxy := wireserver.Proxy{ Host: cnsconfig.WireserverIP, @@ -780,6 +824,45 @@ func main() { logger.Errorf("Failed to create CNS object, err:%v.\n", err) return } + if persistentStateProvider == endpointStateProviderUnified { + var cniStateProvider cns.CNIEndpointStateProvider + if cnsconfig.EnableStateMigration && cnsconfig.InitializeFromCNI { + cniStateProvider = cnipodprovider.NewEndpointStateProvider() + } + persistentState, err = newEndpointStateStartup( + persistentStateProvider, + nil, + func() (*persistentStateStartup, error) { + return newBoltPersistentStateStartup( + rootCtx, + boltPersistentStateConfig{ + paths: statePaths, + mode: boltStartupNormal, + manageEndpointState: cnsconfig.ManageEndpointState, + options: persistentStateOptions, + }, + func(context.Context) error { + return httpRemoteRestService.Start(&config) + }, + productionBoltPersistentStateDependencies( + httpRemoteRestService, + cniStateProvider, + func(context.Context, store.KeyValueStore, store.KeyValueStore) error { return nil }, + ), + ) + }, + ) + if err != nil { + config.Logger.Error("failed to initialize persistent state", zap.Error(err)) + return + } + } + persistentStateCloseHandled := false + defer func() { + if closeErr := persistentState.Close(); closeErr != nil && !persistentStateCloseHandled { + config.Logger.Error("failed to close persistent state", zap.Error(closeErr)) + } + }() // Set CNS options. httpRemoteRestService.SetOption(acn.OptCnsURL, cnsURL) @@ -791,7 +874,7 @@ func main() { httpRemoteRestService.SetOption(acn.OptHttpResponseHeaderTimeout, httpResponseHeaderTimeout) httpRemoteRestService.SetOption(acn.OptProgramSNATIPTables, cnsconfig.ProgramSNATIPTables) httpRemoteRestService.SetOption(acn.OptManageEndpointState, cnsconfig.ManageEndpointState) - httpRemoteRestService.SetOption(acn.OptRestoreStateFromJSON, productionEndpointStateProvider.restoresStateFromJSON()) + httpRemoteRestService.SetOption(acn.OptRestoreStateFromJSON, persistentStateProvider.restoresStateFromJSON()) httpRemoteRestService.SetOption(acn.OptEnableStaleHNSCleanupOnNCCreate, cnsconfig.EnableStaleHNSCleanupOnNCCreate) // Create default ext network if commandline option is set @@ -827,6 +910,24 @@ func main() { return } } + if persistentStateProvider == endpointStateProviderUnified { + if routeErr := httpRemoteRestService.RegisterPersistentStateRoutes( + persistentState.status, + persistentState.snapshot, + cnsconfig.EnablePersistentStateDebug, + ); routeErr != nil { + config.Logger.Error("failed to register persistent state routes", zap.Error(routeErr)) + return + } + } + if restoreErr := persistentState.Restore(rootCtx); restoreErr != nil { + persistentStateCloseHandled = true + config.Logger.Error("failed to restore persistent state", zap.Error(restoreErr)) + return + } + if gateHealthServerOnPersistentState { + startHealthServer() + } // Setting the remote ARP MAC address to 12-34-56-78-9a-bc on windows for external traffic if HNS is enabled err = platform.SetSdnRemoteArpMacAddress(rootCtx) @@ -1025,6 +1126,7 @@ func main() { err = persistentState.Start(rootCtx) if err != nil { + persistentStateCloseHandled = true logger.Errorf("Failed to start CNS, err:%v.\n", err) return } @@ -1428,7 +1530,9 @@ func InitializeCRDState(ctx context.Context, z *zap.Logger, httpRestService cns. } // perform state migration from CNI in case CNS is set to manage the endpoint state and has emty state - if cnsconfig.EnableStateMigration && !httpRestServiceImplementation.EndpointStateStore.Exists() { + if cnsconfig.EnableStateMigration && + cnsconfig.StateStoreBackend != configuration.StateStoreBackendBolt && + !httpRestServiceImplementation.EndpointStateStore.Exists() { if err = PopulateCNSEndpointState(httpRestServiceImplementation.EndpointStateStore); err != nil { return errors.Wrap(err, "failed to create CNS EndpointState From CNI") } @@ -1678,6 +1782,9 @@ func getPodInfoByIPProvider( nodeName string, ) (podInfoByIPProvider cns.PodInfoByIPProvider, err error) { switch { + case cnsconfig.StateStoreBackend == configuration.StateStoreBackendBolt: + logger.Printf("Initializing from unified endpoint state") //nolint:staticcheck // This function still uses the legacy package logger. + podInfoByIPProvider = httpRestServiceImplementation.UnifiedPodInfoByIPProvider() case cnsconfig.ManageEndpointState: logger.Printf("Initializing from self managed endpoint store") podInfoByIPProvider, err = cnspodprovider.New(httpRestServiceImplementation.EndpointStateStore) // get reference to endpoint state store from rest server diff --git a/cns/service/persistent_state.go b/cns/service/persistent_state.go index 39b6a4d99f2..7f4c22332ad 100644 --- a/cns/service/persistent_state.go +++ b/cns/service/persistent_state.go @@ -9,6 +9,7 @@ import ( "fmt" "sync" + "github.com/Azure/azure-container-networking/cns/state" "github.com/Azure/azure-container-networking/platform" "github.com/Azure/azure-container-networking/processlock" "github.com/Azure/azure-container-networking/store" @@ -48,6 +49,10 @@ type persistentStateStartup struct { start func(context.Context) error attachments []persistentStateAttachment locks []processlock.Interface + status func(context.Context) (state.Status, error) + snapshot func(context.Context) (state.Snapshot, error) + restoreOnce sync.Once + restoreErr error closeOnce sync.Once closeErr error } @@ -111,11 +116,21 @@ func newJSONPersistentStateStartup( }) } -func (s *persistentStateStartup) Start(ctx context.Context) error { - for _, attachment := range s.attachments { - if err := attachment.restore(ctx); err != nil { - return errors.Join(err, s.Close()) +func (s *persistentStateStartup) Restore(ctx context.Context) error { + s.restoreOnce.Do(func() { + for _, attachment := range s.attachments { + if err := attachment.restore(ctx); err != nil { + s.restoreErr = errors.Join(err, s.Close()) + return + } } + }) + return s.restoreErr +} + +func (s *persistentStateStartup) Start(ctx context.Context) error { + if err := s.Restore(ctx); err != nil { + return err } if err := s.start(ctx); err != nil { return errors.Join(err, s.Close()) diff --git a/cns/service/persistent_state_test.go b/cns/service/persistent_state_test.go index 315bd7dc03d..009fa7912e1 100644 --- a/cns/service/persistent_state_test.go +++ b/cns/service/persistent_state_test.go @@ -14,6 +14,7 @@ import ( "github.com/Azure/azure-container-networking/processlock" "github.com/Azure/azure-container-networking/store" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -291,6 +292,44 @@ func TestPersistentStateStartup_AttachmentClosesOnceOnShutdown(t *testing.T) { require.Equal(t, 1, lock.unlockCalls) } +func TestPersistentStateStartup_RestoreRunsExactlyOnceBeforeStart(t *testing.T) { + lock := &trackedFileLock{} + restoreCalls := 0 + listenerCalls := 0 + startup, err := newPersistentStateStartup( + testPersistentStatePaths(), + false, + func(context.Context) error { + listenerCalls++ + return nil + }, + persistentStateDependencies{ + createDirectory: func(string) error { return nil }, + newFileLock: func(string) (processlock.Interface, error) { + return lock, nil + }, + openStore: func(path string, _ processlock.Interface) (store.KeyValueStore, error) { + return store.NewMockStore(path), nil + }, + }, + ) + require.NoError(t, err) + require.NoError(t, startup.attach( + func(context.Context) error { + restoreCalls++ + return nil + }, + func() error { return nil }, + )) + + require.NoError(t, startup.Restore(context.Background())) + require.NoError(t, startup.Restore(context.Background())) + require.NoError(t, startup.Start(context.Background())) + assert.Equal(t, 1, restoreCalls) + assert.Equal(t, 1, listenerCalls) + require.NoError(t, startup.Close()) +} + func TestPersistentStateStartup_AttachmentValidation(t *testing.T) { startup := &persistentStateStartup{} require.Error(t, startup.attach(nil, func() error { return nil })) diff --git a/cns/stateprovider/cns/podinfoprovider.go b/cns/stateprovider/cns/podinfoprovider.go index 16c5d7f5f0e..c762e9fe3d3 100644 --- a/cns/stateprovider/cns/podinfoprovider.go +++ b/cns/stateprovider/cns/podinfoprovider.go @@ -27,50 +27,22 @@ func podInfoProvider(endpointStore store.KeyValueStore) (cns.PodInfoByIPProvider if errors.Is(err, store.ErrKeyNotFound) { // Nothing to restore. return cns.PodInfoByIPProviderFunc(func() (map[string]cns.PodInfo, error) { - return endpointStateToPodInfoByIP(state) + return restserver.EndpointStatePodInfoByIP(state) }), err } return nil, fmt.Errorf("failed to read endpoints state from store : %w", err) } return cns.PodInfoByIPProviderFunc(func() (map[string]cns.PodInfo, error) { - return endpointStateToPodInfoByIP(state) + return restserver.EndpointStatePodInfoByIP(state) }), nil } func endpointStateToPodInfoByIP(state map[string]*restserver.EndpointInfo) (map[string]cns.PodInfo, error) { - podInfoByIP := map[string]cns.PodInfo{} - for containerID, endpointInfo := range state { // for each endpoint - for _, ipinfo := range endpointInfo.IfnameToIPMap { // for each IP info object of the endpoint's interfaces - // Only InfraNIC IPs participate in NNC reconcile; delegated NIC IPs - // (FrontendNIC/BackendNIC) are owned by per-pod MTPNC. - if !ipinfo.NICType.IsInfraOrLegacy() { - continue - } - for _, ipv4conf := range ipinfo.IPv4 { // for each IPv4 config of the endpoint's interfaces - if _, ok := podInfoByIP[ipv4conf.IP.String()]; ok { - return nil, errors.Wrap(cns.ErrDuplicateIP, ipv4conf.IP.String()) - } - podInfoByIP[ipv4conf.IP.String()] = cns.NewPodInfo( - containerID, - containerID, - endpointInfo.PodName, - endpointInfo.PodNamespace, - ) - } - for _, ipv6conf := range ipinfo.IPv6 { // for each IPv6 config of the endpoint's interfaces - if _, ok := podInfoByIP[ipv6conf.IP.String()]; ok { - return nil, errors.Wrap(cns.ErrDuplicateIP, ipv6conf.IP.String()) - } - podInfoByIP[ipv6conf.IP.String()] = cns.NewPodInfo( - containerID, - containerID, - endpointInfo.PodName, - endpointInfo.PodNamespace, - ) - } - } + podInfo, err := restserver.EndpointStatePodInfoByIP(state) + if err != nil { + return nil, fmt.Errorf("projecting endpoint state pod information: %w", err) } - return podInfoByIP, nil + return podInfo, nil } // MigrateCNISate returns an endpoint state of CNS by reading the CNI state file diff --git a/docs/persistent-state-observability.md b/docs/persistent-state-observability.md index 69fe9c75a28..697584a1edc 100644 --- a/docs/persistent-state-observability.md +++ b/docs/persistent-state-observability.md @@ -1,24 +1,74 @@ # CNS persistent state observability -The Bolt persistent state engine provides low-cardinality metrics, safe status, and -structured lifecycle events before it becomes a supported runtime mode. R13 does -not activate the backend, register HTTP routes, or add runtime configuration. +The Bolt persistent state engine is an opt-in CNS-owned endpoint-state backend. +This release ships dark: every checked-in configuration keeps +`EnableBoltStateStore=false`, `StateStoreBackend=json`, +`StateStoreMode=normal`, and `EnablePersistentStateDebug=false`. ```mermaid -flowchart LR - Caller[Future R14/R15 runtime boundary] --> DB[Persistent state DB] - DB --> Tx[View and Update metrics] - DB --> Life[Startup, import, rollback, and boot metrics] - DB --> Status[Safe status read] - Status --> Gauges[Metadata and count gauges] - Status --> SafeHandler[Unregistered safe-status handler] - DB --> Snapshot[Logical snapshot] - Snapshot --> Gate{Explicit debug gate} - Gate -->|false by default| NotFound[404] - Gate -->|true| DebugHandler[Unregistered debug handler] - DB --> Logs[Stable lifecycle success/no-op logs] +flowchart TD + Config[Load, default, validate config] --> Choice{Validated backend and mode} + Choice -->|default or cooling| JSON[Open legacy JSON stores] + Choice -->|Bolt normal| Bolt[Open and verify Bolt] + Choice -->|explicit rollback| Export[Open Bolt, export and verify JSON] + Bolt --> Restore[Restore cache projection] + Restore --> Import{CNI ownership import requested?} + Import -->|no| Unified[Activate unified provider] + Import -->|yes| Preflight[Query stateful CNI, preflight, import and verify] + Preflight --> Unified + Export --> JSON + JSON --> Listener[Start CNS listener] + Unified --> Listener ``` +## Supported configuration matrix + +`ManageEndpointState=true` is required whenever the Bolt master flag is enabled. +Bolt is not supported while CNI owns endpoint state. + +| Purpose | Master flag | Backend | Mode | CNS owns endpoint state | Result | +| --- | --- | --- | --- | --- | --- | +| Shipped default | false | json | normal | either | JSON | +| CNS-owned Bolt | true | bolt | normal | true | Bolt | +| Explicit rollback | true | json | rollback-to-json | true | Export Bolt to JSON, then JSON | +| Post-rollback cooling | true | json | normal | true | JSON | +| Master disabled after cooling | false | json | normal | either | JSON | + +All other combinations fail validation, including Bolt with the master flag +disabled, rollback with the master flag disabled, Bolt plus rollback mode, any +enabled master flag with `ManageEndpointState=false`, invalid enum values, debug +outside normal Bolt, and all fault-hook requests. `EnableStateMigration` retains +its existing CNI-to-CNS ownership meaning; it is not a backend selector. + +Configuration is validated before any database or legacy state file is opened. +Normal Bolt startup errors, including lock, schema, authority, import, boot, and +restore failures, stop startup before the CNS listener starts. CNS never falls +back automatically to JSON. After a completed import, Bolt is authoritative and +legacy JSON files are ignored even if they are corrupt. + +## CNI-to-CNS ownership handoff + +Set `EnableStateMigration=true` and `InitializeFromCNI=true` together only for +the import restart while the stateful CNI remains installed and callable. CNS +queries CNI, validates record counts and identities, imports and verifies all +records, restores its cache, and only then activates the unified provider. +Query, preflight, import, or restore failure blocks startup without a partial +cache or fallback. Installing stateless CNI is a separate operator action after +that successful restart; CNS does not install or switch CNI. + +## Rollback sequence + +1. With CNS still owning endpoint state, set the backend to `json` and mode to + `rollback-to-json`, leaving the master flag enabled. +2. Restart CNS. It opens authoritative Bolt state, exports and verifies current + JSON stores, then runs on JSON. +3. Set mode to `normal` while leaving the master flag enabled and restart for + the cooling state. +4. Disable the master flag only after the cooling restart. + +Do not disable the master flag or select JSON normal before completing rollback. +A later re-upgrade imports the current JSON state rather than stale Bolt state. + ## Signals All metric names start with `cns_persistent_state_`. @@ -128,17 +178,24 @@ storage size, bounded invariant state, and aggregate record counts. It never contains the database path, boot value, node identity, pod identity, IP address, endpoint payload, token, or raw Bolt page. -When registered in a future release, a valid GET returns 200 even when +In normal Bolt mode the safe status route is registered on the existing local +CNS transport. A valid GET returns 200 even when `invariantStatus` is `failed`; the bounded failure is the status representation, not a transport failure. Provider failures return 503, canceled requests return 408, method mismatches return 405, and GET requests with bodies return 400. The full logical snapshot contains pod, IP, and endpoint data. Its handler -requires an explicit behavioral `enabled` boolean and returns 404 while disabled. -Authorization tokens are removed before transport. The constructor is available -for future composition, but **no route is registered and the endpoint is not -enabled in R13**. Future callers must default the gate to false and protect the -route as sensitive debug access. +is registered only for normal Bolt when `EnablePersistentStateDebug=true`. +Authorization tokens are removed before transport. This route exposes sensitive +logical state and is not an authentication mechanism; keep it disabled except +during explicitly controlled local diagnosis. + +The test-only fault endpoint is absent unless +`CNS_TEST_FAULT_INJECTION_TOKEN` contains a non-empty, high-entropy token at +process startup. Every request must present that token in +`X-CNS-Test-Fault-Token`. The endpoint remains on the existing local CNS +transport; tokens are never logged, persisted, returned by status, or used as +metric labels. ## Lifecycle logs and error ownership @@ -151,9 +208,9 @@ Successful and no-op lifecycle outcomes use stable messages: Fields are typed and bounded: backend, authority, schema, generation, operation, result, duration, and aggregate record counts. Transaction hot paths do not log. -The state package returns failures without logging them. The R14 startup adapter -and R15 request/runtime handling boundaries own the single error log after they -decide retry, rollback, or process-failure behavior. +The state package returns failures without logging them. The CNS startup and +request/runtime boundaries own the single error log after deciding whether to +fail the process or return the request error. ## Migration and rollback diagnosis @@ -164,8 +221,8 @@ decide retry, rollback, or process-failure behavior. bounded invariant state. 5. Treat schema, authority, or structural invariant failures as unsafe state; do not inspect or publish raw records to diagnose them. -6. At the R14/R15 owner boundary, retain the returned error once and apply the - documented retry or rollback policy. +6. At the owner boundary, retain the returned error once and apply only the + explicit rollback policy; never fall back automatically. ## Soak checklist diff --git a/test/e2e/manifests/cilium/v1.14/cns/configmap.yaml b/test/e2e/manifests/cilium/v1.14/cns/configmap.yaml index dea8ba7a7f7..ae85ec60a31 100644 --- a/test/e2e/manifests/cilium/v1.14/cns/configmap.yaml +++ b/test/e2e/manifests/cilium/v1.14/cns/configmap.yaml @@ -23,6 +23,7 @@ data: }, "ChannelMode": "CRD", "EnableBoltStateStore": false, + "EnablePersistentStateDebug": false, "InitializeFromCNI": false, "StateStoreBackend": "json", "StateStoreMode": "normal", diff --git a/test/integration/manifests/cilium/cns-write-ovly.yaml b/test/integration/manifests/cilium/cns-write-ovly.yaml index 766f418e794..c5450958b76 100644 --- a/test/integration/manifests/cilium/cns-write-ovly.yaml +++ b/test/integration/manifests/cilium/cns-write-ovly.yaml @@ -215,6 +215,7 @@ data: }, "ChannelMode": "CRD", "EnableBoltStateStore": false, + "EnablePersistentStateDebug": false, "InitializeFromCNI": false, "StateStoreBackend": "json", "StateStoreMode": "normal", diff --git a/test/integration/manifests/cnsconfig/azcnichainedciliumconfigmap.yaml b/test/integration/manifests/cnsconfig/azcnichainedciliumconfigmap.yaml index 2c059bc7223..bf8ad19bd42 100644 --- a/test/integration/manifests/cnsconfig/azcnichainedciliumconfigmap.yaml +++ b/test/integration/manifests/cnsconfig/azcnichainedciliumconfigmap.yaml @@ -15,6 +15,7 @@ data: "EnableK8sDevicePlugin": true, "EnableLoggerV2": true, "EnableBoltStateStore": false, + "EnablePersistentStateDebug": false, "EnableStateMigration": true, "EnableSubnetScarcity": false, "InitializeFromCNI": false, diff --git a/test/integration/manifests/cnsconfig/azurecnidualstackoverlaylinuxconfigmap.yaml b/test/integration/manifests/cnsconfig/azurecnidualstackoverlaylinuxconfigmap.yaml index a9b5b8b6e78..8be5f1db1ba 100644 --- a/test/integration/manifests/cnsconfig/azurecnidualstackoverlaylinuxconfigmap.yaml +++ b/test/integration/manifests/cnsconfig/azurecnidualstackoverlaylinuxconfigmap.yaml @@ -29,6 +29,7 @@ data: "EnableCNIConflistGeneration": true, "EnableIPAMv2": true, "EnableBoltStateStore": false, + "EnablePersistentStateDebug": false, "EnableStateMigration": false, "EnableSubnetScarcity": false, "InitializeFromCNI": true, diff --git a/test/integration/manifests/cnsconfig/azurecnidualstackoverlaywindowsconfigmap.yaml b/test/integration/manifests/cnsconfig/azurecnidualstackoverlaywindowsconfigmap.yaml index efb54e63b1b..5d635a62f54 100644 --- a/test/integration/manifests/cnsconfig/azurecnidualstackoverlaywindowsconfigmap.yaml +++ b/test/integration/manifests/cnsconfig/azurecnidualstackoverlaywindowsconfigmap.yaml @@ -29,6 +29,7 @@ data: "EnableCNIConflistGeneration": false, "EnableIPAMv2": true, "EnableBoltStateStore": false, + "EnablePersistentStateDebug": false, "EnableStateMigration": false, "EnableSubnetScarcity": false, "InitializeFromCNI": true, diff --git a/test/integration/manifests/cnsconfig/azurecnioverlaylinuxconfigmap.yaml b/test/integration/manifests/cnsconfig/azurecnioverlaylinuxconfigmap.yaml index 87268938615..a1271bc2389 100644 --- a/test/integration/manifests/cnsconfig/azurecnioverlaylinuxconfigmap.yaml +++ b/test/integration/manifests/cnsconfig/azurecnioverlaylinuxconfigmap.yaml @@ -29,6 +29,7 @@ data: "EnableCNIConflistGeneration": true, "EnableIPAMv2": true, "EnableBoltStateStore": false, + "EnablePersistentStateDebug": false, "EnableStateMigration": false, "EnableSubnetScarcity": false, "InitializeFromCNI": true, diff --git a/test/integration/manifests/cnsconfig/azurecnioverlaywindowsconfigmap.yaml b/test/integration/manifests/cnsconfig/azurecnioverlaywindowsconfigmap.yaml index 0a6fea58ed2..0499d9b1e1c 100644 --- a/test/integration/manifests/cnsconfig/azurecnioverlaywindowsconfigmap.yaml +++ b/test/integration/manifests/cnsconfig/azurecnioverlaywindowsconfigmap.yaml @@ -29,6 +29,7 @@ data: "EnableCNIConflistGeneration": false, "EnableIPAMv2": true, "EnableBoltStateStore": false, + "EnablePersistentStateDebug": false, "EnableStateMigration": false, "EnableSubnetScarcity": false, "InitializeFromCNI": true, diff --git a/test/integration/manifests/cnsconfig/azurestatelesscnioverlaywindowsconfigmap.yaml b/test/integration/manifests/cnsconfig/azurestatelesscnioverlaywindowsconfigmap.yaml index 62e3a08fafe..872fa7d174b 100644 --- a/test/integration/manifests/cnsconfig/azurestatelesscnioverlaywindowsconfigmap.yaml +++ b/test/integration/manifests/cnsconfig/azurestatelesscnioverlaywindowsconfigmap.yaml @@ -29,6 +29,7 @@ data: "EnableCNIConflistGeneration": false, "EnableIPAMv2": true, "EnableBoltStateStore": false, + "EnablePersistentStateDebug": false, "EnableStateMigration": false, "EnableSubnetScarcity": false, "InitializeFromCNI": false, diff --git a/test/integration/manifests/cnsconfig/ciliumconfigmap.yaml b/test/integration/manifests/cnsconfig/ciliumconfigmap.yaml index d3231b82b0a..0491a67b559 100644 --- a/test/integration/manifests/cnsconfig/ciliumconfigmap.yaml +++ b/test/integration/manifests/cnsconfig/ciliumconfigmap.yaml @@ -29,6 +29,7 @@ data: "EnableCNIConflistGeneration": true, "EnableIPAMv2": true, "EnableBoltStateStore": false, + "EnablePersistentStateDebug": false, "EnableStateMigration": false, "EnableSubnetScarcity": false, "InitializeFromCNI": false, diff --git a/test/integration/manifests/cnsconfig/ciliumnodesubnetconfigmap.yaml b/test/integration/manifests/cnsconfig/ciliumnodesubnetconfigmap.yaml index d098a728bf6..2aeaee0cab8 100644 --- a/test/integration/manifests/cnsconfig/ciliumnodesubnetconfigmap.yaml +++ b/test/integration/manifests/cnsconfig/ciliumnodesubnetconfigmap.yaml @@ -29,6 +29,7 @@ data: "EnableCNIConflistGeneration": true, "EnableIPAMv2": true, "EnableBoltStateStore": false, + "EnablePersistentStateDebug": false, "EnableStateMigration": false, "EnableSubnetScarcity": false, "InitializeFromCNI": false, diff --git a/test/integration/manifests/cnsconfig/overlayconfigmap.yaml b/test/integration/manifests/cnsconfig/overlayconfigmap.yaml index 2fc7063c937..6b664959e8e 100644 --- a/test/integration/manifests/cnsconfig/overlayconfigmap.yaml +++ b/test/integration/manifests/cnsconfig/overlayconfigmap.yaml @@ -29,6 +29,7 @@ data: "EnableCNIConflistGeneration": true, "EnableIPAMv2": true, "EnableBoltStateStore": false, + "EnablePersistentStateDebug": false, "EnableStateMigration": false, "EnableSubnetScarcity": false, "InitializeFromCNI": false, diff --git a/test/integration/manifests/cnsconfig/swiftlinuxconfigmap.yaml b/test/integration/manifests/cnsconfig/swiftlinuxconfigmap.yaml index b31def5c51c..066753624ac 100644 --- a/test/integration/manifests/cnsconfig/swiftlinuxconfigmap.yaml +++ b/test/integration/manifests/cnsconfig/swiftlinuxconfigmap.yaml @@ -26,6 +26,7 @@ data: "EnableAsyncPodDelete": true, "EnableIPAMv2": true, "EnableBoltStateStore": false, + "EnablePersistentStateDebug": false, "EnableStateMigration": false, "EnableSubnetScarcity": false, "InitializeFromCNI": true, diff --git a/test/integration/manifests/cnsconfig/swiftwindowsconfigmap.yaml b/test/integration/manifests/cnsconfig/swiftwindowsconfigmap.yaml index eaf77b7b769..58201f9eaf4 100644 --- a/test/integration/manifests/cnsconfig/swiftwindowsconfigmap.yaml +++ b/test/integration/manifests/cnsconfig/swiftwindowsconfigmap.yaml @@ -26,6 +26,7 @@ data: "EnableAsyncPodDelete": true, "EnableIPAMv2": true, "EnableBoltStateStore": false, + "EnablePersistentStateDebug": false, "EnableStateMigration": false, "EnableSubnetScarcity": false, "InitializeFromCNI": true,