diff --git a/cns/restserver/durable_state_adapter.go b/cns/restserver/durable_state_adapter.go index 6b2f0d8f5d..b23e7bafdb 100644 --- a/cns/restserver/durable_state_adapter.go +++ b/cns/restserver/durable_state_adapter.go @@ -61,6 +61,26 @@ type durableStateOperations struct { time.Duration, func(state.Snapshot) error, ) (bool, error) + releaseEndpoint func( + context.Context, + uint64, + state.PodIdentity, + time.Time, + func(state.Snapshot) error, + ) (bool, error) + deleteEndpoint func( + context.Context, + uint64, + string, + func(state.Snapshot) error, + ) (bool, error) + pruneDeleteIntents func( + context.Context, + uint64, + time.Time, + time.Duration, + func(state.Snapshot) error, + ) (int, error) refreshMetrics func(context.Context) (state.Status, error) status func(context.Context) (state.Status, error) close func() error @@ -72,15 +92,16 @@ type durableStateAdapter struct { // mu is acquired before the HTTPRestService lock. Callers must not hold the // service lock; the adapter applies complete projections under that lock. - mu sync.Mutex - projectEndpointState bool - buildProjection func(state.Snapshot) (durableCacheProjection, error) - applyAddProjection func(durableCacheProjection) error - now func() time.Time - projected bool - generation uint64 - closeOnce sync.Once - closeErr error + mu sync.Mutex + projectEndpointState bool + buildProjection func(state.Snapshot) (durableCacheProjection, error) + applyAddProjection func(durableCacheProjection) error + applyDeleteProjection func(durableCacheProjection) error + now func() time.Time + projected bool + generation uint64 + closeOnce sync.Once + closeErr error } type durableServiceMetadata struct { @@ -137,10 +158,13 @@ func newDurableStateAdapter( return nil, errNilDurableStateDB } return newDurableStateAdapterWithOperations(service, durableStateOperations{ - snapshot: db.Snapshot, - replace: db.ReplaceDurableState, - assignEndpoint: db.AssignEndpointIfGeneration, - refreshMetrics: db.RefreshMetrics, + snapshot: db.Snapshot, + replace: db.ReplaceDurableState, + assignEndpoint: db.AssignEndpointIfGeneration, + releaseEndpoint: db.ReleaseEndpointIfGeneration, + deleteEndpoint: db.DeleteEndpointRecordIfGeneration, + pruneDeleteIntents: db.PruneDeleteIntentsIfGeneration, + refreshMetrics: db.RefreshMetrics, updateMetadata: func(ctx context.Context, expectedGeneration uint64, metadata state.Metadata) (bool, error) { err := db.Update(ctx, func(tx *state.WriteTx) error { current, err := tx.Metadata() diff --git a/cns/restserver/ipam.go b/cns/restserver/ipam.go index db5210ded3..9ac91660f2 100644 --- a/cns/restserver/ipam.go +++ b/cns/restserver/ipam.go @@ -454,8 +454,18 @@ func (service *HTTPRestService) ReleaseIPConfigHandlerHelper(ctx context.Context }, }, fmt.Errorf("failed to validate ip config request") //nolint:goerr113 // return error } - // Check if http rest service managed endpoint state is set - if service.Options[common.OptManageEndpointState] == true { + unifiedAdapter := service.selectedUnifiedStateAdapter() + if unifiedAdapter != nil { + if err := unifiedAdapter.releaseIPConfigs(ctx, ipconfigsRequest, podInfo); err != nil { + resp := &cns.IPConfigsResponse{ + Response: cns.Response{ + ReturnCode: unifiedReleaseResponseCode(err), + Message: err.Error(), + }, + } + return resp, fmt.Errorf("releasing unified IP configs: %w", err) + } + } else if service.Options[common.OptManageEndpointState] == true { if err := service.releaseIPConfigsWithDeleteIntent(podInfo); err != nil { resp := &cns.IPConfigsResponse{ Response: cns.Response{ @@ -1221,6 +1231,12 @@ func validateDesiredIPAddresses(desiredIPs []string) error { func (service *HTTPRestService) EndpointHandlerAPI(w http.ResponseWriter, r *http.Request) { opName := "endpointHandler" logger.Printf("[EndpointHandlerAPI] EndpointHandlerAPI received request with http Method %s", r.Method) + if r.Method == http.MethodDelete { + if adapter := service.selectedUnifiedStateAdapter(); adapter != nil { + service.deleteEndpointStateHandler(w, r, adapter) + return + } + } service.Lock() defer service.Unlock() // Check if CNS is managing the CNI statefile @@ -1239,7 +1255,7 @@ func (service *HTTPRestService) EndpointHandlerAPI(w http.ResponseWriter, r *htt case http.MethodPatch: service.UpdateEndpointHandler(w, r) case http.MethodDelete: - service.DeleteEndpointStateHandler(w, r) + service.deleteEndpointStateHandler(w, r, nil) default: //nolint logger.Errorf("[EndpointHandlerAPI] EndpointHandler API expect http Get or Patch or Delete method") @@ -1247,11 +1263,19 @@ func (service *HTTPRestService) EndpointHandlerAPI(w http.ResponseWriter, r *htt } func (service *HTTPRestService) DeleteEndpointStateHandler(w http.ResponseWriter, r *http.Request) { + service.deleteEndpointStateHandler(w, r, service.selectedUnifiedStateAdapter()) +} + +func (service *HTTPRestService) deleteEndpointStateHandler( + w http.ResponseWriter, + r *http.Request, + adapter *durableStateAdapter, +) { opName := "DeleteEndpointStateHandler" logger.Printf("[DeleteEndpointStateHandler] DeleteEndpointState for %s", r.URL.Path) //nolint:staticcheck // reason: using deprecated call until migration to new API endpointID := strings.TrimPrefix(r.URL.Path, cns.EndpointPath) - if service.EndpointStateStore == nil { + if service.EndpointStateStore == nil && adapter == nil { response := cns.Response{ ReturnCode: types.NilEndpointStateStore, Message: "[DeleteEndpointStateHandler] EndpointStateStore is not initialized", @@ -1262,7 +1286,7 @@ func (service *HTTPRestService) DeleteEndpointStateHandler(w http.ResponseWriter } // Delete the endpoint from state - err := service.DeleteEndpointStateHelper(endpointID) + err := service.deleteEndpointState(r.Context(), endpointID, adapter) if err != nil { response := cns.Response{ ReturnCode: types.UnexpectedError, @@ -1310,6 +1334,17 @@ func (service *HTTPRestService) DeleteEndpointStateHelper(endpointID string) err return nil } +func (service *HTTPRestService) deleteEndpointState( + ctx context.Context, + endpointID string, + adapter *durableStateAdapter, +) error { + if adapter != nil { + return adapter.deleteEndpointRecord(ctx, endpointID) + } + return service.DeleteEndpointStateHelper(endpointID) +} + // GetEndpointHandler handles the incoming GetEndpoint requests with http Get method func (service *HTTPRestService) GetEndpointHandler(w http.ResponseWriter, r *http.Request) { opName := "getEndpointState" diff --git a/cns/restserver/unified_add.go b/cns/restserver/unified_add.go index ede6ab61ec..4f6722ae37 100644 --- a/cns/restserver/unified_add.go +++ b/cns/restserver/unified_add.go @@ -79,7 +79,8 @@ func (a *durableStateAdapter) requestIPConfigs( if err != nil { return nil, err } - plan, err := a.service.requestIPConfigsUnifiedLocked(ctx, request, podInfo, snapshot) + now := a.now() + plan, err := a.service.requestIPConfigsUnifiedLocked(ctx, request, podInfo, snapshot, now) if err != nil { return nil, err } @@ -93,7 +94,7 @@ func (a *durableStateAdapter) requestIPConfigs( a.generation, plan.assignment, plan.endpoint, - a.now(), + now, unifiedDeleteIntentTTL, func(candidate state.Snapshot) error { var buildErr error @@ -158,6 +159,7 @@ func (service *HTTPRestService) requestIPConfigsUnifiedLocked( request cns.IPConfigsRequest, podInfo cns.PodInfo, snapshot state.Snapshot, + now time.Time, ) (unifiedAddPlan, error) { if err := ctx.Err(); err != nil { return unifiedAddPlan{}, fmt.Errorf("planning unified endpoint assignment: %w", err) @@ -169,6 +171,14 @@ func (service *HTTPRestService) requestIPConfigsUnifiedLocked( PodName: podInfo.Name(), PodNamespace: podInfo.Namespace(), } + if intent, ok := snapshot.DeleteIntents[requestedPod.InfraContainerID]; ok && + now.Before(intent.CreatedAt.Add(unifiedDeleteIntentTTL)) { + return unifiedAddPlan{}, fmt.Errorf( + "%w: infra container %q", + state.ErrDeleteIntent, + requestedPod.InfraContainerID, + ) + } if existing, ok := snapshot.Assignments[requestedPod.PodKey]; ok { if existing.Pod != requestedPod { return unifiedAddPlan{}, fmt.Errorf( diff --git a/cns/restserver/unified_del.go b/cns/restserver/unified_del.go new file mode 100644 index 0000000000..b80e265127 --- /dev/null +++ b/cns/restserver/unified_del.go @@ -0,0 +1,380 @@ +// Copyright 2026 Microsoft. All rights reserved. +// MIT License + +package restserver + +import ( + "context" + "errors" + "fmt" + "net/netip" + "sort" + "strings" + "time" + + "github.com/Azure/azure-container-networking/cns" + "github.com/Azure/azure-container-networking/cns/state" + "github.com/Azure/azure-container-networking/cns/types" +) + +type unifiedReleasePlan struct { + pod state.PodIdentity + stale bool +} + +type unifiedDeleteCommittedError struct { + err error +} + +var ( + errUnifiedReleaseOperationNil = errors.New("unified DEL: endpoint release operation is nil") + errUnifiedDeleteOperationNil = errors.New("unified DEL: endpoint deletion operation is nil") + errUnifiedPruneOperationNil = errors.New("unified DEL: delete intent prune operation is nil") +) + +func (e *unifiedDeleteCommittedError) Error() string { + return fmt.Sprintf("unified DEL committed but cache projection failed: %v", e.err) +} + +func (e *unifiedDeleteCommittedError) Unwrap() error { + return e.err +} + +func (a *durableStateAdapter) releaseIPConfigs( + ctx context.Context, + request cns.IPConfigsRequest, + podInfo cns.PodInfo, +) error { + if a.store.releaseEndpoint == nil { + return errUnifiedReleaseOperationNil + } + a.mu.Lock() + defer a.mu.Unlock() + a.service.Lock() + defer a.service.Unlock() + + snapshot, err := a.currentSnapshot(ctx) + if err != nil { + return err + } + plan, err := a.service.releaseIPConfigsUnifiedLocked(ctx, request, podInfo, snapshot) + if err != nil { + return err + } + if plan.stale { + return nil + } + + now := a.now() + var projection durableCacheProjection + changed, err := a.store.releaseEndpoint( + ctx, + a.generation, + plan.pod, + now, + func(candidate state.Snapshot) error { + var buildErr error + projection, buildErr = a.buildProjection(candidate) + return buildErr + }, + ) + if err != nil { + return err + } + if !changed { + return nil + } + return a.applyDeleteCommitLocked(ctx, projection) +} + +// releaseIPConfigsUnifiedLocked preflights the complete release without +// mutating service state. The caller owns the service lock. +func (service *HTTPRestService) releaseIPConfigsUnifiedLocked( + ctx context.Context, + request cns.IPConfigsRequest, + podInfo cns.PodInfo, + snapshot state.Snapshot, +) (unifiedReleasePlan, error) { + if err := ctx.Err(); err != nil { + return unifiedReleasePlan{}, fmt.Errorf("preflighting unified endpoint release: %w", err) + } + requestedPod := state.PodIdentity{ + PodKey: strings.TrimSpace(podInfo.Key()), + InfraContainerID: strings.TrimSpace(podInfo.InfraContainerID()), + InterfaceID: strings.TrimSpace(podInfo.InterfaceID()), + PodName: strings.TrimSpace(podInfo.Name()), + PodNamespace: strings.TrimSpace(podInfo.Namespace()), + } + if err := validateUnifiedReleasePod(requestedPod); err != nil { + return unifiedReleasePlan{}, err + } + plan := unifiedReleasePlan{pod: requestedPod} + + if endpoint, ok := snapshot.Endpoints[requestedPod.InfraContainerID]; ok { + if endpoint.PodName != requestedPod.PodName || endpoint.PodNamespace != requestedPod.PodNamespace { + plan.stale = true + return plan, nil + } + if ifname := strings.TrimSpace(request.Ifname); ifname != "" { + if _, ok := endpoint.IfnameToIPMap[ifname]; !ok { + plan.stale = true + return plan, nil + } + } + } + if assignment, ok := snapshot.Assignments[requestedPod.PodKey]; ok && + assignment.Pod.InfraContainerID != requestedPod.InfraContainerID { + plan.stale = true + return plan, nil + } + + assignmentKeys := make([]string, 0) + requestedAssignmentExists := false + currentAddresses := make(map[netip.Addr]struct{}) + for key, assignment := range snapshot.Assignments { + if assignment.Pod.InfraContainerID != requestedPod.InfraContainerID { + continue + } + if assignment.Pod.PodName != requestedPod.PodName || + assignment.Pod.PodNamespace != requestedPod.PodNamespace { + plan.stale = true + return plan, nil + } + assignmentKeys = append(assignmentKeys, key) + requestedAssignmentExists = requestedAssignmentExists || key == requestedPod.PodKey + for _, ipID := range assignment.IPIDs { + record, ok := snapshot.IPs[ipID] + if !ok { + return unifiedReleasePlan{}, fmt.Errorf("%w: assignment IP is missing", state.ErrStaleGeneration) + } + address, err := netip.ParseAddr(record.IPAddress) + if err != nil { + return unifiedReleasePlan{}, fmt.Errorf("%w: assignment IP address is invalid", state.ErrInvalidInput) + } + currentAddresses[address.Unmap()] = struct{}{} + } + } + sort.Strings(assignmentKeys) + if len(assignmentKeys) != 0 && !requestedAssignmentExists { + plan.stale = true + return plan, nil + } + if err := service.validateUnifiedReleaseProjectionLocked(snapshot, assignmentKeys); err != nil { + return unifiedReleasePlan{}, err + } + + requestedAddresses, err := parseUnifiedReleaseAddresses(request.DesiredIPAddresses) + if err != nil { + return unifiedReleasePlan{}, err + } + if len(requestedAddresses) == 0 || len(assignmentKeys) == 0 { + return plan, nil + } + matches := 0 + for address := range requestedAddresses { + if _, ok := currentAddresses[address]; ok { + matches++ + } + } + if matches == 0 { + plan.stale = true + return plan, nil + } + if matches != len(requestedAddresses) || len(requestedAddresses) != len(currentAddresses) { + return unifiedReleasePlan{}, fmt.Errorf( + "%w: release IP set does not match the current assignment", + state.ErrInvalidInput, + ) + } + return plan, nil +} + +func validateUnifiedReleasePod(pod state.PodIdentity) error { + switch { + case pod.PodKey == "": + return fmt.Errorf("%w: pod key is empty", state.ErrInvalidInput) + case pod.InfraContainerID == "": + return fmt.Errorf("%w: infra container ID is empty", state.ErrInvalidInput) + case pod.InterfaceID == "" && pod.PodKey != pod.InfraContainerID: + return fmt.Errorf("%w: pod key must equal infra container ID without an interface ID", state.ErrInvalidInput) + case pod.InterfaceID != "" && pod.PodKey != pod.InterfaceID: + return fmt.Errorf("%w: pod key must equal interface ID", state.ErrInvalidInput) + case pod.PodName == "": + return fmt.Errorf("%w: pod name is empty", state.ErrInvalidInput) + case pod.PodNamespace == "": + return fmt.Errorf("%w: pod namespace is empty", state.ErrInvalidInput) + default: + return nil + } +} + +func (service *HTTPRestService) validateUnifiedReleaseProjectionLocked( + snapshot state.Snapshot, + assignmentKeys []string, +) error { + for _, key := range assignmentKeys { + assignment := snapshot.Assignments[key] + cachedIDs, ok := service.PodIPIDByPodInterfaceKey[key] + if !ok || len(cachedIDs) != len(assignment.IPIDs) { + return fmt.Errorf("%w: projected assignment does not match database", state.ErrStaleGeneration) + } + for index, ipID := range assignment.IPIDs { + if cachedIDs[index] != ipID { + return fmt.Errorf("%w: projected assignment does not match database", state.ErrStaleGeneration) + } + status, ok := service.PodIPConfigState[ipID] + if !ok || status.GetState() != types.Assigned || status.PodInfo == nil || + status.PodInfo.Key() != key { + return fmt.Errorf("%w: projected IP owner does not match database", state.ErrStaleGeneration) + } + } + } + return nil +} + +func parseUnifiedReleaseAddresses(values []string) (map[netip.Addr]struct{}, error) { + addresses := make(map[netip.Addr]struct{}, len(values)) + for _, value := range values { + value = strings.TrimSpace(value) + if value == "" && len(values) == 1 { + continue + } + address, err := netip.ParseAddr(value) + if err != nil { + return nil, fmt.Errorf("%w: invalid release IP address", state.ErrInvalidInput) + } + address = address.Unmap() + if _, ok := addresses[address]; ok { + return nil, fmt.Errorf("%w: duplicate release IP address", state.ErrInvalidInput) + } + addresses[address] = struct{}{} + } + return addresses, nil +} + +func (a *durableStateAdapter) deleteEndpointRecord(ctx context.Context, infraContainerID string) error { + if a.store.deleteEndpoint == nil { + return errUnifiedDeleteOperationNil + } + a.mu.Lock() + defer a.mu.Unlock() + a.service.Lock() + defer a.service.Unlock() + + snapshot, err := a.currentSnapshot(ctx) + if err != nil { + return err + } + if _, ok := snapshot.Endpoints[strings.TrimSpace(infraContainerID)]; !ok { + return ErrEndpointStateNotFound + } + var projection durableCacheProjection + changed, err := a.store.deleteEndpoint( + ctx, + a.generation, + infraContainerID, + func(candidate state.Snapshot) error { + var buildErr error + projection, buildErr = a.buildProjection(candidate) + return buildErr + }, + ) + if err != nil { + return err + } + if !changed { + return ErrEndpointStateNotFound + } + return a.applyDeleteCommitLocked(ctx, projection) +} + +func (a *durableStateAdapter) pruneDeleteIntents(ctx context.Context, now time.Time) (int, error) { + if a.store.pruneDeleteIntents == nil { + return 0, errUnifiedPruneOperationNil + } + a.mu.Lock() + defer a.mu.Unlock() + a.service.Lock() + defer a.service.Unlock() + + if _, err := a.currentSnapshot(ctx); err != nil { + return 0, err + } + var projection durableCacheProjection + count, err := a.store.pruneDeleteIntents( + ctx, + a.generation, + now, + unifiedDeleteIntentTTL, + func(candidate state.Snapshot) error { + var buildErr error + projection, buildErr = a.buildProjection(candidate) + return buildErr + }, + ) + if err != nil || count == 0 { + return count, err + } + if err := a.applyDeleteCommitLocked(ctx, projection); err != nil { + return 0, err + } + return count, nil +} + +func (a *durableStateAdapter) applyDeleteCommitLocked( + ctx context.Context, + projection durableCacheProjection, +) error { + apply := a.applyDeleteProjection + if apply == nil { + apply = func(value durableCacheProjection) error { + a.applyProjectionLocked(value) + return nil + } + } + var postCommitErr error + if err := apply(projection); err != nil { + postCommitErr = a.restoreCommittedDeleteProjectionLocked(ctx, projection, err) + } + if a.store.refreshMetrics != nil { + if _, err := a.store.refreshMetrics(context.WithoutCancel(ctx)); err != nil { + postCommitErr = errors.Join(postCommitErr, fmt.Errorf("refreshing persistent state metrics: %w", err)) + } + } + if postCommitErr != nil { + return &unifiedDeleteCommittedError{err: postCommitErr} + } + return nil +} + +func (a *durableStateAdapter) restoreCommittedDeleteProjectionLocked( + ctx context.Context, + committed durableCacheProjection, + applyErr error, +) error { + restoreErr := error(nil) + snapshot, err := a.store.snapshot(context.WithoutCancel(ctx)) + if err != nil { + restoreErr = fmt.Errorf("reading committed endpoint deletion: %w", err) + } else { + projection, buildErr := a.buildProjection(snapshot) + if buildErr != nil { + restoreErr = fmt.Errorf("building committed endpoint deletion projection: %w", buildErr) + } else { + committed = projection + } + } + a.applyProjectionLocked(committed) + return errors.Join(applyErr, restoreErr) +} + +func unifiedReleaseResponseCode(err error) types.ResponseCode { + switch { + case errors.Is(err, state.ErrInvalidInput): + return types.InvalidRequest + case errors.Is(err, state.ErrStaleGeneration): + return types.InconsistentIPConfigState + default: + return types.UnexpectedError + } +} diff --git a/cns/restserver/unified_del_test.go b/cns/restserver/unified_del_test.go new file mode 100644 index 0000000000..9361a3c6a4 --- /dev/null +++ b/cns/restserver/unified_del_test.go @@ -0,0 +1,528 @@ +// Copyright 2026 Microsoft. All rights reserved. +// MIT License + +package restserver + +import ( + "context" + "errors" + "net" + "path/filepath" + "sync" + "testing" + "time" + + "github.com/Azure/azure-container-networking/cns" + "github.com/Azure/azure-container-networking/cns/state" + "github.com/Azure/azure-container-networking/cns/types" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +var ( + errInjectedReleaseFailure = errors.New("injected release failure") + errInjectedProjectionFailure = errors.New("injected projection failure") +) + +const ( + delTestIPv4Address = "10.0.0.4" + delTestIPv6Address = "2001:db8::4" + delTestIPv4ID = "ip-v4" + delTestIPv6ID = "ip-v6" + delTestIPID1 = "ip-1" + delTestCanceled = "canceled" + delTestInterfaceID = "interface-1" + delTestContainerID = "container-1" + delTestPodName = "pod-1" + delTestNamespace = "namespace-1" + delTestNCIPv4 = "nc-v4" + delTestNCIPv6 = "nc-v6" +) + +func TestUnifiedReleaseSingleAndMultiIP(t *testing.T) { + tests := []struct { + name string + containers map[string][]state.IPRecord + desired []string + }{ + { + name: "single", + containers: map[string][]state.IPRecord{ + delTestNCIPv4: {{ID: delTestIPv4ID, IPAddress: delTestIPv4Address, NCID: delTestNCIPv4, NCVersion: 1}}, + }, + desired: []string{delTestIPv4Address}, + }, + { + name: "dual stack", + containers: map[string][]state.IPRecord{ + delTestNCIPv4: {{ID: delTestIPv4ID, IPAddress: delTestIPv4Address, NCID: delTestNCIPv4, NCVersion: 1}}, + delTestNCIPv6: {{ID: delTestIPv6ID, IPAddress: delTestIPv6Address, NCID: delTestNCIPv6, NCVersion: 1}}, + }, + desired: []string{delTestIPv6Address, delTestIPv4Address}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + service, adapter, db, _ := newUnifiedAddFixture(t, tt.containers, nil) + now := time.Date(2026, time.July, 24, 2, 0, 0, 0, time.UTC) + adapter.now = func() time.Time { return now } + request := unifiedAddRequest(delTestContainerID, delTestInterfaceID, InfraInterfaceName, delTestPodName, delTestNamespace) + request.DesiredIPAddresses = tt.desired + _, err := service.requestIPConfigHandlerHelper(context.Background(), request) + require.NoError(t, err) + beforeRelease := requireUnifiedSnapshot(t, db) + + response, err := service.ReleaseIPConfigHandlerHelper(context.Background(), request) + require.NoError(t, err) + assert.Equal(t, types.Success, response.Response.ReturnCode) + released := requireUnifiedSnapshot(t, db) + assert.Equal(t, beforeRelease.Metadata.Generation+1, released.Metadata.Generation) + assert.Empty(t, released.Assignments) + assert.Empty(t, released.IPOwners) + assert.Contains(t, released.Endpoints, delTestContainerID) + assert.Equal(t, now, released.DeleteIntents[delTestContainerID].CreatedAt) + assert.Empty(t, service.PodIPIDByPodInterfaceKey) + for ipID := range released.IPs { + status := service.PodIPConfigState[ipID] + assert.Equal(t, types.Available, status.GetState()) + } + + adapter.now = func() time.Time { return now.Add(time.Minute) } + response, err = service.ReleaseIPConfigHandlerHelper(context.Background(), request) + require.NoError(t, err) + assert.Equal(t, types.Success, response.Response.ReturnCode) + repeated := requireUnifiedSnapshot(t, db) + assert.Equal(t, released.Metadata.Generation, repeated.Metadata.Generation) + assert.Equal(t, now, repeated.DeleteIntents[delTestContainerID].CreatedAt) + + require.NoError(t, adapter.deleteEndpointRecord(context.Background(), delTestContainerID)) + cleaned := requireUnifiedSnapshot(t, db) + assert.Equal(t, released.Metadata.Generation+1, cleaned.Metadata.Generation) + assert.NotContains(t, cleaned.Endpoints, delTestContainerID) + assert.Equal(t, now, cleaned.DeleteIntents[delTestContainerID].CreatedAt) + }) + } +} + +func TestUnifiedReleaseStaleAndInvalidRequestsAreAtomic(t *testing.T) { + service, _, db, _ := newUnifiedAddFixture(t, map[string][]state.IPRecord{ + delTestNCIPv4: { + {ID: delTestIPID1, IPAddress: delTestIPv4Address, NCID: delTestNCIPv4, NCVersion: 1}, + {ID: "ip-2", IPAddress: primaryIP, NCID: delTestNCIPv4, NCVersion: 1}, + }, + }, nil) + current := unifiedAddRequest(delTestContainerID, delTestInterfaceID, InfraInterfaceName, delTestPodName, delTestNamespace) + current.DesiredIPAddresses = []string{delTestIPv4Address, primaryIP} + _, err := service.requestIPConfigHandlerHelper(context.Background(), current) + require.NoError(t, err) + + tests := []struct { + name string + request cns.IPConfigsRequest + wantCode types.ResponseCode + wantErr bool + }{ + { + name: "old container is stale success", + request: func() cns.IPConfigsRequest { + request := current + request.InfraContainerID = "old-container" + return request + }(), + wantCode: types.Success, + }, + { + name: "old IP set is stale success", + request: func() cns.IPConfigsRequest { + request := current + request.DesiredIPAddresses = []string{"10.0.0.99"} + return request + }(), + wantCode: types.Success, + }, + { + name: "partial multi-IP set is invalid", + request: func() cns.IPConfigsRequest { + request := current + request.DesiredIPAddresses = []string{delTestIPv4Address} + return request + }(), + wantCode: types.InvalidRequest, + wantErr: true, + }, + { + name: "mixed valid and invalid IP is invalid", + request: func() cns.IPConfigsRequest { + request := current + request.DesiredIPAddresses = []string{delTestIPv4Address, "not-an-ip"} + return request + }(), + wantCode: types.InvalidRequest, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + before := requireUnifiedSnapshot(t, db) + response, err := service.ReleaseIPConfigHandlerHelper(context.Background(), tt.request) + if tt.wantErr { + require.Error(t, err) + } else { + require.NoError(t, err) + } + assert.Equal(t, tt.wantCode, response.Response.ReturnCode) + assert.Equal(t, before, requireUnifiedSnapshot(t, db)) + }) + } +} + +func TestUnifiedReleaseCrashRecoveryAndPrune(t *testing.T) { + path := filepath.Join(t.TempDir(), "state.db") + db, err := state.Open(path, state.Options{}) + require.NoError(t, err) + _, err = db.ApplyNetworkContainer(context.Background(), r18NetworkContainer(delTestNCIPv4), []state.IPRecord{{ + ID: delTestIPID1, IPAddress: delTestIPv4Address, NCID: delTestNCIPv4, NCVersion: 1, + }}) + require.NoError(t, err) + service := newUnifiedAddTestService(t) + restore, closeState, err := NewDurableStateLifecycle(service, db, true) + require.NoError(t, err) + require.NoError(t, restore(context.Background())) + request := unifiedAddRequest(delTestContainerID, delTestInterfaceID, InfraInterfaceName, delTestPodName, delTestNamespace) + request.DesiredIPAddresses = []string{delTestIPv4Address} + _, err = service.requestIPConfigHandlerHelper(context.Background(), request) + require.NoError(t, err) + now := time.Date(2026, time.July, 24, 3, 0, 0, 0, time.UTC) + service.unifiedStateAdapter.now = func() time.Time { return now } + _, err = service.ReleaseIPConfigHandlerHelper(context.Background(), request) + require.NoError(t, err) + require.NoError(t, closeState()) + + db, err = state.Open(path, state.Options{}) + require.NoError(t, err) + reopenedService := newUnifiedAddTestService(t) + restore, closeState, err = NewDurableStateLifecycle(reopenedService, db, true) + require.NoError(t, err) + require.NoError(t, restore(context.Background())) + t.Cleanup(func() { require.NoError(t, closeState()) }) + reopened := requireUnifiedSnapshot(t, db) + assert.Empty(t, reopened.Assignments) + assert.Empty(t, reopened.IPOwners) + assert.Contains(t, reopened.Endpoints, delTestContainerID) + assert.Equal(t, now, reopened.DeleteIntents[delTestContainerID].CreatedAt) + + reopenedService.unifiedStateAdapter.now = func() time.Time { return now.Add(time.Minute) } + response, err := reopenedService.requestIPConfigHandlerHelper(context.Background(), request) + require.ErrorIs(t, err, state.ErrDeleteIntent) + assert.Equal(t, types.AddressUnavailable, response.Response.ReturnCode) + require.NoError(t, reopenedService.unifiedStateAdapter.deleteEndpointRecord(context.Background(), delTestContainerID)) + require.NoError(t, closeState()) + + db, err = state.Open(path, state.Options{}) + require.NoError(t, err) + afterDeleteService := newUnifiedAddTestService(t) + restore, closeState, err = NewDurableStateLifecycle(afterDeleteService, db, true) + require.NoError(t, err) + require.NoError(t, restore(context.Background())) + afterDelete := requireUnifiedSnapshot(t, db) + assert.NotContains(t, afterDelete.Endpoints, delTestContainerID) + assert.Equal(t, now, afterDelete.DeleteIntents[delTestContainerID].CreatedAt) + + count, err := afterDeleteService.unifiedStateAdapter.pruneDeleteIntents( + context.Background(), + now.Add(unifiedDeleteIntentTTL), + ) + require.NoError(t, err) + assert.Equal(t, 1, count) + response, err = afterDeleteService.requestIPConfigHandlerHelper(context.Background(), request) + require.NoError(t, err) + assert.Equal(t, types.Success, response.Response.ReturnCode) +} + +func TestUnifiedReleaseConcurrentDuplicate(t *testing.T) { + service, adapter, db, _ := newUnifiedAddFixture(t, map[string][]state.IPRecord{ + delTestNCIPv4: {{ID: delTestIPID1, IPAddress: delTestIPv4Address, NCID: delTestNCIPv4, NCVersion: 1}}, + }, nil) + request := unifiedAddRequest(delTestContainerID, delTestInterfaceID, InfraInterfaceName, delTestPodName, delTestNamespace) + request.DesiredIPAddresses = []string{delTestIPv4Address} + _, err := service.requestIPConfigHandlerHelper(context.Background(), request) + require.NoError(t, err) + now := time.Date(2026, time.July, 24, 4, 0, 0, 0, time.UTC) + adapter.now = func() time.Time { return now } + before := requireUnifiedSnapshot(t, db) + + const callers = 8 + start := make(chan struct{}) + errs := make(chan error, callers) + var ready sync.WaitGroup + ready.Add(callers) + for range callers { + go func() { + ready.Done() + <-start + _, err := service.ReleaseIPConfigHandlerHelper(context.Background(), request) + errs <- err + }() + } + ready.Wait() + close(start) + for range callers { + require.NoError(t, <-errs) + } + after := requireUnifiedSnapshot(t, db) + assert.Equal(t, before.Metadata.Generation+1, after.Metadata.Generation) + assert.Equal(t, now, after.DeleteIntents[delTestContainerID].CreatedAt) + assert.Empty(t, after.Assignments) + assert.Empty(t, after.IPOwners) +} + +func TestUnifiedReleaseLinearizesWithAdd(t *testing.T) { + t.Run("ADD commits first and DEL releases it", func(t *testing.T) { + service, adapter, db, _ := newUnifiedAddFixture(t, map[string][]state.IPRecord{ + delTestNCIPv4: {{ID: delTestIPID1, IPAddress: delTestIPv4Address, NCID: delTestNCIPv4, NCVersion: 1}}, + }, nil) + request := unifiedAddRequest(delTestContainerID, delTestInterfaceID, InfraInterfaceName, delTestPodName, delTestNamespace) + request.DesiredIPAddresses = []string{delTestIPv4Address} + realAssign := adapter.store.assignEndpoint + entered := make(chan struct{}) + unblock := make(chan struct{}) + adapter.store.assignEndpoint = func( + ctx context.Context, + generation uint64, + assignment state.AssignmentRecord, + endpoint state.EndpointRecord, + now time.Time, + ttl time.Duration, + beforeCommit func(state.Snapshot) error, + ) (bool, error) { + close(entered) + <-unblock + return realAssign(ctx, generation, assignment, endpoint, now, ttl, beforeCommit) + } + + addResult := make(chan error, 1) + go func() { + _, err := service.requestIPConfigHandlerHelper(context.Background(), request) + addResult <- err + }() + <-entered + delResult := make(chan error, 1) + go func() { + _, err := service.ReleaseIPConfigHandlerHelper(context.Background(), request) + delResult <- err + }() + close(unblock) + require.NoError(t, <-addResult) + require.NoError(t, <-delResult) + snapshot := requireUnifiedSnapshot(t, db) + assert.Empty(t, snapshot.Assignments) + assert.Empty(t, snapshot.IPOwners) + assert.Contains(t, snapshot.DeleteIntents, delTestContainerID) + }) + + t.Run("DEL commits first and late ADD is blocked", func(t *testing.T) { + service, adapter, db, _ := newUnifiedAddFixture(t, map[string][]state.IPRecord{ + delTestNCIPv4: {{ID: delTestIPID1, IPAddress: delTestIPv4Address, NCID: delTestNCIPv4, NCVersion: 1}}, + }, nil) + request := unifiedAddRequest(delTestContainerID, delTestInterfaceID, InfraInterfaceName, delTestPodName, delTestNamespace) + request.DesiredIPAddresses = []string{delTestIPv4Address} + _, err := service.requestIPConfigHandlerHelper(context.Background(), request) + require.NoError(t, err) + now := time.Date(2026, time.July, 24, 5, 0, 0, 0, time.UTC) + adapter.now = func() time.Time { return now } + realRelease := adapter.store.releaseEndpoint + entered := make(chan struct{}) + unblock := make(chan struct{}) + adapter.store.releaseEndpoint = func( + ctx context.Context, + generation uint64, + pod state.PodIdentity, + now time.Time, + beforeCommit func(state.Snapshot) error, + ) (bool, error) { + close(entered) + <-unblock + return realRelease(ctx, generation, pod, now, beforeCommit) + } + + delResult := make(chan error, 1) + go func() { + _, err := service.ReleaseIPConfigHandlerHelper(context.Background(), request) + delResult <- err + }() + <-entered + type addResult struct { + response *cns.IPConfigsResponse + err error + } + addDone := make(chan addResult, 1) + go func() { + response, err := service.requestIPConfigHandlerHelper(context.Background(), request) + addDone <- addResult{response: response, err: err} + }() + close(unblock) + require.NoError(t, <-delResult) + add := <-addDone + require.ErrorIs(t, add.err, state.ErrDeleteIntent) + assert.Equal(t, types.AddressUnavailable, add.response.Response.ReturnCode) + snapshot := requireUnifiedSnapshot(t, db) + assert.Empty(t, snapshot.Assignments) + assert.Empty(t, snapshot.IPOwners) + assert.Contains(t, snapshot.DeleteIntents, delTestContainerID) + }) +} + +func TestUnifiedReleaseFailureAtomicityAndMapping(t *testing.T) { + tests := []struct { + name string + inject func(*durableStateAdapter) + context func() context.Context + wantCode types.ResponseCode + }{ + { + name: "commit", + inject: func(adapter *durableStateAdapter) { + adapter.store.releaseEndpoint = func( + context.Context, + uint64, + state.PodIdentity, + time.Time, + func(state.Snapshot) error, + ) (bool, error) { + return false, errInjectedReleaseFailure + } + }, + wantCode: types.UnexpectedError, + }, + { + name: "stale generation", + inject: func(adapter *durableStateAdapter) { + adapter.store.releaseEndpoint = func( + context.Context, + uint64, + state.PodIdentity, + time.Time, + func(state.Snapshot) error, + ) (bool, error) { + return false, state.ErrStaleGeneration + } + }, + wantCode: types.InconsistentIPConfigState, + }, + { + name: "candidate callback", + inject: func(adapter *durableStateAdapter) { + adapter.buildProjection = func(state.Snapshot) (durableCacheProjection, error) { + return durableCacheProjection{}, errInjectedReleaseFailure + } + }, + wantCode: types.UnexpectedError, + }, + { + name: delTestCanceled, + inject: func(*durableStateAdapter) {}, + context: canceledUnifiedReleaseContext, + wantCode: types.UnexpectedError, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + service, adapter, db, _ := newUnifiedAddFixture(t, map[string][]state.IPRecord{ + delTestNCIPv4: {{ID: delTestIPID1, IPAddress: delTestIPv4Address, NCID: delTestNCIPv4, NCVersion: 1}}, + }, nil) + request := unifiedAddRequest(delTestContainerID, delTestInterfaceID, InfraInterfaceName, delTestPodName, delTestNamespace) + request.DesiredIPAddresses = []string{delTestIPv4Address} + _, err := service.requestIPConfigHandlerHelper(context.Background(), request) + require.NoError(t, err) + beforeDB := requireUnifiedSnapshot(t, db) + beforeCache := durableCacheFingerprint(service, adapter) + tt.inject(adapter) + ctx := context.Background() + if tt.context != nil { + ctx = tt.context() + } + + response, err := service.ReleaseIPConfigHandlerHelper(ctx, request) + require.Error(t, err) + assert.Equal(t, tt.wantCode, response.Response.ReturnCode) + assert.Equal(t, beforeDB, requireUnifiedSnapshot(t, db)) + assert.Equal(t, beforeCache, durableCacheFingerprint(service, adapter)) + }) + } +} + +func TestUnifiedReleasePostcommitProjectionRestoresCache(t *testing.T) { + service, adapter, db, _ := newUnifiedAddFixture(t, map[string][]state.IPRecord{ + delTestNCIPv4: {{ID: delTestIPID1, IPAddress: delTestIPv4Address, NCID: delTestNCIPv4, NCVersion: 1}}, + }, nil) + request := unifiedAddRequest(delTestContainerID, delTestInterfaceID, InfraInterfaceName, delTestPodName, delTestNamespace) + request.DesiredIPAddresses = []string{delTestIPv4Address} + _, err := service.requestIPConfigHandlerHelper(context.Background(), request) + require.NoError(t, err) + adapter.applyDeleteProjection = func(durableCacheProjection) error { return errInjectedProjectionFailure } + + response, err := service.ReleaseIPConfigHandlerHelper(context.Background(), request) + require.ErrorIs(t, err, errInjectedProjectionFailure) + assert.Equal(t, types.UnexpectedError, response.Response.ReturnCode) + snapshot := requireUnifiedSnapshot(t, db) + assert.Empty(t, snapshot.Assignments) + assert.Empty(t, snapshot.IPOwners) + assert.Contains(t, snapshot.DeleteIntents, delTestContainerID) + status := service.PodIPConfigState[delTestIPID1] + assert.Equal(t, types.Available, status.GetState()) + assert.Empty(t, service.PodIPIDByPodInterfaceKey) + generation, projected := adapter.cacheGeneration() + assert.True(t, projected) + assert.Equal(t, snapshot.Metadata.Generation, generation) +} + +func TestUnifiedReleaseImportedAssignment(t *testing.T) { + service, _, db, _ := newUnifiedAddFixture(t, map[string][]state.IPRecord{ + delTestNCIPv4: {{ID: delTestIPID1, IPAddress: delTestIPv4Address, NCID: delTestNCIPv4, NCVersion: 1}}, + }, func(db *state.DB) { + assignment := state.AssignmentRecord{ + Pod: state.PodIdentity{ + PodKey: delTestInterfaceID, + InfraContainerID: delTestContainerID, + InterfaceID: delTestInterfaceID, + PodName: delTestPodName, + PodNamespace: delTestNamespace, + }, + IPIDs: []string{delTestIPID1}, + } + _, err := db.AssignEndpoint( + context.Background(), + assignment, + state.EndpointRecord{ + PodName: delTestPodName, + PodNamespace: delTestNamespace, + IfnameToIPMap: map[string]*state.IPInfoRecord{ + InfraInterfaceName: { + IPv4: []net.IPNet{testIPNet(delTestIPv4Address + "/24")}, + NetworkContainerID: delTestNCIPv4, + NICType: cns.InfraNIC, + }, + }, + }, + time.Date(2026, time.July, 24, 0, 0, 0, 0, time.UTC), + unifiedDeleteIntentTTL, + ) + require.NoError(t, err) + }) + request := unifiedAddRequest(delTestContainerID, delTestInterfaceID, InfraInterfaceName, delTestPodName, delTestNamespace) + request.DesiredIPAddresses = []string{delTestIPv4Address} + + response, err := service.ReleaseIPConfigHandlerHelper(context.Background(), request) + require.NoError(t, err) + assert.Equal(t, types.Success, response.Response.ReturnCode) + snapshot := requireUnifiedSnapshot(t, db) + assert.Empty(t, snapshot.Assignments) + assert.Empty(t, snapshot.IPOwners) + assert.Contains(t, snapshot.DeleteIntents, delTestContainerID) +} + +func canceledUnifiedReleaseContext() context.Context { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + return ctx +} diff --git a/cns/state/ownership_operations.go b/cns/state/ownership_operations.go index ffef8450c7..79c8d44bfb 100644 --- a/cns/state/ownership_operations.go +++ b/cns/state/ownership_operations.go @@ -234,6 +234,29 @@ func (s *DB) ReleaseEndpoint( ctx context.Context, pod PodIdentity, now time.Time, +) (bool, error) { + return s.releaseEndpoint(ctx, nil, pod, now, nil) +} + +// ReleaseEndpointIfGeneration atomically records the delete intent and releases +// every assignment for the endpoint only at expectedGeneration. beforeCommit +// may validate the complete candidate snapshot before any records are written. +func (s *DB) ReleaseEndpointIfGeneration( + ctx context.Context, + expectedGeneration uint64, + pod PodIdentity, + now time.Time, + beforeCommit func(Snapshot) error, +) (bool, error) { + return s.releaseEndpoint(ctx, &expectedGeneration, pod, now, beforeCommit) +} + +func (s *DB) releaseEndpoint( + ctx context.Context, + expectedGeneration *uint64, + pod PodIdentity, + now time.Time, + beforeCommit func(Snapshot) error, ) (bool, error) { normalizedPod, err := normalizePodIdentity(pod, true) if err != nil { @@ -249,6 +272,14 @@ func (s *DB) ReleaseEndpoint( if snapshotErr != nil { return false, snapshotErr } + if expectedGeneration != nil && current.Metadata.Generation != *expectedGeneration { + return false, fmt.Errorf( + "%w: expected=%d actual=%d", + ErrStaleGeneration, + *expectedGeneration, + current.Metadata.Generation, + ) + } if validationErr := validateReleaseIdentity(current, normalizedPod); validationErr != nil { return false, validationErr } @@ -286,8 +317,25 @@ func (s *DB) ReleaseEndpoint( return false, invalidInput("encoding delete intent", encodeErr) } if len(assignmentKeys) == 0 && intentExists { + if beforeCommit != nil { + if err := beforeCommit(current); err != nil { + return false, fmt.Errorf("validating endpoint release candidate: %w", err) + } + } return false, nil } + if candidate.Metadata.Generation == math.MaxUint64 { + return false, corrupt("generation overflow", nil) + } + candidate.Metadata.Generation++ + if beforeCommit != nil { + if err := beforeCommit(candidate); err != nil { + return false, fmt.Errorf("validating endpoint release candidate: %w", err) + } + } + if err := ctx.Err(); err != nil { + return false, fmt.Errorf("committing endpoint release: %w", err) + } if !intentExists { if writeErr := tx.tx.Bucket(bucketDeleteIntents).Put( @@ -384,6 +432,26 @@ func (s *DB) PatchEndpoint( } func (s *DB) DeleteEndpointRecord(ctx context.Context, infraContainerID string) (bool, error) { + return s.deleteEndpointRecord(ctx, nil, infraContainerID, nil) +} + +// DeleteEndpointRecordIfGeneration removes an endpoint record only at +// expectedGeneration while preserving its delete intent. +func (s *DB) DeleteEndpointRecordIfGeneration( + ctx context.Context, + expectedGeneration uint64, + infraContainerID string, + beforeCommit func(Snapshot) error, +) (bool, error) { + return s.deleteEndpointRecord(ctx, &expectedGeneration, infraContainerID, beforeCommit) +} + +func (s *DB) deleteEndpointRecord( + ctx context.Context, + expectedGeneration *uint64, + infraContainerID string, + beforeCommit func(Snapshot) error, +) (bool, error) { infraContainerID = normalizeID(infraContainerID) if infraContainerID == "" { return false, invalidInput("infra container ID is empty", nil) @@ -393,7 +461,20 @@ func (s *DB) DeleteEndpointRecord(ctx context.Context, infraContainerID string) if snapshotErr != nil { return false, snapshotErr } + if expectedGeneration != nil && current.Metadata.Generation != *expectedGeneration { + return false, fmt.Errorf( + "%w: expected=%d actual=%d", + ErrStaleGeneration, + *expectedGeneration, + current.Metadata.Generation, + ) + } if _, ok := current.Endpoints[infraContainerID]; !ok { + if beforeCommit != nil { + if err := beforeCommit(current); err != nil { + return false, fmt.Errorf("validating endpoint deletion candidate: %w", err) + } + } return false, nil } candidate := cloneSnapshot(current) @@ -401,6 +482,18 @@ func (s *DB) DeleteEndpointRecord(ctx context.Context, infraContainerID string) if validationErr := validateInput(candidate); validationErr != nil { return false, validationErr } + if candidate.Metadata.Generation == math.MaxUint64 { + return false, corrupt("generation overflow", nil) + } + candidate.Metadata.Generation++ + if beforeCommit != nil { + if err := beforeCommit(candidate); err != nil { + return false, fmt.Errorf("validating endpoint deletion candidate: %w", err) + } + } + if err := ctx.Err(); err != nil { + return false, fmt.Errorf("committing endpoint deletion: %w", err) + } if deleteErr := tx.tx.Bucket(bucketEndpoints).Delete([]byte(infraContainerID)); deleteErr != nil { return false, fmt.Errorf("deleting endpoint %q: %w", infraContainerID, deleteErr) } @@ -412,6 +505,28 @@ func (s *DB) PruneDeleteIntents( ctx context.Context, now time.Time, deleteIntentTTL time.Duration, +) (int, error) { + return s.pruneDeleteIntents(ctx, nil, now, deleteIntentTTL, nil) +} + +// PruneDeleteIntentsIfGeneration removes expired intents only at +// expectedGeneration. +func (s *DB) PruneDeleteIntentsIfGeneration( + ctx context.Context, + expectedGeneration uint64, + now time.Time, + deleteIntentTTL time.Duration, + beforeCommit func(Snapshot) error, +) (int, error) { + return s.pruneDeleteIntents(ctx, &expectedGeneration, now, deleteIntentTTL, beforeCommit) +} + +func (s *DB) pruneDeleteIntents( + ctx context.Context, + expectedGeneration *uint64, + now time.Time, + deleteIntentTTL time.Duration, + beforeCommit func(Snapshot) error, ) (int, error) { now, err := normalizeNow(now, deleteIntentTTL) if err != nil { @@ -423,6 +538,14 @@ func (s *DB) PruneDeleteIntents( if snapshotErr != nil { return false, snapshotErr } + if expectedGeneration != nil && current.Metadata.Generation != *expectedGeneration { + return false, fmt.Errorf( + "%w: expected=%d actual=%d", + ErrStaleGeneration, + *expectedGeneration, + current.Metadata.Generation, + ) + } expired := make([]string, 0) for _, containerID := range sortedKeys(current.DeleteIntents) { if !deleteIntentLive(current.DeleteIntents[containerID], now, deleteIntentTTL) { @@ -430,6 +553,11 @@ func (s *DB) PruneDeleteIntents( } } if len(expired) == 0 { + if beforeCommit != nil { + if callbackErr := beforeCommit(current); callbackErr != nil { + return false, fmt.Errorf("validating delete intent prune candidate: %w", callbackErr) + } + } return false, nil } candidate := cloneSnapshot(current) @@ -439,6 +567,18 @@ func (s *DB) PruneDeleteIntents( if validationErr := validateInput(candidate); validationErr != nil { return false, validationErr } + if candidate.Metadata.Generation == math.MaxUint64 { + return false, corrupt("generation overflow", nil) + } + candidate.Metadata.Generation++ + if beforeCommit != nil { + if callbackErr := beforeCommit(candidate); callbackErr != nil { + return false, fmt.Errorf("validating delete intent prune candidate: %w", callbackErr) + } + } + if contextErr := ctx.Err(); contextErr != nil { + return false, fmt.Errorf("committing delete intent prune: %w", contextErr) + } bucket := tx.tx.Bucket(bucketDeleteIntents) for _, containerID := range expired { if deleteErr := bucket.Delete([]byte(containerID)); deleteErr != nil { diff --git a/cns/state/ownership_release_test.go b/cns/state/ownership_release_test.go new file mode 100644 index 0000000000..715c26848f --- /dev/null +++ b/cns/state/ownership_release_test.go @@ -0,0 +1,315 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package state + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + bolterrors "go.etcd.io/bbolt/errors" +) + +var ( + errReleaseCandidateFailure = errors.New("injected release candidate failure") + errDeleteNoopFailure = errors.New("injected no-op callback failure") + errDeleteCandidateFailure = errors.New("injected endpoint delete candidate failure") + errPruneCandidateFailure = errors.New("injected prune candidate failure") +) + +func TestReleaseEndpointIfGenerationPrecommitAndFailures(t *testing.T) { + t.Run("candidate callback failure rolls back", func(t *testing.T) { + db, _ := openTestDB(t) + assignment, endpoint := seedOwnershipInventory(t, db) + _, err := db.AssignEndpoint(context.Background(), assignment, endpoint, testNow, testDeleteIntentTTL) + require.NoError(t, err) + before := requireValidSnapshot(t, db) + callbackCalled := false + + changed, err := db.ReleaseEndpointIfGeneration( + context.Background(), + before.Metadata.Generation, + assignment.Pod, + testNow.Add(time.Minute), + func(candidate Snapshot) error { + callbackCalled = true + assert.Equal(t, before.Metadata.Generation+1, candidate.Metadata.Generation) + assert.Empty(t, candidate.Assignments) + assert.Empty(t, candidate.IPOwners) + assert.Contains(t, candidate.Endpoints, assignment.Pod.InfraContainerID) + assert.Equal(t, testNow.Add(time.Minute), candidate.DeleteIntents[assignment.Pod.InfraContainerID].CreatedAt) + return errReleaseCandidateFailure + }, + ) + require.ErrorIs(t, err, errReleaseCandidateFailure) + assert.False(t, changed) + assert.True(t, callbackCalled) + assert.Equal(t, before, requireValidSnapshot(t, db)) + }) + + t.Run("stale generation does not invoke callback", func(t *testing.T) { + db, _ := openTestDB(t) + assignment, endpoint := seedOwnershipInventory(t, db) + _, err := db.AssignEndpoint(context.Background(), assignment, endpoint, testNow, testDeleteIntentTTL) + require.NoError(t, err) + before := requireValidSnapshot(t, db) + + changed, err := db.ReleaseEndpointIfGeneration( + context.Background(), + before.Metadata.Generation+1, + assignment.Pod, + testNow, + func(Snapshot) error { + t.Fatal("stale generation invoked release callback") + return nil + }, + ) + require.ErrorIs(t, err, ErrStaleGeneration) + assert.False(t, changed) + assert.Equal(t, before, requireValidSnapshot(t, db)) + }) + + t.Run("canceled while waiting for writer gate", func(t *testing.T) { + db, _ := openTestDB(t) + assignment, endpoint := seedOwnershipInventory(t, db) + _, err := db.AssignEndpoint(context.Background(), assignment, endpoint, testNow, testDeleteIntentTTL) + require.NoError(t, err) + before := requireValidSnapshot(t, db) + db.writeGate <- struct{}{} + t.Cleanup(func() { + select { + case <-db.writeGate: + default: + } + }) + ctx, cancel := context.WithCancel(context.Background()) + started := make(chan struct{}) + result := make(chan error, 1) + go func() { + close(started) + _, err := db.ReleaseEndpointIfGeneration( + ctx, + before.Metadata.Generation, + assignment.Pod, + testNow, + nil, + ) + result <- err + }() + <-started + cancel() + require.ErrorIs(t, <-result, context.Canceled) + <-db.writeGate + assert.Equal(t, before, requireValidSnapshot(t, db)) + }) + + t.Run("read only", func(t *testing.T) { + db, path := openTestDB(t) + assignment, endpoint := seedOwnershipInventory(t, db) + _, err := db.AssignEndpoint(context.Background(), assignment, endpoint, testNow, testDeleteIntentTTL) + require.NoError(t, err) + before := requireValidSnapshot(t, db) + require.NoError(t, db.Close()) + readOnly, err := Open(path, Options{ReadOnly: true}) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, readOnly.Close()) }) + + changed, err := readOnly.ReleaseEndpointIfGeneration( + context.Background(), + before.Metadata.Generation, + assignment.Pod, + testNow, + nil, + ) + require.ErrorIs(t, err, bolterrors.ErrDatabaseReadOnly) + assert.False(t, changed) + assert.Equal(t, before, requireValidSnapshot(t, readOnly)) + }) + + t.Run("successful callback commits candidate", func(t *testing.T) { + db, _ := openTestDB(t) + assignment, endpoint := seedOwnershipInventory(t, db) + _, err := db.AssignEndpoint(context.Background(), assignment, endpoint, testNow, testDeleteIntentTTL) + require.NoError(t, err) + before := requireValidSnapshot(t, db) + + changed, err := db.ReleaseEndpointIfGeneration( + context.Background(), + before.Metadata.Generation, + assignment.Pod, + testNow, + func(candidate Snapshot) error { + assert.Empty(t, candidate.Assignments) + assert.Contains(t, candidate.DeleteIntents, assignment.Pod.InfraContainerID) + return nil + }, + ) + require.NoError(t, err) + assert.True(t, changed) + assert.Empty(t, requireValidSnapshot(t, db).Assignments) + }) +} + +func TestDeleteAndPruneIfGenerationPrecommit(t *testing.T) { + t.Run("stale generations do not mutate", func(t *testing.T) { + db, _ := openTestDB(t) + assignment, endpoint := seedOwnershipInventory(t, db) + _, err := db.AssignEndpoint(context.Background(), assignment, endpoint, testNow, testDeleteIntentTTL) + require.NoError(t, err) + before := requireValidSnapshot(t, db) + + changed, err := db.DeleteEndpointRecordIfGeneration( + context.Background(), + before.Metadata.Generation+1, + assignment.Pod.InfraContainerID, + nil, + ) + require.ErrorIs(t, err, ErrStaleGeneration) + assert.False(t, changed) + count, err := db.PruneDeleteIntentsIfGeneration( + context.Background(), + before.Metadata.Generation+1, + testNow, + testDeleteIntentTTL, + nil, + ) + require.ErrorIs(t, err, ErrStaleGeneration) + assert.Zero(t, count) + assert.Equal(t, before, requireValidSnapshot(t, db)) + }) + + t.Run("endpoint delete and repeated no-op callbacks", func(t *testing.T) { + db, _ := openTestDB(t) + assignment, endpoint := seedOwnershipInventory(t, db) + _, err := db.AssignEndpoint(context.Background(), assignment, endpoint, testNow, testDeleteIntentTTL) + require.NoError(t, err) + _, err = db.ReleaseEndpoint(context.Background(), assignment.Pod, testNow) + require.NoError(t, err) + before := requireValidSnapshot(t, db) + callbacks := 0 + + changed, err := db.DeleteEndpointRecordIfGeneration( + context.Background(), + before.Metadata.Generation, + assignment.Pod.InfraContainerID, + func(candidate Snapshot) error { + callbacks++ + assert.Equal(t, before.Metadata.Generation+1, candidate.Metadata.Generation) + return nil + }, + ) + require.NoError(t, err) + assert.True(t, changed) + after := requireValidSnapshot(t, db) + changed, err = db.DeleteEndpointRecordIfGeneration( + context.Background(), + after.Metadata.Generation, + assignment.Pod.InfraContainerID, + func(candidate Snapshot) error { + callbacks++ + assert.Equal(t, after.Metadata.Generation, candidate.Metadata.Generation) + return nil + }, + ) + require.NoError(t, err) + assert.False(t, changed) + assert.Equal(t, 2, callbacks) + changed, err = db.DeleteEndpointRecordIfGeneration( + context.Background(), + after.Metadata.Generation, + assignment.Pod.InfraContainerID, + func(Snapshot) error { return errDeleteNoopFailure }, + ) + require.ErrorIs(t, err, errDeleteNoopFailure) + assert.False(t, changed) + }) + + t.Run("endpoint delete callback failure rolls back", func(t *testing.T) { + db, _ := openTestDB(t) + assignment, endpoint := seedOwnershipInventory(t, db) + _, err := db.AssignEndpoint(context.Background(), assignment, endpoint, testNow, testDeleteIntentTTL) + require.NoError(t, err) + _, err = db.ReleaseEndpoint(context.Background(), assignment.Pod, testNow) + require.NoError(t, err) + before := requireValidSnapshot(t, db) + changed, err := db.DeleteEndpointRecordIfGeneration( + context.Background(), + before.Metadata.Generation, + assignment.Pod.InfraContainerID, + func(candidate Snapshot) error { + assert.NotContains(t, candidate.Endpoints, assignment.Pod.InfraContainerID) + assert.Contains(t, candidate.DeleteIntents, assignment.Pod.InfraContainerID) + return errDeleteCandidateFailure + }, + ) + require.ErrorIs(t, err, errDeleteCandidateFailure) + assert.False(t, changed) + assert.Equal(t, before, requireValidSnapshot(t, db)) + }) + + t.Run("intent prune callback failure rolls back", func(t *testing.T) { + db, _ := openTestDB(t) + require.NoError(t, db.Update(context.Background(), func(tx *WriteTx) error { + return tx.PutDeleteIntent("container-1", DeleteIntent{ + CreatedAt: testNow.Add(-testDeleteIntentTTL), + }) + })) + before := requireValidSnapshot(t, db) + count, err := db.PruneDeleteIntentsIfGeneration( + context.Background(), + before.Metadata.Generation, + testNow, + testDeleteIntentTTL, + func(candidate Snapshot) error { + assert.NotContains(t, candidate.DeleteIntents, "container-1") + return errPruneCandidateFailure + }, + ) + require.ErrorIs(t, err, errPruneCandidateFailure) + assert.Zero(t, count) + assert.Equal(t, before, requireValidSnapshot(t, db)) + }) + + t.Run("intent prune and repeated no-op callbacks", func(t *testing.T) { + db, _ := openTestDB(t) + require.NoError(t, db.Update(context.Background(), func(tx *WriteTx) error { + return tx.PutDeleteIntent("container-1", DeleteIntent{ + CreatedAt: testNow.Add(-testDeleteIntentTTL), + }) + })) + before := requireValidSnapshot(t, db) + callbacks := 0 + count, err := db.PruneDeleteIntentsIfGeneration( + context.Background(), + before.Metadata.Generation, + testNow, + testDeleteIntentTTL, + func(candidate Snapshot) error { + callbacks++ + assert.Empty(t, candidate.DeleteIntents) + return nil + }, + ) + require.NoError(t, err) + assert.Equal(t, 1, count) + after := requireValidSnapshot(t, db) + count, err = db.PruneDeleteIntentsIfGeneration( + context.Background(), + after.Metadata.Generation, + testNow, + testDeleteIntentTTL, + func(candidate Snapshot) error { + callbacks++ + assert.Equal(t, after.Metadata.Generation, candidate.Metadata.Generation) + return nil + }, + ) + require.NoError(t, err) + assert.Zero(t, count) + assert.Equal(t, 2, callbacks) + }) +}