From 6f05d1704cc3017b9d0b5745b8f81d0428505ee8 Mon Sep 17 00:00:00 2001 From: Evan Baker Date: Fri, 24 Jul 2026 12:57:25 +0000 Subject: [PATCH 1/6] test: add Bolt ownership handoff release gate --- .../capture-transition.steps.yaml | 85 ++- .../cni/state-migration/handoff.jobs.yaml | 347 ++++++++++ .../install-components.steps.yaml | 37 +- .../cni/state-migration/lane.stage.yaml | 132 +++- .pipelines/cni/state-migration/pipeline.yaml | 72 +- .../cni/state-migration/transition.steps.yaml | 633 ++++++++++++++++-- .../compare-state-migration-summaries.sh | 317 ++++++--- test/integration/load/load_test.go | 11 + test/integration/state/template_test.go | 610 +++++++++++++++-- test/validate/summary_capture_test.go | 55 ++ test/validate/validate.go | 102 ++- 11 files changed, 2112 insertions(+), 289 deletions(-) create mode 100644 .pipelines/cni/state-migration/handoff.jobs.yaml create mode 100644 test/validate/summary_capture_test.go diff --git a/.pipelines/cni/state-migration/capture-transition.steps.yaml b/.pipelines/cni/state-migration/capture-transition.steps.yaml index 934fbdb5a8..5bc8ae1376 100644 --- a/.pipelines/cni/state-migration/capture-transition.steps.yaml +++ b/.pipelines/cni/state-migration/capture-transition.steps.yaml @@ -23,8 +23,7 @@ parameters: steps: - ${{ if eq(parameters.expectedBackend, 'bolt') }}: - task: AzureCLI@2 - displayName: Capture strict persistent debug responses - condition: always() + displayName: Validate strict persistent state metadata inputs: azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION) scriptLocation: inlineScript @@ -44,40 +43,64 @@ steps: fi found=false + declare -A seenNodes=() while IFS=$'\t' read -r pod node; do [[ -n "$pod" && -n "$node" ]] || continue + if [[ -n "${seenNodes[$node]:-}" ]]; then + echo "multiple CNS pods reported for node $node" + exit 1 + fi + seenNodes[$node]=1 found=true - response="$evidenceDir/persistent-debug/${pod}.json" + response="$evidenceDir/persistent-debug/${node}.json" - if [[ "${{ parameters.os }}" == "windows" ]]; then - kubectl exec -n kube-system "$pod" -c cns-container -- \ - powershell -NoProfile -Command \ - 'Invoke-WebRequest -Uri 127.0.0.1:10090/debug/persistentstate -Method Post -UseBasicParsing -ErrorAction Stop | Select-Object -Expand Content' \ - >"$response" - else - kubectl exec -n kube-system "$pod" -c debug -- \ - bash -c "curl -sf localhost:10090/debug/persistentstate -d '{}'" \ - >"$response" + captured=false + for attempt in 1 2 3; do + if [[ "${{ parameters.os }}" == "windows" ]]; then + if kubectl exec -n kube-system "$pod" -c cns-container -- \ + powershell -NoProfile -Command \ + '$status = (Invoke-WebRequest -Uri 127.0.0.1:10090/debug/persistent-state/status -UseBasicParsing -ErrorAction Stop).Content | ConvertFrom-Json; $snapshot = (Invoke-WebRequest -Uri 127.0.0.1:10090/debug/persistent-state/snapshot -UseBasicParsing -ErrorAction Stop).Content | ConvertFrom-Json; [pscustomobject]@{status=$status;snapshot=$snapshot} | ConvertTo-Json -Depth 100 -Compress' \ + >"$response"; then + captured=true + fi + elif kubectl exec -n kube-system "$pod" -c debug -- \ + bash -c 'jq -n --argjson status "$(curl -sf localhost:10090/debug/persistent-state/status)" --argjson snapshot "$(curl -sf localhost:10090/debug/persistent-state/snapshot)" "{status:\$status,snapshot:\$snapshot}"' \ + >"$response"; then + captured=true + fi + [[ "$captured" == "true" ]] && break + sleep $((attempt * 5)) + done + if [[ "$captured" != "true" ]]; then + echo "persistent state transport failed after three attempts for $node" + exit 1 fi + jq --arg node "$node" '. + {nodeName: $node}' "$response" >"$response.next" + mv "$response.next" "$response" jq -e \ - --arg backend "${{ parameters.expectedBackend }}" \ --arg authority "${{ parameters.expectedAuthority }}" \ --argjson schema "${{ parameters.expectedSchemaVersion }}" \ - '.storage.backend == $backend - and .storage.filePresent == true - and .storage.fileSizeBytes > 0 - and .snapshot.metadata.authority == $authority - and .snapshot.metadata.schemaVersion == $schema - and ((.snapshot.metadata.bootID // "") | length) > 0' \ + '.status.backend == "bbolt" + and .status.authority == $authority + and .status.schemaVersion == $schema + and .status.generation > 0 + and .status.bootPresent == true + and .status.storagePresent == true + and .status.databaseBytes > 0 + and .status.invariantStatus == "healthy" + and .snapshot.Metadata.authority == $authority + and .snapshot.Metadata.schemaVersion == $schema + and .snapshot.Metadata.generation == .status.generation + and ((.snapshot.Metadata.bootID // "") | length) > 0' \ "$response" nodeBootID=$(kubectl get node "$node" -o jsonpath='{.status.nodeInfo.bootID}') - stateBootID=$(jq -r '.snapshot.metadata.bootID' "$response") + stateBootID=$(jq -r '.snapshot.Metadata.bootID' "$response") normalizedNodeBootID=$(tr '[:upper:]' '[:lower:]' <<<"$nodeBootID" | tr -d '{}[:space:]') normalizedStateBootID=$(tr '[:upper:]' '[:lower:]' <<<"$stateBootID" | tr -d '{}[:space:]') if [[ "$normalizedNodeBootID" != "$normalizedStateBootID" ]]; then - echo "persistent boot ID does not match Kubernetes node boot ID for $node" + echo "persistent boot ID $stateBootID does not match Kubernetes node boot ID $nodeBootID for $node" exit 1 fi done < <( @@ -89,10 +112,22 @@ steps: echo "no CNS pods matched selector $selector" exit 1 fi + jq -s \ + --arg backend "${{ parameters.expectedBackend }}" \ + --arg authority "${{ parameters.expectedAuthority }}" \ + --argjson schema "${{ parameters.expectedSchemaVersion }}" \ + '{ + expectedBackend: $backend, + expectedAuthority: $authority, + expectedSchemaVersion: $schema, + nodes: sort_by(.nodeName) + }' \ + "$evidenceDir"/persistent-debug/*.json >"$evidenceDir/persistent-summary.json" - task: AzureCLI@2 displayName: Capture transition state and logs condition: always() + continueOnError: true inputs: azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION) scriptLocation: inlineScript @@ -156,11 +191,5 @@ steps: done if [[ "$captureFailed" == "true" ]]; then - echo "one or more required transition evidence captures failed" - exit 1 + echo "##vso[task.logissue type=warning]one or more best-effort transition evidence captures failed" fi - - - publish: $(Build.SourcesDirectory)/test/integration/logs/state-migration/${{ parameters.lane }}/${{ parameters.transition }} - artifact: state-migration-${{ parameters.lane }}-${{ parameters.transition }} - displayName: Publish transition evidence - condition: always() diff --git a/.pipelines/cni/state-migration/handoff.jobs.yaml b/.pipelines/cni/state-migration/handoff.jobs.yaml new file mode 100644 index 0000000000..dfca422c94 --- /dev/null +++ b/.pipelines/cni/state-migration/handoff.jobs.yaml @@ -0,0 +1,347 @@ +parameters: + - name: lane + type: string + - name: clusterName + type: string + - name: os + type: string + - name: targetNodeCount + type: number + - name: replicasPerNode + type: number + - name: configMapName + type: string + - name: daemonsetName + type: string + - name: expectedAuthority + type: string + - name: expectedSchemaVersion + type: number + - name: transitions + type: object + default: + - job: handoff_cni_json + displayName: 11 Reset stateful CNI-managed JSON + dependsOn: final_same_boot_restart + baselineTransition: 10-final-same-boot-restart + transition: 11-handoff-cni-json + action: rollback-json + backend: json + cni: cniv2 + manageEndpointState: "false" + initializeFromCNI: "true" + enableStateMigration: "false" + stateRelation: exact + bootRelation: none + podRelation: exact + externalFault: false + timeout: 90 + # CNS must import while the stateful CNI executable can still return its + # endpoint store. The following rollout replaces it with stateless CNI + # only after strict CNS endpoint-state validation succeeds. + - job: handoff_ownership_first_import + displayName: 12 Import stateful CNI JSON into CNS JSON + dependsOn: handoff_cni_json + baselineTransition: 11-handoff-cni-json + transition: 12-handoff-cns-json-import + action: configure-state-import + backend: json + cni: cniv2 + manageEndpointState: "true" + initializeFromCNI: "false" + enableStateMigration: "true" + stateRelation: exact + bootRelation: none + podRelation: exact + externalFault: false + timeout: 90 + - job: handoff_ownership_first_stateless + displayName: 13 Activate stateless CNI with CNS-managed JSON + dependsOn: handoff_ownership_first_import + baselineTransition: 12-handoff-cns-json-import + transition: 13-handoff-cns-json-stateless + action: configure-state + backend: json + cni: stateless + manageEndpointState: "true" + initializeFromCNI: "false" + enableStateMigration: "false" + stateRelation: none + bootRelation: none + podRelation: exact + externalFault: false + timeout: 90 + - job: handoff_ownership_first_bolt + displayName: 14 Ownership-first CNS-managed Bolt + dependsOn: handoff_ownership_first_stateless + baselineTransition: 13-handoff-cns-json-stateless + transition: 14-handoff-cns-bolt + action: configure-state + backend: bolt + cni: stateless + manageEndpointState: "true" + initializeFromCNI: "false" + enableStateMigration: "false" + stateRelation: exact + bootRelation: none + podRelation: exact + externalFault: false + timeout: 90 + # CNS has no CNI export. Reverse only before pod churn while the original + # stateful CNI store still describes these exact live endpoints. Restore + # that binary in one rollout before disabling CNS ownership in the next. + - job: handoff_ownership_first_stateful + displayName: 15 Restore stateful CNI binary while CNS remains owner + dependsOn: handoff_ownership_first_bolt + baselineTransition: 14-handoff-cns-bolt + transition: 15-handoff-cns-bolt-stateful + action: configure-state + backend: bolt + cni: cniv2 + manageEndpointState: "true" + initializeFromCNI: "false" + enableStateMigration: "false" + stateRelation: none + bootRelation: same + podRelation: exact + externalFault: false + timeout: 90 + - job: handoff_ownership_first_reverse + displayName: 16 Reverse ownership to stateful CNI before pod churn + dependsOn: handoff_ownership_first_stateful + baselineTransition: 15-handoff-cns-bolt-stateful + transition: 16-handoff-cni-json-reverse + action: rollback-json-and-clear-endpoints + backend: json + cni: cniv2 + manageEndpointState: "false" + initializeFromCNI: "true" + enableStateMigration: "false" + stateRelation: exact + bootRelation: same + podRelation: exact + externalFault: false + timeout: 90 + - job: handoff_backend_first_json + displayName: 17 Reset stateful CNI-managed JSON + dependsOn: handoff_ownership_first_reverse + baselineTransition: 16-handoff-cni-json-reverse + transition: 17-handoff-cni-json + action: rollback-json + backend: json + cni: cniv2 + manageEndpointState: "false" + initializeFromCNI: "true" + enableStateMigration: "false" + stateRelation: exact + bootRelation: none + podRelation: exact + externalFault: false + timeout: 90 + - job: handoff_backend_first_bolt + displayName: 18 Import into CNS Bolt while stateful CNI remains installed + dependsOn: handoff_backend_first_json + baselineTransition: 17-handoff-cni-json + transition: 18-handoff-cns-bolt-import + action: configure-state-import + backend: bolt + cni: cniv2 + manageEndpointState: "true" + initializeFromCNI: "true" + enableStateMigration: "true" + stateRelation: exact + bootRelation: none + podRelation: exact + externalFault: false + timeout: 90 + - job: handoff_backend_first_import + displayName: 19 Confirm CNS-owned Bolt before CNI replacement + dependsOn: handoff_backend_first_bolt + baselineTransition: 18-handoff-cns-bolt-import + transition: 19-handoff-cns-bolt-import + action: configure-state + backend: bolt + cni: cniv2 + manageEndpointState: "true" + initializeFromCNI: "false" + enableStateMigration: "false" + stateRelation: exact + bootRelation: none + podRelation: exact + externalFault: false + timeout: 90 + - job: handoff_backend_first_stateless + displayName: 20 Activate stateless CNI with CNS-managed Bolt + dependsOn: handoff_backend_first_import + baselineTransition: 19-handoff-cns-bolt-import + transition: 20-handoff-cns-bolt-stateless + action: configure-state + backend: bolt + cni: stateless + manageEndpointState: "true" + initializeFromCNI: "false" + enableStateMigration: "false" + stateRelation: none + bootRelation: none + podRelation: exact + externalFault: false + timeout: 90 + - job: handoff_backend_first_stateful + displayName: 21 Restore stateful CNI binary while CNS remains owner + dependsOn: handoff_backend_first_stateless + baselineTransition: 20-handoff-cns-bolt-stateless + transition: 21-handoff-cns-bolt-stateful + action: configure-state + backend: bolt + cni: cniv2 + manageEndpointState: "true" + initializeFromCNI: "false" + enableStateMigration: "false" + stateRelation: none + bootRelation: same + podRelation: exact + externalFault: false + timeout: 90 + - job: handoff_backend_first_reverse + displayName: 22 Reverse ownership to stateful CNI before pod churn + dependsOn: handoff_backend_first_stateful + baselineTransition: 21-handoff-cns-bolt-stateful + transition: 22-handoff-cni-json-reverse + action: rollback-json-and-clear-endpoints + backend: json + cni: cniv2 + manageEndpointState: "false" + initializeFromCNI: "true" + enableStateMigration: "false" + stateRelation: exact + bootRelation: same + podRelation: exact + externalFault: false + timeout: 90 + - job: handoff_direct_json + displayName: 23 Reset stateful CNI-managed JSON + dependsOn: handoff_backend_first_reverse + baselineTransition: 22-handoff-cni-json-reverse + transition: 23-handoff-cni-json + action: rollback-json + backend: json + cni: cniv2 + manageEndpointState: "false" + initializeFromCNI: "true" + enableStateMigration: "false" + stateRelation: exact + bootRelation: none + podRelation: exact + externalFault: false + timeout: 90 + - job: handoff_direct_import + displayName: 24 Direct stateful CNI JSON to CNS Bolt import + dependsOn: handoff_direct_json + baselineTransition: 23-handoff-cni-json + transition: 24-handoff-direct-cns-bolt-import + action: configure-state-import + backend: bolt + cni: cniv2 + manageEndpointState: "true" + initializeFromCNI: "true" + enableStateMigration: "true" + stateRelation: exact + bootRelation: none + podRelation: exact + externalFault: false + timeout: 90 + - job: handoff_direct_stateless + displayName: 25 Activate stateless CNI with CNS-managed Bolt + dependsOn: handoff_direct_import + baselineTransition: 24-handoff-direct-cns-bolt-import + transition: 25-handoff-direct-cns-bolt-stateless + action: configure-state + backend: bolt + cni: stateless + manageEndpointState: "true" + initializeFromCNI: "false" + enableStateMigration: "false" + stateRelation: none + bootRelation: none + podRelation: exact + externalFault: false + timeout: 90 + - job: handoff_faults + displayName: 26 CNS-managed Bolt fault coverage + dependsOn: handoff_direct_stateless + baselineTransition: 25-handoff-direct-cns-bolt-stateless + transition: 26-handoff-faults + action: external-fault + backend: bolt + cni: stateless + manageEndpointState: "true" + initializeFromCNI: "false" + enableStateMigration: "false" + stateRelation: exact + bootRelation: same + podRelation: exact + externalFault: true + timeout: 120 + - job: handoff_restart + displayName: 27 CNS-managed Bolt component restart + dependsOn: handoff_faults + baselineTransition: 26-handoff-faults + transition: 27-handoff-restart + action: same-boot-restart + backend: bolt + cni: stateless + manageEndpointState: "true" + initializeFromCNI: "false" + enableStateMigration: "false" + stateRelation: exact + bootRelation: same + podRelation: exact + externalFault: false + timeout: 90 + - job: handoff_reboot + displayName: 28 CNS-managed Bolt node reboot + dependsOn: handoff_restart + baselineTransition: 27-handoff-restart + transition: 28-handoff-reboot + action: node-reboot + backend: bolt + cni: stateless + manageEndpointState: "true" + initializeFromCNI: "false" + enableStateMigration: "false" + stateRelation: none + bootRelation: changed + podRelation: identity + externalFault: false + timeout: 120 + +jobs: + - ${{ each transition in parameters.transitions }}: + - job: ${{ transition.job }} + displayName: ${{ transition.displayName }} + dependsOn: ${{ transition.dependsOn }} + timeoutInMinutes: ${{ transition.timeout }} + steps: + - template: transition.steps.yaml + parameters: + lane: ${{ parameters.lane }} + transition: ${{ transition.transition }} + action: ${{ transition.action }} + clusterName: ${{ parameters.clusterName }} + os: ${{ parameters.os }} + cni: ${{ transition.cni }} + targetNodeCount: ${{ parameters.targetNodeCount }} + replicasPerNode: ${{ parameters.replicasPerNode }} + configMapName: ${{ parameters.configMapName }} + daemonsetName: ${{ parameters.daemonsetName }} + expectedBackend: ${{ transition.backend }} + expectedAuthority: ${{ parameters.expectedAuthority }} + expectedSchemaVersion: ${{ parameters.expectedSchemaVersion }} + expectedManageEndpointState: ${{ transition.manageEndpointState }} + expectedInitializeFromCNI: ${{ transition.initializeFromCNI }} + expectedEnableStateMigration: ${{ transition.enableStateMigration }} + baselineArtifact: state-migration-$(Build.BuildId)-${{ parameters.lane }}-${{ transition.baselineTransition }} + stateRelation: ${{ transition.stateRelation }} + bootRelation: ${{ transition.bootRelation }} + podRelation: ${{ transition.podRelation }} + injectExternalFault: ${{ transition.externalFault }} diff --git a/.pipelines/cni/state-migration/install-components.steps.yaml b/.pipelines/cni/state-migration/install-components.steps.yaml index 119fc02ce7..61370d4cff 100644 --- a/.pipelines/cni/state-migration/install-components.steps.yaml +++ b/.pipelines/cni/state-migration/install-components.steps.yaml @@ -15,8 +15,13 @@ parameters: type: string - name: initializeFromCNI type: string + - name: enableStateMigration + type: string steps: + - ${{ if eq(parameters.scenario, 'cilium-overlay') }}: + - template: ../../templates/cilium-cli.yaml + - task: AzureCLI@2 displayName: Install migration lane components inputs: @@ -36,7 +41,7 @@ steps: case "${{ parameters.scenario }}" in cilium-overlay) - make -C ./hack/aks deploy-cilium + make -C ./hack/aks deploy-cilium DIR=$(CILIUM_TEMPLATE_DIR) sudo -E env "PATH=$PATH" make test-integration \ AZURE_IPAM_VERSION="$ipamVersion" \ CNS_VERSION="$cnsVersion" \ @@ -99,15 +104,39 @@ steps: done < <(kubectl get nodes -l kubernetes.io/os=windows -o name) fi + config=$(kubectl get configmap/${{ parameters.configMapName }} \ + -n kube-system -o jsonpath='{.data.cns_config\.json}') + config=$(jq \ + --argjson manage "${{ parameters.manageEndpointState }}" \ + --argjson initialize "${{ parameters.initializeFromCNI }}" \ + --argjson migrate "${{ parameters.enableStateMigration }}" \ + '.EnableBoltStateStore = false + | .EnablePersistentStateDebug = false + | .EnablePersistentStateFaults = false + | .StateStoreBackend = "json" + | .StateStoreMode = "normal" + | .ManageEndpointState = $manage + | .InitializeFromCNI = $initialize + | .EnableStateMigration = $migrate' \ + <<<"$config") + payload=$(jq -n --arg config "$config" '{data: {"cns_config.json": $config}}') + kubectl patch configmap/${{ parameters.configMapName }} -n kube-system --type merge -p "$payload" + kubectl rollout restart daemonset/${{ parameters.daemonsetName }} -n kube-system kubectl rollout status daemonset/${{ parameters.daemonsetName }} \ -n kube-system --timeout=20m - config=$(kubectl get configmap/${{ parameters.configMapName }} \ - -n kube-system -o jsonpath='{.data.cns_config\.json}') jq -e \ --argjson manage "${{ parameters.manageEndpointState }}" \ --argjson initialize "${{ parameters.initializeFromCNI }}" \ - '.ManageEndpointState == $manage and .InitializeFromCNI == $initialize' \ + --argjson migrate "${{ parameters.enableStateMigration }}" \ + '.EnableBoltStateStore == false + and .EnablePersistentStateDebug == false + and .EnablePersistentStateFaults == false + and .StateStoreBackend == "json" + and .StateStoreMode == "normal" + and .ManageEndpointState == $manage + and .InitializeFromCNI == $initialize + and .EnableStateMigration == $migrate' \ <<<"$config" kubectl get pods -A -o wide diff --git a/.pipelines/cni/state-migration/lane.stage.yaml b/.pipelines/cni/state-migration/lane.stage.yaml index 840310e2e9..d12cb1738e 100644 --- a/.pipelines/cni/state-migration/lane.stage.yaml +++ b/.pipelines/cni/state-migration/lane.stage.yaml @@ -15,6 +15,8 @@ parameters: type: string - name: region type: string + - name: kubernetesVersion + type: string - name: nodeCount type: number - name: nodeCountWin @@ -35,14 +37,23 @@ parameters: type: string - name: daemonsetName type: string - - name: manageEndpointState + - name: jsonManageEndpointState type: string - name: initializeFromCNI type: string + - name: enableStateMigration + type: string + - name: boltInitializeFromCNI + type: string + - name: boltEnableStateMigration + type: string - name: expectedAuthority type: string - name: expectedSchemaVersion type: number + - name: enableOwnershipHandoff + type: boolean + default: false stages: - stage: ${{ parameters.name }} @@ -51,7 +62,8 @@ stages: - setup - publish_migration_images variables: - commitID: $[ stageDependencies.setup.env.outputs['SetEnvVars.commitID'] ] + buildID: $[ stageDependencies.setup.env.outputs['SetEnvVars.buildID'] ] + KUBECONFIG: "$(Agent.TempDirectory)/kubeconfig-$(Build.BuildId)-$(System.JobId)" pool: name: $(BUILD_POOL_NAME_DEFAULT) jobs: @@ -68,7 +80,7 @@ stages: inlineScript: | set -euo pipefail - cluster="${{ parameters.clusterBaseName }}-$(commitID)" + cluster="${{ parameters.clusterBaseName }}-$(buildID)" make -C ./hack/aks azcfg AZCLI=az REGION=${{ parameters.region }} if az aks show --resource-group "$cluster" --name "$cluster" >/dev/null 2>&1; then @@ -82,6 +94,7 @@ stages: makeArgs=( "AZCLI=az" "REGION=${{ parameters.region }}" + "K8S_VER=${{ parameters.kubernetesVersion }}" "SUB=$(SUB_AZURE_NETWORK_AGENT_BUILD_VALIDATIONS)" "CLUSTER=$cluster" "NODE_COUNT=${{ parameters.nodeCount }}" @@ -108,14 +121,15 @@ stages: steps: - template: install-components.steps.yaml parameters: - clusterName: ${{ parameters.clusterBaseName }}-$(commitID) + clusterName: ${{ parameters.clusterBaseName }}-$(buildID) scenario: ${{ parameters.scenario }} os: ${{ parameters.os }} region: ${{ parameters.region }} configMapName: ${{ parameters.configMapName }} daemonsetName: ${{ parameters.daemonsetName }} - manageEndpointState: ${{ parameters.manageEndpointState }} + manageEndpointState: ${{ parameters.jsonManageEndpointState }} initializeFromCNI: ${{ parameters.initializeFromCNI }} + enableStateMigration: ${{ parameters.enableStateMigration }} - job: json_baseline displayName: 01 JSON baseline @@ -127,7 +141,7 @@ stages: lane: ${{ parameters.name }} transition: 01-json-baseline action: json-baseline - clusterName: ${{ parameters.clusterBaseName }}-$(commitID) + clusterName: ${{ parameters.clusterBaseName }}-$(buildID) os: ${{ parameters.os }} cni: ${{ parameters.cni }} targetNodeCount: ${{ parameters.targetNodeCount }} @@ -137,6 +151,9 @@ stages: expectedBackend: json expectedAuthority: ${{ parameters.expectedAuthority }} expectedSchemaVersion: ${{ parameters.expectedSchemaVersion }} + expectedManageEndpointState: ${{ parameters.jsonManageEndpointState }} + expectedInitializeFromCNI: ${{ parameters.initializeFromCNI }} + expectedEnableStateMigration: ${{ parameters.enableStateMigration }} - job: bolt_migrate displayName: 02 Single-restart Bolt migration @@ -148,7 +165,7 @@ stages: lane: ${{ parameters.name }} transition: 02-bolt-migrate action: bolt-migrate - clusterName: ${{ parameters.clusterBaseName }}-$(commitID) + clusterName: ${{ parameters.clusterBaseName }}-$(buildID) os: ${{ parameters.os }} cni: ${{ parameters.cni }} targetNodeCount: ${{ parameters.targetNodeCount }} @@ -158,7 +175,10 @@ stages: expectedBackend: bolt expectedAuthority: ${{ parameters.expectedAuthority }} expectedSchemaVersion: ${{ parameters.expectedSchemaVersion }} - baselineArtifact: state-migration-${{ parameters.name }}-01-json-baseline + expectedManageEndpointState: "true" + expectedInitializeFromCNI: ${{ parameters.boltInitializeFromCNI }} + expectedEnableStateMigration: ${{ parameters.boltEnableStateMigration }} + baselineArtifact: state-migration-$(Build.BuildId)-${{ parameters.name }}-01-json-baseline stateRelation: exact - job: same_boot_restart @@ -171,7 +191,7 @@ stages: lane: ${{ parameters.name }} transition: 03-same-boot-restart action: same-boot-restart - clusterName: ${{ parameters.clusterBaseName }}-$(commitID) + clusterName: ${{ parameters.clusterBaseName }}-$(buildID) os: ${{ parameters.os }} cni: ${{ parameters.cni }} targetNodeCount: ${{ parameters.targetNodeCount }} @@ -181,21 +201,24 @@ stages: expectedBackend: bolt expectedAuthority: ${{ parameters.expectedAuthority }} expectedSchemaVersion: ${{ parameters.expectedSchemaVersion }} - baselineArtifact: state-migration-${{ parameters.name }}-02-bolt-migrate + expectedManageEndpointState: "true" + expectedInitializeFromCNI: ${{ parameters.boltInitializeFromCNI }} + expectedEnableStateMigration: ${{ parameters.boltEnableStateMigration }} + baselineArtifact: state-migration-$(Build.BuildId)-${{ parameters.name }}-02-bolt-migrate stateRelation: exact bootRelation: same - job: restart_during_scale displayName: 04 Restart during scale dependsOn: same_boot_restart - timeoutInMinutes: 180 + timeoutInMinutes: 90 steps: - template: transition.steps.yaml parameters: lane: ${{ parameters.name }} transition: 04-restart-during-scale action: restart-during-scale - clusterName: ${{ parameters.clusterBaseName }}-$(commitID) + clusterName: ${{ parameters.clusterBaseName }}-$(buildID) os: ${{ parameters.os }} cni: ${{ parameters.cni }} targetNodeCount: ${{ parameters.targetNodeCount }} @@ -205,10 +228,12 @@ stages: expectedBackend: bolt expectedAuthority: ${{ parameters.expectedAuthority }} expectedSchemaVersion: ${{ parameters.expectedSchemaVersion }} - baselineArtifact: state-migration-${{ parameters.name }}-03-same-boot-restart + expectedManageEndpointState: "true" + expectedInitializeFromCNI: ${{ parameters.boltInitializeFromCNI }} + expectedEnableStateMigration: ${{ parameters.boltEnableStateMigration }} + baselineArtifact: state-migration-$(Build.BuildId)-${{ parameters.name }}-03-same-boot-restart stateRelation: changed bootRelation: same - runFaultInjection: ${{ eq(parameters.manageEndpointState, 'true') }} - job: node_reboot displayName: 05 Node reboot @@ -220,7 +245,7 @@ stages: lane: ${{ parameters.name }} transition: 05-node-reboot action: node-reboot - clusterName: ${{ parameters.clusterBaseName }}-$(commitID) + clusterName: ${{ parameters.clusterBaseName }}-$(buildID) os: ${{ parameters.os }} cni: ${{ parameters.cni }} targetNodeCount: ${{ parameters.targetNodeCount }} @@ -230,9 +255,13 @@ stages: expectedBackend: bolt expectedAuthority: ${{ parameters.expectedAuthority }} expectedSchemaVersion: ${{ parameters.expectedSchemaVersion }} - baselineArtifact: state-migration-${{ parameters.name }}-04-restart-during-scale - stateRelation: exact + expectedManageEndpointState: "true" + expectedInitializeFromCNI: ${{ parameters.boltInitializeFromCNI }} + expectedEnableStateMigration: ${{ parameters.boltEnableStateMigration }} + baselineArtifact: state-migration-$(Build.BuildId)-${{ parameters.name }}-04-restart-during-scale + stateRelation: none bootRelation: changed + podRelation: identity - job: os_validation displayName: 06 OS-specific validation @@ -244,7 +273,7 @@ stages: lane: ${{ parameters.name }} transition: 06-os-validation action: os-validation - clusterName: ${{ parameters.clusterBaseName }}-$(commitID) + clusterName: ${{ parameters.clusterBaseName }}-$(buildID) os: ${{ parameters.os }} cni: ${{ parameters.cni }} targetNodeCount: ${{ parameters.targetNodeCount }} @@ -254,7 +283,10 @@ stages: expectedBackend: bolt expectedAuthority: ${{ parameters.expectedAuthority }} expectedSchemaVersion: ${{ parameters.expectedSchemaVersion }} - baselineArtifact: state-migration-${{ parameters.name }}-05-node-reboot + expectedManageEndpointState: "true" + expectedInitializeFromCNI: ${{ parameters.boltInitializeFromCNI }} + expectedEnableStateMigration: ${{ parameters.boltEnableStateMigration }} + baselineArtifact: state-migration-$(Build.BuildId)-${{ parameters.name }}-05-node-reboot stateRelation: exact bootRelation: same @@ -268,7 +300,7 @@ stages: lane: ${{ parameters.name }} transition: 07-rollback-json action: rollback-json - clusterName: ${{ parameters.clusterBaseName }}-$(commitID) + clusterName: ${{ parameters.clusterBaseName }}-$(buildID) os: ${{ parameters.os }} cni: ${{ parameters.cni }} targetNodeCount: ${{ parameters.targetNodeCount }} @@ -278,7 +310,10 @@ stages: expectedBackend: json expectedAuthority: ${{ parameters.expectedAuthority }} expectedSchemaVersion: ${{ parameters.expectedSchemaVersion }} - baselineArtifact: state-migration-${{ parameters.name }}-06-os-validation + expectedManageEndpointState: ${{ parameters.jsonManageEndpointState }} + expectedInitializeFromCNI: ${{ parameters.initializeFromCNI }} + expectedEnableStateMigration: ${{ parameters.enableStateMigration }} + baselineArtifact: state-migration-$(Build.BuildId)-${{ parameters.name }}-06-os-validation stateRelation: exact - job: json_mutation @@ -291,7 +326,7 @@ stages: lane: ${{ parameters.name }} transition: 08-json-mutation action: json-mutation - clusterName: ${{ parameters.clusterBaseName }}-$(commitID) + clusterName: ${{ parameters.clusterBaseName }}-$(buildID) os: ${{ parameters.os }} cni: ${{ parameters.cni }} targetNodeCount: ${{ parameters.targetNodeCount }} @@ -301,7 +336,10 @@ stages: expectedBackend: json expectedAuthority: ${{ parameters.expectedAuthority }} expectedSchemaVersion: ${{ parameters.expectedSchemaVersion }} - baselineArtifact: state-migration-${{ parameters.name }}-07-rollback-json + expectedManageEndpointState: ${{ parameters.jsonManageEndpointState }} + expectedInitializeFromCNI: ${{ parameters.initializeFromCNI }} + expectedEnableStateMigration: ${{ parameters.enableStateMigration }} + baselineArtifact: state-migration-$(Build.BuildId)-${{ parameters.name }}-07-rollback-json stateRelation: changed - job: bolt_reimport @@ -314,7 +352,7 @@ stages: lane: ${{ parameters.name }} transition: 09-bolt-reimport action: bolt-reimport - clusterName: ${{ parameters.clusterBaseName }}-$(commitID) + clusterName: ${{ parameters.clusterBaseName }}-$(buildID) os: ${{ parameters.os }} cni: ${{ parameters.cni }} targetNodeCount: ${{ parameters.targetNodeCount }} @@ -324,7 +362,10 @@ stages: expectedBackend: bolt expectedAuthority: ${{ parameters.expectedAuthority }} expectedSchemaVersion: ${{ parameters.expectedSchemaVersion }} - baselineArtifact: state-migration-${{ parameters.name }}-08-json-mutation + expectedManageEndpointState: "true" + expectedInitializeFromCNI: ${{ parameters.boltInitializeFromCNI }} + expectedEnableStateMigration: ${{ parameters.boltEnableStateMigration }} + baselineArtifact: state-migration-$(Build.BuildId)-${{ parameters.name }}-08-json-mutation stateRelation: exact - job: final_same_boot_restart @@ -337,7 +378,7 @@ stages: lane: ${{ parameters.name }} transition: 10-final-same-boot-restart action: final-same-boot-restart - clusterName: ${{ parameters.clusterBaseName }}-$(commitID) + clusterName: ${{ parameters.clusterBaseName }}-$(buildID) os: ${{ parameters.os }} cni: ${{ parameters.cni }} targetNodeCount: ${{ parameters.targetNodeCount }} @@ -347,10 +388,26 @@ stages: expectedBackend: bolt expectedAuthority: ${{ parameters.expectedAuthority }} expectedSchemaVersion: ${{ parameters.expectedSchemaVersion }} - baselineArtifact: state-migration-${{ parameters.name }}-09-bolt-reimport + expectedManageEndpointState: "true" + expectedInitializeFromCNI: ${{ parameters.boltInitializeFromCNI }} + expectedEnableStateMigration: ${{ parameters.boltEnableStateMigration }} + baselineArtifact: state-migration-$(Build.BuildId)-${{ parameters.name }}-09-bolt-reimport stateRelation: exact bootRelation: same + - ${{ if eq(parameters.enableOwnershipHandoff, true) }}: + - template: handoff.jobs.yaml + parameters: + lane: ${{ parameters.name }} + clusterName: ${{ parameters.clusterBaseName }}-$(buildID) + os: ${{ parameters.os }} + targetNodeCount: ${{ parameters.targetNodeCount }} + replicasPerNode: ${{ parameters.replicasPerNode }} + configMapName: ${{ parameters.configMapName }} + daemonsetName: ${{ parameters.daemonsetName }} + expectedAuthority: ${{ parameters.expectedAuthority }} + expectedSchemaVersion: ${{ parameters.expectedSchemaVersion }} + - job: cleanup displayName: Clean up migration cluster condition: always() @@ -367,10 +424,29 @@ stages: - json_mutation - bolt_reimport - final_same_boot_restart + - ${{ if eq(parameters.enableOwnershipHandoff, true) }}: + - handoff_cni_json + - handoff_ownership_first_import + - handoff_ownership_first_stateless + - handoff_ownership_first_bolt + - handoff_ownership_first_stateful + - handoff_ownership_first_reverse + - handoff_backend_first_json + - handoff_backend_first_bolt + - handoff_backend_first_import + - handoff_backend_first_stateless + - handoff_backend_first_stateful + - handoff_backend_first_reverse + - handoff_direct_json + - handoff_direct_import + - handoff_direct_stateless + - handoff_faults + - handoff_restart + - handoff_reboot steps: - template: ../../templates/delete-cluster.yaml parameters: - clusterName: ${{ parameters.clusterBaseName }}-$(commitID) + clusterName: ${{ parameters.clusterBaseName }}-$(buildID) sub: $(SUB_AZURE_NETWORK_AGENT_BUILD_VALIDATIONS) svcConn: $(BUILD_VALIDATIONS_SERVICE_CONNECTION) deleteResources: "true" diff --git a/.pipelines/cni/state-migration/pipeline.yaml b/.pipelines/cni/state-migration/pipeline.yaml index 1df7bb668d..63acc8acb7 100644 --- a/.pipelines/cni/state-migration/pipeline.yaml +++ b/.pipelines/cni/state-migration/pipeline.yaml @@ -1,10 +1,19 @@ pr: none -trigger: - tags: - include: - - "dropgz/*" - - "azure-ipam/*" - - "v*" +trigger: none + +schedules: + - cron: "0 6 * * 0" + displayName: Weekly ownership handoff validation + branches: + include: + - master + always: false + +variables: + ACN_VERSION: "r22-$(Build.BuildId)" + AZURE_IPAM_VERSION: "r22-$(Build.BuildId)" + CNI_VERSION: "r22-$(Build.BuildId)" + CNS_VERSION: "r22-$(Build.BuildId)" parameters: - name: linuxNodeCount @@ -31,10 +40,12 @@ parameters: - name: region type: string default: "$(LOCATION_AMD64)" + - name: kubernetesVersion + type: string + default: "1.34" - name: expectedSchemaVersion type: number default: 1 - stages: - stage: setup displayName: Set up migration release gate @@ -46,10 +57,10 @@ stages: - bash: | set -euo pipefail go version - commitID="$(make revision)-$(Build.BuildId)" - echo "##vso[task.setvariable variable=commitID;isOutput=true]$commitID" - echo "Migration gate commit ID: $commitID" - echo "Reusable fault injection runs in CNS-managed migration lanes." + buildID="$(Build.BuildId)" + echo "##vso[task.setvariable variable=buildID;isOutput=true]$buildID" + echo "Migration gate isolation ID: $buildID" + echo "Queued source commit: $(Build.SourceVersion)" name: SetEnvVars displayName: Set migration gate variables @@ -127,6 +138,7 @@ stages: os: linux cni: cilium region: ${{ parameters.region }} + kubernetesVersion: ${{ parameters.kubernetesVersion }} nodeCount: ${{ parameters.linuxNodeCount }} nodeCountWin: 0 vmSize: ${{ parameters.linuxVMSize }} @@ -137,21 +149,25 @@ stages: replicasPerNode: ${{ parameters.replicasPerNode }} configMapName: cns-config daemonsetName: azure-cns - manageEndpointState: "true" + jsonManageEndpointState: "true" initializeFromCNI: "false" + enableStateMigration: "false" + boltInitializeFromCNI: "false" + boltEnableStateMigration: "false" expectedAuthority: bolt expectedSchemaVersion: ${{ parameters.expectedSchemaVersion }} - template: lane.stage.yaml parameters: name: linux_podsubnet - displayName: "P0 Linux Azure CNI pod-subnet — CNI-managed" + displayName: "P0 Linux Azure CNI pod-subnet — CNI-controlled JSON, CNS-owned Bolt" clusterBaseName: mig-podsub-lnx clusterType: swift-byocni-up scenario: azure-cni-pod-subnet os: linux cni: cniv2 region: ${{ parameters.region }} + kubernetesVersion: ${{ parameters.kubernetesVersion }} nodeCount: ${{ parameters.linuxNodeCount }} nodeCountWin: 0 vmSize: ${{ parameters.linuxVMSize }} @@ -162,21 +178,25 @@ stages: replicasPerNode: ${{ parameters.replicasPerNode }} configMapName: cns-config daemonsetName: azure-cns - manageEndpointState: "false" + jsonManageEndpointState: "false" initializeFromCNI: "true" + enableStateMigration: "false" + boltInitializeFromCNI: "true" + boltEnableStateMigration: "true" expectedAuthority: bolt expectedSchemaVersion: ${{ parameters.expectedSchemaVersion }} - template: lane.stage.yaml parameters: name: linux_overlay - displayName: "P0 Linux Azure CNI overlay — CNI-managed" + displayName: "P0 Linux Azure CNI overlay — CNI-controlled JSON, CNS-owned Bolt" clusterBaseName: mig-overlay-lnx clusterType: overlay-byocni-up scenario: azure-cni-overlay os: linux cni: cniv2 region: ${{ parameters.region }} + kubernetesVersion: ${{ parameters.kubernetesVersion }} nodeCount: ${{ parameters.linuxNodeCount }} nodeCountWin: 0 vmSize: ${{ parameters.linuxVMSize }} @@ -187,21 +207,25 @@ stages: replicasPerNode: ${{ parameters.replicasPerNode }} configMapName: cns-config daemonsetName: azure-cns - manageEndpointState: "false" + jsonManageEndpointState: "false" initializeFromCNI: "true" + enableStateMigration: "false" + boltInitializeFromCNI: "true" + boltEnableStateMigration: "true" expectedAuthority: bolt expectedSchemaVersion: ${{ parameters.expectedSchemaVersion }} - template: lane.stage.yaml parameters: name: windows_podsubnet - displayName: "P0 Windows Azure CNI pod-subnet — CNI-managed" + displayName: "P0 Windows pod-subnet — CNI JSON source to CNS-owned Bolt handoff" clusterBaseName: mig-podsub-win clusterType: swift-byocni-up scenario: azure-cni-pod-subnet os: windows cni: cniv2 region: ${{ parameters.region }} + kubernetesVersion: ${{ parameters.kubernetesVersion }} nodeCount: ${{ parameters.windowsSystemNodeCount }} nodeCountWin: ${{ parameters.windowsNodeCount }} vmSize: ${{ parameters.windowsSystemVMSize }} @@ -212,10 +236,14 @@ stages: replicasPerNode: ${{ parameters.replicasPerNode }} configMapName: cns-win-config daemonsetName: azure-cns-win - manageEndpointState: "false" + jsonManageEndpointState: "false" initializeFromCNI: "true" + enableStateMigration: "false" + boltInitializeFromCNI: "true" + boltEnableStateMigration: "true" expectedAuthority: bolt expectedSchemaVersion: ${{ parameters.expectedSchemaVersion }} + enableOwnershipHandoff: true - template: lane.stage.yaml parameters: @@ -227,6 +255,7 @@ stages: os: windows cni: stateless region: ${{ parameters.region }} + kubernetesVersion: ${{ parameters.kubernetesVersion }} nodeCount: ${{ parameters.windowsSystemNodeCount }} nodeCountWin: ${{ parameters.windowsNodeCount }} vmSize: ${{ parameters.windowsSystemVMSize }} @@ -237,8 +266,11 @@ stages: replicasPerNode: ${{ parameters.replicasPerNode }} configMapName: cns-win-config daemonsetName: azure-cns-win - manageEndpointState: "true" + jsonManageEndpointState: "true" initializeFromCNI: "false" + enableStateMigration: "false" + boltInitializeFromCNI: "false" + boltEnableStateMigration: "false" expectedAuthority: bolt expectedSchemaVersion: ${{ parameters.expectedSchemaVersion }} @@ -257,5 +289,5 @@ stages: name: $(BUILD_POOL_NAME_DEFAULT) steps: - bash: | - echo "All five P0 state-migration lifecycle lanes passed." + echo "All five lifecycle lanes and all 18 ownership handoff transitions passed." displayName: Confirm release gate diff --git a/.pipelines/cni/state-migration/transition.steps.yaml b/.pipelines/cni/state-migration/transition.steps.yaml index 93b8407b4c..364437ee9e 100644 --- a/.pipelines/cni/state-migration/transition.steps.yaml +++ b/.pipelines/cni/state-migration/transition.steps.yaml @@ -25,6 +25,12 @@ parameters: type: string - name: expectedSchemaVersion type: number + - name: expectedManageEndpointState + type: string + - name: expectedInitializeFromCNI + type: string + - name: expectedEnableStateMigration + type: string - name: baselineArtifact type: string default: "" @@ -34,7 +40,10 @@ parameters: - name: bootRelation type: string default: none - - name: runFaultInjection + - name: podRelation + type: string + default: inherit + - name: injectExternalFault type: boolean default: false @@ -54,7 +63,7 @@ steps: displayName: Prepare transition evidence condition: always() - - ${{ if or(eq(parameters.action, 'json-baseline'), eq(parameters.action, 'bolt-migrate'), eq(parameters.action, 'rollback-json'), eq(parameters.action, 'bolt-reimport')) }}: + - ${{ if or(eq(parameters.action, 'json-baseline'), eq(parameters.action, 'bolt-migrate'), eq(parameters.action, 'rollback-json'), eq(parameters.action, 'bolt-reimport'), eq(parameters.action, 'configure-state'), eq(parameters.action, 'configure-state-import'), eq(parameters.action, 'rollback-json-and-clear-endpoints')) }}: - task: AzureCLI@2 displayName: Apply ${{ parameters.action }} state mode inputs: @@ -71,45 +80,249 @@ steps: configMap="${{ parameters.configMapName }}" daemonset="${{ parameters.daemonsetName }}" action="${{ parameters.action }}" + effectiveCNI="${{ parameters.cni }}" beforeGeneration=$(kubectl get daemonset "$daemonset" -n kube-system -o jsonpath='{.metadata.generation}') + expectedCNISource= + if [[ "${{ parameters.os }}" == "windows" ]]; then + case "$effectiveCNI" in + cniv2) + expectedCNISource=azure-vnet + ;; + stateless) + expectedCNISource=azure-vnet-stateless + ;; + *) + echo "unsupported Windows CNI type: $effectiveCNI" + exit 1 + ;; + esac + fi + patch_config() { local backend=$1 local mode=$2 + local manageEndpointState=$3 + local initializeFromCNI=$4 + local enableStateMigration=$5 + local enableBolt=false + local enableDebug=false local current patched payload + if [[ "$backend" == "bolt" || "$mode" == "rollback-to-json" ]]; then + enableBolt=true + fi + if [[ "$backend" == "bolt" ]]; then + enableDebug=true + fi current=$(kubectl get configmap "$configMap" -n kube-system -o jsonpath='{.data.cns_config\.json}') - patched=$(jq --arg backend "$backend" --arg mode "$mode" \ - '.StateStoreBackend = $backend | .StateStoreMode = $mode' <<<"$current") + patched=$(jq \ + --arg backend "$backend" \ + --arg mode "$mode" \ + --argjson enableBolt "$enableBolt" \ + --argjson enableDebug "$enableDebug" \ + --argjson manageEndpointState "$manageEndpointState" \ + --argjson initializeFromCNI "$initializeFromCNI" \ + --argjson enableStateMigration "$enableStateMigration" \ + '.StateStoreBackend = $backend + | .StateStoreMode = $mode + | .EnableBoltStateStore = $enableBolt + | .EnablePersistentStateDebug = $enableDebug + | .EnablePersistentStateFaults = false + | .ManageEndpointState = $manageEndpointState + | .InitializeFromCNI = $initializeFromCNI + | .EnableStateMigration = $enableStateMigration' \ + <<<"$current") payload=$(jq -n --arg config "$patched" '{data: {"cns_config.json": $config}}') kubectl patch configmap "$configMap" -n kube-system --type merge -p "$payload" } + prepare_endpoint_import() { + local currentConfig alreadyApplied + currentConfig=$(kubectl get configmap "$configMap" -n kube-system -o jsonpath='{.data.cns_config\.json}') + alreadyApplied=$(jq -r \ + --arg backend "${{ parameters.expectedBackend }}" \ + --argjson manageEndpointState "${{ parameters.expectedManageEndpointState }}" \ + --argjson enableStateMigration "${{ parameters.expectedEnableStateMigration }}" \ + '.StateStoreBackend == $backend + and .ManageEndpointState == $manageEndpointState + and .EnableStateMigration == $enableStateMigration' \ + <<<"$currentConfig") + if [[ "$alreadyApplied" == "true" ]]; then + echo "import configuration is already active; preserving imported endpoint state on replay" \ + >"$evidenceDir/endpoint-import-precondition.txt" + return + fi + + if [[ "${{ parameters.expectedBackend }}" != "json" ]]; then + echo "inactive Bolt endpoint state was cleared by the preceding stateful-CNI reverse" \ + >"$evidenceDir/endpoint-import-precondition.txt" + return + fi + + mapfile -t resetPods < <( + kubectl get pods -n kube-system -l app=privileged-daemonset,os=windows \ + -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}' + ) + if (( ${#resetPods[@]} == 0 )); then + echo "no Windows host-process pods found to prepare JSON endpoint import" + exit 1 + fi + : >"$evidenceDir/endpoint-import-precondition.txt" + for pod in "${resetPods[@]}"; do + node=$(kubectl get pod "$pod" -n kube-system -o jsonpath='{.spec.nodeName}') + kubectl exec "$pod" -n kube-system -- powershell -NoProfile -Command \ + '$path = "C:\k\azurecns\azure-endpoints.json"; Remove-Item -Force -ErrorAction SilentlyContinue $path; if (Test-Path $path) { exit 1 }' + echo "$node: removed empty JSON endpoint store before CNI import" | + tee -a "$evidenceDir/endpoint-import-precondition.txt" + done + } + restart_target() { - kubectl rollout restart daemonset "$daemonset" -n kube-system + if [[ -z "$expectedCNISource" ]]; then + kubectl rollout restart daemonset "$daemonset" -n kube-system + else + local daemonsetJSON installerArgs installerCount sourceCount patchedArgs payload + daemonsetJSON=$(kubectl get daemonset "$daemonset" -n kube-system -o json) + installerCount=$(jq '[.spec.template.spec.initContainers[]? | select(.name == "cni-installer")] | length' <<<"$daemonsetJSON") + if (( installerCount != 1 )); then + echo "expected exactly one cni-installer init container, found $installerCount" + exit 1 + fi + installerArgs=$(jq -c '.spec.template.spec.initContainers[] | select(.name == "cni-installer") | .args' <<<"$daemonsetJSON") + sourceCount=$(jq '[.[] | select(. == "azure-vnet" or . == "azure-vnet-stateless")] | length' <<<"$installerArgs") + if (( sourceCount != 1 )); then + echo "expected exactly one stateful/stateless CNI source argument, found $sourceCount" + exit 1 + fi + patchedArgs=$(jq -c --arg source "$expectedCNISource" \ + 'map(if . == "azure-vnet" or . == "azure-vnet-stateless" then $source else . end)' \ + <<<"$installerArgs") + payload=$(jq -n \ + --arg restartedAt "$(date -u +%Y-%m-%dT%H:%M:%S.%NZ)" \ + --argjson args "$patchedArgs" \ + '{ + spec: { + template: { + metadata: {annotations: {"state-migration.azure.com/restartedAt": $restartedAt}}, + spec: {initContainers: [{name: "cni-installer", args: $args}]} + } + } + }') + kubectl patch daemonset "$daemonset" -n kube-system --type strategic -p "$payload" + fi kubectl rollout status daemonset "$daemonset" -n kube-system --timeout=20m } + clear_endpoint_state() { + local pod + mapfile -t targetNodes < <( + kubectl get nodes -l kubernetes.io/os=${{ parameters.os }} -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}' + ) + mapfile -t cnsPods < <( + kubectl get pods -n kube-system -l k8s-app=azure-cns-win \ + -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}' + ) + if (( ${#targetNodes[@]} == 0 || ${#cnsPods[@]} != ${#targetNodes[@]} )); then + echo "expected one Windows CNS pod per target node before clearing endpoint state" >&2 + exit 1 + fi + + : >"$evidenceDir/cns-endpoint-reset.jsonl" + for pod in "${cnsPods[@]}"; do + kubectl exec "$pod" -n kube-system -- powershell -NoProfile -Command ' + $uri = "http://127.0.0.1:10090" + $before = (Invoke-WebRequest -Uri "$uri/debug/persistent-state/snapshot" -UseBasicParsing -ErrorAction Stop).Content | ConvertFrom-Json + $ids = @($before.Endpoints.psobject.Properties | ForEach-Object { $_.Name }) + foreach ($id in $ids) { + $escaped = [uri]::EscapeDataString($id) + $response = (Invoke-WebRequest -Uri "$uri/network/endpoints/$escaped" -Method Delete -UseBasicParsing -ErrorAction Stop).Content | ConvertFrom-Json + if ($response.ReturnCode -ne 0) { throw "failed to delete stale CNS endpoint $id`: $($response.Message)" } + } + $after = (Invoke-WebRequest -Uri "$uri/debug/persistent-state/snapshot" -UseBasicParsing -ErrorAction Stop).Content | ConvertFrom-Json + $remaining = @($after.Endpoints.psobject.Properties | ForEach-Object { $_.Name }) + if ($remaining.Count -ne 0) { throw "CNS endpoint records remain after stateful-CNI reverse" } + [pscustomobject]@{removed = $ids.Count; remaining = $remaining.Count} | ConvertTo-Json -Compress + ' | tee -a "$evidenceDir/cns-endpoint-reset.jsonl" + done + } + case "$action" in json-baseline) [[ "${{ parameters.expectedBackend }}" == "json" ]] - patch_config json normal + patch_config \ + json \ + normal \ + "${{ parameters.expectedManageEndpointState }}" \ + "${{ parameters.expectedInitializeFromCNI }}" \ + "${{ parameters.expectedEnableStateMigration }}" restart_target expectedRestarts=1 ;; bolt-migrate|bolt-reimport) [[ "${{ parameters.expectedBackend }}" == "bolt" ]] - patch_config bolt normal + patch_config \ + bolt \ + normal \ + "${{ parameters.expectedManageEndpointState }}" \ + "${{ parameters.expectedInitializeFromCNI }}" \ + "${{ parameters.expectedEnableStateMigration }}" restart_target expectedRestarts=1 ;; rollback-json) [[ "${{ parameters.expectedBackend }}" == "json" ]] - patch_config json rollback-to-json + patch_config \ + json \ + rollback-to-json \ + true \ + false \ + false restart_target - patch_config json normal + patch_config \ + json \ + normal \ + "${{ parameters.expectedManageEndpointState }}" \ + "${{ parameters.expectedInitializeFromCNI }}" \ + "${{ parameters.expectedEnableStateMigration }}" restart_target expectedRestarts=2 ;; + rollback-json-and-clear-endpoints) + [[ "${{ parameters.expectedBackend }}" == "json" ]] + [[ "${{ parameters.expectedManageEndpointState }}" == "false" ]] + clear_endpoint_state + patch_config json rollback-to-json true false false + restart_target + patch_config \ + json \ + normal \ + "${{ parameters.expectedManageEndpointState }}" \ + "${{ parameters.expectedInitializeFromCNI }}" \ + "${{ parameters.expectedEnableStateMigration }}" + restart_target + expectedRestarts=2 + ;; + configure-state) + patch_config \ + "${{ parameters.expectedBackend }}" \ + normal \ + "${{ parameters.expectedManageEndpointState }}" \ + "${{ parameters.expectedInitializeFromCNI }}" \ + "${{ parameters.expectedEnableStateMigration }}" + restart_target + expectedRestarts=1 + ;; + configure-state-import) + prepare_endpoint_import + patch_config \ + "${{ parameters.expectedBackend }}" \ + normal \ + "${{ parameters.expectedManageEndpointState }}" \ + "${{ parameters.expectedInitializeFromCNI }}" \ + "${{ parameters.expectedEnableStateMigration }}" + restart_target + expectedRestarts=1 + ;; *) echo "unsupported state mode action: $action" exit 1 @@ -123,26 +336,197 @@ steps: exit 1 fi + if [[ -n "$expectedCNISource" ]]; then + daemonsetJSON=$(kubectl get daemonset "$daemonset" -n kube-system -o json) + jq -e \ + --arg source "$expectedCNISource" ' + [.spec.template.spec.initContainers[]? | select(.name == "cni-installer")] as $installers + | ($installers | length) == 1 + and ($installers[0].args | [.[] | select(. == "azure-vnet" or . == "azure-vnet-stateless")]) == [$source] + and ($installers[0].args | index("/k/azurecni/bin/azure-vnet.exe")) != null + ' <<<"$daemonsetJSON" + fi + config=$(kubectl get configmap "$configMap" -n kube-system -o jsonpath='{.data.cns_config\.json}') + expectedEnableBolt=false + expectedDebug=false + if [[ "${{ parameters.expectedBackend }}" == "bolt" ]]; then + expectedEnableBolt=true + expectedDebug=true + fi jq -e \ --arg backend "${{ parameters.expectedBackend }}" \ - '.StateStoreBackend == $backend and .StateStoreMode == "normal"' \ + --argjson enableBolt "$expectedEnableBolt" \ + --argjson enableDebug "$expectedDebug" \ + --argjson manageEndpointState "${{ parameters.expectedManageEndpointState }}" \ + --argjson initializeFromCNI "${{ parameters.expectedInitializeFromCNI }}" \ + --argjson enableStateMigration "${{ parameters.expectedEnableStateMigration }}" \ + '.StateStoreBackend == $backend + and .StateStoreMode == "normal" + and .EnableBoltStateStore == $enableBolt + and .EnablePersistentStateDebug == $enableDebug + and .EnablePersistentStateFaults == false + and .ManageEndpointState == $manageEndpointState + and .InitializeFromCNI == $initializeFromCNI + and .EnableStateMigration == $enableStateMigration' \ <<<"$config" jq -n \ --arg action "$action" \ --arg backend "${{ parameters.expectedBackend }}" \ + --argjson enableBolt "$expectedEnableBolt" \ + --argjson enableDebug "$expectedDebug" \ + --argjson manageEndpointState "${{ parameters.expectedManageEndpointState }}" \ + --argjson initializeFromCNI "${{ parameters.expectedInitializeFromCNI }}" \ + --argjson enableStateMigration "${{ parameters.expectedEnableStateMigration }}" \ + --arg effectiveCNI "$effectiveCNI" \ + --arg expectedCNISource "$expectedCNISource" \ --argjson beforeGeneration "$beforeGeneration" \ --argjson afterGeneration "$afterGeneration" \ --argjson restarts "$expectedRestarts" \ '{ action: $action, expectedBackend: $backend, + expectedEnableBoltStateStore: $enableBolt, + expectedEnablePersistentStateDebug: $enableDebug, + expectedManageEndpointState: $manageEndpointState, + expectedInitializeFromCNI: $initializeFromCNI, + expectedEnableStateMigration: $enableStateMigration, + effectiveCNI: $effectiveCNI, + expectedCNISource: $expectedCNISource, daemonsetGenerationBefore: $beforeGeneration, daemonsetGenerationAfter: $afterGeneration, restartCount: $restarts }' >"$evidenceDir/mode-transition.json" + - ${{ if eq(parameters.action, 'configure-state-import') }}: + - task: AzureCLI@2 + displayName: Preflight imported CNS endpoint state before CNI replacement + inputs: + azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION) + scriptLocation: inlineScript + scriptType: bash + addSpnToEnvironment: true + inlineScript: | + set -euo pipefail + + [[ "${{ parameters.os }}" == "windows" ]] + [[ "${{ parameters.cni }}" == "cniv2" ]] + evidenceDir="$(Build.SourcesDirectory)/test/integration/logs/state-migration/${{ parameters.lane }}/${{ parameters.transition }}" + make -C ./hack/aks set-kubeconf AZCLI=az CLUSTER=${{ parameters.clusterName }} + + mapfile -t nodes < <( + kubectl get nodes -l kubernetes.io/os=windows \ + -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}' + ) + if (( ${#nodes[@]} == 0 )); then + echo "no Windows nodes found for CNS endpoint import preflight" + exit 1 + fi + + for node in "${nodes[@]}"; do + nodeIPs=$(kubectl get node "$node" -o json | + jq -c '[.status.addresses[]? | select(.type == "InternalIP") | .address]') + imported=false + for attempt in $(seq 1 10); do + expected=$(kubectl get pods --all-namespaces \ + --field-selector "spec.nodeName=$node" -o json | + jq -c --argjson nodeIPs "$nodeIPs" ' + [.items[] as $pod + | select($pod.status.phase != "Succeeded" and $pod.status.phase != "Failed") + | $pod.status.podIPs[]?.ip + | select(. as $ip | ($nodeIPs | index($ip) | not)) + | { + podName: $pod.metadata.name, + podNamespace: $pod.metadata.namespace, + ip: . + } + ] | unique_by(.podNamespace, .podName, .ip) | sort_by(.podNamespace, .podName, .ip) + ') + + if [[ "${{ parameters.expectedBackend }}" == "json" ]]; then + statePod=$(kubectl get pods -n kube-system \ + -l app=privileged-daemonset,os=windows -o json | + jq -r --arg node "$node" ' + [.items[] + | select(.spec.nodeName == $node and .status.phase == "Running") + ][0].metadata.name // empty + ') + if [[ -z "$statePod" ]]; then + echo "no Windows host-process state pod found on $node" + exit 1 + fi + state=$(kubectl exec "$statePod" -n kube-system -- powershell -NoProfile -Command \ + 'Get-Content -Raw -ErrorAction Stop C:\k\azurecns\azure-endpoints.json') + actual=$(jq -c ' + [.Endpoints[]? as $endpoint + | $endpoint.IfnameToIPMap[]? + | select((.NICType // "") == "" or .NICType == "InfraNIC") + | (.IPv4[]?.IP, .IPv6[]?.IP) + | { + podName: $endpoint.PodName, + podNamespace: $endpoint.PodNamespace, + ip: . + } + ] | sort_by(.podNamespace, .podName, .ip) + ' <<<"$state") + else + cnsPod=$(kubectl get pods -n kube-system -l k8s-app=azure-cns-win -o json | + jq -r --arg node "$node" ' + [.items[] + | select(.spec.nodeName == $node and .status.phase == "Running") + ][0].metadata.name // empty + ') + if [[ -z "$cnsPod" ]]; then + echo "no Windows CNS pod found on $node" + exit 1 + fi + state=$(kubectl exec "$cnsPod" -n kube-system -- powershell -NoProfile -Command \ + '(Invoke-WebRequest -Uri 127.0.0.1:10090/debug/persistent-state/snapshot -UseBasicParsing -ErrorAction Stop).Content') + actual=$(jq -c ' + [.Endpoints[]? as $endpoint + | $endpoint.ifnameToIPMap[]? + | select((.nicType // "") == "" or .nicType == "InfraNIC") + | (.ipv4[]?.IP, .ipv6[]?.IP) + | { + podName: $endpoint.podName, + podNamespace: $endpoint.podNamespace, + ip: . + } + ] | sort_by(.podNamespace, .podName, .ip) + ' <<<"$state") + fi + + jq -n \ + --arg node "$node" \ + --arg backend "${{ parameters.expectedBackend }}" \ + --argjson attempt "$attempt" \ + --argjson expected "$expected" \ + --argjson actual "$actual" \ + '{ + node: $node, + backend: $backend, + attempt: $attempt, + expected: $expected, + actual: $actual, + exact: ($expected == $actual) + }' >"$evidenceDir/imported-endpoints-$node.json" + + if jq -e -n \ + --argjson expected "$expected" \ + --argjson actual "$actual" \ + '($expected | length) > 0 and $expected == $actual' >/dev/null; then + imported=true + break + fi + sleep 15 + done + if [[ "$imported" != "true" ]]; then + echo "CNS endpoint import on $node did not converge to the exact live pod endpoint set" + exit 1 + fi + done + - ${{ if eq(parameters.action, 'json-baseline') }}: - task: AzureCLI@2 displayName: Seed JSON-authoritative pod state @@ -201,17 +585,6 @@ steps: '{daemonsetGenerationBefore: $beforeGeneration, daemonsetGenerationAfter: $afterGeneration, restartCount: 1}' \ >"$evidenceDir/restart.json" - # Endpoint-commit faults require CNS-owned endpoint state. CNI-managed lanes - # still run the mandatory active-scale restart immediately below. - - ${{ if and(eq(parameters.action, 'restart-during-scale'), eq(parameters.runFaultInjection, true)) }}: - - template: ../load-test-templates/migration-fault-injection-template.yaml - parameters: - clusterName: ${{ parameters.clusterName }} - os: ${{ parameters.os }} - cni: ${{ parameters.cni }} - scenario: all - artifactName: state-migration-${{ parameters.lane }}-04-fault-injection - - ${{ if eq(parameters.action, 'restart-during-scale') }}: - task: AzureCLI@2 displayName: Restart CNS during active pod scale @@ -261,6 +634,43 @@ steps: daemonsetGenerationAfter: $daemonsetGenerationAfter }' >"$evidenceDir/restart-during-scale.json" + - ${{ if and(eq(parameters.action, 'external-fault'), eq(parameters.injectExternalFault, true)) }}: + - task: AzureCLI@2 + displayName: Inject external CNS pod fault + inputs: + azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION) + scriptLocation: inlineScript + scriptType: bash + addSpnToEnvironment: true + inlineScript: | + set -euo pipefail + + evidenceDir="$(Build.SourcesDirectory)/test/integration/logs/state-migration/${{ parameters.lane }}/${{ parameters.transition }}" + make -C ./hack/aks set-kubeconf AZCLI=az CLUSTER=${{ parameters.clusterName }} + if [[ "${{ parameters.os }}" == "windows" ]]; then + selector="k8s-app=azure-cns-win" + else + selector="k8s-app=azure-cns" + fi + pod=$(kubectl get pods -n kube-system -l "$selector" -o json | + jq -r '[.items[] | select(.status.phase == "Running")] | sort_by(.metadata.name) | .[0].metadata.name // empty') + if [[ -z "$pod" ]]; then + echo "no running CNS pod matched selector $selector" + exit 1 + fi + uid=$(kubectl get pod "$pod" -n kube-system -o jsonpath='{.metadata.uid}') + kubectl delete pod "$pod" -n kube-system --wait=false + kubectl rollout status daemonset/${{ parameters.daemonsetName }} -n kube-system --timeout=20m + replacement=$(kubectl get pods -n kube-system -l "$selector" -o json | + jq -r --arg uid "$uid" '[.items[] | select(.metadata.uid != $uid and .status.phase == "Running")] | length') + if (( replacement == 0 )); then + echo "external CNS pod fault did not produce a running replacement" + exit 1 + fi + jq -n --arg pod "$pod" --arg uid "$uid" \ + '{fault: "external-pod-deletion", deletedPod: $pod, deletedUID: $uid}' \ + >"$evidenceDir/external-fault.json" + - ${{ if eq(parameters.action, 'node-reboot') }}: - task: AzureCLI@2 displayName: Reboot cluster nodes and require new boot IDs @@ -275,6 +685,15 @@ steps: evidenceDir="$(Build.SourcesDirectory)/test/integration/logs/state-migration/${{ parameters.lane }}/${{ parameters.transition }}" make -C ./hack/aks set-kubeconf AZCLI=az CLUSTER=${{ parameters.clusterName }} + if [[ "${{ parameters.os }}" == "windows" ]]; then + ( + cd hack/scripts + bash patch-kubeclusterconfig.sh 2>&1 | + tee "$evidenceDir/agentbaker-patch.log" + ) + grep -Fxq "All nodes patched successfully" "$evidenceDir/agentbaker-patch.log" + fi + kubectl get nodes -l kubernetes.io/os=${{ parameters.os }} -o json | jq '[.items[] | {nodeName: .metadata.name, bootID: .status.nodeInfo.bootID}] | sort_by(.nodeName)' \ >"$evidenceDir/node-boots-before.json" @@ -340,6 +759,20 @@ steps: else deployment=load-test fi + + kubectl get pods -n load-test -l load-test=true -o json | + jq '[.items[] + | { + name: .metadata.name, + phase: .status.phase, + ready: any(.status.conditions[]?; .type == "Ready" and .status == "True") + } + | select(.phase != "Running" or .ready != true) + ]' >"$evidenceDir/workload-pods-recreated.json" + mapfile -t stalePods < <(jq -r '.[].name' "$evidenceDir/workload-pods-recreated.json") + if (( ${#stalePods[@]} > 0 )); then + kubectl delete pod -n load-test "${stalePods[@]}" --wait=false + fi kubectl rollout status deployment "$deployment" -n load-test --timeout=30m jq -e -n \ @@ -370,7 +803,23 @@ steps: -l app=privileged-daemonset,os=windows -o name); do found=true kubectl exec -n kube-system "$pod" -- \ - powershell -NoProfile -Command 'Restart-Service hns -Force' + powershell -NoProfile -Command ' + $transient = "Cannot stop service|service has not been started|service cannot accept control messages" + for ($attempt = 1; $attempt -le 3; $attempt++) { + try { + Restart-Service hns -Force -ErrorAction Stop + exit 0 + } catch { + if ($_.Exception.Message -notmatch $transient -or $attempt -eq 3) { throw } + try { + Start-Service hns -ErrorAction Stop + } catch { + if ($_.Exception.Message -notmatch $transient) { throw } + } + Start-Sleep -Seconds (5 * $attempt) + } + } + ' done if [[ "$found" != "true" ]]; then echo "no Windows privileged daemonset pods found for HNS validation" @@ -454,11 +903,61 @@ steps: config=$(kubectl get configmap/${{ parameters.configMapName }} \ -n kube-system -o jsonpath='{.data.cns_config\.json}') + expectedEnableBolt=false + expectedDebug=false + if [[ "${{ parameters.expectedBackend }}" == "bolt" ]]; then + expectedEnableBolt=true + expectedDebug=true + fi jq -e \ --arg backend "${{ parameters.expectedBackend }}" \ - '.StateStoreBackend == $backend and .StateStoreMode == "normal"' \ + --argjson enableBolt "$expectedEnableBolt" \ + --argjson enableDebug "$expectedDebug" \ + --argjson manageEndpointState "${{ parameters.expectedManageEndpointState }}" \ + --argjson initializeFromCNI "${{ parameters.expectedInitializeFromCNI }}" \ + --argjson enableStateMigration "${{ parameters.expectedEnableStateMigration }}" \ + '.StateStoreBackend == $backend + and .StateStoreMode == "normal" + and .EnableBoltStateStore == $enableBolt + and .EnablePersistentStateDebug == $enableDebug + and .EnablePersistentStateFaults == false + and .ManageEndpointState == $manageEndpointState + and .InitializeFromCNI == $initializeFromCNI + and .EnableStateMigration == $enableStateMigration' \ <<<"$config" + effectiveCNI="${{ parameters.cni }}" + expectedCNISource= + if [[ "${{ parameters.os }}" == "windows" ]]; then + case "$effectiveCNI" in + cniv2) + expectedCNISource=azure-vnet + ;; + stateless) + expectedCNISource=azure-vnet-stateless + ;; + *) + echo "unsupported Windows CNI type: $effectiveCNI" + exit 1 + ;; + esac + fi + if [[ -n "$expectedCNISource" ]]; then + daemonsetJSON=$(kubectl get daemonset/${{ parameters.daemonsetName }} -n kube-system -o json) + jq -e \ + --arg source "$expectedCNISource" ' + [.spec.template.spec.initContainers[]? | select(.name == "cni-installer")] as $installers + | ($installers | length) == 1 + and ($installers[0].args | [.[] | select(. == "azure-vnet" or . == "azure-vnet-stateless")]) == [$source] + and ($installers[0].args | index("/k/azurecni/bin/azure-vnet.exe")) != null + ' <<<"$daemonsetJSON" + jq -n \ + --arg effectiveCNI "$effectiveCNI" \ + --arg expectedCNISource "$expectedCNISource" \ + '{effectiveCNI: $effectiveCNI, expectedCNISource: $expectedCNISource}' \ + >"$evidenceDir/cni-type.json" + fi + restartCase=true if [[ "${{ parameters.action }}" == "json-baseline" ]]; then restartCase=false @@ -476,32 +975,19 @@ steps: test -s "$summaryPath" jq -e \ - --arg backend "${{ parameters.expectedBackend }}" \ - --arg authority "${{ parameters.expectedAuthority }}" \ - --argjson schema "${{ parameters.expectedSchemaVersion }}" ' - (.checks | length) > 0 + --arg backend "${{ parameters.expectedBackend }}" ' + .stateBackend == $backend + and (.checks | type == "array" and length > 0) + and ([.checks[] | [.checkName, .nodeName]] | unique | length) == (.checks | length) and all(.checks[]; - .validationPass == true - and .converged == true - and (((.missingIPs // []) | length) == 0) - and (((.unexpectedIPs // []) | length) == 0) - and (((.duplicateIPs // []) | length) == 0) - ) - and ( - if $backend == "bolt" then - ([.checks[] | select((.stateBackend // "") != "")]) as $persistent - | ($persistent | length) > 0 - and all($persistent[]; - .stateBackend == $backend - and .authority == $authority - and .schemaVersion == $schema - and ((.bootID // "") | length) > 0 - and .dbFilePresent == true - and .dbFileSizeBytes > 0 - ) - else - all(.checks[]; (.stateBackend // "") == "") - end + (.checkName | length) > 0 + and (.nodeName | length) > 0 + and .livePodCount >= 0 + and (.expected | type == "array") + and (.actual | type == "array") + and (if .livePodCount > 0 then (.expected | length) > 0 else true end) + and ([.expected[] | {podID, ip}] | sort_by(.podID, .ip)) + == ([.actual[] | {podID, ip}] | sort_by(.podID, .ip)) ) ' "$summaryPath" @@ -526,6 +1012,35 @@ steps: ) ' "$evidenceDir/pod-state.json" >/dev/null + kubectl get nodes -l kubernetes.io/os=${{ parameters.os }} -o json | + jq '[.items[] + | { + nodeName: .metadata.name, + bootID: .status.nodeInfo.bootID + } + ] | sort_by(.nodeName)' >"$evidenceDir/node-boots.json" + jq -e ' + length > 0 + and ([.[].nodeName] | unique | length) == length + and all(.[]; + (.nodeName | type == "string" and length > 0) + and (.bootID | type == "string" and length > 0) + ) + ' "$evidenceDir/node-boots.json" >/dev/null + + - template: capture-transition.steps.yaml + parameters: + lane: ${{ parameters.lane }} + transition: ${{ parameters.transition }} + clusterName: ${{ parameters.clusterName }} + os: ${{ parameters.os }} + configMapName: ${{ parameters.configMapName }} + daemonsetName: ${{ parameters.daemonsetName }} + expectedBackend: ${{ parameters.expectedBackend }} + expectedAuthority: ${{ parameters.expectedAuthority }} + expectedSchemaVersion: ${{ parameters.expectedSchemaVersion }} + summaryPath: $(Build.SourcesDirectory)/test/integration/logs/validate/${{ parameters.lane }}/${{ parameters.transition }}.json + - ${{ if ne(parameters.baselineArtifact, '') }}: - bash: | set -euo pipefail @@ -534,6 +1049,8 @@ steps: candidate="$(Build.SourcesDirectory)/test/integration/logs/validate/${{ parameters.lane }}/${{ parameters.transition }}.json" baselinePodState="$(Pipeline.Workspace)/${{ parameters.baselineArtifact }}/pod-state.json" candidatePodState="$evidenceDir/pod-state.json" + baselineMetadata="$(Pipeline.Workspace)/${{ parameters.baselineArtifact }}/persistent-debug" + candidateMetadata="$evidenceDir/persistent-debug" bash hack/scripts/compare-state-migration-summaries.sh \ "$baseline" \ "$candidate" \ @@ -542,20 +1059,16 @@ steps: "${{ parameters.expectedSchemaVersion }}" \ "${{ parameters.stateRelation }}" \ "${{ parameters.bootRelation }}" \ + "${{ parameters.podRelation }}" \ "$baselinePodState" \ "$candidatePodState" \ + "$baselineMetadata" \ + "$candidateMetadata" \ 2>&1 | tee "$evidenceDir/comparison.txt" displayName: Compare transition summaries - - template: capture-transition.steps.yaml - parameters: - lane: ${{ parameters.lane }} - transition: ${{ parameters.transition }} - clusterName: ${{ parameters.clusterName }} - os: ${{ parameters.os }} - configMapName: ${{ parameters.configMapName }} - daemonsetName: ${{ parameters.daemonsetName }} - expectedBackend: ${{ parameters.expectedBackend }} - expectedAuthority: ${{ parameters.expectedAuthority }} - expectedSchemaVersion: ${{ parameters.expectedSchemaVersion }} - summaryPath: $(Build.SourcesDirectory)/test/integration/logs/validate/${{ parameters.lane }}/${{ parameters.transition }}.json + - publish: $(Build.SourcesDirectory)/test/integration/logs/state-migration/${{ parameters.lane }}/${{ parameters.transition }} + artifact: state-migration-$(Build.BuildId)-${{ parameters.lane }}-${{ parameters.transition }} + displayName: Publish transition evidence + condition: always() + continueOnError: true diff --git a/hack/scripts/compare-state-migration-summaries.sh b/hack/scripts/compare-state-migration-summaries.sh index 3f875e19c3..0fef088e02 100755 --- a/hack/scripts/compare-state-migration-summaries.sh +++ b/hack/scripts/compare-state-migration-summaries.sh @@ -2,8 +2,8 @@ set -euo pipefail -if [[ $# -ne 9 ]]; then - echo "usage: $0 " >&2 +if [[ $# -ne 12 ]]; then + echo "usage: $0 " >&2 exit 2 fi @@ -14,8 +14,11 @@ expected_authority=$4 expected_schema=$5 state_relation=$6 boot_relation=$7 -baseline_pod_state=$8 -candidate_pod_state=$9 +pod_relation=$8 +baseline_pod_state=$9 +candidate_pod_state=${10} +baseline_metadata_dir=${11} +candidate_metadata_dir=${12} for artifact in "$baseline" "$candidate" "$baseline_pod_state" "$candidate_pod_state"; do if [[ ! -s "$artifact" ]]; then @@ -31,7 +34,6 @@ json | bolt) ;; exit 2 ;; esac - case "$state_relation" in none | exact | changed) ;; *) @@ -39,7 +41,6 @@ none | exact | changed) ;; exit 2 ;; esac - case "$boot_relation" in none | same | changed) ;; *) @@ -47,145 +48,259 @@ none | same | changed) ;; exit 2 ;; esac - +if [[ "$pod_relation" == "inherit" ]]; then + pod_relation=$state_relation +fi +case "$pod_relation" in +none | exact | identity | changed) ;; +*) + echo "unsupported pod relation: $pod_relation" >&2 + exit 2 + ;; +esac if [[ ! "$expected_schema" =~ ^[0-9]+$ ]]; then echo "expected schema must be an unsigned integer: $expected_schema" >&2 exit 2 fi -if ! jq -e \ - --arg backend "$expected_backend" \ - --arg authority "$expected_authority" \ - --argjson schema "$expected_schema" ' - (.checks | type == "array" and length > 0) - and all(.checks[]; - .validationPass == true - and .converged == true - and (((.missingIPs // []) | length) == 0) - and (((.unexpectedIPs // []) | length) == 0) - and (((.duplicateIPs // []) | length) == 0) - ) - and ( - if $backend == "bolt" then - ([.checks[] | select((.stateBackend // "") != "")]) as $persistent - | ($persistent | length) > 0 - and all($persistent[]; - .stateBackend == $backend - and .authority == $authority - and .schemaVersion == $schema - and ((.bootID // "") | length) > 0 - and .dbFilePresent == true - and .dbFileSizeBytes > 0 +repo_root=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd) +baseline_backend=$(jq -er '.stateBackend | strings' "$baseline") || { + echo "baseline summary backend is missing or malformed: $baseline" >&2 + exit 1 +} +baseline_abs=$(realpath "$baseline") +candidate_abs=$(realpath "$candidate") +if ! ( + cd "$repo_root" + go run ./test/validate/cmd/summarydiff \ + -baseline "$baseline_abs" \ + -candidate "$baseline_abs" \ + -expected-backend "$baseline_backend" >/dev/null +); then + echo "baseline summary failed strict Go validation: $baseline" >&2 + exit 1 +fi +if ! ( + cd "$repo_root" + go run ./test/validate/cmd/summarydiff \ + -baseline "$candidate_abs" \ + -candidate "$candidate_abs" \ + -expected-backend "$expected_backend" >/dev/null +); then + echo "candidate summary failed strict Go validation: $candidate" >&2 + exit 1 +fi + +validate_summary() { + local path=$1 + local backend=$2 + jq -e --arg backend "$backend" ' + .stateBackend == $backend + and (.checks | type == "array" and length > 0) + and ([.checks[] | [.checkName, .nodeName]] | unique | length) == (.checks | length) + and all(.checks[]; + (.checkName | type == "string" and length > 0) + and (.nodeName | type == "string" and length > 0) + and (.livePodCount | type == "number" and . >= 0 and floor == .) + and (.expected | type == "array") + and (.actual | type == "array") + and (if .livePodCount > 0 then (.expected | length) > 0 else true end) + and ([.expected[] | [.podID, .ip]] | unique | length) == (.expected | length) + and ([.actual[] | [.podID, .ip]] | unique | length) == (.actual | length) + and all(.expected[], .actual[]; + (.podID | type == "string" and length > 0) + and (.ip | type == "string" and test("^[0-9A-Fa-f:.]+$")) ) - else - all(.checks[]; (.stateBackend // "") == "") - end - ) -' "$candidate" >/dev/null; then + and ([.expected[] | .ip] | unique | length) == (.expected | length) + and ([.actual[] | .ip] | unique | length) == (.actual | length) + and ([.expected[] | {podID, ip}] | sort_by(.podID, .ip)) + == ([.actual[] | {podID, ip}] | sort_by(.podID, .ip)) + ) + ' "$path" >/dev/null +} + +if ! validate_summary "$baseline" "$(jq -r '.stateBackend // empty' "$baseline")"; then + echo "baseline summary failed strict validation: $baseline" >&2 + exit 1 +fi +if ! validate_summary "$candidate" "$expected_backend"; then echo "candidate summary failed strict validation: $candidate" >&2 - jq . "$candidate" >&2 || true exit 1 fi if ! jq -e -n \ --slurpfile baseline "$baseline" \ --slurpfile candidate "$candidate" \ - --slurpfile baselinePods "$baseline_pod_state" \ - --slurpfile candidatePods "$candidate_pod_state" \ --arg relation "$state_relation" ' - def normalized_state($summary): + def normalized($summary): [ $summary.checks[] - | select(.checkName != "cns persistent metadata") | { checkName, nodeName, - expectedCount, - actualCount, - missingIPs: ((.missingIPs // []) | sort), - unexpectedIPs: ((.unexpectedIPs // []) | sort), - duplicateIPs: ((.duplicateIPs // []) | sort) + livePodCount, + expected: ([.expected[] | {podID, ip}] | sort_by(.podID, .ip)), + actual: ([.actual[] | {podID, ip}] | sort_by(.podID, .ip)) } ] | sort_by(.checkName, .nodeName); - def normalized_pods($pods): - [ - $pods[] - | { - namespace, - name, - nodeName, - phase, - podIPs: ((.podIPs // []) | sort) - } - ] - | sort_by(.namespace, .name); - - (normalized_state($baseline[0])) as $before - | (normalized_state($candidate[0])) as $after - | (normalized_pods($baselinePods[0])) as $beforePods - | (normalized_pods($candidatePods[0])) as $afterPods - | if $relation == "none" then - true - elif $relation == "exact" then - $before == $after - and ($beforePods | length) > 0 - and $beforePods == $afterPods - else - ($beforePods | length) > 0 - and ($afterPods | length) > 0 - and $beforePods != $afterPods + (normalized($baseline[0])) as $before + | (normalized($candidate[0])) as $after + | if $relation == "none" then true + elif $relation == "exact" then $before == $after + else + ($before | map([.checkName, .nodeName])) == ($after | map([.checkName, .nodeName])) + and ([range(0; $before | length) as $i + | $after[$i].livePodCount >= $before[$i].livePodCount] | all) and $before != $after - end + end ' >/dev/null; then echo "state relation '$state_relation' failed between $baseline and $candidate" >&2 exit 1 fi +validate_pods=' + type == "array" + and length > 0 + and ([.[] | [.namespace, .name]] | unique | length) == length + and all(.[]; + (.namespace | type == "string" and length > 0) + and (.name | type == "string" and length > 0) + and (.nodeName | type == "string" and length > 0) + and .phase == "Running" + and (.podIPs | type == "array" and length > 0) + and ([.podIPs[]] | unique | length) == (.podIPs | length) + ) +' +if ! jq -e "$validate_pods" "$baseline_pod_state" >/dev/null || + ! jq -e "$validate_pods" "$candidate_pod_state" >/dev/null; then + echo "pod state failed strict validation" >&2 + exit 1 +fi + if ! jq -e -n \ - --slurpfile baseline "$baseline" \ - --slurpfile candidate "$candidate" \ - --arg relation "$boot_relation" ' - def boots($summary): - [ - $summary.checks[] - | select((.stateBackend // "") != "") - | {nodeName, bootID} - ] - | sort_by(.nodeName); - - (boots($baseline[0])) as $before - | (boots($candidate[0])) as $after - | if $relation == "none" then - true - elif $relation == "same" then - ($before | length) > 0 - and $before == $after - else - ($before | length) > 0 - and ($before | map(.nodeName)) == ($after | map(.nodeName)) - and ([range(0; $before | length) as $i | $before[$i].bootID != $after[$i].bootID] | all) - end + --slurpfile baselinePods "$baseline_pod_state" \ + --slurpfile candidatePods "$candidate_pod_state" \ + --arg relation "$pod_relation" ' + def normalized($pods): + [$pods[] | {namespace, name, nodeName, phase, podIPs: (.podIPs | sort)}] + | sort_by(.namespace, .name); + def identities($pods): + normalized($pods) | map(del(.podIPs)); + (normalized($baselinePods[0])) as $before + | (normalized($candidatePods[0])) as $after + | if $relation == "none" then true + elif $relation == "exact" then $before == $after + elif $relation == "identity" then identities($baselinePods[0]) == identities($candidatePods[0]) + else ($after | length) >= ($before | length) and $before != $after + end ' >/dev/null; then - echo "boot relation '$boot_relation' failed between $baseline and $candidate" >&2 + echo "pod relation '$pod_relation' failed between $baseline_pod_state and $candidate_pod_state" >&2 exit 1 fi +metadata_json() { + local directory=$1 + if [[ ! -d "$directory" ]]; then + return 1 + fi + local files=("$directory"/*.json) + if [[ ! -e "${files[0]}" ]]; then + return 1 + fi + jq -s 'sort_by(.nodeName)' "${files[@]}" +} + +if [[ "$expected_backend" == "bolt" ]]; then + candidate_metadata=$(metadata_json "$candidate_metadata_dir") || { + echo "candidate persistent metadata is missing: $candidate_metadata_dir" >&2 + exit 1 + } + if ! jq -e \ + --arg authority "$expected_authority" \ + --argjson schema "$expected_schema" ' + type == "array" + and length > 0 + and ([.[].nodeName] | unique | length) == length + and all(.[]; + (.nodeName | type == "string" and length > 0) + and .status.backend == "bbolt" + and .status.authority == $authority + and .status.schemaVersion == $schema + and .status.generation > 0 + and .status.bootPresent == true + and .status.storagePresent == true + and .status.databaseBytes > 0 + and .status.invariantStatus == "healthy" + and .snapshot.Metadata.authority == $authority + and .snapshot.Metadata.schemaVersion == $schema + and .snapshot.Metadata.generation == .status.generation + and ((.snapshot.Metadata.bootID // "") | length) > 0 + ) + ' <<<"$candidate_metadata" >/dev/null; then + echo "candidate persistent metadata failed strict validation" >&2 + exit 1 + fi + + summary_nodes=$(jq -c '[.checks[].nodeName] | unique | sort' "$candidate") + metadata_nodes=$(jq -c '[.[].nodeName] | unique | sort' <<<"$candidate_metadata") + if [[ "$summary_nodes" != "$metadata_nodes" ]]; then + echo "persistent metadata node coverage does not match validation summary" >&2 + exit 1 + fi +fi + +if [[ "$boot_relation" != "none" ]]; then + baseline_boot_state="$(dirname "$baseline_pod_state")/node-boots.json" + candidate_boot_state="$(dirname "$candidate_pod_state")/node-boots.json" + if [[ ! -s "$baseline_boot_state" || ! -s "$candidate_boot_state" ]]; then + echo "node boot evidence is missing for boot relation '$boot_relation'" >&2 + exit 1 + fi + validate_boots=' + type == "array" + and length > 0 + and ([.[].nodeName] | unique | length) == length + and all(.[]; + (.nodeName | type == "string" and length > 0) + and (.bootID | type == "string" and length > 0) + ) + ' + if ! jq -e "$validate_boots" "$baseline_boot_state" >/dev/null || + ! jq -e "$validate_boots" "$candidate_boot_state" >/dev/null; then + echo "node boot evidence failed strict validation" >&2 + exit 1 + fi + if ! jq -e -n \ + --slurpfile before "$baseline_boot_state" \ + --slurpfile after "$candidate_boot_state" \ + --arg relation "$boot_relation" ' + ($before[0] | sort_by(.nodeName)) as $a + | ($after[0] | sort_by(.nodeName)) as $b + | ($a | length) > 0 + and ($a | map(.nodeName)) == ($b | map(.nodeName)) + and if $relation == "same" then $a == $b + else [range(0; $a | length) as $i | $a[$i].bootID != $b[$i].bootID] | all + end + ' >/dev/null; then + echo "boot relation '$boot_relation' failed between metadata captures" >&2 + exit 1 + fi +fi + jq -n \ --arg baseline "$baseline" \ --arg candidate "$candidate" \ - --arg baselinePodState "$baseline_pod_state" \ - --arg candidatePodState "$candidate_pod_state" \ --arg expectedBackend "$expected_backend" \ --arg stateRelation "$state_relation" \ --arg bootRelation "$boot_relation" \ + --arg podRelation "$pod_relation" \ '{ baseline: $baseline, candidate: $candidate, - baselinePodState: $baselinePodState, - candidatePodState: $candidatePodState, expectedBackend: $expectedBackend, stateRelation: $stateRelation, bootRelation: $bootRelation, + podRelation: $podRelation, result: "pass" }' diff --git a/test/integration/load/load_test.go b/test/integration/load/load_test.go index 3c484cf2f7..f460de0dec 100644 --- a/test/integration/load/load_test.go +++ b/test/integration/load/load_test.go @@ -4,6 +4,8 @@ package load import ( "context" + "encoding/json" + "os" "testing" "time" @@ -29,6 +31,8 @@ type TestConfig struct { // PoolLabelSelector optionally restricts state validation to nodes matching // this label selector (e.g. "agentpool=pool1"). Empty validates all nodes. PoolLabelSelector string `env:"POOL_LABEL_SELECTOR" default:""` + SummaryPath string `env:"VALIDATE_SUMMARY_PATH"` + StateBackend string `env:"VALIDATE_STATE_BACKEND" default:"json"` WaitForCNS bool `env:"WAIT_FOR_CNS" default:"true"` } @@ -180,6 +184,13 @@ func TestValidateState(t *testing.T) { err = validator.Validate(ctx) require.NoError(t, err) + if testConfig.SummaryPath != "" { + summary := validator.Summary(validate.StateBackend(testConfig.StateBackend)) + require.NoError(t, validate.CheckSummary(summary, validate.StateBackend(testConfig.StateBackend))) + encoded, err := json.MarshalIndent(summary, "", " ") + require.NoError(t, err) + require.NoError(t, os.WriteFile(testConfig.SummaryPath, append(encoded, '\n'), 0o600)) + } if testConfig.Cleanup { validator.Cleanup(ctx) diff --git a/test/integration/state/template_test.go b/test/integration/state/template_test.go index f42f48cf09..bd02855807 100644 --- a/test/integration/state/template_test.go +++ b/test/integration/state/template_test.go @@ -2,7 +2,10 @@ package state import ( "os" + "os/exec" "path/filepath" + "runtime" + "strconv" "strings" "testing" @@ -10,65 +13,578 @@ import ( "sigs.k8s.io/yaml" ) -func TestMigrationFaultInjectionTemplateContract(t *testing.T) { - path := filepath.Join("..", "..", "..", ".pipelines", "cni", "load-test-templates", "migration-fault-injection-template.yaml") - raw, err := os.ReadFile(path) - require.NoError(t, err) +const repositoryRoot = "../../.." + +type handoffTransition struct { + Job string `json:"job"` + DependsOn string `json:"dependsOn"` + BaselineTransition string `json:"baselineTransition"` + Name string `json:"transition"` + Action string `json:"action"` + Backend string `json:"backend"` + CNI string `json:"cni"` + ManageEndpointState string `json:"manageEndpointState"` + InitializeFromCNI string `json:"initializeFromCNI"` + EnableStateMigration string `json:"enableStateMigration"` + StateRelation string `json:"stateRelation"` + BootRelation string `json:"bootRelation"` + PodRelation string `json:"podRelation"` + ExternalFault bool `json:"externalFault"` +} + +func TestOwnershipHandoffTemplateContract(t *testing.T) { + transitions := readHandoffTransitions(t) + require.Len(t, transitions, 18) + + for index, transition := range transitions { + number := 11 + index + require.True(t, strings.HasPrefix(transition.Name, strconv.Itoa(number)+"-")) + if index == 0 { + require.Equal(t, "final_same_boot_restart", transition.DependsOn) + require.Equal(t, "10-final-same-boot-restart", transition.BaselineTransition) + } else { + require.Equal(t, transitions[index-1].Job, transition.DependsOn) + require.Equal(t, transitions[index-1].Name, transition.BaselineTransition) + } + + if transition.Backend == "bolt" { + require.Equal(t, "true", transition.ManageEndpointState, + "supported Bolt runtime must always be CNS-owned") + if transition.EnableStateMigration == "true" { + require.Equal(t, "true", transition.InitializeFromCNI, + "Bolt import requires the stateful CNI source") + } + } + if transition.CNI == "stateless" { + require.Equal(t, "true", transition.ManageEndpointState) + require.Equal(t, "false", transition.InitializeFromCNI) + } + } + + require.Equal(t, "configure-state-import", transitions[1].Action) + require.Equal(t, "json", transitions[1].Backend) + require.Equal(t, "cniv2", transitions[1].CNI) + require.Equal(t, "true", transitions[1].EnableStateMigration) + require.Equal(t, "stateless", transitions[2].CNI) + require.Equal(t, "bolt", transitions[3].Backend) + + for _, index := range []int{5, 11} { + reverse := transitions[index] + require.Equal(t, "rollback-json-and-clear-endpoints", reverse.Action) + require.Equal(t, "json", reverse.Backend) + require.Equal(t, "cniv2", reverse.CNI) + require.Equal(t, "false", reverse.ManageEndpointState) + require.Equal(t, "exact", reverse.PodRelation) + require.Equal(t, "same", reverse.BootRelation) + } + + require.Equal(t, "configure-state-import", transitions[7].Action) + require.Equal(t, "bolt", transitions[7].Backend) + require.Equal(t, "cniv2", transitions[7].CNI) + require.Equal(t, "true", transitions[7].InitializeFromCNI) + require.Equal(t, "true", transitions[7].EnableStateMigration) + require.Equal(t, "configure-state", transitions[8].Action) + require.Equal(t, "false", transitions[8].InitializeFromCNI) + require.Equal(t, "stateless", transitions[9].CNI) + + require.Equal(t, "configure-state-import", transitions[13].Action) + require.Equal(t, "true", transitions[13].InitializeFromCNI) + require.Equal(t, "stateless", transitions[14].CNI) + require.Equal(t, "external-fault", transitions[15].Action) + require.True(t, transitions[15].ExternalFault) + require.Equal(t, "same-boot-restart", transitions[16].Action) + require.Equal(t, "node-reboot", transitions[17].Action) + require.Equal(t, "identity", transitions[17].PodRelation) + + for _, transition := range transitions[15:] { + require.Equal(t, "bolt", transition.Backend) + require.Equal(t, "true", transition.ManageEndpointState, + "handoff must remain CNS-owned after churn begins") + } +} + +func TestOwnershipHandoffPipelineContract(t *testing.T) { + raw := readFile(t, pipelinePath("pipeline.yaml")) + text := string(raw) + for _, expected := range []string{ + "pr: none", + "trigger: none", + `default: "1.34"`, + `ACN_VERSION: "r22-$(Build.BuildId)"`, + `CNI_VERSION: "r22-$(Build.BuildId)"`, + `CNS_VERSION: "r22-$(Build.BuildId)"`, + `AZURE_IPAM_VERSION: "r22-$(Build.BuildId)"`, + "$(Build.SourceVersion)", + "../../containers/container-template.yaml", + "../../containers/manifest-template.yaml", + } { + require.Contains(t, text, expected) + } + for _, forbidden := range []string{ + "enableFaultInjectionHooks", + "CNI-managed Bolt", + } { + require.NotContains(t, text, forbidden) + } var document struct { - Parameters map[string]any `json:"parameters"` - Steps []map[string]any `json:"steps"` + Stages []struct { + Template string `json:"template"` + Parameters map[string]any `json:"parameters"` + } `json:"stages"` } require.NoError(t, yaml.Unmarshal(raw, &document)) - for _, parameter := range []string{ - "clusterName", - "os", - "cni", - "scenario", - "scaleReplicas", - "timeoutMinutes", - "testTimeoutMinutes", - "taskTimeoutMinutes", - "runID", - "artifactName", - "workloadImage", + lanes := map[string]struct{}{} + handoffLanes := []string{} + for _, stage := range document.Stages { + if stage.Template != "lane.stage.yaml" { + continue + } + name, ok := stage.Parameters["name"].(string) + require.True(t, ok) + lanes[name] = struct{}{} + require.Equal(t, "false", stage.Parameters["enableStateMigration"]) + if stage.Parameters["enableOwnershipHandoff"] == true { + handoffLanes = append(handoffLanes, name) + } + } + require.Len(t, lanes, 5) + require.Equal(t, []string{"windows_podsubnet"}, handoffLanes) +} + +func TestMigrationTemplateSafetyContract(t *testing.T) { + lane := string(readFile(t, pipelinePath("lane.stage.yaml"))) + transition := string(readFile(t, pipelinePath("transition.steps.yaml"))) + capture := string(readFile(t, pipelinePath("capture-transition.steps.yaml"))) + install := string(readFile(t, pipelinePath("install-components.steps.yaml"))) + + for _, expected := range []string{ + `KUBECONFIG: "$(Agent.TempDirectory)/kubeconfig-$(Build.BuildId)-$(System.JobId)"`, + "condition: always()", + "deleteResources: \"true\"", + "handoff_reboot", + "podRelation: identity", + } { + require.Contains(t, lane, expected) + } + for _, expected := range []string{ + ".EnableBoltStateStore = $enableBolt", + ".EnablePersistentStateDebug = $enableDebug", + ".EnablePersistentStateFaults = false", + "rollback-json-and-clear-endpoints", + "external-pod-deletion", + "kubectl delete pod", + "bash patch-kubeclusterconfig.sh", + `grep -Fxq "All nodes patched successfully"`, + "Cannot stop service", + "service cannot accept control messages", + "Start-Service hns -ErrorAction Stop", + "VALIDATE_SUMMARY_PATH", + "VALIDATE_STATE_BACKEND", + "persistent-state/snapshot", } { - require.Contains(t, document.Parameters, parameter) + require.Contains(t, transition, expected) } - require.Contains(t, document.Parameters["runID"], "$(System.JobId)") - require.Contains(t, document.Parameters["artifactName"], "$(System.JobId)") + require.Less(t, + strings.Index(transition, "bash patch-kubeclusterconfig.sh"), + strings.Index(transition, "az vmss restart"), + ) + require.Less(t, + strings.Index(transition, "displayName: Validate strict"), + strings.Index(transition, "template: capture-transition.steps.yaml"), + ) + for _, forbidden := range []string{ + "|| true", + "migration-fault-injection-template", + "faultinjection", + "CNS_TEST_FAULT", + "pkill", + "killall", + "retryCountOnTaskFailure", + } { + require.NotContains(t, transition, forbidden) + } + + require.Contains(t, capture, "Validate strict persistent state metadata") + require.Contains(t, capture, "persistent-state/status") + require.Contains(t, capture, "persistent-state/snapshot") + require.Contains(t, capture, "continueOnError: true") + require.Contains(t, capture, "one or more best-effort transition evidence captures failed") + require.NotContains(t, capture, "retryCountOnTaskFailure") - var inlineScript string - var publishAlways bool - for _, step := range document.Steps { - if inputs, ok := step["inputs"].(map[string]any); ok { - if script, ok := inputs["inlineScript"].(string); ok { - inlineScript = script + for _, expected := range []string{ + ".EnableBoltStateStore = false", + ".EnablePersistentStateDebug = false", + ".EnablePersistentStateFaults = false", + `.StateStoreBackend = "json"`, + } { + require.Contains(t, install, expected) + } +} + +func TestMigrationTemplatesParseAndResolve(t *testing.T) { + files, err := filepath.Glob(pipelinePath("*.yaml")) + require.NoError(t, err) + require.Len(t, files, 6) + localFiles := make(map[string]string, len(files)) + for _, path := range files { + localFiles[filepath.Base(path)] = path + } + for _, sourcePath := range files { + t.Run(filepath.Base(sourcePath), func(t *testing.T) { + raw := readFile(t, sourcePath) + var source any + require.NoError(t, yaml.Unmarshal(raw, &source)) + for _, reference := range collectTemplateReferences(source) { + targetPath, local := localFiles[filepath.Base(reference.Template)] + if !local { + continue + } + requireTemplateParameters(t, targetPath, reference.Parameters) } + }) + } +} + +type templateReference struct { + Template string + Parameters map[string]any +} + +func collectTemplateReferences(value any) []templateReference { + var references []templateReference + switch typed := value.(type) { + case map[string]any: + template, hasTemplate := typed["template"].(string) + if hasTemplate { + parameters, _ := typed["parameters"].(map[string]any) + references = append(references, templateReference{ + Template: template, + Parameters: parameters, + }) + } + for _, nested := range typed { + references = append(references, collectTemplateReferences(nested)...) } - if step["task"] == "PublishPipelineArtifact@1" && step["condition"] == "always()" { - publishAlways = true + case []any: + for _, nested := range typed { + references = append(references, collectTemplateReferences(nested)...) } } - require.NotEmpty(t, inlineScript) - require.True(t, publishAlways) - - for _, value := range []string{ - "MIGRATION_FAULT_SCENARIO", - "MIGRATION_FAULT_OS", - "MIGRATION_FAULT_CNI", - "MIGRATION_FAULT_RUN_ID", - "MIGRATION_FAULT_ARTIFACT_DIR", - "VALIDATE_STATE_BACKEND=bolt", - "export KUBECONFIG=", - "-test-kubeconfig=\"$KUBECONFIG\"", - "./test/integration/state", - "tee \"$artifactDir/go-test.log\"", - } { - require.Contains(t, inlineScript, value) + return references +} + +func requireTemplateParameters(t *testing.T, targetPath string, passed map[string]any) { + t.Helper() + var target struct { + Parameters []map[string]any `json:"parameters"` } - for _, forbidden := range []string{"killall", "pkill", "rollout restart"} { - require.NotContains(t, strings.ToLower(inlineScript), forbidden) + require.NoError(t, yaml.Unmarshal(readFile(t, targetPath), &target)) + declared := make(map[string]bool, len(target.Parameters)) + for _, parameter := range target.Parameters { + name, ok := parameter["name"].(string) + require.True(t, ok, "parameter name in %s", targetPath) + _, optional := parameter["default"] + declared[name] = optional } + for name := range passed { + _, ok := declared[name] + require.True(t, ok, "unexpected parameter %q for %s", name, targetPath) + } + for name, optional := range declared { + if optional { + continue + } + _, ok := passed[name] + require.True(t, ok, "required parameter %q is missing for %s", name, targetPath) + } +} + +func TestCompareStateMigrationSummariesPodIdentity(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("bash validation runs on Linux pipeline agents") + } + + tests := []struct { + name string + candidatePods string + wantErr bool + }{ + { + name: "IP churn preserves identity", + candidatePods: `[{ + "namespace":"load-test","name":"pod-1","nodeName":"node-1", + "phase":"Running","podIPs":["10.0.0.5"] + }]`, + }, + { + name: "pod movement fails", + candidatePods: `[{ + "namespace":"load-test","name":"pod-1","nodeName":"node-2", + "phase":"Running","podIPs":["10.0.0.5"] + }]`, + wantErr: true, + }, + { + name: "phase change fails", + candidatePods: `[{ + "namespace":"load-test","name":"pod-1","nodeName":"node-1", + "phase":"Pending","podIPs":["10.0.0.5"] + }]`, + wantErr: true, + }, + { + name: "name change fails", + candidatePods: `[{ + "namespace":"load-test","name":"pod-2","nodeName":"node-1", + "phase":"Running","podIPs":["10.0.0.5"] + }]`, + wantErr: true, + }, + { + name: "count reduction fails", + candidatePods: `[]`, + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + fixture := newCompareFixture(t) + fixture.write("candidate-pods.json", tt.candidatePods) + output, err := fixture.run("bolt", "none", "none", "identity") + if tt.wantErr { + require.Error(t, err) + return + } + require.NoError(t, err, output) + }) + } +} + +func TestCompareStateMigrationSummariesRejectsInvalidState(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("bash validation runs on Linux pipeline agents") + } + + tests := []struct { + name string + candidate string + relation string + }{ + {name: "malformed", candidate: `{`, relation: "none"}, + {name: "empty checks", candidate: `{"stateBackend":"json","checks":[]}`, relation: "none"}, + { + name: "malformed IP", + candidate: `{"stateBackend":"json","checks":[{ + "checkName":"state","nodeName":"node-1","livePodCount":1, + "expected":[{"podID":"pod-1","ip":"not-an-ip"}], + "actual":[{"podID":"pod-1","ip":"not-an-ip"}] + }]}`, + relation: "none", + }, + { + name: "unknown field", + candidate: `{"stateBackend":"json","unexpected":true,"checks":[{ + "checkName":"state","nodeName":"node-1","livePodCount":1, + "expected":[{"podID":"pod-1","ip":"10.0.0.2"}], + "actual":[{"podID":"pod-1","ip":"10.0.0.2"}] + }]}`, + relation: "none", + }, + { + name: "duplicate check", + candidate: `{"stateBackend":"json","checks":[ + {"checkName":"state","nodeName":"node-1","livePodCount":1,"expected":[{"podID":"pod-1","ip":"10.0.0.2"}],"actual":[{"podID":"pod-1","ip":"10.0.0.2"}]}, + {"checkName":"state","nodeName":"node-1","livePodCount":1,"expected":[{"podID":"pod-1","ip":"10.0.0.2"}],"actual":[{"podID":"pod-1","ip":"10.0.0.2"}]} + ]}`, + relation: "none", + }, + { + name: "duplicate identity", + candidate: `{"stateBackend":"json","checks":[{ + "checkName":"state","nodeName":"node-1","livePodCount":1, + "expected":[{"podID":"pod-1","ip":"10.0.0.2"},{"podID":"pod-1","ip":"10.0.0.2"}], + "actual":[{"podID":"pod-1","ip":"10.0.0.2"},{"podID":"pod-1","ip":"10.0.0.2"}] + }]}`, + relation: "none", + }, + { + name: "reduced exact results", + candidate: `{"stateBackend":"json","checks":[{ + "checkName":"state","nodeName":"node-1","livePodCount":1, + "expected":[{"podID":"pod-1","ip":"10.0.0.2"}], + "actual":[{"podID":"pod-1","ip":"10.0.0.2"}] + }]}`, + relation: "exact", + }, + { + name: "reduced changed results", + candidate: `{"stateBackend":"json","checks":[{ + "checkName":"state","nodeName":"node-1","livePodCount":2, + "expected":[{"podID":"pod-1","ip":"10.0.0.2"},{"podID":"pod-2","ip":"10.0.0.3"}], + "actual":[{"podID":"pod-1","ip":"10.0.0.2"},{"podID":"pod-2","ip":"10.0.0.3"}] + }]}`, + relation: "changed", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + fixture := newCompareFixture(t) + fixture.write("candidate-summary.json", tt.candidate) + output, err := fixture.run("json", tt.relation, "none", "exact") + require.Error(t, err, output) + }) + } +} + +func TestCompareStateMigrationSummariesBootRelations(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("bash validation runs on Linux pipeline agents") + } + + tests := []struct { + name string + candidateBoot string + relation string + wantErr bool + }{ + {name: "same boot", candidateBoot: "boot-1", relation: "same"}, + {name: "changed boot", candidateBoot: "boot-2", relation: "changed"}, + {name: "unchanged rejected", candidateBoot: "boot-1", relation: "changed", wantErr: true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + fixture := newCompareFixture(t) + fixture.write("candidate-node-boots.json", `[{"nodeName":"node-1","bootID":"`+tt.candidateBoot+`"}]`) + output, err := fixture.run("bolt", "none", tt.relation, "exact") + if tt.wantErr { + require.Error(t, err) + return + } + require.NoError(t, err, output) + }) + } +} + +func readHandoffTransitions(t *testing.T) []handoffTransition { + t.Helper() + raw := readFile(t, pipelinePath("handoff.jobs.yaml")) + var document struct { + Parameters []struct { + Name string `json:"name"` + Default []handoffTransition `json:"default"` + } `json:"parameters"` + } + require.NoError(t, yaml.Unmarshal(raw, &document)) + for _, parameter := range document.Parameters { + if parameter.Name == "transitions" { + return parameter.Default + } + } + t.Fatal("transitions parameter is missing") + return nil +} + +func pipelinePath(name string) string { + return filepath.Join(repositoryRoot, ".pipelines", "cni", "state-migration", name) +} + +func readFile(t *testing.T, path string) []byte { + t.Helper() + raw, err := os.ReadFile(path) + require.NoError(t, err) + return raw +} + +type compareFixture struct { + t *testing.T + baselineDir string + candidateDir string +} + +func newCompareFixture(t *testing.T) compareFixture { + t.Helper() + dir, err := os.MkdirTemp(".", ".r22-fixture-") + require.NoError(t, err) + t.Cleanup(func() { + require.NoError(t, os.RemoveAll(dir)) + }) + baselineDir := filepath.Join(dir, "baseline") + candidateDir := filepath.Join(dir, "candidate") + require.NoError(t, os.Mkdir(baselineDir, 0o700)) + require.NoError(t, os.Mkdir(candidateDir, 0o700)) + fixture := compareFixture{ + t: t, + baselineDir: baselineDir, + candidateDir: candidateDir, + } + fixture.write("baseline-summary.json", `{"stateBackend":"bolt","checks":[ + {"checkName":"state","nodeName":"node-1","livePodCount":1,"expected":[{"podID":"pod-1","ip":"10.0.0.2"}],"actual":[{"podID":"pod-1","ip":"10.0.0.2"}]}, + {"checkName":"cache","nodeName":"node-1","livePodCount":1,"expected":[{"podID":"pod-1","ip":"10.0.0.2"}],"actual":[{"podID":"pod-1","ip":"10.0.0.2"}]} + ]}`) + fixture.write("candidate-summary.json", string(readFile(t, fixture.path("baseline-summary.json")))) + fixture.write("baseline-pods.json", `[{ + "namespace":"load-test","name":"pod-1","nodeName":"node-1", + "phase":"Running","podIPs":["10.0.0.4"] + }]`) + fixture.write("candidate-pods.json", string(readFile(t, fixture.path("baseline-pods.json")))) + fixture.write("baseline-node-boots.json", `[{"nodeName":"node-1","bootID":"boot-1"}]`) + fixture.write("candidate-node-boots.json", `[{"nodeName":"node-1","bootID":"boot-1"}]`) + require.NoError(t, os.Mkdir(filepath.Join(baselineDir, "persistent-debug"), 0o700)) + require.NoError(t, os.Mkdir(filepath.Join(candidateDir, "persistent-debug"), 0o700)) + metadata := `{ + "nodeName":"node-1", + "status":{"backend":"bbolt","authority":"bolt","schemaVersion":1,"generation":2,"bootPresent":true,"storagePresent":true,"databaseBytes":4096,"invariantStatus":"healthy"}, + "snapshot":{"Metadata":{"authority":"bolt","schemaVersion":1,"generation":2,"bootID":"boot-1"}} + }` + fixture.write(filepath.Join("baseline-metadata", "node-1.json"), metadata) + fixture.write(filepath.Join("candidate-metadata", "node-1.json"), metadata) + return fixture +} + +func (f compareFixture) write(name, contents string) { + f.t.Helper() + require.NoError(f.t, os.WriteFile(f.path(name), []byte(contents), 0o600)) +} + +func (f compareFixture) path(name string) string { + switch { + case strings.HasPrefix(name, "baseline-metadata/"): + return filepath.Join(f.baselineDir, "persistent-debug", strings.TrimPrefix(name, "baseline-metadata/")) + case strings.HasPrefix(name, "candidate-metadata/"): + return filepath.Join(f.candidateDir, "persistent-debug", strings.TrimPrefix(name, "candidate-metadata/")) + case strings.HasPrefix(name, "baseline-"): + return filepath.Join(f.baselineDir, strings.TrimPrefix(name, "baseline-")) + case strings.HasPrefix(name, "candidate-"): + return filepath.Join(f.candidateDir, strings.TrimPrefix(name, "candidate-")) + default: + f.t.Fatalf("fixture path %q has no baseline or candidate prefix", name) + return "" + } +} + +func (f compareFixture) run(backend, stateRelation, bootRelation, podRelation string) (string, error) { + f.t.Helper() + script := filepath.Join(repositoryRoot, "hack", "scripts", "compare-state-migration-summaries.sh") + command := exec.Command( + "bash", + script, + filepath.Join(f.baselineDir, "summary.json"), + filepath.Join(f.candidateDir, "summary.json"), + backend, + "bolt", + "1", + stateRelation, + bootRelation, + podRelation, + filepath.Join(f.baselineDir, "pods.json"), + filepath.Join(f.candidateDir, "pods.json"), + filepath.Join(f.baselineDir, "persistent-debug"), + filepath.Join(f.candidateDir, "persistent-debug"), + ) + output, err := command.CombinedOutput() + return string(output), err } diff --git a/test/validate/summary_capture_test.go b/test/validate/summary_capture_test.go new file mode 100644 index 0000000000..f6d4c206ba --- /dev/null +++ b/test/validate/summary_capture_test.go @@ -0,0 +1,55 @@ +package validate + +import ( + "net/netip" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestBuildValidationCheckSummary(t *testing.T) { + tests := []struct { + name string + state map[string]string + actualIPs []string + wantErr string + }{ + { + name: "captures sorted exact identities", + state: map[string]string{ + "10.0.0.3": "namespace/pod-b", + "10.0.0.2": "namespace/pod-a", + }, + actualIPs: []string{"10.0.0.3", "10.0.0.2"}, + }, + { + name: "rejects malformed state IP", + state: map[string]string{"bad": "namespace/pod"}, + actualIPs: []string{"10.0.0.2"}, + wantErr: "invalid state IP", + }, + { + name: "rejects unowned live IP", + state: map[string]string{"10.0.0.2": "namespace/pod"}, + actualIPs: []string{"10.0.0.3"}, + wantErr: "has no validated state owner", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + summary, err := buildValidationCheckSummary("state", "node-1", tt.state, tt.actualIPs) + if tt.wantErr != "" { + require.ErrorContains(t, err, tt.wantErr) + return + } + require.NoError(t, err) + require.Equal(t, 2, summary.LivePodCount) + require.Equal(t, []PodIPIdentity{ + {PodID: "namespace/pod-a", IP: netip.MustParseAddr("10.0.0.2")}, + {PodID: "namespace/pod-b", IP: netip.MustParseAddr("10.0.0.3")}, + }, summary.Expected) + require.Equal(t, summary.Expected, summary.Actual) + }) + } +} diff --git a/test/validate/validate.go b/test/validate/validate.go index 2e368ee25a..c326b3758c 100644 --- a/test/validate/validate.go +++ b/test/validate/validate.go @@ -4,6 +4,8 @@ import ( "context" "encoding/json" "log" + "net/netip" + "sort" "time" acnk8s "github.com/Azure/azure-container-networking/test/internal/kubernetes" @@ -52,6 +54,7 @@ type Validator struct { restartCase bool os string poolLabelSelector string + summaries []ValidationCheckSummary } type check struct { @@ -131,6 +134,7 @@ func (v *Validator) Validate(ctx context.Context) error { } func (v *Validator) ValidateStateFile(ctx context.Context) error { + v.summaries = []ValidationCheckSummary{} for _, check := range v.checks { err := v.validateIPs(ctx, check.stateFileIPs, check.cmd, check.name, check.podNamespace, check.podLabelSelector, check.containerName) if err != nil { @@ -150,6 +154,8 @@ func (v *Validator) validateIPs(ctx context.Context, stateFileIps stateFileIpsFu for index := range nodes.Items { node := nodes.Items[index] var comparisonErr error + observedState := map[string]string{} + var livePodIPs []string _, converged, validationErr := runValidationAttempts(ctx, stateValidationAttempts, stateValidationInterval, func() (bool, error) { pod, err := acnk8s.GetPodsByNode(ctx, v.clientset, namespace, labelSelector, node.Name) if err != nil { @@ -170,6 +176,8 @@ func (v *Validator) validateIPs(ctx context.Context, stateFileIps stateFileIpsFu } if len(filePodIps) == 0 && v.restartCase { log.Printf("No pods found on node %s", node.Name) + observedState = map[string]string{} + livePodIPs = nil return true, nil } @@ -177,8 +185,26 @@ func (v *Validator) validateIPs(ctx context.Context, stateFileIps stateFileIpsFu if hasL7PolicyEnabled(ctx, v.clientset, node.Name) { podIps = append(podIps, getCiliumInternalEndpointIPs(ctx, v.clientset, v.config, node.Name)...) } + if len(filePodIps) == 0 && len(podIps) > 0 { + comparisonErr = errors.Errorf( + "state is empty on node %s with %d live pod IPs", + node.Name, + len(podIps), + ) + return false, nil + } + + state := make(map[string]string, len(filePodIps)) + for ip, owner := range filePodIps { + state[ip] = owner + } comparisonErr = compareIPs(filePodIps, podIps) - return comparisonErr == nil, nil + if comparisonErr != nil { + return false, nil + } + observedState = state + livePodIPs = podIps + return true, nil }) if validationErr != nil { return validationErr @@ -186,7 +212,13 @@ func (v *Validator) validateIPs(ctx context.Context, stateFileIps stateFileIpsFu if !converged { return errors.Wrapf(comparisonErr, "State file validation failed for %s on node %s", checkType, node.Name) } + summary, err := buildValidationCheckSummary(checkType, node.Name, observedState, livePodIPs) + if err != nil { + return errors.Wrapf(err, "failed to summarize %s state on node %s", checkType, node.Name) + } + v.summaries = append(v.summaries, summary) } + log.Printf("State file validation for %s passed", checkType) return nil } @@ -231,6 +263,74 @@ func runValidationAttempts( } } +// Summary returns the exact state identities observed by the latest validation. +func (v *Validator) Summary(backend StateBackend) ValidationSummary { + checks := make([]ValidationCheckSummary, len(v.summaries)) + copy(checks, v.summaries) + return ValidationSummary{ + StateBackend: backend, + Checks: checks, + } +} + +func buildValidationCheckSummary( + checkName string, + nodeName string, + state map[string]string, + actualIPs []string, +) (ValidationCheckSummary, error) { + expected := make([]PodIPIdentity, 0, len(state)) + actual := make([]PodIPIdentity, 0, len(actualIPs)) + owners := make(map[string]struct{}, len(state)) + ownerByIP := make(map[netip.Addr]string, len(state)) + + for rawIP, owner := range state { + ip, err := netip.ParseAddr(rawIP) + if err != nil { + return ValidationCheckSummary{}, errors.Wrapf(err, "invalid state IP %q", rawIP) + } + ip = ip.Unmap() + if _, exists := ownerByIP[ip]; exists { + return ValidationCheckSummary{}, errors.Errorf("duplicate state IP %q", ip) + } + ownerByIP[ip] = owner + expected = append(expected, PodIPIdentity{PodID: owner, IP: ip}) + owners[owner] = struct{}{} + } + for _, rawIP := range actualIPs { + ip, err := netip.ParseAddr(rawIP) + if err != nil { + return ValidationCheckSummary{}, errors.Wrapf(err, "invalid live pod IP %q", rawIP) + } + ip = ip.Unmap() + owner, ok := ownerByIP[ip] + if !ok { + return ValidationCheckSummary{}, errors.Errorf("live pod IP %q has no validated state owner", rawIP) + } + actual = append(actual, PodIPIdentity{PodID: owner, IP: ip}) + } + sort.Slice(expected, func(i, j int) bool { + if expected[i].PodID == expected[j].PodID { + return expected[i].IP.Less(expected[j].IP) + } + return expected[i].PodID < expected[j].PodID + }) + sort.Slice(actual, func(i, j int) bool { + if actual[i].PodID == actual[j].PodID { + return actual[i].IP.Less(actual[j].IP) + } + return actual[i].PodID < actual[j].PodID + }) + + return ValidationCheckSummary{ + CheckName: checkName, + NodeName: nodeName, + LivePodCount: len(owners), + Expected: expected, + Actual: actual, + }, nil +} + func validateNodeProperties(nodes *corev1.NodeList, labels map[string]string, expectedIPCount int) error { log.Print("Validating Node properties") From 404840d92e0b22b8e0d27dab09493d11db840e42 Mon Sep 17 00:00:00 2001 From: Evan Baker Date: Fri, 24 Jul 2026 13:27:27 +0000 Subject: [PATCH 2/6] fix: preserve validator semantics in migration gate --- .../cni/state-migration/transition.steps.yaml | 28 ++--- hack/scripts/run-state-migration-validator.sh | 65 ++++++++++ test/integration/state/template_test.go | 115 ++++++++++++++++++ test/validate/summary_capture_test.go | 40 ++++++ test/validate/validate.go | 21 ++-- 5 files changed, 248 insertions(+), 21 deletions(-) create mode 100755 hack/scripts/run-state-migration-validator.sh diff --git a/.pipelines/cni/state-migration/transition.steps.yaml b/.pipelines/cni/state-migration/transition.steps.yaml index 364437ee9e..7fea7e057a 100644 --- a/.pipelines/cni/state-migration/transition.steps.yaml +++ b/.pipelines/cni/state-migration/transition.steps.yaml @@ -958,20 +958,20 @@ steps: >"$evidenceDir/cni-type.json" fi - restartCase=true - if [[ "${{ parameters.action }}" == "json-baseline" ]]; then - restartCase=false - fi - - VALIDATE_SUMMARY_PATH="$summaryPath" \ - VALIDATE_STATEFILE=true \ - VALIDATE_STATE_BACKEND="${{ parameters.expectedBackend }}" \ - VALIDATE_CONVERGENCE_ATTEMPTS=10 \ - VALIDATE_CONVERGENCE_INTERVAL_SECONDS=15 \ - make test-validate-state \ - OS_TYPE=${{ parameters.os }} \ - RESTART_CASE="$restartCase" \ - CNI_TYPE=${{ parameters.cni }} + validatorDiagnostics="$evidenceDir/validator-attempts" + bash hack/scripts/run-state-migration-validator.sh \ + 10 \ + 15 \ + "$validatorDiagnostics" \ + -- \ + env \ + VALIDATE_SUMMARY_PATH="$summaryPath" \ + VALIDATE_STATEFILE=true \ + VALIDATE_STATE_BACKEND="${{ parameters.expectedBackend }}" \ + make test-validate-state \ + OS_TYPE=${{ parameters.os }} \ + RESTART_CASE=false \ + CNI_TYPE=${{ parameters.cni }} test -s "$summaryPath" jq -e \ diff --git a/hack/scripts/run-state-migration-validator.sh b/hack/scripts/run-state-migration-validator.sh new file mode 100755 index 0000000000..853fd43394 --- /dev/null +++ b/hack/scripts/run-state-migration-validator.sh @@ -0,0 +1,65 @@ +#!/usr/bin/env bash + +set -euo pipefail + +if [[ $# -lt 5 || $4 != "--" ]]; then + echo "usage: $0 -- [args...]" >&2 + exit 2 +fi + +attempts=$1 +interval_seconds=$2 +diagnostics_dir=$3 +shift 4 + +if [[ ! "$attempts" =~ ^[1-9][0-9]*$ ]]; then + echo "attempts must be a positive integer: $attempts" >&2 + exit 2 +fi +if [[ ! "$interval_seconds" =~ ^[0-9]+$ ]]; then + echo "interval-seconds must be a nonnegative integer: $interval_seconds" >&2 + exit 2 +fi + +mkdir -p "$diagnostics_dir" +rm -f "$diagnostics_dir"/attempt-*.log +: >"$diagnostics_dir/status.tsv" + +nonretryable_pattern='state is empty|failed to (unmarshal|parse)|invalid (state |live pod )?ip|duplicate|unsupported|corrupt|malformed|schema mismatch|summary .*invalid' +retryable_pattern='state file validation failed|failed to exec into privileged pod|failed to get privileged pod|there are no privileged pods|connection refused|i/o timeout|tls handshake timeout|transport is closing|unexpected eof|not yet converged|timed out waiting' + +for ((attempt = 1; attempt <= attempts; attempt++)); do + log_path=$(printf "%s/attempt-%02d.log" "$diagnostics_dir" "$attempt") + echo "state validator attempt $attempt of $attempts" + + set +e + "$@" 2>&1 | tee "$log_path" + pipeline_status=("${PIPESTATUS[@]}") + set -e + + command_status=${pipeline_status[0]} + tee_status=${pipeline_status[1]} + printf "%d\t%d\n" "$attempt" "$command_status" >>"$diagnostics_dir/status.tsv" + if ((tee_status != 0)); then + echo "writing validator diagnostics failed with status $tee_status" >&2 + exit "$tee_status" + fi + if ((command_status == 0)); then + exit 0 + fi + if grep -Eiq "$nonretryable_pattern" "$log_path"; then + echo "state validator reported a nonretryable semantic failure on attempt $attempt" >&2 + exit "$command_status" + fi + if ! grep -Eiq "$retryable_pattern" "$log_path"; then + echo "state validator failure was not classified as retryable on attempt $attempt" >&2 + exit "$command_status" + fi + if ((attempt == attempts)); then + echo "state validator exhausted $attempts attempts" >&2 + exit "$command_status" + fi + + echo "state validator has not converged; retrying in $interval_seconds seconds" >&2 + sleep "$interval_seconds" +done diff --git a/test/integration/state/template_test.go b/test/integration/state/template_test.go index bd02855807..0915d61244 100644 --- a/test/integration/state/template_test.go +++ b/test/integration/state/template_test.go @@ -182,6 +182,8 @@ func TestMigrationTemplateSafetyContract(t *testing.T) { "Start-Service hns -ErrorAction Stop", "VALIDATE_SUMMARY_PATH", "VALIDATE_STATE_BACKEND", + "run-state-migration-validator.sh", + "RESTART_CASE=false", "persistent-state/snapshot", } { require.Contains(t, transition, expected) @@ -202,6 +204,8 @@ func TestMigrationTemplateSafetyContract(t *testing.T) { "pkill", "killall", "retryCountOnTaskFailure", + "VALIDATE_CONVERGENCE_", + `RESTART_CASE="$restartCase"`, } { require.NotContains(t, transition, forbidden) } @@ -223,6 +227,110 @@ func TestMigrationTemplateSafetyContract(t *testing.T) { } } +func TestStateMigrationValidatorRetry(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("bash validation runs on Linux pipeline agents") + } + + runner := filepath.Join(repositoryRoot, "hack", "scripts", "run-state-migration-validator.sh") + require.NoError(t, exec.Command("bash", "-n", runner).Run()) + + tests := []struct { + name string + mode string + wantAttempts int + wantStatus int + }{ + { + name: "eventual convergence succeeds", + mode: "eventual", + wantAttempts: 3, + }, + { + name: "retryable exhaustion preserves failure", + mode: "exhaust", + wantAttempts: 3, + wantStatus: 7, + }, + { + name: "semantic corruption exits promptly", + mode: "semantic", + wantAttempts: 1, + wantStatus: 9, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + dir, err := os.MkdirTemp(".", ".r22-retry-") + require.NoError(t, err) + t.Cleanup(func() { + require.NoError(t, os.RemoveAll(dir)) + }) + + fixture := filepath.Join(dir, "validator-fixture.sh") + counter := filepath.Join(dir, "counter") + diagnostics := filepath.Join(dir, "diagnostics") + require.NoError(t, os.WriteFile(fixture, []byte(`#!/usr/bin/env bash +set -euo pipefail +mode=$1 +counter=$2 +count=0 +if [[ -s "$counter" ]]; then + count=$(<"$counter") +fi +count=$((count + 1)) +printf '%d' "$count" >"$counter" +case "$mode" in + eventual) + if ((count < 3)); then + echo "State file validation failed for state on node-1" + exit 4 + fi + echo "state converged" + ;; + exhaust) + echo "State file validation failed for state on node-1" + exit 7 + ;; + semantic) + echo "duplicate state IP corruption" + exit 9 + ;; +esac +`), 0o700)) + + command := exec.Command( + "bash", + runner, + "3", + "0", + diagnostics, + "--", + fixture, + tt.mode, + counter, + ) + output, runErr := command.CombinedOutput() + if tt.wantStatus == 0 { + require.NoError(t, runErr, string(output)) + } else { + var exitErr *exec.ExitError + require.ErrorAs(t, runErr, &exitErr, string(output)) + require.Equal(t, tt.wantStatus, exitErr.ExitCode(), string(output)) + } + + attempts := mustAtoi(t, strings.TrimSpace(string(readFile(t, counter)))) + require.Equal(t, tt.wantAttempts, attempts) + logs, err := filepath.Glob(filepath.Join(diagnostics, "attempt-*.log")) + require.NoError(t, err) + require.Len(t, logs, tt.wantAttempts) + status := strings.TrimSpace(string(readFile(t, filepath.Join(diagnostics, "status.tsv")))) + require.Len(t, strings.Split(status, "\n"), tt.wantAttempts) + }) + } +} + func TestMigrationTemplatesParseAndResolve(t *testing.T) { files, err := filepath.Glob(pipelinePath("*.yaml")) require.NoError(t, err) @@ -247,6 +355,13 @@ func TestMigrationTemplatesParseAndResolve(t *testing.T) { } } +func mustAtoi(t *testing.T, value string) int { + t.Helper() + parsed, err := strconv.Atoi(value) + require.NoError(t, err) + return parsed +} + type templateReference struct { Template string Parameters map[string]any diff --git a/test/validate/summary_capture_test.go b/test/validate/summary_capture_test.go index f6d4c206ba..fa05b1fa51 100644 --- a/test/validate/summary_capture_test.go +++ b/test/validate/summary_capture_test.go @@ -53,3 +53,43 @@ func TestBuildValidationCheckSummary(t *testing.T) { }) } } + +func TestEmptyStateValidationMode(t *testing.T) { + tests := []struct { + name string + restartCase bool + wantSkip bool + wantErr bool + }{ + { + name: "existing restart case skips transient empty state", + restartCase: true, + wantSkip: true, + }, + { + name: "strict R22 mode rejects empty state with live pods", + wantErr: true, + }, + { + name: "normal non-restart validation rejects empty state with live pods", + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + skip := skipEmptyStateForRestart(0, tt.restartCase) + require.Equal(t, tt.wantSkip, skip) + if skip { + require.False(t, tt.wantErr) + return + } + err := validateLivePodsHaveState(0, 1) + if tt.wantErr { + require.ErrorContains(t, err, "state is empty with 1 live pod IPs") + return + } + require.NoError(t, err) + }) + } +} diff --git a/test/validate/validate.go b/test/validate/validate.go index c326b3758c..74bf3fea91 100644 --- a/test/validate/validate.go +++ b/test/validate/validate.go @@ -174,7 +174,7 @@ func (v *Validator) validateIPs(ctx context.Context, stateFileIps stateFileIpsFu if err != nil { return false, errors.Wrapf(err, "failed to get pod ips from state file on node %v", node.Name) } - if len(filePodIps) == 0 && v.restartCase { + if skipEmptyStateForRestart(len(filePodIps), v.restartCase) { log.Printf("No pods found on node %s", node.Name) observedState = map[string]string{} livePodIPs = nil @@ -185,12 +185,8 @@ func (v *Validator) validateIPs(ctx context.Context, stateFileIps stateFileIpsFu if hasL7PolicyEnabled(ctx, v.clientset, node.Name) { podIps = append(podIps, getCiliumInternalEndpointIPs(ctx, v.clientset, v.config, node.Name)...) } - if len(filePodIps) == 0 && len(podIps) > 0 { - comparisonErr = errors.Errorf( - "state is empty on node %s with %d live pod IPs", - node.Name, - len(podIps), - ) + if err := validateLivePodsHaveState(len(filePodIps), len(podIps)); err != nil { + comparisonErr = errors.Wrapf(err, "invalid state on node %s", node.Name) return false, nil } @@ -223,6 +219,17 @@ func (v *Validator) validateIPs(ctx context.Context, stateFileIps stateFileIpsFu return nil } +func skipEmptyStateForRestart(stateIPCount int, restartCase bool) bool { + return stateIPCount == 0 && restartCase +} + +func validateLivePodsHaveState(stateIPCount, livePodIPCount int) error { + if stateIPCount == 0 && livePodIPCount > 0 { + return errors.Errorf("state is empty with %d live pod IPs", livePodIPCount) + } + return nil +} + func runValidationAttempts( ctx context.Context, maxAttempts int, From 688b38b74130084016a8db7a85608fe570cdf404 Mon Sep 17 00:00:00 2001 From: GitHub Copilot Date: Fri, 28 Aug 2026 17:57:32 +0000 Subject: [PATCH 3/6] test: keep migration gate fixtures out of the repo tree --- test/integration/state/template_test.go | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/test/integration/state/template_test.go b/test/integration/state/template_test.go index 0915d61244..f099dd7293 100644 --- a/test/integration/state/template_test.go +++ b/test/integration/state/template_test.go @@ -262,11 +262,7 @@ func TestStateMigrationValidatorRetry(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - dir, err := os.MkdirTemp(".", ".r22-retry-") - require.NoError(t, err) - t.Cleanup(func() { - require.NoError(t, os.RemoveAll(dir)) - }) + dir := t.TempDir() fixture := filepath.Join(dir, "validator-fixture.sh") counter := filepath.Join(dir, "counter") @@ -622,11 +618,7 @@ type compareFixture struct { func newCompareFixture(t *testing.T) compareFixture { t.Helper() - dir, err := os.MkdirTemp(".", ".r22-fixture-") - require.NoError(t, err) - t.Cleanup(func() { - require.NoError(t, os.RemoveAll(dir)) - }) + dir := t.TempDir() baselineDir := filepath.Join(dir, "baseline") candidateDir := filepath.Join(dir, "candidate") require.NoError(t, os.Mkdir(baselineDir, 0o700)) From e9ecf6efe8ad98ad26282adf9c305697a3f6a0db Mon Sep 17 00:00:00 2001 From: Audit Dryrun Date: Fri, 28 Aug 2026 22:59:03 +0000 Subject: [PATCH 4/6] test(cni): retain transaction faults in handoff gate --- .pipelines/cni/state-migration/lane.stage.yaml | 3 ++- .../cni/state-migration/transition.steps.yaml | 14 ++++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/.pipelines/cni/state-migration/lane.stage.yaml b/.pipelines/cni/state-migration/lane.stage.yaml index d12cb1738e..d05efe1fff 100644 --- a/.pipelines/cni/state-migration/lane.stage.yaml +++ b/.pipelines/cni/state-migration/lane.stage.yaml @@ -211,7 +211,7 @@ stages: - job: restart_during_scale displayName: 04 Restart during scale dependsOn: same_boot_restart - timeoutInMinutes: 90 + timeoutInMinutes: 180 steps: - template: transition.steps.yaml parameters: @@ -234,6 +234,7 @@ stages: baselineArtifact: state-migration-$(Build.BuildId)-${{ parameters.name }}-03-same-boot-restart stateRelation: changed bootRelation: same + runFaultInjection: ${{ eq(parameters.manageEndpointState, 'true') }} - job: node_reboot displayName: 05 Node reboot diff --git a/.pipelines/cni/state-migration/transition.steps.yaml b/.pipelines/cni/state-migration/transition.steps.yaml index 7fea7e057a..cb18d40c29 100644 --- a/.pipelines/cni/state-migration/transition.steps.yaml +++ b/.pipelines/cni/state-migration/transition.steps.yaml @@ -43,6 +43,9 @@ parameters: - name: podRelation type: string default: inherit + - name: runFaultInjection + type: boolean + default: false - name: injectExternalFault type: boolean default: false @@ -585,6 +588,17 @@ steps: '{daemonsetGenerationBefore: $beforeGeneration, daemonsetGenerationAfter: $afterGeneration, restartCount: 1}' \ >"$evidenceDir/restart.json" + # Endpoint-commit faults require CNS-owned endpoint state. CNI-managed lanes + # still run the mandatory active-scale restart immediately below. + - ${{ if and(eq(parameters.action, 'restart-during-scale'), eq(parameters.runFaultInjection, true)) }}: + - template: ../load-test-templates/migration-fault-injection-template.yaml + parameters: + clusterName: ${{ parameters.clusterName }} + os: ${{ parameters.os }} + cni: ${{ parameters.cni }} + scenario: all + artifactName: state-migration-${{ parameters.lane }}-04-fault-injection + - ${{ if eq(parameters.action, 'restart-during-scale') }}: - task: AzureCLI@2 displayName: Restart CNS during active pod scale From 1298e3f34d096706768e68659ae7222f22d321e5 Mon Sep 17 00:00:00 2001 From: Audit Dryrun Date: Fri, 28 Aug 2026 23:03:11 +0000 Subject: [PATCH 5/6] test(cni): align handoff gate with token-only faults --- .pipelines/cni/state-migration/install-components.steps.yaml | 2 -- .pipelines/cni/state-migration/transition.steps.yaml | 3 --- test/integration/state/template_test.go | 5 ++--- 3 files changed, 2 insertions(+), 8 deletions(-) diff --git a/.pipelines/cni/state-migration/install-components.steps.yaml b/.pipelines/cni/state-migration/install-components.steps.yaml index 61370d4cff..8dfc68fbd4 100644 --- a/.pipelines/cni/state-migration/install-components.steps.yaml +++ b/.pipelines/cni/state-migration/install-components.steps.yaml @@ -112,7 +112,6 @@ steps: --argjson migrate "${{ parameters.enableStateMigration }}" \ '.EnableBoltStateStore = false | .EnablePersistentStateDebug = false - | .EnablePersistentStateFaults = false | .StateStoreBackend = "json" | .StateStoreMode = "normal" | .ManageEndpointState = $manage @@ -131,7 +130,6 @@ steps: --argjson migrate "${{ parameters.enableStateMigration }}" \ '.EnableBoltStateStore == false and .EnablePersistentStateDebug == false - and .EnablePersistentStateFaults == false and .StateStoreBackend == "json" and .StateStoreMode == "normal" and .ManageEndpointState == $manage diff --git a/.pipelines/cni/state-migration/transition.steps.yaml b/.pipelines/cni/state-migration/transition.steps.yaml index cb18d40c29..9c84d4ca0b 100644 --- a/.pipelines/cni/state-migration/transition.steps.yaml +++ b/.pipelines/cni/state-migration/transition.steps.yaml @@ -130,7 +130,6 @@ steps: | .StateStoreMode = $mode | .EnableBoltStateStore = $enableBolt | .EnablePersistentStateDebug = $enableDebug - | .EnablePersistentStateFaults = false | .ManageEndpointState = $manageEndpointState | .InitializeFromCNI = $initializeFromCNI | .EnableStateMigration = $enableStateMigration' \ @@ -368,7 +367,6 @@ steps: and .StateStoreMode == "normal" and .EnableBoltStateStore == $enableBolt and .EnablePersistentStateDebug == $enableDebug - and .EnablePersistentStateFaults == false and .ManageEndpointState == $manageEndpointState and .InitializeFromCNI == $initializeFromCNI and .EnableStateMigration == $enableStateMigration' \ @@ -934,7 +932,6 @@ steps: and .StateStoreMode == "normal" and .EnableBoltStateStore == $enableBolt and .EnablePersistentStateDebug == $enableDebug - and .EnablePersistentStateFaults == false and .ManageEndpointState == $manageEndpointState and .InitializeFromCNI == $initializeFromCNI and .EnableStateMigration == $enableStateMigration' \ diff --git a/test/integration/state/template_test.go b/test/integration/state/template_test.go index f099dd7293..b3cc7835d5 100644 --- a/test/integration/state/template_test.go +++ b/test/integration/state/template_test.go @@ -171,7 +171,6 @@ func TestMigrationTemplateSafetyContract(t *testing.T) { for _, expected := range []string{ ".EnableBoltStateStore = $enableBolt", ".EnablePersistentStateDebug = $enableDebug", - ".EnablePersistentStateFaults = false", "rollback-json-and-clear-endpoints", "external-pod-deletion", "kubectl delete pod", @@ -185,6 +184,8 @@ func TestMigrationTemplateSafetyContract(t *testing.T) { "run-state-migration-validator.sh", "RESTART_CASE=false", "persistent-state/snapshot", + "migration-fault-injection-template.yaml", + "runFaultInjection", } { require.Contains(t, transition, expected) } @@ -198,7 +199,6 @@ func TestMigrationTemplateSafetyContract(t *testing.T) { ) for _, forbidden := range []string{ "|| true", - "migration-fault-injection-template", "faultinjection", "CNS_TEST_FAULT", "pkill", @@ -220,7 +220,6 @@ func TestMigrationTemplateSafetyContract(t *testing.T) { for _, expected := range []string{ ".EnableBoltStateStore = false", ".EnablePersistentStateDebug = false", - ".EnablePersistentStateFaults = false", `.StateStoreBackend = "json"`, } { require.Contains(t, install, expected) From 589aa93614546444c56bd8be6cc89ddd52c2bec2 Mon Sep 17 00:00:00 2001 From: Audit Dryrun Date: Mon, 31 Aug 2026 17:37:26 +0000 Subject: [PATCH 6/6] test(cni): harden handoff validation harness --- test/integration/state/template_test.go | 47 +++++++++++++++---------- test/validate/summary_capture_test.go | 21 ++++++----- test/validate/validate.go | 8 +++-- 3 files changed, 47 insertions(+), 29 deletions(-) diff --git a/test/integration/state/template_test.go b/test/integration/state/template_test.go index b3cc7835d5..aa4923f951 100644 --- a/test/integration/state/template_test.go +++ b/test/integration/state/template_test.go @@ -13,7 +13,13 @@ import ( "sigs.k8s.io/yaml" ) -const repositoryRoot = "../../.." +const ( + repositoryRoot = "../../.." + testBackendBolt = "bolt" + testOSWindows = "windows" + stateRelationNone = "none" + stateRelationChanged = "changed" +) type handoffTransition struct { Job string `json:"job"` @@ -47,7 +53,7 @@ func TestOwnershipHandoffTemplateContract(t *testing.T) { require.Equal(t, transitions[index-1].Name, transition.BaselineTransition) } - if transition.Backend == "bolt" { + if transition.Backend == testBackendBolt { require.Equal(t, "true", transition.ManageEndpointState, "supported Bolt runtime must always be CNS-owned") if transition.EnableStateMigration == "true" { @@ -227,12 +233,12 @@ func TestMigrationTemplateSafetyContract(t *testing.T) { } func TestStateMigrationValidatorRetry(t *testing.T) { - if runtime.GOOS == "windows" { + if runtime.GOOS == testOSWindows { t.Skip("bash validation runs on Linux pipeline agents") } runner := filepath.Join(repositoryRoot, "hack", "scripts", "run-state-migration-validator.sh") - require.NoError(t, exec.Command("bash", "-n", runner).Run()) + require.NoError(t, exec.CommandContext(t.Context(), "bash", "-n", runner).Run()) tests := []struct { name string @@ -293,9 +299,11 @@ case "$mode" in exit 9 ;; esac -`), 0o700)) +`), 0o600)) + require.NoError(t, os.Chmod(fixture, 0o700)) - command := exec.Command( + command := exec.CommandContext( + t.Context(), "bash", runner, "3", @@ -412,7 +420,7 @@ func requireTemplateParameters(t *testing.T, targetPath string, passed map[strin } func TestCompareStateMigrationSummariesPodIdentity(t *testing.T) { - if runtime.GOOS == "windows" { + if runtime.GOOS == testOSWindows { t.Skip("bash validation runs on Linux pipeline agents") } @@ -474,7 +482,7 @@ func TestCompareStateMigrationSummariesPodIdentity(t *testing.T) { } func TestCompareStateMigrationSummariesRejectsInvalidState(t *testing.T) { - if runtime.GOOS == "windows" { + if runtime.GOOS == testOSWindows { t.Skip("bash validation runs on Linux pipeline agents") } @@ -483,8 +491,8 @@ func TestCompareStateMigrationSummariesRejectsInvalidState(t *testing.T) { candidate string relation string }{ - {name: "malformed", candidate: `{`, relation: "none"}, - {name: "empty checks", candidate: `{"stateBackend":"json","checks":[]}`, relation: "none"}, + {name: "malformed", candidate: `{`, relation: stateRelationNone}, + {name: "empty checks", candidate: `{"stateBackend":"json","checks":[]}`, relation: stateRelationNone}, { name: "malformed IP", candidate: `{"stateBackend":"json","checks":[{ @@ -492,7 +500,7 @@ func TestCompareStateMigrationSummariesRejectsInvalidState(t *testing.T) { "expected":[{"podID":"pod-1","ip":"not-an-ip"}], "actual":[{"podID":"pod-1","ip":"not-an-ip"}] }]}`, - relation: "none", + relation: stateRelationNone, }, { name: "unknown field", @@ -501,7 +509,7 @@ func TestCompareStateMigrationSummariesRejectsInvalidState(t *testing.T) { "expected":[{"podID":"pod-1","ip":"10.0.0.2"}], "actual":[{"podID":"pod-1","ip":"10.0.0.2"}] }]}`, - relation: "none", + relation: stateRelationNone, }, { name: "duplicate check", @@ -509,7 +517,7 @@ func TestCompareStateMigrationSummariesRejectsInvalidState(t *testing.T) { {"checkName":"state","nodeName":"node-1","livePodCount":1,"expected":[{"podID":"pod-1","ip":"10.0.0.2"}],"actual":[{"podID":"pod-1","ip":"10.0.0.2"}]}, {"checkName":"state","nodeName":"node-1","livePodCount":1,"expected":[{"podID":"pod-1","ip":"10.0.0.2"}],"actual":[{"podID":"pod-1","ip":"10.0.0.2"}]} ]}`, - relation: "none", + relation: stateRelationNone, }, { name: "duplicate identity", @@ -518,7 +526,7 @@ func TestCompareStateMigrationSummariesRejectsInvalidState(t *testing.T) { "expected":[{"podID":"pod-1","ip":"10.0.0.2"},{"podID":"pod-1","ip":"10.0.0.2"}], "actual":[{"podID":"pod-1","ip":"10.0.0.2"},{"podID":"pod-1","ip":"10.0.0.2"}] }]}`, - relation: "none", + relation: stateRelationNone, }, { name: "reduced exact results", @@ -536,7 +544,7 @@ func TestCompareStateMigrationSummariesRejectsInvalidState(t *testing.T) { "expected":[{"podID":"pod-1","ip":"10.0.0.2"},{"podID":"pod-2","ip":"10.0.0.3"}], "actual":[{"podID":"pod-1","ip":"10.0.0.2"},{"podID":"pod-2","ip":"10.0.0.3"}] }]}`, - relation: "changed", + relation: stateRelationChanged, }, } @@ -551,7 +559,7 @@ func TestCompareStateMigrationSummariesRejectsInvalidState(t *testing.T) { } func TestCompareStateMigrationSummariesBootRelations(t *testing.T) { - if runtime.GOOS == "windows" { + if runtime.GOOS == testOSWindows { t.Skip("bash validation runs on Linux pipeline agents") } @@ -562,8 +570,8 @@ func TestCompareStateMigrationSummariesBootRelations(t *testing.T) { wantErr bool }{ {name: "same boot", candidateBoot: "boot-1", relation: "same"}, - {name: "changed boot", candidateBoot: "boot-2", relation: "changed"}, - {name: "unchanged rejected", candidateBoot: "boot-1", relation: "changed", wantErr: true}, + {name: "changed boot", candidateBoot: "boot-2", relation: stateRelationChanged}, + {name: "unchanged rejected", candidateBoot: "boot-1", relation: stateRelationChanged, wantErr: true}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -675,7 +683,8 @@ func (f compareFixture) path(name string) string { func (f compareFixture) run(backend, stateRelation, bootRelation, podRelation string) (string, error) { f.t.Helper() script := filepath.Join(repositoryRoot, "hack", "scripts", "compare-state-migration-summaries.sh") - command := exec.Command( + command := exec.CommandContext( + f.t.Context(), "bash", script, filepath.Join(f.baselineDir, "summary.json"), diff --git a/test/validate/summary_capture_test.go b/test/validate/summary_capture_test.go index fa05b1fa51..5abab1071d 100644 --- a/test/validate/summary_capture_test.go +++ b/test/validate/summary_capture_test.go @@ -7,6 +7,11 @@ import ( "github.com/stretchr/testify/require" ) +const ( + testCapturedIP1 = "10.0.0.2" + testCapturedIP2 = "10.0.0.3" +) + func TestBuildValidationCheckSummary(t *testing.T) { tests := []struct { name string @@ -17,21 +22,21 @@ func TestBuildValidationCheckSummary(t *testing.T) { { name: "captures sorted exact identities", state: map[string]string{ - "10.0.0.3": "namespace/pod-b", - "10.0.0.2": "namespace/pod-a", + testCapturedIP2: "namespace/pod-b", + testCapturedIP1: "namespace/pod-a", }, - actualIPs: []string{"10.0.0.3", "10.0.0.2"}, + actualIPs: []string{testCapturedIP2, testCapturedIP1}, }, { name: "rejects malformed state IP", state: map[string]string{"bad": "namespace/pod"}, - actualIPs: []string{"10.0.0.2"}, + actualIPs: []string{testCapturedIP1}, wantErr: "invalid state IP", }, { name: "rejects unowned live IP", - state: map[string]string{"10.0.0.2": "namespace/pod"}, - actualIPs: []string{"10.0.0.3"}, + state: map[string]string{testCapturedIP1: "namespace/pod"}, + actualIPs: []string{testCapturedIP2}, wantErr: "has no validated state owner", }, } @@ -46,8 +51,8 @@ func TestBuildValidationCheckSummary(t *testing.T) { require.NoError(t, err) require.Equal(t, 2, summary.LivePodCount) require.Equal(t, []PodIPIdentity{ - {PodID: "namespace/pod-a", IP: netip.MustParseAddr("10.0.0.2")}, - {PodID: "namespace/pod-b", IP: netip.MustParseAddr("10.0.0.3")}, + {PodID: "namespace/pod-a", IP: netip.MustParseAddr(testCapturedIP1)}, + {PodID: "namespace/pod-b", IP: netip.MustParseAddr(testCapturedIP2)}, }, summary.Expected) require.Equal(t, summary.Expected, summary.Actual) }) diff --git a/test/validate/validate.go b/test/validate/validate.go index 74bf3fea91..50cabc24a0 100644 --- a/test/validate/validate.go +++ b/test/validate/validate.go @@ -187,7 +187,7 @@ func (v *Validator) validateIPs(ctx context.Context, stateFileIps stateFileIpsFu } if err := validateLivePodsHaveState(len(filePodIps), len(podIps)); err != nil { comparisonErr = errors.Wrapf(err, "invalid state on node %s", node.Name) - return false, nil + return retryValidationAttempt() } state := make(map[string]string, len(filePodIps)) @@ -196,7 +196,7 @@ func (v *Validator) validateIPs(ctx context.Context, stateFileIps stateFileIpsFu } comparisonErr = compareIPs(filePodIps, podIps) if comparisonErr != nil { - return false, nil + return retryValidationAttempt() } observedState = state livePodIPs = podIps @@ -230,6 +230,10 @@ func validateLivePodsHaveState(stateIPCount, livePodIPCount int) error { return nil } +func retryValidationAttempt() (bool, error) { + return false, nil +} + func runValidationAttempts( ctx context.Context, maxAttempts int,