diff --git a/.pipelines/cni/state-migration/capture-transition.steps.yaml b/.pipelines/cni/state-migration/capture-transition.steps.yaml new file mode 100644 index 0000000000..934fbdb5a8 --- /dev/null +++ b/.pipelines/cni/state-migration/capture-transition.steps.yaml @@ -0,0 +1,166 @@ +parameters: + - name: lane + type: string + - name: transition + type: string + - name: clusterName + type: string + - name: os + type: string + - name: configMapName + type: string + - name: daemonsetName + type: string + - name: expectedBackend + type: string + - name: expectedAuthority + type: string + - name: expectedSchemaVersion + type: number + - name: summaryPath + type: string + +steps: + - ${{ if eq(parameters.expectedBackend, 'bolt') }}: + - task: AzureCLI@2 + displayName: Capture strict persistent debug responses + condition: always() + inputs: + azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION) + scriptLocation: inlineScript + scriptType: bash + addSpnToEnvironment: true + inlineScript: | + set -euo pipefail + + evidenceDir="$(Build.SourcesDirectory)/test/integration/logs/state-migration/${{ parameters.lane }}/${{ parameters.transition }}" + mkdir -p "$evidenceDir/persistent-debug" + make -C ./hack/aks set-kubeconf AZCLI=az CLUSTER=${{ parameters.clusterName }} + + if [[ "${{ parameters.os }}" == "windows" ]]; then + selector="k8s-app=azure-cns-win" + else + selector="k8s-app=azure-cns" + fi + + found=false + while IFS=$'\t' read -r pod node; do + [[ -n "$pod" && -n "$node" ]] || continue + found=true + response="$evidenceDir/persistent-debug/${pod}.json" + + if [[ "${{ parameters.os }}" == "windows" ]]; then + kubectl exec -n kube-system "$pod" -c cns-container -- \ + powershell -NoProfile -Command \ + 'Invoke-WebRequest -Uri 127.0.0.1:10090/debug/persistentstate -Method Post -UseBasicParsing -ErrorAction Stop | Select-Object -Expand Content' \ + >"$response" + else + kubectl exec -n kube-system "$pod" -c debug -- \ + bash -c "curl -sf localhost:10090/debug/persistentstate -d '{}'" \ + >"$response" + fi + + jq -e \ + --arg backend "${{ parameters.expectedBackend }}" \ + --arg authority "${{ parameters.expectedAuthority }}" \ + --argjson schema "${{ parameters.expectedSchemaVersion }}" \ + '.storage.backend == $backend + and .storage.filePresent == true + and .storage.fileSizeBytes > 0 + and .snapshot.metadata.authority == $authority + and .snapshot.metadata.schemaVersion == $schema + and ((.snapshot.metadata.bootID // "") | length) > 0' \ + "$response" + + nodeBootID=$(kubectl get node "$node" -o jsonpath='{.status.nodeInfo.bootID}') + stateBootID=$(jq -r '.snapshot.metadata.bootID' "$response") + normalizedNodeBootID=$(tr '[:upper:]' '[:lower:]' <<<"$nodeBootID" | tr -d '{}[:space:]') + normalizedStateBootID=$(tr '[:upper:]' '[:lower:]' <<<"$stateBootID" | tr -d '{}[:space:]') + if [[ "$normalizedNodeBootID" != "$normalizedStateBootID" ]]; then + echo "persistent boot ID does not match Kubernetes node boot ID for $node" + exit 1 + fi + done < <( + kubectl get pods -n kube-system -l "$selector" -o json | + jq -r '.items[] | [.metadata.name, .spec.nodeName] | @tsv' + ) + + if [[ "$found" != "true" ]]; then + echo "no CNS pods matched selector $selector" + exit 1 + fi + + - task: AzureCLI@2 + displayName: Capture transition state and logs + condition: always() + inputs: + azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION) + scriptLocation: inlineScript + scriptType: bash + addSpnToEnvironment: true + inlineScript: | + set -uo pipefail + + evidenceDir="$(Build.SourcesDirectory)/test/integration/logs/state-migration/${{ parameters.lane }}/${{ parameters.transition }}" + captureFailed=false + if ! mkdir -p "$evidenceDir/cns-logs"; then + captureFailed=true + fi + if ! make -C ./hack/aks set-kubeconf AZCLI=az CLUSTER=${{ parameters.clusterName }}; then + captureFailed=true + fi + + if [[ -s "${{ parameters.summaryPath }}" ]]; then + if ! cp "${{ parameters.summaryPath }}" "$evidenceDir/summary.json"; then + captureFailed=true + fi + else + echo "validation summary is missing: ${{ parameters.summaryPath }}" + captureFailed=true + fi + + if ! kubectl get configmap/${{ parameters.configMapName }} -n kube-system -o yaml \ + >"$evidenceDir/cns-config.yaml"; then + captureFailed=true + fi + if ! kubectl get daemonset/${{ parameters.daemonsetName }} -n kube-system -o yaml \ + >"$evidenceDir/cns-daemonset.yaml"; then + captureFailed=true + fi + if ! kubectl get nodes -o wide >"$evidenceDir/nodes.txt"; then + captureFailed=true + fi + if ! kubectl get pods -A -o wide >"$evidenceDir/pods.txt"; then + captureFailed=true + fi + if ! kubectl get events -A --sort-by=.lastTimestamp >"$evidenceDir/events.txt"; then + captureFailed=true + fi + + if [[ "${{ parameters.os }}" == "windows" ]]; then + selector="k8s-app=azure-cns-win" + else + selector="k8s-app=azure-cns" + fi + podList=$(kubectl get pods -n kube-system -l "$selector" -o name) || captureFailed=true + if [[ -z "$podList" ]]; then + echo "no CNS pods matched selector $selector" + captureFailed=true + fi + for pod in $podList; do + podName=${pod#pod/} + if ! kubectl logs -n kube-system "$podName" --all-containers=true --prefix=true \ + >"$evidenceDir/cns-logs/${podName}.log" 2>&1; then + captureFailed=true + fi + done + + if [[ "$captureFailed" == "true" ]]; then + echo "one or more required transition evidence captures failed" + exit 1 + fi + + - publish: $(Build.SourcesDirectory)/test/integration/logs/state-migration/${{ parameters.lane }}/${{ parameters.transition }} + artifact: state-migration-${{ parameters.lane }}-${{ parameters.transition }} + displayName: Publish transition evidence + condition: always() diff --git a/.pipelines/cni/state-migration/install-components.steps.yaml b/.pipelines/cni/state-migration/install-components.steps.yaml new file mode 100644 index 0000000000..119fc02ce7 --- /dev/null +++ b/.pipelines/cni/state-migration/install-components.steps.yaml @@ -0,0 +1,115 @@ +parameters: + - name: clusterName + type: string + - name: scenario + type: string + - name: os + type: string + - name: region + type: string + - name: configMapName + type: string + - name: daemonsetName + type: string + - name: manageEndpointState + type: string + - name: initializeFromCNI + type: string + +steps: + - task: AzureCLI@2 + displayName: Install migration lane components + inputs: + azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION) + scriptLocation: inlineScript + scriptType: bash + addSpnToEnvironment: true + inlineScript: | + set -euo pipefail + + make -C ./hack/aks azcfg AZCLI=az REGION=${{ parameters.region }} + make -C ./hack/aks set-kubeconf AZCLI=az CLUSTER=${{ parameters.clusterName }} + + cnsVersion=$(make cns-version) + cniVersion=$(make cni-version) + ipamVersion=$(make azure-ipam-version) + + case "${{ parameters.scenario }}" in + cilium-overlay) + make -C ./hack/aks deploy-cilium + sudo -E env "PATH=$PATH" make test-integration \ + AZURE_IPAM_VERSION="$ipamVersion" \ + CNS_VERSION="$cnsVersion" \ + INSTALL_CNS=true \ + INSTALL_OVERLAY=true \ + CNS_IMAGE_REPO=$(CNS_IMAGE_REPO) \ + IPAM_IMAGE_REPO=$(IPAM_IMAGE_REPO) + ;; + azure-cni-pod-subnet) + if [[ "${{ parameters.os }}" == "windows" ]]; then + sudo -E env "PATH=$PATH" make test-load \ + CNS_ONLY=true \ + CNS_VERSION="$cnsVersion" \ + CNI_VERSION="$cniVersion" \ + INSTALL_CNS=true \ + INSTALL_AZURE_VNET=true \ + CNS_IMAGE_REPO=$(CNS_IMAGE_REPO) \ + CNI_IMAGE_REPO=$(CNI_IMAGE_REPO) + else + sudo -E env "PATH=$PATH" make test-integration \ + CNS_VERSION="$cnsVersion" \ + CNI_VERSION="$cniVersion" \ + INSTALL_CNS=true \ + INSTALL_AZURE_VNET=true \ + CNS_IMAGE_REPO=$(CNS_IMAGE_REPO) \ + CNI_IMAGE_REPO=$(CNI_IMAGE_REPO) + fi + ;; + azure-cni-overlay) + sudo -E env "PATH=$PATH" make test-integration \ + CNS_VERSION="$cnsVersion" \ + CNI_VERSION="$cniVersion" \ + INSTALL_CNS=true \ + INSTALL_AZURE_CNI_OVERLAY=true \ + CNS_IMAGE_REPO=$(CNS_IMAGE_REPO) \ + CNI_IMAGE_REPO=$(CNI_IMAGE_REPO) + ;; + azure-cni-stateless-overlay) + sudo -E env "PATH=$PATH" make test-load \ + CNS_ONLY=true \ + CNS_VERSION="$cnsVersion" \ + CNI_VERSION="$cniVersion" \ + INSTALL_CNS=true \ + INSTALL_AZURE_VNET_STATELESS=true \ + CNS_IMAGE_REPO=$(CNS_IMAGE_REPO) \ + CNI_IMAGE_REPO=$(CNI_IMAGE_REPO) + ;; + *) + echo "unsupported migration lane scenario: ${{ parameters.scenario }}" + exit 1 + ;; + esac + + if [[ "${{ parameters.os }}" == "windows" ]]; then + while read -r node; do + nodeName=${node#node/} + if kubectl describe node "$nodeName" | grep -q 'node.cloudprovider.kubernetes.io/uninitialized=true:NoSchedule'; then + kubectl taint node "$nodeName" node.cloudprovider.kubernetes.io/uninitialized=true:NoSchedule- + fi + done < <(kubectl get nodes -l kubernetes.io/os=windows -o name) + fi + + kubectl rollout status daemonset/${{ parameters.daemonsetName }} \ + -n kube-system --timeout=20m + + config=$(kubectl get configmap/${{ parameters.configMapName }} \ + -n kube-system -o jsonpath='{.data.cns_config\.json}') + jq -e \ + --argjson manage "${{ parameters.manageEndpointState }}" \ + --argjson initialize "${{ parameters.initializeFromCNI }}" \ + '.ManageEndpointState == $manage and .InitializeFromCNI == $initialize' \ + <<<"$config" + + kubectl get pods -A -o wide + jq '{ManageEndpointState, InitializeFromCNI, EnableStateMigration, StateStoreBackend, StateStoreMode}' \ + <<<"$config" diff --git a/.pipelines/cni/state-migration/lane.stage.yaml b/.pipelines/cni/state-migration/lane.stage.yaml new file mode 100644 index 0000000000..840310e2e9 --- /dev/null +++ b/.pipelines/cni/state-migration/lane.stage.yaml @@ -0,0 +1,376 @@ +parameters: + - name: name + type: string + - name: displayName + type: string + - name: clusterBaseName + type: string + - name: clusterType + type: string + - name: scenario + type: string + - name: os + type: string + - name: cni + type: string + - name: region + type: string + - name: nodeCount + type: number + - name: nodeCountWin + type: number + - name: vmSize + type: string + - name: vmSizeWin + type: string + - name: osSKU + type: string + - name: osSkuWin + type: string + - name: targetNodeCount + type: number + - name: replicasPerNode + type: number + - name: configMapName + type: string + - name: daemonsetName + type: string + - name: manageEndpointState + type: string + - name: initializeFromCNI + type: string + - name: expectedAuthority + type: string + - name: expectedSchemaVersion + type: number + +stages: + - stage: ${{ parameters.name }} + displayName: ${{ parameters.displayName }} + dependsOn: + - setup + - publish_migration_images + variables: + commitID: $[ stageDependencies.setup.env.outputs['SetEnvVars.commitID'] ] + pool: + name: $(BUILD_POOL_NAME_DEFAULT) + jobs: + - job: create_cluster + displayName: Create migration cluster + timeoutInMinutes: 90 + steps: + - task: AzureCLI@2 + inputs: + azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION) + scriptLocation: inlineScript + scriptType: bash + addSpnToEnvironment: true + inlineScript: | + set -euo pipefail + + cluster="${{ parameters.clusterBaseName }}-$(commitID)" + make -C ./hack/aks azcfg AZCLI=az REGION=${{ parameters.region }} + + if az aks show --resource-group "$cluster" --name "$cluster" >/dev/null 2>&1; then + make -C ./hack/aks down \ + AZCLI=az \ + REGION=${{ parameters.region }} \ + SUB=$(SUB_AZURE_NETWORK_AGENT_BUILD_VALIDATIONS) \ + CLUSTER="$cluster" + fi + + makeArgs=( + "AZCLI=az" + "REGION=${{ parameters.region }}" + "SUB=$(SUB_AZURE_NETWORK_AGENT_BUILD_VALIDATIONS)" + "CLUSTER=$cluster" + "NODE_COUNT=${{ parameters.nodeCount }}" + "VM_SIZE=${{ parameters.vmSize }}" + "OS=${{ parameters.os }}" + "OS_SKU=${{ parameters.osSKU }}" + ) + if [[ "${{ parameters.os }}" == "windows" ]]; then + makeArgs+=( + "NODE_COUNT_WIN=${{ parameters.nodeCountWin }}" + "VM_SIZE_WIN=${{ parameters.vmSizeWin }}" + "OS_SKU_WIN=${{ parameters.osSkuWin }}" + ) + fi + + make -C ./hack/aks "${{ parameters.clusterType }}" "${makeArgs[@]}" + kubectl get nodes -o wide + displayName: Create ${{ parameters.clusterType }} + + - job: install_components + displayName: Install lane components + dependsOn: create_cluster + timeoutInMinutes: 120 + steps: + - template: install-components.steps.yaml + parameters: + clusterName: ${{ parameters.clusterBaseName }}-$(commitID) + scenario: ${{ parameters.scenario }} + os: ${{ parameters.os }} + region: ${{ parameters.region }} + configMapName: ${{ parameters.configMapName }} + daemonsetName: ${{ parameters.daemonsetName }} + manageEndpointState: ${{ parameters.manageEndpointState }} + initializeFromCNI: ${{ parameters.initializeFromCNI }} + + - job: json_baseline + displayName: 01 JSON baseline + dependsOn: install_components + timeoutInMinutes: 90 + steps: + - template: transition.steps.yaml + parameters: + lane: ${{ parameters.name }} + transition: 01-json-baseline + action: json-baseline + clusterName: ${{ parameters.clusterBaseName }}-$(commitID) + os: ${{ parameters.os }} + cni: ${{ parameters.cni }} + targetNodeCount: ${{ parameters.targetNodeCount }} + replicasPerNode: ${{ parameters.replicasPerNode }} + configMapName: ${{ parameters.configMapName }} + daemonsetName: ${{ parameters.daemonsetName }} + expectedBackend: json + expectedAuthority: ${{ parameters.expectedAuthority }} + expectedSchemaVersion: ${{ parameters.expectedSchemaVersion }} + + - job: bolt_migrate + displayName: 02 Single-restart Bolt migration + dependsOn: json_baseline + timeoutInMinutes: 90 + steps: + - template: transition.steps.yaml + parameters: + lane: ${{ parameters.name }} + transition: 02-bolt-migrate + action: bolt-migrate + clusterName: ${{ parameters.clusterBaseName }}-$(commitID) + os: ${{ parameters.os }} + cni: ${{ parameters.cni }} + targetNodeCount: ${{ parameters.targetNodeCount }} + replicasPerNode: ${{ parameters.replicasPerNode }} + configMapName: ${{ parameters.configMapName }} + daemonsetName: ${{ parameters.daemonsetName }} + expectedBackend: bolt + expectedAuthority: ${{ parameters.expectedAuthority }} + expectedSchemaVersion: ${{ parameters.expectedSchemaVersion }} + baselineArtifact: state-migration-${{ parameters.name }}-01-json-baseline + stateRelation: exact + + - job: same_boot_restart + displayName: 03 Same-boot restart + dependsOn: bolt_migrate + timeoutInMinutes: 90 + steps: + - template: transition.steps.yaml + parameters: + lane: ${{ parameters.name }} + transition: 03-same-boot-restart + action: same-boot-restart + clusterName: ${{ parameters.clusterBaseName }}-$(commitID) + os: ${{ parameters.os }} + cni: ${{ parameters.cni }} + targetNodeCount: ${{ parameters.targetNodeCount }} + replicasPerNode: ${{ parameters.replicasPerNode }} + configMapName: ${{ parameters.configMapName }} + daemonsetName: ${{ parameters.daemonsetName }} + expectedBackend: bolt + expectedAuthority: ${{ parameters.expectedAuthority }} + expectedSchemaVersion: ${{ parameters.expectedSchemaVersion }} + baselineArtifact: state-migration-${{ parameters.name }}-02-bolt-migrate + stateRelation: exact + bootRelation: same + + - job: restart_during_scale + displayName: 04 Restart during scale + dependsOn: same_boot_restart + timeoutInMinutes: 180 + steps: + - template: transition.steps.yaml + parameters: + lane: ${{ parameters.name }} + transition: 04-restart-during-scale + action: restart-during-scale + clusterName: ${{ parameters.clusterBaseName }}-$(commitID) + os: ${{ parameters.os }} + cni: ${{ parameters.cni }} + targetNodeCount: ${{ parameters.targetNodeCount }} + replicasPerNode: ${{ parameters.replicasPerNode }} + configMapName: ${{ parameters.configMapName }} + daemonsetName: ${{ parameters.daemonsetName }} + expectedBackend: bolt + expectedAuthority: ${{ parameters.expectedAuthority }} + expectedSchemaVersion: ${{ parameters.expectedSchemaVersion }} + baselineArtifact: state-migration-${{ parameters.name }}-03-same-boot-restart + stateRelation: changed + bootRelation: same + runFaultInjection: ${{ eq(parameters.manageEndpointState, 'true') }} + + - job: node_reboot + displayName: 05 Node reboot + dependsOn: restart_during_scale + timeoutInMinutes: 120 + steps: + - template: transition.steps.yaml + parameters: + lane: ${{ parameters.name }} + transition: 05-node-reboot + action: node-reboot + clusterName: ${{ parameters.clusterBaseName }}-$(commitID) + os: ${{ parameters.os }} + cni: ${{ parameters.cni }} + targetNodeCount: ${{ parameters.targetNodeCount }} + replicasPerNode: ${{ parameters.replicasPerNode }} + configMapName: ${{ parameters.configMapName }} + daemonsetName: ${{ parameters.daemonsetName }} + expectedBackend: bolt + expectedAuthority: ${{ parameters.expectedAuthority }} + expectedSchemaVersion: ${{ parameters.expectedSchemaVersion }} + baselineArtifact: state-migration-${{ parameters.name }}-04-restart-during-scale + stateRelation: exact + bootRelation: changed + + - job: os_validation + displayName: 06 OS-specific validation + dependsOn: node_reboot + timeoutInMinutes: 90 + steps: + - template: transition.steps.yaml + parameters: + lane: ${{ parameters.name }} + transition: 06-os-validation + action: os-validation + clusterName: ${{ parameters.clusterBaseName }}-$(commitID) + os: ${{ parameters.os }} + cni: ${{ parameters.cni }} + targetNodeCount: ${{ parameters.targetNodeCount }} + replicasPerNode: ${{ parameters.replicasPerNode }} + configMapName: ${{ parameters.configMapName }} + daemonsetName: ${{ parameters.daemonsetName }} + expectedBackend: bolt + expectedAuthority: ${{ parameters.expectedAuthority }} + expectedSchemaVersion: ${{ parameters.expectedSchemaVersion }} + baselineArtifact: state-migration-${{ parameters.name }}-05-node-reboot + stateRelation: exact + bootRelation: same + + - job: rollback_json + displayName: 07 Roll back to JSON + dependsOn: os_validation + timeoutInMinutes: 90 + steps: + - template: transition.steps.yaml + parameters: + lane: ${{ parameters.name }} + transition: 07-rollback-json + action: rollback-json + clusterName: ${{ parameters.clusterBaseName }}-$(commitID) + os: ${{ parameters.os }} + cni: ${{ parameters.cni }} + targetNodeCount: ${{ parameters.targetNodeCount }} + replicasPerNode: ${{ parameters.replicasPerNode }} + configMapName: ${{ parameters.configMapName }} + daemonsetName: ${{ parameters.daemonsetName }} + expectedBackend: json + expectedAuthority: ${{ parameters.expectedAuthority }} + expectedSchemaVersion: ${{ parameters.expectedSchemaVersion }} + baselineArtifact: state-migration-${{ parameters.name }}-06-os-validation + stateRelation: exact + + - job: json_mutation + displayName: 08 Mutate JSON-authoritative pod state + dependsOn: rollback_json + timeoutInMinutes: 90 + steps: + - template: transition.steps.yaml + parameters: + lane: ${{ parameters.name }} + transition: 08-json-mutation + action: json-mutation + clusterName: ${{ parameters.clusterBaseName }}-$(commitID) + os: ${{ parameters.os }} + cni: ${{ parameters.cni }} + targetNodeCount: ${{ parameters.targetNodeCount }} + replicasPerNode: ${{ parameters.replicasPerNode }} + configMapName: ${{ parameters.configMapName }} + daemonsetName: ${{ parameters.daemonsetName }} + expectedBackend: json + expectedAuthority: ${{ parameters.expectedAuthority }} + expectedSchemaVersion: ${{ parameters.expectedSchemaVersion }} + baselineArtifact: state-migration-${{ parameters.name }}-07-rollback-json + stateRelation: changed + + - job: bolt_reimport + displayName: 09 Bolt re-upgrade and reimport + dependsOn: json_mutation + timeoutInMinutes: 90 + steps: + - template: transition.steps.yaml + parameters: + lane: ${{ parameters.name }} + transition: 09-bolt-reimport + action: bolt-reimport + clusterName: ${{ parameters.clusterBaseName }}-$(commitID) + os: ${{ parameters.os }} + cni: ${{ parameters.cni }} + targetNodeCount: ${{ parameters.targetNodeCount }} + replicasPerNode: ${{ parameters.replicasPerNode }} + configMapName: ${{ parameters.configMapName }} + daemonsetName: ${{ parameters.daemonsetName }} + expectedBackend: bolt + expectedAuthority: ${{ parameters.expectedAuthority }} + expectedSchemaVersion: ${{ parameters.expectedSchemaVersion }} + baselineArtifact: state-migration-${{ parameters.name }}-08-json-mutation + stateRelation: exact + + - job: final_same_boot_restart + displayName: 10 Final same-boot restart and comparison + dependsOn: bolt_reimport + timeoutInMinutes: 90 + steps: + - template: transition.steps.yaml + parameters: + lane: ${{ parameters.name }} + transition: 10-final-same-boot-restart + action: final-same-boot-restart + clusterName: ${{ parameters.clusterBaseName }}-$(commitID) + os: ${{ parameters.os }} + cni: ${{ parameters.cni }} + targetNodeCount: ${{ parameters.targetNodeCount }} + replicasPerNode: ${{ parameters.replicasPerNode }} + configMapName: ${{ parameters.configMapName }} + daemonsetName: ${{ parameters.daemonsetName }} + expectedBackend: bolt + expectedAuthority: ${{ parameters.expectedAuthority }} + expectedSchemaVersion: ${{ parameters.expectedSchemaVersion }} + baselineArtifact: state-migration-${{ parameters.name }}-09-bolt-reimport + stateRelation: exact + bootRelation: same + + - job: cleanup + displayName: Clean up migration cluster + condition: always() + dependsOn: + - create_cluster + - install_components + - json_baseline + - bolt_migrate + - same_boot_restart + - restart_during_scale + - node_reboot + - os_validation + - rollback_json + - json_mutation + - bolt_reimport + - final_same_boot_restart + steps: + - template: ../../templates/delete-cluster.yaml + parameters: + clusterName: ${{ parameters.clusterBaseName }}-$(commitID) + sub: $(SUB_AZURE_NETWORK_AGENT_BUILD_VALIDATIONS) + svcConn: $(BUILD_VALIDATIONS_SERVICE_CONNECTION) + deleteResources: "true" diff --git a/.pipelines/cni/state-migration/pipeline.yaml b/.pipelines/cni/state-migration/pipeline.yaml new file mode 100644 index 0000000000..1df7bb668d --- /dev/null +++ b/.pipelines/cni/state-migration/pipeline.yaml @@ -0,0 +1,261 @@ +pr: none +trigger: + tags: + include: + - "dropgz/*" + - "azure-ipam/*" + - "v*" + +parameters: + - name: linuxNodeCount + type: number + default: 2 + - name: windowsSystemNodeCount + type: number + default: 2 + - name: windowsNodeCount + type: number + default: 2 + - name: replicasPerNode + type: number + default: 2 + - name: linuxVMSize + type: string + default: "$(VM_SIZE)" + - name: windowsSystemVMSize + type: string + default: "$(VM_SIZE_WINCLUSTER_SYSTEMPOOL)" + - name: windowsVMSize + type: string + default: "$(VM_SIZE_WIN)" + - name: region + type: string + default: "$(LOCATION_AMD64)" + - name: expectedSchemaVersion + type: number + default: 1 + +stages: + - stage: setup + displayName: Set up migration release gate + jobs: + - job: env + pool: + name: $(BUILD_POOL_NAME_DEFAULT) + steps: + - bash: | + set -euo pipefail + go version + commitID="$(make revision)-$(Build.BuildId)" + echo "##vso[task.setvariable variable=commitID;isOutput=true]$commitID" + echo "Migration gate commit ID: $commitID" + echo "Reusable fault injection runs in CNS-managed migration lanes." + name: SetEnvVars + displayName: Set migration gate variables + + - stage: build_migration_images + displayName: Build migration gate images + dependsOn: setup + jobs: + - job: containerize_amd64 + pool: + name: $(BUILD_POOL_NAME_LINUX_AMD64) + strategy: + matrix: + azure_ipam_linux_amd64: + arch: amd64 + name: azure-ipam + os: linux + azure_ipam_windows_amd64: + arch: amd64 + name: azure-ipam + os: windows + cni_linux_amd64: + arch: amd64 + name: cni + os: linux + cni_windows_amd64: + arch: amd64 + name: cni + os: windows + cns_linux_amd64: + arch: amd64 + name: cns + os: linux + cns_windows_amd64: + arch: amd64 + name: cns + os: windows + steps: + - template: ../../containers/container-template.yaml + parameters: + arch: $(arch) + name: $(name) + os: $(os) + + - stage: publish_migration_images + displayName: Publish migration gate manifests + dependsOn: build_migration_images + jobs: + - job: manifest + pool: + name: $(BUILD_POOL_NAME_DEFAULT) + strategy: + matrix: + azure_ipam: + name: azure-ipam + platforms: linux/amd64 windows/amd64 + cni: + name: cni + platforms: linux/amd64 windows/amd64 + cns: + name: cns + platforms: linux/amd64 windows/amd64 + steps: + - template: ../../containers/manifest-template.yaml + parameters: + name: $(name) + platforms: $(platforms) + + - template: lane.stage.yaml + parameters: + name: linux_cilium_overlay + displayName: "P0 Linux Cilium overlay — CNS-managed" + clusterBaseName: mig-cilium-lnx + clusterType: overlay-byocni-nokubeproxy-up + scenario: cilium-overlay + os: linux + cni: cilium + region: ${{ parameters.region }} + nodeCount: ${{ parameters.linuxNodeCount }} + nodeCountWin: 0 + vmSize: ${{ parameters.linuxVMSize }} + vmSizeWin: "" + osSKU: Ubuntu + osSkuWin: Windows2022 + targetNodeCount: ${{ parameters.linuxNodeCount }} + replicasPerNode: ${{ parameters.replicasPerNode }} + configMapName: cns-config + daemonsetName: azure-cns + manageEndpointState: "true" + initializeFromCNI: "false" + expectedAuthority: bolt + expectedSchemaVersion: ${{ parameters.expectedSchemaVersion }} + + - template: lane.stage.yaml + parameters: + name: linux_podsubnet + displayName: "P0 Linux Azure CNI pod-subnet — CNI-managed" + clusterBaseName: mig-podsub-lnx + clusterType: swift-byocni-up + scenario: azure-cni-pod-subnet + os: linux + cni: cniv2 + region: ${{ parameters.region }} + nodeCount: ${{ parameters.linuxNodeCount }} + nodeCountWin: 0 + vmSize: ${{ parameters.linuxVMSize }} + vmSizeWin: "" + osSKU: Ubuntu + osSkuWin: Windows2022 + targetNodeCount: ${{ parameters.linuxNodeCount }} + replicasPerNode: ${{ parameters.replicasPerNode }} + configMapName: cns-config + daemonsetName: azure-cns + manageEndpointState: "false" + initializeFromCNI: "true" + expectedAuthority: bolt + expectedSchemaVersion: ${{ parameters.expectedSchemaVersion }} + + - template: lane.stage.yaml + parameters: + name: linux_overlay + displayName: "P0 Linux Azure CNI overlay — CNI-managed" + clusterBaseName: mig-overlay-lnx + clusterType: overlay-byocni-up + scenario: azure-cni-overlay + os: linux + cni: cniv2 + region: ${{ parameters.region }} + nodeCount: ${{ parameters.linuxNodeCount }} + nodeCountWin: 0 + vmSize: ${{ parameters.linuxVMSize }} + vmSizeWin: "" + osSKU: Ubuntu + osSkuWin: Windows2022 + targetNodeCount: ${{ parameters.linuxNodeCount }} + replicasPerNode: ${{ parameters.replicasPerNode }} + configMapName: cns-config + daemonsetName: azure-cns + manageEndpointState: "false" + initializeFromCNI: "true" + expectedAuthority: bolt + expectedSchemaVersion: ${{ parameters.expectedSchemaVersion }} + + - template: lane.stage.yaml + parameters: + name: windows_podsubnet + displayName: "P0 Windows Azure CNI pod-subnet — CNI-managed" + clusterBaseName: mig-podsub-win + clusterType: swift-byocni-up + scenario: azure-cni-pod-subnet + os: windows + cni: cniv2 + region: ${{ parameters.region }} + nodeCount: ${{ parameters.windowsSystemNodeCount }} + nodeCountWin: ${{ parameters.windowsNodeCount }} + vmSize: ${{ parameters.windowsSystemVMSize }} + vmSizeWin: ${{ parameters.windowsVMSize }} + osSKU: Ubuntu + osSkuWin: Windows2022 + targetNodeCount: ${{ parameters.windowsNodeCount }} + replicasPerNode: ${{ parameters.replicasPerNode }} + configMapName: cns-win-config + daemonsetName: azure-cns-win + manageEndpointState: "false" + initializeFromCNI: "true" + expectedAuthority: bolt + expectedSchemaVersion: ${{ parameters.expectedSchemaVersion }} + + - template: lane.stage.yaml + parameters: + name: windows_stateless_overlay + displayName: "P0 Windows stateless overlay — CNS-managed" + clusterBaseName: mig-stateless-win + clusterType: overlay-byocni-up + scenario: azure-cni-stateless-overlay + os: windows + cni: stateless + region: ${{ parameters.region }} + nodeCount: ${{ parameters.windowsSystemNodeCount }} + nodeCountWin: ${{ parameters.windowsNodeCount }} + vmSize: ${{ parameters.windowsSystemVMSize }} + vmSizeWin: ${{ parameters.windowsVMSize }} + osSKU: Ubuntu + osSkuWin: Windows2022 + targetNodeCount: ${{ parameters.windowsNodeCount }} + replicasPerNode: ${{ parameters.replicasPerNode }} + configMapName: cns-win-config + daemonsetName: azure-cns-win + manageEndpointState: "true" + initializeFromCNI: "false" + expectedAuthority: bolt + expectedSchemaVersion: ${{ parameters.expectedSchemaVersion }} + + - stage: migration_release_gate + displayName: State migration release gate + dependsOn: + - linux_cilium_overlay + - linux_podsubnet + - linux_overlay + - windows_podsubnet + - windows_stateless_overlay + condition: succeeded() + jobs: + - job: passed + pool: + name: $(BUILD_POOL_NAME_DEFAULT) + steps: + - bash: | + echo "All five P0 state-migration lifecycle lanes passed." + displayName: Confirm release gate diff --git a/.pipelines/cni/state-migration/transition.steps.yaml b/.pipelines/cni/state-migration/transition.steps.yaml new file mode 100644 index 0000000000..93b8407b4c --- /dev/null +++ b/.pipelines/cni/state-migration/transition.steps.yaml @@ -0,0 +1,561 @@ +parameters: + - name: lane + type: string + - name: transition + type: string + - name: action + type: string + - name: clusterName + type: string + - name: os + type: string + - name: cni + type: string + - name: targetNodeCount + type: number + - name: replicasPerNode + type: number + - name: configMapName + type: string + - name: daemonsetName + type: string + - name: expectedBackend + type: string + - name: expectedAuthority + type: string + - name: expectedSchemaVersion + type: number + - name: baselineArtifact + type: string + default: "" + - name: stateRelation + type: string + default: none + - name: bootRelation + type: string + default: none + - name: runFaultInjection + type: boolean + default: false + +steps: + - ${{ if ne(parameters.baselineArtifact, '') }}: + - download: current + artifact: ${{ parameters.baselineArtifact }} + displayName: Download preceding transition evidence + + - bash: | + set -euo pipefail + evidenceDir="$(Build.SourcesDirectory)/test/integration/logs/state-migration/${{ parameters.lane }}/${{ parameters.transition }}" + summaryPath="$(Build.SourcesDirectory)/test/integration/logs/validate/${{ parameters.lane }}/${{ parameters.transition }}.json" + rm -rf "$evidenceDir" + rm -f "$summaryPath" + mkdir -p "$evidenceDir" + displayName: Prepare transition evidence + condition: always() + + - ${{ if or(eq(parameters.action, 'json-baseline'), eq(parameters.action, 'bolt-migrate'), eq(parameters.action, 'rollback-json'), eq(parameters.action, 'bolt-reimport')) }}: + - task: AzureCLI@2 + displayName: Apply ${{ parameters.action }} state mode + inputs: + azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION) + scriptLocation: inlineScript + scriptType: bash + addSpnToEnvironment: true + inlineScript: | + set -euo pipefail + + evidenceDir="$(Build.SourcesDirectory)/test/integration/logs/state-migration/${{ parameters.lane }}/${{ parameters.transition }}" + make -C ./hack/aks set-kubeconf AZCLI=az CLUSTER=${{ parameters.clusterName }} + + configMap="${{ parameters.configMapName }}" + daemonset="${{ parameters.daemonsetName }}" + action="${{ parameters.action }}" + beforeGeneration=$(kubectl get daemonset "$daemonset" -n kube-system -o jsonpath='{.metadata.generation}') + + patch_config() { + local backend=$1 + local mode=$2 + local current patched payload + current=$(kubectl get configmap "$configMap" -n kube-system -o jsonpath='{.data.cns_config\.json}') + patched=$(jq --arg backend "$backend" --arg mode "$mode" \ + '.StateStoreBackend = $backend | .StateStoreMode = $mode' <<<"$current") + payload=$(jq -n --arg config "$patched" '{data: {"cns_config.json": $config}}') + kubectl patch configmap "$configMap" -n kube-system --type merge -p "$payload" + } + + restart_target() { + kubectl rollout restart daemonset "$daemonset" -n kube-system + kubectl rollout status daemonset "$daemonset" -n kube-system --timeout=20m + } + + case "$action" in + json-baseline) + [[ "${{ parameters.expectedBackend }}" == "json" ]] + patch_config json normal + restart_target + expectedRestarts=1 + ;; + bolt-migrate|bolt-reimport) + [[ "${{ parameters.expectedBackend }}" == "bolt" ]] + patch_config bolt normal + restart_target + expectedRestarts=1 + ;; + rollback-json) + [[ "${{ parameters.expectedBackend }}" == "json" ]] + patch_config json rollback-to-json + restart_target + patch_config json normal + restart_target + expectedRestarts=2 + ;; + *) + echo "unsupported state mode action: $action" + exit 1 + ;; + esac + + afterGeneration=$(kubectl get daemonset "$daemonset" -n kube-system -o jsonpath='{.metadata.generation}') + generationDelta=$((afterGeneration - beforeGeneration)) + if (( generationDelta != expectedRestarts )); then + echo "$action expected $expectedRestarts daemonset restart(s), generation changed by $generationDelta" + exit 1 + fi + + config=$(kubectl get configmap "$configMap" -n kube-system -o jsonpath='{.data.cns_config\.json}') + jq -e \ + --arg backend "${{ parameters.expectedBackend }}" \ + '.StateStoreBackend == $backend and .StateStoreMode == "normal"' \ + <<<"$config" + + jq -n \ + --arg action "$action" \ + --arg backend "${{ parameters.expectedBackend }}" \ + --argjson beforeGeneration "$beforeGeneration" \ + --argjson afterGeneration "$afterGeneration" \ + --argjson restarts "$expectedRestarts" \ + '{ + action: $action, + expectedBackend: $backend, + daemonsetGenerationBefore: $beforeGeneration, + daemonsetGenerationAfter: $afterGeneration, + restartCount: $restarts + }' >"$evidenceDir/mode-transition.json" + + - ${{ if eq(parameters.action, 'json-baseline') }}: + - task: AzureCLI@2 + displayName: Seed JSON-authoritative pod state + inputs: + azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION) + scriptLocation: inlineScript + scriptType: bash + addSpnToEnvironment: true + inlineScript: | + set -euo pipefail + + evidenceDir="$(Build.SourcesDirectory)/test/integration/logs/state-migration/${{ parameters.lane }}/${{ parameters.transition }}" + make -C ./hack/aks set-kubeconf AZCLI=az CLUSTER=${{ parameters.clusterName }} + kubectl create namespace load-test --dry-run=client -o yaml | kubectl apply -f - + + if [[ "${{ parameters.os }}" == "windows" ]]; then + manifest=test/integration/manifests/noop-deployment-windows.yaml + deployment=win-load-test + else + manifest=test/integration/manifests/noop-deployment-linux.yaml + deployment=load-test + fi + + replicas=$(( ${{ parameters.replicasPerNode }} * ${{ parameters.targetNodeCount }} )) + kubectl apply -f "$manifest" + kubectl scale deployment "$deployment" -n load-test --replicas="$replicas" + kubectl rollout status deployment "$deployment" -n load-test --timeout=30m + jq -n --arg deployment "$deployment" --argjson replicas "$replicas" \ + '{deployment: $deployment, replicas: $replicas}' \ + >"$evidenceDir/workload.json" + + - ${{ if or(eq(parameters.action, 'same-boot-restart'), eq(parameters.action, 'final-same-boot-restart')) }}: + - task: AzureCLI@2 + displayName: Restart CNS on the same node boot + inputs: + azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION) + scriptLocation: inlineScript + scriptType: bash + addSpnToEnvironment: true + inlineScript: | + set -euo pipefail + + evidenceDir="$(Build.SourcesDirectory)/test/integration/logs/state-migration/${{ parameters.lane }}/${{ parameters.transition }}" + make -C ./hack/aks set-kubeconf AZCLI=az CLUSTER=${{ parameters.clusterName }} + beforeGeneration=$(kubectl get daemonset/${{ parameters.daemonsetName }} -n kube-system -o jsonpath='{.metadata.generation}') + kubectl rollout restart daemonset/${{ parameters.daemonsetName }} -n kube-system + kubectl rollout status daemonset/${{ parameters.daemonsetName }} -n kube-system --timeout=20m + afterGeneration=$(kubectl get daemonset/${{ parameters.daemonsetName }} -n kube-system -o jsonpath='{.metadata.generation}') + if (( afterGeneration - beforeGeneration != 1 )); then + echo "same-boot restart must advance daemonset generation exactly once" + exit 1 + fi + jq -n \ + --argjson beforeGeneration "$beforeGeneration" \ + --argjson afterGeneration "$afterGeneration" \ + '{daemonsetGenerationBefore: $beforeGeneration, daemonsetGenerationAfter: $afterGeneration, restartCount: 1}' \ + >"$evidenceDir/restart.json" + + # Endpoint-commit faults require CNS-owned endpoint state. CNI-managed lanes + # still run the mandatory active-scale restart immediately below. + - ${{ if and(eq(parameters.action, 'restart-during-scale'), eq(parameters.runFaultInjection, true)) }}: + - template: ../load-test-templates/migration-fault-injection-template.yaml + parameters: + clusterName: ${{ parameters.clusterName }} + os: ${{ parameters.os }} + cni: ${{ parameters.cni }} + scenario: all + artifactName: state-migration-${{ parameters.lane }}-04-fault-injection + + - ${{ if eq(parameters.action, 'restart-during-scale') }}: + - task: AzureCLI@2 + displayName: Restart CNS during active pod scale + inputs: + azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION) + scriptLocation: inlineScript + scriptType: bash + addSpnToEnvironment: true + inlineScript: | + set -euo pipefail + + evidenceDir="$(Build.SourcesDirectory)/test/integration/logs/state-migration/${{ parameters.lane }}/${{ parameters.transition }}" + make -C ./hack/aks set-kubeconf AZCLI=az CLUSTER=${{ parameters.clusterName }} + + if [[ "${{ parameters.os }}" == "windows" ]]; then + deployment=win-load-test + else + deployment=load-test + fi + + current=$(kubectl get deployment "$deployment" -n load-test -o jsonpath='{.spec.replicas}') + target=$((current + ${{ parameters.targetNodeCount }})) + beforeGeneration=$(kubectl get daemonset/${{ parameters.daemonsetName }} -n kube-system -o jsonpath='{.metadata.generation}') + + kubectl scale deployment "$deployment" -n load-test --replicas="$target" + kubectl rollout restart daemonset/${{ parameters.daemonsetName }} -n kube-system + kubectl rollout status daemonset/${{ parameters.daemonsetName }} -n kube-system --timeout=20m + kubectl rollout status deployment "$deployment" -n load-test --timeout=30m + + afterGeneration=$(kubectl get daemonset/${{ parameters.daemonsetName }} -n kube-system -o jsonpath='{.metadata.generation}') + if (( afterGeneration - beforeGeneration != 1 )); then + echo "scale transition must restart the target CNS daemonset exactly once" + exit 1 + fi + + jq -n \ + --arg deployment "$deployment" \ + --argjson replicasBefore "$current" \ + --argjson replicasAfter "$target" \ + --argjson daemonsetGenerationBefore "$beforeGeneration" \ + --argjson daemonsetGenerationAfter "$afterGeneration" \ + '{ + deployment: $deployment, + replicasBefore: $replicasBefore, + replicasAfter: $replicasAfter, + daemonsetGenerationBefore: $daemonsetGenerationBefore, + daemonsetGenerationAfter: $daemonsetGenerationAfter + }' >"$evidenceDir/restart-during-scale.json" + + - ${{ if eq(parameters.action, 'node-reboot') }}: + - task: AzureCLI@2 + displayName: Reboot cluster nodes and require new boot IDs + inputs: + azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION) + scriptLocation: inlineScript + scriptType: bash + addSpnToEnvironment: true + inlineScript: | + set -euo pipefail + + evidenceDir="$(Build.SourcesDirectory)/test/integration/logs/state-migration/${{ parameters.lane }}/${{ parameters.transition }}" + make -C ./hack/aks set-kubeconf AZCLI=az CLUSTER=${{ parameters.clusterName }} + + kubectl get nodes -l kubernetes.io/os=${{ parameters.os }} -o json | + jq '[.items[] | {nodeName: .metadata.name, bootID: .status.nodeInfo.bootID}] | sort_by(.nodeName)' \ + >"$evidenceDir/node-boots-before.json" + + nodeResourceGroup=$(az aks show \ + --resource-group "${{ parameters.clusterName }}" \ + --name "${{ parameters.clusterName }}" \ + --query nodeResourceGroup -o tsv) + mapfile -t vmsses < <( + kubectl get nodes -l kubernetes.io/os=${{ parameters.os }} -o json | + jq -r ' + .items[].spec.providerID + | split("/") as $parts + | ($parts | map(ascii_downcase) | index("virtualmachinescalesets")) as $index + | select($index != null) + | $parts[$index + 1] + ' | + sort -u + ) + if (( ${#vmsses[@]} == 0 )); then + echo "no target ${{ parameters.os }} VM scale sets found in $nodeResourceGroup" + exit 1 + fi + for vmss in "${vmsses[@]}"; do + az vmss restart -g "$nodeResourceGroup" -n "$vmss" --instance-ids '*' + done + + rebootComplete=false + deadline=$((SECONDS + 1800)) + while (( SECONDS < deadline )); do + kubectl get nodes -l kubernetes.io/os=${{ parameters.os }} -o json | + jq '[.items[] | { + nodeName: .metadata.name, + bootID: .status.nodeInfo.bootID, + ready: any(.status.conditions[]?; .type == "Ready" and .status == "True") + }] | sort_by(.nodeName)' \ + >"$evidenceDir/node-boots-after.json" + + if jq -e -n \ + --slurpfile before "$evidenceDir/node-boots-before.json" \ + --slurpfile after "$evidenceDir/node-boots-after.json" \ + '($before[0] | length) > 0 + and ($before[0] | map(.nodeName)) == ($after[0] | map(.nodeName)) + and all($after[0][]; .ready == true) + and ([range(0; $before[0] | length) as $i + | $before[0][$i].bootID != $after[0][$i].bootID] | all)' \ + >/dev/null; then + rebootComplete=true + break + fi + sleep 20 + done + if [[ "$rebootComplete" != "true" ]]; then + echo "target nodes did not return Ready with new boot IDs within 30 minutes" + exit 1 + fi + + kubectl rollout status daemonset/${{ parameters.daemonsetName }} \ + -n kube-system --timeout=20m + + if [[ "${{ parameters.os }}" == "windows" ]]; then + deployment=win-load-test + else + deployment=load-test + fi + kubectl rollout status deployment "$deployment" -n load-test --timeout=30m + + jq -e -n \ + --slurpfile before "$evidenceDir/node-boots-before.json" \ + --slurpfile after "$evidenceDir/node-boots-after.json" \ + '($before[0] | length) > 0 + and ($before[0] | map(.nodeName)) == ($after[0] | map(.nodeName)) + and ([range(0; $before[0] | length) as $i + | $before[0][$i].bootID != $after[0][$i].bootID] | all)' + + - ${{ if eq(parameters.action, 'os-validation') }}: + - task: AzureCLI@2 + displayName: Run ${{ parameters.os }} service validation + inputs: + azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION) + scriptLocation: inlineScript + scriptType: bash + addSpnToEnvironment: true + inlineScript: | + set -euo pipefail + + evidenceDir="$(Build.SourcesDirectory)/test/integration/logs/state-migration/${{ parameters.lane }}/${{ parameters.transition }}" + make -C ./hack/aks set-kubeconf AZCLI=az CLUSTER=${{ parameters.clusterName }} + + if [[ "${{ parameters.os }}" == "windows" ]]; then + found=false + for pod in $(kubectl get pods -n kube-system \ + -l app=privileged-daemonset,os=windows -o name); do + found=true + kubectl exec -n kube-system "$pod" -- \ + powershell -NoProfile -Command 'Restart-Service hns -Force' + done + if [[ "$found" != "true" ]]; then + echo "no Windows privileged daemonset pods found for HNS validation" + exit 1 + fi + validation=hns-restart + else + # The strict validator below restarts systemd-networkd in the host + # namespace and verifies connectivity on every Linux node. + validation=systemd-networkd-restart + fi + + jq -n --arg validation "$validation" '{osValidation: $validation}' \ + >"$evidenceDir/os-validation.json" + + - ${{ if eq(parameters.action, 'json-mutation') }}: + - task: AzureCLI@2 + displayName: Mutate pods while JSON is authoritative + inputs: + azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION) + scriptLocation: inlineScript + scriptType: bash + addSpnToEnvironment: true + inlineScript: | + set -euo pipefail + + [[ "${{ parameters.expectedBackend }}" == "json" ]] + evidenceDir="$(Build.SourcesDirectory)/test/integration/logs/state-migration/${{ parameters.lane }}/${{ parameters.transition }}" + make -C ./hack/aks set-kubeconf AZCLI=az CLUSTER=${{ parameters.clusterName }} + + config=$(kubectl get configmap/${{ parameters.configMapName }} \ + -n kube-system -o jsonpath='{.data.cns_config\.json}') + jq -e '.StateStoreBackend == "json" and .StateStoreMode == "normal"' <<<"$config" + + if [[ "${{ parameters.os }}" == "windows" ]]; then + deployment=win-load-test + else + deployment=load-test + fi + + before=$(kubectl get deployment "$deployment" -n load-test -o jsonpath='{.spec.replicas}') + peak=$((before + ${{ parameters.targetNodeCount }})) + if (( peak <= before + 1 )); then + peak=$((before + 2)) + fi + after=$((before + 1)) + + kubectl scale deployment "$deployment" -n load-test --replicas="$peak" + kubectl rollout status deployment "$deployment" -n load-test --timeout=30m + kubectl scale deployment "$deployment" -n load-test --replicas="$after" + kubectl rollout status deployment "$deployment" -n load-test --timeout=30m + + jq -n \ + --arg deployment "$deployment" \ + --argjson replicasBefore "$before" \ + --argjson replicasPeak "$peak" \ + --argjson replicasAfter "$after" \ + '{ + authority: "json", + deployment: $deployment, + replicasBefore: $replicasBefore, + replicasPeak: $replicasPeak, + replicasAfter: $replicasAfter + }' >"$evidenceDir/json-mutation.json" + + - task: AzureCLI@2 + displayName: Validate strict ${{ parameters.expectedBackend }} state + inputs: + azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION) + scriptLocation: inlineScript + scriptType: bash + addSpnToEnvironment: true + inlineScript: | + set -euo pipefail + + make -C ./hack/aks set-kubeconf AZCLI=az CLUSTER=${{ parameters.clusterName }} + evidenceDir="$(Build.SourcesDirectory)/test/integration/logs/state-migration/${{ parameters.lane }}/${{ parameters.transition }}" + summaryDir="$(Build.SourcesDirectory)/test/integration/logs/validate/${{ parameters.lane }}" + summaryPath="$summaryDir/${{ parameters.transition }}.json" + mkdir -p "$summaryDir" + + config=$(kubectl get configmap/${{ parameters.configMapName }} \ + -n kube-system -o jsonpath='{.data.cns_config\.json}') + jq -e \ + --arg backend "${{ parameters.expectedBackend }}" \ + '.StateStoreBackend == $backend and .StateStoreMode == "normal"' \ + <<<"$config" + + restartCase=true + if [[ "${{ parameters.action }}" == "json-baseline" ]]; then + restartCase=false + fi + + VALIDATE_SUMMARY_PATH="$summaryPath" \ + VALIDATE_STATEFILE=true \ + VALIDATE_STATE_BACKEND="${{ parameters.expectedBackend }}" \ + VALIDATE_CONVERGENCE_ATTEMPTS=10 \ + VALIDATE_CONVERGENCE_INTERVAL_SECONDS=15 \ + make test-validate-state \ + OS_TYPE=${{ parameters.os }} \ + RESTART_CASE="$restartCase" \ + CNI_TYPE=${{ parameters.cni }} + + test -s "$summaryPath" + jq -e \ + --arg backend "${{ parameters.expectedBackend }}" \ + --arg authority "${{ parameters.expectedAuthority }}" \ + --argjson schema "${{ parameters.expectedSchemaVersion }}" ' + (.checks | length) > 0 + and all(.checks[]; + .validationPass == true + and .converged == true + and (((.missingIPs // []) | length) == 0) + and (((.unexpectedIPs // []) | length) == 0) + and (((.duplicateIPs // []) | length) == 0) + ) + and ( + if $backend == "bolt" then + ([.checks[] | select((.stateBackend // "") != "")]) as $persistent + | ($persistent | length) > 0 + and all($persistent[]; + .stateBackend == $backend + and .authority == $authority + and .schemaVersion == $schema + and ((.bootID // "") | length) > 0 + and .dbFilePresent == true + and .dbFileSizeBytes > 0 + ) + else + all(.checks[]; (.stateBackend // "") == "") + end + ) + ' "$summaryPath" + + kubectl get pods -n load-test -l load-test=true -o json | + jq '[ + .items[] + | { + namespace: .metadata.namespace, + name: .metadata.name, + nodeName: .spec.nodeName, + phase: .status.phase, + podIPs: ((.status.podIPs // []) | map(.ip) | sort) + } + ] | sort_by(.namespace, .name)' \ + >"$evidenceDir/pod-state.json" + jq -e ' + length > 0 + and all(.[]; + .nodeName != "" + and .phase == "Running" + and (.podIPs | length) > 0 + ) + ' "$evidenceDir/pod-state.json" >/dev/null + + - ${{ if ne(parameters.baselineArtifact, '') }}: + - bash: | + set -euo pipefail + evidenceDir="$(Build.SourcesDirectory)/test/integration/logs/state-migration/${{ parameters.lane }}/${{ parameters.transition }}" + baseline="$(Pipeline.Workspace)/${{ parameters.baselineArtifact }}/summary.json" + candidate="$(Build.SourcesDirectory)/test/integration/logs/validate/${{ parameters.lane }}/${{ parameters.transition }}.json" + baselinePodState="$(Pipeline.Workspace)/${{ parameters.baselineArtifact }}/pod-state.json" + candidatePodState="$evidenceDir/pod-state.json" + bash hack/scripts/compare-state-migration-summaries.sh \ + "$baseline" \ + "$candidate" \ + "${{ parameters.expectedBackend }}" \ + "${{ parameters.expectedAuthority }}" \ + "${{ parameters.expectedSchemaVersion }}" \ + "${{ parameters.stateRelation }}" \ + "${{ parameters.bootRelation }}" \ + "$baselinePodState" \ + "$candidatePodState" \ + 2>&1 | tee "$evidenceDir/comparison.txt" + displayName: Compare transition summaries + + - template: capture-transition.steps.yaml + parameters: + lane: ${{ parameters.lane }} + transition: ${{ parameters.transition }} + clusterName: ${{ parameters.clusterName }} + os: ${{ parameters.os }} + configMapName: ${{ parameters.configMapName }} + daemonsetName: ${{ parameters.daemonsetName }} + expectedBackend: ${{ parameters.expectedBackend }} + expectedAuthority: ${{ parameters.expectedAuthority }} + expectedSchemaVersion: ${{ parameters.expectedSchemaVersion }} + summaryPath: $(Build.SourcesDirectory)/test/integration/logs/validate/${{ parameters.lane }}/${{ parameters.transition }}.json diff --git a/hack/scripts/compare-state-migration-summaries.sh b/hack/scripts/compare-state-migration-summaries.sh new file mode 100755 index 0000000000..3f875e19c3 --- /dev/null +++ b/hack/scripts/compare-state-migration-summaries.sh @@ -0,0 +1,191 @@ +#!/usr/bin/env bash + +set -euo pipefail + +if [[ $# -ne 9 ]]; then + echo "usage: $0 " >&2 + exit 2 +fi + +baseline=$1 +candidate=$2 +expected_backend=$3 +expected_authority=$4 +expected_schema=$5 +state_relation=$6 +boot_relation=$7 +baseline_pod_state=$8 +candidate_pod_state=$9 + +for artifact in "$baseline" "$candidate" "$baseline_pod_state" "$candidate_pod_state"; do + if [[ ! -s "$artifact" ]]; then + echo "state migration evidence is missing or empty: $artifact" >&2 + exit 1 + fi +done + +case "$expected_backend" in +json | bolt) ;; +*) + echo "unsupported expected backend: $expected_backend" >&2 + exit 2 + ;; +esac + +case "$state_relation" in +none | exact | changed) ;; +*) + echo "unsupported state relation: $state_relation" >&2 + exit 2 + ;; +esac + +case "$boot_relation" in +none | same | changed) ;; +*) + echo "unsupported boot relation: $boot_relation" >&2 + exit 2 + ;; +esac + +if [[ ! "$expected_schema" =~ ^[0-9]+$ ]]; then + echo "expected schema must be an unsigned integer: $expected_schema" >&2 + exit 2 +fi + +if ! jq -e \ + --arg backend "$expected_backend" \ + --arg authority "$expected_authority" \ + --argjson schema "$expected_schema" ' + (.checks | type == "array" and length > 0) + and all(.checks[]; + .validationPass == true + and .converged == true + and (((.missingIPs // []) | length) == 0) + and (((.unexpectedIPs // []) | length) == 0) + and (((.duplicateIPs // []) | length) == 0) + ) + and ( + if $backend == "bolt" then + ([.checks[] | select((.stateBackend // "") != "")]) as $persistent + | ($persistent | length) > 0 + and all($persistent[]; + .stateBackend == $backend + and .authority == $authority + and .schemaVersion == $schema + and ((.bootID // "") | length) > 0 + and .dbFilePresent == true + and .dbFileSizeBytes > 0 + ) + else + all(.checks[]; (.stateBackend // "") == "") + end + ) +' "$candidate" >/dev/null; then + echo "candidate summary failed strict validation: $candidate" >&2 + jq . "$candidate" >&2 || true + exit 1 +fi + +if ! jq -e -n \ + --slurpfile baseline "$baseline" \ + --slurpfile candidate "$candidate" \ + --slurpfile baselinePods "$baseline_pod_state" \ + --slurpfile candidatePods "$candidate_pod_state" \ + --arg relation "$state_relation" ' + def normalized_state($summary): + [ + $summary.checks[] + | select(.checkName != "cns persistent metadata") + | { + checkName, + nodeName, + expectedCount, + actualCount, + missingIPs: ((.missingIPs // []) | sort), + unexpectedIPs: ((.unexpectedIPs // []) | sort), + duplicateIPs: ((.duplicateIPs // []) | sort) + } + ] + | sort_by(.checkName, .nodeName); + def normalized_pods($pods): + [ + $pods[] + | { + namespace, + name, + nodeName, + phase, + podIPs: ((.podIPs // []) | sort) + } + ] + | sort_by(.namespace, .name); + + (normalized_state($baseline[0])) as $before + | (normalized_state($candidate[0])) as $after + | (normalized_pods($baselinePods[0])) as $beforePods + | (normalized_pods($candidatePods[0])) as $afterPods + | if $relation == "none" then + true + elif $relation == "exact" then + $before == $after + and ($beforePods | length) > 0 + and $beforePods == $afterPods + else + ($beforePods | length) > 0 + and ($afterPods | length) > 0 + and $beforePods != $afterPods + and $before != $after + end +' >/dev/null; then + echo "state relation '$state_relation' failed between $baseline and $candidate" >&2 + exit 1 +fi + +if ! jq -e -n \ + --slurpfile baseline "$baseline" \ + --slurpfile candidate "$candidate" \ + --arg relation "$boot_relation" ' + def boots($summary): + [ + $summary.checks[] + | select((.stateBackend // "") != "") + | {nodeName, bootID} + ] + | sort_by(.nodeName); + + (boots($baseline[0])) as $before + | (boots($candidate[0])) as $after + | if $relation == "none" then + true + elif $relation == "same" then + ($before | length) > 0 + and $before == $after + else + ($before | length) > 0 + and ($before | map(.nodeName)) == ($after | map(.nodeName)) + and ([range(0; $before | length) as $i | $before[$i].bootID != $after[$i].bootID] | all) + end +' >/dev/null; then + echo "boot relation '$boot_relation' failed between $baseline and $candidate" >&2 + exit 1 +fi + +jq -n \ + --arg baseline "$baseline" \ + --arg candidate "$candidate" \ + --arg baselinePodState "$baseline_pod_state" \ + --arg candidatePodState "$candidate_pod_state" \ + --arg expectedBackend "$expected_backend" \ + --arg stateRelation "$state_relation" \ + --arg bootRelation "$boot_relation" \ + '{ + baseline: $baseline, + candidate: $candidate, + baselinePodState: $baselinePodState, + candidatePodState: $candidatePodState, + expectedBackend: $expectedBackend, + stateRelation: $stateRelation, + bootRelation: $bootRelation, + result: "pass" + }' diff --git a/test/integration/state/config.go b/test/integration/state/config.go index 84ca1961ab..980596d265 100644 --- a/test/integration/state/config.go +++ b/test/integration/state/config.go @@ -69,6 +69,8 @@ const ( windowsCNSDaemonSet = "azure-cns-win" linuxCNSLabelSelector = "k8s-app=azure-cns" windowsCNSLabelSelector = "k8s-app=azure-cns-win" + linuxCNSConfigMap = "cns-config" + windowsCNSConfigMap = "cns-win-config" ) type faultConfig struct { @@ -189,6 +191,13 @@ func cnsDaemonSetForOS(osName string) (name, selector string) { return linuxCNSDaemonSet, linuxCNSLabelSelector } +func cnsConfigMapForOS(osName string) string { + if osName == "windows" { + return windowsCNSConfigMap + } + return linuxCNSConfigMap +} + func sanitizeResourceName(value string, maxLength int) string { var builder strings.Builder lastHyphen := false diff --git a/test/integration/state/config_test.go b/test/integration/state/config_test.go index 3e58ec3539..58f2a061e5 100644 --- a/test/integration/state/config_test.go +++ b/test/integration/state/config_test.go @@ -191,6 +191,23 @@ func TestFindCNSContainer(t *testing.T) { } } +func TestCNSConfigMapForOS(t *testing.T) { + tests := []struct { + name string + os string + want string + }{ + {name: "linux", os: faultOSLinux, want: linuxCNSConfigMap}, + {name: "windows", os: faultOSWindows, want: windowsCNSConfigMap}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + require.Equal(t, tt.want, cnsConfigMapForOS(tt.os)) + }) + } +} + func TestSanitizeResourceName(t *testing.T) { tests := []struct { name string diff --git a/test/integration/state/fault_injection_test.go b/test/integration/state/fault_injection_test.go index 5b7d8721cf..4f4d44f684 100644 --- a/test/integration/state/fault_injection_test.go +++ b/test/integration/state/fault_injection_test.go @@ -153,9 +153,10 @@ func newClusterHarness(t *testing.T, ctx context.Context, cfg faultConfig) *clus } func (harness *clusterHarness) validateCNSConfig(ctx context.Context) error { - configMap, err := harness.clientset.CoreV1().ConfigMaps(kubeSystemNamespace).Get(ctx, "cns-config", metav1.GetOptions{}) + configMapName := cnsConfigMapForOS(harness.cfg.OS) + configMap, err := harness.clientset.CoreV1().ConfigMaps(kubeSystemNamespace).Get(ctx, configMapName, metav1.GetOptions{}) if err != nil { - return fmt.Errorf("getting CNS configmap: %w", err) + return fmt.Errorf("getting CNS configmap %q: %w", configMapName, err) } raw := []byte(configMap.Data["cns_config.json"]) if err := validateMigrationCNSConfig(raw); err != nil {