From 77b373a872cf56d38bde107fc41a4c5a700d18ee Mon Sep 17 00:00:00 2001 From: Dawei Huang Date: Wed, 18 Mar 2026 18:39:34 +0000 Subject: [PATCH 1/9] Add ARM64 deb build stage to Azure pipeline SmartSwitch DPU runs on ARM64, so we need an ARM64 deb artifact. - Add BuildArm64 stage that runs after Build succeeds - Uses sonicso1ES-arm64 pool with sonic-slave-bookworm arm64 container - New install-dependencies-arm64.yml template for arm64 deps - Publishes sonic-gnmi.arm64 artifact Signed-off-by: Dawei Huang --- .../templates/install-dependencies-arm64.yml | 78 +++++++++++++++++++ azure-pipelines.yml | 48 ++++++++++++ 2 files changed, 126 insertions(+) create mode 100644 .azure/templates/install-dependencies-arm64.yml diff --git a/.azure/templates/install-dependencies-arm64.yml b/.azure/templates/install-dependencies-arm64.yml new file mode 100644 index 000000000..3816b3fa5 --- /dev/null +++ b/.azure/templates/install-dependencies-arm64.yml @@ -0,0 +1,78 @@ +# Azure DevOps YAML Template: SONiC Dependencies Installation (ARM64) +# +# ARM64 variant of install-dependencies.yml for cross-architecture deb builds. +# Only installs what's needed for `dpkg-buildpackage` — no tests run on ARM64. +# +# Usage: +# - template: .azure/templates/install-dependencies-arm64.yml +# parameters: +# buildBranch: $(BUILD_BRANCH) + +parameters: +- name: buildBranch + type: string + default: $(BUILD_BRANCH) + +steps: +# Download arm64 libyang from common-lib +- task: DownloadPipelineArtifact@2 + inputs: + source: specific + project: build + pipeline: Azure.sonic-buildimage.common_libs + runVersion: 'latestFromBranch' + runBranch: 'refs/heads/${{ parameters.buildBranch }}' + path: $(Build.ArtifactStagingDirectory)/download + artifact: common-lib.arm64 + patterns: | + target/debs/bookworm/libyang-*_1.0*.deb + target/debs/bookworm/libyang_1.0*.deb + displayName: "Download libyang from arm64 common lib" + +- script: | + set -ex + sudo dpkg -i $(find $(Build.ArtifactStagingDirectory)/download -name '*.deb') + displayName: "Install libyang debs" + +# Download arm64 sonic-swss-common +- task: DownloadPipelineArtifact@2 + inputs: + source: specific + project: build + pipeline: Azure.sonic-swss-common + artifact: sonic-swss-common-bookworm.arm64 + runVersion: 'latestFromBranch' + runBranch: 'refs/heads/${{ parameters.buildBranch }}' + displayName: "Download sonic-swss-common (arm64)" + +- script: | + set -ex + sudo dpkg -i libswsscommon_1.0.0_arm64.deb + sudo dpkg -i libswsscommon-dev_1.0.0_arm64.deb + workingDirectory: $(Pipeline.Workspace)/ + displayName: 'Install libswsscommon (arm64)' + +# Download yang wheels (arch-independent) +- task: DownloadPipelineArtifact@2 + inputs: + source: specific + project: build + pipeline: 142 + artifact: sonic-buildimage.vs + runVersion: 'latestFromBranch' + runBranch: 'refs/heads/${{ parameters.buildBranch }}' + patterns: | + target/python-wheels/bookworm/sonic_yang_models*.whl + displayName: "Download yang wheels" + +- script: | + set -ex + sudo pip3 install ../target/python-wheels/bookworm/sonic_yang_models*.whl + displayName: "Install sonic yangs" + +# Install protoc +- script: | + sudo apt-get update + sudo apt-get install -y protobuf-compiler + protoc --version + displayName: 'Install protoc' diff --git a/azure-pipelines.yml b/azure-pipelines.yml index ec18ad566..ca3439b84 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -245,3 +245,51 @@ stages: codeCoverageTool: Cobertura summaryFileLocation: '$(System.DefaultWorkingDirectory)/sonic-gnmi/coverage.xml' displayName: 'Publish coverage' + +- stage: BuildArm64 + dependsOn: Build + condition: succeeded('Build') + jobs: + - job: arm64 + displayName: "arm64 deb build" + timeoutInMinutes: 60 + + pool: + name: sonicso1ES-arm64 + + container: + image: sonicdev-microsoft.azurecr.io:443/sonic-slave-bookworm:$(BUILD_BRANCH)-arm64 + + steps: + - checkout: self + clean: true + submodules: recursive + displayName: 'Checkout code' + + - checkout: sonic-mgmt-common + clean: true + submodules: recursive + displayName: 'Checkout sonic-mgmt-common' + + - checkout: sonic-swss-common + clean: true + submodules: recursive + displayName: 'Checkout sonic-swss-common' + + - template: .azure/templates/install-dependencies-arm64.yml + parameters: + buildBranch: $(BUILD_BRANCH) + + - script: | + set -ex + pushd sonic-mgmt-common + NO_TEST_BINS=1 dpkg-buildpackage -rfakeroot -b -us -uc + popd + + pushd sonic-gnmi + dpkg-buildpackage -rfakeroot -us -uc -b -j$(nproc) && cp ../*.deb $(Build.ArtifactStagingDirectory)/ + displayName: "Build ARM64 deb" + + - publish: $(Build.ArtifactStagingDirectory)/ + artifact: sonic-gnmi.arm64 + displayName: "Archive ARM64 artifacts" From 4bcea8c360c8fc12c578e9cb08287df2c72f5f4c Mon Sep 17 00:00:00 2001 From: Dawei Huang Date: Wed, 18 Mar 2026 18:41:30 +0000 Subject: [PATCH 2/9] Run ARM64 build in parallel with amd64 Build stage Signed-off-by: Dawei Huang --- azure-pipelines.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index ca3439b84..356d489a5 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -247,8 +247,7 @@ stages: displayName: 'Publish coverage' - stage: BuildArm64 - dependsOn: Build - condition: succeeded('Build') + dependsOn: [] jobs: - job: arm64 displayName: "arm64 deb build" From f95b7b7dee0145275d2a9995b82cfc9c1cd6125e Mon Sep 17 00:00:00 2001 From: sigabrtv1-ui Date: Thu, 19 Mar 2026 14:28:42 +0000 Subject: [PATCH 3/9] Fix ARM64 build: install libnl deps before libswsscommon Download libnl debs (libnl-3-200, libnl-genl-3-200, libnl-route-3-200, libnl-nf-3-200) from common-lib.arm64 artifact alongside libyang, matching the pattern used by the amd64 install-dependencies template. Signed-off-by: sigabrtv1-ui Signed-off-by: Dawei Huang --- .azure/templates/install-dependencies-arm64.yml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/.azure/templates/install-dependencies-arm64.yml b/.azure/templates/install-dependencies-arm64.yml index 3816b3fa5..e02a1b3d6 100644 --- a/.azure/templates/install-dependencies-arm64.yml +++ b/.azure/templates/install-dependencies-arm64.yml @@ -27,12 +27,17 @@ steps: patterns: | target/debs/bookworm/libyang-*_1.0*.deb target/debs/bookworm/libyang_1.0*.deb - displayName: "Download libyang from arm64 common lib" + target/debs/bookworm/libnl-3-200_*.deb + target/debs/bookworm/libnl-genl-3-200_*.deb + target/debs/bookworm/libnl-route-3-200_*.deb + target/debs/bookworm/libnl-nf-3-200_*.deb + displayName: "Download libyang and libnl from arm64 common lib" - script: | set -ex + sudo apt-get -y purge libnl-3-dev libnl-route-3-dev || true sudo dpkg -i $(find $(Build.ArtifactStagingDirectory)/download -name '*.deb') - displayName: "Install libyang debs" + displayName: "Install libyang and libnl debs" # Download arm64 sonic-swss-common - task: DownloadPipelineArtifact@2 From 1cfdf4b48d8ec0f44141e4e26c8227e0e64d64f9 Mon Sep 17 00:00:00 2001 From: Dawei Huang Date: Tue, 17 Mar 2026 22:58:31 +0000 Subject: [PATCH 4/9] Fix too_many_pings during DPU SetPackage by coordinating keepalive settings The DPU proxy client was sending keepalive pings every 10s, but the gRPC server uses the default EnforcementPolicy with MinTime=5m. During long-running SetPackage operations (sonic-installer can take minutes), this mismatch causes the server to terminate the connection with GOAWAY/ENHANCE_YOUR_CALM and debug data 'too_many_pings'. Fix: 1. Increase proxy client keepalive interval from 10s to 30s and timeout from 3s to 10s for better resilience during long operations. 2. Add explicit KeepaliveEnforcementPolicy to the gNMI server allowing pings as frequent as every 20s, with a safety margin above the client's 30s interval. Fixes: #619 Related: sonic-net/sonic-mgmt#23054 Signed-off-by: Dawei Huang --- pkg/interceptors/dpuproxy/proxy.go | 11 ++++++++--- telemetry/telemetry.go | 11 +++++++++++ 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/pkg/interceptors/dpuproxy/proxy.go b/pkg/interceptors/dpuproxy/proxy.go index 0645dd1b4..12c348482 100644 --- a/pkg/interceptors/dpuproxy/proxy.go +++ b/pkg/interceptors/dpuproxy/proxy.go @@ -177,13 +177,18 @@ func (p *DPUProxy) getConnection(ctx context.Context, dpuIndex, ipAddress string target := fmt.Sprintf("%s:%s", ipAddress, port) glog.Infof("[DPUProxy] Trying to connect to DPU%s at %s (attempt %d/%d)", dpuIndex, target, i+1, len(portsToTry)) - // Create connection with keepalive settings for long-lived connections + // Create connection with keepalive settings for long-lived connections. + // Use a conservative ping interval to avoid triggering the server's + // default EnforcementPolicy (MinTime=5m). Operations like SetPackage + // can block for minutes during image installation; aggressive pinging + // causes the server to send GOAWAY with "too_many_pings". + // See: https://github.com/sonic-net/sonic-gnmi/issues/619 conn, err := grpc.NewClient( target, grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithKeepaliveParams(keepalive.ClientParameters{ - Time: 10 * time.Second, // Send keepalive ping every 10s - Timeout: 3 * time.Second, // Wait 3s for ping ack before considering connection dead + Time: 30 * time.Second, // Send keepalive ping every 30s + Timeout: 10 * time.Second, // Wait 10s for ping ack before considering connection dead PermitWithoutStream: true, // Send pings even when no active RPCs }), ) diff --git a/telemetry/telemetry.go b/telemetry/telemetry.go index 187b0a5cb..366da0ca0 100644 --- a/telemetry/telemetry.go +++ b/telemetry/telemetry.go @@ -527,6 +527,17 @@ func startGNMIServer(telemetryCfg *TelemetryConfig, cfg *gnmi.Config, serverCont MaxConnectionIdle: time.Duration(*telemetryCfg.IdleConnDuration) * time.Second, // duration in which idle connection will be closed, default is inf } + // Allow clients (e.g. DPU proxy) to send keepalive pings at a + // reasonable rate. Without this the default MinTime is 5 minutes, + // causing "too_many_pings" GOAWAY for clients that ping more + // frequently during long-running operations like SetPackage. + // See: https://github.com/sonic-net/sonic-gnmi/issues/619 + keep_alive_policy := keepalive.EnforcementPolicy{ + MinTime: 20 * time.Second, // Allow pings as frequent as every 20s + PermitWithoutStream: true, // Allow pings when there are no active streams + } + commonOpts = append(commonOpts, grpc.KeepaliveEnforcementPolicy(keep_alive_policy)) + tlsOpts = []grpc.ServerOption{grpc.Creds(credentials.NewTLS(tlsCfg))} if *telemetryCfg.IdleConnDuration > 0 { // non inf case From b29bd227019be5597ce927f9553d0a3f487a736a Mon Sep 17 00:00:00 2001 From: Dawei Huang Date: Wed, 25 Mar 2026 17:29:20 +0000 Subject: [PATCH 5/9] Remove downloaded dependencies from arm64 artifacts Signed-off-by: Dawei Huang --- azure-pipelines.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 356d489a5..3249a68d9 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -289,6 +289,9 @@ stages: dpkg-buildpackage -rfakeroot -us -uc -b -j$(nproc) && cp ../*.deb $(Build.ArtifactStagingDirectory)/ displayName: "Build ARM64 deb" + - script: rm -rf $(Build.ArtifactStagingDirectory)/download + displayName: "Remove downloaded dependencies from artifacts" + - publish: $(Build.ArtifactStagingDirectory)/ artifact: sonic-gnmi.arm64 displayName: "Archive ARM64 artifacts" From a434d619aeb35e8d4e18ede408c438025b318dc4 Mon Sep 17 00:00:00 2001 From: Dawei Huang Date: Fri, 27 Mar 2026 04:11:04 +0000 Subject: [PATCH 6/9] Fix ARM64 build: add ENABLE_TRANSLIB_WRITE and ENABLE_NATIVE_WRITE env vars Signed-off-by: Dawei Huang --- azure-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 3249a68d9..d02ca38c5 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -286,7 +286,7 @@ stages: popd pushd sonic-gnmi - dpkg-buildpackage -rfakeroot -us -uc -b -j$(nproc) && cp ../*.deb $(Build.ArtifactStagingDirectory)/ + ENABLE_TRANSLIB_WRITE=y ENABLE_NATIVE_WRITE=y dpkg-buildpackage -rfakeroot -us -uc -b -j$(nproc) && cp ../*.deb $(Build.ArtifactStagingDirectory)/ displayName: "Build ARM64 deb" - script: rm -rf $(Build.ArtifactStagingDirectory)/download From cadb9abcdff8bf57e998cf1775c850fcdf4b52bc Mon Sep 17 00:00:00 2001 From: sigabrtv1-ui Date: Fri, 27 Mar 2026 17:39:55 +0000 Subject: [PATCH 7/9] ci: deduplicate arm64/amd64 build and dependency installation - Unify install-dependencies.yml with arch parameter (amd64/arm64) instead of maintaining separate install-dependencies-arm64.yml - Add installTestDeps parameter to skip pytest/redis/.NET for build-only jobs - Extract build-deb.yml template for shared checkout + build + publish flow - ARM64 stage now uses build-deb.yml template instead of inline steps - Remove install-dependencies-arm64.yml (merged into unified template) Signed-off-by: sigabrtv1-ui --- .azure/templates/build-deb.yml | 66 ++++++ .../templates/install-dependencies-arm64.yml | 83 -------- .azure/templates/install-dependencies.yml | 201 +++++++++++------- azure-pipelines.yml | 43 +--- 4 files changed, 202 insertions(+), 191 deletions(-) create mode 100644 .azure/templates/build-deb.yml delete mode 100644 .azure/templates/install-dependencies-arm64.yml diff --git a/.azure/templates/build-deb.yml b/.azure/templates/build-deb.yml new file mode 100644 index 000000000..d63b533ab --- /dev/null +++ b/.azure/templates/build-deb.yml @@ -0,0 +1,66 @@ +# Azure DevOps YAML Template: Build SONiC gNMI deb package +# +# Shared template for building sonic-gnmi .deb packages on both amd64 and arm64. +# Handles: checkout → install deps → build mgmt-common → build gnmi → publish artifacts. +# +# Usage: +# - template: .azure/templates/build-deb.yml +# parameters: +# arch: amd64 +# buildBranch: $(BUILD_BRANCH) + +parameters: +- name: buildBranch + type: string + default: $(BUILD_BRANCH) +- name: arch + type: string + default: amd64 + values: + - amd64 + - arm64 + +steps: +# Checkout all required repositories +- checkout: self + clean: true + submodules: recursive + displayName: 'Checkout code' + +- checkout: sonic-mgmt-common + clean: true + submodules: recursive + displayName: 'Checkout sonic-mgmt-common' + +- checkout: sonic-swss-common + clean: true + submodules: recursive + displayName: 'Checkout sonic-swss-common' + +# Install dependencies (architecture-aware) +- template: install-dependencies.yml + parameters: + buildBranch: ${{ parameters.buildBranch }} + arch: ${{ parameters.arch }} + installTestDeps: false + +# Build sonic-mgmt-common and sonic-gnmi +- script: | + set -ex + pushd sonic-mgmt-common + NO_TEST_BINS=1 dpkg-buildpackage -rfakeroot -b -us -uc + popd + + pushd sonic-gnmi + ENABLE_TRANSLIB_WRITE=y ENABLE_NATIVE_WRITE=y dpkg-buildpackage -rfakeroot -us -uc -b -j$(nproc) && cp ../*.deb $(Build.ArtifactStagingDirectory)/ + displayName: "Build ${{ parameters.arch }} deb" + +# Clean up downloaded artifacts from staging (arm64 downloads to staging dir) +- ${{ if eq(parameters.arch, 'arm64') }}: + - script: rm -rf $(Build.ArtifactStagingDirectory)/download + displayName: "Remove downloaded dependencies from artifacts" + +# Publish deb artifacts +- publish: $(Build.ArtifactStagingDirectory)/ + artifact: sonic-gnmi${{ if eq(parameters.arch, 'arm64') }}.arm64${{ endif }} + displayName: "Archive ${{ parameters.arch }} artifacts" diff --git a/.azure/templates/install-dependencies-arm64.yml b/.azure/templates/install-dependencies-arm64.yml deleted file mode 100644 index e02a1b3d6..000000000 --- a/.azure/templates/install-dependencies-arm64.yml +++ /dev/null @@ -1,83 +0,0 @@ -# Azure DevOps YAML Template: SONiC Dependencies Installation (ARM64) -# -# ARM64 variant of install-dependencies.yml for cross-architecture deb builds. -# Only installs what's needed for `dpkg-buildpackage` — no tests run on ARM64. -# -# Usage: -# - template: .azure/templates/install-dependencies-arm64.yml -# parameters: -# buildBranch: $(BUILD_BRANCH) - -parameters: -- name: buildBranch - type: string - default: $(BUILD_BRANCH) - -steps: -# Download arm64 libyang from common-lib -- task: DownloadPipelineArtifact@2 - inputs: - source: specific - project: build - pipeline: Azure.sonic-buildimage.common_libs - runVersion: 'latestFromBranch' - runBranch: 'refs/heads/${{ parameters.buildBranch }}' - path: $(Build.ArtifactStagingDirectory)/download - artifact: common-lib.arm64 - patterns: | - target/debs/bookworm/libyang-*_1.0*.deb - target/debs/bookworm/libyang_1.0*.deb - target/debs/bookworm/libnl-3-200_*.deb - target/debs/bookworm/libnl-genl-3-200_*.deb - target/debs/bookworm/libnl-route-3-200_*.deb - target/debs/bookworm/libnl-nf-3-200_*.deb - displayName: "Download libyang and libnl from arm64 common lib" - -- script: | - set -ex - sudo apt-get -y purge libnl-3-dev libnl-route-3-dev || true - sudo dpkg -i $(find $(Build.ArtifactStagingDirectory)/download -name '*.deb') - displayName: "Install libyang and libnl debs" - -# Download arm64 sonic-swss-common -- task: DownloadPipelineArtifact@2 - inputs: - source: specific - project: build - pipeline: Azure.sonic-swss-common - artifact: sonic-swss-common-bookworm.arm64 - runVersion: 'latestFromBranch' - runBranch: 'refs/heads/${{ parameters.buildBranch }}' - displayName: "Download sonic-swss-common (arm64)" - -- script: | - set -ex - sudo dpkg -i libswsscommon_1.0.0_arm64.deb - sudo dpkg -i libswsscommon-dev_1.0.0_arm64.deb - workingDirectory: $(Pipeline.Workspace)/ - displayName: 'Install libswsscommon (arm64)' - -# Download yang wheels (arch-independent) -- task: DownloadPipelineArtifact@2 - inputs: - source: specific - project: build - pipeline: 142 - artifact: sonic-buildimage.vs - runVersion: 'latestFromBranch' - runBranch: 'refs/heads/${{ parameters.buildBranch }}' - patterns: | - target/python-wheels/bookworm/sonic_yang_models*.whl - displayName: "Download yang wheels" - -- script: | - set -ex - sudo pip3 install ../target/python-wheels/bookworm/sonic_yang_models*.whl - displayName: "Install sonic yangs" - -# Install protoc -- script: | - sudo apt-get update - sudo apt-get install -y protobuf-compiler - protoc --version - displayName: 'Install protoc' diff --git a/.azure/templates/install-dependencies.yml b/.azure/templates/install-dependencies.yml index dc88ec587..b34015835 100644 --- a/.azure/templates/install-dependencies.yml +++ b/.azure/templates/install-dependencies.yml @@ -1,116 +1,175 @@ # Azure DevOps YAML Template: SONiC Dependencies Installation # -# This template contains all the common dependency installation steps shared between -# jobs that require CGO/SONiC dependencies (MemoryLeakJob and IntegrationCIJob). +# Unified template for both amd64 and arm64 architectures. +# Controls architecture-specific behavior via the `arch` parameter. # -# Usage in pipeline jobs: +# Usage: # - template: .azure/templates/install-dependencies.yml # parameters: # buildBranch: $(BUILD_BRANCH) +# arch: amd64 # or arm64 +# installTestDeps: true # install pytest, redis, .NET (for test jobs) # -# Dependencies installed: -# - libyang (from sonic-buildimage artifacts) -# - libnl packages (from sonic-buildimage artifacts) +# Dependencies installed (all architectures): +# - libyang and libnl packages # - sonic-swss-common libraries +# - sonic yang models # - protobuf compiler # -# Note: Building sonic-mgmt-common and sonic-gnmi is NOT included here. -# Only the IntegrationCIJob needs to build and publish artifacts, so that -# step is defined directly in azure-pipelines.yml to avoid duplicate builds. -# -# This eliminates code duplication and ensures consistent dependency setup -# across all jobs that need SONiC/CGO dependencies. +# Additional dependencies when installTestDeps=true (amd64 only): +# - pytest, jsonpatch +# - redis-server +# - .NET SDK 8.0 parameters: - name: buildBranch type: string default: $(BUILD_BRANCH) +- name: arch + type: string + default: amd64 + values: + - amd64 + - arm64 +- name: installTestDeps + type: boolean + default: false steps: -# Download basic dependencies from sonic-buildimage -- task: DownloadPipelineArtifact@2 - inputs: - source: specific - project: build - pipeline: 142 - artifact: sonic-buildimage.vs - runVersion: 'latestFromBranch' - runBranch: 'refs/heads/${{ parameters.buildBranch }}' - patterns: | - target/debs/bookworm/libyang*.deb - target/debs/bookworm/libnl*.deb - target/python-wheels/bookworm/sonic_yang_models*.whl - displayName: "Download bookworm debs" +# === Download libyang + libnl debs === +# amd64: from sonic-buildimage.vs (pipeline 142) +# arm64: from sonic-buildimage.common_libs +- ${{ if eq(parameters.arch, 'amd64') }}: + - task: DownloadPipelineArtifact@2 + inputs: + source: specific + project: build + pipeline: 142 + artifact: sonic-buildimage.vs + runVersion: 'latestFromBranch' + runBranch: 'refs/heads/${{ parameters.buildBranch }}' + patterns: | + target/debs/bookworm/libyang*.deb + target/debs/bookworm/libnl*.deb + target/python-wheels/bookworm/sonic_yang_models*.whl + displayName: "Download bookworm debs" -# Install pytest, jsonpatch, redis and libyang -- script: | - # PYTEST - sudo pip3 install -U pytest - sudo pip3 install -U jsonpatch +- ${{ if eq(parameters.arch, 'arm64') }}: + - task: DownloadPipelineArtifact@2 + inputs: + source: specific + project: build + pipeline: Azure.sonic-buildimage.common_libs + runVersion: 'latestFromBranch' + runBranch: 'refs/heads/${{ parameters.buildBranch }}' + path: $(Build.ArtifactStagingDirectory)/download + artifact: common-lib.arm64 + patterns: | + target/debs/bookworm/libyang-*_1.0*.deb + target/debs/bookworm/libyang_1.0*.deb + target/debs/bookworm/libnl-3-200_*.deb + target/debs/bookworm/libnl-genl-3-200_*.deb + target/debs/bookworm/libnl-route-3-200_*.deb + target/debs/bookworm/libnl-nf-3-200_*.deb + displayName: "Download libyang and libnl from arm64 common lib" - # REDIS - sudo apt-get update - sudo apt-get install -y redis-server - sudo sed -ri 's/^# unixsocket/unixsocket/' /etc/redis/redis.conf - sudo sed -ri 's/^unixsocketperm .../unixsocketperm 777/' /etc/redis/redis.conf - sudo sed -ri 's/redis-server.sock/redis.sock/' /etc/redis/redis.conf - sudo service redis-server start +# === Install test dependencies (amd64 test jobs only) === +- ${{ if and(eq(parameters.arch, 'amd64'), eq(parameters.installTestDeps, true)) }}: + - script: | + # PYTEST + sudo pip3 install -U pytest + sudo pip3 install -U jsonpatch + + # REDIS + sudo apt-get update + sudo apt-get install -y redis-server + sudo sed -ri 's/^# unixsocket/unixsocket/' /etc/redis/redis.conf + sudo sed -ri 's/^unixsocketperm .../unixsocketperm 777/' /etc/redis/redis.conf + sudo sed -ri 's/redis-server.sock/redis.sock/' /etc/redis/redis.conf + sudo service redis-server start + displayName: "Install test dependencies (pytest, redis)" + +# === Install libyang === +- ${{ if eq(parameters.arch, 'amd64') }}: + - script: | + # Note: Must use version-specific pattern to avoid conflicts with libyang3 packages + sudo dpkg -i ../target/debs/bookworm/libyang*1.0.73*.deb + displayName: "Install libyang" - # LIBYANG - # Note: Must use version-specific pattern to avoid conflicts with libyang3 packages - sudo dpkg -i ../target/debs/bookworm/libyang*1.0.73*.deb - displayName: "Install dependency" +- ${{ if eq(parameters.arch, 'arm64') }}: + - script: | + set -ex + sudo apt-get -y purge libnl-3-dev libnl-route-3-dev || true + sudo dpkg -i $(find $(Build.ArtifactStagingDirectory)/download -name '*.deb') + displayName: "Install libyang and libnl debs" + +# === Install sonic yang models === +# amd64: already downloaded above with bookworm debs +# arm64: need separate download +- ${{ if eq(parameters.arch, 'arm64') }}: + - task: DownloadPipelineArtifact@2 + inputs: + source: specific + project: build + pipeline: 142 + artifact: sonic-buildimage.vs + runVersion: 'latestFromBranch' + runBranch: 'refs/heads/${{ parameters.buildBranch }}' + patterns: | + target/python-wheels/bookworm/sonic_yang_models*.whl + displayName: "Download yang wheels" -# Install sonic yangs - script: | - # SONIC YANGS set -ex sudo pip3 install ../target/python-wheels/bookworm/sonic_yang_models*.whl displayName: "Install sonic yangs" -# Install libswsscommon dependencies -- script: | - # LIBSWSSCOMMON dependencies - sudo apt-get -y purge libnl-3-dev libnl-route-3-dev - sudo dpkg -i ../target/debs/bookworm/libnl-3-200_*.deb - sudo dpkg -i ../target/debs/bookworm/libnl-genl-3-200_*.deb - sudo dpkg -i ../target/debs/bookworm/libnl-route-3-200_*.deb - sudo dpkg -i ../target/debs/bookworm/libnl-nf-3-200_*.deb - displayName: "Install libswsscommon dependencies" +# === Install libnl (amd64 only — arm64 installs above with libyang) === +- ${{ if eq(parameters.arch, 'amd64') }}: + - script: | + sudo apt-get -y purge libnl-3-dev libnl-route-3-dev + sudo dpkg -i ../target/debs/bookworm/libnl-3-200_*.deb + sudo dpkg -i ../target/debs/bookworm/libnl-genl-3-200_*.deb + sudo dpkg -i ../target/debs/bookworm/libnl-route-3-200_*.deb + sudo dpkg -i ../target/debs/bookworm/libnl-nf-3-200_*.deb + displayName: "Install libnl dependencies" -# Install .NET Core -- script: | - set -ex - # Install .NET CORE - curl -sSL https://packages.microsoft.com/keys/microsoft.asc | sudo apt-key add - - sudo apt-add-repository https://packages.microsoft.com/debian/12/prod - sudo apt-get update - sudo apt-get install -y dotnet-sdk-8.0 - displayName: "Install .NET CORE" +# === Install .NET Core (amd64 test jobs only) === +- ${{ if and(eq(parameters.arch, 'amd64'), eq(parameters.installTestDeps, true)) }}: + - script: | + set -ex + curl -sSL https://packages.microsoft.com/keys/microsoft.asc | sudo apt-key add - + sudo apt-add-repository https://packages.microsoft.com/debian/12/prod + sudo apt-get update + sudo apt-get install -y dotnet-sdk-8.0 + displayName: "Install .NET CORE" -# Download sonic-swss-common +# === Download and install sonic-swss-common === - task: DownloadPipelineArtifact@2 inputs: source: specific project: build pipeline: Azure.sonic-swss-common - artifact: sonic-swss-common-bookworm + artifact: sonic-swss-common-bookworm${{ if eq(parameters.arch, 'arm64') }}.arm64${{ endif }} runVersion: 'latestFromBranch' runBranch: 'refs/heads/${{ parameters.buildBranch }}' - displayName: "Download sonic-swss-common" + displayName: "Download sonic-swss-common (${{ parameters.arch }})" -# Install libswsscommon packages - script: | set -ex - # LIBSWSSCOMMON - sudo dpkg -i libswsscommon_1.0.0_amd64.deb - sudo dpkg -i libswsscommon-dev_1.0.0_amd64.deb + sudo dpkg -i libswsscommon_1.0.0_${{ parameters.arch }}.deb + sudo dpkg -i libswsscommon-dev_1.0.0_${{ parameters.arch }}.deb + ${{ if eq(parameters.arch, 'amd64') }} sudo dpkg -i python3-swsscommon_1.0.0_amd64.deb + ${{ endif }} workingDirectory: $(Pipeline.Workspace)/ - displayName: 'Install libswsscommon package' + displayName: 'Install libswsscommon (${{ parameters.arch }})' -# Install protoc +# === Install protoc === - script: | + ${{ if eq(parameters.arch, 'arm64') }} + sudo apt-get update + ${{ endif }} sudo apt-get install -y protobuf-compiler protoc --version displayName: 'Install protoc' diff --git a/azure-pipelines.yml b/azure-pipelines.yml index d02ca38c5..8042372f9 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -132,6 +132,8 @@ stages: - template: .azure/templates/install-dependencies.yml parameters: buildBranch: $(BUILD_BRANCH) + arch: amd64 + installTestDeps: true # Memory leak tests with JUnit XML generation @@ -185,14 +187,12 @@ stages: submodules: recursive displayName: 'Checkout sonic-swss-common' - # Integration tests have been separated from pure package tests - # Pure package CI now runs in the separate PureCIJob above - # The steps below are for integration testing with SONiC dependencies - # Install SONiC dependencies using shared template - template: .azure/templates/install-dependencies.yml parameters: buildBranch: $(BUILD_BRANCH) + arch: amd64 + installTestDeps: true # Build sonic-mgmt-common and sonic-gnmi - script: | @@ -260,38 +260,7 @@ stages: image: sonicdev-microsoft.azurecr.io:443/sonic-slave-bookworm:$(BUILD_BRANCH)-arm64 steps: - - checkout: self - clean: true - submodules: recursive - displayName: 'Checkout code' - - - checkout: sonic-mgmt-common - clean: true - submodules: recursive - displayName: 'Checkout sonic-mgmt-common' - - - checkout: sonic-swss-common - clean: true - submodules: recursive - displayName: 'Checkout sonic-swss-common' - - - template: .azure/templates/install-dependencies-arm64.yml + - template: .azure/templates/build-deb.yml parameters: buildBranch: $(BUILD_BRANCH) - - - script: | - set -ex - pushd sonic-mgmt-common - NO_TEST_BINS=1 dpkg-buildpackage -rfakeroot -b -us -uc - popd - - pushd sonic-gnmi - ENABLE_TRANSLIB_WRITE=y ENABLE_NATIVE_WRITE=y dpkg-buildpackage -rfakeroot -us -uc -b -j$(nproc) && cp ../*.deb $(Build.ArtifactStagingDirectory)/ - displayName: "Build ARM64 deb" - - - script: rm -rf $(Build.ArtifactStagingDirectory)/download - displayName: "Remove downloaded dependencies from artifacts" - - - publish: $(Build.ArtifactStagingDirectory)/ - artifact: sonic-gnmi.arm64 - displayName: "Archive ARM64 artifacts" + arch: arm64 From 34d5796b8274ec9f24e1b0b8859003d46d3bf531 Mon Sep 17 00:00:00 2001 From: sigabrtv1-ui Date: Fri, 27 Mar 2026 17:46:44 +0000 Subject: [PATCH 8/9] ci: use common_libs for both amd64 and arm64 libyang/libnl downloads Switch amd64 from downloading the full sonic-buildimage.vs artifact to using Azure.sonic-buildimage.common_libs (artifact: common-lib), matching the pattern used by sonic-sairedis and other SONiC repos. - Both architectures now use the same download step with arch-aware artifact name - Yang wheels still downloaded separately from pipeline 142 (arch-independent) - Eliminates arch-specific branching for libyang/libnl download and install - Smaller artifact downloads for amd64 (common-lib vs full sonic-buildimage.vs) Signed-off-by: sigabrtv1-ui --- .azure/templates/install-dependencies.yml | 115 ++++++++-------------- 1 file changed, 39 insertions(+), 76 deletions(-) diff --git a/.azure/templates/install-dependencies.yml b/.azure/templates/install-dependencies.yml index b34015835..ec6481616 100644 --- a/.azure/templates/install-dependencies.yml +++ b/.azure/templates/install-dependencies.yml @@ -11,9 +11,9 @@ # installTestDeps: true # install pytest, redis, .NET (for test jobs) # # Dependencies installed (all architectures): -# - libyang and libnl packages +# - libyang and libnl packages (from sonic-buildimage.common_libs) # - sonic-swss-common libraries -# - sonic yang models +# - sonic yang models (from sonic-buildimage.vs, arch-independent) # - protobuf compiler # # Additional dependencies when installTestDeps=true (amd64 only): @@ -36,42 +36,25 @@ parameters: default: false steps: -# === Download libyang + libnl debs === -# amd64: from sonic-buildimage.vs (pipeline 142) -# arm64: from sonic-buildimage.common_libs -- ${{ if eq(parameters.arch, 'amd64') }}: - - task: DownloadPipelineArtifact@2 - inputs: - source: specific - project: build - pipeline: 142 - artifact: sonic-buildimage.vs - runVersion: 'latestFromBranch' - runBranch: 'refs/heads/${{ parameters.buildBranch }}' - patterns: | - target/debs/bookworm/libyang*.deb - target/debs/bookworm/libnl*.deb - target/python-wheels/bookworm/sonic_yang_models*.whl - displayName: "Download bookworm debs" - -- ${{ if eq(parameters.arch, 'arm64') }}: - - task: DownloadPipelineArtifact@2 - inputs: - source: specific - project: build - pipeline: Azure.sonic-buildimage.common_libs - runVersion: 'latestFromBranch' - runBranch: 'refs/heads/${{ parameters.buildBranch }}' - path: $(Build.ArtifactStagingDirectory)/download - artifact: common-lib.arm64 - patterns: | - target/debs/bookworm/libyang-*_1.0*.deb - target/debs/bookworm/libyang_1.0*.deb - target/debs/bookworm/libnl-3-200_*.deb - target/debs/bookworm/libnl-genl-3-200_*.deb - target/debs/bookworm/libnl-route-3-200_*.deb - target/debs/bookworm/libnl-nf-3-200_*.deb - displayName: "Download libyang and libnl from arm64 common lib" +# === Download libyang + libnl debs from common_libs === +# amd64 uses artifact "common-lib", arm64 uses "common-lib.arm64" +- task: DownloadPipelineArtifact@2 + inputs: + source: specific + project: build + pipeline: Azure.sonic-buildimage.common_libs + runVersion: 'latestFromBranch' + runBranch: 'refs/heads/${{ parameters.buildBranch }}' + path: $(Build.ArtifactStagingDirectory)/download + artifact: common-lib${{ if eq(parameters.arch, 'arm64') }}.arm64${{ endif }} + patterns: | + target/debs/bookworm/libyang_1.0*.deb + target/debs/bookworm/libyang-*_1.0*.deb + target/debs/bookworm/libnl-3-200_*.deb + target/debs/bookworm/libnl-genl-3-200_*.deb + target/debs/bookworm/libnl-route-3-200_*.deb + target/debs/bookworm/libnl-nf-3-200_*.deb + displayName: "Download libyang and libnl from common_libs (${{ parameters.arch }})" # === Install test dependencies (amd64 test jobs only) === - ${{ if and(eq(parameters.arch, 'amd64'), eq(parameters.installTestDeps, true)) }}: @@ -89,51 +72,31 @@ steps: sudo service redis-server start displayName: "Install test dependencies (pytest, redis)" -# === Install libyang === -- ${{ if eq(parameters.arch, 'amd64') }}: - - script: | - # Note: Must use version-specific pattern to avoid conflicts with libyang3 packages - sudo dpkg -i ../target/debs/bookworm/libyang*1.0.73*.deb - displayName: "Install libyang" - -- ${{ if eq(parameters.arch, 'arm64') }}: - - script: | - set -ex - sudo apt-get -y purge libnl-3-dev libnl-route-3-dev || true - sudo dpkg -i $(find $(Build.ArtifactStagingDirectory)/download -name '*.deb') - displayName: "Install libyang and libnl debs" +# === Install libyang + libnl debs === +- script: | + set -ex + sudo apt-get -y purge libnl-3-dev libnl-route-3-dev || true + sudo dpkg -i $(find $(Build.ArtifactStagingDirectory)/download -name '*.deb') + displayName: "Install libyang and libnl debs" -# === Install sonic yang models === -# amd64: already downloaded above with bookworm debs -# arm64: need separate download -- ${{ if eq(parameters.arch, 'arm64') }}: - - task: DownloadPipelineArtifact@2 - inputs: - source: specific - project: build - pipeline: 142 - artifact: sonic-buildimage.vs - runVersion: 'latestFromBranch' - runBranch: 'refs/heads/${{ parameters.buildBranch }}' - patterns: | - target/python-wheels/bookworm/sonic_yang_models*.whl - displayName: "Download yang wheels" +# === Download and install sonic yang models (arch-independent, from pipeline 142) === +- task: DownloadPipelineArtifact@2 + inputs: + source: specific + project: build + pipeline: 142 + artifact: sonic-buildimage.vs + runVersion: 'latestFromBranch' + runBranch: 'refs/heads/${{ parameters.buildBranch }}' + patterns: | + target/python-wheels/bookworm/sonic_yang_models*.whl + displayName: "Download sonic yang models" - script: | set -ex sudo pip3 install ../target/python-wheels/bookworm/sonic_yang_models*.whl displayName: "Install sonic yangs" -# === Install libnl (amd64 only — arm64 installs above with libyang) === -- ${{ if eq(parameters.arch, 'amd64') }}: - - script: | - sudo apt-get -y purge libnl-3-dev libnl-route-3-dev - sudo dpkg -i ../target/debs/bookworm/libnl-3-200_*.deb - sudo dpkg -i ../target/debs/bookworm/libnl-genl-3-200_*.deb - sudo dpkg -i ../target/debs/bookworm/libnl-route-3-200_*.deb - sudo dpkg -i ../target/debs/bookworm/libnl-nf-3-200_*.deb - displayName: "Install libnl dependencies" - # === Install .NET Core (amd64 test jobs only) === - ${{ if and(eq(parameters.arch, 'amd64'), eq(parameters.installTestDeps, true)) }}: - script: | From e823794e84567beb0b1a981e2b3eb8ae300871f8 Mon Sep 17 00:00:00 2001 From: sigabrtv1-ui Date: Fri, 27 Mar 2026 18:21:01 +0000 Subject: [PATCH 9/9] ci: fix Azure YAML parse errors - no inline directives in strings Azure Pipelines doesn't allow ${{ if }} directives embedded in strings. Replace with explicit parameters (commonLibArtifact, swssCommonArtifact, publishArtifact) and separate conditional blocks for arch-specific steps. Fixes validation errors: - 'The directive if is not allowed in this context' - Directives not supported for expressions embedded within a string Signed-off-by: sigabrtv1-ui --- .azure/templates/build-deb.yml | 20 ++++++-- .azure/templates/install-dependencies.yml | 57 +++++++++++++++-------- azure-pipelines.yml | 3 ++ 3 files changed, 56 insertions(+), 24 deletions(-) diff --git a/.azure/templates/build-deb.yml b/.azure/templates/build-deb.yml index d63b533ab..0fca6c58b 100644 --- a/.azure/templates/build-deb.yml +++ b/.azure/templates/build-deb.yml @@ -19,6 +19,15 @@ parameters: values: - amd64 - arm64 +- name: commonLibArtifact + type: string + default: common-lib +- name: swssCommonArtifact + type: string + default: sonic-swss-common-bookworm +- name: publishArtifact + type: string + default: sonic-gnmi steps: # Checkout all required repositories @@ -43,6 +52,8 @@ steps: buildBranch: ${{ parameters.buildBranch }} arch: ${{ parameters.arch }} installTestDeps: false + commonLibArtifact: ${{ parameters.commonLibArtifact }} + swssCommonArtifact: ${{ parameters.swssCommonArtifact }} # Build sonic-mgmt-common and sonic-gnmi - script: | @@ -55,12 +66,11 @@ steps: ENABLE_TRANSLIB_WRITE=y ENABLE_NATIVE_WRITE=y dpkg-buildpackage -rfakeroot -us -uc -b -j$(nproc) && cp ../*.deb $(Build.ArtifactStagingDirectory)/ displayName: "Build ${{ parameters.arch }} deb" -# Clean up downloaded artifacts from staging (arm64 downloads to staging dir) -- ${{ if eq(parameters.arch, 'arm64') }}: - - script: rm -rf $(Build.ArtifactStagingDirectory)/download - displayName: "Remove downloaded dependencies from artifacts" +# Clean up downloaded artifacts from staging +- script: rm -rf $(Build.ArtifactStagingDirectory)/download + displayName: "Remove downloaded dependencies from artifacts" # Publish deb artifacts - publish: $(Build.ArtifactStagingDirectory)/ - artifact: sonic-gnmi${{ if eq(parameters.arch, 'arm64') }}.arm64${{ endif }} + artifact: ${{ parameters.publishArtifact }} displayName: "Archive ${{ parameters.arch }} artifacts" diff --git a/.azure/templates/install-dependencies.yml b/.azure/templates/install-dependencies.yml index ec6481616..d56689ff4 100644 --- a/.azure/templates/install-dependencies.yml +++ b/.azure/templates/install-dependencies.yml @@ -34,10 +34,15 @@ parameters: - name: installTestDeps type: boolean default: false +- name: commonLibArtifact + type: string + default: common-lib +- name: swssCommonArtifact + type: string + default: sonic-swss-common-bookworm steps: # === Download libyang + libnl debs from common_libs === -# amd64 uses artifact "common-lib", arm64 uses "common-lib.arm64" - task: DownloadPipelineArtifact@2 inputs: source: specific @@ -46,7 +51,7 @@ steps: runVersion: 'latestFromBranch' runBranch: 'refs/heads/${{ parameters.buildBranch }}' path: $(Build.ArtifactStagingDirectory)/download - artifact: common-lib${{ if eq(parameters.arch, 'arm64') }}.arm64${{ endif }} + artifact: ${{ parameters.commonLibArtifact }} patterns: | target/debs/bookworm/libyang_1.0*.deb target/debs/bookworm/libyang-*_1.0*.deb @@ -113,26 +118,40 @@ steps: source: specific project: build pipeline: Azure.sonic-swss-common - artifact: sonic-swss-common-bookworm${{ if eq(parameters.arch, 'arm64') }}.arm64${{ endif }} + artifact: ${{ parameters.swssCommonArtifact }} runVersion: 'latestFromBranch' runBranch: 'refs/heads/${{ parameters.buildBranch }}' displayName: "Download sonic-swss-common (${{ parameters.arch }})" -- script: | - set -ex - sudo dpkg -i libswsscommon_1.0.0_${{ parameters.arch }}.deb - sudo dpkg -i libswsscommon-dev_1.0.0_${{ parameters.arch }}.deb - ${{ if eq(parameters.arch, 'amd64') }} - sudo dpkg -i python3-swsscommon_1.0.0_amd64.deb - ${{ endif }} - workingDirectory: $(Pipeline.Workspace)/ - displayName: 'Install libswsscommon (${{ parameters.arch }})' +# amd64: install libswsscommon + python3-swsscommon +# arm64: install libswsscommon only (no python3 package) +- ${{ if eq(parameters.arch, 'amd64') }}: + - script: | + set -ex + sudo dpkg -i libswsscommon_1.0.0_amd64.deb + sudo dpkg -i libswsscommon-dev_1.0.0_amd64.deb + sudo dpkg -i python3-swsscommon_1.0.0_amd64.deb + workingDirectory: $(Pipeline.Workspace)/ + displayName: 'Install libswsscommon (amd64)' + +- ${{ if eq(parameters.arch, 'arm64') }}: + - script: | + set -ex + sudo dpkg -i libswsscommon_1.0.0_arm64.deb + sudo dpkg -i libswsscommon-dev_1.0.0_arm64.deb + workingDirectory: $(Pipeline.Workspace)/ + displayName: 'Install libswsscommon (arm64)' # === Install protoc === -- script: | - ${{ if eq(parameters.arch, 'arm64') }} - sudo apt-get update - ${{ endif }} - sudo apt-get install -y protobuf-compiler - protoc --version - displayName: 'Install protoc' +- ${{ if eq(parameters.arch, 'arm64') }}: + - script: | + sudo apt-get update + sudo apt-get install -y protobuf-compiler + protoc --version + displayName: 'Install protoc' + +- ${{ if eq(parameters.arch, 'amd64') }}: + - script: | + sudo apt-get install -y protobuf-compiler + protoc --version + displayName: 'Install protoc' diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 8042372f9..29e292762 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -264,3 +264,6 @@ stages: parameters: buildBranch: $(BUILD_BRANCH) arch: arm64 + commonLibArtifact: common-lib.arm64 + swssCommonArtifact: sonic-swss-common-bookworm.arm64 + publishArtifact: sonic-gnmi.arm64