From ed90e3a9564025691761838e61efebc423f80a63 Mon Sep 17 00:00:00 2001 From: quasarea <10334982+quasarea@users.noreply.github.com> Date: Fri, 8 May 2026 21:32:36 +0100 Subject: [PATCH] Bug 4657340: update hosted security validation pipeline --- azure-pipelines.yml | 1152 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 1152 insertions(+) create mode 100644 azure-pipelines.yml diff --git a/azure-pipelines.yml b/azure-pipelines.yml new file mode 100644 index 0000000..77c37ab --- /dev/null +++ b/azure-pipelines.yml @@ -0,0 +1,1152 @@ +# Hybrid validation pipeline for the GitHub-hosted UE customization repo. +# +# This pipeline is intended to be connected to GitHub through the Azure Pipelines GitHub App +# so its checks surface back into GitHub pull requests targeting main. +# +# The hosted test stage can run immediately on the Technology stateful hosted pools. +# The security stage remains opt-in, but now uses the hosted-compatible fake-compile flow +# on the Technology stateful hosted Ubuntu pool instead of generic vmImage selection. +# Templates is consumed from Technology/Templates. +# The Technology project still needs: +# - a self-hosted Windows pool with UE installed and MSBuild available +# - service connections named `Coverity on Polaris` and `BlackDuckScanner` + +resources: + repositories: + - repository: Templates + type: git + name: Technology/Templates + ref: refs/heads/feature/uecf-blackduck-pool-overrides + - repository: antiMalwareTemplate + type: git + name: Architecture/Architecture + ref: refs/heads/antiMalwareVersions/latest + +parameters: + - name: EnableSecurityValidation + type: string + default: "true" + - name: EnableUbuntuBlackDuckComparison + type: string + default: "false" + - name: EnableComponentTemplateBlackDuckComparison + type: string + default: "false" + +variables: + - template: variables.yml@Templates + - name: CoverityProjectTeam + value: .RnD.Polaris.Dabacon-Products.SecurityAdvisors + - name: CoverityCliVersion + value: 2025.9.0 + - name: BlackDuckProjectName + value: AVEVA.UnifiedEngineeringCustomizationFramework + +trigger: + branches: + include: + - main + +pr: + autoCancel: "true" + branches: + include: + - main + +stages: + - stage: InitialiseSource + displayName: Initialise Source + jobs: + - job: Source + displayName: Publish Source + pool: + name: Technology-Microsoft-Hosted-Ubuntu + demands: + - ImageOverride -equals ubuntu-latest + - WorkFolder -equals /mnt/storage/sdc/storage + container: mcr.microsoft.com/dotnet/sdk:9.0 + workspace: + clean: all + steps: + - checkout: self + fetchDepth: "0" + path: Source + + - checkout: Templates + fetchDepth: "1" + path: Templates + + - pwsh: | + $sourceRoot = "$(Pipeline.Workspace)/Source" + $probeRoot = Join-Path $sourceRoot 'detector-coverage-probe' + $classicProbeRoot = Join-Path $probeRoot 'ClassicProbe' + $sdkProbeRoot = Join-Path $probeRoot 'SdkProbe' + + New-Item -Path $probeRoot -ItemType Directory -Force | Out-Null + New-Item -Path $classicProbeRoot -ItemType Directory -Force | Out-Null + New-Item -Path $sdkProbeRoot -ItemType Directory -Force | Out-Null + + $classicPackagesConfig = @( + '' + '' + ' ' + '' + ) -join "`n" + + $classicProject = @( + '' + ' ' + ' Debug' + ' AnyCPU' + ' Library' + ' v4.8.1' + ' ClassicProbe' + ' ClassicProbe' + ' ' + ' ' + ' ' + ' ' + ' ' + ' ' + '' + ) -join "`n" + + $classicSource = @( + 'namespace ClassicProbe' + '{' + ' public class Class1 { }' + '}' + ) -join "`n" + + $sdkProject = @( + '' + ' ' + ' net8.0' + ' disable' + ' disable' + ' ' + ' ' + ' ' + ' ' + '' + ) -join "`n" + + $sdkSource = @( + 'namespace SdkProbe;' + 'public class Class1 { }' + ) -join "`n" + + Set-Content -LiteralPath (Join-Path $classicProbeRoot 'packages.config') -Value $classicPackagesConfig -Encoding utf8 + Set-Content -LiteralPath (Join-Path $classicProbeRoot 'ClassicProbe.csproj') -Value $classicProject -Encoding utf8 + Set-Content -LiteralPath (Join-Path $classicProbeRoot 'Class1.cs') -Value $classicSource -Encoding utf8 + + Set-Content -LiteralPath (Join-Path $sdkProbeRoot 'SdkProbe.csproj') -Value $sdkProject -Encoding utf8 + Set-Content -LiteralPath (Join-Path $sdkProbeRoot 'Class1.cs') -Value $sdkSource -Encoding utf8 + + New-Item -Path (Join-Path $sourceRoot '.artifactignore') -ItemType File -Force | Out-Null + displayName: Prepare Source artifact + + - publish: $(Pipeline.Workspace)/Source + artifact: Source + displayName: Publish Source + + - publish: $(Pipeline.Workspace)/Templates + artifact: Templates + displayName: Publish Templates + + - stage: BuildWithShims + displayName: Build With Shims + dependsOn: InitialiseSource + condition: and(succeeded(), ${{ eq(parameters.EnableSecurityValidation, 'true') }}) + jobs: + - job: BuildWithShims + displayName: Build With Shims + timeoutInMinutes: "120" + pool: + name: Technology-Microsoft-Hosted-Ubuntu + demands: + - ImageOverride -equals ubuntu-latest + - WorkFolder -equals /mnt/storage/sdc/storage + container: mcr.microsoft.com/dotnet/sdk:9.0 + workspace: + clean: all + steps: + - template: steps/retrieve-source.yml@Templates + parameters: + IsIPR: false + ArtifactName: Source + DownloadPath: $(Pipeline.Workspace)/Source + + - download: current + artifact: Templates + displayName: Download Templates artifact + patterns: "**" + + - task: UseDotNet@2 + displayName: Ensure .NET 9 SDK for shim build + retryCountOnTaskFailure: 3 + inputs: + packageType: sdk + version: 9.0.x + + - pwsh: | + $repoRoot = Join-Path $env:PIPELINE_WORKSPACE 'Source' + & (Join-Path $repoRoot 'scripts/Invoke-FakeUECompile.ps1') -Directory templates,Examples -Configuration Release + displayName: Build template and example projects with fake UE references + + - publish: $(Pipeline.Workspace)/Source/shims/artifacts/Release + artifact: Build.Release + displayName: Publish build artifact for component template comparison + + - stage: HostedTests + displayName: Hosted Tests + dependsOn: InitialiseSource + jobs: + - job: Pester + displayName: Pester + pool: + name: Technology-Microsoft-Hosted-Ubuntu + demands: + - ImageOverride -equals ubuntu-latest + - WorkFolder -equals /mnt/storage/sdc/storage + container: mcr.microsoft.com/dotnet/sdk:9.0 + workspace: + clean: all + steps: + - template: steps/retrieve-source.yml@Templates + parameters: + IsIPR: false + ArtifactName: Source + DownloadPath: $(Pipeline.Workspace)/Source + + - pwsh: | + Write-Host "PowerShell version: $($PSVersionTable.PSVersion)" + displayName: Show PowerShell version + + - pwsh: | + if (-not (Get-PackageProvider -Name NuGet -ListAvailable -ErrorAction SilentlyContinue)) { + Write-Host "Installing NuGet provider..." + Install-PackageProvider -Name NuGet -MinimumVersion 2.8.5.201 -Force | Out-Null + } + displayName: Install NuGet provider + + - pwsh: | + $psGallery = Get-PSRepository -Name 'PSGallery' -ErrorAction SilentlyContinue + if ($psGallery -and $psGallery.InstallationPolicy -ne 'Trusted') { + Write-Host "Trusting PSGallery..." + Set-PSRepository -Name 'PSGallery' -InstallationPolicy Trusted + } + displayName: Trust PSGallery + + - pwsh: | + if (-not (Get-Module -ListAvailable -Name Pester)) { + Write-Host "Installing Pester module..." + Install-Module Pester -Scope CurrentUser -Force -AllowClobber -SkipPublisherCheck | Out-Null + } + displayName: Install Pester + + - pwsh: | + ./scripts/Tests/Run-Tests.ps1 -Quick + displayName: Run tests + workingDirectory: $(Pipeline.Workspace)/Source + + - stage: Coverity + displayName: Coverity + dependsOn: InitialiseSource + condition: and(succeeded(), ${{ eq(parameters.EnableSecurityValidation, 'true') }}) + jobs: + - job: CoverityScan + displayName: Coverity Scan + timeoutInMinutes: "180" + pool: + name: Technology-Microsoft-Hosted-Ubuntu + demands: + - ImageOverride -equals ubuntu-latest + - WorkFolder -equals /mnt/storage/sdc/storage + container: mcr.microsoft.com/dotnet/sdk:9.0 + workspace: + clean: all + steps: + - template: steps/retrieve-source.yml@Templates + parameters: + IsIPR: false + ArtifactName: Source + DownloadPath: $(Pipeline.Workspace)/Source + + - download: current + artifact: Templates + displayName: Download Templates artifact + patterns: "**" + + - pwsh: | + dotnet --list-sdks + displayName: List globally installed .NET SDKs + + - task: UseDotNet@2 + displayName: Ensure .NET 9 SDK for Linux + retryCountOnTaskFailure: 3 + inputs: + packageType: sdk + version: 9.0.x + + - pwsh: | + dotnet --info + displayName: Show .NET SDK info + + - pwsh: | + $repoRoot = Join-Path $env:PIPELINE_WORKSPACE 'Source' + & (Join-Path $repoRoot 'scripts\Invoke-FakeUECompile.ps1') -Directory templates,Examples -Configuration Release + displayName: Build template and example projects with fake UE references + + - pwsh: | + $repoRoot = Join-Path $env:PIPELINE_WORKSPACE 'Source' + $paths = @{ + HasShimSources = Test-Path -LiteralPath (Join-Path $repoRoot 'shims') + HasShimManifest = Test-Path -LiteralPath (Join-Path $repoRoot 'shims/shim-projects.psd1') + HasGeneratedShimProjects = Test-Path -LiteralPath (Join-Path $repoRoot '.tmp/generated-shims') + HasShimAssemblies = Test-Path -LiteralPath (Join-Path $repoRoot 'shims/artifacts/Release') + } + + foreach ($entry in $paths.GetEnumerator()) { + $value = if ($entry.Value) { 'true' } else { 'false' } + Write-Host "##vso[task.setvariable variable=$($entry.Key)]$value" + Write-Host "$($entry.Key): $value" + } + displayName: Detect shim artifacts + condition: always() + + - publish: $(Pipeline.Workspace)/Source/shims + artifact: ShimSources + displayName: Publish shim sources + condition: and(always(), eq(variables.HasShimSources, 'true')) + + - publish: $(Pipeline.Workspace)/Source/shims/shim-projects.psd1 + artifact: ShimManifest + displayName: Publish shim manifest + condition: and(always(), eq(variables.HasShimManifest, 'true')) + + - publish: $(Pipeline.Workspace)/Source/.tmp/generated-shims + artifact: GeneratedShimProjects + displayName: Publish generated shim project files + condition: and(always(), eq(variables.HasGeneratedShimProjects, 'true')) + + - publish: $(Pipeline.Workspace)/Source/shims/artifacts/Release + artifact: ShimAssemblies + displayName: Publish compiled shim assemblies + condition: and(always(), eq(variables.HasShimAssemblies, 'true')) + + - pwsh: | + $pwshPath = if ($IsLinux) { '/usr/bin/pwsh' } else { (Get-Command pwsh -ErrorAction Stop).Source } + $repoRoot = Join-Path $env:PIPELINE_WORKSPACE 'Source' + $fakeCompileScriptPath = Join-Path $repoRoot 'scripts/Invoke-FakeUECompile.ps1' + + $branchName = "$env:BUILD_SOURCEBRANCH" -replace '^refs/heads/', '' + $templatesBranch = "$env:TEMPLATES_REF".replace('refs/heads/', '') + $templatesPostfix = ($env:DOBUILDANALYSIS -eq 'true' -and $templatesBranch -and $templatesBranch -ne 'main') ` + ? "@$($templatesBranch)" ` + : '' + $polarisBranch = "$branchName$templatesPostfix" + $commitDate = ([DateTime](git -C $repoRoot log -1 --format=%ci)).ToString('o') + + $cleanCommands = @( + " - shell: ['$pwshPath', '-NoProfile', '-Command', 'Write-Host fake-clean-step']" + ) + + $buildCommands = @( + " - shell: ['$pwshPath', '-NoProfile', '-File', '$fakeCompileScriptPath', '-Directory', 'templates,Examples', '-Configuration', 'Release', '-Rebuild']" + ) + + $polarisContent = @( + 'version: 1' + 'project:' + " name: $(BlackDuckProjectName)" + " branch: $polarisBranch" + " projectDir: $repoRoot" + ' revision:' + " name: $env:BUILD_SOURCEVERSION" + " date: $commitDate" + 'capture:' + ' build:' + ' cleanCommands:' + $cleanCommands + ' buildCommands:' + $buildCommands + 'analyze:' + ' mode: central' + 'install:' + ' coverity:' + " version: $(CoverityCliVersion)" + 'serverUrl: https://osisoft.cop.blackduck.com' + ) -join "`n" + + $polarisPath = Join-Path $env:PIPELINE_WORKSPACE 'polaris.yml' + $polarisContent | Out-File -FilePath $polarisPath -Encoding ascii -Force + Get-Content -Path $polarisPath + displayName: Generate polaris.yml + + - task: Cache@2 + displayName: Restore Coverity CLI cache + retryCountOnTaskFailure: 3 + inputs: + key: "\"CoverityCLI\" | \"$(CoverityCliVersion)\" | \"v1\"" + path: $(Pipeline.Workspace)/Coverity-CLI + cacheHitVar: COVERITY_CLI_RESTORED + + - pwsh: | + $toolPath = "$(Pipeline.Workspace)/Coverity-CLI" + if (-not (Test-Path -Path $toolPath)) { + New-Item -ItemType Directory -Path $toolPath | Out-Null + } + displayName: Ensure Coverity CLI folder + + - task: BlackduckCoverityOnPolaris@2 + displayName: Coverity scan + inputs: + polarisService: Coverity on Polaris + polarisCommand: > + -c $(Pipeline.Workspace)/polaris.yml + analyze + -w + env: + AZURE_DEVOPS_EXT_PAT: $(System.AccessToken) + POLARIS_HOME: $(Pipeline.Workspace)/Coverity-CLI + + - pwsh: | + $repoRoot = Join-Path $env:PIPELINE_WORKSPACE 'Source' + $paths = @{ + HasCoverityArtifacts = Test-Path -LiteralPath (Join-Path $repoRoot '.blackduck') + } + + foreach ($entry in $paths.GetEnumerator()) { + $value = if ($entry.Value) { 'true' } else { 'false' } + Write-Host "##vso[task.setvariable variable=$($entry.Key)]$value" + Write-Host "$($entry.Key): $value" + } + displayName: Detect security artifacts + condition: always() + + - publish: $(Pipeline.Workspace)/Source/.blackduck + artifact: Coverity + displayName: Publish Coverity artifacts + condition: and(always(), eq(variables.HasCoverityArtifacts, 'true')) + + - stage: AntiMalware + displayName: AntiMalware + dependsOn: InitialiseSource + condition: and(succeeded(), ${{ eq(parameters.EnableSecurityValidation, 'true') }}) + jobs: + - job: AntiMalwareScan + displayName: AntiMalware Scan + timeoutInMinutes: "60" + pool: + name: Technology-Microsoft-Hosted-Windows + demands: + - ImageOverride -equals windows-latest + workspace: + clean: all + steps: + - template: steps/retrieve-source.yml@Templates + parameters: + IsIPR: false + ArtifactName: Source + DownloadPath: $(Pipeline.Workspace)/Source + + - pwsh: | + Start-Process powershell.exe -ArgumentList "Update-MpSignature -UpdateSource MicrosoftUpdateServer" -Wait + Write-Output "updated the definitions successfully" + displayName: Antimalware Scan task + + - template: Pipelines/Templates/AntiMalware/AntiMalware.step.yml@antiMalwareTemplate + parameters: + scanType: Directory + scanDirectory: $(Pipeline.Workspace)\Source + artifactName: AntimalwareResults_Source + os: Windows_NT + + - stage: AntiMalwareSarif + displayName: AntiMalware SARIF + dependsOn: AntiMalware + condition: and(not(canceled()), ${{ eq(parameters.EnableSecurityValidation, 'true') }}) + jobs: + - template: jobs/publish-CodeAnalysisLogs.yml@Templates + parameters: + Analysers: [] + DirectSarif: + - AntimalwareResults_Source + UbuntuPoolName: Technology-Microsoft-Hosted-Ubuntu + UbuntuPoolDemands: + - ImageOverride -equals ubuntu-latest + - WorkFolder -equals /mnt/storage/sdc/storage + + - stage: BlackDuckWindows + displayName: Black Duck - Windows + dependsOn: BuildWithShims + condition: and(succeeded(), ${{ eq(parameters.EnableSecurityValidation, 'true') }}) + jobs: + - job: BlackDuckInitialisation + displayName: BlackDuck Initialise + timeoutInMinutes: "10" + pool: + name: Technology-Microsoft-Hosted-Ubuntu + demands: + - ImageOverride -equals ubuntu-latest + - WorkFolder -equals /mnt/storage/sdc/storage + container: mcr.microsoft.com/dotnet/sdk:9.0 + workspace: + clean: all + variables: + - group: BlackDuck on Polaris + steps: + - checkout: none + + - pwsh: | + $templatesBranch = "$env:TEMPLATES_REF".replace('refs/heads/', '') + $templatesPostfix = ($templatesBranch -and $templatesBranch -ne 'main') ` + ? "@$($templatesBranch)" ` + : '' + + $branchVersion = "$env:BUILD_SOURCEBRANCHNAME$templatesPostfix" + + if ($env:BUILD_REASON -eq 'PullRequest' -and $env:SYSTEM_PULLREQUEST_PULLREQUESTID) { + $baseSemanticVersion = "pr-$($env:SYSTEM_PULLREQUEST_PULLREQUESTID)-$($env:BUILD_BUILDID)" + } + else { + $branchName = $env:BUILD_SOURCEBRANCHNAME -replace '[^A-Za-z0-9._-]', '-' + $baseSemanticVersion = "$branchName-$($env:BUILD_BUILDID)" + } + + $semanticVersion = "$baseSemanticVersion-windows$templatesPostfix" + $ubuntuComparisonVersion = "$baseSemanticVersion-linux$templatesPostfix" + $componentComparisonVersion = "$baseSemanticVersion-components$templatesPostfix" + + Write-Host "##vso[task.setvariable variable=BranchVersion]$branchVersion" + Write-Host "##vso[task.setvariable variable=BranchVersion;isOutput=true]$branchVersion" + Write-Host "##vso[task.setvariable variable=SemanticVersion]$semanticVersion" + Write-Host "##vso[task.setvariable variable=SemanticVersion;isOutput=true]$semanticVersion" + Write-Host "##vso[task.setvariable variable=UbuntuComparisonVersion]$ubuntuComparisonVersion" + Write-Host "##vso[task.setvariable variable=UbuntuComparisonVersion;isOutput=true]$ubuntuComparisonVersion" + Write-Host "##vso[task.setvariable variable=ComponentComparisonVersion]$componentComparisonVersion" + Write-Host "##vso[task.setvariable variable=ComponentComparisonVersion;isOutput=true]$componentComparisonVersion" + displayName: Calculate Black Duck versions + name: CalculateVersion + + - pwsh: | + $headers = @{ + Authorization = "token $env:BLACKDUCK_ACCESS_TOKEN" + Accept = 'application/vnd.blackducksoftware.user-4+json' + } + + $uri = "$env:BLACKDUCK_URL/api/tokens/authenticate" + $response = Invoke-RestMethod -Method POST -Uri $uri -Headers $headers + + Write-Host "##vso[task.setvariable variable=BaseUrl]$env:BLACKDUCK_URL" + Write-Host "##vso[task.setvariable variable=BaseUrl;isOutput=true]$env:BLACKDUCK_URL" + Write-Host "##vso[task.setvariable variable=BearerToken]$($response.bearerToken)" + Write-Host "##vso[task.setvariable variable=BearerToken;isOutput=true]$($response.bearerToken)" + displayName: Authenticate to Black Duck + name: Authenticate + retryCountOnTaskFailure: 3 + env: + BLACKDUCK_URL: https://aveva.app.blackduck.com + BLACKDUCK_ACCESS_TOKEN: $(BLACKDUCK_ACCESS_TOKEN) + + - pwsh: | + $headers = @{ + Authorization = "Bearer $env:BearerToken" + Accept = 'application/vnd.blackducksoftware.project-detail-7+json' + } + + $projectName = '$(BlackDuckProjectName)' + $limit = 100 + $offset = 0 + $allProjects = @() + + do { + $uri = "$env:BaseUrl/api/projects?limit=$limit&offset=$offset&q=name:$projectName" + $response = Invoke-RestMethod -Method GET -Uri $uri -Headers $headers + if ($response.items) { + $allProjects += $response.items + } + $offset += $limit + } while ($offset -lt $response.totalCount) + + $matchedProject = $allProjects | Where-Object { $_.name -eq $projectName } + + if (-not $matchedProject) { + Write-Host "Project '$projectName' not found in Black Duck." + Write-Host "##vso[task.setvariable variable=ProjectExists]false" + Write-Host "##vso[task.setvariable variable=ProjectExists;isOutput=true]false" + Write-Host "##vso[task.logissue type=error]Black Duck project '$projectName' must exist before the scan runs. Create it or grant the scanner access to the existing project." + exit 1 + } + + $projectId = $matchedProject._meta.href.Split('/')[-1] + Write-Host "##vso[task.setvariable variable=ProjectId]$projectId" + Write-Host "##vso[task.setvariable variable=ProjectId;isOutput=true]$projectId" + Write-Host "##vso[task.setvariable variable=ProjectExists]true" + Write-Host "##vso[task.setvariable variable=ProjectExists;isOutput=true]true" + displayName: Get Black Duck project ID + name: ProjectId + retryCountOnTaskFailure: 3 + env: + BearerToken: $(BearerToken) + BaseUrl: $(BaseUrl) + + - job: BlackDuckClone + displayName: BlackDuck Clone + dependsOn: BlackDuckInitialisation + condition: and(succeeded(), eq(dependencies.BlackDuckInitialisation.outputs['ProjectId.ProjectExists'], 'true')) + timeoutInMinutes: "10" + pool: + name: Technology-Microsoft-Hosted-Ubuntu + demands: + - ImageOverride -equals ubuntu-latest + - WorkFolder -equals /mnt/storage/sdc/storage + container: mcr.microsoft.com/dotnet/sdk:9.0 + workspace: + clean: all + variables: + - group: BlackDuck on Polaris + - name: BearerToken + value: $[ dependencies.BlackDuckInitialisation.outputs['Authenticate.BearerToken'] ] + - name: BaseUrl + value: $[ dependencies.BlackDuckInitialisation.outputs['Authenticate.BaseUrl'] ] + - name: ProjectId + value: $[ dependencies.BlackDuckInitialisation.outputs['ProjectId.ProjectId'] ] + - name: ProjectExists + value: $[ dependencies.BlackDuckInitialisation.outputs['ProjectId.ProjectExists'] ] + - name: BranchVersion + value: $[ dependencies.BlackDuckInitialisation.outputs['CalculateVersion.BranchVersion'] ] + - name: SemanticVersion + value: $[ dependencies.BlackDuckInitialisation.outputs['CalculateVersion.SemanticVersion'] ] + - name: UbuntuComparisonVersion + value: $[ dependencies.BlackDuckInitialisation.outputs['CalculateVersion.UbuntuComparisonVersion'] ] + steps: + - checkout: none + + - template: steps/blackduck-clone-version.yml@Templates + parameters: + BearerToken: $(BearerToken) + BaseUrl: $(BaseUrl) + ProjectId: $(ProjectId) + SourceVersion: main + TargetVersion: $(BranchVersion) + Phase: DEVELOPMENT + RemoveTargetVersion: false + + - template: steps/blackduck-clone-version.yml@Templates + parameters: + BearerToken: $(BearerToken) + BaseUrl: $(BaseUrl) + ProjectId: $(ProjectId) + SourceVersion: $(BranchVersion) + TargetVersion: $(SemanticVersion) + Phase: PLANNING + RemoveTargetVersion: true + + - template: steps/blackduck-clone-version.yml@Templates + parameters: + BearerToken: $(BearerToken) + BaseUrl: $(BaseUrl) + ProjectId: $(ProjectId) + SourceVersion: $(BranchVersion) + TargetVersion: $(UbuntuComparisonVersion) + Phase: PLANNING + RemoveTargetVersion: false + + - job: BlackDuckScan + displayName: Black Duck Scan + dependsOn: + - BlackDuckInitialisation + - BlackDuckClone + condition: and(succeeded(), eq(dependencies.BlackDuckInitialisation.outputs['ProjectId.ProjectExists'], 'true')) + timeoutInMinutes: "120" + pool: + name: Technology-Microsoft-Hosted-Windows + demands: + - ImageOverride -equals windows-latest + container: mcr.microsoft.com/dotnet/sdk:9.0-windowsservercore-ltsc2022 + workspace: + clean: all + variables: + - name: SemanticVersion + value: $[ dependencies.BlackDuckInitialisation.outputs['CalculateVersion.SemanticVersion'] ] + steps: + - template: steps/retrieve-source.yml@Templates + parameters: + IsIPR: false + ArtifactName: Source + DownloadPath: $(Pipeline.Workspace)/Source + + - pwsh: | + $workdir = "$(Pipeline.Workspace)\JDK" + New-Item -Path $workdir -ItemType Directory -Force | Out-Null + + $url = "https://github.com/adoptium/temurin8-binaries/releases/download/jdk8u452-b09/OpenJDK8U-jdk_x64_windows_hotspot_8u452b09.zip" + $zipPath = "$workdir\Temurin8.zip" + + Write-Host "Invoke-WebRequest -Uri $url -OutFile $zipPath" + [Net.ServicePointManager]::SecurityProtocol = "Tls12" + Invoke-WebRequest -Uri $url -OutFile $zipPath + + Write-Host "Expand-Archive -LiteralPath $zipPath -DestinationPath $workdir -Force" + Expand-Archive -LiteralPath $zipPath -DestinationPath $workdir -Force + displayName: Download Temurin JDK 8 + + - pwsh: | + $jdkFolder = Get-ChildItem -Path "$(Pipeline.Workspace)\JDK" -Directory | + Where-Object { $_.Name -like 'jdk8*' } | + Select-Object -First 1 + + if (-not $jdkFolder) { + Write-Error "Cannot locate a jdk8* folder under $(Pipeline.Workspace)\JDK" + exit 1 + } + + $javaHome = $jdkFolder.FullName + Write-Host "##vso[task.setvariable variable=JAVA_HOME]$javaHome" + Write-Host "##vso[task.setvariable variable=PATH]$javaHome\bin;$($env:PATH)" + Write-Host "JAVA_HOME is now set to $javaHome" + displayName: Configure JAVA_HOME for Temurin JDK 8 + + - pwsh: | + Write-Host "Verifying Java..." + java -version + displayName: Verify Java Installation + + - task: ArchiveFiles@2 + displayName: Archive sources and build outputs + inputs: + rootFolderOrFile: $(Pipeline.Workspace)\Source + includeRootFolder: true + archiveType: zip + archiveFile: $(Build.ArtifactStagingDirectory)\blackduck-input.zip + replaceExistingArchive: true + verbose: true + + - task: BlackDuckDetectTask@10 + displayName: Black Duck scan + retryCountOnTaskFailure: 0 + inputs: + BlackDuckScaService: BlackDuckScanner + DetectVersion: latest + DetectFolder: $(Pipeline.Workspace)\detect + DetectArguments: | + --detect.project.name=$(BlackDuckProjectName) + --detect.project.version.name=$(SemanticVersion) + --detect.source.path=$(Pipeline.Workspace)\Source + --detect.binary.scan.file.path=$(Build.ArtifactStagingDirectory)\blackduck-input.zip + --detect.cleanup=true + --detect.output.path=$(Build.ArtifactStagingDirectory)\blackduck-logs + --detect.detector.search.depth=10 + --detect.detector.search.continue=true + --detect.blackduck.signature.scanner.upload.source.mode=true + --detect.blackduck.signature.scanner.copyright.search=true + --detect.blackduck.signature.scanner.license.search=true + --logging.level.detect=DEBUG + --detect.diagnostic=true + --detect.timeout=1800 + --detect.blackduck.signature.scanner.arguments= --fs-wait-time=90 + --detect.wait.for.results=true + --detect.wait.for.results.timeout=6000000 + + - pwsh: | + $artifactStaging = $env:BUILD_ARTIFACTSTAGINGDIRECTORY + $paths = @{ + HasBlackDuckLogs = Test-Path -LiteralPath (Join-Path $artifactStaging 'blackduck-logs') + HasBlackDuckInput = Test-Path -LiteralPath (Join-Path $artifactStaging 'blackduck-input.zip') + } + + foreach ($entry in $paths.GetEnumerator()) { + $value = if ($entry.Value) { 'true' } else { 'false' } + Write-Host "##vso[task.setvariable variable=$($entry.Key)]$value" + Write-Host "$($entry.Key): $value" + } + displayName: Detect Black Duck artifacts + condition: always() + + - publish: $(Build.ArtifactStagingDirectory)/blackduck-logs + artifact: BlackDuckLogs + displayName: Publish Black Duck logs + condition: and(always(), eq(variables.HasBlackDuckLogs, 'true')) + + - publish: $(Build.ArtifactStagingDirectory)/blackduck-input.zip + artifact: BlackDuckInput + displayName: Publish Black Duck input archive + condition: and(always(), eq(variables.HasBlackDuckInput, 'true')) + + - stage: BlackDuckUbuntu + displayName: Black Duck - Ubuntu + dependsOn: BuildWithShims + condition: and(succeeded(), ${{ eq(parameters.EnableSecurityValidation, 'true') }}) + jobs: + - job: BlackDuckInitialisation + displayName: BlackDuck Initialise + timeoutInMinutes: "10" + pool: + name: Technology-Microsoft-Hosted-Ubuntu + demands: + - ImageOverride -equals ubuntu-latest + - WorkFolder -equals /mnt/storage/sdc/storage + container: mcr.microsoft.com/dotnet/sdk:9.0 + workspace: + clean: all + variables: + - group: BlackDuck on Polaris + steps: + - checkout: none + + - pwsh: | + $templatesBranch = "$env:TEMPLATES_REF".replace('refs/heads/', '') + $templatesPostfix = ($templatesBranch -and $templatesBranch -ne 'main') ` + ? "@$($templatesBranch)" ` + : '' + + $branchVersion = "$env:BUILD_SOURCEBRANCHNAME$templatesPostfix" + + if ($env:BUILD_REASON -eq 'PullRequest' -and $env:SYSTEM_PULLREQUEST_PULLREQUESTID) { + $baseSemanticVersion = "pr-$($env:SYSTEM_PULLREQUEST_PULLREQUESTID)-$($env:BUILD_BUILDID)" + } + else { + $branchName = $env:BUILD_SOURCEBRANCHNAME -replace '[^A-Za-z0-9._-]', '-' + $baseSemanticVersion = "$branchName-$($env:BUILD_BUILDID)" + } + + $ubuntuComparisonVersion = "$baseSemanticVersion-linux$templatesPostfix" + + Write-Host "##vso[task.setvariable variable=BranchVersion]$branchVersion" + Write-Host "##vso[task.setvariable variable=BranchVersion;isOutput=true]$branchVersion" + Write-Host "##vso[task.setvariable variable=UbuntuComparisonVersion]$ubuntuComparisonVersion" + Write-Host "##vso[task.setvariable variable=UbuntuComparisonVersion;isOutput=true]$ubuntuComparisonVersion" + displayName: Calculate Black Duck versions + name: CalculateVersion + + - pwsh: | + $headers = @{ + Authorization = "token $env:BLACKDUCK_ACCESS_TOKEN" + Accept = 'application/vnd.blackducksoftware.user-4+json' + } + + $uri = "$env:BLACKDUCK_URL/api/tokens/authenticate" + $response = Invoke-RestMethod -Method POST -Uri $uri -Headers $headers + + Write-Host "##vso[task.setvariable variable=BaseUrl]$env:BLACKDUCK_URL" + Write-Host "##vso[task.setvariable variable=BaseUrl;isOutput=true]$env:BLACKDUCK_URL" + Write-Host "##vso[task.setvariable variable=BearerToken]$($response.bearerToken)" + Write-Host "##vso[task.setvariable variable=BearerToken;isOutput=true]$($response.bearerToken)" + displayName: Authenticate to Black Duck + name: Authenticate + retryCountOnTaskFailure: 3 + env: + BLACKDUCK_URL: https://aveva.app.blackduck.com + BLACKDUCK_ACCESS_TOKEN: $(BLACKDUCK_ACCESS_TOKEN) + + - pwsh: | + $headers = @{ + Authorization = "Bearer $env:BearerToken" + Accept = 'application/vnd.blackducksoftware.project-detail-7+json' + } + + $projectName = '$(BlackDuckProjectName)' + $limit = 100 + $offset = 0 + $allProjects = @() + + do { + $uri = "$env:BaseUrl/api/projects?limit=$limit&offset=$offset&q=name:$projectName" + $response = Invoke-RestMethod -Method GET -Uri $uri -Headers $headers + if ($response.items) { + $allProjects += $response.items + } + $offset += $limit + } while ($offset -lt $response.totalCount) + + $matchedProject = $allProjects | Where-Object { $_.name -eq $projectName } + + if (-not $matchedProject) { + Write-Host "Project '$projectName' not found in Black Duck." + Write-Host "##vso[task.setvariable variable=ProjectExists]false" + Write-Host "##vso[task.setvariable variable=ProjectExists;isOutput=true]false" + Write-Host "##vso[task.logissue type=error]Black Duck project '$projectName' must exist before the scan runs. Create it or grant the scanner access to the existing project." + exit 1 + } + + $projectId = $matchedProject._meta.href.Split('/')[-1] + Write-Host "##vso[task.setvariable variable=ProjectId]$projectId" + Write-Host "##vso[task.setvariable variable=ProjectId;isOutput=true]$projectId" + Write-Host "##vso[task.setvariable variable=ProjectExists]true" + Write-Host "##vso[task.setvariable variable=ProjectExists;isOutput=true]true" + displayName: Get Black Duck project ID + name: ProjectId + retryCountOnTaskFailure: 3 + env: + BearerToken: $(BearerToken) + BaseUrl: $(BaseUrl) + + - job: BlackDuckClone + displayName: BlackDuck Clone + dependsOn: BlackDuckInitialisation + condition: and(succeeded(), eq(dependencies.BlackDuckInitialisation.outputs['ProjectId.ProjectExists'], 'true')) + timeoutInMinutes: "10" + pool: + name: Technology-Microsoft-Hosted-Ubuntu + demands: + - ImageOverride -equals ubuntu-latest + - WorkFolder -equals /mnt/storage/sdc/storage + container: mcr.microsoft.com/dotnet/sdk:9.0 + workspace: + clean: all + variables: + - group: BlackDuck on Polaris + - name: BearerToken + value: $[ dependencies.BlackDuckInitialisation.outputs['Authenticate.BearerToken'] ] + - name: BaseUrl + value: $[ dependencies.BlackDuckInitialisation.outputs['Authenticate.BaseUrl'] ] + - name: ProjectId + value: $[ dependencies.BlackDuckInitialisation.outputs['ProjectId.ProjectId'] ] + - name: ProjectExists + value: $[ dependencies.BlackDuckInitialisation.outputs['ProjectId.ProjectExists'] ] + - name: BranchVersion + value: $[ dependencies.BlackDuckInitialisation.outputs['CalculateVersion.BranchVersion'] ] + - name: UbuntuComparisonVersion + value: $[ dependencies.BlackDuckInitialisation.outputs['CalculateVersion.UbuntuComparisonVersion'] ] + steps: + - checkout: none + + - template: steps/blackduck-clone-version.yml@Templates + parameters: + BearerToken: $(BearerToken) + BaseUrl: $(BaseUrl) + ProjectId: $(ProjectId) + SourceVersion: main + TargetVersion: $(BranchVersion) + Phase: DEVELOPMENT + RemoveTargetVersion: false + + - template: steps/blackduck-clone-version.yml@Templates + parameters: + BearerToken: $(BearerToken) + BaseUrl: $(BaseUrl) + ProjectId: $(ProjectId) + SourceVersion: $(BranchVersion) + TargetVersion: $(UbuntuComparisonVersion) + Phase: PLANNING + RemoveTargetVersion: true + + - job: BlackDuckScanUbuntu + displayName: Black Duck Scan Ubuntu + dependsOn: + - BlackDuckInitialisation + - BlackDuckClone + condition: and(succeeded(), eq(dependencies.BlackDuckInitialisation.outputs['ProjectId.ProjectExists'], 'true'), ${{ eq(parameters.EnableUbuntuBlackDuckComparison, 'true') }}) + timeoutInMinutes: "120" + pool: + name: Technology-Microsoft-Hosted-Ubuntu + demands: + - ImageOverride -equals ubuntu-latest + - WorkFolder -equals /mnt/storage/sdc/storage + container: mcr.microsoft.com/dotnet/sdk:9.0 + workspace: + clean: all + variables: + - group: BlackDuck on Polaris + - name: UbuntuComparisonVersion + value: $[ dependencies.BlackDuckInitialisation.outputs['CalculateVersion.UbuntuComparisonVersion'] ] + steps: + - template: steps/retrieve-source.yml@Templates + parameters: + IsIPR: false + ArtifactName: Source + DownloadPath: $(Pipeline.Workspace)/Source + + - pwsh: | + $sourceRoot = "$(Pipeline.Workspace)/Source" + $archivePath = "$(Build.ArtifactStagingDirectory)/blackduck-input-ubuntu.zip" + + if (Test-Path -LiteralPath $archivePath) { + Remove-Item -LiteralPath $archivePath -Force + } + + Compress-Archive -Path $sourceRoot -DestinationPath $archivePath -CompressionLevel Optimal + Write-Host "Created archive at $archivePath" + displayName: Archive sources and build outputs (Ubuntu) + + - pwsh: | + $diagnosticPath = "$(Build.ArtifactStagingDirectory)/blackduck-network-ubuntu.txt" + $targets = @( + 'detect.blackduck.com', + 'aveva.app.blackduck.com' + ) + + $lines = New-Object System.Collections.Generic.List[string] + + foreach ($target in $targets) { + $lines.Add("=== $target ===") + + try { + $addresses = [System.Net.Dns]::GetHostAddresses($target) | ForEach-Object { $_.IPAddressToString } + $lines.Add("DNS: $($addresses -join ', ')") + } + catch { + $lines.Add("DNS ERROR: $($_.Exception.Message)") + } + + try { + $pingOutput = & ping -c 2 $target 2>&1 + $lines.Add('PING:') + foreach ($line in $pingOutput) { + $lines.Add([string]$line) + } + } + catch { + $lines.Add("PING ERROR: $($_.Exception.Message)") + } + + try { + $response = Invoke-WebRequest -Uri ("https://{0}" -f $target) -Method Head -SkipHttpErrorCheck -TimeoutSec 30 + $lines.Add("HTTPS: StatusCode=$($response.StatusCode)") + } + catch { + $lines.Add("HTTPS ERROR: $($_.Exception.Message)") + } + + $lines.Add('') + } + + Set-Content -LiteralPath $diagnosticPath -Value $lines -Encoding utf8 + Get-Content -LiteralPath $diagnosticPath + displayName: Diagnose Black Duck network reachability (Ubuntu) + condition: always() + + - task: Cache@2 + displayName: Restore JDK cache (Ubuntu) + retryCountOnTaskFailure: 3 + inputs: + key: 'jdk | temurin17 | $(Agent.OS) | v1' + path: $(Pipeline.Workspace)/JDK-Ubuntu + cacheHitVar: JDK_UBUNTU_RESTORED + + - pwsh: | + if ($env:JDK_UBUNTU_RESTORED -eq 'true') { + Write-Host 'JDK cache restored.' + exit 0 + } + + $workdir = "$(Pipeline.Workspace)/JDK-Ubuntu" + $archivePath = Join-Path $workdir 'Temurin17.tar.gz' + $url = 'https://github.com/adoptium/temurin17-binaries/releases/download/jdk-17.0.14%2B7/OpenJDK17U-jdk_x64_linux_hotspot_17.0.14_7.tar.gz' + + New-Item -Path $workdir -ItemType Directory -Force | Out-Null + Invoke-WebRequest -Uri $url -OutFile $archivePath + & tar -xzf $archivePath -C $workdir + displayName: Download Temurin JDK 17 (Ubuntu) + condition: ne(variables.JDK_UBUNTU_RESTORED, 'true') + + - pwsh: | + $jdkFolder = Get-ChildItem -Path "$(Pipeline.Workspace)/JDK-Ubuntu" -Directory | + Where-Object { $_.Name -like 'jdk-17*' -or $_.Name -like 'jdk17*' } | + Select-Object -First 1 + + if (-not $jdkFolder) { + throw 'Unable to locate extracted Temurin JDK 17 folder under $(Pipeline.Workspace)/JDK-Ubuntu.' + } + + $javaHome = $jdkFolder.FullName + Write-Host "##vso[task.setvariable variable=JAVA_HOME]$javaHome" + Write-Host "##vso[task.prependpath]$javaHome/bin" + & "$javaHome/bin/java" -version + displayName: Configure JAVA_HOME (Ubuntu) + + - pwsh: | + $detectRoot = "$(Pipeline.Workspace)/detect-ubuntu" + $outputPath = "$(Build.ArtifactStagingDirectory)/blackduck-logs-ubuntu" + $detectScript = Join-Path $detectRoot 'detect10.sh' + $sourcePath = "$(Pipeline.Workspace)/Source" + $binaryScanPath = "$(Build.ArtifactStagingDirectory)/blackduck-input-ubuntu.zip" + + New-Item -Path $detectRoot -ItemType Directory -Force | Out-Null + New-Item -Path $outputPath -ItemType Directory -Force | Out-Null + + Invoke-WebRequest -Uri 'https://detect.blackduck.com/detect10.sh' -OutFile $detectScript + & chmod +x $detectScript + + $arguments = @( + '--blackduck.url=https://aveva.app.blackduck.com' + "--blackduck.api.token=$env:BLACKDUCK_ACCESS_TOKEN" + '--detect.project.name=$(BlackDuckProjectName)' + '--detect.project.version.name=$(UbuntuComparisonVersion)' + "--detect.source.path=$sourcePath" + "--detect.binary.scan.file.path=$binaryScanPath" + '--detect.cleanup=true' + "--detect.output.path=$outputPath" + '--detect.detector.search.depth=10' + '--detect.detector.search.continue=true' + '--detect.blackduck.signature.scanner.upload.source.mode=true' + '--detect.blackduck.signature.scanner.copyright.search=true' + '--detect.blackduck.signature.scanner.license.search=true' + '--logging.level.detect=DEBUG' + '--detect.diagnostic=true' + '--detect.timeout=1800' + '--detect.blackduck.signature.scanner.arguments=--fs-wait-time=90' + '--detect.wait.for.results=true' + '--detect.wait.for.results.timeout=6000000' + ) + + & bash $detectScript @arguments + displayName: Black Duck scan (Ubuntu manual Detect) + retryCountOnTaskFailure: 3 + env: + BLACKDUCK_ACCESS_TOKEN: $(BLACKDUCK_ACCESS_TOKEN) + JAVA_HOME: $(JAVA_HOME) + + - pwsh: | + $artifactStaging = $env:BUILD_ARTIFACTSTAGINGDIRECTORY + $paths = @{ + HasBlackDuckLogsUbuntu = Test-Path -LiteralPath (Join-Path $artifactStaging 'blackduck-logs-ubuntu') + HasBlackDuckInputUbuntu = Test-Path -LiteralPath (Join-Path $artifactStaging 'blackduck-input-ubuntu.zip') + HasBlackDuckNetworkDiagnosticsUbuntu = Test-Path -LiteralPath (Join-Path $artifactStaging 'blackduck-network-ubuntu.txt') + } + + foreach ($entry in $paths.GetEnumerator()) { + $value = if ($entry.Value) { 'true' } else { 'false' } + Write-Host "##vso[task.setvariable variable=$($entry.Key)]$value" + Write-Host "$($entry.Key): $value" + } + displayName: Detect Black Duck artifacts (Ubuntu) + condition: always() + + - publish: $(Build.ArtifactStagingDirectory)/blackduck-logs-ubuntu + artifact: BlackDuckLogsUbuntu + displayName: Publish Black Duck logs (Ubuntu) + condition: and(always(), eq(variables.HasBlackDuckLogsUbuntu, 'true')) + + - publish: $(Build.ArtifactStagingDirectory)/blackduck-input-ubuntu.zip + artifact: BlackDuckInputUbuntu + displayName: Publish Black Duck input archive (Ubuntu) + condition: and(always(), eq(variables.HasBlackDuckInputUbuntu, 'true')) + + - publish: $(Build.ArtifactStagingDirectory)/blackduck-network-ubuntu.txt + artifact: BlackDuckNetworkDiagnosticsUbuntu + displayName: Publish Black Duck network diagnostics (Ubuntu) + condition: and(always(), eq(variables.HasBlackDuckNetworkDiagnosticsUbuntu, 'true')) + + - ${{ if eq(parameters.EnableSecurityValidation, 'true') }}: + - stage: ComponentTemplateBlackDuckComparison + displayName: Component Template Black Duck Comparison + dependsOn: BuildWithShims + condition: succeeded() + jobs: + - template: component-blackduck.yml@Templates + parameters: + BuildArtifact: Build.Release + BlackDuckGroup: DabaconGroup + UbuntuPoolName: Technology-Microsoft-Hosted-Ubuntu + UbuntuPoolDemands: + - ImageOverride -equals ubuntu-latest + - WorkFolder -equals /mnt/storage/sdc/storage + WindowsPoolName: Technology-Microsoft-Hosted-Windows + WindowsPoolDemands: + - ImageOverride -equals windows-latest + IsIPR: false + ProjectName: $(BlackDuckProjectName) + PackageVersion: $(Build.SourceBranchName)-$(Build.BuildId)-components