From d211fe4dd048360a7cf82d46be94d9b6bb8932c0 Mon Sep 17 00:00:00 2001 From: Derek Keeler Date: Tue, 10 Jun 2025 09:43:52 -0700 Subject: [PATCH 1/9] Udpate pipeline to classify as a 'release job' for internal Microsoft compliance. --- .devops/gctoolkit-release.yml | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/.devops/gctoolkit-release.yml b/.devops/gctoolkit-release.yml index 2e4a69f2..b50dbb36 100644 --- a/.devops/gctoolkit-release.yml +++ b/.devops/gctoolkit-release.yml @@ -57,10 +57,15 @@ extends: stages: - stage: Release_GCToolkit jobs: - - job: build_gctoolkit + - job: build_release_gctoolkit workspace: clean: all - displayName: Build with Maven + displayName: Build and Release GCToolkit with Maven + templateContext: + type: releaseJob + isProduction: true + # inputs: // no inputs for this release job, we build the library and release it. + steps: - checkout: gctoolkit path: gctoolkit @@ -222,7 +227,7 @@ extends: folderlocation: '$(Build.ArtifactStagingDirectory)/staging/com/microsoft/gctoolkit' waitforreleasecompletion: true owners: 'dekeeler@microsoft.com' - approvers: 'milderhc@microsoft.com,john.oliver@microsoft.com,dagrieve@microsoft.com,kirk.pepperdine@microsoft.com' + approvers: 'maverbur@microsoft.com,john.oliver@microsoft.com' serviceendpointurl: 'https://api.esrp.microsoft.com' mainpublisher: 'ESRPRELPACMAN' domaintenantid: '33e01921-4d64-4f8c-a055-5bdaffd5e33d' From 2511940bcac8c6b664b345057561b07a80d4fa09 Mon Sep 17 00:00:00 2001 From: Derek Keeler Date: Tue, 10 Jun 2025 09:58:18 -0700 Subject: [PATCH 2/9] Separate the release tasks from the build tasks (requirement of 1ES template). --- .devops/gctoolkit-release.yml | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/.devops/gctoolkit-release.yml b/.devops/gctoolkit-release.yml index b50dbb36..7b3e442d 100644 --- a/.devops/gctoolkit-release.yml +++ b/.devops/gctoolkit-release.yml @@ -57,14 +57,10 @@ extends: stages: - stage: Release_GCToolkit jobs: - - job: build_release_gctoolkit + - job: build_gctoolkit workspace: clean: all - displayName: Build and Release GCToolkit with Maven - templateContext: - type: releaseJob - isProduction: true - # inputs: // no inputs for this release job, we build the library and release it. + displayName: Build GCToolkit with Maven steps: - checkout: gctoolkit @@ -205,11 +201,28 @@ extends: workingDirectory: $(Build.ArtifactStagingDirectory) displayName: Create sha256sums + # Upload the staged artifacts to this pipeline for subsequent release job. - task: 1ES.PublishPipelineArtifact@1 inputs: targetPath: '$(Build.ArtifactStagingDirectory)/staging' artifactName: staged-artifacts + # Release jobs have to be separated from build jobs for our internal release service to be compliant + - job: release_gctoolkit_to_maven_central + dependsOn: build_gctoolkit + workspace: + clean: all + displayName: Release GCToolkit to Maven Central + templateContext: + type: releaseJob + isProduction: true + inputs: # Pull the staged artifacts from the build job. + - input: PublishPipelineArtifact + artifactName: staged-artifacts + targetPath: '$(Build.ArtifactStagingDirectory)/staging' + + steps: + - task: EsrpRelease@8 inputs: connectedservicename: 'JEG-Tooling-Prod' From 77b8084e11b64dda17a40767bd7bb5fea1cb606b Mon Sep 17 00:00:00 2001 From: Derek Keeler Date: Tue, 10 Jun 2025 10:06:03 -0700 Subject: [PATCH 3/9] Added missing template context for the build job. --- .devops/gctoolkit-release.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.devops/gctoolkit-release.yml b/.devops/gctoolkit-release.yml index 7b3e442d..5f46d870 100644 --- a/.devops/gctoolkit-release.yml +++ b/.devops/gctoolkit-release.yml @@ -61,6 +61,11 @@ extends: workspace: clean: all displayName: Build GCToolkit with Maven + templateContext: + outputs: + - output: pipelineArtifact + targetPath: $(Build.ArtifactStagingDirectory)/staging + artifactName: staged-artifacts steps: - checkout: gctoolkit @@ -201,11 +206,6 @@ extends: workingDirectory: $(Build.ArtifactStagingDirectory) displayName: Create sha256sums - # Upload the staged artifacts to this pipeline for subsequent release job. - - task: 1ES.PublishPipelineArtifact@1 - inputs: - targetPath: '$(Build.ArtifactStagingDirectory)/staging' - artifactName: staged-artifacts # Release jobs have to be separated from build jobs for our internal release service to be compliant - job: release_gctoolkit_to_maven_central From e2033a71f7eabea652b1cd1ce94a516898df66b0 Mon Sep 17 00:00:00 2001 From: Derek Keeler Date: Tue, 10 Jun 2025 10:10:00 -0700 Subject: [PATCH 4/9] Fix syntax problem --- .devops/gctoolkit-release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.devops/gctoolkit-release.yml b/.devops/gctoolkit-release.yml index 5f46d870..f22674b6 100644 --- a/.devops/gctoolkit-release.yml +++ b/.devops/gctoolkit-release.yml @@ -217,7 +217,7 @@ extends: type: releaseJob isProduction: true inputs: # Pull the staged artifacts from the build job. - - input: PublishPipelineArtifact + - input: pipelineArtifact artifactName: staged-artifacts targetPath: '$(Build.ArtifactStagingDirectory)/staging' From 5c64eb9834823bf922e415700c72aab9a423eefe Mon Sep 17 00:00:00 2001 From: Derek Keeler Date: Tue, 10 Jun 2025 11:18:19 -0700 Subject: [PATCH 5/9] Upgrade pool used to Azure Linux (3.0) as Mariner 2.0 is being deprecated in July. --- .devops/gctoolkit-release.yml | 2 +- .devops/weekly-build.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.devops/gctoolkit-release.yml b/.devops/gctoolkit-release.yml index f22674b6..e10e4cab 100644 --- a/.devops/gctoolkit-release.yml +++ b/.devops/gctoolkit-release.yml @@ -42,7 +42,7 @@ extends: template: v1/1ES.Official.PipelineTemplate.yml@1esPipelines parameters: pool: - name: JEG-mariner2.0-x64-release + name: JEG-azurelinux-x64-release os: linux sdl: sourceAnalysisPool: diff --git a/.devops/weekly-build.yml b/.devops/weekly-build.yml index cbf2e9a6..11400f7e 100644 --- a/.devops/weekly-build.yml +++ b/.devops/weekly-build.yml @@ -12,7 +12,7 @@ variables: JAVA_HOME_11_X64: /usr/lib/jvm/msopenjdk-11 pool: - name: JEG-mariner2.0-x64-release + name: JEG-azurelinux-x64-release steps: - task: JavaToolInstaller@0 From c7f15ecb830d4313c5bc87af349fccc00fabfc14 Mon Sep 17 00:00:00 2001 From: Derek Keeler Date: Tue, 10 Jun 2025 13:06:47 -0700 Subject: [PATCH 6/9] TEMP: See what tools are hosted... --- .devops/gctoolkit-release.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/.devops/gctoolkit-release.yml b/.devops/gctoolkit-release.yml index e10e4cab..b6aec446 100644 --- a/.devops/gctoolkit-release.yml +++ b/.devops/gctoolkit-release.yml @@ -72,6 +72,18 @@ extends: path: gctoolkit clean: true + # Find out what is installed on the agents... + - bash: | + echo "Installed Tools and versions:" + find /opt/hostedtoolcache -mindepth 1 -maxdepth 2 -type d | while read dir; do + if [[ "$dir" == *jdk* ]]; then + echo "👉 $dir" + else + echo "$dir" + fi + done + displayName: 'Show installed tools' + # Use modern Java to build - task: JavaToolInstaller@0 inputs: From 19e854ab5ea071e8601eb13e2ad364fd45d87cea Mon Sep 17 00:00:00 2001 From: Derek Keeler Date: Tue, 10 Jun 2025 15:49:17 -0700 Subject: [PATCH 7/9] TEMP: Show more depth in tools --- .devops/gctoolkit-release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.devops/gctoolkit-release.yml b/.devops/gctoolkit-release.yml index b6aec446..aeaa2766 100644 --- a/.devops/gctoolkit-release.yml +++ b/.devops/gctoolkit-release.yml @@ -75,7 +75,7 @@ extends: # Find out what is installed on the agents... - bash: | echo "Installed Tools and versions:" - find /opt/hostedtoolcache -mindepth 1 -maxdepth 2 -type d | while read dir; do + find /opt/hostedtoolcache -mindepth 1 -type d | while read dir; do if [[ "$dir" == *jdk* ]]; then echo "👉 $dir" else From 073fab693d851e80dbd36210beb117106f4d9af7 Mon Sep 17 00:00:00 2001 From: Derek Keeler Date: Tue, 17 Jun 2025 10:54:01 -0700 Subject: [PATCH 8/9] Use our test pool to ensure the new Azure Linux machines are set up properly --- .devops/gctoolkit-release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.devops/gctoolkit-release.yml b/.devops/gctoolkit-release.yml index aeaa2766..ac81d2f0 100644 --- a/.devops/gctoolkit-release.yml +++ b/.devops/gctoolkit-release.yml @@ -42,7 +42,7 @@ extends: template: v1/1ES.Official.PipelineTemplate.yml@1esPipelines parameters: pool: - name: JEG-azurelinux-x64-release + name: JEG-test-pool os: linux sdl: sourceAnalysisPool: From 452af2a9dc7e50cac2358a4800e7e4e627088976 Mon Sep 17 00:00:00 2001 From: Derek Keeler Date: Thu, 26 Jun 2025 16:02:31 -0700 Subject: [PATCH 9/9] Corrections for Azure Linux agents - Disable guardian anti-malware scan, not supported on AzLinux yet. - Dry-runs should not fail the build. --- .devops/gctoolkit-release.yml | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/.devops/gctoolkit-release.yml b/.devops/gctoolkit-release.yml index aeaa2766..e3d1a4d0 100644 --- a/.devops/gctoolkit-release.yml +++ b/.devops/gctoolkit-release.yml @@ -62,6 +62,12 @@ extends: clean: all displayName: Build GCToolkit with Maven templateContext: + # Disable Defender for Linux since it is not supported by Azure Linux. + # More info here: https://eng.ms/docs/cloud-ai-platform/devdiv/one-engineering-system-1es/1es-docs/1es-pipeline-templates/features/sdlanalysis/antimalware + sdl: + antimalwareScan: + enabled: false + justificationForDisabling: Disabling Defender for Linux as its not supported in Azure Linux 3 outputs: - output: pipelineArtifact targetPath: $(Build.ArtifactStagingDirectory)/staging @@ -234,7 +240,7 @@ extends: targetPath: '$(Build.ArtifactStagingDirectory)/staging' steps: - + # ESRP Release task docs at aka.ms/esrp under 'ESRP Portal Help' - task: EsrpRelease@8 inputs: connectedservicename: 'JEG-Tooling-Prod' @@ -244,6 +250,8 @@ extends: clientid: '516af6d8-6ab4-4069-8f64-b18c64d16688' intent: 'PackageDistribution' # Test with contentype PyPI to avoid publishing to Maven Central + # NOTE: This is the guidance given in ESRP portal for testing + # the flow during dry-runs, see aka.ms/esrp. ${{ if eq(parameters.release_type, 'release') }}: contenttype: 'Maven' ${{ else }}: @@ -257,3 +265,7 @@ extends: mainpublisher: 'ESRPRELPACMAN' domaintenantid: '33e01921-4d64-4f8c-a055-5bdaffd5e33d' displayName: 'Publish to Maven Central' + # For non-release runs, allow this task to fail (it should!) so the + # pipeline does not appear to fail when it doesn't. + continueOnError: ${{ ne(parameters.release_type, 'release') }} +