diff --git a/.github/workflows/verify_library_generation.yaml b/.github/workflows/verify_library_generation.yaml index cb229bbfba..93b8409d39 100644 --- a/.github/workflows/verify_library_generation.yaml +++ b/.github/workflows/verify_library_generation.yaml @@ -78,6 +78,8 @@ jobs: scandir: 'hermetic_build' format: tty severity: error + ignore_paths: + .kokoro library-generation-lint-python: runs-on: ubuntu-22.04 needs: should-run-library-generation-tests diff --git a/hermetic_build/library_generation/owlbot/templates/java_library/.github/sync-repo-settings.yaml b/hermetic_build/library_generation/owlbot/templates/java_library/.github/sync-repo-settings.yaml index bbfd4c0314..f1c3e6d096 100644 --- a/hermetic_build/library_generation/owlbot/templates/java_library/.github/sync-repo-settings.yaml +++ b/hermetic_build/library_generation/owlbot/templates/java_library/.github/sync-repo-settings.yaml @@ -49,8 +49,9 @@ branchProtectionRules: - "Kokoro - Test: Integration" - "cla/google" - "OwlBot Post Processor" - - "Kokoro - Test: Java GraalVM Native Image" - - "Kokoro - Test: Java 17 GraalVM Native Image" + - "Kokoro - Test: Java GraalVM Native Image A" + - "Kokoro - Test: Java GraalVM Native Image B" + - "Kokoro - Test: Java GraalVM Native Image C" # List of explicit permissions to add (additive only) permissionRules: - team: yoshi-admins diff --git a/hermetic_build/library_generation/owlbot/templates/java_library/.github/workflows/renovate_config_check.yaml b/hermetic_build/library_generation/owlbot/templates/java_library/.github/workflows/renovate_config_check.yaml index 7c5ec7865e..47b9e87c98 100644 --- a/hermetic_build/library_generation/owlbot/templates/java_library/.github/workflows/renovate_config_check.yaml +++ b/hermetic_build/library_generation/owlbot/templates/java_library/.github/workflows/renovate_config_check.yaml @@ -7,7 +7,7 @@ on: jobs: renovate_bot_config_validation: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 steps: - name: Checkout code @@ -16,7 +16,7 @@ jobs: - name: Set up Node.js uses: actions/setup-node@v4 with: - node-version: '20' + node-version: '22' - name: Install Renovate and Config Validator run: | diff --git a/hermetic_build/library_generation/owlbot/templates/java_library/.github/workflows/update_generation_config.yaml b/hermetic_build/library_generation/owlbot/templates/java_library/.github/workflows/update_generation_config.yaml index 3c5051538a..6f8c13f005 100644 --- a/hermetic_build/library_generation/owlbot/templates/java_library/.github/workflows/update_generation_config.yaml +++ b/hermetic_build/library_generation/owlbot/templates/java_library/.github/workflows/update_generation_config.yaml @@ -21,7 +21,7 @@ on: {% raw %} jobs: update-generation-config: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 env: # the branch into which the pull request is merged base_branch: main diff --git a/hermetic_build/library_generation/owlbot/templates/java_library/.kokoro/build.sh b/hermetic_build/library_generation/owlbot/templates/java_library/.kokoro/build.sh index eda70322e3..afcccadd9d 100755 --- a/hermetic_build/library_generation/owlbot/templates/java_library/.kokoro/build.sh +++ b/hermetic_build/library_generation/owlbot/templates/java_library/.kokoro/build.sh @@ -1,6 +1,5 @@ #!/bin/bash - -# Copyright 2018 Google LLC +# Copyright 2025 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -16,13 +15,125 @@ set -eo pipefail -cd github/synthtool +## Get the directory of the build script +scriptDir=$(realpath $(dirname "${BASH_SOURCE[0]}")) +## cd to the parent directory, i.e. the root of the git repo +cd ${scriptDir}/.. + +# include common functions +source ${scriptDir}/common.sh + +# Print out Maven & Java version +mvn -version +echo ${JOB_TYPE} + +# attempt to install 3 times with exponential backoff (starting with 10 seconds) +retry_with_backoff 3 10 \ + mvn install -B -V -ntp \ + -DskipTests=true \ + -Dclirr.skip=true \ + -Denforcer.skip=true \ + -Dmaven.javadoc.skip=true \ + -Dgcloud.download.skip=true \ + -T 1C + +# if GOOGLE_APPLICATION_CREDENTIALS is specified as a relative path, prepend Kokoro root directory onto it +if [[ ! -z "${GOOGLE_APPLICATION_CREDENTIALS}" && "${GOOGLE_APPLICATION_CREDENTIALS}" != /* ]]; then + export GOOGLE_APPLICATION_CREDENTIALS=$(realpath ${KOKORO_GFILE_DIR}/${GOOGLE_APPLICATION_CREDENTIALS}) +fi +{% if metadata['repo']['api_shortname'] == 'storage' %} + +export TEST_UNIVERSE_DOMAIN_CREDENTIAL=$(realpath ${KOKORO_GFILE_DIR}/secret_manager/client-library-test-universe-domain-credential) +export TEST_UNIVERSE_DOMAIN=$(gcloud secrets versions access latest --project cloud-devrel-kokoro-resources --secret=client-library-test-universe-domain) +export TEST_UNIVERSE_PROJECT_ID=$(gcloud secrets versions access latest --project cloud-devrel-kokoro-resources --secret=client-library-test-universe-project-id) +export TEST_UNIVERSE_LOCATION=$(gcloud secrets versions access latest --project cloud-devrel-kokoro-resources --secret=client-library-test-universe-storage-location) +{% endif %} + +RETURN_CODE=0 +set +e + +case ${JOB_TYPE} in +test) + echo "SUREFIRE_JVM_OPT: ${SUREFIRE_JVM_OPT}" + mvn test -B -ntp -Dclirr.skip=true -Denforcer.skip=true ${SUREFIRE_JVM_OPT} + RETURN_CODE=$? + ;; +lint) + mvn com.coveo:fmt-maven-plugin:check -B -ntp + RETURN_CODE=$? + ;; +javadoc) + mvn javadoc:javadoc javadoc:test-javadoc -B -ntp + RETURN_CODE=$? + ;; +integration) + mvn -B ${INTEGRATION_TEST_ARGS} \ + -ntp \ + -Penable-integration-tests \ + -DtrimStackTrace=false \ + -Dclirr.skip=true \ + -Denforcer.skip=true \ + -Dcheckstyle.skip=true \ + -DskipUnitTests=true \ + -fae \ + verify + RETURN_CODE=$? + ;; +graalvm) + # Run Unit and Integration Tests with Native Image + mvn -B ${INTEGRATION_TEST_ARGS} -ntp -Pnative test + RETURN_CODE=$? + ;; +samples) + SAMPLES_DIR=samples + # only run ITs in snapshot/ on presubmit PRs. run ITs in all 3 samples/ subdirectories otherwise. + if [[ ! -z ${KOKORO_GITHUB_PULL_REQUEST_NUMBER} ]] + then + SAMPLES_DIR=samples/snapshot + fi + + if [[ -f ${SAMPLES_DIR}/pom.xml ]] + then + for FILE in ${KOKORO_GFILE_DIR}/secret_manager/*-samples-secrets; do + [[ -f "$FILE" ]] || continue + source "$FILE" + done + + pushd ${SAMPLES_DIR} + mvn -B \ + -ntp \ + -DtrimStackTrace=false \ + -Dclirr.skip=true \ + -Denforcer.skip=true \ + -fae \ + verify + RETURN_CODE=$? + popd + else + echo "no sample pom.xml found - skipping sample tests" + fi + ;; +clirr) + mvn -B -ntp -Denforcer.skip=true clirr:check + RETURN_CODE=$? + ;; +*) + ;; +esac + +if [ "${REPORT_COVERAGE}" == "true" ] +then + bash ${KOKORO_GFILE_DIR}/codecov.sh +fi -# Disable buffering, so that the logs stream through. -export PYTHONUNBUFFERED=1 +# fix output location of logs +bash .kokoro/coerce_logs.sh -# Run tests -nox -s lint test +if [[ "${ENABLE_FLAKYBOT}" == "true" ]] +then + chmod +x ${KOKORO_GFILE_DIR}/linux_amd64/flakybot + ${KOKORO_GFILE_DIR}/linux_amd64/flakybot -repo=googleapis/java-{{ metadata['repo']['api_shortname'] }} +fi -# remove all files, preventing kokoro from trying to sync them. -rm -rf * +echo "exiting with ${RETURN_CODE}" +exit ${RETURN_CODE} \ No newline at end of file diff --git a/hermetic_build/library_generation/owlbot/templates/java_library/.kokoro/presubmit/graalvm-a.cfg b/hermetic_build/library_generation/owlbot/templates/java_library/.kokoro/presubmit/graalvm-a.cfg new file mode 100644 index 0000000000..7846dd80f9 --- /dev/null +++ b/hermetic_build/library_generation/owlbot/templates/java_library/.kokoro/presubmit/graalvm-a.cfg @@ -0,0 +1,38 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Configure the docker image for kokoro-trampoline. +env_vars: { + key: "TRAMPOLINE_IMAGE" + value: "gcr.io/cloud-devrel-public-resources/graalvm_sdk_platform_b:3.45.1" +} + +env_vars: { + key: "JOB_TYPE" + value: "graalvm" +} + +# TODO: remove this after we've migrated all tests and scripts +env_vars: { + key: "GCLOUD_PROJECT" + value: "gcloud-devel" +} + +env_vars: { + key: "GOOGLE_CLOUD_PROJECT" + value: "gcloud-devel" +} + +env_vars: { + key: "GOOGLE_APPLICATION_CREDENTIALS" + value: "secret_manager/java-it-service-account" +} + +env_vars: { + key: "SECRET_MANAGER_KEYS" + value: "java-it-service-account" +} + +env_vars: { + key: "IT_SERVICE_ACCOUNT_EMAIL" + value: "it-service-account@gcloud-devel.iam.gserviceaccount.com" +} \ No newline at end of file diff --git a/hermetic_build/library_generation/owlbot/templates/java_library/.kokoro/presubmit/graalvm-b.cfg b/hermetic_build/library_generation/owlbot/templates/java_library/.kokoro/presubmit/graalvm-b.cfg new file mode 100644 index 0000000000..7846dd80f9 --- /dev/null +++ b/hermetic_build/library_generation/owlbot/templates/java_library/.kokoro/presubmit/graalvm-b.cfg @@ -0,0 +1,38 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Configure the docker image for kokoro-trampoline. +env_vars: { + key: "TRAMPOLINE_IMAGE" + value: "gcr.io/cloud-devrel-public-resources/graalvm_sdk_platform_b:3.45.1" +} + +env_vars: { + key: "JOB_TYPE" + value: "graalvm" +} + +# TODO: remove this after we've migrated all tests and scripts +env_vars: { + key: "GCLOUD_PROJECT" + value: "gcloud-devel" +} + +env_vars: { + key: "GOOGLE_CLOUD_PROJECT" + value: "gcloud-devel" +} + +env_vars: { + key: "GOOGLE_APPLICATION_CREDENTIALS" + value: "secret_manager/java-it-service-account" +} + +env_vars: { + key: "SECRET_MANAGER_KEYS" + value: "java-it-service-account" +} + +env_vars: { + key: "IT_SERVICE_ACCOUNT_EMAIL" + value: "it-service-account@gcloud-devel.iam.gserviceaccount.com" +} \ No newline at end of file diff --git a/hermetic_build/library_generation/owlbot/templates/java_library/.kokoro/presubmit/graalvm-c.cfg b/hermetic_build/library_generation/owlbot/templates/java_library/.kokoro/presubmit/graalvm-c.cfg new file mode 100644 index 0000000000..7846dd80f9 --- /dev/null +++ b/hermetic_build/library_generation/owlbot/templates/java_library/.kokoro/presubmit/graalvm-c.cfg @@ -0,0 +1,38 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Configure the docker image for kokoro-trampoline. +env_vars: { + key: "TRAMPOLINE_IMAGE" + value: "gcr.io/cloud-devrel-public-resources/graalvm_sdk_platform_b:3.45.1" +} + +env_vars: { + key: "JOB_TYPE" + value: "graalvm" +} + +# TODO: remove this after we've migrated all tests and scripts +env_vars: { + key: "GCLOUD_PROJECT" + value: "gcloud-devel" +} + +env_vars: { + key: "GOOGLE_CLOUD_PROJECT" + value: "gcloud-devel" +} + +env_vars: { + key: "GOOGLE_APPLICATION_CREDENTIALS" + value: "secret_manager/java-it-service-account" +} + +env_vars: { + key: "SECRET_MANAGER_KEYS" + value: "java-it-service-account" +} + +env_vars: { + key: "IT_SERVICE_ACCOUNT_EMAIL" + value: "it-service-account@gcloud-devel.iam.gserviceaccount.com" +} \ No newline at end of file diff --git a/hermetic_build/library_generation/setup.py b/hermetic_build/library_generation/setup.py index b58b54e8ab..cce0a27d8b 100755 --- a/hermetic_build/library_generation/setup.py +++ b/hermetic_build/library_generation/setup.py @@ -25,10 +25,9 @@ "owlbot/templates/clirr/*.j2", "owlbot/templates/poms/*.j2", "owlbot/templates/java_library/.github/**/*", - # TODO: uncomment this line after https://github.com/googleapis/sdk-platform-java/pull/3723 - # has been merged. - # "owlbot/templates/java_library/.kokoro/**/*", "owlbot/templates/java_library/**/*", + "owlbot/templates/java_library/.kokoro/build.sh", + "owlbot/templates/java_library/.kokoro/presubmit/graalvm-*.cfg", ], "synthtool": ["owlbot/synthtool/**/*"], },