diff --git a/.buildkite/advisory.yml b/.buildkite/advisory.yml deleted file mode 100644 index 2fb1e591b..000000000 --- a/.buildkite/advisory.yml +++ /dev/null @@ -1,79 +0,0 @@ -# Uploaded only by the advisory-upload step when -# BUILDKITE_BRANCH=main and BUILDKITE_SOURCE=schedule. -# Do not add this file to the default pipeline.yml: a step with -# `if:` still appears as broken on push and PR builds. -# A separately uploaded pipeline does not inherit agents/env from -# pipeline.yml. Repeat those blocks here. - -agents: - queue: "${MAESTRO_CI_QUEUE:-linux-large}" - image: "${MAESTRO_CI_IMAGE:-evalops-platform-ci-v6}" - -env: - CI: "true" - CARGO_INCREMENTAL: "0" - CARGO_PROFILE_DEV_DEBUG: "0" - CARGO_BUILD_JOBS: "4" - CARGO_PROFILE_DEV_CODEGEN_UNITS: "16" - CARGO_PROFILE_TEST_CODEGEN_UNITS: "16" - CARGO_TARGET_DIR: ".buildkite/cache/cargo-target" - npm_config_cache: ".buildkite/cache/npm" - npm_config_fetch_retries: "1" - npm_config_fetch_timeout: "30000" - -steps: - - label: ":orb: Hosted Orb delegation acceptance (required when gated)" - key: "hosted-orb-delegation-live" - priority: 10 - timeout_in_minutes: 30 - agents: - queue: "${MAESTRO_CI_PROTOCOL_QUEUE:-linux-medium}" - image: "${MAESTRO_CI_PROTOCOL_IMAGE:-evalops-platform-ci-v6}" - command: | - set -euo pipefail - timeout --signal=TERM --kill-after=10s 5m npm ci --ignore-scripts - if [[ "${MAESTRO_HOSTED_ORB_LIVE_SMOKE:-}" != "1" ]]; then - echo "Optional hosted Orb acceptance was not requested; skipping." - exit 0 - fi - npm run smoke:hosted-orb-delegation - - - label: ":bar_chart: Coverage (advisory)" - key: "coverage" - priority: 10 - soft_fail: true - timeout_in_minutes: 90 - concurrency: 1 - concurrency_group: "${MAESTRO_CI_ADVISORY_CONCURRENCY_GROUP:-maestro-advisory-workloads}" - retry: - automatic: - - exit_status: -1 - signal_reason: none - limit: 1 - - signal_reason: agent_stop - limit: 1 - cache: - name: "maestro-coverage-main" - size: "50g" - paths: [".buildkite/cache"] - command: scripts/run-buildkite-coverage.sh - - - label: ":stopwatch: Performance baseline (advisory)" - key: "perf-baseline" - priority: 10 - soft_fail: true - timeout_in_minutes: 60 - concurrency: 1 - concurrency_group: "${MAESTRO_CI_ADVISORY_CONCURRENCY_GROUP:-maestro-advisory-workloads}" - retry: - automatic: - - exit_status: -1 - signal_reason: none - limit: 1 - - signal_reason: agent_stop - limit: 1 - cache: - name: "maestro-perf-main" - size: "30g" - paths: [".buildkite/cache"] - command: scripts/run-buildkite-perf.sh diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml deleted file mode 100644 index f89968b41..000000000 --- a/.buildkite/pipeline.yml +++ /dev/null @@ -1,370 +0,0 @@ -# The former Hetzner fleet is retired; use the established hosted Rust pool. -agents: - queue: "${MAESTRO_CI_QUEUE:-linux-large}" - image: "${MAESTRO_CI_IMAGE:-evalops-platform-ci-v6}" - -env: - CI: "true" - CARGO_INCREMENTAL: "0" - CARGO_PROFILE_DEV_DEBUG: "0" - # [profile.dev] defaults to 128 codegen units. Cap both crate - # parallelism and per-crate CGUs so one rustc cannot exhaust the - # self-hosted worker. - CARGO_BUILD_JOBS: "4" - CARGO_PROFILE_DEV_CODEGEN_UNITS: "16" - CARGO_PROFILE_TEST_CODEGEN_UNITS: "16" - CARGO_TARGET_DIR: ".buildkite/cache/cargo-target" - npm_config_cache: ".buildkite/cache/npm" - npm_config_fetch_retries: "1" - npm_config_fetch_timeout: "30000" - -steps: - # Node-only lock. Heavy jobs depend on this so a stale - # protocol-compatibility-manifest.json finishes the build as failed - # in seconds. Combined GitHub status buildkite/maestro-internal-ci - # stays pending while rust-tests still runs; #3497 merged in that - # window because lint had already failed and the rest of the - # pipeline had not. - - label: ":lock: Protocol compatibility lock" - key: "protocol-contracts" - priority: 80 - timeout_in_minutes: 10 - # Node-only. Hosted linux-medium, separate from the linux-large rust-tests - # workers. Those workers were full on #3499/#3501 and left this - # lock scheduled while GitHub stayed green. - agents: - queue: "${MAESTRO_CI_PROTOCOL_QUEUE:-linux-medium}" - image: "${MAESTRO_CI_PROTOCOL_IMAGE:-evalops-platform-ci-v6}" - retry: - automatic: - - exit_status: -1 - signal_reason: none - limit: 1 - - signal_reason: agent_stop - limit: 1 - command: | - set -euo pipefail - timeout --signal=TERM --kill-after=10s 5m npm ci --ignore-scripts - npm run check:workspace-contract - npm run check:protocol-manifest - npm run check:runtime-passport - npm run check:rust-only-runtime - npm run check:helm-probes - npm run check:hook-dispatch - npm run check:session-transfer - npm run check:hosted-orb-delegation - npm run check:macos-signature - - - group: ":rust: Native validation" - depends_on: "protocol-contracts" - if: build.env("MAESTRO_CI_SUPPLY_CHAIN_ONLY") != "true" - steps: - - label: ":lint-roller: Contracts, format, and Clippy" - key: "lint" - priority: 50 - timeout_in_minutes: 90 - concurrency: 3 - concurrency_group: "${MAESTRO_CI_CONCURRENCY_GROUP:-maestro-heavy-workloads-v6}" - retry: - automatic: - - exit_status: -1 - signal_reason: none - limit: 1 - - exit_status: 137 - limit: 1 - - signal_reason: agent_stop - limit: 1 - cache: - name: "maestro-lint-${BUILDKITE_PULL_REQUEST}" - size: "50g" - paths: [".buildkite/cache"] - command: | - .buildkite/run-with-mise "rust@1.95.0" bash -s <<'MAESTRO_CI' - set -euo pipefail - timeout --signal=TERM --kill-after=10s 5m npm ci --ignore-scripts - timeout --signal=TERM --kill-after=30s 45m npm run check - node --test scripts/check-doc-paths.test.mjs scripts/version.test.mjs - if [[ -f scripts/measure-ci-build-latency.test.mjs ]]; then - node --test scripts/measure-ci-build-latency.test.mjs - fi - if [[ -d test/internal ]]; then - npm run test:internal - fi - timeout --signal=TERM --kill-after=30s 30m npm run lint - node --test scripts/check-advisory-expiry.test.mjs - node scripts/check-advisory-expiry.mjs - MAESTRO_CI - - - label: ":test_tube: Full Rust workspace" - key: "rust-tests" - priority: 50 - timeout_in_minutes: 90 - concurrency: 3 - concurrency_group: "${MAESTRO_CI_CONCURRENCY_GROUP:-maestro-heavy-workloads-v6}" - retry: - automatic: - - exit_status: -1 - signal_reason: none - limit: 1 - - exit_status: 137 - limit: 1 - - signal_reason: agent_stop - limit: 1 - agents: - queue: "${MAESTRO_CI_QUEUE:-linux-large}" - image: "${MAESTRO_CI_IMAGE:-evalops-platform-ci-v6}" - env: - CARGO_BUILD_JOBS: "4" - cache: - name: "maestro-tests-${BUILDKITE_PULL_REQUEST}" - size: "50g" - paths: [".buildkite/cache"] - command: | - .buildkite/run-with-mise "rust@1.95.0" bash -s <<'MAESTRO_CI' - set -euo pipefail - export MAESTRO_TRUSTED_RUNNER_WORKSPACE_ROOTS="$$(pwd -P)" - maestro_test_home="$$(mktemp -d)" - export MAESTRO_HOME="$$maestro_test_home" - export MAESTRO_SUBAGENTS_DIR="$$maestro_test_home/subagents" - nextest_version="0.9.143" - nextest_archive="$$(mktemp)" - nextest_dir="$$(mktemp -d)" - trap 'rm -rf "$$maestro_test_home"; rm -f "$$nextest_archive"; rm -rf "$$nextest_dir"' EXIT - timeout --signal=TERM --kill-after=10s 2m curl --fail --location --silent --show-error \ - "https://github.com/nextest-rs/nextest/releases/download/cargo-nextest-$$nextest_version/cargo-nextest-$$nextest_version-x86_64-unknown-linux-gnu.tar.gz" \ - --output "$$nextest_archive" - printf '%s %s\n' \ - 66786b9abe23920d022a182d1416b1bbc8130dd4872a9553d76985a1708dcd1e \ - "$$nextest_archive" | sha256sum --check --status - tar -C "$$nextest_dir" -xzf "$$nextest_archive" - export PATH="$$nextest_dir:$$PATH" - # This focused contract exercises the offline denial path and - # child-environment stripping before the full suite. - timeout --signal=TERM --kill-after=30s 10m cargo test --locked -p maestro-local-host ci_auth_conformance - timeout --signal=TERM --kill-after=30s 45m cargo nextest run --profile buildkite --workspace --locked --no-fail-fast - timeout --signal=TERM --kill-after=30s 20m cargo test --workspace --locked --doc - export MAESTRO_TUI_BIN="$$CARGO_TARGET_DIR/debug/maestro-tui" - test -x "$$MAESTRO_TUI_BIN" || timeout --signal=TERM --kill-after=30s 30m cargo build --locked -p maestro-tui - npm run smoke:tui - MAESTRO_CI - - - label: ":package: Native build and conformance" - key: "native-release" - priority: 50 - timeout_in_minutes: 90 - concurrency: 3 - concurrency_group: "${MAESTRO_CI_CONCURRENCY_GROUP:-maestro-heavy-workloads-v6}" - retry: - automatic: - - exit_status: -1 - signal_reason: none - limit: 1 - - signal_reason: agent_stop - limit: 1 - agents: - queue: "${MAESTRO_CI_QUEUE:-linux-large}" - image: "${MAESTRO_CI_IMAGE:-evalops-platform-ci-v6}" - cache: - name: "maestro-release-${BUILDKITE_PULL_REQUEST}" - size: "50g" - paths: [".buildkite/cache"] - command: | - .buildkite/run-with-mise "rust@1.95.0" bash -s <<'MAESTRO_CI' - set -euo pipefail - timeout --signal=TERM --kill-after=10s 5m npm ci --ignore-scripts - timeout --signal=TERM --kill-after=30s 45m npm run build - npm run smoke:release-native-only - node scripts/run-runtime-conformance.mjs --binary "$$CARGO_TARGET_DIR/fast-validation/maestro" - MAESTRO_CI - - - group: ":repeat: Integration and replay" - depends_on: "protocol-contracts" - if: build.env("MAESTRO_CI_SUPPLY_CHAIN_ONLY") != "true" - steps: - - label: ":docker: Control-plane integration" - key: "integration" - priority: 50 - timeout_in_minutes: 60 - concurrency: 3 - concurrency_group: "${MAESTRO_CI_CONCURRENCY_GROUP:-maestro-heavy-workloads-v6}" - retry: - automatic: - - exit_status: -1 - signal_reason: none - limit: 1 - - signal_reason: agent_stop - limit: 1 - agents: - queue: "${MAESTRO_CI_INTEGRATION_QUEUE:-linux-medium}" - image: "${MAESTRO_CI_INTEGRATION_IMAGE:-evalops-platform-ci-v6}" - cache: - name: "maestro-integration-${BUILDKITE_PULL_REQUEST}" - size: "50g" - paths: [".buildkite/cache"] - command: | - .buildkite/run-with-mise "rust@1.95.0" bash -s <<'MAESTRO_CI' - set -euo pipefail - suffix="$${BUILDKITE_JOB_ID//-/}" - redis="maestro-redis-$$suffix" - postgres="maestro-postgres-$$suffix" - trap 'docker rm -f "$$redis" "$$postgres" >/dev/null 2>&1 || true' EXIT - redis_image="mirror.gcr.io/library/redis:7-alpine" - postgres_image="mirror.gcr.io/library/postgres:16-alpine" - timeout --signal=TERM --kill-after=10s 2m docker pull "$$redis_image" - timeout --signal=TERM --kill-after=10s 2m docker pull "$$postgres_image" - docker run -d --rm --name "$$redis" -p 127.0.0.1::6379 "$$redis_image" - docker run -d --rm --name "$$postgres" -e POSTGRES_USER=maestro -e POSTGRES_PASSWORD=maestro -e POSTGRES_DB=maestro -p 127.0.0.1::5432 "$$postgres_image" - redis_port="$$(docker port "$$redis" 6379/tcp | sed 's/.*://')" - postgres_port="$$(docker port "$$postgres" 5432/tcp | sed 's/.*://')" - for _ in $$(seq 1 60); do docker exec "$$postgres" pg_isready -U maestro >/dev/null 2>&1 && break; sleep 2; done - MAESTRO_REDIS_URL="redis://127.0.0.1:$$redis_port" \ - MAESTRO_DATABASE_URL="postgresql://maestro:maestro@127.0.0.1:$$postgres_port/maestro" \ - PGPASSWORD=maestro timeout --signal=TERM --kill-after=30s 30m cargo test --locked -p maestro-runtime-gateway - POSTGRES_DB=maestro \ - POSTGRES_HOST=127.0.0.1 \ - POSTGRES_PASSWORD=maestro \ - POSTGRES_PORT="$$postgres_port" \ - POSTGRES_USER=maestro \ - PGPASSWORD=maestro \ - timeout --signal=TERM --kill-after=30s 30m cargo test --locked -p maestro-tui --test tools_integration - MAESTRO_CI - - - label: ":repeat: Deterministic scenario replay" - key: "scenario-replay" - priority: 50 - timeout_in_minutes: 45 - concurrency: 3 - concurrency_group: "${MAESTRO_CI_CONCURRENCY_GROUP:-maestro-heavy-workloads-v6}" - retry: - automatic: - - exit_status: -1 - signal_reason: none - limit: 1 - - signal_reason: agent_stop - limit: 1 - cache: - name: "maestro-replay-${BUILDKITE_PULL_REQUEST}" - size: "30g" - paths: [".buildkite/cache"] - command: | - .buildkite/run-with-mise "rust@1.95.0" bash -s <<'MAESTRO_CI' - set -euo pipefail - timeout --signal=TERM --kill-after=10s 5m npm ci --ignore-scripts - timeout --signal=TERM --kill-after=30s 30m cargo build --locked -p maestro-scenario - node --test scripts/scenario-replay-governance.test.mjs - MAESTRO_BIN="$$CARGO_TARGET_DIR/debug/maestro-scenario" npm run check:scenario-replay-gate -- --junit-dir tmp/scenario-replay - MAESTRO_CI - - - label: ":pipeline: CI configuration contracts" - key: "ci-contracts" - depends_on: "protocol-contracts" - priority: 50 - if: build.env("MAESTRO_CI_SUPPLY_CHAIN_ONLY") != "true" - timeout_in_minutes: 30 - concurrency: 3 - concurrency_group: "${MAESTRO_CI_CONCURRENCY_GROUP:-maestro-heavy-workloads-v6}" - retry: - automatic: - - exit_status: -1 - signal_reason: none - limit: 1 - - signal_reason: agent_stop - limit: 1 - cache: - name: "maestro-ci-contracts-${BUILDKITE_PULL_REQUEST}" - size: "20g" - paths: [".buildkite/cache"] - command: | - set -euo pipefail - timeout --signal=TERM --kill-after=10s 5m npm ci --ignore-scripts - test ! -d src/hooks - node --test scripts/check-buildkite-pipeline.test.mjs - node scripts/check-rust-only-runtime.mjs - node scripts/check-hook-dispatch-coverage.mjs - if [[ -f scripts/workflow-contracts.test.mjs ]]; then - node --test scripts/workflow-contracts.test.mjs - fi - - - group: ":shield: Auxiliary validation" - depends_on: "protocol-contracts" - steps: - - label: ":github: Workflow and shell tooling" - key: "workflow-tooling" - priority: 50 - if: build.env("MAESTRO_CI_SUPPLY_CHAIN_ONLY") != "true" - timeout_in_minutes: 30 - concurrency: 3 - concurrency_group: "${MAESTRO_CI_CONCURRENCY_GROUP:-maestro-heavy-workloads-v6}" - retry: - automatic: - - exit_status: -1 - signal_reason: none - limit: 1 - - signal_reason: agent_stop - limit: 1 - cache: - name: "maestro-ci-tools-${BUILDKITE_PULL_REQUEST}" - size: "20g" - paths: [".buildkite/cache"] - command: scripts/run-buildkite-ci-tooling.sh - - - label: ":lock: Supply chain" - key: "supply-chain" - priority: 50 - agents: - queue: "${MAESTRO_CI_SUPPLY_CHAIN_QUEUE:-linux-medium}" - image: "${MAESTRO_CI_SUPPLY_CHAIN_IMAGE:-evalops-platform-ci-v6}" - timeout_in_minutes: 45 - concurrency: 3 - concurrency_group: "${MAESTRO_CI_CONCURRENCY_GROUP:-maestro-heavy-workloads-v6}" - retry: - automatic: - - exit_status: -1 - signal_reason: none - limit: 1 - - signal_reason: agent_stop - limit: 1 - cache: - name: "maestro-supply-chain-${BUILDKITE_PULL_REQUEST}" - size: "20g" - paths: [".buildkite/cache"] - command: .buildkite/run-with-mise "rust@1.95.0 gh@2.88.1" scripts/run-buildkite-supply-chain.sh - - - label: ":idea: JetBrains plugin" - key: "jetbrains-plugin" - priority: 50 - agents: - queue: "${MAESTRO_CI_JETBRAINS_QUEUE:-linux-medium}" - image: "${MAESTRO_CI_JETBRAINS_IMAGE:-evalops-platform-ci-v6}" - if: build.env("MAESTRO_CI_SUPPLY_CHAIN_ONLY") != "true" - timeout_in_minutes: 45 - concurrency: 3 - concurrency_group: "${MAESTRO_CI_JETBRAINS_CONCURRENCY_GROUP:-maestro-jetbrains-heavy-workloads}" - retry: - automatic: - - exit_status: -1 - signal_reason: none - limit: 1 - - exit_status: 137 - limit: 1 - - signal_reason: agent_stop - limit: 1 - cache: - name: "maestro-jetbrains-${BUILDKITE_PULL_REQUEST}" - size: "20g" - paths: [".buildkite/cache"] - command: scripts/run-buildkite-jetbrains.sh - - # Do not list coverage/perf here. A step with `if:` is still created - # and shows as broken on push and PR. Upload them only on the daily - # 05:17 UTC schedule (BUILDKITE_SOURCE=schedule). - - label: ":pipeline: Upload scheduled advisory steps" - key: "advisory-upload" - priority: 60 - timeout_in_minutes: 5 - soft_fail: true - command: | - set -euo pipefail - if [[ "$${BUILDKITE_BRANCH}" == "main" && "$${BUILDKITE_SOURCE}" == "schedule" && -f .buildkite/advisory.yml ]]; then - buildkite-agent pipeline upload .buildkite/advisory.yml - fi diff --git a/.buildkite/run-with-mise b/.buildkite/run-with-mise deleted file mode 100755 index 419e96c9f..000000000 --- a/.buildkite/run-with-mise +++ /dev/null @@ -1,75 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -if [[ $# -lt 2 ]]; then - echo "usage: $0 '' command [args...]" >&2 - exit 2 -fi - -tool_list="$1" -shift - -repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd -P)" -cd "${repo_root}" - -mise_version="2026.4.24" -mise_sha256="4ecf49b825741e1e3e8ff8c92ee242cf728da951d1b3592ef1c2f080201fa454" -cache_root="${BUILDKITE_BUILD_CHECKOUT_PATH:-${repo_root}}/.buildkite/cache" -runtime_root="${cache_root}/runtime" -mkdir -p "${runtime_root}/bin" "${runtime_root}/mise/data" "${runtime_root}/mise/cache" \ - "${runtime_root}/cargo/bin" "${runtime_root}/rustup" "${cache_root}/tmp" - -export MISE_DATA_DIR="${MISE_DATA_DIR:-${runtime_root}/mise/data}" -export MISE_CACHE_DIR="${MISE_CACHE_DIR:-${runtime_root}/mise/cache}" -export CARGO_HOME="${CARGO_HOME:-${runtime_root}/cargo}" -export RUSTUP_HOME="${RUSTUP_HOME:-${runtime_root}/rustup}" -export RUNNER_TEMP="${RUNNER_TEMP:-${cache_root}/tmp}" -export CARGO_TARGET_DIR="${CARGO_TARGET_DIR:-${cache_root}/cargo-target}" -export npm_config_cache="${npm_config_cache:-${cache_root}/npm}" -export RUSTUP_INIT_SKIP_PATH_CHECK=yes -export MISE_EXEC_AUTO_INSTALL=false -export PATH="${runtime_root}/bin:${CARGO_HOME}/bin:${PATH}" - -mise_reports_pinned_version() { - [[ -x "$1" ]] && "$1" --version 2>/dev/null | grep -Eq "(^|[[:space:]])${mise_version}([[:space:]]|$)" -} - -mise_bin="${EVALOPS_MISE_BIN:-}" -if [[ -n "${mise_bin}" ]] && ! mise_reports_pinned_version "${mise_bin}"; then - echo "Ignoring EVALOPS_MISE_BIN=${mise_bin}: expected mise ${mise_version}." >&2 - mise_bin="" -fi -if [[ -z "${mise_bin}" ]] && command -v mise >/dev/null 2>&1; then - discovered_mise="$(command -v mise)" - if mise_reports_pinned_version "${discovered_mise}"; then - mise_bin="${discovered_mise}" - fi -fi -if [[ -z "${mise_bin}" ]]; then - mise_bin="${runtime_root}/bin/mise-${mise_version}" -fi - -if [[ ! -x "${mise_bin}" ]]; then - mise_download="${cache_root}/mise-v${mise_version}-linux-x64" - if ! curl -fsSLo "${mise_download}" --retry 5 --retry-all-errors --retry-delay 3 \ - --connect-timeout 15 \ - "https://github.com/jdx/mise/releases/download/v${mise_version}/mise-v${mise_version}-linux-x64"; then - curl -fsSLo "${mise_download}" --retry 5 --retry-all-errors --retry-delay 3 \ - --connect-timeout 15 \ - "https://mise.jdx.dev/v${mise_version}/mise-v${mise_version}-linux-x64" - fi - printf '%s %s\n' "${mise_sha256}" "${mise_download}" | sha256sum --check --status - install -m 0755 "${mise_download}" "${mise_bin}" -fi - -read -r -a tools <<< "${tool_list}" -for tool in "${tools[@]}"; do - "${mise_bin}" install "${tool}" -done - -if [[ " ${tool_list} " == *" rust@"* ]]; then - "${mise_bin}" exec "${tools[@]}" -- rustup component add rustfmt clippy -fi - -export EVALOPS_MISE_BIN="${mise_bin}" -"${mise_bin}" exec "${tools[@]}" -- "$@" diff --git a/.config/nextest.toml b/.config/nextest.toml index 61c6f0340..30fcf9e29 100644 --- a/.config/nextest.toml +++ b/.config/nextest.toml @@ -1,7 +1,7 @@ -[profile.buildkite] +[profile.ci] slow-timeout = { period = "5m", terminate-after = 2 } -[[profile.buildkite.overrides]] +[[profile.ci.overrides]] filter = 'binary(pty_e2e)' test-group = 'pty-e2e' diff --git a/.github/actionlint.yaml b/.github/actionlint.yaml index c1a058cd8..3600dd7d3 100644 --- a/.github/actionlint.yaml +++ b/.github/actionlint.yaml @@ -1,4 +1,6 @@ self-hosted-runner: labels: + - evalops-internal-arc + - evalops-internal-arc-light - evalops-maestro-internal-rbe - evalops-maestro-rbe diff --git a/.github/workflows/check-release-workflow-contract.test.mjs b/.github/workflows/check-release-workflow-contract.test.mjs index 2f9c8f15f..64950f3d5 100644 --- a/.github/workflows/check-release-workflow-contract.test.mjs +++ b/.github/workflows/check-release-workflow-contract.test.mjs @@ -1078,17 +1078,17 @@ test("current release workflow satisfies the parsed contract", async () => { assert.deepEqual(await checkReleaseWorkflow(), []); }); -test("required Buildkite tooling lane runs the release workflow contracts", async () => { - const buildkiteTooling = await readFile( - new URL("../../scripts/run-buildkite-ci-tooling.sh", import.meta.url), +test("required CI tooling lane runs the release workflow contracts", async () => { + const ciTooling = await readFile( + new URL("../../scripts/run-ci-tooling.sh", import.meta.url), "utf8", ); assert.match( - buildkiteTooling, + ciTooling, /node --test \.github\/workflows\/check-release-workflow-contract\.test\.mjs/u, ); assert.match( - buildkiteTooling, + ciTooling, /node \.github\/workflows\/check-release-workflow-contract\.mjs/u, ); }); diff --git a/.github/workflows/maestro-ci.yml b/.github/workflows/maestro-ci.yml new file mode 100644 index 000000000..00d725ea4 --- /dev/null +++ b/.github/workflows/maestro-ci.yml @@ -0,0 +1,549 @@ +name: maestro-ci + +# GitHub Actions maestro-ci pipeline on owned runners. +# The terminal job named `maestro-ci` is the required check-run. +# +# Trusted-head gate: this file is the public evalops/maestro workflow as well +# as the internal source of truth. An unrestricted `pull_request` trigger would +# let a fork execute contributor-controlled scripts, Cargo build scripts, npm +# commands, and Gradle on `evalops-internal-arc*` runners. Use the same +# selector as mono `.github/workflows/sandboxwich-publish.yml`: skip the job +# unless this is not a pull_request or the head repo is this repository. +# Fork PRs never reach internal runners. `require-internal-pr` still blocks +# merging mirrored-source forks. + +on: + pull_request: + types: [opened, synchronize, reopened, ready_for_review] + push: + branches: [main] + schedule: + # Daily 05:17 UTC advisory coverage and perf jobs. + - cron: "17 5 * * *" + workflow_dispatch: + inputs: + hosted_orb_live_smoke: + description: "Set to 1 to run hosted Orb live acceptance" + required: false + default: "" + +permissions: + contents: read + +concurrency: + group: maestro-ci-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +env: + CI: "true" + CARGO_INCREMENTAL: "0" + CARGO_PROFILE_DEV_DEBUG: "0" + CARGO_BUILD_JOBS: "4" + CARGO_PROFILE_DEV_CODEGEN_UNITS: "16" + CARGO_PROFILE_TEST_CODEGEN_UNITS: "16" + npm_config_fetch_retries: "1" + npm_config_fetch_timeout: "30000" + FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true + RUST_TOOLCHAIN: "1.95.0" + +jobs: + protocol-contracts: + if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository + runs-on: ${{ vars.MAESTRO_CI_LIGHT_RUNNER || 'evalops-internal-arc-light' }} + timeout-minutes: 10 + steps: + - name: Reset runner workspace + run: find "$GITHUB_WORKSPACE" -mindepth 1 -maxdepth 1 -exec rm -rf {} + + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version-file: .node-version + cache: npm + - name: Protocol compatibility lock + run: | + set -euo pipefail + timeout --signal=TERM --kill-after=10s 5m npm ci --ignore-scripts + npm run check:workspace-contract + npm run check:protocol-manifest + npm run check:runtime-passport + npm run check:rust-only-runtime + npm run check:helm-probes + npm run check:hook-dispatch + npm run check:session-transfer + npm run check:hosted-orb-delegation + npm run check:macos-signature + + lint: + needs: protocol-contracts + if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository + runs-on: ${{ vars.MAESTRO_CI_HEAVY_RUNNER || 'evalops-internal-arc' }} + timeout-minutes: 90 + steps: + - name: Reset runner workspace + run: find "$GITHUB_WORKSPACE" -mindepth 1 -maxdepth 1 -exec rm -rf {} + + - name: Set Cargo target directory + run: echo "CARGO_TARGET_DIR=${RUNNER_TEMP}/cargo-target" >> "$GITHUB_ENV" + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version-file: .node-version + cache: npm + - uses: ./.github/actions/setup-rust + with: + toolchain: ${{ env.RUST_TOOLCHAIN }} + components: rustfmt,clippy + cache-group: lint + use-sccache: "true" + - name: Contracts, format, and Clippy + run: | + set -euo pipefail + timeout --signal=TERM --kill-after=10s 5m npm ci --ignore-scripts + timeout --signal=TERM --kill-after=30s 45m npm run check + node --test scripts/check-doc-paths.test.mjs scripts/version.test.mjs + if [[ -f scripts/measure-ci-build-latency.test.mjs ]]; then + node --test scripts/measure-ci-build-latency.test.mjs + fi + if [[ -d test/internal ]]; then + npm run test:internal + fi + timeout --signal=TERM --kill-after=30s 30m npm run lint + node --test scripts/check-advisory-expiry.test.mjs + node scripts/check-advisory-expiry.mjs + + rust-tests: + needs: protocol-contracts + if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository + runs-on: ${{ vars.MAESTRO_CI_HEAVY_RUNNER || 'evalops-internal-arc' }} + timeout-minutes: 90 + strategy: + fail-fast: false + matrix: + partition: [1, 2, 3, 4] + env: + CARGO_BUILD_JOBS: "4" + steps: + - name: Reset runner workspace + run: find "$GITHUB_WORKSPACE" -mindepth 1 -maxdepth 1 -exec rm -rf {} + + - name: Set Cargo target directory + run: echo "CARGO_TARGET_DIR=${RUNNER_TEMP}/cargo-target" >> "$GITHUB_ENV" + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version-file: .node-version + cache: npm + - uses: ./.github/actions/setup-rust + with: + toolchain: ${{ env.RUST_TOOLCHAIN }} + cache-group: rust-tests + install-cargo-nextest: "true" + use-sccache: "true" + - name: Prove CI machine auth fails closed + if: matrix.partition == 1 + run: | + set -euo pipefail + MAESTRO_TRUSTED_RUNNER_WORKSPACE_ROOTS="$(pwd -P)" + export MAESTRO_TRUSTED_RUNNER_WORKSPACE_ROOTS + maestro_test_home="$(mktemp -d "${RUNNER_TEMP}/maestro-home.XXXXXX")" + export MAESTRO_HOME="$maestro_test_home" + export MAESTRO_SUBAGENTS_DIR="$maestro_test_home/subagents" + trap 'rm -rf "$maestro_test_home"' EXIT + timeout --signal=TERM --kill-after=30s 10m cargo test --locked -p maestro-local-host ci_auth_conformance + - name: Run nextest partition + env: + NEXTEST_PARTITION: ${{ matrix.partition }} + run: | + set -euo pipefail + timeout --signal=TERM --kill-after=30s 45m scripts/run-nextest-partition.sh "${NEXTEST_PARTITION}/4" + - name: Doctests and TUI smoke + if: matrix.partition == 1 + run: | + set -euo pipefail + timeout --signal=TERM --kill-after=30s 20m cargo test --workspace --locked --doc + export MAESTRO_TUI_BIN="${CARGO_TARGET_DIR}/debug/maestro-tui" + test -x "$MAESTRO_TUI_BIN" || timeout --signal=TERM --kill-after=30s 30m cargo build --locked -p maestro-tui + npm run smoke:tui + + native-release: + needs: protocol-contracts + if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository + runs-on: ${{ vars.MAESTRO_CI_HEAVY_RUNNER || 'evalops-internal-arc' }} + timeout-minutes: 90 + steps: + - name: Reset runner workspace + run: find "$GITHUB_WORKSPACE" -mindepth 1 -maxdepth 1 -exec rm -rf {} + + - name: Set Cargo target directory + run: echo "CARGO_TARGET_DIR=${RUNNER_TEMP}/cargo-target" >> "$GITHUB_ENV" + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version-file: .node-version + cache: npm + - uses: ./.github/actions/setup-rust + with: + toolchain: ${{ env.RUST_TOOLCHAIN }} + cache-group: native-release + use-sccache: "true" + - name: Native build and conformance + run: | + set -euo pipefail + timeout --signal=TERM --kill-after=10s 5m npm ci --ignore-scripts + timeout --signal=TERM --kill-after=30s 45m npm run build + npm run smoke:release-native-only + node scripts/run-runtime-conformance.mjs --binary "${CARGO_TARGET_DIR}/fast-validation/maestro" + + integration: + needs: protocol-contracts + if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository + runs-on: ${{ vars.MAESTRO_CI_HEAVY_RUNNER || 'evalops-internal-arc' }} + timeout-minutes: 60 + steps: + - name: Reset runner workspace + run: find "$GITHUB_WORKSPACE" -mindepth 1 -maxdepth 1 -exec rm -rf {} + + - name: Set Cargo target directory + run: echo "CARGO_TARGET_DIR=${RUNNER_TEMP}/cargo-target" >> "$GITHUB_ENV" + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version-file: .node-version + cache: npm + - uses: ./.github/actions/setup-rust + with: + toolchain: ${{ env.RUST_TOOLCHAIN }} + cache-group: integration + use-sccache: "true" + - name: Control-plane integration + env: + INTEGRATION_SUFFIX: ${{ github.run_id }}-${{ github.run_attempt }} + run: | + set -euo pipefail + suffix="${INTEGRATION_SUFFIX//-/}" + redis="maestro-redis-${suffix}" + postgres="maestro-postgres-${suffix}" + trap 'docker rm -f "$redis" "$postgres" >/dev/null 2>&1 || true' EXIT + redis_image="mirror.gcr.io/library/redis:7-alpine" + postgres_image="mirror.gcr.io/library/postgres:16-alpine" + timeout --signal=TERM --kill-after=10s 2m docker pull "$redis_image" + timeout --signal=TERM --kill-after=10s 2m docker pull "$postgres_image" + docker run -d --rm --name "$redis" -p 127.0.0.1::6379 "$redis_image" + docker run -d --rm --name "$postgres" -e POSTGRES_USER=maestro -e POSTGRES_PASSWORD=maestro -e POSTGRES_DB=maestro -p 127.0.0.1::5432 "$postgres_image" + redis_port="$(docker port "$redis" 6379/tcp | sed 's/.*://')" + postgres_port="$(docker port "$postgres" 5432/tcp | sed 's/.*://')" + for _ in $(seq 1 60); do docker exec "$postgres" pg_isready -U maestro >/dev/null 2>&1 && break; sleep 2; done + MAESTRO_REDIS_URL="redis://127.0.0.1:${redis_port}" \ + MAESTRO_DATABASE_URL="postgresql://maestro:maestro@127.0.0.1:${postgres_port}/maestro" \ + PGPASSWORD=maestro timeout --signal=TERM --kill-after=30s 30m cargo test --locked -p maestro-runtime-gateway + POSTGRES_DB=maestro \ + POSTGRES_HOST=127.0.0.1 \ + POSTGRES_PASSWORD=maestro \ + POSTGRES_PORT="$postgres_port" \ + POSTGRES_USER=maestro \ + PGPASSWORD=maestro \ + timeout --signal=TERM --kill-after=30s 30m cargo test --locked -p maestro-tui --test tools_integration + + scenario-replay: + needs: protocol-contracts + if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository + runs-on: ${{ vars.MAESTRO_CI_HEAVY_RUNNER || 'evalops-internal-arc' }} + timeout-minutes: 45 + steps: + - name: Reset runner workspace + run: find "$GITHUB_WORKSPACE" -mindepth 1 -maxdepth 1 -exec rm -rf {} + + - name: Set Cargo target directory + run: echo "CARGO_TARGET_DIR=${RUNNER_TEMP}/cargo-target" >> "$GITHUB_ENV" + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version-file: .node-version + cache: npm + - uses: ./.github/actions/setup-rust + with: + toolchain: ${{ env.RUST_TOOLCHAIN }} + cache-group: scenario-replay + use-sccache: "true" + - name: Deterministic scenario replay + run: | + set -euo pipefail + timeout --signal=TERM --kill-after=10s 5m npm ci --ignore-scripts + timeout --signal=TERM --kill-after=30s 30m cargo build --locked -p maestro-scenario + node --test scripts/scenario-replay-governance.test.mjs + MAESTRO_BIN="${CARGO_TARGET_DIR}/debug/maestro-scenario" npm run check:scenario-replay-gate -- --junit-dir tmp/scenario-replay + + ci-contracts: + needs: protocol-contracts + if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository + runs-on: ${{ vars.MAESTRO_CI_LIGHT_RUNNER || 'evalops-internal-arc-light' }} + timeout-minutes: 30 + steps: + - name: Reset runner workspace + run: find "$GITHUB_WORKSPACE" -mindepth 1 -maxdepth 1 -exec rm -rf {} + + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version-file: .node-version + cache: npm + - name: CI configuration contracts + run: | + set -euo pipefail + timeout --signal=TERM --kill-after=10s 5m npm ci --ignore-scripts + test ! -d src/hooks + node --test scripts/check-ci-concurrency.test.mjs + node scripts/check-rust-only-runtime.mjs + node scripts/check-hook-dispatch-coverage.mjs + if [[ -f scripts/workflow-contracts.test.mjs ]]; then + node --test scripts/workflow-contracts.test.mjs + fi + + workflow-tooling: + needs: protocol-contracts + if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository + runs-on: ${{ vars.MAESTRO_CI_LIGHT_RUNNER || 'evalops-internal-arc-light' }} + timeout-minutes: 30 + steps: + - name: Reset runner workspace + run: find "$GITHUB_WORKSPACE" -mindepth 1 -maxdepth 1 -exec rm -rf {} + + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version-file: .node-version + cache: npm + - name: Workflow and shell tooling + run: bash scripts/run-ci-tooling.sh + + supply-chain: + needs: protocol-contracts + if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository + # deny.toml PRs call GET /pulls/{n} and GET /issues/{n}/timeline through GH_TOKEN. + # Job-level permissions replace the workflow default, so contents: read stays explicit. + permissions: + contents: read + pull-requests: read + issues: read + runs-on: ${{ vars.MAESTRO_CI_LIGHT_RUNNER || 'evalops-internal-arc-light' }} + timeout-minutes: 45 + steps: + - name: Reset runner workspace + run: find "$GITHUB_WORKSPACE" -mindepth 1 -maxdepth 1 -exec rm -rf {} + + - name: Set Cargo target directory + run: echo "CARGO_TARGET_DIR=${RUNNER_TEMP}/cargo-target" >> "$GITHUB_ENV" + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 0 + persist-credentials: false + - name: Fetch pull-request base branch + if: github.event_name == 'pull_request' + env: + BASE_REF: ${{ github.base_ref }} + run: | + set -euo pipefail + git fetch --no-tags origin "+refs/heads/${BASE_REF}:refs/remotes/origin/${BASE_REF}" + - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version-file: .node-version + cache: npm + - uses: ./.github/actions/setup-rust + with: + toolchain: ${{ env.RUST_TOOLCHAIN }} + cache-group: supply-chain + use-sccache: "true" + - name: Supply chain + env: + GH_TOKEN: ${{ github.token }} + MAESTRO_CI_PULL_REQUEST: ${{ github.event.pull_request.number || 'false' }} + MAESTRO_CI_BASE_BRANCH: ${{ github.base_ref || 'main' }} + MAESTRO_CI_COMMIT: ${{ github.event.pull_request.head.sha || github.sha }} + run: bash scripts/run-ci-supply-chain.sh + + jetbrains-plugin: + needs: protocol-contracts + if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository + runs-on: ${{ vars.MAESTRO_CI_LIGHT_RUNNER || 'evalops-internal-arc-light' }} + timeout-minutes: 45 + steps: + - name: Reset runner workspace + run: find "$GITHUB_WORKSPACE" -mindepth 1 -maxdepth 1 -exec rm -rf {} + + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: actions/setup-java@de7274f081f381c8f8158605e0321c36c376e2e6 # v6.0.1 + with: + distribution: temurin + java-version: "21" + - name: JetBrains plugin + run: bash scripts/run-ci-jetbrains.sh + + linux-check: + needs: protocol-contracts + if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository + runs-on: ${{ vars.MAESTRO_CI_HEAVY_RUNNER || 'evalops-internal-arc' }} + timeout-minutes: 90 + steps: + - name: Reset runner workspace + run: find "$GITHUB_WORKSPACE" -mindepth 1 -maxdepth 1 -exec rm -rf {} + + - name: Set Cargo target directory + run: echo "CARGO_TARGET_DIR=${RUNNER_TEMP}/cargo-target" >> "$GITHUB_ENV" + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version-file: .node-version + cache: npm + - uses: ./.github/actions/setup-rust + with: + toolchain: ${{ env.RUST_TOOLCHAIN }} + components: rustfmt,clippy + cache-group: linux-check + use-sccache: "true" + - name: Native Linux validation + run: | + set -euo pipefail + timeout --signal=TERM --kill-after=10s 5m npm ci --ignore-scripts + bash scripts/ci-linux-check.sh + + coverage: + if: (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) && (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') + continue-on-error: true + runs-on: ${{ vars.MAESTRO_CI_HEAVY_RUNNER || 'evalops-internal-arc' }} + timeout-minutes: 90 + steps: + - name: Reset runner workspace + run: find "$GITHUB_WORKSPACE" -mindepth 1 -maxdepth 1 -exec rm -rf {} + + - name: Set Cargo target directory + run: echo "CARGO_TARGET_DIR=${RUNNER_TEMP}/cargo-target" >> "$GITHUB_ENV" + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: ./.github/actions/setup-rust + with: + toolchain: ${{ env.RUST_TOOLCHAIN }} + cache-group: coverage + install-cargo-llvm-cov: "true" + install-cargo-nextest: "true" + use-sccache: "true" + - name: Coverage (advisory) + run: bash scripts/run-ci-coverage.sh + - name: Upload coverage report + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: coverage-report-${{ github.run_id }}-${{ github.run_attempt }} + path: coverage-report/** + if-no-files-found: warn + retention-days: 14 + + perf-baseline: + if: (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) && (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') + continue-on-error: true + runs-on: ${{ vars.MAESTRO_CI_HEAVY_RUNNER || 'evalops-internal-arc' }} + timeout-minutes: 60 + steps: + - name: Reset runner workspace + run: find "$GITHUB_WORKSPACE" -mindepth 1 -maxdepth 1 -exec rm -rf {} + + - name: Set Cargo target directory + run: echo "CARGO_TARGET_DIR=${RUNNER_TEMP}/cargo-target" >> "$GITHUB_ENV" + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: ./.github/actions/setup-rust + with: + toolchain: ${{ env.RUST_TOOLCHAIN }} + cache-group: perf + use-sccache: "true" + - name: Performance baseline (advisory) + run: bash scripts/run-ci-perf.sh + + hosted-orb-delegation-live: + if: (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) && (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') + # Tolerate failure only when the live smoke is not requested. A gated failure + # must keep a failing `needs` result so the terminal maestro-ci job fails closed. + continue-on-error: ${{ !(github.event.inputs.hosted_orb_live_smoke == '1' || vars.MAESTRO_HOSTED_ORB_LIVE_SMOKE == '1') }} + runs-on: ${{ vars.MAESTRO_CI_LIGHT_RUNNER || 'evalops-internal-arc-light' }} + timeout-minutes: 30 + steps: + - name: Reset runner workspace + run: find "$GITHUB_WORKSPACE" -mindepth 1 -maxdepth 1 -exec rm -rf {} + + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version-file: .node-version + cache: npm + - name: Hosted Orb delegation acceptance + env: + MAESTRO_HOSTED_ORB_LIVE_SMOKE: ${{ github.event.inputs.hosted_orb_live_smoke || vars.MAESTRO_HOSTED_ORB_LIVE_SMOKE || '' }} + run: | + set -euo pipefail + timeout --signal=TERM --kill-after=10s 5m npm ci --ignore-scripts + if [[ "${MAESTRO_HOSTED_ORB_LIVE_SMOKE:-}" != "1" ]]; then + echo "Optional hosted Orb acceptance was not requested; skipping." + exit 0 + fi + npm run smoke:hosted-orb-delegation + + maestro-ci: + name: maestro-ci + if: always() && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) + needs: + - protocol-contracts + - lint + - rust-tests + - native-release + - integration + - scenario-replay + - ci-contracts + - workflow-tooling + - supply-chain + - jetbrains-plugin + - linux-check + - coverage + - perf-baseline + - hosted-orb-delegation-live + runs-on: ${{ vars.MAESTRO_CI_LIGHT_RUNNER || 'evalops-internal-arc-light' }} + timeout-minutes: 5 + steps: + - name: Require every needed maestro-ci job + env: + HEAD_SHA: ${{ github.event.pull_request.head.sha || github.sha }} + NEEDS_JSON: ${{ toJSON(needs) }} + run: | + set -euo pipefail + python3 - <<'PY' + import json + import os + import sys + + needs = json.loads(os.environ["NEEDS_JSON"]) + sha = os.environ["HEAD_SHA"] + bad = [] + skipped = [] + for name, body in sorted(needs.items()): + result = (body or {}).get("result") + if result == "success": + continue + if result == "skipped": + skipped.append(name) + continue + bad.append(f"{name}={result or ''}") + if bad: + print(f"maestro-ci failed for {sha}: {', '.join(bad)}") + sys.exit(1) + extra = f" skipped={','.join(skipped)}" if skipped else "" + print(f"maestro-ci passed for {sha}{extra}") + PY diff --git a/.github/workflows/release-branch-admission.test.mjs b/.github/workflows/release-branch-admission.test.mjs index 5e4d3a49a..bd1992fa0 100644 --- a/.github/workflows/release-branch-admission.test.mjs +++ b/.github/workflows/release-branch-admission.test.mjs @@ -16,7 +16,7 @@ const rules = ["pull_request", "non_fast_forward", "deletion", "required_linear_ rules.push({type:"required_status_checks", parameters:{strict_required_status_checks_policy:true, required_status_checks:[ {context:"require-internal-pr", integration_id:15368}, {context:"unresolved-review-threads / unresolved-review-threads", integration_id:15368}, - {context:"buildkite/maestro-ci", integration_id:805657}, + {context:"maestro-ci", integration_id:15368}, ]}}); test("release admission requires protected history and all pinned checks", () => { diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index f87b3e11e..b93f4ddf5 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -97,7 +97,7 @@ jobs: .parameters.strict_required_status_checks_policy == true and any(.parameters.required_status_checks[]; .context == "require-internal-pr" and .integration_id == 15368) and any(.parameters.required_status_checks[]; .context == "unresolved-review-threads / unresolved-review-threads" and .integration_id == 15368) and - any(.parameters.required_status_checks[]; .context == "buildkite/maestro-ci" and .integration_id == 805657)) + any(.parameters.required_status_checks[]; .context == "maestro-ci" and .integration_id == 15368)) ' <<<"$rules" >/dev/null timeout 60s git fetch --no-tags origin "refs/heads/$release_branch" fi diff --git a/AGENTS.md b/AGENTS.md index a0cbe3bd6..5f0799b5b 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -9,8 +9,8 @@ originate in `evalops/maestro-internal` and arrive through the protected mirror workflow; do not edit mirrored source directly here. For contribution setup and checks, use `CONTRIBUTING.md` and -`docs/CONTRIBUTOR_RUNBOOK.md`. Validation is defined in the mirrored -`.buildkite/pipeline.yml` and `.buildkite/advisory.yml`; public GitHub workflows are limited to review policy, +`docs/CONTRIBUTOR_RUNBOOK.md`. Validation is defined in `.github/workflows/maestro-ci.yml` on owned GitHub +Actions runners; public GitHub workflows also cover review policy, source provenance, release publishing, and repository automation. Do not add private repository documentation, credentials, internal agent diff --git a/packages/jetbrains-plugin/README.md b/packages/jetbrains-plugin/README.md index 523bfdecd..99085b220 100644 --- a/packages/jetbrains-plugin/README.md +++ b/packages/jetbrains-plugin/README.md @@ -99,7 +99,7 @@ When connected to the Deixic Code server, the plugin provides IDE-aware tools: The JetBrains plugin is kept as an active package. Pull requests that touch `packages/jetbrains-plugin/`, the shared headless proto, or the JetBrains -Buildkite `jetbrains-plugin` step, which executes +GitHub Actions `jetbrains-plugin` job, which executes `./gradlew check buildPlugin --no-daemon`. ### Build diff --git a/packages/tui-rs/src/config_cli.rs b/packages/tui-rs/src/config_cli.rs index 345d27289..6d2c71fe8 100644 --- a/packages/tui-rs/src/config_cli.rs +++ b/packages/tui-rs/src/config_cli.rs @@ -2328,7 +2328,7 @@ fn format_bytes(bytes: u64) -> String { fn stdin_is_tty() -> bool { // Unit tests must stay non-interactive even when the test runner allocates - // a pseudo-terminal (as Buildkite does for this workspace). + // a pseudo-terminal (as GitHub Actions does for this workspace). if cfg!(test) { return false; } diff --git a/scripts/build-profile-contract.test.mjs b/scripts/build-profile-contract.test.mjs index 68387702f..58eaa74d9 100644 --- a/scripts/build-profile-contract.test.mjs +++ b/scripts/build-profile-contract.test.mjs @@ -20,7 +20,7 @@ const materialize = read("scripts/materialize-native-package.mjs"); const releaseBuilder = read("scripts/build-release-binary.mjs"); const dockerfile = read("Dockerfile"); const updateCli = read("packages/tui-rs/src/update_cli.rs"); -const ci = read(".buildkite/pipeline.yml"); +const ci = read(".github/workflows/maestro-ci.yml"); const release = read(".github/workflows/release.yml"); const ghcr = read(".github/workflows/ghcr-publish.yml"); const mirrorPath = new URL("../.github/workflows/public-release-mirror.yml", import.meta.url); diff --git a/scripts/check-buildkite-pipeline.test.mjs b/scripts/check-buildkite-pipeline.test.mjs deleted file mode 100644 index 10bd87bed..000000000 --- a/scripts/check-buildkite-pipeline.test.mjs +++ /dev/null @@ -1,343 +0,0 @@ -import assert from "node:assert/strict"; -import { access, readFile } from "node:fs/promises"; -import test from "node:test"; -import { mkdtempSync, mkdirSync, writeFileSync, rmSync } from 'node:fs'; -import { tmpdir } from 'node:os'; -import { join } from 'node:path'; -import { fileURLToPath } from 'node:url'; -import { spawnSync } from 'node:child_process'; -const root = new URL("../", import.meta.url); -const pipeline = await readFile(new URL(".buildkite/pipeline.yml", root), "utf8"); -const advisory = await readFile(new URL(".buildkite/advisory.yml", root), "utf8"); -const nextest = await readFile(new URL(".config/nextest.toml", root), "utf8"); -const tooling = await readFile(new URL("scripts/run-buildkite-ci-tooling.sh", root), "utf8"); -const jetbrains = await readFile(new URL("scripts/run-buildkite-jetbrains.sh", root), "utf8"); -const coverage = await readFile(new URL("scripts/run-buildkite-coverage.sh", root), "utf8"); -const miseRunner = await readFile(new URL(".buildkite/run-with-mise", root), "utf8"); -const a2aTmuxSmoke = await readFile(new URL("scripts/smoke-maestro-a2a-tmux.sh", root), "utf8"); - -test("Buildkite routes jobs through the configured Maestro worker pool", () => { - assert.doesNotMatch(pipeline + advisory, /hetzner-linux-heavy/, "retired worker pool must not strand validation jobs"); - assert.match(pipeline, /queue: "\$\{MAESTRO_CI_QUEUE:-linux-large\}"/); - assert.match(pipeline, /image: "\$\{MAESTRO_CI_IMAGE:-evalops-platform-ci-v6\}"/); - assert.match(pipeline, /queue: "\$\{MAESTRO_CI_JETBRAINS_QUEUE:-linux-medium\}"/); - assert.match(pipeline, /image: "\$\{MAESTRO_CI_JETBRAINS_IMAGE:-evalops-platform-ci-v6\}"/); - assert.match(pipeline, /queue: "\$\{MAESTRO_CI_INTEGRATION_QUEUE:-linux-medium\}"/); - assert.match(pipeline, /image: "\$\{MAESTRO_CI_INTEGRATION_IMAGE:-evalops-platform-ci-v6\}"/); -}); - -test("Rust lanes bootstrap the pinned toolchain before invoking Cargo", () => { - assert.equal((pipeline.match(/\.buildkite\/run-with-mise "rust@1\.95\.0"/g) ?? []).length, 5); - assert.equal( - (pipeline.match(/\.buildkite\/run-with-mise "rust@1\.95\.0 gh@2\.88\.1"/g) ?? []).length, - 1, - ); - assert.match(miseRunner, /mise_version="2026\.4\.24"/); - assert.match(miseRunner, /4ecf49b825741e1e3e8ff8c92ee242cf728da951d1b3592ef1c2f080201fa454/); - assert.match(miseRunner, /sha256sum --check --status/); - assert.match(miseRunner, /exec "\$\{tools\[@\]\}" -- "\$@"/); -}); - -test("Buildkite bounds shared worker concurrency and infrastructure retries", () => { - assert.equal( - pipeline.match(/priority: 50/gu)?.length, - 9, - "required repository validation lanes should outrank advisory and stale default-priority work", - ); - assert.equal((pipeline.match(/concurrency: 3/g) ?? []).length, 9); - assert.equal((pipeline.match(/MAESTRO_CI_CONCURRENCY_GROUP/g) ?? []).length, 8); - assert.equal((pipeline.match(/MAESTRO_CI_JETBRAINS_CONCURRENCY_GROUP/g) ?? []).length, 1); - assert.equal((pipeline.match(/MAESTRO_CI_ADVISORY_CONCURRENCY_GROUP/g) ?? []).length, 0); - assert.equal((advisory.match(/MAESTRO_CI_ADVISORY_CONCURRENCY_GROUP/g) ?? []).length, 2); - assert.equal((pipeline.match(/exit_status: -1/g) ?? []).length, 10); - assert.equal((pipeline.match(/signal_reason: none/g) ?? []).length, 10); - assert.equal((pipeline.match(/signal_reason: agent_stop/g) ?? []).length, 10); - assert.equal((advisory.match(/exit_status: -1/g) ?? []).length, 2); - assert.equal((advisory.match(/signal_reason: none/g) ?? []).length, 2); - assert.equal((advisory.match(/signal_reason: agent_stop/g) ?? []).length, 2); - assert.doesNotMatch(pipeline, /exit_status: "\*"/); -}); - -test("Buildkite network and long-running operations are bounded", () => { - assert.equal((pipeline.match(/5m npm ci --ignore-scripts/g) ?? []).length, 5); - assert.match(pipeline, /npm_config_fetch_retries: "1"/); - assert.match(pipeline, /npm_config_fetch_timeout: "30000"/); - assert.equal((pipeline.match(/2m docker pull/g) ?? []).length, 2); - for (const command of [ - "45m npm run check", - "30m npm run lint", - "45m cargo nextest run --profile buildkite --workspace --locked --no-fail-fast", - "20m cargo test --workspace --locked --doc", - "30m cargo build --locked -p maestro-tui", - "45m npm run build", - "30m cargo test --locked -p maestro-runtime-gateway", - "30m cargo test --locked -p maestro-tui --test tools_integration", - "30m cargo build --locked -p maestro-scenario", - ]) { - assert.match(pipeline, new RegExp(command.replaceAll("-", "\\-"))); - } -}); - -test("Buildkite publishes only the offline machine-auth contract", () => { - assert.doesNotMatch( - pipeline, - /live exchange remains disabled|Identity owns|durable replay store/, - ); -}); - -test("advisory coverage uses nextest and an isolated instrumented target dir", () => { - assert.match(coverage, /CARGO_TARGET_DIR="\$\{repo_root\}\/\.buildkite\/cache\/cargo-target-cov"/); - assert.match(coverage, /cargo llvm-cov nextest/); - assert.match(coverage, /--lib/); - assert.match(coverage, /--no-clean/); - assert.doesNotMatch(coverage, /llvm-cov nextest[\s\S]*--no-report/); - assert.match(coverage, /--ignore-run-fail/); - assert.match(coverage, /--profile buildkite/); - assert.match( - coverage, - /cargo-llvm-cov-x86_64-unknown-linux-gnu\.tar\.gz/, - ); - assert.match( - coverage, - /b068f7c98841aacb9c4f382b4a0c184ae82f49b56a32d442b429b2961c73be15/, - ); - assert.doesNotMatch(coverage, /cargo install cargo-llvm-cov/); - assert.doesNotMatch(coverage, /cargo llvm-cov --workspace --locked --no-report/); - assert.doesNotMatch(coverage, /--html/); -}); - -test("advisory coverage and perf are not in the default pipeline", () => { - assert.doesNotMatch(pipeline, /key: "coverage"/); - assert.doesNotMatch(pipeline, /key: "perf-baseline"/); - assert.match(pipeline, /key: "advisory-upload"/); - assert.match(pipeline, /soft_fail: true/); - assert.match( - pipeline, - /BUILDKITE_SOURCE\}" == "schedule" && -f \.buildkite\/advisory\.yml[\s\S]*pipeline upload \.buildkite\/advisory\.yml/, - ); - assert.match(advisory, /queue: "\$\{MAESTRO_CI_QUEUE:-linux-large\}"/); - assert.match(advisory, /image: "\$\{MAESTRO_CI_IMAGE:-evalops-platform-ci-v6\}"/); - assert.match(advisory, /CARGO_TARGET_DIR: "\.buildkite\/cache\/cargo-target"/); - assert.match(advisory, /key: "coverage"[\s\S]*priority: 10/); - assert.match(advisory, /key: "perf-baseline"[\s\S]*priority: 10/); - assert.doesNotMatch(advisory, /^\s*if:/mu); -}); - -test("Buildkite covers every migrated validation family", () => { - for (const key of [ - "protocol-contracts", "lint", "rust-tests", "native-release", "integration", - "scenario-replay", "ci-contracts", "workflow-tooling", "supply-chain", - "jetbrains-plugin", "advisory-upload", - ]) { - assert.match(pipeline, new RegExp(`key: "${key}"`)); - } - for (const key of ["coverage", "perf-baseline"]) { - assert.match(advisory, new RegExp(`key: "${key}"`)); - } - for (const script of ["ci-tooling", "supply-chain", "jetbrains"]) { - assert.match(pipeline, new RegExp(`scripts/run-buildkite-${script}\\.sh`)); - } - for (const script of ["coverage", "perf"]) { - assert.match(advisory, new RegExp(`scripts/run-buildkite-${script}\\.sh`)); - } -}); - -test("workflow tooling installs pinned binaries without requiring Go", () => { - assert.doesNotMatch(tooling, /go install/); - assert.match(tooling, /actionlint_1\.7\.9_\$\{actionlint_platform\}\.tar\.gz/); - assert.match(tooling, /233b280d05e100837f4af1433c7b40a5dcb306e3aa68fb4f17f8a7f45a7df7b4/); - assert.match(tooling, /sha256sum --check --status/); - assert.match(tooling, /shasum -a 256 --check --status/); - assert.match(tooling, /! -f \.github\/workflows\/sync-public-release-mirror\.yml/); -}); - -test("JetBrains validation fails fast and retries only its stuck-JVM exit", () => { - assert.match(pipeline, /key: "jetbrains-plugin"[\s\S]*exit_status: 137[\s\S]*limit: 1/); - assert.match(jetbrains, /10m \\\n\s+\.\/gradlew check buildPlugin --no-daemon \\/); - assert.match(jetbrains, /org\.gradle\.workers\.max=1/); - assert.match( - jetbrains, - /org\.gradle\.jvmargs="-Xmx1g -XX:MaxMetaspaceSize=512m -XX:\+ExitOnOutOfMemoryError"/, - ); -}); - -test("protocol lock fails the build before heavy jobs start", () => { - assert.match(pipeline, /key: "protocol-contracts"/); - assert.match(pipeline, /npm run check:protocol-manifest/); - assert.equal((pipeline.match(/depends_on: "protocol-contracts"/g) ?? []).length, 4); -}); - -test("protocol lock does not share the heavy Rust queue", () => { - const lock = pipeline.split('key: "protocol-contracts"')[1]?.split('key: "')[0] ?? ""; - const rust = pipeline.split('key: "rust-tests"')[1]?.split('key: "')[0] ?? ""; - assert.match(lock, /queue: "\$\{MAESTRO_CI_PROTOCOL_QUEUE:-linux-medium\}"/); - assert.match(lock, /image: "\$\{MAESTRO_CI_PROTOCOL_IMAGE:-evalops-platform-ci-v6\}"/); - assert.match(rust, /queue: "\$\{MAESTRO_CI_QUEUE:-linux-large\}"/); - assert.doesNotMatch(lock, /queue: "\$\{MAESTRO_CI_QUEUE:-linux-large\}"/); -}); - -test("rust-tests caps compile jobs and retries OOM SIGKILL", () => { - assert.match(pipeline, /key: "rust-tests"[\s\S]*?CARGO_BUILD_JOBS: "4"/); - assert.match(pipeline, /key: "rust-tests"[\s\S]*?exit_status: 137/); -}); - -test("CI caps rustc codegen units so one crate cannot SIGKILL the worker", () => { - assert.match(pipeline, /CARGO_PROFILE_DEV_CODEGEN_UNITS: "16"/); - assert.match(pipeline, /CARGO_PROFILE_TEST_CODEGEN_UNITS: "16"/); - assert.match(advisory, /CARGO_PROFILE_DEV_CODEGEN_UNITS: "16"/); - assert.match(advisory, /CARGO_PROFILE_TEST_CODEGEN_UNITS: "16"/); - assert.match(pipeline, /CARGO_BUILD_JOBS: "4"/); - assert.match(pipeline, /key: "lint"[\s\S]*?exit_status: 137/); -}); - -test("internal-only contracts are conditional in the shared public pipeline", () => { - assert.match(pipeline, /if \[\[ -d test\/internal \]\]; then\n\s+npm run test:internal\n\s+fi/); - assert.match(pipeline, /if \[\[ -f scripts\/measure-ci-build-latency\.test\.mjs \]\]; then/); -}); - -test("public projections skip the internal mirror workflow contract", () => { - assert.match( - tooling, - /check-sync-public-release-mirror-workflow\.test\.mjs[\s\S]*?\.github\/workflows\/sync-public-release-mirror\.yml/, - ); -}); - -test("tag-release contract stays green when the public tree omits public-release-mirror.yml", async () => { - const contract = await readFile(new URL("scripts/tag-release-contract.test.mjs", root), "utf8"); - assert.match(contract, /existsSync\(/); - assert.match(contract, /public-release-mirror\.yml/); - assert.match(contract, /dispatch-public-release:/); -}); - -test("Rust validation uses the pinned canonical nextest split", () => { - assert.match(pipeline, /nextest_version="0\.9\.143"/); - assert.match(pipeline, /66786b9abe23920d022a182d1416b1bbc8130dd4872a9553d76985a1708dcd1e/); - assert.match(pipeline, /cargo nextest run --profile buildkite --workspace --locked --no-fail-fast/); - assert.match(pipeline, /cargo test --workspace --locked --doc/); - assert.match(nextest, /filter = 'binary\(pty_e2e\)'/); - assert.match(nextest, /test-group = 'pty-e2e'/); - assert.match(nextest, /\[test-groups\.pty-e2e\]\nmax-threads = 1/); -}); - -test("rust-tests explicitly proves CI machine auth fails closed without privileged tokens", () => { - const rust = pipeline.split('key: "rust-tests"')[1]?.split('key: "')[0] ?? ""; - assert.match(rust, /cargo test --locked -p maestro-local-host ci_auth_conformance/); - assert.doesNotMatch(rust, /ACTIONS_ID_TOKEN_REQUEST_TOKEN/); -}); - -test("integration containers are removed by their actual names", () => { - assert.match(pipeline, /trap 'docker rm -f "\$\$redis" "\$\$postgres"/); -}); - -test("Docker integration and supply-chain checks select their required runtimes", () => { - const integration = pipeline.split('key: "integration"')[1]?.split('key: "')[0] ?? ""; - const integrationAgents = integration.split("agents:")[1]?.split("cache:")[0] ?? ""; - const supplyChain = pipeline.split('key: "supply-chain"')[1]?.split('key: "')[0] ?? ""; - assert.match(integration, /queue: "\$\{MAESTRO_CI_INTEGRATION_QUEUE:-linux-medium\}"/); - assert.doesNotMatch(integration, /queue: "\$\{MAESTRO_CI_QUEUE:-linux-large\}"/); - assert.doesNotMatch(integrationAgents, /#/); - assert.match(integration, /docker pull/); - assert.match(supplyChain, /queue: "\$\{MAESTRO_CI_SUPPLY_CHAIN_QUEUE:-linux-medium\}"/); - assert.match(supplyChain, /image: "\$\{MAESTRO_CI_SUPPLY_CHAIN_IMAGE:-evalops-platform-ci-v6\}"/); - assert.match( - supplyChain, - /command: \.buildkite\/run-with-mise "rust@1\.95\.0 gh@2\.88\.1" scripts\/run-buildkite-supply-chain\.sh/, - ); -}); - -test("A2A tmux smoke atomically reserves its session before resetting durable task databases", () => { - const cleanup = a2aTmuxSmoke.split("cleanup() {")[1]?.split("a2a_cli() {")[0] ?? ""; - const startup = a2aTmuxSmoke.split('cd "$ROOT_DIR"')[1] ?? ""; - const reservation = startup.indexOf('tmux new-session -d -s "$SESSION_NAME" -n peer-a "sleep 300"'); - const resetStart = startup.indexOf("rm -f"); - const respawn = startup.indexOf('tmux respawn-window -k -t "$SESSION_NAME:peer-a"'); - assert.match(cleanup, /if \[\[ "\$OWNS_SESSION" == "1" \]\]; then/); - assert.match(startup, /if tmux new-session[^\n]+; then\n\tOWNS_SESSION=1/); - assert.ok(reservation >= 0, "smoke must atomically reserve the tmux session"); - assert.ok(resetStart > reservation, "state reset must follow session reservation"); - assert.ok(respawn > resetStart, "peer A must start after state reset"); - assert.doesNotMatch(startup, /tmux has-session/); - const reset = startup.slice(resetStart, respawn); - for (const tasks of ["TASKS_A", "TASKS_B"]) { - for (const suffix of ["", ".sqlite3", ".sqlite3-wal", ".sqlite3-shm"]) { - assert.ok(reset.includes(`"$${tasks}${suffix}"`), `reset must remove $${tasks}${suffix}`); - } - } -}); - -test("legacy GitHub validation workflows are absent", async () => { - const names = [ - "actionlint.yml", "ci.yml", "coverage.yml", "evals.yml", "hooks.yml", - "integration.yml", "jetbrains-plugin.yml", "perf-baselines.yml", - "required-checks-invariant.yml", "scenario-replay.yml", "shellcheck.yml", - "supply-chain.yml", - ]; - for (const name of names) { - await assert.rejects(access(new URL(`.github/workflows/${name}`, root))); - } -}); - - -const runner = fileURLToPath(new URL('./run-buildkite-jetbrains.sh', import.meta.url)); - -test('JetBrains CI detaches Gradle stdin from the controlling terminal', { - skip: process.platform !== 'linux', -}, () => { - const root = mkdtempSync(join(tmpdir(), 'maestro-gradle-stdin-')); - try { - const bin = join(root, 'bin'); - const plugin = join(root, 'packages/jetbrains-plugin'); - mkdirSync(bin); - mkdirSync(plugin, { recursive: true }); - writeFileSync(join(bin, 'java'), '#!/bin/sh\necho \'openjdk version "21.0.12"\' >&2\n', { mode: 0o755 }); - // Shorten only the deadline; retain GNU timeout's real process-group behavior. - writeFileSync(join(bin, 'timeout'), `#!/bin/bash -args=("$@") -for i in "\${!args[@]}"; do - [[ "\${args[$i]}" != 10m ]] || args[$i]=1s - [[ "\${args[$i]}" != --kill-after=30s ]] || args[$i]=--kill-after=1s -done -exec /usr/bin/timeout "\${args[@]}" -`, { mode: 0o755 }); - writeFileSync(join(plugin, 'gradlew'), '#!/bin/sh\nexec "$TEST_NODE" "$TEST_READER"\n', { mode: 0o755 }); - const reader = join(root, 'read-stdin.cjs'); - writeFileSync(reader, `const fs = require('node:fs'); -fs.readSync(0, Buffer.alloc(1), 0, 1, null); -console.log('gradle stdin reached EOF'); -`); - const result = spawnSync('script', ['-q', '-e', '-c', `bash '${runner.replaceAll("'", "'\\''")}'`, '/dev/null'], { - cwd: root, - env: { ...process.env, PATH: `${bin}:${process.env.PATH}`, TEST_NODE: process.execPath, TEST_READER: reader, BUILDKITE_BUILD_CHECKOUT_PATH: root }, - input: '', encoding: 'utf8', timeout: 5000, - }); - assert.ifError(result.error); - assert.equal(result.status, 0, result.stdout + result.stderr); - assert.match(result.stdout, /gradle stdin reached EOF/); - } finally { - rmSync(root, { recursive: true, force: true }); - } -}); - - -test("tool bootstrap rejects an installed but unusable actionlint shim", () => { - const directory = mkdtempSync(join(tmpdir(), 'maestro-tool-shim-')); - try { - const bin = join(directory, 'bin'); - mkdirSync(bin); - writeFileSync(join(bin, 'curl'), '#!/bin/sh\necho bootstrap-download-requested\nexit 77\n', { mode: 0o755 }); - const bootstrap = tooling.slice(0, tooling.indexOf('if ! shellcheck --version')); - assert.ok(bootstrap.includes('if ! actionlint --version')); - for (const usable of [false, true]) { - writeFileSync(join(bin, 'actionlint'), `#!/bin/sh\n[ "$1" = "--version" ] || exit 90\nexit ${usable ? 0 : 1}\n`, { mode: 0o755 }); - const result = spawnSync('bash', ['-c', bootstrap], { - cwd: directory, - env: { ...process.env, BUILDKITE_BUILD_CHECKOUT_PATH: directory, PATH: `${bin}:${process.env.PATH}` }, - encoding: 'utf8', - }); - assert.equal(result.status, usable ? 0 : 77, result.stderr); - assert.equal(result.stdout.includes('bootstrap-download-requested'), !usable); - } - } finally { - rmSync(directory, { recursive: true, force: true }); - } -}); diff --git a/scripts/check-ci-concurrency.test.mjs b/scripts/check-ci-concurrency.test.mjs index 3abc0657f..d473c9924 100644 --- a/scripts/check-ci-concurrency.test.mjs +++ b/scripts/check-ci-concurrency.test.mjs @@ -1,30 +1,470 @@ import assert from "node:assert/strict"; -import { readFileSync } from "node:fs"; +import { mkdtempSync, mkdirSync, readFileSync, writeFileSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { spawnSync } from "node:child_process"; import test from "node:test"; -const pipeline = readFileSync( - new URL("../.buildkite/pipeline.yml", import.meta.url), +const workflow = readFileSync( + new URL("../.github/workflows/maestro-ci.yml", import.meta.url), "utf8", ); -const advisory = readFileSync( - new URL("../.buildkite/advisory.yml", import.meta.url), +const nextest = readFileSync( + new URL("../.config/nextest.toml", import.meta.url), "utf8", ); +const tooling = readFileSync( + new URL("./run-ci-tooling.sh", import.meta.url), + "utf8", +); +const jetbrains = readFileSync( + new URL("./run-ci-jetbrains.sh", import.meta.url), + "utf8", +); +const coverage = readFileSync( + new URL("./run-ci-coverage.sh", import.meta.url), + "utf8", +); +const a2aTmuxSmoke = readFileSync( + new URL("./smoke-maestro-a2a-tmux.sh", import.meta.url), + "utf8", +); + +test("maestro-ci concurrency is per ref and cancels in-progress PR runs", () => { + assert.match( + workflow, + /group: maestro-ci-\$\{\{ github\.event\.pull_request\.number \|\| github\.ref \}\}/, + ); + assert.match( + workflow, + /cancel-in-progress: \$\{\{ github\.event_name == 'pull_request' \}\}/, + ); +}); + +test("maestro-ci uses owned runner labels through vars indirection", () => { + assert.match( + workflow, + /vars\.MAESTRO_CI_HEAVY_RUNNER \|\| 'evalops-internal-arc'/, + ); + assert.match( + workflow, + /vars\.MAESTRO_CI_LIGHT_RUNNER \|\| 'evalops-internal-arc-light'/, + ); + assert.doesNotMatch(workflow, /runs-on:\s*ubuntu-/); + assert.doesNotMatch(workflow, /runs-on:\s*macos-/); + assert.doesNotMatch(workflow, /runs-on:\s*windows-/); + assert.doesNotMatch(workflow, /hetzner-linux-heavy/); +}); + +test("heavy Rust and Docker jobs use the heavy owned runner", () => { + for (const job of [ + "lint", + "rust-tests", + "native-release", + "integration", + "scenario-replay", + "linux-check", + "coverage", + "perf-baseline", + ]) { + const block = workflow.split(` ${job}:`)[1]?.split(/\n [a-z]/)[0] ?? ""; + assert.match( + block, + /vars\.MAESTRO_CI_HEAVY_RUNNER \|\| 'evalops-internal-arc'/, + `${job} must run on the heavy owned runner`, + ); + } +}); + +test("light checks use the light owned runner", () => { + for (const job of [ + "protocol-contracts", + "ci-contracts", + "workflow-tooling", + "supply-chain", + "jetbrains-plugin", + "hosted-orb-delegation-live", + "maestro-ci", + ]) { + const block = workflow.split(` ${job}:`)[1]?.split(/\n [a-z]/)[0] ?? ""; + assert.match( + block, + /vars\.MAESTRO_CI_LIGHT_RUNNER \|\| 'evalops-internal-arc-light'/, + `${job} must run on the light owned runner`, + ); + } +}); + +const trustedHead = + /github\.event_name != 'pull_request' \|\| github\.event\.pull_request\.head\.repo\.full_name == github\.repository/; + +test("advisory coverage and perf are scheduled, not required on push or PR", () => { + assert.match( + workflow, + /coverage:[\s\S]*?if: \(github\.event_name != 'pull_request' \|\| github\.event\.pull_request\.head\.repo\.full_name == github\.repository\) && \(github\.event_name == 'schedule' \|\| github\.event_name == 'workflow_dispatch'\)/, + ); + assert.match( + workflow, + /perf-baseline:[\s\S]*?if: \(github\.event_name != 'pull_request' \|\| github\.event\.pull_request\.head\.repo\.full_name == github\.repository\) && \(github\.event_name == 'schedule' \|\| github\.event_name == 'workflow_dispatch'\)/, + ); + assert.match(workflow, /coverage:[\s\S]*?continue-on-error: true/); + assert.match(workflow, /perf-baseline:[\s\S]*?continue-on-error: true/); +}); + +test("fork pull requests never start jobs on internal runners", () => { + for (const job of [ + "protocol-contracts", + "lint", + "rust-tests", + "native-release", + "integration", + "scenario-replay", + "ci-contracts", + "workflow-tooling", + "supply-chain", + "jetbrains-plugin", + "linux-check", + "coverage", + "perf-baseline", + "hosted-orb-delegation-live", + "maestro-ci", + ]) { + const block = workflow.split(` ${job}:`)[1]?.split(/\n [a-z]/)[0] ?? ""; + assert.match(block, trustedHead, `${job} must skip fork pull requests`); + } + assert.doesNotMatch( + workflow, + /^ if: github\.event_name == 'schedule' \|\| github\.event_name == 'workflow_dispatch'$/m, + "advisory jobs must keep the trusted-head conjunct", + ); +}); + +test("maestro-ci covers every migrated validation family", () => { + for (const job of [ + "protocol-contracts", + "lint", + "rust-tests", + "native-release", + "integration", + "scenario-replay", + "ci-contracts", + "workflow-tooling", + "supply-chain", + "jetbrains-plugin", + "linux-check", + "coverage", + "perf-baseline", + "hosted-orb-delegation-live", + "maestro-ci", + ]) { + assert.match(workflow, new RegExp(`^ ${job}:`, "m"), `missing job ${job}`); + } + assert.match(workflow, /bash scripts\/run-ci-tooling\.sh/); + assert.match(workflow, /bash scripts\/run-ci-supply-chain\.sh/); + assert.match(workflow, /bash scripts\/run-ci-jetbrains\.sh/); + assert.match(workflow, /bash scripts\/run-ci-coverage\.sh/); + assert.match(workflow, /bash scripts\/run-ci-perf\.sh/); + assert.match(workflow, /bash scripts\/ci-linux-check\.sh/); + assert.match(workflow, /scripts\/run-nextest-partition\.sh/); + assert.match(workflow, /actions\/setup-java@de7274f081f381c8f8158605e0321c36c376e2e6/); + assert.match(workflow, /use-sccache: "true"/); + assert.match( + workflow, + /vars\.MAESTRO_HOSTED_ORB_LIVE_SMOKE \|\| ''/, + ); + const hostedOrb = workflow.split(" hosted-orb-delegation-live:")[1]?.split(/\n [a-z]/)[0] ?? ""; + assert.match( + hostedOrb, + /continue-on-error:\s*\$\{\{\s*!\(github\.event\.inputs\.hosted_orb_live_smoke == '1' \|\| vars\.MAESTRO_HOSTED_ORB_LIVE_SMOKE == '1'\)\s*\}\}/, + ); + assert.doesNotMatch(hostedOrb, /^ continue-on-error:\s*true\b/m); +}); + +test("supply-chain can read the pull request and issue timeline for deny.toml checks", () => { + const workflowPermissions = workflow.split("\njobs:")[0] ?? ""; + assert.match(workflowPermissions, /^permissions:\n contents: read$/m); + assert.doesNotMatch(workflowPermissions, /pull-requests:/); + assert.doesNotMatch(workflowPermissions, /issues:/); + const block = workflow.split(" supply-chain:")[1]?.split(/\n [a-z]/)[0] ?? ""; + assert.match(block, /^ permissions:\n contents: read\n pull-requests: read\n issues: read$/m); + assert.match(block, /GH_TOKEN: \$\{\{ github\.token \}\}/); + assert.match(block, /bash scripts\/run-ci-supply-chain\.sh/); +}); -test("CI uses one configurable Buildkite concurrency group", () => { - assert.match(pipeline, /MAESTRO_CI_CONCURRENCY_GROUP:-maestro-heavy-workloads-v6/); +test("terminal maestro-ci job fails unless needed jobs succeeded or skipped", () => { + assert.match( + workflow, + /^ maestro-ci:\n name: maestro-ci\n if: always\(\) && \(github\.event_name != 'pull_request' \|\| github\.event\.pull_request\.head\.repo\.full_name == github\.repository\)/m, + ); + assert.match(workflow, /result == "skipped"/); + assert.match(workflow, /maestro-ci failed for/); + const terminal = workflow.split(" maestro-ci:")[1] ?? ""; + assert.match(terminal, /for name, body in sorted\(needs\.items\(\)\):/); + assert.doesNotMatch( + terminal, + /if name in \(|hosted-orb-delegation-live.*continue-on-error|result == "failure"/, + ); +}); + +test("nextest runs as a four-way hash partition matrix", () => { + assert.match(workflow, /matrix:\n partition: \[1, 2, 3, 4\]/); + assert.match( + workflow, + /scripts\/run-nextest-partition\.sh "\$\{NEXTEST_PARTITION\}\/4"/, + ); + assert.match(nextest, /\[profile\.ci\]/); + assert.match(nextest, /filter = 'binary\(pty_e2e\)'/); + assert.match(nextest, /test-group = 'pty-e2e'/); + assert.match(nextest, /\[test-groups\.pty-e2e\]\nmax-threads = 1/); +}); + +test("CI caps rustc codegen units and compile jobs", () => { + assert.match(workflow, /CARGO_PROFILE_DEV_CODEGEN_UNITS: "16"/); + assert.match(workflow, /CARGO_PROFILE_TEST_CODEGEN_UNITS: "16"/); + assert.match(workflow, /CARGO_BUILD_JOBS: "4"/); + assert.match(workflow, /RUST_TOOLCHAIN: "1\.95\.0"/); +}); + +test("network and long-running operations are bounded", () => { + assert.equal((workflow.match(/5m npm ci --ignore-scripts/g) ?? []).length, 7); + assert.match(workflow, /npm_config_fetch_retries: "1"/); + assert.match(workflow, /npm_config_fetch_timeout: "30000"/); + assert.equal((workflow.match(/2m docker pull/g) ?? []).length, 2); + for (const command of [ + "45m npm run check", + "30m npm run lint", + "20m cargo test --workspace --locked --doc", + "30m cargo build --locked -p maestro-tui", + "45m npm run build", + "30m cargo test --locked -p maestro-runtime-gateway", + "30m cargo test --locked -p maestro-tui --test tools_integration", + "30m cargo build --locked -p maestro-scenario", + ]) { + assert.match(workflow, new RegExp(command.replaceAll("-", "\\-"))); + } +}); + +test("protocol lock fails the build before heavy jobs start", () => { + assert.match(workflow, /^ protocol-contracts:/m); + assert.match(workflow, /npm run check:protocol-manifest/); + assert.equal( + (workflow.match(/needs: protocol-contracts/g) ?? []).length, + 10, + "heavy jobs must wait for the protocol lock", + ); +}); + +test("rust-tests explicitly proves CI machine auth fails closed without privileged tokens", () => { + assert.match( + workflow, + /cargo test --locked -p maestro-local-host ci_auth_conformance/, + ); + assert.doesNotMatch(workflow, /ACTIONS_ID_TOKEN_REQUEST_TOKEN/); +}); + +test("workflow tooling installs pinned binaries without requiring Go", () => { + assert.doesNotMatch(tooling, /go install/); + assert.match(tooling, /actionlint_1\.7\.9_\$\{actionlint_platform\}\.tar\.gz/); + assert.match(tooling, /233b280d05e100837f4af1433c7b40a5dcb306e3aa68fb4f17f8a7f45a7df7b4/); + assert.match(tooling, /sha256sum --check --status/); + assert.match(tooling, /shasum -a 256 --check --status/); + assert.match(tooling, /! -f \.github\/workflows\/sync-public-release-mirror\.yml/); +}); + +test("JetBrains validation bounds Gradle workers and heap", () => { + assert.match(jetbrains, /10m \\\n\s+\.\/gradlew check buildPlugin --no-daemon \\/); + assert.match(jetbrains, /org\.gradle\.workers\.max=1/); + assert.match( + jetbrains, + /org\.gradle\.jvmargs="-Xmx1g -XX:MaxMetaspaceSize=512m -XX:\+ExitOnOutOfMemoryError"/, + ); }); -test("CI bounds each Buildkite lane to available worker capacity", () => { - assert.equal((pipeline.match(/concurrency: 3/g) ?? []).length, 9); +test("advisory coverage uses nextest and an isolated instrumented target dir", () => { + assert.match(coverage, /CARGO_TARGET_DIR="\$\{cache_root\}\/cargo-target-cov"/); + assert.match(coverage, /cargo llvm-cov nextest/); + assert.match(coverage, /--lib/); + assert.match(coverage, /--no-clean/); + assert.doesNotMatch(coverage, /llvm-cov nextest[\s\S]*--no-report/); + assert.match(coverage, /--ignore-run-fail/); + assert.match(coverage, /--profile ci/); + assert.match(coverage, /cargo-llvm-cov-x86_64-unknown-linux-gnu\.tar\.gz/); + assert.match( + coverage, + /b068f7c98841aacb9c4f382b4a0c184ae82f49b56a32d442b429b2961c73be15/, + ); + assert.doesNotMatch(coverage, /cargo install cargo-llvm-cov/); + assert.doesNotMatch(coverage, /--html/); +}); + +test("internal-only contracts are conditional in the shared public pipeline", () => { + assert.match( + workflow, + /if \[\[ -d test\/internal \]\]; then\n\s+npm run test:internal\n\s+fi/, + ); + assert.match( + workflow, + /if \[\[ -f scripts\/measure-ci-build-latency\.test\.mjs \]\]; then/, + ); }); -test("advisory coverage and perf use a separate one-slot pool", () => { - assert.doesNotMatch(pipeline, /MAESTRO_CI_ADVISORY_CONCURRENCY_GROUP/); - assert.equal((pipeline.match(/^\s*concurrency: 1$/gmu) ?? []).length, 0); +test("public projections skip the internal mirror workflow contract", () => { assert.match( - advisory, - /MAESTRO_CI_ADVISORY_CONCURRENCY_GROUP:-maestro-advisory-workloads/, + tooling, + /check-sync-public-release-mirror-workflow\.test\.mjs[\s\S]*?\.github\/workflows\/sync-public-release-mirror\.yml/, ); - assert.equal((advisory.match(/^\s*concurrency: 1$/gmu) ?? []).length, 2); +}); + +test("A2A tmux smoke atomically reserves its session before resetting durable task databases", () => { + const cleanup = a2aTmuxSmoke.split("cleanup() {")[1]?.split("a2a_cli() {")[0] ?? ""; + const startup = a2aTmuxSmoke.split('cd "$ROOT_DIR"')[1] ?? ""; + const reservation = startup.indexOf( + 'tmux new-session -d -s "$SESSION_NAME" -n peer-a "sleep 300"', + ); + const resetStart = startup.indexOf("rm -f"); + const respawn = startup.indexOf('tmux respawn-window -k -t "$SESSION_NAME:peer-a"'); + assert.match(cleanup, /if \[\[ "\$OWNS_SESSION" == "1" \]\]; then/); + assert.match(startup, /if tmux new-session[^\n]+; then\n\tOWNS_SESSION=1/); + assert.ok(reservation >= 0, "smoke must atomically reserve the tmux session"); + assert.ok(resetStart > reservation, "state reset must follow session reservation"); + assert.ok(respawn > resetStart, "peer A must start after state reset"); + assert.doesNotMatch(startup, /tmux has-session/); + const reset = startup.slice(resetStart, respawn); + for (const tasks of ["TASKS_A", "TASKS_B"]) { + for (const suffix of ["", ".sqlite3", ".sqlite3-wal", ".sqlite3-shm"]) { + assert.ok(reset.includes(`"$${tasks}${suffix}"`), `reset must remove $${tasks}${suffix}`); + } + } +}); + +test("legacy GitHub validation workflows are absent", () => { + const names = [ + "actionlint.yml", + "ci.yml", + "coverage.yml", + "evals.yml", + "hooks.yml", + "integration.yml", + "jetbrains-plugin.yml", + "perf-baselines.yml", + "required-checks-invariant.yml", + "scenario-replay.yml", + "shellcheck.yml", + "supply-chain.yml", + ]; + for (const name of names) { + try { + readFileSync(new URL(`../.github/workflows/${name}`, import.meta.url)); + assert.fail(`legacy workflow ${name} must not exist`); + } catch (error) { + assert.equal(error.code, "ENOENT"); + } + } +}); + +const runner = fileURLToPath(new URL("./run-ci-jetbrains.sh", import.meta.url)); +const hasScript = spawnSync("bash", ["-lc", "command -v script"], { + encoding: "utf8", +}).status === 0; + +test( + "JetBrains CI detaches Gradle stdin from the controlling terminal", + { skip: process.platform !== "linux" || !hasScript }, + () => { + const root = mkdtempSync(join(tmpdir(), "maestro-gradle-stdin-")); + try { + const bin = join(root, "bin"); + const plugin = join(root, "packages/jetbrains-plugin"); + mkdirSync(bin); + mkdirSync(plugin, { recursive: true }); + writeFileSync( + join(bin, "java"), + '#!/bin/sh\necho \'openjdk version "21.0.12"\' >&2\n', + { mode: 0o755 }, + ); + writeFileSync( + join(bin, "timeout"), + `#!/bin/bash +args=("$@") +for i in "\${!args[@]}"; do + [[ "\${args[$i]}" != 10m ]] || args[$i]=1s + [[ "\${args[$i]}" != --kill-after=30s ]] || args[$i]=--kill-after=1s +done +exec /usr/bin/timeout "\${args[@]}" +`, + { mode: 0o755 }, + ); + writeFileSync( + join(plugin, "gradlew"), + '#!/bin/sh\nexec "$TEST_NODE" "$TEST_READER"\n', + { mode: 0o755 }, + ); + const reader = join(root, "read-stdin.cjs"); + writeFileSync( + reader, + `const fs = require('node:fs'); +fs.readSync(0, Buffer.alloc(1), 0, 1, null); +console.log('gradle stdin reached EOF'); +`, + ); + const result = spawnSync( + "script", + ["-q", "-e", "-c", `bash '${runner.replaceAll("'", "'\\''")}'`, "/dev/null"], + { + cwd: root, + env: { + ...process.env, + PATH: `${bin}:${process.env.PATH}`, + TEST_NODE: process.execPath, + TEST_READER: reader, + MAESTRO_CI_CACHE_ROOT: root, + }, + input: "", + encoding: "utf8", + timeout: 5000, + }, + ); + assert.ifError(result.error); + assert.equal(result.status, 0, result.stdout + result.stderr); + assert.match(result.stdout, /gradle stdin reached EOF/); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }, +); + +test("tool bootstrap rejects an installed but unusable actionlint shim", () => { + const directory = mkdtempSync(join(tmpdir(), "maestro-tool-shim-")); + try { + const bin = join(directory, "bin"); + mkdirSync(bin); + writeFileSync( + join(bin, "curl"), + "#!/bin/sh\necho bootstrap-download-requested\nexit 77\n", + { mode: 0o755 }, + ); + const bootstrap = tooling.slice(0, tooling.indexOf("if ! shellcheck --version")); + assert.ok(bootstrap.includes("if ! actionlint --version")); + for (const usable of [false, true]) { + writeFileSync( + join(bin, "actionlint"), + `#!/bin/sh\n[ "$1" = "--version" ] || exit 90\nexit ${usable ? 0 : 1}\n`, + { mode: 0o755 }, + ); + const result = spawnSync("bash", ["-c", bootstrap], { + cwd: directory, + env: { + ...process.env, + MAESTRO_CI_CACHE_ROOT: directory, + PATH: `${bin}:${process.env.PATH}`, + }, + encoding: "utf8", + }); + assert.equal(result.status, usable ? 0 : 77, result.stderr); + assert.equal(result.stdout.includes("bootstrap-download-requested"), !usable); + } + } finally { + rmSync(directory, { recursive: true, force: true }); + } }); diff --git a/scripts/check-integration-required-gate.test.mjs b/scripts/check-integration-required-gate.test.mjs index 1aa0300b6..7bd74893f 100644 --- a/scripts/check-integration-required-gate.test.mjs +++ b/scripts/check-integration-required-gate.test.mjs @@ -93,12 +93,15 @@ test("requires the integration suite to succeed for relevant paths", () => { ); }); -test("Buildkite retains the complete integration suite", () => { - const pipeline = readFileSync(new URL("../.buildkite/pipeline.yml", import.meta.url), "utf8"); - assert.match(pipeline, /key: "integration"/); - assert.match(pipeline, /cargo test --locked -p maestro-runtime-gateway/); - assert.match(pipeline, /cargo test --locked -p maestro-tui --test tools_integration/); - assert.match(pipeline, /trap 'docker rm -f "\$\$redis" "\$\$postgres"/); +test("maestro-ci retains the complete integration suite", () => { + const workflow = readFileSync( + new URL("../.github/workflows/maestro-ci.yml", import.meta.url), + "utf8", + ); + assert.match(workflow, /^ integration:/m); + assert.match(workflow, /cargo test --locked -p maestro-runtime-gateway/); + assert.match(workflow, /cargo test --locked -p maestro-tui --test tools_integration/); + assert.match(workflow, /trap 'docker rm -f "\$redis" "\$postgres"/); }); test("accepts only one structurally valid approved setup-node step", () => { diff --git a/scripts/check-new-deps-supply-chain.mjs b/scripts/check-new-deps-supply-chain.mjs index 04c79b49e..ab9db1659 100644 --- a/scripts/check-new-deps-supply-chain.mjs +++ b/scripts/check-new-deps-supply-chain.mjs @@ -8,7 +8,7 @@ * fresh RUSTSEC advisory landing against a dependency this PR never touched * should not spuriously red an unrelated PR — that's the same "gate that * fails for the wrong reason" problem this repo is trying to get away from. - * The Buildkite supply-chain lane (`scripts/run-buildkite-supply-chain.sh`) + * The CI supply-chain lane (`scripts/run-ci-supply-chain.sh`) * is where advisories against the existing tree get enforced unconditionally. * * This script narrows PR-time enforcement to findings whose *own* flagged @@ -287,7 +287,7 @@ function main() { if (preexisting > 0) { console.log( `${preexisting} finding(s) are against dependencies this PR did not add or change — not failing this PR lane. ` + - "They are enforced by the full Buildkite supply-chain check and visible in this job's output.", + "They are enforced by the full CI supply-chain check and visible in this job's output.", ); } diff --git a/scripts/check-new-deps-supply-chain.test.mjs b/scripts/check-new-deps-supply-chain.test.mjs index 4653f99b3..b7e1273dc 100644 --- a/scripts/check-new-deps-supply-chain.test.mjs +++ b/scripts/check-new-deps-supply-chain.test.mjs @@ -15,7 +15,7 @@ import { } from "./check-new-deps-supply-chain.mjs"; const SUPPLY_CHAIN_SCRIPT_URL = new URL( - "./run-buildkite-supply-chain.sh", + "./run-ci-supply-chain.sh", import.meta.url, ); const SUPPLY_CHAIN_SCRIPT = existsSync(SUPPLY_CHAIN_SCRIPT_URL) @@ -41,14 +41,14 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "deadbeef" `; -test("Buildkite supply-chain validation is strict and shared with the public projection", () => { - assert.ok(SUPPLY_CHAIN_SCRIPT, "Buildkite supply-chain script must exist"); +test("CI supply-chain validation is strict and shared with the public projection", () => { + assert.ok(SUPPLY_CHAIN_SCRIPT, "CI supply-chain script must exist"); assert.match(SUPPLY_CHAIN_SCRIPT, /cargo deny fetch db/); assert.match(SUPPLY_CHAIN_SCRIPT, /cargo deny check --disable-fetch/); assert.match(SUPPLY_CHAIN_SCRIPT, /check-new-deps-supply-chain\.test\.mjs/); - assert.match(SUPPLY_CHAIN_SCRIPT, /BUILDKITE_PULL_REQUEST_BASE_BRANCH/); + assert.match(SUPPLY_CHAIN_SCRIPT, /MAESTRO_CI_BASE_BRANCH/); assert.match(SUPPLY_CHAIN_SCRIPT, /supply-chain-policy-approved/); - assert.match(SUPPLY_CHAIN_SCRIPT, /pr\.head\?\.sha !== process\.env\.BUILDKITE_COMMIT/); + assert.match(SUPPLY_CHAIN_SCRIPT, /pr\.head\?\.sha !== process\.env\.MAESTRO_CI_COMMIT/); assert.match(SUPPLY_CHAIN_SCRIPT, /event\.event === "committed"/); assert.match(SUPPLY_CHAIN_SCRIPT, /approvalIndex <= headCommitIndex/); assert.match(SUPPLY_CHAIN_SCRIPT, /gh api --paginate --slurp/); diff --git a/scripts/ci-linux-check.sh b/scripts/ci-linux-check.sh old mode 100644 new mode 100755 index 79584b92b..e7962406c --- a/scripts/ci-linux-check.sh +++ b/scripts/ci-linux-check.sh @@ -13,7 +13,8 @@ fi script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)" maestro_root="$(cd "${script_dir}/.." && pwd -P)" -tool_root="${BUILDKITE_BUILD_CHECKOUT_PATH:-${maestro_root}}/.buildkite/cache/ci-tools" +cache_root="${MAESTRO_CI_CACHE_ROOT:-${RUNNER_TEMP:-${maestro_root}/.cache}/maestro-ci}" +tool_root="${cache_root}/ci-tools" mkdir -p "${tool_root}/bin" export PATH="${tool_root}/bin:${PATH}" diff --git a/scripts/generate-protocol-compatibility-manifest.test.mjs b/scripts/generate-protocol-compatibility-manifest.test.mjs index 6880cd72b..8d3c678e3 100644 --- a/scripts/generate-protocol-compatibility-manifest.test.mjs +++ b/scripts/generate-protocol-compatibility-manifest.test.mjs @@ -759,11 +759,14 @@ test("cosign DSSE verification selects the exact decoded predicate", () => { }); test("required Rust validation rejects compatibility manifest drift", () => { - const pipeline = readFileSync(resolve(ROOT, ".buildkite/pipeline.yml"), "utf8"); - assert.match(pipeline, /npm run check/); - assert.match(pipeline, /key: "protocol-contracts"/); - assert.match(pipeline, /npm run check:protocol-manifest/); - assert.match(pipeline, /depends_on: "protocol-contracts"/); + const workflow = readFileSync( + resolve(ROOT, ".github/workflows/maestro-ci.yml"), + "utf8", + ); + assert.match(workflow, /npm run check/); + assert.match(workflow, /^ protocol-contracts:/m); + assert.match(workflow, /npm run check:protocol-manifest/); + assert.match(workflow, /needs: protocol-contracts/); const packageJson = JSON.parse(readFileSync(resolve(ROOT, "package.json"), "utf8")); assert.match(packageJson.scripts.check, /check:protocol-manifest/); }); diff --git a/scripts/hosted-orb-delegation-acceptance.test.mjs b/scripts/hosted-orb-delegation-acceptance.test.mjs index 85b2c8081..ada78db76 100644 --- a/scripts/hosted-orb-delegation-acceptance.test.mjs +++ b/scripts/hosted-orb-delegation-acceptance.test.mjs @@ -57,3 +57,25 @@ test("the public validator rejects live-gate metadata", async () => { contract.live_gate = { enabled: true }; assert.throws(() => validateContract(contract), /live-gate inventory/); }); + +test("the managed hosted acceptance is a hard gate when explicitly configured", async () => { + const workflow = await readFile( + new URL("../.github/workflows/maestro-ci.yml", import.meta.url), + "utf8", + ); + const block = + workflow.split(" hosted-orb-delegation-live:")[1]?.split(/\n [a-z]/)[0] ?? + ""; + assert.match(block, /Optional hosted Orb acceptance was not requested; skipping\./); + assert.match(block, /npm run smoke:hosted-orb-delegation/); + assert.match( + block, + /if: \(github\.event_name != 'pull_request' \|\| github\.event\.pull_request\.head\.repo\.full_name == github\.repository\) && \(github\.event_name == 'schedule' \|\| github\.event_name == 'workflow_dispatch'\)/, + ); + assert.match(workflow, /vars\.MAESTRO_HOSTED_ORB_LIVE_SMOKE \|\| ''/); + assert.match( + block, + /continue-on-error:\s*\$\{\{\s*!\(github\.event\.inputs\.hosted_orb_live_smoke == '1' \|\| vars\.MAESTRO_HOSTED_ORB_LIVE_SMOKE == '1'\)\s*\}\}/, + ); + assert.doesNotMatch(block, /^ continue-on-error:\s*true\b/m); +}); diff --git a/scripts/run-buildkite-coverage.sh b/scripts/run-ci-coverage.sh similarity index 85% rename from scripts/run-buildkite-coverage.sh rename to scripts/run-ci-coverage.sh index 1cafd5680..c5dcb8537 100755 --- a/scripts/run-buildkite-coverage.sh +++ b/scripts/run-ci-coverage.sh @@ -12,11 +12,12 @@ set -euo pipefail # Keep this job on an isolated target dir. Sharing CARGO_TARGET_DIR with # lint/test mixes instrumented and plain incremental artifacts and forces a # full rebuild. -repo_root="${BUILDKITE_BUILD_CHECKOUT_PATH:-$(pwd)}" -tool_root="${repo_root}/.buildkite/cache/cargo-tools" +repo_root="$(pwd)" +cache_root="${MAESTRO_CI_CACHE_ROOT:-${RUNNER_TEMP:-${repo_root}/.cache}/maestro-ci}" +tool_root="${cache_root}/cargo-tools" mkdir -p "$tool_root/bin" export PATH="$tool_root/bin:$PATH" -export CARGO_TARGET_DIR="${repo_root}/.buildkite/cache/cargo-target-cov" +export CARGO_TARGET_DIR="${cache_root}/cargo-target-cov" install_github_crate_bin() { local name="$1" @@ -52,9 +53,9 @@ fi # --lib: one instrumented test harness per crate. No tests/*.rs, no bins, # no doctests. nextest for the same parallelism as rust-tests. -# cargo-llvm-cov 0.9.0 rejects combining a deferred report with --no-clean -# (Buildkite 353). Keep --no-clean so the isolated target dir stays -# incremental; generate the lcov/summary reports in the commands below. +# cargo-llvm-cov 0.9.0 rejects combining a deferred report with --no-clean. +# Keep --no-clean so the isolated target dir stays incremental; generate the +# lcov/summary reports in the commands below. timeout --signal=TERM --kill-after=30s 25m cargo llvm-cov nextest \ --workspace \ --lib \ @@ -62,10 +63,9 @@ timeout --signal=TERM --kill-after=30s 25m cargo llvm-cov nextest \ --no-clean \ --ignore-run-fail \ -- \ - --profile buildkite \ + --profile ci \ --no-fail-fast cargo llvm-cov report --summary-only mkdir -p coverage-report cargo llvm-cov report --lcov --output-path coverage-report/lcov.info -buildkite-agent artifact upload 'coverage-report/**/*' diff --git a/scripts/run-buildkite-jetbrains.sh b/scripts/run-ci-jetbrains.sh similarity index 91% rename from scripts/run-buildkite-jetbrains.sh rename to scripts/run-ci-jetbrains.sh index 185c94d51..c5871c44f 100755 --- a/scripts/run-buildkite-jetbrains.sh +++ b/scripts/run-ci-jetbrains.sh @@ -1,7 +1,8 @@ #!/usr/bin/env bash set -euo pipefail -tool_root="${BUILDKITE_BUILD_CHECKOUT_PATH:-$(pwd)}/.buildkite/cache/jetbrains-tools" +cache_root="${MAESTRO_CI_CACHE_ROOT:-${RUNNER_TEMP:-$(pwd)/.cache}/maestro-ci}" +tool_root="${cache_root}/jetbrains-tools" jdk_root="$tool_root/jdk-21" if ! java -version 2>&1 | head -1 | grep -Eq 'version "21([.]|\")'; then case "$(uname -s)-$(uname -m)" in @@ -33,7 +34,7 @@ java -version cd packages/jetbrains-plugin # Bound the Gradle JVM. Empty jvmargs let HotSpot pick a huge ergonomic # heap on the heavy workers, and the host OOM-killer then SIGKILLs the -# job (Buildkite 114/351, exit 137) mid-:test. Cold Kotlin/IntelliJ builds +# job (CI 114/351, exit 137) mid-:test. Cold Kotlin/IntelliJ builds # exhausted the former 256 MiB metaspace cap (public build 332); allow # 512 MiB while retaining the 1 GiB heap and single worker. Keep the 10m timeout so a # stuck IntelliJ download still dies cleanly. Detach stdin: timeout puts Gradle diff --git a/scripts/run-buildkite-perf.sh b/scripts/run-ci-perf.sh similarity index 100% rename from scripts/run-buildkite-perf.sh rename to scripts/run-ci-perf.sh diff --git a/scripts/run-buildkite-supply-chain.sh b/scripts/run-ci-supply-chain.sh similarity index 92% rename from scripts/run-buildkite-supply-chain.sh rename to scripts/run-ci-supply-chain.sh index df9d022d9..58a907149 100755 --- a/scripts/run-buildkite-supply-chain.sh +++ b/scripts/run-ci-supply-chain.sh @@ -11,7 +11,8 @@ changed_inputs="$supply_chain_tmp/changed-dependency-inputs" deny_report="$supply_chain_tmp/deny-report.jsonl" base_deny_report="$supply_chain_tmp/base-deny-report.jsonl" -tool_root="${BUILDKITE_BUILD_CHECKOUT_PATH:-$(pwd)}/.buildkite/cache/cargo-tools" +cache_root="${MAESTRO_CI_CACHE_ROOT:-${RUNNER_TEMP:-$(pwd)/.cache}/maestro-ci}" +tool_root="${cache_root}/cargo-tools" mkdir -p "$tool_root" export CARGO_INSTALL_ROOT="$tool_root" export PATH="$tool_root/bin:$PATH" @@ -23,13 +24,14 @@ timeout --signal=TERM --kill-after=10s 2m cargo deny fetch db node --test scripts/check-new-deps-supply-chain.test.mjs scripts/check-advisory-expiry.test.mjs node scripts/check-advisory-expiry.mjs -pull_request="${BUILDKITE_PULL_REQUEST:-false}" +pull_request="${MAESTRO_CI_PULL_REQUEST:-false}" if [[ "$pull_request" == "false" || -z "$pull_request" ]]; then timeout --signal=TERM --kill-after=30s 20m cargo deny check --disable-fetch exit 0 fi -base_branch="${BUILDKITE_PULL_REQUEST_BASE_BRANCH:-main}" +base_branch="${MAESTRO_CI_BASE_BRANCH:-${GITHUB_BASE_REF:-main}}" +export MAESTRO_CI_COMMIT="${MAESTRO_CI_COMMIT:-${GITHUB_SHA:-}}" timeout --signal=TERM --kill-after=10s 2m git fetch --no-tags origin "+refs/heads/$base_branch:refs/remotes/origin/$base_branch" base_sha="$(git merge-base HEAD "origin/$base_branch")" git show "$base_sha:Cargo.lock" > "$base_lockfile" @@ -40,9 +42,9 @@ test -s "$base_deny" policy_changed=false if ! git diff --quiet "$base_sha" HEAD -- deny.toml; then policy_changed=true - repo_slug="$(printf '%s' "${BUILDKITE_REPO:-}" | sed -E 's#(git@github.com:|https://github.com/)##; s#\.git$##')" + repo_slug="${GITHUB_REPOSITORY:-}" [[ "$repo_slug" == */* ]] || { - echo "could not derive GitHub repository from BUILDKITE_REPO" >&2 + echo "could not derive GitHub repository from GITHUB_REPOSITORY" >&2 exit 1 } if [[ -n "${GH_TOKEN:-}" ]]; then @@ -69,7 +71,7 @@ if ! git diff --quiet "$base_sha" HEAD -- deny.toml; then --location -H "Accept: application/vnd.github+json" -H "X-GitHub-Api-Version: 2022-11-28" - -H "User-Agent: evalops-maestro-buildkite" + -H "User-Agent: evalops-maestro-ci" ) timeout --signal=TERM --kill-after=10s 60s curl "${curl_args[@]}" \ "$github_api/pulls/$pull_request" > "$pr_json" @@ -119,15 +121,15 @@ NODE readFileSync(process.env.SUPPLY_CHAIN_TIMELINE_JSON, "utf8"), ).flat(); const label = "supply-chain-policy-approved"; - if (pr.head?.sha !== process.env.BUILDKITE_COMMIT) { - throw new Error("Buildkite commit is not the current pull-request head"); + if (pr.head?.sha !== process.env.MAESTRO_CI_COMMIT) { + throw new Error("CI commit is not the current pull-request head"); } if (!pr.labels?.some((entry) => entry.name === label)) { throw new Error(`${label} is not currently applied`); } const headCommitIndex = timeline.findLastIndex( (event) => - event.event === "committed" && event.sha === process.env.BUILDKITE_COMMIT, + event.event === "committed" && event.sha === process.env.MAESTRO_CI_COMMIT, ); const approvalIndex = timeline.findLastIndex( (event) => event.event === "labeled" && event.label?.name === label, diff --git a/scripts/run-buildkite-ci-tooling.sh b/scripts/run-ci-tooling.sh similarity index 97% rename from scripts/run-buildkite-ci-tooling.sh rename to scripts/run-ci-tooling.sh index cd6b7154b..07e28e0aa 100755 --- a/scripts/run-buildkite-ci-tooling.sh +++ b/scripts/run-ci-tooling.sh @@ -1,7 +1,8 @@ #!/usr/bin/env bash set -euo pipefail -tool_root="${BUILDKITE_BUILD_CHECKOUT_PATH:-$(pwd)}/.buildkite/cache/ci-tools" +cache_root="${MAESTRO_CI_CACHE_ROOT:-${RUNNER_TEMP:-$(pwd)/.cache}/maestro-ci}" +tool_root="${cache_root}/ci-tools" mkdir -p "$tool_root/bin" export PATH="$tool_root/bin:$PATH" diff --git a/scripts/run-nextest-partition.sh b/scripts/run-nextest-partition.sh index c1f844a5f..a0fcd216e 100755 --- a/scripts/run-nextest-partition.sh +++ b/scripts/run-nextest-partition.sh @@ -26,7 +26,7 @@ export MAESTRO_HOME="${maestro_test_home}" export MAESTRO_SUBAGENTS_DIR="${maestro_test_home}/subagents" cargo nextest run \ - --profile buildkite \ + --profile ci \ --workspace \ --locked \ --no-fail-fast \