diff --git a/.rwx/acceptance.yml b/.rwx/acceptance.yml new file mode 100644 index 00000000000..b7bacb3ec0a --- /dev/null +++ b/.rwx/acceptance.yml @@ -0,0 +1,322 @@ +# CircleCI's memory-all acceptance jobs: op-node and kona-node, on four shards +# each. CircleCI retains all required gates. +on: + cache-rebuild: + if: ${{ event.git.branch == 'develop' }} + target: [go, contracts, kona, op-reth, prestate, sp1-executor, acceptance-tools] + init: + commit-sha: ${{ event.git.sha }} + branch: ${{ event.git.branch }} + cache-warm: "true" + cache-epoch: v1 + build-probe: "" + target-cache-mode: keep + cli: + init: + commit-sha: ${{ event.git.sha }} + branch: ${{ event.git.branch }} + cache-warm: "false" + cache-epoch: v1 + build-probe: "" + target-cache-mode: keep + +# Shared cache baseline: only develop may write; PRs restore it read-only. +tool-cache: + vault: optimism-op-reth-shadow + +base: + image: ubuntu:24.04@sha256:008173c23f95b170204355c12626cb5a965d779a7e1283b09e9cffbb1bf33ca3 + config: rwx/base 1.2.0 + arch: x86_64 + +defaults: + runner: + cpus: 2 + memory: 8gb + +tasks: + - key: code + call: git/clone 2.2.0 + with: + repository: https://github.com/ethereum-optimism/optimism.git + ref: ${{ init.commit-sha }} + preserve-git-dir: true + fetch-full-depth: true + submodules: false + + - key: bootstrap-inputs + use: code + run: 'true' + filter: [mise.toml, ops/ci/runtime] + outputs: + filesystem: + filter: + workspace: [mise.toml, ops/ci/runtime/apt-install.sh, ops/ci/runtime/rwx-prepare.sh, ops/ci/runtime/rwx-contracts-prepare.sh, ops/ci/runtime/rwx-rust-prepare.sh, ops/ci/runtime/op-reth-shadow.sh, ops/ci/runtime/op-reth-report.py, ops/ci/runtime/rust-target-cache.py] + system: [] + + - key: mise + call: mise/install 1.1.0 + with: + mise-version: "2026.2.2" + install: "false" + + # The filtered bootstrap layer keeps the checkout's Git history out of + # reusable tools. Language toolchains are added only by their own workload. + - key: tools + use: [mise, bootstrap-inputs] + call: ${{ run.dir }}/packages/toolchain-common.yml + + - key: go-build-tools + use: tools + call: ${{ run.dir }}/packages/toolchain-go.yml + with: + mode: go + + - key: route + use: [code, tools] + run: bash ops/ci/runtime/rwx-metadata.sh + env: + CI_CACHE_WARM: ${{ init.cache-warm }} + CI_BRANCH: ${{ init.branch }} + CI_COMMIT_SHA: ${{ init.commit-sha }} + cache: false + timeout: 10m + outputs: + filesystem: false + + - key: contract-tools + use: [bootstrap-inputs, tools] + call: ${{ run.dir }}/packages/toolchain-foundry.yml + with: + mode: anvil + + - key: rust-tools + use: [bootstrap-inputs, tools] + call: ${{ run.dir }}/packages/toolchain-rust.yml + with: + mode: release + + - key: helper-tests + use: [code, go-build-tools] + if: ${{ init.cache-warm != 'true' && tasks.route.values.run-main == 'true' }} + run: | + mise exec -- python3 -m unittest discover -s ops/ci/tests -p 'test_acceptance*.py' + cache: false + timeout: 10m + outputs: + filesystem: false + + - key: source + use: [code, tools] + if: ${{ init.cache-warm == 'true' || tasks.route.values.run-main == 'true' }} + run: | + git submodule sync --recursive + git submodule update --init --recursive --jobs 8 + git rev-parse HEAD >packages/contracts-bedrock/.gitcommit + timeout: 20m + + # A dependency build checks out the commit that last changed its build inputs, so it + # is a cache hit for later commits and for the other workloads of the run. + - key: dependency-plan + use: code + run: python3 ops/ci/runtime/go-dependency-source.py plan + outputs: + filesystem: false + + - key: go + use: go-build-tools + if: ${{ init.cache-warm == 'true' || tasks.route.values.run-main == 'true' }} + call: ${{ run.dir }}/packages/go-go.yml + with: + commit-sha: ${{ tasks.dependency-plan.values.go }} + cache-epoch: ${{ init.cache-epoch }} + build-probe: ${{ init.build-probe }} + target-cache-mode: ${{ init.target-cache-mode }} + + - key: contracts + use: [go-build-tools, contract-tools] + if: ${{ init.cache-warm == 'true' || tasks.route.values.run-main == 'true' }} + call: ${{ run.dir }}/packages/go-contracts.yml + with: + commit-sha: ${{ tasks.dependency-plan.values.contracts }} + cache-epoch: ${{ init.cache-epoch }} + build-probe: ${{ init.build-probe }} + target-cache-mode: ${{ init.target-cache-mode }} + + - key: kona + use: rust-tools + if: ${{ init.cache-warm == 'true' || tasks.route.values.run-main == 'true' }} + call: ${{ run.dir }}/packages/go-kona.yml + with: + commit-sha: ${{ tasks.dependency-plan.values.kona }} + cache-epoch: ${{ init.cache-epoch }} + build-probe: ${{ init.build-probe }} + target-cache-mode: ${{ init.target-cache-mode }} + + - key: op-reth + use: rust-tools + if: ${{ init.cache-warm == 'true' || tasks.route.values.run-main == 'true' }} + call: ${{ run.dir }}/packages/go-op-reth.yml + with: + commit-sha: ${{ tasks.dependency-plan.values.op-reth }} + cache-epoch: ${{ init.cache-epoch }} + build-probe: ${{ init.build-probe }} + target-cache-mode: ${{ init.target-cache-mode }} + + - key: prestate + use: go-build-tools + if: ${{ init.cache-warm == 'true' || tasks.route.values.run-main == 'true' }} + call: ${{ run.dir }}/packages/go-prestate.yml + with: + commit-sha: ${{ tasks.dependency-plan.values.prestate }} + cache-epoch: ${{ init.cache-epoch }} + build-probe: ${{ init.build-probe }} + target-cache-mode: ${{ init.target-cache-mode }} + + - key: sp1-executor + use: rust-tools + if: ${{ init.cache-warm == 'true' || tasks.route.values.run-main == 'true' }} + call: ${{ run.dir }}/packages/go-sp1-executor.yml + with: + commit-sha: ${{ tasks.dependency-plan.values.sp1-executor }} + cache-epoch: ${{ init.cache-epoch }} + build-probe: ${{ init.build-probe }} + target-cache-mode: ${{ init.target-cache-mode }} + + - key: acceptance-tools + use: [go-build-tools, contract-tools] + filter: [mise.toml, ops/ci/runtime/apt-install.sh] + run: | + bash ops/ci/runtime/apt-install.sh --no-install-recommends eatmydata + mise install go:github.com/ethereum/go-ethereum/cmd/geth + printf '%s\n' "$(mise bin-paths | paste -sd: -):$PATH" >"$RWX_ENV/PATH" + timeout: 30m + + # Each verdict task is one node of CircleCI's memory-all-opn-op-reth-fusaka + # or memory-all-kona-op-reth-fusaka job. Shard indexes 0 to 3 cover every test. + - key: verdict-opn-0 + use: [source, acceptance-tools, go.build, contracts.build, kona.build, op-reth.build, prestate.build, sp1-executor.build] + if: ${{ init.cache-warm != 'true' && tasks.route.values.run-main == 'true' }} + call: ${{ run.dir }}/packages/acceptance-verdict.yml + with: + commit-sha: ${{ init.commit-sha }} + branch: ${{ init.branch }} + shard-index: '0' + cl-kind: op-node + cache-key: acceptance-runtime-opn-0-${{ init.cache-epoch }} + + - key: verdict-opn-1 + use: [source, acceptance-tools, go.build, contracts.build, kona.build, op-reth.build, prestate.build, sp1-executor.build] + if: ${{ init.cache-warm != 'true' && tasks.route.values.run-main == 'true' }} + call: ${{ run.dir }}/packages/acceptance-verdict.yml + with: + commit-sha: ${{ init.commit-sha }} + branch: ${{ init.branch }} + shard-index: '1' + cl-kind: op-node + cache-key: acceptance-runtime-opn-1-${{ init.cache-epoch }} + + - key: verdict-opn-2 + use: [source, acceptance-tools, go.build, contracts.build, kona.build, op-reth.build, prestate.build, sp1-executor.build] + if: ${{ init.cache-warm != 'true' && tasks.route.values.run-main == 'true' }} + call: ${{ run.dir }}/packages/acceptance-verdict.yml + with: + commit-sha: ${{ init.commit-sha }} + branch: ${{ init.branch }} + shard-index: '2' + cl-kind: op-node + cache-key: acceptance-runtime-opn-2-${{ init.cache-epoch }} + + - key: verdict-opn-3 + use: [source, acceptance-tools, go.build, contracts.build, kona.build, op-reth.build, prestate.build, sp1-executor.build] + if: ${{ init.cache-warm != 'true' && tasks.route.values.run-main == 'true' }} + call: ${{ run.dir }}/packages/acceptance-verdict.yml + with: + commit-sha: ${{ init.commit-sha }} + branch: ${{ init.branch }} + shard-index: '3' + cl-kind: op-node + cache-key: acceptance-runtime-opn-3-${{ init.cache-epoch }} + + - key: verdict-kona-0 + use: [source, acceptance-tools, go.build, contracts.build, kona.build, op-reth.build, prestate.build, sp1-executor.build] + if: ${{ init.cache-warm != 'true' && tasks.route.values.run-main == 'true' }} + call: ${{ run.dir }}/packages/acceptance-verdict.yml + with: + commit-sha: ${{ init.commit-sha }} + branch: ${{ init.branch }} + shard-index: '0' + cl-kind: kona-node + cache-key: acceptance-runtime-kona-0-${{ init.cache-epoch }} + + - key: verdict-kona-1 + use: [source, acceptance-tools, go.build, contracts.build, kona.build, op-reth.build, prestate.build, sp1-executor.build] + if: ${{ init.cache-warm != 'true' && tasks.route.values.run-main == 'true' }} + call: ${{ run.dir }}/packages/acceptance-verdict.yml + with: + commit-sha: ${{ init.commit-sha }} + branch: ${{ init.branch }} + shard-index: '1' + cl-kind: kona-node + cache-key: acceptance-runtime-kona-1-${{ init.cache-epoch }} + + - key: verdict-kona-2 + use: [source, acceptance-tools, go.build, contracts.build, kona.build, op-reth.build, prestate.build, sp1-executor.build] + if: ${{ init.cache-warm != 'true' && tasks.route.values.run-main == 'true' }} + call: ${{ run.dir }}/packages/acceptance-verdict.yml + with: + commit-sha: ${{ init.commit-sha }} + branch: ${{ init.branch }} + shard-index: '2' + cl-kind: kona-node + cache-key: acceptance-runtime-kona-2-${{ init.cache-epoch }} + + - key: verdict-kona-3 + use: [source, acceptance-tools, go.build, contracts.build, kona.build, op-reth.build, prestate.build, sp1-executor.build] + if: ${{ init.cache-warm != 'true' && tasks.route.values.run-main == 'true' }} + call: ${{ run.dir }}/packages/acceptance-verdict.yml + with: + commit-sha: ${{ init.commit-sha }} + branch: ${{ init.branch }} + shard-index: '3' + cl-kind: kona-node + cache-key: acceptance-runtime-kona-3-${{ init.cache-epoch }} + + - key: main-gate-receipt + use: [code, tools] + after: ${{ (verdict-kona-0.succeeded || verdict-kona-0.failed || verdict-kona-0.skipped) && (verdict-kona-1.succeeded || verdict-kona-1.failed || verdict-kona-1.skipped) && (verdict-kona-2.succeeded || verdict-kona-2.failed || verdict-kona-2.skipped) && (verdict-kona-3.succeeded || verdict-kona-3.failed || verdict-kona-3.skipped) && (verdict-opn-0.succeeded || verdict-opn-0.failed || verdict-opn-0.skipped) && (verdict-opn-1.succeeded || verdict-opn-1.failed || verdict-opn-1.skipped) && (verdict-opn-2.succeeded || verdict-opn-2.failed || verdict-opn-2.skipped) && (verdict-opn-3.succeeded || verdict-opn-3.failed || verdict-opn-3.skipped) }} + if: ${{ init.cache-warm != 'true' }} + run: python3 ops/ci/runtime/pr-gate.py receipt main-acceptance + env: + GROUP_SELECTED: ${{ tasks.route.values.run-main }} + TASK_VERDICT_KONA_0_SUCCEEDED: ${{ tasks.verdict-kona-0.succeeded }} + TASK_VERDICT_KONA_0_FAILED: ${{ tasks.verdict-kona-0.failed }} + TASK_VERDICT_KONA_0_SKIPPED: ${{ tasks.verdict-kona-0.skipped }} + TASK_VERDICT_KONA_1_SUCCEEDED: ${{ tasks.verdict-kona-1.succeeded }} + TASK_VERDICT_KONA_1_FAILED: ${{ tasks.verdict-kona-1.failed }} + TASK_VERDICT_KONA_1_SKIPPED: ${{ tasks.verdict-kona-1.skipped }} + TASK_VERDICT_KONA_2_SUCCEEDED: ${{ tasks.verdict-kona-2.succeeded }} + TASK_VERDICT_KONA_2_FAILED: ${{ tasks.verdict-kona-2.failed }} + TASK_VERDICT_KONA_2_SKIPPED: ${{ tasks.verdict-kona-2.skipped }} + TASK_VERDICT_KONA_3_SUCCEEDED: ${{ tasks.verdict-kona-3.succeeded }} + TASK_VERDICT_KONA_3_FAILED: ${{ tasks.verdict-kona-3.failed }} + TASK_VERDICT_KONA_3_SKIPPED: ${{ tasks.verdict-kona-3.skipped }} + TASK_VERDICT_OPN_0_SUCCEEDED: ${{ tasks.verdict-opn-0.succeeded }} + TASK_VERDICT_OPN_0_FAILED: ${{ tasks.verdict-opn-0.failed }} + TASK_VERDICT_OPN_0_SKIPPED: ${{ tasks.verdict-opn-0.skipped }} + TASK_VERDICT_OPN_1_SUCCEEDED: ${{ tasks.verdict-opn-1.succeeded }} + TASK_VERDICT_OPN_1_FAILED: ${{ tasks.verdict-opn-1.failed }} + TASK_VERDICT_OPN_1_SKIPPED: ${{ tasks.verdict-opn-1.skipped }} + TASK_VERDICT_OPN_2_SUCCEEDED: ${{ tasks.verdict-opn-2.succeeded }} + TASK_VERDICT_OPN_2_FAILED: ${{ tasks.verdict-opn-2.failed }} + TASK_VERDICT_OPN_2_SKIPPED: ${{ tasks.verdict-opn-2.skipped }} + TASK_VERDICT_OPN_3_SUCCEEDED: ${{ tasks.verdict-opn-3.succeeded }} + TASK_VERDICT_OPN_3_FAILED: ${{ tasks.verdict-opn-3.failed }} + TASK_VERDICT_OPN_3_SKIPPED: ${{ tasks.verdict-opn-3.skipped }} + cache: false + timeout: 10m + outputs: + filesystem: false + artifacts: + - key: receipt + path: .ci/pr-gates/groups/main-acceptance diff --git a/.rwx/cannon-go.yml b/.rwx/cannon-go.yml new file mode 100644 index 00000000000..7470fb2da04 --- /dev/null +++ b/.rwx/cannon-go.yml @@ -0,0 +1,190 @@ +# Complete Cannon Go PR workload; Circle still owns its required gate. +on: + cache-rebuild: + if: ${{ event.git.branch == 'develop' }} + target: [modules, contracts] + init: + commit-sha: ${{ event.git.sha }} + branch: ${{ event.git.branch }} + cache-warm: "true" + cache-epoch: v1 + cli: + init: + commit-sha: ${{ event.git.sha }} + branch: ${{ event.git.branch }} + cache-warm: "false" + cache-epoch: v1 + +tool-cache: + vault: optimism-op-reth-shadow + +base: + image: ubuntu:24.04@sha256:008173c23f95b170204355c12626cb5a965d779a7e1283b09e9cffbb1bf33ca3 + config: rwx/base 1.2.0 + arch: x86_64 + +defaults: + runner: + cpus: 2 + memory: 8gb + +tasks: + - key: code + call: git/clone 2.2.0 + with: + repository: https://github.com/ethereum-optimism/optimism.git + ref: ${{ init.commit-sha }} + preserve-git-dir: true + fetch-full-depth: true + submodules: false + + - key: bootstrap-inputs + use: code + run: 'true' + filter: [mise.toml, ops/ci/runtime] + outputs: + filesystem: + filter: + workspace: [mise.toml, ops/ci/runtime/apt-install.sh, ops/ci/runtime/rwx-prepare.sh, ops/ci/runtime/rwx-contracts-prepare.sh, ops/ci/runtime/rwx-rust-prepare.sh, ops/ci/runtime/op-reth-shadow.sh, ops/ci/runtime/op-reth-report.py, ops/ci/runtime/rust-target-cache.py] + system: [] + + - key: mise + call: mise/install 1.1.0 + with: + mise-version: "2026.2.2" + install: "false" + + - key: tools + use: [mise, bootstrap-inputs] + call: ${{ run.dir }}/packages/toolchain-common.yml + + - key: route + use: [code, tools] + run: bash ops/ci/runtime/rwx-metadata.sh + env: + CI_CACHE_WARM: ${{ init.cache-warm }} + CI_BRANCH: ${{ init.branch }} + CI_COMMIT_SHA: ${{ init.commit-sha }} + cache: false + timeout: 10m + outputs: + filesystem: false + + - key: go-tools + use: tools + if: ${{ init.cache-warm == 'true' || tasks.route.values.run-main == 'true' }} + call: ${{ run.dir }}/packages/toolchain-go.yml + with: + mode: go + + - key: contract-tools + use: [bootstrap-inputs, tools] + if: ${{ init.cache-warm == 'true' || tasks.route.values.run-main == 'true' }} + call: ${{ run.dir }}/packages/toolchain-foundry.yml + with: + mode: anvil + + - key: source + use: [code, tools] + if: ${{ init.cache-warm == 'true' || tasks.route.values.run-main == 'true' }} + run: | + git submodule sync --recursive + git -c protocol.file.allow=never submodule update --init --recursive --jobs 8 + git rev-parse HEAD >packages/contracts-bedrock/.gitcommit + timeout: 20m + + - key: modules + use: go-tools + if: ${{ init.cache-warm == 'true' || tasks.route.values.run-main == 'true' }} + call: ${{ run.dir }}/packages/go-modules.yml + with: + commit-sha: ${{ tasks.dependency-plan.values.go-modules }} + branch: ${{ init.branch }} + cache-epoch: ${{ init.cache-epoch }} + + # A dependency build checks out the commit that last changed its build inputs, so it + # is a cache hit for later commits and for the other workloads of the run. + - key: dependency-plan + use: code + run: python3 ops/ci/runtime/go-dependency-source.py plan + outputs: + filesystem: false + + - key: contracts + use: [go-tools, contract-tools] + if: ${{ init.cache-warm == 'true' || tasks.route.values.run-main == 'true' }} + call: ${{ run.dir }}/packages/go-contracts.yml + with: + commit-sha: ${{ tasks.dependency-plan.values.contracts }} + cache-epoch: ${{ init.cache-epoch }} + build-probe: '' + target-cache-mode: keep + + - key: helper-tests + use: [code, go-tools, contract-tools] + if: ${{ init.cache-warm != 'true' && tasks.route.values.run-main == 'true' }} + run: RWX_LIVE_CONTRACT_DEPENDENCIES_FIXTURE=1 python3 -m unittest discover -s ops/ci/tests -p 'test_go_dependencies.py' + cache: false + timeout: 15m + outputs: + filesystem: false + + # CircleCI's cannon-go-lint-and-test with skip_slow_tests: true: its lint and + # test commands, after the restores that replace attach_workspace. gotestsum's + # exit status is the verdict. + - key: tests + use: [source, go-tools, contract-tools] + if: ${{ init.cache-warm != 'true' && tasks.route.values.run-main == 'true' }} + run: | + export GOMODCACHE="$PWD/.ci/go-cache/pr-checks/modules" + export GOCACHE="$PWD/.ci/go-cache/cannon-go/build" + mkdir -p "$GOCACHE" tmp/test-results tmp/testlogs + python3 ops/ci/runtime/go-artifacts.py restore go-modules "$MODULE_ARTIFACT" + python3 ops/ci/runtime/go-artifacts.py restore contracts "$CONTRACT_ARTIFACT" + cd cannon + just lint + SKIP_SLOW_TESTS=true ../ops/scripts/gotestsum-split.sh --format=testname \ + --junitfile=../tmp/test-results/cannon-64.xml --jsonfile=../tmp/testlogs/log-64.json \ + -- -timeout=10m -parallel="$(nproc)" ./... + env: + CI: "true" + CI_COMMIT_SHA: ${{ init.commit-sha }} + MODULE_ARTIFACT: ${{ tasks.modules.tasks.build.artifacts.dependency }} + CONTRACT_ARTIFACT: ${{ tasks.contracts.tasks.build.artifacts.dependency }} + cache: false + tool-cache: cannon-go-compiler-${{ init.cache-epoch }} + timeout: 30m + runner: + cpus: 8 + memory: 16gb + outputs: + filesystem: + filter: + workspace: [.ci/go-cache/cannon-go/build] + system: [] + test-results: + - path: tmp/test-results/cannon-64.xml + artifacts: + - key: junit + path: tmp/test-results + - key: test-logs + path: tmp/testlogs + + # Exact Main dependency states; unrelated workload failures stay outside this gate. + - key: main-gate-receipt + use: [code, tools] + after: ${{ (tests.succeeded || tests.failed || tests.skipped) }} + if: ${{ init.cache-warm != 'true' }} + run: python3 ops/ci/runtime/pr-gate.py receipt main-cannon + env: + GROUP_SELECTED: ${{ tasks.route.values.run-main }} + TASK_TESTS_SUCCEEDED: ${{ tasks.tests.succeeded }} + TASK_TESTS_FAILED: ${{ tasks.tests.failed }} + TASK_TESTS_SKIPPED: ${{ tasks.tests.skipped }} + cache: false + timeout: 10m + outputs: + filesystem: false + artifacts: + - key: receipt + path: .ci/pr-gates/groups/main-cannon diff --git a/.rwx/contract-l2-fork.yml b/.rwx/contract-l2-fork.yml new file mode 100644 index 00000000000..92e4bbb7793 --- /dev/null +++ b/.rwx/contract-l2-fork.yml @@ -0,0 +1,192 @@ +# OP Mainnet L2 fork tests and the NUT bundle check in an optional shadow; CircleCI keeps its gates. +on: + cache-rebuild: + if: ${{ event.git.branch == 'develop' }} + target: compile + init: + commit-sha: ${{ event.git.sha }} + branch: ${{ event.git.branch }} + cache-warm: "true" + cache-epoch: v1 + fork-block: latest + cli: + init: + commit-sha: ${{ event.git.sha }} + branch: ${{ event.git.branch }} + cache-warm: "false" + cache-epoch: v1 + fork-block: latest + +tool-cache: + vault: optimism-op-reth-shadow + +base: + image: ubuntu:24.04@sha256:008173c23f95b170204355c12626cb5a965d779a7e1283b09e9cffbb1bf33ca3 + config: rwx/base 1.2.0 + arch: x86_64 + +defaults: + runner: + cpus: 2 + memory: 8gb + +tasks: + - key: code + call: git/clone 2.2.0 + with: + repository: https://github.com/ethereum-optimism/optimism.git + ref: ${{ init.commit-sha }} + preserve-git-dir: true + fetch-full-depth: true + submodules: false + + - key: bootstrap-inputs + use: code + run: 'true' + filter: [mise.toml, ops/ci/runtime] + outputs: + filesystem: + filter: + workspace: [mise.toml, ops/ci/runtime/apt-install.sh, ops/ci/runtime/rwx-prepare.sh, ops/ci/runtime/rwx-contracts-prepare.sh, ops/ci/runtime/rwx-rust-prepare.sh, ops/ci/runtime/op-reth-shadow.sh, ops/ci/runtime/op-reth-report.py, ops/ci/runtime/rust-target-cache.py] + system: [] + + - key: mise + call: mise/install 1.1.0 + with: + mise-version: "2026.2.2" + install: "false" + + - key: tools + use: [mise, bootstrap-inputs] + call: ${{ run.dir }}/packages/toolchain-common.yml + + - key: route + use: [code, tools] + run: bash ops/ci/runtime/rwx-metadata.sh + env: + CI_CACHE_WARM: ${{ init.cache-warm }} + CI_BRANCH: ${{ init.branch }} + CI_COMMIT_SHA: ${{ init.commit-sha }} + cache: false + timeout: 10m + outputs: + filesystem: false + + - key: go-tools + use: tools + if: ${{ init.cache-warm == 'true' || tasks.route.values.run-contracts == 'true' }} + call: ${{ run.dir }}/packages/toolchain-go.yml + with: + mode: go + + - key: contract-tools + use: [bootstrap-inputs, tools] + if: ${{ init.cache-warm == 'true' || tasks.route.values.run-contracts == 'true' }} + call: ${{ run.dir }}/packages/toolchain-foundry.yml + with: + mode: compiler + + # A dependency build checks out the commit that last changed its build inputs, so it + # is a cache hit for later commits and for the other workloads of the run. + - key: dependency-plan + use: code + run: python3 ops/ci/runtime/go-dependency-source.py plan + outputs: + filesystem: false + + - key: go-modules + use: go-tools + if: ${{ init.cache-warm == 'true' || tasks.route.values.run-contracts == 'true' }} + call: ${{ run.dir }}/packages/go-modules.yml + with: + commit-sha: ${{ tasks.dependency-plan.values.go-modules }} + branch: ${{ init.branch }} + cache-epoch: ${{ init.cache-epoch }} + + - key: source + use: [code, tools] + if: ${{ init.cache-warm == 'true' || tasks.route.values.run-contracts == 'true' }} + run: | + git submodule sync --recursive + git -c protocol.file.allow=never submodule update --init --recursive --jobs 8 + timeout: 20m + + - key: helper-tests + use: [code, tools] + if: ${{ init.cache-warm != 'true' && tasks.route.values.run-contracts == 'true' }} + run: python3 -m unittest discover -s ops/ci/tests -p 'test_contract_l2_fork.py' + cache: false + timeout: 10m + outputs: + filesystem: false + + - key: compile + use: [source, go-tools, contract-tools, go-modules] + # As in packages/contracts-compile.yml: the files that the build reads. + filter: [mise.toml, go.mod, go.sum, cannon, 'op-*', packages/contracts-bedrock, ops/ci/runtime/ci-report.py, ops/ci/runtime/contract-upgrades.py, ops/ci/runtime/contract-l2-fork.py, ops/ci/runtime/rpc-pacer.py, .ci/go-cache/pr-checks/modules, .ci/go-cache/l2-fork/compile] + if: ${{ init.cache-warm == 'true' || tasks.route.values.run-contracts == 'true' }} + run: | + export GOMODCACHE="$PWD/.ci/go-cache/pr-checks/modules" + export GOCACHE="$PWD/.ci/go-cache/l2-fork/compile" + python3 ops/ci/runtime/contract-l2-fork.py prepare + tool-cache: l2-fork-compiled-ci-${{ init.cache-epoch }} + timeout: 40m + runner: + cpus: 4 + memory: 32gb + outputs: + filesystem: + filter: + workspace: [packages/contracts-bedrock/forge-artifacts, packages/contracts-bedrock/cache/solidity-files-cache.json, packages/contracts-bedrock/artifacts/build-info, packages/contracts-bedrock/scripts/go-ffi/go-ffi, .ci/go-cache/l2-fork/compile] + system: [/root/.svm, /home/*/.svm] + artifacts: + - key: reports + path: .ci/contract-l2-fork/prepare + + - key: verdict + use: [source, go-tools, contract-tools, go-modules, compile] + if: ${{ init.cache-warm != 'true' && tasks.route.values.run-contracts == 'true' }} + run: | + export GOMODCACHE="$PWD/.ci/go-cache/pr-checks/modules" + export GOCACHE="$PWD/.ci/go-cache/l2-fork/runtime" + python3 ops/ci/runtime/contract-l2-fork.py run --fork-block "$CI_L2_FORK_BLOCK" + env: + CI_L2_FORK_BLOCK: ${{ init.fork-block }} + cache: false + tool-cache: l2-fork-runtime-state-${{ init.cache-epoch }} + timeout: 60m + runner: + cpus: 16 + memory: 32gb + outputs: + filesystem: + filter: + workspace: [.ci/go-cache/l2-fork/runtime] + system: [/root/.foundry/cache/rpc, /home/*/.foundry/cache/rpc] + test-results: + - path: .ci/contract-l2-fork/run/results.xml + options: + language: Solidity + framework: Foundry + artifacts: + - key: reports + path: .ci/contract-l2-fork/run + + # Bind every complete Contracts prerequisite to fresh engine terminal states. + - key: contracts-gate-receipt + use: [code, tools] + after: ${{ (verdict.succeeded || verdict.failed || verdict.skipped) }} + if: ${{ init.cache-warm != 'true' }} + run: python3 ops/ci/runtime/pr-gate.py receipt contracts-l2-fork + env: + GROUP_SELECTED: ${{ tasks.route.values.run-contracts }} + TASK_VERDICT_SUCCEEDED: ${{ tasks.verdict.succeeded }} + TASK_VERDICT_FAILED: ${{ tasks.verdict.failed }} + TASK_VERDICT_SKIPPED: ${{ tasks.verdict.skipped }} + cache: false + timeout: 10m + outputs: + filesystem: false + artifacts: + - key: receipt + path: .ci/pr-gates/groups/contracts-l2-fork diff --git a/.rwx/contract-upgrades.yml b/.rwx/contract-upgrades.yml new file mode 100644 index 00000000000..1402e2fca2e --- /dev/null +++ b/.rwx/contract-upgrades.yml @@ -0,0 +1,241 @@ +# Complete L1 upgrade matrices; Circle retains its required gates. +on: + cache-rebuild: + if: ${{ event.git.branch == 'develop' }} + target: [compile] + init: + commit-sha: ${{ event.git.sha }} + branch: ${{ event.git.branch }} + cache-warm: "true" + cache-epoch: v1 + cli: + init: + commit-sha: ${{ event.git.sha }} + branch: ${{ event.git.branch }} + cache-warm: "false" + cache-epoch: v1 + +tool-cache: + vault: optimism-op-reth-shadow + +base: + image: ubuntu:24.04@sha256:008173c23f95b170204355c12626cb5a965d779a7e1283b09e9cffbb1bf33ca3 + config: rwx/base 1.2.0 + arch: x86_64 + +defaults: + runner: + cpus: 2 + memory: 8gb + +tasks: + - key: code + call: git/clone 2.2.0 + with: + repository: https://github.com/ethereum-optimism/optimism.git + ref: ${{ init.commit-sha }} + preserve-git-dir: true + fetch-full-depth: true + submodules: false + + - key: bootstrap-inputs + use: code + run: 'true' + filter: [mise.toml, ops/ci/runtime] + outputs: + filesystem: + filter: + workspace: [mise.toml, ops/ci/runtime/apt-install.sh, ops/ci/runtime/rwx-prepare.sh, ops/ci/runtime/rwx-contracts-prepare.sh, ops/ci/runtime/rwx-rust-prepare.sh, ops/ci/runtime/op-reth-shadow.sh, ops/ci/runtime/op-reth-report.py, ops/ci/runtime/rust-target-cache.py] + system: [] + + - key: mise + call: mise/install 1.1.0 + with: + mise-version: "2026.2.2" + install: "false" + + - key: tools + use: [mise, bootstrap-inputs] + call: ${{ run.dir }}/packages/toolchain-common.yml + + - key: route + use: [code, tools] + run: bash ops/ci/runtime/rwx-metadata.sh + env: + CI_CACHE_WARM: ${{ init.cache-warm }} + CI_BRANCH: ${{ init.branch }} + CI_COMMIT_SHA: ${{ init.commit-sha }} + cache: false + timeout: 10m + outputs: + filesystem: false + + - key: go-tools + use: tools + if: ${{ init.cache-warm == 'true' || tasks.route.values.run-contracts == 'true' }} + call: ${{ run.dir }}/packages/toolchain-go.yml + with: + mode: go + + - key: contract-tools + use: [bootstrap-inputs, tools] + if: ${{ init.cache-warm == 'true' || tasks.route.values.run-contracts == 'true' }} + call: ${{ run.dir }}/packages/toolchain-foundry.yml + with: + mode: compiler + + # A dependency build checks out the commit that last changed its build inputs, so it + # is a cache hit for later commits and for the other workloads of the run. + - key: dependency-plan + use: code + run: python3 ops/ci/runtime/go-dependency-source.py plan + outputs: + filesystem: false + + - key: go-modules + use: go-tools + if: ${{ init.cache-warm == 'true' || tasks.route.values.run-contracts == 'true' }} + call: ${{ run.dir }}/packages/go-modules.yml + with: + commit-sha: ${{ tasks.dependency-plan.values.go-modules }} + branch: ${{ init.branch }} + cache-epoch: ${{ init.cache-epoch }} + + # One initialized checkout supplies the submodule sources to every compile and verdict task. + - key: source + use: [code, tools] + if: ${{ init.cache-warm == 'true' || tasks.route.values.run-contracts == 'true' }} + run: | + git submodule sync --recursive + git -c protocol.file.allow=never submodule update --init --recursive --jobs 8 + timeout: 20m + + - key: helper-tests + use: [code, go-tools, contract-tools] + if: ${{ init.cache-warm != 'true' && tasks.route.values.run-contracts == 'true' }} + run: | + RWX_LIVE_FORGE_FIXTURE=1 python3 -m unittest discover -s ops/ci/tests -p 'test_contract_upgrades.py' + cache: false + timeout: 10m + outputs: + filesystem: false + + # One build for every verdict task below: see packages/contracts-compile.yml. + - key: compile + use: [source, go-tools, contract-tools, go-modules] + if: ${{ init.cache-warm == 'true' || tasks.route.values.run-contracts == 'true' }} + call: ${{ run.dir }}/packages/contracts-compile.yml + with: + branch: ${{ init.branch }} + cache-epoch: ${{ init.cache-epoch }} + + - key: verdict-feature-main + use: [source, go-tools, contract-tools, go-modules, compile] + if: ${{ init.cache-warm != 'true' && tasks.route.values.run-contracts == 'true' }} + call: ${{ run.dir }}/packages/contract-upgrades-verdict.yml + with: + branch: ${{ init.branch }} + cache-key: contract-upgrades-rpc-feature-main-${{ init.cache-epoch }} + variant: feature-main + + - key: verdict-feature-CUSTOM_GAS_TOKEN + use: [source, go-tools, contract-tools, go-modules, compile] + if: ${{ init.cache-warm != 'true' && tasks.route.values.run-contracts == 'true' }} + call: ${{ run.dir }}/packages/contract-upgrades-verdict.yml + with: + branch: ${{ init.branch }} + cache-key: contract-upgrades-rpc-feature-CUSTOM_GAS_TOKEN-${{ init.cache-epoch }} + variant: feature-CUSTOM_GAS_TOKEN + + - key: verdict-feature-INTEROP + use: [source, go-tools, contract-tools, go-modules, compile] + if: ${{ init.cache-warm != 'true' && tasks.route.values.run-contracts == 'true' }} + call: ${{ run.dir }}/packages/contract-upgrades-verdict.yml + with: + branch: ${{ init.branch }} + cache-key: contract-upgrades-rpc-feature-INTEROP-${{ init.cache-epoch }} + variant: feature-INTEROP + + - key: verdict-feature-BRIDGE_HOOK + use: [source, go-tools, contract-tools, go-modules, compile] + if: ${{ init.cache-warm != 'true' && tasks.route.values.run-contracts == 'true' }} + call: ${{ run.dir }}/packages/contract-upgrades-verdict.yml + with: + branch: ${{ init.branch }} + cache-key: contract-upgrades-rpc-feature-BRIDGE_HOOK-${{ init.cache-epoch }} + variant: feature-BRIDGE_HOOK + + - key: verdict-feature-OP_PRIVATE + use: [source, go-tools, contract-tools, go-modules, compile] + if: ${{ init.cache-warm != 'true' && tasks.route.values.run-contracts == 'true' }} + call: ${{ run.dir }}/packages/contract-upgrades-verdict.yml + with: + branch: ${{ init.branch }} + cache-key: contract-upgrades-rpc-feature-OP_PRIVATE-${{ init.cache-epoch }} + variant: feature-OP_PRIVATE + + - key: verdict-chain-op + use: [source, go-tools, contract-tools, go-modules, compile] + if: ${{ init.cache-warm != 'true' && tasks.route.values.run-contracts == 'true' }} + call: ${{ run.dir }}/packages/contract-upgrades-verdict.yml + with: + branch: ${{ init.branch }} + cache-key: contract-upgrades-rpc-chain-op-${{ init.cache-epoch }} + variant: chain-op + + - key: verdict-chain-ink + use: [source, go-tools, contract-tools, go-modules, compile] + if: ${{ init.cache-warm != 'true' && tasks.route.values.run-contracts == 'true' }} + call: ${{ run.dir }}/packages/contract-upgrades-verdict.yml + with: + branch: ${{ init.branch }} + cache-key: contract-upgrades-rpc-chain-ink-${{ init.cache-epoch }} + variant: chain-ink + + - key: verdict-chain-unichain + use: [source, go-tools, contract-tools, go-modules, compile] + if: ${{ init.cache-warm != 'true' && tasks.route.values.run-contracts == 'true' }} + call: ${{ run.dir }}/packages/contract-upgrades-verdict.yml + with: + branch: ${{ init.branch }} + cache-key: contract-upgrades-rpc-chain-unichain-${{ init.cache-epoch }} + variant: chain-unichain + + - key: contracts-gate-receipt + use: [code, tools] + after: ${{ (verdict-chain-ink.succeeded || verdict-chain-ink.failed || verdict-chain-ink.skipped) && (verdict-chain-op.succeeded || verdict-chain-op.failed || verdict-chain-op.skipped) && (verdict-chain-unichain.succeeded || verdict-chain-unichain.failed || verdict-chain-unichain.skipped) && (verdict-feature-BRIDGE_HOOK.succeeded || verdict-feature-BRIDGE_HOOK.failed || verdict-feature-BRIDGE_HOOK.skipped) && (verdict-feature-CUSTOM_GAS_TOKEN.succeeded || verdict-feature-CUSTOM_GAS_TOKEN.failed || verdict-feature-CUSTOM_GAS_TOKEN.skipped) && (verdict-feature-INTEROP.succeeded || verdict-feature-INTEROP.failed || verdict-feature-INTEROP.skipped) && (verdict-feature-OP_PRIVATE.succeeded || verdict-feature-OP_PRIVATE.failed || verdict-feature-OP_PRIVATE.skipped) && (verdict-feature-main.succeeded || verdict-feature-main.failed || verdict-feature-main.skipped) }} + if: ${{ init.cache-warm != 'true' }} + run: python3 ops/ci/runtime/pr-gate.py receipt contracts-upgrades + env: + TASK_VERDICT_FEATURE_BRIDGE_HOOK_SUCCEEDED: ${{ tasks.verdict-feature-BRIDGE_HOOK.succeeded }} + TASK_VERDICT_FEATURE_BRIDGE_HOOK_FAILED: ${{ tasks.verdict-feature-BRIDGE_HOOK.failed }} + TASK_VERDICT_FEATURE_BRIDGE_HOOK_SKIPPED: ${{ tasks.verdict-feature-BRIDGE_HOOK.skipped }} + GROUP_SELECTED: ${{ tasks.route.values.run-contracts }} + TASK_VERDICT_CHAIN_INK_SUCCEEDED: ${{ tasks.verdict-chain-ink.succeeded }} + TASK_VERDICT_CHAIN_INK_FAILED: ${{ tasks.verdict-chain-ink.failed }} + TASK_VERDICT_CHAIN_INK_SKIPPED: ${{ tasks.verdict-chain-ink.skipped }} + TASK_VERDICT_CHAIN_OP_SUCCEEDED: ${{ tasks.verdict-chain-op.succeeded }} + TASK_VERDICT_CHAIN_OP_FAILED: ${{ tasks.verdict-chain-op.failed }} + TASK_VERDICT_CHAIN_OP_SKIPPED: ${{ tasks.verdict-chain-op.skipped }} + TASK_VERDICT_CHAIN_UNICHAIN_SUCCEEDED: ${{ tasks.verdict-chain-unichain.succeeded }} + TASK_VERDICT_CHAIN_UNICHAIN_FAILED: ${{ tasks.verdict-chain-unichain.failed }} + TASK_VERDICT_CHAIN_UNICHAIN_SKIPPED: ${{ tasks.verdict-chain-unichain.skipped }} + TASK_VERDICT_FEATURE_CUSTOM_GAS_TOKEN_SUCCEEDED: ${{ tasks.verdict-feature-CUSTOM_GAS_TOKEN.succeeded }} + TASK_VERDICT_FEATURE_CUSTOM_GAS_TOKEN_FAILED: ${{ tasks.verdict-feature-CUSTOM_GAS_TOKEN.failed }} + TASK_VERDICT_FEATURE_CUSTOM_GAS_TOKEN_SKIPPED: ${{ tasks.verdict-feature-CUSTOM_GAS_TOKEN.skipped }} + TASK_VERDICT_FEATURE_INTEROP_SUCCEEDED: ${{ tasks.verdict-feature-INTEROP.succeeded }} + TASK_VERDICT_FEATURE_INTEROP_FAILED: ${{ tasks.verdict-feature-INTEROP.failed }} + TASK_VERDICT_FEATURE_INTEROP_SKIPPED: ${{ tasks.verdict-feature-INTEROP.skipped }} + TASK_VERDICT_FEATURE_OP_PRIVATE_SUCCEEDED: ${{ tasks.verdict-feature-OP_PRIVATE.succeeded }} + TASK_VERDICT_FEATURE_OP_PRIVATE_FAILED: ${{ tasks.verdict-feature-OP_PRIVATE.failed }} + TASK_VERDICT_FEATURE_OP_PRIVATE_SKIPPED: ${{ tasks.verdict-feature-OP_PRIVATE.skipped }} + TASK_VERDICT_FEATURE_MAIN_SUCCEEDED: ${{ tasks.verdict-feature-main.succeeded }} + TASK_VERDICT_FEATURE_MAIN_FAILED: ${{ tasks.verdict-feature-main.failed }} + TASK_VERDICT_FEATURE_MAIN_SKIPPED: ${{ tasks.verdict-feature-main.skipped }} + cache: false + timeout: 10m + outputs: + filesystem: false + artifacts: + - key: receipt + path: .ci/pr-gates/groups/contracts-upgrades diff --git a/.rwx/contracts.yml b/.rwx/contracts.yml new file mode 100644 index 00000000000..81284cd978e --- /dev/null +++ b/.rwx/contracts.yml @@ -0,0 +1,341 @@ +# Complete standard contracts in an optional shadow; CircleCI keeps its gates. +on: + cache-rebuild: + if: ${{ event.git.branch == 'develop' }} + target: [compile] + init: + commit-sha: ${{ event.git.sha }} + branch: ${{ event.git.branch }} + cache-warm: "true" + cache-epoch: v1 + cli: + init: + commit-sha: ${{ event.git.sha }} + branch: ${{ event.git.branch }} + cache-warm: "false" + cache-epoch: v1 + +# Cache-only vault; no RPC or publishing credentials. +tool-cache: + vault: optimism-op-reth-shadow + +base: + image: ubuntu:24.04@sha256:008173c23f95b170204355c12626cb5a965d779a7e1283b09e9cffbb1bf33ca3 + config: rwx/base 1.2.0 + arch: x86_64 + +defaults: + runner: + cpus: 2 + memory: 8gb + +tasks: + - key: code + call: git/clone 2.2.0 + with: + repository: https://github.com/ethereum-optimism/optimism.git + ref: ${{ init.commit-sha }} + preserve-git-dir: true + fetch-full-depth: true + submodules: false + + - key: bootstrap-inputs + use: code + run: 'true' + filter: [mise.toml, ops/ci/runtime] + outputs: + filesystem: + filter: + workspace: [mise.toml, ops/ci/runtime/apt-install.sh, ops/ci/runtime/rwx-prepare.sh, ops/ci/runtime/rwx-contracts-prepare.sh, ops/ci/runtime/rwx-rust-prepare.sh, ops/ci/runtime/op-reth-shadow.sh, ops/ci/runtime/op-reth-report.py, ops/ci/runtime/rust-target-cache.py] + system: [] + + - key: mise + call: mise/install 1.1.0 + with: + mise-version: "2026.2.2" + install: "false" + + # The filtered bootstrap layer keeps the checkout's Git history out of + # reusable tools. Language toolchains are added only by their own workload. + - key: tools + use: [mise, bootstrap-inputs] + call: ${{ run.dir }}/packages/toolchain-common.yml + + - key: route + use: [code, tools] + run: bash ops/ci/runtime/rwx-metadata.sh + env: + CI_CACHE_WARM: ${{ init.cache-warm }} + CI_BRANCH: ${{ init.branch }} + CI_COMMIT_SHA: ${{ init.commit-sha }} + cache: false + timeout: 10m + outputs: + filesystem: false + + - key: go-tools + use: tools + if: ${{ init.cache-warm == 'true' || tasks.route.values.run-contracts == 'true' }} + call: ${{ run.dir }}/packages/toolchain-go.yml + with: + mode: go + + - key: contract-tools + use: [bootstrap-inputs, tools] + if: ${{ init.cache-warm == 'true' || tasks.route.values.run-contracts == 'true' }} + call: ${{ run.dir }}/packages/toolchain-foundry.yml + with: + mode: compiler + + # A dependency build checks out the commit that last changed its build inputs, so it + # is a cache hit for later commits and for the other workloads of the run. + - key: dependency-plan + use: code + run: python3 ops/ci/runtime/go-dependency-source.py plan + outputs: + filesystem: false + + - key: go-modules + use: go-tools + if: ${{ init.cache-warm == 'true' || tasks.route.values.run-contracts == 'true' }} + call: ${{ run.dir }}/packages/go-modules.yml + with: + commit-sha: ${{ tasks.dependency-plan.values.go-modules }} + branch: ${{ init.branch }} + cache-epoch: ${{ init.cache-epoch }} + + # Fetch the moving develop branch once per run. The `modified` compile and + # verdict tasks read origin/develop from this layer, so they diff the same commit. + - key: modified-baseline + use: source + if: ${{ init.cache-warm == 'true' || tasks.route.values.run-contracts == 'true' }} + run: | + git fetch --no-tags origin +refs/heads/develop:refs/remotes/origin/develop + git rev-parse --verify 'origin/develop^{commit}' + cache: false + timeout: 10m + + # One initialized checkout supplies the submodule sources to every compile and verdict task. + - key: source + use: [code, tools] + if: ${{ init.cache-warm == 'true' || tasks.route.values.run-contracts == 'true' }} + run: | + git submodule sync --recursive + git -c protocol.file.allow=never submodule update --init --recursive --jobs 8 + timeout: 20m + + - key: helper-tests + use: [code, go-tools, contract-tools, go-modules] + if: ${{ init.cache-warm != 'true' && tasks.route.values.run-contracts == 'true' }} + run: | + export GOMODCACHE="$PWD/.ci/go-cache/pr-checks/modules" + RWX_LIVE_CONTRACT_SUITE_FIXTURE=1 python3 -m unittest discover -s ops/ci/tests -p 'test_contract_suites.py' + python3 ops/ci/tests/test_rwx_contracts_prepare.py + cache: false + timeout: 15m + outputs: + filesystem: false + + # One build for every verdict task below: see packages/contracts-compile.yml. + - key: compile + use: [source, go-tools, contract-tools, go-modules] + if: ${{ init.cache-warm == 'true' || tasks.route.values.run-contracts == 'true' }} + call: ${{ run.dir }}/packages/contracts-compile.yml + with: + branch: ${{ init.branch }} + cache-epoch: ${{ init.cache-epoch }} + + - key: verdict-standard-main + use: [source, go-tools, contract-tools, go-modules, compile] + if: ${{ init.cache-warm != 'true' && tasks.route.values.run-contracts == 'true' }} + call: ${{ run.dir }}/packages/contracts-verdict.yml + with: + branch: ${{ init.branch }} + cache-key: contract-suites-runtime-standard-main-${{ init.cache-epoch }} + suite: standard + feature: main + + - key: verdict-standard-CUSTOM_GAS_TOKEN + use: [source, go-tools, contract-tools, go-modules, compile] + if: ${{ init.cache-warm != 'true' && tasks.route.values.run-contracts == 'true' }} + call: ${{ run.dir }}/packages/contracts-verdict.yml + with: + branch: ${{ init.branch }} + cache-key: contract-suites-runtime-standard-CUSTOM_GAS_TOKEN-${{ init.cache-epoch }} + suite: standard + feature: CUSTOM_GAS_TOKEN + + - key: verdict-standard-INTEROP + use: [source, go-tools, contract-tools, go-modules, compile] + if: ${{ init.cache-warm != 'true' && tasks.route.values.run-contracts == 'true' }} + call: ${{ run.dir }}/packages/contracts-verdict.yml + with: + branch: ${{ init.branch }} + cache-key: contract-suites-runtime-standard-INTEROP-${{ init.cache-epoch }} + suite: standard + feature: INTEROP + + - key: verdict-standard-BRIDGE_HOOK + use: [source, go-tools, contract-tools, go-modules, compile] + if: ${{ init.cache-warm != 'true' && tasks.route.values.run-contracts == 'true' }} + call: ${{ run.dir }}/packages/contracts-verdict.yml + with: + branch: ${{ init.branch }} + cache-key: contract-suites-runtime-standard-BRIDGE_HOOK-${{ init.cache-epoch }} + suite: standard + feature: BRIDGE_HOOK + + - key: verdict-standard-OP_PRIVATE + use: [source, go-tools, contract-tools, go-modules, compile] + if: ${{ init.cache-warm != 'true' && tasks.route.values.run-contracts == 'true' }} + call: ${{ run.dir }}/packages/contracts-verdict.yml + with: + branch: ${{ init.branch }} + cache-key: contract-suites-runtime-standard-OP_PRIVATE-${{ init.cache-epoch }} + suite: standard + feature: OP_PRIVATE + + - key: compile-modified-main + use: [modified-baseline, go-tools, contract-tools, go-modules] + if: ${{ init.cache-warm == 'true' || tasks.route.values.run-contracts == 'true' }} + call: ${{ run.dir }}/packages/contracts-compile-modified.yml + with: + branch: ${{ init.branch }} + cache-key: contract-suites-modified-main-${{ init.cache-epoch }} + suite: modified + feature: main + + - key: verdict-modified-main + use: [modified-baseline, go-tools, contract-tools, go-modules, compile-modified-main] + if: ${{ init.cache-warm != 'true' && tasks.route.values.run-contracts == 'true' }} + call: ${{ run.dir }}/packages/contracts-verdict.yml + with: + branch: ${{ init.branch }} + cache-key: contract-suites-runtime-modified-main-${{ init.cache-epoch }} + suite: modified + feature: main + + - key: compile-modified-CUSTOM_GAS_TOKEN + use: [modified-baseline, go-tools, contract-tools, go-modules] + if: ${{ init.cache-warm == 'true' || tasks.route.values.run-contracts == 'true' }} + call: ${{ run.dir }}/packages/contracts-compile-modified.yml + with: + branch: ${{ init.branch }} + cache-key: contract-suites-modified-CUSTOM_GAS_TOKEN-${{ init.cache-epoch }} + suite: modified + feature: CUSTOM_GAS_TOKEN + + - key: verdict-modified-CUSTOM_GAS_TOKEN + use: [modified-baseline, go-tools, contract-tools, go-modules, compile-modified-CUSTOM_GAS_TOKEN] + if: ${{ init.cache-warm != 'true' && tasks.route.values.run-contracts == 'true' }} + call: ${{ run.dir }}/packages/contracts-verdict.yml + with: + branch: ${{ init.branch }} + cache-key: contract-suites-runtime-modified-CUSTOM_GAS_TOKEN-${{ init.cache-epoch }} + suite: modified + feature: CUSTOM_GAS_TOKEN + + - key: compile-modified-INTEROP + use: [modified-baseline, go-tools, contract-tools, go-modules] + if: ${{ init.cache-warm == 'true' || tasks.route.values.run-contracts == 'true' }} + call: ${{ run.dir }}/packages/contracts-compile-modified.yml + with: + branch: ${{ init.branch }} + cache-key: contract-suites-modified-INTEROP-${{ init.cache-epoch }} + suite: modified + feature: INTEROP + + - key: verdict-modified-INTEROP + use: [modified-baseline, go-tools, contract-tools, go-modules, compile-modified-INTEROP] + if: ${{ init.cache-warm != 'true' && tasks.route.values.run-contracts == 'true' }} + call: ${{ run.dir }}/packages/contracts-verdict.yml + with: + branch: ${{ init.branch }} + cache-key: contract-suites-runtime-modified-INTEROP-${{ init.cache-epoch }} + suite: modified + feature: INTEROP + + - key: compile-modified-BRIDGE_HOOK + use: [modified-baseline, go-tools, contract-tools, go-modules] + if: ${{ init.cache-warm == 'true' || tasks.route.values.run-contracts == 'true' }} + call: ${{ run.dir }}/packages/contracts-compile-modified.yml + with: + branch: ${{ init.branch }} + cache-key: contract-suites-modified-BRIDGE_HOOK-${{ init.cache-epoch }} + suite: modified + feature: BRIDGE_HOOK + + - key: compile-modified-OP_PRIVATE + use: [modified-baseline, go-tools, contract-tools, go-modules] + if: ${{ init.cache-warm == 'true' || tasks.route.values.run-contracts == 'true' }} + call: ${{ run.dir }}/packages/contracts-compile-modified.yml + with: + branch: ${{ init.branch }} + cache-key: contract-suites-modified-OP_PRIVATE-${{ init.cache-epoch }} + suite: modified + feature: OP_PRIVATE + + - key: verdict-modified-BRIDGE_HOOK + use: [modified-baseline, go-tools, contract-tools, go-modules, compile-modified-BRIDGE_HOOK] + if: ${{ init.cache-warm != 'true' && tasks.route.values.run-contracts == 'true' }} + call: ${{ run.dir }}/packages/contracts-verdict.yml + with: + branch: ${{ init.branch }} + cache-key: contract-suites-runtime-modified-BRIDGE_HOOK-${{ init.cache-epoch }} + suite: modified + feature: BRIDGE_HOOK + + - key: verdict-modified-OP_PRIVATE + use: [modified-baseline, go-tools, contract-tools, go-modules, compile-modified-OP_PRIVATE] + if: ${{ init.cache-warm != 'true' && tasks.route.values.run-contracts == 'true' }} + call: ${{ run.dir }}/packages/contracts-verdict.yml + with: + branch: ${{ init.branch }} + cache-key: contract-suites-runtime-modified-OP_PRIVATE-${{ init.cache-epoch }} + suite: modified + feature: OP_PRIVATE + + - key: contracts-gate-receipt + use: [code, tools] + after: ${{ (verdict-modified-BRIDGE_HOOK.succeeded || verdict-modified-BRIDGE_HOOK.failed || verdict-modified-BRIDGE_HOOK.skipped) && (verdict-modified-CUSTOM_GAS_TOKEN.succeeded || verdict-modified-CUSTOM_GAS_TOKEN.failed || verdict-modified-CUSTOM_GAS_TOKEN.skipped) && (verdict-modified-INTEROP.succeeded || verdict-modified-INTEROP.failed || verdict-modified-INTEROP.skipped) && (verdict-modified-OP_PRIVATE.succeeded || verdict-modified-OP_PRIVATE.failed || verdict-modified-OP_PRIVATE.skipped) && (verdict-modified-main.succeeded || verdict-modified-main.failed || verdict-modified-main.skipped) && (verdict-standard-BRIDGE_HOOK.succeeded || verdict-standard-BRIDGE_HOOK.failed || verdict-standard-BRIDGE_HOOK.skipped) && (verdict-standard-CUSTOM_GAS_TOKEN.succeeded || verdict-standard-CUSTOM_GAS_TOKEN.failed || verdict-standard-CUSTOM_GAS_TOKEN.skipped) && (verdict-standard-INTEROP.succeeded || verdict-standard-INTEROP.failed || verdict-standard-INTEROP.skipped) && (verdict-standard-OP_PRIVATE.succeeded || verdict-standard-OP_PRIVATE.failed || verdict-standard-OP_PRIVATE.skipped) && (verdict-standard-main.succeeded || verdict-standard-main.failed || verdict-standard-main.skipped) }} + if: ${{ init.cache-warm != 'true' }} + run: python3 ops/ci/runtime/pr-gate.py receipt contracts-suites + env: + TASK_VERDICT_STANDARD_BRIDGE_HOOK_SUCCEEDED: ${{ tasks.verdict-standard-BRIDGE_HOOK.succeeded }} + TASK_VERDICT_STANDARD_BRIDGE_HOOK_FAILED: ${{ tasks.verdict-standard-BRIDGE_HOOK.failed }} + TASK_VERDICT_STANDARD_BRIDGE_HOOK_SKIPPED: ${{ tasks.verdict-standard-BRIDGE_HOOK.skipped }} + TASK_VERDICT_MODIFIED_BRIDGE_HOOK_SUCCEEDED: ${{ tasks.verdict-modified-BRIDGE_HOOK.succeeded }} + TASK_VERDICT_MODIFIED_BRIDGE_HOOK_FAILED: ${{ tasks.verdict-modified-BRIDGE_HOOK.failed }} + TASK_VERDICT_MODIFIED_BRIDGE_HOOK_SKIPPED: ${{ tasks.verdict-modified-BRIDGE_HOOK.skipped }} + GROUP_SELECTED: ${{ tasks.route.values.run-contracts }} + TASK_VERDICT_MODIFIED_CUSTOM_GAS_TOKEN_SUCCEEDED: ${{ tasks.verdict-modified-CUSTOM_GAS_TOKEN.succeeded }} + TASK_VERDICT_MODIFIED_CUSTOM_GAS_TOKEN_FAILED: ${{ tasks.verdict-modified-CUSTOM_GAS_TOKEN.failed }} + TASK_VERDICT_MODIFIED_CUSTOM_GAS_TOKEN_SKIPPED: ${{ tasks.verdict-modified-CUSTOM_GAS_TOKEN.skipped }} + TASK_VERDICT_MODIFIED_INTEROP_SUCCEEDED: ${{ tasks.verdict-modified-INTEROP.succeeded }} + TASK_VERDICT_MODIFIED_INTEROP_FAILED: ${{ tasks.verdict-modified-INTEROP.failed }} + TASK_VERDICT_MODIFIED_INTEROP_SKIPPED: ${{ tasks.verdict-modified-INTEROP.skipped }} + TASK_VERDICT_MODIFIED_OP_PRIVATE_SUCCEEDED: ${{ tasks.verdict-modified-OP_PRIVATE.succeeded }} + TASK_VERDICT_MODIFIED_OP_PRIVATE_FAILED: ${{ tasks.verdict-modified-OP_PRIVATE.failed }} + TASK_VERDICT_MODIFIED_OP_PRIVATE_SKIPPED: ${{ tasks.verdict-modified-OP_PRIVATE.skipped }} + TASK_VERDICT_MODIFIED_MAIN_SUCCEEDED: ${{ tasks.verdict-modified-main.succeeded }} + TASK_VERDICT_MODIFIED_MAIN_FAILED: ${{ tasks.verdict-modified-main.failed }} + TASK_VERDICT_MODIFIED_MAIN_SKIPPED: ${{ tasks.verdict-modified-main.skipped }} + TASK_VERDICT_STANDARD_CUSTOM_GAS_TOKEN_SUCCEEDED: ${{ tasks.verdict-standard-CUSTOM_GAS_TOKEN.succeeded }} + TASK_VERDICT_STANDARD_CUSTOM_GAS_TOKEN_FAILED: ${{ tasks.verdict-standard-CUSTOM_GAS_TOKEN.failed }} + TASK_VERDICT_STANDARD_CUSTOM_GAS_TOKEN_SKIPPED: ${{ tasks.verdict-standard-CUSTOM_GAS_TOKEN.skipped }} + TASK_VERDICT_STANDARD_INTEROP_SUCCEEDED: ${{ tasks.verdict-standard-INTEROP.succeeded }} + TASK_VERDICT_STANDARD_INTEROP_FAILED: ${{ tasks.verdict-standard-INTEROP.failed }} + TASK_VERDICT_STANDARD_INTEROP_SKIPPED: ${{ tasks.verdict-standard-INTEROP.skipped }} + TASK_VERDICT_STANDARD_OP_PRIVATE_SUCCEEDED: ${{ tasks.verdict-standard-OP_PRIVATE.succeeded }} + TASK_VERDICT_STANDARD_OP_PRIVATE_FAILED: ${{ tasks.verdict-standard-OP_PRIVATE.failed }} + TASK_VERDICT_STANDARD_OP_PRIVATE_SKIPPED: ${{ tasks.verdict-standard-OP_PRIVATE.skipped }} + TASK_VERDICT_STANDARD_MAIN_SUCCEEDED: ${{ tasks.verdict-standard-main.succeeded }} + TASK_VERDICT_STANDARD_MAIN_FAILED: ${{ tasks.verdict-standard-main.failed }} + TASK_VERDICT_STANDARD_MAIN_SKIPPED: ${{ tasks.verdict-standard-main.skipped }} + cache: false + timeout: 10m + outputs: + filesystem: false + artifacts: + - key: receipt + path: .ci/pr-gates/groups/contracts-suites diff --git a/.rwx/go-tests.yml b/.rwx/go-tests.yml new file mode 100644 index 00000000000..bbe24fa84c1 --- /dev/null +++ b/.rwx/go-tests.yml @@ -0,0 +1,668 @@ +# Full aggregate Go workload. CircleCI retains all required gates. +on: + cache-rebuild: + if: ${{ event.git.branch == 'develop' }} + target: [go, contracts, kona, op-reth, prestate, compile-0, compile-1, compile-2, compile-3, compile-4, compile-5, compile-6, compile-7, compile-8, compile-9, compile-10, compile-11] + init: + commit-sha: ${{ event.git.sha }} + branch: ${{ event.git.branch }} + cache-warm: "true" + cache-epoch: v1 + build-probe: "" + target-cache-mode: keep + cli: + init: + commit-sha: ${{ event.git.sha }} + branch: ${{ event.git.branch }} + cache-warm: "false" + cache-epoch: v1 + build-probe: "" + target-cache-mode: keep + +# Shared cache baseline: only develop may write; PRs restore it read-only. +tool-cache: + vault: optimism-op-reth-shadow + +base: + image: ubuntu:24.04@sha256:008173c23f95b170204355c12626cb5a965d779a7e1283b09e9cffbb1bf33ca3 + config: rwx/base 1.2.0 + arch: x86_64 + +defaults: + runner: + cpus: 2 + memory: 8gb + +tasks: + - key: code + call: git/clone 2.2.0 + with: + repository: https://github.com/ethereum-optimism/optimism.git + ref: ${{ init.commit-sha }} + preserve-git-dir: true + fetch-full-depth: true + submodules: false + + - key: bootstrap-inputs + use: code + run: 'true' + filter: [mise.toml, ops/ci/runtime] + outputs: + filesystem: + filter: + workspace: [mise.toml, ops/ci/runtime/apt-install.sh, ops/ci/runtime/rwx-prepare.sh, ops/ci/runtime/rwx-contracts-prepare.sh, ops/ci/runtime/rwx-rust-prepare.sh, ops/ci/runtime/op-reth-shadow.sh, ops/ci/runtime/op-reth-report.py, ops/ci/runtime/rust-target-cache.py] + system: [] + artifacts: + - key: ci-scripts + path: ops/ci/runtime + + - key: mise + call: mise/install 1.1.0 + with: + mise-version: "2026.2.2" + install: "false" + + # The filtered bootstrap layer keeps the checkout's Git history out of + # reusable tools. Language toolchains are added only by their own workload. + - key: tools + use: [mise, bootstrap-inputs] + call: ${{ run.dir }}/packages/toolchain-common.yml + + - key: go-build-tools + use: tools + call: ${{ run.dir }}/packages/toolchain-go.yml + with: + mode: go + + - key: route + use: [code, tools] + run: bash ops/ci/runtime/rwx-metadata.sh + env: + CI_CACHE_WARM: ${{ init.cache-warm }} + CI_BRANCH: ${{ init.branch }} + CI_COMMIT_SHA: ${{ init.commit-sha }} + cache: false + timeout: 10m + outputs: + filesystem: false + + - key: contract-tools + use: [bootstrap-inputs, tools] + call: ${{ run.dir }}/packages/toolchain-foundry.yml + with: + mode: anvil + + - key: helper-tests + use: [code, go-build-tools] + if: ${{ init.cache-warm != 'true' }} + run: mise exec -- python3 -m unittest discover -s ops/ci/tests -p 'test_go*.py' + cache: false + timeout: 10m + outputs: + filesystem: false + + # This is a runtime preflight: only it and fresh verdicts receive RPC inputs. + - key: rpc-preflight + use: tools + if: ${{ init.cache-warm != 'true' && tasks.route.values.run-main == 'true' }} + run: python3 "$CI_SCRIPTS/go-rpc-preflight.py" + filter: + ${{ tasks.bootstrap-inputs.artifacts.ci-scripts }}: [go-rpc-preflight.py] + env: + CI_SCRIPTS: ${{ tasks.bootstrap-inputs.artifacts.ci-scripts }} + OP_CI_MAINNET_L1_ARCHIVE_RPC_URL: ${{ vaults.optimism-go-tests-rpc-shadow.secrets.OP_CI_MAINNET_L1_ARCHIVE_RPC_URL }} + OP_CI_SEPOLIA_L1_ARCHIVE_RPC_URL: ${{ vaults.optimism-go-tests-rpc-shadow.secrets.OP_CI_SEPOLIA_L1_ARCHIVE_RPC_URL }} + cache: false + outputs: + filesystem: false + + - key: source + use: [code, tools] + after: ${{ rpc-preflight.succeeded || rpc-preflight.skipped }} + if: ${{ init.cache-warm == 'true' || tasks.route.values.run-main == 'true' }} + run: | + git submodule sync --recursive + git submodule update --init --recursive --jobs 8 + git rev-parse HEAD >packages/contracts-bedrock/.gitcommit + timeout: 20m + + # A producer checks out the commit that last changed its build inputs + # (dependency-plan), so its cache key holds for later commits and for every + # workload that calls it. A consumer checks that those inputs equal HEAD's + # before it restores the output. + - key: dependency-plan + use: code + run: python3 ops/ci/runtime/go-dependency-source.py plan + outputs: + filesystem: false + + - key: rust-tools + use: [bootstrap-inputs, tools] + if: ${{ init.cache-warm == 'true' || tasks.route.values.run-main == 'true' }} + call: ${{ run.dir }}/packages/toolchain-rust.yml + with: + mode: release + + - key: go + use: go-build-tools + if: ${{ init.cache-warm == 'true' || tasks.route.values.run-main == 'true' }} + call: ${{ run.dir }}/packages/go-go.yml + with: + commit-sha: ${{ tasks.dependency-plan.values.go }} + cache-epoch: ${{ init.cache-epoch }} + build-probe: ${{ init.build-probe }} + target-cache-mode: ${{ init.target-cache-mode }} + + - key: contracts + use: [go-build-tools, contract-tools] + if: ${{ init.cache-warm == 'true' || tasks.route.values.run-main == 'true' }} + call: ${{ run.dir }}/packages/go-contracts.yml + with: + commit-sha: ${{ tasks.dependency-plan.values.contracts }} + cache-epoch: ${{ init.cache-epoch }} + build-probe: ${{ init.build-probe }} + target-cache-mode: ${{ init.target-cache-mode }} + + - key: kona + use: rust-tools + if: ${{ init.cache-warm == 'true' || tasks.route.values.run-main == 'true' }} + call: ${{ run.dir }}/packages/go-kona.yml + with: + commit-sha: ${{ tasks.dependency-plan.values.kona }} + cache-epoch: ${{ init.cache-epoch }} + build-probe: ${{ init.build-probe }} + target-cache-mode: ${{ init.target-cache-mode }} + + - key: op-reth + use: rust-tools + if: ${{ init.cache-warm == 'true' || tasks.route.values.run-main == 'true' }} + call: ${{ run.dir }}/packages/go-op-reth.yml + with: + commit-sha: ${{ tasks.dependency-plan.values.op-reth }} + cache-epoch: ${{ init.cache-epoch }} + build-probe: ${{ init.build-probe }} + target-cache-mode: ${{ init.target-cache-mode }} + + - key: prestate + use: go-build-tools + if: ${{ init.cache-warm == 'true' || tasks.route.values.run-main == 'true' }} + call: ${{ run.dir }}/packages/go-prestate.yml + with: + commit-sha: ${{ tasks.dependency-plan.values.prestate }} + cache-epoch: ${{ init.cache-epoch }} + build-probe: ${{ init.build-probe }} + target-cache-mode: ${{ init.target-cache-mode }} + + # The compiler seeds are reusable in the same way as the dependency builds: each + # task starts without a checkout, checks out the planned commit itself and passes + # on only what it built. Current-source compilation and every verdict still use + # the discovery of the current commit. + - key: compiler-preparation + if: ${{ init.cache-warm == 'true' || tasks.route.values.run-main == 'true' }} + use: [go-build-tools, contracts.build] + filter: [mise.toml, .ci/go-tests/dependencies/contracts] + run: | + git init -q . + git remote add origin https://github.com/ethereum-optimism/optimism.git + git fetch -q origin "$CI_COMMIT_SHA" + git checkout -qf FETCH_HEAD + git submodule update --init --recursive --jobs 8 + export GOMODCACHE="$PWD/.ci/go-cache/full/modules" GOCACHE="$PWD/.ci/go-cache/full/build" + go mod download + just sync-superchain-go + git diff --exit-code op-core/superchain/bundle/superchain-configs.zip.sha256 + just cannon + python3 ops/ci/runtime/go-artifacts.py restore contracts .ci/go-tests/dependencies/contracts + mise exec -- bash ops/ci/runtime/go-full-tests.sh discover + env: + CI_COMMIT_SHA: ${{ tasks.dependency-plan.values.compiler }} + COMPILER_CACHE_EPOCH: ${{ init.cache-epoch }} + timeout: 30m + outputs: + filesystem: + filter: + workspace: &go-prepared [.ci/go-tests/manifest.json, .ci/go-cache/full/modules, cannon/bin, cannon/multicannon/embeds, cannon/testdata/bin, op-core/superchain/bundle/superchain-configs.zip, packages/contracts-bedrock/cache, packages/contracts-bedrock/artifacts, packages/contracts-bedrock/forge-artifacts, op-deployer/pkg/deployer/artifacts/forge-artifacts] + system: [] + + - &compiler-seed + key: compiler-seed-0 + if: ${{ init.cache-warm == 'true' || tasks.route.values.run-main == 'true' }} + use: compiler-preparation + filter: [mise.toml, *go-prepared] + run: | + git init -q . + git remote add origin https://github.com/ethereum-optimism/optimism.git + git fetch -q origin "$CI_COMMIT_SHA" + git checkout -qf FETCH_HEAD + git submodule update --init --recursive --jobs 8 + mise exec -- bash ops/ci/runtime/go-full-tests.sh build + env: &compiler-seed-env + CI_COMMIT_SHA: ${{ tasks.dependency-plan.values.compiler }} + CI_SHARD_INDEX: '0' + COMPILER_CACHE_EPOCH: ${{ init.cache-epoch }} + timeout: 60m + runner: + cpus: 8 + memory: 16gb + outputs: + filesystem: + filter: + workspace: [.ci/go-cache/full/build] + system: [] + + - <<: *compiler-seed + key: compiler-seed-1 + env: + <<: *compiler-seed-env + CI_SHARD_INDEX: '1' + + - <<: *compiler-seed + key: compiler-seed-2 + env: + <<: *compiler-seed-env + CI_SHARD_INDEX: '2' + + - <<: *compiler-seed + key: compiler-seed-3 + env: + <<: *compiler-seed-env + CI_SHARD_INDEX: '3' + + - <<: *compiler-seed + key: compiler-seed-4 + env: + <<: *compiler-seed-env + CI_SHARD_INDEX: '4' + + - <<: *compiler-seed + key: compiler-seed-5 + env: + <<: *compiler-seed-env + CI_SHARD_INDEX: '5' + + - <<: *compiler-seed + key: compiler-seed-6 + env: + <<: *compiler-seed-env + CI_SHARD_INDEX: '6' + + - <<: *compiler-seed + key: compiler-seed-7 + env: + <<: *compiler-seed-env + CI_SHARD_INDEX: '7' + + - <<: *compiler-seed + key: compiler-seed-8 + env: + <<: *compiler-seed-env + CI_SHARD_INDEX: '8' + + - <<: *compiler-seed + key: compiler-seed-9 + env: + <<: *compiler-seed-env + CI_SHARD_INDEX: '9' + + - <<: *compiler-seed + key: compiler-seed-10 + env: + <<: *compiler-seed-env + CI_SHARD_INDEX: '10' + + - <<: *compiler-seed + key: compiler-seed-11 + env: + <<: *compiler-seed-env + CI_SHARD_INDEX: '11' + + - key: discovery + use: [go-build-tools, go.build, contracts.build, source] + if: ${{ init.cache-warm == 'true' || tasks.route.values.run-main == 'true' }} + run: | + if ! git cat-file -e "$CI_COMMIT_SHA^{commit}"; then git fetch --no-tags origin "$CI_COMMIT_SHA"; fi + git reset --hard "$CI_COMMIT_SHA" + git submodule sync --recursive + git submodule update --init --recursive --jobs 8 + git rev-parse HEAD >packages/contracts-bedrock/.gitcommit + python3 ops/ci/runtime/go-artifacts.py restore go .ci/go-tests/dependencies/go + python3 ops/ci/runtime/go-artifacts.py restore contracts .ci/go-tests/dependencies/contracts + mise exec -- bash ops/ci/runtime/go-full-tests.sh discover + env: + CI_COMMIT_SHA: ${{ init.commit-sha }} + outputs: + filesystem: + filter: + workspace: *go-prepared + system: [] + filter: ['!.ci/go-cache/full/dependencies'] + + - key: compile-0 + use: [go-build-tools, compiler-seed-0, source, discovery] + if: ${{ (init.cache-warm == 'true' || tasks.route.values.run-main == 'true') }} + call: ${{ run.dir }}/packages/go-test-compile.yml + with: + commit-sha: ${{ init.commit-sha }} + shard-index: '0' + build-probe: ${{ init.build-probe }} + cache-key: go-full-compile-0-${{ init.cache-epoch }} + + - key: verdict-0 + use: [go-build-tools, contract-tools, go.build, contracts.build, kona.build, op-reth.build, prestate.build, source, discovery, compile-0.build] + if: ${{ init.cache-warm != 'true' && tasks.route.values.run-main == 'true' }} + call: ${{ run.dir }}/packages/go-test-verdict.yml + with: + commit-sha: ${{ init.commit-sha }} + branch: ${{ init.branch }} + shard-index: '0' + cache-key: go-full-runtime-0-${{ init.cache-epoch }} + + - key: compile-1 + use: [go-build-tools, compiler-seed-1, source, discovery] + if: ${{ (init.cache-warm == 'true' || tasks.route.values.run-main == 'true') }} + call: ${{ run.dir }}/packages/go-test-compile.yml + with: + commit-sha: ${{ init.commit-sha }} + shard-index: '1' + build-probe: ${{ init.build-probe }} + cache-key: go-full-compile-1-${{ init.cache-epoch }} + + - key: verdict-1 + use: [go-build-tools, contract-tools, go.build, contracts.build, kona.build, op-reth.build, prestate.build, source, discovery, compile-1.build] + if: ${{ init.cache-warm != 'true' && tasks.route.values.run-main == 'true' }} + call: ${{ run.dir }}/packages/go-test-verdict.yml + with: + commit-sha: ${{ init.commit-sha }} + branch: ${{ init.branch }} + shard-index: '1' + cache-key: go-full-runtime-1-${{ init.cache-epoch }} + + - key: compile-2 + use: [go-build-tools, compiler-seed-2, source, discovery] + if: ${{ (init.cache-warm == 'true' || tasks.route.values.run-main == 'true') }} + call: ${{ run.dir }}/packages/go-test-compile.yml + with: + commit-sha: ${{ init.commit-sha }} + shard-index: '2' + build-probe: ${{ init.build-probe }} + cache-key: go-full-compile-2-${{ init.cache-epoch }} + + - key: verdict-2 + use: [go-build-tools, contract-tools, go.build, contracts.build, kona.build, op-reth.build, prestate.build, source, discovery, compile-2.build] + if: ${{ init.cache-warm != 'true' && tasks.route.values.run-main == 'true' }} + call: ${{ run.dir }}/packages/go-test-verdict.yml + with: + commit-sha: ${{ init.commit-sha }} + branch: ${{ init.branch }} + shard-index: '2' + cache-key: go-full-runtime-2-${{ init.cache-epoch }} + + - key: compile-3 + use: [go-build-tools, compiler-seed-3, source, discovery] + if: ${{ (init.cache-warm == 'true' || tasks.route.values.run-main == 'true') }} + call: ${{ run.dir }}/packages/go-test-compile.yml + with: + commit-sha: ${{ init.commit-sha }} + shard-index: '3' + build-probe: ${{ init.build-probe }} + cache-key: go-full-compile-3-${{ init.cache-epoch }} + + - key: verdict-3 + use: [go-build-tools, contract-tools, go.build, contracts.build, kona.build, op-reth.build, prestate.build, source, discovery, compile-3.build] + if: ${{ init.cache-warm != 'true' && tasks.route.values.run-main == 'true' }} + call: ${{ run.dir }}/packages/go-test-verdict.yml + with: + commit-sha: ${{ init.commit-sha }} + branch: ${{ init.branch }} + shard-index: '3' + cache-key: go-full-runtime-3-${{ init.cache-epoch }} + + - key: compile-4 + use: [go-build-tools, compiler-seed-4, source, discovery] + if: ${{ (init.cache-warm == 'true' || tasks.route.values.run-main == 'true') }} + call: ${{ run.dir }}/packages/go-test-compile.yml + with: + commit-sha: ${{ init.commit-sha }} + shard-index: '4' + build-probe: ${{ init.build-probe }} + cache-key: go-full-compile-4-${{ init.cache-epoch }} + + - key: verdict-4 + use: [go-build-tools, contract-tools, go.build, contracts.build, kona.build, op-reth.build, prestate.build, source, discovery, compile-4.build] + if: ${{ init.cache-warm != 'true' && tasks.route.values.run-main == 'true' }} + call: ${{ run.dir }}/packages/go-test-verdict.yml + with: + commit-sha: ${{ init.commit-sha }} + branch: ${{ init.branch }} + shard-index: '4' + cache-key: go-full-runtime-4-${{ init.cache-epoch }} + + - key: compile-5 + use: [go-build-tools, compiler-seed-5, source, discovery] + if: ${{ (init.cache-warm == 'true' || tasks.route.values.run-main == 'true') }} + call: ${{ run.dir }}/packages/go-test-compile.yml + with: + commit-sha: ${{ init.commit-sha }} + shard-index: '5' + build-probe: ${{ init.build-probe }} + cache-key: go-full-compile-5-${{ init.cache-epoch }} + + - key: verdict-5 + use: [go-build-tools, contract-tools, go.build, contracts.build, kona.build, op-reth.build, prestate.build, source, discovery, compile-5.build] + if: ${{ init.cache-warm != 'true' && tasks.route.values.run-main == 'true' }} + call: ${{ run.dir }}/packages/go-test-verdict.yml + with: + commit-sha: ${{ init.commit-sha }} + branch: ${{ init.branch }} + shard-index: '5' + cache-key: go-full-runtime-5-${{ init.cache-epoch }} + + - key: compile-6 + use: [go-build-tools, compiler-seed-6, source, discovery] + if: ${{ (init.cache-warm == 'true' || tasks.route.values.run-main == 'true') }} + call: ${{ run.dir }}/packages/go-test-compile.yml + with: + commit-sha: ${{ init.commit-sha }} + shard-index: '6' + build-probe: ${{ init.build-probe }} + cache-key: go-full-compile-6-${{ init.cache-epoch }} + + - key: verdict-6 + use: [go-build-tools, contract-tools, go.build, contracts.build, kona.build, op-reth.build, prestate.build, source, discovery, compile-6.build] + if: ${{ init.cache-warm != 'true' && tasks.route.values.run-main == 'true' }} + call: ${{ run.dir }}/packages/go-test-verdict.yml + with: + commit-sha: ${{ init.commit-sha }} + branch: ${{ init.branch }} + shard-index: '6' + cache-key: go-full-runtime-6-${{ init.cache-epoch }} + + - key: compile-7 + use: [go-build-tools, compiler-seed-7, source, discovery] + if: ${{ (init.cache-warm == 'true' || tasks.route.values.run-main == 'true') }} + call: ${{ run.dir }}/packages/go-test-compile.yml + with: + commit-sha: ${{ init.commit-sha }} + shard-index: '7' + build-probe: ${{ init.build-probe }} + cache-key: go-full-compile-7-${{ init.cache-epoch }} + + - key: verdict-7 + use: [go-build-tools, contract-tools, go.build, contracts.build, kona.build, op-reth.build, prestate.build, source, discovery, compile-7.build] + if: ${{ init.cache-warm != 'true' && tasks.route.values.run-main == 'true' }} + call: ${{ run.dir }}/packages/go-test-verdict.yml + with: + commit-sha: ${{ init.commit-sha }} + branch: ${{ init.branch }} + shard-index: '7' + cache-key: go-full-runtime-7-${{ init.cache-epoch }} + + - key: compile-8 + use: [go-build-tools, compiler-seed-8, source, discovery] + if: ${{ (init.cache-warm == 'true' || tasks.route.values.run-main == 'true') }} + call: ${{ run.dir }}/packages/go-test-compile.yml + with: + commit-sha: ${{ init.commit-sha }} + shard-index: '8' + build-probe: ${{ init.build-probe }} + cache-key: go-full-compile-8-${{ init.cache-epoch }} + + - key: verdict-8 + use: [go-build-tools, contract-tools, go.build, contracts.build, kona.build, op-reth.build, prestate.build, source, discovery, compile-8.build] + if: ${{ init.cache-warm != 'true' && tasks.route.values.run-main == 'true' }} + call: ${{ run.dir }}/packages/go-test-verdict.yml + with: + commit-sha: ${{ init.commit-sha }} + branch: ${{ init.branch }} + shard-index: '8' + cache-key: go-full-runtime-8-${{ init.cache-epoch }} + + - key: compile-9 + use: [go-build-tools, compiler-seed-9, source, discovery] + if: ${{ (init.cache-warm == 'true' || tasks.route.values.run-main == 'true') }} + call: ${{ run.dir }}/packages/go-test-compile.yml + with: + commit-sha: ${{ init.commit-sha }} + shard-index: '9' + build-probe: ${{ init.build-probe }} + cache-key: go-full-compile-9-${{ init.cache-epoch }} + + - key: verdict-9 + use: [go-build-tools, contract-tools, go.build, contracts.build, kona.build, op-reth.build, prestate.build, source, discovery, compile-9.build] + if: ${{ init.cache-warm != 'true' && tasks.route.values.run-main == 'true' }} + call: ${{ run.dir }}/packages/go-test-verdict.yml + with: + commit-sha: ${{ init.commit-sha }} + branch: ${{ init.branch }} + shard-index: '9' + cache-key: go-full-runtime-9-${{ init.cache-epoch }} + + - key: compile-10 + use: [go-build-tools, compiler-seed-10, source, discovery] + if: ${{ (init.cache-warm == 'true' || tasks.route.values.run-main == 'true') }} + call: ${{ run.dir }}/packages/go-test-compile.yml + with: + commit-sha: ${{ init.commit-sha }} + shard-index: '10' + build-probe: ${{ init.build-probe }} + cache-key: go-full-compile-10-${{ init.cache-epoch }} + + - key: verdict-10 + use: [go-build-tools, contract-tools, go.build, contracts.build, kona.build, op-reth.build, prestate.build, source, discovery, compile-10.build] + if: ${{ init.cache-warm != 'true' && tasks.route.values.run-main == 'true' }} + call: ${{ run.dir }}/packages/go-test-verdict.yml + with: + commit-sha: ${{ init.commit-sha }} + branch: ${{ init.branch }} + shard-index: '10' + cache-key: go-full-runtime-10-${{ init.cache-epoch }} + + - key: compile-11 + use: [go-build-tools, compiler-seed-11, source, discovery] + if: ${{ (init.cache-warm == 'true' || tasks.route.values.run-main == 'true') }} + call: ${{ run.dir }}/packages/go-test-compile.yml + with: + commit-sha: ${{ init.commit-sha }} + shard-index: '11' + build-probe: ${{ init.build-probe }} + cache-key: go-full-compile-11-${{ init.cache-epoch }} + + - key: verdict-11 + use: [go-build-tools, contract-tools, go.build, contracts.build, kona.build, op-reth.build, prestate.build, source, discovery, compile-11.build] + if: ${{ init.cache-warm != 'true' && tasks.route.values.run-main == 'true' }} + call: ${{ run.dir }}/packages/go-test-verdict.yml + with: + commit-sha: ${{ init.commit-sha }} + branch: ${{ init.branch }} + shard-index: '11' + cache-key: go-full-runtime-11-${{ init.cache-epoch }} + + # op-e2e/actions has its own Circle job (go-actions-tests); list-test-packages + # excludes it from the general verdicts, so it never runs twice. + - key: actions-verdict + use: [go-build-tools, go.build, contracts.build, op-reth.build, source] + if: ${{ init.cache-warm != 'true' && tasks.route.values.run-main == 'true' }} + run: | + if ! git cat-file -e "$CI_COMMIT_SHA^{commit}"; then git fetch --no-tags origin "$CI_COMMIT_SHA"; fi + git reset --hard "$CI_COMMIT_SHA" + git submodule sync --recursive + git submodule update --init --recursive --jobs 8 + git rev-parse HEAD >packages/contracts-bedrock/.gitcommit + python3 ops/ci/runtime/go-artifacts.py restore contracts .ci/go-tests/dependencies/contracts + python3 ops/ci/runtime/go-artifacts.py restore op-reth .ci/go-tests/dependencies/op-reth + export RUST_BINARY_PATH_OP_RETH_TEST_ENGINE="$PWD/rust/target/release/op-reth-test-engine" + test -x "$RUST_BINARY_PATH_OP_RETH_TEST_ENGINE" + export GOMODCACHE="$PWD/.ci/go-cache/full/modules" GOCACHE="$PWD/.ci/go-cache/actions/run" + mkdir -p "$GOCACHE" + TEST_TIMEOUT=30m mise exec -- just go-actions-tests-ci + env: + CI_COMMIT_SHA: ${{ init.commit-sha }} + CIRCLE_BRANCH: ${{ init.branch }} + cache: false + timeout: 60m + runner: + cpus: 16 + memory: 32gb + outputs: + filesystem: false + test-results: + - path: tmp/test-results/results.xml + artifacts: + - key: junit + path: tmp/test-results + - key: test-logs + path: tmp/testlogs + + - key: main-gate-receipt + use: [code, tools] + after: ${{ (actions-verdict.succeeded || actions-verdict.failed || actions-verdict.skipped) && (verdict-0.succeeded || verdict-0.failed || verdict-0.skipped) && (verdict-1.succeeded || verdict-1.failed || verdict-1.skipped) && (verdict-10.succeeded || verdict-10.failed || verdict-10.skipped) && (verdict-11.succeeded || verdict-11.failed || verdict-11.skipped) && (verdict-2.succeeded || verdict-2.failed || verdict-2.skipped) && (verdict-3.succeeded || verdict-3.failed || verdict-3.skipped) && (verdict-4.succeeded || verdict-4.failed || verdict-4.skipped) && (verdict-5.succeeded || verdict-5.failed || verdict-5.skipped) && (verdict-6.succeeded || verdict-6.failed || verdict-6.skipped) && (verdict-7.succeeded || verdict-7.failed || verdict-7.skipped) && (verdict-8.succeeded || verdict-8.failed || verdict-8.skipped) && (verdict-9.succeeded || verdict-9.failed || verdict-9.skipped) }} + if: ${{ init.cache-warm != 'true' }} + run: python3 ops/ci/runtime/pr-gate.py receipt main-go + env: + GROUP_SELECTED: ${{ tasks.route.values.run-main }} + TASK_ACTIONS_VERDICT_SUCCEEDED: ${{ tasks.actions-verdict.succeeded }} + TASK_ACTIONS_VERDICT_FAILED: ${{ tasks.actions-verdict.failed }} + TASK_ACTIONS_VERDICT_SKIPPED: ${{ tasks.actions-verdict.skipped }} + TASK_VERDICT_0_SUCCEEDED: ${{ tasks.verdict-0.succeeded }} + TASK_VERDICT_0_FAILED: ${{ tasks.verdict-0.failed }} + TASK_VERDICT_0_SKIPPED: ${{ tasks.verdict-0.skipped }} + TASK_VERDICT_1_SUCCEEDED: ${{ tasks.verdict-1.succeeded }} + TASK_VERDICT_1_FAILED: ${{ tasks.verdict-1.failed }} + TASK_VERDICT_1_SKIPPED: ${{ tasks.verdict-1.skipped }} + TASK_VERDICT_10_SUCCEEDED: ${{ tasks.verdict-10.succeeded }} + TASK_VERDICT_10_FAILED: ${{ tasks.verdict-10.failed }} + TASK_VERDICT_10_SKIPPED: ${{ tasks.verdict-10.skipped }} + TASK_VERDICT_11_SUCCEEDED: ${{ tasks.verdict-11.succeeded }} + TASK_VERDICT_11_FAILED: ${{ tasks.verdict-11.failed }} + TASK_VERDICT_11_SKIPPED: ${{ tasks.verdict-11.skipped }} + TASK_VERDICT_2_SUCCEEDED: ${{ tasks.verdict-2.succeeded }} + TASK_VERDICT_2_FAILED: ${{ tasks.verdict-2.failed }} + TASK_VERDICT_2_SKIPPED: ${{ tasks.verdict-2.skipped }} + TASK_VERDICT_3_SUCCEEDED: ${{ tasks.verdict-3.succeeded }} + TASK_VERDICT_3_FAILED: ${{ tasks.verdict-3.failed }} + TASK_VERDICT_3_SKIPPED: ${{ tasks.verdict-3.skipped }} + TASK_VERDICT_4_SUCCEEDED: ${{ tasks.verdict-4.succeeded }} + TASK_VERDICT_4_FAILED: ${{ tasks.verdict-4.failed }} + TASK_VERDICT_4_SKIPPED: ${{ tasks.verdict-4.skipped }} + TASK_VERDICT_5_SUCCEEDED: ${{ tasks.verdict-5.succeeded }} + TASK_VERDICT_5_FAILED: ${{ tasks.verdict-5.failed }} + TASK_VERDICT_5_SKIPPED: ${{ tasks.verdict-5.skipped }} + TASK_VERDICT_6_SUCCEEDED: ${{ tasks.verdict-6.succeeded }} + TASK_VERDICT_6_FAILED: ${{ tasks.verdict-6.failed }} + TASK_VERDICT_6_SKIPPED: ${{ tasks.verdict-6.skipped }} + TASK_VERDICT_7_SUCCEEDED: ${{ tasks.verdict-7.succeeded }} + TASK_VERDICT_7_FAILED: ${{ tasks.verdict-7.failed }} + TASK_VERDICT_7_SKIPPED: ${{ tasks.verdict-7.skipped }} + TASK_VERDICT_8_SUCCEEDED: ${{ tasks.verdict-8.succeeded }} + TASK_VERDICT_8_FAILED: ${{ tasks.verdict-8.failed }} + TASK_VERDICT_8_SKIPPED: ${{ tasks.verdict-8.skipped }} + TASK_VERDICT_9_SUCCEEDED: ${{ tasks.verdict-9.succeeded }} + TASK_VERDICT_9_FAILED: ${{ tasks.verdict-9.failed }} + TASK_VERDICT_9_SKIPPED: ${{ tasks.verdict-9.skipped }} + cache: false + timeout: 10m + outputs: + filesystem: false + artifacts: + - key: receipt + path: .ci/pr-gates/groups/main-go diff --git a/.rwx/kontrol-build.yml b/.rwx/kontrol-build.yml new file mode 100644 index 00000000000..da87fd0f094 --- /dev/null +++ b/.rwx/kontrol-build.yml @@ -0,0 +1,177 @@ +# Full Kontrol summary generation and proof build; Circle retains required ownership. +on: + cache-rebuild: + if: ${{ event.git.branch == 'develop' }} + target: [contracts, image] + init: + commit-sha: ${{ event.git.sha }} + branch: ${{ event.git.branch }} + cache-warm: "true" + cache-epoch: v1 + cli: + init: + commit-sha: ${{ event.git.sha }} + branch: ${{ event.git.branch }} + cache-warm: "false" + cache-epoch: v1 + +tool-cache: + vault: optimism-op-reth-shadow + +base: + image: ubuntu:24.04@sha256:008173c23f95b170204355c12626cb5a965d779a7e1283b09e9cffbb1bf33ca3 + config: rwx/base 1.2.0 + arch: x86_64 + +defaults: + runner: + cpus: 2 + memory: 8gb + +tasks: + - key: code + call: git/clone 2.2.0 + with: + repository: https://github.com/ethereum-optimism/optimism.git + ref: ${{ init.commit-sha }} + preserve-git-dir: true + fetch-full-depth: true + submodules: false + + - key: bootstrap-inputs + use: code + run: 'true' + filter: [mise.toml, ops/ci/runtime] + outputs: + filesystem: + filter: + workspace: [mise.toml, ops/ci/runtime/apt-install.sh, ops/ci/runtime/rwx-prepare.sh, ops/ci/runtime/rwx-contracts-prepare.sh, ops/ci/runtime/rwx-rust-prepare.sh, ops/ci/runtime/op-reth-shadow.sh, ops/ci/runtime/op-reth-report.py, ops/ci/runtime/rust-target-cache.py] + system: [] + + - key: mise + call: mise/install 1.1.0 + with: + mise-version: "2026.2.2" + install: "false" + + - key: tools + use: [mise, bootstrap-inputs] + call: ${{ run.dir }}/packages/toolchain-common.yml + + - key: route + use: [code, tools] + run: bash ops/ci/runtime/rwx-metadata.sh + env: + CI_CACHE_WARM: ${{ init.cache-warm }} + CI_BRANCH: ${{ init.branch }} + CI_COMMIT_SHA: ${{ init.commit-sha }} + cache: false + timeout: 10m + outputs: + filesystem: false + + - key: contract-tools + use: [bootstrap-inputs, tools] + if: ${{ init.cache-warm == 'true' || tasks.route.values.run-main == 'true' }} + call: ${{ run.dir }}/packages/toolchain-foundry.yml + with: + mode: anvil + + - key: go-tools + use: tools + if: ${{ init.cache-warm == 'true' || tasks.route.values.run-main == 'true' }} + call: ${{ run.dir }}/packages/toolchain-go.yml + with: + mode: go + + # Pull the image that test/kontrol/scripts/common.sh runs, so that this task + # retries a registry failure and the verdict task does not download it. + - key: image + use: [code, tools] + if: ${{ init.cache-warm == 'true' || tasks.route.values.run-main == 'true' }} + filter: [mise.toml] + run: | + image="runtimeverificationinc/kontrol:ubuntu-jammy-$(yq '.tools.kontrol' mise.toml)" + for attempt in 1 2 3 4 5; do + if docker pull "$image"; then exit 0; fi + sleep "$((2 ** attempt))" + done + exit 1 + docker: preserve-data + timeout: 30m + # The task changes Docker data only; it hands no file to later tasks. + outputs: + filesystem: + filter: + workspace: [] + system: [] + + - key: source + use: [code, tools] + if: ${{ init.cache-warm == 'true' || tasks.route.values.run-main == 'true' }} + run: | + git submodule sync --recursive + git -c protocol.file.allow=never submodule update --init --recursive --jobs 8 + git rev-parse HEAD >packages/contracts-bedrock/.gitcommit + timeout: 20m + + # A dependency build checks out the commit that last changed its build inputs, so it + # is a cache hit for later commits and for the other workloads of the run. + - key: dependency-plan + use: code + run: python3 ops/ci/runtime/go-dependency-source.py plan + outputs: + filesystem: false + + # CircleCI's contracts-bedrock-build, which check-kontrol-build requires. + - key: contracts + use: [go-tools, contract-tools] + if: ${{ init.cache-warm == 'true' || tasks.route.values.run-main == 'true' }} + call: ${{ run.dir }}/packages/go-contracts.yml + with: + commit-sha: ${{ tasks.dependency-plan.values.contracts }} + cache-epoch: ${{ init.cache-epoch }} + build-probe: '' + target-cache-mode: keep + + # CircleCI's check-kontrol-build. The exit status of its two recipes is the + # verdict. Like CircleCI, the task sets no FOUNDRY_PROFILE. + - key: check + use: [source, go-tools, contract-tools, image] + if: ${{ init.cache-warm != 'true' && tasks.route.values.run-main == 'true' }} + run: | + python3 ops/ci/runtime/go-artifacts.py restore contracts "$CONTRACT_ARTIFACT" + cd packages/contracts-bedrock + just kontrol-summary-fp + just forge-build ./test/kontrol/proofs + env: + CI: "true" + CI_COMMIT_SHA: ${{ init.commit-sha }} + CONTRACT_ARTIFACT: ${{ tasks.contracts.tasks.build.artifacts.dependency }} + docker: true + cache: false + timeout: 90m + runner: + cpus: 2 + memory: 8gb + outputs: + filesystem: false + + # Exact Main dependency states; unrelated workload failures stay outside this gate. + - key: main-gate-receipt + use: [code, tools] + after: ${{ (check.succeeded || check.failed || check.skipped) }} + if: ${{ init.cache-warm != 'true' }} + run: python3 ops/ci/runtime/pr-gate.py receipt main-kontrol + env: + GROUP_SELECTED: ${{ tasks.route.values.run-main }} + TASK_CHECK_SUCCEEDED: ${{ tasks.check.succeeded }} + TASK_CHECK_FAILED: ${{ tasks.check.failed }} + TASK_CHECK_SKIPPED: ${{ tasks.check.skipped }} + cache: false + timeout: 10m + outputs: + filesystem: false + artifacts: + - key: receipt + path: .ci/pr-gates/groups/main-kontrol diff --git a/.rwx/nut-prefork.yml b/.rwx/nut-prefork.yml new file mode 100644 index 00000000000..69b6f6d9ba1 --- /dev/null +++ b/.rwx/nut-prefork.yml @@ -0,0 +1,206 @@ +# Complete NUT pre-fork state regeneration; Circle still owns its required gate. +on: + cache-rebuild: + if: ${{ event.git.branch == 'develop' }} + target: [modules, contracts, superchain, op-reth] + init: + commit-sha: ${{ event.git.sha }} + branch: ${{ event.git.branch }} + cache-warm: "true" + cache-epoch: v1 + cli: + init: + commit-sha: ${{ event.git.sha }} + branch: ${{ event.git.branch }} + cache-warm: "false" + cache-epoch: v1 + +tool-cache: + vault: optimism-op-reth-shadow + +base: + image: ubuntu:24.04@sha256:008173c23f95b170204355c12626cb5a965d779a7e1283b09e9cffbb1bf33ca3 + config: rwx/base 1.2.0 + arch: x86_64 + +defaults: + runner: + cpus: 2 + memory: 8gb + +tasks: + - key: code + call: git/clone 2.2.0 + with: + repository: https://github.com/ethereum-optimism/optimism.git + ref: ${{ init.commit-sha }} + preserve-git-dir: true + fetch-full-depth: true + submodules: false + + - key: bootstrap-inputs + use: code + run: 'true' + filter: [mise.toml, ops/ci/runtime] + outputs: + filesystem: + filter: + workspace: [mise.toml, ops/ci/runtime/apt-install.sh, ops/ci/runtime/rwx-prepare.sh, ops/ci/runtime/rwx-contracts-prepare.sh, ops/ci/runtime/rwx-rust-prepare.sh, ops/ci/runtime/op-reth-shadow.sh, ops/ci/runtime/op-reth-report.py, ops/ci/runtime/rust-target-cache.py] + system: [] + + - key: mise + call: mise/install 1.1.0 + with: + mise-version: "2026.2.2" + install: "false" + + - key: tools + use: [mise, bootstrap-inputs] + call: ${{ run.dir }}/packages/toolchain-common.yml + + - key: route + use: [code, tools] + run: bash ops/ci/runtime/rwx-metadata.sh + env: + CI_CACHE_WARM: ${{ init.cache-warm }} + CI_BRANCH: ${{ init.branch }} + CI_COMMIT_SHA: ${{ init.commit-sha }} + cache: false + timeout: 10m + outputs: + filesystem: false + + - key: go-tools + use: tools + if: ${{ init.cache-warm == 'true' || tasks.route.values.run-main == 'true' }} + call: ${{ run.dir }}/packages/toolchain-go.yml + with: + mode: go + + - key: contract-tools + use: [bootstrap-inputs, tools] + if: ${{ init.cache-warm == 'true' || tasks.route.values.run-main == 'true' }} + call: ${{ run.dir }}/packages/toolchain-foundry.yml + with: + mode: anvil + + - key: rust-tools + use: [bootstrap-inputs, tools] + if: ${{ init.cache-warm == 'true' || tasks.route.values.run-main == 'true' }} + call: ${{ run.dir }}/packages/toolchain-rust.yml + with: + mode: release + + - key: source + use: [code, tools] + if: ${{ init.cache-warm == 'true' || tasks.route.values.run-main == 'true' }} + run: | + git submodule sync --recursive + git -c protocol.file.allow=never submodule update --init --recursive --jobs 8 + git rev-parse HEAD >packages/contracts-bedrock/.gitcommit + timeout: 20m + + - key: modules + use: go-tools + if: ${{ init.cache-warm == 'true' || tasks.route.values.run-main == 'true' }} + call: ${{ run.dir }}/packages/go-modules.yml + with: + commit-sha: ${{ tasks.dependency-plan.values.go-modules }} + branch: ${{ init.branch }} + cache-epoch: ${{ init.cache-epoch }} + + # A dependency build checks out the commit that last changed its build inputs, so it + # is a cache hit for later commits and for the other workloads of the run. + - key: dependency-plan + use: code + run: python3 ops/ci/runtime/go-dependency-source.py plan + outputs: + filesystem: false + + - key: contracts + use: [go-tools, contract-tools] + if: ${{ init.cache-warm == 'true' || tasks.route.values.run-main == 'true' }} + call: ${{ run.dir }}/packages/go-contracts.yml + with: + commit-sha: ${{ tasks.dependency-plan.values.contracts }} + cache-epoch: ${{ init.cache-epoch }} + build-probe: '' + target-cache-mode: keep + + # CircleCI's prep-superchain: regenerate the bundle, then fail if its + # committed checksum changed. + - key: superchain + use: [source, go-tools] + if: ${{ init.cache-warm == 'true' || tasks.route.values.run-main == 'true' }} + run: | + just sync-superchain-go + git diff --exit-code op-core/superchain/bundle/superchain-configs.zip.sha256 + timeout: 15m + outputs: + filesystem: + filter: + workspace: [op-core/superchain/bundle/superchain-configs.zip] + system: [] + + # The generator runs the L2 chain on op-reth-test-engine. + - key: op-reth + use: rust-tools + if: ${{ init.cache-warm == 'true' || tasks.route.values.run-main == 'true' }} + call: ${{ run.dir }}/packages/go-op-reth.yml + with: + commit-sha: ${{ tasks.dependency-plan.values.op-reth }} + cache-epoch: ${{ init.cache-epoch }} + build-probe: '' + target-cache-mode: keep + + # CircleCI's check-nut-prefork-states: its three command lines, after the + # restores that replace attach_workspace. The recipe's exit status is the verdict. + - key: tests + use: [source, go-tools, contract-tools, superchain] + if: ${{ init.cache-warm != 'true' && tasks.route.values.run-main == 'true' }} + run: | + export GOMODCACHE="$PWD/.ci/go-cache/pr-checks/modules" + export GOCACHE="$PWD/.ci/go-cache/nut-prefork/build" + mkdir -p "$GOCACHE" + python3 ops/ci/runtime/go-artifacts.py restore go-modules "$MODULE_ARTIFACT" + python3 ops/ci/runtime/go-artifacts.py restore contracts "$CONTRACT_ARTIFACT" + python3 ops/ci/runtime/go-artifacts.py restore op-reth "$RETH_ARTIFACT" + export RUST_BINARY_PATH_OP_RETH_TEST_ENGINE="$PWD/rust/target/release/op-reth-test-engine" + test -x "$RUST_BINARY_PATH_OP_RETH_TEST_ENGINE" + just _check-nut-prefork-states + env: + CI: "true" + CI_COMMIT_SHA: ${{ init.commit-sha }} + MODULE_ARTIFACT: ${{ tasks.modules.tasks.build.artifacts.dependency }} + CONTRACT_ARTIFACT: ${{ tasks.contracts.tasks.build.artifacts.dependency }} + RETH_ARTIFACT: ${{ tasks.op-reth.tasks.build.artifacts.dependency }} + cache: false + tool-cache: nut-prefork-compiler-${{ init.cache-epoch }} + timeout: 45m + runner: + cpus: 2 + memory: 8gb + outputs: + filesystem: + filter: + workspace: [.ci/go-cache/nut-prefork/build] + system: [] + + # Exact Main dependency states; unrelated workload failures stay outside this gate. + - key: main-gate-receipt + use: [code, tools] + after: ${{ (tests.succeeded || tests.failed || tests.skipped) }} + if: ${{ init.cache-warm != 'true' }} + run: python3 ops/ci/runtime/pr-gate.py receipt main-nut-prefork + env: + GROUP_SELECTED: ${{ tasks.route.values.run-main }} + TASK_TESTS_SUCCEEDED: ${{ tasks.tests.succeeded }} + TASK_TESTS_FAILED: ${{ tasks.tests.failed }} + TASK_TESTS_SKIPPED: ${{ tasks.tests.skipped }} + cache: false + timeout: 10m + outputs: + filesystem: false + artifacts: + - key: receipt + path: .ci/pr-gates/groups/main-nut-prefork diff --git a/.rwx/nut-provenance.yml b/.rwx/nut-provenance.yml new file mode 100644 index 00000000000..96fdd98f6f4 --- /dev/null +++ b/.rwx/nut-provenance.yml @@ -0,0 +1,167 @@ +# Full recorded-source NUT provenance regeneration; Circle still owns its required gate. +on: + cache-rebuild: + if: ${{ event.git.branch == 'develop' }} + target: [modules, historical-tools] + init: + commit-sha: ${{ event.git.sha }} + branch: ${{ event.git.branch }} + cache-warm: "true" + cache-epoch: v1 + cli: + init: + commit-sha: ${{ event.git.sha }} + branch: ${{ event.git.branch }} + cache-warm: "false" + cache-epoch: v1 + +tool-cache: + vault: optimism-op-reth-shadow + +base: + image: ubuntu:24.04@sha256:008173c23f95b170204355c12626cb5a965d779a7e1283b09e9cffbb1bf33ca3 + config: rwx/base 1.2.0 + arch: x86_64 + +defaults: + runner: + cpus: 2 + memory: 8gb + +tasks: + - key: code + call: git/clone 2.2.0 + with: + repository: https://github.com/ethereum-optimism/optimism.git + ref: ${{ init.commit-sha }} + preserve-git-dir: true + fetch-full-depth: true + submodules: false + + - key: bootstrap-inputs + use: code + run: 'true' + filter: [mise.toml, ops/ci/runtime] + outputs: + filesystem: + filter: + workspace: [mise.toml, ops/ci/runtime/apt-install.sh, ops/ci/runtime/rwx-prepare.sh, ops/ci/runtime/rwx-contracts-prepare.sh, ops/ci/runtime/rwx-rust-prepare.sh, ops/ci/runtime/op-reth-shadow.sh, ops/ci/runtime/op-reth-report.py, ops/ci/runtime/rust-target-cache.py] + system: [] + + - key: mise + call: mise/install 1.1.0 + with: + mise-version: "2026.2.2" + install: "false" + + - key: tools + use: [mise, bootstrap-inputs] + call: ${{ run.dir }}/packages/toolchain-common.yml + + - key: route + use: [code, tools] + run: bash ops/ci/runtime/rwx-metadata.sh + env: + CI_CACHE_WARM: ${{ init.cache-warm }} + CI_BRANCH: ${{ init.branch }} + CI_COMMIT_SHA: ${{ init.commit-sha }} + cache: false + timeout: 10m + outputs: + filesystem: false + + - key: go-tools + use: tools + if: ${{ init.cache-warm == 'true' || tasks.route.values.run-main == 'true' }} + call: ${{ run.dir }}/packages/toolchain-go.yml + with: + mode: go + + # A dependency build checks out the commit that last changed its build inputs, so it + # is a cache hit for later commits and for the other workloads of the run. + - key: dependency-plan + use: code + run: python3 ops/ci/runtime/go-dependency-source.py plan + outputs: + filesystem: false + + - key: modules + use: go-tools + if: ${{ init.cache-warm == 'true' || tasks.route.values.run-main == 'true' }} + call: ${{ run.dir }}/packages/go-modules.yml + with: + commit-sha: ${{ tasks.dependency-plan.values.go-modules }} + branch: ${{ init.branch }} + cache-epoch: ${{ init.cache-epoch }} + + # The contract tool layer has the Solidity compilers that CircleCI installs. + - key: contract-tools + use: [bootstrap-inputs, tools] + if: ${{ init.cache-warm == 'true' || tasks.route.values.run-main == 'true' }} + call: ${{ run.dir }}/packages/toolchain-foundry.yml + with: + mode: compiler + + # Install the Forge and Just versions pinned at each locked bundle's recorded + # commit. mise auto-install is off in RWX, so the verdict task cannot. + - key: historical-tools + use: [code, contract-tools] + if: ${{ init.cache-warm == 'true' || tasks.route.values.run-main == 'true' }} + run: python3 ops/ci/runtime/nut-provenance.py --install-tools + tool-cache: nut-provenance-historical-tools-${{ init.cache-epoch }} + timeout: 30m + + - key: helper-tests + use: [code, tools] + if: ${{ init.cache-warm != 'true' && tasks.route.values.run-main == 'true' }} + run: python3 -m unittest discover -s ops/ci/tests -p 'test_nut_provenance.py' -v + cache: false + timeout: 15m + outputs: + filesystem: false + + # CircleCI's nut-provenance-verify, for every locked fork instead of the + # changed ones. The verifier's exit status is the verdict. + - key: verify + use: [code, historical-tools, go-tools] + if: ${{ init.cache-warm != 'true' && tasks.route.values.run-main == 'true' }} + run: | + export GOMODCACHE="$PWD/.ci/go-cache/pr-checks/modules" + export GOCACHE="$PWD/.ci/go-cache/nut-provenance/build" + mkdir -p "$GOCACHE" + python3 ops/ci/runtime/go-artifacts.py restore go-modules "$MODULE_ARTIFACT" + python3 ops/ci/runtime/nut-provenance.py + env: + CI: "true" + CI_COMMIT_SHA: ${{ init.commit-sha }} + MODULE_ARTIFACT: ${{ tasks.modules.tasks.build.artifacts.dependency }} + cache: false + tool-cache: nut-provenance-go-compiler-${{ init.cache-epoch }} + timeout: 45m + runner: + cpus: 16 + memory: 32gb + outputs: + filesystem: + filter: + workspace: [.ci/go-cache/nut-provenance/build] + system: [] + + # Exact Main dependency states; unrelated workload failures stay outside this gate. + - key: main-gate-receipt + use: [code, tools] + after: ${{ (verify.succeeded || verify.failed || verify.skipped) }} + if: ${{ init.cache-warm != 'true' }} + run: python3 ops/ci/runtime/pr-gate.py receipt main-nut-provenance + env: + GROUP_SELECTED: ${{ tasks.route.values.run-main }} + TASK_VERIFY_SUCCEEDED: ${{ tasks.verify.succeeded }} + TASK_VERIFY_FAILED: ${{ tasks.verify.failed }} + TASK_VERIFY_SKIPPED: ${{ tasks.verify.skipped }} + cache: false + timeout: 10m + outputs: + filesystem: false + artifacts: + - key: receipt + path: .ci/pr-gates/groups/main-nut-provenance diff --git a/.rwx/op-reth.yml b/.rwx/op-reth.yml new file mode 100644 index 00000000000..57b6566b449 --- /dev/null +++ b/.rwx/op-reth.yml @@ -0,0 +1,368 @@ +# Three op-reth workloads: integration tests, compact codec and superchain snapshot. +# CircleCI retains the required checks. +on: + cache-rebuild: + if: ${{ event.git.branch == 'develop' }} + target: [integration-build, codec-base-build, codec-head-build, snapshot-build] + init: + commit-sha: ${{ event.git.sha }} + branch: ${{ event.git.branch }} + cache-epoch: v1 + build-probe: "" + target-cache-mode: keep + codec-base-sha: ${{ event.git.sha }} + cache-warm: "true" + # Automatic pushes are coordinated by pr-gates.yml; CLI and warming remain. + cli: + init: + commit-sha: ${{ event.git.sha }} + branch: ${{ event.git.branch }} + cache-epoch: v1 + build-probe: "" + target-cache-mode: keep + codec-base-sha: "" + cache-warm: "false" + +base: + image: ubuntu:24.04@sha256:008173c23f95b170204355c12626cb5a965d779a7e1283b09e9cffbb1bf33ca3 + config: rwx/base 1.2.0 + arch: x86_64 + +# Cache-only vault: no publishing credentials. Only develop may write; +# PRs restore the protected shared baseline and compile changes locally. +tool-cache: + vault: optimism-op-reth-shadow + +defaults: + runner: + cpus: 2 + memory: 8gb + +tasks: + - key: code + call: git/clone 2.2.0 + with: + repository: https://github.com/ethereum-optimism/optimism.git + ref: ${{ init.commit-sha }} + preserve-git-dir: true + fetch-full-depth: true + submodules: false + + - key: bootstrap-inputs + use: code + run: 'true' + filter: [mise.toml, ops/ci/runtime] + outputs: + filesystem: + filter: + workspace: [mise.toml, ops/ci/runtime/apt-install.sh, ops/ci/runtime/rwx-prepare.sh, ops/ci/runtime/rwx-contracts-prepare.sh, ops/ci/runtime/rwx-rust-prepare.sh, ops/ci/runtime/op-reth-shadow.sh, ops/ci/runtime/op-reth-report.py, ops/ci/runtime/rust-target-cache.py] + system: [] + + - key: mise + call: mise/install 1.1.0 + with: + mise-version: "2026.2.2" + install: "false" + + # The filtered bootstrap layer keeps the checkout's Git history out of + # reusable tools. Language toolchains are added only by their own workload. + - key: tools + use: [mise, bootstrap-inputs] + call: ${{ run.dir }}/packages/toolchain-common.yml + + - key: route + use: [code, tools] + run: bash ops/ci/runtime/rwx-metadata.sh + env: + CI_CACHE_WARM: ${{ init.cache-warm }} + CI_BRANCH: ${{ init.branch }} + CI_COMMIT_SHA: ${{ init.commit-sha }} + cache: false + timeout: 10m + outputs: + filesystem: false + + - key: helper-tests + use: [code, tools] + if: ${{ init.cache-warm != 'true' && tasks.route.values.run-rust-ci == 'true' }} + run: | + python3 ops/ci/tests/test_op_reth_shadow.py + cache: false + timeout: 10m + outputs: + filesystem: false + + - key: rust-tools + use: [bootstrap-inputs, tools] + if: ${{ init.cache-warm == 'true' || tasks.route.values.run-rust-ci == 'true' }} + call: ${{ run.dir }}/packages/toolchain-rust.yml + with: + mode: release + + - key: rust-dependencies + use: [code, rust-tools] + if: ${{ init.cache-warm == 'true' || tasks.route.values.run-rust-ci == 'true' }} + run: | + export CARGO_HOME="$PWD/.ci/rust-cache/cargo" + cd rust + attempt=0 + until cargo fetch --locked; do + attempt=$((attempt + 1)) + if [[ "$attempt" -ge 5 ]]; then exit 1; fi + sleep "$((2 ** attempt))" + done + filter: [rust, mise.toml] + tool-cache: op-reth-dependencies-${{ init.cache-epoch }} + timeout: 30m + outputs: + filesystem: + filter: + workspace: [.ci/rust-cache/cargo] + system: [] + + # Keep full source, Git metadata and the pinned submodule as build inputs. + - key: head-source + use: [code, rust-dependencies] + if: ${{ init.cache-warm == 'true' || tasks.route.values.run-rust-ci == 'true' }} + run: bash /usr/local/lib/optimism-ci/op-reth-shadow.sh source + timeout: 15m + + - key: codec-base-revision + use: [code, tools] + if: ${{ init.cache-warm == 'true' || tasks.route.values.run-rust-ci == 'true' }} + run: | + revision="$CODEC_BASE_SHA" + if [[ -z "$revision" ]]; then + revision="$(git ls-remote --exit-code origin refs/heads/develop | cut -f1)" + fi + [[ "$revision" =~ ^[0-9a-f]{40}$ ]] || { echo 'Expected an immutable develop SHA.' >&2; exit 1; } + mkdir -p .ci + printf '%s\n' "$revision" >"$RWX_VALUES/sha" + printf '%s\n' "$revision" >.ci/op-reth-codec-base-sha.txt + env: + CODEC_BASE_SHA: ${{ init.codec-base-sha }} + cache: false + timeout: 5m + outputs: + filesystem: false + artifacts: + - key: codec-base-revision + path: .ci/op-reth-codec-base-sha.txt + + - key: codec-base-code + if: ${{ init.cache-warm == 'true' || tasks.route.values.run-rust-ci == 'true' }} + call: git/clone 2.2.0 + with: + repository: https://github.com/ethereum-optimism/optimism.git + ref: ${{ tasks.codec-base-revision.values.sha }} + preserve-git-dir: true + fetch-full-depth: true + submodules: false + + - key: codec-base-source + # Baseline checkout wins over the PR manifest tree in the dependency layer. + use: [rust-dependencies, codec-base-code] + if: ${{ init.cache-warm == 'true' || tasks.route.values.run-rust-ci == 'true' }} + run: bash /usr/local/lib/optimism-ci/op-reth-shadow.sh source + timeout: 15m + + # Each producer has its own mutable cache, avoiding concurrent writers. RWX + # content hits skip compilation; misses restore Cargo targets and local + # sccache entries, then Cargo checks source, flags, profiles and lockfiles. + # BUILD_PROBE forces a content miss for measurements without disabling the + # tool cache. A new CACHE_EPOCH gives genuinely cold build/compile caches. + - key: integration-build + use: head-source + if: ${{ init.cache-warm == 'true' || tasks.route.values.run-rust-ci == 'true' }} + run: bash /usr/local/lib/optimism-ci/op-reth-shadow.sh integration-build + env: &build-env + CACHE_EPOCH: ${{ init.cache-epoch }} + BUILD_PROBE: ${{ init.build-probe }} + TARGET_CACHE_MODE: ${{ init.target-cache-mode }} + tool-cache: op-reth-integration-${{ init.cache-epoch }} + timeout: 60m + runner: &build-runner + cpus: 16 + memory: 32gb + outputs: + filesystem: + filter: + workspace: [rust/target, rust/op-reth/crates/chainspec/res/superchain-configs.tar, .ci/rust-cache, .ci/op-reth/integration-build] + system: [] + artifacts: + - key: integration-build-report + path: .ci/op-reth/integration-build + + - key: integration + # Artifact references create dependencies without inheriting Cargo/sccache + # filesystem layers. Input filters affect cache keys, not layer downloads. + use: [code, rust-tools] + if: ${{ tasks.route.values.run-rust-ci == 'true' }} + run: | + mkdir -p .ci/op-reth/integration-build + cp -a "$INTEGRATION_ARTIFACT/." .ci/op-reth/integration-build/ + bash /usr/local/lib/optimism-ci/op-reth-shadow.sh integration + env: + INTEGRATION_ARTIFACT: ${{ tasks.integration-build.artifacts.integration-build-report }} + cache: false + timeout: 60m + runner: *build-runner + outputs: + filesystem: false + test-results: + # Captain has no nextest-specific parser. Infer its generic JUnit parser. + - path: .ci/op-reth/integration/junit.xml + artifacts: + - key: integration-reports + path: .ci/op-reth/integration + + - key: codec-base-build + use: codec-base-source + if: ${{ init.cache-warm == 'true' || tasks.route.values.run-rust-ci == 'true' }} + run: bash /usr/local/lib/optimism-ci/op-reth-shadow.sh codec-base-build + env: + # The PR SHA is deliberately absent: an unchanged pinned develop build + # can content-hit when only the PR changes. + CACHE_EPOCH: ${{ init.cache-epoch }} + BUILD_PROBE: ${{ init.build-probe }} + TARGET_CACHE_MODE: ${{ init.target-cache-mode }} + tool-cache: op-reth-codec-base-${{ init.cache-epoch }} + timeout: 60m + runner: + cpus: 8 + memory: 16gb + outputs: + filesystem: + filter: + workspace: [rust/target, rust/op-reth/crates/chainspec/res/superchain-configs.tar, .ci/rust-cache, .ci/op-reth/codec-base-build] + system: [] + artifacts: + - key: codec-base-build-report + path: .ci/op-reth/codec-base-build + + - key: codec-head-build + use: head-source + if: ${{ init.cache-warm == 'true' || tasks.route.values.run-rust-ci == 'true' }} + run: bash /usr/local/lib/optimism-ci/op-reth-shadow.sh codec-head-build + env: *build-env + tool-cache: op-reth-codec-head-${{ init.cache-epoch }} + timeout: 60m + runner: + cpus: 8 + memory: 16gb + outputs: + filesystem: + filter: + workspace: [rust/target, rust/op-reth/crates/chainspec/res/superchain-configs.tar, .ci/rust-cache, .ci/op-reth/codec-head-build] + system: [] + artifacts: + - key: codec-head-build-report + path: .ci/op-reth/codec-head-build + + # Generate random baseline vectors afresh on every run. Only the immutable + # baseline compiler output is reusable, preserving CircleCI's sample cadence. + - key: codec-vectors + use: [code, rust-tools] + if: ${{ tasks.route.values.run-rust-ci == 'true' }} + run: | + mkdir -p .ci/op-reth/codec-base-build + cp -a "$CODEC_BASE_ARTIFACT/." .ci/op-reth/codec-base-build/ + bash /usr/local/lib/optimism-ci/op-reth-shadow.sh codec-vectors + env: + CODEC_BASE_ARTIFACT: ${{ tasks.codec-base-build.artifacts.codec-base-build-report }} + cache: false + timeout: 10m + runner: + cpus: 8 + memory: 16gb + outputs: + filesystem: false + artifacts: + - key: codec-vectors + path: testdata/micro/compact + - key: codec-vectors-report + path: .ci/op-reth/codec-vectors + + - key: codec + # Keep the PR checkout and mount only the head binary and fresh vectors. + use: [code, rust-tools] + if: ${{ tasks.route.values.run-rust-ci == 'true' }} + run: | + mkdir -p .ci/op-reth/codec-head-build + cp -a "$CODEC_HEAD_ARTIFACT/." .ci/op-reth/codec-head-build/ + rm -rf testdata/micro/compact + mkdir -p testdata/micro/compact + cp -a "$CODEC_VECTORS/." testdata/micro/compact/ + bash /usr/local/lib/optimism-ci/op-reth-shadow.sh codec + env: + CODEC_HEAD_ARTIFACT: ${{ tasks.codec-head-build.artifacts.codec-head-build-report }} + CODEC_VECTORS: ${{ tasks.codec-vectors.artifacts.codec-vectors }} + cache: false + timeout: 10m + runner: + cpus: 8 + memory: 16gb + outputs: + filesystem: false + artifacts: + - key: codec-verification + path: .ci/op-reth/codec + + - key: snapshot-build + use: head-source + if: ${{ init.cache-warm == 'true' || tasks.route.values.run-rust-ci == 'true' }} + run: bash /usr/local/lib/optimism-ci/op-reth-shadow.sh snapshot-build + env: *build-env + tool-cache: op-reth-snapshot-${{ init.cache-epoch }} + timeout: 30m + runner: + cpus: 4 + memory: 8gb + outputs: + filesystem: + filter: + workspace: [rust/target, rust/op-reth/crates/chainspec/res/superchain-configs.tar, .ci/rust-cache, .ci/op-reth/snapshot-build] + system: [] + artifacts: + - key: snapshot-build-report + path: .ci/op-reth/snapshot-build + + - key: snapshot + use: snapshot-build + if: ${{ tasks.route.values.run-rust-ci == 'true' }} + run: bash /usr/local/lib/optimism-ci/op-reth-shadow.sh snapshot + cache: false + timeout: 30m + runner: + cpus: 4 + memory: 8gb + outputs: + filesystem: false + artifacts: + - key: snapshot-verification + path: .ci/op-reth/snapshot + + # Terminal task states preserve exact Rust dependencies without Main-only failures. + - key: rust-gate-receipt + use: [code, tools] + if: ${{ init.cache-warm != 'true' }} + after: ${{ (codec.succeeded || codec.failed || codec.skipped) && (integration.succeeded || integration.failed || integration.skipped) && (snapshot.succeeded || snapshot.failed || snapshot.skipped) }} + run: python3 ops/ci/runtime/pr-gate.py receipt op-reth-rust + env: + GROUP_SELECTED: ${{ tasks.route.values.run-rust-ci }} + TASK_CODEC_SUCCEEDED: ${{ tasks.codec.succeeded }} + TASK_CODEC_FAILED: ${{ tasks.codec.failed }} + TASK_CODEC_SKIPPED: ${{ tasks.codec.skipped }} + TASK_INTEGRATION_SUCCEEDED: ${{ tasks.integration.succeeded }} + TASK_INTEGRATION_FAILED: ${{ tasks.integration.failed }} + TASK_INTEGRATION_SKIPPED: ${{ tasks.integration.skipped }} + TASK_SNAPSHOT_SUCCEEDED: ${{ tasks.snapshot.succeeded }} + TASK_SNAPSHOT_FAILED: ${{ tasks.snapshot.failed }} + TASK_SNAPSHOT_SKIPPED: ${{ tasks.snapshot.skipped }} + cache: false + timeout: 10m + outputs: + filesystem: false + artifacts: + - key: receipt + path: .ci/pr-gates/groups/op-reth-rust diff --git a/.rwx/packages/acceptance-verdict.yml b/.rwx/packages/acceptance-verdict.yml new file mode 100644 index 00000000000..5f1ab36f4ad --- /dev/null +++ b/.rwx/packages/acceptance-verdict.yml @@ -0,0 +1,51 @@ +# One node of CircleCI's op-acceptance-tests job. The memory-all jobs run it on +# four nodes of 16 vCPU and 32 GiB (2xlarge.gen2). +package: + parameters: + commit-sha: + required: true + branch: + required: true + shard-index: + required: true + cl-kind: + required: true + cache-key: + required: true +tasks: + - key: run + use: package.use + docker: true + run: | + for dependency in go contracts kona op-reth prestate sp1-executor; do + python3 ops/ci/runtime/go-artifacts.py restore "$dependency" ".ci/go-tests/dependencies/$dependency" + done + mise exec -- bash ops/ci/runtime/acceptance-tests.sh + env: + CI_COMMIT_SHA: ${{ params.commit-sha }} + CIRCLE_BRANCH: ${{ params.branch }} + CI_SHARD_INDEX: ${{ params.shard-index }} + DEVSTACK_L1_FORK: fusaka + DEVSTACK_L2CL_KIND: ${{ params.cl-kind }} + cache: false + tool-cache: ${{ params.cache-key }} + timeout: 120m + runner: + cpus: 16 + memory: 32gb + outputs: + filesystem: + filter: + workspace: [.ci/go-cache/acceptance/run] + system: [] + # The recipe's own reports, which CircleCI stores with store_test_results + # and store_artifacts: one JUnit file for each test, and the run's logs. + test-results: + - path: op-acceptance-tests/results/results-*.xml + artifacts: + - key: logs + path: op-acceptance-tests/logs + # Keep the producer's module cache so tests do not refetch modules; exclude compiler state. + # Exclude no tracked file: go-artifacts.py restores a build of an older commit only into + # a checkout whose build inputs are unmodified. + filter: ['!.ci/go-cache/full/build', '!.ci/go-cache/full/dependencies', '!.ci/go-cache/acceptance/run', '!.ci/rust-cache', '!rust/target', '!packages/contracts-bedrock/cache', '!packages/contracts-bedrock/artifacts', '!packages/contracts-bedrock/forge-artifacts'] diff --git a/.rwx/packages/contract-upgrades-verdict.yml b/.rwx/packages/contract-upgrades-verdict.yml new file mode 100644 index 00000000000..571778f2c65 --- /dev/null +++ b/.rwx/packages/contract-upgrades-verdict.yml @@ -0,0 +1,43 @@ +package: + parameters: + branch: + required: true + cache-key: + required: true + variant: + required: true +tasks: + - key: run + use: package.use + run: | + export GOMODCACHE="$PWD/.ci/go-cache/pr-checks/modules" + export GOCACHE="$PWD/.ci/go-cache/contracts-compile/build" + python3 ops/ci/runtime/contract-upgrades.py run --variant "$CI_CONTRACT_VARIANT" + env: + CI_BRANCH: ${{ params.branch }} + OP_CI_MAINNET_L1_ARCHIVE_RPC_URL: ${{ vaults.optimism-go-tests-rpc-shadow.secrets.OP_CI_MAINNET_L1_ARCHIVE_RPC_URL }} + CI_CONTRACT_VARIANT: ${{ params.variant }} + # Fresh verdicts retain read-only access to protected fork-state caches. + RWX_RUN_ID: + cache-key: included + RWX_TASK_ATTEMPT_NUMBER: + cache-key: included + tool-cache: ${{ params.cache-key }} + timeout: 60m + runner: + cpus: 16 + memory: 32gb + outputs: + filesystem: + filter: + workspace: [] + system: [~/.foundry/cache/rpc] + test-results: + # RWX does not expand an expression in this path. This task has one variant's directory. + - path: .ci/contract-upgrades/*/run/results.xml + options: + language: Solidity + framework: Foundry + artifacts: + - key: reports + path: .ci/contract-upgrades/${{ params.variant }}/run diff --git a/.rwx/packages/contracts-compile-modified.yml b/.rwx/packages/contracts-compile-modified.yml new file mode 100644 index 00000000000..e616c8a4598 --- /dev/null +++ b/.rwx/packages/contracts-compile-modified.yml @@ -0,0 +1,34 @@ +package: + parameters: + branch: + required: true + cache-key: + required: true + suite: + required: true + feature: + required: true +tasks: + - key: build + use: package.use + run: | + export GOMODCACHE="$PWD/.ci/go-cache/pr-checks/modules" + export GOCACHE="$PWD/.ci/go-cache/contract-suites/build" + python3 ops/ci/runtime/contract-suites.py prepare --suite "$CI_CONTRACT_SUITE" --feature "$CI_CONTRACT_FEATURE" + env: + CI_BRANCH: ${{ params.branch }} + CI_CONTRACT_SUITE: ${{ params.suite }} + CI_CONTRACT_FEATURE: ${{ params.feature }} + tool-cache: ${{ params.cache-key }} + timeout: 40m + runner: + cpus: 16 + memory: 32gb + outputs: + filesystem: + filter: + workspace: [packages/contracts-bedrock/forge-artifacts, packages/contracts-bedrock/cache/solidity-files-cache.json, packages/contracts-bedrock/artifacts/build-info, packages/contracts-bedrock/scripts/go-ffi/go-ffi, .ci/go-cache/contract-suites/build] + system: [] + artifacts: + - key: reports + path: .ci/contract-suites/${{ params.suite }}-${{ params.feature }}/prepare diff --git a/.rwx/packages/contracts-compile.yml b/.rwx/packages/contracts-compile.yml new file mode 100644 index 00000000000..baa9d5fc01c --- /dev/null +++ b/.rwx/packages/contracts-compile.yml @@ -0,0 +1,35 @@ +# One `forge build` for the standard contract suites and the upgrade variants of a commit. +# CircleCI builds in each of those jobs. The builds are the same: the compiler reads the +# Foundry profile, and the tests read the feature flags and the fork variables. +package: + parameters: + branch: + required: true + cache-epoch: + required: true +tasks: + - key: build + use: package.use + # What the build reads: the contracts, the Go module and the Go sources of the FFI helper, + # and its scripts. A commit that changes none of these keeps the cache key. The list names + # what to include: with a list of exclusions only, RWX gives each commit a new key. + filter: [mise.toml, go.mod, go.sum, cannon, 'op-*', packages/contracts-bedrock, ops/ci/runtime/ci-report.py, ops/ci/runtime/contract-upgrades.py, .ci/go-cache/pr-checks/modules, .ci/go-cache/contracts-compile/build] + run: | + export GOMODCACHE="$PWD/.ci/go-cache/pr-checks/modules" + export GOCACHE="$PWD/.ci/go-cache/contracts-compile/build" + python3 ops/ci/runtime/contract-upgrades.py compile + env: + CI_BRANCH: ${{ params.branch }} + tool-cache: contracts-compile-${{ params.cache-epoch }} + timeout: 40m + runner: + cpus: 16 + memory: 32gb + outputs: + filesystem: + filter: + workspace: [packages/contracts-bedrock/forge-artifacts, packages/contracts-bedrock/cache/solidity-files-cache.json, packages/contracts-bedrock/artifacts/build-info, packages/contracts-bedrock/scripts/go-ffi/go-ffi, .ci/go-cache/contracts-compile/build] + system: [] + artifacts: + - key: reports + path: .ci/contracts-compile diff --git a/.rwx/packages/contracts-verdict.yml b/.rwx/packages/contracts-verdict.yml new file mode 100644 index 00000000000..13192519325 --- /dev/null +++ b/.rwx/packages/contracts-verdict.yml @@ -0,0 +1,46 @@ +package: + parameters: + branch: + required: true + cache-key: + required: true + suite: + required: true + feature: + required: true +tasks: + - key: run + use: package.use + run: | + export GOMODCACHE="$PWD/.ci/go-cache/pr-checks/modules" + export GOCACHE="$PWD/.ci/go-cache/contract-suites-runtime/build" + python3 ops/ci/runtime/contract-suites.py run --suite "$CI_CONTRACT_SUITE" --feature "$CI_CONTRACT_FEATURE" + env: + CI_BRANCH: ${{ params.branch }} + RWX_RUN_ID: + cache-key: included + RWX_TASK_ATTEMPT_NUMBER: + cache-key: included + CI_CONTRACT_SUITE: ${{ params.suite }} + CI_CONTRACT_FEATURE: ${{ params.feature }} + cache: false + tool-cache: ${{ params.cache-key }} + timeout: 60m + runner: + cpus: 16 + memory: 32gb + outputs: + filesystem: + filter: + workspace: [.ci/go-cache/contract-suites-runtime/build] + system: [] + test-results: + # `just test` writes one JUnit report per EVM pass: results-amsterdam.xml and results-l2.xml. + # RWX does not expand an expression in this path. This task has one suite's directory. + - path: .ci/contract-suites/*/run/results-*.xml + options: + language: Solidity + framework: Foundry + artifacts: + - key: reports + path: .ci/contract-suites/${{ params.suite }}-${{ params.feature }}/run diff --git a/.rwx/packages/go-contracts.yml b/.rwx/packages/go-contracts.yml new file mode 100644 index 00000000000..f57eb0355dd --- /dev/null +++ b/.rwx/packages/go-contracts.yml @@ -0,0 +1,41 @@ +package: + parameters: + commit-sha: + cache-epoch: + build-probe: + target-cache-mode: +tasks: + # The tool tasks in package.use carry the caller's checkout. The filter keeps it out of + # this task: the cache key then holds only the tools, the command and the commit, and + # the task checks that commit out itself. The checkout stays out of the filtered output, + # so a task that uses this build gets the build outputs and no other checkout. + - key: build + use: package.use + filter: [mise.toml, packages/contracts-bedrock/cache, packages/contracts-bedrock/artifacts, packages/contracts-bedrock/forge-artifacts, .ci/go-cache/full, .ci/go-tests/dependencies/contracts] + run: | + git init -q . + git remote add origin https://github.com/ethereum-optimism/optimism.git + git fetch -q origin "$CI_COMMIT_SHA" + git checkout -qf FETCH_HEAD + git submodule update --init --recursive --jobs 8 + git rev-parse HEAD >packages/contracts-bedrock/.gitcommit + mise exec -- bash ops/ci/runtime/go-dependencies.sh contracts + env: + CI_GO_REUSABLE_DEPENDENCY: "1" + CI_COMMIT_SHA: ${{ params.commit-sha }} + BUILD_PROBE: ${{ params.build-probe }} + TARGET_CACHE_MODE: ${{ params.target-cache-mode }} + # Keep artifacts from the former full test-graph build out of this baseline. + tool-cache: go-full-contracts-ci-source-scripts-${{ params.cache-epoch }} + timeout: 90m + runner: + cpus: 4 + memory: 8gb + outputs: + filesystem: + filter: + workspace: [packages/contracts-bedrock/cache, packages/contracts-bedrock/artifacts, packages/contracts-bedrock/forge-artifacts, .ci/go-cache/full, .ci/go-tests/dependencies/contracts] + system: [] + artifacts: + - key: dependency + path: .ci/go-tests/dependencies/contracts diff --git a/.rwx/packages/go-go.yml b/.rwx/packages/go-go.yml new file mode 100644 index 00000000000..fdcf9137d4f --- /dev/null +++ b/.rwx/packages/go-go.yml @@ -0,0 +1,40 @@ +package: + parameters: + commit-sha: + cache-epoch: + build-probe: + target-cache-mode: +tasks: + # The tool tasks in package.use carry the caller's checkout. The filter keeps it out of + # this task: the cache key then holds only the tools, the command and the commit, and + # the task checks that commit out itself. The checkout stays out of the filtered output, + # so a task that uses this build gets the build outputs and no other checkout. + - key: build + use: package.use + filter: [mise.toml, .ci/go-cache/full, .ci/go-tests/dependencies/go] + run: | + git init -q . + git remote add origin https://github.com/ethereum-optimism/optimism.git + git fetch -q origin "$CI_COMMIT_SHA" + git checkout -qf FETCH_HEAD + git submodule update --init --recursive --jobs 8 + git rev-parse HEAD >packages/contracts-bedrock/.gitcommit + mise exec -- bash ops/ci/runtime/go-dependencies.sh go + env: + CI_GO_REUSABLE_DEPENDENCY: "1" + CI_COMMIT_SHA: ${{ params.commit-sha }} + BUILD_PROBE: ${{ params.build-probe }} + TARGET_CACHE_MODE: ${{ params.target-cache-mode }} + tool-cache: go-full-dependencies-${{ params.cache-epoch }} + timeout: 90m + runner: + cpus: 2 + memory: 4gb + outputs: + filesystem: + filter: + workspace: [.ci/go-cache/full, .ci/go-tests/dependencies/go] + system: [] + artifacts: + - key: dependency + path: .ci/go-tests/dependencies/go diff --git a/.rwx/packages/go-kona.yml b/.rwx/packages/go-kona.yml new file mode 100644 index 00000000000..2a12b50a264 --- /dev/null +++ b/.rwx/packages/go-kona.yml @@ -0,0 +1,40 @@ +package: + parameters: + commit-sha: + cache-epoch: + build-probe: + target-cache-mode: +tasks: + # The tool tasks in package.use carry the caller's checkout. The filter keeps it out of + # this task: the cache key then holds only the tools, the command and the commit, and + # the task checks that commit out itself. The checkout stays out of the filtered output, + # so a task that uses this build gets the build outputs and no other checkout. + - key: build + use: package.use + filter: [mise.toml, rust/target, .ci/rust-cache, .ci/go-tests/dependencies/kona] + run: | + git init -q . + git remote add origin https://github.com/ethereum-optimism/optimism.git + git fetch -q origin "$CI_COMMIT_SHA" + git checkout -qf FETCH_HEAD + git submodule update --init --recursive --jobs 8 + git rev-parse HEAD >packages/contracts-bedrock/.gitcommit + mise exec -- bash ops/ci/runtime/go-dependencies.sh kona + env: + CI_GO_REUSABLE_DEPENDENCY: "1" + CI_COMMIT_SHA: ${{ params.commit-sha }} + BUILD_PROBE: ${{ params.build-probe }} + TARGET_CACHE_MODE: ${{ params.target-cache-mode }} + tool-cache: kona-release-${{ params.cache-epoch }} + timeout: 90m + runner: + cpus: 16 + memory: 32gb + outputs: + filesystem: + filter: + workspace: [rust/target, .ci/rust-cache, .ci/go-tests/dependencies/kona] + system: [] + artifacts: + - key: dependency + path: .ci/go-tests/dependencies/kona diff --git a/.rwx/packages/go-modules.yml b/.rwx/packages/go-modules.yml new file mode 100644 index 00000000000..db11f430284 --- /dev/null +++ b/.rwx/packages/go-modules.yml @@ -0,0 +1,46 @@ +# The Go module set, downloaded once like CircleCI's prep-go-modules job. Callers +# read GOMODCACHE at .ci/go-cache/pr-checks/modules or restore the `dependency` +# artifact. Modules only: Cannon's Circle job does not embed generated Cannon binaries. +package: + parameters: + commit-sha: + branch: + cache-epoch: +tasks: + # As in go-go.yml: the filter and the checkout in the task keep the cache key to the tools, + # the command and the planned commit, which is the one that last changed go.mod or go.sum. + - key: build + use: package.use + filter: [mise.toml, .ci/go-cache/pr-checks/modules, .ci/go-tests/dependencies/go-modules] + run: | + git init -q . + git remote add origin https://github.com/ethereum-optimism/optimism.git + git fetch -q origin "$CI_COMMIT_SHA" + git checkout -qf FETCH_HEAD + export GOMODCACHE="$PWD/.ci/go-cache/pr-checks/modules" + mkdir -p "$GOMODCACHE" + # proxy.golang.org returns transient errors. Like CircleCI, retry with backoff. + n=0 + until go mod download; do + n=$((n + 1)) + if [[ "$n" -ge 5 ]]; then echo "go mod download failed after $n attempts" >&2; exit 1; fi + sleep "$((5 * 2 ** (n - 1)))" + done + go mod verify + python3 ops/ci/runtime/go-artifacts.py pack go-modules .ci/go-cache/pr-checks/modules + env: + CI_GO_REUSABLE_DEPENDENCY: "1" + CI_COMMIT_SHA: ${{ params.commit-sha }} + tool-cache: pr-checks-go-modules-${{ params.cache-epoch }} + timeout: 30m + runner: + cpus: 2 + memory: 4gb + outputs: + filesystem: + filter: + workspace: [.ci/go-cache/pr-checks/modules, .ci/go-tests/dependencies/go-modules] + system: [] + artifacts: + - key: dependency + path: .ci/go-tests/dependencies/go-modules diff --git a/.rwx/packages/go-op-reth.yml b/.rwx/packages/go-op-reth.yml new file mode 100644 index 00000000000..f2a7077a03a --- /dev/null +++ b/.rwx/packages/go-op-reth.yml @@ -0,0 +1,40 @@ +package: + parameters: + commit-sha: + cache-epoch: + build-probe: + target-cache-mode: +tasks: + # The tool tasks in package.use carry the caller's checkout. The filter keeps it out of + # this task: the cache key then holds only the tools, the command and the commit, and + # the task checks that commit out itself. The checkout stays out of the filtered output, + # so a task that uses this build gets the build outputs and no other checkout. + - key: build + use: package.use + filter: [mise.toml, rust/target, .ci/rust-cache, .ci/go-tests/dependencies/op-reth] + run: | + git init -q . + git remote add origin https://github.com/ethereum-optimism/optimism.git + git fetch -q origin "$CI_COMMIT_SHA" + git checkout -qf FETCH_HEAD + git submodule update --init --recursive --jobs 8 + git rev-parse HEAD >packages/contracts-bedrock/.gitcommit + mise exec -- bash ops/ci/runtime/go-dependencies.sh op-reth + env: + CI_GO_REUSABLE_DEPENDENCY: "1" + CI_COMMIT_SHA: ${{ params.commit-sha }} + BUILD_PROBE: ${{ params.build-probe }} + TARGET_CACHE_MODE: ${{ params.target-cache-mode }} + tool-cache: op-reth-release-${{ params.cache-epoch }} + timeout: 90m + runner: + cpus: 16 + memory: 32gb + outputs: + filesystem: + filter: + workspace: [rust/target, .ci/rust-cache, .ci/go-tests/dependencies/op-reth] + system: [] + artifacts: + - key: dependency + path: .ci/go-tests/dependencies/op-reth diff --git a/.rwx/packages/go-prestate.yml b/.rwx/packages/go-prestate.yml new file mode 100644 index 00000000000..b109565c1d2 --- /dev/null +++ b/.rwx/packages/go-prestate.yml @@ -0,0 +1,41 @@ +package: + parameters: + commit-sha: + cache-epoch: + build-probe: + target-cache-mode: +tasks: + # The tool tasks in package.use carry the caller's checkout. The filter keeps it out of + # this task: the cache key then holds only the tools, the command and the commit, and + # the task checks that commit out itself. The checkout stays out of the filtered output, + # so a task that uses this build gets the build outputs and no other checkout. + - key: build + use: package.use + filter: [mise.toml, rust/kona/prestate-artifacts-*, .ci/go-tests/dependencies/prestate] + run: | + git init -q . + git remote add origin https://github.com/ethereum-optimism/optimism.git + git fetch -q origin "$CI_COMMIT_SHA" + git checkout -qf FETCH_HEAD + git submodule update --init --recursive --jobs 8 + git rev-parse HEAD >packages/contracts-bedrock/.gitcommit + mise exec -- bash ops/ci/runtime/go-dependencies.sh prestate + env: + CI_GO_REUSABLE_DEPENDENCY: "1" + CI_COMMIT_SHA: ${{ params.commit-sha }} + BUILD_PROBE: ${{ params.build-probe }} + TARGET_CACHE_MODE: ${{ params.target-cache-mode }} + tool-cache: go-full-prestate-${{ params.cache-epoch }} + docker: preserve-data + timeout: 90m + runner: + cpus: 4 + memory: 16gb + outputs: + filesystem: + filter: + workspace: [rust/kona/prestate-artifacts-*, .ci/go-tests/dependencies/prestate] + system: [] + artifacts: + - key: dependency + path: .ci/go-tests/dependencies/prestate diff --git a/.rwx/packages/go-sp1-executor.yml b/.rwx/packages/go-sp1-executor.yml new file mode 100644 index 00000000000..d70fcf03ee8 --- /dev/null +++ b/.rwx/packages/go-sp1-executor.yml @@ -0,0 +1,40 @@ +package: + parameters: + commit-sha: + cache-epoch: + build-probe: + target-cache-mode: +tasks: + # The tool tasks in package.use carry the caller's checkout. The filter keeps it out of + # this task: the cache key then holds only the tools, the command and the commit, and + # the task checks that commit out itself. The checkout stays out of the filtered output, + # so a task that uses this build gets the build outputs and no other checkout. + - key: build + use: package.use + filter: [mise.toml, rust/target, .ci/rust-cache, .ci/go-tests/dependencies/sp1-executor] + run: | + git init -q . + git remote add origin https://github.com/ethereum-optimism/optimism.git + git fetch -q origin "$CI_COMMIT_SHA" + git checkout -qf FETCH_HEAD + git submodule update --init --recursive --jobs 8 + git rev-parse HEAD >packages/contracts-bedrock/.gitcommit + mise exec -- bash ops/ci/runtime/go-dependencies.sh sp1-executor + env: + CI_GO_REUSABLE_DEPENDENCY: "1" + CI_COMMIT_SHA: ${{ params.commit-sha }} + BUILD_PROBE: ${{ params.build-probe }} + TARGET_CACHE_MODE: ${{ params.target-cache-mode }} + tool-cache: sp1-executor-release-${{ params.cache-epoch }} + timeout: 90m + runner: + cpus: 16 + memory: 32gb + outputs: + filesystem: + filter: + workspace: [rust/target, .ci/rust-cache, .ci/go-tests/dependencies/sp1-executor] + system: [] + artifacts: + - key: dependency + path: .ci/go-tests/dependencies/sp1-executor diff --git a/.rwx/packages/go-test-compile.yml b/.rwx/packages/go-test-compile.yml new file mode 100644 index 00000000000..0bdeb5aadad --- /dev/null +++ b/.rwx/packages/go-test-compile.yml @@ -0,0 +1,34 @@ +package: + parameters: + commit-sha: + required: true + shard-index: + required: true + build-probe: + required: true + cache-key: + required: true +tasks: + - key: build + use: package.use + run: | + if ! git cat-file -e "$CI_COMMIT_SHA^{commit}"; then git fetch --no-tags origin "$CI_COMMIT_SHA"; fi + git reset --hard "$CI_COMMIT_SHA" + git submodule sync --recursive + git submodule update --init --recursive --jobs 8 + git rev-parse HEAD >packages/contracts-bedrock/.gitcommit + mise exec -- bash ops/ci/runtime/go-full-tests.sh build + env: + CI_COMMIT_SHA: ${{ params.commit-sha }} + CI_SHARD_INDEX: ${{ params.shard-index }} + BUILD_PROBE: ${{ params.build-probe }} + tool-cache: ${{ params.cache-key }} + timeout: 60m + runner: + cpus: 8 + memory: 16gb + outputs: + filesystem: + filter: + workspace: [.ci/go-tests/build] + system: [] diff --git a/.rwx/packages/go-test-verdict.yml b/.rwx/packages/go-test-verdict.yml new file mode 100644 index 00000000000..67fafe41764 --- /dev/null +++ b/.rwx/packages/go-test-verdict.yml @@ -0,0 +1,60 @@ +package: + parameters: + commit-sha: + required: true + branch: + required: true + shard-index: + required: true + cache-key: + required: true +tasks: + - key: run + use: package.use + docker: true + run: | + if ! git cat-file -e "$CI_COMMIT_SHA^{commit}"; then git fetch --no-tags origin "$CI_COMMIT_SHA"; fi + git reset --hard "$CI_COMMIT_SHA" + git submodule sync --recursive + git submodule update --init --recursive --jobs 8 + git rev-parse HEAD >packages/contracts-bedrock/.gitcommit + # The go and contracts restores also supply op-deployer's embedded files + # (superchain configs, forge artifacts) for tests that build it at runtime. + for dependency in go contracts kona op-reth prestate; do + python3 ops/ci/runtime/go-artifacts.py restore "$dependency" ".ci/go-tests/dependencies/$dependency" + done + export RUST_BINARY_PATH_OP_RETH="$PWD/rust/target/release/op-reth" + # A missing engine must fail instead of building one locally, as on Circle. + export RUST_BINARY_PATH_OP_RETH_TEST_ENGINE="$PWD/rust/target/release/op-reth-test-engine" + export REQUIRE_RUST_ENGINE=1 + mise exec -- bash ops/ci/runtime/go-full-tests.sh run + env: + CI_COMMIT_SHA: ${{ params.commit-sha }} + CIRCLE_BRANCH: ${{ params.branch }} + CI_SHARD_INDEX: ${{ params.shard-index }} + OP_CI_MAINNET_L1_ARCHIVE_RPC_URL: ${{ vaults.optimism-go-tests-rpc-shadow.secrets.OP_CI_MAINNET_L1_ARCHIVE_RPC_URL }} + OP_CI_SEPOLIA_L1_ARCHIVE_RPC_URL: ${{ vaults.optimism-go-tests-rpc-shadow.secrets.OP_CI_SEPOLIA_L1_ARCHIVE_RPC_URL }} + cache: false + tool-cache: ${{ params.cache-key }} + timeout: 60m + runner: + cpus: 8 + memory: 16gb + outputs: + filesystem: + filter: + workspace: [.ci/go-cache/full/run] + system: [] + test-results: + - path: tmp/testlogs/native.json + options: + language: Go + framework: go test + artifacts: + - key: junit + path: tmp/test-results + - key: go-json + path: tmp/testlogs/log.json + - key: test-logs + path: tmp/testlogs + filter: ['!.ci/go-cache/full/build', '!.ci/go-cache/full/dependencies', '!.ci/go-cache/full/run', '!.ci/rust-cache', '!rust/target', '!packages/contracts-bedrock/cache', '!packages/contracts-bedrock/artifacts', '!packages/contracts-bedrock/forge-artifacts', '!op-deployer/pkg/deployer/artifacts/forge-artifacts'] diff --git a/.rwx/packages/rust-e2e-contracts.yml b/.rwx/packages/rust-e2e-contracts.yml new file mode 100644 index 00000000000..0ba56b78b25 --- /dev/null +++ b/.rwx/packages/rust-e2e-contracts.yml @@ -0,0 +1,27 @@ +package: + parameters: + commit-sha: + cache-epoch: + build-probe: + target-cache-mode: +tasks: + - key: build + use: package.use + run: mise exec -- bash ops/ci/runtime/go-dependencies.sh contracts-e2e + env: + CI_COMMIT_SHA: ${{ params.commit-sha }} + BUILD_PROBE: ${{ params.build-probe }} + TARGET_CACHE_MODE: ${{ params.target-cache-mode }} + tool-cache: go-rust-e2e-contracts-ci-${{ params.cache-epoch }} + timeout: 90m + runner: + cpus: 4 + memory: 8gb + outputs: + filesystem: + filter: + workspace: [packages/contracts-bedrock/cache, packages/contracts-bedrock/artifacts, packages/contracts-bedrock/forge-artifacts, .ci/go-cache/full, .ci/go-tests/dependencies/contracts-e2e] + system: [] + artifacts: + - key: dependency + path: .ci/go-tests/dependencies/contracts-e2e diff --git a/.rwx/packages/rust-e2e-release.yml b/.rwx/packages/rust-e2e-release.yml new file mode 100644 index 00000000000..6eda52e4e13 --- /dev/null +++ b/.rwx/packages/rust-e2e-release.yml @@ -0,0 +1,28 @@ +package: + parameters: + commit-sha: + cache-epoch: +tasks: + - key: build + use: package.use + run: bash ops/ci/runtime/rust-e2e-release.sh binaries + env: + CI_COMMIT_SHA: ${{ params.commit-sha }} + # The gate receipt reads this task, so it runs in every run. The tool cache keeps it incremental. + RWX_RUN_ID: + cache-key: included + RWX_TASK_ATTEMPT_NUMBER: + cache-key: included + tool-cache: rust-e2e-workspace-release-${{ params.cache-epoch }} + timeout: 120m + runner: + cpus: 16 + memory: 32gb + outputs: + filesystem: + filter: + workspace: [.ci/rust-cache, rust/target] + system: [] + artifacts: + - key: dependency + path: .ci/go-tests/dependencies/rust-e2e-release diff --git a/.rwx/packages/toolchain-common.yml b/.rwx/packages/toolchain-common.yml new file mode 100644 index 00000000000..28fdee9e5da --- /dev/null +++ b/.rwx/packages/toolchain-common.yml @@ -0,0 +1,9 @@ +package: true +tasks: + - key: prepare + use: package.use + run: | + bash ops/ci/runtime/rwx-prepare.sh + filter: + workspace: [mise.toml, ops/ci/runtime/apt-install.sh, ops/ci/runtime/rwx-prepare.sh] + timeout: 30m diff --git a/.rwx/packages/toolchain-foundry.yml b/.rwx/packages/toolchain-foundry.yml new file mode 100644 index 00000000000..a94dafe92fa --- /dev/null +++ b/.rwx/packages/toolchain-foundry.yml @@ -0,0 +1,30 @@ +package: + parameters: + mode: + required: true +tasks: + - key: prepare + use: package.use + run: | + bash ops/ci/runtime/rwx-contracts-prepare.sh tools + case "$FOUNDRY_TOOL_MODE" in + compiler) ;; + anvil) mise install anvil ;; + analysis) + mise install uv + PATH="$(mise bin-paths | paste -sd: -):$PATH" + export PATH + # The pinned Semgrep's MCP dependency imports a Pydantic private API + # removed in later releases. Keep this constraint in its own tool env. + semgrep_version="$(mise tool pipx:semgrep --requested)" + mise install "pipx:semgrep[uvx_args=--with pydantic==2.11.10]@$semgrep_version" + printf '%s:%s\n' "$(mise bin-paths | paste -sd: -)" "$PATH" >"$RWX_ENV/PATH" + mise exec -- semgrep --version + ;; + *) echo "Unknown Foundry tool mode: $FOUNDRY_TOOL_MODE" >&2; exit 1 ;; + esac + env: + FOUNDRY_TOOL_MODE: ${{ params.mode }} + filter: + workspace: [mise.toml, ops/ci/runtime/apt-install.sh, ops/ci/runtime/rwx-prepare.sh, ops/ci/runtime/rwx-contracts-prepare.sh] + timeout: 30m diff --git a/.rwx/packages/toolchain-go.yml b/.rwx/packages/toolchain-go.yml new file mode 100644 index 00000000000..1948a765d00 --- /dev/null +++ b/.rwx/packages/toolchain-go.yml @@ -0,0 +1,16 @@ +# Pinned Go tool layers; language installation remains separate from source and verdicts. +package: + parameters: + mode: + required: true +tasks: + - key: prepare + use: package.use + run: bash ops/ci/runtime/rwx-prepare.sh "$GO_TOOL_MODE" + env: + GO_TOOL_MODE: ${{ params.mode }} + # Without a filter the task has the whole checkout in its cache key and runs again for each + # commit. Each task that uses it, a dependency build included, then gets a new cache key too. + filter: + workspace: [mise.toml, ops/ci/runtime/rwx-prepare.sh] + timeout: 30m diff --git a/.rwx/packages/toolchain-rust.yml b/.rwx/packages/toolchain-rust.yml new file mode 100644 index 00000000000..f2f91df06d6 --- /dev/null +++ b/.rwx/packages/toolchain-rust.yml @@ -0,0 +1,29 @@ +package: + parameters: + mode: + required: true +tasks: + - key: prepare + use: package.use + run: | + bash ops/ci/runtime/rwx-prepare.sh rust + mise exec -- bash ops/ci/runtime/rwx-rust-prepare.sh + case "$RUST_TOOL_MODE" in + release) ;; + e2e) bash ops/ci/runtime/apt-install.sh m4 file ;; + workspace|sp1) + bash ops/ci/runtime/apt-install.sh m4 file + if [[ "$RUST_TOOL_MODE" == workspace ]]; then + mise install github:est31/cargo-udeps github:taiki-e/cargo-hack + else + mise install github:succinctlabs/sp1 + fi + printf '%s:%s\n' "$(mise bin-paths | paste -sd: -)" "$PATH" >"$RWX_ENV/PATH" + ;; + *) echo "Unknown Rust tool mode: $RUST_TOOL_MODE" >&2; exit 1 ;; + esac + env: + RUST_TOOL_MODE: ${{ params.mode }} + filter: + workspace: [mise.toml, ops/ci/runtime/apt-install.sh, ops/ci/runtime/rwx-prepare.sh, ops/ci/runtime/rwx-rust-prepare.sh, ops/ci/runtime/op-reth-shadow.sh, ops/ci/runtime/op-reth-report.py, ops/ci/runtime/rust-target-cache.py] + timeout: 30m diff --git a/.rwx/pr-checks.yml b/.rwx/pr-checks.yml new file mode 100644 index 00000000000..3d5523d339e --- /dev/null +++ b/.rwx/pr-checks.yml @@ -0,0 +1,494 @@ +# The small checks of CircleCI's main workflow and its contracts fast check. +# CircleCI owns the required gates. Each check task runs the command of the +# CircleCI job that its comment names, from .circleci/continue/main.yml. +on: + cache-rebuild: + if: ${{ event.git.branch == 'develop' }} + target: [go-modules, contract-tools, main-tools, static-tools, go-lint-tools] + init: + commit-sha: ${{ event.git.sha }} + branch: ${{ event.git.branch }} + cache-warm: "true" + cache-epoch: v1 + cli: + init: + commit-sha: ${{ event.git.sha }} + branch: ${{ event.git.branch }} + cache-warm: "false" + cache-epoch: v1 + +tool-cache: + vault: optimism-op-reth-shadow + +base: + image: ubuntu:24.04@sha256:008173c23f95b170204355c12626cb5a965d779a7e1283b09e9cffbb1bf33ca3 + config: rwx/base 1.2.0 + arch: x86_64 + +defaults: + runner: + cpus: 2 + memory: 8gb + +tasks: + - key: code + call: git/clone 2.2.0 + with: + repository: https://github.com/ethereum-optimism/optimism.git + ref: ${{ init.commit-sha }} + preserve-git-dir: true + fetch-full-depth: true + submodules: false + + - key: bootstrap-inputs + use: code + run: 'true' + filter: [mise.toml, ops/ci/runtime] + outputs: + filesystem: + filter: + workspace: [mise.toml, ops/ci/runtime/apt-install.sh, ops/ci/runtime/rwx-prepare.sh, ops/ci/runtime/rwx-contracts-prepare.sh, ops/ci/runtime/rwx-rust-prepare.sh, ops/ci/runtime/op-reth-shadow.sh, ops/ci/runtime/op-reth-report.py, ops/ci/runtime/rust-target-cache.py] + system: [] + + - key: mise + call: mise/install 1.1.0 + with: + mise-version: "2026.2.2" + install: "false" + + - key: tools + use: [mise, bootstrap-inputs] + call: ${{ run.dir }}/packages/toolchain-common.yml + + - key: route + use: [code, tools] + run: bash ops/ci/runtime/rwx-metadata.sh + env: + CI_CACHE_WARM: ${{ init.cache-warm }} + CI_BRANCH: ${{ init.branch }} + CI_COMMIT_SHA: ${{ init.commit-sha }} + cache: false + timeout: 10m + outputs: + filesystem: false + + - key: go-tools + use: tools + if: ${{ init.cache-warm == 'true' || tasks.route.values.run-main == 'true' || tasks.route.values.run-contracts == 'true' }} + call: ${{ run.dir }}/packages/toolchain-go.yml + with: + mode: go + + - key: contract-tools + use: [bootstrap-inputs, tools] + if: ${{ init.cache-warm == 'true' || tasks.route.values.run-contracts == 'true' }} + call: ${{ run.dir }}/packages/toolchain-foundry.yml + with: + mode: analysis + + - key: main-tools + use: go-tools + filter: [mise.toml] + if: ${{ init.cache-warm == 'true' || tasks.route.values.run-main == 'true' }} + run: | + mise install ripgrep mockery + printf '%s:%s\n' "$(mise bin-paths | paste -sd: -)" "$PATH" >"$RWX_ENV/PATH" + timeout: 30m + + - key: static-tools + use: tools + filter: [mise.toml] + if: ${{ init.cache-warm == 'true' || tasks.route.values.run-main == 'true' }} + run: | + mise install uv shellcheck shfmt + PATH="$(mise bin-paths | paste -sd: -):$PATH" + export PATH + # The pinned Semgrep's MCP dependency imports a Pydantic private API + # removed in later releases, as in packages/toolchain-foundry.yml. + semgrep_version="$(mise tool pipx:semgrep --requested)" + mise install "pipx:semgrep[uvx_args=--with pydantic==2.11.10]@$semgrep_version" + printf '%s:%s\n' "$(mise bin-paths | paste -sd: -)" "$PATH" >"$RWX_ENV/PATH" + timeout: 30m + + # Mirrors CircleCI's shell-check job. That job has its find expression inline, + # so this task reads the command from the job and the two cannot drift. + - key: shell-check + use: [code, static-tools] + if: ${{ init.cache-warm != 'true' && tasks.route.values.run-main == 'true' }} + run: | + command="$(yq -r '.jobs["shell-check"].steps[] | select(.run.name? == "Run shellcheck") | .run.command' .circleci/continue/main.yml)" + if [[ -z "$command" || "$command" == null ]]; then echo "CircleCI's shell-check command is missing" >&2; exit 1; fi + bash -eo pipefail -c "$command" + cache: false + timeout: 20m + runner: + cpus: 4 + memory: 8gb + outputs: + filesystem: false + + # Mirrors CircleCI's shfmt-check job. + - key: shfmt-check + use: [code, static-tools] + if: ${{ init.cache-warm != 'true' && tasks.route.values.run-main == 'true' }} + run: just shfmt-check + cache: false + timeout: 10m + outputs: + filesystem: false + + # Mirrors CircleCI's semgrep-scan job as semgrep-test. Like that job, a branch + # other than develop sets SEMGREP_BASELINE_REF to develop. `semgrep scan` reads + # no other variable of that job's Semgrep environment; `semgrep ci` reads the rest. + - key: semgrep-test + use: [code, static-tools] + if: ${{ init.cache-warm != 'true' && tasks.route.values.run-main == 'true' }} + run: | + if [[ "$CI_BRANCH" != develop ]]; then + git fetch --no-tags origin develop + git update-ref refs/heads/develop FETCH_HEAD + export SEMGREP_BASELINE_REF=develop + fi + semgrep scan --test --config .semgrep/rules/ .semgrep/tests/ + env: + CI_BRANCH: ${{ init.branch }} + cache: false + timeout: 20m + runner: + cpus: 8 + memory: 16gb + outputs: + filesystem: false + + # Mirrors CircleCI's semgrep-scan job as semgrep-scan-local, with the same + # baseline as semgrep-test. + - key: semgrep-scan-local + use: [code, static-tools] + if: ${{ init.cache-warm != 'true' && tasks.route.values.run-main == 'true' }} + run: | + if [[ "$CI_BRANCH" != develop ]]; then + git fetch --no-tags origin develop + git update-ref refs/heads/develop FETCH_HEAD + export SEMGREP_BASELINE_REF=develop + fi + semgrep scan --timeout=100 --config .semgrep/rules/ --error . + env: + CI_BRANCH: ${{ init.branch }} + cache: false + timeout: 30m + runner: + cpus: 8 + memory: 16gb + outputs: + filesystem: false + + # Mirrors CircleCI's prep-go-modules job. Later tasks read the module cache at + # .ci/go-cache/pr-checks/modules. + # A dependency build checks out the commit that last changed its build inputs, so it + # is a cache hit for later commits and for the other workloads of the run. + - key: dependency-plan + use: code + run: python3 ops/ci/runtime/go-dependency-source.py plan + outputs: + filesystem: false + + - key: go-modules + use: go-tools + if: ${{ init.cache-warm == 'true' || tasks.route.values.run-main == 'true' || tasks.route.values.run-contracts == 'true' }} + call: ${{ run.dir }}/packages/go-modules.yml + with: + commit-sha: ${{ tasks.dependency-plan.values.go-modules }} + branch: ${{ init.branch }} + cache-epoch: ${{ init.cache-epoch }} + + # Mirrors CircleCI's contracts-bedrock-checks-fast job, which required-contracts-ci + # requires as contracts-bedrock-checks-fast-feature-tests. + - key: contracts-fast + use: [code, go-tools, go-modules, contract-tools] + if: ${{ init.cache-warm != 'true' && tasks.route.values.run-contracts == 'true' }} + run: | + export GOMODCACHE="$PWD/.ci/go-cache/pr-checks/modules" + export GOCACHE="$PWD/.ci/go-cache/pr-checks/fast-build" + git submodule sync --recursive + git -c protocol.file.allow=never submodule update --init --recursive --jobs 8 + cd packages/contracts-bedrock + # scripts/checks/check-semver-diff.sh compares with origin/. + # The clone of one commit can lack that ref. + source scripts/ops/get-target-branch.sh + git fetch --no-tags origin "+refs/heads/$TARGET_BRANCH:refs/remotes/origin/$TARGET_BRANCH" + forge --version + just check-fast + env: + # checks.yaml formats the sources in place unless CI is set. + CI: "true" + # scripts/ops/get-target-branch.sh reads it to find a merge-queue base. + CIRCLE_BRANCH: ${{ init.branch }} + RWX_RUN_ID: + cache-key: included + RWX_TASK_ATTEMPT_NUMBER: + cache-key: included + tool-cache: pr-checks-contracts-fast-${{ init.cache-epoch }} + timeout: 60m + runner: + cpus: 16 + memory: 32gb + outputs: + filesystem: + filter: + workspace: [packages/contracts-bedrock/forge-artifacts, packages/contracts-bedrock/cache/solidity-files-cache.json, packages/contracts-bedrock/artifacts/build-info, .ci/go-cache/pr-checks/fast-build] + system: [] + + # Mirrors CircleCI's todo-issues job as todo-issues-check (check_closed: false). + # No gate requires it. ripgrep reads a stdin that is not a terminal instead of + # searching the checkout, so stdin is /dev/null. + - key: todo-issues-check + use: [code, main-tools] + if: ${{ init.cache-warm != 'true' && tasks.route.values.run-main == 'true' }} + run: ./ops/scripts/todo-checker.sh --verbose --strict packages/contracts-bedrock/.gitcommit + timeout: 20m + + - key: rust-dependencies + use: [source, rust-tools] + if: ${{ init.cache-warm == 'true' || tasks.route.values.run-rust-e2e-ci == 'true' }} + run: | + export CARGO_HOME="$PWD/.ci/rust-cache/cargo" + cd rust + attempt=0 + until cargo fetch --locked; do + attempt=$((attempt + 1)) + if [[ "$attempt" -ge 5 ]]; then exit 1; fi + sleep "$((2 ** attempt))" + done + filter: [rust, mise.toml] + tool-cache: op-reth-dependencies-${{ init.cache-epoch }} + timeout: 30m + outputs: + filesystem: + filter: + workspace: [.ci/rust-cache/cargo] + system: [] + + - key: head-source + use: [source, rust-dependencies] + if: ${{ init.cache-warm == 'true' || tasks.route.values.run-rust-e2e-ci == 'true' }} + run: 'true' + + # A dependency build checks out the commit that last changed its build inputs, so it + # is a cache hit for later commits and for the other workloads of the run. + - key: dependency-plan + use: code + run: python3 ops/ci/runtime/go-dependency-source.py plan + outputs: + filesystem: false + + - key: go + use: go-build-tools + if: ${{ init.cache-warm == 'true' || tasks.route.values.run-rust-e2e-ci == 'true' }} + call: ${{ run.dir }}/packages/go-go.yml + with: + commit-sha: ${{ tasks.dependency-plan.values.go }} + cache-epoch: ${{ init.cache-epoch }} + build-probe: ${{ init.build-probe }} + target-cache-mode: ${{ init.target-cache-mode }} + + - key: contracts + use: [source, contract-tools, go-build-tools] + if: ${{ init.cache-warm == 'true' || tasks.route.values.run-rust-e2e-ci == 'true' }} + call: ${{ run.dir }}/packages/rust-e2e-contracts.yml + with: + commit-sha: ${{ init.commit-sha }} + cache-epoch: ${{ init.cache-epoch }} + build-probe: ${{ init.build-probe }} + target-cache-mode: ${{ init.target-cache-mode }} + + # CircleCI's rust-e2e-binaries: the binaries that the verdict tasks run. + - key: binaries + use: [rust-tools, head-source] + if: ${{ init.cache-warm == 'true' || tasks.route.values.run-rust-e2e-ci == 'true' }} + call: ${{ run.dir }}/packages/rust-e2e-release.yml + with: + commit-sha: ${{ init.commit-sha }} + cache-epoch: ${{ init.cache-epoch }} + + # CircleCI's rust-workspace-release: every other binary of the release workspace. It + # starts from the binaries build, so the two tasks compile each unit once. No verdict + # waits for it, so it has a smaller runner. + - key: release + use: [rust-tools, head-source, binaries.build] + if: ${{ init.cache-warm != 'true' && tasks.route.values.run-rust-e2e-ci == 'true' }} + run: bash ops/ci/runtime/rust-e2e-release.sh workspace + env: + CI_COMMIT_SHA: ${{ init.commit-sha }} + RWX_RUN_ID: + cache-key: included + RWX_TASK_ATTEMPT_NUMBER: + cache-key: included + timeout: 120m + runner: + cpus: 8 + memory: 16gb + outputs: + filesystem: false + + - key: prestate + use: go-build-tools + if: ${{ init.cache-warm == 'true' || tasks.route.values.run-rust-e2e-ci == 'true' }} + call: ${{ run.dir }}/packages/go-prestate.yml + with: + commit-sha: ${{ tasks.dependency-plan.values.prestate }} + cache-epoch: ${{ init.cache-epoch }} + build-probe: ${{ init.build-probe }} + target-cache-mode: ${{ init.target-cache-mode }} + + - key: runtime-tools + use: [go-build-tools, contract-tools] + filter: [mise.toml] + if: ${{ init.cache-warm == 'true' || tasks.route.values.run-rust-e2e-ci == 'true' }} + run: | + mise install go:github.com/ethereum/go-ethereum/cmd/geth + printf '%s\n' "$(mise bin-paths | paste -sd: -):$PATH" >"$RWX_ENV/PATH" + timeout: 30m + + - &compile + key: compile-proof + use: [source, go-build-tools] + if: ${{ init.cache-warm == 'true' || tasks.route.values.run-rust-e2e-ci == 'true' }} + run: | + python3 ops/ci/runtime/go-artifacts.py restore go "$GO_ARTIFACT" + python3 ops/ci/runtime/go-artifacts.py restore contracts-e2e "$CONTRACT_ARTIFACT" + bash ops/ci/runtime/rust-e2e.sh compile proof + env: + CI_COMMIT_SHA: ${{ init.commit-sha }} + GO_ARTIFACT: ${{ tasks.go.tasks.build.artifacts.dependency }} + CONTRACT_ARTIFACT: ${{ tasks.contracts.tasks.build.artifacts.dependency }} + tool-cache: rust-e2e-go-compile-${{ init.cache-epoch }} + timeout: 60m + runner: + cpus: 8 + memory: 16gb + outputs: &compile-outputs + filesystem: + filter: + workspace: [.ci/go-cache/rust-e2e/compile] + system: [] + artifacts: + - key: compiled + path: .ci/rust-e2e/compiled/proof + + - <<: *compile + key: compile-restart + run: | + python3 ops/ci/runtime/go-artifacts.py restore go "$GO_ARTIFACT" + python3 ops/ci/runtime/go-artifacts.py restore contracts-e2e "$CONTRACT_ARTIFACT" + bash ops/ci/runtime/rust-e2e.sh compile restart + outputs: + <<: *compile-outputs + artifacts: + - key: compiled + path: .ci/rust-e2e/compiled/restart + + - <<: *compile + key: compile-simple-kona + run: | + python3 ops/ci/runtime/go-artifacts.py restore go "$GO_ARTIFACT" + python3 ops/ci/runtime/go-artifacts.py restore contracts-e2e "$CONTRACT_ARTIFACT" + bash ops/ci/runtime/rust-e2e.sh compile simple-kona + outputs: + <<: *compile-outputs + artifacts: + - key: compiled + path: .ci/rust-e2e/compiled/simple-kona + + - <<: *compile + key: compile-simple-kona-sequencer + run: | + python3 ops/ci/runtime/go-artifacts.py restore go "$GO_ARTIFACT" + python3 ops/ci/runtime/go-artifacts.py restore contracts-e2e "$CONTRACT_ARTIFACT" + bash ops/ci/runtime/rust-e2e.sh compile simple-kona-sequencer + outputs: + <<: *compile-outputs + artifacts: + - key: compiled + path: .ci/rust-e2e/compiled/simple-kona-sequencer + + - <<: *compile + key: compile-op-reth + run: | + python3 ops/ci/runtime/go-artifacts.py restore go "$GO_ARTIFACT" + python3 ops/ci/runtime/go-artifacts.py restore contracts-e2e "$CONTRACT_ARTIFACT" + bash ops/ci/runtime/rust-e2e.sh compile op-reth + outputs: + <<: *compile-outputs + artifacts: + - key: compiled + path: .ci/rust-e2e/compiled/op-reth + + # Run/attempt identity forces fresh verdicts while preserving runtime Go caches. + - &verdict-proof + key: verdict-proof-0 + use: [source, runtime-tools] + if: ${{ init.cache-warm != 'true' && tasks.route.values.run-rust-e2e-ci == 'true' }} + docker: true + run: | + python3 ops/ci/runtime/go-artifacts.py restore go "$GO_ARTIFACT" + python3 ops/ci/runtime/go-artifacts.py restore contracts-e2e "$CONTRACT_ARTIFACT" + python3 ops/ci/runtime/go-artifacts.py restore rust-e2e-release "$RELEASE_ARTIFACT" + bash ops/ci/runtime/rust-e2e.sh run proof + env: &verdict-proof-env + CI_COMMIT_SHA: ${{ init.commit-sha }} + CI_SHARD_INDEX: "0" + PARALLEL: "8" + RWX_RUN_ID: + cache-key: included + RWX_TASK_ATTEMPT_NUMBER: + cache-key: included + CI_E2E_COMPILED: ${{ tasks.compile-proof.artifacts.compiled }} + GO_ARTIFACT: ${{ tasks.go.tasks.build.artifacts.dependency }} + CONTRACT_ARTIFACT: ${{ tasks.contracts.tasks.build.artifacts.dependency }} + RELEASE_ARTIFACT: ${{ tasks.binaries.tasks.build.artifacts.dependency }} + tool-cache: rust-e2e-runtime-proof-${{ init.cache-epoch }} + timeout: 90m + runner: + cpus: 8 + memory: 16gb + outputs: + filesystem: + filter: + workspace: [.ci/go-cache/rust-e2e/run] + system: [] + test-results: + - path: .ci/rust-e2e/reports/proof/native.json + options: + language: Go + framework: go test + artifacts: + - key: reports + path: .ci/rust-e2e/reports/proof + + - <<: *verdict-proof + key: verdict-proof-1 + env: + <<: *verdict-proof-env + CI_SHARD_INDEX: "1" + + - <<: *verdict-proof + key: verdict-proof-2 + env: + <<: *verdict-proof-env + CI_SHARD_INDEX: "2" + + - <<: *verdict-proof + key: verdict-proof-3 + env: + <<: *verdict-proof-env + CI_SHARD_INDEX: "3" + + - <<: *verdict-proof + key: verdict-proof-4 + env: + <<: *verdict-proof-env + CI_SHARD_INDEX: "4" + + - <<: *verdict-proof + key: verdict-proof-5 + env: + <<: *verdict-proof-env + CI_SHARD_INDEX: "5" + + - <<: *verdict-proof + key: verdict-proof-6 + env: + <<: *verdict-proof-env + CI_SHARD_INDEX: "6" + + - <<: *verdict-proof + key: verdict-proof-7 + env: + <<: *verdict-proof-env + CI_SHARD_INDEX: "7" + + - &verdict-node + key: verdict-restart-0 + use: [source, runtime-tools] + if: ${{ init.cache-warm != 'true' && tasks.route.values.run-rust-e2e-ci == 'true' }} + docker: true + run: | + python3 ops/ci/runtime/go-artifacts.py restore go "$GO_ARTIFACT" + python3 ops/ci/runtime/go-artifacts.py restore contracts-e2e "$CONTRACT_ARTIFACT" + python3 ops/ci/runtime/go-artifacts.py restore rust-e2e-release "$RELEASE_ARTIFACT" + python3 ops/ci/runtime/go-artifacts.py restore prestate "$PRESTATE_ARTIFACT" + bash ops/ci/runtime/rust-e2e.sh run restart + env: &verdict-node-env + CI_COMMIT_SHA: ${{ init.commit-sha }} + CI_SHARD_INDEX: "0" + RWX_RUN_ID: + cache-key: included + RWX_TASK_ATTEMPT_NUMBER: + cache-key: included + CI_E2E_COMPILED: ${{ tasks.compile-restart.artifacts.compiled }} + GO_ARTIFACT: ${{ tasks.go.tasks.build.artifacts.dependency }} + CONTRACT_ARTIFACT: ${{ tasks.contracts.tasks.build.artifacts.dependency }} + RELEASE_ARTIFACT: ${{ tasks.binaries.tasks.build.artifacts.dependency }} + PRESTATE_ARTIFACT: ${{ tasks.prestate.tasks.build.artifacts.dependency }} + tool-cache: rust-e2e-runtime-restart-${{ init.cache-epoch }} + timeout: 60m + runner: + cpus: 8 + memory: 16gb + outputs: &verdict-node-outputs + filesystem: + filter: + workspace: [.ci/go-cache/rust-e2e/run] + system: [] + test-results: + - path: .ci/rust-e2e/reports/restart/native.json + options: + language: Go + framework: go test + artifacts: + - key: reports + path: .ci/rust-e2e/reports/restart + + - <<: *verdict-node + key: verdict-restart-1 + env: + <<: *verdict-node-env + CI_SHARD_INDEX: "1" + + - <<: *verdict-node + key: verdict-restart-2 + env: + <<: *verdict-node-env + CI_SHARD_INDEX: "2" + + - <<: *verdict-node + key: verdict-simple-kona-0 + run: | + python3 ops/ci/runtime/go-artifacts.py restore go "$GO_ARTIFACT" + python3 ops/ci/runtime/go-artifacts.py restore contracts-e2e "$CONTRACT_ARTIFACT" + python3 ops/ci/runtime/go-artifacts.py restore rust-e2e-release "$RELEASE_ARTIFACT" + python3 ops/ci/runtime/go-artifacts.py restore prestate "$PRESTATE_ARTIFACT" + bash ops/ci/runtime/rust-e2e.sh run simple-kona + env: + <<: *verdict-node-env + CI_E2E_COMPILED: ${{ tasks.compile-simple-kona.artifacts.compiled }} + tool-cache: rust-e2e-runtime-simple-kona-${{ init.cache-epoch }} + outputs: + <<: *verdict-node-outputs + test-results: + - path: .ci/rust-e2e/reports/simple-kona/native.json + options: + language: Go + framework: go test + artifacts: + - key: reports + path: .ci/rust-e2e/reports/simple-kona + + - <<: *verdict-node + key: verdict-simple-kona-sequencer-0 + run: | + python3 ops/ci/runtime/go-artifacts.py restore go "$GO_ARTIFACT" + python3 ops/ci/runtime/go-artifacts.py restore contracts-e2e "$CONTRACT_ARTIFACT" + python3 ops/ci/runtime/go-artifacts.py restore rust-e2e-release "$RELEASE_ARTIFACT" + python3 ops/ci/runtime/go-artifacts.py restore prestate "$PRESTATE_ARTIFACT" + bash ops/ci/runtime/rust-e2e.sh run simple-kona-sequencer + env: + <<: *verdict-node-env + CI_E2E_COMPILED: ${{ tasks.compile-simple-kona-sequencer.artifacts.compiled }} + tool-cache: rust-e2e-runtime-simple-kona-sequencer-${{ init.cache-epoch }} + outputs: + <<: *verdict-node-outputs + test-results: + - path: .ci/rust-e2e/reports/simple-kona-sequencer/native.json + options: + language: Go + framework: go test + artifacts: + - key: reports + path: .ci/rust-e2e/reports/simple-kona-sequencer + + - <<: *verdict-node + key: verdict-op-reth-0 + run: | + python3 ops/ci/runtime/go-artifacts.py restore go "$GO_ARTIFACT" + python3 ops/ci/runtime/go-artifacts.py restore contracts-e2e "$CONTRACT_ARTIFACT" + python3 ops/ci/runtime/go-artifacts.py restore rust-e2e-release "$RELEASE_ARTIFACT" + bash ops/ci/runtime/rust-e2e.sh run op-reth + env: + CI_COMMIT_SHA: ${{ init.commit-sha }} + CI_SHARD_INDEX: "0" + RWX_RUN_ID: + cache-key: included + RWX_TASK_ATTEMPT_NUMBER: + cache-key: included + CI_E2E_COMPILED: ${{ tasks.compile-op-reth.artifacts.compiled }} + GO_ARTIFACT: ${{ tasks.go.tasks.build.artifacts.dependency }} + CONTRACT_ARTIFACT: ${{ tasks.contracts.tasks.build.artifacts.dependency }} + RELEASE_ARTIFACT: ${{ tasks.binaries.tasks.build.artifacts.dependency }} + tool-cache: rust-e2e-runtime-op-reth-${{ init.cache-epoch }} + outputs: + <<: *verdict-node-outputs + test-results: + - path: .ci/rust-e2e/reports/op-reth/native.json + options: + language: Go + framework: go test + artifacts: + - key: reports + path: .ci/rust-e2e/reports/op-reth + + - key: rust-e2e-gate-receipt + use: [code, tools] + after: ${{ (binaries.succeeded || binaries.failed || binaries.skipped) && (release.succeeded || release.failed || release.skipped) && (verdict-op-reth-0.succeeded || verdict-op-reth-0.failed || verdict-op-reth-0.skipped) && (verdict-proof-0.succeeded || verdict-proof-0.failed || verdict-proof-0.skipped) && (verdict-proof-1.succeeded || verdict-proof-1.failed || verdict-proof-1.skipped) && (verdict-proof-2.succeeded || verdict-proof-2.failed || verdict-proof-2.skipped) && (verdict-proof-3.succeeded || verdict-proof-3.failed || verdict-proof-3.skipped) && (verdict-proof-4.succeeded || verdict-proof-4.failed || verdict-proof-4.skipped) && (verdict-proof-5.succeeded || verdict-proof-5.failed || verdict-proof-5.skipped) && (verdict-proof-6.succeeded || verdict-proof-6.failed || verdict-proof-6.skipped) && (verdict-proof-7.succeeded || verdict-proof-7.failed || verdict-proof-7.skipped) && (verdict-restart-0.succeeded || verdict-restart-0.failed || verdict-restart-0.skipped) && (verdict-restart-1.succeeded || verdict-restart-1.failed || verdict-restart-1.skipped) && (verdict-restart-2.succeeded || verdict-restart-2.failed || verdict-restart-2.skipped) && (verdict-simple-kona-0.succeeded || verdict-simple-kona-0.failed || verdict-simple-kona-0.skipped) && (verdict-simple-kona-sequencer-0.succeeded || verdict-simple-kona-sequencer-0.failed || verdict-simple-kona-sequencer-0.skipped) }} + if: ${{ init.cache-warm != 'true' }} + run: python3 ops/ci/runtime/pr-gate.py receipt rust-e2e + env: + GROUP_SELECTED: ${{ tasks.route.values.run-rust-e2e-ci }} + TASK_BINARIES_SUCCEEDED: ${{ tasks.binaries.succeeded }} + TASK_BINARIES_FAILED: ${{ tasks.binaries.failed }} + TASK_BINARIES_SKIPPED: ${{ tasks.binaries.skipped }} + TASK_RELEASE_SUCCEEDED: ${{ tasks.release.succeeded }} + TASK_RELEASE_FAILED: ${{ tasks.release.failed }} + TASK_RELEASE_SKIPPED: ${{ tasks.release.skipped }} + TASK_VERDICT_OP_RETH_0_SUCCEEDED: ${{ tasks.verdict-op-reth-0.succeeded }} + TASK_VERDICT_OP_RETH_0_FAILED: ${{ tasks.verdict-op-reth-0.failed }} + TASK_VERDICT_OP_RETH_0_SKIPPED: ${{ tasks.verdict-op-reth-0.skipped }} + TASK_VERDICT_PROOF_0_SUCCEEDED: ${{ tasks.verdict-proof-0.succeeded }} + TASK_VERDICT_PROOF_0_FAILED: ${{ tasks.verdict-proof-0.failed }} + TASK_VERDICT_PROOF_0_SKIPPED: ${{ tasks.verdict-proof-0.skipped }} + TASK_VERDICT_PROOF_1_SUCCEEDED: ${{ tasks.verdict-proof-1.succeeded }} + TASK_VERDICT_PROOF_1_FAILED: ${{ tasks.verdict-proof-1.failed }} + TASK_VERDICT_PROOF_1_SKIPPED: ${{ tasks.verdict-proof-1.skipped }} + TASK_VERDICT_PROOF_2_SUCCEEDED: ${{ tasks.verdict-proof-2.succeeded }} + TASK_VERDICT_PROOF_2_FAILED: ${{ tasks.verdict-proof-2.failed }} + TASK_VERDICT_PROOF_2_SKIPPED: ${{ tasks.verdict-proof-2.skipped }} + TASK_VERDICT_PROOF_3_SUCCEEDED: ${{ tasks.verdict-proof-3.succeeded }} + TASK_VERDICT_PROOF_3_FAILED: ${{ tasks.verdict-proof-3.failed }} + TASK_VERDICT_PROOF_3_SKIPPED: ${{ tasks.verdict-proof-3.skipped }} + TASK_VERDICT_PROOF_4_SUCCEEDED: ${{ tasks.verdict-proof-4.succeeded }} + TASK_VERDICT_PROOF_4_FAILED: ${{ tasks.verdict-proof-4.failed }} + TASK_VERDICT_PROOF_4_SKIPPED: ${{ tasks.verdict-proof-4.skipped }} + TASK_VERDICT_PROOF_5_SUCCEEDED: ${{ tasks.verdict-proof-5.succeeded }} + TASK_VERDICT_PROOF_5_FAILED: ${{ tasks.verdict-proof-5.failed }} + TASK_VERDICT_PROOF_5_SKIPPED: ${{ tasks.verdict-proof-5.skipped }} + TASK_VERDICT_PROOF_6_SUCCEEDED: ${{ tasks.verdict-proof-6.succeeded }} + TASK_VERDICT_PROOF_6_FAILED: ${{ tasks.verdict-proof-6.failed }} + TASK_VERDICT_PROOF_6_SKIPPED: ${{ tasks.verdict-proof-6.skipped }} + TASK_VERDICT_PROOF_7_SUCCEEDED: ${{ tasks.verdict-proof-7.succeeded }} + TASK_VERDICT_PROOF_7_FAILED: ${{ tasks.verdict-proof-7.failed }} + TASK_VERDICT_PROOF_7_SKIPPED: ${{ tasks.verdict-proof-7.skipped }} + TASK_VERDICT_RESTART_0_SUCCEEDED: ${{ tasks.verdict-restart-0.succeeded }} + TASK_VERDICT_RESTART_0_FAILED: ${{ tasks.verdict-restart-0.failed }} + TASK_VERDICT_RESTART_0_SKIPPED: ${{ tasks.verdict-restart-0.skipped }} + TASK_VERDICT_RESTART_1_SUCCEEDED: ${{ tasks.verdict-restart-1.succeeded }} + TASK_VERDICT_RESTART_1_FAILED: ${{ tasks.verdict-restart-1.failed }} + TASK_VERDICT_RESTART_1_SKIPPED: ${{ tasks.verdict-restart-1.skipped }} + TASK_VERDICT_RESTART_2_SUCCEEDED: ${{ tasks.verdict-restart-2.succeeded }} + TASK_VERDICT_RESTART_2_FAILED: ${{ tasks.verdict-restart-2.failed }} + TASK_VERDICT_RESTART_2_SKIPPED: ${{ tasks.verdict-restart-2.skipped }} + TASK_VERDICT_SIMPLE_KONA_0_SUCCEEDED: ${{ tasks.verdict-simple-kona-0.succeeded }} + TASK_VERDICT_SIMPLE_KONA_0_FAILED: ${{ tasks.verdict-simple-kona-0.failed }} + TASK_VERDICT_SIMPLE_KONA_0_SKIPPED: ${{ tasks.verdict-simple-kona-0.skipped }} + TASK_VERDICT_SIMPLE_KONA_SEQUENCER_0_SUCCEEDED: ${{ tasks.verdict-simple-kona-sequencer-0.succeeded }} + TASK_VERDICT_SIMPLE_KONA_SEQUENCER_0_FAILED: ${{ tasks.verdict-simple-kona-sequencer-0.failed }} + TASK_VERDICT_SIMPLE_KONA_SEQUENCER_0_SKIPPED: ${{ tasks.verdict-simple-kona-sequencer-0.skipped }} + cache: false + timeout: 10m + outputs: + filesystem: false + artifacts: + - key: receipt + path: .ci/pr-gates/groups/rust-e2e diff --git a/.rwx/rust.yml b/.rwx/rust.yml new file mode 100644 index 00000000000..6175aa53f4f --- /dev/null +++ b/.rwx/rust.yml @@ -0,0 +1,1008 @@ +# Rust workspace builds, tests, cross-compilation and source checks. Circle retains required gates. +on: + cache-rebuild: + if: ${{ event.git.branch == 'develop' }} + target: [tests-cache, tests-build, build, clippy, docs, no-std, udeps, features-build-0, features-build-1, features-build-2, features-build-3, features-build-4, features-build-5, features-build-6, features-build-7, features-build-8, features-build-9, wasm-unknown, wasm-wasi, registry, cannon-env, cannon-go, cannon-witness, cannon-build, rust-fmt-tools] + init: + commit-sha: ${{ event.git.sha }} + branch: ${{ event.git.branch }} + cache-epoch: v1 + cache-warm: "true" + # Automatic pushes are coordinated by pr-gates.yml; CLI and warming remain. + cli: + init: + commit-sha: ${{ event.git.sha }} + branch: ${{ event.git.branch }} + cache-epoch: v1 + cache-warm: "false" + +base: + image: ubuntu:24.04@sha256:008173c23f95b170204355c12626cb5a965d779a7e1283b09e9cffbb1bf33ca3 + config: rwx/base 1.2.0 + arch: x86_64 + +# Cache-only vault: no publishing credentials. Only develop may write; +# PRs restore the protected shared baseline and compile changes locally. +tool-cache: + vault: optimism-op-reth-shadow + +defaults: + runner: + cpus: 2 + memory: 8gb + +tasks: + - key: code + call: git/clone 2.2.0 + with: + repository: https://github.com/ethereum-optimism/optimism.git + ref: ${{ init.commit-sha }} + preserve-git-dir: true + fetch-full-depth: true + submodules: false + + - key: bootstrap-inputs + use: code + run: 'true' + filter: [mise.toml, ops/ci/runtime] + outputs: + filesystem: + filter: + workspace: [mise.toml, ops/ci/runtime/apt-install.sh, ops/ci/runtime/rwx-prepare.sh, ops/ci/runtime/rwx-contracts-prepare.sh, ops/ci/runtime/rwx-rust-prepare.sh, ops/ci/runtime/op-reth-shadow.sh, ops/ci/runtime/op-reth-report.py, ops/ci/runtime/rust-target-cache.py] + system: [] + + - key: mise + call: mise/install 1.1.0 + with: + mise-version: "2026.2.2" + install: "false" + + # The filtered bootstrap layer keeps the checkout's Git history out of + # reusable tools. Language toolchains are added only by their own workload. + - key: tools + use: [mise, bootstrap-inputs] + call: ${{ run.dir }}/packages/toolchain-common.yml + + - key: route + use: [code, tools] + run: bash ops/ci/runtime/rwx-metadata.sh + env: + CI_CACHE_WARM: ${{ init.cache-warm }} + CI_BRANCH: ${{ init.branch }} + CI_COMMIT_SHA: ${{ init.commit-sha }} + cache: false + timeout: 10m + outputs: + filesystem: false + + - key: helper-tests + use: [code, tools] + if: ${{ init.cache-warm != 'true' && tasks.route.values.run-rust-ci == 'true' }} + run: | + python3 -m unittest discover -s ops/ci/tests -p 'test_rust_workspace.py' + python3 -m unittest discover -s ops/ci/tests -p 'test_rust_compiler_baseline.py' + python3 -m unittest discover -s ops/ci/tests -p 'test_rust_dependencies.py' + python3 -m unittest discover -s ops/ci/tests -p 'test_rwx_materialize_linker.py' + python3 -m unittest discover -s ops/ci/tests -p 'test_rust_target_cache.py' + python3 -m unittest discover -s ops/ci/tests -p 'test_rust_cannon.py' + cache: false + timeout: 10m + outputs: + filesystem: false + + - key: rust-tools + use: [bootstrap-inputs, tools] + if: ${{ init.cache-warm == 'true' || tasks.route.values.run-rust-ci == 'true' }} + call: ${{ run.dir }}/packages/toolchain-rust.yml + with: + mode: workspace + + - key: wasm-tools + use: rust-tools + filter: [mise.toml] + if: ${{ init.cache-warm == 'true' || tasks.route.values.run-rust-ci == 'true' }} + run: rustup target add wasm32-unknown-unknown wasm32-wasip1 + timeout: 30m + + - key: lint-tools + use: rust-tools + filter: [mise.toml] + if: ${{ tasks.route.values.run-rust-ci == 'true' }} + run: | + mise install github:ggwpez/zepter github:crate-ci/typos github:EmbarkStudios/cargo-deny + printf '%s:%s\n' "$(mise bin-paths | paste -sd: -)" "$PATH" >"$RWX_ENV/PATH" + timeout: 30m + + - key: go-tools + use: rust-tools + if: ${{ init.cache-warm == 'true' || tasks.route.values.run-rust-ci == 'true' }} + call: ${{ run.dir }}/packages/toolchain-go.yml + with: + mode: go + + # Fetch from the immutable compiler ancestor, so source-only edits reuse the + # same Cargo registry/checkouts even when PRs cannot write shared tool caches. + - key: rust-dependencies + use: compiler-source + if: ${{ init.cache-warm == 'true' || tasks.route.values.run-rust-ci == 'true' }} + run: python3 ops/ci/runtime/rust-dependencies.py + env: + COMPILER_CACHE_EPOCH: ${{ init.cache-epoch }} + timeout: 30m + outputs: + filesystem: + filter: + workspace: [.ci/rust-cache/cargo] + system: [] + artifacts: + - key: report + path: .ci/rust-dependencies + + # Keep full source, Git metadata and the pinned submodule as build inputs. + - key: head-source + use: [rust-dependencies, code] + if: ${{ init.cache-warm == 'true' || tasks.route.values.run-rust-ci == 'true' }} + run: | + git reset --hard "$CI_COMMIT_SHA" + git clean -fdx -- rust + bash /usr/local/lib/optimism-ci/op-reth-shadow.sh source + env: + CI_COMMIT_SHA: ${{ init.commit-sha }} + timeout: 15m + + # Anchor compiler preparation at the last compiler/configuration change. + # Ordinary source edits reuse these immutable task results without granting + # PRs shared-cache writes. Fresh HEAD workers below still run every verdict. + - key: compiler-base + use: code + run: python3 ops/ci/runtime/rust-target-cache.py base > "$RWX_VALUES/sha" + outputs: + filesystem: false + + - key: compiler-code + call: git/clone 2.2.0 + with: + repository: https://github.com/ethereum-optimism/optimism.git + ref: ${{ tasks.compiler-base.values.sha }} + preserve-git-dir: true + fetch-full-depth: true + submodules: false + + # Build the ancestor toolchain from ancestor inputs too. Depending on the + # HEAD toolchain would carry changing checkout metadata into the cache key. + - key: compiler-tools + use: [mise, compiler-code] + if: ${{ init.cache-warm == 'true' || tasks.route.values.run-rust-ci == 'true' }} + call: ${{ run.dir }}/packages/toolchain-common.yml + + - key: compiler-rust-tools + use: [compiler-code, compiler-tools] + if: ${{ init.cache-warm == 'true' || tasks.route.values.run-rust-ci == 'true' }} + call: ${{ run.dir }}/packages/toolchain-rust.yml + with: + mode: workspace + + - key: compiler-source + use: [compiler-code, compiler-rust-tools] + if: ${{ init.cache-warm == 'true' || tasks.route.values.run-rust-ci == 'true' }} + run: | + git reset --hard "$COMPILER_BASE_SHA" + git clean -fdx -- rust + bash /usr/local/lib/optimism-ci/op-reth-shadow.sh source + env: + COMPILER_BASE_SHA: ${{ tasks.compiler-base.values.sha }} + timeout: 15m + + - key: tests-cache + use: rust-dependencies + if: ${{ init.cache-warm == 'true' || tasks.route.values.run-rust-ci == 'true' }} + run: python3 ops/ci/runtime/rust-compiler-baseline.py tests-build + env: &compiler-base-env + COMPILER_CACHE_EPOCH: ${{ init.cache-epoch }} + timeout: 90m + runner: &test-runner + cpus: 16 + memory: 32gb + disk: 150gb + outputs: + filesystem: &test-compiler-output + filter: + workspace: [rust/target, .ci/rust-cache, rust/op-reth/crates/chainspec/res/superchain-configs.tar] + system: [] + artifacts: + - key: report + path: .ci/rust-workspace/compiler-baseline/tests-build + + - key: tests-build + use: [tests-cache, head-source] + if: ${{ init.cache-warm == 'true' || tasks.route.values.run-rust-ci == 'true' }} + run: bash ops/ci/runtime/rust-workspace.sh tests-build + # Compile the current source afresh using the ancestor's compiler state. + cache: false + timeout: 90m + runner: *test-runner + outputs: + filesystem: *test-compiler-output + artifacts: + - key: report + path: .ci/rust-workspace/tests-build + + # Execution-only nextest workers receive the test archive, never the full + # compiler snapshot. + - key: tests-unit + use: head-source + if: ${{ tasks.route.values.run-rust-ci == 'true' }} + run: bash ops/ci/runtime/rust-workspace.sh tests-unit + env: + TEST_ARCHIVE: ${{ tasks.tests-build.artifacts.report }} + RWX_RUN_ID: + cache-key: included + RWX_TASK_ATTEMPT_NUMBER: + cache-key: included + cache: false + timeout: 90m + runner: *test-runner + outputs: + filesystem: false + test-results: + - path: .ci/rust-workspace/tests-unit/junit.xml + artifacts: + - key: report + path: .ci/rust-workspace/tests-unit + + # Compiler-dependent verdicts inherit the producer's immutable filesystem. + # Publish reports only: never pack, restore or republish the compiler tree. + - key: tests-compiler + use: [head-source, tests-build] + if: ${{ tasks.route.values.run-rust-ci == 'true' }} + run: bash ops/ci/runtime/rust-workspace.sh tests-compiler + env: + RWX_RUN_ID: + cache-key: included + RWX_TASK_ATTEMPT_NUMBER: + cache-key: included + cache: false + timeout: 90m + runner: *test-runner + outputs: + filesystem: false + artifacts: + - key: report + path: .ci/rust-workspace/tests-compiler + + - key: build + use: head-source + if: ${{ init.cache-warm == 'true' || tasks.route.values.run-rust-ci == 'true' }} + run: bash ops/ci/runtime/rust-workspace.sh build + env: &fresh-rust-env + RWX_RUN_ID: + cache-key: included + RWX_TASK_ATTEMPT_NUMBER: + cache-key: included + tool-cache: rust-workspace-build-${{ init.cache-epoch }} + timeout: 90m + runner: + cpus: 16 + memory: 32gb + outputs: + filesystem: &compiler-output + filter: + workspace: [rust/target, .ci/rust-cache, rust/op-reth/crates/chainspec/res/superchain-configs.tar] + system: [] + artifacts: + - key: report + path: .ci/rust-workspace/build + + - key: docs + use: head-source + if: ${{ init.cache-warm == 'true' || tasks.route.values.run-rust-ci == 'true' }} + run: bash ops/ci/runtime/rust-workspace.sh docs + env: *fresh-rust-env + tool-cache: rust-workspace-docs-${{ init.cache-epoch }} + timeout: 90m + runner: + cpus: 8 + memory: 16gb + outputs: + filesystem: *compiler-output + artifacts: + - key: report + path: .ci/rust-workspace/docs + + - key: clippy + use: head-source + if: ${{ init.cache-warm == 'true' || tasks.route.values.run-rust-ci == 'true' }} + run: bash ops/ci/runtime/rust-workspace.sh clippy + env: *fresh-rust-env + tool-cache: rust-workspace-clippy-${{ init.cache-epoch }} + timeout: 90m + runner: + cpus: 16 + memory: 32gb + outputs: + filesystem: *compiler-output + artifacts: + - key: report + path: .ci/rust-workspace/clippy + + - key: no-std + use: head-source + if: ${{ init.cache-warm == 'true' || tasks.route.values.run-rust-ci == 'true' }} + run: bash ops/ci/runtime/rust-workspace.sh no-std + env: *fresh-rust-env + tool-cache: rust-workspace-no-std-${{ init.cache-epoch }} + timeout: 90m + runner: + cpus: 2 + memory: 4gb + outputs: + filesystem: *compiler-output + artifacts: + - key: report + path: .ci/rust-workspace/no-std + + - key: udeps + use: head-source + if: ${{ init.cache-warm == 'true' || tasks.route.values.run-rust-ci == 'true' }} + run: bash ops/ci/runtime/rust-workspace.sh udeps + env: *fresh-rust-env + tool-cache: rust-workspace-udeps-${{ init.cache-epoch }} + timeout: 90m + runner: + cpus: 16 + memory: 32gb + outputs: + filesystem: *compiler-output + artifacts: + - key: report + path: .ci/rust-workspace/udeps + + # Reuse compiler preparation; the separate consumer always checks this PR afresh. + - &features-build + key: features-build-0 + use: rust-dependencies + if: ${{ init.cache-warm == 'true' || tasks.route.values.run-rust-ci == 'true' }} + run: python3 ops/ci/runtime/rust-compiler-baseline.py features + env: + <<: *compiler-base-env + CI_RUST_PARTITION_INDEX: "0" + CI_RUST_PARTITION_TOTAL: "10" + timeout: 120m + runner: + cpus: 16 + memory: 32gb + outputs: &features-build-outputs + filesystem: *compiler-output + artifacts: + - key: report + path: .ci/rust-workspace/compiler-baseline/features-0 + + # 16 GB: on 8 GB, one partition failed once with ENOMEM while sccache opened a file, and + # passed on a retry. + - &features + key: features-0 + use: [features-build-0, head-source] + if: ${{ init.cache-warm != 'true' && tasks.route.values.run-rust-ci == 'true' }} + run: bash ops/ci/runtime/rust-workspace.sh features + env: + <<: *fresh-rust-env + CI_RUST_PARTITION_INDEX: "0" + CI_RUST_PARTITION_TOTAL: "10" + cache: false + timeout: 120m + runner: + cpus: 4 + memory: 16gb + outputs: &features-outputs + filesystem: false + artifacts: + - key: report + path: .ci/rust-workspace/features-0 + + - <<: *features-build + key: features-build-1 + env: + <<: *compiler-base-env + CI_RUST_PARTITION_INDEX: "1" + CI_RUST_PARTITION_TOTAL: "10" + outputs: + <<: *features-build-outputs + artifacts: + - key: report + path: .ci/rust-workspace/compiler-baseline/features-1 + + - <<: *features + key: features-1 + use: [features-build-1, head-source] + env: + <<: *fresh-rust-env + CI_RUST_PARTITION_INDEX: "1" + CI_RUST_PARTITION_TOTAL: "10" + outputs: + <<: *features-outputs + artifacts: + - key: report + path: .ci/rust-workspace/features-1 + + - <<: *features-build + key: features-build-2 + env: + <<: *compiler-base-env + CI_RUST_PARTITION_INDEX: "2" + CI_RUST_PARTITION_TOTAL: "10" + outputs: + <<: *features-build-outputs + artifacts: + - key: report + path: .ci/rust-workspace/compiler-baseline/features-2 + + - <<: *features + key: features-2 + use: [features-build-2, head-source] + env: + <<: *fresh-rust-env + CI_RUST_PARTITION_INDEX: "2" + CI_RUST_PARTITION_TOTAL: "10" + outputs: + <<: *features-outputs + artifacts: + - key: report + path: .ci/rust-workspace/features-2 + + - <<: *features-build + key: features-build-3 + env: + <<: *compiler-base-env + CI_RUST_PARTITION_INDEX: "3" + CI_RUST_PARTITION_TOTAL: "10" + outputs: + <<: *features-build-outputs + artifacts: + - key: report + path: .ci/rust-workspace/compiler-baseline/features-3 + + - <<: *features + key: features-3 + use: [features-build-3, head-source] + env: + <<: *fresh-rust-env + CI_RUST_PARTITION_INDEX: "3" + CI_RUST_PARTITION_TOTAL: "10" + outputs: + <<: *features-outputs + artifacts: + - key: report + path: .ci/rust-workspace/features-3 + + - <<: *features-build + key: features-build-4 + env: + <<: *compiler-base-env + CI_RUST_PARTITION_INDEX: "4" + CI_RUST_PARTITION_TOTAL: "10" + outputs: + <<: *features-build-outputs + artifacts: + - key: report + path: .ci/rust-workspace/compiler-baseline/features-4 + + - <<: *features + key: features-4 + use: [features-build-4, head-source] + env: + <<: *fresh-rust-env + CI_RUST_PARTITION_INDEX: "4" + CI_RUST_PARTITION_TOTAL: "10" + outputs: + <<: *features-outputs + artifacts: + - key: report + path: .ci/rust-workspace/features-4 + + - <<: *features-build + key: features-build-5 + env: + <<: *compiler-base-env + CI_RUST_PARTITION_INDEX: "5" + CI_RUST_PARTITION_TOTAL: "10" + outputs: + <<: *features-build-outputs + artifacts: + - key: report + path: .ci/rust-workspace/compiler-baseline/features-5 + + - <<: *features + key: features-5 + use: [features-build-5, head-source] + env: + <<: *fresh-rust-env + CI_RUST_PARTITION_INDEX: "5" + CI_RUST_PARTITION_TOTAL: "10" + outputs: + <<: *features-outputs + artifacts: + - key: report + path: .ci/rust-workspace/features-5 + + - <<: *features-build + key: features-build-6 + env: + <<: *compiler-base-env + CI_RUST_PARTITION_INDEX: "6" + CI_RUST_PARTITION_TOTAL: "10" + outputs: + <<: *features-build-outputs + artifacts: + - key: report + path: .ci/rust-workspace/compiler-baseline/features-6 + + - <<: *features + key: features-6 + use: [features-build-6, head-source] + env: + <<: *fresh-rust-env + CI_RUST_PARTITION_INDEX: "6" + CI_RUST_PARTITION_TOTAL: "10" + outputs: + <<: *features-outputs + artifacts: + - key: report + path: .ci/rust-workspace/features-6 + + - <<: *features-build + key: features-build-7 + env: + <<: *compiler-base-env + CI_RUST_PARTITION_INDEX: "7" + CI_RUST_PARTITION_TOTAL: "10" + outputs: + <<: *features-build-outputs + artifacts: + - key: report + path: .ci/rust-workspace/compiler-baseline/features-7 + + - <<: *features + key: features-7 + use: [features-build-7, head-source] + env: + <<: *fresh-rust-env + CI_RUST_PARTITION_INDEX: "7" + CI_RUST_PARTITION_TOTAL: "10" + outputs: + <<: *features-outputs + artifacts: + - key: report + path: .ci/rust-workspace/features-7 + + - <<: *features-build + key: features-build-8 + env: + <<: *compiler-base-env + CI_RUST_PARTITION_INDEX: "8" + CI_RUST_PARTITION_TOTAL: "10" + outputs: + <<: *features-build-outputs + artifacts: + - key: report + path: .ci/rust-workspace/compiler-baseline/features-8 + + - <<: *features + key: features-8 + use: [features-build-8, head-source] + env: + <<: *fresh-rust-env + CI_RUST_PARTITION_INDEX: "8" + CI_RUST_PARTITION_TOTAL: "10" + outputs: + <<: *features-outputs + artifacts: + - key: report + path: .ci/rust-workspace/features-8 + + - <<: *features-build + key: features-build-9 + env: + <<: *compiler-base-env + CI_RUST_PARTITION_INDEX: "9" + CI_RUST_PARTITION_TOTAL: "10" + outputs: + <<: *features-build-outputs + artifacts: + - key: report + path: .ci/rust-workspace/compiler-baseline/features-9 + + - <<: *features + key: features-9 + use: [features-build-9, head-source] + env: + <<: *fresh-rust-env + CI_RUST_PARTITION_INDEX: "9" + CI_RUST_PARTITION_TOTAL: "10" + outputs: + <<: *features-outputs + artifacts: + - key: report + path: .ci/rust-workspace/features-9 + + - key: wasm-unknown + use: [head-source, wasm-tools] + if: ${{ init.cache-warm == 'true' || tasks.route.values.run-rust-ci == 'true' }} + run: bash ops/ci/runtime/rust-workspace.sh wasm-unknown + env: *fresh-rust-env + tool-cache: rust-workspace-wasm-unknown-${{ init.cache-epoch }} + timeout: 90m + runner: + cpus: 2 + memory: 4gb + outputs: + filesystem: *compiler-output + test-results: + - path: .ci/rust-workspace/wasm-unknown/checks.junit.xml + artifacts: + - key: report + path: .ci/rust-workspace/wasm-unknown + + - key: wasm-wasi + use: [head-source, wasm-tools] + if: ${{ init.cache-warm == 'true' || tasks.route.values.run-rust-ci == 'true' }} + run: bash ops/ci/runtime/rust-workspace.sh wasm-wasi + env: *fresh-rust-env + tool-cache: rust-workspace-wasm-wasi-${{ init.cache-epoch }} + timeout: 90m + runner: + cpus: 2 + memory: 4gb + outputs: + filesystem: *compiler-output + test-results: + - path: .ci/rust-workspace/wasm-wasi/checks.junit.xml + artifacts: + - key: report + path: .ci/rust-workspace/wasm-wasi + + - key: zepter + use: [head-source, lint-tools] + if: ${{ tasks.route.values.run-rust-ci == 'true' }} + run: bash ops/ci/runtime/rust-workspace.sh zepter + env: *fresh-rust-env + tool-cache: rust-workspace-zepter-${{ init.cache-epoch }} + timeout: 90m + runner: + cpus: 2 + memory: 4gb + outputs: + filesystem: *compiler-output + test-results: + - path: .ci/rust-workspace/zepter/checks.junit.xml + artifacts: + - key: report + path: .ci/rust-workspace/zepter + + - key: deny + use: [head-source, lint-tools] + if: ${{ tasks.route.values.run-rust-ci == 'true' }} + run: bash ops/ci/runtime/rust-workspace.sh deny + env: *fresh-rust-env + cache: false + timeout: 90m + runner: + cpus: 2 + memory: 4gb + outputs: + filesystem: false + test-results: + - path: .ci/rust-workspace/deny/checks.junit.xml + artifacts: + - key: report + path: .ci/rust-workspace/deny + + - key: typos + use: [head-source, lint-tools] + if: ${{ tasks.route.values.run-rust-ci == 'true' }} + run: bash ops/ci/runtime/rust-workspace.sh typos + env: *fresh-rust-env + tool-cache: rust-workspace-typos-${{ init.cache-epoch }} + timeout: 90m + runner: + cpus: 2 + memory: 4gb + outputs: + filesystem: *compiler-output + test-results: + - path: .ci/rust-workspace/typos/checks.junit.xml + artifacts: + - key: report + path: .ci/rust-workspace/typos + + - key: registry + use: head-source + if: ${{ init.cache-warm == 'true' || tasks.route.values.run-rust-ci == 'true' }} + run: bash ops/ci/runtime/rust-workspace.sh registry + env: *fresh-rust-env + tool-cache: rust-workspace-registry-${{ init.cache-epoch }} + timeout: 90m + runner: + cpus: 2 + memory: 4gb + outputs: + filesystem: *compiler-output + test-results: + - path: .ci/rust-workspace/registry/checks.junit.xml + artifacts: + - key: report + path: .ci/rust-workspace/registry + + - key: interop + use: [head-source, go-tools] + if: ${{ tasks.route.values.run-rust-ci == 'true' }} + run: bash ops/ci/runtime/rust-workspace.sh interop + env: *fresh-rust-env + tool-cache: rust-workspace-interop-${{ init.cache-epoch }} + timeout: 90m + runner: + cpus: 2 + memory: 4gb + outputs: + filesystem: + filter: + workspace: [rust/target, .ci/rust-cache, .ci/interop-go, rust/op-reth/crates/chainspec/res/superchain-configs.tar] + system: [] + test-results: + - path: .ci/rust-workspace/interop/checks.junit.xml + artifacts: + - key: report + path: .ci/rust-workspace/interop + + # Preserve Docker/BuildKit data through the normal use dependency, without + # exporting image tarballs. The original recipe builds all configured ELFs. + - key: cannon-env + use: [head-source, cannon-witness] + if: ${{ init.cache-warm == 'true' || tasks.route.values.run-rust-ci == 'true' }} + run: bash ops/ci/runtime/rust-cannon.sh env + docker: preserve-data + tool-cache: rust-cannon-env-${{ init.cache-epoch }} + timeout: 90m + runner: + cpus: 8 + memory: 32gb + outputs: + filesystem: *compiler-output + artifacts: + - key: report + path: .ci/rust-workspace/cannon-env + + - key: cannon-go + use: [head-source, go-tools] + if: ${{ init.cache-warm == 'true' || tasks.route.values.run-rust-ci == 'true' }} + run: bash ops/ci/runtime/rust-cannon.sh go + tool-cache: rust-cannon-go-${{ init.cache-epoch }} + timeout: 30m + runner: + cpus: 8 + memory: 32gb + outputs: + filesystem: + filter: + workspace: [cannon/bin, cannon/multicannon/embeds, .ci/cannon-cache/go] + system: [] + artifacts: + - key: report + path: .ci/rust-workspace/cannon-go + + # Fail a missing/corrupt public fixture before the expensive environment build. + - key: cannon-witness + use: head-source + if: ${{ init.cache-warm == 'true' || tasks.route.values.run-rust-ci == 'true' }} + run: bash ops/ci/runtime/rust-cannon.sh witness + timeout: 15m + outputs: + filesystem: + filter: + workspace: [rust/kona/bin/client/testdata/*.tar.zst, rust/kona/bin/client/testdata/*.etag] + system: [] + artifacts: + - key: report + path: .ci/rust-workspace/cannon-witness + + - key: cannon-lint + use: [head-source, cannon-env] + if: ${{ tasks.route.values.run-rust-ci == 'true' }} + run: bash ops/ci/runtime/rust-cannon.sh lint + env: *fresh-rust-env + docker: preserve-data + tool-cache: rust-cannon-lint-${{ init.cache-epoch }} + timeout: 90m + runner: + cpus: 8 + memory: 32gb + outputs: + filesystem: *compiler-output + test-results: + - path: .ci/rust-workspace/cannon-lint/checks.junit.xml + artifacts: + - key: report + path: .ci/rust-workspace/cannon-lint + + - key: cannon-build + use: [head-source, cannon-env] + if: ${{ init.cache-warm == 'true' || tasks.route.values.run-rust-ci == 'true' }} + run: bash ops/ci/runtime/rust-cannon.sh build + env: *fresh-rust-env + docker: preserve-data + tool-cache: rust-cannon-build-${{ init.cache-epoch }} + timeout: 90m + runner: + cpus: 16 + memory: 32gb + outputs: + filesystem: *compiler-output + test-results: + - path: .ci/rust-workspace/cannon-build/checks.junit.xml + artifacts: + - key: report + path: .ci/rust-workspace/cannon-build + + - key: cannon-offline + use: [head-source, go-tools, cannon-witness, cannon-go, cannon-build] + if: ${{ tasks.route.values.run-rust-ci == 'true' }} + run: bash ops/ci/runtime/rust-cannon.sh offline + env: *fresh-rust-env + docker: preserve-data + tool-cache: rust-cannon-offline-${{ init.cache-epoch }} + timeout: 90m + runner: + cpus: 16 + memory: 32gb + outputs: + filesystem: + filter: + workspace: [rust/target, .ci/rust-cache, .ci/cannon-cache] + system: [] + test-results: + - path: .ci/rust-workspace/cannon-offline/checks.junit.xml + artifacts: + - key: report + path: .ci/rust-workspace/cannon-offline + + - key: rust-fmt-tools + use: tools + filter: [mise.toml, ops/ci/runtime/rwx-prepare.sh] + if: ${{ init.cache-warm == 'true' || tasks.route.values.run-rust-ci == 'true' }} + run: bash ops/ci/runtime/rwx-prepare.sh rust + timeout: 30m + + # Mirrors CircleCI's rust-fmt job. + - key: rust-fmt + use: [code, rust-fmt-tools] + if: ${{ init.cache-warm != 'true' && tasks.route.values.run-rust-ci == 'true' }} + run: | + # Run CircleCI's own toolchain pin check, read from its config, so it cannot drift. + pins="$(mise exec -- yq -r '.jobs["rust-ci-fmt"].steps[] | select(.run.name? == "Check Rust toolchain pins agree") | .run.command' .circleci/continue/rust-ci.yml)" + if [[ -z "$pins" || "$pins" == null ]]; then echo "CircleCI's Rust toolchain pin check is missing" >&2; exit 1; fi + mise exec -- bash -eo pipefail -c "$pins" + mise exec -- just update-superchain-registry-submodule + cd rust + mise exec -- just fmt-check + mise exec -- just check-upstream-mirrors + cache: false + timeout: 20m + runner: + cpus: 2 + memory: 4gb + outputs: + filesystem: false + + # Mirrors CircleCI's rust-reth-tag-pin job. The script needs only git, Python and the GitHub API. + - key: rust-reth-tag-pin + use: [code, rust-fmt-tools] + if: ${{ init.cache-warm != 'true' && tasks.route.values.run-rust-ci == 'true' }} + run: | + cd rust + mise exec -- python3 scripts/upstream-mirrors.py --check-tag-pin + cache: false + timeout: 10m + outputs: + filesystem: false + + # Terminal task states preserve exact Rust dependencies without Main-only failures. + - key: rust-gate-receipt + use: [code, tools] + if: ${{ init.cache-warm != 'true' }} + after: ${{ (build.succeeded || build.failed || build.skipped) && (cannon-build.succeeded || cannon-build.failed || cannon-build.skipped) && (cannon-lint.succeeded || cannon-lint.failed || cannon-lint.skipped) && (cannon-offline.succeeded || cannon-offline.failed || cannon-offline.skipped) && (clippy.succeeded || clippy.failed || clippy.skipped) && (deny.succeeded || deny.failed || deny.skipped) && (docs.succeeded || docs.failed || docs.skipped) && (features-0.succeeded || features-0.failed || features-0.skipped) && (features-1.succeeded || features-1.failed || features-1.skipped) && (features-2.succeeded || features-2.failed || features-2.skipped) && (features-3.succeeded || features-3.failed || features-3.skipped) && (features-4.succeeded || features-4.failed || features-4.skipped) && (features-5.succeeded || features-5.failed || features-5.skipped) && (features-6.succeeded || features-6.failed || features-6.skipped) && (features-7.succeeded || features-7.failed || features-7.skipped) && (features-8.succeeded || features-8.failed || features-8.skipped) && (features-9.succeeded || features-9.failed || features-9.skipped) && (interop.succeeded || interop.failed || interop.skipped) && (no-std.succeeded || no-std.failed || no-std.skipped) && (registry.succeeded || registry.failed || registry.skipped) && (tests-compiler.succeeded || tests-compiler.failed || tests-compiler.skipped) && (tests-unit.succeeded || tests-unit.failed || tests-unit.skipped) && (typos.succeeded || typos.failed || typos.skipped) && (udeps.succeeded || udeps.failed || udeps.skipped) && (wasm-unknown.succeeded || wasm-unknown.failed || wasm-unknown.skipped) && (wasm-wasi.succeeded || wasm-wasi.failed || wasm-wasi.skipped) && (zepter.succeeded || zepter.failed || zepter.skipped) && (rust-fmt.succeeded || rust-fmt.failed || rust-fmt.skipped) && (rust-reth-tag-pin.succeeded || rust-reth-tag-pin.failed || rust-reth-tag-pin.skipped) }} + run: python3 ops/ci/runtime/pr-gate.py receipt rust-workspace + env: + GROUP_SELECTED: ${{ tasks.route.values.run-rust-ci }} + TASK_BUILD_SUCCEEDED: ${{ tasks.build.succeeded }} + TASK_BUILD_FAILED: ${{ tasks.build.failed }} + TASK_BUILD_SKIPPED: ${{ tasks.build.skipped }} + TASK_CANNON_BUILD_SUCCEEDED: ${{ tasks.cannon-build.succeeded }} + TASK_CANNON_BUILD_FAILED: ${{ tasks.cannon-build.failed }} + TASK_CANNON_BUILD_SKIPPED: ${{ tasks.cannon-build.skipped }} + TASK_CANNON_LINT_SUCCEEDED: ${{ tasks.cannon-lint.succeeded }} + TASK_CANNON_LINT_FAILED: ${{ tasks.cannon-lint.failed }} + TASK_CANNON_LINT_SKIPPED: ${{ tasks.cannon-lint.skipped }} + TASK_CANNON_OFFLINE_SUCCEEDED: ${{ tasks.cannon-offline.succeeded }} + TASK_CANNON_OFFLINE_FAILED: ${{ tasks.cannon-offline.failed }} + TASK_CANNON_OFFLINE_SKIPPED: ${{ tasks.cannon-offline.skipped }} + TASK_CLIPPY_SUCCEEDED: ${{ tasks.clippy.succeeded }} + TASK_CLIPPY_FAILED: ${{ tasks.clippy.failed }} + TASK_CLIPPY_SKIPPED: ${{ tasks.clippy.skipped }} + TASK_DOCS_SUCCEEDED: ${{ tasks.docs.succeeded }} + TASK_DOCS_FAILED: ${{ tasks.docs.failed }} + TASK_DOCS_SKIPPED: ${{ tasks.docs.skipped }} + TASK_DENY_SUCCEEDED: ${{ tasks.deny.succeeded }} + TASK_DENY_FAILED: ${{ tasks.deny.failed }} + TASK_DENY_SKIPPED: ${{ tasks.deny.skipped }} + TASK_FEATURES_0_SUCCEEDED: ${{ tasks.features-0.succeeded }} + TASK_FEATURES_0_FAILED: ${{ tasks.features-0.failed }} + TASK_FEATURES_0_SKIPPED: ${{ tasks.features-0.skipped }} + TASK_FEATURES_1_SUCCEEDED: ${{ tasks.features-1.succeeded }} + TASK_FEATURES_1_FAILED: ${{ tasks.features-1.failed }} + TASK_FEATURES_1_SKIPPED: ${{ tasks.features-1.skipped }} + TASK_FEATURES_2_SUCCEEDED: ${{ tasks.features-2.succeeded }} + TASK_FEATURES_2_FAILED: ${{ tasks.features-2.failed }} + TASK_FEATURES_2_SKIPPED: ${{ tasks.features-2.skipped }} + TASK_FEATURES_3_SUCCEEDED: ${{ tasks.features-3.succeeded }} + TASK_FEATURES_3_FAILED: ${{ tasks.features-3.failed }} + TASK_FEATURES_3_SKIPPED: ${{ tasks.features-3.skipped }} + TASK_FEATURES_4_SUCCEEDED: ${{ tasks.features-4.succeeded }} + TASK_FEATURES_4_FAILED: ${{ tasks.features-4.failed }} + TASK_FEATURES_4_SKIPPED: ${{ tasks.features-4.skipped }} + TASK_FEATURES_5_SUCCEEDED: ${{ tasks.features-5.succeeded }} + TASK_FEATURES_5_FAILED: ${{ tasks.features-5.failed }} + TASK_FEATURES_5_SKIPPED: ${{ tasks.features-5.skipped }} + TASK_FEATURES_6_SUCCEEDED: ${{ tasks.features-6.succeeded }} + TASK_FEATURES_6_FAILED: ${{ tasks.features-6.failed }} + TASK_FEATURES_6_SKIPPED: ${{ tasks.features-6.skipped }} + TASK_FEATURES_7_SUCCEEDED: ${{ tasks.features-7.succeeded }} + TASK_FEATURES_7_FAILED: ${{ tasks.features-7.failed }} + TASK_FEATURES_7_SKIPPED: ${{ tasks.features-7.skipped }} + TASK_FEATURES_8_SUCCEEDED: ${{ tasks.features-8.succeeded }} + TASK_FEATURES_8_FAILED: ${{ tasks.features-8.failed }} + TASK_FEATURES_8_SKIPPED: ${{ tasks.features-8.skipped }} + TASK_FEATURES_9_SUCCEEDED: ${{ tasks.features-9.succeeded }} + TASK_FEATURES_9_FAILED: ${{ tasks.features-9.failed }} + TASK_FEATURES_9_SKIPPED: ${{ tasks.features-9.skipped }} + TASK_INTEROP_SUCCEEDED: ${{ tasks.interop.succeeded }} + TASK_INTEROP_FAILED: ${{ tasks.interop.failed }} + TASK_INTEROP_SKIPPED: ${{ tasks.interop.skipped }} + TASK_NO_STD_SUCCEEDED: ${{ tasks.no-std.succeeded }} + TASK_NO_STD_FAILED: ${{ tasks.no-std.failed }} + TASK_NO_STD_SKIPPED: ${{ tasks.no-std.skipped }} + TASK_REGISTRY_SUCCEEDED: ${{ tasks.registry.succeeded }} + TASK_REGISTRY_FAILED: ${{ tasks.registry.failed }} + TASK_REGISTRY_SKIPPED: ${{ tasks.registry.skipped }} + TASK_TESTS_COMPILER_SUCCEEDED: ${{ tasks.tests-compiler.succeeded }} + TASK_TESTS_COMPILER_FAILED: ${{ tasks.tests-compiler.failed }} + TASK_TESTS_COMPILER_SKIPPED: ${{ tasks.tests-compiler.skipped }} + TASK_TESTS_UNIT_SUCCEEDED: ${{ tasks.tests-unit.succeeded }} + TASK_TESTS_UNIT_FAILED: ${{ tasks.tests-unit.failed }} + TASK_TESTS_UNIT_SKIPPED: ${{ tasks.tests-unit.skipped }} + TASK_TYPOS_SUCCEEDED: ${{ tasks.typos.succeeded }} + TASK_TYPOS_FAILED: ${{ tasks.typos.failed }} + TASK_TYPOS_SKIPPED: ${{ tasks.typos.skipped }} + TASK_UDEPS_SUCCEEDED: ${{ tasks.udeps.succeeded }} + TASK_UDEPS_FAILED: ${{ tasks.udeps.failed }} + TASK_UDEPS_SKIPPED: ${{ tasks.udeps.skipped }} + TASK_WASM_UNKNOWN_SUCCEEDED: ${{ tasks.wasm-unknown.succeeded }} + TASK_WASM_UNKNOWN_FAILED: ${{ tasks.wasm-unknown.failed }} + TASK_WASM_UNKNOWN_SKIPPED: ${{ tasks.wasm-unknown.skipped }} + TASK_WASM_WASI_SUCCEEDED: ${{ tasks.wasm-wasi.succeeded }} + TASK_WASM_WASI_FAILED: ${{ tasks.wasm-wasi.failed }} + TASK_WASM_WASI_SKIPPED: ${{ tasks.wasm-wasi.skipped }} + TASK_ZEPTER_SUCCEEDED: ${{ tasks.zepter.succeeded }} + TASK_ZEPTER_FAILED: ${{ tasks.zepter.failed }} + TASK_ZEPTER_SKIPPED: ${{ tasks.zepter.skipped }} + TASK_RUST_FMT_SUCCEEDED: ${{ tasks.rust-fmt.succeeded }} + TASK_RUST_FMT_FAILED: ${{ tasks.rust-fmt.failed }} + TASK_RUST_FMT_SKIPPED: ${{ tasks.rust-fmt.skipped }} + TASK_RUST_RETH_TAG_PIN_SUCCEEDED: ${{ tasks.rust-reth-tag-pin.succeeded }} + TASK_RUST_RETH_TAG_PIN_FAILED: ${{ tasks.rust-reth-tag-pin.failed }} + TASK_RUST_RETH_TAG_PIN_SKIPPED: ${{ tasks.rust-reth-tag-pin.skipped }} + cache: false + timeout: 10m + outputs: + filesystem: false + artifacts: + - key: receipt + path: .ci/pr-gates/groups/rust-workspace diff --git a/.rwx/sp1-guest.yml b/.rwx/sp1-guest.yml new file mode 100644 index 00000000000..5d0a7357a40 --- /dev/null +++ b/.rwx/sp1-guest.yml @@ -0,0 +1,173 @@ +# Complete SP1 guest ELF build and fresh original checks; Circle retains required ownership. +on: + cache-rebuild: + if: ${{ event.git.branch == 'develop' }} + target: [build] + init: + commit-sha: ${{ event.git.sha }} + branch: ${{ event.git.branch }} + cache-warm: "true" + cache-epoch: v1 + target-cache-mode: keep + cli: + init: + commit-sha: ${{ event.git.sha }} + branch: ${{ event.git.branch }} + cache-warm: "false" + cache-epoch: v1 + target-cache-mode: keep + +tool-cache: + vault: optimism-op-reth-shadow + +base: + image: ubuntu:24.04@sha256:008173c23f95b170204355c12626cb5a965d779a7e1283b09e9cffbb1bf33ca3 + config: rwx/base 1.2.0 + arch: x86_64 + +defaults: + runner: + cpus: 2 + memory: 8gb + +tasks: + - key: code + call: git/clone 2.2.0 + with: + repository: https://github.com/ethereum-optimism/optimism.git + ref: ${{ init.commit-sha }} + preserve-git-dir: true + fetch-full-depth: true + submodules: false + + - key: bootstrap-inputs + use: code + run: 'true' + filter: [mise.toml, ops/ci/runtime] + outputs: + filesystem: + filter: + workspace: [mise.toml, ops/ci/runtime/apt-install.sh, ops/ci/runtime/rwx-prepare.sh, ops/ci/runtime/rwx-contracts-prepare.sh, ops/ci/runtime/rwx-rust-prepare.sh, ops/ci/runtime/op-reth-shadow.sh, ops/ci/runtime/op-reth-report.py, ops/ci/runtime/rust-target-cache.py] + system: [] + + - key: mise + call: mise/install 1.1.0 + with: + mise-version: "2026.2.2" + install: "false" + + - key: tools + use: [mise, bootstrap-inputs] + call: ${{ run.dir }}/packages/toolchain-common.yml + + - key: route + use: [code, tools] + run: bash ops/ci/runtime/rwx-metadata.sh + env: + CI_CACHE_WARM: ${{ init.cache-warm }} + CI_BRANCH: ${{ init.branch }} + CI_COMMIT_SHA: ${{ init.commit-sha }} + cache: false + timeout: 10m + outputs: + filesystem: false + + - key: rust-tools + use: [bootstrap-inputs, tools] + if: ${{ init.cache-warm == 'true' || tasks.route.values.run-main == 'true' }} + call: ${{ run.dir }}/packages/toolchain-rust.yml + with: + mode: sp1 + + # CircleCI's sp1-toolchain-restore-or-install command. The filter lists the + # files that the script and its cargo and just calls read. + - key: toolchain + use: [code, rust-tools] + if: ${{ init.cache-warm == 'true' || tasks.route.values.run-main == 'true' }} + run: bash ops/ci/runtime/sp1-guest-toolchain.sh + filter: + - mise.toml + - rust/Cargo.toml + - rust/Cargo.lock + - rust/kona/sp1/programs/Cargo.toml + - rust/kona/sp1/programs/Cargo.lock + - rust/kona/sp1/justfile + - rust/rust-toolchain.toml + - rust/.cargo/config.toml + - ops/ci/runtime/sp1-guest-toolchain.sh + timeout: 40m + + # CircleCI's steps up to the ELF marker check. The artifact hands the ELFs + # and vkeys.toml to the check task. + - key: build + use: [code, toolchain] + if: ${{ init.cache-warm == 'true' || tasks.route.values.run-main == 'true' }} + run: python3 ops/ci/runtime/sp1-guest.py --phase build --features test-config-fallback + env: + CI: "true" + CARGO_INCREMENTAL: "0" + SP1_GUEST_TARGET_MODE: ${{ init.target-cache-mode }} + tool-cache: sp1-guest-canonical-elf-${{ init.cache-epoch }} + timeout: 90m + runner: + cpus: 8 + memory: 16gb + outputs: + filesystem: + filter: + workspace: [.ci/sp1-cache/cargo, .ci/sp1-cache/elf-target, .ci/sp1-cache/elf-sccache] + system: [] + artifacts: + - key: dependency + path: .ci/sp1-guest/dependency + + - key: helper-tests + use: [code, tools] + if: ${{ init.cache-warm != 'true' && tasks.route.values.run-main == 'true' }} + run: python3 -m unittest discover -s ops/ci/tests -p 'test_sp1_guest.py' + cache: false + timeout: 15m + outputs: + filesystem: false + + # CircleCI's test, lint and vkey crate steps. The exit status of each recipe + # is the verdict. + - key: check + use: [code, toolchain] + if: ${{ init.cache-warm != 'true' && tasks.route.values.run-main == 'true' }} + run: python3 ops/ci/runtime/sp1-guest.py --phase checks --features test-config-fallback --artifact "$SP1_GUEST_ARTIFACT" + env: + CI: "true" + CARGO_INCREMENTAL: "0" + SP1_GUEST_TARGET_MODE: ${{ init.target-cache-mode }} + SP1_GUEST_ARTIFACT: ${{ tasks.build.artifacts.dependency }} + tool-cache: sp1-guest-checks-${{ init.cache-epoch }} + cache: false + timeout: 90m + runner: + cpus: 8 + memory: 16gb + outputs: + filesystem: + filter: + workspace: [.ci/sp1-cache/cargo, .ci/sp1-cache/checks-target, .ci/sp1-cache/checks-sccache] + system: [] + + # Exact Main dependency states; unrelated workload failures stay outside this gate. + - key: main-gate-receipt + use: [code, tools] + after: ${{ (check.succeeded || check.failed || check.skipped) }} + if: ${{ init.cache-warm != 'true' }} + run: python3 ops/ci/runtime/pr-gate.py receipt main-sp1 + env: + GROUP_SELECTED: ${{ tasks.route.values.run-main }} + TASK_CHECK_SUCCEEDED: ${{ tasks.check.succeeded }} + TASK_CHECK_FAILED: ${{ tasks.check.failed }} + TASK_CHECK_SKIPPED: ${{ tasks.check.skipped }} + cache: false + timeout: 10m + outputs: + filesystem: false + artifacts: + - key: receipt + path: .ci/pr-gates/groups/main-sp1 diff --git a/AGENTS.md b/AGENTS.md index 8e55cbbcc10..7dc733c4ff8 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -98,6 +98,7 @@ and the relevant language guide below. More detailed guidance for AI agents can - [docs/ai/ci-ops.md](docs/ai/ci-ops.md) - CI/CD operations - [docs/ai/ci-config-review.md](docs/ai/ci-config-review.md) - Reviewing changes to CI config (`.circleci/`, `.github/workflows/`), or to anything a CI job runs: gate coverage, required checks, path filtering, caching, jobs that never run on a PR, cost cuts that must not regress coverage, maintainability or security, plus general CircleCI/GHA best practices +- [docs/ai/rwx-shadow-ci.md](docs/ai/rwx-shadow-ci.md) - The optional RWX shadow CI (`.rwx/`, `ops/ci/`): when it runs, how to turn it off, routing, gates and how to compare it with CircleCI - [docs/ai/docker.md](docs/ai/docker.md) - Docker image builds: making every external fetch (apt/apk/curl/wget) retry so registry/CDN blips don't flake CI - [docs/ai/contract-dev.md](docs/ai/contract-dev.md) - Smart contract development - [docs/ai/standard-validator-review.md](docs/ai/standard-validator-review.md) - Reviewing `StandardValidator` for assertions it should make but doesn't: cross-game symmetry, diff-driven coverage, read-versus-assert, plus the false-positive traps (pass-through getters, implementation-pinned immutables) that make naive gap-hunting noisy. Pairs with the `standard-validator-reviewer` agent diff --git a/docs/ai/rwx-shadow-ci.md b/docs/ai/rwx-shadow-ci.md new file mode 100644 index 00000000000..c10fd602249 --- /dev/null +++ b/docs/ai/rwx-shadow-ci.md @@ -0,0 +1,315 @@ +# RWX shadow CI + +RWX (rwx.com) runs an optional shadow of the CircleCI PR workloads so the team can +compare the two providers before any decision to migrate. CircleCI stays the only +required CI: it owns `ci-gate`, `required-contracts-ci`, `required-rust-ci` and +`required-rust-e2e`, and nothing in RWX gates a merge. RWX changes nothing that +CircleCI runs. + +## When RWX runs + +| Trigger | Definition | Condition | +| --- | --- | --- | +| GitHub push | `.rwx/pr-gates.yml` | Any branch push to this repository; tag pushes start no run | +| Cache rebuild | Every definition with `on.cache-rebuild` | Branch is `develop`; RWX starts these to re-warm caches | +| CLI | Every definition | An operator runs `rwx run` | + +Like CircleCI, RWX runs on every branch push, so a PR from a branch of this +repository gets RWX's optional checks next to CircleCI's. A new push to a branch +cancels the older RWX run on that branch; `develop` runs every commit. + +A fork PR cannot push to this repository, so it starts no run by itself. When a +maintainer comments `/ci authorize`, bailiff pushes the authorized commit to an +`external-fork/` branch of this repository (see +`ops/book/src/ci/pr-authorization.md`). That is a branch push. It starts an RWX +run next to CircleCI's, and the run can read the test vault. Before you +authorize a fork PR, review its changes under `.rwx/` and `ops/ci/` as you +review `.circleci/`: RWX reads its definitions from the pushed commit, so those +files decide what runs with that access. Push no other unreviewed commit to a +branch of this repository. + +RWX reads the definitions from the pushed commit. A trigger change therefore +takes effect for the next push that contains it. + +## Turning RWX off + +- **From the repository:** set the `if:` of the `on.github.push` trigger in + `.rwx/pr-gates.yml` to `false`, or delete the `github:` block, and merge. Later + pushes of branches that contain the change start no runs; older branches keep + their old trigger until they merge develop. The `cache-rebuild` triggers stop being useful without pushes; + delete them too for a full stop. +- **Immediately, without a merge:** a GitHub organization owner suspends the RWX + GitHub App installation, or removes this repository from its access list. +- **Permanently:** delete `.rwx/`, `ops/ci/` and this guide, and remove the + guide's index line from `AGENTS.md`. Nothing else depends on them. + +## Layout + +| Path | Content | +| --- | --- | +| `.rwx/pr-gates.yml` | Coordinator. Calls one definition per workload, then mirrors the CircleCI gates as optional checks | +| `.rwx/*.yml` | One definition per workload | +| `.rwx/packages/` | Reusable tool, build and verdict packages that the definitions call | +| `ops/ci/runtime/` | Scripts that RWX tasks run, the gate policy `pr-gates.json` and the Go test timing seed | +| `ops/ci/tests/` | Unit tests for `ops/ci/runtime/` | +| `ops/ci/migration/` | Transitional tools tied to CircleCI: the gate alignment check and the CI result comparer | + +A task that mirrors a CircleCI job runs the command of that job, and its exit +status is the verdict. Follow that shape for a new task: `run:` the recipe, add +the environment that the CircleCI job sets, and name the job in a comment. Do +not wrap the command in a script that re-checks what the recipe or RWX already +guarantees. "Known differences" lists where a task departs from this. + +## Routing + +`ops/ci/runtime/rwx-metadata.sh` selects workloads with CircleCI's own routing +data and policy (`.circleci/routing.yml`, +`.circleci/scripts/collect-params.sh` and +`.circleci/scripts/compute-workflow-conditions.sh`), which it runs read-only. +RWX adds one rule: a change under `.rwx/` or `ops/ci/`, or an uncommitted change +uploaded by `rwx run`, selects every workload. On `develop` every workload runs, +as on CircleCI. There the route task fails if the policy leaves one of the four +shadowed workflows out: a flag that CircleCI renamed would otherwise read as +"not selected" on every branch, and each receipt would pass with its tasks +skipped. + +RWX compares every branch with `develop`. CircleCI compares a PR with its target +branch. The two providers therefore select the same workloads for a PR into +`develop`; for a PR into a release branch, RWX can select more. + +## Gates + +`ops/ci/runtime/pr-gates.json` lists the prerequisites of `ci-gate`, +`required-contracts-ci`, `required-rust-ci` and `required-rust-e2e`, and the RWX +tasks that mirror each one. `ops/ci/migration/circle-alignment.py`, which runs in RWX's +`ci-unit-tests` task, fails when one of those gates' `requires:` lists no longer +names the same jobs, and prints each difference. After a CircleCI gate change, +update `pr-gates.json`. `ci-gate` requires `ci-gate-proposals`, the gate of the +release branches; the policy of `ci-gate` names the jobs that +`ci-gate-proposals` requires, and RWX has no separate mirror of that gate. + +Each workload has a receipt task (`ops/ci/runtime/pr-gate.py receipt`). It +passes if the route selected the workload and every mirrored task succeeded, or +the route did not select it and every mirrored task was skipped. The +coordinator has one status task per gate (`pr-gate.py status`). It passes if +every receipt of the gate succeeded, and it records each CircleCI prerequisite +with its outcome. `ops/ci/tests/test_pr_gate.py` fails when `pr-gates.json` and +the definitions name different tasks. + +CircleCI splits the release build that `required-rust-e2e` requires across two +jobs, so that the E2E jobs wait only for the binaries they run. RWX has the +same two tasks: `binaries`, and `release`, which starts from the output of +`binaries` and builds the rest. `JOBS` in +`ops/ci/runtime/rust-e2e.py` holds the package, shard count and timeout of each +E2E suite. After a CircleCI E2E change, update it together with `pr-gates.json` +and `.rwx/rust-e2e.yml`. + +The RWX checks (`optimism-rwx-pr-coordinator` and the `optimism-*-shadow` +checks) are not required by any ruleset. + +## Caches and credentials + +- Tool caches use the `optimism-op-reth-shadow` vault. Only + `refs/heads/develop` may write. Other branches restore the `develop` baseline + read-only and compile their own changes. +- The Go tests and the L1 contract upgrade tests read archive RPC URLs from the + test-only `optimism-go-tests-rpc-shadow` vault. A branch needs a grant on this + vault for those tasks to run. In a grant's branch pattern, `*` matches one path + segment and `**` matches nothing, so the vault grants `refs/heads/*`, + `refs/heads/*/*` and `refs/heads/*/*/*` (merge-queue branches have two + slashes). A branch with more slashes gets no access. +- RWX holds no production publishing credentials. + +Check the grants with `rwx vaults access list --vault --json`. + +## Comparing with CircleCI + +RWX does not change CircleCI, so CircleCI produces only its normal artifacts. +Compare RWX results with CircleCI's test results for the same commit and +workload. `ops/ci/migration/compare-ci.py` takes the report files of each +provider, one per shard, and prints the test cases that are missing from the +candidate, extra in it, or different: + +```bash +python3 ops/ci/migration/compare-ci.py --baseline circle-tests.json \ + --candidate rwx-shard-0.xml rwx-shard-1.xml +``` + +A report is JUnit XML, `go test -json` events (as gotestsum retains them), or +one complete response of CircleCI's job test-metadata API. That API returns a +large job in pages; join the `items` of every page first. The exit status is 0 +for equivalent, 1 for different, 2 for rejected input and 3 when the two sides +have no case in common. The tool does not check that both sides are from the +same commit. Nothing collects the report files yet: download them by hand. + +The lint, check and build jobs publish no per-test results on either provider. +For those, compare the task's command and its conclusion. + +Compare wall time and cost per workload, including setup and cache transfer, +rather than per test. + +## Speed and cost + +RWX bills each vCPU by the second. A larger runner costs the same for the same +work only when the task uses its CPUs. Choose a runner size, a shard count and +a cache key for RWX from a measurement on RWX. A CircleCI resource class or +`parallelism` value is not a default. The largest runner that this RWX +organization can start has 16 CPUs. + +- **Dependency builds.** `ops/ci/runtime/go-dependency-source.py plan` names, + for each dependency, the commit that last changed its build inputs. The + dependencies are the Go dependency, the contracts, kona, op-reth, the SP1 + executor, the prestate and the Go compiler seed. Every workload passes that + commit and the same tool tasks to the build package (`.rwx/packages/go-*.yml`), + and the build checks the commit out itself. The build then has one cache key: + a later commit that leaves the inputs unchanged reuses it, and so do the other + workloads of the run. A task passes on every layer that it uses, and the tool + tasks carry the checkout of the run. So the build has an input `filter` that + leaves it only `mise.toml` and its own outputs, and it writes its checkout + itself: the run's checkout stays out of the build's cache key, and the build's + checkout stays out of its filtered output. The Go compiler seeds work the same + way. Each tool package, and each tool task that a workload defines itself, has + an input `filter` for the same reason: a tool task with the checkout in its + cache key runs again for each commit, delays the tasks that wait for it, and + gives each of them a new cache key. + Before a task restores an output, `go-artifacts.py` checks that the inputs + equal those of the task's commit, and that the task's checkout has no + modified input. `SCOPES` in the script lists the inputs; add a file there when + a build starts to read it. Of `ops/ci/runtime`, a scope names only the scripts + that the build runs, and `test_go_dependency_source.py` fails when a script + that they reach is missing. `.rwx` and the tool scripts are not inputs, + because RWX's own cache key covers a task's definition and its tool tasks. + `test_rwx_local_packages.py` fails when a workload builds a dependency in + another way. +- **Contract compiles.** The shared build of the standard suites and the upgrade + variants, and the L2 fork compile, have an input `filter` that names what a + compile reads: the contracts, the Go module and the Go trees of the FFI + helper, and its scripts. A commit that changes none of these reuses the build. + Name the files to include: with exclusions only, RWX gives the task a new + cache key for each commit, even when every remaining file is unchanged. The Go + module set is a dependency build of its own, keyed on `go.mod` and `go.sum`. +- **Rust baselines.** `ops/ci/runtime/rust-target-cache.py base` names the + commit in the same way for the nextest archive and the cargo-hack partitions. +- **Runner sizes.** A task on the critical path of a gate gets the size at + which it stops scaling; a task off that path stays small. Measured: clippy + takes 9:12 on 4 CPUs and 2:50 on 16, udeps 9:16 and 2:43, the workspace build + 9:57 on 8 CPUs and 6:14 on 16, and the slowest cargo-hack baseline 31:08 on 4 + CPUs and 17:52 on 16. no-std is no faster on 16 CPUs than on 8. The L2 fork + compile spends eight of its nine minutes in one solc process, so it has 4 + CPUs. +- **Floors.** More shards do not shorten the main gate: one acceptance test, + `TestExecuteStep_CannonKona`, runs for six minutes, after each shard has + listed the tests for one minute. The four longest Go test packages each have + a shard to themselves. After a change to a compile input, the contracts gate + waits for the L2 fork compile, nine minutes in one solc process, and then for + the paced fork tests. After any other change it waits for the tests only. + +## Known differences from CircleCI + +- **Skipped jobs.** CircleCI halts some jobs unless their paths changed + (`check-changed`): Cannon Go, the Kontrol build, NUT provenance, the fast + contract checks, both generated-mock checks and the forge version check. RWX + runs them whenever their workflow is selected. +- **Base branch.** RWX uses `develop` for routing and for the op-reth codec + baseline. The fast contract checks find a merge-queue base, and otherwise use + `develop`. CircleCI uses the PR's target branch. +- **Go tests.** RWX compiles the test binaries in one task and runs them in a + fresh verdict task with `-count=1`, with the flags of `just go-tests-ci`. It + places packages on shards with the committed timings in + `ops/ci/runtime/go-tests-timings.json`; CircleCI uses its own timing history. +- **Rust E2E.** RWX compiles each suite once and runs the binaries with the + flags of the recipes. It shards tests by count. +- **Rust unit and op-reth integration tests.** RWX builds a nextest archive in + one task and runs it in another, with the recipe's test filter. +- **Acceptance sharding.** CircleCI splits tests across shards by historical + timing (`circleci tests split` in `ops/scripts/shard-tests.sh`). RWX puts + `ops/ci/runtime/acceptance-circleci/circleci` first on `PATH`; it gives each + shard the lines of the recipe's test list whose position, modulo the shard + count, is the shard index. +- **Contract tests.** RWX compiles in a task of its own before the test recipe. + The five standard suites and the eight upgrade variants share one build: + CircleCI builds in each job, and the builds are the same, because the compiler + reads the Foundry profile and only the tests read the feature flags and the + fork variables. + On `develop`, CircleCI shards the standard suite; RWX does not. +- **L2 fork tests.** CircleCI forks a private RPC. RWX has no such URL in a + vault, so it forks the public `https://mainnet.optimism.io` through + `ops/ci/runtime/rpc-pacer.py`, which starts at most two requests per second. + At forge's own rate the public RPC answers HTTP 429 for longer than forge + retries. The recipe and its arguments are CircleCI's. To remove the pacer, + add the private URL to an RWX vault and pass it as `L2_FORK_RPC_URL`. +- **cargo-hack partitions.** RWX seeds the package shuffle with the hash of + `rust/Cargo.lock` instead of the CircleCI workflow ID, so each partition + checks different crates. All partitions together check the same set. +- **NUT provenance.** RWX verifies every fork, each with the forge and just + versions pinned at the fork's recorded commit. CircleCI verifies only forks + whose lock entry changed against the PR base. +- **SP1 guest.** RWX builds the ELFs in a clone at CircleCI's paths + (`/home/circleci/project`), so the bytes can match CircleCI's. + `ops/ci/runtime/sp1-guest-toolchain.sh` is a hand copy of CircleCI's + toolchain command; nothing compares the two. +- **Flaky-test report.** Not shadowed: it reads CircleCI's Insights API, which + needs a CircleCI token. +- **Selector upload.** Not shadowed: the CircleCI job publishes to the public + selector registry, and no gate requires it. + +## Validating a change + +RWX definitions must lint without errors. A warning that a pinned package has a +newer version is informational; bump packages in their own change. + +```bash +rwx lint --dir .rwx +``` + +The lint does not check the order of a `use` list. Name a task after the tasks +that it uses itself. RWX applies a task above the tasks it uses whatever the +list says, so another order has no effect, and each run prints a warning for +the task. + +Write a `test-results` path without a `${{ }}` expression: a literal path or a +glob. RWX does not expand an expression there, finds no file, and reports no +tests without a warning. It does expand one in an `artifacts` path. + +Unit tests (Python 3.12+, with `yq`, `jq`, `just` and `git` on `PATH`; +`mise exec yq jq just --` provides the pinned versions): + +```bash +python3 -m unittest discover -s ops/ci/tests -p 'test_*.py' +python3 -m unittest discover -s ops/ci/migration/tests -p 'test_*.py' +python3 ops/ci/migration/circle-alignment.py +``` + +In RWX, a workload's `helper-tests` task runs the tests of that workload's +scripts, and `ci-unit-tests` in `pr-gates.yml` runs the shared ones. A workload +whose tasks run recipes directly has no scripts and no `helper-tests`. A change +under `.rwx/` or `ops/ci/` selects every workload, so all of them run. +`test_ci_layout.py` fails when no task runs a test file; add a new test file to +its workload's `helper-tests`. Tests that need a live toolchain +(`RWX_LIVE_*_FIXTURE=1`) skip locally unless you set the variable. + +Several task families in the definitions are YAML merges: the first task of the +family has an anchor, and each later task lists `<<: *anchor` first and then +only the fields that differ. + +A hosted run costs money. An authenticated operator can start one from a pushed +commit with `rwx run .rwx/pr-gates.yml --wait`; a branch push also starts one. +Either one cancels a run of `pr-gates.yml` that is still in progress for the +same branch, even when the new run names only some tasks with `--target`. + +To try a change on RWX without a CircleCI pipeline, push the commits under a +scratch branch whose tip is an empty commit with `[skip ci]` in its message, then +start the run from the local branch. CircleCI and RWX's push trigger both skip +that tip, and the run clones the commit below it: + +```bash +tip=$(git commit-tree 'HEAD^{tree}' -p HEAD -m '[skip ci] scratch tip') +git push --force origin "${tip}:refs/heads/" +rwx run .rwx/pr-gates.yml --wait +``` + +Commit first. `rwx run` sends uncommitted and unpushed changes as a patch, and +the tasks that reset their checkout or check a build's inputs do not accept one. +A change to `.rwx` alone needs no push, because the CLI uploads that directory. +A workload definition, such as `.rwx/acceptance.yml`, has no concurrency pool, +so a run of it alone cancels nothing. diff --git a/ops/book/src/ci/pr-authorization.md b/ops/book/src/ci/pr-authorization.md index 09e3573bcc7..9695a507346 100644 --- a/ops/book/src/ci/pr-authorization.md +++ b/ops/book/src/ci/pr-authorization.md @@ -27,6 +27,13 @@ Note: Bailiff does not push to the source branch on the fork. It pushes to its o branch. It'll look like `external-fork/`. For this reason, **re-running CI jobs on the source branch will not work.** Instead, re-run the commit on the `external-fork` branch or push a new commit and re-authorize. +An authorized commit runs with CI secrets. Bailiff's push starts CircleCI, and it also starts the optional RWX shadow +(see `docs/ai/rwx-shadow-ci.md`), whose runs on an `external-fork` branch can read a test-only vault of archive RPC URLs. +Before you authorize, read the PR's changes to the files that decide what CI runs: + +- `.circleci/`, and the scripts and recipes that its jobs run. +- `.rwx/` and `ops/ci/`. RWX reads its definitions from the pushed commit. + Bailiff will post a PR status if it succeeded. That status looks like this: diff --git a/ops/ci/migration/ci-test-results.py b/ops/ci/migration/ci-test-results.py new file mode 100644 index 00000000000..9334939a2d7 --- /dev/null +++ b/ops/ci/migration/ci-test-results.py @@ -0,0 +1,202 @@ +"""Decode original Go JSON and JUnit reports without provider parity policy. + +Callers supply the package scope. Decoding retains retries, skips and package +failures; callers decide whether coverage and verdict histories are acceptable. +""" +import json +import math +from pathlib import Path +import re +import xml.etree.ElementTree as ET + +OUTCOMES = {"pass", "fail", "error", "skip"} + + +RESULTS = { + "success": "pass", "passed": "pass", "pass": "pass", "system-out": "pass", + "failure": "fail", "failed": "fail", "fail": "fail", + "error": "error", "skipped": "skip", "skip": "skip", +} + + +def required_text(value, label): + if not isinstance(value, str) or not value.strip() or value != value.strip(): + raise ValueError(f"{label} must be a nonempty string without surrounding whitespace") + return value + + +def number(value, label): + if value is None: + return None + if isinstance(value, bool) or not isinstance(value, (int, float)) or not math.isfinite(value) or value < 0: + raise ValueError(f"{label} must be a finite nonnegative number or null") + return value + + +def json_object(value, label): + if not isinstance(value, dict): + raise ValueError(f"{label} must be an object") + return value + + +def no_duplicate_keys(pairs): + result = {} + for key, value in pairs: + if key in result: + raise ValueError(f"duplicate JSON key: {key}") + result[key] = value + return result + + +def read_json(path): + return json.loads(Path(path).read_text(encoding="utf-8"), object_pairs_hook=no_duplicate_keys, + parse_constant=lambda value: (_ for _ in ()).throw(ValueError(f"invalid JSON number: {value}"))) + + +def skip_reason(message): + """Remove reporter decoration, preserving the reason rather than gas/timing.""" + if message is None or message == "": + return None + if not isinstance(message, str): + raise ValueError("skip message must be a string or null") + if message.lstrip().startswith("[SKIP]"): + return None + if "[SKIP:" in message: + message = message.split("[SKIP:", 1)[0] + message = re.sub(r"^\s*skipped:\s*", "", message) + if message.strip() == "skipped": + return None + lines = [] + for line in message.splitlines(): + line = line.strip() + if not line or re.match(r"(?:=== (?:RUN|PAUSE|CONT)|--- (?:SKIP|PASS|FAIL))\b", line): + continue + line = re.sub(r"^\S+\.go:\d+:\s*", "", line) + lines.append(line) + return " ".join(" ".join(lines).split()) or None + + +def in_scope(suite, metadata): + prefix = metadata.get("package_prefix") + return prefix is None or suite == prefix or suite.startswith(prefix + "/") + + +def make_case(suite, name, outcome, duration, reason, source, attempts=None): + if outcome not in OUTCOMES: + raise ValueError(f"unsupported outcome {outcome!r}") + return { + "feature": source["feature"], "suite": required_text(suite, "case suite"), + "name": required_text(name, "case name"), "outcome": outcome, + "skip_reason": reason if outcome == "skip" else None, + "duration_seconds": number(duration, "case duration_seconds"), + "attempts": attempts, "source_id": source["id"], + } + + +def junit_cases(path, source, metadata): + # No entity expansion/DTDs: reports do not need external or custom entities. + content = path.read_text(encoding="utf-8") + if re.search(r" 1: + raise ValueError("ambiguous JUnit testcase verdict") + outcome = RESULTS[verdicts[0].tag] if verdicts else "pass" + reason = None + if outcome == "skip": + child = verdicts[0] + reason = skip_reason(child.get("message") or child.text) + duration = case.get("time") + cases.append(make_case(suite, case.get("name"), outcome, float(duration) if duration is not None else None, + reason, source)) + return cases + + +def go_cases(path, source, metadata): + attempts, active, output = {}, {}, {} + package_terminals = [] + observed_packages = set() + for line_number, line in enumerate(path.read_text(encoding="utf-8").splitlines(), 1): + if not line.strip(): + continue + try: + event = json.loads(line, object_pairs_hook=no_duplicate_keys) + except ValueError as error: + raise ValueError(f"Go JSON line {line_number}: {error}") from error + json_object(event, "Go event") + action = event.get("Action") + if action not in ("start", "run", "pause", "cont", "output", "pass", "fail", "skip", "build-output", "build-fail"): + raise ValueError(f"unknown Go event action {action!r}") + if action in ("build-output", "build-fail"): + # Go 1.24+ reports build events by ImportPath (for example "p [p.test]"), + # not Package. The package's own fail event, with FailedBuild, carries + # the verdict, so the terminal-verdict checks below still apply. + required_text(event.get("ImportPath"), "Go build event ImportPath") + continue + suite = required_text(event.get("Package"), "Go event Package") + if not in_scope(suite, metadata): + continue + observed_packages.add(suite) + name = event.get("Test") + if not name: + if action in ("pass", "fail", "skip"): + package_terminals.append((suite, action)) + continue + required_text(name, "Go event Test") + key = (suite, name) + if action == "run": + if key in active: + raise ValueError(f"Go test started twice without a verdict: {key}") + active[key] = True + output[key] = [] + elif action == "output": + value = event.get("Output") + if not isinstance(value, str): + raise ValueError("Go output event must contain string Output") + if key in active: + output[key].append(value) + elif action in ("pass", "fail", "skip"): + if key not in active: + raise ValueError(f"Go verdict without matching run event: {key}") + attempt = {"outcome": action, "duration_seconds": number(event.get("Elapsed"), "Go Elapsed"), + "skip_reason": skip_reason("".join(output[key])) if action == "skip" else None} + attempts.setdefault(key, []).append(attempt) + del active[key] + if active: + raise ValueError("Go JSON has tests without terminal verdicts") + # A build/TestMain failure can coexist with passing case records. Preserve it + # outside the case set so case parity cannot turn a failed run green. + final_packages = dict(package_terminals) + if observed_packages != set(final_packages): + raise ValueError("Go JSON has packages without terminal verdicts") + source["observed_packages"] = sorted(observed_packages) + failures = [suite for suite, action in final_packages.items() if action == "fail"] + package_history = {} + for suite, action in package_terminals: + package_history.setdefault(suite, []).append(action) + source["package_attempts"] = [{"suite": suite, "outcomes": history} for suite, history in sorted(package_history.items())] + cases = [] + for (suite, name), history in attempts.items(): + final = history[-1] + cases.append(make_case(suite, name, final["outcome"], final["duration_seconds"], final["skip_reason"], source, history)) + return cases, sorted(set(failures)) diff --git a/ops/ci/migration/circle-alignment.py b/ops/ci/migration/circle-alignment.py new file mode 100644 index 00000000000..d6492e87bb6 --- /dev/null +++ b/ops/ci/migration/circle-alignment.py @@ -0,0 +1,96 @@ +#!/usr/bin/env python3 +"""Check that RWX's gate policy names the prerequisites of CircleCI's required gates. + +For each gate, CircleCI's `requires:` list must name the same jobs as RWX's +policy, ops/ci/runtime/pr-gates.json, each as `terminal`. A row of the policy +names one job. Where a gate requires another gate job of its workflow, as +`ci-gate` requires `ci-gate-proposals`, the policy names the jobs that the +inner gate requires. The order of the list has no meaning in CircleCI, so the +check ignores it. The check prints each difference and exits with 1. +""" +import collections +import json +from pathlib import Path +import subprocess +import sys + +ROOT = Path(__file__).resolve().parents[3] +MANIFEST = 'ops/ci/runtime/pr-gates.json' +# Gate: (CircleCI config fragment, workflow that runs the gate with its prerequisites). +GATES = { + 'ci-gate': ('.circleci/continue/main.yml', 'main'), + 'required-contracts-ci': ('.circleci/continue/main.yml', 'contracts-feature-tests'), + 'required-rust-ci': ('.circleci/continue/rust-ci.yml', 'rust-ci'), + 'required-rust-e2e': ('.circleci/continue/rust-e2e.yml', 'rust-e2e-ci'), +} +HINT = 'After a CircleCI gate change, update ops/ci/runtime/pr-gates.json.' + + +def rwx_prerequisites(): + """Return {gate: [name of each CircleCI job that RWX's policy mirrors]}.""" + manifest = json.loads((ROOT / MANIFEST).read_text()) + return {gate: [row['name'] for row in policy['dependencies']] + for gate, policy in manifest['gates'].items()} + + +def circle_prerequisites(gate): + """Return [(job, requirement)] from the `requires:` list of the gate in CircleCI. + + A workflow lists a job as `key` or as `{key: parameters}`, and `name:` in the + parameters renames the job. The gate is the one job whose resulting name is + `gate`. A `requires:` entry is `job`, which requires `success`, or + `{job: requirement}`. An entry that requires, as `terminal`, another job of + the gate's own kind stands for the `requires:` list of that job: the gate + waits for those jobs through it. Raises ValueError if the workflow does not + run exactly one job with the gate's name, or runs it or such an inner gate + with `always-succeed`. + """ + if gate not in GATES: + raise ValueError(f'{gate}: RWX has a policy for this gate, but GATES names no CircleCI workflow for it') + config, workflow = GATES[gate] + document = json.loads(subprocess.check_output(['yq', '-o=json', '.', str(ROOT / config)], text=True)) + jobs = collections.defaultdict(list) + for job in document.get('workflows', {}).get(workflow, {}).get('jobs', []): + key, parameters = next(iter(job.items())) if isinstance(job, dict) else (job, {}) + jobs[parameters.get('name', key)].append((key, parameters)) + if len(jobs[gate]) != 1: + raise ValueError(f'{gate}: workflow {workflow} of {config} has {len(jobs[gate])} jobs with this name; expected 1') + kind = jobs[gate][0][0] + + def prerequisites(name): + parameters = jobs[name][0][1] + if parameters.get('always-succeed', False): + raise ValueError(f'{name}: workflow {workflow} of {config} runs the gate with always-succeed, so it has no prerequisites') + found = [] + for entry in parameters.get('requires', []): + job, requirement = next(iter(entry.items())) if isinstance(entry, dict) else (entry, 'success') + inner = requirement == 'terminal' and [key for key, _ in jobs[job]] == [kind] + found += prerequisites(job) if inner else [(job, requirement)] + return found + return prerequisites(gate) + + +def problems(): + """Return one line for each difference; an empty list means that every gate is aligned.""" + lines = [] + for gate, wanted in rwx_prerequisites().items(): + try: + circle = circle_prerequisites(gate) + except ValueError as error: + lines.append(str(error)) + continue + names, wanted = collections.Counter(job for job, _ in circle), collections.Counter(wanted) + found = [f' only in CircleCI: {job}' for job in sorted((names - wanted).elements())] + found += [f' only in RWX: {job}' for job in sorted((wanted - names).elements())] + found += [f' not `terminal` in CircleCI: {job} ({requirement})' for job, requirement in circle if requirement != 'terminal'] + if found: + config, workflow = GATES[gate] + lines += [f"{gate}: `requires:` in workflow {workflow} of {config} differs from RWX's policy:", *found] + return lines + + +if __name__ == '__main__': + found = problems() + if found: + print('\n'.join([*found, HINT]), file=sys.stderr) + sys.exit(1 if found else 0) diff --git a/ops/ci/migration/compare-ci.py b/ops/ci/migration/compare-ci.py new file mode 100644 index 00000000000..87f68d95ee9 --- /dev/null +++ b/ops/ci/migration/compare-ci.py @@ -0,0 +1,142 @@ +#!/usr/bin/env python3 +"""Compare the test cases that two CI providers report. + +Each side is one or more report files, one per shard. A report is JUnit XML, +`go test -json` events, or one complete response of CircleCI's job +test-metadata API. The tool takes the format from the `.xml` extension or from +the content. It joins the reports of a side into +{(suite, name): (outcome, skip reason)}. It prints the cases that are missing +from the candidate, extra in the candidate, or different. A Go package that +fails outside its test cases is the case `(package)` of that package. + +The caller selects reports of the same commit and workload. The tool does not +check the commit, and it does not check that a provider found every test. + +Exit status: 0 equivalent, 1 different, 2 rejected input, 3 incomparable (the +sides have no case in common). +""" +import argparse +import importlib.util +import json +from pathlib import Path +import sys +import xml.etree.ElementTree as ET + +EQUIVALENT, DIFFERENT, REJECTED, INCOMPARABLE = 0, 1, 2, 3 +PACKAGE = '(package)' + + +def helper(name): + spec = importlib.util.spec_from_file_location(name.replace('-', '_'), Path(__file__).with_name(name + '.py')) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +T = helper('ci-test-results') + + +def report_format(path): + """Return `junit`, `go-json` or `circleci-tests` for one report file.""" + text = path.read_text(encoding='utf-8').lstrip() + if path.suffix == '.xml' or text.startswith('<'): + return 'junit' + try: + first = json.loads(text.split('\n', 1)[0]) + except ValueError: + return 'circleci-tests' # One JSON document that spans several lines. + return 'go-json' if isinstance(first, dict) and 'Action' in first else 'circleci-tests' + + +def circleci_cases(path, source): + """Decode one complete response of CircleCI's job test-metadata API. + + The API returns a large job in pages. Before the call, the caller joins the + `items` of every page into one response, or into one bare list. + """ + data = T.read_json(path) + if isinstance(data, dict): + if data.get('next_page_token'): + raise ValueError(f'CircleCI tests response is one page of several; join every page first: {path}') + data = data.get('items') + if not isinstance(data, list): + raise ValueError(f'CircleCI tests response has no `items` list: {path}') + cases = [] + for row in data: + T.json_object(row, 'CircleCI test') + outcome = T.RESULTS.get(row.get('result')) + if outcome is None: + raise ValueError(f'unknown CircleCI test result {row.get("result")!r}: {path}') + cases.append(T.make_case(row.get('classname'), row.get('name'), outcome, row.get('run_time'), + T.skip_reason(row.get('message')) if outcome == 'skip' else None, source)) + return cases + + +def load(paths): + """Return {(suite, name): (outcome, skip reason)} for the reports of one side. + + Raises ValueError if a report has no case, or if two reports of the side + contain the same case. + """ + result = {} + for path in paths: + source, failed = {'feature': None, 'id': str(path)}, [] + kind = report_format(path) + if kind == 'junit': + cases = T.junit_cases(path, source, {}) + elif kind == 'go-json': + cases, failed = T.go_cases(path, source, {}) + else: + cases = circleci_cases(path, source) + if not cases and not failed: + raise ValueError(f'report contains no test case: {path}') + # Test-level shards run the same package, so a package can fail in each. + result.update(((suite, PACKAGE), ('fail', None)) for suite in failed) + for case in cases: + key = case['suite'], case['name'] + if key in result: + raise ValueError(f'one side reports a case twice: {key[0]} {key[1]} ({path})') + result[key] = case['outcome'], case['skip_reason'] + return result + + +def compare(baseline, candidate): + """Return the sorted case keys that are missing, extra and different in the candidate.""" + shared = baseline.keys() & candidate.keys() + return {'missing': sorted(baseline.keys() - shared), 'extra': sorted(candidate.keys() - shared), + 'different': sorted(key for key in shared if baseline[key] != candidate[key])} + + +def verdict(value): + outcome, reason = value + return outcome if reason is None else f'{outcome} ({reason})' + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + for side in ('baseline', 'candidate'): + parser.add_argument('--' + side, type=Path, nargs='+', required=True, metavar='REPORT', + help=f'report files of the {side} provider, one per shard') + args = parser.parse_args(argv) + try: + baseline, candidate = load(args.baseline), load(args.candidate) + except (OSError, ValueError, ET.ParseError) as error: + print(f'CI comparison rejected input: {error}', file=sys.stderr) + return REJECTED + sizes = f'{len(baseline)} baseline cases, {len(candidate)} candidate cases' + if not baseline.keys() & candidate.keys(): + print(f'CI comparison: incomparable ({sizes}, none in common)') + return INCOMPARABLE + report = compare(baseline, candidate) + print(f'CI comparison: {"different" if any(report.values()) else "equivalent"} ({sizes})') + for suite, name in report['missing']: + print(f'missing from candidate: {suite} {name}: {verdict(baseline[suite, name])}') + for suite, name in report['extra']: + print(f'extra in candidate: {suite} {name}: {verdict(candidate[suite, name])}') + for suite, name in report['different']: + print(f'different: {suite} {name}: baseline {verdict(baseline[suite, name])}, candidate {verdict(candidate[suite, name])}') + return DIFFERENT if any(report.values()) else EQUIVALENT + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/ops/ci/migration/tests/fixtures/ci-comparison/circleci-foundry-skips.json b/ops/ci/migration/tests/fixtures/ci-comparison/circleci-foundry-skips.json new file mode 100644 index 00000000000..717a24f224a --- /dev/null +++ b/ops/ci/migration/tests/fixtures/ci-comparison/circleci-foundry-skips.json @@ -0,0 +1,37 @@ +{ + "sha": "cb48622213e9aa8408de9904baafac784441ecc6", + "provenance": { + "url": "https://circleci.com/gh/ethereum-optimism/optimism/5622529", + "source": "Selected original CircleCI API records corresponding to native Foundry fixture identities. Different SHA; parsing fixture only, not a provider equivalence claim." + }, + "items": [ + { + "classname": "test/L1/L1StandardBridge.t.sol:L1StandardBridge_DepositETHTo_Test", + "name": "testFuzz_depositETHTo_withCustomGasToken_reverts(address,uint256,uint32)", + "result": "skipped", + "run_time": 0.07, + "message": "Skipping test because SYS_FEATURE__CUSTOM_GAS_TOKEN is disabled[SKIP: Skipping test because SYS_FEATURE__CUSTOM_GAS_TOKEN is disabled] testFuzz_depositETHTo_withCustomGasToken_reverts(address,uint256,uint32) (runs: 0, \u03bc: 0, ~: 0)" + }, + { + "classname": "test/L1/OPContractsManagerStandardValidator.t.sol:OPContractsManagerStandardValidator_AnchorStateRegistry_Test", + "name": "setUp()", + "result": "skipped", + "run_time": 0, + "message": "skipped: Skipping: standard configs incompatible with SUPER_ROOT_GAMES_MIGRATION[SKIP: skipped: Skipping: standard configs incompatible with SUPER_ROOT_GAMES_MIGRATION] setUp() (gas: 0)" + }, + { + "classname": "test/L1/ResourceMetering.t.sol:ArtifactResourceMetering_Metered_Test", + "name": "test_meter_generateArtifact_succeeds()", + "result": "skipped", + "run_time": 0, + "message": "[SKIP] test_meter_generateArtifact_succeeds() (gas: 0)" + }, + { + "classname": "test/L1/opcm/OPContractsManagerV2.t.sol:OPContractsManagerV2_UpgradeSuperchain_Test", + "name": "setUp()", + "result": "skipped", + "run_time": 0, + "message": "skipped[SKIP: skipped] setUp() (gas: 0)" + } + ] +} diff --git a/ops/ci/migration/tests/fixtures/ci-comparison/circleci-go-cases.json b/ops/ci/migration/tests/fixtures/ci-comparison/circleci-go-cases.json new file mode 100644 index 00000000000..1396fef8006 --- /dev/null +++ b/ops/ci/migration/tests/fixtures/ci-comparison/circleci-go-cases.json @@ -0,0 +1,30 @@ +{ + "sha": "cb48622213e9aa8408de9904baafac784441ecc6", + "provenance": { + "url": "https://circleci.com/gh/ethereum-optimism/optimism/5622514", + "source": "CircleCI test metadata API, selected real records; intentionally incomplete fixture, not a full runtime report" + }, + "items": [ + { + "classname": "github.com/ethereum-optimism/optimism/op-node/rollup", + "name": "TestChainSpec_CanyonForkActivation/Genesis", + "result": "success", + "run_time": 0, + "message": "" + }, + { + "classname": "github.com/ethereum-optimism/optimism/op-node/rollup/derive", + "name": "TestBatchStages/BatchQueue/SingularBatch/OverlappingSpanBatch", + "result": "skipped", + "run_time": 0, + "message": "=== RUN TestBatchStages/BatchQueue/SingularBatch/OverlappingSpanBatch\n batch_queue_test.go:569: only applicable to span batches\n--- SKIP: TestBatchStages/BatchQueue/SingularBatch/OverlappingSpanBatch (0.00s)\n" + }, + { + "classname": "github.com/ethereum-optimism/optimism/op-node/rollup/derive", + "name": "TestBatchStages/BatchStage/SingularBatch/OverlappingSpanBatch", + "result": "skipped", + "run_time": 0, + "message": "=== RUN TestBatchStages/BatchStage/SingularBatch/OverlappingSpanBatch\n batch_queue_test.go:569: only applicable to span batches\n--- SKIP: TestBatchStages/BatchStage/SingularBatch/OverlappingSpanBatch (0.00s)\n" + } + ] +} diff --git a/ops/ci/migration/tests/fixtures/ci-comparison/rwx-foundry-skips.xml b/ops/ci/migration/tests/fixtures/ci-comparison/rwx-foundry-skips.xml new file mode 100644 index 00000000000..55d550c16ed --- /dev/null +++ b/ops/ci/migration/tests/fixtures/ci-comparison/rwx-foundry-skips.xml @@ -0,0 +1,18 @@ + + + + [SKIP: Skipping test because SYS_FEATURE__CUSTOM_GAS_TOKEN is disabled] testFuzz_depositETHTo_withCustomGasToken_reverts(address,uint256,uint32) (runs: 0, μ: 0, ~: 0) + + + + [SKIP: skipped: Skipping: standard configs incompatible with SUPER_ROOT_GAMES_MIGRATION] setUp() (gas: 0) + + + + [SKIP] test_meter_generateArtifact_succeeds() (gas: 0) + + + + [SKIP: skipped] setUp() (gas: 0) + + \ No newline at end of file diff --git a/ops/ci/migration/tests/test_circle_alignment.py b/ops/ci/migration/tests/test_circle_alignment.py new file mode 100644 index 00000000000..89b56312e08 --- /dev/null +++ b/ops/ci/migration/tests/test_circle_alignment.py @@ -0,0 +1,80 @@ +"""Change copies of the real CircleCI gate fragments and of RWX's gate policy.""" +import importlib.util +import json +from pathlib import Path +import re +import shutil +import tempfile +import unittest +from unittest.mock import patch + +SPEC = importlib.util.spec_from_file_location('alignment', Path(__file__).resolve().parents[1] / 'circle-alignment.py') +A = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(A) + + +class AlignmentTests(unittest.TestCase): + def setUp(self): + temp = tempfile.TemporaryDirectory() + self.addCleanup(temp.cleanup) + self.root = Path(temp.name) + for name in {A.MANIFEST, *(config for config, _ in A.GATES.values())}: + (self.root / name).parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(A.ROOT / name, self.root / name) + handle = patch.object(A, 'ROOT', self.root) + handle.start() + self.addCleanup(handle.stop) + + def test_added_removed_renamed_and_nonterminal_prerequisites_are_named(self): + self.assertEqual(A.problems(), []) + for gate, (config, workflow) in A.GATES.items(): + job = A.rwx_prerequisites()[gate][0] + path = self.root / config + source = path.read_text() + line = re.search(rf'^ *- {re.escape(job)}: terminal\n', source, re.MULTILINE).group() + indent = line[:len(line) - len(line.lstrip())] + header = f"{gate}: `requires:` in workflow {workflow} of {config} differs from RWX's policy:" + for change, expected in ((line + indent + '- future-job: terminal\n', [' only in CircleCI: future-job']), + ('', [f' only in RWX: {job}']), + (line.replace(job, job + '-renamed'), [f' only in CircleCI: {job}-renamed', f' only in RWX: {job}']), + (line.replace(': terminal', ''), [f' not `terminal` in CircleCI: {job} (success)'])): + path.write_text(source.replace(line, change, 1)) + with self.subTest(gate=gate, change=change): + self.assertEqual(A.problems(), [header, *expected]) + path.write_text(source) + + def test_order_is_ignored_and_a_gate_without_a_circleci_workflow_is_named(self): + path = self.root / A.MANIFEST + manifest = json.loads(path.read_text()) + for policy in manifest['gates'].values(): + policy['dependencies'].reverse() + path.write_text(json.dumps(manifest)) + self.assertEqual(A.problems(), []) + manifest['gates']['future-gate'] = {'dependencies': [{'name': 'future-job'}]} + path.write_text(json.dumps(manifest)) + self.assertEqual(A.problems(), ['future-gate: RWX has a policy for this gate, but GATES names no CircleCI workflow for it']) + + def test_gate_is_the_one_job_with_the_gate_name_after_a_rename(self): + gate = 'required-rust-ci' + config, workflow = A.GATES[gate] + + def prerequisites(jobs): + (self.root / config).write_text(f'workflows:\n {workflow}:\n jobs:\n{jobs}') + return A.circle_prerequisites(gate) + other = f' - {gate}:\n name: fan-in\n requires: [{{build: terminal}}]\n' + renamed = f' - utils-gate:\n name: {gate}\n requires: [lint, {{test: terminal}}]\n' + self.assertEqual(prerequisites(' - build\n' + other + renamed), [('lint', 'success'), ('test', 'terminal')]) + for message, jobs in (('has 0 jobs with this name', other), + ('has 2 jobs with this name', renamed + f' - {gate}:\n requires: []\n'), + ('always-succeed', f' - {gate}:\n always-succeed: true\n')): + with self.subTest(message=message), self.assertRaisesRegex(ValueError, message): + prerequisites(jobs) + self.assertIn(f'{gate}: workflow {workflow} of {config} runs the gate with always-succeed, so it has no prerequisites', A.problems()) + # A gate that requires another job of its own kind waits for that job's prerequisites. + inner = f' - {gate}:\n name: inner\n requires: [{{build: terminal}}, lint]\n' + outer = f' - {gate}:\n requires: [{{inner: terminal}}, {{test: terminal}}]\n' + self.assertEqual(prerequisites(inner + outer), [('build', 'terminal'), ('lint', 'success'), ('test', 'terminal')]) + # Without `terminal`, the gate would not run after a failure of the inner gate: name it. + self.assertEqual(prerequisites(inner + outer.replace('{inner: terminal}', 'inner')), [('inner', 'success'), ('test', 'terminal')]) + with self.assertRaisesRegex(ValueError, 'inner: .* always-succeed'): + prerequisites(inner.replace('requires:', 'always-succeed: true\n requires:') + outer) diff --git a/ops/ci/migration/tests/test_compare_ci.py b/ops/ci/migration/tests/test_compare_ci.py new file mode 100644 index 00000000000..cb27e09674b --- /dev/null +++ b/ops/ci/migration/tests/test_compare_ci.py @@ -0,0 +1,208 @@ +#!/usr/bin/env python3 +"""Compare real reporter fixtures and decode synthetic Go and JUnit edge cases. + +The fixtures hold selected real records, not complete runs. The edge-case tests +are the only direct tests of ops/ci/migration/ci-test-results.py. +""" +import contextlib +import importlib.util +import io +import json +from pathlib import Path +import tempfile +import unittest + +SPEC = importlib.util.spec_from_file_location('compare_ci', Path(__file__).resolve().parents[1] / 'compare-ci.py') +CI = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(CI) +T = CI.T +FIXTURES = Path(__file__).parent / 'fixtures/ci-comparison' +GO_API = FIXTURES / 'circleci-go-cases.json' +FOUNDRY_API = FIXTURES / 'circleci-foundry-skips.json' +FOUNDRY_JUNIT = FIXTURES / 'rwx-foundry-skips.xml' +PACKAGE = 'github.com/ethereum-optimism/optimism/op-node/rollup' +RUN = {'Action': 'run', 'Package': PACKAGE, 'Test': 'TestSynthetic'} +PASS = {'Action': 'pass', 'Package': PACKAGE, 'Test': 'TestSynthetic'} +PACKAGE_PASS = {'Action': 'pass', 'Package': PACKAGE} +PACKAGE_FAIL = {'Action': 'fail', 'Package': PACKAGE} + + +class ReportTest(unittest.TestCase): + def setUp(self): + temp = tempfile.TemporaryDirectory() + self.addCleanup(temp.cleanup) + self.root = Path(temp.name) + + def write(self, name, text): + path = self.root / name + path.write_text(text) + return path + + def api(self, name, items, **fields): + return self.write(name, json.dumps({'items': items, **fields})) + + def events(self, name, events): + return self.write(name, ''.join(json.dumps(event) + '\n' for event in events)) + + +class ComparisonTest(ReportTest): + def run_cli(self, baseline, candidate): + output = io.StringIO() + with contextlib.redirect_stdout(output), contextlib.redirect_stderr(output): + status = CI.main(['--baseline', *map(str, baseline), '--candidate', *map(str, candidate)]) + return status, output.getvalue() + + def test_real_circleci_go_records_keep_subtest_identity_and_skip_reason(self): + cases = CI.load([GO_API]) + self.assertEqual(len(cases), 3) + skipped = {key: reason for key, (outcome, reason) in cases.items() if outcome == 'skip'} + self.assertEqual(len(skipped), 2) + self.assertEqual(set(skipped.values()), {'only applicable to span batches'}) + self.assertTrue(all('/' in name for _, name in skipped)) + + def test_real_foundry_api_and_junit_records_are_equivalent(self): + # The two fixtures come from different commits. The test compares the + # reporter conversion only: suite identity and skip reasons. + api, junit = CI.load([FOUNDRY_API]), CI.load([FOUNDRY_JUNIT]) + self.assertEqual(api, junit) + self.assertEqual(len(junit), 4) + reasons = [reason for _, reason in junit.values()] + self.assertEqual(reasons.count(None), 2) + self.assertIn('Skipping: standard configs incompatible with SUPER_ROOT_GAMES_MIGRATION', reasons) + self.assertTrue(all(suite.startswith('test/') and ':' in suite for suite, _ in junit)) + status, output = self.run_cli([FOUNDRY_API], [FOUNDRY_JUNIT]) + self.assertEqual(status, CI.EQUIVALENT, output) + self.assertIn('equivalent (4 baseline cases, 4 candidate cases)', output) + + def test_equal_counts_do_not_hide_missing_extra_and_different_cases(self): + first, second, third = T.read_json(GO_API)['items'] + candidate = self.api('candidate.json', [{**first, 'name': 'TestSubstitute'}, {**second, 'result': 'failure'}, third]) + self.assertEqual(CI.compare(CI.load([GO_API]), CI.load([candidate])), + {'missing': [(first['classname'], first['name'])], + 'extra': [(first['classname'], 'TestSubstitute')], + 'different': [(second['classname'], second['name'])]}) + status, output = self.run_cli([GO_API], [candidate]) + self.assertEqual(status, CI.DIFFERENT, output) + self.assertIn(f"missing from candidate: {first['classname']} {first['name']}: pass", output) + self.assertIn(f"extra in candidate: {first['classname']} TestSubstitute: pass", output) + self.assertIn('baseline skip (only applicable to span batches), candidate fail', output) + + def test_cases_and_packages_that_fail_on_both_sides_are_equivalent(self): + first = T.read_json(GO_API)['items'][0] + failing = self.api('failing.json', [{**first, 'result': 'failure'}]) + self.assertEqual(self.run_cli([failing], [failing])[0], CI.EQUIVALENT) + package_fails = self.events('fails.json', [RUN, PASS, PACKAGE_FAIL]) + package_passes = self.events('passes.json', [RUN, PASS, PACKAGE_PASS]) + self.assertEqual(self.run_cli([package_fails], [package_fails])[0], CI.EQUIVALENT) + status, output = self.run_cli([package_fails], [package_passes]) + self.assertEqual(status, CI.DIFFERENT, output) + self.assertIn(f'missing from candidate: {PACKAGE} (package): fail', output) + + def test_shards_are_joined_and_a_case_in_two_reports_is_rejected(self): + items = T.read_json(GO_API)['items'] + shards = [self.api('shard-0.json', items[:1]), self.write('shard-1.json', json.dumps(items[1:]))] + self.assertEqual(CI.load(shards), CI.load([GO_API])) + self.assertEqual(self.run_cli(shards, [GO_API])[0], CI.EQUIVALENT) + with self.assertRaisesRegex(ValueError, 'reports a case twice'): + CI.load([GO_API, shards[0]]) + self.assertEqual(self.run_cli([GO_API, shards[0]], [GO_API])[0], CI.REJECTED) + # Test-level shards run the same package. Its failure in each shard is one failure. + failed = [self.events(f'go-{index}.json', [{**RUN, 'Test': name}, {**PASS, 'Test': name}, PACKAGE_FAIL]) + for index, name in enumerate(('TestOne', 'TestTwo'))] + self.assertEqual(CI.load(failed), {(PACKAGE, 'TestOne'): ('pass', None), (PACKAGE, 'TestTwo'): ('pass', None), + (PACKAGE, CI.PACKAGE): ('fail', None)}) + + def test_incomplete_empty_or_malformed_responses_are_rejected(self): + row = T.read_json(GO_API)['items'][0] + for message, path in (('one page of several', self.api('page.json', [row], next_page_token='still-more')), + ('no test case', self.api('empty.json', [])), + ('no `items` list', self.write('error.json', '{"message": "Not Found"}')), + ('unknown CircleCI test result', self.api('result.json', [{**row, 'result': 'mystery'}])), + ('finite nonnegative', self.api('time.json', [{**row, 'run_time': -1}])), + ('duplicate JSON key', self.write('keys.json', '{"items": [], "items": []}'))): + with self.subTest(path=path.name): + with self.assertRaisesRegex(ValueError, message): + CI.load([path]) + self.assertEqual(self.run_cli([path], [GO_API])[0], CI.REJECTED) + self.assertEqual(self.run_cli([self.root / 'absent.json'], [GO_API])[0], CI.REJECTED) + + def test_sides_without_a_common_case_are_incomparable(self): + status, output = self.run_cli([GO_API], [FOUNDRY_JUNIT]) + self.assertEqual(status, CI.INCOMPARABLE, output) + self.assertIn('incomparable (3 baseline cases, 4 candidate cases, none in common)', output) + + def test_format_follows_the_xml_extension_or_the_content(self): + self.assertEqual(CI.report_format(FOUNDRY_JUNIT), 'junit') + self.assertEqual(CI.report_format(self.write('junit.txt', '\n')), 'junit') + self.assertEqual(CI.report_format(GO_API), 'circleci-tests') + self.assertEqual(CI.report_format(self.api('compact.json', [])), 'circleci-tests') + events = self.events('events.json', [RUN, PASS, PACKAGE_PASS]) + self.assertEqual(CI.report_format(events), 'go-json') + junit = self.write('junit.xml', f'') + self.assertEqual(self.run_cli([events], [junit])[0], CI.EQUIVALENT) + + +class ParserTest(ReportTest): + def go_cases(self, events): + source = {'feature': None, 'id': 'events'} + cases, failures = T.go_cases(self.events('events.json', events), source, {}) + return cases, failures, source + + def test_go_retry_keeps_every_attempt_and_the_final_verdict(self): + # Synthetic events in the order that gotestsum writes them when it reruns a failed test. + events = [{'Action': action, 'Package': PACKAGE, 'Test': 'TestSynthetic', 'Elapsed': 1} + for action in ('run', 'fail', 'run', 'pass')] + events.insert(2, PACKAGE_FAIL) + events.append(PACKAGE_PASS) + cases, failures, source = self.go_cases(events) + self.assertEqual([case['outcome'] for case in cases], ['pass']) + self.assertEqual([attempt['outcome'] for attempt in cases[0]['attempts']], ['fail', 'pass']) + self.assertEqual(failures, []) + self.assertEqual(source['package_attempts'], [{'suite': PACKAGE, 'outcomes': ['fail', 'pass']}]) + + def test_go_package_failure_outside_passing_cases_is_kept(self): + cases, failures, _ = self.go_cases([RUN, PASS, PACKAGE_FAIL]) + self.assertEqual([case['outcome'] for case in cases], ['pass']) + self.assertEqual(failures, [PACKAGE]) + + def test_go_build_events_identified_by_import_path_keep_the_package_verdict(self): + # Go 1.24+ emits build events with ImportPath and no Package field. + passing = [{'ImportPath': PACKAGE + ' [' + PACKAGE + '.test]', 'Action': 'build-output', 'Output': '# compiling\n'}, + RUN, PASS, PACKAGE_PASS] + cases, failures, _ = self.go_cases(passing) + self.assertEqual([case['outcome'] for case in cases], ['pass']) + self.assertEqual(failures, []) + broken = PACKAGE + '/broken' + target = broken + ' [' + broken + '.test]' + failed = passing + [{'ImportPath': target, 'Action': 'build-output', 'Output': 'x.go:1: undefined: y\n'}, + {'ImportPath': target, 'Action': 'build-fail'}, + {'Action': 'start', 'Package': broken}, + {'Action': 'fail', 'Package': broken, 'FailedBuild': target}] + _, failures, source = self.go_cases(failed) + self.assertEqual(failures, [broken]) + self.assertEqual(source['observed_packages'], [PACKAGE, broken]) + with self.assertRaisesRegex(ValueError, 'ImportPath'): + self.go_cases([{'Action': 'build-fail'}] + passing) + + def test_go_package_without_tests_is_observed(self): + _, failures, source = self.go_cases([RUN, PASS, PACKAGE_PASS, {'Action': 'skip', 'Package': PACKAGE + '/no-tests'}]) + self.assertEqual(source['observed_packages'], [PACKAGE, PACKAGE + '/no-tests']) + self.assertEqual(failures, []) + + def test_truncated_or_ambiguous_go_events_are_rejected(self): + for message, events in (('tests without terminal verdicts', [RUN]), ('without matching run', [PASS]), + ('started twice', [RUN, RUN]), ('packages without terminal verdicts', [RUN, PASS])): + with self.subTest(events=events), self.assertRaisesRegex(ValueError, message): + self.go_cases(events) + + def test_malformed_junit_is_rejected(self): + for message, xml in (('root must be', ''), + ('classname or parent suite name', ''), + ('DTDs/entities', ''), + ('ambiguous', f'')): + with self.subTest(xml=xml), self.assertRaisesRegex(ValueError, message): + T.junit_cases(self.write('results.xml', xml), {'feature': None, 'id': 'junit'}, {}) + + +if __name__ == '__main__': + unittest.main() diff --git a/ops/ci/runtime/acceptance-circleci/circleci b/ops/ci/runtime/acceptance-circleci/circleci new file mode 100755 index 00000000000..d0543fde14e --- /dev/null +++ b/ops/ci/runtime/acceptance-circleci/circleci @@ -0,0 +1,22 @@ +#!/usr/bin/env bash +# CircleCI CLI stand-in. acceptance-tests.sh puts it first on PATH for RWX +# verdicts. It serves only the `circleci tests split ` +# call in ops/scripts/shard-tests.sh; holds every test name that the +# recipe listed, one per line. Every shard lists the same names in the same +# order. The shim prints each line whose zero-based position, modulo +# CIRCLE_NODE_TOTAL, is CIRCLE_NODE_INDEX, so each name goes to exactly one +# shard and placement options have no effect. It fails for every other invocation. +set -euo pipefail +unsupported() { + echo "circleci: the RWX acceptance shim does not support: circleci $*" >&2 + exit 1 +} +[[ $# -ge 3 && $1 == tests && $2 == split && ${!#} != -* ]] || unsupported "$@" +for option in "${@:3:$#-3}"; do + case "$option" in + --split-by=* | --timings-type=* | --time-default=*) ;; + *) unsupported "$@" ;; + esac +done +exec awk -v shard="$CIRCLE_NODE_INDEX" -v shards="$CIRCLE_NODE_TOTAL" \ + '(NR - 1) % shards == shard' "${!#}" diff --git a/ops/ci/runtime/acceptance-tests.sh b/ops/ci/runtime/acceptance-tests.sh new file mode 100755 index 00000000000..dcfbdbd2e83 --- /dev/null +++ b/ops/ci/runtime/acceptance-tests.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash +# Run shard CI_SHARD_INDEX of CircleCI's op-acceptance-tests job with the +# unchanged recipe. The caller restores the build outputs that the job attaches. +set -euo pipefail +cd "$(dirname "${BASH_SOURCE[0]}")/../../.." +# CircleCI's memory-all jobs set acceptance_test_jobs: "16" and LOG_LEVEL=info. +export ACCEPTANCE_TEST_JOBS=16 LOG_LEVEL=info +export GOMODCACHE="$PWD/.ci/go-cache/full/modules" +export GOCACHE="$PWD/.ci/go-cache/acceptance/run" +export GOROOT +GOROOT="$(go env GOROOT)" +export PATH="$GOROOT/bin:$PATH" +mkdir -p "$GOCACHE" +export RUST_BINARY_PATH_KONA_HOST="$PWD/rust/target/release/kona-host" +export RUST_BINARY_PATH_KONA_CLIENT="$PWD/rust/target/release/kona-client" +export RUST_BINARY_PATH_KONA_NODE="$PWD/rust/target/release/kona-node" +export RUST_BINARY_PATH_OP_ZK_PROPOSER="$PWD/rust/target/release/op-zk-proposer" +export RUST_BINARY_PATH_OP_RETH="$PWD/rust/target/release/op-reth" +export RUST_BINARY_PATH_KONA_SP1_SUPER_RANGE_EXECUTOR="$PWD/.circleci-cache/rust-binaries/kona-sp1-super-range-executor" +for binary in "$RUST_BINARY_PATH_KONA_HOST" "$RUST_BINARY_PATH_KONA_CLIENT" \ + "$RUST_BINARY_PATH_KONA_NODE" "$RUST_BINARY_PATH_OP_ZK_PROPOSER" \ + "$RUST_BINARY_PATH_OP_RETH" \ + "$RUST_BINARY_PATH_KONA_SP1_SUPER_RANGE_EXECUTOR"; do + test -x "$binary" || { + echo "Missing executable runtime dependency: $binary" >&2 + exit 1 + } +done +# Fail before the suite starts when the pinned geth is not installed. +test -x "$(mise which geth)" +# Like CircleCI's "Disable fsync for all subprocesses" step. +if [[ -f /usr/lib/x86_64-linux-gnu/libeatmydata.so ]]; then + export LD_PRELOAD=libeatmydata.so +fi +# The recipe shards through ops/scripts/shard-tests.sh. Its `circleci tests +# split` call reaches the shim, which prints this shard's share of the tests. +# CircleCI's memory-all jobs have parallelism: 4, and .rwx/acceptance.yml has +# one verdict task for each of the four shards. +export CIRCLE_NODE_INDEX="$CI_SHARD_INDEX" CIRCLE_NODE_TOTAL=4 +export PATH="$PWD/ops/ci/runtime/acceptance-circleci:$PATH" +cd op-acceptance-tests +just acceptance-test diff --git a/ops/ci/runtime/apt-install.sh b/ops/ci/runtime/apt-install.sh new file mode 100755 index 00000000000..ae1c76af988 --- /dev/null +++ b/ops/ci/runtime/apt-install.sh @@ -0,0 +1,67 @@ +#!/usr/bin/env bash +# apt-get update + install. Arguments are passed to `apt-get install`. +set -euo pipefail + +# Timeouts, so a mirror that stops responding errors out instead of being waited on. +APT_OPTS=(-o Acquire::Retries=3 -o Acquire::http::Timeout=20 -o Acquire::https::Timeout=20) + +export NEEDRESTART_MODE=a +export DEBIAN_FRONTEND=noninteractive + +# Copy of each list as the machine image shipped it. The suffix is not one apt +# reads (it only picks up *.list and *.sources), so the copies sit inertly next +# to the rewritten files until the fallback below needs them. Written once: some +# jobs call this script more than once, and re-copying would capture the +# already-rewritten file and make the fallback a no-op. +BACKUP_SUFFIX=".shipped-mirrors" +# Set once archive.ubuntu.com has been found unusable, so later calls in the +# same job don't rewrite back onto it. +FALLBACK_MARKER="/tmp/.apt-install-mirror-fallback" + +SOURCE_FILES=() +for f in /etc/apt/sources.list /etc/apt/sources.list.d/*.sources /etc/apt/sources.list.d/*.list; do + if [ -f "$f" ]; then + SOURCE_FILES+=("$f") + fi +done + +# The machine images point apt at the single-region EC2 mirror +# (us-east-1.ec2.archive.ubuntu.com), which is one set of AWS hosts with no +# routing of its own: when it degrades, apt has nowhere else to go, and it is only +# nearby when the runner happens to be in that region. archive.ubuntu.com is +# Cloudflare anycast, so redundancy, geo-routing and caching are the CDN's job. +# Third-party lists keep their own URIs. +if [ ! -f "$FALLBACK_MARKER" ]; then + for f in "${SOURCE_FILES[@]}"; do + if [ ! -f "$f$BACKUP_SUFFIX" ]; then + sudo cp -p -- "$f" "$f$BACKUP_SUFFIX" + fi + sudo sed -i -E "s#https?://[A-Za-z0-9.-]+\.ec2\.archive\.ubuntu\.com/ubuntu#http://archive.ubuntu.com/ubuntu#g" "$f" + done +fi + +# The CDN is a single point of failure of its own: Canonical has had +# archive.ubuntu.com outages that fail every job installing a package through it. +# The EC2 mirror the image shipped is usually still serving when that happens, so +# put it back and try once more instead of failing the job outright. +restore_shipped_mirrors() { + for f in "${SOURCE_FILES[@]}"; do + if [ -f "$f$BACKUP_SUFFIX" ]; then + sudo cp -p -- "$f$BACKUP_SUFFIX" "$f" + fi + done + touch "$FALLBACK_MARKER" +} + +apt_update_and_install() { + sudo -E apt-get "${APT_OPTS[@]}" update --error-on=any + sudo -E apt-get "${APT_OPTS[@]}" install -y "$@" +} + +if apt_update_and_install "$@"; then + exit 0 +fi + +echo "apt failed against archive.ubuntu.com; retrying with the mirrors the machine image shipped." >&2 +restore_shipped_mirrors +apt_update_and_install "$@" diff --git a/ops/ci/runtime/cannon-witness.json b/ops/ci/runtime/cannon-witness.json new file mode 100644 index 00000000000..f5b6bd5828e --- /dev/null +++ b/ops/ci/runtime/cannon-witness.json @@ -0,0 +1,11 @@ +{ + "block_number": "47600000", + "l2_claim": "0x6ac07d241d170ddd504532a2c8127bb85426e604a360f8ba2d976be3cc0aa7f0", + "l2_output_root": "0x9c799a1b767f43d6c654f83e18ae20bfc2953e94e5093ef59664ae8244ffdc96", + "l2_head": "0x0084ac82023844b6f286783f3ea0ffddb8e597211ffbe01b56d8873f36a8bacb", + "l1_head": "0xf82badb3d56a53c373b501ec24e1e9ab2797feaab552c0d683849b1234308a2d", + "l2_chain_id": "11155420", + "filename": "op-sepolia-47600000-witness.tar.zst", + "release_url": "https://github.com/ethereum-optimism/chain-test-data/releases/download/kona-2026-08-13-op-sepolia-47600000-v2", + "sha256": "2ac1e74305ef1e62b0ca7ef90f193c680dc03c0b7c3db3af4c3a5c810cdc0b67" +} diff --git a/ops/ci/runtime/ci-report.py b/ops/ci/runtime/ci-report.py new file mode 100644 index 00000000000..3998b4b9dcd --- /dev/null +++ b/ops/ci/runtime/ci-report.py @@ -0,0 +1,148 @@ +"""Suite-independent report bytes, artifact checks and subprocess evidence. + +Callers choose required files, report layout, working directory and redaction. +Callers own selection, source/settings binding, retries and verdict rules. +""" +from concurrent.futures import ThreadPoolExecutor +from contextlib import contextmanager +import hashlib +import json +import os +from pathlib import Path +import re +import signal +import subprocess +import sys +import time + +def read(path): + return json.loads(Path(path).read_text()) + + +def write(path, value): + Path(path).write_text(json.dumps(value, indent=2, sort_keys=True) + '\n') + + +def digest(path): + with Path(path).open('rb') as source: + return hashlib.file_digest(source, 'sha256').hexdigest() + + +def command(argv, *, cwd=None): + return subprocess.check_output(argv, cwd=cwd, text=True).strip() + + +def file_hashes(directory, *, exclude=()): + directory = Path(directory) + return {str(path.relative_to(directory)): digest(path) + for path in sorted(directory.rglob('*')) + if path.is_file() and path.name not in exclude} + + +def verify_files(directory, hashes, *, required=(), label='report'): + """Verify listed artifact bytes without creating or changing any file.""" + if not isinstance(hashes, dict) or not set(required) <= hashes.keys(): + raise ValueError('Incomplete original file manifest: ' + label) + directory = Path(directory) + for name, sha in hashes.items(): + if not isinstance(name, str) or not isinstance(sha, str): + raise ValueError('Invalid original report path or hash') + path = Path(name) + if path.is_absolute() or '..' in path.parts or not re.fullmatch('[0-9a-f]{64}', sha): + raise ValueError('Invalid original report path or hash') + target = directory / path + if target.is_symlink() or any(p.is_symlink() for p in target.parents if p.is_relative_to(directory)): + raise ValueError('Linked original report: ' + label + '/' + name) + if digest(target) != sha: + raise ValueError('Missing or corrupt original report: ' + label + '/' + name) + return hashes + + +@contextmanager +def forwarding_signals(child): + """Forward cancellation to the child's process group; restore parent handlers.""" + previous = {} + def cancel(signum, _frame): + try: + os.killpg(child.pid, signum) + except ProcessLookupError: + pass + try: + for signum in (signal.SIGINT, signal.SIGTERM): + previous[signum] = signal.signal(signum, cancel) + yield + finally: + for signum, handler in previous.items(): + signal.signal(signum, handler) + + +def stage(directory, name, argv, *, cwd, layout='combined', stdout_json=False, + stdout_file=None, stdin=None, redactor=lambda data: data): + """Run once, retain original streams and signal, and return a shell exit code. + + combined preserves Rust stage logs and optional separate stdout. split + preserves contract stdout/stderr files and redacts before saving or echoing. + Suite adapters choose the layout; this helper never retries the command. + """ + if layout not in ('combined', 'split'): + raise ValueError('Unknown stage report layout') + if stdout_file is not None and (stdout_json or Path(stdout_file).name != stdout_file or layout != 'combined'): + raise ValueError('Invalid separated original stdout destination') + directory = Path(directory) + started = time.time() + row = {'argv': [redactor(arg.encode()).decode() for arg in argv], 'cwd': str(cwd), + 'started_at': started, 'exit_code': None} + if stdout_file is not None: + row['stdout_file'] = stdout_file + if stdin == subprocess.DEVNULL: + row['stdin'] = 'devnull' + record = directory / (name + '.stage.json') + write(record, row) + + def copy(stream, path, echo): + with path.open('wb') as output: + for line in iter(stream.readline, b''): + line = redactor(line) + output.write(line); output.flush() + if echo: + sys.stdout.buffer.write(line); sys.stdout.buffer.flush() + + if layout == 'split': + stdout = directory / (name + ('.json' if stdout_json else '.log')) + stderr = directory / (name + '.stderr.log') + child = subprocess.Popen(argv, cwd=cwd, stdin=stdin, stdout=subprocess.PIPE, + stderr=subprocess.PIPE, start_new_session=True) + try: + with forwarding_signals(child), ThreadPoolExecutor(max_workers=2) as pool: + readers = [pool.submit(copy, child.stdout, stdout, not stdout_json), + pool.submit(copy, child.stderr, stderr, True)] + status = child.wait() + for reader in readers: + reader.result() + finally: + child.stdout.close(); child.stderr.close() + row.update(stdout_sha256=digest(stdout), stderr_sha256=digest(stderr)) + else: + separate = stdout_json or stdout_file is not None + log = directory / (name + '.log') + stdout = directory / (stdout_file or name + '.json') + with log.open('wb') as output, (stdout.open('wb') if separate else open(os.devnull, 'wb')) as out: + child = subprocess.Popen(argv, cwd=cwd, stdin=stdin, start_new_session=True, + stdout=out if separate else subprocess.PIPE, + stderr=output if separate else subprocess.STDOUT) + try: + with forwarding_signals(child): + if not separate: + for line in iter(child.stdout.readline, b''): + output.write(line); output.flush() + sys.stdout.buffer.write(line); sys.stdout.buffer.flush() + status = child.wait() + finally: + if child.stdout is not None: + child.stdout.close() + row['log_sha256'] = digest(log) + if stdout_file is not None: + row['stdout_sha256'] = digest(stdout) + row.update(exit_code=status, elapsed_seconds=time.time() - started) + write(record, row) + return status if status >= 0 else 128 - status diff --git a/ops/ci/runtime/contract-l2-fork.py b/ops/ci/runtime/contract-l2-fork.py new file mode 100644 index 00000000000..b8c93362a7b --- /dev/null +++ b/ops/ci/runtime/contract-l2-fork.py @@ -0,0 +1,40 @@ +#!/usr/bin/env python3 +"""Run the OP Mainnet L2 fork tests as CircleCI's contracts-bedrock-tests-l2-fork job does.""" +import argparse +import importlib.util +import os +from pathlib import Path +import sys + + +def helper(name): + spec = importlib.util.spec_from_file_location(name.replace('-', '_'), Path(__file__).with_name(name + '.py')) + module = importlib.util.module_from_spec(spec); spec.loader.exec_module(module) + return module + + +UP = helper('contract-upgrades') +PACER = helper('rpc-pacer') +RPC = 'https://mainnet.optimism.io' + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('mode', choices=('prepare', 'run')); parser.add_argument('--fork-block', default='latest') + args = parser.parse_args() + UP.configure('main', FOUNDRY_PROFILE='ci') + directory = UP.report('contract-l2-fork', args.mode) + if args.mode == 'prepare': return UP.build(directory) + # Like the CircleCI job, `latest` forks the current head. + block = UP.REPORT.command(['cast', 'block-number', '--rpc-url', RPC]) if args.fork_block == 'latest' else args.fork_block + print('Forking OP Mainnet at block ' + block, flush=True) + os.environ.update(L2_FORK_BLOCK_NUMBER=block, JUNIT_TEST_PATH=str(directory / 'results.xml')) + status = UP.stage(directory, 'nut-bundle-check', ['just', 'nut-bundle-check-no-build']) + if status: return status + # CircleCI forks a private RPC. The public one needs paced requests: see rpc-pacer.py. + with PACER.serve(RPC) as endpoint: + os.environ['L2_FORK_RPC_URL'] = endpoint + return UP.run_tests(directory, 'test-l2-fork-upgrade') + + +if __name__ == '__main__': sys.exit(main()) diff --git a/ops/ci/runtime/contract-suites.py b/ops/ci/runtime/contract-suites.py new file mode 100644 index 00000000000..8d944088452 --- /dev/null +++ b/ops/ci/runtime/contract-suites.py @@ -0,0 +1,42 @@ +#!/usr/bin/env python3 +"""Run a contract test suite as CircleCI's contracts-bedrock-tests job does. + +The `standard` suite is contracts-bedrock-tests. The `modified` suite is +contracts-bedrock-tests-heavy-fuzz-modified. +""" +import argparse +import importlib.util +import os +from pathlib import Path +import sys + +SPEC = importlib.util.spec_from_file_location('contract_upgrades', Path(__file__).with_name('contract-upgrades.py')) +UP = importlib.util.module_from_spec(SPEC); SPEC.loader.exec_module(UP) +# Each job's CircleCI `test_list` parameter. It prints the test files to run, one per line. +# The `modified-baseline` task fetches origin/develop once, so every task of a run diffs the same commit. +FILE_COMMANDS = { + 'standard': ['find', 'test', '-name', '*.t.sol'], + 'modified': ['bash', '-eo', 'pipefail', '-c', + "git diff origin/develop...HEAD --name-only --no-renames --diff-filter=AM -- './test/**/*.t.sol' | sed 's|packages/contracts-bedrock/||'"], +} + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('mode', choices=('prepare', 'run')); parser.add_argument('--suite', choices=FILE_COMMANDS, required=True) + parser.add_argument('--feature', choices=UP.FEATURES, required=True) + args = parser.parse_args() + UP.configure(args.feature, FOUNDRY_PROFILE='ciheavy' if args.suite == 'modified' else UP.profile()) + directory = UP.report('contract-suites', args.suite + '-' + args.feature, args.mode) + status = UP.stage(directory, 'files', FILE_COMMANDS[args.suite]) + if status: return status + files = (directory / 'files.log').read_text().strip() + if not files: + # Like the CircleCI job, stop with success. With no TEST_FILES, `just test` runs every test. + print('No test files to run.', flush=True); return 0 + if args.mode == 'prepare': return UP.build(directory) + os.environ.update(TEST_FILES=files, JUNIT_TEST_DIR=str(directory)) + return UP.run_tests(directory, 'test') or UP.stage(directory, 'lint-test-names', ['just', 'lint-forge-tests-check-no-build']) + + +if __name__ == '__main__': sys.exit(main()) diff --git a/ops/ci/runtime/contract-upgrades.py b/ops/ci/runtime/contract-upgrades.py new file mode 100644 index 00000000000..01b3c8d4add --- /dev/null +++ b/ops/ci/runtime/contract-upgrades.py @@ -0,0 +1,117 @@ +#!/usr/bin/env python3 +"""Run the L1 contract upgrade tests as CircleCI's contracts-bedrock-tests-upgrade job does. + +The contract suite and L2 fork runners import the shared steps of this module. +""" +import argparse +import html +import importlib.util +import os +from pathlib import Path +import shutil +import sys +import urllib.parse + +SPEC = importlib.util.spec_from_file_location('ci_report', Path(__file__).with_name('ci-report.py')) +REPORT = importlib.util.module_from_spec(SPEC); SPEC.loader.exec_module(REPORT) +ROOT = Path(__file__).resolve().parents[3] +CONTRACTS = ROOT / 'packages/contracts-bedrock' +# CircleCI's features_matrix, with the variable that its setup-features command exports. +FEATURES = {'main': None, + 'CUSTOM_GAS_TOKEN': 'SYS_FEATURE__CUSTOM_GAS_TOKEN', + 'INTEROP': 'SYS_FEATURE__INTEROP', + 'BRIDGE_HOOK': 'DEV_FEATURE__BRIDGE_HOOK', + 'OP_PRIVATE': 'DEV_FEATURE__OP_PRIVATE'} +# Variant to (FORK_OP_CHAIN, feature): CircleCI's feature matrix on OP Mainnet, then its chain matrix. +VARIANTS = {**{'feature-' + feature: ('op', feature) for feature in FEATURES}, + **{'chain-' + chain: (chain, 'main') for chain in ('op', 'ink', 'unichain')}} + + +class Redactor: + """Mask authentication data before streaming or saving RPC-dependent output.""" + def __init__(self, url): + parsed = urllib.parse.urlsplit(url) + if parsed.scheme not in ('http', 'https') or not parsed.hostname: raise ValueError('Invalid test-only RPC configuration') + parts = [url, html.escape(url, quote=True), urllib.parse.quote(url, safe="")] + parts += [v for _, v in urllib.parse.parse_qsl(parsed.query) if v] + parts += [v for v in (parsed.username, parsed.password) if v] + path_parts = [v for v in parsed.path.split('/') if v] + # Hosted RPC keys commonly occupy the final path segment. Error pages + # can echo that key alone, including keys shorter than sixteen bytes. + parts += [v for v in path_parts if len(v) >= 16] + path_parts[-1:] + parts += [html.escape(v, quote=True) for v in parts] + [urllib.parse.quote(v, safe="") for v in parts] + self.values = sorted({v.encode() for v in parts}, key=len, reverse=True) + def __call__(self, data, replacement=b''): + for value in self.values: data = data.replace(value, replacement) + return data + def xml(self, data): + # XML text and attributes must remain parseable after authentication is + # removed. The plain-log marker would introduce an unclosed XML tag. + return self(data, b'<test-only-rpc>') + + +def profile(): + """Return the Foundry profile that CircleCI uses on this branch.""" + return 'ci' if os.environ['CI_BRANCH'] == 'develop' else 'liteci' + + +def configure(feature, **environment): + """Export the CircleCI job's `environment:` values and the flag of its feature.""" + os.environ.update(environment, CI='true') + if FEATURES[feature]: os.environ[FEATURES[feature]] = 'true' + + +def report(*names): + """Return a new, empty directory under .ci for the logs and reports of this task's stages.""" + directory = ROOT.joinpath('.ci', *names) + shutil.rmtree(directory, ignore_errors=True); directory.mkdir(parents=True) + return directory + + +def stage(directory, name, argv, redactor=lambda data: data): + return REPORT.stage(directory, name, argv, cwd=CONTRACTS, layout='split', redactor=redactor) + + +def build(directory): + """Compile the Go FFI helper and the contracts with the exported profile.""" + return stage(directory, 'go-ffi', ['just', 'build-go-ffi']) or stage(directory, 'contracts-build', ['forge', 'build']) + + +def run_tests(directory, recipe, arguments=(), redactor=lambda data: data): + """Run a test recipe. After a test failure, run its rerun recipe, which prints the traces.""" + status = stage(directory, 'tests', ['just', recipe, *arguments], redactor) + if status and status < 128: + # CircleCI's trace step sets no JUnit path. With one set, the recipe sends + # forge's output, traces included, to the report file. + for name in ('JUNIT_TEST_PATH', 'JUNIT_TEST_DIR'): os.environ.pop(name, None) + stage(directory, 'rerun', ['just', recipe + '-rerun', *arguments], redactor) + return status + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('mode', choices=('compile', 'run')); parser.add_argument('--variant', choices=VARIANTS) + args = parser.parse_args() + if args.mode == 'compile': + # One build serves every variant and the standard contract suites: the compiler reads + # the profile, and the tests read the feature flags and the fork variables. + configure('main', FOUNDRY_PROFILE=profile()) + return build(report('contracts-compile')) + if not args.variant: parser.error('run requires --variant') + chain, feature = VARIANTS[args.variant] + configure(feature, FOUNDRY_PROFILE=profile(), FOUNDRY_FUZZ_SEED='42424242', FOUNDRY_FUZZ_RUNS='1', + FORK_OP_CHAIN=chain, FORK_BASE_CHAIN='mainnet') + directory = report('contract-upgrades', args.variant, 'run') + # Like CircleCI, the recipe gets the archive URL itself: it pins the block and retries fork reads. + url = os.environ.get('OP_CI_MAINNET_L1_ARCHIVE_RPC_URL', '') + try: redactor = Redactor(url) + except ValueError: sys.exit('OP_CI_MAINNET_L1_ARCHIVE_RPC_URL is not an http(s) URL. Does this branch have a grant on the RPC vault?') + junit = directory / 'results.xml' + os.environ.update(ETH_RPC_URL=url, JUNIT_TEST_PATH=str(junit)) + try: return run_tests(directory, 'test-upgrade', redactor=redactor) + finally: + # Failure reasons in the report can quote the URL. + if junit.exists(): junit.write_bytes(redactor.xml(junit.read_bytes())) + + +if __name__ == '__main__': sys.exit(main()) diff --git a/ops/ci/runtime/go-artifacts.py b/ops/ci/runtime/go-artifacts.py new file mode 100755 index 00000000000..5c2279c9d08 --- /dev/null +++ b/ops/ci/runtime/go-artifacts.py @@ -0,0 +1,103 @@ +#!/usr/bin/env python3 +"""Pack the outputs of a dependency build, and restore them in a task that consumes them. + +pack ... Write files.tar.gz and metadata.json to .ci/go-tests/dependencies/. +restore Extract the artifact in if it fits this checkout. +""" +import argparse +import hashlib +import importlib.util +import json +import os +from pathlib import Path +import subprocess +import tarfile +import sys + +ROOT = Path(__file__).resolve().parents[3] +KINDS = ('go', 'go-modules', 'contracts', 'contracts-e2e', 'rust-e2e-release', 'kona', 'op-reth', 'prestate', 'sp1-executor') + +def dependency_source(): + spec = importlib.util.spec_from_file_location('go_dependency_source', ROOT / 'ops/ci/runtime/go-dependency-source.py') + helper = importlib.util.module_from_spec(spec) + spec.loader.exec_module(helper) + return helper + + +def digest(path): + with path.open('rb') as stream: + return hashlib.file_digest(stream, 'sha256').hexdigest() + +def pack(kind, paths): + actual_sha = subprocess.check_output(['git', 'rev-parse', 'HEAD'], cwd=ROOT, text=True).strip() + if actual_sha != os.environ['CI_COMMIT_SHA']: + raise ValueError(f"Dependency revision {actual_sha} differs from requested {os.environ['CI_COMMIT_SHA']}") + output = ROOT / '.ci/go-tests/dependencies' / kind + output.mkdir(parents=True, exist_ok=True) + files = {} + for name in paths: + path = ROOT / name + if not path.exists(): + raise ValueError(f'Missing dependency output: {name}') + for item in sorted(path.rglob('*')) if path.is_dir() else [path]: + if item.is_file(): + files[str(item.relative_to(ROOT))] = digest(item) + if not files: + raise ValueError('Empty dependency artifact') + archive = output / 'files.tar.gz' + with tarfile.open(archive, 'w:gz') as stream: + for name in paths: + stream.add(ROOT / name, arcname=name) + metadata = {'kind': kind, 'commit_sha': actual_sha, 'files': files, 'archive_sha256': digest(archive)} + if os.environ.get('CI_GO_REUSABLE_DEPENDENCY') == '1': + source = dependency_source() + metadata['build_inputs_sha256'] = source.fingerprint(kind) + source.verify(kind, metadata) + (output / 'metadata.json').write_text(json.dumps(metadata, indent=2) + '\n') + +def restore(kind, artifact): + metadata = json.loads((artifact / 'metadata.json').read_text()) + expected = {'kind': kind, 'commit_sha': os.environ['CI_COMMIT_SHA']} + if 'build_inputs_sha256' in metadata: + actual = subprocess.check_output(['git', 'rev-parse', 'HEAD'], cwd=ROOT, text=True).strip() + if actual != os.environ['CI_COMMIT_SHA']: + raise ValueError(f"Consumer checkout {actual} differs from requested {os.environ['CI_COMMIT_SHA']}") + # Same-commit consumers may intentionally project the workspace (e.g. + # acceptance tests). Their artifact still has the strict source binding. + # Relax that binding only after verifying an ancestor's complete inputs. + if metadata.get('commit_sha') != expected['commit_sha']: + dependency_source().verify(kind, metadata) + expected.pop('commit_sha') + if any(metadata.get(key) != value for key, value in expected.items()): + raise ValueError('Dependency kind or revision mismatch') + archive = artifact / 'files.tar.gz' + if metadata.get('archive_sha256') != digest(archive): + raise ValueError('Dependency archive changed') + files = metadata['files'] + with tarfile.open(archive) as stream: + def member_to_extract(member, destination): + member = tarfile.data_filter(member, destination) + if member is None: return None + path = ROOT / member.name + # The producer's layer can already hold these files, and Go module + # directories are read-only. Do not write a file that is identical. + if (member.isfile() and path.is_file() and not path.is_symlink() + and digest(path) == files.get(member.name) + and path.stat().st_mode & 0o111 == member.mode & 0o111): + return None + return member + stream.extractall(ROOT, filter=member_to_extract) + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument('command', choices=['pack', 'restore']) + parser.add_argument('kind', choices=KINDS) + parser.add_argument('paths', nargs='+') + args = parser.parse_args() + try: + if args.command == 'pack': pack(args.kind, args.paths) + elif len(args.paths) == 1: restore(args.kind, Path(args.paths[0])) + else: raise ValueError('Restore requires exactly one artifact directory') + except (ValueError, OSError, KeyError, subprocess.CalledProcessError) as error: + print(f'ERROR: {error}', file=sys.stderr) + sys.exit(1) diff --git a/ops/ci/runtime/go-compiled-tests.py b/ops/ci/runtime/go-compiled-tests.py new file mode 100644 index 00000000000..764c4eac5a4 --- /dev/null +++ b/ops/ci/runtime/go-compiled-tests.py @@ -0,0 +1,188 @@ +#!/usr/bin/env python3 +"""Compile the Go test binaries of one shard, and run them. + +build Compile the packages of shard CI_SHARD_INDEX into `.ci/go-tests/build`. +restore Make `.ci/go-tests/manifest.json` the manifest that `build` used. +run Run the binaries of shard CI_SHARD_INDEX and print their test2json + events. gotestsum calls `run`, and calls it again with a test filter + and a package for each rerun of a failed test. + +`go-suite.py` writes the manifest. Binary `.test` belongs to package `n` of +the shard's package list. +""" + +from concurrent.futures import ThreadPoolExecutor +import argparse +from datetime import datetime, timezone +import json +import os +import re +import shutil +import signal +from pathlib import Path +import subprocess +import sys +import threading + + +ROOT = Path(__file__).resolve().parents[3] +MODULE = "github.com/ethereum-optimism/optimism" +MANIFEST = ROOT / ".ci/go-tests/manifest.json" +BUILD = ROOT / ".ci/go-tests/build" +PRINT_LOCK = threading.Lock() +PROCESSES = set() +PROCESS_LOCK = threading.Lock() + + +def backup_timeout(value): + # go test grants at least one extra minute beyond the testing package's + # alarm, bounding TestMain cleanup and inherited output pipes as well. + units = {'ns': 1e-9, 'us': 1e-6, 'µs': 1e-6, 'ms': 1e-3, 's': 1, 'm': 60, 'h': 3600} + if value == '0': + return None + pattern = r'(\d+(?:\.\d*)?|\.\d+)(ns|us|µs|ms|s|m|h)' + pieces = re.findall(pattern, value) + if not pieces or ''.join(number + unit for number, unit in pieces) != value: + raise ValueError('Invalid Go package timeout') + seconds = sum(float(number) * units[unit] for number, unit in pieces) + return seconds + 60 if seconds else None + + +def build(): + manifest = json.loads(MANIFEST.read_text()) + # A compiler-seed layer carries the manifest of an older commit. This task + # must compile the packages that the discovery task selected for its commit. + if manifest["commit_sha"] != os.environ["CI_COMMIT_SHA"]: + raise ValueError("Go discovery belongs to a different commit") + packages = manifest["shards"][int(os.environ["CI_SHARD_INDEX"])] + BUILD.mkdir(parents=True, exist_ok=True) + subprocess.run(["go", "build", "-o", str(BUILD / "test2json"), "cmd/test2json"], cwd=ROOT, check=True) + + def compile_package(item): + index, package = item + output = BUILD / f"{index}.test" + output.unlink(missing_ok=True) + # go test -c compiles only. TestMain and init functions never run here. + subprocess.run(["go", "test", "-c", "-p=4", "-tags=ci", "-o", str(output), package], cwd=ROOT, check=True) + # For a package without test files, go test -c succeeds and writes no binary. + if package not in manifest["no_tests"] and not output.is_file(): + raise ValueError(f"Compilation did not produce a test binary: {package}") + + with ThreadPoolExecutor(max_workers=4) as pool: + list(pool.map(compile_package, enumerate(packages))) + shutil.copyfile(MANIFEST, BUILD / "manifest.json") + + +def restore_discovery(): + """Replace the workspace manifest with the one that `build` stored with the binaries. + + The verdict task inherits compiler-seed layers through its compile task. + Such a layer carries the manifest of an older commit, and that file can + replace the one from the discovery task. + """ + shutil.copyfile(BUILD / "manifest.json", MANIFEST) + + +def emit(line): + with PRINT_LOCK: + sys.stdout.write(line) + sys.stdout.flush() + + +def cancel(signum, _frame): + with PROCESS_LOCK: + for process in PROCESSES: + try: + os.killpg(process.pid, signal.SIGTERM) + except ProcessLookupError: + pass + raise SystemExit(128 + signum) + + +def execute_binary(reporter, binary, package, directory, parallel, timeout, run_filter=None): + """Run a fresh binary with ordinary Go signal handling and cleanup bounds.""" + delay = backup_timeout(timeout) + # test2json command mode ignores INT/QUIT before spawning its child. + # Reset those dispositions before exec, as ordinary go test does; + # otherwise subprocess fixtures inherit SIG_IGN and cannot stop. + launch = 'import os,signal,sys; signal.signal(signal.SIGINT,signal.SIG_DFL); signal.signal(signal.SIGQUIT,signal.SIG_DFL); os.execv(sys.argv[1],sys.argv[1:])' + command = [str(reporter), "-t", "-p", package, sys.executable, "-c", launch, str(binary), + "-test.v=test2json", "-test.count=1", + "-test.timeout=" + timeout, "-test.paniconexit0"] + if parallel is not None: + command.append("-test.parallel=" + str(parallel)) + if run_filter is not None: + command.append("-test.run=" + run_filter) + with PROCESS_LOCK: + process = subprocess.Popen(command, cwd=directory, stdout=subprocess.PIPE, text=True, start_new_session=True) + PROCESSES.add(process) + def expire(): + emit(json.dumps({'Time': datetime.now(timezone.utc).isoformat(), 'Action': 'output', 'Package': package, + 'Output': '*** Test process exceeded the package timeout plus one-minute cleanup grace.\n'}) + '\n') + try: + os.killpg(process.pid, signal.SIGKILL) + except ProcessLookupError: + pass + watchdog = threading.Timer(delay, expire) if delay is not None else None + if watchdog: + watchdog.daemon = True + watchdog.start() + try: + for line in process.stdout: + emit(line) + return process.wait() + finally: + if watchdog: + watchdog.cancel() + process.stdout.close() + with PROCESS_LOCK: + PROCESSES.discard(process) + +def run(args): + manifest = json.loads(MANIFEST.read_text()) + packages = manifest["shards"][int(os.environ["CI_SHARD_INDEX"])] + binaries = {package: None if package in manifest["no_tests"] else BUILD / f"{index}.test" + for index, package in enumerate(packages)} + # gotestsum's raw-command reruns append -test.run= and a package. + # Only a rerun may narrow the selection; the initial attempt runs it all. + run_filter = None + if args: + if len(args) != 2 or not args[0].startswith("-test.run=") or args[1] not in binaries: + raise ValueError("Unexpected gotestsum rerun arguments") + run_filter, packages = args[0][len("-test.run="):], [args[1]] + + def run_package(package): + if binaries[package] is None: + now = datetime.now(timezone.utc).isoformat() + for action in ("start", "skip"): + emit(json.dumps({"Time": now, "Action": action, "Package": package}) + "\n") + return 0 + # go test runs each binary in its package directory, where its fixtures are. + directory = ROOT / package.removeprefix(MODULE).lstrip("/") + return execute_binary(BUILD / "test2json", binaries[package], package, directory, + os.environ["PARALLEL"], os.environ["TEST_TIMEOUT"], run_filter) + + # At most four packages run at a time, like the recipe's `go test -p=4`. + with ThreadPoolExecutor(max_workers=4) as pool: + statuses = list(pool.map(run_package, packages)) + return 1 if any(status != 0 for status in statuses) else 0 + + +if __name__ == "__main__": + # Preserve the selected executable's SDK, rather than a parent's mise root. + os.environ.pop('GOROOT', None) + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("command", choices=["build", "restore", "run"]) + options, arguments = parser.parse_known_args() + try: + signal.signal(signal.SIGTERM, cancel) + signal.signal(signal.SIGINT, cancel) + if options.command == "build": + build() + elif options.command == "restore": + restore_discovery() + else: + sys.exit(run(arguments)) + except (ValueError, KeyError, IndexError, OSError, subprocess.CalledProcessError) as error: + print(f"ERROR: {error}", file=sys.stderr) + sys.exit(1) diff --git a/ops/ci/runtime/go-dependencies.sh b/ops/ci/runtime/go-dependencies.sh new file mode 100755 index 00000000000..47099cdb7a1 --- /dev/null +++ b/ops/ci/runtime/go-dependencies.sh @@ -0,0 +1,80 @@ +#!/usr/bin/env bash +# Build the aggregate Circle Go workload's dependencies without executing tests. +set -euo pipefail +cd "$(dirname "${BASH_SOURCE[0]}")/../../.." +job="${1:?Pass a dependency name}" +export GOMODCACHE="$PWD/.ci/go-cache/full/modules" GOCACHE="$PWD/.ci/go-cache/full/dependencies" +mkdir -p "$GOMODCACHE" "$GOCACHE" + +# Runs `cargo build --locked --profile release` in rust/ with the given cargo +# arguments. Like Circle's rust-build, release builds that only feed CI tests +# skip LTO. Writes the sccache statistics next to the dependency's artifact. +rust_release() { + export CARGO_HOME="$PWD/.ci/rust-cache/cargo" CARGO_TARGET_DIR="$PWD/rust/target" + export CARGO_INCREMENTAL=0 RUSTC_WRAPPER=sccache SCCACHE_DIR="$PWD/.ci/rust-cache/sccache" + export SCCACHE_BASEDIRS="$PWD" SCCACHE_CACHE_SIZE=10G SCCACHE_IDLE_TIMEOUT=0 + mkdir -p "$CARGO_HOME" "$SCCACHE_DIR" ".ci/go-tests/dependencies/$job" + sccache --start-server + sccache --zero-stats + trap 'sccache --show-stats --stats-format json >".ci/go-tests/dependencies/$job/sccache.json"; sccache --stop-server' EXIT + python3 ops/ci/runtime/rust-target-cache.py prepare + (cd rust && CARGO_PROFILE_RELEASE_LTO=off mold -run cargo build --locked --profile release "$@") + python3 ops/ci/runtime/rust-target-cache.py commit +} + +case "$job" in + go) + attempt=0 + until go mod download; do + attempt=$((attempt + 1)) + [[ "$attempt" -lt 5 ]] || exit 1 + sleep "$((2 ** attempt))" + done + just sync-superchain-go + git diff --exit-code op-core/superchain/bundle/superchain-configs.zip.sha256 + just cannon + (cd cannon && just diff-hello-elf) + python3 ops/ci/runtime/go-artifacts.py pack go \ + op-core/superchain/bundle/superchain-configs.zip cannon/bin cannon/multicannon/embeds cannon/testdata/bin + ;; + contracts | contracts-e2e) + git submodule update --init --recursive -- packages/contracts-bedrock/lib + export FOUNDRY_PROFILE=ci + # Match Circle's producer, contracts-bedrock-build with `--skip test`. Forge's + # "test" alias skips .t.sol files while retaining non-test helper contracts + # under test/ (e.g. MockSP1Verifier). Like Circle, this build does not deny + # warnings: that would turn off the compile cache that the script cache + # warm-up checks. check-fast's dev build is the warning gate. + (cd packages/contracts-bedrock && just forge-build --skip test) + (cd op-deployer && just copy-contract-artifacts) + python3 ops/ci/runtime/go-artifacts.py pack "$job" packages/contracts-bedrock/cache \ + packages/contracts-bedrock/artifacts packages/contracts-bedrock/forge-artifacts \ + op-deployer/pkg/deployer/artifacts/forge-artifacts + ;; + kona) + if [[ "${TARGET_CACHE_MODE:-keep}" == sccache-only ]]; then rm -rf rust/target; fi + rust_release --features default \ + --package kona-host --package kona-client --package kona-node --package op-zk-proposer + python3 ops/ci/runtime/go-artifacts.py pack kona rust/target/release/kona-host rust/target/release/kona-client \ + rust/target/release/kona-node rust/target/release/op-zk-proposer + ;; + sp1-executor) + rust_release --all-features --package kona-sp1-super-range-executor --bin kona-sp1-super-range-executor + mkdir -p .circleci-cache/rust-binaries + cp rust/target/release/kona-sp1-super-range-executor .circleci-cache/rust-binaries/ + python3 ops/ci/runtime/go-artifacts.py pack sp1-executor .circleci-cache/rust-binaries/kona-sp1-super-range-executor + ;; + op-reth) + bash /usr/local/lib/optimism-ci/op-reth-shadow.sh source + bash /usr/local/lib/optimism-ci/op-reth-shadow.sh release-build + python3 ops/ci/runtime/go-artifacts.py pack op-reth rust/target/release/op-reth rust/target/release/op-reth-test-engine + ;; + prestate) + just reproducible-prestate + python3 ops/ci/runtime/go-artifacts.py pack prestate rust/kona/prestate-artifacts-* + ;; + *) + echo 'Unknown dependency' >&2 + exit 1 + ;; +esac diff --git a/ops/ci/runtime/go-dependency-source.py b/ops/ci/runtime/go-dependency-source.py new file mode 100644 index 00000000000..ae25bd0e83a --- /dev/null +++ b/ops/ci/runtime/go-dependency-source.py @@ -0,0 +1,78 @@ +#!/usr/bin/env python3 +"""Bind reusable Go helpers to all tracked build inputs, independently of PR SHA.""" +import argparse +import hashlib +import json +import os +from pathlib import Path +import subprocess + +ROOT = Path(__file__).resolve().parents[3] +RUNTIME = 'ops/ci/runtime/' +# The scripts that a dependency build runs, and the scripts that the Go compiler seeds run +# as well. test_go_dependency_source.py fails when one of them reaches a script that is not +# listed. RWX's own cache key covers the rest: the task definition in `.rwx`, and the tool +# scripts through the tool tasks. +BUILD_SCRIPTS = [RUNTIME + name for name in ( + 'go-artifacts.py', 'go-dependencies.sh', 'go-dependency-source.py', 'op-reth-report.py', 'op-reth-shadow.sh', + 'rust-target-cache.py')] +COMPILER_SCRIPTS = [RUNTIME + name for name in ( + 'go-compiled-tests.py', 'go-full-tests.sh', 'go-package-shards.py', 'go-report.py', 'go-suite.py', 'go-tests-timings.json')] +# The files that every build reads. +COMMON = ['mise.toml', '.gitmodules', 'superchain-registry', 'justfile', 'justfiles', *BUILD_SCRIPTS, + 'go.mod', 'go.sum', 'package.json', 'pnpm-lock.yaml', 'pnpm-workspace.yaml', '.npmrc'] +RUST = ['rust', 'op-core/nuts/bundles', 'op-core/superchain'] +SCOPES = { + 'go-modules': ['mise.toml', 'go.mod', 'go.sum', *BUILD_SCRIPTS], + 'go': COMMON + [':(glob)**/*.go', 'cannon', 'op-core/superchain'], + 'contracts': COMMON + ['packages/contracts-bedrock', 'op-deployer/justfile', + 'op-deployer/pkg/deployer/artifacts', 'op-service/ioutil'], + 'kona': COMMON + RUST, + 'op-reth': COMMON + RUST, + 'sp1-executor': COMMON + RUST, + 'prestate': COMMON + RUST + ['cannon', 'op-service', 'op-preimage', 'ops/scripts', + 'Dockerfile*', '.dockerignore'], + 'compiler': COMMON + COMPILER_SCRIPTS + ['packages/contracts-bedrock', 'op-deployer/justfile', + 'op-deployer/pkg/deployer/artifacts', 'op-service/ioutil', + ':(glob)**/go.mod', ':(glob)**/go.sum'], +} + +def git(*args): + return subprocess.check_output(['git', *args], cwd=ROOT) + +def fingerprint(kind, revision='HEAD'): + # Tree diff includes names, modes, blob IDs, and submodule commits. A changed + # helper/configuration must invalidate reuse just like changed source does. + empty = subprocess.check_output(['git', 'hash-object', '-t', 'tree', '--stdin'], input=b'', cwd=ROOT).decode().strip() + return hashlib.sha256(git('diff-tree', '--no-commit-id', '--raw', '--no-abbrev', '-r', '-z', empty, revision, '--', *SCOPES[kind])).hexdigest() + +def ancestor(kind): + sha = git('log', '-1', '--format=%H', 'HEAD', '--', *SCOPES[kind]).decode().strip() + if not sha or fingerprint(kind, sha) != fingerprint(kind): + sha = git('rev-parse', 'HEAD').decode().strip() + return sha + +def verify(kind, metadata): + sha = metadata.get('commit_sha', '') + if len(sha) != 40 or any(c not in '0123456789abcdef' for c in sha): + raise ValueError('Invalid dependency source revision') + subprocess.run(['git', 'merge-base', '--is-ancestor', sha, 'HEAD'], cwd=ROOT, check=True) + modified = git('status', '--porcelain', '--untracked-files=no', '--', *SCOPES[kind]).decode(errors='replace').strip() + if modified: + raise ValueError('Modified dependency build inputs: ' + modified[:2000]) + expected = fingerprint(kind) + if metadata.get('build_inputs_sha256') != expected or fingerprint(kind, sha) != expected: + raise ValueError('Dependency build inputs differ from current source') + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('command', choices=['plan', 'fingerprint']) + parser.add_argument('kind', choices=SCOPES, nargs='?') + args = parser.parse_args() + if args.command == 'plan': + values = Path(os.environ['RWX_VALUES']) + for kind in SCOPES: + (values / kind).write_text(ancestor(kind)) + else: + if not args.kind: parser.error('fingerprint requires a kind') + print(fingerprint(args.kind)) diff --git a/ops/ci/runtime/go-full-tests.sh b/ops/ci/runtime/go-full-tests.sh new file mode 100755 index 00000000000..483d75fa07f --- /dev/null +++ b/ops/ci/runtime/go-full-tests.sh @@ -0,0 +1,59 @@ +#!/usr/bin/env bash +# The Go test workload of .rwx/go-tests.yml: discover the packages, compile one +# shard's test binaries, or run them. Compilation and the run are separate tasks. +set -euo pipefail +cd "$(dirname "${BASH_SOURCE[0]}")/../../.." +phase="${1:?Expected discover, build or run}" +# .rwx/go-tests.yml has one compile task and one verdict task per shard. +shards=12 +# Each phase has its own writable build cache. Test binaries inherit the run +# cache, module cache and toolchain, e.g. for the deployer CLI's TestMain build. +export GOMODCACHE="$PWD/.ci/go-cache/full/modules" +export GOCACHE="$PWD/.ci/go-cache/full/$phase" +mkdir -p "$GOMODCACHE" "$GOCACHE" +# Unlike `go test`, invoking compiled binaries does not replace mise's GOROOT +# when GOTOOLCHAIN selects the newer version required by go.mod. +unset GOROOT +export GOROOT +GOROOT="$(go env GOROOT)" +export PATH="$GOROOT/bin:$PATH" +case "$phase" in + discover) + python3 ops/ci/runtime/go-suite.py --total "$shards" --timings ops/ci/runtime/go-tests-timings.json + ;; + build) + python3 ops/ci/runtime/go-compiled-tests.py build + ;; + run) + mkdir -p tmp/test-results tmp/testlogs + # Compiler-seed layers can carry an older commit's manifest. Use the one + # that this shard's binaries were compiled from. + python3 ops/ci/runtime/go-compiled-tests.py restore + # The environment and gotestsum options of the `_go-tests-ci-internal` + # recipe in the root justfile. The restored `go` dependency already holds + # cannon/testdata/bin/hello.64.elf, which the recipe builds at this point. + export ENABLE_KURTOSIS=true OP_E2E_CANNON_ENABLED=false OP_E2E_USE_HTTP=true ENABLE_ANVIL=true + PARALLEL="$(nproc 2> /dev/null || sysctl -n hw.ncpu 2> /dev/null || echo 4)" + export PARALLEL TEST_TIMEOUT="${TEST_TIMEOUT:-40m}" + export OP_TESTLOG_FILE_LOGGER_OUTDIR="$PWD/tmp/testlogs" + # shellcheck disable=SC1091 + source ops/scripts/source-ci-archive-rpcs.sh + export NAT_INTEROP_LOADTEST_TARGET=10 NAT_INTEROP_LOADTEST_TIMEOUT=30s + status=0 + # --format=pkgname keeps full output in the JSON and per-test artifacts + # without exhausting live-log quotas. + ./ops/scripts/gotestsum-split.sh --format=pkgname \ + --junitfile="tmp/test-results/results-${CI_SHARD_INDEX}.xml" --jsonfile=tmp/testlogs/log.json \ + --rerun-fails=3 --rerun-fails-max-failures=50 --raw-command \ + -- python3 ops/ci/runtime/go-compiled-tests.py run || status=$? + # RWX reads the shortened copy. The go-json artifact keeps the complete log. + if [[ -f tmp/testlogs/log.json ]]; then + python3 ops/ci/runtime/go-report.py tmp/testlogs/log.json tmp/testlogs/native.json || [[ "$status" -ne 0 ]] || status=1 + fi + exit "$status" + ;; + *) + echo 'Expected discover, build or run' >&2 + exit 1 + ;; +esac diff --git a/ops/ci/runtime/go-package-shards.py b/ops/ci/runtime/go-package-shards.py new file mode 100644 index 00000000000..23385dc7707 --- /dev/null +++ b/ops/ci/runtime/go-package-shards.py @@ -0,0 +1,39 @@ +#!/usr/bin/env python3 +"""Read `go list -json` output and place Go packages on shards.""" + +import json +import statistics + + +def read_go_packages(source): + """Return the package objects in `go list -e -json` output. Fail if a package has an error.""" + decoder, offset, packages = json.JSONDecoder(), 0, [] + # go list prints concatenated JSON objects, not a JSON array. + while offset < len(source): + if source[offset].isspace(): + offset += 1 + continue + package, offset = decoder.raw_decode(source, offset) + # go list -e exits successfully even for missing embeds or dependencies. + if package.get("Error") or package.get("DepsErrors"): + raise ValueError(f"go list reported an error for {package.get('ImportPath', '')}") + packages.append(package) + return packages + + +def assign(packages, total, durations): + """Return `total` sorted package lists. Each package is in exactly one list. + + `durations` maps a package to its seconds. A package without a duration gets + the median of the known ones. The longest package goes first, onto the shard + with the smallest load. Durations change the placement only. + """ + known = [durations[package] for package in packages if package in durations] + fallback = max(0.001, statistics.median(known)) if known else 1.0 + weights = {package: durations.get(package, fallback) for package in packages} + shards, loads = [[] for _ in range(total)], [0.0] * total + for package in sorted(packages, key=lambda package: (-weights[package], package)): + index = min(range(total), key=lambda index: (loads[index], len(shards[index]), index)) + shards[index].append(package) + loads[index] += weights[package] + return [sorted(shard) for shard in shards] diff --git a/ops/ci/runtime/go-report.py b/ops/ci/runtime/go-report.py new file mode 100644 index 00000000000..fd6aaba820d --- /dev/null +++ b/ops/ci/runtime/go-report.py @@ -0,0 +1,72 @@ +#!/usr/bin/env python3 +"""Produce a bounded native Go report while retaining the original JSON unchanged.""" +import argparse +from collections import deque +import json +from pathlib import Path + +LIMIT = 64 * 1024 + + +def events(path): + with path.open() as stream: + for line in stream: + event = json.loads(line) + if not isinstance(event, dict) or not isinstance(event.get('Action'), str): + raise ValueError('Invalid original Go event') + yield event + + +def key(event): + return event.get('Package'), event.get('Test') + + +def project(source, destination): + # Retain failure and skip messages from every attempt, including recovered failures. + interesting = {key(e) for e in events(source) if e['Action'] in ('fail', 'skip')} + buffered, sizes, truncated = {}, {}, set() + destination.parent.mkdir(parents=True, exist_ok=True) + with destination.open('w') as output: + def emit(event): + output.write(json.dumps(event, separators=(',', ':')) + '\n') + + def flush(identity, terminal): + if identity in truncated: + emit({**terminal, 'Action': 'output', 'Output': '[Native report output shortened; complete original events are in the go-json artifact.]\n'}) + for event in buffered.pop(identity, []): + emit(event) + sizes.pop(identity, None) + truncated.discard(identity) + + for event in events(source): + identity = key(event) + if event['Action'] == 'output': + if identity not in interesting: + continue + queue = buffered.setdefault(identity, deque()) + size = len(event.get('Output', '').encode()) + if size > LIMIT: + event = {**event, 'Output': event['Output'].encode()[-LIMIT:].decode(errors='replace')} + size = len(event['Output'].encode()) + truncated.add(identity) + queue.append(event) + sizes[identity] = sizes.get(identity, 0) + size + while sizes[identity] > LIMIT and queue: + sizes[identity] -= len(queue.popleft().get('Output', '').encode()) + truncated.add(identity) + continue + if event['Action'] in ('pass', 'fail', 'skip'): + flush(identity, event) + emit(event) + # Partial/canceled reports stay partial: never synthesize terminal verdicts. + for identity, queue in list(buffered.items()): + if queue: + flush(identity, queue[-1]) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('source', type=Path) + parser.add_argument('destination', type=Path) + args = parser.parse_args() + project(args.source, args.destination) diff --git a/ops/ci/runtime/go-rpc-preflight.py b/ops/ci/runtime/go-rpc-preflight.py new file mode 100644 index 00000000000..765733bbb6d --- /dev/null +++ b/ops/ci/runtime/go-rpc-preflight.py @@ -0,0 +1,38 @@ +#!/usr/bin/env python3 +"""Check archive inputs without logging URLs, credentials or provider bodies.""" +import json +import os +import urllib.parse +import urllib.error +import urllib.request + +NAMES = ('OP_CI_MAINNET_L1_ARCHIVE_RPC_URL', 'OP_CI_SEPOLIA_L1_ARCHIVE_RPC_URL') + +def check(): + for name in NAMES: + value = os.environ.get(name, '') + url = urllib.parse.urlparse(value) + if url.scheme not in ('http', 'https') or not url.netloc: + raise ValueError('An archive RPC input is unavailable or invalid') + request = urllib.request.Request(value, headers={'Content-Type': 'application/json', 'User-Agent': 'Go-http-client/1.1'}, + data=json.dumps({'jsonrpc': '2.0', 'id': 1, 'method': 'eth_getBalance', + 'params': ['0x0000000000000000000000000000000000000000', '0x1']}).encode()) + try: + with urllib.request.urlopen(request, timeout=30) as response: + result = json.load(response) + if result.get('id') != 1 or not isinstance(result.get('result'), str) or not result['result'].startswith('0x') or result.get('error'): + code = result.get('error', {}).get('code') + reason = 'RPC error ' + str(code) if isinstance(code, int) else 'invalid JSON-RPC response' + raise ValueError(reason) + except Exception as error: + # Report only fixed labels and numeric codes; provider exception text + # can contain a credential-bearing URL or the response body. + reason = ('HTTP ' + str(error.code)) if isinstance(error, urllib.error.HTTPError) else (str(error) if isinstance(error, ValueError) and str(error).startswith(('RPC error ', 'invalid JSON-RPC')) else type(error).__name__) + raise ValueError(name + ' historical-state probe failed: ' + reason) from None + print('Both archive RPC historical-state probes passed') + +if __name__ == '__main__': + try: + check() + except ValueError as error: + raise SystemExit(str(error)) diff --git a/ops/ci/runtime/go-suite.py b/ops/ci/runtime/go-suite.py new file mode 100644 index 00000000000..c30abb94451 --- /dev/null +++ b/ops/ci/runtime/go-suite.py @@ -0,0 +1,59 @@ +#!/usr/bin/env python3 +"""Discover the Go test packages and place them on shards. + +The script writes `.ci/go-tests/manifest.json` with these keys: + commit_sha The commit that this task tests (CI_COMMIT_SHA). + shards One sorted package list per shard. Each package is in exactly one list. + no_tests The packages that have no test files with `-tags=ci`. +""" + +import argparse +import importlib.util +import json +import os +from pathlib import Path +import subprocess +import sys + + +ROOT = Path(__file__).resolve().parents[3] +SPEC = importlib.util.spec_from_file_location("shards", Path(__file__).with_name("go-package-shards.py")) +SHARDS = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(SHARDS) + + +def discover(total, timings): + # The selection and the build tag of the `_go-tests-ci-internal` recipe. + selected = subprocess.check_output(["just", "list-test-packages"], cwd=ROOT, text=True).split() + if not selected: + raise ValueError("list-test-packages produced no packages") + listing = subprocess.check_output(["go", "list", "-e", "-tags=ci", "-json", *selected], cwd=ROOT, text=True) + packages = SHARDS.read_go_packages(listing) + no_tests = sorted(package["ImportPath"] for package in packages + if not (package.get("TestGoFiles") or package.get("XTestGoFiles"))) + durations = json.loads(timings.read_text())["packages"] + manifest = {"commit_sha": os.environ["CI_COMMIT_SHA"], "no_tests": no_tests, + "shards": SHARDS.assign([package["ImportPath"] for package in packages], total, durations)} + output = ROOT / ".ci/go-tests" + output.mkdir(parents=True, exist_ok=True) + (output / "manifest.json").write_text(json.dumps(manifest, indent=2) + "\n") + + +def main(): + # A selected Go toolchain can be newer than mise's exported GOROOT. + # Let the executable infer its matching SDK. + os.environ.pop('GOROOT', None) + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--total", type=int, required=True, help="the number of shards") + parser.add_argument("--timings", type=Path, required=True, help="a JSON file with package durations") + args = parser.parse_args() + try: + discover(args.total, args.timings) + except (ValueError, KeyError, OSError, subprocess.CalledProcessError) as error: + print(f"ERROR: {error}", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/ops/ci/runtime/go-tests-timings.json b/ops/ci/runtime/go-tests-timings.json new file mode 100644 index 00000000000..d5c2f8e5bc5 --- /dev/null +++ b/ops/ci/runtime/go-tests-timings.json @@ -0,0 +1,455 @@ +{ + "version": 1, + "source_job": 5625852, + "commit_sha": "eca6169554d544044675bb37b49a5fc94ec17b30", + "note": "Historical Circle package terminal elapsed values; cached packages are estimates, never selection filters.", + "unit": "seconds", + "packages": { + "github.com/ethereum-optimism/optimism/docs/public-docs/scripts/gen-deploy-config": 0, + "github.com/ethereum-optimism/optimism/docs/public-docs/scripts/gen-flags": 0, + "github.com/ethereum-optimism/optimism/linter/analyzers/bigint": 0.713, + "github.com/ethereum-optimism/optimism/op-alt-da": 0.05, + "github.com/ethereum-optimism/optimism/op-alt-da/bindings": 0, + "github.com/ethereum-optimism/optimism/op-alt-da/cmd/daserver": 0, + "github.com/ethereum-optimism/optimism/op-batcher/batcher": 27.245, + "github.com/ethereum-optimism/optimism/op-batcher/batcher/throttler": 0.458, + "github.com/ethereum-optimism/optimism/op-batcher/cmd": 0, + "github.com/ethereum-optimism/optimism/op-batcher/compressor": 0, + "github.com/ethereum-optimism/optimism/op-batcher/config": 0, + "github.com/ethereum-optimism/optimism/op-batcher/flags": 1.148, + "github.com/ethereum-optimism/optimism/op-batcher/metrics": 0.009, + "github.com/ethereum-optimism/optimism/op-batcher/rpc": 0, + "github.com/ethereum-optimism/optimism/op-chain-ops/addresses": 0.084, + "github.com/ethereum-optimism/optimism/op-chain-ops/clients": 0, + "github.com/ethereum-optimism/optimism/op-chain-ops/cmd/check-jovian": 0, + "github.com/ethereum-optimism/optimism/op-chain-ops/cmd/check-karst": 0.004, + "github.com/ethereum-optimism/optimism/op-chain-ops/cmd/check-karst/karsttest": 0, + "github.com/ethereum-optimism/optimism/op-chain-ops/cmd/check-lagoon": 0.21, + "github.com/ethereum-optimism/optimism/op-chain-ops/cmd/check-lagoon/checks": 0.019, + "github.com/ethereum-optimism/optimism/op-chain-ops/cmd/check-lagoon/sdmcheck": 0.268, + "github.com/ethereum-optimism/optimism/op-chain-ops/cmd/check-output-root": 0, + "github.com/ethereum-optimism/optimism/op-chain-ops/cmd/check-prestate": 0, + "github.com/ethereum-optimism/optimism/op-chain-ops/cmd/check-prestate/prestate": 0.028, + "github.com/ethereum-optimism/optimism/op-chain-ops/cmd/check-prestate/registry": 0, + "github.com/ethereum-optimism/optimism/op-chain-ops/cmd/check-prestate/types": 0.05, + "github.com/ethereum-optimism/optimism/op-chain-ops/cmd/check-prestate/util": 0, + "github.com/ethereum-optimism/optimism/op-chain-ops/cmd/check-super-root": 0, + "github.com/ethereum-optimism/optimism/op-chain-ops/cmd/deposit-hash": 0, + "github.com/ethereum-optimism/optimism/op-chain-ops/cmd/gnosis": 0, + "github.com/ethereum-optimism/optimism/op-chain-ops/cmd/interop-smoke": 0, + "github.com/ethereum-optimism/optimism/op-chain-ops/cmd/receipt-reference-builder": 0, + "github.com/ethereum-optimism/optimism/op-chain-ops/cmd/sdm-workload": 0, + "github.com/ethereum-optimism/optimism/op-chain-ops/cmd/unclaimed-credits": 0, + "github.com/ethereum-optimism/optimism/op-chain-ops/cmd/withdrawal": 0, + "github.com/ethereum-optimism/optimism/op-chain-ops/crossdomain": 0, + "github.com/ethereum-optimism/optimism/op-chain-ops/crossdomain/bindings": 0, + "github.com/ethereum-optimism/optimism/op-chain-ops/devkeys": 0, + "github.com/ethereum-optimism/optimism/op-chain-ops/foundry": 0.019, + "github.com/ethereum-optimism/optimism/op-chain-ops/genesis": 0.077, + "github.com/ethereum-optimism/optimism/op-chain-ops/genesis/beacondeposit": 0, + "github.com/ethereum-optimism/optimism/op-chain-ops/interopbridge": 0, + "github.com/ethereum-optimism/optimism/op-chain-ops/interopgen": 0.353, + "github.com/ethereum-optimism/optimism/op-chain-ops/interopgen/config": 0.008, + "github.com/ethereum-optimism/optimism/op-chain-ops/interopsmoke": 0, + "github.com/ethereum-optimism/optimism/op-chain-ops/opcmregistry": 0.007, + "github.com/ethereum-optimism/optimism/op-chain-ops/pkg/sdm": 0.372, + "github.com/ethereum-optimism/optimism/op-chain-ops/script": 0.142, + "github.com/ethereum-optimism/optimism/op-chain-ops/script/addresses": 0, + "github.com/ethereum-optimism/optimism/op-chain-ops/script/consolegen": 0, + "github.com/ethereum-optimism/optimism/op-chain-ops/script/forking": 0, + "github.com/ethereum-optimism/optimism/op-chain-ops/solc": 0.004, + "github.com/ethereum-optimism/optimism/op-chain-ops/srcmap": 0, + "github.com/ethereum-optimism/optimism/op-challenger": 0.022, + "github.com/ethereum-optimism/optimism/op-challenger/cmd": 7.21, + "github.com/ethereum-optimism/optimism/op-challenger/config": 0.005, + "github.com/ethereum-optimism/optimism/op-challenger/flags": 1.2650000000000001, + "github.com/ethereum-optimism/optimism/op-challenger/game": 0.319, + "github.com/ethereum-optimism/optimism/op-challenger/game/client": 0.015, + "github.com/ethereum-optimism/optimism/op-challenger/game/fault": 0.332, + "github.com/ethereum-optimism/optimism/op-challenger/game/fault/claims": 0.052, + "github.com/ethereum-optimism/optimism/op-challenger/game/fault/contracts": 0.462, + "github.com/ethereum-optimism/optimism/op-challenger/game/fault/contracts/gameargs": 0.01, + "github.com/ethereum-optimism/optimism/op-challenger/game/fault/contracts/metrics": 0, + "github.com/ethereum-optimism/optimism/op-challenger/game/fault/preimages": 1.533, + "github.com/ethereum-optimism/optimism/op-challenger/game/fault/responder": 0.01, + "github.com/ethereum-optimism/optimism/op-challenger/game/fault/solver": 1.724, + "github.com/ethereum-optimism/optimism/op-challenger/game/fault/test": 0, + "github.com/ethereum-optimism/optimism/op-challenger/game/fault/trace": 0.172, + "github.com/ethereum-optimism/optimism/op-challenger/game/fault/trace/alphabet": 0, + "github.com/ethereum-optimism/optimism/op-challenger/game/fault/trace/cannon": 7.039, + "github.com/ethereum-optimism/optimism/op-challenger/game/fault/trace/outputs": 0.015, + "github.com/ethereum-optimism/optimism/op-challenger/game/fault/trace/prestates": 0.293, + "github.com/ethereum-optimism/optimism/op-challenger/game/fault/trace/split": 0.278, + "github.com/ethereum-optimism/optimism/op-challenger/game/fault/trace/super": 0.025, + "github.com/ethereum-optimism/optimism/op-challenger/game/fault/trace/utils": 0, + "github.com/ethereum-optimism/optimism/op-challenger/game/fault/trace/vm": 0.083, + "github.com/ethereum-optimism/optimism/op-challenger/game/fault/types": 0.002, + "github.com/ethereum-optimism/optimism/op-challenger/game/generic": 0.018, + "github.com/ethereum-optimism/optimism/op-challenger/game/keccak": 0.026, + "github.com/ethereum-optimism/optimism/op-challenger/game/keccak/fetcher": 0.397, + "github.com/ethereum-optimism/optimism/op-challenger/game/keccak/matrix": 0.211, + "github.com/ethereum-optimism/optimism/op-challenger/game/keccak/merkle": 0, + "github.com/ethereum-optimism/optimism/op-challenger/game/keccak/types": 0, + "github.com/ethereum-optimism/optimism/op-challenger/game/registry": 0, + "github.com/ethereum-optimism/optimism/op-challenger/game/scheduler": 0.313, + "github.com/ethereum-optimism/optimism/op-challenger/game/scheduler/test": 0.121, + "github.com/ethereum-optimism/optimism/op-challenger/game/types": 0.001, + "github.com/ethereum-optimism/optimism/op-challenger/game/zk": 0.017, + "github.com/ethereum-optimism/optimism/op-challenger/kvstore": 0.68, + "github.com/ethereum-optimism/optimism/op-challenger/metrics": 0.049, + "github.com/ethereum-optimism/optimism/op-challenger/runner": 0.116, + "github.com/ethereum-optimism/optimism/op-challenger/sender": 0.011, + "github.com/ethereum-optimism/optimism/op-challenger/tools": 0, + "github.com/ethereum-optimism/optimism/op-challenger/version": 0, + "github.com/ethereum-optimism/optimism/op-conductor/client": 0.014, + "github.com/ethereum-optimism/optimism/op-conductor/client/mocks": 0, + "github.com/ethereum-optimism/optimism/op-conductor/cmd": 0.057, + "github.com/ethereum-optimism/optimism/op-conductor/conductor": 0.95, + "github.com/ethereum-optimism/optimism/op-conductor/consensus": 1.6019999999999999, + "github.com/ethereum-optimism/optimism/op-conductor/consensus/mocks": 0, + "github.com/ethereum-optimism/optimism/op-conductor/flags": 0.014, + "github.com/ethereum-optimism/optimism/op-conductor/health": 12.017, + "github.com/ethereum-optimism/optimism/op-conductor/health/mocks": 0.053, + "github.com/ethereum-optimism/optimism/op-conductor/metrics": 0, + "github.com/ethereum-optimism/optimism/op-conductor/rpc": 0, + "github.com/ethereum-optimism/optimism/op-conductor/rpc/ws": 0, + "github.com/ethereum-optimism/optimism/op-core/devfeatures": 0.174, + "github.com/ethereum-optimism/optimism/op-core/eip1559": 0, + "github.com/ethereum-optimism/optimism/op-core/fees": 0, + "github.com/ethereum-optimism/optimism/op-core/forks": 0.222, + "github.com/ethereum-optimism/optimism/op-core/interop": 0, + "github.com/ethereum-optimism/optimism/op-core/interop/depset": 0, + "github.com/ethereum-optimism/optimism/op-core/interop/messages": 0.81, + "github.com/ethereum-optimism/optimism/op-core/nuts": 0, + "github.com/ethereum-optimism/optimism/op-core/nuts/state": 0, + "github.com/ethereum-optimism/optimism/op-core/params": 0, + "github.com/ethereum-optimism/optimism/op-core/predeploys": 0, + "github.com/ethereum-optimism/optimism/op-core/superchain": 0.528, + "github.com/ethereum-optimism/optimism/op-core/types": 0.001, + "github.com/ethereum-optimism/optimism/op-deployer/cmd/op-deployer": 0, + "github.com/ethereum-optimism/optimism/op-deployer/pkg/cli": 0.156, + "github.com/ethereum-optimism/optimism/op-deployer/pkg/deployer": 22.384, + "github.com/ethereum-optimism/optimism/op-deployer/pkg/deployer/artifacts": 0.019, + "github.com/ethereum-optimism/optimism/op-deployer/pkg/deployer/artifacts/cmd/mktar": 0, + "github.com/ethereum-optimism/optimism/op-deployer/pkg/deployer/bootstrap": 98.839, + "github.com/ethereum-optimism/optimism/op-deployer/pkg/deployer/broadcaster": 0.256, + "github.com/ethereum-optimism/optimism/op-deployer/pkg/deployer/clean": 1.893, + "github.com/ethereum-optimism/optimism/op-deployer/pkg/deployer/flags": 0, + "github.com/ethereum-optimism/optimism/op-deployer/pkg/deployer/inspect": 12.185, + "github.com/ethereum-optimism/optimism/op-deployer/pkg/deployer/integration_test": 479.528, + "github.com/ethereum-optimism/optimism/op-deployer/pkg/deployer/integration_test/cli": 603.179, + "github.com/ethereum-optimism/optimism/op-deployer/pkg/deployer/integration_test/shared": 0, + "github.com/ethereum-optimism/optimism/op-deployer/pkg/deployer/manage": 72.735, + "github.com/ethereum-optimism/optimism/op-deployer/pkg/deployer/opcm": 7.244, + "github.com/ethereum-optimism/optimism/op-deployer/pkg/deployer/pipeline": 69.412, + "github.com/ethereum-optimism/optimism/op-deployer/pkg/deployer/standard": 0.18, + "github.com/ethereum-optimism/optimism/op-deployer/pkg/deployer/state": 0.002, + "github.com/ethereum-optimism/optimism/op-deployer/pkg/deployer/testutil": 0, + "github.com/ethereum-optimism/optimism/op-deployer/pkg/deployer/upgrade": 0.046, + "github.com/ethereum-optimism/optimism/op-deployer/pkg/deployer/upgrade/embedded": 0, + "github.com/ethereum-optimism/optimism/op-deployer/pkg/deployer/validate": 0, + "github.com/ethereum-optimism/optimism/op-deployer/pkg/deployer/verify": 0.62, + "github.com/ethereum-optimism/optimism/op-deployer/pkg/deployer/version": 0, + "github.com/ethereum-optimism/optimism/op-deployer/pkg/env": 0, + "github.com/ethereum-optimism/optimism/op-devstack/devtest": 0.006, + "github.com/ethereum-optimism/optimism/op-devstack/devtest/metrics": 0.434, + "github.com/ethereum-optimism/optimism/op-devstack/dsl": 6.037, + "github.com/ethereum-optimism/optimism/op-devstack/dsl/contract": 0, + "github.com/ethereum-optimism/optimism/op-devstack/dsl/disputemon": 0, + "github.com/ethereum-optimism/optimism/op-devstack/dsl/poller": 0.058, + "github.com/ethereum-optimism/optimism/op-devstack/dsl/proofs": 0.796, + "github.com/ethereum-optimism/optimism/op-devstack/dsl/zkproposer": 0.464, + "github.com/ethereum-optimism/optimism/op-devstack/presets": 0.029, + "github.com/ethereum-optimism/optimism/op-devstack/shared/challenger": 0, + "github.com/ethereum-optimism/optimism/op-devstack/shared/rustbin": 0.142, + "github.com/ethereum-optimism/optimism/op-devstack/stack": 0.02, + "github.com/ethereum-optimism/optimism/op-devstack/sysgo": 0.554, + "github.com/ethereum-optimism/optimism/op-dispute-mon": 0.011, + "github.com/ethereum-optimism/optimism/op-dispute-mon/cmd": 1.263, + "github.com/ethereum-optimism/optimism/op-dispute-mon/config": 0.01, + "github.com/ethereum-optimism/optimism/op-dispute-mon/flags": 0.028, + "github.com/ethereum-optimism/optimism/op-dispute-mon/metrics": 0.229, + "github.com/ethereum-optimism/optimism/op-dispute-mon/mon": 0.451, + "github.com/ethereum-optimism/optimism/op-dispute-mon/mon/bonds": 0.01, + "github.com/ethereum-optimism/optimism/op-dispute-mon/mon/extract": 0.415, + "github.com/ethereum-optimism/optimism/op-dispute-mon/mon/transform": 0, + "github.com/ethereum-optimism/optimism/op-dispute-mon/mon/types": 0.541, + "github.com/ethereum-optimism/optimism/op-dispute-mon/version": 0, + "github.com/ethereum-optimism/optimism/op-dripper/bindings": 0, + "github.com/ethereum-optimism/optimism/op-dripper/cmd": 0.059, + "github.com/ethereum-optimism/optimism/op-dripper/dripper": 0, + "github.com/ethereum-optimism/optimism/op-dripper/flags": 0, + "github.com/ethereum-optimism/optimism/op-dripper/metrics": 0, + "github.com/ethereum-optimism/optimism/op-e2e": 0, + "github.com/ethereum-optimism/optimism/op-e2e/bindings": 0, + "github.com/ethereum-optimism/optimism/op-e2e/bindingspreview": 0, + "github.com/ethereum-optimism/optimism/op-e2e/config": 0.11, + "github.com/ethereum-optimism/optimism/op-e2e/config/secrets": 0, + "github.com/ethereum-optimism/optimism/op-e2e/e2eutils": 3.44, + "github.com/ethereum-optimism/optimism/op-e2e/e2eutils/batcher": 0, + "github.com/ethereum-optimism/optimism/op-e2e/e2eutils/blobstore": 0, + "github.com/ethereum-optimism/optimism/op-e2e/e2eutils/challenger": 0, + "github.com/ethereum-optimism/optimism/op-e2e/e2eutils/contracts/bindings/delegatecallproxy": 0, + "github.com/ethereum-optimism/optimism/op-e2e/e2eutils/contracts/bindings/emit": 0, + "github.com/ethereum-optimism/optimism/op-e2e/e2eutils/contracts/bindings/inbox": 0, + "github.com/ethereum-optimism/optimism/op-e2e/e2eutils/contracts/bindings/invoker": 0, + "github.com/ethereum-optimism/optimism/op-e2e/e2eutils/disputegame": 0.05, + "github.com/ethereum-optimism/optimism/op-e2e/e2eutils/disputegame/preimage": 0, + "github.com/ethereum-optimism/optimism/op-e2e/e2eutils/el": 0.02, + "github.com/ethereum-optimism/optimism/op-e2e/e2eutils/fakebeacon": 1.991, + "github.com/ethereum-optimism/optimism/op-e2e/e2eutils/geth": 0, + "github.com/ethereum-optimism/optimism/op-e2e/e2eutils/intentbuilder": 0.124, + "github.com/ethereum-optimism/optimism/op-e2e/e2eutils/opnode": 0, + "github.com/ethereum-optimism/optimism/op-e2e/e2eutils/receipts": 0, + "github.com/ethereum-optimism/optimism/op-e2e/e2eutils/reth": 0.058, + "github.com/ethereum-optimism/optimism/op-e2e/e2eutils/services": 0, + "github.com/ethereum-optimism/optimism/op-e2e/e2eutils/setuputils": 0, + "github.com/ethereum-optimism/optimism/op-e2e/e2eutils/transactions": 0, + "github.com/ethereum-optimism/optimism/op-e2e/e2eutils/wait": 0, + "github.com/ethereum-optimism/optimism/op-e2e/faultproofs": 186.816, + "github.com/ethereum-optimism/optimism/op-e2e/interop": 4.989, + "github.com/ethereum-optimism/optimism/op-e2e/opgeth": 23.257, + "github.com/ethereum-optimism/optimism/op-e2e/system/altda": 23.777, + "github.com/ethereum-optimism/optimism/op-e2e/system/bridge": 129.561, + "github.com/ethereum-optimism/optimism/op-e2e/system/conductor": 9.803, + "github.com/ethereum-optimism/optimism/op-e2e/system/contracts": 3.16, + "github.com/ethereum-optimism/optimism/op-e2e/system/da": 62.49, + "github.com/ethereum-optimism/optimism/op-e2e/system/e2esys": 0, + "github.com/ethereum-optimism/optimism/op-e2e/system/fees": 37.013, + "github.com/ethereum-optimism/optimism/op-e2e/system/fjord": 0.03, + "github.com/ethereum-optimism/optimism/op-e2e/system/helpers": 0, + "github.com/ethereum-optimism/optimism/op-e2e/system/isthmus": 15.094, + "github.com/ethereum-optimism/optimism/op-e2e/system/p2p": 27.857, + "github.com/ethereum-optimism/optimism/op-e2e/system/proofs": 23.301, + "github.com/ethereum-optimism/optimism/op-e2e/system/runcfg": 19.576999999999998, + "github.com/ethereum-optimism/optimism/op-e2e/system/verifier": 40.945, + "github.com/ethereum-optimism/optimism/op-faucet/cmd": 0, + "github.com/ethereum-optimism/optimism/op-faucet/config": 0, + "github.com/ethereum-optimism/optimism/op-faucet/faucet": 0.043, + "github.com/ethereum-optimism/optimism/op-faucet/faucet/backend": 0.048, + "github.com/ethereum-optimism/optimism/op-faucet/faucet/backend/config": 0.027, + "github.com/ethereum-optimism/optimism/op-faucet/faucet/backend/types": 0, + "github.com/ethereum-optimism/optimism/op-faucet/faucet/frontend": 0, + "github.com/ethereum-optimism/optimism/op-faucet/flags": 0.012, + "github.com/ethereum-optimism/optimism/op-faucet/metrics": 0.009, + "github.com/ethereum-optimism/optimism/op-fetcher/cmd": 0, + "github.com/ethereum-optimism/optimism/op-fetcher/pkg/fetcher/fetch": 0.079, + "github.com/ethereum-optimism/optimism/op-fetcher/pkg/fetcher/fetch/script": 0.075, + "github.com/ethereum-optimism/optimism/op-interop-filter/cmd": 0, + "github.com/ethereum-optimism/optimism/op-interop-filter/filter": 0.227, + "github.com/ethereum-optimism/optimism/op-interop-filter/flags": 0.012, + "github.com/ethereum-optimism/optimism/op-interop-filter/metrics": 0, + "github.com/ethereum-optimism/optimism/op-interop-mon/cmd": 0, + "github.com/ethereum-optimism/optimism/op-interop-mon/flags": 0, + "github.com/ethereum-optimism/optimism/op-interop-mon/metrics": 0, + "github.com/ethereum-optimism/optimism/op-interop-mon/monitor": 0.129, + "github.com/ethereum-optimism/optimism/op-node": 0.017, + "github.com/ethereum-optimism/optimism/op-node/benchmarks": 0.012, + "github.com/ethereum-optimism/optimism/op-node/bindings": 0, + "github.com/ethereum-optimism/optimism/op-node/bindings/preview": 0, + "github.com/ethereum-optimism/optimism/op-node/chaincfg": 0, + "github.com/ethereum-optimism/optimism/op-node/cmd": 0, + "github.com/ethereum-optimism/optimism/op-node/cmd/batch_decoder": 0, + "github.com/ethereum-optimism/optimism/op-node/cmd/batch_decoder/fetch": 0, + "github.com/ethereum-optimism/optimism/op-node/cmd/batch_decoder/reassemble": 0, + "github.com/ethereum-optimism/optimism/op-node/cmd/genesis": 0.012, + "github.com/ethereum-optimism/optimism/op-node/cmd/interop-deposits-dump": 0, + "github.com/ethereum-optimism/optimism/op-node/cmd/networks": 0, + "github.com/ethereum-optimism/optimism/op-node/cmd/p2p": 0, + "github.com/ethereum-optimism/optimism/op-node/config": 0, + "github.com/ethereum-optimism/optimism/op-node/flags": 0.021, + "github.com/ethereum-optimism/optimism/op-node/metrics": 0.02, + "github.com/ethereum-optimism/optimism/op-node/metrics/metered": 0.012, + "github.com/ethereum-optimism/optimism/op-node/node": 0.075, + "github.com/ethereum-optimism/optimism/op-node/node/runcfg": 0.013, + "github.com/ethereum-optimism/optimism/op-node/node/safedb": 0.039, + "github.com/ethereum-optimism/optimism/op-node/node/tracer": 0, + "github.com/ethereum-optimism/optimism/op-node/p2p": 8.135, + "github.com/ethereum-optimism/optimism/op-node/p2p/cli": 0.017, + "github.com/ethereum-optimism/optimism/op-node/p2p/gating": 0.009, + "github.com/ethereum-optimism/optimism/op-node/p2p/gating/mocks": 0, + "github.com/ethereum-optimism/optimism/op-node/p2p/mocks": 0, + "github.com/ethereum-optimism/optimism/op-node/p2p/monitor": 0.006, + "github.com/ethereum-optimism/optimism/op-node/p2p/monitor/mocks": 0, + "github.com/ethereum-optimism/optimism/op-node/p2p/store": 0.008, + "github.com/ethereum-optimism/optimism/op-node/params": 0, + "github.com/ethereum-optimism/optimism/op-node/rollup": 0.22, + "github.com/ethereum-optimism/optimism/op-node/rollup/async": 0.061, + "github.com/ethereum-optimism/optimism/op-node/rollup/attributes": 0.021, + "github.com/ethereum-optimism/optimism/op-node/rollup/conductor": 0, + "github.com/ethereum-optimism/optimism/op-node/rollup/confdepth": 0.012, + "github.com/ethereum-optimism/optimism/op-node/rollup/derive": 21.957, + "github.com/ethereum-optimism/optimism/op-node/rollup/derive/mocks": 0, + "github.com/ethereum-optimism/optimism/op-node/rollup/derive/params": 0, + "github.com/ethereum-optimism/optimism/op-node/rollup/derive/test": 0, + "github.com/ethereum-optimism/optimism/op-node/rollup/driver": 0.023, + "github.com/ethereum-optimism/optimism/op-node/rollup/engine": 0.038, + "github.com/ethereum-optimism/optimism/op-node/rollup/finality": 0.018, + "github.com/ethereum-optimism/optimism/op-node/rollup/sequencing": 7.217, + "github.com/ethereum-optimism/optimism/op-node/rollup/status": 0.013, + "github.com/ethereum-optimism/optimism/op-node/rollup/sync": 30.87, + "github.com/ethereum-optimism/optimism/op-node/rollup/test": 0, + "github.com/ethereum-optimism/optimism/op-node/superchain": 0, + "github.com/ethereum-optimism/optimism/op-node/version": 0, + "github.com/ethereum-optimism/optimism/op-node/withdrawals": 0.022, + "github.com/ethereum-optimism/optimism/op-preimage": 0, + "github.com/ethereum-optimism/optimism/op-proposer/cmd": 0, + "github.com/ethereum-optimism/optimism/op-proposer/contracts": 0.462, + "github.com/ethereum-optimism/optimism/op-proposer/flags": 0.013, + "github.com/ethereum-optimism/optimism/op-proposer/metrics": 0.026, + "github.com/ethereum-optimism/optimism/op-proposer/proposer": 0.016, + "github.com/ethereum-optimism/optimism/op-proposer/proposer/rpc": 0, + "github.com/ethereum-optimism/optimism/op-proposer/proposer/source": 0.014, + "github.com/ethereum-optimism/optimism/op-service": 0, + "github.com/ethereum-optimism/optimism/op-service/accounting": 0, + "github.com/ethereum-optimism/optimism/op-service/apis": 0, + "github.com/ethereum-optimism/optimism/op-service/bgpo": 0.118, + "github.com/ethereum-optimism/optimism/op-service/bigs": 0, + "github.com/ethereum-optimism/optimism/op-service/binary": 0.004, + "github.com/ethereum-optimism/optimism/op-service/buffer": 0, + "github.com/ethereum-optimism/optimism/op-service/cliapp": 0, + "github.com/ethereum-optimism/optimism/op-service/client": 4.129, + "github.com/ethereum-optimism/optimism/op-service/client/mocks": 0, + "github.com/ethereum-optimism/optimism/op-service/cliflags": 0, + "github.com/ethereum-optimism/optimism/op-service/cliiface": 0, + "github.com/ethereum-optimism/optimism/op-service/cliutil": 0, + "github.com/ethereum-optimism/optimism/op-service/clock": 0.68, + "github.com/ethereum-optimism/optimism/op-service/closer": 0, + "github.com/ethereum-optimism/optimism/op-service/crypto": 0.018, + "github.com/ethereum-optimism/optimism/op-service/ctxinterrupt": 0, + "github.com/ethereum-optimism/optimism/op-service/dial": 0.019, + "github.com/ethereum-optimism/optimism/op-service/endpoint": 0, + "github.com/ethereum-optimism/optimism/op-service/enum": 0, + "github.com/ethereum-optimism/optimism/op-service/errutil": 0, + "github.com/ethereum-optimism/optimism/op-service/eth": 1.153, + "github.com/ethereum-optimism/optimism/op-service/eth/safety": 0, + "github.com/ethereum-optimism/optimism/op-service/event": 2.233, + "github.com/ethereum-optimism/optimism/op-service/flags": 0, + "github.com/ethereum-optimism/optimism/op-service/github/release": 0, + "github.com/ethereum-optimism/optimism/op-service/gnosis": 0, + "github.com/ethereum-optimism/optimism/op-service/gnosis/integration_test": 20.481, + "github.com/ethereum-optimism/optimism/op-service/httputil": 0, + "github.com/ethereum-optimism/optimism/op-service/ioutil": 0.015, + "github.com/ethereum-optimism/optimism/op-service/jsonrpc": 0.063, + "github.com/ethereum-optimism/optimism/op-service/jsonutil": 0, + "github.com/ethereum-optimism/optimism/op-service/kzg": 57.211, + "github.com/ethereum-optimism/optimism/op-service/locks": 0, + "github.com/ethereum-optimism/optimism/op-service/log": 0.066, + "github.com/ethereum-optimism/optimism/op-service/log/logcli": 0, + "github.com/ethereum-optimism/optimism/op-service/log/logfilter": 0.007, + "github.com/ethereum-optimism/optimism/op-service/logmods": 0, + "github.com/ethereum-optimism/optimism/op-service/logpipe": 0.005, + "github.com/ethereum-optimism/optimism/op-service/metrics": 0.009, + "github.com/ethereum-optimism/optimism/op-service/metrics/doc": 0, + "github.com/ethereum-optimism/optimism/op-service/metrics/test": 0, + "github.com/ethereum-optimism/optimism/op-service/oppprof": 0, + "github.com/ethereum-optimism/optimism/op-service/plan": 0, + "github.com/ethereum-optimism/optimism/op-service/ptr": 0, + "github.com/ethereum-optimism/optimism/op-service/queue": 0, + "github.com/ethereum-optimism/optimism/op-service/retry": 0, + "github.com/ethereum-optimism/optimism/op-service/rpc": 1.022, + "github.com/ethereum-optimism/optimism/op-service/safego": 0, + "github.com/ethereum-optimism/optimism/op-service/serialize": 0.001, + "github.com/ethereum-optimism/optimism/op-service/signer": 0.078, + "github.com/ethereum-optimism/optimism/op-service/slices": 0, + "github.com/ethereum-optimism/optimism/op-service/solabi": 0, + "github.com/ethereum-optimism/optimism/op-service/sources": 2.835, + "github.com/ethereum-optimism/optimism/op-service/sources/batching": 0.013, + "github.com/ethereum-optimism/optimism/op-service/sources/batching/rpcblock": 0, + "github.com/ethereum-optimism/optimism/op-service/sources/batching/test": 0, + "github.com/ethereum-optimism/optimism/op-service/sources/caching": 0, + "github.com/ethereum-optimism/optimism/op-service/sources/mocks": 0, + "github.com/ethereum-optimism/optimism/op-service/tasks": 0, + "github.com/ethereum-optimism/optimism/op-service/testlog": 0.379, + "github.com/ethereum-optimism/optimism/op-service/testreq": 0, + "github.com/ethereum-optimism/optimism/op-service/testutils": 0, + "github.com/ethereum-optimism/optimism/op-service/testutils/depguard": 0.523, + "github.com/ethereum-optimism/optimism/op-service/testutils/devnet": 0, + "github.com/ethereum-optimism/optimism/op-service/testutils/fuzzerutils": 0, + "github.com/ethereum-optimism/optimism/op-service/testutils/mockrpc": 0, + "github.com/ethereum-optimism/optimism/op-service/testutils/tcpproxy": 0.136, + "github.com/ethereum-optimism/optimism/op-service/tls": 0, + "github.com/ethereum-optimism/optimism/op-service/tls/certman": 0.018, + "github.com/ethereum-optimism/optimism/op-service/tri": 0, + "github.com/ethereum-optimism/optimism/op-service/txinclude": 0, + "github.com/ethereum-optimism/optimism/op-service/txintent": 0.013, + "github.com/ethereum-optimism/optimism/op-service/txintent/bindings": 0.014, + "github.com/ethereum-optimism/optimism/op-service/txintent/contractio": 0, + "github.com/ethereum-optimism/optimism/op-service/txmgr": 18.133, + "github.com/ethereum-optimism/optimism/op-service/txmgr/metrics": 0, + "github.com/ethereum-optimism/optimism/op-service/txmgr/mocks": 0, + "github.com/ethereum-optimism/optimism/op-service/txplan": 0.024, + "github.com/ethereum-optimism/optimism/op-supernode/cmd": 0.019, + "github.com/ethereum-optimism/optimism/op-supernode/config": 0.016, + "github.com/ethereum-optimism/optimism/op-supernode/flags": 0.017, + "github.com/ethereum-optimism/optimism/op-supernode/supernode": 0.026, + "github.com/ethereum-optimism/optimism/op-supernode/supernode/activity": 0, + "github.com/ethereum-optimism/optimism/op-supernode/supernode/activity/heartbeat": 0, + "github.com/ethereum-optimism/optimism/op-supernode/supernode/activity/internal/syncstatus": 0, + "github.com/ethereum-optimism/optimism/op-supernode/supernode/activity/interop": 0.212, + "github.com/ethereum-optimism/optimism/op-supernode/supernode/activity/interop/raftwallogdb": 0.003, + "github.com/ethereum-optimism/optimism/op-supernode/supernode/activity/supernode": 0.018, + "github.com/ethereum-optimism/optimism/op-supernode/supernode/activity/superroot": 0.045, + "github.com/ethereum-optimism/optimism/op-supernode/supernode/chain_container": 37.615, + "github.com/ethereum-optimism/optimism/op-supernode/supernode/chain_container/engine_controller": 28.562, + "github.com/ethereum-optimism/optimism/op-supernode/supernode/chain_container/virtual_node": 0.056, + "github.com/ethereum-optimism/optimism/op-supernode/supernode/resources": 0, + "github.com/ethereum-optimism/optimism/op-sync-tester/cmd": 0.015, + "github.com/ethereum-optimism/optimism/op-sync-tester/config": 0, + "github.com/ethereum-optimism/optimism/op-sync-tester/flags": 0.01, + "github.com/ethereum-optimism/optimism/op-sync-tester/metrics": 0.01, + "github.com/ethereum-optimism/optimism/op-sync-tester/synctester": 0.012, + "github.com/ethereum-optimism/optimism/op-sync-tester/synctester/backend": 0.027, + "github.com/ethereum-optimism/optimism/op-sync-tester/synctester/backend/config": 0.01, + "github.com/ethereum-optimism/optimism/op-sync-tester/synctester/backend/elsync": 0.007, + "github.com/ethereum-optimism/optimism/op-sync-tester/synctester/backend/session": 0, + "github.com/ethereum-optimism/optimism/op-sync-tester/synctester/backend/types": 0.005, + "github.com/ethereum-optimism/optimism/op-sync-tester/synctester/frontend": 0, + "github.com/ethereum-optimism/optimism/op-test-sequencer/cmd": 0.019, + "github.com/ethereum-optimism/optimism/op-test-sequencer/config": 0.017, + "github.com/ethereum-optimism/optimism/op-test-sequencer/flags": 0.015, + "github.com/ethereum-optimism/optimism/op-test-sequencer/metrics": 0, + "github.com/ethereum-optimism/optimism/op-test-sequencer/sequencer": 0.049, + "github.com/ethereum-optimism/optimism/op-test-sequencer/sequencer/backend": 0.008, + "github.com/ethereum-optimism/optimism/op-test-sequencer/sequencer/backend/work": 0.009, + "github.com/ethereum-optimism/optimism/op-test-sequencer/sequencer/backend/work/builders/fakepos": 0, + "github.com/ethereum-optimism/optimism/op-test-sequencer/sequencer/backend/work/builders/noopbuilder": 0.008, + "github.com/ethereum-optimism/optimism/op-test-sequencer/sequencer/backend/work/builders/standardbuilder": 0.013, + "github.com/ethereum-optimism/optimism/op-test-sequencer/sequencer/backend/work/committers/noopcommitter": 0.007, + "github.com/ethereum-optimism/optimism/op-test-sequencer/sequencer/backend/work/committers/standardcommitter": 0.023, + "github.com/ethereum-optimism/optimism/op-test-sequencer/sequencer/backend/work/config": 0.021, + "github.com/ethereum-optimism/optimism/op-test-sequencer/sequencer/backend/work/publishers/nooppublisher": 0.007, + "github.com/ethereum-optimism/optimism/op-test-sequencer/sequencer/backend/work/publishers/standardpublisher": 0.028, + "github.com/ethereum-optimism/optimism/op-test-sequencer/sequencer/backend/work/sequencers/fullseq": 0.008, + "github.com/ethereum-optimism/optimism/op-test-sequencer/sequencer/backend/work/sequencers/noopseq": 0.01, + "github.com/ethereum-optimism/optimism/op-test-sequencer/sequencer/backend/work/signers/localkey": 0.007, + "github.com/ethereum-optimism/optimism/op-test-sequencer/sequencer/backend/work/signers/noopsigner": 0.009, + "github.com/ethereum-optimism/optimism/op-test-sequencer/sequencer/frontend": 0.011, + "github.com/ethereum-optimism/optimism/op-test-sequencer/sequencer/seqtypes": 0.005, + "github.com/ethereum-optimism/optimism/op-up": 8.326, + "github.com/ethereum-optimism/optimism/op-wheel": 0, + "github.com/ethereum-optimism/optimism/op-wheel/cmd": 0, + "github.com/ethereum-optimism/optimism/op-wheel/engine": 0.011, + "github.com/ethereum-optimism/optimism/ops/prestate-reproducibility/prestates": 0, + "github.com/ethereum-optimism/optimism/ops/prestate-reproducibility/prestates/kona-sp1-versions": 0.006, + "github.com/ethereum-optimism/optimism/ops/prestate-reproducibility/prestates/verify": 0, + "github.com/ethereum-optimism/optimism/ops/scripts/check-nut-locks": 0, + "github.com/ethereum-optimism/optimism/ops/scripts/check-op-geth-version": 0.005, + "github.com/ethereum-optimism/optimism/ops/scripts/nut-provenance-verify": 0, + "github.com/ethereum-optimism/optimism/ops/scripts/nut-snapshot-for": 0, + "github.com/ethereum-optimism/optimism/ops/scripts/prefork-state-dump": 0, + "github.com/ethereum-optimism/optimism/packages/contracts-bedrock/scripts/autogen/generate-semver-lock": 0, + "github.com/ethereum-optimism/optimism/packages/contracts-bedrock/scripts/autogen/generate-snapshots": 0, + "github.com/ethereum-optimism/optimism/packages/contracts-bedrock/scripts/check-runner": 0, + "github.com/ethereum-optimism/optimism/packages/contracts-bedrock/scripts/checks/common": 0.008, + "github.com/ethereum-optimism/optimism/packages/contracts-bedrock/scripts/checks/interfaces": 0, + "github.com/ethereum-optimism/optimism/packages/contracts-bedrock/scripts/checks/reinitializer": 0, + "github.com/ethereum-optimism/optimism/packages/contracts-bedrock/scripts/checks/spacers": 0, + "github.com/ethereum-optimism/optimism/packages/contracts-bedrock/scripts/checks/strict-pragma": 0, + "github.com/ethereum-optimism/optimism/packages/contracts-bedrock/scripts/checks/test-validation": 0.007, + "github.com/ethereum-optimism/optimism/packages/contracts-bedrock/scripts/checks/unused-imports": 0.005, + "github.com/ethereum-optimism/optimism/packages/contracts-bedrock/scripts/checks/valid-semver-check": 0, + "github.com/ethereum-optimism/optimism/packages/contracts-bedrock/scripts/go-ffi": 0.001, + "github.com/ethereum-optimism/optimism/packages/contracts-bedrock/snapshots": 0 + } +} diff --git a/ops/ci/runtime/nut-provenance.py b/ops/ci/runtime/nut-provenance.py new file mode 100644 index 00000000000..b29d29e0373 --- /dev/null +++ b/ops/ci/runtime/nut-provenance.py @@ -0,0 +1,65 @@ +#!/usr/bin/env python3 +"""Verify the provenance of every locked NUT bundle. + +CircleCI verifies only the forks whose lock entry changed. This script verifies +every fork in fork_lock.toml. For each fork it runs the verifier that CircleCI +runs, with the Forge and Just versions that mise.toml pins at the commit the +lock records for that fork. The verifier owns every check of the lock entry. + +RWX turns mise auto-install off. `--install-tools` installs those versions in +an earlier task, so the verdict task installs nothing. +""" +import argparse +from pathlib import Path +import subprocess +import sys +import tomllib + +ROOT = Path(__file__).resolve().parents[3] +LOCK = 'op-core/nuts/fork_lock.toml' +TOOLS = 'ops/ci/runtime/nut-tools.sh' +VERIFIER = ['go', 'run', './ops/scripts/nut-provenance-verify'] + + +def generator_pins(root=ROOT): + """Return the Forge and Just versions of each locked fork. + + The versions come from mise.toml at the commit that the lock records for + the fork, not from the mise.toml of the checkout. + """ + pins = {} + for fork, entry in sorted(tomllib.loads((root / LOCK).read_text()).items()): + recorded = subprocess.check_output(['git', 'show', entry['commit'] + ':mise.toml'], cwd=root, text=True) + tools = tomllib.loads(recorded)['tools'] + pins[fork] = {'forge': tools['forge'], 'just': tools['just']} + return pins + + +def install_tools(root=ROOT): + """Install the extra Solidity compiler and each distinct Forge and Just pair.""" + subprocess.run(['bash', TOOLS, 'solc'], cwd=root, check=True) + for forge, just in sorted({(pin['forge'], pin['just']) for pin in generator_pins(root).values()}): + subprocess.run(['bash', TOOLS, 'generators', forge, just], cwd=root, check=True) + + +def verify(root=ROOT): + """Run the verifier for every locked fork. + + Return 0 if every verifier passes. Otherwise stop at the first verifier + that fails and return its exit status. + """ + for fork, pin in generator_pins(root).items(): + argv = ['mise', 'exec', 'forge@' + pin['forge'], 'just@' + pin['just'], '--', *VERIFIER, fork] + status = subprocess.call(argv, cwd=root, stdin=subprocess.DEVNULL) + if status: + return status + return 0 + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--install-tools', action='store_true') + if parser.parse_args().install_tools: + install_tools() + else: + sys.exit(verify()) diff --git a/ops/ci/runtime/nut-tools.sh b/ops/ci/runtime/nut-tools.sh new file mode 100644 index 00000000000..e59bf62b738 --- /dev/null +++ b/ops/ci/runtime/nut-tools.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash +# Install the tools that the historical NUT bundle generators need beyond the +# contract tool layer. nut-provenance.py calls this script. +set -euo pipefail + +retry_install() { + local attempt status + for attempt in 1 2 3 4 5; do + if "$@"; then return 0; else status=$?; fi + if [[ "${attempt}" == 5 ]]; then return "${status}"; fi + sleep "$((2 ** attempt))" + done +} + +case "${1:-}" in + solc) + # The contract tool layer installs the four versions that CircleCI installs. + # The historical generators were also observed to compile with 0.8.30. + # Install it here, so that the verdict task downloads no compiler. + if ! svm which 0.8.30; then + retry_install svm install 0.8.30 + fi + ;; + generators) + if ! [[ "${2:-}" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ && "${3:-}" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + echo "The recorded commit must pin exact Forge and Just versions: '${2:-}' '${3:-}'" >&2 + exit 1 + fi + retry_install mise install "forge@${2}" "just@${3}" + ;; + *) + echo "Usage: $0 solc|generators FORGE_VERSION JUST_VERSION" >&2 + exit 1 + ;; +esac diff --git a/ops/ci/runtime/op-reth-report.py b/ops/ci/runtime/op-reth-report.py new file mode 100644 index 00000000000..4a626aa7809 --- /dev/null +++ b/ops/ci/runtime/op-reth-report.py @@ -0,0 +1,95 @@ +#!/usr/bin/env python3 +"""Prepare the superchain bundle and reject incomplete integration test evidence.""" + +import hashlib +import json +import re +from pathlib import Path +import sys +import xml.etree.ElementTree as ET + + +def digest(path): + with Path(path).open("rb") as source: + return hashlib.file_digest(source, "sha256").hexdigest() + + +def write(directory, name, data): + Path(directory, name).write_text(json.dumps(data, indent=2, sort_keys=True) + "\n") + + +def prepare_superchain(): + directory = Path("rust/op-reth/crates/chainspec/res") + expected = (directory / "superchain-configs.tar.sha256").read_text().split()[0] + if not re.fullmatch(r"[0-9a-f]{64}", expected): + raise ValueError("Invalid committed superchain archive checksum") + archive = directory / "superchain-configs.tar" + if archive.exists() and digest(archive) != expected: + archive.unlink() + + +def integration_report(directory): + directory = Path(directory) + discovery = json.loads((directory / "discovery.json").read_text()) + expected = {} + ignored = set() + for suite_id, suite in discovery["rust-suites"].items(): + for name, case in suite["testcases"].items(): + key = (suite_id, name) + match = case.get("filter-match", {}) + if case["ignored"] is True and match == {"status": "mismatch", "reason": "ignored"}: + ignored.add(key) + elif case["ignored"] is not False or match != {"status": "matches"}: + raise ValueError(f"Unexpected test filter: {key}") + expected[key] = key in ignored + if not expected: + raise ValueError("Empty integration test discovery") + if discovery.get("test-count", len(expected)) != len(expected): + raise ValueError("Integration discovery count differs from its case inventory") + actual = {} + flaky = [] + for case in ET.parse(directory / "junit.xml").iter("testcase"): + key = (case.attrib.get("classname", ""), case.attrib["name"]) + if key in actual: + raise ValueError(f"Duplicate test verdict: {key}") + outcome = "pass" + if case.find("skipped") is not None: + outcome = "skip" + elif case.find("failure") is not None or case.find("error") is not None: + outcome = "fail" + actual[key] = outcome + if case.find("flakyFailure") is not None or case.find("flakyError") is not None: + flaky.append(key) + # Pinned nextest omits ignored tests from JUnit. Their authoritative list + # entry must explicitly say "ignored"; every runnable test still needs one + # verdict. Never infer a skip merely because a verdict is missing. + missing = set(expected) - ignored - set(actual) + extra = set(actual) - set(expected) + summary = {"discovered": len(expected), "reported": len(actual), + "missing": sorted(missing), "extra": sorted(extra), + "outcomes": {s: list(actual.values()).count(s) for s in ("pass", "skip", "fail")}, + "ignored_cases": [{"suite": s, "name": n, "reason": "ignored"} + for s, n in sorted(ignored)], + "retried_cases": sorted(flaky)} + summary["outcomes"]["skip"] += len(ignored - set(actual)) + write(directory, "coverage.json", summary) + if missing or extra: + raise ValueError(f"Incomplete integration evidence: {len(missing)} missing, {len(extra)} extra") + for key, outcome in actual.items(): + if (outcome == "skip") != expected[key]: + raise ValueError(f"Unexpected integration skip status: {key}") + print(json.dumps(summary, sort_keys=True)) + + +def main(args): + operation, *rest = args + if operation == "integration": + integration_report(*rest) + elif operation == "prepare-superchain": + prepare_superchain() + else: + raise ValueError(f"Unknown report operation: {operation}") + + +if __name__ == "__main__": + main(sys.argv[1:]) diff --git a/ops/ci/runtime/op-reth-shadow.sh b/ops/ci/runtime/op-reth-shadow.sh new file mode 100644 index 00000000000..bd107dad0ca --- /dev/null +++ b/ops/ci/runtime/op-reth-shadow.sh @@ -0,0 +1,127 @@ +#!/usr/bin/env bash +# Compile with caches; execute fresh verdicts separately. Called at repo root. +set -euo pipefail +REPO_ROOT="$(pwd)" +REPORT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/op-reth-report.py" +[[ -f rust/Cargo.lock && -d .git || -f rust/Cargo.lock && -f .git ]] || { + echo 'Run op-reth-shadow.sh from the repository root.' >&2 + exit 1 +} +export CARGO_HOME="${REPO_ROOT}/.ci/rust-cache/cargo" +export CARGO_TARGET_DIR="${REPO_ROOT}/rust/target" +export CARGO_INCREMENTAL=0 RUSTC_WRAPPER=sccache +export SCCACHE_DIR="${REPO_ROOT}/.ci/rust-cache/sccache" +export SCCACHE_CACHE_SIZE=10G SCCACHE_IDLE_TIMEOUT=0 SCCACHE_LOG=warn +export SCCACHE_BASEDIRS="${REPO_ROOT}" +job="${1:?Pass a build or verdict job}" +report_dir="${REPO_ROOT}/.ci/op-reth/${job}" +mkdir -p "$report_dir" + +finish() { + local status=$? diagnostics=0 + trap - EXIT + if [[ "$job" == *-build ]]; then + sccache --show-stats --stats-format json > "$report_dir/sccache.json" || diagnostics=$? + sccache --stop-server > "$report_dir/sccache-stop.log" 2>&1 || diagnostics=$? + fi + if [[ "$job" == *-build && "$status" == 0 ]]; then + python3 "$(dirname "$REPORT")/rust-target-cache.py" commit || diagnostics=$? + fi + if [[ "$status" == 0 && "$diagnostics" != 0 ]]; then status=$diagnostics; fi + exit "$status" +} +trap finish EXIT + +if [[ "$job" == *-build ]]; then + case "${TARGET_CACHE_MODE:-keep}" in + keep) ;; + sccache-only) rm -rf "${REPO_ROOT}/rust/target" ;; + *) + echo 'TARGET_CACHE_MODE must be keep or sccache-only.' >&2 + exit 1 + ;; + esac + mkdir -p "$CARGO_HOME" "$SCCACHE_DIR" + sccache --start-server + sccache --zero-stats + # Keep a checksum-matching bundle without touching its mtime. Removing it on + # every build dirties chainspec and forces downstream recompilation/relinking. + # A stale bundle is removed so build.rs regenerates from the pinned submodule. + python3 "$REPORT" prepare-superchain + python3 "$(dirname "$REPORT")/rust-target-cache.py" prepare +fi + +case "$job" in + source) + just update-superchain-registry-submodule + ;; + release-build) + # Like Circle's rust-build, release builds that only feed CI tests skip LTO. + (cd rust && CARGO_PROFILE_RELEASE_LTO=off mold -run cargo build --locked --profile release \ + --package op-reth --package op-reth-test-engine --features default) \ + 2>&1 | tee "$report_dir/build.log" + ;; + integration-build) + (cd rust && mold -run cargo nextest archive --locked -p reth-optimism-node \ + --archive-file "$report_dir/tests.tar.zst") 2>&1 | tee "$report_dir/build.log" + ;; + integration) + # Extract manually to a stable path so JUnit and metadata survive nextest's + # temporary-directory cleanup. Runtime config and source remain from the PR. + unpacked="$report_dir/unpacked" + rm -rf "$unpacked" + mkdir -p "$unpacked" + tar --zstd -xf .ci/op-reth/integration-build/tests.tar.zst -C "$unpacked" + nextest_args=(--binaries-metadata "$unpacked/target/nextest/binaries-metadata.json" + --cargo-metadata "$unpacked/target/nextest/cargo-metadata.json" + --target-dir-remap "$unpacked/target" --workspace-remap "$REPO_ROOT/rust") + # Nextest's report store is relative to the remapped workspace, not the + # binary target-dir remap. Keep the committed config and collect its path. + junit="$REPO_ROOT/rust/target/nextest/default/junit.xml" + rm -f "$junit" + (cd rust && cargo nextest list "${nextest_args[@]}" --message-format json) > "$report_dir/discovery.json" + status=0 + (cd rust && RUST_BACKTRACE=1 cargo nextest run "${nextest_args[@]}") \ + 2>&1 | tee "$report_dir/tests.log" || status=$? + if [[ -s "$junit" ]]; then cp "$junit" "$report_dir/junit.xml"; fi + report_status=0 + python3 "$REPORT" integration "$report_dir" || report_status=$? + # The reusable archive is a producer artifact; verdict artifacts only need + # discovery, original JUnit, coverage and logs, not a second copy of binaries. + rm -rf "$unpacked" + # A reporting failure must never replace the original failing verdict. + if [[ "$status" != 0 ]]; then exit "$status"; fi + exit "$report_status" + ;; + codec-base-build | codec-head-build) + (cd rust && mold -run cargo build --locked --bin op-reth --features dev \ + --manifest-path op-reth/bin/Cargo.toml) 2>&1 | tee "$report_dir/build.log" + cp rust/target/debug/op-reth "$report_dir/op-reth" + ;; + codec-vectors) + rm -rf testdata/micro/compact + .ci/op-reth/codec-base-build/op-reth test-vectors compact --write 2>&1 | tee "$report_dir/generate.log" + ;; + codec) + .ci/op-reth/codec-head-build/op-reth test-vectors compact --read 2>&1 | tee "$report_dir/read.log" + ;; + snapshot-build) + (cd rust && OP_RETH_SYNC_SUPERCHAIN=0 mold -run cargo build --locked \ + -p reth-optimism-chainspec --features superchain-configs) 2>&1 | tee "$report_dir/build.log" + ;; + snapshot) + # The producer always leaves Cargo's recorded env at 0. Switching to 1 is + # declared by build.rs, forcing fresh regeneration even on a content hit. + unset RUSTC_WRAPPER + (cd rust && OP_RETH_SYNC_SUPERCHAIN=1 mold -run cargo build --locked \ + -p reth-optimism-chainspec --features superchain-configs) 2>&1 | tee "$report_dir/regenerate.log" + git diff --exit-code -- \ + rust/op-reth/crates/chainspec/res/superchain-configs.tar.sha256 \ + rust/op-reth/crates/chainspec/src/superchain/chain_specs.rs \ + > "$report_dir/snapshot.diff" + ;; + *) + echo "Unknown op-reth shadow job: $job" >&2 + exit 1 + ;; +esac diff --git a/ops/ci/runtime/pr-gate.py b/ops/ci/runtime/pr-gate.py new file mode 100644 index 00000000000..6033357cd11 --- /dev/null +++ b/ops/ci/runtime/pr-gate.py @@ -0,0 +1,111 @@ +#!/usr/bin/env python3 +"""Report the RWX mirror of a CircleCI gate from the states of its prerequisite tasks. + +ops/ci/runtime/pr-gates.json names, for each CircleCI gate, the jobs that the +gate requires and the RWX tasks that mirror each job. Tasks of one workload +definition form a group. + +`receipt GROUP` runs in the workload. RWX passes it the state of each task of +the group and whether the route selected the workload. The receipt passes if +the workload is selected and every task succeeded, or the workload is not +selected and every task was skipped. + +`status GATE` runs in the coordinator. RWX passes it the state of each receipt +of the gate. The status passes if every receipt succeeded. RWX fails a custom +status check when one of its tasks is skipped, so this task has no `if:`. + +Each command writes its inputs and its result under .ci/pr-gates/ and exits 0 +only on a pass. +""" +import argparse +import json +import os +from pathlib import Path +import shutil +import sys + +ROOT = Path(__file__).resolve().parents[3] +MANIFEST = ROOT / 'ops/ci/runtime/pr-gates.json' +STATES = ('succeeded', 'failed', 'skipped') + + +def variable(prefix, name, state): + """Return the environment variable that carries one engine state of one task.""" + return '_'.join((prefix, name.replace('-', '_'), state)).upper() + + +def flag(name): + value = os.environ[name] + if value not in ('true', 'false'): + raise ValueError(f'{name} is {value!r}; expected true or false') + return value == 'true' + + +def states(prefix, names): + """Return {name: {state: bool}} from the environment. + + Raises ValueError unless each task has exactly one true state. A task that + has not finished has none, so a task that does not wait for it fails here. + """ + result = {} + for name in names: + result[name] = {state: flag(variable(prefix, name, state)) for state in STATES} + if sum(result[name].values()) != 1: + raise ValueError(f'{name} is not in exactly one of the states {", ".join(STATES)}') + return result + + +def gates(): + return json.loads(MANIFEST.read_text())['gates'] + + +def group_tasks(group): + """Return the tasks of a group. A task mirrors one CircleCI job, so no task repeats.""" + tasks = [task for gate in gates().values() for row in gate['dependencies'] if row['group'] == group for task in row['tasks']] + if not tasks or len(tasks) != len(set(tasks)): + raise ValueError(f'pr-gates.json has no task, or a repeated task, for group {group}') + return sorted(tasks) + + +def report(kind, name, record, passed): + """Write the record of one command and return the exit status.""" + directory = ROOT / '.ci/pr-gates' / kind / name + shutil.rmtree(directory, ignore_errors=True) + directory.mkdir(parents=True) + (directory / 'verdict.json').write_text(json.dumps({**record, 'passed': passed}, indent=2, sort_keys=True) + '\n') + return 0 if passed else 1 + + +def receipt(group): + selected = flag('GROUP_SELECTED') + found = states('TASK', group_tasks(group)) + wanted = 'succeeded' if selected else 'skipped' + unexpected = sorted(task for task, row in found.items() if not row[wanted]) + if unexpected: + print(f'{group}: the route {"selected" if selected else "did not select"} this workload, ' + f'but these tasks are not {wanted}: {", ".join(unexpected)}', file=sys.stderr) + return report('groups', group, {'group': group, 'selected': selected, 'tasks': found}, not unexpected) + + +def status(gate): + dependencies = gates()[gate]['dependencies'] + selected = flag('GATE_SELECTED') + found = states('GROUP', sorted({row['group'] for row in dependencies})) + failed = sorted(group for group, row in found.items() if not row['succeeded']) + if failed: + print(f'{gate}: these receipts did not succeed: {", ".join(failed)}', file=sys.stderr) + # A receipt succeeds for a selected workload whose tasks passed, and for an + # unselected workload whose tasks were all skipped. + outcome = 'passed' if selected else 'safe_skip' + record = {'gate': gate, 'selected': selected, 'receipts': found, + 'dependencies': [{**row, 'outcome': outcome if found[row['group']]['succeeded'] else 'failed'} for row in dependencies]} + return report('status', gate, record, not failed) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + commands = parser.add_subparsers(dest='command', required=True) + commands.add_parser('receipt').add_argument('name', metavar='GROUP') + commands.add_parser('status').add_argument('name', metavar='GATE') + arguments = parser.parse_args() + sys.exit({'receipt': receipt, 'status': status}[arguments.command](arguments.name)) diff --git a/ops/ci/runtime/pr-gates.json b/ops/ci/runtime/pr-gates.json new file mode 100644 index 00000000000..b403260e624 --- /dev/null +++ b/ops/ci/runtime/pr-gates.json @@ -0,0 +1,90 @@ +{ + "gates": { + "required-rust-ci": { + "dependencies": [ + {"name": "rust-fmt", "group": "rust-workspace", "tasks": ["rust-fmt"]}, + {"name": "rust-reth-tag-pin", "group": "rust-workspace", "tasks": ["rust-reth-tag-pin"]}, + {"name": "rust-clippy", "group": "rust-workspace", "tasks": ["clippy"]}, + {"name": "rust-deny", "group": "rust-workspace", "tasks": ["deny"]}, + {"name": "rust-typos", "group": "rust-workspace", "tasks": ["typos"]}, + {"name": "rust-zepter", "group": "rust-workspace", "tasks": ["zepter"]}, + {"name": "rust-tests", "group": "rust-workspace", "tasks": ["tests-unit", "tests-compiler"]}, + {"name": "interop-deposits-diff", "group": "rust-workspace", "tasks": ["interop"]}, + {"name": "rust-docs", "group": "rust-workspace", "tasks": ["docs"]}, + {"name": "rust-udeps", "group": "rust-workspace", "tasks": ["udeps"]}, + {"name": "rust-cargo-hack", "group": "rust-workspace", "tasks": ["features-0", "features-1", "features-2", "features-3", "features-4", "features-5", "features-6", "features-7", "features-8", "features-9"]}, + {"name": "rust-build", "group": "rust-workspace", "tasks": ["build"]}, + {"name": "rust-check-no-std", "group": "rust-workspace", "tasks": ["no-std"]}, + {"name": "rust-wasm-unknown", "group": "rust-workspace", "tasks": ["wasm-unknown"]}, + {"name": "rust-wasm-wasi", "group": "rust-workspace", "tasks": ["wasm-wasi"]}, + {"name": "op-reth-compact-codec", "group": "op-reth-rust", "tasks": ["codec"]}, + {"name": "op-reth-integration-tests", "group": "op-reth-rust", "tasks": ["integration"]}, + {"name": "kona-lint-cannon", "group": "rust-workspace", "tasks": ["cannon-lint"]}, + {"name": "kona-build-fpvm-cannon-client", "group": "rust-workspace", "tasks": ["cannon-build"]}, + {"name": "kona-host-client-offline-cannon", "group": "rust-workspace", "tasks": ["cannon-offline"]}, + {"name": "kona-registry-snapshot-check", "group": "rust-workspace", "tasks": ["registry"]}, + {"name": "op-reth-superchain-snapshot-check", "group": "op-reth-rust", "tasks": ["snapshot"]} + ] + }, + "ci-gate": { + "dependencies": [ + {"name": "check-kontrol-build", "group": "main-kontrol", "tasks": ["check"]}, + {"name": "shell-check", "group": "main-checks", "tasks": ["shell-check"]}, + {"name": "shfmt-check", "group": "main-checks", "tasks": ["shfmt-check"]}, + {"name": "go-lint", "group": "main-checks", "tasks": ["go-lint"]}, + {"name": "semgrep-scan-local", "group": "main-checks", "tasks": ["semgrep-scan-local"]}, + {"name": "semgrep-test", "group": "main-checks", "tasks": ["semgrep-test"]}, + {"name": "go-tests", "group": "main-go", "tasks": ["verdict-0", "verdict-1", "verdict-2", "verdict-3", "verdict-4", "verdict-5", "verdict-6", "verdict-7", "verdict-8", "verdict-9", "verdict-10", "verdict-11"]}, + {"name": "go-actions-tests", "group": "main-go", "tasks": ["actions-verdict"]}, + {"name": "memory-all-opn-op-reth-fusaka", "group": "main-acceptance", "tasks": ["verdict-opn-0", "verdict-opn-1", "verdict-opn-2", "verdict-opn-3"]}, + {"name": "memory-all-kona-op-reth-fusaka", "group": "main-acceptance", "tasks": ["verdict-kona-0", "verdict-kona-1", "verdict-kona-2", "verdict-kona-3"]}, + {"name": "kona-build-sp1-elfs", "group": "main-sp1", "tasks": ["check"]}, + {"name": "cannon-go-lint-and-test", "group": "main-cannon", "tasks": ["tests"]}, + {"name": "check-generated-mocks-op-node", "group": "main-checks", "tasks": ["check-generated-mocks-op-node"]}, + {"name": "check-generated-mocks-op-service", "group": "main-checks", "tasks": ["check-generated-mocks-op-service"]}, + {"name": "check-nut-locks", "group": "main-checks", "tasks": ["check-nut-locks"]}, + {"name": "check-nut-prefork-states", "group": "main-nut-prefork", "tasks": ["tests"]}, + {"name": "check-op-geth-version", "group": "main-checks", "tasks": ["check-op-geth-version"]}, + {"name": "opgeth-ratchet", "group": "main-checks", "tasks": ["opgeth-ratchet"]}, + {"name": "op-deployer-forge-version", "group": "main-checks", "tasks": ["op-deployer-forge-version"]}, + {"name": "l2-chains-sync-check", "group": "main-checks", "tasks": ["l2-chains-sync-check"]}, + {"name": "nut-provenance-verify", "group": "main-nut-provenance", "tasks": ["verify"]} + ] + }, + "required-contracts-ci": { + "dependencies": [ + {"name": "contracts-bedrock-tests main", "group": "contracts-suites", "tasks": ["verdict-standard-main"]}, + {"name": "contracts-bedrock-tests CUSTOM_GAS_TOKEN", "group": "contracts-suites", "tasks": ["verdict-standard-CUSTOM_GAS_TOKEN"]}, + {"name": "contracts-bedrock-tests INTEROP", "group": "contracts-suites", "tasks": ["verdict-standard-INTEROP"]}, + {"name": "contracts-bedrock-tests BRIDGE_HOOK", "group": "contracts-suites", "tasks": ["verdict-standard-BRIDGE_HOOK"]}, + {"name": "contracts-bedrock-tests OP_PRIVATE", "group": "contracts-suites", "tasks": ["verdict-standard-OP_PRIVATE"]}, + {"name": "contracts-bedrock-tests-heavy-fuzz-modified main", "group": "contracts-suites", "tasks": ["verdict-modified-main"]}, + {"name": "contracts-bedrock-tests-heavy-fuzz-modified CUSTOM_GAS_TOKEN", "group": "contracts-suites", "tasks": ["verdict-modified-CUSTOM_GAS_TOKEN"]}, + {"name": "contracts-bedrock-tests-heavy-fuzz-modified INTEROP", "group": "contracts-suites", "tasks": ["verdict-modified-INTEROP"]}, + {"name": "contracts-bedrock-tests-heavy-fuzz-modified BRIDGE_HOOK", "group": "contracts-suites", "tasks": ["verdict-modified-BRIDGE_HOOK"]}, + {"name": "contracts-bedrock-tests-heavy-fuzz-modified OP_PRIVATE", "group": "contracts-suites", "tasks": ["verdict-modified-OP_PRIVATE"]}, + {"name": "contracts-bedrock-tests-upgrade op-mainnet main", "group": "contracts-upgrades", "tasks": ["verdict-feature-main"]}, + {"name": "contracts-bedrock-tests-upgrade op-mainnet CUSTOM_GAS_TOKEN", "group": "contracts-upgrades", "tasks": ["verdict-feature-CUSTOM_GAS_TOKEN"]}, + {"name": "contracts-bedrock-tests-upgrade op-mainnet INTEROP", "group": "contracts-upgrades", "tasks": ["verdict-feature-INTEROP"]}, + {"name": "contracts-bedrock-tests-upgrade op-mainnet BRIDGE_HOOK", "group": "contracts-upgrades", "tasks": ["verdict-feature-BRIDGE_HOOK"]}, + {"name": "contracts-bedrock-tests-upgrade op-mainnet OP_PRIVATE", "group": "contracts-upgrades", "tasks": ["verdict-feature-OP_PRIVATE"]}, + {"name": "contracts-bedrock-tests-upgrade op-mainnet", "group": "contracts-upgrades", "tasks": ["verdict-chain-op"]}, + {"name": "contracts-bedrock-tests-upgrade ink-mainnet", "group": "contracts-upgrades", "tasks": ["verdict-chain-ink"]}, + {"name": "contracts-bedrock-tests-upgrade unichain-mainnet", "group": "contracts-upgrades", "tasks": ["verdict-chain-unichain"]}, + {"name": "contracts-bedrock-tests-l2-fork op-mainnet", "group": "contracts-l2-fork", "tasks": ["verdict"]}, + {"name": "contracts-bedrock-checks-fast-feature-tests", "group": "contracts-checks", "tasks": ["contracts-fast"]} + ] + }, + "required-rust-e2e": { + "dependencies": [ + {"name": "rust-e2e-binaries", "group": "rust-e2e", "tasks": ["binaries"]}, + {"name": "rust-workspace-release", "group": "rust-e2e", "tasks": ["release"]}, + {"name": "rust-e2e-simple-kona", "group": "rust-e2e", "tasks": ["verdict-simple-kona-0"]}, + {"name": "rust-e2e-simple-kona-sequencer", "group": "rust-e2e", "tasks": ["verdict-simple-kona-sequencer-0"]}, + {"name": "rust-e2e-restart", "group": "rust-e2e", "tasks": ["verdict-restart-0", "verdict-restart-1", "verdict-restart-2"]}, + {"name": "op-reth-e2e-sysgo-tests", "group": "rust-e2e", "tasks": ["verdict-op-reth-0"]}, + {"name": "kona-proof-action-single", "group": "rust-e2e", "tasks": ["verdict-proof-0", "verdict-proof-1", "verdict-proof-2", "verdict-proof-3", "verdict-proof-4", "verdict-proof-5", "verdict-proof-6", "verdict-proof-7"]} + ] + } + } +} diff --git a/ops/ci/runtime/rpc-pacer.py b/ops/ci/runtime/rpc-pacer.py new file mode 100644 index 00000000000..846ead3a6d1 --- /dev/null +++ b/ops/ci/runtime/rpc-pacer.py @@ -0,0 +1,61 @@ +#!/usr/bin/env python3 +"""Forward JSON-RPC requests to one upstream URL at a bounded rate. + +The public OP Mainnet RPC limits requests per IP address. At forge's own rate it answers +HTTP 429 for longer than forge retries, and the fork tests fail in setUp. The pacer starts +at most RATE requests per second. It does not retry: the client receives each upstream +status and body unchanged, and owns the retries. +""" +from contextlib import contextmanager +import http.client +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +import threading +import time +import urllib.error +import urllib.request + +# A hosted run made the 420 requests of the fork tests at this rate without one HTTP 429. +RATE = 2 +FORWARDED = ('Content-Type', 'User-Agent') + + +@contextmanager +def serve(upstream, rate=RATE, clock=time.monotonic, wait=time.sleep): + """Yield the URL of a local endpoint that forwards each POST body to `upstream`.""" + lock = threading.Lock() + following = [clock()] + + def pace(): + # Each request takes the next start time, so concurrent clients share one budget. + with lock: + start = max(following[0], clock()) + following[0] = start + 1 / rate + wait(max(0, start - clock())) + + class Handler(BaseHTTPRequestHandler): + def do_POST(self): + body = self.rfile.read(int(self.headers['Content-Length'])) + request = urllib.request.Request(upstream, data=body, headers={ + name: self.headers[name] for name in FORWARDED if self.headers[name]}) + pace() + try: + with urllib.request.urlopen(request, timeout=30) as response: + status, answer = response.status, response.read() + except urllib.error.HTTPError as failure: + with failure: status, answer = failure.code, failure.read() + # No answer from the upstream. Forge retries HTTP 503. + except (OSError, http.client.HTTPException): status, answer = 503, b'' + self.send_response(status) + self.send_header('Content-Type', 'application/json') + self.send_header('Content-Length', str(len(answer))) + self.end_headers() + self.wfile.write(answer) + + def log_message(self, *_): pass + + server = ThreadingHTTPServer(('127.0.0.1', 0), Handler) + worker = threading.Thread(target=server.serve_forever, daemon=True) + worker.start() + try: yield 'http://127.0.0.1:' + str(server.server_port) + finally: + server.shutdown(); server.server_close(); worker.join() diff --git a/ops/ci/runtime/rust-cannon-report.py b/ops/ci/runtime/rust-cannon-report.py new file mode 100644 index 00000000000..ffb6af9b3d6 --- /dev/null +++ b/ops/ci/runtime/rust-cannon-report.py @@ -0,0 +1,106 @@ +#!/usr/bin/env python3 +"""Record the settings and exit status of one Cannon job, and check the pinned witness.""" +import hashlib +import json +import os +from pathlib import Path +import re +import subprocess +import sys +import time +import xml.etree.ElementTree as ET + +CONFIG = Path('ops/ci/runtime/cannon-witness.json') +DOCKERFILE = Path('rust/kona/docker/fpvm-prestates/cannon-repro.dockerfile') +# The JUnit case of each job. The witness and offline jobs name their case after the witness. +CASES = {'env': 'build/kona-cannon-env', 'go': 'go/cannon', 'lint': 'clippy/kona-std-fpvm', + 'build': 'build/kona-client'} + + +def write(path, data): + path.write_text(json.dumps(data, indent=2, sort_keys=True) + '\n') + + +def read(path): + return json.loads(path.read_text()) + + +def digest(path): + with Path(path).open('rb') as source: + return hashlib.file_digest(source, 'sha256').hexdigest() + + +def command(*args, cwd=None): + return subprocess.check_output(args, cwd=cwd, text=True).strip() + + +def witness_config(): + data = read(CONFIG) + if not re.fullmatch('[0-9a-f]{64}', data['sha256']) or Path(data['filename']).name != data['filename']: + raise ValueError('Invalid witness archive pin') + if not data['release_url'].startswith('https://github.com/ethereum-optimism/chain-test-data/releases/download/'): + raise ValueError('Witness must use a pinned chain-test-data release') + for key in ('l2_claim', 'l2_output_root', 'l2_head', 'l1_head'): + if not re.fullmatch('0x[0-9a-f]{64}', data[key]): + raise ValueError('Invalid witness boundary hash') + for key in ('block_number', 'l2_chain_id'): + if not re.fullmatch('[1-9][0-9]*', data[key]): + raise ValueError('Invalid witness chain or block') + return data + + +def begin(directory, job): + settings = {'suite': 'rust-cannon', 'job': job, 'source_sha': command('git', 'rev-parse', 'HEAD'), + 'rwx_run_id': os.environ.get('RWX_RUN_ID'), + 'rwx_task_attempt': os.environ.get('RWX_TASK_ATTEMPT_NUMBER'), + 'rustc': command('rustc', '--version'), 'cargo': command('cargo', '--version'), + 'started_at': time.time()} + if job in ('go', 'offline'): + settings['go'] = command('go', 'version') + if job in ('witness', 'offline'): + settings['witness'] = witness_config() + write(directory / 'settings.json', settings) + + +def check_witness(): + """Fail unless the fetched witness archive has the pinned checksum.""" + data = witness_config() + if digest(Path('rust/kona/bin/client/testdata') / data['filename']) != data['sha256']: + raise ValueError('Pinned Cannon witness archive checksum mismatch') + + +def finish(directory, status): + """Write the job's exit status and stage records. The recipes own the verdict.""" + settings = read(directory / 'settings.json') + job = settings['job'] + stages = {p.name.removesuffix('.stage.json'): read(p) for p in directory.glob('*.stage.json')} + if job == 'witness': + name = 'archive/' + settings['witness']['filename'] + elif job == 'offline': + name = 'kona-client/' + settings['witness']['l2_chain_id'] + '/' + settings['witness']['block_number'] + else: + name = CASES[job] + suite = ET.Element('testsuite', name='cannon-' + job, tests='1') + case = ET.SubElement(suite, 'testcase', classname='cannon-' + job, name=name) + if status: + ET.SubElement(case, 'failure', message='See original Cannon stages and final.json') + ET.ElementTree(suite).write(directory / 'checks.junit.xml', encoding='utf-8', xml_declaration=True) + write(directory / 'final.json', {'exit_code': status, 'stages': stages}) + return status + + +if __name__ == '__main__': + mode = sys.argv[1] + if mode == 'config': + config = witness_config() + keys = ['block_number', 'l2_claim', 'l2_output_root', 'l2_head', 'l1_head', 'l2_chain_id', 'filename', 'release_url'] + print('\n'.join(config[k] for k in keys)) + elif mode == 'base': + print(re.search(r'^FROM (\S+) AS kona-build-env$', DOCKERFILE.read_text(), re.M).group(1)) + elif mode == 'witness': + check_witness() + else: + directory = Path(sys.argv[2]).resolve() + if mode == 'begin': begin(directory, sys.argv[3]) + elif mode == 'finish': sys.exit(finish(directory, int(sys.argv[3]))) + else: raise ValueError('Unknown Cannon report mode') diff --git a/ops/ci/runtime/rust-cannon.sh b/ops/ci/runtime/rust-cannon.sh new file mode 100644 index 00000000000..da4589dcda6 --- /dev/null +++ b/ops/ci/runtime/rust-cannon.sh @@ -0,0 +1,89 @@ +#!/usr/bin/env bash +# Run one of Circle's Cannon recipes in an RWX task. A task receives the output +# of its producer tasks through the `use` layers that .rwx/rust.yml gives it. +set -euo pipefail +HELPERS="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT="$(git rev-parse --show-toplevel)" +cd "$ROOT" +job="${1:?Pass env, go, witness, lint, build or offline}" +case "$job" in env | go | witness | lint | build | offline) ;; +*) + echo "Unknown Cannon job: $job" >&2 + exit 1 + ;; +esac +report="$ROOT/.ci/rust-workspace/cannon-$job" +rm -rf "$report" +mkdir -p "$report" rust/target +python3 "$HELPERS/rust-cannon-report.py" begin "$report" "$job" +finish() { + local status=$? diagnostics=0 + trap - EXIT + if [[ "$job" == offline ]]; then + sccache --show-stats --stats-format json > "$report/sccache.json" || diagnostics=$? + sccache --stop-server > "$report/sccache-stop.log" 2>&1 || diagnostics=$? + fi + if [[ ("$job" == env || "$job" == lint || "$job" == build || "$job" == offline) && "$status" == 0 ]]; then + python3 "$HELPERS/rust-target-cache.py" commit || diagnostics=$? + fi + if [[ "$status" == 0 && "$diagnostics" != 0 ]]; then status=$diagnostics; fi + python3 "$HELPERS/rust-cannon-report.py" finish "$report" "$status" || status=$? + exit "$status" +} +trap finish EXIT +trap 'exit 130' INT +trap 'exit 143' TERM +stage_at() { python3 "$HELPERS/rust-workspace-report.py" stage-at "$report" "$@"; } +export CARGO_HOME="$ROOT/.ci/rust-cache/cargo" CARGO_TARGET_DIR="$ROOT/rust/target" +export GOPATH="$ROOT/.ci/cannon-cache/go/modules" GOCACHE="$ROOT/.ci/cannon-cache/go/build" +mkdir -p "$CARGO_HOME" "$GOPATH" "$GOCACHE" +if [[ "$job" == env || "$job" == lint || "$job" == build || "$job" == offline ]]; then + # Stamp COPY inputs before Docker builds as well as bound runtime sources. + # Normalized checkout mtimes must not make old BuildKit Cargo targets fresh. + python3 "$HELPERS/rust-target-cache.py" prepare > "$report/cache-source.json" +fi +if [[ "$job" == offline ]]; then + export CARGO_INCREMENTAL=0 RUSTC_WRAPPER=sccache SCCACHE_DIR="$ROOT/.ci/cannon-cache/sccache" + export SCCACHE_BASEDIRS="$ROOT" SCCACHE_CACHE_SIZE=10G SCCACHE_IDLE_TIMEOUT=0 + mkdir -p "$SCCACHE_DIR" + sccache --start-server + sccache --zero-stats +fi +config=() +if [[ "$job" == witness || "$job" == offline ]]; then + while IFS= read -r value; do config+=("$value"); done < <(python3 "$HELPERS/rust-cannon-report.py" config) + if [[ "${#config[@]}" != 8 ]]; then + echo 'Missing Cannon witness configuration.' >&2 + exit 1 + fi +fi +pull_base() { + local base attempt=0 + base="$(python3 "$HELPERS/rust-cannon-report.py" base)" + # Retry only a registry fetch. Compilation/test failures keep their first exit. + until docker pull "$base"; do + attempt=$((attempt + 1)) + [[ "$attempt" -lt 5 ]] || return 1 + sleep "$((2 ** attempt))" + done +} +case "$job" in + env) + export -f pull_base + export HELPERS + stage_at . base-pull bash -c pull_base + stage_at rust/kona env just build-kona-env + ;; + go) stage_at cannon go just cannon ;; + witness) + stage_at . witness bash rust/kona/bin/client/scripts/fetch-witness-tar.sh "${config[6]}" "${config[7]}" + python3 "$HELPERS/rust-cannon-report.py" witness + ;; + lint) stage_at rust/kona lint just lint-cannon ;; + build) stage_at rust/kona build just build-cannon-client ;; + offline) + # The recipe runs the Cannon binary that the cannon-go task built. + export PATH="$ROOT/cannon/bin:$PATH" + stage_at rust/kona/bin/client offline just run-client-cannon-offline "${config[@]}" + ;; +esac diff --git a/ops/ci/runtime/rust-compiler-baseline.py b/ops/ci/runtime/rust-compiler-baseline.py new file mode 100644 index 00000000000..61085481261 --- /dev/null +++ b/ops/ci/runtime/rust-compiler-baseline.py @@ -0,0 +1,41 @@ +#!/usr/bin/env python3 +"""Prepare an optional ancestor compiler cache; never provide a HEAD verdict.""" +import json +import os +from pathlib import Path +import shutil +import subprocess +import sys + + +def prepare(job, root): + if job not in ('tests-build', 'features'): + raise ValueError('Expected tests-build or features') + index = os.environ.get('CI_RUST_PARTITION_INDEX', '0') + name = 'features-' + index if job == 'features' else job + report = root / '.ci/rust-workspace' / name + result = subprocess.run(['bash', str(root / 'ops/ci/runtime/rust-workspace.sh'), job], cwd=root, + env={**os.environ, 'CI_RUST_BASELINE': 'true'}) + # An intermediate ancestor can be broken even though HEAD fixes it. Retain + # its original failure and discard partial targets, then let fresh HEAD + # workers compile normally. A fallback is never reported as a warm cache. + evidence = root / '.ci/rust-workspace/compiler-baseline' / name + evidence.mkdir(parents=True, exist_ok=True) + if report.exists(): + for path in report.iterdir(): + if path.is_file() and path.suffix in ('.json', '.log', '.xml'): + shutil.copyfile(path, evidence / path.name) + target = root / 'rust/target' + if result.returncode and target.exists(): + # Retain originals even if cleanup fails, but never claim a clean + # fallback or allow downstream compilation with failed cleanup. + shutil.rmtree(target) + (evidence / 'compiler-baseline.json').write_text(json.dumps({ + 'source_sha': subprocess.check_output(['git', 'rev-parse', 'HEAD'], cwd=root, text=True).strip(), + 'build_exit_code': result.returncode, 'usable': result.returncode == 0, + 'fallback': 'none' if result.returncode == 0 else 'clean HEAD compilation', + }, indent=2) + '\n') + + +if __name__ == '__main__': + prepare(sys.argv[1], Path.cwd()) diff --git a/ops/ci/runtime/rust-dependencies.py b/ops/ci/runtime/rust-dependencies.py new file mode 100644 index 00000000000..b0d4926861a --- /dev/null +++ b/ops/ci/runtime/rust-dependencies.py @@ -0,0 +1,43 @@ +#!/usr/bin/env python3 +"""Fetch immutable ancestor dependencies as an optional compiler-cache hint.""" +import hashlib +import json +import os +from pathlib import Path +import subprocess +import time + + +def fetch(root): + report = root / '.ci/rust-dependencies' + report.mkdir(parents=True, exist_ok=True) + home = root / '.ci/rust-cache/cargo' + home.mkdir(parents=True, exist_ok=True) + attempts = [] + with (report / 'fetch.log').open('wb') as log: + for attempt in range(1, 6): + started = time.monotonic() + result = subprocess.run(['cargo', 'fetch', '--locked'], cwd=root / 'rust', + env={**os.environ, 'CARGO_HOME': str(home)}, + stdout=log, stderr=subprocess.STDOUT) + attempts.append({'attempt': attempt, 'exit_code': result.returncode, + 'elapsed_seconds': time.monotonic() - started}) + if result.returncode == 0: + break + if attempt < 5: + time.sleep(2 ** attempt) + # A broken intermediate manifest must not block a valid HEAD. Cargo's fresh + # HEAD commands still fetch/verify missing dependencies and own the verdict. + usable = attempts[-1]['exit_code'] == 0 + evidence = {'source_sha': subprocess.check_output(['git', 'rev-parse', 'HEAD'], cwd=root, text=True).strip(), + 'lock_sha256': hashlib.sha256((root / 'rust/Cargo.lock').read_bytes()).hexdigest(), + 'usable': usable, 'attempts': attempts, + 'original_log_sha256': hashlib.sha256((report / 'fetch.log').read_bytes()).hexdigest()} + (report / 'result.json').write_text(json.dumps(evidence, indent=2) + '\n') + print(json.dumps(evidence)) + if not usable: + print('Ancestor dependency prefetch unavailable; fresh HEAD Cargo commands must fetch dependencies.') + + +if __name__ == '__main__': + fetch(Path.cwd()) diff --git a/ops/ci/runtime/rust-e2e-release.sh b/ops/ci/runtime/rust-e2e-release.sh new file mode 100644 index 00000000000..f9df9f1841a --- /dev/null +++ b/ops/ci/runtime/rust-e2e-release.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash +# Build one half of the default-feature release workspace, split as CircleCI splits it: +# binaries rust-e2e-binaries: every library, and the four binaries that the E2E +# verdict tasks run. Packs those four binaries. +# workspace rust-workspace-release: every library, and every other binary. +# Both halves select the whole workspace, so features unify as in one +# `cargo build --workspace`. The verdict tasks wait only for the first half. +set -euo pipefail +cd "$(git rev-parse --show-toplevel)" +part="${1:?Pass binaries or workspace}" +runtime=(op-reth op-reth-test-engine kona-node kona-host) +export CI_COMMIT_SHA="${CI_COMMIT_SHA:?Missing source SHA}" +export CARGO_HOME="$PWD/.ci/rust-cache/cargo" CARGO_TARGET_DIR="$PWD/rust/target" +export CARGO_INCREMENTAL=0 RUSTC_WRAPPER=sccache SCCACHE_DIR="$PWD/.ci/rust-cache/sccache" +export SCCACHE_BASEDIRS="$PWD" SCCACHE_CACHE_SIZE=10G SCCACHE_IDLE_TIMEOUT=0 +mkdir -p "$CARGO_HOME" "$CARGO_TARGET_DIR" "$SCCACHE_DIR" .ci/go-tests/dependencies/rust-e2e-release +targets=(--lib) +case "$part" in + binaries) + for name in "${runtime[@]}"; do targets+=(--bin "$name"); done + ;; + workspace) + # Assignments, so that a failure of either command stops the build. + metadata="$(cd rust && cargo metadata --no-deps --format-version 1 --locked)" + names="$(python3 -c 'import json, sys +for package in json.load(sys.stdin)["packages"]: + for target in package["targets"]: + if "bin" in target["kind"]: print(target["name"])' <<< "$metadata")" + for name in $names; do + case " ${runtime[*]} " in + *" $name "*) ;; + *) targets+=(--bin "$name") ;; + esac + done + ;; + *) + echo 'Expected binaries or workspace' >&2 + exit 1 + ;; +esac +sccache --start-server +sccache --zero-stats +trap 'sccache --show-stats --stats-format json >.ci/go-tests/dependencies/rust-e2e-release/sccache.json; sccache --stop-server' EXIT +python3 ops/ci/runtime/op-reth-report.py prepare-superchain +python3 ops/ci/runtime/rust-target-cache.py prepare +# Like Circle's rust-build: --locked, and no LTO for release builds that only feed CI tests. +(cd rust && CARGO_PROFILE_RELEASE_LTO=off mold -run cargo build --locked --profile release --workspace --features default "${targets[@]}") +python3 ops/ci/runtime/rust-target-cache.py commit +if [[ "$part" == binaries ]]; then + python3 ops/ci/runtime/go-artifacts.py pack rust-e2e-release "${runtime[@]/#/rust/target/release/}" +fi diff --git a/ops/ci/runtime/rust-e2e.py b/ops/ci/runtime/rust-e2e.py new file mode 100644 index 00000000000..40236c692c6 --- /dev/null +++ b/ops/ci/runtime/rust-e2e.py @@ -0,0 +1,182 @@ +#!/usr/bin/env python3 +"""Compile, discover and freshly run the complete Rust E2E Go workloads.""" +import argparse +from collections import Counter +import importlib.util +import os +from pathlib import Path +import re +import shutil +import signal +import sys + +ROOT = Path(__file__).resolve().parents[3] +PREFIX = 'github.com/ethereum-optimism/optimism/' + + +def helper(name): + spec = importlib.util.spec_from_file_location(name.replace('-', '_'), Path(__file__).with_name(name + '.py')) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +GO = helper('go-compiled-tests') +PROJECT = helper('go-report') +REPORT = helper('ci-report') +# One entry per CircleCI E2E job. Each comment names the job and the recipe that it runs. +# RWX runs the compiled suite, not the recipe. `package` and `timeout` copy that recipe, +# `shards` copies the job's `parallelism`, and environment() copies the recipe's exports. +JOBS = { + # kona-proof-action-single. rust/kona: just action-tests-single-run + 'proof': {'package': 'rust/kona/tests/proofs', 'shards': 8, 'timeout': '60m'}, + # rust-e2e-restart. rust/kona: just test-e2e-sysgo-run node node/restart + 'restart': {'package': 'rust/kona/tests/node/restart', 'shards': 3, 'timeout': '40m'}, + # rust-e2e-simple-kona. rust/kona: just test-e2e-sysgo-run node node/common simple-kona + 'simple-kona': {'package': 'rust/kona/tests/node/common', 'shards': 1, 'timeout': '40m'}, + # rust-e2e-simple-kona-sequencer. rust/kona: just test-e2e-sysgo-run node node/common simple-kona-sequencer + 'simple-kona-sequencer': {'package': 'rust/kona/tests/node/common', 'shards': 1, 'timeout': '40m'}, + # op-reth-e2e-sysgo-tests. rust/op-reth/tests: just test-e2e-sysgo + 'op-reth': {'package': 'rust/op-reth/tests/proofs/core', 'shards': 1, 'timeout': '40m'}, +} + + +read = REPORT.read + + +write = REPORT.write + + +def environment(job): + values = {'DISABLE_OP_E2E_LEGACY': 'true', 'DEVSTACK_ORCHESTRATOR': 'sysgo'} if job != 'proof' else {} + if job == 'proof': + values.update(KONA_HOST_PATH=str(ROOT / 'rust/target/release/kona-host'), + RUST_BINARY_PATH_OP_RETH_TEST_ENGINE=str(ROOT / 'rust/target/release/op-reth-test-engine')) + elif job == 'op-reth': + values.update(OP_RETH_ENABLE_PROOF_HISTORY='true', + RUST_BINARY_PATH_OP_RETH=str(ROOT / 'rust/target/release/op-reth')) + else: + sequencer = job == 'simple-kona-sequencer' + values.update(KONA_SEQUENCER_WITH_RETH=str(int(sequencer)), KONA_VALIDATOR_WITH_RETH='1', + OP_SEQUENCER_WITH_RETH=str(int(not sequencer)), OP_VALIDATOR_WITH_RETH=str(int(not sequencer)), + RUST_BINARY_PATH_OP_RETH=str(ROOT / 'rust/target/release/op-reth'), + RUST_BINARY_PATH_KONA_NODE=str(ROOT / 'rust/target/release/kona-node')) + return values + + +def partition(names, total): + """Assign each name to exactly one of `total` shards, round-robin in the given order.""" + return [names[index::total] for index in range(total)] + + +def discovery(job, directory): + """Return the tests that `job` runs and the listed names that it leaves out. + + The compiled suite wrote its test list to `listing.log`. Raise ValueError + when a name is listed twice or when no test remains. + """ + names = [line for line in (directory / 'listing.log').read_text().splitlines() + if re.fullmatch(r'(Test|Example|Fuzz)\w*', line)] + if len(set(names)) != len(names): raise ValueError('Duplicate E2E listed test') + # Original Circle sharding keeps ^Test names only. Unsharded packages run + # examples and fuzz seeds too; record that distinction instead of hiding it. + excluded = [n for n in names if JOBS[job]['shards'] > 1 and not n.startswith('Test')] + selected = sorted(set(names) - set(excluded)) + if not selected: raise ValueError('Empty E2E test discovery') + return selected, excluded + + +def compile_suite(job): + directory = ROOT / '.ci/rust-e2e/compiled' / job + shutil.rmtree(directory, ignore_errors=True); directory.mkdir(parents=True) + package = JOBS[job]['package'] + for name, command, cwd in ( + ('compile', ['go', 'test', '-c', '-o', str(directory / 'suite.test'), './' + package], ROOT), + ('reporter', ['go', 'build', '-o', str(directory / 'test2json'), 'cmd/test2json'], ROOT), + ('listing', [str(directory / 'suite.test'), '-test.list=.', '-test.timeout=5m'], ROOT / package)): + status = REPORT.stage(directory, name, command, cwd=str(cwd)) + if status: return status + names, excluded = discovery(job, directory) + write(directory / 'manifest.json', {'tests': names, 'excluded_listing': excluded, + 'shards': partition(names, JOBS[job]['shards'])}) + return 0 + + +def parallel_flag(job): + # Only the proof recipe passes -parallel. The node and reth recipes keep Go's default. + return int(os.environ['PARALLEL']) if job == 'proof' else None + + +def select(directory, reports, index): + shards = read(directory / 'manifest.json')['shards'] + if not 0 <= index < len(shards): raise ValueError('Invalid E2E shard index') + write(reports / 'selection.json', {'shard_index': index, 'assigned_tests': shards[index]}) + (reports / 'assigned.txt').write_text(''.join(name + '\n' for name in shards[index])) + + +def execute(job, directory, names): + """Run `names` from the compiled suite in one fresh process and print its test2json events.""" + package = JOBS[job]['package'] + pattern = '^(' + '|'.join(re.escape(n) for n in names) + ')$' + return GO.execute_binary(directory / 'test2json', directory / 'suite.test', PREFIX + package, + ROOT / package, parallel_flag(job), JOBS[job]['timeout'], pattern) + + +def report(job, reports, status): + errors, cases, terminals = [], {}, Counter() + record = read(reports / 'selection.json') if (reports / 'selection.json').exists() else None + logs = sorted((reports / 'events').glob('*.json')) + combined = reports / 'original.json' + combined.write_bytes(b''.join(p.read_bytes() for p in logs)) + try: + if record is None: raise ValueError('Missing E2E selection') + if not logs and record['assigned_tests']: raise ValueError('Missing original E2E events') + for event in PROJECT.events(combined): + if event.get('Package') != PREFIX + JOBS[job]['package']: raise ValueError('Unexpected E2E event package') + name, action = event.get('Test'), event['Action'] + if name and action in ('pass', 'skip', 'fail'): + if '/' not in name: terminals[name] += 1 + if name in cases: raise ValueError('Duplicate E2E case verdict; retries are disabled') + cases[name] = {'outcome': action, 'elapsed': event.get('Elapsed')} + coverage = {'assigned': record['assigned_tests'], 'missing': sorted(set(record['assigned_tests']) - terminals.keys()), + 'extra': sorted(terminals.keys() - set(record['assigned_tests'])), + 'duplicates': sorted(n for n, count in terminals.items() if count != 1), 'cases': cases} + write(reports / 'coverage.json', coverage) + if coverage['missing'] or coverage['extra'] or coverage['duplicates']: raise ValueError('Incomplete or duplicate E2E test verdicts') + if status == 0 and any(c['outcome'] == 'fail' for c in cases.values()): raise ValueError('Successful E2E command contains failed verdicts') + PROJECT.project(combined, reports / 'native.json') + except (ValueError, OSError, KeyError) as error: errors.append(str(error)) + # A reporting failure never replaces the first command status and never + # fabricates a terminal test verdict. Partial and canceled originals stay. + write(reports / 'final.json', {'exit_code': status, 'report_errors': errors}) + if errors: print('\n'.join(errors), file=sys.stderr) + return status or bool(errors) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('command', choices=['compile', 'select', 'execute', 'report', 'env']) + parser.add_argument('job', choices=JOBS) + parser.add_argument('arguments', nargs='*') + args = parser.parse_args() + job = args.job + directory = Path(os.environ.get('CI_E2E_COMPILED', ROOT / '.ci/rust-e2e/compiled' / job)) + reports = ROOT / '.ci/rust-e2e/reports' / job + if args.command == 'env': + # NUL pairs are consumed by Bash without eval or command substitution. + for key, value in environment(job).items(): sys.stdout.buffer.write(key.encode() + b'\0' + value.encode() + b'\0') + elif args.command == 'compile': return compile_suite(job) + elif args.command == 'select': + reports.mkdir(parents=True, exist_ok=True) + select(directory, reports, int(os.environ['CI_SHARD_INDEX'])) + elif args.command == 'execute': + signal.signal(signal.SIGINT, GO.cancel); signal.signal(signal.SIGTERM, GO.cancel) + return execute(job, directory, args.arguments) + elif args.command == 'report': return report(job, reports, int(args.arguments[0])) + return 0 + + +if __name__ == '__main__': + try: sys.exit(main()) + except (ValueError, OSError, KeyError) as error: + print(f'ERROR: {error}', file=sys.stderr); sys.exit(1) diff --git a/ops/ci/runtime/rust-e2e.sh b/ops/ci/runtime/rust-e2e.sh new file mode 100644 index 00000000000..fe9a25432a8 --- /dev/null +++ b/ops/ci/runtime/rust-e2e.sh @@ -0,0 +1,66 @@ +#!/usr/bin/env bash +# Shared full Rust E2E workloads. Compiler outputs are reusable; verdicts are fresh. +set -euo pipefail +cd "$(git rev-parse --show-toplevel)" +mode="${1:?Pass compile or run}" +job="${2:?Pass the E2E job name}" +export GOMODCACHE="$PWD/.ci/go-cache/full/modules" +export GOCACHE="$PWD/.ci/go-cache/rust-e2e/$mode" +mkdir -p "$GOMODCACHE" "$GOCACHE" +if [[ "$mode" == compile ]]; then + python3 ops/ci/runtime/rust-e2e.py compile "$job" + exit +fi +if [[ "$mode" != run ]]; then + echo 'Unknown E2E mode' >&2 + exit 1 +fi +export CI_SHARD_INDEX="${CI_SHARD_INDEX:-0}" +export PARALLEL="${PARALLEL:-$(nproc 2> /dev/null || sysctl -n hw.ncpu)}" +report="$PWD/.ci/rust-e2e/reports/$job" +rm -rf "$report" +mkdir -p "$report/events" "$report/junit" +finish() { + local status=$? + trap - EXIT + python3 ops/ci/runtime/rust-e2e.py report "$job" "$status" || status=$? + exit "$status" +} +trap finish EXIT +trap 'exit 130' INT +trap 'exit 143' TERM +python3 ops/ci/runtime/rust-e2e.py env "$job" > "$report/environment.nul" +while IFS= read -r -d '' key && IFS= read -r -d '' value; do export "$key=$value"; done < "$report/environment.nul" +python3 ops/ci/runtime/rust-e2e.py select "$job" +if [[ "$job" == op-reth ]]; then + python3 ops/ci/runtime/rust-workspace-report.py stage-at "$report" rust/op-reth/tests proof-contracts just build-contracts +fi +if [[ ! -s "$report/assigned.txt" ]]; then + echo 'Validated empty E2E shard; no tests assigned.' + exit 0 +fi +run_tests() { + local name="$1" + shift + local status=0 split_status=0 + gotestsum --format=testname --junitfile="$report/junit/$name.xml" --jsonfile="$report/events/$name.json" \ + --raw-command -- python3 ops/ci/runtime/rust-e2e.py execute "$job" "$@" || status=$? + bash ops/scripts/split-test-logs.sh "$report/events/$name.json" || split_status=$? + if [[ "$status" == 0 ]]; then status=$split_status; fi + return "$status" +} +if [[ "$job" == proof ]]; then + # Circle xargs executes each top-level test in a new process, even after + # another test failed. Keep the first failure and collect every invocation. + first_status=0 + while IFS= read -r name; do + status=0 + run_tests "$name" "$name" || status=$? + if [[ "$first_status" == 0 && "$status" != 0 ]]; then first_status=$status; fi + done < "$report/assigned.txt" + exit "$first_status" +else + names=() + while IFS= read -r name; do names+=("$name"); done < "$report/assigned.txt" + run_tests package "${names[@]}" +fi diff --git a/ops/ci/runtime/rust-target-cache.py b/ops/ci/runtime/rust-target-cache.py new file mode 100755 index 00000000000..04702a9ab79 --- /dev/null +++ b/ops/ci/runtime/rust-target-cache.py @@ -0,0 +1,203 @@ +#!/usr/bin/env python3 +"""Keep Cargo source freshness correct when restoring targets across checkouts.""" +import hashlib +import json +import os +from pathlib import Path +import shutil +import stat +import subprocess +import sys +import tempfile +import time + +GENERATED_SOURCES = ('rust/op-reth/crates/chainspec/res/superchain-configs.tar',) +# The scripts that the compiler baselines and their tool tasks run. test_go_dependency_source.py +# fails when one of them reaches a script that is not listed. +BASELINE_SCRIPTS = ('apt-install.sh', 'ci-report.py', 'op-reth-report.py', 'op-reth-shadow.sh', + 'rust-compiler-baseline.py', 'rust-dependencies.py', 'rust-target-cache.py', + 'rust-workspace-report.py', 'rust-workspace.sh', 'rwx-materialize-linker.py', + 'rwx-prepare.sh', 'rwx-rust-prepare.sh') + + +def whole_seconds(mtime_ns): + # Cache restores can quantize target/ mtimes to whole-second resolution. + return mtime_ns // 10**9 * 10**9 + + +def wait_until_after(stamp): + # Cargo's build timestamps must not precede a refreshed input. Recheck the + # wall clock after sleeping, including an early wake or a clock adjustment. + while True: + remaining = stamp - time.time_ns() + if remaining < 0: + return + time.sleep((remaining + 1) / 10**9) + + +def generated_sources(root): + # prepare-superchain/build.rs own checksum verification and materialization. + # Cargo declares this ignored archive as a build-script input. Its timestamp + # must survive a cache restore just like a tracked source file's timestamp. + return {name: {'sha256': hashlib.sha256((root / name).read_bytes()).hexdigest()} + for name in GENERATED_SOURCES if (root / name).is_file()} + + +def sources(root): + listing = subprocess.check_output(['git', 'ls-files', '-z', '--cached', '--others', + '--exclude-standard', '--', 'rust', 'mise.toml', + 'op-core/nuts/bundles'], cwd=root) + paths = sorted(set(name.decode() for name in listing.split(b'\0') if name)) + files = [root / name for name in paths if (root / name).is_file() + and not name.startswith('rust/target/')] + entries = {} + digest = hashlib.sha256(b'rwx-cargo-source-v3\0') + for path in files: + name = str(path.relative_to(root)) + content = hashlib.sha256(path.read_bytes()).hexdigest() + entries[name] = {'sha256': content} + digest.update(name.encode() + b'\0') + digest.update(bytes.fromhex(content)) + return entries, digest.hexdigest() + + +def compiler_base(root): + """Anchor immutable compiler preparation at the last build-input change. + + Source-only PR edits reuse this ancestor's compiler filesystem. Fresh workers + always overlay HEAD and run Cargo, which checks changed source dependencies. + No compiler-cache writer permission is needed to reuse a task result. + `.rwx` is not an input: RWX's own cache key covers the task definition. + Of ops/ci/runtime, only BASELINE_SCRIPTS are inputs. + """ + paths = ['mise.toml', 'rust/Cargo.lock', ':(glob)rust/**/Cargo.toml', + 'rust/.cargo', 'rust/.config', 'rust/rust-toolchain.toml', 'rust/justfile', + *('ops/ci/runtime/' + name for name in BASELINE_SCRIPTS), + 'rust/op-reth/crates/chainspec/res/superchain-configs.tar.sha256'] + anchor = subprocess.check_output(['git', 'log', '-1', '--format=%H', 'HEAD', '--', + *paths], cwd=root, text=True).strip() + # Merge history can have a different ordering than the final tree. Never + # prepare a baseline with different build inputs from the current checkout. + if subprocess.run(['git', 'diff', '--quiet', anchor, 'HEAD', '--', *paths], cwd=root).returncode: + return subprocess.check_output(['git', 'rev-parse', 'HEAD'], cwd=root, text=True).strip() + return anchor + + +def detach_metadata(target): + """Publish independent host metadata files with identical bytes and mtimes. + + Rust hardlinks deps/*.rmeta to incremental session metadata. A restored + shared entry returned ESTALE during a hosted rebuild. Keep incremental + data, but do not publish that shared inode as a writable compiler output. + """ + count = size = 0 + for path in target.glob('*/deps/*.rmeta'): + entry = path.lstat() + if entry.st_nlink <= 1 or not stat.S_ISREG(entry.st_mode): + continue + fd, name = tempfile.mkstemp(prefix='.rwx-metadata-', dir=path.parent) + os.close(fd) + temporary = Path(name) + try: + shutil.copyfile(path, temporary) + shutil.copystat(path, temporary) + temporary.replace(path) + finally: + temporary.unlink(missing_ok=True) + count += 1 + size += entry.st_size + return {'detached_metadata_files': count, 'detached_metadata_bytes': size} + + +def manage(phase, root, target): + target.mkdir(parents=True, exist_ok=True) + state = target / '.rwx-source-fingerprint.json' + pending = target / '.rwx-source-pending.json' + entries, digest = sources(root) + generated = generated_sources(root) + if not entries: + raise ValueError('No Rust build source inputs') + if phase == 'prepare': + previous = json.loads(state.read_text()) if state.exists() else {} + # A pending manifest means a prior build did not commit. Its partial + # targets may be newer than the last successful source, even after a + # source rollback. Refresh all inputs before reusing those targets. + reusable = previous.get('version') == 3 and not pending.exists() + old_entries = previous.get('files', {}) if reusable else {} + old_generated = previous.get('generated_files', {}) if reusable else {} + changed = previous.get('source_sha256') != digest or not reusable + changed |= {name: row['sha256'] for name, row in old_generated.items()} != { + name: row['sha256'] for name, row in generated.items()} + # A restored target may have lost its subsecond timestamp. Put changed + # inputs strictly into the next second so Cargo observes the change. + stamp = whole_seconds(time.time_ns()) + 10**9 + restored = 0 + restored_generated = 0 + refreshed_generated = 0 + recorded_generated = 0 + for current, old_files in ((entries, old_entries), (generated, old_generated)): + for name, entry in current.items(): + old = old_files.get(name, {}) + if old.get('sha256') == entry['sha256'] and type(old.get('mtime_ns')) is int: + entry['mtime_ns'] = whole_seconds(old['mtime_ns']) + if current is entries: restored += 1 + else: restored_generated += 1 + elif current is generated and not old: + # The producer owns a newly materialized asset's timestamp. + # Record it without changing a checksum-matching bundle. + entry['mtime_ns'] = (root / name).stat().st_mtime_ns + recorded_generated += 1 + continue + else: + entry['mtime_ns'] = stamp + if current is generated: refreshed_generated += 1 + os.utime(root / name, ns=(entry['mtime_ns'], entry['mtime_ns'])) + # Unchanged inputs normally need no wait. Include their restored stamps + # too so a backwards clock adjustment cannot start Cargo before them. + wait_until_after(max(entry['mtime_ns'] for current in (entries, generated) + for entry in current.values())) + if changed: + state.unlink(missing_ok=True) + # Commit this per-file map only after the caller completes its build. + pending.write_text(json.dumps({'version': 3, 'source_sha256': digest, + 'files': entries, 'generated_files': generated}) + '\n') + print(json.dumps({'source_sha256': digest, 'source_changed': changed, + 'restored_source_files': restored, + 'refreshed_source_files': len(entries) - restored, + 'restored_generated_files': restored_generated, + 'refreshed_generated_files': refreshed_generated, + 'recorded_generated_files': recorded_generated})) + elif phase == 'commit': + if not pending.exists(): + raise ValueError('Rust source preparation is missing') + prepared = json.loads(pending.read_text()) + if prepared.get('source_sha256') != digest: + # Name the files, so that a recipe output in the source tree is visible. + before = {name: row['sha256'] for name, row in prepared.get('files', {}).items()} + after = {name: row['sha256'] for name, row in entries.items()} + changed = sorted(name for name in before.keys() | after.keys() + if before.get(name) != after.get(name)) + more = f' (+{len(changed) - 10} more)' if len(changed) > 10 else '' + raise ValueError('Rust source changed during build: ' + ' '.join(changed[:10]) + more) + for name, entry in prepared.get('generated_files', {}).items(): + if generated.get(name, {}).get('sha256') != entry['sha256']: + raise ValueError('Generated Rust input changed during build: ' + name) + # A cold build may create the checksum-verified archive after prepare. + # Record the timestamp Cargo actually used, without refreshing it now. + for name, entry in generated.items(): + entry['mtime_ns'] = (root / name).stat().st_mtime_ns + prepared['generated_files'] = generated + publication = detach_metadata(target) + pending.write_text(json.dumps(prepared) + '\n') + # A failed build never reaches commit, so partial targets stay untrusted. + pending.replace(state) + print(json.dumps(publication)) + else: + raise ValueError('Expected prepare or commit') + + +if __name__ == '__main__': + if sys.argv[1] == 'base': + print(compiler_base(Path.cwd())) + else: + manage(sys.argv[1], Path.cwd(), Path(os.environ['CARGO_TARGET_DIR'])) diff --git a/ops/ci/runtime/rust-workspace-report.py b/ops/ci/runtime/rust-workspace-report.py new file mode 100644 index 00000000000..a6d7b4ca67a --- /dev/null +++ b/ops/ci/runtime/rust-workspace-report.py @@ -0,0 +1,77 @@ +#!/usr/bin/env python3 +"""Record the stages, tool versions and exit status of one Rust workspace job.""" +import importlib.util +import json +import os +from pathlib import Path +import sys +import time +import xml.etree.ElementTree as ET + + +SPEC = importlib.util.spec_from_file_location('ci_report', Path(__file__).with_name('ci-report.py')) +REPORT = importlib.util.module_from_spec(SPEC); SPEC.loader.exec_module(REPORT) + +digest = REPORT.digest +write = REPORT.write + +# Jobs whose command writes no test report. RWX shows one case for each of them. +CHECK_JOBS = ('wasm-unknown', 'wasm-wasi', 'zepter', 'typos', 'deny', 'registry', 'interop') +# A tool that only one job uses, and the command that prints its version. +JOB_TOOLS = {'deny': ('cargo', 'deny', '--version'), 'zepter': ('zepter', '--version'), 'typos': ('typos', '--version'), + 'wasm-unknown': ('cargo', 'hack', '--version'), + 'wasm-wasi': ('cargo', 'hack', '--version'), 'interop': ('go', 'version')} + + +def command(*args): + return REPORT.command(args) + + +def begin(directory, job): + settings = {'suite': 'rust-workspace', 'job': job, 'source_sha': command('git', 'rev-parse', 'HEAD'), + 'rwx_run_id': os.environ.get('RWX_RUN_ID'), + 'rwx_task_attempt': os.environ.get('RWX_TASK_ATTEMPT_NUMBER'), + 'rustc': command('rustc', '--version'), 'cargo': command('cargo', '--version'), + 'nextest': command('cargo', 'nextest', '--version'), + 'started_at': time.time(), 'cpus': os.cpu_count()} + if job in JOB_TOOLS: + settings['extra_tool'] = command(*JOB_TOOLS[job]) + write(directory / 'settings.json', settings) + + +def stage(directory, name, args, stdout_json=False, cwd='rust', stdin=None, stdout_file=None): + return REPORT.stage(directory, name, args, cwd=cwd, stdout_json=stdout_json, + stdin=stdin, stdout_file=stdout_file) + + +def check_junit(directory, job, passed): + suite = ET.Element('testsuite', name=job, tests='1') + case = ET.SubElement(suite, 'testcase', classname=job, name=job) + if not passed: + ET.SubElement(case, 'failure', message='See original stage logs and final.json') + ET.ElementTree(suite).write(directory / 'checks.junit.xml', encoding='utf-8', xml_declaration=True) + + +def finish(directory, status): + """Write the job's exit status and stage records. The caller's commands own the verdict.""" + job = json.loads((directory / 'settings.json').read_text())['job'] + stages = {p.stem.removesuffix('.stage'): json.loads(p.read_text()) for p in directory.glob('*.stage.json')} + if job in CHECK_JOBS: + check_junit(directory, job, status == 0) + write(directory / 'final.json', {'exit_code': status, 'stages': stages}) + return status + + +if __name__ == '__main__': + mode, name = sys.argv[1:3] + directory = Path(name).resolve() + if mode == 'begin': + begin(directory, sys.argv[3]) + elif mode in ('stage', 'json-stage'): + sys.exit(stage(directory, sys.argv[3], sys.argv[4:], mode == 'json-stage')) + elif mode == 'stage-at': + sys.exit(stage(directory, sys.argv[4], sys.argv[5:], cwd=sys.argv[3])) + elif mode == 'finish': + sys.exit(finish(directory, int(sys.argv[3]))) + else: + raise ValueError('Unknown Rust report mode') diff --git a/ops/ci/runtime/rust-workspace.sh b/ops/ci/runtime/rust-workspace.sh new file mode 100644 index 00000000000..e87823b43b2 --- /dev/null +++ b/ops/ci/runtime/rust-workspace.sh @@ -0,0 +1,138 @@ +#!/usr/bin/env bash +# Rust workspace commands and evidence for RWX tasks. Run from the monorepo root. +set -euo pipefail +HELPERS="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT="$(git rev-parse --show-toplevel)" +cd "$ROOT" +job="${1:?Pass a Rust workspace job}" +case "$job" in tests-build | tests-unit | tests-compiler | docs | clippy | build | features | no-std | udeps | wasm-unknown | wasm-wasi | zepter | typos | deny | registry | interop) ;; +*) + echo "Unknown Rust workspace job: $job" >&2 + exit 1 + ;; +esac +index="${CI_RUST_PARTITION_INDEX:-0}" +total="${CI_RUST_PARTITION_TOTAL:-10}" +if [[ "$job" == features ]]; then + if ! [[ "$index" =~ ^(0|[1-9][0-9]*)$ && "$total" =~ ^[1-9][0-9]*$ ]] || ((index >= total)); then + echo 'Expected a nonnegative feature index below a positive partition total.' >&2 + exit 1 + fi + report="$ROOT/.ci/rust-workspace/features-$index" +else + report="$ROOT/.ci/rust-workspace/$job" +fi +rm -rf "$report" +mkdir -p "$report" +export CARGO_INCREMENTAL=0 +if [[ "$job" == clippy ]]; then export RUSTFLAGS=-Dwarnings; fi +export CARGO_HOME="$ROOT/.ci/rust-cache/cargo" CARGO_TARGET_DIR="$ROOT/rust/target" +export RUSTC_WRAPPER=sccache SCCACHE_DIR="$ROOT/.ci/rust-cache/sccache" +export SCCACHE_CACHE_SIZE=10G SCCACHE_IDLE_TIMEOUT=0 SCCACHE_BASEDIRS="$ROOT" SCCACHE_LOG=warn +mkdir -p "$CARGO_HOME" "$SCCACHE_DIR" "$CARGO_TARGET_DIR" +sccache --start-server +sccache --zero-stats +python3 "$HELPERS/rust-workspace-report.py" begin "$report" "$job" +finish() { + local status=$? diagnostics=0 + trap - EXIT + # Original verdicts are copied into the report artifact, never compiler caches. + rm -rf "$ROOT/rust/target/nextest/default" + sccache --show-stats --stats-format json > "$report/sccache.json" || diagnostics=$? + sccache --stop-server > "$report/sccache-stop.log" 2>&1 || diagnostics=$? + if [[ "$status" == 0 ]]; then + python3 "$HELPERS/rust-target-cache.py" commit > "$report/cache-publication.json" || diagnostics=$? + fi + if [[ "$status" == 0 && "$diagnostics" != 0 ]]; then status=$diagnostics; fi + python3 "$HELPERS/rust-workspace-report.py" finish "$report" "$status" || status=$? + exit "$status" +} +trap finish EXIT +trap 'exit 130' INT +trap 'exit 143' TERM +stage() { python3 "$HELPERS/rust-workspace-report.py" stage "$report" "$@"; } +stage_at() { python3 "$HELPERS/rust-workspace-report.py" stage-at "$report" "$@"; } +json_stage() { python3 "$HELPERS/rust-workspace-report.py" json-stage "$report" "$@"; } +python3 "$HELPERS/op-reth-report.py" prepare-superchain +python3 "$HELPERS/rust-target-cache.py" prepare > "$report/cache-source.json" +case "$job" in + tests-build) + stage archive mold -run cargo nextest archive --workspace --all-features --locked --archive-file "$report/tests.tar.zst" + if [[ "${CI_RUST_BASELINE:-false}" == true ]]; then + # A cold chainspec build generates a watched archive after Cargo stamps + # its build-script fingerprint. Settle that input before publishing a + # reusable compiler seed. This is preparation; HEAD creates its own archive. + json_stage compiler-settle cargo nextest list --workspace --all-features --locked --message-format json + fi + # Compile the extra test once; its fresh verdict remains in the runtime task. + stage beacon-build cargo test --profile fast-build --locked -p kona-providers-alloy \ + test_filtered_beacon_blobs_deserializes_on_small_stack --no-run + ;; + tests-unit) + rm -f "$ROOT/rust/target/nextest/default/junit.xml" + producer="${TEST_ARCHIVE:?TEST_ARCHIVE must identify the compiled producer artifact}" + # Extract at the producer's target directory. Every RWX task in this run + # has the same workspace path, so nextest's recorded paths and compile-time + # paths such as env!("CARGO_BIN_EXE_lokahi") resolve without a remap. + tar --zstd -xf "$producer/tests.tar.zst" -C "$ROOT/rust" + status=0 + stage unit cargo nextest run --binaries-metadata "$ROOT/rust/target/nextest/binaries-metadata.json" \ + --cargo-metadata "$ROOT/rust/target/nextest/cargo-metadata.json" -E '!test(test_online)' || status=$? + # RWX reads nextest's own JUnit report from the report directory, also after a test failure. + if [[ -f "$ROOT/rust/target/nextest/default/junit.xml" ]]; then + cp "$ROOT/rust/target/nextest/default/junit.xml" "$report/junit.xml" + fi + exit "$status" + ;; + tests-compiler) + # CircleCI's rust-tests job runs `just test`. The tests-unit job runs its `test-unit` + # recipe from a nextest archive; these are the other recipes that `just test` runs. + stage beacon just test-beacon-blob-stack + stage doctests just test-docs + stage custom-embeds just test-custom-embeds + ;; + docs) stage docs just lint-docs ;; + clippy) stage clippy cargo clippy --workspace --all-targets --all-features --locked ;; + build) stage build mold -run cargo build --locked --profile dev --workspace --features default ;; + features) + # Circle's recipes and partition argument. One seed shuffles every partition. + # The ancestor compiler producer has the same manifests and Cargo.lock, so + # its partition holds the same crates. + partition="" + if [[ "$total" != 1 ]]; then partition="$((index + 1))/$total"; fi + seed="$(sha256sum rust/Cargo.lock | cut -d' ' -f1)" + stage features just hack "$partition" true "$seed" + stage feature-tests just hack-tests-default "$partition" true "$seed" + ;; + no-std) stage no-std just check-no-std ;; + udeps) stage udeps just check-udeps ;; + wasm-unknown | wasm-wasi) + # Native build scripts must not lazily reopen restored GCC/glibc inputs + # while Cargo links them concurrently. Preserve byte/inode/read evidence. + stage_at . host-linker sudo python3 "$HELPERS/rwx-materialize-linker.py" + target=wasm32-wasip1 + packages=(-p op-alloy-consensus -p op-alloy-rpc-types-engine -p alloy-op-evm) + if [[ "$job" == wasm-unknown ]]; then + target=wasm32-unknown-unknown + packages=(-p op-alloy-consensus -p op-alloy-rpc-types -p op-alloy-rpc-types-engine -p alloy-op-evm --no-default-features) + fi + stage wasm-target rustup target add "$target" + stage wasm cargo hack build --target "$target" "${packages[@]}" + ;; + zepter) stage zepter zepter run check ;; + typos) stage typos typos ;; + deny) stage deny just deny ;; + registry) + # A restored target may already have KONA_SYNC_SUPERCHAIN=true. Cleaning + # just this crate forces its build script to regenerate all three snapshots. + stage registry-clean cargo clean -p kona-registry + stage_at rust/kona registry env KONA_SYNC_SUPERCHAIN=true cargo build -p kona-registry + stage_at . registry-diff git diff --exit-code -- rust/kona/crates/protocol/registry/etc/ + ;; + interop) + export GOPATH="$ROOT/.ci/interop-go/gopath" GOCACHE="$ROOT/.ci/interop-go/build" + mkdir -p "$GOPATH" "$GOCACHE" + stage_at . superchain-go just build-superchain-go + stage_at . interop bash ops/scripts/test-interop-deposits-diff.sh + ;; +esac diff --git a/ops/ci/runtime/rwx-contracts-prepare.sh b/ops/ci/runtime/rwx-contracts-prepare.sh new file mode 100644 index 00000000000..77a184f1a61 --- /dev/null +++ b/ops/ci/runtime/rwx-contracts-prepare.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash +# Install the pinned Foundry tools and solc versions for the contract tasks. +set -euo pipefail + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)" +cd "${REPO_ROOT}" + +retry_download() { + local attempt status + for attempt in 1 2 3 4 5; do + if "$@"; then return 0; else status=$?; fi + if [[ "${attempt}" == 5 ]]; then return "${status}"; fi + echo "Download failed; retrying attempt $((attempt + 1))/5." >&2 + sleep "$((2 ** attempt))" + done +} + +if [[ "${1:-}" != tools ]]; then + echo "Usage: $0 tools" >&2 + exit 1 +fi +: "${RWX_ENV:?RWX_ENV must name the exported-environment directory}" +# The shared bootstrap already trusted this disposable checkout. Keep each +# version in mise.toml and install only the additional contract tools. +retry_download mise install forge cast svm-rs +PATH="$(mise bin-paths | paste -sd: -):${PATH}" +export PATH +# These are the exact compiler versions installed by the CircleCI command. +for version in 0.8.15 0.8.19 0.8.25 0.8.28; do + if ! svm which "${version}"; then + retry_download svm install "${version}" + fi +done +mkdir -p "${RWX_ENV}" +printf '%s\n' "${PATH}" > "${RWX_ENV}/PATH" diff --git a/ops/ci/runtime/rwx-materialize-linker.py b/ops/ci/runtime/rwx-materialize-linker.py new file mode 100644 index 00000000000..ba17f9a26f5 --- /dev/null +++ b/ops/ci/runtime/rwx-materialize-linker.py @@ -0,0 +1,106 @@ +#!/usr/bin/env python3 +"""Detach native linker inputs from restored RWX layers before WASM builds.""" +import errno +import hashlib +import json +import os +from pathlib import Path +import stat +import subprocess +import sys +import tempfile +import time + + +# WASM build scripts still link native executables. Ask the image's default C +# driver for its inputs rather than pinning a GCC version or following every +# file in /usr/lib. Include the targets of glibc/GCC linker scripts as well. +INPUTS = ( + 'crt1.o', 'Scrt1.o', 'crti.o', 'crtn.o', 'crtbegin.o', 'crtbeginS.o', + 'crtbeginT.o', 'crtend.o', 'crtendS.o', 'libgcc.a', 'libgcc_eh.a', + 'libgcc_s.so', 'libgcc_s.so.1', 'libc.so', 'libc.so.6', 'libc_nonshared.a', + 'libm.so', 'libm.so.6', 'libmvec.so.1', 'libpthread.a', 'libdl.a', + 'librt.a', 'libutil.a', 'ld-linux-x86-64.so.2', +) + + +def linker_inputs(): + paths = set() + for name in INPUTS: + value = subprocess.check_output( + ['/usr/bin/cc', '-print-file-name=' + name], text=True, timeout=30).strip() + path = Path(value) + if not path.is_absolute(): + raise ValueError('C driver could not locate linker input: ' + name) + path = path.resolve(strict=True) + if not path.is_relative_to('/usr/lib'): + raise ValueError('Linker input outside /usr/lib: ' + str(path)) + paths.add(path) + return sorted(paths) + + +def materialize(path, record): + started = time.monotonic() + record.update(path=str(path), read_attempts=0, read_timeouts=0) + try: + for attempt in range(3): + record['read_attempts'] += 1 + try: + before = path.stat() + if not stat.S_ISREG(before.st_mode) or before.st_size > 32 * 1024 * 1024: + raise ValueError('Unexpected linker input type or size: ' + str(path)) + data = path.read_bytes() + after = path.stat() + if (before.st_ino, before.st_size, before.st_mtime_ns) != ( + after.st_ino, after.st_size, after.st_mtime_ns) or len(data) != before.st_size: + raise ValueError('Linker input changed while reading: ' + str(path)) + break + except OSError as error: + # Retry only the filesystem failure observed in hosted builds. + # Never retry Cargo, the linker, or a compilation/test verdict. + if error.errno != errno.ETIMEDOUT: + raise + record['read_timeouts'] += 1 + if attempt == 2: + raise + time.sleep(attempt + 1) + expected = hashlib.sha256(data).hexdigest() + fd, temporary = tempfile.mkstemp(prefix='.rwx-linker-', dir=path.parent) + try: + with os.fdopen(fd, 'wb') as target: + target.write(data) + os.fchown(target.fileno(), before.st_uid, before.st_gid) + os.fchmod(target.fileno(), stat.S_IMODE(before.st_mode)) + # A real write to a fresh inode is intentional: reading the old file + # alone only warms a page cache that the kernel can evict again. + if hashlib.sha256(Path(temporary).read_bytes()).hexdigest() != expected: + raise ValueError('Local linker input digest mismatch: ' + str(path)) + os.utime(temporary, ns=(before.st_atime_ns, before.st_mtime_ns)) + os.replace(temporary, path) + finally: + Path(temporary).unlink(missing_ok=True) + current = path.stat() + if current.st_ino == before.st_ino or current.st_nlink != 1: + raise ValueError('Linker input was not detached: ' + str(path)) + record.update(bytes=len(data), sha256=expected, original_inode=before.st_ino, + local_inode=current.st_ino, detached=True) + finally: + record['seconds'] = time.monotonic() - started + + +def main(): + report = {'schema': 1, 'files': [], 'success': False} + try: + for path in linker_inputs(): + record = {} + report['files'].append(record) + materialize(path, record) + report['success'] = True + except (OSError, ValueError, subprocess.SubprocessError) as error: + report['error'] = str(error) + print(json.dumps(report, sort_keys=True), flush=True) + return 0 if report['success'] else 1 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/ops/ci/runtime/rwx-metadata.sh b/ops/ci/runtime/rwx-metadata.sh new file mode 100755 index 00000000000..303db98fb5e --- /dev/null +++ b/ops/ci/runtime/rwx-metadata.sh @@ -0,0 +1,106 @@ +#!/usr/bin/env bash +# Adapt RWX event metadata to CircleCI's workflow routing, read-only. +# +# RWX selects workloads with the CircleCI routing data and policy +# (.circleci/routing.yml, .circleci/scripts/collect-params.sh and +# compute-workflow-conditions.sh), so the two providers cannot drift. This +# adapter supplies the inputs those scripts document and adds one RWX rule: +# a change under .rwx/ or ops/ci/ selects every workload, like a .circleci/ +# change does for CircleCI. +set -euo pipefail + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)" +cd "${REPO_ROOT}" + +: "${CI_COMMIT_SHA:?CI_COMMIT_SHA must identify the checked-out commit}" +: "${RWX_VALUES:?RWX_VALUES must name the task output-values directory}" +CI_BRANCH="${CI_BRANCH:-}" +CI_BASE_REVISION="${CI_BASE_REVISION:-develop}" + +if [[ ! "${CI_COMMIT_SHA}" =~ ^[0-9a-f]{40}$ ]]; then + echo "ERROR: CI_COMMIT_SHA must be a full commit SHA." >&2 + exit 1 +fi +if [[ "$(git rev-parse HEAD)" != "${CI_COMMIT_SHA}" ]]; then + echo "ERROR: checked-out HEAD differs from CI_COMMIT_SHA." >&2 + exit 1 +fi +if [[ "${CI_CACHE_WARM:-false}" == true ]]; then + # A referenced task must succeed even when an `if` expression could bypass + # its value. Keep a real routing dependency for warm runs, but select no + # verdicts. Compiler tasks explicitly opt in through their warm flag. + if [[ "${CI_BRANCH}" != develop ]]; then + echo 'ERROR: cache warming is restricted to develop.' >&2 + exit 1 + fi + mkdir -p .ci "${RWX_VALUES}" + printf '%s\n' '{"cache-rebuild":true,"c-run_main":false,"c-run_rust_ci":false,"c-run_rust_e2e_ci":false,"c-run_contracts_feature_tests":false}' \ + > .ci/pipeline-parameters.json + printf 'false\n' > "${RWX_VALUES}/run-main" + printf 'false\n' > "${RWX_VALUES}/run-rust-ci" + printf 'false\n' > "${RWX_VALUES}/run-rust-e2e-ci" + printf 'false\n' > "${RWX_VALUES}/run-contracts" + echo "Prepared compiler-only cache warming for ${CI_COMMIT_SHA}." + exit 0 +fi +if [[ -z "${CI_BRANCH}" ]]; then + echo "ERROR: a push must identify its branch." >&2 + exit 1 +fi +git check-ref-format --branch "${CI_BASE_REVISION}" > /dev/null + +# Fetch the protected comparison branch explicitly. A shallow checkout must +# include enough history to find the merge base; an unavailable base fails the +# task rather than treating an incomplete diff as a docs-only change. +base_refspec="+refs/heads/${CI_BASE_REVISION}:refs/remotes/origin/${CI_BASE_REVISION}" +if [[ "$(git rev-parse --is-shallow-repository)" == true ]]; then + git fetch --no-tags --unshallow origin "${CI_COMMIT_SHA}" "${base_refspec}" +else + git fetch --no-tags origin "${base_refspec}" +fi +git merge-base "origin/${CI_BASE_REVISION}" HEAD > /dev/null + +mkdir -p .ci "${RWX_VALUES}" +# collect-params.sh always appends to this fixed file; start from a clean one. +PARAMS=/tmp/pipeline-parameters.json +printf '{}\n' > "${PARAMS}" +BASE_REVISION="${CI_BASE_REVISION}" bash .circleci/scripts/collect-params.sh detect +BASE_REVISION="${CI_BASE_REVISION}" bash .circleci/scripts/collect-params.sh detect_all + +# The RWX rule. `rwx run` can also upload an uncommitted patch, which the +# committed diff above cannot see: treat any such change like a CI change, so +# a local experiment runs every workload instead of a docs-only shortcut. +committed="$(git diff --name-only --no-renames "origin/${CI_BASE_REVISION}...HEAD")" +uncommitted="$(git status --porcelain --untracked-files=all -- . ':(exclude).ci')" +if grep -qE '^(\.rwx|ops/ci)/' <<< "${committed}" || [[ -n "${uncommitted}" ]]; then + echo "RWX or uncommitted CI change: selecting every workload." + jq '. + {"c-circleci_changed": true, "c-contracts_changed": true, "c-rust_changes_detected": true, "c-only_docs_changes": false}' \ + "${PARAMS}" > "${PARAMS}.next" + mv "${PARAMS}.next" "${PARAMS}" +fi + +TRIGGER_SOURCE=webhook BRANCH="${CI_BRANCH}" TAG="" SCHEDULE_NAME="" \ + CIRCLE_PROJECT_USERNAME=ethereum-optimism CIRCLE_PROJECT_REPONAME=optimism OUTPUT="${PARAMS}" \ + bash .circleci/scripts/compute-workflow-conditions.sh +# The policy writes only the workflows it enables. RWX route tasks and gate +# aggregates require these four flags as booleans, so write false explicitly. +# The four route values below are therefore always "true" or "false". +jq '{"c-run_main": false, "c-run_rust_ci": false, "c-run_rust_e2e_ci": false, "c-run_contracts_feature_tests": false} + .' \ + "${PARAMS}" > .ci/pipeline-parameters.json + +# On develop the policy enables every workflow that RWX shadows. A flag that is +# not true there means that CircleCI renamed it or stopped writing it. The +# default above would then report "not selected" on every branch, and each +# receipt would pass with all of its tasks skipped. +if [[ "${CI_BRANCH}" == develop ]]; then + disabled="$(jq -r '[("c-run_main", "c-run_rust_ci", "c-run_rust_e2e_ci", "c-run_contracts_feature_tests") as $flag | select(.[$flag] != true) | $flag] | join(" ")' .ci/pipeline-parameters.json)" + if [[ -n "${disabled}" ]]; then + echo "ERROR: develop runs every shadowed workflow, but the routing policy did not enable: ${disabled}. Did CircleCI rename a flag?" >&2 + exit 1 + fi +fi + +jq -r '."c-run_main"' .ci/pipeline-parameters.json > "${RWX_VALUES}/run-main" +jq -r '."c-run_rust_ci"' .ci/pipeline-parameters.json > "${RWX_VALUES}/run-rust-ci" +jq -r '."c-run_rust_e2e_ci"' .ci/pipeline-parameters.json > "${RWX_VALUES}/run-rust-e2e-ci" +jq -r '."c-run_contracts_feature_tests"' .ci/pipeline-parameters.json > "${RWX_VALUES}/run-contracts" diff --git a/ops/ci/runtime/rwx-prepare.sh b/ops/ci/runtime/rwx-prepare.sh new file mode 100755 index 00000000000..182c313dc01 --- /dev/null +++ b/ops/ci/runtime/rwx-prepare.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash +# Prepare the pinned tools that RWX tasks need on the Ubuntu base. +set -euo pipefail + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)" +cd "${REPO_ROOT}" +: "${RWX_ENV:?RWX_ENV must name the exported-environment directory}" + +# Tool downloads come from GitHub releases and dl.google.com, which time out +# intermittently. mise retries each download and allows slower transfers. +# The common layer exports both settings, so later mise installs retry too. +export MISE_HTTP_RETRIES=5 MISE_HTTP_TIMEOUT=90s + +# Language toolchains are independent layers. In particular, Go/Foundry tasks +# must not download both Rust toolchains or the Go linter to run their tests. +case "${1:-common}" in + common) + bash ops/ci/runtime/apt-install.sh \ + build-essential ca-certificates pkg-config unzip zip + # Trust only the disposable RWX checkout; local development is unchanged. + mise trust --yes mise.toml + mise install just jq yq python + ;; + go) mise install go gotestsum ;; + go-lint) mise install go golangci-lint ;; + rust) mise install rust ;; + *) + echo "Usage: $0 [common|go|go-lint|rust]" >&2 + exit 1 + ;; +esac +mkdir -p "${RWX_ENV}" +printf '%s:%s\n' "$(mise bin-paths | paste -sd: -)" "${PATH}" > "${RWX_ENV}/PATH" +# Later mise exec calls must use this prepared toolset rather than installing +# unrelated tools from the root configuration in every verdict task. +printf 'false\n' > "${RWX_ENV}/MISE_AUTO_INSTALL" +printf 'false\n' > "${RWX_ENV}/MISE_EXEC_AUTO_INSTALL" +printf '%s\n' "${MISE_HTTP_RETRIES}" > "${RWX_ENV}/MISE_HTTP_RETRIES" +printf '%s\n' "${MISE_HTTP_TIMEOUT}" > "${RWX_ENV}/MISE_HTTP_TIMEOUT" diff --git a/ops/ci/runtime/rwx-rust-prepare.sh b/ops/ci/runtime/rwx-rust-prepare.sh new file mode 100644 index 00000000000..28f1b862e8d --- /dev/null +++ b/ops/ci/runtime/rwx-rust-prepare.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash +# Install op-reth's pinned build tools in a reusable RWX filesystem layer. +set -euo pipefail +cd "$(dirname "${BASH_SOURCE[0]}")/../../.." +: "${RWX_ENV:?RWX_ENV must name the exported-environment directory}" + +bash ops/ci/runtime/apt-install.sh clang llvm-dev libclang-dev libssl-dev cmake zstd +mise install mold protoc github:nextest-rs/nextest +PATH="$(mise bin-paths | paste -sd: -):${PATH}" +export PATH + +# Separate from CircleCI's GCS cache: no cloud credentials or production writer. +SCCACHE_VERSION=0.18.0 +SCCACHE_SHA256=45f1447fbe231e3037bde351ef70677dd212216c8d62ae7ca409fecc4d6acc89 +download="$(mktemp -d)" +trap 'rm -rf "$download"' EXIT +curl -fsSL --retry 5 --retry-delay 2 \ + "https://github.com/mozilla/sccache/releases/download/v${SCCACHE_VERSION}/sccache-v${SCCACHE_VERSION}-x86_64-unknown-linux-musl.tar.gz" \ + -o "$download/sccache.tar.gz" +printf '%s %s\n' "$SCCACHE_SHA256" "$download/sccache.tar.gz" | sha256sum -c - +tar -xzf "$download/sccache.tar.gz" -C "$download" +sudo install -m 0755 "$download/sccache-v${SCCACHE_VERSION}-x86_64-unknown-linux-musl/sccache" /usr/local/bin/sccache + +# Installing the helpers in the tool layer lets the pinned develop checkout use +# the same implementation without overlaying PR source onto the baseline. +sudo mkdir -p /usr/local/lib/optimism-ci +sudo install -m 0755 ops/ci/runtime/op-reth-shadow.sh /usr/local/lib/optimism-ci/op-reth-shadow.sh +sudo install -m 0755 ops/ci/runtime/rust-target-cache.py /usr/local/lib/optimism-ci/rust-target-cache.py +sudo install -m 0755 ops/ci/runtime/op-reth-report.py /usr/local/lib/optimism-ci/op-reth-report.py +printf '%s\n' "$PATH" > "$RWX_ENV/PATH" +printf '%s\n' "${RUSTUP_HOME:?mise exec must export its Rust toolchain directory}" > "$RWX_ENV/RUSTUP_HOME" +python3 - << 'PY' > "$RWX_ENV/RUSTUP_TOOLCHAIN" +import tomllib +with open('mise.toml', 'rb') as source: + print(tomllib.load(source)['tools']['rust'][0]['version']) +PY +sccache --version +protoc --version +mold --version +cargo nextest --version diff --git a/ops/ci/runtime/sp1-guest-native-build.py b/ops/ci/runtime/sp1-guest-native-build.py new file mode 100644 index 00000000000..e86fd10f2a5 --- /dev/null +++ b/ops/ci/runtime/sp1-guest-native-build.py @@ -0,0 +1,56 @@ +#!/usr/bin/env python3 +"""Build the SP1 guest ELFs in a clone at the paths that CircleCI uses. + +The clone and its Cargo home have CircleCI's paths, so that ELF bytes that +embed those paths match CircleCI's build of the same commit. +""" +import os +from pathlib import Path +import shutil +import subprocess +import sys + +ROOT = Path(__file__).resolve().parents[3] +WORKSPACE = Path('/home/circleci/project') +CARGO = Path('/data/mise-data/.cargo') +TARGET = WORKSPACE/'rust/kona/sp1/programs/target' + + +def execute(features=""): + """Run the build phase of sp1-guest.py in the clone and copy its hand-off directory here. + + Preconditions: WORKSPACE and CARGO do not exist, and this checkout has no + uncommitted change to a tracked file. The clone contains only the commit, + so the build would not contain such a change. + + Return the exit status of the build phase, or 1 if a precondition fails. + The tool cache keeps Cargo home and the target directory inside this + checkout. This function copies them to CARGO and TARGET before the build + and, if the build passes, back. + """ + cache=ROOT/'.ci/sp1-cache' + output=ROOT/'.ci/sp1-guest/dependency';shutil.rmtree(output,ignore_errors=True) + if (WORKSPACE.exists() or WORKSPACE.is_symlink() or CARGO.exists() or CARGO.is_symlink() + or WORKSPACE.resolve()!=WORKSPACE or CARGO.resolve()!=CARGO): + print('Native SP1 canonical directories are already occupied',file=sys.stderr);return 1 + if subprocess.call(['git','diff','--quiet','HEAD'],cwd=ROOT): + print('The clone would not contain the uncommitted changes of this checkout',file=sys.stderr);return 1 + sha=subprocess.check_output(['git','rev-parse','HEAD'],cwd=ROOT,text=True).strip() + for argv in (['sudo','mkdir','-p',str(WORKSPACE),str(CARGO)], + ['sudo','chown',str(os.getuid())+':'+str(os.getgid()),str(WORKSPACE),str(CARGO)], + ['git','clone','--shared','--no-checkout',str(ROOT),str(WORKSPACE)], + ['git','-C',str(WORKSPACE),'checkout','--detach',sha], + ['mise','trust','--yes',str(WORKSPACE/'mise.toml')]): + subprocess.run(argv,cwd=ROOT,check=True,stdin=subprocess.DEVNULL) + for source,target in [(cache/'cargo',CARGO),(cache/'elf-target',TARGET)]: + if source.exists():shutil.copytree(source,target,symlinks=True,dirs_exist_ok=True) + else:target.mkdir(parents=True,exist_ok=True) + status=subprocess.call([sys.executable,'ops/ci/runtime/sp1-guest.py','--phase','build','--features',features], + cwd=WORKSPACE,env={**os.environ,'SP1_GUEST_NATIVE_CACHE_ROOT':str(cache)}) + built=WORKSPACE/'.ci/sp1-guest/dependency' + if built.is_dir():shutil.copytree(built,output) + else:output.mkdir(parents=True) + if not status: + for source,target in [(CARGO,cache/'cargo'),(TARGET,cache/'elf-target')]: + shutil.rmtree(target,ignore_errors=True);shutil.copytree(source,target,symlinks=True) + return status diff --git a/ops/ci/runtime/sp1-guest-toolchain.sh b/ops/ci/runtime/sp1-guest-toolchain.sh new file mode 100644 index 00000000000..d81ba844a8a --- /dev/null +++ b/ops/ci/runtime/sp1-guest-toolchain.sh @@ -0,0 +1,71 @@ +#!/usr/bin/env bash +# Copy of the body of CircleCI's sp1-toolchain-restore-or-install command +# (.circleci/continue/rust-e2e.yml): SP1 version pin checks, then the Succinct +# toolchain installation. CircleCI does not run this file and nothing compares +# the two, so apply a change of that command here by hand. +set -euo pipefail +cd "$(dirname "${BASH_SOURCE[0]}")/../../.." + +install_sp1_toolchain() { + (cd rust/kona/sp1 && just install-sp1-toolchain) +} + +mise_sp1_version="$(yq -r '.tools."github:succinctlabs/sp1".version // ""' mise.toml)" +root_manifest_sp1_sdk="$(yq -r '.workspace.dependencies."sp1-sdk".version // ""' rust/Cargo.toml)" +guest_manifest_sp1_lib="$(yq -r '.workspace.dependencies."sp1-lib".version // ""' rust/kona/sp1/programs/Cargo.toml)" +guest_manifest_sp1_zkvm="$(yq -r '.workspace.dependencies."sp1-zkvm".version // ""' rust/kona/sp1/programs/Cargo.toml)" +sp1_tag="$(cd rust/kona/sp1 && just --evaluate SP1_TAG 2> /dev/null || printf '')" + +lock_data_dir="$(mktemp -d)" +trap 'rm -rf "$lock_data_dir"' EXIT +root_lock_sp1_sdk="" +if yq -p=toml -o=json '.package // []' rust/Cargo.lock > "$lock_data_dir/root.json"; then + if ! root_lock_sp1_sdk="$(jq -er '[.[] | select(.name == "sp1-sdk") | .version] | unique | if length == 1 then .[0] else error("expected exactly one resolved sp1-sdk version") end' "$lock_data_dir/root.json")"; then + root_lock_sp1_sdk="" + fi +fi + +guest_lock_sp1_lib="" +guest_lock_sp1_zkvm="" +if yq -p=toml -o=json '.package // []' rust/kona/sp1/programs/Cargo.lock > "$lock_data_dir/guest.json"; then + if ! guest_lock_sp1_lib="$(jq -er '[.[] | select(.name == "sp1-lib") | .version] | unique | if length == 1 then .[0] else error("expected exactly one resolved sp1-lib version") end' "$lock_data_dir/guest.json")"; then + guest_lock_sp1_lib="" + fi + if ! guest_lock_sp1_zkvm="$(jq -er '[.[] | select(.name == "sp1-zkvm") | .version] | unique | if length == 1 then .[0] else error("expected exactly one resolved sp1-zkvm version") end' "$lock_data_dir/guest.json")"; then + guest_lock_sp1_zkvm="" + fi +fi + +if [ "$root_manifest_sp1_sdk" != "$mise_sp1_version" ] \ + || [ "$guest_manifest_sp1_lib" != "$mise_sp1_version" ] \ + || [ "$guest_manifest_sp1_zkvm" != "$mise_sp1_version" ] \ + || [ "$root_lock_sp1_sdk" != "$mise_sp1_version" ] \ + || [ "$guest_lock_sp1_lib" != "$mise_sp1_version" ] \ + || [ "$guest_lock_sp1_zkvm" != "$mise_sp1_version" ] \ + || [ "$sp1_tag" != "v${mise_sp1_version}" ]; then + printf '%s\n' \ + "ERROR: SP1 version drift; mise.toml cargo-prove is canonical." \ + " mise.toml cargo-prove: ${mise_sp1_version}" \ + " rust/Cargo.toml sp1-sdk: ${root_manifest_sp1_sdk}" \ + " rust/Cargo.lock resolved sp1-sdk: ${root_lock_sp1_sdk}" \ + " rust/kona/sp1/programs/Cargo.toml sp1-lib: ${guest_manifest_sp1_lib}" \ + " rust/kona/sp1/programs/Cargo.toml sp1-zkvm: ${guest_manifest_sp1_zkvm}" \ + " rust/kona/sp1/programs/Cargo.lock resolved sp1-lib: ${guest_lock_sp1_lib}" \ + " rust/kona/sp1/programs/Cargo.lock resolved sp1-zkvm: ${guest_lock_sp1_zkvm}" \ + " rust/kona/sp1/justfile SP1_TAG: ${sp1_tag} (expected v${mise_sp1_version})" >&2 + exit 1 +fi + +toolchain_dir="$(find "$HOME/.sp1/toolchains" -mindepth 1 -maxdepth 1 -type d -print -quit 2> /dev/null || true)" +if [ -n "$toolchain_dir" ]; then + rustup toolchain remove succinct || true + if rustup toolchain link succinct "$toolchain_dir" && rustc +succinct --version > /dev/null; then + echo "Restored SP1 toolchain from $toolchain_dir" + else + echo "Cached SP1 toolchain is invalid; reinstalling" + install_sp1_toolchain + fi +else + install_sp1_toolchain +fi +rustc +succinct --version diff --git a/ops/ci/runtime/sp1-guest.py b/ops/ci/runtime/sp1-guest.py new file mode 100644 index 00000000000..5965e0c03a0 --- /dev/null +++ b/ops/ci/runtime/sp1-guest.py @@ -0,0 +1,197 @@ +#!/usr/bin/env python3 +"""Run the steps of CircleCI's kona-build-sp1-elfs job in two RWX tasks. + +`--phase build` checks the guest lock, builds the guest ELFs and applies +CircleCI's two ELF checks. `--phase checks` receives those ELFs and runs the +guest tests, the guest lints and the vkey crate checks. The exit status of +each recipe is the verdict. +""" +import argparse +import hashlib +import importlib.util +import json +import os +from pathlib import Path +import re +import shutil +import subprocess +import sys +import tomllib + +ROOT = Path(__file__).resolve().parents[3] +RUST = ROOT / 'rust' +SP1 = RUST / 'kona/sp1' +REPORTS = ROOT / '.ci/sp1-guest' +# The guest programs that build-elfs-native builds, as CircleCI's marker check lists them. +PROGRAMS = ('log-commitment', 'super-range', 'super-aggregation') +TEST_CONFIG_MARKER = b'KONA_SP1_UNSAFE_TEST_CONFIG_FALLBACK{fd6d88e711058eef5eff1512237c8ad3}' + + +def helper(name): + spec = importlib.util.spec_from_file_location(name.replace('-', '_'), Path(__file__).with_name(name + '.py')) + value = importlib.util.module_from_spec(spec); spec.loader.exec_module(value) + return value + + +REPORT = helper('ci-report') + + +def command(*argv): return REPORT.command(argv, cwd=ROOT) + + +def run(*argv, cwd=RUST): + """Run one command. Raise CalledProcessError if it fails.""" + subprocess.run(argv, cwd=cwd, check=True, stdin=subprocess.DEVNULL) + + +def tools(): + version = tomllib.loads((ROOT / 'mise.toml').read_text())['tools']['github:succinctlabs/sp1']['version'] + result = {name: command(*argv) for name, argv in { + 'rustc': ['rustc', '--version'], 'cargo': ['cargo', '--version'], + 'cargo-prove': ['cargo', 'prove', '--version'], 'succinct-rustc': ['rustc', '+succinct', '--version'], + 'nightly-rustfmt': ['cargo', '+' + tomllib.loads((ROOT / 'mise.toml').read_text())['tools']['rust'][1]['version'], 'fmt', '--version'], + 'just': ['just', '--version'], 'protoc': ['protoc', '--version']}.items()} + directory=Path(command('mise','where','github:succinctlabs/sp1')).resolve() + binary=Path(shutil.which('cargo-prove') or '/missing-cargo-prove').resolve() + if directory.name!=version or not binary.is_file() or not binary.is_relative_to(directory): + raise ValueError('Cargo-prove executable does not come from the pinned mise installation') + # This release reports a Git descriptor ('sp1'), not the numeric SDK version. + # Retain that original descriptor and verify the actual installed binary bytes. + result['cargo-prove-sha256']=REPORT.digest(binary) + rustc = Path(command('rustup', 'which', '--toolchain', 'succinct', 'rustc')) + if not rustc.is_file(): raise ValueError('Missing actual Succinct compiler') + result['succinct-rustc-sha256'] = REPORT.digest(rustc) + library = rustc.parent.parent/'lib' + result['succinct-library-sha256'] = {str(p.relative_to(library)):REPORT.digest(p) for p in sorted(library.rglob('*')) if p.is_file()} + if not result['succinct-library-sha256']:raise ValueError('Missing actual Succinct linker and standard-library inputs') + result['sp1-version'] = version + return result + + +def check_elfs(directory, sha, features): + """Apply CircleCI's two checks to the built ELFs. Raise ValueError if one fails. + + 1. vkeys.toml has the names that its consumers read. + 2. Each ELF embeds exactly one build marker, and that marker names `sha`, + with the suffix -test if the build selected features. The super-range + ELF has the test-config-fallback marker if, and only if, the build + selected features. + """ + # CircleCI's "Check vkeys manifest names expected by consumers" step. + if not {'super-aggregation', 'log-commitment'} <= set(tomllib.loads((directory / 'vkeys.toml').read_text())): + raise ValueError('vkeys manifest lacks a name that its consumers read') + expected = 'KONA_SP1_BUILD{git_sha=' + sha + ('-test' if features else '') + '}' + for name in PROGRAMS: + data = (directory / (name + '-elf')).read_bytes() + markers = set(m.decode() for m in re.findall(rb'KONA_SP1_BUILD\{git_sha=[0-9A-Za-z._-]*\}', data)) + if markers != {expected}: + raise ValueError('Guest ELF has missing, extra or stale source markers') + if name == 'super-range' and (TEST_CONFIG_MARKER in data) != bool(features): + raise ValueError('Guest ELF test-config-fallback marker differs from selected features') + + +def cache_start(phase, features): + """Point Cargo and sccache at the tool-cache directories of the phase and start sccache. + + If the compiler inputs differ from the ones that built the target + directory of the phase, delete that directory first. + """ + cache = ROOT / '.ci/sp1-cache' + target = cache / (phase + '-target') + cargo_home = cache/'cargo' + if phase=='elf': + # The clone at CircleCI's paths keeps sccache data in the checkout that started it. + native=helper('sp1-guest-native-build') + cache=Path(os.environ['SP1_GUEST_NATIVE_CACHE_ROOT']) + target=native.TARGET;cargo_home=native.CARGO + mode = os.environ.get('SP1_GUEST_TARGET_MODE', 'keep') + if mode not in ('keep', 'sccache-only'): raise ValueError('Invalid SP1 compiler-target probe mode') + identity={'phase':phase,'features':features,'tools':tools(),'rustflags':'' if phase=='elf' else '-Dwarnings','incremental':'0'} + identity_sha256=hashlib.sha256(json.dumps(identity,sort_keys=True).encode()).hexdigest() + identity_path=target/'.sp1-compiler-inputs.json' + if not identity_path.is_file() or REPORT.read(identity_path)!=identity: + shutil.rmtree(target,ignore_errors=True) + if mode == 'sccache-only': shutil.rmtree(target, ignore_errors=True) + target.mkdir(parents=True,exist_ok=True);REPORT.write(identity_path,identity) + os.environ.update(CARGO_HOME=str(cargo_home), CARGO_TARGET_DIR=str(target), + RUSTC_WRAPPER='sccache', SCCACHE_DIR=str(cache/(phase+'-sccache')/identity_sha256), + SCCACHE_BASEDIRS=str(ROOT), SCCACHE_CACHE_SIZE='10G', SCCACHE_IDLE_TIMEOUT='0') + for name in ('CARGO_HOME', 'CARGO_TARGET_DIR', 'SCCACHE_DIR'): + Path(os.environ[name]).mkdir(parents=True, exist_ok=True) + run('sccache', '--start-server', cwd=ROOT) + run('sccache', '--zero-stats', cwd=ROOT) + run('python3', 'ops/ci/runtime/rust-target-cache.py', 'prepare', cwd=ROOT) + + +def cache_finish(status): + """Print the sccache statistics and stop sccache. Commit the target cache if nothing failed. + + Return `status` if it is not 0. Otherwise return the exit status of the + first cache command that fails, or 0. + """ + for argv in (['sccache', '--show-stats'], ['sccache', '--stop-server']): + code = subprocess.call(argv, cwd=ROOT) + status = status or code + if not status: + status = subprocess.call(['python3', 'ops/ci/runtime/rust-target-cache.py', 'commit'], cwd=ROOT) + return status + + +def cached(phase, features, steps): + """Run `steps` between cache_start and cache_finish. Return the exit status of the phase.""" + cache_start(phase, features) + try: + steps() + status = 0 + except subprocess.CalledProcessError as error: + # A negative status is a signal number; report it as a shell does. + status = error.returncode if error.returncode > 0 else 128 - error.returncode + except (OSError, ValueError) as error: + print(error, file=sys.stderr) + status = 1 + return cache_finish(status) + + +def build(features): + """Build the guest ELFs and write them to the hand-off directory. Return the exit status. + + sp1-guest-native-build.py owns the build location. If this checkout is not + at that location, delegate to it; it runs this phase again inside its clone. + """ + native = helper('sp1-guest-native-build') + if ROOT != native.WORKSPACE: + return native.execute(features) + files = REPORTS / 'dependency/files' + shutil.rmtree(files.parent, ignore_errors=True) + files.mkdir(parents=True) + + def steps(): + # CircleCI checks the committed guest lock first: build-elfs-native regenerates it in place. + run('just', 'check-sp1-guest-lock') + run('just', 'build-elfs-native', features, cwd=SP1) + check_elfs(SP1 / 'elf', command('git', 'rev-parse', 'HEAD'), features) + for name in (*(program + '-elf' for program in PROGRAMS), 'vkeys.toml'): + shutil.copyfile(SP1 / 'elf' / name, files / name) + return cached('elf', features, steps) + + +def checks(artifact, features): + """Run CircleCI's test, lint and vkey crate steps on the ELFs in `artifact`. Return the exit status.""" + def steps(): + # The guest lints and the vkey crates read the ELFs and vkeys.toml that the build task made. + (SP1 / 'elf').mkdir(exist_ok=True) + for path in (artifact / 'files').iterdir(): + shutil.copyfile(path, SP1 / 'elf' / path.name) + os.environ['RUSTFLAGS'] = '-Dwarnings' + for recipe in ('test-sp1-guest', 'lint-sp1-guest', 'check-vkey-crates'): + run('just', recipe) + return cached('checks', features, steps) + + +if __name__=='__main__': + parser=argparse.ArgumentParser(description=__doc__) + parser.add_argument('--phase',choices=('build','checks'),required=True) + parser.add_argument('--features',choices=('', 'test-config-fallback'),default='') + parser.add_argument('--artifact',type=Path,default=REPORTS/'dependency') + args=parser.parse_args() + sys.exit(build(args.features) if args.phase=='build' else checks(args.artifact,args.features)) diff --git a/ops/ci/tests/ci_test_fixtures.py b/ops/ci/tests/ci_test_fixtures.py new file mode 100644 index 00000000000..97ac062a65c --- /dev/null +++ b/ops/ci/tests/ci_test_fixtures.py @@ -0,0 +1,14 @@ +"""Shared helpers for the ops/ci unit tests.""" +from contextlib import contextmanager +import os +from pathlib import Path + + +@contextmanager +def preserve_working_directory(): + """Restore the test process after a CLI runner changes its working directory.""" + previous = Path.cwd() + try: + yield + finally: + os.chdir(previous) diff --git a/ops/ci/tests/test_acceptance_runner.py b/ops/ci/tests/test_acceptance_runner.py new file mode 100644 index 00000000000..ebe5071bec0 --- /dev/null +++ b/ops/ci/tests/test_acceptance_runner.py @@ -0,0 +1,156 @@ +#!/usr/bin/env python3 +"""Execute the unchanged Just entrypoint through the RWX wrapper and its circleci shim.""" +import json +import os +from pathlib import Path +import shutil +import subprocess +import tempfile +import unittest + +ROOT = Path(__file__).resolve().parents[3] +PACKAGE = 'github.com/ethereum-optimism/optimism/op-acceptance-tests/tests/base' + + +@unittest.skipUnless(shutil.which('just'), 'Just required') +class AcceptanceRunnerTest(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + self.root = Path(self.temp.name) + for directory in ['op-acceptance-tests/tests', 'ops/ci/runtime', 'ops/scripts', 'bin']: + (self.root / directory).mkdir(parents=True) + justfile = (ROOT / 'op-acceptance-tests/justfile').read_text().splitlines() + justfile[0] = 'REPO_ROOT := "' + str(self.root) + '"' + (self.root / 'op-acceptance-tests/justfile').write_text('\n'.join(justfile) + '\n') + shutil.copyfile(ROOT / 'ops/ci/runtime/acceptance-tests.sh', self.root / 'ops/ci/runtime/acceptance-tests.sh') + shutil.copytree(ROOT / 'ops/ci/runtime/acceptance-circleci', self.root / 'ops/ci/runtime/acceptance-circleci') + for name in ['gotestsum-split.sh', 'shard-tests.sh']: + shutil.copy2(ROOT / 'ops/scripts' / name, self.root / 'ops/scripts' / name) + # The real split-test-logs.sh runs `python3 - `. An endpoint security + # tool can treat that file as a script, so a laptop run uses a stand-in. + # No assertion below reads the per-test logs. + stand_in = self.root / 'ops/scripts/split-test-logs.sh' + stand_in.write_text('#!/bin/sh\nexit 0\n'); stand_in.chmod(0o755) + for binary in ['rust/target/release/' + name for name in ['kona-host', 'kona-client', 'kona-node', 'op-zk-proposer', 'op-reth']] \ + + ['.circleci-cache/rust-binaries/kona-sp1-super-range-executor']: + path = self.root / binary + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text('#!/bin/sh\nexit 0\n'); path.chmod(0o755) + # The recipe lists plain test names with `go test -list`. + self.script('go', '''import os, sys +p, names = os.environ['FIXTURE_PACKAGE'], os.environ.get('FIXTURE_TESTS', 'TestOne TestTwo').split() +if sys.argv[1] == 'env': print('/fixture/go') +elif os.environ.get('LISTING_FAIL'): sys.exit(2) +else: print('\\n'.join(names) + '\\nok \\t' + p + '\\t0.01s') +''') + self.script('gotestsum', '''import json, os, re, sys +from pathlib import Path +a=sys.argv[1:] +def value(key): + for i,s in enumerate(a): + if s.startswith(key+'='): return s.split('=',1)[1] + if s == key: return a[i+1] +p, status = os.environ['FIXTURE_PACKAGE'], int(os.environ.get('VERDICT_STATUS','0')) +# The recipe runs one test per call and checks the events as go test writes them. +pattern, verdict = value('-run'), 'fail' if status else 'pass' +events = [] +for n in os.environ.get('FIXTURE_TESTS', 'TestOne TestTwo').split(): + if re.search(pattern, n): events += [{'Action':'run','Package':p,'Test':n}, {'Action':'output','Package':p,'Test':n,'Output':'--- '+verdict.upper()+': '+n+' (0.00s)\\n'}, {'Action':verdict,'Package':p,'Test':n}] +j=Path(value('--jsonfile')); j.parent.mkdir(parents=True,exist_ok=True); j.write_text(''.join(json.dumps(e,separators=(',',':'))+'\\n' for e in events)) +x=Path(value('--junitfile')); x.parent.mkdir(parents=True,exist_ok=True); x.write_text('') +Path(os.environ['FIXTURE_ARGS']).write_text(json.dumps(a)) +sys.exit(status) +''') + self.script('geth', "print('Geth fixture pinned version')") + self.script('mise', "import os; print(os.environ['FIXTURE_GETH'])") + self.env = {**os.environ, 'PATH': str(self.root / 'bin') + ':' + os.environ['PATH'], + 'CI_SHARD_INDEX': '0', 'FIXTURE_PACKAGE': PACKAGE, + 'FIXTURE_ARGS': str(self.root / 'args.json'), 'FIXTURE_GETH': str(self.root / 'bin/geth')} + + def script(self, name, body): + path = self.root / 'bin' / name + path.write_text('#!' + shutil.which('python3') + '\n' + body) + path.chmod(0o755) + + def wrapper(self, **env): + return subprocess.run(['bash', str(self.root / 'ops/ci/runtime/acceptance-tests.sh')], + env={**self.env, **env}, text=True, capture_output=True) + + def split(self, listed, index, total): + shim = self.root / 'ops/ci/runtime/acceptance-circleci/circleci' + result = subprocess.run([str(shim), 'tests', 'split', '--split-by=timings', '--timings-type=testname', '--time-default=1m', str(listed)], + env={**self.env, 'CIRCLE_NODE_INDEX': str(index), 'CIRCLE_NODE_TOTAL': str(total)}, + text=True, capture_output=True) + self.assertEqual(result.returncode, 0, result.stderr) + return result.stdout.splitlines() + + def test_runtime_supplies_verified_rust_paths_and_pinned_geth(self): + self.script('just', "import json, os; from pathlib import Path; Path(os.environ['FIXTURE_ARGS']).write_text(json.dumps({k:v for k,v in os.environ.items() if k.startswith('RUST_BINARY_PATH_')}))") + result = self.wrapper() + self.assertEqual(result.returncode, 0, result.stderr) + paths = json.loads((self.root / 'args.json').read_text()) + self.assertEqual(len(paths), 6) + self.assertTrue(all(Path(value).is_file() for value in paths.values())) + (self.root / 'args.json').unlink() + (self.root / 'bin/geth').unlink() + missing = self.wrapper() + self.assertNotEqual(missing.returncode, 0) + self.assertFalse((self.root / 'args.json').exists()) + + def test_verdict_runs_only_its_shard_and_writes_the_reports_that_rwx_collects(self): + result = self.wrapper() + self.assertEqual(result.returncode, 0, result.stderr + result.stdout) + # CircleCI's memory-all jobs run 16 tests at once on each of 4 nodes. + self.assertIn('TEST JOBS: 16\n', result.stdout) + self.assertIn('Node 0/4 running 1 tests', result.stdout) + args = json.loads((self.root / 'args.json').read_text()) + self.assertIn('-count=1', args) + self.assertEqual(args[args.index('-run') + 1:], ['^TestOne$', PACKAGE]) + # The verdict task's outputs name the files that the recipe writes. + definition = json.loads(subprocess.check_output( + ['yq', '-o=json', '.', str(ROOT / '.rwx/packages/acceptance-verdict.yml')], text=True)) + outputs = definition['tasks'][0]['outputs'] + junit = list(self.root.glob(outputs['test-results'][0]['path'])) + self.assertEqual([path.name for path in junit], ['results-0-' + PACKAGE.replace('/', '.') + '.TestOne.xml']) + self.assertTrue(list((self.root / outputs['artifacts'][0]['path']).glob('testrun-*/all.log'))) + + def test_failing_verdict_fails_the_wrapper(self): + result = self.wrapper(VERDICT_STATUS='1') + self.assertNotEqual(result.returncode, 0) + self.assertTrue((self.root / 'args.json').exists()) + + def test_shim_gives_every_listed_name_to_exactly_one_shard(self): + listed = self.root / 'listed.txt' + for count in (1, 4, 7): + names = ['Test%02d' % number for number in range(count)] + listed.write_text(''.join(name + '\n' for name in names)) + for total in (2, 4): + with self.subTest(count=count, total=total): + shards = [self.split(listed, index, total) for index in range(total)] + self.assertEqual(sorted(name for shard in shards for name in shard), names) + + def test_empty_shard_fails_without_running_tests(self): + # The recipe treats an empty shard as a failed split. + result = self.wrapper(CI_SHARD_INDEX='3') + self.assertNotEqual(result.returncode, 0) + self.assertIn('no tests assigned', result.stdout + result.stderr) + self.assertFalse((self.root / 'args.json').exists()) + + def test_listing_failure_never_runs_verdict(self): + result = self.wrapper(LISTING_FAIL='1') + self.assertNotEqual(result.returncode, 0) + self.assertFalse((self.root / 'args.json').exists()) + + def test_shim_rejects_every_other_circleci_invocation(self): + shim, listed = self.root / 'ops/ci/runtime/acceptance-circleci/circleci', self.root / 'listed.txt' + listed.write_text('TestOne\nTestTwo\n') + for argv in (['tests', 'glob', 'op-acceptance-tests/**'], ['tests', 'split'], ['tests', 'split', '--split-by=timings'], + ['tests', 'split', '--index=1', str(listed)], ['tests', 'split', str(listed), str(listed)], ['step', 'halt']): + with self.subTest(argv=argv): + result = subprocess.run([str(shim), *argv], env=self.env, text=True, capture_output=True) + self.assertNotEqual(result.returncode, 0) + self.assertIn('does not support', result.stderr) + + +if __name__ == '__main__': unittest.main() diff --git a/ops/ci/tests/test_ci_layout.py b/ops/ci/tests/test_ci_layout.py new file mode 100644 index 00000000000..c6c44b23a87 --- /dev/null +++ b/ops/ci/tests/test_ci_layout.py @@ -0,0 +1,68 @@ +"""Enforce removable migration tooling and native-only runtime inputs.""" +import ast +from pathlib import Path +import shutil +import subprocess +import tempfile +import unittest + +ROOT=Path(__file__).resolve().parents[3] +CI=ROOT/'ops/ci' + + +class LayoutTests(unittest.TestCase): + def test_all_helper_files_have_an_owner(self): + self.assertFalse([p.name for p in CI.iterdir() if p.is_file()]) + owners={p.name for p in CI.iterdir() if p.is_dir() and p.name!='__pycache__'} + self.assertTrue({'runtime','tests'} <= owners <= {'runtime','tests','migration'}) + + def test_runtime_and_permanent_tests_do_not_import_migration(self): + for directory in (CI/'runtime',CI/'tests'): + for path in directory.glob('*.py'): + tree=ast.parse(path.read_text()) + for node in ast.walk(tree): + if isinstance(node,(ast.Import,ast.ImportFrom)): + names=[n.name for n in node.names] if isinstance(node,ast.Import) else [node.module or ''] + self.assertFalse(any('migration' in name or 'compare_' in name for name in names),str(path)) + if isinstance(node,ast.Call): + function=node.func + is_path=isinstance(function,ast.Attribute) and function.attr in ('with_name','spec_from_file_location') + is_helper=isinstance(function,ast.Name) and function.id in ('helper','module') + if is_path or is_helper: + for value in node.args: + if isinstance(value,ast.Constant) and isinstance(value.value,str): + self.assertFalse(value.value.startswith(('compare-','migration/')),str(path)) + + def test_every_test_file_runs_in_some_rwx_task(self): + # No task fails for a test file that no task runs. Match the two forms + # the definitions use: unittest discovery and a direct script run. + import fnmatch,json,re + commands=[] + def walk(node): + if isinstance(node,dict): + for key,value in node.items(): + if key=='run' and isinstance(value,str):commands.append(value) + else:walk(value) + elif isinstance(node,list): + for value in node:walk(value) + for path in sorted((ROOT/'.rwx').rglob('*.yml')): + walk(json.loads(subprocess.check_output(['yq','-o=json','explode(.)',str(path)],text=True))) + tests=sorted(p.relative_to(ROOT).as_posix() for d in (CI/'tests',CI/'migration/tests') for p in d.glob('test_*.py')) + run=set() + for command in commands: + for directory,pattern in re.findall(r"discover -s (\S+) -p '([^']+)'",command): + run.update(t for t in tests if Path(t).parent.as_posix()==directory and fnmatch.fnmatch(Path(t).name,pattern)) + run.update(re.findall(r'(ops/ci/(?:migration/)?tests/test_\w+\.py)',command)) + self.assertEqual(sorted(set(tests)-run),[]) + + def test_every_runtime_module_imports_without_circle_or_migration(self): + with tempfile.TemporaryDirectory() as tmp: + root=Path(tmp); target=root/'ops/ci/runtime' + shutil.copytree(CI/'runtime', target, ignore=shutil.ignore_patterns('__pycache__')) + self.assertFalse((root/'.circleci').exists()) + self.assertFalse((root/'ops/ci/migration').exists()) + code='import importlib.util,sys;s=importlib.util.spec_from_file_location("runtime",sys.argv[1]);m=importlib.util.module_from_spec(s);s.loader.exec_module(m)' + for path in sorted(target.glob('*.py')): + with self.subTest(module=path.name): + result=subprocess.run(['python3','-c',code,str(path)],cwd=root,capture_output=True,text=True,timeout=10) + self.assertEqual(result.returncode,0,result.stderr) diff --git a/ops/ci/tests/test_ci_report.py b/ops/ci/tests/test_ci_report.py new file mode 100644 index 00000000000..cdfbe92d952 --- /dev/null +++ b/ops/ci/tests/test_ci_report.py @@ -0,0 +1,76 @@ +"""Exercise shared mechanics at the report and subprocess trust boundaries.""" +import importlib.util +import json +from pathlib import Path +import subprocess +import sys +import tempfile +import unittest + +SPEC = importlib.util.spec_from_file_location('ci_report', Path(__file__).resolve().parents[1] / 'runtime' / 'ci-report.py') +REPORT = importlib.util.module_from_spec(SPEC); SPEC.loader.exec_module(REPORT) + + +class ReportTests(unittest.TestCase): + def test_required_corrupt_and_unsafe_originals_are_rejected(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + path = root / 'original.json'; path.write_text('original evidence') + hashes = REPORT.file_hashes(root) + self.assertEqual(REPORT.verify_files(root, hashes, required={'original.json'}), hashes) + path.write_text('corrupted') + with self.assertRaisesRegex(ValueError, 'corrupt'): + REPORT.verify_files(root, hashes) + for manifest in ({}, {'../outside': 'a'*64}, {'/outside': 'a'*64}, {'original.json': 'invalid'}): + with self.subTest(manifest=manifest), self.assertRaises(ValueError): + REPORT.verify_files(root, manifest, required={'original.json'}) + + def test_missing_empty_original_is_rejected_without_modifying_the_directory(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + with self.assertRaises(OSError): + REPORT.verify_files(root, {"nested/empty.log": "a" * 64}) + self.assertEqual(list(root.iterdir()), []) + + def test_linked_artifact_bytes_are_not_accepted(self): + with tempfile.TemporaryDirectory() as tmp: + root=Path(tmp); outside=root/'outside'; outside.mkdir() + (outside/'same.log').write_text('original') + report=root/'report'; report.mkdir() + (report/'linked').symlink_to(outside, target_is_directory=True) + with self.assertRaisesRegex(ValueError, 'Linked'): + REPORT.verify_files(report, {'linked/same.log': REPORT.digest(outside/'same.log')}) + + def test_split_drains_large_stderr_without_contaminating_original_json(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + # Exceeds a pipe buffer. A serial reader would deadlock before JSON. + code = 'import sys;sys.stderr.write("x"*200000);print("{\\"complete\\":true}");sys.exit(17)' + harness = ('import importlib.util,sys;from pathlib import Path;' + 's=importlib.util.spec_from_file_location("r",sys.argv[1]);' + 'r=importlib.util.module_from_spec(s);s.loader.exec_module(r);' + 'sys.exit(r.stage(Path(sys.argv[2]),"failure",[sys.executable,"-c",sys.argv[3]],' + 'cwd=sys.argv[2],layout="split",stdout_json=True))') + result = subprocess.run([sys.executable, '-c', harness, str(Path(__file__).resolve().parents[1] / 'runtime' / 'ci-report.py'), tmp, code], + capture_output=True, timeout=10) + self.assertEqual(result.returncode, 17, result.stderr.decode()) + self.assertEqual(REPORT.read(root / 'failure.json'), {'complete': True}) + self.assertEqual((root / 'failure.stderr.log').stat().st_size, 200000) + stage = REPORT.read(root / 'failure.stage.json') + self.assertEqual(stage['exit_code'], 17) + self.assertEqual(stage['stderr_sha256'], REPORT.digest(root / 'failure.stderr.log')) + + def test_combined_separate_stdout_preserves_devnull_and_stage_schema(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + status = REPORT.stage(root, 'original', [sys.executable, '-c', + 'import sys;assert sys.stdin.read()=="";print("bytes");print("diagnostic",file=sys.stderr)'], + cwd=tmp, stdout_file='original.stdout', stdin=subprocess.DEVNULL) + self.assertEqual(status, 0) + self.assertEqual((root / 'original.stdout').read_text(), 'bytes\n') + self.assertEqual((root / 'original.log').read_text(), 'diagnostic\n') + row = REPORT.read(root / 'original.stage.json') + self.assertEqual(row['stdin'], 'devnull') + self.assertEqual(row['stdout_file'], 'original.stdout') + self.assertEqual(row['stdout_sha256'], REPORT.digest(root / 'original.stdout')) + self.assertEqual(row['log_sha256'], REPORT.digest(root / 'original.log')) diff --git a/ops/ci/tests/test_contract_l2_fork.py b/ops/ci/tests/test_contract_l2_fork.py new file mode 100644 index 00000000000..e57870505a8 --- /dev/null +++ b/ops/ci/tests/test_contract_l2_fork.py @@ -0,0 +1,120 @@ +#!/usr/bin/env python3 +"""Exercise the L2 fork runner's job environment, fork block, recipes and reruns, and the RPC pacer.""" +from contextlib import contextmanager +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +import importlib.util +from pathlib import Path +import tempfile +import threading +import unittest +from unittest import mock +import urllib.error +import urllib.request + +from test_contract_upgrades import RUNTIME, drive + +SPEC = importlib.util.spec_from_file_location('l2', RUNTIME / 'contract-l2-fork.py') +L = importlib.util.module_from_spec(SPEC); SPEC.loader.exec_module(L) +TESTS = ['just', 'test-l2-fork-upgrade'] + + +@contextmanager +def paced(upstream): + yield 'paced:' + upstream + + +class L2Tests(unittest.TestCase): + def setUp(self): + temporary = tempfile.TemporaryDirectory(); self.addCleanup(temporary.cleanup) + self.root = Path(temporary.name) + + def run_fork(self, statuses=(), block='latest'): + with mock.patch.object(L.UP.REPORT, 'command', return_value='66') as command, mock.patch.object(L.PACER, 'serve', paced): + status, calls = drive(L, self.root, ['run', '--fork-block', block], statuses) + return status, calls, command + + def test_latest_forks_the_current_head_then_checks_the_bundle_and_runs_the_tests(self): + status, calls, command = self.run_fork() + self.assertEqual(status, 0) + command.assert_called_once_with(['cast', 'block-number', '--rpc-url', L.RPC]) + self.assertEqual([call[:2] for call in calls], [('nut-bundle-check', ['just', 'nut-bundle-check-no-build']), ('tests', TESTS)]) + expected = {'CI': 'true', 'FOUNDRY_PROFILE': 'ci', 'L2_FORK_BLOCK_NUMBER': '66', + 'JUNIT_TEST_PATH': str(self.root / '.ci/contract-l2-fork/run/results.xml')} + # Only the tests read the fork, and they read it through the pacer. + self.assertEqual([call[2] for call in calls], [expected, {**expected, 'L2_FORK_RPC_URL': 'paced:https://mainnet.optimism.io'}]) + + def test_given_fork_block_is_used_without_a_request_to_the_rpc(self): + status, calls, command = self.run_fork(block='123') + command.assert_not_called() + self.assertEqual((status, calls[1][2]['L2_FORK_BLOCK_NUMBER']), (0, '123')) + + def test_nut_bundle_failure_prevents_the_tests(self): + status, calls, _ = self.run_fork([17]) + self.assertEqual((status, [call[0] for call in calls]), (17, ['nut-bundle-check'])) + + def test_test_failure_is_kept_and_the_rerun_writes_no_junit(self): + status, calls, _ = self.run_fork([0, 23, 0]) + self.assertEqual(status, 23) + self.assertEqual([call[1] for call in calls], [['just', 'nut-bundle-check-no-build'], TESTS, + ['just', 'test-l2-fork-upgrade-rerun']]) + self.assertEqual([call[2].get('L2_FORK_RPC_URL') for call in calls[1:]], ['paced:' + L.RPC] * 2) + # Like CircleCI's trace step, the rerun writes no JUnit, so forge prints its traces. + report = str(self.root / '.ci/contract-l2-fork/run/results.xml') + self.assertEqual([call[2].get('JUNIT_TEST_PATH') for call in calls], [report, report, None]) + + def test_prepare_builds_with_the_ci_profile_and_no_rpc(self): + status, calls = drive(L, self.root, ['prepare']) + self.assertEqual(status, 0) + expected = {'CI': 'true', 'FOUNDRY_PROFILE': 'ci'} + self.assertEqual(calls, [('go-ffi', ['just', 'build-go-ffi'], expected), ('contracts-build', ['forge', 'build'], expected)]) + + +class PacerTests(unittest.TestCase): + def setUp(self): + # A stand-in upstream that records each request and answers with the next (status, body). + self.seen = seen = []; self.answers = answers = [] + + class Upstream(BaseHTTPRequestHandler): + def do_POST(self): + seen.append((self.headers['User-Agent'], self.headers['Content-Type'], self.rfile.read(int(self.headers['Content-Length'])))) + status, body = answers.pop(0) + self.send_response(status); self.send_header('Content-Length', str(len(body))); self.end_headers(); self.wfile.write(body) + + def log_message(self, *_): pass + + server = ThreadingHTTPServer(('127.0.0.1', 0), Upstream) + threading.Thread(target=server.serve_forever, daemon=True).start() + self.addCleanup(server.server_close); self.addCleanup(server.shutdown) + self.upstream = 'http://127.0.0.1:' + str(server.server_port) + + def post(self, endpoint, body): + request = urllib.request.Request(endpoint, data=body, headers={'Content-Type': 'application/json', 'User-Agent': 'forge'}) + try: + with urllib.request.urlopen(request, timeout=10) as response: return response.status, response.read() + except urllib.error.HTTPError as failure: + with failure: return failure.code, failure.read() + + def test_client_receives_each_upstream_answer_unchanged_and_the_pacer_does_not_retry(self): + self.answers += [(200, b'{"result":"0x1"}'), (429, b'{"error":"slow down"}')] + with L.PACER.serve(self.upstream, wait=lambda seconds: None) as endpoint: + self.assertEqual(self.post(endpoint, b'{"id":1}'), (200, b'{"result":"0x1"}')) + # Forge owns the retries: it must see the rate limit answer itself. + self.assertEqual(self.post(endpoint, b'{"id":2}'), (429, b'{"error":"slow down"}')) + self.assertEqual(self.seen, [('forge', 'application/json', b'{"id":1}'), ('forge', 'application/json', b'{"id":2}')]) + + def test_requests_start_one_interval_apart(self): + now = [100.0]; waits = [] + def wait(seconds): waits.append(seconds); now[0] += seconds + self.answers += [(200, b'{}')] * 3 + with L.PACER.serve(self.upstream, rate=2, clock=lambda: now[0], wait=wait) as endpoint: + for _ in range(3): self.post(endpoint, b'{}') + # A request that arrives after its start time does not wait. + now[0] += 5; self.answers.append((200, b'{}')); self.post(endpoint, b'{}') + self.assertEqual(waits, [0, 0.5, 0.5, 0]) + + def test_unreachable_upstream_is_an_http_503_that_forge_retries(self): + with L.PACER.serve('http://127.0.0.1:1', wait=lambda seconds: None) as endpoint: + self.assertEqual(self.post(endpoint, b'{}'), (503, b'')) + + +if __name__ == '__main__': unittest.main() diff --git a/ops/ci/tests/test_contract_suites.py b/ops/ci/tests/test_contract_suites.py new file mode 100644 index 00000000000..0e1770bace5 --- /dev/null +++ b/ops/ci/tests/test_contract_suites.py @@ -0,0 +1,160 @@ +#!/usr/bin/env python3 +"""Exercise the suite runner's file selection, job environment, recipes and reruns.""" +import importlib.util +import json +import os +from pathlib import Path +import re +import shutil +import subprocess +import tempfile +import unittest +import xml.etree.ElementTree as ET + +from test_contract_upgrades import RUNTIME, drive, project + +SPEC = importlib.util.spec_from_file_location('suites', RUNTIME / 'contract-suites.py') +CS = importlib.util.module_from_spec(SPEC); SPEC.loader.exec_module(CS) +FIND = ['find', 'test', '-name', '*.t.sol'] + + +def listing(text): + """Return a stage effect that makes the stubbed test_list command print text.""" + def effect(directory, name): + if name == 'files': (directory / 'files.log').write_text(text) + return effect + + +def git(root, *arguments): + subprocess.check_call(['git', '-c', 'user.name=Fixture', '-c', 'user.email=fixture@example.invalid', *arguments], + cwd=root, stdout=subprocess.DEVNULL) + + +class SuiteTests(unittest.TestCase): + def setUp(self): + temporary = tempfile.TemporaryDirectory(); self.addCleanup(temporary.cleanup) + self.root = Path(temporary.name) + + def test_run_gives_just_test_the_file_list_then_lints_the_test_names(self): + status, calls = drive(CS, self.root, ['run', '--suite', 'standard', '--feature', 'CUSTOM_GAS_TOKEN'], + environment={'CI_BRANCH': 'pilot'}, effect=listing('test/L1/A.t.sol\ntest/L2/B.t.sol\n')) + self.assertEqual(status, 0) + self.assertEqual([call[:2] for call in calls], [('files', FIND), ('tests', ['just', 'test']), + ('lint-test-names', ['just', 'lint-forge-tests-check-no-build'])]) + # Like the CircleCI step, the recipe gets the files in TEST_FILES and a directory for its per-pass JUnit reports. + expected = {'CI_BRANCH': 'pilot', 'CI': 'true', 'FOUNDRY_PROFILE': 'liteci', 'SYS_FEATURE__CUSTOM_GAS_TOKEN': 'true', + 'TEST_FILES': 'test/L1/A.t.sol\ntest/L2/B.t.sol', + 'JUNIT_TEST_DIR': str(self.root / '.ci/contract-suites/standard-CUSTOM_GAS_TOKEN/run')} + self.assertEqual(calls[1][2], expected) + + def test_failed_tests_are_rerun_for_traces_without_junit_and_skip_the_lint(self): + arguments = ['run', '--suite', 'standard', '--feature', 'main'] + status, calls = drive(CS, self.root, arguments, [0, 1, 1], {'CI_BRANCH': 'pilot'}, listing('test/L1/A.t.sol\n')) + self.assertEqual(status, 1) + self.assertEqual([call[:2] for call in calls], [('files', FIND), ('tests', ['just', 'test']), ('rerun', ['just', 'test-rerun'])]) + self.assertEqual(['JUNIT_TEST_DIR' in call[2] for call in calls], [False, True, False]) + # A failed lint fails the task after passing tests. A failed test_list command stops it at once. + for statuses, expected in [([0, 0, 5], (5, ['files', 'tests', 'lint-test-names'])), ([2], (2, ['files']))]: + status, calls = drive(CS, self.root, arguments, statuses, {'CI_BRANCH': 'pilot'}, listing('test/L1/A.t.sol\n')) + self.assertEqual((status, [call[0] for call in calls]), expected) + + def test_prepare_builds_with_the_profile_of_the_circleci_job(self): + for suite, branch, profile in [('standard', 'pilot', 'liteci'), ('standard', 'develop', 'ci'), + ('modified', 'pilot', 'ciheavy'), ('modified', 'develop', 'ciheavy')]: + with self.subTest(suite=suite, branch=branch): + status, calls = drive(CS, self.root, ['prepare', '--suite', suite, '--feature', 'main'], + environment={'CI_BRANCH': branch}, effect=listing('test/L1/A.t.sol\n')) + self.assertEqual((status, [call[1] for call in calls]), + (0, [CS.FILE_COMMANDS[suite], ['just', 'build-go-ffi'], ['forge', 'build']])) + self.assertEqual({call[2]['FOUNDRY_PROFILE'] for call in calls}, {profile}) + + def test_empty_file_list_stops_with_success_before_any_build_or_test(self): + # With no TEST_FILES, `just test` would run every test with the heavy profile. + for mode in ('prepare', 'run'): + with self.subTest(mode=mode): + status, calls = drive(CS, self.root, [mode, '--suite', 'modified', '--feature', 'main'], + environment={'CI_BRANCH': 'pilot'}, effect=listing('\n')) + self.assertEqual((status, [call[0] for call in calls]), (0, ['files'])) + + def test_modified_file_list_is_circleci_heavy_fuzz_test_list(self): + # contracts-bedrock-tests-heavy-fuzz-modified owns the changed-file selection. + config = (CS.UP.ROOT / '.circleci/continue/main.yml').read_text() + match = re.search(r"^( *)test_list: (git diff .*\n(?:\1 .*\n)*)", config, re.M) + self.assertIsNotNone(match) + self.assertEqual(' '.join(match[2].split()), CS.FILE_COMMANDS['modified'][-1]) + + def test_real_diff_against_the_fetched_develop_selects_only_changed_test_files(self): + # git is real. just and forge are stubs that record their arguments and TEST_FILES. + contracts, invoke = project(self.root, '', {'test/unit/Fixture.t.sol': '// fixture\n', 'test/unit/Other.t.sol': '// other\n', + 'test/unit/Support.sol': '// not a test\n'}) + tools = self.root / 'bin'; tools.mkdir(); log = self.root / 'calls.log' + for name in ('just', 'forge'): + (tools / name).write_text('#!/bin/sh\necho "' + name + ' $*|$TEST_FILES" >> "$CALL_LOG"\n'); (tools / name).chmod(0o755) + git(self.root, 'init', '-q'); git(self.root, 'add', '.'); git(self.root, 'commit', '-qm', 'base') + git(self.root, 'update-ref', 'refs/remotes/origin/develop', 'HEAD') + def run(mode): + result = invoke('contract-suites.py', mode, '--suite', 'modified', '--feature', 'main', + PATH=str(tools) + os.pathsep + os.environ['PATH'], CALL_LOG=str(log)) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + return result.stdout + for mode in ('prepare', 'run'): self.assertIn('No test files to run.', run(mode)) + self.assertFalse(log.exists()) + for name in ('test/unit/Fixture.t.sol', 'test/unit/Support.sol'): (contracts / name).write_text('// changed\n') + git(self.root, 'commit', '-qam', 'change') + run('prepare'); run('run') + self.assertEqual(log.read_text().splitlines(), ['just build-go-ffi|', 'forge build|', 'just test|test/unit/Fixture.t.sol', + 'just lint-forge-tests-check-no-build|test/unit/Fixture.t.sol']) + + +@unittest.skipUnless(os.environ.get('RWX_LIVE_CONTRACT_SUITE_FIXTURE') == '1', 'Opt-in fixture: needs the pinned just, go and forge, and the Go modules') +class LiveSuiteTests(unittest.TestCase): + def test_real_recipes_run_both_evm_passes_lint_and_rerun_a_failure(self): + temporary = tempfile.TemporaryDirectory(); self.addCleanup(temporary.cleanup); root = Path(temporary.name) + contracts, invoke = project(root, '[profile.default]\nsrc="src"\ntest="test"\nout="forge-artifacts"\nsolc="0.8.15"\nffi=true\nast=true\n' + '[profile.liteci]\noptimizer=false\n[profile.ciheavy]\noptimizer=false\n', { + 'src/unit/Fixture.sol': '// SPDX-License-Identifier: MIT\npragma solidity 0.8.15; interface Vm {function skip(bool,string calldata) external;function ffi(string[] calldata) external returns(bytes memory);}contract Fixture {uint256 public value;uint256 public copied;function runtime(uint256 a) public returns(uint256){value=a;copied=a;return a;}}', + 'test/unit/Fixture.t.sol': '// SPDX-License-Identifier: MIT\npragma solidity 0.8.15; import {Fixture,Vm} from "../../src/unit/Fixture.sol";contract Fixture_Runtime_Test {Vm constant vm=Vm(address(uint160(uint256(keccak256("hevm cheat code")))));function test_runtime_succeeds() public {string[] memory args=new string[](1);args[0]="./scripts/go-ffi/go-ffi";assert(abi.decode(vm.ffi(args),(uint256))==1);}function testFuzz_runtime_succeeds(uint256 a) public {assert(new Fixture().runtime(a)==a);}function test_runtime_disabled_succeeds() public {vm.skip(true,"fixture disabled");}}', + 'src/L2/InvariantFixture.sol': '// SPDX-License-Identifier: MIT\npragma solidity 0.8.15; import {Fixture} from "../unit/Fixture.sol";contract InvariantFixture is Fixture {}', + # A default-EVM test makes the standard suite run both EVM passes. + 'test/L2/InvariantFixture.t.sol': '// SPDX-License-Identifier: MIT\npragma solidity 0.8.15; import {InvariantFixture} from "../../src/L2/InvariantFixture.sol";contract InvariantFixture_Runtime_Test {InvariantFixture fixture;function setUp() public {fixture=new InvariantFixture();}function targetContracts() public view returns(address[] memory targets){targets=new address[](1);targets[0]=address(fixture);}function invariant_runtime_succeeds() public view {assert(fixture.value()==fixture.copied());}}'}) + # The lint recipe runs the real test-name checker, so the project needs its Go packages. + real = CS.UP.ROOT + text = subprocess.check_output(['go', 'list', '-deps', '-json', './packages/contracts-bedrock/scripts/checks/test-validation'], cwd=real, text=True) + decoder, offset = json.JSONDecoder(), 0 + while offset < len(text): + if text[offset].isspace(): offset += 1; continue + row, offset = decoder.raw_decode(text, offset) + if not row.get('Module', {}).get('Main'): continue + source = Path(row['Dir']) + for name in [f.name for f in source.glob('*.go')] + row.get('EmbedFiles', []): + target = root / source.relative_to(real) / name; target.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(source / name, target) + for name in ('go.mod', 'go.sum', 'packages/contracts-bedrock/scripts/checks/test-validation/exclusions.toml'): + shutil.copy2(real / name, root / name) + git(root, 'init', '-q'); git(root, 'add', '.'); git(root, 'commit', '-qm', 'base') + git(root, 'update-ref', 'refs/remotes/origin/develop', 'HEAD') + path = contracts / 'test/unit/Fixture.t.sol'; path.write_text(path.read_text() + '\n// Changed test file.\n') + git(root, 'commit', '-qam', 'changed test') + amsterdam = {'test_runtime_succeeds()', 'testFuzz_runtime_succeeds(uint256)', 'test_runtime_disabled_succeeds()'} + marker = root / 'ffi-calls' + for suite, passes in [('standard', {'results-amsterdam.xml': amsterdam, 'results-l2.xml': {'invariant_runtime_succeeds()'}}), + ('modified', {'results-amsterdam.xml': amsterdam})]: + with self.subTest(suite=suite): + before = len(marker.read_text()) if marker.exists() else 0 + for mode in ('prepare', 'run'): + result = invoke('contract-suites.py', mode, '--suite', suite, '--feature', 'main', GOFLAGS='-mod=readonly') + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + report = root / '.ci/contract-suites' / (suite + '-main') / 'run' + reports = {p.name: {case.get('name') for case in ET.parse(p).iter('testcase')} for p in report.glob('results-*.xml')} + self.assertEqual(reports, passes) + self.assertEqual(len(marker.read_text()), before + 1) + self.assertIn('All contract test validations passed', (report / 'lint-test-names.log').read_text()) + result = invoke('contract-suites.py', 'run', '--suite', 'standard', '--feature', 'main', GOFLAGS='-mod=readonly', CI_FIXTURE_FAIL='1') + self.assertNotEqual(result.returncode, 0) + report = root / '.ci/contract-suites/standard-main/run' + self.assertIn('', 'reverted: ')) + for private in (b'secret-token-of-32-characters', b'query-token-value'): self.assertNotIn(private, report) + + def test_run_without_an_archive_url_stops_before_any_stage(self): + for environment in ({'CI_BRANCH': 'pilot'}, {'CI_BRANCH': 'pilot', 'OP_CI_MAINNET_L1_ARCHIVE_RPC_URL': 'file:///secret'}): + status, calls = drive(UP, self.root, ['run', '--variant', 'chain-op'], environment=environment) + self.assertIn('OP_CI_MAINNET_L1_ARCHIVE_RPC_URL', status); self.assertNotIn('secret', status); self.assertEqual(calls, []) + + def test_short_reflected_credentials_are_redacted(self): + url = 'https://user:pass@provider.invalid/v2/short-token?apikey=abc123' + redactor = UP.Redactor(url) + for value in (url, 'user', 'pass', 'short-token', 'abc123'): + self.assertNotIn(value.encode(), redactor(value.encode())) + + def test_redaction_covers_url_xml_encoded_credentials_and_fragments(self): + value = 'https://user:password-token@example.invalid/api/path-token-of-32-characters?key=query-token-value&chain=1' + mask = UP.Redactor(value) + for data in (value, UP.html.escape(value), UP.urllib.parse.quote(value, safe=''), 'password-token', 'query-token-value', 'path-token-of-32-characters'): + self.assertNotIn(data.encode(), mask(data.encode())) + with self.assertRaises(ValueError): UP.Redactor('file:///secret') + + def test_real_stage_redacts_both_streams_before_saving_and_keeps_the_exit_status(self): + url = 'https://example.invalid/a-test-secret-token' + child = 'import sys;print(sys.argv[1]);print(sys.argv[1],file=sys.stderr);sys.exit(17)' + with patch.object(UP, 'CONTRACTS', self.root): + self.assertEqual(UP.stage(self.root, 'intentional', [sys.executable, '-c', child, url], UP.Redactor(url)), 17) + for name in ('intentional.log', 'intentional.stderr.log'): + self.assertEqual((self.root / name).read_text(), '\n') + for file in self.root.iterdir(): self.assertNotIn(b'a-test-secret-token', file.read_bytes()) + + def test_real_process_group_cancellation_returns_the_signal_and_keeps_the_partial_log(self): + # run_tests skips the rerun after a cancellation: stage reports 128 plus the signal. + source = ('import importlib.util,os,signal,sys,threading;from pathlib import Path;' + 's=importlib.util.spec_from_file_location("u",sys.argv[1]);u=importlib.util.module_from_spec(s);s.loader.exec_module(u);' + 'u.CONTRACTS=Path(sys.argv[2]);' + 'threading.Timer(0.4,lambda:os.kill(os.getpid(),signal.SIGTERM)).start();' + 'sys.exit(u.stage(Path(sys.argv[2]),"canceled",[sys.executable,"-u","-c","import time;print(123,flush=True);time.sleep(20)"]))') + result = subprocess.run([sys.executable, '-c', source, UP.__file__, str(self.root)], capture_output=True, timeout=10) + self.assertEqual(result.returncode, 143, result.stderr.decode()) + self.assertEqual((self.root / 'canceled.log').read_text().strip(), '123') + self.assertEqual(json.loads((self.root / 'canceled.stage.json').read_text())['exit_code'], -signal.SIGTERM) + + +@unittest.skipUnless(os.environ.get('RWX_LIVE_FORGE_FIXTURE') == '1', 'Opt-in fixture: needs the pinned just, go, forge and anvil') +class LiveForgeTests(unittest.TestCase): + def test_real_recipes_fork_through_the_archive_url_and_redact_it_from_a_failure(self): + temporary = tempfile.TemporaryDirectory(); self.addCleanup(temporary.cleanup); root = Path(temporary.name) + _, invoke = project(root, '[profile.default]\nsrc="src"\ntest="test"\nout="forge-artifacts"\nsolc="0.8.15"\n' + 'ffi=true\nno_storage_caching=true\n[profile.liteci]\noptimizer=true\n', {'test/L1/Fixture.t.sol': + 'pragma solidity 0.8.15; ' + VM + 'contract Fixture is Cheats {' + # A failing helper makes the failure reason quote the archive URL. + 'function test_ffi() public {require(ffi()==1,vm.envString("FORK_RPC_URL"));} ' + 'function test_fork() public {vm.createSelectFork(vm.envString("FORK_RPC_URL"),66);assert(block.number==66);} ' + 'function testFuzz_fresh(uint256 a) public pure {assert(a==a);} ' + 'function test_skip() public {vm.skip(true);}}'}) + (root / 'go.mod').write_text('module example.invalid/upgrade-fixture\n\ngo 1.22\n') + # The local chain has no block at 00:00 UTC, so the pinned-block lookup of the recipe gets a fixed answer. + tools = root / 'bin'; tools.mkdir(); (tools / 'cast').write_text('#!/bin/sh\necho 66\n'); (tools / 'cast').chmod(0o755) + with socket.socket() as listener: + listener.bind(('127.0.0.1', 0)); port = listener.getsockname()[1] + anvil = subprocess.Popen(['anvil', '--silent', '--port', str(port), '--chain-id', '1'], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + self.addCleanup(anvil.wait, 10); self.addCleanup(anvil.terminate) + def call(body): + request = urllib.request.Request('http://127.0.0.1:' + str(port), data=body, headers={'Content-Type': 'application/json'}) + with urllib.request.urlopen(request, timeout=5) as response: return response.read() + for _ in range(100): + try: call(b'{"jsonrpc":"2.0","id":1,"method":"anvil_mine","params":["0x42"]}'); break + except OSError: time.sleep(0.05) + else: self.fail('Anvil did not start') + # This relay gives the chain a URL with a key in its path and its query, like a hosted archive RPC. + class Rpc(BaseHTTPRequestHandler): + def do_POST(self): + response = call(self.rfile.read(int(self.headers['Content-Length']))) + self.send_response(200); self.send_header('Content-Type', 'application/json') + self.send_header('Content-Length', str(len(response))); self.end_headers(); self.wfile.write(response) + def log_message(self, *_): pass + server = ThreadingHTTPServer(('127.0.0.1', 0), Rpc); thread = threading.Thread(target=server.serve_forever, daemon=True); thread.start() + self.addCleanup(server.server_close); self.addCleanup(thread.join, 5); self.addCleanup(server.shutdown) + environment = {'PATH': str(tools) + os.pathsep + os.environ['PATH'], + 'OP_CI_MAINNET_L1_ARCHIVE_RPC_URL': 'http://127.0.0.1:' + str(server.server_port) + '/short-token?apikey=abc123'} + for arguments in (['compile'], ['run', '--variant', 'feature-main']): + result = invoke('contract-upgrades.py', *arguments, **environment) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + report = root / '.ci/contract-upgrades/feature-main/run' + cases = {case.get('name'): case for case in ET.parse(report / 'results.xml').iter('testcase')} + self.assertEqual(set(cases), {'test_ffi()', 'test_fork()', 'testFuzz_fresh(uint256)', 'test_skip()'}) + self.assertEqual([name for name, case in cases.items() if case.find('skipped') is not None], ['test_skip()']) + self.assertFalse(any(case.find('failure') is not None for case in cases.values())) + self.assertTrue((root / 'ffi-calls').read_text()) + result = invoke('contract-upgrades.py', 'run', '--variant', 'feature-main', CI_FIXTURE_FAIL='1', **environment) + self.assertNotEqual(result.returncode, 0) + self.assertIn('', (report / 'rerun.log').read_text()) + for private in ('short-token', 'abc123'): self.assertNotIn(private, saved) + + +if __name__ == '__main__': unittest.main() diff --git a/ops/ci/tests/test_go_artifacts.py b/ops/ci/tests/test_go_artifacts.py new file mode 100644 index 00000000000..3016bfe55d8 --- /dev/null +++ b/ops/ci/tests/test_go_artifacts.py @@ -0,0 +1,133 @@ +#!/usr/bin/env python3 +"""Check that dependency artifacts restore, and that stale or corrupt ones are rejected.""" +import importlib.util +import json +import os +import shutil +import subprocess +from pathlib import Path +import tempfile +import unittest +from unittest.mock import patch + +SPEC = importlib.util.spec_from_file_location('artifacts', Path(__file__).resolve().parents[1] / 'runtime' / 'go-artifacts.py') +ART = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(ART) + +class GoArtifactsTest(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + self.root = Path(self.temp.name) + (self.root / 'fixture').mkdir() + (self.root / 'fixture/binary').write_text('compiled') + self.addCleanup(patch.stopall) + patch.object(ART, 'ROOT', self.root).start() + patch.dict(os.environ, CI_COMMIT_SHA='a'*40).start() + # The checkout is at the requested commit: `git rev-parse HEAD` prints it. + patch.object(ART.subprocess, 'check_output', return_value='a'*40).start() + ART.pack('go', ['fixture']) + self.artifact = self.root / '.ci/go-tests/dependencies/go' + + def test_restores_the_packed_files(self): + (self.root / 'fixture/binary').unlink() + ART.restore('go', self.artifact) + self.assertEqual((self.root / 'fixture/binary').read_text(), 'compiled') + + def test_artifact_of_another_revision_or_kind_is_rejected(self): + metadata = self.artifact / 'metadata.json' + original = json.loads(metadata.read_text()) + metadata.write_text(json.dumps({**original, 'commit_sha': 'b'*40})) + with self.assertRaises(ValueError): ART.restore('go', self.artifact) + metadata.write_text(json.dumps(original)) + with self.assertRaises(ValueError): ART.restore('contracts', self.artifact) + with patch.dict(os.environ, CI_COMMIT_SHA='c'*40): + with self.assertRaises(ValueError): ART.restore('go', self.artifact) + + def test_missing_and_corrupt_archives_are_rejected(self): + archive = self.artifact / 'files.tar.gz' + archive.write_bytes(b'corrupt') + with self.assertRaises(ValueError): ART.restore('go', self.artifact) + archive.unlink() + with self.assertRaises(OSError): ART.restore('go', self.artifact) + + def test_missing_output_and_revision_fail_at_producer(self): + with self.assertRaises(ValueError): ART.pack('go', ['absent']) + with patch.dict(os.environ, CI_COMMIT_SHA='b'*40): + with self.assertRaises(ValueError): ART.pack('go', ['fixture']) + + def test_readonly_materialized_cache_is_verified_without_overwriting(self): + binary = self.root / 'fixture/binary'; directory = binary.parent + binary.chmod(0o444); directory.chmod(0o555) + self.addCleanup(directory.chmod, 0o755) + inode, changed = binary.stat().st_ino, binary.stat().st_mtime_ns + ART.restore('go', self.artifact); ART.restore('go', self.artifact) + self.assertEqual(binary.stat().st_ino, inode) + self.assertEqual(binary.stat().st_mtime_ns, changed) + self.assertEqual(binary.stat().st_mode & 0o777, 0o444) + self.assertEqual(directory.stat().st_mode & 0o777, 0o555) + self.assertEqual(binary.read_text(), 'compiled') + + def test_same_bytes_do_not_hide_lost_executable_permissions(self): + binary = self.root / 'fixture/binary' + binary.chmod(0o755) + ART.pack('go', ['fixture']) + binary.chmod(0o644) + ART.restore('go', self.artifact) + self.assertEqual(binary.stat().st_mode & 0o777, 0o755) + + def test_changed_destination_file_is_replaced(self): + binary = self.root / 'fixture/binary' + binary.write_text('stale') + ART.restore('go', self.artifact) + self.assertEqual(binary.read_text(), 'compiled') + +class ReusableFilteredConsumerTest(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + self.root = Path(self.temp.name) + runtime = self.root / 'ops/ci/runtime' + runtime.mkdir(parents=True) + shutil.copy2(Path(ART.__file__).with_name('go-dependency-source.py'), runtime / 'go-dependency-source.py') + (self.root / 'mise.toml').write_text('pinned') + (self.root / 'justfiles').mkdir() + (self.root / 'justfiles/producer.just').write_text('build input') + (self.root / '.gitignore').write_text('.ci/\nfixture/\n') + (self.root / 'fixture').mkdir() + (self.root / 'fixture/binary').write_text('compiled') + self.git('init', '-q') + self.git('add', '.') + self.git('-c', 'user.name=Fixture', '-c', 'user.email=fixture@example.invalid', 'commit', '-qm', 'source') + self.sha = self.git('rev-parse', 'HEAD').strip() + self.addCleanup(patch.stopall) + patch.object(ART, 'ROOT', self.root).start() + patch.dict(os.environ, CI_COMMIT_SHA=self.sha, CI_GO_REUSABLE_DEPENDENCY='1').start() + ART.pack('go', ['fixture']) + self.artifact = self.root / '.ci/go-tests/dependencies/go' + + def git(self, *args): + return subprocess.check_output(['git', *args], cwd=self.root, text=True) + + def test_same_commit_projection_restores_but_ancestor_projection_fails_closed(self): + omitted = self.root / 'justfiles/producer.just' + omitted.unlink() # A runtime input filter omits a build-only tracked file. + ART.restore('go', self.artifact) + self.assertEqual((self.root / 'fixture/binary').read_text(), 'compiled') + self.git('checkout', '--', 'justfiles/producer.just') + (self.root / 'docs.txt').write_text('unrelated edit') + self.git('add', 'docs.txt') + self.git('-c', 'user.name=Fixture', '-c', 'user.email=fixture@example.invalid', 'commit', '-qm', 'unrelated') + with patch.dict(os.environ, CI_COMMIT_SHA=self.git('rev-parse', 'HEAD').strip()): + ART.restore('go', self.artifact) # Complete, identical ancestor inputs. + omitted.unlink() + with self.assertRaisesRegex(ValueError, 'Modified dependency build inputs'): + ART.restore('go', self.artifact) + + def test_reusable_producer_rejects_modified_tracked_inputs(self): + (self.root / 'justfiles/producer.just').write_text('dirty producer input') + with self.assertRaisesRegex(ValueError, 'Modified dependency build inputs'): + ART.pack('go', ['fixture']) + + +if __name__ == '__main__': unittest.main() diff --git a/ops/ci/tests/test_go_compiled_tests.py b/ops/ci/tests/test_go_compiled_tests.py new file mode 100644 index 00000000000..47d22d8398e --- /dev/null +++ b/ops/ci/tests/test_go_compiled_tests.py @@ -0,0 +1,253 @@ +#!/usr/bin/env python3 +"""Check compilation of a shard, binary execution, rerun selection and cancellation.""" + +import json +import os +from pathlib import Path +import shutil +import subprocess +import signal +import time +import tempfile +import unittest + + +SCRIPTS = Path(__file__).resolve().parents[1] / 'runtime' +MODULE = "github.com/ethereum-optimism/optimism" +PREFIX = MODULE + "/op-node/rollup" +# Stands in for the Go command: `go build -o cmd/test2json` and `go test -c ... -o `. +GO = '''#!/usr/bin/env python3 +import json, os, pathlib, sys +args = sys.argv[1:] +with open(os.environ['GO_CALLS'], 'a') as calls: + calls.write(json.dumps({'args': args, 'goroot': os.environ.get('GOROOT')}) + '\\n') +package = args[-1] +if package == os.environ.get('GO_FAIL_PACKAGE'): + sys.exit(1) +if not package.endswith('/mocks') and package != os.environ.get('GO_NO_BINARY_PACKAGE'): + pathlib.Path(args[args.index('-o') + 1]).write_text('compiled ' + package) +''' + + +class CompiledGoTest(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + self.root = Path(self.temp.name).resolve() + scripts = self.root / "ops/ci/runtime" + scripts.mkdir(parents=True) + shutil.copy2(SCRIPTS / "go-compiled-tests.py", scripts / "go-compiled-tests.py") + self.build = self.root / ".ci/go-tests/build" + self.build.mkdir(parents=True) + self.packages = [PREFIX, PREFIX + "/derive", PREFIX + "/derive/mocks"] + self.manifest = {"commit_sha": "a" * 40, "shards": [self.packages], "no_tests": [self.packages[2]]} + self.save_manifest() + self.executable(self.build / "test2json", '''#!/usr/bin/env python3 +import os, signal, sys +signal.signal(signal.SIGINT, signal.SIG_IGN) +signal.signal(signal.SIGQUIT, signal.SIG_IGN) +os.environ['REPORT_PACKAGE'] = sys.argv[3] +os.execv(sys.argv[4], sys.argv[4:]) +''') + for index, package in enumerate(self.packages): + directory = self.root / package.removeprefix(MODULE + "/") + directory.mkdir(parents=True, exist_ok=True) + (directory / "fixture.txt").write_text(package) + if package in self.manifest["no_tests"]: + continue + self.executable(self.build / f"{index}.test", '''#!/usr/bin/env python3 +import json, os, pathlib, sys +package = os.environ['REPORT_PACKAGE'] +assert pathlib.Path('fixture.txt').read_text() == package +with open(os.environ['BINARY_CALLS'], 'a') as calls: + calls.write(json.dumps({'package': package, 'args': sys.argv[1:], 'goroot': os.environ.get('GOROOT')}) + '\\n') +print(json.dumps({'Action': 'run', 'Package': package, 'Test': 'TestFresh'})) +failed = os.environ.get('FAIL_PACKAGE') == package +print(json.dumps({'Action': 'fail' if failed else 'pass', 'Package': package, 'Test': 'TestFresh'})) +print(json.dumps({'Action': 'fail' if failed else 'pass', 'Package': package, 'Elapsed': 0.01})) +sys.exit(1 if failed else 0) +''') + tools = self.root / "bin" + tools.mkdir() + self.executable(tools / "go", GO) + self.calls = self.root / "calls.jsonl" + self.go_calls = self.root / "go-calls.jsonl" + self.env = {**os.environ, "CI_COMMIT_SHA": "a" * 40, "CI_SHARD_INDEX": "0", "PARALLEL": "8", "TEST_TIMEOUT": "40m", + "BINARY_CALLS": str(self.calls), "GO_CALLS": str(self.go_calls), + "PATH": str(tools) + os.pathsep + os.environ["PATH"]} + + def executable(self, path, source): + path.write_text(source) + path.chmod(0o755) + + def save_manifest(self): + (self.build.parent / "manifest.json").write_text(json.dumps(self.manifest)) + + def command(self, *args): + return ["python3", str(self.root / "ops/ci/runtime/go-compiled-tests.py"), *args] + + def run_helper(self, *args, **env): + return subprocess.run(self.command(*args), env={**self.env, **env}, capture_output=True, text=True) + + def recorded(self, path): + return [json.loads(line) for line in path.read_text().splitlines()] + + def test_build_compiles_the_shard_and_keeps_its_manifest(self): + for binary in self.build.glob("*.test"): + binary.unlink() + (self.build / "2.test").write_text("binary of an earlier shard layout") + result = self.run_helper("build", GOROOT="/stale-parent-sdk") + self.assertEqual(result.returncode, 0, result.stderr) + calls = self.recorded(self.go_calls) + self.assertEqual(calls[0]["args"], ["build", "-o", str(self.build / "test2json"), "cmd/test2json"]) + self.assertEqual(sorted(call["args"] for call in calls[1:]), + [["test", "-c", "-p=4", "-tags=ci", "-o", str(self.build / f"{index}.test"), package] + for index, package in enumerate(self.packages)]) + self.assertEqual([call["goroot"] for call in calls], [None] * 4) + self.assertEqual(sorted(path.name for path in self.build.glob("*.test")), ["0.test", "1.test"]) + self.assertEqual(json.loads((self.build / "manifest.json").read_text()), self.manifest) + self.assertFalse(self.calls.exists(), "Compilation must run no test binary") + + def test_build_fails_for_another_commits_manifest_a_compile_error_or_a_missing_binary(self): + result = self.run_helper("build", CI_COMMIT_SHA="b" * 40) + self.assertNotEqual(result.returncode, 0) + self.assertIn("different commit", result.stderr) + self.assertFalse(self.go_calls.exists()) + for variable in ("GO_FAIL_PACKAGE", "GO_NO_BINARY_PACKAGE"): + with self.subTest(variable=variable): + self.assertNotEqual(self.run_helper("build", **{variable: self.packages[1]}).returncode, 0) + self.assertFalse((self.build / "manifest.json").exists()) + + def test_restore_replaces_an_older_manifest_with_the_compiled_one(self): + shutil.copyfile(self.build.parent / "manifest.json", self.build / "manifest.json") + self.manifest = {"commit_sha": "0" * 40, "shards": [[PREFIX + "/removed"]], "no_tests": []} + self.save_manifest() + self.assertNotEqual(self.run_helper("run").returncode, 0) + self.assertFalse(self.calls.exists()) + result = self.run_helper("restore") + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual((self.build.parent / "manifest.json").read_bytes(), (self.build / "manifest.json").read_bytes()) + self.assertEqual(self.run_helper("run").returncode, 0) + self.assertEqual(len(self.recorded(self.calls)), 2) + + def test_every_attempt_executes_binaries_and_preserves_flags_cwd_and_no_test_packages(self): + for _ in range(2): + result = self.run_helper("run", GOROOT="/stale-parent-sdk") + self.assertEqual(result.returncode, 0, result.stderr) + events = [json.loads(line) for line in result.stdout.splitlines()] + self.assertEqual(sorted({event["Package"] for event in events}), self.packages) + self.assertEqual([event["Action"] for event in events if event["Package"] == self.packages[2]], ["start", "skip"]) + calls = self.recorded(self.calls) + self.assertEqual(len(calls), 4) + for call in calls: + self.assertIn("-test.count=1", call["args"]) + self.assertIn("-test.parallel=8", call["args"]) + self.assertIn("-test.timeout=40m", call["args"]) + self.assertIn("-test.paniconexit0", call["args"]) + self.assertFalse(any(arg.startswith("-test.run") for arg in call["args"])) + # A test binary that runs `go` must find the SDK of the go on PATH. + self.assertIsNone(call["goroot"]) + + def test_run_executes_only_its_own_shard(self): + self.manifest["shards"] = [[self.packages[2]], self.packages[:2]] + self.save_manifest() + result = self.run_helper("run", CI_SHARD_INDEX="1") + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(sorted(call["package"] for call in self.recorded(self.calls)), self.packages[:2]) + self.calls.unlink() + result = self.run_helper("run", CI_SHARD_INDEX="0") + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual({json.loads(line)["Package"] for line in result.stdout.splitlines()}, {self.packages[2]}) + self.assertFalse(self.calls.exists()) + self.assertNotEqual(self.run_helper("run", CI_SHARD_INDEX="2").returncode, 0) + + def test_failures_are_retained_and_only_gotestsum_reruns_can_select_cases(self): + failed = self.run_helper("run", FAIL_PACKAGE=self.packages[1]) + self.assertEqual(failed.returncode, 1) + self.assertIn('"Action": "fail"', failed.stdout) + self.calls.unlink() + rerun = self.run_helper("run", "-test.run=^TestFresh$/^nested$", self.packages[1]) + self.assertEqual(rerun.returncode, 0, rerun.stderr) + calls = self.recorded(self.calls) + self.assertEqual(len(calls), 1) + self.assertEqual(calls[0]["package"], self.packages[1]) + self.assertIn("-test.run=^TestFresh$/^nested$", calls[0]["args"]) + + def test_other_rerun_arguments_fail_before_execution(self): + for args in (("-run=TestFresh",), ("-run=TestFresh", PREFIX + "/outside"), ("-short", self.packages[0]), + ("-test.run=TestFresh", PREFIX + "/outside")): + with self.subTest(args=args): + self.assertNotEqual(self.run_helper("run", *args).returncode, 0) + self.assertFalse(self.calls.exists()) + + def test_invalid_package_timeout_fails_before_spawning_binaries(self): + result = self.run_helper('run', TEST_TIMEOUT='invalid') + self.assertNotEqual(result.returncode, 0) + self.assertFalse(self.calls.exists()) + + def test_cancellation_terminates_active_binary_process_group(self): + pidfile = self.root / "binary.pid" + self.executable(self.build / "0.test", "#!/usr/bin/env python3\nimport os,time,pathlib\npathlib.Path(" + repr(str(pidfile)) + ").write_text(str(os.getpid()))\ntime.sleep(300)\n") + process = subprocess.Popen(self.command('run'), env=self.env, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + try: + deadline = time.monotonic() + 10 + while not pidfile.exists() and process.poll() is None and time.monotonic() < deadline: + time.sleep(0.02) + self.assertTrue(pidfile.exists()) + child = int(pidfile.read_text()) + process.send_signal(signal.SIGTERM) + _, stderr = process.communicate(timeout=10) + self.assertEqual(process.returncode, 143, stderr) + with self.assertRaises(ProcessLookupError): + os.kill(child, 0) + finally: + if process.poll() is None: + process.kill() + process.communicate() + if pidfile.exists(): + try: os.kill(int(pidfile.read_text()), signal.SIGKILL) + except ProcessLookupError: pass + + def test_subprocess_interrupt_disposition_matches_go_test(self): + self.executable(self.build / '0.test', '''#!/usr/bin/env python3 +import json, signal, subprocess +p = subprocess.Popen(['/bin/sleep', '30']) +try: + p.send_signal(signal.SIGINT) + p.wait(timeout=2) + assert p.returncode == -signal.SIGINT +finally: + if p.poll() is None: p.kill(); p.wait() +''') + result = self.run_helper('run') + self.assertEqual(result.returncode, 0, result.stderr) + + @unittest.skipUnless(shutil.which("gotestsum"), "Pinned gotestsum is exercised on Linux") + def test_actual_gotestsum_stops_retries_above_fifty_failures(self): + self.executable(self.build / "0.test", "#!/usr/bin/env python3\nimport json,os\np=os.environ['REPORT_PACKAGE']\nwith open(os.environ['BINARY_CALLS'],'a') as f:f.write(p+'\\n')\nfor i in range(51):\n for action in ['run','fail']: print(json.dumps({'Action':action,'Package':p,'Test':'TestFailure'+str(i)}))\nprint(json.dumps({'Action':'fail','Package':p,'Elapsed':0.01}))\nraise SystemExit(1)\n") + command = ['gotestsum', '--rerun-fails=3', '--rerun-fails-max-failures=50', + '--jsonfile='+str(self.root/'many-failures.json'), '--raw-command', '--', *self.command('run')] + result = subprocess.run(command, env=self.env, capture_output=True, text=True) + self.assertNotEqual(result.returncode, 0) + self.assertEqual(self.calls.read_text().splitlines().count(self.packages[0]), 1, result.stdout + result.stderr) + + @unittest.skipUnless(shutil.which("gotestsum"), "Pinned gotestsum is exercised in the hosted Go helper task") + def test_actual_gotestsum_preserves_three_failed_reruns_and_reports(self): + report = self.root / "attempts.json" + junit = self.root / "junit.xml" + result = subprocess.run(["gotestsum", "--rerun-fails=3", "--rerun-fails-max-failures=50", + "--jsonfile=" + str(report), "--junitfile=" + str(junit), "--raw-command", "--", *self.command("run")], + env={**self.env, "FAIL_PACKAGE": self.packages[1]}, text=True, capture_output=True) + self.assertNotEqual(result.returncode, 0, result.stdout + result.stderr) + failed_calls = [call for call in self.recorded(self.calls) if call["package"] == self.packages[1]] + self.assertEqual(len(failed_calls), 4, result.stdout + result.stderr) + for call in failed_calls[1:]: + self.assertIn("-test.run=^TestFresh$", call["args"]) + events = [json.loads(line) for line in report.read_text().splitlines()] + failures = [event for event in events if event.get("Test") == "TestFresh" and event["Action"] == "fail"] + self.assertEqual(len(failures), 4) + self.assertTrue(junit.is_file()) + + +if __name__ == "__main__": + unittest.main() diff --git a/ops/ci/tests/test_go_dependencies.py b/ops/ci/tests/test_go_dependencies.py new file mode 100644 index 00000000000..b2505d84906 --- /dev/null +++ b/ops/ci/tests/test_go_dependencies.py @@ -0,0 +1,95 @@ +"""Run the contract producer with real Forge and verify its Go artifact handoff.""" +import importlib.util +import json +import os +from pathlib import Path +import shutil +import subprocess +import tempfile +import unittest +from unittest.mock import patch + +ROOT = Path(__file__).resolve().parents[3] + + +@unittest.skipUnless(os.environ.get('RWX_LIVE_CONTRACT_DEPENDENCIES_FIXTURE') == '1', + 'Opt-in pinned Forge/Go contract producer fixture') +class ContractDependenciesTests(unittest.TestCase): + def test_skips_executable_solidity_tests_but_preserves_helpers_and_bound_artifacts(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) / 'project'; root.mkdir() + def run(*argv, cwd=root): + return subprocess.run(argv, cwd=cwd, text=True, capture_output=True, check=True) + run('git', 'init', '-q') + contracts = root / 'packages/contracts-bedrock'; contracts.mkdir(parents=True) + # A real initialized submodule exercises the production update command. + module = Path(temporary) / 'module'; module.mkdir() + run('git', 'init', '-q', cwd=module); (module / 'library.txt').write_text('fixture') + run('git', 'add', '.', cwd=module) + run('git', '-c', 'user.name=CI fixture', '-c', 'user.email=ci@example.invalid', + 'commit', '-qm', 'Fixture library', cwd=module) + run('git', '-c', 'protocol.file.allow=always', 'submodule', 'add', '-q', + str(module), 'packages/contracts-bedrock/lib') + recipe = (ROOT / 'packages/contracts-bedrock/justfile').read_text() + recipe = recipe[recipe.index('forge-build *ARGS:'):recipe.index('# Developer build command')] + files = { + 'mise.toml': '# Pinned tools are inherited from the prepared worker.\n', + '.gitignore': '.ci/\ntmp/\n**/forge-artifacts/\n**/cache/\n**/artifacts/\n', + 'packages/contracts-bedrock/justfile': recipe, + 'packages/contracts-bedrock/foundry.toml': '[profile.default]\nsrc="src"\nscript="scripts"\nout="forge-artifacts"\nsolc="0.8.28"\nbuild_info=true\nbuild_info_path="artifacts/build-info"\n', + 'packages/contracts-bedrock/src/Protocol.sol': '// SPDX-License-Identifier: MIT\npragma solidity 0.8.28;\ncontract Protocol {}\n', + # forge-build warms the script cache of every op-deployer script (one that + # defines runWithBytes) and fails when there is none. + 'packages/contracts-bedrock/scripts/deploy/DeploySuperchain.s.sol': '// SPDX-License-Identifier: MIT\npragma solidity 0.8.28;\ncontract DeploySuperchain { function runWithBytes(bytes memory) public {} }\n', + # The warm-up compiles the script graph through Deploy.s.sol and fails without it. + 'packages/contracts-bedrock/scripts/deploy/Deploy.s.sol': '// SPDX-License-Identifier: MIT\npragma solidity 0.8.28;\ncontract Deploy { function run() public {} }\n', + 'packages/contracts-bedrock/scripts/ops/warm-script-cache.sh': (ROOT / 'packages/contracts-bedrock/scripts/ops/warm-script-cache.sh').read_text(), + # The alias must retain .sol helpers inside test/, not exclude the directory. + 'packages/contracts-bedrock/test/dispute/zk/MockSP1Verifier.sol': '// SPDX-License-Identifier: MIT\npragma solidity 0.8.28;\ncontract MockSP1Verifier { function verify() external pure returns (bool) { return true; } }\n', + # An intentionally invalid executable test proves the producer excludes .t.sol. + 'packages/contracts-bedrock/test/Unneeded.t.sol': 'this file must not be compiled\n', + 'op-deployer/justfile': 'copy-contract-artifacts:\n go run copy.go\n', + 'op-deployer/copy.go': 'package main\nimport("encoding/json";"os")\nfunc main(){\n data,err:=os.ReadFile("../packages/contracts-bedrock/forge-artifacts/MockSP1Verifier.sol/MockSP1Verifier.json");if err!=nil{panic(err)}\n var artifact struct{Bytecode struct{Object string}};if err=json.Unmarshal(data,&artifact);err!=nil{panic(err)};if len(artifact.Bytecode.Object)<4{panic("missing deployable mock bytecode")}\n if err=os.MkdirAll("pkg/deployer/artifacts/forge-artifacts",0755);err!=nil{panic(err)};if err=os.WriteFile("pkg/deployer/artifacts/forge-artifacts/mock.json",data,0644);err!=nil{panic(err)}\n}\n', + } + for name, text in files.items(): + path = root / name; path.parent.mkdir(parents=True, exist_ok=True); path.write_text(text) + (contracts / 'scripts/ops/warm-script-cache.sh').chmod(0o755) + for name in ('go-dependencies.sh', 'go-artifacts.py'): + path = root / 'ops/ci/runtime' / name; path.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(ROOT / 'ops/ci/runtime' / name, path) + run('git', 'add', '.') + run('git', '-c', 'user.name=CI fixture', '-c', 'user.email=ci@example.invalid', + 'commit', '-qm', 'Contract dependency fixture') + sha = run('git', 'rev-parse', 'HEAD').stdout.strip() + env = {**os.environ, 'CI_COMMIT_SHA': sha} + result = subprocess.run(['bash', 'ops/ci/runtime/go-dependencies.sh', 'contracts'], + cwd=root, env=env, text=True, capture_output=True, timeout=120) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + artifact = root / '.ci/go-tests/dependencies/contracts' + metadata = json.loads((artifact / 'metadata.json').read_text()) + self.assertTrue(any('MockSP1Verifier.sol/MockSP1Verifier.json' in name for name in metadata['files'])) + self.assertFalse(any('Unneeded.t.sol' in name for name in metadata['files'])) + # Cold restoration must supply the same deployable helper. + spec = importlib.util.spec_from_file_location('artifact_fixture', root / 'ops/ci/runtime/go-artifacts.py') + module = importlib.util.module_from_spec(spec); spec.loader.exec_module(module) + with patch.dict(os.environ, CI_COMMIT_SHA=sha): + shutil.rmtree(contracts / 'forge-artifacts'); module.restore('contracts', artifact) + # Like Circle's contracts-bedrock-build, a compiler warning does not fail + # the producer; check-fast's dev build is the warning gate. + (contracts / 'src/Protocol.sol').write_text('// SPDX-License-Identifier: MIT\npragma solidity 0.8.28;\ncontract Protocol { function value(uint256 unused) external pure returns(uint256) { return 42; } }\n') + run('git', '-c', 'user.name=CI fixture', '-c', 'user.email=ci@example.invalid', + 'commit', '-qam', 'Compiler warning') + env['CI_COMMIT_SHA'] = run('git', 'rev-parse', 'HEAD').stdout.strip() + warning = subprocess.run(['bash', 'ops/ci/runtime/go-dependencies.sh', 'contracts'], + cwd=root, env=env, text=True, capture_output=True, timeout=120) + self.assertEqual(warning.returncode, 0, warning.stdout + warning.stderr) + retained = (artifact / 'metadata.json').read_bytes() + (contracts / 'src/Protocol.sol').write_text('// SPDX-License-Identifier: MIT\npragma solidity 0.8.28;\ncontract Protocol { function value() external pure returns(uint256) { return missing; } }\n') + failure = subprocess.run(['bash', 'ops/ci/runtime/go-dependencies.sh', 'contracts'], + cwd=root, env=env, text=True, capture_output=True, timeout=120) + self.assertNotEqual(failure.returncode, 0, failure.stdout + failure.stderr) + self.assertIn('missing', (failure.stdout + failure.stderr).lower()) + self.assertEqual((artifact / 'metadata.json').read_bytes(), retained) + + +if __name__ == '__main__': unittest.main() diff --git a/ops/ci/tests/test_go_dependency_source.py b/ops/ci/tests/test_go_dependency_source.py new file mode 100644 index 00000000000..c10bc59f9ae --- /dev/null +++ b/ops/ci/tests/test_go_dependency_source.py @@ -0,0 +1,84 @@ +"""Check when a dependency that an ancestor commit built may be reused.""" +import importlib.util +from pathlib import Path +import subprocess +import tempfile +import unittest +from unittest.mock import patch + +SPEC = importlib.util.spec_from_file_location('go_dependency_source', Path(__file__).resolve().parents[1] / 'runtime' / 'go-dependency-source.py') +SOURCE = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(SOURCE) + + +class SourceIdentityTests(unittest.TestCase): + def test_go_only_edit_reuses_rust_helper_but_source_and_build_inputs_invalidate(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + def git(*args): + return subprocess.check_output(['git', *args], cwd=root, text=True).strip() + git('init', '-q') + def commit(path, text): + p=root/path;p.parent.mkdir(parents=True,exist_ok=True);p.write_text(text) + git('add', '.');git('-c','user.name=Fixture','-c','user.email=fixture@example.invalid','commit','-qm','fixture') + return git('rev-parse','HEAD') + commit('mise.toml', '# toolchain') + old=commit('rust/kona/src/lib.rs', 'old rust') + with patch.object(SOURCE, 'ROOT', root): + for kind in SOURCE.SCOPES: + self.assertEqual(len(SOURCE.fingerprint(kind)), 64) + metadata={'commit_sha':old,'build_inputs_sha256':SOURCE.fingerprint('kona')} + commit('op-service/errutil/errors.go', 'new go') + # RWX's own cache key covers a task definition, so a .rwx change keeps the ancestor. + commit('.rwx/go-tests.yml', 'changed task definition') + SOURCE.verify('kona', metadata) + for kind in ('kona', 'op-reth', 'sp1-executor'): + self.assertEqual(SOURCE.ancestor(kind), old) + prestate = SOURCE.fingerprint('prestate') + commit('cannon/mipsevm/state.go', 'changed cannon') + self.assertNotEqual(prestate, SOURCE.fingerprint('prestate')) + # Matching content alone cannot authorize an unrelated revision. + with self.assertRaises((ValueError, subprocess.CalledProcessError)): + SOURCE.verify('kona', {**metadata,'commit_sha':'0'*40}) + commit('rust/kona/src/lib.rs', 'new rust') + with self.assertRaises(ValueError): SOURCE.verify('kona', metadata) + fresh={'commit_sha':git('rev-parse','HEAD'),'build_inputs_sha256':SOURCE.fingerprint('kona')} + (root/'mise.toml').write_text('uncommitted tool change') + with self.assertRaises(ValueError): SOURCE.verify('kona',fresh) + git('checkout','--','mise.toml') + commit('ops/ci/runtime/pr-gate.py', 'a script that no build runs') + SOURCE.verify('kona', fresh) + commit('ops/ci/runtime/go-dependencies.sh', 'changed build script') + with self.assertRaises(ValueError): SOURCE.verify('kona', fresh) + + def test_scopes_list_every_script_that_a_build_reaches(self): + runtime = Path(SOURCE.__file__).parent + names = [path.name for path in runtime.iterdir() if path.is_file()] + + def reached(roots): + """Return the scripts that `roots` name, directly or through another script.""" + seen, pending = set(roots), list(roots) + while pending: + current = pending.pop() + # These two declare the lists under test. They name scripts as data and run none. + if current in ('go-dependency-source.py', 'rust-target-cache.py'): continue + text = (runtime / current).read_text() + for name in names: + stem = name.rsplit('.', 1)[0] + if name not in seen and (name in text or f"helper('{stem}')" in text): + seen.add(name); pending.append(name) + return seen + + listed = lambda paths: {path.removeprefix(SOURCE.RUNTIME) for path in paths} + # .rwx/packages/go-*.yml run go-dependencies.sh; the compiler seeds also run go-full-tests.sh. + self.assertLessEqual(reached(['go-dependencies.sh']), listed(SOURCE.BUILD_SCRIPTS)) + self.assertLessEqual(reached(['go-dependencies.sh', 'go-full-tests.sh']), + listed(SOURCE.BUILD_SCRIPTS + SOURCE.COMPILER_SCRIPTS)) + for kind, scope in SOURCE.SCOPES.items(): + self.assertLessEqual(set(SOURCE.BUILD_SCRIPTS), set(scope), kind) + self.assertLessEqual(set(SOURCE.COMPILER_SCRIPTS), set(SOURCE.SCOPES['compiler'])) + # .rwx/rust.yml: the baselines, and the tool tasks of their own checkout. + spec = importlib.util.spec_from_file_location('rust_target_cache', runtime / 'rust-target-cache.py') + cache = importlib.util.module_from_spec(spec); spec.loader.exec_module(cache) + self.assertLessEqual(reached(['rust-compiler-baseline.py', 'rust-dependencies.py', 'rwx-prepare.sh', 'rwx-rust-prepare.sh']), + set(cache.BASELINE_SCRIPTS)) diff --git a/ops/ci/tests/test_go_native_isolation.py b/ops/ci/tests/test_go_native_isolation.py new file mode 100644 index 00000000000..216cfd06740 --- /dev/null +++ b/ops/ci/tests/test_go_native_isolation.py @@ -0,0 +1,68 @@ +"""Discover, compile and run a Go fixture module with the actual Go, Just and gotestsum.""" +import importlib.util +import json +import os +from pathlib import Path +import shutil +import subprocess +import sys +import tempfile +import unittest + +RUNTIME=Path(__file__).resolve().parents[1]/'runtime' +SPEC=importlib.util.spec_from_file_location('go_report',RUNTIME/'go-report.py') +REPORT=importlib.util.module_from_spec(SPEC);SPEC.loader.exec_module(REPORT) +MODULE='github.com/ethereum-optimism/optimism' + + +@unittest.skipUnless(all(shutil.which(t) for t in ('go','just','gotestsum')), 'requires actual Go, Just and gotestsum') +class NativeIsolationTests(unittest.TestCase): + def test_compilation_fresh_verdicts_fixture_paths_and_original_failure_reporting(self): + with tempfile.TemporaryDirectory() as tmp: + root=Path(tmp);helpers=root/'ops/ci/runtime';helpers.mkdir(parents=True) + for name in ('go-compiled-tests.py','go-suite.py','go-package-shards.py'): + shutil.copy2(RUNTIME/name,helpers/name) + files={'go.mod':'module '+MODULE+'\n\ngo 1.26.0\n', + 'justfile':'list-test-packages:\n go list -tags=ci ./fixture/...\n', + 'timings.json':'{"packages": {}}\n', + 'fixture/empty/empty.go':'package empty\n', + 'fixture/tests/fixture.txt':'original fixture', + 'fixture/tests/fixture_test.go':'''package fixture +import("os";"testing") +func TestFresh(t *testing.T) { + data,e:=os.ReadFile("fixture.txt");if e!=nil || string(data)!="original fixture" {t.Fatalf("working directory: %s %v",data,e)} + f,e:=os.OpenFile("../../.ci/calls",os.O_CREATE|os.O_WRONLY|os.O_APPEND,0600);if e!=nil {t.Fatal(e)};defer f.Close();f.WriteString("fresh\\n") + if os.Getenv("GO_ISOLATED_FAILURE")=="1" {t.Fatal("original intentional Go failure")} +} +'''} + for name,data in files.items(): + path=root/name;path.parent.mkdir(parents=True,exist_ok=True);path.write_text(data) + env=dict(os.environ,CI_COMMIT_SHA='a'*40,CI_SHARD_INDEX='0',PARALLEL='8',TEST_TIMEOUT='40m') + def execute(args,**extra): + result=subprocess.run(args,cwd=root,env=env|extra,capture_output=True,text=True,timeout=120) + return result + discovered=execute([sys.executable,str(helpers/'go-suite.py'),'--total','1','--timings','timings.json']) + self.assertEqual(discovered.returncode,0,discovered.stderr) + manifest=root/'.ci/go-tests/manifest.json' + self.assertEqual(json.loads(manifest.read_text()),{'commit_sha':'a'*40,'no_tests':[MODULE+'/fixture/empty'], + 'shards':[[MODULE+'/fixture/empty',MODULE+'/fixture/tests']]}) + built=execute([sys.executable,str(helpers/'go-compiled-tests.py'),'build']) + self.assertEqual(built.returncode,0,built.stderr) + self.assertFalse((root/'.ci/calls').exists(),'Compilation must execute zero tests') + # Package 0 has no test files, so only package 1 has a binary. + self.assertEqual(sorted(p.name for p in (root/'.ci/go-tests/build').glob('*.test')),['1.test']) + manifest.write_text('the manifest of an older commit') + restored=execute([sys.executable,str(helpers/'go-compiled-tests.py'),'restore']) + self.assertEqual(restored.returncode,0,restored.stderr) + for label,failing in [('fresh-one',False),('fresh-two',False),('intentional-failure',True)]: + output=root/'.ci'/label;output.mkdir() + result=execute(['gotestsum','--format=standard-quiet','--jsonfile='+str(output/'original.json'), + '--junitfile='+str(output/'junit.xml'),'--raw-command','--',sys.executable, + str(helpers/'go-compiled-tests.py'),'run'],GO_ISOLATED_FAILURE='1' if failing else '0') + self.assertEqual(result.returncode,1 if failing else 0,result.stdout+result.stderr) + REPORT.project(output/'original.json',output/'native.json') + self.assertIn('TestFresh',(output/'original.json').read_text()) + if failing: + self.assertIn('original intentional Go failure',(output/'original.json').read_text()) + self.assertIn('', "") + duplicate = JUNIT.replace("", '') + for junit in (missing, duplicate): + with self.subTest(junit=junit): + self.assertNotEqual(self.run_job("integration", JUNIT_XML=junit).returncode, 0) + + def test_filtered_or_unexpectedly_skipped_test_fails(self): + self.build_integration() + discovery = copy.deepcopy(DISCOVERY) + discovery["rust-suites"]["reth-optimism-node::e2e"]["testcases"]["passes"]["filter-match"]["status"] = "mismatch" + self.assertNotEqual(self.run_job("integration", DISCOVERY_JSON=json.dumps(discovery)).returncode, 0) + junit = JUNIT.replace('name="passes"/>', 'name="passes">') + self.assertNotEqual(self.run_job("integration", JUNIT_XML=junit).returncode, 0) + + def test_only_authoritative_ignored_cases_may_omit_verdicts(self): + self.build_integration() + result = self.run_job("integration") + self.assertEqual(result.returncode, 0, result.stdout) + coverage = self.report("integration", "coverage") + self.assertEqual((coverage["discovered"], coverage["reported"]), (2, 1)) + self.assertEqual(coverage["ignored_cases"], [{"suite": "reth-optimism-node::e2e", "name": "ignored", "reason": "ignored"}]) + discovery = copy.deepcopy(DISCOVERY) + discovery["rust-suites"]["reth-optimism-node::e2e"]["testcases"]["ignored"]["filter-match"]["reason"] = "expression" + self.assertNotEqual(self.run_job("integration", DISCOVERY_JSON=json.dumps(discovery)).returncode, 0) + + def test_reported_ignored_case_must_be_skipped(self): + self.build_integration() + reported = '' + junit = JUNIT.replace("", reported + "") + self.assertEqual(self.run_job("integration", JUNIT_XML=junit).returncode, 0) + self.assertNotEqual(self.run_job("integration", JUNIT_XML=junit.replace("", "")).returncode, 0) + + def test_extracted_binaries_are_not_duplicated_in_verdict_artifacts(self): + self.build_integration() + self.assertEqual(self.run_job("integration").returncode, 0) + report = self.root / ".ci/op-reth/integration" + self.assertTrue((report / "junit.xml").is_file()) + self.assertFalse((report / "unpacked").exists()) + + def test_retried_failure_is_visible(self): + self.build_integration() + junit = JUNIT.replace('name="passes"/>', 'name="passes">first attempt failed') + result = self.run_job("integration", JUNIT_XML=junit) + self.assertEqual(result.returncode, 0, result.stdout) + self.assertEqual(self.report("integration", "coverage")["retried_cases"], [["reth-optimism-node::e2e", "passes"]]) + + def test_snapshot_changes_sync_flag_and_checks_regenerated_files(self): + self.assertEqual(self.run_job("snapshot-build").returncode, 0) + self.assertEqual(self.run_job("snapshot", DIFF_EXIT="1").returncode, 1) + builds = [c for c in self.calls("cargo") if c["args"][:1] == ["build"]] + self.assertEqual([c["sync"] for c in builds], ["0", "1"]) + self.assertIsNone(builds[-1]["wrapper"]) + + def test_failed_submodule_setup_propagates(self): + self.assertEqual(self.run_job("source", SOURCE_EXIT="17").returncode, 17) + + +if __name__ == "__main__": + unittest.main() diff --git a/ops/ci/tests/test_pr_gate.py b/ops/ci/tests/test_pr_gate.py new file mode 100644 index 00000000000..ed700a12471 --- /dev/null +++ b/ops/ci/tests/test_pr_gate.py @@ -0,0 +1,215 @@ +"""Check the gate policy's wiring in the RWX definitions, and the receipt and status verdicts.""" +import importlib.util +import json +import os +from pathlib import Path +import re +import subprocess +import tempfile +import unittest +from unittest.mock import patch + +ROOT = Path(__file__).resolve().parents[3] +SPEC = importlib.util.spec_from_file_location('pr_gate', ROOT / 'ops/ci/runtime/pr-gate.py') +G = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(G) +RECEIPT = re.compile(r'python3 ops/ci/runtime/pr-gate\.py receipt (\S+)') + + +def definition(path): + return json.loads(subprocess.check_output(['yq', '-o=json', 'explode(.)', str(path)], text=True)) + + +def tasks(path): + return {task['key']: task for task in definition(path)['tasks']} + + +def terminal(key): + return f'({key}.succeeded || {key}.failed || {key}.skipped)' + + +def groups(gate): + return sorted({row['group'] for row in G.gates()[gate]['dependencies']}) + + +class WiringTests(unittest.TestCase): + """pr-gates.json and the definitions must name the same tasks. A hosted run would fail on a difference.""" + + @classmethod + def setUpClass(cls): + # {group: (definition file name, receipt task)} + cls.receipts = {} + for path in sorted((ROOT / '.rwx').glob('*.yml')): + for task in tasks(path).values(): + match = RECEIPT.fullmatch(task['run'].strip()) if isinstance(task.get('run'), str) else None + if match: + assert match[1] not in cls.receipts, match[1] + cls.receipts[match[1]] = (path.name, task) + + def test_each_group_has_one_receipt_that_reads_every_task_of_the_group(self): + self.assertEqual(set(self.receipts), {group for gate in G.gates() for group in groups(gate)}) + for group, (name, receipt) in self.receipts.items(): + with self.subTest(group=group): + names = G.group_tasks(group) + workload = tasks(ROOT / '.rwx' / name) + self.assertLessEqual(set(names), set(workload)) + wanted = {G.variable('TASK', key, state): '${{ tasks.%s.%s }}' % (key, state) for key in names for state in G.STATES} + self.assertEqual({key: value for key, value in receipt['env'].items() if key.startswith('TASK_')}, wanted) + # The receipt waits for every task, and runs whatever their states are. + self.assertEqual(sorted(receipt['after'].removeprefix('${{ ').removesuffix(' }}').split(' && ')), + sorted(terminal(key) for key in names)) + self.assertRegex(receipt['env']['GROUP_SELECTED'], r'^\$\{\{ tasks\.route\.values\.run-[a-z0-9-]+ \}\}$') + self.assertIs(receipt['cache'], False) + + def test_each_gate_has_one_status_task_that_reads_every_receipt_of_the_gate(self): + coordinator = definition(ROOT / '.rwx/pr-gates.yml') + graph = {task['key']: task for task in coordinator['tasks']} + calls = {task['call'].removeprefix('${{ run.dir }}/'): key for key, task in graph.items() + if str(task.get('call', '')).startswith('${{ run.dir }}/')} + checks = coordinator['on']['github']['push']['status-checks']['custom'] + for gate in G.gates(): + with self.subTest(gate=gate): + found = [task for task in graph.values() if task.get('run') == 'python3 ops/ci/runtime/pr-gate.py status ' + gate] + self.assertEqual(len(found), 1) + status = found[0] + wanted, waits = {}, set() + for group in groups(gate): + name, receipt = self.receipts[group] + waits.add(terminal(calls[name])) + for state in G.STATES: + wanted[G.variable('GROUP', group, state)] = '${{ tasks.%s.tasks.%s.%s }}' % (calls[name], receipt['key'], state) + self.assertEqual({key: value for key, value in status['env'].items() if key.startswith('GROUP_')}, wanted) + self.assertEqual(set(status['after'].removeprefix('${{ ').removesuffix(' }}').split(' && ')), waits) + # RWX fails a custom status check when one of its tasks is skipped. + self.assertNotIn('if', status) + self.assertIs(status['cache'], False) + self.assertEqual([check['tasks'] for check in checks].count(status['key']), 1) + + def test_each_workload_has_one_status_check_that_names_its_call(self): + # `rwx lint` accepts a check that names no task, and a run from the CLI reports no checks. + coordinator = definition(ROOT / '.rwx/pr-gates.yml') + checks = coordinator['on']['github']['push']['status-checks']['custom'] + named = [check['tasks'] for check in checks] + self.assertLessEqual(set(named), {task['key'] for task in coordinator['tasks']}) + calls = [task['key'] for task in coordinator['tasks'] if str(task.get('call', '')).startswith('${{ run.dir }}/') and 'init' in task] + self.assertEqual(sorted(key for key in named if key.startswith('native-')), sorted(calls)) + self.assertEqual(len({check['name'] for check in checks}), len(checks)) + # The coordinator is the one definition that a push starts. + for path in sorted((ROOT / '.rwx').glob('*.yml')): + self.assertEqual('github' in definition(path)['on'], path.name == 'pr-gates.yml', path.name) + + def test_each_gated_task_runs_in_every_run(self): + # RWX reuses the result of a task whose inputs it has seen. A gated task + # must run anyway: it sets `cache: false`, or it includes the run id and + # the attempt number in its cache key, which keeps a tool cache usable. + for group, (name, _) in self.receipts.items(): + workload = tasks(ROOT / '.rwx' / name) + for key in G.group_tasks(group): + task = workload[key] + if 'call' in task: + package = definition(ROOT / '.rwx' / task['call'].removeprefix('${{ run.dir }}/')) + leaves = [leaf for leaf in package['tasks'] if 'run' in leaf] + self.assertEqual(len(leaves), 1, key) + task = leaves[0] + fresh = task.get('cache') is False or all( + task.get('env', {}).get(variable) == {'cache-key': 'included'} for variable in ('RWX_RUN_ID', 'RWX_TASK_ATTEMPT_NUMBER')) + self.assertTrue(fresh, f'{name}: {key}') + + +class VerdictTests(unittest.TestCase): + def setUp(self): + temp = tempfile.TemporaryDirectory() + self.addCleanup(temp.cleanup) + self.root = Path(temp.name) + patcher = patch.object(G, 'ROOT', self.root) + patcher.start() + self.addCleanup(patcher.stop) + + def run_command(self, command, name, prefix, states, **flags): + """Run a command with one state per task. Return its exit status and its record.""" + environment = {G.variable(prefix, key, state): str(state == actual).lower() + for key, actual in states.items() for state in G.STATES} + environment.update(flags) + with patch.dict(os.environ, environment, clear=True): + status = command(name) + kind = 'groups' if command is G.receipt else 'status' + return status, json.loads((self.root / '.ci/pr-gates' / kind / name / 'verdict.json').read_text()) + + def test_receipt_passes_when_selected_tasks_succeeded_or_unselected_tasks_were_skipped(self): + group = 'rust-workspace' + names = G.group_tasks(group) + self.assertGreater(len(names), 20) + for selected, state, expected in [('true', 'succeeded', 0), ('true', 'failed', 1), ('true', 'skipped', 1), + ('false', 'skipped', 0), ('false', 'succeeded', 1), ('false', 'failed', 1)]: + with self.subTest(selected=selected, state=state): + status, record = self.run_command(G.receipt, group, 'TASK', dict.fromkeys(names, state), GROUP_SELECTED=selected) + self.assertEqual(status, expected) + self.assertEqual(record['passed'], expected == 0) + self.assertEqual(record['selected'], selected == 'true') + self.assertTrue(all(row[state] for row in record['tasks'].values())) + + def test_receipt_fails_when_one_task_of_many_differs(self): + group = 'main-go' + names = G.group_tasks(group) + for selected, good, bad in [('true', 'succeeded', 'skipped'), ('true', 'succeeded', 'failed'), ('false', 'skipped', 'succeeded')]: + with self.subTest(selected=selected, bad=bad): + states = {**dict.fromkeys(names, good), names[-1]: bad} + status, record = self.run_command(G.receipt, group, 'TASK', states, GROUP_SELECTED=selected) + self.assertEqual(status, 1) + self.assertFalse(record['passed']) + self.assertTrue(record['tasks'][names[-1]][bad]) + + def test_receipt_rejects_states_that_rwx_did_not_supply_as_one_terminal_state(self): + group = 'main-kontrol' + environment = {'GROUP_SELECTED': 'true', 'TASK_CHECK_SUCCEEDED': 'true', 'TASK_CHECK_FAILED': 'false', 'TASK_CHECK_SKIPPED': 'false'} + with patch.dict(os.environ, environment, clear=True): + self.assertEqual(G.receipt(group), 0) + # A task that has not finished has no true state: the receipt did not wait for it. + for change in ({'TASK_CHECK_SUCCEEDED': 'false'}, {'TASK_CHECK_SKIPPED': 'true'}, {'TASK_CHECK_FAILED': ''}, + {'TASK_CHECK_FAILED': 'True'}, {'GROUP_SELECTED': 'yes'}): + with self.subTest(change=change), patch.dict(os.environ, {**environment, **change}, clear=True): + with self.assertRaises(ValueError): + G.receipt(group) + # A receipt without the variable of a task fails: the definition must pass every state. + del environment['TASK_CHECK_SKIPPED'] + with patch.dict(os.environ, environment, clear=True), self.assertRaises(KeyError): + G.receipt(group) + + def test_status_passes_only_when_every_receipt_succeeded(self): + for gate in G.gates(): + names = groups(gate) + requires = [row['name'] for row in G.gates()[gate]['dependencies']] + for selected, outcome in [('true', 'passed'), ('false', 'safe_skip')]: + with self.subTest(gate=gate, selected=selected): + status, record = self.run_command(G.status, gate, 'GROUP', dict.fromkeys(names, 'succeeded'), GATE_SELECTED=selected) + self.assertEqual(status, 0) + self.assertTrue(record['passed']) + # The record names each CircleCI prerequisite once, in the policy's order. + self.assertEqual([row['name'] for row in record['dependencies']], requires) + self.assertEqual({row['outcome'] for row in record['dependencies']}, {outcome}) + for state in ('failed', 'skipped'): + with self.subTest(gate=gate, state=state): + states = {**dict.fromkeys(names, 'succeeded'), names[0]: state} + status, record = self.run_command(G.status, gate, 'GROUP', states, GATE_SELECTED='true') + self.assertEqual(status, 1) + self.assertFalse(record['passed']) + outcomes = {row['group']: row['outcome'] for row in record['dependencies']} + self.assertEqual(outcomes[names[0]], 'failed') + self.assertEqual({outcomes[name] for name in names[1:]} or {'passed'}, {'passed'}) + + def test_policy_names_each_circleci_job_and_each_task_once(self): + seen = {} + for gate, policy in G.gates().items(): + names = [row['name'] for row in policy['dependencies']] + self.assertEqual(len(set(names)), len(names)) + for row in policy['dependencies']: + self.assertTrue(row['tasks']) + for task in row['tasks']: + self.assertNotIn((row['group'], task), seen) + seen[row['group'], task] = row['name'] + with self.assertRaisesRegex(ValueError, 'no task'): + G.group_tasks('unknown-group') + + +if __name__ == '__main__': + unittest.main() diff --git a/ops/ci/tests/test_rust_cannon.py b/ops/ci/tests/test_rust_cannon.py new file mode 100644 index 00000000000..914421930d7 --- /dev/null +++ b/ops/ci/tests/test_rust_cannon.py @@ -0,0 +1,155 @@ +"""Run the Cannon job shell against stub tools and check its exit status and report.""" +import hashlib +import importlib.util +import json +import os +from pathlib import Path +import shutil +import signal +import subprocess +import tempfile +import time +import unittest +import xml.etree.ElementTree as ET + +REPO = Path(__file__).resolve().parents[3] +SPEC = importlib.util.spec_from_file_location('cannon_report', Path(__file__).resolve().parents[1] / 'runtime' / 'rust-cannon-report.py') +REPORT = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(REPORT) +CONFIG = json.loads((REPO / REPORT.CONFIG).read_text()) +KEYS = ('block_number', 'l2_claim', 'l2_output_root', 'l2_head', 'l1_head', 'l2_chain_id', 'filename', 'release_url') + + +class ReportTests(unittest.TestCase): + def test_finish_keeps_the_exit_status_and_reports_one_guest_case(self): + with tempfile.TemporaryDirectory() as temp: + report = Path(temp) + REPORT.write(report / 'settings.json', {'job': 'offline', 'witness': CONFIG}) + REPORT.write(report / 'offline.stage.json', {'exit_code': 7}) + for status in (7, 0): + self.assertEqual(REPORT.finish(report, status), status) + final = REPORT.read(report / 'final.json') + self.assertEqual((final['exit_code'], final['stages']['offline']['exit_code']), (status, 7)) + cases = ET.parse(report / 'checks.junit.xml').findall('.//testcase') + self.assertEqual([case.get('name') for case in cases], + ['kona-client/' + CONFIG['l2_chain_id'] + '/' + CONFIG['block_number']]) + self.assertEqual(cases[0].find('failure') is not None, bool(status)) + + +class RunnerTests(unittest.TestCase): + def setUp(self): + temp = tempfile.TemporaryDirectory(); self.addCleanup(temp.cleanup) + self.root = Path(temp.name) + # The two ignore files keep the witness archive and the recipe's final + # Cannon state out of the Rust source check, as in the repository. + for name in ('ops/ci/runtime/rust-cannon.sh', 'ops/ci/runtime/rust-cannon-report.py', + 'ops/ci/runtime/rust-workspace-report.py', 'ops/ci/runtime/ci-report.py', + 'ops/ci/runtime/rust-target-cache.py', str(REPORT.DOCKERFILE), + 'rust/kona/bin/client/scripts/fetch-witness-tar.sh', + 'rust/kona/bin/client/testdata/.gitignore', 'rust/kona/.gitignore'): + target = self.root / name; target.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(REPO / name, target) + self.config = CONFIG | {'sha256': hashlib.sha256(b'fixture witness').hexdigest()} + REPORT.write(self.root / REPORT.CONFIG, self.config) + stubs = self.root / 'stubs'; stubs.mkdir() + stub = stubs / 'stub' + stub.write_text('''#!/usr/bin/env python3 +import gzip,json,os,pathlib,sys,time +root=pathlib.Path(os.environ['FIXTURE_ROOT']); tool=pathlib.Path(sys.argv[0]).name; args=sys.argv[1:] +if tool in ('rustc','cargo','go'): print(tool+' pinned'); sys.exit(0) +if tool=='sccache': + if '--show-stats' in args: print('{}') + sys.exit(0) +if tool=='curl': + path=pathlib.Path(args[args.index('-o')+1]); path.parent.mkdir(parents=True,exist_ok=True); path.write_bytes(b'fixture witness') + sys.exit(int(os.environ.get('CURL_EXIT','0'))) +if tool=='docker': + (root/'docker.json').write_text(json.dumps(args)); sys.exit(0) +if tool=='just': + (root/'invocations.json').write_text(json.dumps(args)) + if args[0]=='run-client-cannon-offline': + if os.environ.get('WAIT_FOR_CANCEL'): + (root/'running').touch(); print('guest running',flush=True); time.sleep(60) + status=int(os.environ.get('OFFLINE_EXIT','0')) + if status: print('original compile or runtime failure',flush=True); sys.exit(status) + (root/'rust/kona/out.bin.gz').write_bytes(gzip.compress(b'final state')) + sys.exit(0) +''') + stub.chmod(0o755) + for name in ('just', 'docker', 'rustc', 'cargo', 'go', 'curl', 'sccache'): (stubs / name).symlink_to(stub) + subprocess.run(['git', 'init', '-q'], cwd=self.root, check=True) + subprocess.run(['git', 'add', '.'], cwd=self.root, check=True) + subprocess.run(['git', '-c', 'user.name=Fixture', '-c', 'user.email=fixture@example.test', 'commit', '-qm', 'fixture'], cwd=self.root, check=True) + self.env = dict(os.environ, PATH=str(stubs) + ':' + os.environ['PATH'], + RWX_RUN_ID='a' * 32, RWX_TASK_ATTEMPT_NUMBER='1', FIXTURE_ROOT=str(self.root)) + + def run_job(self, job, **env): + return subprocess.run(['bash', 'ops/ci/runtime/rust-cannon.sh', job], cwd=self.root, + env=self.env | env, capture_output=True, text=True, timeout=15) + + def final(self, job): + return REPORT.read(self.root / '.ci/rust-workspace' / ('cannon-' + job) / 'final.json') + + def test_offline_passes_the_pinned_witness_to_the_recipe(self): + run = self.run_job('offline') + self.assertEqual(run.returncode, 0, run.stdout + run.stderr) + args = json.loads((self.root / 'invocations.json').read_text()) + self.assertEqual(args, ['run-client-cannon-offline'] + [CONFIG[key] for key in KEYS]) + self.assertEqual(self.final('offline')['exit_code'], 0) + # Git ignores the recipe's final Cannon state, so the target cache commit accepts the run. + self.assertTrue((self.root / 'rust/kona/out.bin.gz').exists()) + self.assertTrue((self.root / 'rust/target/.rwx-source-fingerprint.json').exists()) + + def test_first_failure_not_retried_and_report_collected(self): + run = self.run_job('offline', OFFLINE_EXIT='7') + self.assertEqual(run.returncode, 7, run.stdout + run.stderr) + final = self.final('offline') + self.assertEqual(final['exit_code'], 7) + self.assertEqual(final['stages']['offline']['exit_code'], 7) + report = self.root / '.ci/rust-workspace/cannon-offline' + self.assertIn('original compile or runtime failure', (report / 'offline.log').read_text()) + self.assertIsNotNone(ET.parse(report / 'checks.junit.xml').find('.//failure')) + + def test_witness_fetch_failure_and_pin_mismatch_fail_the_witness_job(self): + run = self.run_job('witness') + self.assertEqual(run.returncode, 0, run.stdout + run.stderr) + self.assertEqual(self.final('witness')['stages']['witness']['argv'][-2:], [CONFIG['filename'], CONFIG['release_url']]) + run = self.run_job('witness', CURL_EXIT='22') + self.assertEqual(run.returncode, 22) + self.assertEqual({name: stage['exit_code'] for name, stage in self.final('witness')['stages'].items()}, {'witness': 22}) + REPORT.write(self.root / REPORT.CONFIG, self.config | {'sha256': '0' * 64}) + run = self.run_job('witness') + self.assertNotEqual(run.returncode, 0) + self.assertIn('checksum mismatch', run.stderr) + self.assertEqual(self.final('witness')['stages']['witness']['exit_code'], 0) + + def test_env_pulls_the_base_image_before_the_recipe(self): + run = self.run_job('env') + self.assertEqual(run.returncode, 0, run.stdout + run.stderr) + base = subprocess.check_output(['python3', 'ops/ci/runtime/rust-cannon-report.py', 'base'], cwd=self.root, text=True).strip() + self.assertEqual(json.loads((self.root / 'docker.json').read_text()), ['pull', base]) + self.assertEqual(json.loads((self.root / 'invocations.json').read_text()), ['build-kona-env']) + self.assertEqual(list(self.final('env')['stages']), ['base-pull', 'env']) + + @unittest.skipUnless(os.name == 'posix', 'POSIX cancellation') + def test_cancellation_preserves_signal_and_original_stream(self): + process = subprocess.Popen(['bash', 'ops/ci/runtime/rust-cannon.sh', 'offline'], cwd=self.root, + env=self.env | {'WAIT_FOR_CANCEL': '1'}, stdout=subprocess.PIPE, stderr=subprocess.PIPE, + text=True, start_new_session=True) + try: + deadline = time.monotonic() + 10 + while not (self.root / 'running').exists() and process.poll() is None and time.monotonic() < deadline: + time.sleep(.05) + self.assertTrue((self.root / 'running').exists()) + os.killpg(process.pid, signal.SIGTERM) + process.communicate(timeout=10) + final = self.final('offline') + self.assertEqual(final['exit_code'], 143) + self.assertEqual(final['stages']['offline']['exit_code'], -signal.SIGTERM) + self.assertIn('guest running', (self.root / '.ci/rust-workspace/cannon-offline/offline.log').read_text()) + finally: + if process.poll() is None: + os.killpg(process.pid, signal.SIGKILL); process.communicate() + + +if __name__ == '__main__': unittest.main() diff --git a/ops/ci/tests/test_rust_compiler_baseline.py b/ops/ci/tests/test_rust_compiler_baseline.py new file mode 100644 index 00000000000..a4d7e68a3e4 --- /dev/null +++ b/ops/ci/tests/test_rust_compiler_baseline.py @@ -0,0 +1,47 @@ +import importlib.util +import json +from pathlib import Path +import subprocess +import tempfile +import unittest +from unittest.mock import patch + +SPEC = importlib.util.spec_from_file_location('baseline', Path(__file__).resolve().parents[1] / 'runtime/rust-compiler-baseline.py') +BASELINE = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(BASELINE) + + +class BaselineTests(unittest.TestCase): + def test_optional_failed_ancestor_retains_failure_and_discards_partial_targets(self): + for status in (0, 101): + with self.subTest(status=status), tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + target = root / 'rust/target'; target.mkdir(parents=True) + (target / '.rwx-source-pending.json').write_text('partial') + report = root / '.ci/rust-workspace/tests-build'; report.mkdir(parents=True) + (report / 'final.json').write_text(json.dumps({'exit_code': status})) + (report / 'archive.log').write_text('original compiler output') + (report / 'tests.tar.zst').write_bytes(b'large binary archive') + with patch.object(BASELINE.subprocess, 'run', return_value=subprocess.CompletedProcess([], status)) as build, \ + patch.object(BASELINE.subprocess, 'check_output', return_value='a'*40+'\n'): + BASELINE.prepare('tests-build', root) + self.assertEqual(build.call_args.kwargs['env']['CI_RUST_BASELINE'], 'true') + self.assertEqual(target.exists(), status == 0) + evidence = root / '.ci/rust-workspace/compiler-baseline/tests-build' + self.assertEqual(json.loads((evidence / 'final.json').read_text())['exit_code'], status) + self.assertEqual((evidence / 'archive.log').read_text(), 'original compiler output') + self.assertFalse((evidence / 'tests.tar.zst').exists()) + binding = json.loads((evidence / 'compiler-baseline.json').read_text()) + self.assertEqual(binding['usable'], status == 0) + self.assertEqual(binding['build_exit_code'], status) + self.assertEqual(binding['source_sha'], 'a'*40) + + def test_failed_cleanup_cannot_claim_a_clean_fallback(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + (root / 'rust/target').mkdir(parents=True) + with patch.object(BASELINE.subprocess, 'run', return_value=subprocess.CompletedProcess([], 101)), \ + patch.object(BASELINE.shutil, 'rmtree', side_effect=PermissionError('retained target')): + with self.assertRaises(PermissionError): + BASELINE.prepare('tests-build', root) + self.assertFalse((root / '.ci/rust-workspace/compiler-baseline/tests-build/compiler-baseline.json').exists()) diff --git a/ops/ci/tests/test_rust_dependencies.py b/ops/ci/tests/test_rust_dependencies.py new file mode 100644 index 00000000000..eed77c43de6 --- /dev/null +++ b/ops/ci/tests/test_rust_dependencies.py @@ -0,0 +1,37 @@ +import hashlib +import importlib.util +import json +from pathlib import Path +import subprocess +import tempfile +import unittest +from unittest.mock import patch + +SPEC = importlib.util.spec_from_file_location('dependencies', Path(__file__).resolve().parents[1] / 'runtime/rust-dependencies.py') +DEPS = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(DEPS) + + +class DependencyTests(unittest.TestCase): + def test_prefetch_preserves_attempts_and_failed_ancestor_remains_optional(self): + for exits in ([101, 0], [101] * 5): + with self.subTest(exits=exits), tempfile.TemporaryDirectory() as tmp: + root = Path(tmp); (root / 'rust').mkdir() + (root / 'rust/Cargo.lock').write_text('locked parent dependencies') + values = iter(exits) + def run(args, **kwargs): + status = next(values) + self.assertEqual(args, ['cargo', 'fetch', '--locked']) + self.assertEqual(kwargs['env']['CARGO_HOME'], str(root / '.ci/rust-cache/cargo')) + kwargs['stdout'].write(f'original fetch exit {status}\n'.encode()) + return subprocess.CompletedProcess(args, status) + with patch.object(DEPS.subprocess, 'run', side_effect=run), \ + patch.object(DEPS.subprocess, 'check_output', return_value='a'*40), \ + patch.object(DEPS.time, 'sleep'): + DEPS.fetch(root) + report = root / '.ci/rust-dependencies' + value = json.loads((report / 'result.json').read_text()) + self.assertEqual([a['exit_code'] for a in value['attempts']], exits) + self.assertEqual(value['usable'], exits[-1] == 0) + self.assertEqual(value['original_log_sha256'], hashlib.sha256((report / 'fetch.log').read_bytes()).hexdigest()) + self.assertIn(b'original fetch exit 101', (report / 'fetch.log').read_bytes()) diff --git a/ops/ci/tests/test_rust_e2e.py b/ops/ci/tests/test_rust_e2e.py new file mode 100644 index 00000000000..0c3f3177ee2 --- /dev/null +++ b/ops/ci/tests/test_rust_e2e.py @@ -0,0 +1,229 @@ +#!/usr/bin/env python3 +"""Exercise complete E2E discovery, partitions, fresh binaries and failure reports.""" +import importlib.util +import json +import os +from pathlib import Path +import re +import shutil +import signal +import subprocess +import tempfile +import time +import unittest +from unittest.mock import patch + +SCRIPTS = Path(__file__).resolve().parents[1] / 'runtime' +SPEC = importlib.util.spec_from_file_location('rust_e2e', Path(__file__).resolve().parents[1] / 'runtime' / 'rust-e2e.py') +E2E = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(E2E) + + +class E2ETests(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory(); self.addCleanup(self.tmp.cleanup) + self.root = Path(self.tmp.name) + + def listing(self, names=('TestNew', 'TestOld')): + (self.root / 'listing.log').write_text('\n'.join(names) + '\n') + + def test_every_discovered_test_is_assigned_to_exactly_one_shard(self): + self.listing() + names, excluded = E2E.discovery('proof', self.root) + self.assertEqual(names, ['TestNew', 'TestOld']); self.assertEqual(excluded, []) + shards = E2E.partition(names, 8) + self.assertEqual(sorted(n for shard in shards for n in shard), names) + self.assertEqual(sum(not shard for shard in shards), 6) + many = ['Test%02d' % index for index in range(37)] + shards = E2E.partition(many, 8) + self.assertEqual(sorted(n for shard in shards for n in shard), many) + self.assertEqual([len(shard) for shard in shards], [5, 5, 5, 5, 5, 4, 4, 4]) + + def test_empty_or_duplicate_listing_never_becomes_empty_success(self): + for mode in ('empty', 'duplicate'): + self.listing(names=() if mode == 'empty' else ('TestNew', 'TestNew')) + with self.subTest(mode=mode), self.assertRaises(ValueError): E2E.discovery('proof', self.root) + + def test_selection_writes_one_shard_and_rejects_an_unknown_shard(self): + E2E.write(self.root / 'manifest.json', {'shards': [['TestA', 'TestC'], ['TestB'], []]}) + E2E.select(self.root, self.root, 0) + self.assertEqual((self.root / 'assigned.txt').read_text(), 'TestA\nTestC\n') + self.assertEqual(E2E.read(self.root / 'selection.json'), {'shard_index': 0, 'assigned_tests': ['TestA', 'TestC']}) + E2E.select(self.root, self.root, 2) + self.assertEqual((self.root / 'assigned.txt').read_text(), '') + for index in (-1, 3): + with self.subTest(index=index), self.assertRaises(ValueError): E2E.select(self.root, self.root, index) + + def test_examples_and_fuzz_are_explicit_in_original_sharding_scope(self): + self.listing(('TestNew', 'ExampleDemo', 'FuzzInputs')) + names, excluded = E2E.discovery('proof', self.root) + self.assertEqual(names, ['TestNew']); self.assertEqual(excluded, ['ExampleDemo', 'FuzzInputs']) + self.assertEqual(E2E.discovery('simple-kona', self.root)[0], ['ExampleDemo', 'FuzzInputs', 'TestNew']) + + def test_original_failure_and_missing_verdicts_are_retained(self): + (self.root / 'events').mkdir() + E2E.write(self.root / 'selection.json', {'assigned_tests': ['TestOriginal', 'TestMissing']}) + events = [{'Action': 'run', 'Package': E2E.PREFIX + E2E.JOBS['proof']['package'], 'Test': 'TestOriginal'}, + {'Action': 'fail', 'Package': E2E.PREFIX + E2E.JOBS['proof']['package'], 'Test': 'TestOriginal'}] + (self.root / 'events/failure.json').write_text('\n'.join(json.dumps(e) for e in events) + '\n') + self.assertEqual(E2E.report('proof', self.root, 17), 17) + final = E2E.read(self.root / 'final.json') + self.assertEqual(final['exit_code'], 17) + self.assertTrue(final['report_errors']) + self.assertEqual(E2E.read(self.root / 'coverage.json')['missing'], ['TestMissing']) + self.assertEqual((self.root / 'original.json').read_bytes(), (self.root / 'events/failure.json').read_bytes()) + + def test_complete_verdicts_report_success_and_a_native_report(self): + (self.root / 'events').mkdir() + E2E.write(self.root / 'selection.json', {'assigned_tests': ['TestOriginal']}) + package = E2E.PREFIX + E2E.JOBS['proof']['package'] + events = [{'Action': 'run', 'Package': package, 'Test': 'TestOriginal'}, + {'Action': 'pass', 'Package': package, 'Test': 'TestOriginal', 'Elapsed': 1.5}, + {'Action': 'pass', 'Package': package, 'Elapsed': 1.5}] + (self.root / 'events/TestOriginal.json').write_text('\n'.join(json.dumps(e) for e in events) + '\n') + self.assertEqual(E2E.report('proof', self.root, 0), 0) + self.assertEqual(E2E.read(self.root / 'final.json'), {'exit_code': 0, 'report_errors': []}) + self.assertEqual(E2E.read(self.root / 'coverage.json')['cases'], {'TestOriginal': {'outcome': 'pass', 'elapsed': 1.5}}) + self.assertIn('TestOriginal', (self.root / 'native.json').read_text()) + + def test_configurations_preserve_validator_and_sequencer_roles(self): + a, b = E2E.environment('simple-kona'), E2E.environment('simple-kona-sequencer') + self.assertEqual(a['KONA_SEQUENCER_WITH_RETH'], '0'); self.assertEqual(b['KONA_SEQUENCER_WITH_RETH'], '1') + self.assertEqual(a['OP_VALIDATOR_WITH_RETH'], '1'); self.assertEqual(b['OP_VALIDATOR_WITH_RETH'], '0') + self.assertEqual(E2E.environment('op-reth')['OP_RETH_ENABLE_PROOF_HISTORY'], 'true') + self.assertEqual(E2E.JOBS['proof']['timeout'], '60m') + + def test_op_reth_environment_matches_owning_recipe(self): + # RWX runs the compiled suite without the recipe, so it must export + # exactly what test-e2e-sysgo exports, and nothing the recipe dropped. + recipe = (SCRIPTS.parents[2] / 'rust/op-reth/tests/justfile').read_text() + body = re.search(r'^test-e2e-sysgo:.*\n((?:[ \t]+.*\n|\n)*)', recipe, re.M) + self.assertIsNotNone(body) + source = str(SCRIPTS.parents[2] / 'rust/op-reth/tests') + exported = {name: os.path.normpath(value.strip('"').replace('{{SOURCE_DIR}}', source)) + if '{{SOURCE_DIR}}' in value else value.strip('"') + for name, value in re.findall(r'^\s*export (\w+)=(\S+)$', body[1], re.M)} + self.assertEqual(E2E.environment('op-reth'), exported) + + def test_only_proofs_use_explicit_host_parallelism(self): + with patch.dict(os.environ, PARALLEL='32'): + self.assertEqual(E2E.parallel_flag('proof'), 32) + for job in ('restart', 'simple-kona', 'simple-kona-sequencer', 'op-reth'): + self.assertIsNone(E2E.parallel_flag(job)) + + +@unittest.skipUnless(os.environ.get('RWX_LIVE_GO_FIXTURE') == '1', 'Opt-in real Go/gotestsum fixture') +class LiveE2ETests(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory(); self.addCleanup(self.tmp.cleanup) + self.root = Path(self.tmp.name) + for name in ('rust-e2e.py', 'rust-e2e.sh', 'go-compiled-tests.py', 'rust-workspace-report.py', 'ci-report.py', 'go-report.py'): + path = self.root / 'ops/ci/runtime' / name; path.parent.mkdir(parents=True, exist_ok=True); shutil.copyfile(SCRIPTS / name, path) + path = self.root / 'ops/scripts/split-test-logs.sh'; path.parent.mkdir(); shutil.copyfile(SCRIPTS.parents[1] / 'scripts/split-test-logs.sh', path) + (self.root / 'go.mod').write_text('module github.com/ethereum-optimism/optimism\n\ngo 1.24.0\n') + (self.root / '.gitignore').write_text('.ci/\nexecuted\n') + package = self.root / 'rust/kona/tests/node/common'; package.mkdir(parents=True) + (package / 'fixture.txt').write_text('correct package directory') + (package / 'fixture_test.go').write_text('''package common +import ("testing"; "os"; "time") +func TestFresh(t *testing.T) { b,e:=os.ReadFile("fixture.txt"); if e!=nil || string(b)!="correct package directory" { t.Fatal("fixture path") }; f,e:=os.OpenFile("../../../../../executed",os.O_CREATE|os.O_APPEND|os.O_WRONLY,0600); if e!=nil { t.Fatal(e) }; defer f.Close(); f.WriteString("fresh\\n") } +func TestSkip(t *testing.T) { t.Skip("original skip reason") } +func TestFailure(t *testing.T) { if os.Getenv("E2E_INTENTIONAL_FAILURE")=="1" { t.Fatal("original failure") } } +func TestSlow(t *testing.T) { if os.Getenv("E2E_WAIT")=="1" { time.Sleep(time.Minute) } } +''') + for args in (['init', '-q'], ['config', 'user.email', 'fixture@example.invalid'], ['config', 'user.name', 'CI Fixture'], ['add', '.'], ['commit', '-qm', 'fixture']): + subprocess.run(['git', *args], cwd=self.root, check=True) + self.env = {**os.environ, 'CI_SHARD_INDEX': '0', 'PARALLEL': '2'} + result = self.run_mode('compile') + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + + def run_mode(self, mode, **env): + return subprocess.run(['bash', 'ops/ci/runtime/rust-e2e.sh', mode, 'simple-kona'], cwd=self.root, + env={**self.env, **env}, text=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + + def test_fresh_execution_twice_original_skip_and_fixture_paths(self): + for _ in range(2): + result = self.run_mode('run') + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertEqual((self.root / 'executed').read_text(), 'fresh\nfresh\n') + report = self.root / '.ci/rust-e2e/reports/simple-kona' + self.assertEqual(E2E.read(report / 'coverage.json')['cases']['TestSkip']['outcome'], 'skip') + self.assertIn('original skip reason', (report / 'original.json').read_text()) + + def test_real_failure_retains_original_json_junit_and_failing_status(self): + result = self.run_mode('run', E2E_INTENTIONAL_FAILURE='1') + self.assertNotEqual(result.returncode, 0) + report = self.root / '.ci/rust-e2e/reports/simple-kona' + final = E2E.read(report / 'final.json') + self.assertNotEqual(final['exit_code'], 0); self.assertEqual(final['report_errors'], []) + self.assertEqual(E2E.read(report / 'coverage.json')['cases']['TestFailure']['outcome'], 'fail') + self.assertIn('original failure', (report / 'original.json').read_text()) + self.assertIn('&2; exit 17\n') + (tests / 'justfile').write_text('SOURCE_DIR := justfile_directory()\nbuild-contracts:\n' + ' cd "{{SOURCE_DIR}}/proofs/contracts" && forge build\n') + (contracts / 'foundry.toml').write_text('[profile.default]\nsrc="src"\nout="artifacts"\n') + (contracts / 'src/Fixture.sol').write_text('// SPDX-License-Identifier: MIT\npragma solidity 0.8.15;\n' + 'contract Fixture { function value() public pure returns (uint256) { return 42; } }\n') + (package / 'fixture_test.go').write_text('''package core +import ("testing"; "os"; "encoding/json") +func TestRuntimeForge(t *testing.T) { + b,e:=os.ReadFile("../contracts/artifacts/Fixture.sol/Fixture.json"); if e!=nil { t.Fatal(e) } + var artifact struct { Bytecode struct { Object string } }; if e=json.Unmarshal(b,&artifact); e!=nil { t.Fatal(e) } + if len(artifact.Bytecode.Object)<4 { t.Fatal("Missing actual compiled fixture bytecode") } +} +''') + for args in (['add', '.'], ['commit', '-qm', 'nested Forge fixture']): + subprocess.run(['git', *args], cwd=self.root, check=True) + for mode in ('compile', 'run'): + result = subprocess.run(['bash', 'ops/ci/runtime/rust-e2e.sh', mode, 'op-reth'], cwd=self.root, + env=self.env, text=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + report = self.root / '.ci/rust-e2e/reports/op-reth' + stage = E2E.read(report / 'proof-contracts.stage.json') + self.assertEqual(stage['cwd'], 'rust/op-reth/tests') + self.assertEqual(stage['argv'], ['just', 'build-contracts']) + self.assertEqual(E2E.read(report / 'coverage.json')['cases']['TestRuntimeForge']['outcome'], 'pass') + + def test_proof_runs_each_assigned_test_in_its_own_process_with_explicit_parallelism(self): + package = self.root / 'rust/kona/tests/proofs'; package.mkdir(parents=True) + (package / 'fixture_test.go').write_text('package proofs\nimport ("flag"; "fmt"; "os"; "testing")\n' + 'func record(t *testing.T) { f,e:=os.OpenFile("../../../../executed",os.O_CREATE|os.O_APPEND|os.O_WRONLY,0600); if e!=nil { t.Fatal(e) }; defer f.Close(); ' + 'fmt.Fprintf(f, "%s %d %s\\n", t.Name(), os.Getpid(), flag.Lookup("test.parallel").Value) }\n' + + ''.join('func Test%d(t *testing.T) { record(t) }\n' % index for index in range(9))) + for mode in ('compile', 'run'): + result = subprocess.run(['bash', 'ops/ci/runtime/rust-e2e.sh', mode, 'proof'], cwd=self.root, + env=self.env, text=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + # Nine tests on eight shards: shard 0 runs the first and the ninth. + rows = [line.split() for line in (self.root / 'executed').read_text().splitlines()] + self.assertEqual([(name, parallel) for name, _, parallel in rows], [('Test0', '2'), ('Test8', '2')]) + self.assertNotEqual(rows[0][1], rows[1][1]) + report = self.root / '.ci/rust-e2e/reports/proof' + self.assertEqual(sorted(p.name for p in (report / 'junit').iterdir()), ['Test0.xml', 'Test8.xml']) + self.assertEqual(E2E.read(report / 'final.json'), {'exit_code': 0, 'report_errors': []}) + + def test_cancellation_preserves_partial_reports_and_cannot_pass(self): + process = subprocess.Popen(['bash', 'ops/ci/runtime/rust-e2e.sh', 'run', 'simple-kona'], cwd=self.root, + env={**self.env, 'E2E_WAIT': '1'}, start_new_session=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + path = self.root / '.ci/rust-e2e/reports/simple-kona/events/package.json' + deadline = time.monotonic() + 20 + while time.monotonic() < deadline: + if path.exists() and '"Test":"TestSlow"' in path.read_text(): break + if process.poll() is not None: break + time.sleep(.1) + os.killpg(process.pid, signal.SIGTERM) + out, err = process.communicate(timeout=20) + self.assertNotEqual(process.returncode, 0, out + err) + report = self.root / '.ci/rust-e2e/reports/simple-kona' + final = E2E.read(report / 'final.json') + self.assertNotEqual(final['exit_code'], 0) + self.assertIn('TestSlow', (report / 'original.json').read_text()) + + +if __name__ == '__main__': unittest.main() diff --git a/ops/ci/tests/test_rust_e2e_release.py b/ops/ci/tests/test_rust_e2e_release.py new file mode 100644 index 00000000000..3692009cd96 --- /dev/null +++ b/ops/ci/tests/test_rust_e2e_release.py @@ -0,0 +1,110 @@ +#!/usr/bin/env python3 +"""Run the E2E release build script with stub tools and check what it hands to the verdicts.""" +import json +import os +from pathlib import Path +import shutil +import subprocess +import tempfile +import unittest + +SCRIPTS = Path(__file__).resolve().parents[1] / 'runtime' +BINARIES = ('kona-host', 'kona-node', 'op-reth', 'op-reth-test-engine') + +STUB = r'''#!/usr/bin/env python3 +import json +import os +from pathlib import Path +import sys +command, args = Path(sys.argv[0]).name, sys.argv[1:] +with open(os.environ['CALL_LOG'], 'a') as log: + log.write(json.dumps({'command': command, 'args': args, 'cwd': os.getcwd(), + 'lto': os.environ.get('CARGO_PROFILE_RELEASE_LTO')}) + '\n') +if command == 'mold': + os.execvp(args[1], args[1:]) +elif command == 'sccache': + if args[:1] == ['--show-stats']: print('{}') +elif args == ['--version']: + print('pinned tool') +elif command == 'cargo' and args[:1] == ['metadata']: + names = os.environ['BUILT_BINARIES'].split() + print(json.dumps({'packages': [{'targets': [{'name': name, 'kind': ['bin']} for name in names] + + [{'name': 'library', 'kind': ['lib']}]}]})) +elif command == 'cargo' and args[:1] == ['build']: + release = Path(os.environ['CARGO_TARGET_DIR']) / 'release' + release.mkdir(parents=True, exist_ok=True) + for name in os.environ['BUILT_BINARIES'].split(): + (release / name).write_text('compiled ' + name) + sys.exit(int(os.environ.get('BUILD_EXIT', '0'))) +else: + sys.exit('Unexpected command: ' + command + ' ' + repr(args)) +''' + + +class ReleaseBuildTests(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory(); self.addCleanup(self.tmp.cleanup) + self.root = Path(self.tmp.name).resolve() / 'repo' + for name in ('rust-e2e-release.sh', 'op-reth-report.py', 'rust-target-cache.py', 'go-artifacts.py'): + path = self.root / 'ops/ci/runtime' / name; path.parent.mkdir(parents=True, exist_ok=True); shutil.copyfile(SCRIPTS / name, path) + pin = self.root / 'rust/op-reth/crates/chainspec/res/superchain-configs.tar.sha256' + pin.parent.mkdir(parents=True); pin.write_text('0' * 64 + ' superchain-configs.tar\n') + (self.root / 'rust/Cargo.lock').write_text('pinned dependencies\n') + (self.root / 'mise.toml').write_text('# Pinned tools are stubbed.\n') + (self.root / '.gitignore').write_text('.ci/\nrust/target/\n') + for args in (['init', '-q'], ['config', 'user.email', 'fixture@example.invalid'], ['config', 'user.name', 'CI Fixture'], ['add', '.'], ['commit', '-qm', 'fixture']): + subprocess.run(['git', *args], cwd=self.root, check=True) + tools = self.root.parent / 'bin'; tools.mkdir() + for name in ('cargo', 'mold', 'rustc', 'sccache'): + (tools / name).write_text(STUB); (tools / name).chmod(0o755) + self.log = self.root.parent / 'calls.jsonl' + sha = subprocess.check_output(['git', 'rev-parse', 'HEAD'], cwd=self.root, text=True).strip() + self.env = {**os.environ, 'PATH': str(tools) + os.pathsep + os.environ['PATH'], 'CALL_LOG': str(self.log), + 'CI_COMMIT_SHA': sha, 'BUILT_BINARIES': ' '.join(BINARIES + ('utility',))} + self.metadata = self.root / '.ci/go-tests/dependencies/rust-e2e-release/metadata.json' + + def build(self, part='binaries', **env): + return subprocess.run(['bash', 'ops/ci/runtime/rust-e2e-release.sh', part], cwd=self.root, env={**self.env, **env}, + text=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + + def builds(self): + return [call for call in map(json.loads, self.log.read_text().splitlines()) + if call['command'] == 'cargo' and call['args'][:1] == ['build']] + + WORKSPACE = ['build', '--locked', '--profile', 'release', '--workspace', '--features', 'default', '--lib'] + + def test_binaries_half_builds_and_packs_the_four_runtime_binaries(self): + result = self.build() + self.assertEqual(result.returncode, 0, result.stdout) + builds = self.builds() + # CircleCI's rust-e2e-binaries: every library, and the binaries of its `binary` parameter. + self.assertEqual([call['args'] for call in builds], + [self.WORKSPACE + ['--bin', 'op-reth', '--bin', 'op-reth-test-engine', '--bin', 'kona-node', '--bin', 'kona-host']]) + self.assertEqual((builds[0]['cwd'], builds[0]['lto']), (str(self.root / 'rust'), 'off')) + metadata = json.loads(self.metadata.read_text()) + self.assertEqual((metadata['kind'], metadata['commit_sha']), ('rust-e2e-release', self.env['CI_COMMIT_SHA'])) + self.assertEqual(set(metadata['files']), {'rust/target/release/' + name for name in BINARIES}) + + def test_workspace_half_builds_every_other_binary_and_packs_nothing(self): + result = self.build('workspace') + self.assertEqual(result.returncode, 0, result.stdout) + # CircleCI's rust-workspace-release: every library, and each binary that the other half leaves out. + self.assertEqual([call['args'] for call in self.builds()], [self.WORKSPACE + ['--bin', 'utility']]) + self.assertFalse(self.metadata.exists()) + self.assertNotEqual(self.build('everything').returncode, 0) + + def test_failed_build_keeps_its_status_and_packs_nothing(self): + result = self.build(BUILD_EXIT='23') + self.assertEqual(result.returncode, 23, result.stdout) + self.assertFalse(self.metadata.exists()) + + def test_missing_runtime_binary_fails(self): + for missing in BINARIES: + shutil.rmtree(self.root / 'rust/target', ignore_errors=True) + result = self.build(BUILT_BINARIES=' '.join(name for name in BINARIES if name != missing)) + with self.subTest(missing=missing): + self.assertNotEqual(result.returncode, 0, result.stdout) + self.assertFalse(self.metadata.exists()) + + +if __name__ == '__main__': unittest.main() diff --git a/ops/ci/tests/test_rust_target_cache.py b/ops/ci/tests/test_rust_target_cache.py new file mode 100644 index 00000000000..6c2c745ffc3 --- /dev/null +++ b/ops/ci/tests/test_rust_target_cache.py @@ -0,0 +1,390 @@ +import contextlib +import io +import importlib.util +import json +import os +from pathlib import Path +import shutil +import subprocess +import tempfile +import unittest +from unittest.mock import patch + +SPEC = importlib.util.spec_from_file_location('rust_cache', Path(__file__).resolve().parents[1] / 'runtime' / 'rust-target-cache.py') +CACHE = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(CACHE) + + +class RustTargetCacheTest(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + self.root = Path(self.temp.name) + subprocess.run(['git', 'init', '-q', str(self.root)], check=True) + self.source = self.root / 'rust/src/lib.rs' + self.source.parent.mkdir(parents=True) + self.source.write_text('pub fn old() {}') + (self.root / 'mise.toml').write_text('[tools]\nrust="1.95.0"\n') + subprocess.run(['git', 'add', '.'], cwd=self.root, check=True) + self.target = self.root / 'rust/target' + self.target.mkdir() + (self.root / '.git/info/exclude').write_text('rust/target/\n') + + def run_phase(self, phase): + with contextlib.redirect_stdout(io.StringIO()): + CACHE.manage(phase, self.root, self.target) + + def test_compiler_anchor_survives_source_edits_and_advances_for_build_inputs(self): + def commit(message): + subprocess.run(['git', 'add', '.'], cwd=self.root, check=True) + subprocess.run(['git', '-c', 'user.name=Fixture', '-c', 'user.email=fixture@example.invalid', + 'commit', '-qm', message], cwd=self.root, check=True) + return subprocess.check_output(['git', 'rev-parse', 'HEAD'], cwd=self.root, text=True).strip() + initial = commit('initial build inputs') + self.assertEqual(CACHE.compiler_base(self.root), initial) + self.source.write_text('pub fn source_changed() {}') + commit('ordinary source edit') + self.assertEqual(CACHE.compiler_base(self.root), initial) + # RWX's own cache key covers a task definition, so a .rwx change keeps the anchor. + # So does a script that no baseline runs. + for name in ('.rwx/rust.yml', 'ops/ci/runtime/pr-gate.py'): + path = self.root / name; path.parent.mkdir(parents=True, exist_ok=True) + path.write_text('not a baseline input') + commit(name) + self.assertEqual(CACHE.compiler_base(self.root), initial) + for name in ('rust/Cargo.lock', 'rust/local/Cargo.toml', 'ops/ci/runtime/rust-workspace.sh', + 'rust/.cargo/config.toml'): + path = self.root / name; path.parent.mkdir(parents=True, exist_ok=True) + path.write_text('changed build input') + changed = commit(name) + self.assertEqual(CACHE.compiler_base(self.root), changed) + self.source.write_text(self.source.read_text() + '\n// ordinary source edit') + commit('source after build change') + self.assertEqual(CACHE.compiler_base(self.root), changed) + + def test_changed_content_refreshes_normalized_mtimes_and_keeps_targets(self): + self.run_phase('prepare'); self.run_phase('commit') + binary = self.target / 'compiled-dependency' + binary.write_text('retained object') + old = self.source.stat().st_mtime_ns + toolchain = self.root / 'mise.toml' + toolchain_stamp = toolchain.stat().st_mtime_ns + self.source.write_text('pub trait Factory {}') + os.utime(self.source, ns=(old, old)) + self.run_phase('prepare') + self.assertGreater(self.source.stat().st_mtime_ns, old) + self.assertEqual(toolchain.stat().st_mtime_ns, CACHE.whole_seconds(toolchain_stamp)) + self.assertTrue(binary.exists()) + self.assertFalse((self.target / '.rwx-source-fingerprint.json').exists()) + + def test_unchanged_source_restores_stable_mtimes_for_target_reuse(self): + self.run_phase('prepare'); self.run_phase('commit') + stamp = self.source.stat().st_mtime_ns + os.utime(self.source, ns=(1000000000, 1000000000)) + self.run_phase('prepare') + self.assertEqual(self.source.stat().st_mtime_ns, CACHE.whole_seconds(stamp)) + + def test_restore_rounds_recorded_mtimes_down_to_whole_seconds(self): + self.run_phase('prepare'); self.run_phase('commit') + state = self.target / '.rwx-source-fingerprint.json' + recorded = json.loads(state.read_text()) + recorded['files']['rust/src/lib.rs']['mtime_ns'] = 1234567890123456789 + state.write_text(json.dumps(recorded)) + self.run_phase('prepare') + self.assertEqual(self.source.stat().st_mtime_ns, 1234567890000000000) + + def test_changed_inputs_use_next_second_and_wait_strictly_past_it(self): + name = CACHE.GENERATED_SOURCES[0] + with (self.root / '.git/info/exclude').open('a') as f: f.write(name + '\n') + archive = self.root / name; archive.parent.mkdir(parents=True) + archive.write_bytes(b'original generated input') + self.run_phase('prepare'); self.run_phase('commit') + self.source.write_text('pub fn changed() {}') + archive.write_bytes(b'changed generated input') + base = CACHE.whole_seconds(self.source.stat().st_mtime_ns) + 10**9 + stamp = base + 10**9 + with patch.object(CACHE.time, 'time_ns', side_effect=[base + 123456789, base + 500000000, stamp, stamp + 1]), \ + patch.object(CACHE.time, 'sleep') as sleep: + self.run_phase('prepare') + self.assertEqual(self.source.stat().st_mtime_ns, stamp) + self.assertEqual(archive.stat().st_mtime_ns, stamp) + self.assertEqual([call.args[0] for call in sleep.call_args_list], [0.500000001, 0.000000001]) + pending = json.loads((self.target / '.rwx-source-pending.json').read_text()) + self.assertEqual(pending['files']['rust/src/lib.rs']['mtime_ns'], stamp) + self.assertEqual(pending['generated_files'][name]['mtime_ns'], stamp) + + def test_unchanged_inputs_do_not_wait_for_unused_next_second(self): + self.run_phase('prepare'); self.run_phase('commit') + stamp = self.source.stat().st_mtime_ns + with patch.object(CACHE.time, 'time_ns', return_value=stamp + 10**9), \ + patch.object(CACHE.time, 'sleep') as sleep: + self.run_phase('prepare') + sleep.assert_not_called() + self.assertEqual(self.source.stat().st_mtime_ns, stamp) + + def test_restored_inputs_wait_after_a_backwards_clock_adjustment(self): + self.run_phase('prepare'); self.run_phase('commit') + stamp = self.source.stat().st_mtime_ns + with patch.object(CACHE.time, 'time_ns', side_effect=[stamp - 1, stamp - 1, stamp + 1]), \ + patch.object(CACHE.time, 'sleep') as sleep: + self.run_phase('prepare') + sleep.assert_called_once_with(0.000000002) + self.assertEqual(self.source.stat().st_mtime_ns, stamp) + + def test_failed_build_retries_refresh_source_again(self): + self.run_phase('prepare') + os.utime(self.source, ns=(1000000000, 1000000000)) + self.run_phase('prepare') + self.assertGreater(self.source.stat().st_mtime_ns, 1000000000) + + def test_ignored_generated_input_restores_timestamp_and_rejects_mid_build_change(self): + name = CACHE.GENERATED_SOURCES[0] + with (self.root / '.git/info/exclude').open('a') as f: f.write(name + '\n') + archive = self.root / name; archive.parent.mkdir(parents=True) + archive.write_bytes(b'caller-verified generated input') + os.utime(archive, ns=(1000000000000, 1000000000000)) + self.run_phase('prepare') + self.assertEqual(archive.stat().st_mtime_ns, 1000000000000) + self.run_phase('commit') + stamp = archive.stat().st_mtime_ns + os.utime(archive, ns=(1000000000, 1000000000)) + self.run_phase('prepare') + self.assertEqual(archive.stat().st_mtime_ns, stamp) + archive.write_bytes(b'changed while compiling') + with self.assertRaisesRegex(ValueError, 'Generated Rust input changed'): + self.run_phase('commit') + + def test_commit_names_source_files_that_changed_during_the_build(self): + with self.assertRaisesRegex(ValueError, 'Rust source preparation is missing'): + self.run_phase('commit') + self.run_phase('prepare') + output = self.root / 'rust/kona/out.bin.gz' + output.parent.mkdir(parents=True) + output.write_bytes(b'recipe output that git does not ignore') + with self.assertRaisesRegex(ValueError, 'Rust source changed during build: rust/kona/out.bin.gz$'): + self.run_phase('commit') + output.unlink() + self.run_phase('commit') + + def test_cold_build_records_newly_materialized_generated_input(self): + name = CACHE.GENERATED_SOURCES[0] + with (self.root / '.git/info/exclude').open('a') as f: f.write(name + '\n') + self.run_phase('prepare') + archive = self.root / name; archive.parent.mkdir(parents=True) + archive.write_bytes(b'materialized by checksum-verifying build.rs') + stamp = archive.stat().st_mtime_ns + self.run_phase('commit') + state = json.loads((self.target / '.rwx-source-fingerprint.json').read_text()) + self.assertEqual(state['generated_files'][name]['mtime_ns'], stamp) + self.assertEqual(archive.stat().st_mtime_ns, stamp) + self.run_phase('prepare') + self.assertEqual(archive.stat().st_mtime_ns, CACHE.whole_seconds(stamp)) + + def test_commit_detaches_incremental_metadata_without_changing_bytes_or_mtime(self): + metadata = self.target / 'debug/deps/liblocal.rmeta' + sibling = self.target / 'debug/incremental/local/session/metadata.rmeta' + metadata.parent.mkdir(parents=True) + sibling.parent.mkdir(parents=True) + metadata.write_bytes(b'compiled metadata') + os.utime(metadata, ns=(1234567890123456789, 1234567890123456789)) + os.link(metadata, sibling) + stamp = metadata.stat().st_mtime_ns + untouched = metadata.with_name('libexternal.rmeta') + untouched.write_bytes(b'independent metadata') + inode = untouched.stat().st_ino + self.run_phase('prepare'); self.run_phase('commit') + self.assertEqual(metadata.read_bytes(), sibling.read_bytes()) + self.assertEqual(metadata.stat().st_mtime_ns, stamp) + self.assertEqual(metadata.stat().st_mode, sibling.stat().st_mode) + self.assertNotEqual(metadata.stat().st_ino, sibling.stat().st_ino) + self.assertEqual(metadata.stat().st_nlink, 1) + self.assertEqual(untouched.stat().st_ino, inode) + metadata.write_bytes(b'rebuilt metadata') + self.assertEqual(sibling.read_bytes(), b'compiled metadata') + self.assertEqual(list(metadata.parent.glob('.rwx-metadata-*')), []) + + def test_failed_build_then_rollback_does_not_restore_successful_timestamp(self): + self.run_phase('prepare'); self.run_phase('commit') + old = self.source.stat().st_mtime_ns + self.source.write_text('pub fn changed() {}') + self.run_phase('prepare') + self.source.write_text('pub fn old() {}') + self.run_phase('prepare') + self.assertGreater(self.source.stat().st_mtime_ns, old) + + def test_legacy_cache_refreshes_once_then_preserves_per_file_timestamps(self): + (self.target / '.rwx-source-fingerprint.json').write_text(json.dumps({ + 'source_sha256': 'legacy', 'source_mtime_ns': 1000000000})) + self.run_phase('prepare'); self.run_phase('commit') + stamp = self.source.stat().st_mtime_ns + state = json.loads((self.target / '.rwx-source-fingerprint.json').read_text()) + self.assertEqual(state['version'], 3) + self.run_phase('prepare') + self.assertEqual(self.source.stat().st_mtime_ns, CACHE.whole_seconds(stamp)) + + def test_added_deleted_and_reintroduced_files_preserve_other_timestamps(self): + self.run_phase('prepare'); self.run_phase('commit') + original = self.source.stat().st_mtime_ns + new = self.source.with_name('new.rs') + new.write_text('pub struct New;') + self.run_phase('prepare'); self.run_phase('commit') + added = new.stat().st_mtime_ns + new.unlink() + self.run_phase('prepare'); self.run_phase('commit') + state = json.loads((self.target / '.rwx-source-fingerprint.json').read_text()) + self.assertNotIn('rust/src/new.rs', state['files']) + new.write_text('pub struct New;') + os.utime(new, ns=(added, added)) + self.run_phase('prepare') + self.assertGreater(new.stat().st_mtime_ns, added) + self.assertEqual(self.source.stat().st_mtime_ns, CACHE.whole_seconds(original)) + + def test_commit_rejects_source_changed_during_build(self): + self.run_phase('prepare') + self.source.write_text('changed during compilation') + with self.assertRaises(ValueError): self.run_phase('commit') + + def test_new_untracked_source_and_toolchain_changes_invalidate(self): + self.run_phase('prepare'); self.run_phase('commit') + before = json.loads((self.target / '.rwx-source-fingerprint.json').read_text()) + (self.source.parent / 'new.rs').write_text('pub struct New;') + (self.root / 'mise.toml').write_text('[tools]\nrust="1.96.0"\n') + self.run_phase('prepare'); self.run_phase('commit') + self.assertNotEqual(before, json.loads((self.target / '.rwx-source-fingerprint.json').read_text())) + + def test_embedded_nut_bundle_refreshes_sources_and_rejects_mid_build_change(self): + bundle = self.root / 'op-core/nuts/bundles/fixture.json' + bundle.parent.mkdir(parents=True); bundle.write_text('old embedded payload') + self.run_phase('prepare'); self.run_phase('commit') + old = bundle.stat().st_mtime_ns + source_stamp = self.source.stat().st_mtime_ns + before = json.loads((self.target / '.rwx-source-fingerprint.json').read_text())['source_sha256'] + bundle.write_text('new embedded payload'); os.utime(bundle, ns=(old, old)) + self.run_phase('prepare') + self.assertGreater(bundle.stat().st_mtime_ns, old) + self.assertEqual(self.source.stat().st_mtime_ns, CACHE.whole_seconds(source_stamp)) + self.assertNotEqual(before, json.loads((self.target / '.rwx-source-pending.json').read_text())['source_sha256']) + bundle.write_text('changed while building') + with self.assertRaises(ValueError): self.run_phase('commit') + + @unittest.skipUnless(shutil.which('cargo'), 'Cargo required for embedded bundle freshness reproduction') + def test_cargo_rebuilds_external_embedded_bundle_and_reuses_unchanged_target(self): + bundle = self.root / 'op-core/nuts/bundles/fixture.txt' + bundle.parent.mkdir(parents=True); bundle.write_text('old bundle') + (self.root / 'rust/Cargo.toml').write_text('[package]\nname="external-bundle"\nversion="0.1.0"\nedition="2021"\n') + self.source.write_text('') + (self.source.parent / 'main.rs').write_text('fn main() { print!("{}", include_str!("../../op-core/nuts/bundles/fixture.txt")); }') + env = {**os.environ, 'CARGO_TARGET_DIR': str(self.target)} + def build(): return subprocess.run(['cargo', 'build', '--offline'], cwd=self.root / 'rust', env=env, text=True, capture_output=True) + subprocess.run(['cargo', 'generate-lockfile', '--offline'], cwd=self.root / 'rust', env=env, check=True, capture_output=True) + self.run_phase('prepare') + first = build(); self.assertEqual(first.returncode, 0, first.stderr); self.run_phase('commit') + binary = self.target / 'debug/external-bundle' + self.assertEqual(subprocess.check_output([str(binary)], text=True), 'old bundle') + old = bundle.stat().st_mtime_ns + bundle.write_text('new bundle'); os.utime(bundle, ns=(old, old)) + stale = build(); self.assertEqual(stale.returncode, 0, stale.stderr) + self.assertEqual(subprocess.check_output([str(binary)], text=True), 'old bundle') + self.run_phase('prepare') + fresh = build(); self.assertEqual(fresh.returncode, 0, fresh.stderr); self.run_phase('commit') + self.assertEqual(subprocess.check_output([str(binary)], text=True), 'new bundle') + self.run_phase('prepare') + reused = build(); self.assertEqual(reused.returncode, 0, reused.stderr) + self.assertNotIn('Compiling external-bundle', reused.stderr) + + @unittest.skipUnless(shutil.which('cargo'), 'Cargo required for cache freshness reproduction') + def test_cargo_rebuilds_changed_crate_and_keeps_unrelated_crate_fresh(self): + for incremental in ('0', '1'): + with self.subTest(incremental=incremental): + target = self.target / incremental + for name in ('provider', 'unrelated', 'consumer'): + folder = self.root / 'rust' / name + (folder / 'src').mkdir(parents=True, exist_ok=True) + dependencies = '[dependencies]\nprovider={path="../provider"}\nunrelated={path="../unrelated"}\n' if name == 'consumer' else '' + (folder / 'Cargo.toml').write_text(f'[package]\nname="{name}"\nversion="0.1.0"\nedition="2021"\n{dependencies}') + (self.root / 'rust/Cargo.toml').write_text('[workspace]\nmembers=["provider","unrelated","consumer"]\nresolver="2"\n') + provider = self.root / 'rust/provider/src/lib.rs' + unrelated = self.root / 'rust/unrelated/src/lib.rs' + provider.write_text('pub fn value() -> u8 { 1 }') + unrelated.write_text('pub fn value() -> u8 { 10 }') + (self.root / 'rust/consumer/src/main.rs').write_text('fn main() { print!("{}", provider::value() + unrelated::value()); }') + env = {**os.environ, 'CARGO_TARGET_DIR': str(target)} + if incremental == '1': + env.pop('CARGO_INCREMENTAL', None) + env.pop('CARGO_BUILD_INCREMENTAL', None) + env['CARGO_PROFILE_DEV_INCREMENTAL'] = 'true' + else: + env['CARGO_INCREMENTAL'] = '0' + def build(): + result = subprocess.run(['cargo', 'build', '--offline', '--message-format=json'], + cwd=self.root / 'rust', env=env, text=True, capture_output=True) + self.assertEqual(result.returncode, 0, result.stderr) + return {row['target']['name']: row['fresh'] for row in + (json.loads(line) for line in result.stdout.splitlines()) + if row['reason'] == 'compiler-artifact'} + subprocess.run(['cargo', 'generate-lockfile', '--offline'], cwd=self.root / 'rust', + env=env, check=True, capture_output=True) + with contextlib.redirect_stdout(io.StringIO()): + CACHE.manage('prepare', self.root, target) + build() + CACHE.manage('commit', self.root, target) + def restore_whole_second_targets(): + for path in target.rglob('*'): + if path.is_file() and not path.is_symlink(): + rounded = CACHE.whole_seconds(path.stat().st_mtime_ns) + os.utime(path, ns=(rounded, rounded)) + restore_whole_second_targets() + with contextlib.redirect_stdout(io.StringIO()): + CACHE.manage('prepare', self.root, target) + self.assertTrue(all(build().values()), 'unchanged restored targets must stay fresh') + CACHE.manage('commit', self.root, target) + stamp = unrelated.stat().st_mtime_ns + old = provider.stat().st_mtime_ns + # Reproduce normalized checkout mtimes, including a changed file + # whose timestamp would otherwise falsely match the cached source. + provider.write_text('pub fn value() -> u8 { 2 }') + for path in (provider, unrelated): + os.utime(path, ns=(old, old)) + with contextlib.redirect_stdout(io.StringIO()): + CACHE.manage('prepare', self.root, target) + artifacts = build() + CACHE.manage('commit', self.root, target) + self.assertFalse(artifacts['provider']) + self.assertFalse(artifacts['consumer']) + self.assertTrue(artifacts['unrelated']) + self.assertEqual(unrelated.stat().st_mtime_ns, CACHE.whole_seconds(stamp)) + self.assertEqual(subprocess.check_output([str(target / 'debug/consumer')], text=True), '12') + restore_whole_second_targets() + with contextlib.redirect_stdout(io.StringIO()): + CACHE.manage('prepare', self.root, target) + self.assertTrue(all(build().values()), 'changed targets must stay fresh on the next restore') + CACHE.manage('commit', self.root, target) + + @unittest.skipUnless(shutil.which('cargo'), 'Cargo required for cache freshness reproduction') + def test_cargo_rebuilds_changed_dependency_with_older_checkout_mtime(self): + for name in ['provider','consumer']: + folder=self.root/'rust'/name; (folder/'src').mkdir(parents=True) + (folder/'Cargo.toml').write_text('[package]\nname="'+name+'"\nversion="0.1.0"\nedition="2021"\n'+('[dependencies]\nprovider={path="../provider"}\n' if name=='consumer' else '')) + (self.root/'rust/Cargo.toml').write_text('[workspace]\nmembers=["provider","consumer"]\nresolver="2"\n') + provider=self.root/'rust/provider/src/lib.rs'; consumer=self.root/'rust/consumer/src/lib.rs' + provider.write_text('pub fn old() {}'); consumer.write_text('pub fn caller() {}') + env={**os.environ,'CARGO_TARGET_DIR':str(self.target)} + def check(): return subprocess.run(['cargo','check','--offline','-p','consumer'],cwd=self.root/'rust',env=env,text=True,capture_output=True) + subprocess.run(['cargo','generate-lockfile','--offline'],cwd=self.root/'rust',env=env,check=True,capture_output=True) + self.run_phase('prepare'); self.assertEqual(check().returncode,0); self.run_phase('commit') + old=provider.stat().st_mtime_ns + provider.write_text('pub trait Factory {}'); os.utime(provider,ns=(old,old)) + consumer.write_text('pub fn caller() {}') + stale=check() + self.assertNotEqual(stale.returncode,0,stale.stderr) + self.assertIn('Factory',stale.stderr) + self.run_phase('prepare') + fresh=check(); self.assertEqual(fresh.returncode,0,fresh.stderr) + self.run_phase('commit') + os.utime(provider,ns=(1000000000,1000000000)) + self.run_phase('prepare') + reused=check(); self.assertEqual(reused.returncode,0,reused.stderr) + self.assertNotIn('Checking provider',reused.stderr) + + +if __name__ == '__main__': unittest.main() diff --git a/ops/ci/tests/test_rust_workspace.py b/ops/ci/tests/test_rust_workspace.py new file mode 100644 index 00000000000..219ebb3421c --- /dev/null +++ b/ops/ci/tests/test_rust_workspace.py @@ -0,0 +1,207 @@ +#!/usr/bin/env python3 +"""Run the Rust workspace job shell and its stage recorder against stub tools.""" +import importlib.util +import json +import os +from pathlib import Path +import shlex +import signal +import shutil +import subprocess +import sys +import tempfile +import time +import unittest + +SCRIPTS = Path(__file__).resolve().parents[1] / 'runtime' +REPO = Path(__file__).resolve().parents[3] +SPEC = importlib.util.spec_from_file_location('rust_workspace_report', SCRIPTS / 'rust-workspace-report.py') +REPORT = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(REPORT) + + +class ReportTests(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + self.root = Path(self.temp.name) + + def test_finish_keeps_the_original_exit_and_stage_records(self): + REPORT.write(self.root / 'settings.json', {'job': 'tests-compiler'}) + REPORT.write(self.root / 'doctests.stage.json', {'exit_code': 7}) + self.assertEqual(REPORT.finish(self.root, 7), 7) + final = json.loads((self.root / 'final.json').read_text()) + self.assertEqual((final['exit_code'], final['stages']['doctests']['exit_code']), (7, 7)) + # Only a job whose command writes no test report gets a JUnit case. + self.assertFalse((self.root / 'checks.junit.xml').exists()) + + def test_check_job_junit_follows_the_original_script_exit(self): + REPORT.write(self.root / 'settings.json', {'job': 'interop'}) + self.assertEqual(REPORT.finish(self.root, 0), 0) + self.assertNotIn('failure', (self.root / 'checks.junit.xml').read_text()) + self.assertEqual(REPORT.finish(self.root, 1), 1) + self.assertIn('failure', (self.root / 'checks.junit.xml').read_text()) + + +@unittest.skipUnless(shutil.which('just'), 'requires the pinned just tool') +class JobCommandTests(unittest.TestCase): + """Run the shared shell; Just dry-runs the unchanged recipes, so arity errors fail.""" + + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + root = self.root = Path(self.temp.name) + for name in ('ops/ci/runtime/rust-workspace.sh', 'ops/ci/runtime/rust-workspace-report.py', + 'ops/ci/runtime/ci-report.py', 'ops/ci/runtime/op-reth-report.py', + 'ops/ci/runtime/rust-target-cache.py', 'mise.toml', 'rust/justfile'): + (root / name).parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(REPO / name, root / name) + (root / 'rust/Cargo.lock').write_text('version = 4\n') + self.snapshot = root / 'rust/kona/crates/protocol/registry/etc/chainList.json' + self.snapshot.parent.mkdir(parents=True) + self.snapshot.write_text('[]\n') + checksum = root / 'rust/op-reth/crates/chainspec/res/superchain-configs.tar.sha256' + checksum.parent.mkdir(parents=True) + checksum.write_text('0' * 64 + ' superchain-configs.tar\n') + for args in (['init', '-q'], ['add', '.'], ['-c', 'user.name=Fixture', '-c', 'user.email=fixture@example.invalid', 'commit', '-qm', 'fixture']): + subprocess.run(['git', *args], cwd=root, check=True, capture_output=True) + shims = root / 'bin' + shims.mkdir() + # The Cargo stub stands for nextest only: it writes a JUnit report and exits as the test asks. + nextest = ('if [[ "$1 $2" == "nextest run" ]]; then mkdir -p "$CARGO_TARGET_DIR/nextest/default"; ' + 'echo "" > "$CARGO_TARGET_DIR/nextest/default/junit.xml"; exit "${NEXTEST_EXIT:-0}"; fi') + for name, body in (('just', 'exec ' + shlex.quote(shutil.which('just')) + ' --dry-run "$@"'), + ('cargo', nextest), ('rustc', 'true'), ('sccache', 'true'), ('tar', 'true')): + (shims / name).write_text('#!/usr/bin/env bash\n' + body + '\n') + (shims / name).chmod(0o755) + self.env = os.environ | {'PATH': str(shims) + os.pathsep + os.environ['PATH'], + 'RWX_RUN_ID': 'a' * 32, 'RWX_TASK_ATTEMPT_NUMBER': '1'} + + def run_job(self, job, **env): + return subprocess.run(['bash', 'ops/ci/runtime/rust-workspace.sh', job], cwd=self.root, + env=self.env | env, capture_output=True, text=True) + + def report(self, name): + return self.root / '.ci/rust-workspace' / name + + def test_partitions_pass_original_recipe_arguments_with_one_lockfile_seed(self): + seed = REPORT.digest(self.root / 'rust/Cargo.lock') + for index, total, partition in (('2', '10', '3/10'), ('0', '1', '')): + with self.subTest(total=total): + result = self.run_job('features', CI_RUST_PARTITION_INDEX=index, CI_RUST_PARTITION_TOTAL=total) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + report = self.report('features-' + index) + for stage, recipe in (('features', 'hack'), ('feature-tests', 'hack-tests-default')): + record = json.loads((report / (stage + '.stage.json')).read_text()) + self.assertEqual((record['argv'], record['cwd']), (['just', recipe, partition, 'true', seed], 'rust')) + self.assertEqual('--partition ' + partition in (report / (stage + '.log')).read_text(), bool(partition)) + + def test_unit_job_exports_the_nextest_junit_and_keeps_the_nextest_exit(self): + archive = self.root / 'archive' + archive.mkdir() + for status in (0, 7): + with self.subTest(status=status): + result = self.run_job('tests-unit', TEST_ARCHIVE=str(archive), NEXTEST_EXIT=str(status)) + self.assertEqual(result.returncode, status, result.stdout + result.stderr) + report = self.report('tests-unit') + self.assertEqual((report / 'junit.xml').read_text(), '\n') + record = json.loads((report / 'unit.stage.json').read_text()) + self.assertEqual((record['exit_code'], record['argv'][:3], record['argv'][-2:]), + (status, ['cargo', 'nextest', 'run'], ['-E', '!test(test_online)'])) + self.assertEqual(json.loads((report / 'final.json').read_text())['exit_code'], status) + + def test_compiler_tests_run_the_original_recipes(self): + result = self.run_job('tests-compiler') + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + final = json.loads((self.report('tests-compiler') / 'final.json').read_text()) + self.assertEqual(final['exit_code'], 0) + stages = {name: record['argv'] for name, record in final['stages'].items()} + self.assertEqual(stages, {'beacon': ['just', 'test-beacon-blob-stack'], 'doctests': ['just', 'test-docs'], + 'custom-embeds': ['just', 'test-custom-embeds']}) + # `just test` is what CircleCI's rust-tests job runs. When that recipe gains a part, + # the job runs it at once; RWX runs it only after this job names it. + dump = json.loads(subprocess.check_output([shutil.which('just'), '--justfile', str(REPO / 'rust/justfile'), '--dump', '--dump-format', 'json'], text=True)) + parts = [dependency['recipe'] for dependency in dump['recipes']['test']['dependencies']] + self.assertEqual(sorted(parts), sorted(['test-unit', *(argv[1] for argv in stages.values())])) + + def test_registry_verdict_is_the_diff_of_the_regenerated_snapshots(self): + result = self.run_job('registry') + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + report = self.report('registry') + final = json.loads((report / 'final.json').read_text()) + self.assertEqual(list(final['stages']), ['registry', 'registry-clean', 'registry-diff']) + self.assertEqual(final['stages']['registry']['cwd'], 'rust/kona') + self.assertNotIn('failure', (report / 'checks.junit.xml').read_text()) + # The stub Cargo regenerates nothing. A snapshot that differs from the commit stands for stale output. + self.snapshot.write_text('[1]\n') + result = self.run_job('registry') + self.assertEqual(result.returncode, 1, result.stdout + result.stderr) + final = json.loads((report / 'final.json').read_text()) + self.assertEqual((final['exit_code'], final['stages']['registry-diff']['exit_code']), (1, 1)) + self.assertIn('failure', (report / 'checks.junit.xml').read_text()) + + +class StageTests(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + self.root = Path(self.temp.name) + (self.root / 'rust').mkdir() + self.report = self.root / 'reports' + self.report.mkdir() + + def run_stage(self, code): + return subprocess.run([sys.executable, str(SCRIPTS / 'rust-workspace-report.py'), + 'stage', str(self.report), 'failure', sys.executable, '-c', code], + cwd=self.root, capture_output=True) + + def test_real_failing_process_retains_original_log(self): + result = self.run_stage('print("original failure", flush=True); raise SystemExit(7)') + self.assertEqual(result.returncode, 7) + self.assertIn('original failure', (self.report / 'failure.log').read_text()) + self.assertEqual(json.loads((self.report / 'failure.stage.json').read_text())['exit_code'], 7) + + def test_explicit_working_directory_is_used_and_recorded(self): + result = subprocess.run([sys.executable, str(SCRIPTS / 'rust-workspace-report.py'), + 'stage-at', str(self.report), '.', 'root', sys.executable, '-c', + 'import os; print(os.getcwd())'], cwd=self.root, capture_output=True) + self.assertEqual(result.returncode, 0) + self.assertEqual((self.report / 'root.log').read_text().strip(), str(self.root.resolve())) + self.assertEqual(json.loads((self.report / 'root.stage.json').read_text())['cwd'], '.') + + def test_cancellation_reaches_child_group(self): + process = subprocess.Popen([sys.executable, str(SCRIPTS / 'rust-workspace-report.py'), + 'stage', str(self.report), 'cancel', sys.executable, '-c', + 'import time; print("running", flush=True); time.sleep(60)'], + cwd=self.root, stdout=subprocess.DEVNULL) + try: + for _ in range(100): + path = self.report / 'cancel.log' + if path.exists() and 'running' in path.read_text(): + break + time.sleep(.02) + else: + self.fail('Child did not start') + process.send_signal(signal.SIGTERM) + self.assertEqual(process.wait(timeout=5), 143) + self.assertEqual(json.loads((self.report / 'cancel.stage.json').read_text())['exit_code'], -signal.SIGTERM) + finally: + if process.poll() is None: + process.kill() + process.wait() + + +class ConfigurationTests(unittest.TestCase): + @unittest.skipUnless(shutil.which('yq'), 'requires the pinned yq tool') + def test_feature_tasks_cover_every_partition_once(self): + """Each hand-written task pair must check its own cargo-hack partition; a copied index leaves one unchecked.""" + tasks = {task['key']: task for task in json.loads(subprocess.check_output( + ['yq', '-o=json', '.tasks', str(REPO / '.rwx/rust.yml')], text=True))} + for index in range(10): + for key in (f'features-build-{index}', f'features-{index}'): + env = tasks[key]['env'] + self.assertEqual((env['CI_RUST_PARTITION_INDEX'], env['CI_RUST_PARTITION_TOTAL']), (str(index), '10'), key) + + +if __name__ == '__main__': + unittest.main() diff --git a/ops/ci/tests/test_rwx_contracts_prepare.py b/ops/ci/tests/test_rwx_contracts_prepare.py new file mode 100644 index 00000000000..8bb51fc4894 --- /dev/null +++ b/ops/ci/tests/test_rwx_contracts_prepare.py @@ -0,0 +1,114 @@ +#!/usr/bin/env python3 +"""Exercise the RWX contract tool installation, its download retries and failures.""" + +import json +import os +from pathlib import Path +import shutil +import subprocess +import tempfile +import unittest + + +SCRIPT = Path(__file__).resolve().parents[1] / 'runtime' / 'rwx-contracts-prepare.sh' + +# Commands are stubbed at the process boundary: the real script decides which +# tools and compilers to install, without downloading anything. +STUB = r'''#!/usr/bin/env python3 +import json +import os +from pathlib import Path +import sys + +command = Path(sys.argv[0]).name +args = sys.argv[1:] +env = os.environ +with Path(env["CALL_LOG"]).open("a") as log: + log.write(json.dumps({"command": command, "args": args}) + "\n") + +if command == "mise" and args == ["bin-paths"]: + print(env["STUB_BIN"]) + sys.exit(0) +elif command == "mise" and args == ["install", "forge", "cast", "svm-rs"]: + label = "mise" +elif command == "svm" and len(args) == 2 and args[0] == "which": + sys.exit(0 if args[1] in env.get("INSTALLED_SOLC", "").split(",") else 1) +elif command == "svm" and len(args) == 2 and args[0] == "install": + label = "svm-" + args[1] +elif command == "sleep": + sys.exit(0) +else: + sys.exit("Unexpected command: " + command + " " + repr(args)) + +counter = Path(env["COUNTERS"]) / label +count = int(counter.read_text()) + 1 if counter.exists() else 1 +counter.write_text(str(count)) +if count <= int(env.get("FAIL_" + label.upper().replace("-", "_").replace(".", "_"), "0")): + sys.exit(17) +print("Prepared " + label) +''' + + +class RwxContractsPrepareTest(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + self.root = Path(self.temp.name) + self.script = self.root / "ops/ci/runtime" / SCRIPT.name + self.script.parent.mkdir(parents=True) + shutil.copyfile(SCRIPT, self.script) + self.bin = self.root / "bin" + self.bin.mkdir() + for name in ("mise", "svm", "sleep"): + path = self.bin / name + path.write_text(STUB, encoding="utf-8") + path.chmod(0o755) + self.log = self.root / "calls.jsonl" + self.counters = self.root / "counters" + self.counters.mkdir() + + def run_helper(self, mode="tools", **overrides): + env = dict(os.environ) + env.update( + PATH=str(self.bin) + os.pathsep + os.environ["PATH"], + CALL_LOG=str(self.log), COUNTERS=str(self.counters), STUB_BIN=str(self.bin), + RWX_ENV=str(self.root / "rwx-env"), + ) + env.update(overrides) + return subprocess.run(["bash", str(self.script), mode], cwd=self.root, env=env, text=True, capture_output=True) + + def calls(self): + return [json.loads(line) for line in self.log.read_text().splitlines()] if self.log.exists() else [] + + def test_tools_install_only_pinned_contract_tools_and_exact_missing_compilers(self): + result = self.run_helper(FAIL_MISE="1", FAIL_SVM_0_8_25="1", INSTALLED_SOLC="0.8.15") + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + calls = self.calls() + installs = [call["args"] for call in calls if call["command"] == "mise" and call["args"][0] == "install"] + self.assertEqual(installs, [["install", "forge", "cast", "svm-rs"]] * 2) + self.assertEqual([call["args"][1] for call in calls if call["command"] == "svm" and call["args"][0] == "which"], + ["0.8.15", "0.8.19", "0.8.25", "0.8.28"]) + self.assertEqual([call["args"][1] for call in calls if call["command"] == "svm" and call["args"][0] == "install"], + ["0.8.19", "0.8.25", "0.8.25", "0.8.28"]) + self.assertTrue((self.root / "rwx-env/PATH").is_file()) + + def test_failed_tool_or_compiler_download_stops_bootstrap(self): + for overrides, expected_command in (({"FAIL_MISE": "5"}, "mise"), ({"FAIL_SVM_0_8_15": "5"}, "svm")): + with self.subTest(overrides=overrides): + for path in self.counters.iterdir(): + path.unlink() + self.log.unlink(missing_ok=True) + result = self.run_helper(**overrides) + self.assertEqual(result.returncode, 17, result.stdout + result.stderr) + downloads = [call for call in self.calls() if call["command"] == expected_command and call["args"][0] == "install"] + self.assertEqual(len(downloads), 5) + self.assertFalse((self.root / "rwx-env/PATH").exists()) + + def test_unknown_mode_installs_nothing(self): + result = self.run_helper("source") + self.assertEqual((result.returncode, self.calls()), (1, [])) + self.assertIn("Usage:", result.stderr) + + +if __name__ == "__main__": + unittest.main() diff --git a/ops/ci/tests/test_rwx_local_packages.py b/ops/ci/tests/test_rwx_local_packages.py new file mode 100644 index 00000000000..6c29e241c95 --- /dev/null +++ b/ops/ci/tests/test_rwx_local_packages.py @@ -0,0 +1,240 @@ +"""Check the chosen DAG and the native package input boundary caught by the probe.""" +import json +import os +import importlib.util +import re +from pathlib import Path +import subprocess +import shutil +import tempfile +import unittest + +ROOT = Path(__file__).resolve().parents[3] + + +def definition(name): + return json.loads(subprocess.check_output( + ['yq', '-o=json', 'explode(.)', str(ROOT / '.rwx' / name)], text=True)) + + +def tasks(name): + return {task['key']: task for task in definition(name)['tasks']} + + +class LocalPackageTests(unittest.TestCase): + def test_each_of_twelve_go_verdicts_waits_for_only_its_own_compilation(self): + graph = tasks('go-tests.yml') + for phase in ('compile', 'verdict'): + self.assertEqual({key for key in graph if key.startswith(phase + '-')}, + {phase + '-' + str(index) for index in range(12)}) + for index in range(12): + compile_task = graph['compile-' + str(index)] + # The seed uses the tools, so RWX applies it above them. `discovery` uses `source`, + # so RWX applies it above `source`. + self.assertEqual(compile_task['use'], ['go-build-tools', 'compiler-seed-' + str(index), 'source', 'discovery']) + verdict = graph['verdict-' + str(index)] + self.assertEqual([key for key in verdict['use'] if key.startswith('compile-')], + ['compile-' + str(index) + '.build']) + self.assertTrue({'go.build', 'contracts.build', 'kona.build', 'op-reth.build', + 'prestate.build', 'discovery', 'source'} <= set(verdict['use'])) + leaf = tasks('packages/go-test-verdict.yml')['run'] + self.assertIs(leaf['cache'], False) + self.assertTrue({'!.ci/go-cache/full/build', '!.ci/go-cache/full/dependencies', '!.ci/rust-cache', '!rust/target'} <= set(leaf['filter'])) + self.assertEqual(leaf['outputs']['filesystem']['filter']['workspace'], ['.ci/go-cache/full/run']) + + def test_every_workload_builds_a_dependency_at_the_planned_commit_with_the_same_tools(self): + # A dependency build has one cache key for all workloads and for later commits only if + # each caller passes the planned commit and the same tool tasks, and nothing else. + tools = {'go': ['go'], 'contracts': ['go', 'anvil'], 'kona': ['release'], 'op-reth': ['release'], + 'sp1-executor': ['release'], 'prestate': ['go'], 'modules': ['go']} + calls = 0 + for path in sorted((ROOT / '.rwx').glob('*.yml')): + graph = tasks(path.name) + for key, task in graph.items(): + match = re.fullmatch(r'\$\{\{ run\.dir \}\}/packages/go-(' + '|'.join(tools) + r')\.yml', task.get('call', '')) + if not match: continue + kind, calls = match.group(1), calls + 1 + planned = {'modules': 'go-modules'}.get(kind, kind) + self.assertEqual(task['with']['commit-sha'], '${{ tasks.dependency-plan.values.' + planned + ' }}', (path.name, key)) + used = task['use'] if isinstance(task['use'], list) else [task['use']] + self.assertEqual([graph[tool]['with']['mode'] for tool in used], tools[kind], (path.name, key)) + self.assertEqual(calls, 24) + for kind in tools: + build = tasks('packages/go-' + kind + '.yml')['build'] + self.assert_starts_without_a_checkout(build, build['outputs']['filesystem']['filter']['workspace']) + self.assertEqual((build['use'], build['env']['CI_COMMIT_SHA']), ('package.use', '${{ params.commit-sha }}')) + # The Go compiler seeds follow the same contract. + graph = tasks('go-tests.yml') + prepared = graph['compiler-preparation']['outputs']['filesystem']['filter']['workspace'] + self.assert_starts_without_a_checkout(graph['compiler-preparation'], ['.ci/go-tests/dependencies/contracts']) + self.assertEqual(graph['discovery']['outputs']['filesystem']['filter']['workspace'], prepared) + for index in range(12): + seed = graph['compiler-seed-' + str(index)] + self.assertEqual(seed['use'], 'compiler-preparation') + self.assert_starts_without_a_checkout(seed, prepared) + self.assertEqual(seed['outputs']['filesystem']['filter']['workspace'], ['.ci/go-cache/full/build']) + + def test_contract_compiles_name_the_files_that_they_read(self): + # A compile with `.git` among its inputs has a new cache key for each commit. RWX keys a + # task on file contents only for an include list, so each compile names what it reads. + shared = tasks('packages/contracts-compile.yml')['build'] + fork = tasks('contract-l2-fork.yml')['compile'] + scripts = lambda task: {name for name in task['filter'] if name.startswith('ops/ci/runtime/')} + self.assertEqual(scripts(shared), {'ops/ci/runtime/ci-report.py', 'ops/ci/runtime/contract-upgrades.py'}) + self.assertEqual(scripts(fork), scripts(shared) | {'ops/ci/runtime/contract-l2-fork.py', 'ops/ci/runtime/rpc-pacer.py'}) + self.assertEqual(set(shared['env']), {'CI_BRANCH'}) + # The FFI helper is Go. Follow its imports: each package must be under a listed tree. + prefix, seen, pending = 'github.com/ethereum-optimism/optimism/', set(), ['packages/contracts-bedrock/scripts/go-ffi'] + while pending: + directory = pending.pop() + if directory in seen: continue + seen.add(directory) + for source in (ROOT / directory).glob('*.go'): + if source.name.endswith('_test.go'): continue + for found in re.findall('"' + re.escape(prefix) + '([^"]+)"', source.read_text()): + if (ROOT / found).is_dir(): pending.append(found) + self.assertGreater(len(seen), 10) + for task in (shared, fork): + self.assertFalse([name for name in task['filter'] if name.startswith('!')]) + self.assertTrue({'mise.toml', 'go.mod', 'go.sum', 'packages/contracts-bedrock', '.ci/go-cache/pr-checks/modules'} <= set(task['filter'])) + for directory in seen: + top = directory.split('/')[0] + self.assertTrue(top in task['filter'] or (top.startswith('op-') and 'op-*' in task['filter']) + or directory.startswith('packages/contracts-bedrock'), directory) + verdicts = 0 + for path in ('contracts.yml', 'contract-upgrades.yml'): + graph = tasks(path) + self.assertEqual(graph['compile']['call'], '${{ run.dir }}/packages/contracts-compile.yml') + for key, task in graph.items(): + if key.startswith('verdict-') and '-modified-' not in key: + verdicts += 1 + self.assertEqual(task['use'][-1], 'compile', (path, key)) + self.assertEqual(verdicts, 13) + + def assert_starts_without_a_checkout(self, task, inputs): + """The task sees mise.toml and `inputs` only, checks out CI_COMMIT_SHA itself and filters its output. + + A task passes on every layer that it uses, and the tool tasks carry the checkout of the + run. Without the input filter, that checkout is in the task's cache key and under its + own checkout. Without the output filter, the task's checkout reaches each consumer. + """ + flat = [name for entry in task['filter'] for name in (entry if isinstance(entry, list) else [entry])] + self.assertEqual(flat, ['mise.toml', *inputs]) + self.assertIn('git fetch -q origin "$CI_COMMIT_SHA"\ngit checkout -qf FETCH_HEAD\n', task['run']) + self.assertEqual(task['outputs']['filesystem']['filter']['system'], []) + + def test_acceptance_keeps_both_variants_and_excludes_compiler_state(self): + graph = tasks('acceptance.yml') + # Like Circle's memory-all jobs: 4 nodes each, on 32 GiB (2xlarge.gen2). + self.assertEqual({key for key in graph if key.startswith('verdict-')}, + {'verdict-' + variant + '-' + str(index) + for variant in ('opn', 'kona') for index in range(4)}) + for variant, cl in [('opn', 'op-node'), ('kona', 'kona-node')]: + for index in range(4): + verdict = graph['verdict-' + variant + '-' + str(index)] + # The circleci shim gives each shard index its own share of the tests. + self.assertEqual((verdict['with']['cl-kind'], verdict['with']['shard-index']), (cl, str(index))) + leaf = tasks('packages/acceptance-verdict.yml')['run'] + self.assertIs(leaf['cache'], False) + self.assertEqual(leaf['runner']['memory'], '32gb') + # Compiler state stays out; the producer's module cache stays in, as in Circle. + self.assertTrue({'!.ci/go-cache/full/build', '!.ci/go-cache/full/dependencies', + '!.ci/go-cache/acceptance/run'} <= set(leaf['filter'])) + self.assertFalse([rule for rule in leaf['filter'] + if '.ci/go-cache/full/modules'.startswith(rule.lstrip('!').rstrip('/'))]) + self.assertIn('!.ci/rust-cache', leaf['filter']) + self.assertIn('!rust/target', leaf['filter']) + # The task does not reset its checkout, so each excluded path must hold no tracked file. + # A hidden tracked file is a modified build input to go-artifacts.py. + hidden = subprocess.check_output(['git', 'ls-files', '--', *(rule.lstrip('!') for rule in leaf['filter'])], cwd=ROOT, text=True) + self.assertEqual(hidden, '') + + def test_artifact_mounts_and_credentials_are_not_package_value_arguments(self): + # Native RWX rejected an artifact expression in with even though lint + # accepted it. Bytes now arrive through filtered package.use inputs. + for path in sorted((ROOT / '.rwx').glob('*.yml')): + for task in tasks(path.name).values(): + if task.get('call', '').startswith('${{ run.dir }}/packages/'): + values = json.dumps(task.get('with', {})) + with self.subTest(path=path.name, task=task['key']): + self.assertNotIn('.artifacts.', values) + self.assertNotIn('vaults.', values) + + def test_output_filesystem_filters_and_test_results_have_no_parameterized_paths(self): + # RWX does not expand an expression in these paths, and gives no warning. A filter then + # omits the path, and a task reports no tests. RWX does expand an artifact path. + for name in ('contracts-compile', 'contracts-compile-modified'): + leaf = tasks('packages/' + name + '.yml')['build'] + filters = leaf['outputs']['filesystem']['filter']['workspace'] + self.assertNotIn('${{', json.dumps(filters)) + declared = 0 + for path in sorted((ROOT / '.rwx').rglob('*.yml')): + for task in tasks(path.relative_to(ROOT / '.rwx').as_posix()).values(): + results = task.get('outputs', {}).get('test-results', []) + declared += len(results) + with self.subTest(path=path.name, task=task['key']): + self.assertNotIn('${{', json.dumps(results)) + self.assertGreater(declared, 20) + + def test_nested_references_are_bound_to_calls_with_the_declared_child(self): + # The first hosted run caught a reference to a .build child on an + # unchanged Rust E2E command. Lint did not reject that reference. + for path in sorted((ROOT / '.rwx').glob('*.yml')): + graph = tasks(path.name) + for parent, child in set(re.findall(r'tasks\.([\w-]+)\.tasks\.([\w-]+)', path.read_text())): + with self.subTest(path=path.name, parent=parent, child=child): + self.assertIn('call', graph[parent]) + called = graph[parent]['call'].removeprefix('${{ run.dir }}/') + self.assertIn(child, tasks(called)) + + def test_shared_tools_use_narrow_files_without_checkout_history(self): + for path in sorted((ROOT / '.rwx').glob('*.yml')): + graph = tasks(path.name) + if graph.get('tools', {}).get('call') != '${{ run.dir }}/packages/toolchain-common.yml': + continue + self.assertEqual(graph['tools']['use'], ['mise', 'bootstrap-inputs']) + files = graph['bootstrap-inputs']['outputs']['filesystem']['filter']['workspace'] + self.assertIn('mise.toml', files) + self.assertTrue(all(name == 'mise.toml' or name.startswith('ops/ci/runtime/') + for name in files)) + self.assertNotIn('.git', files) + for name in ('common', 'go', 'foundry', 'rust'): + self.assertNotIn('vaults.', json.dumps(definition('packages/toolchain-' + name + '.yml'))) + # A tool task without this filter has the checkout in its cache key. It then runs + # for each commit, and gives every task that uses it a new cache key. + files = tasks('packages/toolchain-' + name + '.yml')['prepare']['filter']['workspace'] + self.assertIn('mise.toml', files) + self.assertTrue(all(name == 'mise.toml' or name.startswith('ops/ci/runtime/') for name in files)) + # The same holds for a tool task that a workload defines itself. + inline = {'acceptance.yml': ['acceptance-tools'], 'rust-e2e.yml': ['runtime-tools'], + 'rust.yml': ['wasm-tools', 'lint-tools', 'rust-fmt-tools'], + 'pr-checks.yml': ['main-tools', 'static-tools', 'go-lint-tools']} + for path, keys in inline.items(): + graph = tasks(path) + for key in keys: + files = graph[key]['filter'] + self.assertIn('mise.toml', files, (path, key)) + self.assertTrue(all(name == 'mise.toml' or name.startswith('ops/ci/runtime/') for name in files), (path, key)) + + + def test_tool_preparation_retries_downloads_and_exports_the_policy(self): + # Tool downloads time out intermittently. Each install sees mise's retry + # policy, and the exported environment carries it to later installs. + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + binaries = root / 'bin'; binaries.mkdir() + log = root / 'mise.log' + stub = binaries / 'mise' + stub.write_text('#!/bin/bash\nprintf "%s retries=%s timeout=%s\\n" "$*" "${MISE_HTTP_RETRIES:-}" "${MISE_HTTP_TIMEOUT:-}" >> "$STUB_LOG"\n') + stub.chmod(0o755) + exports = root / 'exports' + env = dict(os.environ, PATH=str(binaries) + os.pathsep + os.environ['PATH'], RWX_ENV=str(exports), STUB_LOG=str(log)) + env.pop('MISE_HTTP_RETRIES', None); env.pop('MISE_HTTP_TIMEOUT', None) + subprocess.run(['bash', str(ROOT / 'ops/ci/runtime/rwx-prepare.sh'), 'go'], env=env, check=True) + self.assertIn('install go gotestsum retries=5 timeout=90s', log.read_text().splitlines()) + self.assertEqual((exports / 'MISE_HTTP_RETRIES').read_text(), '5\n') + self.assertEqual((exports / 'MISE_HTTP_TIMEOUT').read_text(), '90s\n') + + +if __name__ == '__main__': + unittest.main() diff --git a/ops/ci/tests/test_rwx_materialize_linker.py b/ops/ci/tests/test_rwx_materialize_linker.py new file mode 100644 index 00000000000..bb33612b877 --- /dev/null +++ b/ops/ci/tests/test_rwx_materialize_linker.py @@ -0,0 +1,124 @@ +import errno +import importlib.util +import os +from pathlib import Path +import stat +import tempfile +import unittest +from unittest.mock import patch + + +SPEC = importlib.util.spec_from_file_location( + 'linker', Path(__file__).parents[1] / 'runtime/rwx-materialize-linker.py') +LINKER = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(LINKER) + + +class MaterializeLinkerTests(unittest.TestCase): + def setUp(self): + self.directory = tempfile.TemporaryDirectory() + self.addCleanup(self.directory.cleanup) + self.path = Path(self.directory.name) / 'crt.o' + self.path.write_bytes(b'original linker input') + self.path.chmod(0o640) + os.utime(self.path, ns=(1_700_000_000_000_000_000, 1_700_000_001_123_456_789)) + + def test_detaches_hardlink_preserving_bytes_owner_mode_and_mtime(self): + alias = self.path.with_name('alias') + os.link(self.path, alias) + before = self.path.stat() + record = {} + LINKER.materialize(self.path, record) + after = self.path.stat() + self.assertEqual(self.path.read_bytes(), alias.read_bytes()) + self.assertEqual(alias.stat().st_ino, before.st_ino) + self.assertNotEqual(after.st_ino, before.st_ino) + self.assertEqual(after.st_nlink, 1) + self.assertEqual((after.st_uid, after.st_gid, stat.S_IMODE(after.st_mode), after.st_atime_ns, after.st_mtime_ns), + (before.st_uid, before.st_gid, stat.S_IMODE(before.st_mode), before.st_atime_ns, before.st_mtime_ns)) + self.assertTrue(record['detached']) + self.assertEqual(record['read_attempts'], 1) + self.assertEqual(list(self.path.parent.glob('.rwx-linker-*')), []) + + def test_retries_only_timed_out_reads_then_copies_original_bytes(self): + original = Path.read_bytes + calls = [] + + def read(path): + if path == self.path: + calls.append(path) + if len(calls) == 1: + raise OSError(errno.ETIMEDOUT, 'Connection timed out') + return original(path) + + record = {} + with patch.object(Path, 'read_bytes', read), patch.object(LINKER.time, 'sleep') as sleep: + LINKER.materialize(self.path, record) + self.assertEqual(record['read_attempts'], 2) + self.assertEqual(record['read_timeouts'], 1) + self.assertEqual(self.path.read_bytes(), b'original linker input') + sleep.assert_called_once_with(1) + + def test_persistent_timeout_is_bounded_and_does_not_replace_input(self): + before = self.path.stat() + record = {} + with patch.object(Path, 'read_bytes', side_effect=OSError(errno.ETIMEDOUT, 'timeout')), \ + patch.object(LINKER.time, 'sleep'): + with self.assertRaises(OSError): + LINKER.materialize(self.path, record) + self.assertEqual(record['read_attempts'], 3) + self.assertEqual(record['read_timeouts'], 3) + self.assertEqual(self.path.stat().st_ino, before.st_ino) + + def test_other_read_errors_fail_immediately(self): + for code in (errno.EACCES, errno.ENOENT, errno.EIO): + with self.subTest(code=code): + record = {} + with patch.object(Path, 'read_bytes', side_effect=OSError(code, 'failure')), \ + patch.object(LINKER.time, 'sleep') as sleep: + with self.assertRaises(OSError): + LINKER.materialize(self.path, record) + self.assertEqual(record['read_attempts'], 1) + sleep.assert_not_called() + + def test_corrupt_local_copy_never_replaces_input(self): + before = self.path.stat() + original = Path.read_bytes + + def read(path): + return original(path) if path == self.path else b'corrupt copy' + + with patch.object(Path, 'read_bytes', read): + with self.assertRaisesRegex(ValueError, 'digest mismatch'): + LINKER.materialize(self.path, {}) + self.assertEqual(self.path.stat().st_ino, before.st_ino) + self.assertEqual(self.path.read_bytes(), b'original linker input') + self.assertEqual(list(self.path.parent.glob('.rwx-linker-*')), []) + + def test_failed_replace_keeps_original_and_cleans_temporary(self): + before = self.path.stat() + with patch.object(LINKER.os, 'replace', side_effect=OSError(errno.ENOSPC, 'disk full')): + with self.assertRaises(OSError): + LINKER.materialize(self.path, {}) + self.assertEqual(self.path.stat().st_ino, before.st_ino) + self.assertEqual(list(self.path.parent.glob('.rwx-linker-*')), []) + + def test_rejects_missing_driver_input_and_outside_system_root(self): + for value in ('crti.o', str(self.path)): + with self.subTest(value=value), patch.object(LINKER.subprocess, 'check_output', return_value=value): + with self.assertRaises(ValueError): + LINKER.linker_inputs() + + def test_reports_partial_failure_and_returns_failure_status(self): + with patch.object(LINKER, 'linker_inputs', return_value=[self.path]), \ + patch.object(LINKER, 'materialize', side_effect=OSError(errno.EIO, 'failure')), \ + patch('builtins.print') as output: + self.assertEqual(LINKER.main(), 1) + import json + report = json.loads(output.call_args.args[0]) + self.assertFalse(report['success']) + self.assertIn('failure', report['error']) + + +if __name__ == '__main__': + unittest.main() diff --git a/ops/ci/tests/test_rwx_metadata.py b/ops/ci/tests/test_rwx_metadata.py new file mode 100644 index 00000000000..f387fd94115 --- /dev/null +++ b/ops/ci/tests/test_rwx_metadata.py @@ -0,0 +1,222 @@ +"""Exercise RWX routing through CircleCI's routing scripts with real Git histories.""" + +import json +import os +from pathlib import Path +import shutil +import subprocess +import tempfile +import unittest + + +ROOT = Path(__file__).resolve().parents[3] +# The adapter reads CircleCI's routing data and policy without modifying them. +ROUTING_FILES = ( + "ops/ci/runtime/rwx-metadata.sh", + ".circleci/routing.yml", + ".circleci/scripts/collect-params.sh", + ".circleci/scripts/compute-workflow-conditions.sh", + ".circleci/scripts/workflow-helpers.sh", +) + + +CONSUMED = ("c-run_main", "c-run_rust_ci", "c-run_rust_e2e_ci", "c-run_contracts_feature_tests") + + +class RwxMetadataTests(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + self.root = Path(self.temp.name) + self.upstream = self.root / "upstream" + self.checkout = self.root / "checkout" + self.values = self.root / "values" + self.upstream.mkdir() + self.git("init", "--initial-branch=develop", cwd=self.upstream) + self.configure_git(self.upstream) + for name in ROUTING_FILES: + destination = self.upstream / name + destination.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(ROOT / name, destination) + self.write("docs/public-docs/example.md", "original\n", self.upstream) + self.git("add", ".", cwd=self.upstream) + self.git("commit", "-m", "initial", cwd=self.upstream) + self.git("clone", str(self.upstream), str(self.checkout), cwd=self.root) + self.configure_git(self.checkout) + self.git("switch", "-c", "rwx/example") + + def git(self, *args, cwd=None): + return subprocess.run( + ["git", *args], + cwd=cwd or self.checkout, + check=True, + text=True, + capture_output=True, + ).stdout.strip() + + def configure_git(self, directory): + self.git("config", "user.name", "CI fixture", cwd=directory) + self.git("config", "user.email", "ci-fixture@example.invalid", cwd=directory) + + def write(self, name, contents="changed\n", directory=None): + path = (directory or self.checkout) / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(contents, encoding="utf-8") + + def commit(self, *names): + self.git("add", "--", *names) + self.git("commit", "-m", "fixture change") + + def run_metadata(self, **overrides): + env = dict(os.environ) + for name in ("BRANCH", "TAG", "TRIGGER_SOURCE", "SCHEDULE_NAME", "BASE_REVISION", "OUTPUT"): + env.pop(name, None) + env.update( + CI_BRANCH="rwx/example", + CI_COMMIT_SHA=self.git("rev-parse", "HEAD"), + CI_BASE_REVISION="develop", + RWX_VALUES=str(self.values), + ) + env.update(overrides) + return subprocess.run( + ["bash", "ops/ci/runtime/rwx-metadata.sh"], + cwd=self.checkout, + env=env, + text=True, + capture_output=True, + ) + + def assert_routes(self, main, rust, **overrides): + result = self.run_metadata(**overrides) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertEqual((self.values / "run-main").read_text().strip(), str(main).lower()) + self.assertEqual((self.values / "run-rust-ci").read_text().strip(), str(rust).lower()) + params = json.loads((self.checkout / ".ci/pipeline-parameters.json").read_text()) + # Route tasks and gate aggregates read these as booleans, never as absent keys. + for name in CONSUMED: + self.assertIs(type(params[name]), bool, name) + self.assertEqual((self.values / "run-rust-e2e-ci").read_text().strip(), str(params['c-run_rust_e2e_ci']).lower()) + self.assertEqual((self.values / "run-contracts").read_text().strip(), str(params['c-run_contracts_feature_tests']).lower()) + return params + + def test_docs_only(self): + self.write("docs/public-docs/example.md") + self.commit("docs/public-docs/example.md") + params = self.assert_routes(False, False) + self.assertTrue(params["c-run_ci_gate_skip"]) + + def test_cache_warming_has_a_successful_route_but_selects_no_verdicts(self): + self.write("op-node/rollup/example.go") + self.commit("op-node/rollup/example.go") + params = self.assert_routes(False, False, CI_CACHE_WARM="true", CI_BRANCH="develop") + self.assertTrue(params["cache-rebuild"]) + self.assertFalse(params["c-run_contracts_feature_tests"]) + self.assertFalse(params["c-run_rust_e2e_ci"]) + for branch in ("rwx/example", "external-fork/untrusted"): + with self.subTest(branch=branch): + self.assertNotEqual(self.run_metadata(CI_CACHE_WARM="true", CI_BRANCH=branch).returncode, 0) + + def test_cache_warming_still_rejects_a_different_checked_out_sha(self): + result = self.run_metadata(CI_CACHE_WARM="true", CI_BRANCH="develop", CI_COMMIT_SHA="a" * 40) + self.assertNotEqual(result.returncode, 0) + self.assertFalse((self.values / "run-main").exists()) + + def test_docs_and_unclassified_code_run_main(self): + self.write("docs/public-docs/example.md") + self.write("new-component/main.go") + self.commit("docs/public-docs/example.md", "new-component/main.go") + self.assert_routes(True, False) + + def test_code_routes_like_circleci(self): + self.write("op-node/example.go") + self.commit("op-node/example.go") + params = self.assert_routes(True, False) + self.assertIs(params["c-run_contracts_feature_tests"], False) + self.assertTrue(params["c-run_contracts_feature_tests_short"]) + self.write("rust/kona/example.rs") + self.commit("rust/kona/example.rs") + self.assert_routes(True, True) + + def test_rwx_and_ops_ci_changes_run_all_suites(self): + for name in (".rwx/pr-gates.yml", "ops/ci/runtime/example.py"): + with self.subTest(name=name): + self.write(name) + self.commit(name) + params = self.assert_routes(True, True) + self.assertTrue(params["c-run_contracts_feature_tests"]) + self.assertTrue(params["c-run_rust_e2e_ci"]) + + def test_cli_patch_runs_all_suites(self): + self.write("docs/public-docs/example.md") + self.commit("docs/public-docs/example.md") + # An uncommitted docs edit is still unseen by the committed diff, so it + # selects every workload rather than the docs-only shortcut. + self.write("docs/public-docs/example.md", "uncommitted\n") + self.assert_routes(True, True) + self.git("checkout", "--", "docs/public-docs/example.md") + self.write("new-component/untracked.go") + self.assert_routes(True, True) + + def test_generated_ci_directory_is_not_a_cli_patch(self): + self.write("docs/public-docs/example.md") + self.commit("docs/public-docs/example.md") + self.write(".ci/previous/report.json", "{}\n") + self.assert_routes(False, False) + + def test_develop_empty_diff_runs_full_postmerge_set(self): + self.git("switch", "develop") + params = self.assert_routes(True, True, CI_BRANCH="develop") + self.assertTrue(params["c-run_kona_publish_prestates"]) + self.assertTrue(params["c-run_contracts_feature_tests"]) + + def test_develop_fails_when_the_policy_stops_enabling_a_shadowed_workflow(self): + # After a rename in CircleCI's policy the adapter's default reports "not selected", and + # every receipt passes with all of its tasks skipped. develop enables every workflow. + policy = ".circleci/scripts/compute-workflow-conditions.sh" + text = (self.checkout / policy).read_text() + self.assertIn("run rust_ci\n", text) + self.write(policy, text.replace("run rust_ci\n", "run rust_ci_renamed\n")) + self.commit(policy) + # A PR branch may leave a workflow out, so the renamed flag passes there. + self.assert_routes(True, False) + self.git("switch", "develop") + self.git("merge", "--ff-only", "rwx/example") + values = self.root / "develop-values" + result = self.run_metadata(CI_BRANCH="develop", RWX_VALUES=str(values)) + self.assertNotEqual(result.returncode, 0, result.stdout) + self.assertIn("did not enable: c-run_rust_ci.", result.stderr) + # The task fails before it reports a route, so no workload reads "not selected". + self.assertEqual(list(values.iterdir()), []) + + def test_a_push_without_a_branch_is_rejected(self): + result = self.run_metadata(CI_BRANCH="") + self.assertNotEqual(result.returncode, 0) + self.assertFalse(self.values.exists()) + + def test_wrong_sha_fails_before_reporting_values(self): + result = self.run_metadata(CI_COMMIT_SHA="0" * 40) + self.assertNotEqual(result.returncode, 0) + self.assertIn("differs from CI_COMMIT_SHA", result.stderr) + self.assertFalse(self.values.exists()) + + def test_missing_base_fails_before_reporting_values(self): + result = self.run_metadata(CI_BASE_REVISION="missing") + self.assertNotEqual(result.returncode, 0) + self.assertFalse(self.values.exists()) + + def test_shallow_checkout_fetches_merge_base(self): + shutil.rmtree(self.checkout) + self.git( + "clone", "--depth=1", self.upstream.as_uri(), str(self.checkout), cwd=self.root + ) + self.configure_git(self.checkout) + self.git("switch", "-c", "rwx/example") + self.write("docs/public-docs/example.md") + self.commit("docs/public-docs/example.md") + self.git("push", "origin", "HEAD:refs/heads/rwx/example") + self.assert_routes(False, False) + self.assertEqual(self.git("rev-parse", "--is-shallow-repository"), "false") + + +if __name__ == "__main__": + unittest.main() diff --git a/ops/ci/tests/test_sp1_guest.py b/ops/ci/tests/test_sp1_guest.py new file mode 100644 index 00000000000..65f40efc5cc --- /dev/null +++ b/ops/ci/tests/test_sp1_guest.py @@ -0,0 +1,126 @@ +from contextlib import redirect_stderr +import importlib.util +import io +import json +import os +from pathlib import Path +import re +import shutil +import subprocess +import tempfile +import unittest +from unittest.mock import patch + +SPEC = importlib.util.spec_from_file_location('sp1_guest', Path(__file__).resolve().parents[1] / 'runtime' / 'sp1-guest.py') +G = importlib.util.module_from_spec(SPEC); SPEC.loader.exec_module(G) + + +class GuestTests(unittest.TestCase): + def test_occupied_canonical_workspace_fails_without_reading_or_changing_its_files(self): + native=G.helper('sp1-guest-native-build') + with tempfile.TemporaryDirectory() as temp: + base=Path(temp);root=base/'source';root.mkdir() + occupied=base/'occupied';occupied.mkdir();sentinel=occupied/'original.txt';sentinel.write_bytes(b'original occupied data\n') + with patch.object(native,'ROOT',root),patch.object(native,'WORKSPACE',occupied),patch.object(native,'CARGO',base/'cargo'): + self.assertEqual(native.execute(),1) + self.assertEqual(sentinel.read_bytes(),b'original occupied data\n') + self.assertFalse((root/'.ci/sp1-guest/dependency/files').exists()) + + def test_uncommitted_change_fails_before_the_clone_that_would_not_contain_it(self): + native=G.helper('sp1-guest-native-build') + with tempfile.TemporaryDirectory() as temp: + # Resolved, so that the absent canonical paths pass the first precondition. + base=Path(temp).resolve();root=base/'source';root.mkdir();tracked=root/'tracked.txt';tracked.write_text('committed\n') + for argv in (['git','init','-q'],['git','add','.'], + ['git','-c','user.name=CI fixture','-c','user.email=ci-fixture@example.invalid','commit','-qm','fixture']): + subprocess.run(argv,cwd=root,check=True,stdout=subprocess.DEVNULL) + tracked.write_text('uncommitted\n') + prepare=AssertionError('The build must not prepare the clone') + with patch.object(native,'ROOT',root),patch.object(native,'WORKSPACE',base/'workspace'),patch.object(native,'CARGO',base/'cargo'),patch.object(native.subprocess,'run',side_effect=prepare),redirect_stderr(io.StringIO()) as stderr: + self.assertEqual(native.execute(),1) + self.assertIn('uncommitted',stderr.getvalue()) + self.assertFalse((base/'workspace').exists()) + + def test_guest_feature_markers_and_source_suffix_bind_actual_elf_bytes(self): + with tempfile.TemporaryDirectory() as temp: + path = Path(temp); sha = 'a'*40; programs = list(G.PROGRAMS) + def write(features, source, extra=b''): + (path/'vkeys.toml').write_text('git_sha="'+source+'"\nsp1_toolchain_tag="v6.0.0"\n'+ + ''.join(n+'="0x'+'b'*64+'"\n' for n in programs)) + for n in programs: + data = ('KONA_SP1_BUILD{git_sha='+source+'}').encode() + extra + if features and n=='super-range': data += G.TEST_CONFIG_MARKER + (path/(n+'-elf')).write_bytes(data) + write(True, sha+'-test') + G.check_elfs(path, sha, 'test-config-fallback') + write(False, sha+'-test') + with self.assertRaisesRegex(ValueError, 'test-config'): G.check_elfs(path, sha, 'test-config-fallback') + write(True, sha) + with self.assertRaisesRegex(ValueError, 'source'): G.check_elfs(path, sha, 'test-config-fallback') + with self.assertRaisesRegex(ValueError, 'test-config'): G.check_elfs(path, sha, '') + write(False, sha) + G.check_elfs(path, sha, '') + # A second, different marker fails: CircleCI compares every distinct marker. + write(False, sha, extra=b'KONA_SP1_BUILD{git_sha=unknown}') + with self.assertRaisesRegex(ValueError, 'source'): G.check_elfs(path, sha, '') + # CircleCI greps the manifest for the names its consumers read. + (path/'log-commitment-elf').unlink(); programs.remove('log-commitment') + write(False, sha) + with self.assertRaisesRegex(ValueError, 'consumers'): G.check_elfs(path, sha, '') + + def test_checks_receive_the_built_elfs_and_run_the_circleci_steps_in_order(self): + with tempfile.TemporaryDirectory() as temp: + base = Path(temp); files = base/'artifact/files'; files.mkdir(parents=True); recipes = [] + for name in ('vkeys.toml', 'super-range-elf'): (files/name).write_text(name) + def run(*argv, cwd=G.RUST): + recipes.append((argv, cwd, os.environ['RUSTFLAGS'], sorted(p.name for p in (base/'sp1/elf').iterdir()))) + with patch.object(G,'SP1',base/'sp1'), patch.object(G,'cache_start') as start, patch.object(G,'cache_finish',side_effect=lambda status: status), patch.object(G,'run',side_effect=run), patch.dict(os.environ): + (base/'sp1').mkdir() + self.assertEqual(G.checks(base/'artifact', 'test-config-fallback'), 0) + start.assert_called_once_with('checks', 'test-config-fallback') + self.assertEqual(recipes, [(('just', recipe), G.RUST, '-Dwarnings', ['super-range-elf', 'vkeys.toml']) + for recipe in ('test-sp1-guest', 'lint-sp1-guest', 'check-vkey-crates')]) + + def test_failing_recipe_keeps_its_exit_status_and_the_cache_still_finishes(self): + for returncode, expected in ((101, 101), (-15, 143)): + def steps(): raise subprocess.CalledProcessError(returncode, ['just', 'test-sp1-guest']) + with self.subTest(returncode=returncode), patch.object(G,'cache_start'), patch.object(G,'cache_finish',side_effect=lambda status: status) as finish: + self.assertEqual(G.cached('checks', '', steps), expected) + finish.assert_called_once_with(expected) + + def test_pin_checks_reject_manifest_and_lock_drift_before_installation(self): + tools = [shutil.which(name) for name in ('yq', 'jq', 'just')] + if not all(tools): self.skipTest('The pin checks need yq, jq and just') + script = 'ops/ci/runtime/sp1-guest-toolchain.sh' + # Only the files that the toolchain task transfers: its filter must suffice for the pin checks. + inputs = json.loads(subprocess.check_output( + ['yq', '-o=json', '.tasks[] | select(.key == "toolchain") | .filter', str(G.ROOT/'.rwx/sp1-guest.yml')], text=True)) + self.assertIn(script, inputs) + with tempfile.TemporaryDirectory() as temp: + root = Path(temp)/'source'; home = Path(temp)/'home'; home.mkdir() + for name in inputs: + target = root/name; target.parent.mkdir(parents=True, exist_ok=True); shutil.copyfile(G.ROOT/name, target) + # Without cargo and rustup, a run that passes the pin checks cannot install or relink a toolchain. + path = os.pathsep.join([*dict.fromkeys(str(Path(tool).parent) for tool in tools), '/usr/bin', '/bin']) + if shutil.which('cargo', path=path) or shutil.which('rustup', path=path): + self.skipTest('yq, jq or just share a directory with cargo or rustup') + def run(): + return subprocess.run(['bash', script], cwd=root, env={'PATH': path, 'HOME': str(home)}, capture_output=True, text=True) + self.assertNotIn('SP1 version drift', run().stderr) + manifest = root/'rust/Cargo.toml'; original = manifest.read_text() + changed, count = re.subn('(sp1-sdk\\s*=\\s*\\{[^}\\n]*version\\s*=\\s*")[^"]+', '\\g<1>0.0.0', original) + self.assertEqual(count, 1) + manifest.write_text(changed) + result = run() + self.assertEqual(result.returncode, 1) + self.assertIn('ERROR: SP1 version drift', result.stderr) + self.assertIn('rust/Cargo.toml sp1-sdk: 0.0.0', result.stderr) + manifest.write_text(original) + (root/'rust/kona/sp1/programs/Cargo.lock').write_text('invalid original TOML = [\n') + result = run() + self.assertEqual(result.returncode, 1) + self.assertIn('ERROR: SP1 version drift', result.stderr) + self.assertIn('resolved sp1-lib: ', result.stderr) + + +if __name__ == '__main__': unittest.main()