From ab1bf6fe66c6ae9581de9f867a2f80334508d993 Mon Sep 17 00:00:00 2001 From: Matt Solomon Date: Mon, 16 Nov 2026 14:40:59 -0800 Subject: [PATCH 1/9] ci: split memory-all tests on PRs by develop's timings CircleCI splits tests by the timings of the branch's own last green run of the job. After a rebase that run can predate long tests added on develop, so TestExecuteStep_CannonKona (about 400 s) weighs nothing and lands on a full node, which then runs 10 to 12 minutes instead of about 7.5. Develop's memory-all jobs now save the timings CircleCI gave their split to a cache. On PR branches, go-binaries-for-sysgo, which both memory-all jobs already wait for, restores the latest copy once and persists it, so every node splits from the same file. shard-tests.sh passes it with --timings-file when present. No file means the split works as before. Develop, tags and the merge queue never restore it, so trusted runs read nothing a branch could have written. Co-Authored-By: Claude Opus 5.5 --- .circleci/continue/main.yml | 39 +++++++++++++++++++++++++++++++++++++ ops/scripts/shard-tests.sh | 11 ++++++++++- 2 files changed, 49 insertions(+), 1 deletion(-) diff --git a/.circleci/continue/main.yml b/.circleci/continue/main.yml index 9f126babfad..cdc92605c0a 100644 --- a/.circleci/continue/main.yml +++ b/.circleci/continue/main.yml @@ -1699,6 +1699,28 @@ jobs: root: . paths: - "cannon/bin" + # develop's latest timings for the memory-all test splits (see shard-tests.sh). + # Restored once here, in a job they wait for, so every node gets the same file. + # Not on develop, tags or the merge queue: a branch that edits its CI config can + # write any cache, so trusted runs keep splitting by CircleCI's own timings. + - unless: + condition: + or: + - equal: ["develop", << pipeline.git.branch >>] + - equal: ["", << pipeline.git.branch >>] + - matches: + pattern: "^gh-readonly-queue/.*$" + value: << pipeline.git.branch >> + steps: + - restore_cache: + key: shard-timings-v1-memory-all-opn-op-reth-fusaka- + - restore_cache: + key: shard-timings-v1-memory-all-kona-op-reth-fusaka- + - run: mkdir -p /tmp/shard-timings + - persist_to_workspace: + root: /tmp + paths: + - shard-timings check-op-geth-version: docker: @@ -2294,6 +2316,23 @@ jobs: steps: - notify-failures-on-develop: mentions: "@protocol-oncall" + # On develop, save the timings CircleCI gave this split, for other branches' + # splits. With none it halts instead, since an empty cache would hide the + # last good one, so this must stay the last step. + - when: + condition: + equal: [develop, << pipeline.git.branch >>] + steps: + - run: + name: Copy shard timings + command: | + SRC="$CIRCLE_INTERNAL_TASK_DATA/circle-test-results/results.json" + mkdir -p /tmp/shard-timings + { [ -s "$SRC" ] && cp "$SRC" "/tmp/shard-timings/$CIRCLE_JOB.json"; } || circleci-agent step halt + - save_cache: + key: shard-timings-v1-{{ .Environment.CIRCLE_JOB }}-{{ .Revision }} + paths: + - /tmp/shard-timings # Aggregator job - allows downstream jobs and required checks to depend on a single memory-all job. memory-all: diff --git a/ops/scripts/shard-tests.sh b/ops/scripts/shard-tests.sh index 5d2f652e295..0204ee1dd06 100755 --- a/ops/scripts/shard-tests.sh +++ b/ops/scripts/shard-tests.sh @@ -37,6 +37,15 @@ fi # Canceled runs can leave partial history. CircleCI's tiny default weight for # missing timings can put every unknown test on the same otherwise-light shard. # Give unknown acceptance tests a realistic setup/runtime budget instead. +# CircleCI's own timings come from this branch's last green run, which can predate +# long tests added on develop since. Use develop's when the workflow provides them +# (the same file on every node, so each test still lands on exactly one node). +TIMINGS_FILE="$(dirname "$0")/../../shard-timings/${CIRCLE_JOB:-}.json" +set -- +if [ -s "$TIMINGS_FILE" ]; then + echo "Splitting by develop's timings from $TIMINGS_FILE" >&2 + set -- --timings-file "$TIMINGS_FILE" +fi # `awk 1` ends the last line with a newline, since callers' `read` and `wc -l` would # miss a test without one. -circleci tests split --split-by=timings --timings-type=testname --time-default=1m "$ALL_TESTS_FILE" | awk 1 +circleci tests split --split-by=timings --timings-type=testname --time-default=1m "$@" "$ALL_TESTS_FILE" | awk 1 From e60c8650c334ae97bc7f7d4a1f66149ca8013e42 Mon Sep 17 00:00:00 2001 From: Matt Solomon Date: Mon, 16 Nov 2026 15:14:47 -0800 Subject: [PATCH 2/9] ops: keep the time-default comment next to its split Co-Authored-By: Claude Opus 5.5 --- ops/scripts/shard-tests.sh | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/ops/scripts/shard-tests.sh b/ops/scripts/shard-tests.sh index 0204ee1dd06..31b0a920038 100755 --- a/ops/scripts/shard-tests.sh +++ b/ops/scripts/shard-tests.sh @@ -34,9 +34,6 @@ if [ -z "${CIRCLE_NODE_TOTAL:-}" ] || [ "${CIRCLE_NODE_TOTAL:-1}" -le 1 ]; then exit 0 fi -# Canceled runs can leave partial history. CircleCI's tiny default weight for -# missing timings can put every unknown test on the same otherwise-light shard. -# Give unknown acceptance tests a realistic setup/runtime budget instead. # CircleCI's own timings come from this branch's last green run, which can predate # long tests added on develop since. Use develop's when the workflow provides them # (the same file on every node, so each test still lands on exactly one node). @@ -46,6 +43,10 @@ if [ -s "$TIMINGS_FILE" ]; then echo "Splitting by develop's timings from $TIMINGS_FILE" >&2 set -- --timings-file "$TIMINGS_FILE" fi + +# Canceled runs can leave partial history. CircleCI's tiny default weight for +# missing timings can put every unknown test on the same otherwise-light shard. +# Give unknown acceptance tests a realistic setup/runtime budget instead. # `awk 1` ends the last line with a newline, since callers' `read` and `wc -l` would # miss a test without one. circleci tests split --split-by=timings --timings-type=testname --time-default=1m "$@" "$ALL_TESTS_FILE" | awk 1 From d35285a0741703d80dec1fef1cc1542039b65ad7 Mon Sep 17 00:00:00 2001 From: Matt Solomon Date: Mon, 16 Nov 2026 15:27:28 -0800 Subject: [PATCH 3/9] ci: keep restored shard timings out of go-tests The restored timings dir is persisted to the workspace, which go-tests attaches at the repo root and tests with `go list ./...`. A cache any branch can write could put a _test.go there. Name the dir _shard-timings, which Go skips, and keep only plain .json files after the restore. Bump the key to v2 for the new layout. Also skip the restore on proposal branches, which run post-merge like develop, and note that the restore keys follow acceptance_l1_forks. Co-Authored-By: Claude Opus 5.5 --- .circleci/continue/main.yml | 33 +++++++++++++++++++++++---------- ops/scripts/shard-tests.sh | 2 +- 2 files changed, 24 insertions(+), 11 deletions(-) diff --git a/.circleci/continue/main.yml b/.circleci/continue/main.yml index cdc92605c0a..a4b4e96578e 100644 --- a/.circleci/continue/main.yml +++ b/.circleci/continue/main.yml @@ -1701,8 +1701,8 @@ jobs: - "cannon/bin" # develop's latest timings for the memory-all test splits (see shard-tests.sh). # Restored once here, in a job they wait for, so every node gets the same file. - # Not on develop, tags or the merge queue: a branch that edits its CI config can - # write any cache, so trusted runs keep splitting by CircleCI's own timings. + # Not on develop, tags, the merge queue or proposal branches: a branch that edits + # its CI config can write any cache, so trusted runs keep CircleCI's own timings. - unless: condition: or: @@ -1711,16 +1711,28 @@ jobs: - matches: pattern: "^gh-readonly-queue/.*$" value: << pipeline.git.branch >> + - matches: + pattern: "^proposal/.*$" + value: << pipeline.git.branch >> steps: + # The fork must follow acceptance_l1_forks. A job with no file splits as today. - restore_cache: - key: shard-timings-v1-memory-all-opn-op-reth-fusaka- + key: shard-timings-v2-memory-all-opn-op-reth-fusaka- - restore_cache: - key: shard-timings-v1-memory-all-kona-op-reth-fusaka- - - run: mkdir -p /tmp/shard-timings + key: shard-timings-v2-memory-all-kona-op-reth-fusaka- + # The workspace is attached at the repo root, where go-tests runs every + # package. Go skips dirs starting with "_", and only plain .json files pass. + - run: + name: Keep only the timing files + command: | + D=/tmp/_shard-timings + [ -d "$D" ] && [ ! -L "$D" ] || rm -rf "$D" + mkdir -p "$D" + find "$D" -mindepth 1 -maxdepth 1 ! \( -type f -name '*.json' \) -exec rm -rf {} + - persist_to_workspace: root: /tmp paths: - - shard-timings + - _shard-timings check-op-geth-version: docker: @@ -2327,12 +2339,12 @@ jobs: name: Copy shard timings command: | SRC="$CIRCLE_INTERNAL_TASK_DATA/circle-test-results/results.json" - mkdir -p /tmp/shard-timings - { [ -s "$SRC" ] && cp "$SRC" "/tmp/shard-timings/$CIRCLE_JOB.json"; } || circleci-agent step halt + mkdir -p /tmp/_shard-timings + { [ -s "$SRC" ] && cp "$SRC" "/tmp/_shard-timings/$CIRCLE_JOB.json"; } || circleci-agent step halt - save_cache: - key: shard-timings-v1-{{ .Environment.CIRCLE_JOB }}-{{ .Revision }} + key: shard-timings-v2-{{ .Environment.CIRCLE_JOB }}-{{ .Revision }} paths: - - /tmp/shard-timings + - /tmp/_shard-timings # Aggregator job - allows downstream jobs and required checks to depend on a single memory-all job. memory-all: @@ -2839,6 +2851,7 @@ workflows: image: <> matrix: parameters: + # A new fork also needs its shard-timings restore in go-binaries-for-sysgo. l1_fork: &acceptance_l1_forks - fusaka parallelism: 4 diff --git a/ops/scripts/shard-tests.sh b/ops/scripts/shard-tests.sh index 31b0a920038..e8d385d2032 100755 --- a/ops/scripts/shard-tests.sh +++ b/ops/scripts/shard-tests.sh @@ -37,7 +37,7 @@ fi # CircleCI's own timings come from this branch's last green run, which can predate # long tests added on develop since. Use develop's when the workflow provides them # (the same file on every node, so each test still lands on exactly one node). -TIMINGS_FILE="$(dirname "$0")/../../shard-timings/${CIRCLE_JOB:-}.json" +TIMINGS_FILE="$(dirname "$0")/../../_shard-timings/${CIRCLE_JOB:-}.json" set -- if [ -s "$TIMINGS_FILE" ]; then echo "Splitting by develop's timings from $TIMINGS_FILE" >&2 From 36df6ffd1d68c9ed213c10a6d40b2e188e7fb152 Mon Sep 17 00:00:00 2001 From: Matt Solomon Date: Mon, 16 Nov 2026 15:32:32 -0800 Subject: [PATCH 4/9] ci: call the timings cleanup tools by absolute path A restored cache can write ~/bin, which comes first in PATH on cimg/base, so the cleanup after the restore names root-owned binaries directly. Co-Authored-By: Claude Opus 5.5 --- .circleci/continue/main.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.circleci/continue/main.yml b/.circleci/continue/main.yml index a4b4e96578e..db1c8216025 100644 --- a/.circleci/continue/main.yml +++ b/.circleci/continue/main.yml @@ -1722,13 +1722,14 @@ jobs: key: shard-timings-v2-memory-all-kona-op-reth-fusaka- # The workspace is attached at the repo root, where go-tests runs every # package. Go skips dirs starting with "_", and only plain .json files pass. + # Absolute paths, since a cache can write to ~/bin, which is first in PATH. - run: name: Keep only the timing files command: | D=/tmp/_shard-timings - [ -d "$D" ] && [ ! -L "$D" ] || rm -rf "$D" - mkdir -p "$D" - find "$D" -mindepth 1 -maxdepth 1 ! \( -type f -name '*.json' \) -exec rm -rf {} + + [ -d "$D" ] && [ ! -L "$D" ] || /bin/rm -rf "$D" + /bin/mkdir -p "$D" + /usr/bin/find "$D" -mindepth 1 -maxdepth 1 ! \( -type f -name '*.json' \) -exec /bin/rm -rf {} + - persist_to_workspace: root: /tmp paths: From 6b621902302b6f781ab53a8e99093ab6b3e47261 Mon Sep 17 00:00:00 2001 From: Matt Solomon Date: Mon, 16 Nov 2026 15:37:32 -0800 Subject: [PATCH 5/9] ci: run the timings cleanup under sh The orb's BASH_ENV runs ~/.local/bin/mise at the start of every bash step, and a restored cache can write that file. sh never reads BASH_ENV. Co-Authored-By: Claude Opus 5.5 --- .circleci/continue/main.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.circleci/continue/main.yml b/.circleci/continue/main.yml index db1c8216025..915e77b3e3f 100644 --- a/.circleci/continue/main.yml +++ b/.circleci/continue/main.yml @@ -1722,9 +1722,11 @@ jobs: key: shard-timings-v2-memory-all-kona-op-reth-fusaka- # The workspace is attached at the repo root, where go-tests runs every # package. Go skips dirs starting with "_", and only plain .json files pass. - # Absolute paths, since a cache can write to ~/bin, which is first in PATH. + # A cache can write ~/bin (first in PATH) and the mise that BASH_ENV runs, + # so use sh, which skips BASH_ENV, and call tools by absolute path. - run: name: Keep only the timing files + shell: /bin/sh -eu command: | D=/tmp/_shard-timings [ -d "$D" ] && [ ! -L "$D" ] || /bin/rm -rf "$D" From de9248567f33ab789b2df8575b9b542ced924875 Mon Sep 17 00:00:00 2001 From: Matt Solomon Date: Mon, 16 Nov 2026 16:08:33 -0800 Subject: [PATCH 6/9] ci: restore shard timings in their own job with no context go-binaries-for-sysgo holds the GitHub token context, so restoring a cache any branch can write there needed a hardened cleanup. Move the restore into memory-all-shard-timings, which has no context and runs no command after the restore: the persist glob keeps only the .json files. Project env vars still reach every job, which is why nothing runs after the restore. go-tests no longer gets the directory at all, since it does not require this job. Co-Authored-By: Claude Opus 5.5 --- .circleci/continue/main.yml | 43 +++++++++++++++++++------------------ 1 file changed, 22 insertions(+), 21 deletions(-) diff --git a/.circleci/continue/main.yml b/.circleci/continue/main.yml index 915e77b3e3f..4088c5da95e 100644 --- a/.circleci/continue/main.yml +++ b/.circleci/continue/main.yml @@ -1699,10 +1699,21 @@ jobs: root: . paths: - "cannon/bin" - # develop's latest timings for the memory-all test splits (see shard-tests.sh). - # Restored once here, in a job they wait for, so every node gets the same file. - # Not on develop, tags, the merge queue or proposal branches: a branch that edits - # its CI config can write any cache, so trusted runs keep CircleCI's own timings. + + # develop's latest timings for the memory-all test splits (see shard-tests.sh). + # Restored once, in a job both memory-all jobs wait for, so every node gets the same + # file. Not on develop, tags, the merge queue or proposal branches: a branch that + # edits its CI config can write any cache, so trusted runs keep CircleCI's own timings. + # No context. Project env vars still reach every job, so no command runs after the + # restore: the persist glob keeps only the .json files, in a dir Go skips ("_"). + memory-all-shard-timings: + docker: + - image: <> + resource_class: small.gen2 + steps: + - run: + name: Create the timings dir + command: mkdir -p /tmp/_shard-timings && touch /tmp/_shard-timings/none.json - unless: condition: or: @@ -1720,22 +1731,10 @@ jobs: key: shard-timings-v2-memory-all-opn-op-reth-fusaka- - restore_cache: key: shard-timings-v2-memory-all-kona-op-reth-fusaka- - # The workspace is attached at the repo root, where go-tests runs every - # package. Go skips dirs starting with "_", and only plain .json files pass. - # A cache can write ~/bin (first in PATH) and the mise that BASH_ENV runs, - # so use sh, which skips BASH_ENV, and call tools by absolute path. - - run: - name: Keep only the timing files - shell: /bin/sh -eu - command: | - D=/tmp/_shard-timings - [ -d "$D" ] && [ ! -L "$D" ] || /bin/rm -rf "$D" - /bin/mkdir -p "$D" - /usr/bin/find "$D" -mindepth 1 -maxdepth 1 ! \( -type f -name '*.json' \) -exec /bin/rm -rf {} + - - persist_to_workspace: - root: /tmp - paths: - - _shard-timings + - persist_to_workspace: + root: /tmp + paths: + - _shard-timings/*.json check-op-geth-version: docker: @@ -2843,6 +2842,7 @@ workflows: - prep-go-modules context: - circleci-repo-readonly-authenticated-github-token + - memory-all-shard-timings # Both run on the job's default 2xlarge.gen2 (16 vCPU / 32 GiB). The tests mostly wait on # blocks, so 4 nodes running 16 at once take as long as 8 nodes running 8. At 16, anonymous # memory peaked at about 11.4 GiB of 32 per node (Nov 2026 bench). memory.peak reads 32 GiB @@ -2854,7 +2854,7 @@ workflows: image: <> matrix: parameters: - # A new fork also needs its shard-timings restore in go-binaries-for-sysgo. + # A new fork also needs its restore in memory-all-shard-timings. l1_fork: &acceptance_l1_forks - fusaka parallelism: 4 @@ -2871,6 +2871,7 @@ workflows: - rust-op-reth-binary - rust-sp1-super-range-executor - go-binaries-for-sysgo + - memory-all-shard-timings - prep-go-modules # for the gitignored superchain-configs.zip op-acceptance-tests //go:embeds - prep-superchain From 15e98a43ca1f74c249aac1ac32620f0db566b571 Mon Sep 17 00:00:00 2001 From: Matt Solomon Date: Mon, 16 Nov 2026 16:31:26 -0800 Subject: [PATCH 7/9] ci: keep cache-planted tar and gzip away from the timings persist persist_to_workspace runs tar and gzip from the image PATH, which starts with ~/bin and ~/.local/bin. Make both root-owned before the restore. Co-Authored-By: Claude Opus 5.5 --- .circleci/continue/main.yml | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/.circleci/continue/main.yml b/.circleci/continue/main.yml index 4088c5da95e..9a15212486c 100644 --- a/.circleci/continue/main.yml +++ b/.circleci/continue/main.yml @@ -1704,8 +1704,8 @@ jobs: # Restored once, in a job both memory-all jobs wait for, so every node gets the same # file. Not on develop, tags, the merge queue or proposal branches: a branch that # edits its CI config can write any cache, so trusted runs keep CircleCI's own timings. - # No context. Project env vars still reach every job, so no command runs after the - # restore: the persist glob keeps only the .json files, in a dir Go skips ("_"). + # No context, but project env vars reach every job. No run step follows the restore, + # and the persist glob keeps only the .json files, in a dir Go skips ("_"). memory-all-shard-timings: docker: - image: <> @@ -1713,7 +1713,11 @@ jobs: steps: - run: name: Create the timings dir - command: mkdir -p /tmp/_shard-timings && touch /tmp/_shard-timings/none.json + # persist_to_workspace runs tar and gzip from the image PATH, which starts with + # these two dirs. Root owns them, so a restored cache can't plant either there. + command: | + mkdir -p /tmp/_shard-timings && touch /tmp/_shard-timings/none.json + sudo install -d -o root -g root -m 755 ~/bin ~/.local/bin - unless: condition: or: From c04846ce4eb930f48fa39dcf6e8bd162bdaf769e Mon Sep 17 00:00:00 2001 From: Matt Solomon Date: Mon, 16 Nov 2026 16:55:22 -0800 Subject: [PATCH 8/9] ci: fetch develop's shard timings from CircleCI's API instead of a cache A branch that edits its CI config can write any cache, so restoring one needed hardening. The project's API is public, so fetch the latest green develop run's test results instead and convert them to the split's format. An optional CIRCLE_TIMINGS_TOKEN keeps it working if the repo goes private. Any failure writes no file and the split works as today. Co-Authored-By: Claude Opus 5.5 --- .circleci/continue/main.yml | 88 ++++++++++++++++++++++--------------- 1 file changed, 53 insertions(+), 35 deletions(-) diff --git a/.circleci/continue/main.yml b/.circleci/continue/main.yml index 9a15212486c..cc1636963e2 100644 --- a/.circleci/continue/main.yml +++ b/.circleci/continue/main.yml @@ -1700,12 +1700,10 @@ jobs: paths: - "cannon/bin" - # develop's latest timings for the memory-all test splits (see shard-tests.sh). - # Restored once, in a job both memory-all jobs wait for, so every node gets the same - # file. Not on develop, tags, the merge queue or proposal branches: a branch that - # edits its CI config can write any cache, so trusted runs keep CircleCI's own timings. - # No context, but project env vars reach every job. No run step follows the restore, - # and the persist glob keeps only the .json files, in a dir Go skips ("_"). + # develop's latest timings for the memory-all test splits (see shard-tests.sh), from + # CircleCI's API. Fetched once, in a job both memory-all jobs wait for, so every node + # gets the same file. Not on develop, tags, the merge queue or proposal branches, which + # keep CircleCI's own timings. Any failure writes no file, and the split works as today. memory-all-shard-timings: docker: - image: <> @@ -1713,11 +1711,7 @@ jobs: steps: - run: name: Create the timings dir - # persist_to_workspace runs tar and gzip from the image PATH, which starts with - # these two dirs. Root owns them, so a restored cache can't plant either there. - command: | - mkdir -p /tmp/_shard-timings && touch /tmp/_shard-timings/none.json - sudo install -d -o root -g root -m 755 ~/bin ~/.local/bin + command: mkdir -p /tmp/_shard-timings && touch /tmp/_shard-timings/.keep - unless: condition: or: @@ -1730,15 +1724,56 @@ jobs: pattern: "^proposal/.*$" value: << pipeline.git.branch >> steps: - # The fork must follow acceptance_l1_forks. A job with no file splits as today. - - restore_cache: - key: shard-timings-v2-memory-all-opn-op-reth-fusaka- - - restore_cache: - key: shard-timings-v2-memory-all-kona-op-reth-fusaka- + - run: + name: Fetch develop's timings + # The API needs no token while the project is public. If it goes private, set + # CIRCLE_TIMINGS_TOKEN (a read-only project API token) as a project env var. + environment: + # The fork must follow acceptance_l1_forks. A job with no file splits as today. + JOBS: memory-all-opn-op-reth-fusaka memory-all-kona-op-reth-fusaka + command: | + # Any failure logs one line and passes, leaving no file for the jobs not yet done. + trap 'echo "Could not fetch develop timings, memory-all splits as today"; exit 0' ERR + API=https://circleci.com/api + SLUG=gh/ethereum-optimism/optimism + CURL=(curl -fsS --max-time 10 --retry 2 --retry-delay 1) + if [ -n "${CIRCLE_TIMINGS_TOKEN:-}" ]; then CURL+=(-H "Circle-Token: $CIRCLE_TIMINGS_TOKEN"); fi + cd "$(mktemp -d)" + # Green develop jobs, newest first. A develop pipeline adds about 100. + for offset in 0 100 200; do + [ "$SECONDS" -lt 60 ] + "${CURL[@]}" -o jobs.json "$API/v1.1/project/$SLUG/tree/develop?filter=successful&shallow=true&limit=100&offset=$offset" + jq -r '.[] | "\(.workflows.job_name) \(.build_num)"' jobs.json >> jobs.txt + if awk -v want="$JOBS" 'BEGIN { n = split(want, w, " ") } { seen[$1] = 1 } + END { for (i = 1; i <= n; i++) if (!seen[w[i]]) exit 1 }' jobs.txt; then break; fi + done + for job in $JOBS; do + num=$(awk -v j="$job" '$1 == j && $2 > n { n = $2 } END { print n + 0 }' jobs.txt) + if [ "$num" -eq 0 ]; then echo "No green develop run of $job found"; continue; fi + : > tests.jsonl + token= + while :; do + [ "$SECONDS" -lt 60 ] + "${CURL[@]}" -o page.json "$API/v2/project/$SLUG/$num/tests${token:+?page-token=$token}" + jq -c '.items[]' page.json >> tests.jsonl + token=$(jq -r '.next_page_token // empty | @uri' page.json) + [ -n "$token" ] || break + done + # The split gets top-level names from `go test -list`. A parent's run_time includes + # its subtests, so those are dropped. A name in several packages runs on one node, + # so its copies are summed. + jq -s '{tests: [map(select(.name | contains("/") | not)) | group_by(.name)[] + | {name: .[0].name, classname: .[0].classname, result: "success", + run_time: (map(.run_time // 0) | add)}]}' tests.jsonl > timings.json + n=$(jq '.tests | length' timings.json) + if [ "$n" -eq 0 ]; then echo "No tests in develop job $num"; continue; fi + mv timings.json "/tmp/_shard-timings/$job.json" + echo "Wrote $n test timings from develop job $num to $job.json" + done - persist_to_workspace: root: /tmp paths: - - _shard-timings/*.json + - _shard-timings check-op-geth-version: docker: @@ -2334,23 +2369,6 @@ jobs: steps: - notify-failures-on-develop: mentions: "@protocol-oncall" - # On develop, save the timings CircleCI gave this split, for other branches' - # splits. With none it halts instead, since an empty cache would hide the - # last good one, so this must stay the last step. - - when: - condition: - equal: [develop, << pipeline.git.branch >>] - steps: - - run: - name: Copy shard timings - command: | - SRC="$CIRCLE_INTERNAL_TASK_DATA/circle-test-results/results.json" - mkdir -p /tmp/_shard-timings - { [ -s "$SRC" ] && cp "$SRC" "/tmp/_shard-timings/$CIRCLE_JOB.json"; } || circleci-agent step halt - - save_cache: - key: shard-timings-v2-{{ .Environment.CIRCLE_JOB }}-{{ .Revision }} - paths: - - /tmp/_shard-timings # Aggregator job - allows downstream jobs and required checks to depend on a single memory-all job. memory-all: @@ -2858,7 +2876,7 @@ workflows: image: <> matrix: parameters: - # A new fork also needs its restore in memory-all-shard-timings. + # A new fork also needs its name in memory-all-shard-timings. l1_fork: &acceptance_l1_forks - fusaka parallelism: 4 From c5fdd2ef180146c53a13575945c68eae3979cf4b Mon Sep 17 00:00:00 2001 From: Matt Solomon Date: Mon, 16 Nov 2026 17:08:22 -0800 Subject: [PATCH 9/9] ci: never block memory-all on the timings job, cap curl retry waits Review fixes: memory-all requires the timings job as terminal, so any failure there falls back to today's split. --retry-max-time stops a long Retry-After from stalling the fetch. v2 needs a personal token. Co-Authored-By: Claude Opus 5.5 --- .circleci/continue/main.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.circleci/continue/main.yml b/.circleci/continue/main.yml index cc1636963e2..27068f658a6 100644 --- a/.circleci/continue/main.yml +++ b/.circleci/continue/main.yml @@ -1727,7 +1727,7 @@ jobs: - run: name: Fetch develop's timings # The API needs no token while the project is public. If it goes private, set - # CIRCLE_TIMINGS_TOKEN (a read-only project API token) as a project env var. + # CIRCLE_TIMINGS_TOKEN (a personal API token, v2 rejects project tokens) as a project env var. environment: # The fork must follow acceptance_l1_forks. A job with no file splits as today. JOBS: memory-all-opn-op-reth-fusaka memory-all-kona-op-reth-fusaka @@ -1736,7 +1736,7 @@ jobs: trap 'echo "Could not fetch develop timings, memory-all splits as today"; exit 0' ERR API=https://circleci.com/api SLUG=gh/ethereum-optimism/optimism - CURL=(curl -fsS --max-time 10 --retry 2 --retry-delay 1) + CURL=(curl -fsS --max-time 10 --retry 2 --retry-delay 1 --retry-max-time 10) if [ -n "${CIRCLE_TIMINGS_TOKEN:-}" ]; then CURL+=(-H "Circle-Token: $CIRCLE_TIMINGS_TOKEN"); fi cd "$(mktemp -d)" # Green develop jobs, newest first. A develop pipeline adds about 100. @@ -2893,7 +2893,8 @@ workflows: - rust-op-reth-binary - rust-sp1-super-range-executor - go-binaries-for-sysgo - - memory-all-shard-timings + # terminal: with no timings file the split runs as before. + - memory-all-shard-timings: terminal - prep-go-modules # for the gitignored superchain-configs.zip op-acceptance-tests //go:embeds - prep-superchain