diff --git a/.circleci/continue/main.yml b/.circleci/continue/main.yml index 9f126babfad..27068f658a6 100644 --- a/.circleci/continue/main.yml +++ b/.circleci/continue/main.yml @@ -1700,6 +1700,81 @@ jobs: paths: - "cannon/bin" + # develop's latest timings for the memory-all test splits (see shard-tests.sh), from + # CircleCI's API. Fetched once, in a job both memory-all jobs wait for, so every node + # gets the same file. Not on develop, tags, the merge queue or proposal branches, which + # keep CircleCI's own timings. Any failure writes no file, and the split works as today. + memory-all-shard-timings: + docker: + - image: <> + resource_class: small.gen2 + steps: + - run: + name: Create the timings dir + command: mkdir -p /tmp/_shard-timings && touch /tmp/_shard-timings/.keep + - unless: + condition: + or: + - equal: ["develop", << pipeline.git.branch >>] + - equal: ["", << pipeline.git.branch >>] + - matches: + pattern: "^gh-readonly-queue/.*$" + value: << pipeline.git.branch >> + - matches: + pattern: "^proposal/.*$" + value: << pipeline.git.branch >> + steps: + - run: + name: Fetch develop's timings + # The API needs no token while the project is public. If it goes private, set + # CIRCLE_TIMINGS_TOKEN (a personal API token, v2 rejects project tokens) as a project env var. + environment: + # The fork must follow acceptance_l1_forks. A job with no file splits as today. + JOBS: memory-all-opn-op-reth-fusaka memory-all-kona-op-reth-fusaka + command: | + # Any failure logs one line and passes, leaving no file for the jobs not yet done. + trap 'echo "Could not fetch develop timings, memory-all splits as today"; exit 0' ERR + API=https://circleci.com/api + SLUG=gh/ethereum-optimism/optimism + CURL=(curl -fsS --max-time 10 --retry 2 --retry-delay 1 --retry-max-time 10) + if [ -n "${CIRCLE_TIMINGS_TOKEN:-}" ]; then CURL+=(-H "Circle-Token: $CIRCLE_TIMINGS_TOKEN"); fi + cd "$(mktemp -d)" + # Green develop jobs, newest first. A develop pipeline adds about 100. + for offset in 0 100 200; do + [ "$SECONDS" -lt 60 ] + "${CURL[@]}" -o jobs.json "$API/v1.1/project/$SLUG/tree/develop?filter=successful&shallow=true&limit=100&offset=$offset" + jq -r '.[] | "\(.workflows.job_name) \(.build_num)"' jobs.json >> jobs.txt + if awk -v want="$JOBS" 'BEGIN { n = split(want, w, " ") } { seen[$1] = 1 } + END { for (i = 1; i <= n; i++) if (!seen[w[i]]) exit 1 }' jobs.txt; then break; fi + done + for job in $JOBS; do + num=$(awk -v j="$job" '$1 == j && $2 > n { n = $2 } END { print n + 0 }' jobs.txt) + if [ "$num" -eq 0 ]; then echo "No green develop run of $job found"; continue; fi + : > tests.jsonl + token= + while :; do + [ "$SECONDS" -lt 60 ] + "${CURL[@]}" -o page.json "$API/v2/project/$SLUG/$num/tests${token:+?page-token=$token}" + jq -c '.items[]' page.json >> tests.jsonl + token=$(jq -r '.next_page_token // empty | @uri' page.json) + [ -n "$token" ] || break + done + # The split gets top-level names from `go test -list`. A parent's run_time includes + # its subtests, so those are dropped. A name in several packages runs on one node, + # so its copies are summed. + jq -s '{tests: [map(select(.name | contains("/") | not)) | group_by(.name)[] + | {name: .[0].name, classname: .[0].classname, result: "success", + run_time: (map(.run_time // 0) | add)}]}' tests.jsonl > timings.json + n=$(jq '.tests | length' timings.json) + if [ "$n" -eq 0 ]; then echo "No tests in develop job $num"; continue; fi + mv timings.json "/tmp/_shard-timings/$job.json" + echo "Wrote $n test timings from develop job $num to $job.json" + done + - persist_to_workspace: + root: /tmp + paths: + - _shard-timings + check-op-geth-version: docker: - image: <> @@ -2789,6 +2864,7 @@ workflows: - prep-go-modules context: - circleci-repo-readonly-authenticated-github-token + - memory-all-shard-timings # Both run on the job's default 2xlarge.gen2 (16 vCPU / 32 GiB). The tests mostly wait on # blocks, so 4 nodes running 16 at once take as long as 8 nodes running 8. At 16, anonymous # memory peaked at about 11.4 GiB of 32 per node (Nov 2026 bench). memory.peak reads 32 GiB @@ -2800,6 +2876,7 @@ workflows: image: <> matrix: parameters: + # A new fork also needs its name in memory-all-shard-timings. l1_fork: &acceptance_l1_forks - fusaka parallelism: 4 @@ -2816,6 +2893,8 @@ workflows: - rust-op-reth-binary - rust-sp1-super-range-executor - go-binaries-for-sysgo + # terminal: with no timings file the split runs as before. + - memory-all-shard-timings: terminal - prep-go-modules # for the gitignored superchain-configs.zip op-acceptance-tests //go:embeds - prep-superchain diff --git a/ops/scripts/shard-tests.sh b/ops/scripts/shard-tests.sh index 5d2f652e295..e8d385d2032 100755 --- a/ops/scripts/shard-tests.sh +++ b/ops/scripts/shard-tests.sh @@ -34,9 +34,19 @@ if [ -z "${CIRCLE_NODE_TOTAL:-}" ] || [ "${CIRCLE_NODE_TOTAL:-1}" -le 1 ]; then exit 0 fi +# CircleCI's own timings come from this branch's last green run, which can predate +# long tests added on develop since. Use develop's when the workflow provides them +# (the same file on every node, so each test still lands on exactly one node). +TIMINGS_FILE="$(dirname "$0")/../../_shard-timings/${CIRCLE_JOB:-}.json" +set -- +if [ -s "$TIMINGS_FILE" ]; then + echo "Splitting by develop's timings from $TIMINGS_FILE" >&2 + set -- --timings-file "$TIMINGS_FILE" +fi + # Canceled runs can leave partial history. CircleCI's tiny default weight for # missing timings can put every unknown test on the same otherwise-light shard. # Give unknown acceptance tests a realistic setup/runtime budget instead. # `awk 1` ends the last line with a newline, since callers' `read` and `wc -l` would # miss a test without one. -circleci tests split --split-by=timings --timings-type=testname --time-default=1m "$ALL_TESTS_FILE" | awk 1 +circleci tests split --split-by=timings --timings-type=testname --time-default=1m "$@" "$ALL_TESTS_FILE" | awk 1