diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 56eebdc..a9de790 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -1,14 +1,18 @@ -# The long differential run, nightly. It does not block merges. +# The long runs, nightly. They do not block merges. # -# This is the only job that compares memory. ci.yml's differential-smoke stops -# well short of the first RAM_HASH_INTERVAL boundary, so `ramhash` and `fbhash` -# are exercised here and nowhere else. A failure here needs a divergence-report -# issue opened by hand; nothing files one automatically. +# deep-diff is the only job that compares memory. ci.yml's differential-smoke +# stops well short of the first RAM_HASH_INTERVAL boundary, so `ramhash` and +# `fbhash` are exercised here and nowhere else. A failure there needs a +# divergence-report issue opened by hand. # -# Throughput is deliberately not benchmarked here. A shared runner cannot give -# a timing a quiet machine gives, and a gate that fails for reasons unrelated to -# the change is worse than no gate. Run `bench-canonical-throughput` on a quiet -# box instead. +# native-regression runs `native diff --record` on each main commit not yet +# recorded and judges each line against the one before it. native-record +# appends the lines to the `regression-data` branch. Cost is judged only +# between a parent and its child measured one after the other in the same +# job, never against a number from another runner. +# +# Throughput is not benchmarked here. A shared runner cannot give the timing a +# quiet machine gives. Run `bench-canonical-throughput` on a quiet box instead. name: nightly @@ -25,6 +29,9 @@ on: # 91.6 minutes, a 3.28x margin inside the 300-minute timeout, and # crosses 4 RAM_HASH_INTERVAL boundaries. default: "5000000" + max_commits: + description: "Main commits native-regression walks after the last recorded one" + default: "8" concurrency: group: nightly-${{ github.ref }} @@ -37,6 +44,10 @@ env: # Local-only. This database holds an emulator's RAM and nothing secret. CLICKHOUSE_PASSWORD: clickdoom CLICKHOUSE_DATABASE: clickdoom + # The branch the native-regression lines are kept on, and the file on it. + # The branch is never merged into main. + REGRESSION_BRANCH: regression-data + REGRESSION_PATH: native/bench/regression/results.jsonl jobs: deep-diff: @@ -65,3 +76,147 @@ jobs: ./target/release/clickdoom emulation diff "$INSTRUCTIONS" --host localhost --port 8123 \ --bin rom/build/doom-rv32im.bin --manifest rom/build/manifest.json \ --refemu-bin target/release/refemu + + native-regression: + # Reads the repository and writes nothing to it. What it found goes to + # native-record as an artifact. + runs-on: ubuntu-latest + permissions: + contents: read + timeout-minutes: 330 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 0 + persist-credentials: false + - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 + - name: Start the pinned ClickHouse + run: make up + - name: Build ROM + run: make -C rom + - name: Build the judge + # The walk rebuilds target/release/clickdoom for every commit, so the + # binary that judges the night is kept apart. + run: | + cargo build --locked --release -p clickdoom-driver + cp target/release/clickdoom "$RUNNER_TEMP/clickdoom-judge" + - name: Name main's probe trace + id: probe + run: | + echo "key=$(git rev-parse origin/main:rom/PINNED_HASH)-$(git rev-parse origin/main:refemu/probe)" >>"$GITHUB_OUTPUT" + - name: Restore the probe traces + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: ${{ runner.temp }}/traces + key: probe-trace-${{ steps.probe.outputs.key }} + - name: Read what is recorded + env: + GH_TOKEN: ${{ github.token }} + REPO: ${{ github.repository }} + run: | + if gh api "repos/$REPO/branches/$REGRESSION_BRANCH" --silent 2>"$RUNNER_TEMP/err"; then + gh api -H "Accept: application/vnd.github.raw" \ + "repos/$REPO/contents/$REGRESSION_PATH?ref=$REGRESSION_BRANCH" >"$RUNNER_TEMP/history.jsonl" + echo "$(wc -l <"$RUNNER_TEMP/history.jsonl") line(s) recorded" + elif grep -q "HTTP 404" "$RUNNER_TEMP/err"; then + echo "no $REGRESSION_BRANCH branch yet" + : >"$RUNNER_TEMP/history.jsonl" + else + cat "$RUNNER_TEMP/err" >&2 + exit 1 + fi + - name: Walk the commits not yet recorded + env: + MAX_COMMITS: ${{ github.event.inputs.max_commits || '8' }} + TRACES: ${{ runner.temp }}/traces + CLICKDOOM_DATABASE: regress + run: scripts/native-regression-walk.sh "$RUNNER_TEMP/history.jsonl" regression + - name: Judge the night + run: | + if [ ! -s regression/night.jsonl ]; then + echo "nothing walked, so nothing to judge" + exit 0 + fi + history=() + if [ -s "$RUNNER_TEMP/history.jsonl" ]; then + history=(--history "$RUNNER_TEMP/history.jsonl") + fi + status=0 + "$RUNNER_TEMP/clickdoom-judge" native regress regression/night.jsonl "${history[@]}" \ + || status=$? + case "$status" in + 0) ;; + 3) echo "::warning::a recorded line regressed; the lines above name it" ;; + *) exit "$status" ;; + esac + - name: Upload what the night found + if: ${{ !cancelled() }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: native-regression + path: regression/ + if-no-files-found: error + + native-record: + # Writes the branch, and runs no repository code: it checks nothing out, + # and reads only the artifact native-regression uploaded. + needs: native-regression + runs-on: ubuntu-latest + permissions: + contents: write # appends to the regression-data branch + timeout-minutes: 10 + env: + GH_TOKEN: ${{ github.token }} + REPO: ${{ github.repository }} + RUN_ID: ${{ github.run_id }} + steps: + - name: Download what the night found + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: native-regression + path: regression + - name: Append the new lines to the data branch + run: | + if [ ! -s regression/night.jsonl ]; then + echo "nothing walked, so nothing to record" + exit 0 + fi + file_sha="" + if gh api "repos/$REPO/branches/$REGRESSION_BRANCH" --silent 2>err; then + gh api -H "Accept: application/vnd.github.raw" \ + "repos/$REPO/contents/$REGRESSION_PATH?ref=$REGRESSION_BRANCH" >history.jsonl + file_sha=$(gh api "repos/$REPO/contents/$REGRESSION_PATH?ref=$REGRESSION_BRANCH" --jq .sha) + elif grep -q "HTTP 404" err; then + : >history.jsonl + else + cat err >&2 + exit 1 + fi + # A commit the branch already holds is not written twice. The walk + # measures the last recorded commit again as the next one's parent. + jq -c -n --slurpfile old history.jsonl --slurpfile new regression/night.jsonl ' + reduce $new[] as $line ({seen: ($old | map(.commit)), out: []}; + if (.seen | index($line.commit)) then . else + .seen += [$line.commit] | .out += [$line] end) | .out[]' >fresh.jsonl + count=$(wc -l results.jsonl + message="regression: record $count commit(s) from run $RUN_ID" + if [ -n "$file_sha" ]; then + gh api -X PUT "repos/$REPO/contents/$REGRESSION_PATH" --silent \ + -f message="$message" -f branch="$REGRESSION_BRANCH" -f sha="$file_sha" \ + -f content="$(base64 -w0 results.jsonl)" + else + # The branch is an orphan: its one file and no history from main. + blob=$(gh api "repos/$REPO/git/blobs" -f encoding=base64 \ + -f content="$(base64 -w0 results.jsonl)" --jq .sha) + tree=$(gh api "repos/$REPO/git/trees" -f "tree[][path]=$REGRESSION_PATH" \ + -f "tree[][mode]=100644" -f "tree[][type]=blob" -f "tree[][sha]=$blob" --jq .sha) + commit=$(gh api "repos/$REPO/git/commits" -f message="$message" -f tree="$tree" --jq .sha) + gh api "repos/$REPO/git/refs" --silent \ + -f ref="refs/heads/$REGRESSION_BRANCH" -f sha="$commit" + fi + echo "recorded $count line(s) on $REGRESSION_BRANCH" diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 4334c70..8661b34 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -48,7 +48,10 @@ what runs there. Expect upwards of ten minutes, most of it the ROM build and the differential smoke. It is necessary and not sufficient. The nightly deep-diff is the only run that -compares memory, and no pull request makes it. +compares memory, and no pull request makes it. The nightly also diffs native +mode against the reference emulator on each new main commit, and names the +commit whose first refusal or first divergence moved earlier, or whose tic +statement got slower than its parent's. Check by exit code. A pipeline reports only its last command's status, so `make gates | tail` can hide a failure. diff --git a/DEVELOPING.md b/DEVELOPING.md index 565c4f6..7087e62 100644 --- a/DEVELOPING.md +++ b/DEVELOPING.md @@ -31,7 +31,7 @@ ROM build and the smoke diff. `lint` also runs `check-adr`, `actionlint` and `check-bare`, which have no CI job of their own. -The benches, `fuzz`, the milestone targets and the nightly deep-diff sit outside +The benches, `fuzz`, the milestone targets and the nightly jobs sit outside `gates`, by cost or by what they need. A timing run needs a quiet machine, and the deep-diff takes hours. @@ -177,6 +177,32 @@ The melt's pass count per frame comes from the reference run and is loaded as data from `driver/melt/demo3.tsv`; its provenance is in that directory's README. +### What the nightly records + +`nightly.yml`'s `native-regression` job runs `scripts/native-regression-walk.sh` +over the main commits not yet recorded, oldest first, at most `max_commits` a +night (default 8). Each commit is built in its own worktree and run with +`native diff --record` against a probe trace generated from that commit's ROM +and probe. A diff that stops at a refusal compares nothing, so the walk runs a +second diff over the tics before the refused one and keeps its line with the +first diff's refusal. `driver/src/native/record.rs` documents the line. A commit +that does not build or run gets a line with `error`, and the walk goes on. + +`clickdoom native regress` judges the night's lines against the recorded ones, +and its `--help` states the rules. Cost is judged only between a parent and a +child measured one after the other in the same job, so the walk measures the +last recorded commit again first. + +The `native-record` job appends the new lines to +`native/bench/regression/results.jsonl` on the `regression-data` branch, which +is never merged into main. It is the only job with write access, and it runs no +repository code. With `max_commits` at 0 the walk measures only the last +recorded commit. + +The walk runs locally against `make up`: +`scripts/native-regression-walk.sh HISTORY OUT`, with `HISTORY` a copy of the +branch's file. + ## Benchmarks Timings need a quiet machine, and the numbers in `docs/experiments/` were diff --git a/scripts/native-regression-walk.sh b/scripts/native-regression-walk.sh new file mode 100755 index 0000000..2ae9b5a --- /dev/null +++ b/scripts/native-regression-walk.sh @@ -0,0 +1,182 @@ +#!/usr/bin/env bash +# +# Records native mode's parity and cost for each main commit not yet +# recorded, oldest first: one `clickdoom native diff --record` line per +# commit. +# +# scripts/native-regression-walk.sh HISTORY OUT +# +# HISTORY holds the lines recorded so far and may be absent. The walk starts +# at the last commit it records, measured again so the commit after it has a +# parent measured on the same machine, and follows TIP's first parents from +# there. With no HISTORY, or when its last commit is not an ancestor of TIP, +# it measures TIP's first parent and TIP. +# +# Each commit is checked out in its own worktree, built, loaded into a fresh +# database and diffed against a probe trace generated from that commit's ROM +# and probe. The first diff runs SEARCH_TICS tics to find the first refused +# tic. When its line compared nothing, a second diff runs over the tics +# before the refusal, and the line kept is the second one carrying the +# first one's refusal. That line has to end below the refusal and compare at +# least one tic. +# +# A commit that does not build, load or run gets a line with `error` instead, +# and the walk goes on. OUT/night.jsonl gets one line per commit walked, and +# OUT/logs/ each commit's output. +# +# Environment: +# TIP the commit to walk up to, default origin/main +# MAX_COMMITS commits to walk after the parent, default 8 +# BUDGET_MINUTES no commit starts once the walk has run this long, default 240 +# SEARCH_TICS the first diff's span, default 2000 +# TRACES where probe traces are kept by ROM and probe version, +# default OUT/traces +# CLICKDOOM_DATABASE the database each commit loads, default regress +# CH_HOST, CH_HTTP_PORT and CLICKHOUSE_PASSWORD, as for make +set -euo pipefail +cd "$(dirname "$0")/.." + +if [ "$#" -ne 2 ]; then + echo "usage: $0 HISTORY OUT" >&2 + exit 2 +fi +history=$1 +mkdir -p "$2" +out=$(cd "$2" && pwd) +max_commits=${MAX_COMMITS:-8} +budget_minutes=${BUDGET_MINUTES:-240} +search_tics=${SEARCH_TICS:-2000} +traces=${TRACES:-$out/traces} +tip=${TIP:-origin/main} +conn=(--host "${CH_HOST:-localhost}" --port "${CH_HTTP_PORT:-8123}" + --database "${CLICKDOOM_DATABASE:-regress}") +export CARGO_TARGET_DIR="$PWD/target" + +mkdir -p "$out/logs" "$traces" +night="$out/night.jsonl" +: >"$night" +work=$(mktemp -d) +trap 'rm -rf "$work"; git worktree prune' EXIT + +last="" +if [ -s "$history" ]; then + last=$(tail -n 1 "$history" | jq -r .commit) +fi +if [ -n "$last" ] && git merge-base --is-ancestor "$last" "$tip" 2>/dev/null; then + parent=$last + mapfile -t newer < <(git rev-list --first-parent --reverse "$last..$tip") +else + parent=$(git rev-parse "$tip^") + newer=("$(git rev-parse "$tip")") +fi +if [ "${#newer[@]}" -eq 0 ]; then + echo "nothing on $tip after $last" + exit 0 +fi +if [ "${#newer[@]}" -gt "$max_commits" ]; then + echo "${#newer[@]} commits after $parent; walking the oldest $max_commits" + newer=("${newer[@]:0:$max_commits}") +fi + +# The ROM this job built, which a commit pinning the same hash reuses. +built_rom="" +if [ -f rom/build/doom-rv32im.bin ]; then + built_rom=$(sha256sum rom/build/doom-rv32im.bin | cut -d' ' -f1) +fi + +# Records why the step that called it failed, for the error line. +fail() { + printf '%s' "$1" >"$work/why" + return 1 +} + +# One diff of SPAN tics in WT, writing its line to RECORD. Exit 3 is a +# refusal or a divergence, which the line records. Any other exit is a +# failure. +diff_span() { + local wt=$1 bin=$2 trace=$3 span=$4 record=$5 status=0 + (cd "$wt" && "$bin/clickdoom" native diff "$span" --probe "$trace" \ + "${conn[@]}" --record "$record") || status=$? + if [ "$status" -ne 0 ] && [ "$status" -ne 3 ]; then + fail "native diff $span exited $status" + elif [ ! -f "$record" ] || [ "$(wc -l <"$record")" -ne 1 ]; then + fail "native diff $span wrote no line" + fi +} + +# Builds and measures SHA, and writes the line to keep to $work/SHA.line. +measure() { + local sha=$1 wt="$work/$1" bin="$work/bin/$1" + git worktree add --detach --quiet "$wt" "$sha" || fail "checkout failed" || return + (cd "$wt" && cargo build --locked --release -p clickdoom-driver -p refemu) \ + || fail "cargo build failed" || return + mkdir -p "$bin" + cp "$CARGO_TARGET_DIR/release/clickdoom" "$CARGO_TARGET_DIR/release/refemu" "$bin/" + + if [ "$(cat "$wt/rom/PINNED_HASH")" = "$built_rom" ]; then + mkdir -p "$wt/rom/build" + cp rom/build/doom-rv32im.bin rom/build/doom-rv32im.elf rom/build/manifest.json "$wt/rom/build/" + else + make -C "$wt/rom" || fail "the ROM build failed" || return + fi + + # A trace depends only on the ROM and on the probe that dumps it. + local rom_blob probe_tree trace + rom_blob=$(git rev-parse "$sha:rom/PINNED_HASH") || fail "no rom/PINNED_HASH" || return + probe_tree=$(git rev-parse "$sha:refemu/probe") || fail "no refemu/probe" || return + trace="$traces/$rom_blob-$probe_tree.tsv" + if [ ! -f "$trace" ]; then + make -C "$wt" gen-probe-trace REFEMU="$bin/refemu" \ + || fail "make gen-probe-trace failed" || return + cp "$wt/refemu/reference_traces/demo3/probe.$(cut -c1-12 "$wt/rom/PINNED_HASH").tsv" "$trace" \ + || fail "make gen-probe-trace wrote no trace" || return + fi + + (cd "$wt" && "$bin/clickdoom" native load --fresh "${conn[@]}") \ + || fail "native load failed" || return + + local search="$work/$sha.search.jsonl" span="$work/$sha.span.jsonl" + local line="$work/$sha.line" refused + diff_span "$wt" "$bin" "$trace" "$search_tics" "$search" || return + if jq -e '.compared_through != null' "$search" >/dev/null; then + cp "$search" "$line" + else + refused=$(jq -r '.first_refused_tic' "$search") + if [ "$refused" -le 1 ]; then + fail "tic $refused refused, so no tic can be compared" + return + fi + diff_span "$wt" "$bin" "$trace" "$((refused - 1))" "$span" || return + jq -e --argjson refused "$refused" \ + '.first_refused_tic == null and .compared_through < $refused and .compared_tics > 0' \ + "$span" >/dev/null \ + || fail "native diff $((refused - 1)) did not compare the tics before the refusal at $refused" \ + || return + jq -c -s '.[1] + {first_refused_tic: .[0].first_refused_tic, first_refused_bits: .[0].first_refused_bits}' \ + "$search" "$span" >"$line" + fi + [ "$(jq -r .commit "$line")" = "$sha" ] \ + || fail "the line names commit $(jq -r .commit "$line"), not $sha" +} + +for sha in "$parent" "${newer[@]}"; do + if [ "$SECONDS" -ge "$((budget_minutes * 60))" ]; then + echo "stopping after $budget_minutes minutes; the next run starts from the last line written" + break + fi + short=${sha:0:12} + echo "::group::$short" + rm -f "$work/why" + if measure "$sha" 2>&1 | tee "$out/logs/$short.log"; then + cat "$work/$sha.line" >>"$night" + echo "$short: $(cat "$work/$sha.line")" + else + why=$(cat "$work/why" 2>/dev/null || echo "failed; see logs/$short.log") + echo "::error::$short: $why" + jq -n -c --arg commit "$sha" --arg error "$why" --arg run_id "${GITHUB_RUN_ID:-}" \ + '{commit: $commit, error: $error, run_id: (if $run_id == "" then null else $run_id end)}' >>"$night" + fi + git worktree remove --force "$work/$sha" 2>/dev/null || true + echo "::endgroup::" +done +echo "$(wc -l <"$night") line(s) in $night"