From 656f47c0930b61da7b0f1fb47241d69dbe31ac56 Mon Sep 17 00:00:00 2001 From: Ely Erin Fox Date: Sun, 4 Oct 2026 15:03:55 -0700 Subject: [PATCH 1/2] CI gpu-scheduling: interleaved best-of-3 timings, session-reuse check instead of a chunk-spread race The two develop failures of multi-GPU scheduling (runs 37220358649 and 37226241965) were not the timing assertions, which passed with their usual margins (ladder 0.78-0.79x against 0.95, 10 s single file 1.06x against 1.10), nor card contention (the x2 pod holds both devbox cards through the device plugin; per-card jobs queued meanwhile ran on the A310). They were the reuse step's 'every card took a chunk' check on the 70 s H.264 rung: that encode is decode-bound, the A750 finishes each chunk before the next is decoded, and the speed gate rightly kept the A380 out in 2 of 6 runs, finishing faster (7.7 s against 8.1-8.7 s). - The reuse step now asserts what it is for: every card that encoded built one session and reused it for each further chunk, nothing failed to reset, and at least one session was reused. - Every timing assertion compares within the run, best of three interleaved rounds with the plan order rotated per round: speed_aware_timings.py now also times the encode-bound archive ladders (0.85x), and the single-shot 0.95x / 0.85x checks in the ladder and busy steps become correctness-only. One job per card: best of three. Every round, each plan's spread and the paired ratios go to the step summary and speed_aware_timings.json. - Limits unchanged (0.95, 1.10, 0.85, 1.15): six develop runs measured 0.78-0.79, 1.06-1.07, 0.66-0.71 and 0.96-1.00. --- .github/scripts/multi_gpu_check.py | 38 ++++- .github/scripts/speed_aware_timings.py | 187 +++++++++++++++++-------- .github/workflows/ci.yml | 104 ++++++++------ 3 files changed, 228 insertions(+), 101 deletions(-) diff --git a/.github/scripts/multi_gpu_check.py b/.github/scripts/multi_gpu_check.py index c1bb651..9875bc4 100644 --- a/.github/scripts/multi_gpu_check.py +++ b/.github/scripts/multi_gpu_check.py @@ -15,6 +15,10 @@ multi_gpu_check.py chunks LOG CARDS.json rivet's log: the ladder workers' `rung chunk done` lines name every card. + multi_gpu_check.py reuse LOG + rivet's log: every card that encoded built one encoder session and + reused it (reset) for each further chunk it took, no reset failed or + was unsupported, and at least one session was reused. multi_gpu_check.py faster LABEL SECONDS BASELINE_SECONDS MAX_RATIO SECONDS <= BASELINE_SECONDS * MAX_RATIO. multi_gpu_check.py timings CARDS.json OUT.json @@ -105,6 +109,38 @@ def chunks(log, cards_json): fail(f"{log}: no chunk was encoded on card(s) {missing}: {counts}") +def reuse(log): + ansi = re.compile(r"\x1b\[[0-9;]*m") + chunks, slots = {}, {} + with open(log, errors="replace") as f: + for line in f: + line = ansi.sub("", line) + if "rung chunk done" in line: + m = re.search(r"gpu_index=Some\((\d+)\)", line) + key = m.group(1) if m else "none" + chunks[key] = chunks.get(key, 0) + 1 + elif "built vs reused" in line: + fields = dict(re.findall(r"(\w+)=(\S+)", line.split("built vs reused", 1)[1])) + slots[fields["slot"]] = fields + print(f"chunks per card: {chunks}") + if not slots: + fail(f"{log}: no `built vs reused` line") + reused = 0 + for slot, f in sorted(slots.items()): + gpu = f["gpu_index"] + built, used = int(f["built"]), int(f["reused"]) + print(f"slot {slot} (gpu {gpu}): {chunks.get(gpu, 0)} chunk(s), built {built}, reused {used}") + for bad in ("reset_failed", "reset_unsupported", "evicted"): + if int(f[bad]) != 0: + fail(f"{log}: slot {slot} (gpu {gpu}) {bad}={f[bad]}") + if built + used != chunks.get(gpu, 0) or built > 1: + fail(f"{log}: slot {slot} (gpu {gpu}) took {chunks.get(gpu, 0)} chunk(s) " + f"with {built} session(s) built and {used} reused: each card builds one and reuses it") + reused += used + if reused < 1: + fail(f"{log}: no encoder session was reused: {slots}") + + def faster(label, seconds, baseline, ratio): seconds, baseline, ratio = float(seconds), float(baseline), float(ratio) print(f"{label}: {seconds:.1f}s against {baseline:.1f}s, {baseline / seconds:.2f}x") @@ -133,7 +169,7 @@ def timings(cards_json, out_json): print(f"| {name} | {r['wall_s']} | " + " | ".join(str(r["video_busy_s"][i]) for i in order) + " |") -COMMANDS = {"devices": devices, "pinned": pinned, "all": every, "chunks": chunks, "faster": faster, "timings": timings} +COMMANDS = {"devices": devices, "pinned": pinned, "all": every, "chunks": chunks, "reuse": reuse, "faster": faster, "timings": timings} if __name__ == "__main__": if len(sys.argv) < 2 or sys.argv[1] not in COMMANDS: diff --git a/.github/scripts/speed_aware_timings.py b/.github/scripts/speed_aware_timings.py index 99559b3..03a7418 100644 --- a/.github/scripts/speed_aware_timings.py +++ b/.github/scripts/speed_aware_timings.py @@ -1,21 +1,44 @@ #!/usr/bin/env python3 -"""Times rivet on each card alone and on all of them, and checks that the -cards together are no slower than the fastest card alone. +"""Times rivet on each card alone and on all of them, interleaved, and checks +that the cards together beat (or, for a short file, keep up with) the +fastest card alone. speed_aware_timings.py RIVET SRC_LADDER SRC_SINGLE [--base RIVET_BASE] - [--vendor intel] [--repeat N] - -For every plan — each card pinned (`--encode gpu:N --decode gpu:N`), and all -the cards (`--encode family:VENDOR`, decode `auto` and `whole`) — it runs a -three-rung AV1 HLS ladder on SRC_LADDER and a one-rung H.265 single file on -SRC_SINGLE and on SRC_LADDER (reported only), takes the best of N wall times, and prints a Markdown table -(`--base` adds the same runs with another build, e.g. develop's, as a -"before" column). With two or more cards it then asserts, for this build: - -- the ladder on all the cards beats the fastest card alone (<= 0.95x); -- the single rung on all the cards is no slower than the fastest card alone - (<= 1.10x): a short single file is one or two chunks, which no split can - make faster than one fast card, but must not make slower either. + [--vendor intel] [--repeat N] [--json OUT.json] + +Jobs, each run under every plan: + +- `ladder`: a three-rung AV1 HLS ladder on SRC_LADDER. Plans: each card + pinned (`--encode gpu:N --decode gpu:N`), and all the cards + (`--encode family:VENDOR`, decode `auto` — the default — and `whole`). +- `single`: a one-rung H.265 single file on SRC_SINGLE; same plans. +- `single-long`: the same on SRC_LADDER (reported only: two chunks of + unequal length on cards 3x apart). +- `busy-av1`, `busy-h265`: the ladder at `--video-speed archive` with the + decode on the fastest card, so the encode is the limit. Plans: the + fastest card's encoder alone, and every card's (`family:VENDOR`). + +The measurement is built to compare within one run, not across runs: + +- **Interleaved**: N rounds, and every round runs every plan of a job once, + in an order rotated from round to round. Load that comes and goes on the + host (another pod's CPU work, a clock ramp) lands on all the plans alike + instead of on whichever plan happened to run while it lasted. +- **Best of N**: each plan's time is its fastest round. Outside load only + ever slows a run, so the minimum is the closest to the hardware's own + time; the ratios are of these minima. +- **Recorded**: every round's time, each plan's spread ((max - min) / min) + and every round's paired ratio go to the step summary and to OUT.json. + +With two or more cards it then asserts, for this build: + +- `ladder`: all the cards at most 0.95x the fastest card alone. devbox + (A750 + A380) measures 0.78-0.79x run after run, so 0.95 leaves room for + noise and still fails the moment a second card stops paying for itself. +- `single`: at most 1.10x. A 10 s file is one or two chunks, which no split + can make faster than one fast card, but which must not be made slower + either; devbox measures 1.06-1.07x (a fixed ~60 ms of opening both cards). +- `busy-av1`, `busy-h265`: at most 0.85x. devbox measures 0.66-0.71x. Exits 0 with "skipped" on a host with fewer than two cards of the vendor. """ @@ -23,12 +46,14 @@ import argparse import json import os -import shutil +import statistics import subprocess import sys import tempfile import time +LIMITS = {"ladder": 0.95, "single": 1.10, "single-long": None, "busy-av1": 0.85, "busy-h265": 0.85} + def cards(rivet, vendor): out = subprocess.run([rivet, "devices", "--json"], check=True, capture_output=True, text=True).stdout @@ -58,8 +83,26 @@ def fmt(t): return "failed" if t != t else f"{t:.2f} s" -def best_of(rivet, args, log, repeat): - return min(run(rivet, args, log) for _ in range(repeat)) +def rounds(jobs, plans, builds, repeat, samples): + """Runs every (job, plan, build) `repeat` times: round by round, each + job's plans in an order rotated by the round number.""" + for r in range(repeat): + for job, job_args in jobs: + cells = [(plan, plan_args, build, binary) for plan, plan_args in plans[job] for build, binary in builds] + k = r % len(cells) + for plan, plan_args, build, binary in cells[k:] + cells[:k]: + tag = f"{job}-{plan}-{build}".replace(" ", "-").replace(",", "") + try: + t = run(binary, job_args + plan_args, f"timing-{tag}.log") + except SystemExit as e: + # The "before" build is only a comparison: its failure is + # reported, not fatal. + if build != "before": + raise + print(f"before build failed: {e}") + t = float("nan") + samples.setdefault((job, plan, build), []).append(t) + print(f"round {r + 1}/{repeat} {job:11} {plan:26} {build:6} {t:6.2f} s", flush=True) def main(): @@ -69,67 +112,89 @@ def main(): ap.add_argument("single_src") ap.add_argument("--base") ap.add_argument("--vendor", default="intel") - ap.add_argument("--repeat", type=int, default=2) + ap.add_argument("--repeat", type=int, default=3) + ap.add_argument("--json", default="speed_aware_timings.json") a = ap.parse_args() found = cards(a.rivet, a.vendor) if len(found) < 2: print(f"skipped: {len(found)} {a.vendor} card(s) on this host") return - ladder = [a.ladder_src, "--mode", "hls", "--codec", "av1", "--rung", "1920x1080", "--rung", "1280x720", "--rung", "854x480"] + rungs = ["--rung", "1920x1080", "--rung", "1280x720", "--rung", "854x480"] + ladder = [a.ladder_src, "--mode", "hls", "--codec", "av1", *rungs] single = [a.single_src, "--codec", "h265", "--rung", "1920x1080"] single_long = [a.ladder_src, "--codec", "h265", "--rung", "1920x1080"] - plans = [(f"gpu {g} alone", ["--encode", f"gpu:{g}", "--decode", f"gpu:{g}"]) for g in found] - plans += [ + alone = [(f"gpu {g} alone", ["--encode", f"gpu:{g}", "--decode", f"gpu:{g}"]) for g in found] + together = [ ("all cards, decode auto", ["--encode", f"family:{a.vendor}", "--decode", "auto"]), ("all cards, decode whole", ["--encode", f"family:{a.vendor}", "--decode", "whole"]), ] builds = [("after", a.rivet)] + ([("before", a.base)] if a.base else []) - times = {} - jobs = (("ladder", ladder), ("single", single), ("single-long", single_long)) - for job, job_args in jobs: - for plan, plan_args in plans: - for build, binary in builds: - tag = f"{job}-{plan}-{build}".replace(" ", "-").replace(",", "") - try: - times[(job, plan, build)] = best_of(binary, job_args + plan_args, f"timing-{tag}.log", a.repeat) - except SystemExit as e: - # The "before" build is only a comparison: its failure is - # reported, not fatal. - if build != "before": - raise - print(f"before build failed: {e}") - times[(job, plan, build)] = float("nan") - print(f"{job:6} {plan:26} {build:6} {times[(job, plan, build)]:6.2f} s", flush=True) - - head = "| job | plan | " + " | ".join(b for b, _ in builds) + " |" - lines = [head, "|" + "---|" * (2 + len(builds))] - for job, _ in jobs: - for plan, _ in plans: - lines.append(f"| {job} | {plan} | " + " | ".join(fmt(times[(job, plan, b)]) for b, _ in builds) + " |") + samples = {} + + def best(job, plan, build="after"): + return min(samples[(job, plan, build)]) + + # Phase 1: every card alone and all of them, for each job. + jobs = [("ladder", ladder), ("single", single), ("single-long", single_long)] + rounds(jobs, {job: alone + together for job, _ in jobs}, builds, a.repeat, samples) + alone_ladder = {g: best("ladder", f"gpu {g} alone") for g in found} + fast = min(alone_ladder, key=alone_ladder.get) + print(f"the fastest card (ladder, best of {a.repeat}): gpu {fast}") + + # Phase 2: encode-bound, on the fastest card alone and on all of them. + busy_jobs = [(f"busy-{codec}", [a.ladder_src, "--mode", "hls", "--codec", codec, *rungs, + "--video-speed", "archive", "--decode", f"gpu:{fast}"]) for codec in ("av1", "h265")] + busy_plans = [(f"gpu {fast} alone", ["--encode", f"gpu:{fast}"]), ("all cards", ["--encode", f"family:{a.vendor}"])] + rounds(busy_jobs, {job: busy_plans for job, _ in busy_jobs}, builds, a.repeat, samples) + jobs += busy_jobs + + # Every round of every plan, its best and its spread. + head = "| job | plan | build | rounds (s) | best | spread |" + lines = [head, "|---|---|---|---|---|---|"] + for (job, plan, build), ts in samples.items(): + ok = [t for t in ts if t == t] + spread = f"{(max(ok) - min(ok)) / min(ok) * 100:.1f}%" if ok else "-" + lines.append(f"| {job} | {plan} | {build} | " + ", ".join(f"{t:.2f}" for t in ts) + + f" | {fmt(min(ok) if ok else float('nan'))} | {spread} |") table = "\n".join(lines) print(table) summary = os.environ.get("GITHUB_STEP_SUMMARY") - if summary: - with open(summary, "a") as f: - f.write("\n### Speed-aware scheduling: wall times (best of %d)\n\n%s\n" % (a.repeat, table)) - - failures = [] - # single-long (the ladder source as one rung: two chunks) is reported, - # not judged: two chunks of unequal length on cards 3x apart. - for job, limit in (("ladder", 0.95), ("single", 1.10), ("single-long", None)): - alone = {g: times[(job, f"gpu {g} alone", "after")] for g in found} - fastest = min(alone, key=alone.get) - together = times[(job, "all cards, decode auto", "after")] - ratio = together / alone[fastest] - verdict = "reported" if limit is None else ("ok" if ratio <= limit else "FAIL") - line = f"{job}: all cards {together:.2f} s vs gpu {fastest} alone {alone[fastest]:.2f} s = {ratio:.2f}x (limit {limit}x) {verdict}" - print(line) + + def out(text): if summary: with open(summary, "a") as f: - f.write(f"- {line}\n") - if limit is not None and ratio > limit: + f.write(text) + + out(f"\n### Speed-aware scheduling: wall times, {a.repeat} interleaved rounds\n\n{table}\n\n") + + # The judgement: best of N on all cards against best of N on the fastest + # card alone, both from this run. The paired ratios (round r against + # round r) are reported as the measure of how noisy this run was. + failures, verdicts = [], {} + for job, _ in jobs: + limit = LIMITS[job] + if job.startswith("busy-"): + fastest, all_plan = f"gpu {fast} alone", "all cards" + else: + fastest = min((f"gpu {g} alone" for g in found), key=lambda p: best(job, p)) + all_plan = "all cards, decode auto" + ratio = best(job, all_plan) / best(job, fastest) + paired = [t / s for t, s in zip(samples[(job, all_plan, "after")], samples[(job, fastest, "after")])] + verdict = "reported" if limit is None else ("ok" if ratio <= limit else "FAIL") + line = (f"{job}: all cards {best(job, all_plan):.2f} s vs {fastest} {best(job, fastest):.2f} s " + f"= {ratio:.3f}x (limit {limit}x; rounds {', '.join(f'{p:.3f}' for p in paired)}) {verdict}") + print(line) + out(f"- {line}\n") + verdicts[job] = {"all": all_plan, "fastest": fastest, "ratio": ratio, "paired": paired, + "limit": limit, "verdict": verdict} + if verdict == "FAIL": failures.append(line) + + with open(a.json, "w") as f: + json.dump({"repeat": a.repeat, "cards": found, + "samples": [{"job": j, "plan": p, "build": b, "wall_s": ts} for (j, p, b), ts in samples.items()], + "verdicts": verdicts}, f, indent=1) if failures: raise SystemExit("\n".join(failures)) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a3b5b0f..3a2e126 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -336,6 +336,16 @@ jobs: # replaces, so a burst of pushes would lose whole runs instead of only # their scheduling job. A one-card job from another run can still hold a # devbox card for a while; the pod then waits for it. + # + # No other GPU work can share the cards while it runs, so no concurrency + # group with the per-card jobs is needed (and one would cost: a group + # runs one member and keeps one pending, cancelling the rest). The pod + # requests gpu.intel.com/i915: 2 on devbox, which has exactly two, and + # the device plugin runs with -shared-dev-num 1, so while it holds them + # no other pod (safewords-runners-gpu, or another org's GPU runners, + # all of which request i915) is given either card: per-card jobs queued + # meanwhile run on vulpes-zerda's A310 or wait. Other pods can still + # load devbox's CPU, which is what the interleaved timings are for. needs: [gpu-build, gpu-devices, gpu-codec] if: ${{ !cancelled() && github.event_name != 'pull_request' && needs.gpu-build.result == 'success' }} concurrency: @@ -375,13 +385,13 @@ jobs: # them (`--encode family:intel`, the default decode plan): on all, # chunks done on every card and every card's video engines busy. # Every rendition decodes; the full-size one close to the source. - # - # With the default plans the cards together must beat the fastest - # card alone (at most 0.95x its time): devbox's A380 (PCIe 3.0 x2 - # behind the chipset) is 2-3.5x slower at 1080p than its A750 (4.0 - # x16), and the work is split by speed — the decode stays off a card - # far behind the fastest, and the tail of the encode runs on the - # fast one (multigpu::speed). + # devbox's A380 (PCIe 3.0 x2 behind the chipset) is 2-3.5x slower at + # 1080p than its A750 (4.0 x16), and the work is split by speed — the + # decode stays off a card far behind the fastest, and the tail of the + # encode runs on the fast one (multigpu::speed). These runs are single + # and only reported (scheduling.json); the timing judgement (at most + # 0.95x the fastest card alone) is the interleaved best of three in + # "Cards of different speeds share the work by speed". env: CARDS: ${{ steps.cards.outputs.cards }} GPUS: ${{ steps.cards.outputs.gpus }} @@ -397,22 +407,19 @@ jobs: done $ASSERT all usage-ladder-family-intel.json cards.json $ASSERT chunks ladder-family-intel.log cards.json - wall() { python3 -c "import json,sys; print(json.load(open(sys.argv[1]))['wall_s'])" "$1"; } - fast=$(python3 -c "import json; c=json.load(open('cards.json')); print(min(c, key=lambda i: json.load(open(f'usage-ladder-gpu-{i}.json'))['wall_s']))") - $ASSERT faster "default AV1 ladder on all cards vs card $fast alone" "$(wall usage-ladder-family-intel.json)" "$(wall "usage-ladder-gpu-$fast.json")" 0.95 full=$(awk '/RESOLUTION=1920x1080/{getline; print; exit}' ladder-family-intel/master.m3u8) $CHECK "ladder-family-intel/$full" --ref src1080.mp4 --codec av1 --frames 900 --size 1920x1080 --min-psnr 30 - - name: When the encode is the bottleneck, all the cards beat the fastest one alone + - name: When the encode is the bottleneck, every card does chunks # The encoders at `--video-speed archive` and the decode on the - # fastest card, so the encode is the limit: the ladder on every card - # must take at most 0.85x the fastest card's time alone (devbox: - # about 0.7x). + # fastest card, so the encode is the limit: pinned stays pinned, and + # on all cards every card encodes. Timed (at most 0.85x the fastest + # card alone; devbox: about 0.7x) in the interleaved runs of the + # next step. env: CARDS: ${{ steps.cards.outputs.cards }} run: | if [ "$CARDS" -lt 2 ]; then echo "skipped: $CARDS card on this host" | tee -a "$GITHUB_STEP_SUMMARY"; exit 0; fi - wall() { python3 -c "import json,sys; print(json.load(open(sys.argv[1]))['wall_s'])" "$1"; } fast=$(python3 -c "import json; c=json.load(open('cards.json')); print(min(c, key=lambda i: json.load(open(f'usage-ladder-gpu-{i}.json'))['wall_s']))") echo "the fastest card is $fast" for codec in av1 h265; do @@ -425,38 +432,48 @@ jobs: $ASSERT pinned "usage-busy-$codec-gpu-$fast.json" cards.json "$fast" $ASSERT all "usage-busy-$codec-family-intel.json" cards.json $ASSERT chunks "busy-$codec-family-intel.log" cards.json - $ASSERT faster "$codec archive ladder on all cards vs card $fast alone" \ - "$(wall "usage-busy-$codec-family-intel.json")" "$(wall "usage-busy-$codec-gpu-$fast.json")" 0.85 for p in $(find "busy-$codec-family-intel/video" -name '*.m3u8' | sort); do $CHECK "$p" --codec "$codec" --frames 900; done done - name: Cards of different speeds share the work by speed - # Each card alone and all of them together (decode auto and whole): - # a three-rung AV1 HLS ladder, a one-rung H.265 single file of 10 s, - # and the same of 30 s (reported only). The ladder on all cards must - # take at most 0.95x the fastest card's time alone, the 10 s rung - # at most 1.10x (one chunk: it encodes serially on the fastest card). - # Then a 70 s rung through the chunk engine (four chunks on two - # cards), where a card's encoder session is reset and reused for its - # next chunk, for H.265, H.264 and AV1. + # The timing assertions, all compared within this run: each card + # alone and all of them together (decode auto and whole), three + # rounds, every round running every plan in a rotated order, best of + # the three (speed_aware_timings.py). A three-rung AV1 HLS ladder: + # all cards at most 0.95x the fastest card alone. A one-rung H.265 + # single file of 10 s: at most 1.10x (one chunk: it encodes serially + # on the fastest card). The same of 30 s: reported only. The ladder + # at `--video-speed archive`, AV1 and H.265: at most 0.85x. Every + # round's time, each plan's spread and the ratios go to the summary + # and to speed_aware_timings.json. + # + # Then a 70 s rung through the chunk engine (four chunks), where a + # card's encoder session is reset and reused for its next chunk, for + # H.265, H.264 and AV1: every card that encodes builds one session + # and reuses it. Which cards take chunks is the speed gate's call + # and is not asserted here (the steps above assert the spread): for + # H.264 the A750 encodes a chunk faster than the decode delivers the + # next, so the gate rightly leaves the A380 out in some runs, and + # the job is faster for it (devbox: 7.7 s against 8.1-8.7 s). env: CARDS: ${{ steps.cards.outputs.cards }} run: | if [ "$CARDS" -lt 2 ]; then echo "skipped: $CARDS card on this host" | tee -a "$GITHUB_STEP_SUMMARY"; exit 0; fi bin/synth_clip src1080s.mp4 --size 1920x1080 --fps 30 --seconds 10 --noise 6 --bitrate 20000000 - python3 .github/scripts/speed_aware_timings.py $RIVET src1080.mp4 src1080s.mp4 --repeat 2 + python3 .github/scripts/speed_aware_timings.py $RIVET src1080.mp4 src1080s.mp4 --repeat 3 bin/synth_clip src1080x.mp4 --size 1920x1080 --fps 30 --seconds 70 --noise 6 --bitrate 20000000 for codec in h265 h264 av1; do $RIVET transcode src1080x.mp4 -o "reuse-$codec.mp4" --codec "$codec" --rung 1920x1080 --encode family:intel 2> "reuse-$codec.log" || { tail -60 "reuse-$codec.log"; exit 1; } - $ASSERT chunks "reuse-$codec.log" cards.json - grep -h "built vs reused" "reuse-$codec.log" | sed 's/.*sessions: //' + $ASSERT reuse "reuse-$codec.log" $CHECK "reuse-$codec.mp4" --codec "$codec" --frames 2100 done - name: One job per card, all at once # Independent jobs pinned to their own cards, side by side: each # stays on its card, every output decodes, and none is slowed by the - # others (at most 1.15x its time alone). + # others (at most 1.15x its time alone; devbox: 0.96-1.00x). Three + # rounds of alone-then-together; each card's best of three together + # against its best of three alone, every round in the summary. env: CARDS: ${{ steps.cards.outputs.cards }} GPUS: ${{ steps.cards.outputs.gpus }} @@ -464,18 +481,25 @@ jobs: if [ "$CARDS" -lt 2 ]; then echo "skipped: $CARDS card on this host" | tee -a "$GITHUB_STEP_SUMMARY"; exit 0; fi job() { $USAGE "usage-$2.json" -- $RIVET transcode src1080.mp4 -o "$2.mp4" --codec h265 --rung 1920x1080 \ --encode "gpu:$1" --decode "gpu:$1" 2> "$2.log"; } - for n in $GPUS; do job "$n" "alone-$n" || { tail -50 "alone-$n.log"; exit 1; }; done - pids="" - for n in $GPUS; do job "$n" "together-$n" & pids="$pids $!"; done - for pid in $pids; do wait "$pid" || { tail -50 together-*.log; exit 1; }; done wall() { python3 -c "import json,sys; print(json.load(open(sys.argv[1]))['wall_s'])" "$1"; } - for n in $GPUS; do - $ASSERT pinned "usage-alone-$n.json" cards.json "$n" - $ASSERT pinned "usage-together-$n.json" cards.json "$n" - for t in "alone-$n" "together-$n"; do - $CHECK "$t.mp4" --ref src1080.mp4 --codec h265 --frames 900 --size 1920x1080 --min-psnr 30 + for round in 1 2 3; do + for n in $GPUS; do job "$n" "alone-$n" || { tail -50 "alone-$n.log"; exit 1; }; done + pids="" + for n in $GPUS; do job "$n" "together-$n" & pids="$pids $!"; done + for pid in $pids; do wait "$pid" || { tail -50 together-*.log; exit 1; }; done + for n in $GPUS; do + $ASSERT pinned "usage-alone-$n.json" cards.json "$n" + $ASSERT pinned "usage-together-$n.json" cards.json "$n" + for t in "alone-$n" "together-$n"; do + [ "$round" = 1 ] && $CHECK "$t.mp4" --ref src1080.mp4 --codec h265 --frames 900 --size 1920x1080 --min-psnr 30 + wall "usage-$t.json" >> "walls-$t.txt" + done done - $ASSERT faster "card $n alongside the others vs alone" "$(wall "usage-together-$n.json")" "$(wall "usage-alone-$n.json")" 1.15 + done + for n in $GPUS; do + echo "card $n alone: $(paste -sd' ' "walls-alone-$n.txt") s; alongside the others: $(paste -sd' ' "walls-together-$n.txt") s" | tee -a "$GITHUB_STEP_SUMMARY" + $ASSERT faster "card $n alongside the others vs alone (best of 3)" \ + "$(sort -g "walls-together-$n.txt" | head -1)" "$(sort -g "walls-alone-$n.txt" | head -1)" 1.15 done - name: Per-card timings @@ -491,6 +515,8 @@ jobs: path: | *.log usage-*.json + walls-*.txt + speed_aware_timings.json scheduling.json devices.json cards.json From 7aeb88b49a0690978a42c5957cf6d4f87b3f10e0 Mon Sep 17 00:00:00 2001 From: Ely Erin Fox Date: Sun, 4 Oct 2026 16:14:21 -0700 Subject: [PATCH 2/2] CI gpu-scheduling: no concurrency group; the one-runner set and the device plugin already serialise it The group kept one pending scheduling job and cancelled the others ('a higher priority waiting request for safewords-runners-gpu-x2 exists'): develop run 37240219796 (#63) lost its scheduling job that way. --- .github/workflows/ci.yml | 34 +++++++++++++++------------------- 1 file changed, 15 insertions(+), 19 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3a2e126..4ef887e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -329,28 +329,24 @@ jobs: name: multi-GPU scheduling (both of devbox's Intel cards) # Its runner needs both of devbox's cards free at once, so it runs after # this run's per-codec jobs have let go of them (whatever they concluded: - # a codec failure must not hide the scheduling results), and one at a - # time across runs. The group is on this job, not the workflow: a - # workflow-level group would also queue every run's per-codec jobs behind - # the previous run, and GitHub cancels a queued member that a newer one - # replaces, so a burst of pushes would lose whole runs instead of only - # their scheduling job. A one-card job from another run can still hold a - # devbox card for a while; the pod then waits for it. + # a codec failure must not hide the scheduling results). A one-card job + # from another run can still hold a devbox card for a while; the pod + # then waits for it. # - # No other GPU work can share the cards while it runs, so no concurrency - # group with the per-card jobs is needed (and one would cost: a group - # runs one member and keeps one pending, cancelling the rest). The pod - # requests gpu.intel.com/i915: 2 on devbox, which has exactly two, and - # the device plugin runs with -shared-dev-num 1, so while it holds them - # no other pod (safewords-runners-gpu, or another org's GPU runners, - # all of which request i915) is given either card: per-card jobs queued - # meanwhile run on vulpes-zerda's A310 or wait. Other pods can still - # load devbox's CPU, which is what the interleaved timings are for. + # The cards are its alone while it runs, with no concurrency group: the + # pod requests gpu.intel.com/i915: 2 on devbox, which has exactly two, + # and the device plugin runs with -shared-dev-num 1, so no other pod + # (safewords-runners-gpu, or another org's GPU runners, all of which + # request i915) is given either card meanwhile; per-card jobs queued + # then run on vulpes-zerda's A310 or wait. The runner set has one + # runner, so scheduling jobs of several runs simply queue for it. A + # concurrency group (there was one) keeps only one pending member and + # cancels the rest ("a higher priority waiting request ... exists"), + # which lost the scheduling job of every develop push in a burst but + # the last. Other pods can still load devbox's CPU (it is also a CI + # node), which is what the interleaved best-of-3 timings are for. needs: [gpu-build, gpu-devices, gpu-codec] if: ${{ !cancelled() && github.event_name != 'pull_request' && needs.gpu-build.result == 'success' }} - concurrency: - group: safewords-runners-gpu-x2 - cancel-in-progress: false runs-on: safewords-runners-gpu-x2 env: NO_COLOR: "1"