diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6e3a787..cb12307 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -60,3 +60,41 @@ jobs: - name: Run agents-audit on this repo run: node packages/agents-audit-compat/dist/cli.js scan . --fail-on error + + # Compatibility gate. `agents-audit` is a frozen bridge, so anything touching + # its command surface, exit codes, output or exports is measured against the + # frozen pre-migration source rather than only against our own tests. + # + # Separate job: it clones and builds a second repository, which the unit-test + # matrix should not pay for, and it does not need the Node matrix. + parity: + name: Compatibility parity vs frozen source + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: pnpm/action-setup@v4 + + - uses: actions/setup-node@v4 + with: + node-version: 22 + cache: pnpm + + # The old side is pinned to an immutable commit, so the SHA is a perfect + # cache key — it can never go stale for the wrong reason. + - name: Restore frozen-source cache + uses: actions/cache@v4 + with: + path: .parity-cache + key: parity-e47eb1b8556c4f361db9a78190a2f36b400756e8-${{ runner.os }}-node22 + + - run: pnpm install --no-frozen-lockfile + + - name: agents-audit runtime parity + run: migration/parity-agents-audit-runtime.sh + + - name: agents-audit packed-artifact parity + run: migration/parity-agents-audit-pack.sh + + - name: DataHub adapter parity + run: node migration/parity-datahub-shim.mjs diff --git a/migration/parity-agents-audit-pack.sh b/migration/parity-agents-audit-pack.sh old mode 100644 new mode 100755 index 77878ab..41036df --- a/migration/parity-agents-audit-pack.sh +++ b/migration/parity-agents-audit-pack.sh @@ -39,6 +39,7 @@ fi echo echo "=== 3. IDENTITY FIELDS ===" +GATE=0 python3 - "$OUT/old/manifest.json" "$OUT/new/manifest.json" <<'PY' import json, sys o = json.load(open(sys.argv[1])); n = json.load(open(sys.argv[2])) @@ -50,6 +51,7 @@ for f in fields: ok += same print(f" {'OK ' if same else 'DIFF'} {f}: {json.dumps(o.get(f))}" + ("" if same else f" -> {json.dumps(n.get(f))}")) print(f" identity fields identical: {ok}/{total}") +identity_ok = (ok == total) print() print(" runtime dependencies:") od, nd = o.get("dependencies",{}), n.get("dependencies",{}) @@ -57,10 +59,25 @@ for k in sorted(set(od)|set(nd)): same = od.get(k) == nd.get(k) print(f" {'OK ' if same else 'DIFF'} {k}: {od.get(k)}" + ("" if same else f" -> {nd.get(k)}")) print(f" dependency surface identical: {od == nd}") + +# Gate on identity only. These eleven fields ARE the compatibility contract — +# a consumer resolves the package through them. Dependencies deliberately +# changed in META-247 (agents-audit now depends on @workspacejson/cli), and the +# tsup chunk filename is content-hashed, so neither is a compatibility signal. +if not identity_ok: + print("\n GATE: FAIL — a packed identity field changed. This breaks how") + print(" consumers resolve `agents-audit`. It is not a cosmetic difference.") + sys.exit(1) +print("\n GATE: PASS — all packed identity fields match the frozen source.") PY +GATE=$? echo echo "=== 4. TARBALL HASHES ===" echo " old sha256: $(cat "$OUT/old/tarball.sha256")" echo " new sha256: $(cat "$OUT/new/tarball.sha256")" echo " published agents-audit@0.4.4 sha256: c7d302901f7df8b4890eeb0b925ae40b8b90868c49aa87a5b6df52f3ae08df2c" + +# The Python block above is the gate; `set -e` is deliberately not used here, so +# propagate its result explicitly rather than exiting on the last echo. +exit "$GATE" diff --git a/migration/parity-agents-audit-runtime.sh b/migration/parity-agents-audit-runtime.sh old mode 100644 new mode 100755 index fc187bc..2f8387a --- a/migration/parity-agents-audit-runtime.sh +++ b/migration/parity-agents-audit-runtime.sh @@ -68,12 +68,18 @@ run_side () { # $1=side $2=case $3...=args if [ "$side" = "old" ]; then bin="$OLD_DIR/node_modules/.bin/agents-audit"; else bin="$NEW_DIR/node_modules/.bin/agents-audit"; fi local out exit out=$(cd "$dir" && "$bin" "$@" 2>&1); exit=$? - # Normalize volatile content: timestamps, uuids, absolute paths, durations, versions of node + # Normalize volatile content: anything that differs purely because the two + # sides cannot execute at the same instant — timestamps, uuids, absolute + # paths, durations. `temporalWeight` belongs here too: it is a time-decayed + # float in @workspacejson/rules, so a sub-millisecond gap between the old and + # new runs yields 1 vs 0.9999999998842592. That is engine nondeterminism, not + # a migration difference, and it made this check flake roughly 1 run in 7. out=$(printf '%s' "$out" \ | sed -E 's/[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9:.-]+Z?//g' \ | sed -E "s|$RUN/[A-Za-z0-9_-]+||g" \ | sed -E 's/[0-9]+(\.[0-9]+)?ms//g' \ | sed -E 's/"durationMs": [0-9]+/"durationMs": /g' \ + | sed -E 's/"temporalWeight": [0-9]+(\.[0-9]+)?/"temporalWeight": /g' \ | sed -E 's/[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}//g') printf '%s|%s' "$exit" "$out" printf '%s' "$out" > "$RUN/$case-$side.out" @@ -255,5 +261,74 @@ fi echo echo "==============================================================" echo " RESULT: $PASS passed, $FAIL failed (total $((PASS+FAIL)))" -if [ "$FAIL" -gt 0 ]; then printf ' FAILED: %s\n' "${FAILED[@]}"; fi +if [ "$FAIL" -gt 0 ]; then printf ' differs: %s\n' "${FAILED[@]}"; fi echo "==============================================================" + +# --- gate ------------------------------------------------------------------- +# Compare the set of differing checks against the ratified baseline. This is +# what makes the harness a gate rather than a report: it fails when the set +# CHANGES, in either direction. A harness that always exits 0 is not coverage. + +BASELINE="$PARITY_LIB_DIR/parity-expected-differences.txt" +EXPECTED=() +if [ -f "$BASELINE" ]; then + while IFS= read -r line || [ -n "$line" ]; do + line="${line%%#*}" + line="$(printf '%s' "$line" | sed 's/^[[:space:]]*//; s/[[:space:]]*$//')" + [ -n "$line" ] && EXPECTED+=("$line") + done < "$BASELINE" +else + echo "ERROR: missing baseline $BASELINE" >&2 + exit 1 +fi + +parity_contains () { # $1=needle; rest=haystack + local needle="$1"; shift + local item + for item in "$@"; do [ "$item" = "$needle" ] && return 0; done + return 1 +} + +UNEXPECTED=(); STALE=() +for actual in ${FAILED[@]+"${FAILED[@]}"}; do + parity_contains "$actual" ${EXPECTED[@]+"${EXPECTED[@]}"} || UNEXPECTED+=("$actual") +done +for want in ${EXPECTED[@]+"${EXPECTED[@]}"}; do + parity_contains "$want" ${FAILED[@]+"${FAILED[@]}"} || STALE+=("$want") +done + +echo +echo " GATE" +echo " ----" +echo " expected differences: ${#EXPECTED[@]} (migration/parity-expected-differences.txt)" +echo " observed differences: $FAIL" + +GATE=0 + +if [ "${#UNEXPECTED[@]}" -gt 0 ]; then + GATE=1 + echo + echo " REGRESSION — these checks differ and are NOT in the baseline:" + printf ' - %s\n' "${UNEXPECTED[@]}" + echo + echo " Either the change was unintended, or it is a deliberate decision that" + echo " needs a human ruling and a baseline entry citing the issue." +fi + +if [ "${#STALE[@]}" -gt 0 ]; then + GATE=1 + echo + echo " STALE BASELINE — these are listed as expected but now MATCH:" + printf ' - %s\n' "${STALE[@]}" + echo + echo " A ratified difference disappeared. Either a decision was reverted, or" + echo " the difference was resolved and the baseline entry should be removed." +fi + +if [ "$GATE" -eq 0 ]; then + echo + echo " PASS — observed differences exactly match the ratified baseline." +fi +echo "==============================================================" + +exit "$GATE" diff --git a/migration/parity-expected-differences.txt b/migration/parity-expected-differences.txt new file mode 100644 index 0000000..5f667f3 --- /dev/null +++ b/migration/parity-expected-differences.txt @@ -0,0 +1,32 @@ +# Expected parity differences — the compatibility baseline. +# +# Each entry is a parity check that is KNOWN to differ between the frozen +# pre-migration source and this repository, together with the ratified decision +# that authorizes it. Anything not listed here must match. +# +# The gate is exact in BOTH directions: +# +# a check fails that is not listed -> REGRESSION. Something changed that +# was supposed to stay compatible. +# a listed check starts passing -> STALE BASELINE. A ratified decision +# was reverted, or the difference was +# resolved and this file is now lying. +# +# Do not add a line here to make CI green. Adding a line is a claim that a +# human decided the difference is correct, and it should cite the issue that +# decided it. +# +# Format: one check label per line, matching the harness's FAILED label exactly. +# Everything after # is a comment. + +# META-236 ratified that vendor promotion never enters the neutral package and +# is removed or made opt-in in the compatibility package. `agents-audit scan` +# now points at the real producer command instead of vreko.dev. Exit code is +# unchanged and `scan --json` remains byte-identical, so machine-readable +# consumers are unaffected. +agents-audit scan . + +# Same ruling. `renderMissingArtifactNotice` was added alongside the retained +# `renderVrekoUpsell`, so the export set grew. Purely additive — all nine +# historical exports are still present. A removal here would be a real break. +exports