From 294a6af29aaa2832bb89634835a29a329c812e7c Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 13 Jun 2026 15:09:09 +0000 Subject: [PATCH] Add automated R-parity check-and-fix workflow Adds the "fix if required" half of the fidelity chain alongside the existing detection workflow, with hard guardrails and a human-merge policy. - .github/workflows/parity-autofix.yml: sets up Python + live R at the recorded R commit, reproduces the divergence via the live-R parity runner, and only when behavior genuinely drifted hands a structured report to anthropics/claude-code-action. The agent may edit src/nns/** only, must classify the root cause (Python port bug -> fix; R behavior change or native change -> escalate, never touch extern/NNS-core, tools/NNS, or the cache), must verify against live R, and opens a SEPARATE parity-correction PR. If live R cannot be installed, it opens an escalation PR instead of guessing. Never auto-merges. - inspect-r-api-update.yml: chains to parity-autofix via an nns-parity-divergence repository_dispatch (gated on the optional PARITY_DISPATCH_TOKEN secret; prints the manual trigger otherwise). - docs/parity_autofix.md: setup (ANTHROPIC_API_KEY via /install-github-app, optional PARITY_APP_TOKEN so fix PRs trigger CI, optional PARITY_DISPATCH_TOKEN for auto-chaining), guardrails, and human-merge policy. Automation only; no runtime/package behavior changes. --- .github/workflows/inspect-r-api-update.yml | 34 ++++ .github/workflows/parity-autofix.yml | 213 +++++++++++++++++++++ docs/parity_autofix.md | 90 +++++++++ 3 files changed, 337 insertions(+) create mode 100644 .github/workflows/parity-autofix.yml create mode 100644 docs/parity_autofix.md diff --git a/.github/workflows/inspect-r-api-update.yml b/.github/workflows/inspect-r-api-update.yml index 04b233d7..4aa6f0f3 100644 --- a/.github/workflows/inspect-r-api-update.yml +++ b/.github/workflows/inspect-r-api-update.yml @@ -127,6 +127,40 @@ jobs: echo "- DESCRIPTION changed: \`${{ steps.payload.outputs.description_changed }}\`" } >> sync/last_live_r_parity_report.md + - name: Dispatch parity autofix for live-R verification + if: steps.payload.outputs.fresh_cache != 'true' + shell: bash + env: + DISPATCH_TOKEN: ${{ secrets.PARITY_DISPATCH_TOKEN }} + run: | + set -euo pipefail + # The cache-based gates below only prove parity against the committed + # cache. The parity-autofix workflow owns live-R verification (it sets + # up R) and opens a separate, human-reviewed fix PR if behavior drifted. + tests=$(jq '.parity_tests | length' sync/last_r_api_plan.json) + if [ "${tests}" -eq 0 ]; then + echo "No mapped parity tests for this change; not dispatching parity autofix." + exit 0 + fi + if [ -z "${DISPATCH_TOKEN:-}" ]; then + echo "PARITY_DISPATCH_TOKEN not set; skipping auto-chain to parity-autofix." + echo "Run parity-autofix manually with r_commit=${{ steps.payload.outputs.r_commit }} r_version=${{ steps.payload.outputs.r_version }}." + exit 0 + fi + payload=$(jq -n \ + --arg rc "${{ steps.payload.outputs.r_commit }}" \ + --arg rv "${{ steps.payload.outputs.r_version }}" \ + --arg rh "${{ steps.payload.outputs.r_src_tree_hash }}" \ + --slurpfile cf changed_files.json \ + '{event_type:"nns-parity-divergence", client_payload:{r_commit:$rc, r_version:$rv, r_src_tree_hash:$rh, changed_files:($cf[0] // [])}}') + curl -sSf -X POST \ + -H "Accept: application/vnd.github+json" \ + -H "Authorization: Bearer ${DISPATCH_TOKEN}" \ + "https://api.github.com/repos/${{ github.repository }}/dispatches" \ + -d "${payload}" + echo "Dispatched nns-parity-divergence for live-R parity autofix." + + - name: Run standard gates if no fresh cache was required if: steps.payload.outputs.fresh_cache != 'true' run: | diff --git a/.github/workflows/parity-autofix.yml b/.github/workflows/parity-autofix.yml new file mode 100644 index 00000000..67a77cee --- /dev/null +++ b/.github/workflows/parity-autofix.yml @@ -0,0 +1,213 @@ +name: Parity autofix + +# Automated R-behavior parity fix loop for NNS-python. +# +# This is the "fix if required" half of the fidelity chain. The detection half +# lives in inspect-r-api-update.yml. When live-R parity for a changed R API +# diverges, this workflow re-derives the divergence against live R and hands a +# structured report to an agent (anthropics/claude-code-action) that may draft a +# fix to src/nns/** ONLY, then opens a SEPARATE parity-correction PR. +# +# Human-merge policy: this workflow never merges. A human reviews every fix PR. +# +# Trigger model: +# * repository_dispatch [nns-parity-divergence] - emitted by +# inspect-r-api-update.yml when live parity diverges (needs a PAT, see +# docs/parity_autofix.md), or by upstream automation. +# * workflow_dispatch - manual fallback. + +on: + repository_dispatch: + types: [nns-parity-divergence] + workflow_dispatch: + inputs: + r_commit: + required: true + type: string + r_version: + required: true + type: string + r_src_tree_hash: + required: false + default: "unknown" + type: string + model: + description: Claude model id for the autofix agent + required: false + default: claude-sonnet-4-6 + type: string + +permissions: + contents: write + pull-requests: write + issues: write + +jobs: + parity-autofix: + runs-on: ubuntu-latest + steps: + - name: Check out NNS-python + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Resolve payload + id: payload + shell: bash + run: | + set -euo pipefail + if [ "${{ github.event_name }}" = "repository_dispatch" ]; then + echo "r_commit=${{ github.event.client_payload.r_commit }}" >> "$GITHUB_OUTPUT" + echo "r_version=${{ github.event.client_payload.r_version }}" >> "$GITHUB_OUTPUT" + echo "r_src_tree_hash=${{ github.event.client_payload.r_src_tree_hash }}" >> "$GITHUB_OUTPUT" + echo "model=claude-sonnet-4-6" >> "$GITHUB_OUTPUT" + echo '${{ toJson(github.event.client_payload.changed_files) }}' > changed_files.json + else + echo "r_commit=${{ inputs.r_commit }}" >> "$GITHUB_OUTPUT" + echo "r_version=${{ inputs.r_version }}" >> "$GITHUB_OUTPUT" + echo "r_src_tree_hash=${{ inputs.r_src_tree_hash }}" >> "$GITHUB_OUTPUT" + echo "model=${{ inputs.model }}" >> "$GITHUB_OUTPUT" + echo '[]' > changed_files.json + fi + + - name: Check out upstream R NNS + uses: actions/checkout@v4 + with: + repository: OVVO-Financial/NNS + ref: ${{ steps.payload.outputs.r_commit }} + path: upstream/NNS + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.11" + + - name: Set up R + uses: r-lib/actions/setup-r@v2 + with: + r-version: "release" + + - name: Install Python build and test tools + run: | + python -m pip install -U pip + python -m pip install build scikit-build-core nanobind pytest ruff mypy numpy scipy + python -m pip install hypothesis pytest-benchmark pytest-xdist + + - name: Install package editable + run: python -m pip install -e . --force-reinstall + + - name: Install R dependencies for NNS (best effort) + uses: r-lib/actions/setup-r-dependencies@v2 + continue-on-error: true + with: + working-directory: tools/NNS + dependencies: '"hard"' + extra-packages: any::jsonlite + + - name: Plan R API parity review + run: | + python scripts/plan_r_api_parity_review.py \ + --changed-files-json changed_files.json \ + --map sync/r_api_map.json \ + --out sync/last_r_api_inspection.md \ + --json-out sync/last_r_api_plan.json + + - name: Install live R NNS from upstream checkout (best effort) + id: r_install + continue-on-error: true + run: | + # install_local_r_nns.py installs from the vendored tools/NNS by + # default; the upstream checkout is the recorded truth for this commit. + python scripts/install_local_r_nns.py + + - name: Run live R parity to reproduce divergence + id: live_parity + continue-on-error: true + run: | + python scripts/run_live_r_parity_for_changed_api.py \ + --plan sync/last_r_api_plan.json \ + --r-checkout upstream/NNS \ + --skip-install \ + --out sync/last_live_r_parity_report.md + + - name: Decide whether a fix is required + id: gate + shell: bash + run: | + outcome="${{ steps.live_parity.outcome }}" + r_ok="${{ steps.r_install.outcome }}" + echo "live_parity_outcome=${outcome}" >> "$GITHUB_OUTPUT" + echo "r_install_outcome=${r_ok}" >> "$GITHUB_OUTPUT" + if [ "${outcome}" = "success" ]; then + echo "needs_fix=false" >> "$GITHUB_OUTPUT" + echo "Live R parity passed; no fix required." + elif [ "${r_ok}" != "success" ]; then + # Cannot verify against live R -> do NOT let the agent guess a fix. + echo "needs_fix=false" >> "$GITHUB_OUTPUT" + echo "escalate=true" >> "$GITHUB_OUTPUT" + echo "Live R install failed; escalating instead of auto-fixing." + else + echo "needs_fix=true" >> "$GITHUB_OUTPUT" + echo "Live R parity diverged; dispatching autofix agent." + fi + + - name: Run parity-fix agent + if: steps.gate.outputs.needs_fix == 'true' + uses: anthropics/claude-code-action@v1 + with: + anthropic_api_key: ${{ secrets.ANTHROPIC_API_KEY }} + # Use the GitHub App token if configured so the resulting PR triggers + # native-backend-ci (PRs opened with the default GITHUB_TOKEN do not). + github_token: ${{ secrets.PARITY_APP_TOKEN || github.token }} + prompt: | + You are fixing an R-behavior parity divergence in OVVO-Financial/NNS-python. + + Context: + - R behavioral truth: OVVO-Financial/NNS at commit ${{ steps.payload.outputs.r_commit }} (version ${{ steps.payload.outputs.r_version }}), checked out at upstream/NNS and installed as live R. + - The divergence report is at sync/last_live_r_parity_report.md. The parity plan is at sync/last_r_api_plan.json and the R->Python map is at sync/r_api_map.json. + + Hard rules (do not violate): + 1. Edit files under src/nns/** ONLY. NEVER edit extern/NNS-core/**, tools/NNS/**, tests/_r_cache.json, or any test/cache file to make a check pass. + 2. Classify the root cause first and act accordingly: + - Python port bug (a wrapper/default/return-shape/algorithm in src/nns/** drifted from R): fix it in src/nns/**. + - R changed behavior (the divergence is because upstream R itself changed): DO NOT edit code to chase a cache value. Stop and open the PR/issue as an escalation describing the change; cache regeneration is a separate, reviewed step. + - Native kernel change (the difference originates in C++ src/** / NNS-core): DO NOT edit. Native code enters Python only through accepted NNS-core commits. Stop and escalate. + 3. Verify any fix against LIVE R, not the committed cache. The fix must keep ALL of these green: + - python -m pytest -q -n 0 tests/invariants + - the mapped parity tests in the plan (run with -n 0), against live R + - ruff check . ; mypy ; python -m build + Do not weaken tests or tolerances. + 4. Keep the change minimal and in the style of the surrounding code. + + Deliverable: + - Create a NEW branch named parity-fix/${{ steps.payload.outputs.r_commit }} and open a SEPARATE parity-correction pull request into the default branch. Do NOT merge it. + - The PR body MUST contain, for each divergence: the function, the arguments, the R output, the Python output, the first divergent intermediate, the affected source files, and the proposed fix (or, for escalations, why no code fix was made and what must happen upstream / in NNS-core). + - Title: "Parity fix: R NNS ${{ steps.payload.outputs.r_commit }}". + claude_args: | + --model ${{ steps.payload.outputs.model }} + --max-turns 40 + --allowedTools Edit,Read,Write,Glob,Grep,Bash(python:*),Bash(python -m pytest:*),Bash(ruff:*),Bash(mypy:*),Bash(pip:*),Bash(Rscript:*),Bash(git:*),Bash(gh:*) + --disallowedTools Bash(rm:*) + + - name: Escalate when live R could not verify + if: steps.gate.outputs.escalate == 'true' + uses: peter-evans/create-pull-request@v6 + with: + branch: parity-escalation-${{ steps.payload.outputs.r_commit }} + title: Parity escalation (live R unavailable) ${{ steps.payload.outputs.r_commit }} + body: | + Live R NNS could not be installed in CI for R commit + `${{ steps.payload.outputs.r_commit }}` (version + `${{ steps.payload.outputs.r_version }}`), so a parity fix was NOT + attempted automatically — fidelity must be verified against live R, + never guessed. + + Reports: + - `sync/last_r_api_inspection.md` + - `sync/last_r_api_plan.json` + - `sync/last_live_r_parity_report.md` + + Action required: run `scripts/install_local_r_nns.py` in an + environment with R available, reproduce the divergence, and apply a + reviewed parity fix. Native differences must route through NNS-core. + commit-message: Parity escalation ${{ steps.payload.outputs.r_commit }} diff --git a/docs/parity_autofix.md b/docs/parity_autofix.md new file mode 100644 index 00000000..89d07fa1 --- /dev/null +++ b/docs/parity_autofix.md @@ -0,0 +1,90 @@ +# Automated parity check and fix + +`NNS-python` automates both halves of R-behavior fidelity: + +1. **Check** — when upstream `OVVO-Financial/NNS` changes an R API, the + `inspect-r-api-update` workflow plans which Python modules / parity tests are + affected and records a report. +2. **Fix if required** — the `parity-autofix` workflow verifies the affected + public behavior against **live R** at the recorded R commit and, if behavior + drifted, hands a structured divergence report to an agent + (`anthropics/claude-code-action`) that drafts a fix and opens a **separate, + human-reviewed** parity-correction PR. + +```text +NNS R API change + -> inspect-r-api-update.yml (plan + cache gates + inspection PR) + -> dispatch nns-parity-divergence + -> parity-autofix.yml (live-R verify -> fix or escalate -> PR) + -> human review + merge +``` + +## What the agent may and may not do + +Hard rules enforced in the agent prompt and the workflow gate: + +* Edit **`src/nns/**` only**. Never edit `extern/NNS-core/**`, `tools/NNS/**`, + or `tests/_r_cache.json` to make a check pass. +* Classify the root cause and act accordingly: + * **Python port bug** → fix in `src/nns/**`. + * **R changed behavior** → do not chase a cache value; escalate (cache + regeneration is a separate, reviewed step). + * **Native kernel change** → do not edit; native code enters Python only + through accepted `NNS-core` commits. Escalate. +* Verify against **live R**, not the committed cache. If live R cannot be + installed in CI, the workflow opens an **escalation PR** instead of letting + the agent guess a fix. +* **Human-merge only.** The workflow never merges; every fix PR is reviewed. + +## Required setup + +### 1. Anthropic credentials (required) + +The autofix agent needs an API key exposed as the `ANTHROPIC_API_KEY` repo +secret. + +Easiest path — from Claude Code, run: + +```text +/install-github-app +``` + +This installs the official Claude GitHub App, adds the `ANTHROPIC_API_KEY` +secret, and (optionally) scaffolds a workflow. You must be a repo admin. + +Manual path: + +1. Repo **Settings → Secrets and variables → Actions → New repository secret**. +2. Name `ANTHROPIC_API_KEY`, value = your key from `console.anthropic.com`. + +Bedrock / Vertex are supported via the action's `use_bedrock` / `use_vertex` +inputs with OIDC (`id-token: write`) instead of a static key; see the +[cloud providers docs](https://github.com/anthropics/claude-code-action/blob/main/docs/cloud-providers.md). + +### 2. Fix-PR token so CI runs on the fix (recommended) + +A PR opened with the default `GITHUB_TOKEN` does **not** trigger other workflows +(GitHub's recursion guard), so `native-backend-ci` would not run on the fix PR. +To get CI on fix PRs, let the agent open the PR with a token that does trigger +workflows: + +* Installing the **GitHub App** (above) already provides this, or +* add a fine-grained PAT / GitHub App token as the `PARITY_APP_TOKEN` secret; + the workflow passes it to the action as `github_token`. If unset, it falls + back to `github.token`. + +### 3. Auto-chain token (optional) + +`inspect-r-api-update` chains to `parity-autofix` via a `repository_dispatch`, +which the default `GITHUB_TOKEN` cannot emit. To enable the automatic chain, add +a PAT with `contents: write` (or `repo`) scope as the `PARITY_DISPATCH_TOKEN` +secret. Without it, `inspect-r-api-update` prints the manual trigger command and +you run `parity-autofix` yourself via **workflow_dispatch** (inputs: `r_commit`, +`r_version`). + +## Model + +The agent model defaults to `claude-sonnet-4-6` and is overridable via the +`workflow_dispatch` `model` input. Use a more capable model (e.g. +`claude-opus-4-8`) for harder divergences. Every fix is still verified by the +full gate set against live R and reviewed by a human before merge.