diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml new file mode 100644 index 0000000..6156725 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -0,0 +1,67 @@ +name: Bug report +description: Report a reproducible problem using synthetic or sanitized data. +title: "[Bug]: " +labels: [bug] +body: + - type: markdown + attributes: + value: | + Do not attach private datasets, prompts, responses, credentials, model weights, or unsanitized Regression Cards. Use [private vulnerability reporting](https://github.com/Labeeb2339/sftguard/security/advisories/new) for a security or privacy failure. + - type: input + id: version + attributes: + label: SFTGuard version or commit + placeholder: 0.1.0 or a full Git commit + validations: + required: true + - type: dropdown + id: component + attributes: + label: Component + options: + - Python CLI or library + - Dataset audit + - Tokenizer or loss mask + - Regression Card or gate + - Static report viewer + - CI or packaging + - Documentation + validations: + required: true + - type: input + id: environment + attributes: + label: Sanitized environment + description: Operating system, Python or Node version, and relevant package versions. Remove usernames and private paths. + placeholder: Windows 11, Python 3.12.4 + validations: + required: true + - type: textarea + id: steps + attributes: + label: Reproduction steps + description: Use the smallest synthetic fixture that demonstrates the problem. + validations: + required: true + - type: textarea + id: expected + attributes: + label: Expected behavior + validations: + required: true + - type: textarea + id: actual + attributes: + label: Actual behavior + description: Quote only sanitized output; do not paste a real record or secret candidate. + validations: + required: true + - type: checkboxes + id: privacy + attributes: + label: Privacy check + options: + - label: I used synthetic or sanitized data and removed credentials, raw records, local usernames, and private paths. + required: true + - label: This report is not a vulnerability or private-data exposure that should be submitted privately. + required: true diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 0000000..4ec9197 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1,8 @@ +blank_issues_enabled: false +contact_links: + - name: Private security report + url: https://github.com/Labeeb2339/sftguard/security/advisories/new + about: Report vulnerabilities, secret exposure, or private-data leakage without opening a public issue. + - name: Safe sharing guidance + url: https://github.com/Labeeb2339/sftguard/blob/main/docs/SHARING.md + about: Check what can be included safely in a public issue or reproduction. diff --git a/.github/ISSUE_TEMPLATE/feature_request.yml b/.github/ISSUE_TEMPLATE/feature_request.yml new file mode 100644 index 0000000..142f76e --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.yml @@ -0,0 +1,46 @@ +name: Feature request +description: Propose a bounded addition to SFTGuard's correctness or regression contract. +title: "[Feature]: " +labels: [enhancement] +body: + - type: markdown + attributes: + value: | + SFTGuard does not train models, predict final fine-tuning performance, certify safety, or automatically publish reports. Explain how the proposal fits the local-first, fail-closed boundary. + - type: textarea + id: problem + attributes: + label: Problem and user evidence + description: Describe the observable pipeline failure or missing release evidence. + validations: + required: true + - type: textarea + id: proposal + attributes: + label: Proposed behavior + description: Include the expected PASS, FAIL, and ABSTAIN behavior. + validations: + required: true + - type: textarea + id: validation + attributes: + label: Validation plan + description: State the synthetic fixture, negative control, and measurable gate. + validations: + required: true + - type: textarea + id: privacy + attributes: + label: Privacy and sharing impact + description: Identify any new input, artifact field, network access, or disclosure risk. + validations: + required: true + - type: checkboxes + id: boundaries + attributes: + label: Boundary check + options: + - label: The proposal does not require posting private data or reports to an external service. + required: true + - label: Missing or untrustworthy evidence will not be converted into PASS. + required: true diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..8cb70df --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,37 @@ +version: 2 +updates: + - package-ecosystem: pip + directory: / + schedule: + interval: weekly + day: monday + time: "03:00" + timezone: Asia/Kuala_Lumpur + open-pull-requests-limit: 5 + labels: [dependencies, python] + + - package-ecosystem: npm + directory: /web + schedule: + interval: weekly + day: monday + time: "03:15" + timezone: Asia/Kuala_Lumpur + open-pull-requests-limit: 5 + labels: [dependencies, web] + groups: + npm-minor-and-patch: + update-types: [minor, patch] + + - package-ecosystem: github-actions + directory: / + schedule: + interval: weekly + day: monday + time: "03:30" + timezone: Asia/Kuala_Lumpur + open-pull-requests-limit: 5 + labels: [dependencies, github-actions] + groups: + actions-minor-and-patch: + update-types: [minor, patch] diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 0000000..a4dc474 --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,24 @@ +## Change + +Describe the user-visible behavior and why it belongs inside SFTGuard's bounded +correctness or regression contract. + +## Evidence + +List the deterministic tests, synthetic fixtures, commands, and observed +results. Do not attach private datasets or an unsanitized report. + +## Privacy and security + +Describe new inputs, outputs, paths, dependencies, or network behavior. State +whether any artifact schema or safe-sharing guidance changed. + +## Checklist + +- [ ] Python tests pass on the supported versions, when applicable. +- [ ] The Node 22 viewer checks and production build pass, when applicable. +- [ ] New behavior has a deterministic synthetic test and negative control. +- [ ] `PASS`, `FAIL`, and `ABSTAIN` remain distinct and fail closed. +- [ ] Artifacts and fixtures contain no raw private records, credentials, or model weights. +- [ ] No telemetry, automatic upload, or automatic posting was introduced. +- [ ] Documentation and the frozen evidence boundary remain accurate. diff --git a/.github/release.yml b/.github/release.yml new file mode 100644 index 0000000..7f88699 --- /dev/null +++ b/.github/release.yml @@ -0,0 +1,24 @@ +changelog: + exclude: + labels: + - skip-changelog + categories: + - title: Security and privacy + labels: + - security + - privacy + - title: Detectors and regression gates + labels: + - detector + - regression + - title: Viewer and reports + labels: + - web + - reporting + - title: Documentation and maintenance + labels: + - documentation + - dependencies + - title: Other changes + labels: + - "*" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..a14f0ea --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,79 @@ +name: CI + +on: + push: + branches: [main] + pull_request: + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: ci-${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + python: + name: Python ${{ matrix.python-version }} / ${{ matrix.os }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, windows-latest] + python-version: ["3.11", "3.12"] + steps: + - uses: actions/checkout@v7 + with: + fetch-depth: 0 + - uses: actions/setup-python@v6 + with: + python-version: ${{ matrix.python-version }} + cache: pip + cache-dependency-path: pyproject.toml + - name: Install package and test tools + run: >- + python -m pip install --upgrade pip + && python -m pip install -e ".[dev]" build + - name: Check installed dependencies + run: python -m pip check + - name: Compile Python sources + run: python -m compileall -q src tests scripts + - name: Lint and format-check Python + run: >- + python -m ruff check src tests/python scripts + && python -m ruff format --check src tests/python scripts + - name: Run Python tests + run: python -m pytest + - name: Verify checked-in evidence integrity + if: ${{ hashFiles('evidence/v0.1.0-confirmatory.json') != '' }} + run: python scripts/confirmatory_evidence.py verify --integrity-only + - name: Replay evidence from the implementation commit + if: ${{ matrix.python-version == '3.11' && hashFiles('evidence/v0.1.0-confirmatory.json') != '' }} + run: python scripts/confirmatory_evidence.py verify + - name: Smoke-test the CLI + run: sftguard --help + - name: Build source and wheel distributions + run: python -m build + + web: + name: Node 22 / ${{ matrix.os }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, windows-latest] + defaults: + run: + working-directory: web + steps: + - uses: actions/checkout@v7 + - uses: actions/setup-node@v7 + with: + node-version: "22" + cache: npm + cache-dependency-path: web/package-lock.json + - name: Install locked dependencies + run: npm ci + - name: Type-check, test, and build the viewer + run: npm run check diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml new file mode 100644 index 0000000..59d2f37 --- /dev/null +++ b/.github/workflows/codeql.yml @@ -0,0 +1,34 @@ +name: CodeQL + +on: + push: + branches: [main] + pull_request: + branches: [main] + schedule: + - cron: "37 4 * * 4" + workflow_dispatch: + +permissions: + contents: read + security-events: write + +jobs: + analyze: + name: Analyze ${{ matrix.language }} + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + language: [python, javascript-typescript] + steps: + - uses: actions/checkout@v7 + - name: Initialize CodeQL + uses: github/codeql-action/init@v4 + with: + languages: ${{ matrix.language }} + build-mode: none + - name: Analyze + uses: github/codeql-action/analyze@v4 + with: + category: /language:${{ matrix.language }} diff --git a/.github/workflows/dependency-audit.yml b/.github/workflows/dependency-audit.yml new file mode 100644 index 0000000..5d23b43 --- /dev/null +++ b/.github/workflows/dependency-audit.yml @@ -0,0 +1,58 @@ +name: Dependency audit + +on: + push: + branches: [main] + paths: + - pyproject.toml + - web/package.json + - web/package-lock.json + - .github/workflows/dependency-audit.yml + pull_request: + paths: + - pyproject.toml + - web/package.json + - web/package-lock.json + - .github/workflows/dependency-audit.yml + schedule: + - cron: "19 3 * * 1" + workflow_dispatch: + +permissions: + contents: read + +jobs: + python: + name: Python dependency audit + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + - uses: actions/setup-python@v6 + with: + python-version: "3.12" + cache: pip + cache-dependency-path: pyproject.toml + - name: Install the project auditor + run: >- + python -m pip install --upgrade pip + && python -m pip install pip-audit + - name: Audit declared project dependencies + run: python -m pip_audit --strict . + + web: + name: npm dependency audit + runs-on: ubuntu-latest + defaults: + run: + working-directory: web + steps: + - uses: actions/checkout@v7 + - uses: actions/setup-node@v7 + with: + node-version: "22" + cache: npm + cache-dependency-path: web/package-lock.json + - name: Install locked dependencies + run: npm ci + - name: Audit production and development dependencies + run: npm audit --audit-level=high diff --git a/.github/workflows/pages.yml b/.github/workflows/pages.yml new file mode 100644 index 0000000..5c91abc --- /dev/null +++ b/.github/workflows/pages.yml @@ -0,0 +1,56 @@ +name: Deploy static report viewer + +on: + push: + branches: [main] + paths: + - web/** + - evidence/** + - .github/workflows/pages.yml + workflow_dispatch: + +permissions: + contents: read + pages: write + id-token: write + +concurrency: + group: pages + cancel-in-progress: false + +jobs: + build: + name: Build Pages artifact + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + - name: Read Pages configuration + id: pages + uses: actions/configure-pages@v6 + - uses: actions/setup-node@v7 + with: + node-version: "22" + cache: npm + cache-dependency-path: web/package-lock.json + - name: Install locked dependencies + working-directory: web + run: npm ci + - name: Build for the repository Pages base path + working-directory: web + run: npm run build -- --base "${{ steps.pages.outputs.base_path }}/" + - name: Upload static site only + uses: actions/upload-pages-artifact@v5 + with: + path: web/dist + + deploy: + name: Deploy to GitHub Pages + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} + runs-on: ubuntu-latest + needs: build + steps: + - name: Deploy Pages artifact + id: deployment + uses: actions/deploy-pages@v5 diff --git a/.gitignore b/.gitignore index 975f2d6..11f206f 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,8 @@ __pycache__/ *.py[cod] *.egg-info/ .pytest_cache/ +.pytest-tmp/ +.pytest-tmp-*/ .ruff_cache/ .coverage htmlcov/ @@ -12,6 +14,7 @@ build/ web/node_modules/ web/dist/ web/.vite/ +*.tsbuildinfo .env .env.* diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..745ccdd --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,26 @@ +# Changelog + +All notable changes to SFTGuard are documented here. The project follows +[Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## [Unreleased] + +## [0.1.0] - 2026-07-19 + +### Added + +- Preregistered v0.1 product and fault-injection protocols. +- Dependency-light `audit`, `gate`, `benchmark`, and `demo` CLI workflows. +- Bounded JSON/JSONL parsing, redacted data findings, and exported mask checks. +- Paired target/retention Regression Cards with deterministic seed-cluster + bootstrap intervals and a fixed work budget. +- One-shot, fresh-process confirmatory evidence generation and replay tooling. +- Checked-in v0.1 confirmatory evidence: 270/270 known synthetic faults + detected, 0/30 clean controls flagged, and byte-identical replay from the + frozen implementation commit. +- Static browser-local report viewer for native SFTGuard artifacts. +- Cross-platform CI, CodeQL, dependency audits, Pages deployment, community + templates, citation metadata, and security documentation. + +[Unreleased]: https://github.com/Labeeb2339/sftguard/compare/v0.1.0...HEAD +[0.1.0]: https://github.com/Labeeb2339/sftguard/releases/tag/v0.1.0 diff --git a/CITATION.cff b/CITATION.cff new file mode 100644 index 0000000..96ce0c1 --- /dev/null +++ b/CITATION.cff @@ -0,0 +1,25 @@ +cff-version: 1.2.0 +message: >- + If SFTGuard supports your work, cite the software and the exact released + version or commit used. +title: SFTGuard +type: software +authors: + - given-names: Muhammad Labeeb Aryan + name-particle: bin + family-names: Mohd Lokman +repository-code: https://github.com/Labeeb2339/sftguard +url: https://github.com/Labeeb2339/sftguard +license: Apache-2.0 +version: 0.1.0 +date-released: 2026-07-19 +abstract: >- + SFTGuard is a local-first correctness and regression gate for supervised + fine-tuning pipelines. It audits bounded pipeline evidence and emits + reproducible PASS, FAIL, or ABSTAIN artifacts without claiming to train, + predict, or certify a model. +keywords: + - supervised fine-tuning + - regression testing + - dataset auditing + - reproducibility diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 0000000..23ef9ea --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,50 @@ +# Code of conduct + +## Our pledge + +We pledge to make participation in SFTGuard a harassment-free experience for +everyone, regardless of age, body size, visible or invisible disability, +ethnicity, sex characteristics, gender identity and expression, level of +experience, education, socioeconomic status, nationality, personal appearance, +race, caste, color, religion, or sexual identity and orientation. + +We will act and interact in ways that contribute to an open, welcoming, +diverse, inclusive, and healthy community. + +## Expected behavior + +Examples of behavior that supports the community include: + +- demonstrating empathy and kindness; +- respecting differing opinions and experiences; +- giving and accepting constructive feedback; +- taking responsibility, apologizing, and learning from mistakes; +- focusing on what is best for the project and community. + +Unacceptable behavior includes harassment, sexualized language or attention, +trolling or insulting comments, threats, sustained disruption, publishing +someone else's private information, and other conduct that would reasonably be +considered inappropriate in a professional setting. + +## Enforcement + +Maintainers may remove, edit, or reject contributions and communication that do +not follow this code, and may temporarily or permanently restrict participation +for harmful behavior. Enforcement decisions should be proportionate, private +where practical, and focused on community safety. + +Do not place personal details in a public issue. Use private contact information +published by the repository owner when available. If no private project contact +is available, use [GitHub Support's abuse-reporting channel](https://support.github.com/contact/report-abuse) +rather than publishing sensitive allegations. + +## Scope + +This code applies in project spaces and when an individual is officially +representing the project in public spaces. + +## Attribution + +This policy is adapted from the +[Contributor Covenant, version 2.1](https://www.contributor-covenant.org/version/2/1/code_of_conduct.html), +available under the [CC BY 4.0 license](https://creativecommons.org/licenses/by/4.0/). diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..f6416b2 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,66 @@ +# Contributing to SFTGuard + +SFTGuard welcomes small, reviewable contributions that preserve its +fail-closed and local-first boundaries. Before beginning a large change, open a +feature request describing the intended evidence and failure behavior. + +## Privacy first + +Use synthetic fixtures in issues, pull requests, tests, and screenshots. Never +submit private training data, prompts, responses, access tokens, model weights, +or reports containing them. Redact local usernames and absolute paths. Security +and privacy vulnerabilities belong in a private report under +[SECURITY.md](SECURITY.md), not a public issue. + +SFTGuard does not automatically publish reports or post results to external +services. Contributions must not introduce telemetry, remote report processing, +or automatic uploads without an explicit design review and an opt-in boundary. + +## Development setup + +Python 3.11 or 3.12 and Node.js 22 are supported for development. + +```bash +python -m venv .venv +python -m pip install --upgrade pip +python -m pip install -e ".[dev]" +python -m pytest +``` + +Build and test the static viewer separately: + +```bash +cd web +npm ci +npm run check +``` + +The repository CI is the authority for the complete supported-platform matrix. +If a command changes, update the workflow and this file in the same pull +request. + +## Change requirements + +- Keep input parsing bounded and avoid reading entire untrusted files when a + streaming implementation is possible. +- Never construct shell commands from dataset values. +- Preserve `PASS`, `FAIL`, and `ABSTAIN` as distinct outcomes. Missing evidence + must not become a pass. +- Keep public artifacts free of raw prompt, response, and injected-secret text. +- Add deterministic tests for every detector, gate, parser, and schema change. +- Document new report fields and state whether they are safe to share. +- Do not describe synthetic validation as a real-world safety guarantee. + +The confirmatory seed range in +[docs/FAULT_INJECTION_PROTOCOL.md](docs/FAULT_INJECTION_PROTOCOL.md) is frozen. +Do not tune rules or thresholds after inspecting confirmatory results. A failed +gate should remain visible with `carryForward: false`. + +## Pull requests + +Create a focused branch, run the relevant Python and web checks, and complete +the pull-request checklist. Explain the user-visible behavior, evidence, and +privacy impact. Generated build directories, local artifacts, datasets, and +model files must remain untracked. + +By participating, you agree to follow [CODE_OF_CONDUCT.md](CODE_OF_CONDUCT.md). diff --git a/README.md b/README.md index 83e4b99..c1fb4de 100644 --- a/README.md +++ b/README.md @@ -1,30 +1,262 @@ # SFTGuard -SFTGuard is a local-first correctness and regression gate for supervised -fine-tuning pipelines. +[![CI](https://github.com/Labeeb2339/sftguard/actions/workflows/ci.yml/badge.svg)](https://github.com/Labeeb2339/sftguard/actions/workflows/ci.yml) +[![CodeQL](https://github.com/Labeeb2339/sftguard/actions/workflows/codeql.yml/badge.svg)](https://github.com/Labeeb2339/sftguard/actions/workflows/codeql.yml) +[![License](https://img.shields.io/github/license/Labeeb2339/sftguard)](LICENSE) -The project is currently under preregistered development. The v0.1 protocol is -public before the confirmatory fault matrix is executed. Implementation, -checked-in evidence, and a browser report will follow on a review branch. +**Fail-closed preflight and regression CI for supervised fine-tuning.** -## Intended v0.1 boundary +![SFTGuard confirmatory dashboard](docs/assets/sftguard-confirmatory-dashboard.png) -SFTGuard will: +SFTGuard catches a narrow set of expensive-to-discover pipeline mistakes before +an adapter is released: malformed chat records, conflicting or leaked examples, +credential-shaped text, broken EOS handling, prompt tokens included in loss, and +answer-destroying truncation. It then gates paired base-versus-adapter scores +against explicit target and retention budgets. -- inspect chat records, template output, and supervised-token masks; -- detect deterministic data and pipeline faults such as leakage, duplicated or - conflicting examples, missing EOS handling, and answer-destroying truncation; -- compare base and adapter evaluation records against explicit target and - retention budgets; -- emit reproducible `PASS`, `FAIL`, or `ABSTAIN` artifacts without embedding - dataset text. +It runs locally, emits content-addressed JSON, and returns `PASS`, `FAIL`, or +`ABSTAIN`. Missing evidence never becomes a silent pass. -SFTGuard will **not** train a model, prove that a dataset is high quality, -predict final fine-tuning performance, or guarantee that an adapter is safe. +## Why this exists -Read the frozen [product specification](docs/PRODUCT_SPEC.md) and -[fault-injection protocol](docs/FAULT_INJECTION_PROTOCOL.md). +A training job can complete while optimizing the wrong tokens, leaking an +evaluation prompt into training, or degrading a capability that was not the +fine-tuning target. Training loss alone does not answer those questions. +SFTGuard turns the checks that *can* be established deterministically into a +small release contract: -## License +```text +local JSONL + exported token/mask evidence ──> audit ──> PASS / FAIL / ABSTAIN +paired base + adapter evaluation records ───> gate ──> Regression Card + └──> CI exit code + viewer +``` -Apache-2.0. +## Try it in two minutes + +SFTGuard requires Python 3.11 or newer and has no runtime dependencies. + +```bash +git clone https://github.com/Labeeb2339/sftguard.git +cd sftguard +python -m venv .venv + +# Windows PowerShell +.venv\Scripts\Activate.ps1 + +# macOS / Linux +# source .venv/bin/activate + +python -m pip install -e . +sftguard demo --output artifacts/local/demo.json +``` + +The demo is deterministic and uses synthetic records only. Exit codes are +`0 = PASS`, `1 = FAIL`, and `2 = ABSTAIN`. + +Audit the checked-in safe fixtures: + +```bash +sftguard audit \ + --train examples/clean-train.jsonl \ + --eval examples/clean-eval.jsonl \ + --output artifacts/local/audit.json +``` + +Gate paired base and adapter evaluation evidence: + +```bash +sftguard gate \ + --base examples/base-eval.jsonl \ + --adapter examples/adapter-eval.jsonl \ + --contract examples/regression-contract.json \ + --output artifacts/local/regression-card.json +``` + +## Exact mask evidence, not a heuristic + +The data checks accept ordinary `messages` records. Template, EOS, truncation, +and loss-mask claims require an `_sftguard` annotation exported from the exact +tokenization/collation path used for training: + +```json +{ + "messages": [ + {"role": "user", "content": "Classify this."}, + {"role": "assistant", "content": "class_a"} + ], + "_sftguard": { + "token_ids": [10, 20, 30, 2], + "token_roles": ["user", "user", "assistant", "assistant"], + "loss_mask": [false, false, true, true], + "eos_token_id": 2, + "original_assistant_tokens": 2, + "truncated": false + } +} +``` + +v0.1 intentionally does **not** guess an assistant mask or claim direct +integration with Transformers, TRL, Axolotl, or LLaMA-Factory. If trustworthy +mask evidence is missing or malformed, the full audit returns `ABSTAIN`. The +`MaskInspector` protocol in `src/sftguard/tokenizer.py` is the extension point +for trainer-specific exporters. + +For compatible Hugging Face templates, a future adapter can use +`apply_chat_template(..., tokenize=True, return_dict=True, +return_assistant_tokens_mask=True)` only when the template marks assistant spans +with `{% generation %}`. Until that adapter is implemented and tested, it is not +part of SFTGuard's claims. + +## Regression Cards + +The gate pairs base and adapter measurements by seed, sample ID, group, and +metric. A contract declares: + +- a minimum confidence-bound improvement for each `target` metric; +- a maximum confidence-bound regression for each `retention` metric; and +- the bootstrap draw count, confidence level, and deterministic seed. + +SFTGuard normalizes lower-is-better metrics, resamples seed clusters with a +frozen xorshift32 stream, and applies the threshold to the lower confidence +bound. Unpaired, duplicated, undeclared, non-finite, or incomplete evidence +returns `ABSTAIN`. + +The fixed joint work budget also returns `ABSTAIN` before an extreme +draw-count × pair-count request can consume unbounded CPU time. + +### Put the gate in GitHub Actions + +```yaml +name: SFT preflight +on: [push, pull_request] + +jobs: + sftguard: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + - uses: actions/setup-python@v6 + with: + python-version: "3.11" + - run: python -m pip install git+https://github.com/Labeeb2339/sftguard.git + - run: >- + sftguard audit + --train data/train.jsonl + --eval data/eval.jsonl + --output artifacts/sftguard-audit.json +``` + +The command's exit code blocks `FAIL` and `ABSTAIN` by default. Keep private +datasets out of Actions artifacts unless their disclosure is explicitly safe. + +## Static report viewer + +The viewer is a static React application with no analytics, report endpoint, +external font, or server-side storage. An imported report stays in the current +browser tab. + +```bash +cd web +npm ci +npm run dev +``` + +It accepts the native audit, Regression Card, development-benchmark, and frozen +confirmatory schemas. The viewer treats a supplied digest as **declared but +unverified** because it cannot promise byte-for-byte parity with Python's +canonical JSON serializer. Use the Python verifier for an integrity attestation. + +## Preregistered evidence + +The v0.1 fault protocol was committed before confirmatory seeds were generated +or inspected. It freezes nine single-fault templates, seeds `4100–4129`, six +carry-forward gates, and the allowed claim. The implementation and one-shot +evidence runner are committed and pushed before the confirmatory run. + +- [Frozen protocol](docs/FAULT_INJECTION_PROTOCOL.md) +- [Product specification](docs/PRODUCT_SPEC.md) +- [Evidence runner](scripts/confirmatory_evidence.py) +- [Evidence reproduction guide](docs/EVIDENCE.md) +- [Confirmatory result summary](docs/RESULTS.md) +- [Related work and boundaries](docs/RELATED_WORK.md) + +The matrix is an **internal detector regression**: each seed changes synthetic +markers, while each fault class keeps the same structural injection template. +It is not 30 independent real-world datasets, an external benchmark, a safety +certification, or proof that SFTGuard catches unknown failures. + +Checked-in confirmatory evidence is added only after the runner verifies a clean +and pushed implementation commit and reproduces the artifact in two fresh +processes. The result is preserved even if a frozen gate fails. + +### v0.1 confirmatory result + +All six frozen carry-forward gates passed without post-result changes: + +| Measure | Frozen gate | Observed | +| --- | ---: | ---: | +| Critical fault recall | 100% per class | **100%** | +| Macro recall, nine fault classes | ≥ 90% | **100%** | +| Clean-control false-positive rate | ≤ 10% | **0 / 30 (0%)** | +| Fault injections detected | — | **270 / 270** | +| Fresh-process reproduction | byte-identical | **matched** | +| Raw synthetic text in report | none | **none detected** | + +`carry_forward: true` across 300 synthetic cases. Protocol commit: +`d4c74cefca6729124142635dc8893b5e3ddec2d3`. Implementation commit: +`b0f168ab4d6a959d4e28611e345a250c14ca2ea9`. + +- [Confirmatory artifact](evidence/v0.1.0-confirmatory.json) +- [Evidence manifest](evidence/v0.1.0-confirmatory.manifest.json) +- Raw artifact SHA-256: + `961b2842d7791f6f3af7bec5bce6105e433b68d41bf6f7b82a9f3a98e1f695ab` +- Canonical payload SHA-256: + `eb55eaa16a492881ff1bcec8812b2ef37bb2ed6055b6b90a1da408386f632e3a` + +This supports the preregistered synthetic-detector statement only. It is not a +claim that 300 varied real-world fine-tuning pipelines were evaluated. + +## Privacy and security boundary + +- Files are bounded and parsed as untrusted UTF-8 JSON/JSONL. +- Findings contain signal categories and bounded locations, not dataset text. +- Reports contain aggregate values and hashes; hashes are identifiers, **not + anonymization**. +- The repository contains no model weights, adapters, private datasets, + private prompt/response content, or real credentials. Example conversations + and credential-shaped test strings are synthetic. +- Do not publish a real report until you have followed the + [safe-sharing checklist](docs/SHARING.md). + +See [SECURITY.md](SECURITY.md) for private vulnerability reports. + +## What SFTGuard does not prove + +SFTGuard does not train a model, score dataset quality, predict fine-tuning +success, detect every secret, replace a real evaluation harness, or establish +that an adapter is generally safe. A Regression Card is valid only for the +records and thresholds supplied to it. + +## Develop + +```bash +python -m pip install -e ".[dev]" +python -m pytest +python -m ruff check src tests/python scripts +python -m ruff format --check src tests/python scripts + +cd web +npm ci +npm run check +``` + +Contributions that add a detector should include a minimal fault fixture, a +clean control, redaction tests, and a precise claim boundary. Start with +[CONTRIBUTING.md](CONTRIBUTING.md) and the [roadmap](docs/ROADMAP.md). + +Planning to share the project? Use the truthful, limitation-preserving +[Reddit and Discord drafts](docs/ANNOUNCEMENT.md). + +## License and citation + +Apache-2.0. Citation metadata is available in [CITATION.cff](CITATION.cff). diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000..2b6b663 --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,55 @@ +# Security policy + +## Supported versions + +Security fixes are applied to the latest released version and the `main` branch. +Pre-release branches may change without a compatibility guarantee. + +## Report a vulnerability privately + +Do not open a public issue for a vulnerability, exposed credential, private +dataset, or personally identifying record. Use GitHub's +[private vulnerability reporting](https://github.com/Labeeb2339/sftguard/security/advisories/new) +for this repository. Include only the minimum synthetic information needed to +reproduce the problem. + +If a credential has already been exposed, revoke or rotate it with its provider +before changing local files. Do not paste the credential into a report, commit, +chat, screenshot, or Regression Card. Rewriting Git history is a separate, +potentially disruptive action and should be planned only after rotation and +current-file redaction. + +Please allow the maintainer a reasonable period to reproduce and address a +report before public disclosure. There is no guaranteed response or remediation +time. Good-faith reports that avoid privacy violations and service disruption +are welcome. + +## Local-data privacy boundary + +SFTGuard is designed for local analysis. The command-line tool must not upload +datasets, prompts, responses, model weights, or generated reports. Its public +static viewer has no report-upload endpoint, analytics, or server-side report +processing; a report selected in the viewer remains in that browser context. + +Generated artifacts are intended to contain aggregates, configuration, +content-addressing information, and bounded findings rather than raw dataset +text. This is a design boundary, not a guarantee that every user-supplied file +is safe to publish. Before sharing any artifact, follow +[docs/SHARING.md](docs/SHARING.md). Treat hashes as stable identifiers that can +still reveal equality or permit guessing when the underlying data is simple. + +When reporting a suspected privacy failure: + +- name the affected command, version, and output field; +- describe the data category, not the sensitive value; +- use synthetic reproduction data whenever possible; +- remove local usernames, absolute paths, access tokens, and model credentials; +- do not attach the original dataset or an unsanitized report. + +## Security scope + +Reports about unsafe parsing, path traversal, command construction, secret or +raw-text leakage, unbounded resource use, cross-site scripting in the viewer, +and dependency compromise are in scope. A `PASS` Regression Card is not a +safety certification and does not show that a model, adapter, or dataset is +secure. diff --git a/docs/ANNOUNCEMENT.md b/docs/ANNOUNCEMENT.md new file mode 100644 index 0000000..7eb1f31 --- /dev/null +++ b/docs/ANNOUNCEMENT.md @@ -0,0 +1,70 @@ +# Share copy + +These drafts describe the checked-in v0.1 result without calling it a +breakthrough, benchmark win, or safety certification. Check each community's +current self-promotion rules before posting and do not mass-post identical copy. + +## Reddit + +**Title** + +> I built SFTGuard: a fail-closed preflight and regression gate for supervised fine-tuning + +**Body** + +> I kept seeing fine-tuning checks discussed separately, so I built a small +> local tool that turns the deterministic ones into a release gate. +> +> SFTGuard audits chat JSONL plus exported token/loss-mask evidence for schema, +> role order, exact duplicates, conflicting answers, train/eval leakage, +> credential-shaped text, EOS handling, prompt supervision, and destructive +> truncation. It also compares paired base-vs-adapter metrics against explicit +> target and retention budgets. Missing evidence returns ABSTAIN instead of +> silently passing. +> +> I preregistered the synthetic fault protocol before running its confirmatory +> seeds. On the frozen internal matrix it detected 270/270 injected faults, had +> 0/30 clean-control false positives, and reproduced the artifact byte-for-byte +> from the recorded implementation commit. +> +> Important limitation: this is internal regression evidence using one +> structural template per known fault class—not an external benchmark, model +> quality result, or proof that it catches unknown failures. +> +> I would especially value technical feedback on the exported mask-evidence +> contract and the paired retention gate. +> +> Repo: https://github.com/Labeeb2339/sftguard +> +> Static viewer: https://labeeb2339.github.io/sftguard/ + +## Discord + +> I built **SFTGuard**, a local fail-closed preflight + regression gate for SFT +> pipelines. It checks chat data and exported loss-mask evidence, then gates +> paired base-vs-adapter target/retention metrics with PASS / FAIL / ABSTAIN. +> +> Frozen synthetic result: 270/270 known fault injections detected, 0/30 clean +> controls flagged, exact implementation replay matched. This is internal +> detector regression evidence, not an external benchmark or safety claim. +> +> Feedback on the mask-evidence format and regression contract would help: +> https://github.com/Labeeb2339/sftguard + +## Optional author line + +Use this only if you want to identify yourself as a student; otherwise omit it: + +> I’m a student learning how to make AI tooling claims reproducible instead of +> relying on demos alone. + +## Before posting + +- Open the public repo and Pages link in a signed-out window. +- Use only [`docs/assets/sftguard-confirmatory-dashboard.png`](assets/sftguard-confirmatory-dashboard.png); + do not capture private tabs, + local file paths, Discord messages, tokens, or notifications. +- Keep the limitation paragraph intact. +- Ask for specific technical feedback; do not trade stars, spam, or imply that + stars validate the research claim. +- If a community bans project promotion, do not post there. diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md new file mode 100644 index 0000000..1925009 --- /dev/null +++ b/docs/ARCHITECTURE.md @@ -0,0 +1,105 @@ +# Architecture + +This document describes the intended v0.1 boundaries. The frozen behavior is +defined by [PRODUCT_SPEC.md](PRODUCT_SPEC.md); implementation details may evolve +only within that contract. + +## System boundary + +SFTGuard is an evidence gate around a supervised fine-tuning workflow. It does +not train an adapter. The Python package audits local records and paired +evaluation evidence, writes a content-addressed Regression Card, and exits with +a machine-readable status. The static web application renders an explicitly +selected card without sending it to a report-processing service. + +```mermaid +flowchart LR + A["Local JSONL records"] --> B["Bounded parser and schema audit"] + T["Trusted exported token/mask evidence"] --> C["Template and loss-mask audit"] + B --> C + C --> D["Deterministic findings"] + E["Paired base and adapter evaluations"] --> F["Target and retention gates"] + D --> G["Status reducer"] + F --> G + G --> H["Content-addressed Regression Card"] + H --> I["Static browser viewer"] + H --> J["CI exit code and artifact validation"] +``` + +## Components + +### Bounded input layer + +The CLI accepts local JSONL and evaluation files subject to explicit file, +line, record, and field-size limits. Parsing is line-oriented. Dataset values +are treated as untrusted data and are never interpolated into shell commands. +Malformed, oversized, or incomplete inputs produce a finding or `ABSTAIN` +rather than an implicit pass. + +### Dataset, template, and mask audits + +Deterministic checks cover the preregistered schema, ordering, duplication, +conflict, leakage, EOS, truncation, supervision-mask, and credential-shaped +fault classes. When a tokenizer cannot expose a trustworthy assistant-token +mask, the template-dependent checks abstain. A heuristic mask must not be +reported as verified evidence. + +### Regression gate + +The regression layer consumes supplied, paired base-versus-adapter evaluation +records. It evaluates target improvement and retention budgets defined by the +user's evaluation contract and reports deterministic bootstrap intervals where +configured. It does not infer model quality from training loss and does not +predict the outcome of a future run. + +### Status reducer + +The reducer preserves three terminal states: + +- `PASS`: every required check ran and met its frozen gate; +- `FAIL`: at least one completed required check violated a gate; +- `ABSTAIN`: required evidence was unavailable or untrustworthy. + +`ABSTAIN` has release-blocking semantics. Mixed results retain the individual +finding states so that missing evidence is not hidden behind an aggregate. + +### Artifact layer + +Canonical JSON serialization supports byte-stable hashing and independent +verification. Public artifacts contain configuration, provenance, aggregates, +bounded findings, and content identifiers—not raw prompts, responses, injected +secrets, model weights, or access credentials. Exact protocol and +implementation commits are part of confirmatory evidence. + +### Static report viewer + +The viewer is a build-time static React application. It has no dataset or report +upload endpoint, telemetry, external fonts, or server-side rendering of an +imported Regression Card. File selection is an explicit user action; imported +content remains in the browser. The viewer treats all report strings as data +and must not render supplied HTML. + +## Trust boundaries + +| Boundary | Trusted assumption | Required response when it fails | +| --- | --- | --- | +| Dataset and evaluation files | Files are untrusted and may be malformed or large | Bound work; emit a finding or abstain | +| Exported mask evidence | The producer exported the exact training token and loss mask | Validate structure; abstain when evidence is absent or malformed | +| Regression contract | Metrics and pair identifiers are supplied honestly | Validate shape, pairing, and work budget; do not invent missing pairs | +| Browser artifact reader | A report may be modified or hostile | Allow-list aggregate fields, escape values, and label the declared digest unverified | +| Python evidence verifier | Checked-in evidence and manifest may be modified | Recompute canonical and raw hashes; fail on any mismatch | +| GitHub Pages | Application assets are public | Never bundle private reports or datasets | + +## Determinism and evidence + +The synthetic fault matrix validates SFTGuard's own bounded detectors. Its +confirmatory result is written only after the protocol and implementation are +committed, and a fresh process must reproduce the canonical payload. This +evidence does not establish coverage of unknown real-world faults. + +## Deployment + +Python and web checks run on Linux and Windows. Dependency audits and static +analysis run separately from behavior tests. GitHub Pages deploys only the +contents of `web/dist`; local Regression Cards are not part of that directory +and are not deployed automatically. diff --git a/docs/EVIDENCE.md b/docs/EVIDENCE.md new file mode 100644 index 0000000..7a704e3 --- /dev/null +++ b/docs/EVIDENCE.md @@ -0,0 +1,65 @@ +# Reproducing the v0.1 evidence + +The confirmatory bundle is generated once from the frozen implementation and +then retained whether its gates pass or fail. The runner does not overwrite an +existing result. + +## Phase separation + +1. Commit and publish the [fault protocol](FAULT_INJECTION_PROTOCOL.md). +2. Implement and test only with development seeds `0–9`. +3. Commit and push the complete detector, runner, tests, and frozen thresholds. +4. From that clean pushed commit, execute the confirmatory seeds `4100–4129`. +5. Commit the resulting artifact and manifest without changing a detector or + threshold in response to the result. + +The runner enforces steps 3 and 4 by refusing a dirty tree, an implementation +commit not represented by a local `origin/*` ref, or existing evidence paths. + +## Generate the bundle once + +```bash +python scripts/confirmatory_evidence.py generate +``` + +Two fresh Python processes independently produce the complete JSON bytes. The +runner writes files only if those bytes match and both artifacts pass their +canonical integrity check. + +The bundle contains: + +- `evidence/v0.1.0-confirmatory.json` — aggregate gates, per-fault recall, + per-case identifiers/statuses, provenance, and no synthetic record text; +- `evidence/v0.1.0-confirmatory.manifest.json` — raw file SHA-256, canonical + payload SHA-256, protocol commit, and implementation commit. + +The raw file digest lives in the manifest because a file cannot contain its own +SHA-256 without changing that digest. + +## Verify integrity + +This check needs only the two evidence files: + +```bash +python scripts/confirmatory_evidence.py verify --integrity-only +``` + +## Replay the exact implementation + +With full Git history available, this creates a temporary detached worktree at +the recorded implementation commit and regenerates the bytes: + +```bash +python scripts/confirmatory_evidence.py verify +``` + +Replay requires the same Python major/minor version recorded by the artifact. +CI performs the exact replay on Python 3.11 and integrity checks across the +supported matrix. + +## Interpretation + +A passing artifact supports only the claim frozen in the protocol. The nine +injections are structural templates repeated across deterministic synthetic +markers. The result is internal detector regression evidence, not an external +dataset benchmark or a claim about unknown fine-tuning failures. diff --git a/docs/RELATED_WORK.md b/docs/RELATED_WORK.md new file mode 100644 index 0000000..1dcf302 --- /dev/null +++ b/docs/RELATED_WORK.md @@ -0,0 +1,81 @@ +# Related work and project boundary + +This comparison defines SFTGuard's scope; it is not an exhaustive novelty +search. SFTGuard is a local correctness and regression gate around a fine-tuning +pipeline. It does not replace trainers, parameter-efficient tuning methods, +data-selection research, or general model benchmarks. + +## Training frameworks + +[TRL](https://github.com/huggingface/trl) provides post-training algorithms and +trainers, including supervised fine-tuning. Its official +[SFTTrainer documentation](https://huggingface.co/docs/trl/main/en/sft_trainer) +defines dataset formats, chat-template application, token-level loss, and +assistant/completion masking. SFTGuard inspects evidence about those semantics; +it does not implement or accelerate the training loop. + +[LLaMA-Factory](https://github.com/hiyouga/LLaMA-Factory) and +[Axolotl](https://github.com/axolotl-ai-cloud/axolotl) are configurable +fine-tuning systems spanning model loading, dataset preparation, distributed +training, parameter-efficient methods, and evaluation integrations. SFTGuard is +designed to sit beside such a system: it can audit exported local records and +gate supplied base-versus-adapter results, but it does not launch their jobs or +claim configuration parity with them. + +## Parameter-efficient fine-tuning + +[PEFT](https://github.com/huggingface/peft) implements LoRA and other +parameter-efficient methods. Its +[LoRA API](https://huggingface.co/docs/peft/main/en/package_reference/lora) +supports target modules, per-module rank and alpha patterns, multiple +initialization methods, quantization-aware workflows, and automatic target +selection. SFTGuard does not propose a new adapter, choose an optimal rank, or +allocate modules. It checks pipeline correctness and evaluates an explicit +release contract for an adapter that another tool produced. + +## Prediction and experiment screening + +[TuneAhead](https://arxiv.org/abs/2606.17660) predicts full fine-tuning +performance using static dataset descriptors and features from a short probe +run. SFTGuard deliberately makes a different claim: it does not estimate a +future score. It verifies bounded, observable properties and returns `ABSTAIN` +when the evidence required for a gate is missing. + +## Data selection and processing + +[LESS](https://github.com/princeton-nlp/LESS) selects influential examples for +targeted instruction tuning using optimizer-aware low-rank gradient features. +[DataInf](https://github.com/ykwon0407/DataInf) estimates data influence for +LoRA-tuned language and diffusion models. These methods rank training examples +by estimated utility. SFTGuard does not rank examples or assert that a dataset +is high quality; its data checks target deterministic conditions such as schema +errors, exact duplication, conflicting answers, split leakage, and bounded +privacy signals. + +[Data-Juicer](https://github.com/datajuicer/data-juicer) is a broad data +processing system with filtering, analysis, transformation, and distributed +execution operators for foundation-model data. SFTGuard is narrower: it uses a +small frozen audit contract and emits a release decision artifact. It is not a +general cleaning or data-transformation pipeline. + +## Evaluation and forgetting + +[lm-evaluation-harness](https://github.com/EleutherAI/lm-evaluation-harness) +runs a large catalog of language-model evaluations. +[TRACE](https://github.com/BeyonderXX/TRACE), introduced in the +[TRACE paper](https://arxiv.org/abs/2310.06762), benchmarks continual learning +across multiple tasks and measures degradation in general and +instruction-following abilities. SFTGuard does not reproduce those benchmark +suites. It consumes paired evaluation records chosen by the user and applies +explicit target and retention budgets; the resulting Regression Card is valid +only for that supplied contract. + +## Practical distinction + +SFTGuard's intended contribution is the integration of four conservative +software checks: exact training-data semantics where they can be verified, +deterministic fault findings, paired regression budgets, and shareable +content-addressed provenance. Existing projects above can provide the trainer, +adapter, data pipeline, selector, or benchmark. SFTGuard records whether a +specific proposed release met its declared evidence gates, without converting +missing evidence into success or claiming that the release is safe in general. diff --git a/docs/RESULTS.md b/docs/RESULTS.md new file mode 100644 index 0000000..6e0bc2d --- /dev/null +++ b/docs/RESULTS.md @@ -0,0 +1,50 @@ +# v0.1 confirmatory result + +Status: `PASS` + +Carry forward: `true` + +The result was generated after the protocol and implementation were committed +and pushed. No detector or threshold was changed after confirmatory execution. +Post-result changes are limited to checked-in evidence, documentation, +packaging metadata, Pages routing, and rendering that evidence in the static +viewer; the recorded implementation remains the replay authority. + +## Frozen gates + +| Gate | Result | +| --- | ---: | +| 100% recall for every critical fault class | pass — 30/30 for each class | +| Macro recall across all nine fault classes ≥ 90% | pass — 100% | +| Clean-control false-positive rate ≤ 10% | pass — 0/30 (0%) | +| Expected and observed primary signal recorded | pass | +| Fresh process reproduces result bytes | pass | +| Raw prompts, responses, and injected secrets absent | pass | + +Across 30 seeds, all 270 injected cases emitted their required primary signal. +All 30 clean controls emitted no signal. The exact implementation replay +matched the checked-in artifact byte-for-byte. + +## Provenance + +| Item | Value | +| --- | --- | +| Protocol commit | `d4c74cefca6729124142635dc8893b5e3ddec2d3` | +| Implementation commit | `b0f168ab4d6a959d4e28611e345a250c14ca2ea9` | +| Python | CPython 3.11 | +| Raw artifact SHA-256 | `961b2842d7791f6f3af7bec5bce6105e433b68d41bf6f7b82a9f3a98e1f695ab` | +| Canonical payload SHA-256 | `eb55eaa16a492881ff1bcec8812b2ef37bb2ed6055b6b90a1da408386f632e3a` | + +See the [artifact](../evidence/v0.1.0-confirmatory.json), +[manifest](../evidence/v0.1.0-confirmatory.manifest.json), and +[reproduction guide](EVIDENCE.md). + +## Allowed claim + +> SFTGuard detected the preregistered synthetic fault matrix with the reported +> false-positive rate and reproduced its content-addressed evidence. + +This is internal detector regression evidence. Each class uses one structural +fault template across deterministic marker variations. The result does not show +coverage of unknown faults, model quality, or safety, and it is not an external +fine-tuning benchmark. diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md new file mode 100644 index 0000000..b98490e --- /dev/null +++ b/docs/ROADMAP.md @@ -0,0 +1,33 @@ +# Roadmap + +SFTGuard grows through evidence, not feature count. + +## v0.1 — correctness and regression gate + +- Local JSONL audit with bounded parsing and redacted findings. +- Exact inspection of trusted, exported token-role and loss-mask evidence. +- Base-versus-adapter Regression Cards. +- Deterministic synthetic fault matrix. +- Static report viewer and GitHub Action integration. + +## v0.2 — real training integration + +- TRL callback for checkpoint-level target and retention evaluation. +- Tested Transformers chat-template mask exporter for compatible templates. +- LLaMA-Factory and Axolotl result importers. +- One small, redistributable real-model benchmark with pinned revisions. + +## Research track + +- Fault-injection benchmark across multiple trainers and model families. +- False-alarm and fault-detection calibration on independently authored cases. +- Retention-contract design for structured output, multilingual behavior, and + safety probes. +- Post-merge and post-quantization Regression Cards. + +## Deliberately out of scope + +- Hosted dataset upload or model training. +- A new LoRA rank allocator or optimizer. +- Predicted final performance without full evaluation. +- Automatic claims that an adapter is safe or production-ready. diff --git a/docs/SHARING.md b/docs/SHARING.md new file mode 100644 index 0000000..9c9aaa3 --- /dev/null +++ b/docs/SHARING.md @@ -0,0 +1,71 @@ +# Sharing SFTGuard results safely + +SFTGuard is local-first. It does not automatically post, upload, or publish a +dataset, Regression Card, screenshot, or result. Sharing is a separate decision +made by the person who controls the underlying data. + +## Usually appropriate to share + +After inspection, these fields are generally intended for reproducibility: + +- SFTGuard version and implementation commit; +- protocol version, configuration, and frozen thresholds; +- aggregate finding counts and `PASS`, `FAIL`, or `ABSTAIN` status; +- synthetic benchmark seeds and synthetic evidence hashes; +- Python, Node.js, operating-system, and dependency versions; +- evaluation metric names, pair counts, and aggregate intervals when the + evaluation contract permits disclosure. + +Hashes are identifiers, not anonymization. A dataset or row hash can reveal +that two parties used the same content and may be guessable for short or common +records. Do not publish a hash when even dataset identity or equality is +sensitive. + +## Do not share + +- raw prompts, responses, system messages, labels, or private evaluation data; +- credentials, tokens, cookies, private keys, connection strings, or `.env` + files; +- names, contact details, health, school, financial, or other identifying data; +- absolute local paths that reveal usernames or private project names; +- proprietary model weights, adapter files, or licensed data without + permission; +- an unsanitized report, terminal log, screenshot, issue attachment, or browser + export. + +Finding locations should use bounded record indexes and field categories, not +the sensitive value itself. A secret-pattern finding should say where and what +category was detected without copying the candidate secret. + +## Pre-publication checklist + +1. Work from a copy of the intended artifact; keep the original private. +2. Validate the card schema and canonical hash. +3. Search the copy for raw prompt/response fields, credentials, local paths, + emails, phone numbers, and other identifiers. +4. Confirm that no data sample was embedded in metadata, error text, or a + screenshot. +5. Check the dataset and model licenses and any competition or institutional + disclosure rules. +6. State the exact evidence boundary. A synthetic pass is not a safety + certification or a claim about unknown faults. +7. When uncertain, share only the aggregate conclusion and reproduction code, + or do not publish the artifact. + +## Browser viewer and GitHub Pages + +The public Pages site contains only static application assets from `web/dist`. +Selecting a local report reads it into the current browser; the project has no +report upload endpoint or analytics. Browser extensions, modified builds, and +the surrounding device are outside SFTGuard's control, so sensitive reports +should still be opened only in a trusted local environment. + +Never commit a real Regression Card merely to view it on Pages. Use synthetic +fixtures for documentation, tests, issues, and demonstrations. + +## Security reports and support requests + +Use synthetic reproduction data in public issues. For suspected leakage or +another vulnerability, follow [SECURITY.md](../SECURITY.md) and report it +privately. Rotate an exposed credential before discussing code remediation, and +never include the credential value in the report. diff --git a/docs/assets/sftguard-confirmatory-dashboard.png b/docs/assets/sftguard-confirmatory-dashboard.png new file mode 100644 index 0000000..1f43b0b Binary files /dev/null and b/docs/assets/sftguard-confirmatory-dashboard.png differ diff --git a/evidence/v0.1.0-confirmatory.json b/evidence/v0.1.0-confirmatory.json new file mode 100644 index 0000000..f05a1d6 --- /dev/null +++ b/evidence/v0.1.0-confirmatory.json @@ -0,0 +1,4046 @@ +{ + "carry_forward": true, + "cases": [ + { + "audit_status": "PASS", + "case_sha256": "5f57f7b095e2563bdfe60e10e6a808433a09811b55752bd32aaf7cf79b001eb8", + "expected_primary_signal": null, + "fault_id": null, + "kind": "clean_control", + "observed_primary_signal": null, + "observed_signal_ids": [], + "primary_expectation_met": true, + "seed": 4100 + }, + { + "audit_status": "FAIL", + "case_sha256": "8c8e6297ef27bb9f5125037ba7f3eea04794bdc14ef2168404b008b53b4dcbff", + "expected_primary_signal": "invalid_role", + "fault_id": "schema.invalid_role", + "kind": "fault", + "observed_primary_signal": "invalid_role", + "observed_signal_ids": [ + "invalid_role", + "role_order" + ], + "primary_expectation_met": true, + "seed": 4100 + }, + { + "audit_status": "FAIL", + "case_sha256": "a7de518da70ae0f64ee7d96fcc3552a6a6945bd011024e0c25335fae67b4bbb3", + "expected_primary_signal": "role_order", + "fault_id": "schema.role_order", + "kind": "fault", + "observed_primary_signal": "role_order", + "observed_signal_ids": [ + "role_order" + ], + "primary_expectation_met": true, + "seed": 4100 + }, + { + "audit_status": "FAIL", + "case_sha256": "0597243d651815b44146970c3dc9744c5a43fd11c98aeb3d39a26fce9f5b3531", + "expected_primary_signal": "missing_eos", + "fault_id": "template.missing_eos", + "kind": "fault", + "observed_primary_signal": "missing_eos", + "observed_signal_ids": [ + "missing_eos" + ], + "primary_expectation_met": true, + "seed": 4100 + }, + { + "audit_status": "FAIL", + "case_sha256": "4ad0fcdb6576067ffdfe1d4b0216a2a58dff462bcc251edb496e07ddffb8ba95", + "expected_primary_signal": "prompt_token_supervised", + "fault_id": "mask.prompt_supervised", + "kind": "fault", + "observed_primary_signal": "prompt_token_supervised", + "observed_signal_ids": [ + "prompt_token_supervised" + ], + "primary_expectation_met": true, + "seed": 4100 + }, + { + "audit_status": "FAIL", + "case_sha256": "f986bc41a0b596c99c23127dd74dba3576fafad82fc021bf0e47ffe012724188", + "expected_primary_signal": "assistant_fully_truncated", + "fault_id": "truncation.answer_removed", + "kind": "fault", + "observed_primary_signal": "assistant_fully_truncated", + "observed_signal_ids": [ + "assistant_fully_truncated" + ], + "primary_expectation_met": true, + "seed": 4100 + }, + { + "audit_status": "PASS", + "case_sha256": "16a6b1085342447e14607a74f722fdc4a69d06a01843fb01938ecdef3a3fd8a1", + "expected_primary_signal": "exact_duplicate", + "fault_id": "data.exact_duplicate", + "kind": "fault", + "observed_primary_signal": "exact_duplicate", + "observed_signal_ids": [ + "exact_duplicate" + ], + "primary_expectation_met": true, + "seed": 4100 + }, + { + "audit_status": "FAIL", + "case_sha256": "042de45aa942895b11a6999a1df92ae7bf9a1e6c447cc0eccd32c67ee08dd8f0", + "expected_primary_signal": "prompt_conflict", + "fault_id": "data.prompt_conflict", + "kind": "fault", + "observed_primary_signal": "prompt_conflict", + "observed_signal_ids": [ + "prompt_conflict" + ], + "primary_expectation_met": true, + "seed": 4100 + }, + { + "audit_status": "FAIL", + "case_sha256": "359feff53fd877e5fd246484a82239cc64908dffd028f8294eed53122f7b20d4", + "expected_primary_signal": "train_eval_leakage", + "fault_id": "data.train_eval_leakage", + "kind": "fault", + "observed_primary_signal": "train_eval_leakage", + "observed_signal_ids": [ + "train_eval_leakage" + ], + "primary_expectation_met": true, + "seed": 4100 + }, + { + "audit_status": "FAIL", + "case_sha256": "f56ba366d44170e2e7019bc90c2de2501b803756b356f5f14c2668c2d4ffc0ac", + "expected_primary_signal": "secret_pattern", + "fault_id": "privacy.secret_pattern", + "kind": "fault", + "observed_primary_signal": "secret_pattern", + "observed_signal_ids": [ + "pii_like", + "secret_pattern" + ], + "primary_expectation_met": true, + "seed": 4100 + }, + { + "audit_status": "PASS", + "case_sha256": "120af0f7bef4f9f196721ec14a55a10cc8878a89c73d4e89386eec45d96b8103", + "expected_primary_signal": null, + "fault_id": null, + "kind": "clean_control", + "observed_primary_signal": null, + "observed_signal_ids": [], + "primary_expectation_met": true, + "seed": 4101 + }, + { + "audit_status": "FAIL", + "case_sha256": "acb5d289c02e4f9a2ae791b163ee53f4b8821ded0cbb5658626e13bf98061585", + "expected_primary_signal": "invalid_role", + "fault_id": "schema.invalid_role", + "kind": "fault", + "observed_primary_signal": "invalid_role", + "observed_signal_ids": [ + "invalid_role", + "role_order" + ], + "primary_expectation_met": true, + "seed": 4101 + }, + { + "audit_status": "FAIL", + "case_sha256": "3fd48a916b7961f34b8e799f28ba46f3f549c670dbaba442030db61391b12395", + "expected_primary_signal": "role_order", + "fault_id": "schema.role_order", + "kind": "fault", + "observed_primary_signal": "role_order", + "observed_signal_ids": [ + "role_order" + ], + "primary_expectation_met": true, + "seed": 4101 + }, + { + "audit_status": "FAIL", + "case_sha256": "36bb69f44f76c9f41ade010d997058efde81854cb1a942f6d2901e25c3ffc28b", + "expected_primary_signal": "missing_eos", + "fault_id": "template.missing_eos", + "kind": "fault", + "observed_primary_signal": "missing_eos", + "observed_signal_ids": [ + "missing_eos" + ], + "primary_expectation_met": true, + "seed": 4101 + }, + { + "audit_status": "FAIL", + "case_sha256": "b61e7a38c7c46599e4ce106fb6dfd06dc26b057299561a59873b892679315747", + "expected_primary_signal": "prompt_token_supervised", + "fault_id": "mask.prompt_supervised", + "kind": "fault", + "observed_primary_signal": "prompt_token_supervised", + "observed_signal_ids": [ + "prompt_token_supervised" + ], + "primary_expectation_met": true, + "seed": 4101 + }, + { + "audit_status": "FAIL", + "case_sha256": "775dd68b9272d1179f575b239c41b1881d58011ac0cbaefb0e135a499bae55c4", + "expected_primary_signal": "assistant_fully_truncated", + "fault_id": "truncation.answer_removed", + "kind": "fault", + "observed_primary_signal": "assistant_fully_truncated", + "observed_signal_ids": [ + "assistant_fully_truncated" + ], + "primary_expectation_met": true, + "seed": 4101 + }, + { + "audit_status": "PASS", + "case_sha256": "c23421a5aba39766532f9f64bea5cd28b85ac42b2b2dcee22904cca34c9d8c19", + "expected_primary_signal": "exact_duplicate", + "fault_id": "data.exact_duplicate", + "kind": "fault", + "observed_primary_signal": "exact_duplicate", + "observed_signal_ids": [ + "exact_duplicate" + ], + "primary_expectation_met": true, + "seed": 4101 + }, + { + "audit_status": "FAIL", + "case_sha256": "8f78488a8cbd048f3ad41be28707a60717270620ba896cdfd1073ea937983066", + "expected_primary_signal": "prompt_conflict", + "fault_id": "data.prompt_conflict", + "kind": "fault", + "observed_primary_signal": "prompt_conflict", + "observed_signal_ids": [ + "prompt_conflict" + ], + "primary_expectation_met": true, + "seed": 4101 + }, + { + "audit_status": "FAIL", + "case_sha256": "b66cfe8ff43543ddc543ae2ca5074fd634b0ac889debcff1270a13f30d4e1eeb", + "expected_primary_signal": "train_eval_leakage", + "fault_id": "data.train_eval_leakage", + "kind": "fault", + "observed_primary_signal": "train_eval_leakage", + "observed_signal_ids": [ + "train_eval_leakage" + ], + "primary_expectation_met": true, + "seed": 4101 + }, + { + "audit_status": "FAIL", + "case_sha256": "0730548b36ad1704412297fdb158b51c39ded538bfae2a69c74343c68e5469d2", + "expected_primary_signal": "secret_pattern", + "fault_id": "privacy.secret_pattern", + "kind": "fault", + "observed_primary_signal": "secret_pattern", + "observed_signal_ids": [ + "secret_pattern" + ], + "primary_expectation_met": true, + "seed": 4101 + }, + { + "audit_status": "PASS", + "case_sha256": "f31c71167af0fd046d6c777ede394ab3db0d1adbe54d53d3ca4a450fffb421c7", + "expected_primary_signal": null, + "fault_id": null, + "kind": "clean_control", + "observed_primary_signal": null, + "observed_signal_ids": [], + "primary_expectation_met": true, + "seed": 4102 + }, + { + "audit_status": "FAIL", + "case_sha256": "b1d213d79e5af97317cc8837a2739aaf308511dfc0269c209806850212fbe141", + "expected_primary_signal": "invalid_role", + "fault_id": "schema.invalid_role", + "kind": "fault", + "observed_primary_signal": "invalid_role", + "observed_signal_ids": [ + "invalid_role", + "role_order" + ], + "primary_expectation_met": true, + "seed": 4102 + }, + { + "audit_status": "FAIL", + "case_sha256": "367a5991a0512eb801b5a6d6571f7e2f2e9e01aed81172d89f00c7cbc44fbedd", + "expected_primary_signal": "role_order", + "fault_id": "schema.role_order", + "kind": "fault", + "observed_primary_signal": "role_order", + "observed_signal_ids": [ + "role_order" + ], + "primary_expectation_met": true, + "seed": 4102 + }, + { + "audit_status": "FAIL", + "case_sha256": "453e63e7af0b039c0a8b3ff01f1e8cd5041ca7fac7d215d8638ef4c9d3671f48", + "expected_primary_signal": "missing_eos", + "fault_id": "template.missing_eos", + "kind": "fault", + "observed_primary_signal": "missing_eos", + "observed_signal_ids": [ + "missing_eos" + ], + "primary_expectation_met": true, + "seed": 4102 + }, + { + "audit_status": "FAIL", + "case_sha256": "fafa1ea2b069ad8548bfe8ee128c231fe6ed0f171ad63de2a94f459e30db9a19", + "expected_primary_signal": "prompt_token_supervised", + "fault_id": "mask.prompt_supervised", + "kind": "fault", + "observed_primary_signal": "prompt_token_supervised", + "observed_signal_ids": [ + "prompt_token_supervised" + ], + "primary_expectation_met": true, + "seed": 4102 + }, + { + "audit_status": "FAIL", + "case_sha256": "254ec14b57f9114db01514ff91b51136ddb449c1e93ec23d2cb4e3b3a5803c10", + "expected_primary_signal": "assistant_fully_truncated", + "fault_id": "truncation.answer_removed", + "kind": "fault", + "observed_primary_signal": "assistant_fully_truncated", + "observed_signal_ids": [ + "assistant_fully_truncated" + ], + "primary_expectation_met": true, + "seed": 4102 + }, + { + "audit_status": "PASS", + "case_sha256": "a35f60a6af78c040dbcd16d045b75461eb991ae38c87d667d4adec9ac0c5e1e3", + "expected_primary_signal": "exact_duplicate", + "fault_id": "data.exact_duplicate", + "kind": "fault", + "observed_primary_signal": "exact_duplicate", + "observed_signal_ids": [ + "exact_duplicate" + ], + "primary_expectation_met": true, + "seed": 4102 + }, + { + "audit_status": "FAIL", + "case_sha256": "89b6a0312678a385d9d3e060f72dc30e7bd1ecf46544e0c0e2106edd08139e14", + "expected_primary_signal": "prompt_conflict", + "fault_id": "data.prompt_conflict", + "kind": "fault", + "observed_primary_signal": "prompt_conflict", + "observed_signal_ids": [ + "prompt_conflict" + ], + "primary_expectation_met": true, + "seed": 4102 + }, + { + "audit_status": "FAIL", + "case_sha256": "1f7fcbbab56e0ee8776fbc4b0b1c5254ffcbcb6e9a2244443f7b151fb96fe08e", + "expected_primary_signal": "train_eval_leakage", + "fault_id": "data.train_eval_leakage", + "kind": "fault", + "observed_primary_signal": "train_eval_leakage", + "observed_signal_ids": [ + "train_eval_leakage" + ], + "primary_expectation_met": true, + "seed": 4102 + }, + { + "audit_status": "FAIL", + "case_sha256": "9759bcabf8ca5a20249df2cd3893565df983da09325095301d2d64333a54a4ad", + "expected_primary_signal": "secret_pattern", + "fault_id": "privacy.secret_pattern", + "kind": "fault", + "observed_primary_signal": "secret_pattern", + "observed_signal_ids": [ + "secret_pattern" + ], + "primary_expectation_met": true, + "seed": 4102 + }, + { + "audit_status": "PASS", + "case_sha256": "7b4d254c97c716ce74ef7a36fa0d6f53c2371e0dad7955cfcfc679e9f124fe80", + "expected_primary_signal": null, + "fault_id": null, + "kind": "clean_control", + "observed_primary_signal": null, + "observed_signal_ids": [], + "primary_expectation_met": true, + "seed": 4103 + }, + { + "audit_status": "FAIL", + "case_sha256": "dee111a8f5014d0d3a9dcdb266d2e6313265c45f6fa893929863977f456b7530", + "expected_primary_signal": "invalid_role", + "fault_id": "schema.invalid_role", + "kind": "fault", + "observed_primary_signal": "invalid_role", + "observed_signal_ids": [ + "invalid_role", + "role_order" + ], + "primary_expectation_met": true, + "seed": 4103 + }, + { + "audit_status": "FAIL", + "case_sha256": "932de8dcca5274c7fa28d543a7dd26ac20f1b308adce0077e48400c3139a68ef", + "expected_primary_signal": "role_order", + "fault_id": "schema.role_order", + "kind": "fault", + "observed_primary_signal": "role_order", + "observed_signal_ids": [ + "role_order" + ], + "primary_expectation_met": true, + "seed": 4103 + }, + { + "audit_status": "FAIL", + "case_sha256": "c45202f0ec58f33c7aba100d955571db340da0dfde2f119130dcc8be4c264f05", + "expected_primary_signal": "missing_eos", + "fault_id": "template.missing_eos", + "kind": "fault", + "observed_primary_signal": "missing_eos", + "observed_signal_ids": [ + "missing_eos" + ], + "primary_expectation_met": true, + "seed": 4103 + }, + { + "audit_status": "FAIL", + "case_sha256": "8512832b406abd33bd75d779e3f637d5094548b2f1323c1cdf65447254bea926", + "expected_primary_signal": "prompt_token_supervised", + "fault_id": "mask.prompt_supervised", + "kind": "fault", + "observed_primary_signal": "prompt_token_supervised", + "observed_signal_ids": [ + "prompt_token_supervised" + ], + "primary_expectation_met": true, + "seed": 4103 + }, + { + "audit_status": "FAIL", + "case_sha256": "9ef256075af045c8e1d79bdad7895326766050559a6c6d3c6c3bf53e199915e8", + "expected_primary_signal": "assistant_fully_truncated", + "fault_id": "truncation.answer_removed", + "kind": "fault", + "observed_primary_signal": "assistant_fully_truncated", + "observed_signal_ids": [ + "assistant_fully_truncated" + ], + "primary_expectation_met": true, + "seed": 4103 + }, + { + "audit_status": "PASS", + "case_sha256": "a356c506d51c444b5ff2aa409adc75ecbd3ca5b66ef6b8d57598841cb5af03b4", + "expected_primary_signal": "exact_duplicate", + "fault_id": "data.exact_duplicate", + "kind": "fault", + "observed_primary_signal": "exact_duplicate", + "observed_signal_ids": [ + "exact_duplicate" + ], + "primary_expectation_met": true, + "seed": 4103 + }, + { + "audit_status": "FAIL", + "case_sha256": "e5bed19e47c3622936a09530537ec9da6d0105a2ed37ecb0361e195d36964372", + "expected_primary_signal": "prompt_conflict", + "fault_id": "data.prompt_conflict", + "kind": "fault", + "observed_primary_signal": "prompt_conflict", + "observed_signal_ids": [ + "prompt_conflict" + ], + "primary_expectation_met": true, + "seed": 4103 + }, + { + "audit_status": "FAIL", + "case_sha256": "5ac22a11bd9776e2465d32285e37f49a29ebb751486b0a91acbdbc2d2f1955c9", + "expected_primary_signal": "train_eval_leakage", + "fault_id": "data.train_eval_leakage", + "kind": "fault", + "observed_primary_signal": "train_eval_leakage", + "observed_signal_ids": [ + "train_eval_leakage" + ], + "primary_expectation_met": true, + "seed": 4103 + }, + { + "audit_status": "FAIL", + "case_sha256": "a8f80a191416c92decb18e0d1870d8de8d07bbb7f8f641c8eedd36685a9a1fd3", + "expected_primary_signal": "secret_pattern", + "fault_id": "privacy.secret_pattern", + "kind": "fault", + "observed_primary_signal": "secret_pattern", + "observed_signal_ids": [ + "secret_pattern" + ], + "primary_expectation_met": true, + "seed": 4103 + }, + { + "audit_status": "PASS", + "case_sha256": "6b5bc5465d5fc19760bacd625ebffb6ed07df20734fac9b02f529a55d50784cc", + "expected_primary_signal": null, + "fault_id": null, + "kind": "clean_control", + "observed_primary_signal": null, + "observed_signal_ids": [], + "primary_expectation_met": true, + "seed": 4104 + }, + { + "audit_status": "FAIL", + "case_sha256": "b9a58b5f0f72bfd71d4615c5dfab7d51eddeb328d73583f1a89f7da05f8d6c9d", + "expected_primary_signal": "invalid_role", + "fault_id": "schema.invalid_role", + "kind": "fault", + "observed_primary_signal": "invalid_role", + "observed_signal_ids": [ + "invalid_role", + "role_order" + ], + "primary_expectation_met": true, + "seed": 4104 + }, + { + "audit_status": "FAIL", + "case_sha256": "8cd4438af5c34ec74a32d496b5b17740978284a18caf24cf94823383430d83d0", + "expected_primary_signal": "role_order", + "fault_id": "schema.role_order", + "kind": "fault", + "observed_primary_signal": "role_order", + "observed_signal_ids": [ + "role_order" + ], + "primary_expectation_met": true, + "seed": 4104 + }, + { + "audit_status": "FAIL", + "case_sha256": "de51b9acbd8bf3125dd7e3e4b1fd985d0f64cea1ce09c0504da8627c323b13ab", + "expected_primary_signal": "missing_eos", + "fault_id": "template.missing_eos", + "kind": "fault", + "observed_primary_signal": "missing_eos", + "observed_signal_ids": [ + "missing_eos" + ], + "primary_expectation_met": true, + "seed": 4104 + }, + { + "audit_status": "FAIL", + "case_sha256": "4dd1e127e2b5d71cd1911ca1034f46d498d9e363dd047b6a09d182113e0d4d87", + "expected_primary_signal": "prompt_token_supervised", + "fault_id": "mask.prompt_supervised", + "kind": "fault", + "observed_primary_signal": "prompt_token_supervised", + "observed_signal_ids": [ + "prompt_token_supervised" + ], + "primary_expectation_met": true, + "seed": 4104 + }, + { + "audit_status": "FAIL", + "case_sha256": "56e33d0a805d7211522f1a4105e4c7e77188fdfc85b656641f67cb4ff4146016", + "expected_primary_signal": "assistant_fully_truncated", + "fault_id": "truncation.answer_removed", + "kind": "fault", + "observed_primary_signal": "assistant_fully_truncated", + "observed_signal_ids": [ + "assistant_fully_truncated" + ], + "primary_expectation_met": true, + "seed": 4104 + }, + { + "audit_status": "PASS", + "case_sha256": "7d1d77341d0e6d9ce8cc78567510b0c1e014ddb9c0a1125c8c4a1d3c6ee82cf2", + "expected_primary_signal": "exact_duplicate", + "fault_id": "data.exact_duplicate", + "kind": "fault", + "observed_primary_signal": "exact_duplicate", + "observed_signal_ids": [ + "exact_duplicate" + ], + "primary_expectation_met": true, + "seed": 4104 + }, + { + "audit_status": "FAIL", + "case_sha256": "6786ed7b373b369d4ed06b2a7e217c23fabdb695cd16c4f3ff19dd6f3da57cef", + "expected_primary_signal": "prompt_conflict", + "fault_id": "data.prompt_conflict", + "kind": "fault", + "observed_primary_signal": "prompt_conflict", + "observed_signal_ids": [ + "prompt_conflict" + ], + "primary_expectation_met": true, + "seed": 4104 + }, + { + "audit_status": "FAIL", + "case_sha256": "9e7cf17b027e99c4bb9af650744fa662d0136faf9194b370aabd02fccdf9ce7d", + "expected_primary_signal": "train_eval_leakage", + "fault_id": "data.train_eval_leakage", + "kind": "fault", + "observed_primary_signal": "train_eval_leakage", + "observed_signal_ids": [ + "train_eval_leakage" + ], + "primary_expectation_met": true, + "seed": 4104 + }, + { + "audit_status": "FAIL", + "case_sha256": "a0d4a629a532d604a37ec2b3e728e130afff454208a24293c45416fec6894d32", + "expected_primary_signal": "secret_pattern", + "fault_id": "privacy.secret_pattern", + "kind": "fault", + "observed_primary_signal": "secret_pattern", + "observed_signal_ids": [ + "pii_like", + "secret_pattern" + ], + "primary_expectation_met": true, + "seed": 4104 + }, + { + "audit_status": "PASS", + "case_sha256": "0357c77ddb7b8942eea6dd48c13d13498a34a483ce8d32def788df3903512f7e", + "expected_primary_signal": null, + "fault_id": null, + "kind": "clean_control", + "observed_primary_signal": null, + "observed_signal_ids": [], + "primary_expectation_met": true, + "seed": 4105 + }, + { + "audit_status": "FAIL", + "case_sha256": "d535374ca00d74a1aa5706ba649d0be9509ae7497c84e3ed67fd23966e9b12d8", + "expected_primary_signal": "invalid_role", + "fault_id": "schema.invalid_role", + "kind": "fault", + "observed_primary_signal": "invalid_role", + "observed_signal_ids": [ + "invalid_role", + "role_order" + ], + "primary_expectation_met": true, + "seed": 4105 + }, + { + "audit_status": "FAIL", + "case_sha256": "78b332a3e31c530aed0bac4efaeba2b12d21a857a13c807b0e4d612e7ae00ecd", + "expected_primary_signal": "role_order", + "fault_id": "schema.role_order", + "kind": "fault", + "observed_primary_signal": "role_order", + "observed_signal_ids": [ + "role_order" + ], + "primary_expectation_met": true, + "seed": 4105 + }, + { + "audit_status": "FAIL", + "case_sha256": "301b0b7143a981ec5d5ef2f49f3368ea724fbf84887ff2fe9f81693df423446a", + "expected_primary_signal": "missing_eos", + "fault_id": "template.missing_eos", + "kind": "fault", + "observed_primary_signal": "missing_eos", + "observed_signal_ids": [ + "missing_eos" + ], + "primary_expectation_met": true, + "seed": 4105 + }, + { + "audit_status": "FAIL", + "case_sha256": "c97406f120eb6cd1eca3ac659806bc3d6ee365b1e735352d75aeca0a51437d97", + "expected_primary_signal": "prompt_token_supervised", + "fault_id": "mask.prompt_supervised", + "kind": "fault", + "observed_primary_signal": "prompt_token_supervised", + "observed_signal_ids": [ + "prompt_token_supervised" + ], + "primary_expectation_met": true, + "seed": 4105 + }, + { + "audit_status": "FAIL", + "case_sha256": "c5eefbae3a9dd5582f6a34192366a73fd0db77f960316eb4d763449f5a585d90", + "expected_primary_signal": "assistant_fully_truncated", + "fault_id": "truncation.answer_removed", + "kind": "fault", + "observed_primary_signal": "assistant_fully_truncated", + "observed_signal_ids": [ + "assistant_fully_truncated" + ], + "primary_expectation_met": true, + "seed": 4105 + }, + { + "audit_status": "PASS", + "case_sha256": "573b28256db635507d75ee178391c283a1676402c903b1d323bee0f9e4e7ec21", + "expected_primary_signal": "exact_duplicate", + "fault_id": "data.exact_duplicate", + "kind": "fault", + "observed_primary_signal": "exact_duplicate", + "observed_signal_ids": [ + "exact_duplicate" + ], + "primary_expectation_met": true, + "seed": 4105 + }, + { + "audit_status": "FAIL", + "case_sha256": "4be6046ea204fe4a9d05b91a7e38c2eb1e71cc23d08bfa1a3f94aaa1f8a929d1", + "expected_primary_signal": "prompt_conflict", + "fault_id": "data.prompt_conflict", + "kind": "fault", + "observed_primary_signal": "prompt_conflict", + "observed_signal_ids": [ + "prompt_conflict" + ], + "primary_expectation_met": true, + "seed": 4105 + }, + { + "audit_status": "FAIL", + "case_sha256": "0d89095827d8dd1a028f8ef5fa5714562e099d741c147350fc7e1a1a7e700b45", + "expected_primary_signal": "train_eval_leakage", + "fault_id": "data.train_eval_leakage", + "kind": "fault", + "observed_primary_signal": "train_eval_leakage", + "observed_signal_ids": [ + "train_eval_leakage" + ], + "primary_expectation_met": true, + "seed": 4105 + }, + { + "audit_status": "FAIL", + "case_sha256": "3a05f12c090dffb830820a8a4ba3a0a0a51efaa334f2f00753163b6450f54e86", + "expected_primary_signal": "secret_pattern", + "fault_id": "privacy.secret_pattern", + "kind": "fault", + "observed_primary_signal": "secret_pattern", + "observed_signal_ids": [ + "secret_pattern" + ], + "primary_expectation_met": true, + "seed": 4105 + }, + { + "audit_status": "PASS", + "case_sha256": "35bfa7f3397b36cfb88a64209ada36f1561059efddcda8b15f0cb61f0ba1eada", + "expected_primary_signal": null, + "fault_id": null, + "kind": "clean_control", + "observed_primary_signal": null, + "observed_signal_ids": [], + "primary_expectation_met": true, + "seed": 4106 + }, + { + "audit_status": "FAIL", + "case_sha256": "388b6d77a2733a0b00964b8785f6e997712c2fb2de1b32080b09667dba828c35", + "expected_primary_signal": "invalid_role", + "fault_id": "schema.invalid_role", + "kind": "fault", + "observed_primary_signal": "invalid_role", + "observed_signal_ids": [ + "invalid_role", + "role_order" + ], + "primary_expectation_met": true, + "seed": 4106 + }, + { + "audit_status": "FAIL", + "case_sha256": "0b11d0596e798287903bc1d3adcb723fb4d65aae8282f0dfa77df0fba3155557", + "expected_primary_signal": "role_order", + "fault_id": "schema.role_order", + "kind": "fault", + "observed_primary_signal": "role_order", + "observed_signal_ids": [ + "role_order" + ], + "primary_expectation_met": true, + "seed": 4106 + }, + { + "audit_status": "FAIL", + "case_sha256": "d211f8ebeb51fdbbdeb82ae8d2442b139019305315faa4d895b848b379115f89", + "expected_primary_signal": "missing_eos", + "fault_id": "template.missing_eos", + "kind": "fault", + "observed_primary_signal": "missing_eos", + "observed_signal_ids": [ + "missing_eos" + ], + "primary_expectation_met": true, + "seed": 4106 + }, + { + "audit_status": "FAIL", + "case_sha256": "f2694b79c5a0dfc59d2e974b04bba447ed3592e6497b0eb067ab6e6be0c8ab82", + "expected_primary_signal": "prompt_token_supervised", + "fault_id": "mask.prompt_supervised", + "kind": "fault", + "observed_primary_signal": "prompt_token_supervised", + "observed_signal_ids": [ + "prompt_token_supervised" + ], + "primary_expectation_met": true, + "seed": 4106 + }, + { + "audit_status": "FAIL", + "case_sha256": "a7c0768edfea63f94a95a8e3a02e06989722fe0a3dbfd4718a555112181a8406", + "expected_primary_signal": "assistant_fully_truncated", + "fault_id": "truncation.answer_removed", + "kind": "fault", + "observed_primary_signal": "assistant_fully_truncated", + "observed_signal_ids": [ + "assistant_fully_truncated" + ], + "primary_expectation_met": true, + "seed": 4106 + }, + { + "audit_status": "PASS", + "case_sha256": "f14f86467ad8e543a32cb971f4cc25ef8acf94b58870442df9a8d58dfefbffbf", + "expected_primary_signal": "exact_duplicate", + "fault_id": "data.exact_duplicate", + "kind": "fault", + "observed_primary_signal": "exact_duplicate", + "observed_signal_ids": [ + "exact_duplicate" + ], + "primary_expectation_met": true, + "seed": 4106 + }, + { + "audit_status": "FAIL", + "case_sha256": "bbbd1410e779b083ec3d9244180f4b5836d47d568ffb4bbfec8de2b257d33ae0", + "expected_primary_signal": "prompt_conflict", + "fault_id": "data.prompt_conflict", + "kind": "fault", + "observed_primary_signal": "prompt_conflict", + "observed_signal_ids": [ + "prompt_conflict" + ], + "primary_expectation_met": true, + "seed": 4106 + }, + { + "audit_status": "FAIL", + "case_sha256": "33f97bc06672fef67cabfd447de3e170be89df4867fe0caac23de6570a70b6cd", + "expected_primary_signal": "train_eval_leakage", + "fault_id": "data.train_eval_leakage", + "kind": "fault", + "observed_primary_signal": "train_eval_leakage", + "observed_signal_ids": [ + "train_eval_leakage" + ], + "primary_expectation_met": true, + "seed": 4106 + }, + { + "audit_status": "FAIL", + "case_sha256": "69f0c24c572488595af7a3ebad44a7eb73bf7f1b9e8ac5ab88a555bbb47d07c1", + "expected_primary_signal": "secret_pattern", + "fault_id": "privacy.secret_pattern", + "kind": "fault", + "observed_primary_signal": "secret_pattern", + "observed_signal_ids": [ + "secret_pattern" + ], + "primary_expectation_met": true, + "seed": 4106 + }, + { + "audit_status": "PASS", + "case_sha256": "81f5b10709af8749c64f2c74b4c121f56f063aab38b25a94fad2be049643454f", + "expected_primary_signal": null, + "fault_id": null, + "kind": "clean_control", + "observed_primary_signal": null, + "observed_signal_ids": [], + "primary_expectation_met": true, + "seed": 4107 + }, + { + "audit_status": "FAIL", + "case_sha256": "957d37519b287f6288f437ef940656179f843840e783afdb9ba0176ddfbdcbbd", + "expected_primary_signal": "invalid_role", + "fault_id": "schema.invalid_role", + "kind": "fault", + "observed_primary_signal": "invalid_role", + "observed_signal_ids": [ + "invalid_role", + "role_order" + ], + "primary_expectation_met": true, + "seed": 4107 + }, + { + "audit_status": "FAIL", + "case_sha256": "c824d2e040d07850d875e6a09edd91f102df9358f8b6d7140b41fdce6e146ea6", + "expected_primary_signal": "role_order", + "fault_id": "schema.role_order", + "kind": "fault", + "observed_primary_signal": "role_order", + "observed_signal_ids": [ + "role_order" + ], + "primary_expectation_met": true, + "seed": 4107 + }, + { + "audit_status": "FAIL", + "case_sha256": "e3f42e545ebdcd701932f8d3aada843dbd084537d3cfa7b14846ea3ce8bbadc6", + "expected_primary_signal": "missing_eos", + "fault_id": "template.missing_eos", + "kind": "fault", + "observed_primary_signal": "missing_eos", + "observed_signal_ids": [ + "missing_eos" + ], + "primary_expectation_met": true, + "seed": 4107 + }, + { + "audit_status": "FAIL", + "case_sha256": "539b776c87f1f131537f1ad25bb03ae4155eeacfc8a941b0b66ecf401e124c79", + "expected_primary_signal": "prompt_token_supervised", + "fault_id": "mask.prompt_supervised", + "kind": "fault", + "observed_primary_signal": "prompt_token_supervised", + "observed_signal_ids": [ + "prompt_token_supervised" + ], + "primary_expectation_met": true, + "seed": 4107 + }, + { + "audit_status": "FAIL", + "case_sha256": "1501d0b8811558fa19287c34e128caab1fab28fb5f7308f99f888f0ba7edef50", + "expected_primary_signal": "assistant_fully_truncated", + "fault_id": "truncation.answer_removed", + "kind": "fault", + "observed_primary_signal": "assistant_fully_truncated", + "observed_signal_ids": [ + "assistant_fully_truncated" + ], + "primary_expectation_met": true, + "seed": 4107 + }, + { + "audit_status": "PASS", + "case_sha256": "87ba242d74cfe2c8fedc44a4dfec37a5e14126d21f8137e0c9ea4ce34709cb69", + "expected_primary_signal": "exact_duplicate", + "fault_id": "data.exact_duplicate", + "kind": "fault", + "observed_primary_signal": "exact_duplicate", + "observed_signal_ids": [ + "exact_duplicate" + ], + "primary_expectation_met": true, + "seed": 4107 + }, + { + "audit_status": "FAIL", + "case_sha256": "a7175063e375b753d54089a270b189ecd3a33791c2f17f75c2c963cb053ce8f2", + "expected_primary_signal": "prompt_conflict", + "fault_id": "data.prompt_conflict", + "kind": "fault", + "observed_primary_signal": "prompt_conflict", + "observed_signal_ids": [ + "prompt_conflict" + ], + "primary_expectation_met": true, + "seed": 4107 + }, + { + "audit_status": "FAIL", + "case_sha256": "c177e4613e011f0ea32ed030fe199c25d715594d1ba71d0504556778f5aab5cc", + "expected_primary_signal": "train_eval_leakage", + "fault_id": "data.train_eval_leakage", + "kind": "fault", + "observed_primary_signal": "train_eval_leakage", + "observed_signal_ids": [ + "train_eval_leakage" + ], + "primary_expectation_met": true, + "seed": 4107 + }, + { + "audit_status": "FAIL", + "case_sha256": "5d35601df6a100740e2c4de5e16652c64a90bb4d093672417dec5b95216c4d45", + "expected_primary_signal": "secret_pattern", + "fault_id": "privacy.secret_pattern", + "kind": "fault", + "observed_primary_signal": "secret_pattern", + "observed_signal_ids": [ + "secret_pattern" + ], + "primary_expectation_met": true, + "seed": 4107 + }, + { + "audit_status": "PASS", + "case_sha256": "3f55c9481bed5ccd26a3c4961acd704fba2f1c29d44ff91adb4dbf951a2e551e", + "expected_primary_signal": null, + "fault_id": null, + "kind": "clean_control", + "observed_primary_signal": null, + "observed_signal_ids": [], + "primary_expectation_met": true, + "seed": 4108 + }, + { + "audit_status": "FAIL", + "case_sha256": "4e4bf52788b2088ae42448f7b128e58c7afd35b08d989dbee51e245d5602b0b3", + "expected_primary_signal": "invalid_role", + "fault_id": "schema.invalid_role", + "kind": "fault", + "observed_primary_signal": "invalid_role", + "observed_signal_ids": [ + "invalid_role", + "role_order" + ], + "primary_expectation_met": true, + "seed": 4108 + }, + { + "audit_status": "FAIL", + "case_sha256": "7d1093d24803feaabc083226aa28078c649174c79ed5e16d7812942e40c6db88", + "expected_primary_signal": "role_order", + "fault_id": "schema.role_order", + "kind": "fault", + "observed_primary_signal": "role_order", + "observed_signal_ids": [ + "role_order" + ], + "primary_expectation_met": true, + "seed": 4108 + }, + { + "audit_status": "FAIL", + "case_sha256": "23ac621787cfb11972865e4a03a2b305d96e25ea570145d2f5b074fb6c89fb17", + "expected_primary_signal": "missing_eos", + "fault_id": "template.missing_eos", + "kind": "fault", + "observed_primary_signal": "missing_eos", + "observed_signal_ids": [ + "missing_eos" + ], + "primary_expectation_met": true, + "seed": 4108 + }, + { + "audit_status": "FAIL", + "case_sha256": "7210d38db56ef25b9d6405162656d4f2d29516382ed1540b13dd0c3a7fe3ccf5", + "expected_primary_signal": "prompt_token_supervised", + "fault_id": "mask.prompt_supervised", + "kind": "fault", + "observed_primary_signal": "prompt_token_supervised", + "observed_signal_ids": [ + "prompt_token_supervised" + ], + "primary_expectation_met": true, + "seed": 4108 + }, + { + "audit_status": "FAIL", + "case_sha256": "dc990136da2c86f76cff299f0d7cb418ffa06f1bc2f0e0396cd3829ab72a1d41", + "expected_primary_signal": "assistant_fully_truncated", + "fault_id": "truncation.answer_removed", + "kind": "fault", + "observed_primary_signal": "assistant_fully_truncated", + "observed_signal_ids": [ + "assistant_fully_truncated" + ], + "primary_expectation_met": true, + "seed": 4108 + }, + { + "audit_status": "PASS", + "case_sha256": "4da630ea56f99fe5abdf2a052d52a84d9c1139cff98fafa2150bcb1fce5addeb", + "expected_primary_signal": "exact_duplicate", + "fault_id": "data.exact_duplicate", + "kind": "fault", + "observed_primary_signal": "exact_duplicate", + "observed_signal_ids": [ + "exact_duplicate" + ], + "primary_expectation_met": true, + "seed": 4108 + }, + { + "audit_status": "FAIL", + "case_sha256": "b946eed36e29dd2dcb2fd99b7f5d89b8c3bc406d1fbc902e6445d25bf9d81942", + "expected_primary_signal": "prompt_conflict", + "fault_id": "data.prompt_conflict", + "kind": "fault", + "observed_primary_signal": "prompt_conflict", + "observed_signal_ids": [ + "prompt_conflict" + ], + "primary_expectation_met": true, + "seed": 4108 + }, + { + "audit_status": "FAIL", + "case_sha256": "7e50faf6d57a47afa4e4788bddf5c8a2a56dc0f030dacdb88d1416e092309b17", + "expected_primary_signal": "train_eval_leakage", + "fault_id": "data.train_eval_leakage", + "kind": "fault", + "observed_primary_signal": "train_eval_leakage", + "observed_signal_ids": [ + "train_eval_leakage" + ], + "primary_expectation_met": true, + "seed": 4108 + }, + { + "audit_status": "FAIL", + "case_sha256": "e84634accd0546bc1f7428633989550641c6d535425d70bcbc79cbf70a51eb59", + "expected_primary_signal": "secret_pattern", + "fault_id": "privacy.secret_pattern", + "kind": "fault", + "observed_primary_signal": "secret_pattern", + "observed_signal_ids": [ + "secret_pattern" + ], + "primary_expectation_met": true, + "seed": 4108 + }, + { + "audit_status": "PASS", + "case_sha256": "16ab3ae2999883dfff98fe5c1b9f25bf145d9c111243f61aa59fce5f3393131e", + "expected_primary_signal": null, + "fault_id": null, + "kind": "clean_control", + "observed_primary_signal": null, + "observed_signal_ids": [], + "primary_expectation_met": true, + "seed": 4109 + }, + { + "audit_status": "FAIL", + "case_sha256": "404fad990a87680a5c453030d12b521731161751155602a1679da464a01758a6", + "expected_primary_signal": "invalid_role", + "fault_id": "schema.invalid_role", + "kind": "fault", + "observed_primary_signal": "invalid_role", + "observed_signal_ids": [ + "invalid_role", + "role_order" + ], + "primary_expectation_met": true, + "seed": 4109 + }, + { + "audit_status": "FAIL", + "case_sha256": "54f646751f4d9728aaf5f1b1acf4b387abef8839ca5b3a84418f865fbca2d414", + "expected_primary_signal": "role_order", + "fault_id": "schema.role_order", + "kind": "fault", + "observed_primary_signal": "role_order", + "observed_signal_ids": [ + "role_order" + ], + "primary_expectation_met": true, + "seed": 4109 + }, + { + "audit_status": "FAIL", + "case_sha256": "a8cc437e757c12ac3e8583007c76524da7cdfe3e73c94491dcd72b0497b6315c", + "expected_primary_signal": "missing_eos", + "fault_id": "template.missing_eos", + "kind": "fault", + "observed_primary_signal": "missing_eos", + "observed_signal_ids": [ + "missing_eos" + ], + "primary_expectation_met": true, + "seed": 4109 + }, + { + "audit_status": "FAIL", + "case_sha256": "b4cc2d8e8d880c4c56ca51fffd59f7c84f34c171ea027c93b4eb05ddf3e3cce8", + "expected_primary_signal": "prompt_token_supervised", + "fault_id": "mask.prompt_supervised", + "kind": "fault", + "observed_primary_signal": "prompt_token_supervised", + "observed_signal_ids": [ + "prompt_token_supervised" + ], + "primary_expectation_met": true, + "seed": 4109 + }, + { + "audit_status": "FAIL", + "case_sha256": "3bbcffb09f5f270c189e5809d5536313092ee4b4d0b0b63262a0333d0ad7feef", + "expected_primary_signal": "assistant_fully_truncated", + "fault_id": "truncation.answer_removed", + "kind": "fault", + "observed_primary_signal": "assistant_fully_truncated", + "observed_signal_ids": [ + "assistant_fully_truncated" + ], + "primary_expectation_met": true, + "seed": 4109 + }, + { + "audit_status": "PASS", + "case_sha256": "44a4315abd61560ed375319dbfc32b7935db02f8fdae9ed13628d0859fa46c83", + "expected_primary_signal": "exact_duplicate", + "fault_id": "data.exact_duplicate", + "kind": "fault", + "observed_primary_signal": "exact_duplicate", + "observed_signal_ids": [ + "exact_duplicate" + ], + "primary_expectation_met": true, + "seed": 4109 + }, + { + "audit_status": "FAIL", + "case_sha256": "87fe08cfdc001af12f52cda0e976e33ac176594bc7dad1f3355b8aa6ba6e5725", + "expected_primary_signal": "prompt_conflict", + "fault_id": "data.prompt_conflict", + "kind": "fault", + "observed_primary_signal": "prompt_conflict", + "observed_signal_ids": [ + "prompt_conflict" + ], + "primary_expectation_met": true, + "seed": 4109 + }, + { + "audit_status": "FAIL", + "case_sha256": "ebb6d84aade1e122483ccf316fe832d696d25643507f2ade49312c2c53523220", + "expected_primary_signal": "train_eval_leakage", + "fault_id": "data.train_eval_leakage", + "kind": "fault", + "observed_primary_signal": "train_eval_leakage", + "observed_signal_ids": [ + "train_eval_leakage" + ], + "primary_expectation_met": true, + "seed": 4109 + }, + { + "audit_status": "FAIL", + "case_sha256": "91e6a1877a4110abbeb9c4bfee9e7f5114364a92e81bece8a1847b9e3da4d34c", + "expected_primary_signal": "secret_pattern", + "fault_id": "privacy.secret_pattern", + "kind": "fault", + "observed_primary_signal": "secret_pattern", + "observed_signal_ids": [ + "pii_like", + "secret_pattern" + ], + "primary_expectation_met": true, + "seed": 4109 + }, + { + "audit_status": "PASS", + "case_sha256": "13e22dbd3f550c00d041d6bda9b9af93182d420ec5ce262c7eb1e5be0895596f", + "expected_primary_signal": null, + "fault_id": null, + "kind": "clean_control", + "observed_primary_signal": null, + "observed_signal_ids": [], + "primary_expectation_met": true, + "seed": 4110 + }, + { + "audit_status": "FAIL", + "case_sha256": "a0e045acc17b0cde75dd141deba704525bcf43005e5ac33d003a4aee92ba4dd7", + "expected_primary_signal": "invalid_role", + "fault_id": "schema.invalid_role", + "kind": "fault", + "observed_primary_signal": "invalid_role", + "observed_signal_ids": [ + "invalid_role", + "role_order" + ], + "primary_expectation_met": true, + "seed": 4110 + }, + { + "audit_status": "FAIL", + "case_sha256": "eb7574be69350248970eeefe1c00f22bbe89359d67065d9a44c36b30beeccc4c", + "expected_primary_signal": "role_order", + "fault_id": "schema.role_order", + "kind": "fault", + "observed_primary_signal": "role_order", + "observed_signal_ids": [ + "role_order" + ], + "primary_expectation_met": true, + "seed": 4110 + }, + { + "audit_status": "FAIL", + "case_sha256": "976129c7dfed8f68337072dbbe37ae13f09331212b383a2db767c741842fb8da", + "expected_primary_signal": "missing_eos", + "fault_id": "template.missing_eos", + "kind": "fault", + "observed_primary_signal": "missing_eos", + "observed_signal_ids": [ + "missing_eos" + ], + "primary_expectation_met": true, + "seed": 4110 + }, + { + "audit_status": "FAIL", + "case_sha256": "43241a51afff04a02b750c7cbc5745815455ff9bdd9971e6347e46176211a5b7", + "expected_primary_signal": "prompt_token_supervised", + "fault_id": "mask.prompt_supervised", + "kind": "fault", + "observed_primary_signal": "prompt_token_supervised", + "observed_signal_ids": [ + "prompt_token_supervised" + ], + "primary_expectation_met": true, + "seed": 4110 + }, + { + "audit_status": "FAIL", + "case_sha256": "53e251e6f05f21939c1268553e01a64b60375e4d03c045974ad8c0891c969d5d", + "expected_primary_signal": "assistant_fully_truncated", + "fault_id": "truncation.answer_removed", + "kind": "fault", + "observed_primary_signal": "assistant_fully_truncated", + "observed_signal_ids": [ + "assistant_fully_truncated" + ], + "primary_expectation_met": true, + "seed": 4110 + }, + { + "audit_status": "PASS", + "case_sha256": "44459ba9ab22aa4d9a50b2f43fda8aaa5b1e821ac485aa6872552f11293b5ead", + "expected_primary_signal": "exact_duplicate", + "fault_id": "data.exact_duplicate", + "kind": "fault", + "observed_primary_signal": "exact_duplicate", + "observed_signal_ids": [ + "exact_duplicate" + ], + "primary_expectation_met": true, + "seed": 4110 + }, + { + "audit_status": "FAIL", + "case_sha256": "48974ab2b2bfa90fa343fb85d51d494beb52158af719ebe7986cbafe7de115df", + "expected_primary_signal": "prompt_conflict", + "fault_id": "data.prompt_conflict", + "kind": "fault", + "observed_primary_signal": "prompt_conflict", + "observed_signal_ids": [ + "prompt_conflict" + ], + "primary_expectation_met": true, + "seed": 4110 + }, + { + "audit_status": "FAIL", + "case_sha256": "e5180f667c629783ea010b565639f925584acc429f1df2f1de466e57e1c8c7f1", + "expected_primary_signal": "train_eval_leakage", + "fault_id": "data.train_eval_leakage", + "kind": "fault", + "observed_primary_signal": "train_eval_leakage", + "observed_signal_ids": [ + "train_eval_leakage" + ], + "primary_expectation_met": true, + "seed": 4110 + }, + { + "audit_status": "FAIL", + "case_sha256": "875c951d9750064bc5098aa338d0dc980121af1fdda5683646aecd63f313d486", + "expected_primary_signal": "secret_pattern", + "fault_id": "privacy.secret_pattern", + "kind": "fault", + "observed_primary_signal": "secret_pattern", + "observed_signal_ids": [ + "secret_pattern" + ], + "primary_expectation_met": true, + "seed": 4110 + }, + { + "audit_status": "PASS", + "case_sha256": "7dfddc4a22ceb0e044e816a6156957ac53db4ca228c3f3189ae97dceca363a73", + "expected_primary_signal": null, + "fault_id": null, + "kind": "clean_control", + "observed_primary_signal": null, + "observed_signal_ids": [], + "primary_expectation_met": true, + "seed": 4111 + }, + { + "audit_status": "FAIL", + "case_sha256": "ef72dedad0a502abd977e88f3b7c033a5c84a5580e55a1a975c66a2aaffd8c6b", + "expected_primary_signal": "invalid_role", + "fault_id": "schema.invalid_role", + "kind": "fault", + "observed_primary_signal": "invalid_role", + "observed_signal_ids": [ + "invalid_role", + "role_order" + ], + "primary_expectation_met": true, + "seed": 4111 + }, + { + "audit_status": "FAIL", + "case_sha256": "eb64c69ddc03e8ea030186f2d7c537671a53117dfb93ee8eff4ae6437d85b708", + "expected_primary_signal": "role_order", + "fault_id": "schema.role_order", + "kind": "fault", + "observed_primary_signal": "role_order", + "observed_signal_ids": [ + "role_order" + ], + "primary_expectation_met": true, + "seed": 4111 + }, + { + "audit_status": "FAIL", + "case_sha256": "6f48b88c5d405e7223316c2daae172f7c6474444b0fdf881cad1b4b20fe1f656", + "expected_primary_signal": "missing_eos", + "fault_id": "template.missing_eos", + "kind": "fault", + "observed_primary_signal": "missing_eos", + "observed_signal_ids": [ + "missing_eos" + ], + "primary_expectation_met": true, + "seed": 4111 + }, + { + "audit_status": "FAIL", + "case_sha256": "e4b108d581e197a5d0a8dde4576982638ccc5041a3d148678fb82f63d321f88e", + "expected_primary_signal": "prompt_token_supervised", + "fault_id": "mask.prompt_supervised", + "kind": "fault", + "observed_primary_signal": "prompt_token_supervised", + "observed_signal_ids": [ + "prompt_token_supervised" + ], + "primary_expectation_met": true, + "seed": 4111 + }, + { + "audit_status": "FAIL", + "case_sha256": "af5d14eab27905b91977d52cce4115f05d97d845f00f92f71d53dfcaf1f9cb74", + "expected_primary_signal": "assistant_fully_truncated", + "fault_id": "truncation.answer_removed", + "kind": "fault", + "observed_primary_signal": "assistant_fully_truncated", + "observed_signal_ids": [ + "assistant_fully_truncated" + ], + "primary_expectation_met": true, + "seed": 4111 + }, + { + "audit_status": "PASS", + "case_sha256": "4a8e75bedbba5833d132a74734e1fff6ede0f10a683b81320304ff73bbe52aa6", + "expected_primary_signal": "exact_duplicate", + "fault_id": "data.exact_duplicate", + "kind": "fault", + "observed_primary_signal": "exact_duplicate", + "observed_signal_ids": [ + "exact_duplicate" + ], + "primary_expectation_met": true, + "seed": 4111 + }, + { + "audit_status": "FAIL", + "case_sha256": "6ea9a18dd86d50add2721abcef3d377e79b82cb4148812d1b49ea7735ee44cab", + "expected_primary_signal": "prompt_conflict", + "fault_id": "data.prompt_conflict", + "kind": "fault", + "observed_primary_signal": "prompt_conflict", + "observed_signal_ids": [ + "prompt_conflict" + ], + "primary_expectation_met": true, + "seed": 4111 + }, + { + "audit_status": "FAIL", + "case_sha256": "2cb7fdedf4c31080700661df79230100ff4850f9d074f540682af35a237a3ddf", + "expected_primary_signal": "train_eval_leakage", + "fault_id": "data.train_eval_leakage", + "kind": "fault", + "observed_primary_signal": "train_eval_leakage", + "observed_signal_ids": [ + "train_eval_leakage" + ], + "primary_expectation_met": true, + "seed": 4111 + }, + { + "audit_status": "FAIL", + "case_sha256": "f50dbc3742daa867f38e4f41b50a3fc32ebfa4a349d973283d5d1584f0438504", + "expected_primary_signal": "secret_pattern", + "fault_id": "privacy.secret_pattern", + "kind": "fault", + "observed_primary_signal": "secret_pattern", + "observed_signal_ids": [ + "pii_like", + "secret_pattern" + ], + "primary_expectation_met": true, + "seed": 4111 + }, + { + "audit_status": "PASS", + "case_sha256": "66c3c8e28b6242fd3adaa982629e024e0786474c81d3dd1b4ff9fdd98c60d875", + "expected_primary_signal": null, + "fault_id": null, + "kind": "clean_control", + "observed_primary_signal": null, + "observed_signal_ids": [], + "primary_expectation_met": true, + "seed": 4112 + }, + { + "audit_status": "FAIL", + "case_sha256": "d8f75a1ddb11959ebc5d680d465a345d7016b32d96712be2e50d8a5cd4e57502", + "expected_primary_signal": "invalid_role", + "fault_id": "schema.invalid_role", + "kind": "fault", + "observed_primary_signal": "invalid_role", + "observed_signal_ids": [ + "invalid_role", + "role_order" + ], + "primary_expectation_met": true, + "seed": 4112 + }, + { + "audit_status": "FAIL", + "case_sha256": "e807cd36d8d23773c4266fc1b08ef77db809c612af46ae408fdd55637b6bcf53", + "expected_primary_signal": "role_order", + "fault_id": "schema.role_order", + "kind": "fault", + "observed_primary_signal": "role_order", + "observed_signal_ids": [ + "role_order" + ], + "primary_expectation_met": true, + "seed": 4112 + }, + { + "audit_status": "FAIL", + "case_sha256": "b3ba48bc8895baccde7036e798f554d2993ea52a92e33fdd2869744b68d93b01", + "expected_primary_signal": "missing_eos", + "fault_id": "template.missing_eos", + "kind": "fault", + "observed_primary_signal": "missing_eos", + "observed_signal_ids": [ + "missing_eos" + ], + "primary_expectation_met": true, + "seed": 4112 + }, + { + "audit_status": "FAIL", + "case_sha256": "baa2b53b0563345ba7364eac8c69e8ef939131b40d46c425d3a242125590cfad", + "expected_primary_signal": "prompt_token_supervised", + "fault_id": "mask.prompt_supervised", + "kind": "fault", + "observed_primary_signal": "prompt_token_supervised", + "observed_signal_ids": [ + "prompt_token_supervised" + ], + "primary_expectation_met": true, + "seed": 4112 + }, + { + "audit_status": "FAIL", + "case_sha256": "2433f33f22bc9e2f89b357c47903ff37bb8622249f1197d95338d826068a0b60", + "expected_primary_signal": "assistant_fully_truncated", + "fault_id": "truncation.answer_removed", + "kind": "fault", + "observed_primary_signal": "assistant_fully_truncated", + "observed_signal_ids": [ + "assistant_fully_truncated" + ], + "primary_expectation_met": true, + "seed": 4112 + }, + { + "audit_status": "PASS", + "case_sha256": "8c4bbc28b53f1de0f445b862cf781381aeb5fa9d880959f4050098365f13ef3e", + "expected_primary_signal": "exact_duplicate", + "fault_id": "data.exact_duplicate", + "kind": "fault", + "observed_primary_signal": "exact_duplicate", + "observed_signal_ids": [ + "exact_duplicate" + ], + "primary_expectation_met": true, + "seed": 4112 + }, + { + "audit_status": "FAIL", + "case_sha256": "551a9c8d2f2d57a27dc2bc7d0907824f603ac2f9436b621390f8e882a4138134", + "expected_primary_signal": "prompt_conflict", + "fault_id": "data.prompt_conflict", + "kind": "fault", + "observed_primary_signal": "prompt_conflict", + "observed_signal_ids": [ + "prompt_conflict" + ], + "primary_expectation_met": true, + "seed": 4112 + }, + { + "audit_status": "FAIL", + "case_sha256": "ae7626b6c200b045e878b2baf287a71cd2de15e7e7bbbf16ac3e2d9cda5c7914", + "expected_primary_signal": "train_eval_leakage", + "fault_id": "data.train_eval_leakage", + "kind": "fault", + "observed_primary_signal": "train_eval_leakage", + "observed_signal_ids": [ + "train_eval_leakage" + ], + "primary_expectation_met": true, + "seed": 4112 + }, + { + "audit_status": "FAIL", + "case_sha256": "57887a784b6993c35bdccfd1c009c02bc26599c03e9b56cac3c86e2fc0eca709", + "expected_primary_signal": "secret_pattern", + "fault_id": "privacy.secret_pattern", + "kind": "fault", + "observed_primary_signal": "secret_pattern", + "observed_signal_ids": [ + "secret_pattern" + ], + "primary_expectation_met": true, + "seed": 4112 + }, + { + "audit_status": "PASS", + "case_sha256": "3a54d9b34bcab93e0a87aae007e57a8e0ba8f67bc1d2724b2a5a703174f900da", + "expected_primary_signal": null, + "fault_id": null, + "kind": "clean_control", + "observed_primary_signal": null, + "observed_signal_ids": [], + "primary_expectation_met": true, + "seed": 4113 + }, + { + "audit_status": "FAIL", + "case_sha256": "fab8ce95fd8f7f3e18b04ea9b47735ca3d3c6b04bafca85b04173ea228a421a5", + "expected_primary_signal": "invalid_role", + "fault_id": "schema.invalid_role", + "kind": "fault", + "observed_primary_signal": "invalid_role", + "observed_signal_ids": [ + "invalid_role", + "role_order" + ], + "primary_expectation_met": true, + "seed": 4113 + }, + { + "audit_status": "FAIL", + "case_sha256": "07fffcf0033b9431c2d528cde516b8b1699e4cd80164eab96b0e579b3714bae4", + "expected_primary_signal": "role_order", + "fault_id": "schema.role_order", + "kind": "fault", + "observed_primary_signal": "role_order", + "observed_signal_ids": [ + "role_order" + ], + "primary_expectation_met": true, + "seed": 4113 + }, + { + "audit_status": "FAIL", + "case_sha256": "25d532430f368a47b2bb68886da036df129f54c25420d34eaad9476d17ebbf26", + "expected_primary_signal": "missing_eos", + "fault_id": "template.missing_eos", + "kind": "fault", + "observed_primary_signal": "missing_eos", + "observed_signal_ids": [ + "missing_eos" + ], + "primary_expectation_met": true, + "seed": 4113 + }, + { + "audit_status": "FAIL", + "case_sha256": "1fad6a0bedbebe24c4c0fcd35e30ac7bfe37466101b3153a82dce868ee709f71", + "expected_primary_signal": "prompt_token_supervised", + "fault_id": "mask.prompt_supervised", + "kind": "fault", + "observed_primary_signal": "prompt_token_supervised", + "observed_signal_ids": [ + "prompt_token_supervised" + ], + "primary_expectation_met": true, + "seed": 4113 + }, + { + "audit_status": "FAIL", + "case_sha256": "b94134ebf5075d0ebdf5f1e575e46099675afa57d5f43149fbcfd4e220f65e32", + "expected_primary_signal": "assistant_fully_truncated", + "fault_id": "truncation.answer_removed", + "kind": "fault", + "observed_primary_signal": "assistant_fully_truncated", + "observed_signal_ids": [ + "assistant_fully_truncated" + ], + "primary_expectation_met": true, + "seed": 4113 + }, + { + "audit_status": "PASS", + "case_sha256": "d238e79b8da9726496b5d47d61054d5efe5782357c939a9e003047c1b0a6ae7f", + "expected_primary_signal": "exact_duplicate", + "fault_id": "data.exact_duplicate", + "kind": "fault", + "observed_primary_signal": "exact_duplicate", + "observed_signal_ids": [ + "exact_duplicate" + ], + "primary_expectation_met": true, + "seed": 4113 + }, + { + "audit_status": "FAIL", + "case_sha256": "fa1feeb189e2a127a285a90d3a79e15820b13acbc0158173ff5e39b80f9977dc", + "expected_primary_signal": "prompt_conflict", + "fault_id": "data.prompt_conflict", + "kind": "fault", + "observed_primary_signal": "prompt_conflict", + "observed_signal_ids": [ + "prompt_conflict" + ], + "primary_expectation_met": true, + "seed": 4113 + }, + { + "audit_status": "FAIL", + "case_sha256": "f406f8bf9510703f4464930841dbd918bddf8ec2891312f308d63f99ae827caf", + "expected_primary_signal": "train_eval_leakage", + "fault_id": "data.train_eval_leakage", + "kind": "fault", + "observed_primary_signal": "train_eval_leakage", + "observed_signal_ids": [ + "train_eval_leakage" + ], + "primary_expectation_met": true, + "seed": 4113 + }, + { + "audit_status": "FAIL", + "case_sha256": "bc2133f04751ad6baa0e1538988e5adeed67c4b1c87ea476f414ca3ceaffffe9", + "expected_primary_signal": "secret_pattern", + "fault_id": "privacy.secret_pattern", + "kind": "fault", + "observed_primary_signal": "secret_pattern", + "observed_signal_ids": [ + "secret_pattern" + ], + "primary_expectation_met": true, + "seed": 4113 + }, + { + "audit_status": "PASS", + "case_sha256": "37edd88f5a6a51c0d079669442b7918e13acfb3053b0c2b2b04baeb1a6b8be04", + "expected_primary_signal": null, + "fault_id": null, + "kind": "clean_control", + "observed_primary_signal": null, + "observed_signal_ids": [], + "primary_expectation_met": true, + "seed": 4114 + }, + { + "audit_status": "FAIL", + "case_sha256": "19ab6a4737cb0bbb576754384935431f9ea290c832a370629471dea31bdd597a", + "expected_primary_signal": "invalid_role", + "fault_id": "schema.invalid_role", + "kind": "fault", + "observed_primary_signal": "invalid_role", + "observed_signal_ids": [ + "invalid_role", + "role_order" + ], + "primary_expectation_met": true, + "seed": 4114 + }, + { + "audit_status": "FAIL", + "case_sha256": "ec6198da48a0cf47a598cb82a58325fd60bb21dd312c966a3fb52fac874d7628", + "expected_primary_signal": "role_order", + "fault_id": "schema.role_order", + "kind": "fault", + "observed_primary_signal": "role_order", + "observed_signal_ids": [ + "role_order" + ], + "primary_expectation_met": true, + "seed": 4114 + }, + { + "audit_status": "FAIL", + "case_sha256": "5ad9284041c1cdfecee063dbc61e7b2d23f387e36deb574d312d1d05e05bb3cb", + "expected_primary_signal": "missing_eos", + "fault_id": "template.missing_eos", + "kind": "fault", + "observed_primary_signal": "missing_eos", + "observed_signal_ids": [ + "missing_eos" + ], + "primary_expectation_met": true, + "seed": 4114 + }, + { + "audit_status": "FAIL", + "case_sha256": "72c7b124c34d89907014e8520892b8f8c9fc208e71c295d87be62e95e67f1817", + "expected_primary_signal": "prompt_token_supervised", + "fault_id": "mask.prompt_supervised", + "kind": "fault", + "observed_primary_signal": "prompt_token_supervised", + "observed_signal_ids": [ + "prompt_token_supervised" + ], + "primary_expectation_met": true, + "seed": 4114 + }, + { + "audit_status": "FAIL", + "case_sha256": "10bcb0ee3a4a896dc949c423d04516be468d4b300de74a6bc5d61f12829bf33f", + "expected_primary_signal": "assistant_fully_truncated", + "fault_id": "truncation.answer_removed", + "kind": "fault", + "observed_primary_signal": "assistant_fully_truncated", + "observed_signal_ids": [ + "assistant_fully_truncated" + ], + "primary_expectation_met": true, + "seed": 4114 + }, + { + "audit_status": "PASS", + "case_sha256": "0b153fd092054585f9e42b7e067768a155b58f330b3ff9cc8475135d8c4d7a2f", + "expected_primary_signal": "exact_duplicate", + "fault_id": "data.exact_duplicate", + "kind": "fault", + "observed_primary_signal": "exact_duplicate", + "observed_signal_ids": [ + "exact_duplicate" + ], + "primary_expectation_met": true, + "seed": 4114 + }, + { + "audit_status": "FAIL", + "case_sha256": "a580a0124311660e8e71b23e3d4524e3bb7d9728938c089e081ae85323ac0abb", + "expected_primary_signal": "prompt_conflict", + "fault_id": "data.prompt_conflict", + "kind": "fault", + "observed_primary_signal": "prompt_conflict", + "observed_signal_ids": [ + "prompt_conflict" + ], + "primary_expectation_met": true, + "seed": 4114 + }, + { + "audit_status": "FAIL", + "case_sha256": "97c8be3f733831e8875daa15c3191ad8db916faa2e79811b7fcf6ea38334ae96", + "expected_primary_signal": "train_eval_leakage", + "fault_id": "data.train_eval_leakage", + "kind": "fault", + "observed_primary_signal": "train_eval_leakage", + "observed_signal_ids": [ + "train_eval_leakage" + ], + "primary_expectation_met": true, + "seed": 4114 + }, + { + "audit_status": "FAIL", + "case_sha256": "1baa6ec0933cb9ab5ead20da872704959e47719c5e1f343ccc88cb7eaeb0f0f1", + "expected_primary_signal": "secret_pattern", + "fault_id": "privacy.secret_pattern", + "kind": "fault", + "observed_primary_signal": "secret_pattern", + "observed_signal_ids": [ + "secret_pattern" + ], + "primary_expectation_met": true, + "seed": 4114 + }, + { + "audit_status": "PASS", + "case_sha256": "bbef328b37eefea4cab9e39000fb9a80543e05ad3c58d3b5035c332ccbb6a4bb", + "expected_primary_signal": null, + "fault_id": null, + "kind": "clean_control", + "observed_primary_signal": null, + "observed_signal_ids": [], + "primary_expectation_met": true, + "seed": 4115 + }, + { + "audit_status": "FAIL", + "case_sha256": "28e9f96f3239b2e11744226be8926d97016c8c2062720a642e1d35169ace75de", + "expected_primary_signal": "invalid_role", + "fault_id": "schema.invalid_role", + "kind": "fault", + "observed_primary_signal": "invalid_role", + "observed_signal_ids": [ + "invalid_role", + "role_order" + ], + "primary_expectation_met": true, + "seed": 4115 + }, + { + "audit_status": "FAIL", + "case_sha256": "7b43a727e7f5b064315e905556a05e6e82d560fa5b53320790f430483b44b26a", + "expected_primary_signal": "role_order", + "fault_id": "schema.role_order", + "kind": "fault", + "observed_primary_signal": "role_order", + "observed_signal_ids": [ + "role_order" + ], + "primary_expectation_met": true, + "seed": 4115 + }, + { + "audit_status": "FAIL", + "case_sha256": "5dd9544480802ef6a5d0f5ed5272a6f462f9e334863ede7ab4b96fedd8e97da3", + "expected_primary_signal": "missing_eos", + "fault_id": "template.missing_eos", + "kind": "fault", + "observed_primary_signal": "missing_eos", + "observed_signal_ids": [ + "missing_eos" + ], + "primary_expectation_met": true, + "seed": 4115 + }, + { + "audit_status": "FAIL", + "case_sha256": "a6426429f1dde802b67152b70d7a951a220039633665ded659405fad3b7f694d", + "expected_primary_signal": "prompt_token_supervised", + "fault_id": "mask.prompt_supervised", + "kind": "fault", + "observed_primary_signal": "prompt_token_supervised", + "observed_signal_ids": [ + "prompt_token_supervised" + ], + "primary_expectation_met": true, + "seed": 4115 + }, + { + "audit_status": "FAIL", + "case_sha256": "6ed2116fb16d55c4045b1ed9f751596e72ca6b81d3501a1306b2870e247a5c71", + "expected_primary_signal": "assistant_fully_truncated", + "fault_id": "truncation.answer_removed", + "kind": "fault", + "observed_primary_signal": "assistant_fully_truncated", + "observed_signal_ids": [ + "assistant_fully_truncated" + ], + "primary_expectation_met": true, + "seed": 4115 + }, + { + "audit_status": "PASS", + "case_sha256": "1946e5194c6b0215a036f9ce87881450d7769562561d20c353b1171fe3ec0e5e", + "expected_primary_signal": "exact_duplicate", + "fault_id": "data.exact_duplicate", + "kind": "fault", + "observed_primary_signal": "exact_duplicate", + "observed_signal_ids": [ + "exact_duplicate" + ], + "primary_expectation_met": true, + "seed": 4115 + }, + { + "audit_status": "FAIL", + "case_sha256": "54803dddf060c5df66ae3a294d2200a9ad74b634cec0e6714a27bb5128041e6d", + "expected_primary_signal": "prompt_conflict", + "fault_id": "data.prompt_conflict", + "kind": "fault", + "observed_primary_signal": "prompt_conflict", + "observed_signal_ids": [ + "prompt_conflict" + ], + "primary_expectation_met": true, + "seed": 4115 + }, + { + "audit_status": "FAIL", + "case_sha256": "67707cb5152ebb39e404484a72579c5d85e9b832bfc3b84f0709347ab5ce2f3c", + "expected_primary_signal": "train_eval_leakage", + "fault_id": "data.train_eval_leakage", + "kind": "fault", + "observed_primary_signal": "train_eval_leakage", + "observed_signal_ids": [ + "train_eval_leakage" + ], + "primary_expectation_met": true, + "seed": 4115 + }, + { + "audit_status": "FAIL", + "case_sha256": "4feff72c16b99af980e07f73af5d09f0653e59064d29572af5c119703d145efd", + "expected_primary_signal": "secret_pattern", + "fault_id": "privacy.secret_pattern", + "kind": "fault", + "observed_primary_signal": "secret_pattern", + "observed_signal_ids": [ + "secret_pattern" + ], + "primary_expectation_met": true, + "seed": 4115 + }, + { + "audit_status": "PASS", + "case_sha256": "544fe66e06df9d8eae8b514f82d188c5805f54c6b44e8c9e99fde81743ae6088", + "expected_primary_signal": null, + "fault_id": null, + "kind": "clean_control", + "observed_primary_signal": null, + "observed_signal_ids": [], + "primary_expectation_met": true, + "seed": 4116 + }, + { + "audit_status": "FAIL", + "case_sha256": "dd1f2be80def6c5c35d27b765bdc260a5f87fc891b04da9f808b5b1aac84d375", + "expected_primary_signal": "invalid_role", + "fault_id": "schema.invalid_role", + "kind": "fault", + "observed_primary_signal": "invalid_role", + "observed_signal_ids": [ + "invalid_role", + "role_order" + ], + "primary_expectation_met": true, + "seed": 4116 + }, + { + "audit_status": "FAIL", + "case_sha256": "1c87f9a38cca9996a642bd468eca53ef29035199a2538771bd53332fc208458b", + "expected_primary_signal": "role_order", + "fault_id": "schema.role_order", + "kind": "fault", + "observed_primary_signal": "role_order", + "observed_signal_ids": [ + "role_order" + ], + "primary_expectation_met": true, + "seed": 4116 + }, + { + "audit_status": "FAIL", + "case_sha256": "eb65d44235588c2701caf61b97096fd6be929066a7a816811f3ade539c07d5d5", + "expected_primary_signal": "missing_eos", + "fault_id": "template.missing_eos", + "kind": "fault", + "observed_primary_signal": "missing_eos", + "observed_signal_ids": [ + "missing_eos" + ], + "primary_expectation_met": true, + "seed": 4116 + }, + { + "audit_status": "FAIL", + "case_sha256": "e82ee26e27e79d60f29af20bb9a8ac0c3a9ed7e2e173106e2ae7b9dff2e31990", + "expected_primary_signal": "prompt_token_supervised", + "fault_id": "mask.prompt_supervised", + "kind": "fault", + "observed_primary_signal": "prompt_token_supervised", + "observed_signal_ids": [ + "prompt_token_supervised" + ], + "primary_expectation_met": true, + "seed": 4116 + }, + { + "audit_status": "FAIL", + "case_sha256": "8c80ab26509dd24f248d856d9f9ae4ea35b417bdb03bc8afe02746b80e0745f1", + "expected_primary_signal": "assistant_fully_truncated", + "fault_id": "truncation.answer_removed", + "kind": "fault", + "observed_primary_signal": "assistant_fully_truncated", + "observed_signal_ids": [ + "assistant_fully_truncated" + ], + "primary_expectation_met": true, + "seed": 4116 + }, + { + "audit_status": "PASS", + "case_sha256": "db01b70329733c65d34c0fb90ccb607a01edbf63798ed62d5d86c605150cf728", + "expected_primary_signal": "exact_duplicate", + "fault_id": "data.exact_duplicate", + "kind": "fault", + "observed_primary_signal": "exact_duplicate", + "observed_signal_ids": [ + "exact_duplicate" + ], + "primary_expectation_met": true, + "seed": 4116 + }, + { + "audit_status": "FAIL", + "case_sha256": "8d8ffe170bb76fb2c6e6e2e8454c1621d186da05346a1bca47cb9217b6e20a53", + "expected_primary_signal": "prompt_conflict", + "fault_id": "data.prompt_conflict", + "kind": "fault", + "observed_primary_signal": "prompt_conflict", + "observed_signal_ids": [ + "prompt_conflict" + ], + "primary_expectation_met": true, + "seed": 4116 + }, + { + "audit_status": "FAIL", + "case_sha256": "f61c7b068547b6b5e13ca75f6a003bc0a83a39ea30ec64f7f34772f765b84827", + "expected_primary_signal": "train_eval_leakage", + "fault_id": "data.train_eval_leakage", + "kind": "fault", + "observed_primary_signal": "train_eval_leakage", + "observed_signal_ids": [ + "train_eval_leakage" + ], + "primary_expectation_met": true, + "seed": 4116 + }, + { + "audit_status": "FAIL", + "case_sha256": "96d3e596f7e837b8099b34b07cb8d5fe1f456b82617411bbf29d38a6e594ced0", + "expected_primary_signal": "secret_pattern", + "fault_id": "privacy.secret_pattern", + "kind": "fault", + "observed_primary_signal": "secret_pattern", + "observed_signal_ids": [ + "secret_pattern" + ], + "primary_expectation_met": true, + "seed": 4116 + }, + { + "audit_status": "PASS", + "case_sha256": "e3578eac6138c96ea5f83a44128a892d5b532df56beb797b336f7b416f4b4cd8", + "expected_primary_signal": null, + "fault_id": null, + "kind": "clean_control", + "observed_primary_signal": null, + "observed_signal_ids": [], + "primary_expectation_met": true, + "seed": 4117 + }, + { + "audit_status": "FAIL", + "case_sha256": "5761871ebadca00ffeb6ce694a8a99b02617b188a26ae96b06def327fbdfa022", + "expected_primary_signal": "invalid_role", + "fault_id": "schema.invalid_role", + "kind": "fault", + "observed_primary_signal": "invalid_role", + "observed_signal_ids": [ + "invalid_role", + "role_order" + ], + "primary_expectation_met": true, + "seed": 4117 + }, + { + "audit_status": "FAIL", + "case_sha256": "26332e6164d02075e6f8a63979be5d2dacc14a6d1357d2d34bf9a595a6e2a3e8", + "expected_primary_signal": "role_order", + "fault_id": "schema.role_order", + "kind": "fault", + "observed_primary_signal": "role_order", + "observed_signal_ids": [ + "role_order" + ], + "primary_expectation_met": true, + "seed": 4117 + }, + { + "audit_status": "FAIL", + "case_sha256": "1020bd0cbbb199b2f53e320429c1a0ddc8e44cd99ec9f3fb3834e5e7a82b54d5", + "expected_primary_signal": "missing_eos", + "fault_id": "template.missing_eos", + "kind": "fault", + "observed_primary_signal": "missing_eos", + "observed_signal_ids": [ + "missing_eos" + ], + "primary_expectation_met": true, + "seed": 4117 + }, + { + "audit_status": "FAIL", + "case_sha256": "d00ff25cf3db7f51ed80fec4de8418a1f6562673a25d818c6c96ca195a6099bf", + "expected_primary_signal": "prompt_token_supervised", + "fault_id": "mask.prompt_supervised", + "kind": "fault", + "observed_primary_signal": "prompt_token_supervised", + "observed_signal_ids": [ + "prompt_token_supervised" + ], + "primary_expectation_met": true, + "seed": 4117 + }, + { + "audit_status": "FAIL", + "case_sha256": "dfa3cbbca0df85b4ae04b734512d5f1bead010cda38c196231ab430c1775dcea", + "expected_primary_signal": "assistant_fully_truncated", + "fault_id": "truncation.answer_removed", + "kind": "fault", + "observed_primary_signal": "assistant_fully_truncated", + "observed_signal_ids": [ + "assistant_fully_truncated" + ], + "primary_expectation_met": true, + "seed": 4117 + }, + { + "audit_status": "PASS", + "case_sha256": "9697c78c821fc1da30e790c182c52545ebeee817f7b985e217a37cb06c202a0b", + "expected_primary_signal": "exact_duplicate", + "fault_id": "data.exact_duplicate", + "kind": "fault", + "observed_primary_signal": "exact_duplicate", + "observed_signal_ids": [ + "exact_duplicate" + ], + "primary_expectation_met": true, + "seed": 4117 + }, + { + "audit_status": "FAIL", + "case_sha256": "f64f6bfd907b608a7e888f9683c2eba0b1d19082254f8a05360728e5e9f2c07e", + "expected_primary_signal": "prompt_conflict", + "fault_id": "data.prompt_conflict", + "kind": "fault", + "observed_primary_signal": "prompt_conflict", + "observed_signal_ids": [ + "prompt_conflict" + ], + "primary_expectation_met": true, + "seed": 4117 + }, + { + "audit_status": "FAIL", + "case_sha256": "bddeee3f5fb20185acd8a7ff345dac9e4beee1dba2d98fe4eccd0442eebf5092", + "expected_primary_signal": "train_eval_leakage", + "fault_id": "data.train_eval_leakage", + "kind": "fault", + "observed_primary_signal": "train_eval_leakage", + "observed_signal_ids": [ + "train_eval_leakage" + ], + "primary_expectation_met": true, + "seed": 4117 + }, + { + "audit_status": "FAIL", + "case_sha256": "3ea3daec9df1f75436493190b62b231f2c422f0c27cf277ea022c0fe72d7a5e4", + "expected_primary_signal": "secret_pattern", + "fault_id": "privacy.secret_pattern", + "kind": "fault", + "observed_primary_signal": "secret_pattern", + "observed_signal_ids": [ + "secret_pattern" + ], + "primary_expectation_met": true, + "seed": 4117 + }, + { + "audit_status": "PASS", + "case_sha256": "215b2b8a2b69cdccb8d86e09367e4c638ef672bc56001bf6af85612fa1d8110a", + "expected_primary_signal": null, + "fault_id": null, + "kind": "clean_control", + "observed_primary_signal": null, + "observed_signal_ids": [], + "primary_expectation_met": true, + "seed": 4118 + }, + { + "audit_status": "FAIL", + "case_sha256": "34602eff64f63429a792fe5b7f4dd70c8b531dd92dce1bcf9cdc89e735acac05", + "expected_primary_signal": "invalid_role", + "fault_id": "schema.invalid_role", + "kind": "fault", + "observed_primary_signal": "invalid_role", + "observed_signal_ids": [ + "invalid_role", + "role_order" + ], + "primary_expectation_met": true, + "seed": 4118 + }, + { + "audit_status": "FAIL", + "case_sha256": "c327b0d7b79a982d094bcb155dc619291b81a07c9405d0cb893ff488fd99cce1", + "expected_primary_signal": "role_order", + "fault_id": "schema.role_order", + "kind": "fault", + "observed_primary_signal": "role_order", + "observed_signal_ids": [ + "role_order" + ], + "primary_expectation_met": true, + "seed": 4118 + }, + { + "audit_status": "FAIL", + "case_sha256": "31b4610f45804044abffa2747bdc0ab0e76c2afc6c2dfec79ae37bfe569c8737", + "expected_primary_signal": "missing_eos", + "fault_id": "template.missing_eos", + "kind": "fault", + "observed_primary_signal": "missing_eos", + "observed_signal_ids": [ + "missing_eos" + ], + "primary_expectation_met": true, + "seed": 4118 + }, + { + "audit_status": "FAIL", + "case_sha256": "27b3db8b00cd853d71beefbb7c22bd38aa6f526614586daa3e44a4bf79bb405a", + "expected_primary_signal": "prompt_token_supervised", + "fault_id": "mask.prompt_supervised", + "kind": "fault", + "observed_primary_signal": "prompt_token_supervised", + "observed_signal_ids": [ + "prompt_token_supervised" + ], + "primary_expectation_met": true, + "seed": 4118 + }, + { + "audit_status": "FAIL", + "case_sha256": "ff6ae3e4a32a5bed0338e57bbb281aace312b55554ac6a73f4edfdf5c25b9300", + "expected_primary_signal": "assistant_fully_truncated", + "fault_id": "truncation.answer_removed", + "kind": "fault", + "observed_primary_signal": "assistant_fully_truncated", + "observed_signal_ids": [ + "assistant_fully_truncated" + ], + "primary_expectation_met": true, + "seed": 4118 + }, + { + "audit_status": "PASS", + "case_sha256": "bf2ac15a331d2219cf33e51ac9bf6f12d68ac2ecb000e97ed4cc88610a95e7db", + "expected_primary_signal": "exact_duplicate", + "fault_id": "data.exact_duplicate", + "kind": "fault", + "observed_primary_signal": "exact_duplicate", + "observed_signal_ids": [ + "exact_duplicate" + ], + "primary_expectation_met": true, + "seed": 4118 + }, + { + "audit_status": "FAIL", + "case_sha256": "1ab7211f6c6d93209b26f92c378773996d72f376819e147ab3c7741c8c24848a", + "expected_primary_signal": "prompt_conflict", + "fault_id": "data.prompt_conflict", + "kind": "fault", + "observed_primary_signal": "prompt_conflict", + "observed_signal_ids": [ + "prompt_conflict" + ], + "primary_expectation_met": true, + "seed": 4118 + }, + { + "audit_status": "FAIL", + "case_sha256": "f9995f5b0657d034246c90353857e919463a3ae791613aead70a12425dfe38d4", + "expected_primary_signal": "train_eval_leakage", + "fault_id": "data.train_eval_leakage", + "kind": "fault", + "observed_primary_signal": "train_eval_leakage", + "observed_signal_ids": [ + "train_eval_leakage" + ], + "primary_expectation_met": true, + "seed": 4118 + }, + { + "audit_status": "FAIL", + "case_sha256": "1b605516e660fed7390537d33aa27bb0905d559f2d7ee54d41bee88de3308c9f", + "expected_primary_signal": "secret_pattern", + "fault_id": "privacy.secret_pattern", + "kind": "fault", + "observed_primary_signal": "secret_pattern", + "observed_signal_ids": [ + "secret_pattern" + ], + "primary_expectation_met": true, + "seed": 4118 + }, + { + "audit_status": "PASS", + "case_sha256": "578d5488ca1f03f66a2e51daa018cbfb4eb3695257cf37f19dce266d5b47f957", + "expected_primary_signal": null, + "fault_id": null, + "kind": "clean_control", + "observed_primary_signal": null, + "observed_signal_ids": [], + "primary_expectation_met": true, + "seed": 4119 + }, + { + "audit_status": "FAIL", + "case_sha256": "dbc2f38040fc5c4aaf389baf680e8134d3f5d673ff28e5d05e7ab1d36cb41b01", + "expected_primary_signal": "invalid_role", + "fault_id": "schema.invalid_role", + "kind": "fault", + "observed_primary_signal": "invalid_role", + "observed_signal_ids": [ + "invalid_role", + "role_order" + ], + "primary_expectation_met": true, + "seed": 4119 + }, + { + "audit_status": "FAIL", + "case_sha256": "cc4d2d5deebbedd2f65e5e00fe290c4a9ba88f880c1c912e38a2f5389c500976", + "expected_primary_signal": "role_order", + "fault_id": "schema.role_order", + "kind": "fault", + "observed_primary_signal": "role_order", + "observed_signal_ids": [ + "role_order" + ], + "primary_expectation_met": true, + "seed": 4119 + }, + { + "audit_status": "FAIL", + "case_sha256": "6aba84bee2b373f47df6d634a47753019219b31071ab1eae5231b7d0c92ebce2", + "expected_primary_signal": "missing_eos", + "fault_id": "template.missing_eos", + "kind": "fault", + "observed_primary_signal": "missing_eos", + "observed_signal_ids": [ + "missing_eos" + ], + "primary_expectation_met": true, + "seed": 4119 + }, + { + "audit_status": "FAIL", + "case_sha256": "49d52d6c85a7251a6b8f7e3048d3a12ab3f5758d6b94457d2c7c1f66d0ef74c3", + "expected_primary_signal": "prompt_token_supervised", + "fault_id": "mask.prompt_supervised", + "kind": "fault", + "observed_primary_signal": "prompt_token_supervised", + "observed_signal_ids": [ + "prompt_token_supervised" + ], + "primary_expectation_met": true, + "seed": 4119 + }, + { + "audit_status": "FAIL", + "case_sha256": "4fc10dad18380fb400341fb4f3c31d43200aa2f61f0dd610c4613892f7949311", + "expected_primary_signal": "assistant_fully_truncated", + "fault_id": "truncation.answer_removed", + "kind": "fault", + "observed_primary_signal": "assistant_fully_truncated", + "observed_signal_ids": [ + "assistant_fully_truncated" + ], + "primary_expectation_met": true, + "seed": 4119 + }, + { + "audit_status": "PASS", + "case_sha256": "26509933959d9b6bb31fed0040277069c4d501b594f26f3d10a421c9de87794e", + "expected_primary_signal": "exact_duplicate", + "fault_id": "data.exact_duplicate", + "kind": "fault", + "observed_primary_signal": "exact_duplicate", + "observed_signal_ids": [ + "exact_duplicate" + ], + "primary_expectation_met": true, + "seed": 4119 + }, + { + "audit_status": "FAIL", + "case_sha256": "b6b6ef537498600353c38b4250cc1c0634e1ef73a9fda8f5f8135f5d25b1c606", + "expected_primary_signal": "prompt_conflict", + "fault_id": "data.prompt_conflict", + "kind": "fault", + "observed_primary_signal": "prompt_conflict", + "observed_signal_ids": [ + "prompt_conflict" + ], + "primary_expectation_met": true, + "seed": 4119 + }, + { + "audit_status": "FAIL", + "case_sha256": "9d5ec3586d48dd994eb523b543561e5b28e1ea7eee902e17a2fb7982e406903a", + "expected_primary_signal": "train_eval_leakage", + "fault_id": "data.train_eval_leakage", + "kind": "fault", + "observed_primary_signal": "train_eval_leakage", + "observed_signal_ids": [ + "train_eval_leakage" + ], + "primary_expectation_met": true, + "seed": 4119 + }, + { + "audit_status": "FAIL", + "case_sha256": "9169c633041ba1391853585cde0fb9596a8eb37c07710f90b5ed790dad8da86a", + "expected_primary_signal": "secret_pattern", + "fault_id": "privacy.secret_pattern", + "kind": "fault", + "observed_primary_signal": "secret_pattern", + "observed_signal_ids": [ + "secret_pattern" + ], + "primary_expectation_met": true, + "seed": 4119 + }, + { + "audit_status": "PASS", + "case_sha256": "bbcd8caa00e3132793f45b5c8f33cd53516d15dca283a5764043c05fbb8d3bf3", + "expected_primary_signal": null, + "fault_id": null, + "kind": "clean_control", + "observed_primary_signal": null, + "observed_signal_ids": [], + "primary_expectation_met": true, + "seed": 4120 + }, + { + "audit_status": "FAIL", + "case_sha256": "4fa740c406c8363a90ea8aedc2978350951b8e15db19384dc15709321fa196c9", + "expected_primary_signal": "invalid_role", + "fault_id": "schema.invalid_role", + "kind": "fault", + "observed_primary_signal": "invalid_role", + "observed_signal_ids": [ + "invalid_role", + "role_order" + ], + "primary_expectation_met": true, + "seed": 4120 + }, + { + "audit_status": "FAIL", + "case_sha256": "578ff72f19c615e74982be92aa0ca95910660d2b4771d0e1119103bec0dcf12a", + "expected_primary_signal": "role_order", + "fault_id": "schema.role_order", + "kind": "fault", + "observed_primary_signal": "role_order", + "observed_signal_ids": [ + "role_order" + ], + "primary_expectation_met": true, + "seed": 4120 + }, + { + "audit_status": "FAIL", + "case_sha256": "7544417a8c3fc5269e10a179fb154fb8325128a50110ff6d8aa215df819ecc0f", + "expected_primary_signal": "missing_eos", + "fault_id": "template.missing_eos", + "kind": "fault", + "observed_primary_signal": "missing_eos", + "observed_signal_ids": [ + "missing_eos" + ], + "primary_expectation_met": true, + "seed": 4120 + }, + { + "audit_status": "FAIL", + "case_sha256": "e2b3b672895770e376fd76c5d0aa4e9b669a4096b00f8bce567381283d042170", + "expected_primary_signal": "prompt_token_supervised", + "fault_id": "mask.prompt_supervised", + "kind": "fault", + "observed_primary_signal": "prompt_token_supervised", + "observed_signal_ids": [ + "prompt_token_supervised" + ], + "primary_expectation_met": true, + "seed": 4120 + }, + { + "audit_status": "FAIL", + "case_sha256": "38df8f121fd79663e94b9fc02600c98931f4a09e9acd33ea44cd8e0e30c784bd", + "expected_primary_signal": "assistant_fully_truncated", + "fault_id": "truncation.answer_removed", + "kind": "fault", + "observed_primary_signal": "assistant_fully_truncated", + "observed_signal_ids": [ + "assistant_fully_truncated" + ], + "primary_expectation_met": true, + "seed": 4120 + }, + { + "audit_status": "PASS", + "case_sha256": "6756cb61538b98de82b7b43102d5d706b5b0e5193c53f730aa89738141f4c062", + "expected_primary_signal": "exact_duplicate", + "fault_id": "data.exact_duplicate", + "kind": "fault", + "observed_primary_signal": "exact_duplicate", + "observed_signal_ids": [ + "exact_duplicate" + ], + "primary_expectation_met": true, + "seed": 4120 + }, + { + "audit_status": "FAIL", + "case_sha256": "048e1dd2077ac83a6f0f57d60a175ca05d833b6b158c039f9cf069cd57f18d51", + "expected_primary_signal": "prompt_conflict", + "fault_id": "data.prompt_conflict", + "kind": "fault", + "observed_primary_signal": "prompt_conflict", + "observed_signal_ids": [ + "prompt_conflict" + ], + "primary_expectation_met": true, + "seed": 4120 + }, + { + "audit_status": "FAIL", + "case_sha256": "7563f724286e0fd0b8f7865c9aae6f5e32c1a48005b084dc6832a0047c53311d", + "expected_primary_signal": "train_eval_leakage", + "fault_id": "data.train_eval_leakage", + "kind": "fault", + "observed_primary_signal": "train_eval_leakage", + "observed_signal_ids": [ + "train_eval_leakage" + ], + "primary_expectation_met": true, + "seed": 4120 + }, + { + "audit_status": "FAIL", + "case_sha256": "867695627e032e4647af711b708ec63ab192f8c916da84639591f4e0c82c3b06", + "expected_primary_signal": "secret_pattern", + "fault_id": "privacy.secret_pattern", + "kind": "fault", + "observed_primary_signal": "secret_pattern", + "observed_signal_ids": [ + "secret_pattern" + ], + "primary_expectation_met": true, + "seed": 4120 + }, + { + "audit_status": "PASS", + "case_sha256": "d97b7d1917ba56d7feb21f6003d888c25495df43edf0512dcf9a0611a7b7a289", + "expected_primary_signal": null, + "fault_id": null, + "kind": "clean_control", + "observed_primary_signal": null, + "observed_signal_ids": [], + "primary_expectation_met": true, + "seed": 4121 + }, + { + "audit_status": "FAIL", + "case_sha256": "411af9edda759f8439213951e037409284f2631a959e8f1fc751d4b664565a7e", + "expected_primary_signal": "invalid_role", + "fault_id": "schema.invalid_role", + "kind": "fault", + "observed_primary_signal": "invalid_role", + "observed_signal_ids": [ + "invalid_role", + "role_order" + ], + "primary_expectation_met": true, + "seed": 4121 + }, + { + "audit_status": "FAIL", + "case_sha256": "1f77e9b8573165cdad20e21f050b20654a5ead43cf0535ba1e6b51eee4d701f2", + "expected_primary_signal": "role_order", + "fault_id": "schema.role_order", + "kind": "fault", + "observed_primary_signal": "role_order", + "observed_signal_ids": [ + "role_order" + ], + "primary_expectation_met": true, + "seed": 4121 + }, + { + "audit_status": "FAIL", + "case_sha256": "132739f8e130a177843ebf49911f6bc1a651fd9d6bebcc7626349b46cba0a230", + "expected_primary_signal": "missing_eos", + "fault_id": "template.missing_eos", + "kind": "fault", + "observed_primary_signal": "missing_eos", + "observed_signal_ids": [ + "missing_eos" + ], + "primary_expectation_met": true, + "seed": 4121 + }, + { + "audit_status": "FAIL", + "case_sha256": "ce87a3034e82cf4a6a60d443c45e3b3f743dda499d5d9f9780e7a2c1c5684797", + "expected_primary_signal": "prompt_token_supervised", + "fault_id": "mask.prompt_supervised", + "kind": "fault", + "observed_primary_signal": "prompt_token_supervised", + "observed_signal_ids": [ + "prompt_token_supervised" + ], + "primary_expectation_met": true, + "seed": 4121 + }, + { + "audit_status": "FAIL", + "case_sha256": "f0a8e6bd6ab4b6e333c76d79f714c5e7982d06970115eb388b2e85e62ac193ac", + "expected_primary_signal": "assistant_fully_truncated", + "fault_id": "truncation.answer_removed", + "kind": "fault", + "observed_primary_signal": "assistant_fully_truncated", + "observed_signal_ids": [ + "assistant_fully_truncated" + ], + "primary_expectation_met": true, + "seed": 4121 + }, + { + "audit_status": "PASS", + "case_sha256": "b19b7835729877ee3cd7e875db97cd8c588013b4f9fd561c0f2c23ad1a0a5957", + "expected_primary_signal": "exact_duplicate", + "fault_id": "data.exact_duplicate", + "kind": "fault", + "observed_primary_signal": "exact_duplicate", + "observed_signal_ids": [ + "exact_duplicate" + ], + "primary_expectation_met": true, + "seed": 4121 + }, + { + "audit_status": "FAIL", + "case_sha256": "a92282fc9334146eabf38a1f71dac494f50cd8d7920167ae30d3a0df67e4e0f1", + "expected_primary_signal": "prompt_conflict", + "fault_id": "data.prompt_conflict", + "kind": "fault", + "observed_primary_signal": "prompt_conflict", + "observed_signal_ids": [ + "prompt_conflict" + ], + "primary_expectation_met": true, + "seed": 4121 + }, + { + "audit_status": "FAIL", + "case_sha256": "be34c3262556af1cc65bad81e503c05e353245281822d4fa2f5cb49c90d55a93", + "expected_primary_signal": "train_eval_leakage", + "fault_id": "data.train_eval_leakage", + "kind": "fault", + "observed_primary_signal": "train_eval_leakage", + "observed_signal_ids": [ + "train_eval_leakage" + ], + "primary_expectation_met": true, + "seed": 4121 + }, + { + "audit_status": "FAIL", + "case_sha256": "a84d79305a8d11bc1c6c30ab20914475e8aa868783e2e65cb861495f2dc23e71", + "expected_primary_signal": "secret_pattern", + "fault_id": "privacy.secret_pattern", + "kind": "fault", + "observed_primary_signal": "secret_pattern", + "observed_signal_ids": [ + "secret_pattern" + ], + "primary_expectation_met": true, + "seed": 4121 + }, + { + "audit_status": "PASS", + "case_sha256": "3821ce93d22fad6a9b4aca153694f02360ec8e48dfb556cb152cbd06fd21408e", + "expected_primary_signal": null, + "fault_id": null, + "kind": "clean_control", + "observed_primary_signal": null, + "observed_signal_ids": [], + "primary_expectation_met": true, + "seed": 4122 + }, + { + "audit_status": "FAIL", + "case_sha256": "e9289e9839d261916b40faf59bf063d16f8938d5e2b0d7d2786b840092b0464d", + "expected_primary_signal": "invalid_role", + "fault_id": "schema.invalid_role", + "kind": "fault", + "observed_primary_signal": "invalid_role", + "observed_signal_ids": [ + "invalid_role", + "role_order" + ], + "primary_expectation_met": true, + "seed": 4122 + }, + { + "audit_status": "FAIL", + "case_sha256": "dd5ac2328dd6fad35f0f26e5d720c40aff8b3cc5016d150d4448570ca3c5dbfb", + "expected_primary_signal": "role_order", + "fault_id": "schema.role_order", + "kind": "fault", + "observed_primary_signal": "role_order", + "observed_signal_ids": [ + "role_order" + ], + "primary_expectation_met": true, + "seed": 4122 + }, + { + "audit_status": "FAIL", + "case_sha256": "8961470746342496bfebc6fd31724c058a1d8fab7c5820e2b42a6a9df9349f52", + "expected_primary_signal": "missing_eos", + "fault_id": "template.missing_eos", + "kind": "fault", + "observed_primary_signal": "missing_eos", + "observed_signal_ids": [ + "missing_eos" + ], + "primary_expectation_met": true, + "seed": 4122 + }, + { + "audit_status": "FAIL", + "case_sha256": "a8117a923396d5a8ed32f5ba72a44d0155ca24fbfc4463da9054a8395b0111ce", + "expected_primary_signal": "prompt_token_supervised", + "fault_id": "mask.prompt_supervised", + "kind": "fault", + "observed_primary_signal": "prompt_token_supervised", + "observed_signal_ids": [ + "prompt_token_supervised" + ], + "primary_expectation_met": true, + "seed": 4122 + }, + { + "audit_status": "FAIL", + "case_sha256": "64c642d6283e190e279d3b1318a15c4d7e0c9f6a0cc9530aae2e17119e23fdec", + "expected_primary_signal": "assistant_fully_truncated", + "fault_id": "truncation.answer_removed", + "kind": "fault", + "observed_primary_signal": "assistant_fully_truncated", + "observed_signal_ids": [ + "assistant_fully_truncated" + ], + "primary_expectation_met": true, + "seed": 4122 + }, + { + "audit_status": "PASS", + "case_sha256": "8fd20fa9287512bbad63bba5b89f085d2424c42f57b513350d7d7c86e665dd1b", + "expected_primary_signal": "exact_duplicate", + "fault_id": "data.exact_duplicate", + "kind": "fault", + "observed_primary_signal": "exact_duplicate", + "observed_signal_ids": [ + "exact_duplicate" + ], + "primary_expectation_met": true, + "seed": 4122 + }, + { + "audit_status": "FAIL", + "case_sha256": "c3f5855921f6652f2cb1e83e7c08361d25743f3402beecb047ebb6c5878465d4", + "expected_primary_signal": "prompt_conflict", + "fault_id": "data.prompt_conflict", + "kind": "fault", + "observed_primary_signal": "prompt_conflict", + "observed_signal_ids": [ + "prompt_conflict" + ], + "primary_expectation_met": true, + "seed": 4122 + }, + { + "audit_status": "FAIL", + "case_sha256": "a40d2bf83ad856814248cfc88b99e405c2ea85860e129e5ae5f571eb0d49f48b", + "expected_primary_signal": "train_eval_leakage", + "fault_id": "data.train_eval_leakage", + "kind": "fault", + "observed_primary_signal": "train_eval_leakage", + "observed_signal_ids": [ + "train_eval_leakage" + ], + "primary_expectation_met": true, + "seed": 4122 + }, + { + "audit_status": "FAIL", + "case_sha256": "50366833aa7c4c849fda79972226f8498c530e8e6f5571b1ca019c5fd9cbf42e", + "expected_primary_signal": "secret_pattern", + "fault_id": "privacy.secret_pattern", + "kind": "fault", + "observed_primary_signal": "secret_pattern", + "observed_signal_ids": [ + "secret_pattern" + ], + "primary_expectation_met": true, + "seed": 4122 + }, + { + "audit_status": "PASS", + "case_sha256": "646fcb1faa8b3f173249614e8cca1446b62a791587830432888729e308b014bd", + "expected_primary_signal": null, + "fault_id": null, + "kind": "clean_control", + "observed_primary_signal": null, + "observed_signal_ids": [], + "primary_expectation_met": true, + "seed": 4123 + }, + { + "audit_status": "FAIL", + "case_sha256": "798c037ab41b6b924eca3a20e953c579dfcfe2ef7d0530b5a47f1e9de97e7e85", + "expected_primary_signal": "invalid_role", + "fault_id": "schema.invalid_role", + "kind": "fault", + "observed_primary_signal": "invalid_role", + "observed_signal_ids": [ + "invalid_role", + "role_order" + ], + "primary_expectation_met": true, + "seed": 4123 + }, + { + "audit_status": "FAIL", + "case_sha256": "2585d8d02cd838a171dc13e2b955cb71cc9607e10f06591717e73ccca1e9cf94", + "expected_primary_signal": "role_order", + "fault_id": "schema.role_order", + "kind": "fault", + "observed_primary_signal": "role_order", + "observed_signal_ids": [ + "role_order" + ], + "primary_expectation_met": true, + "seed": 4123 + }, + { + "audit_status": "FAIL", + "case_sha256": "9277cd52353d1b28a444f9b6045186649dad9dfeaf1116c135d55f7bcf488384", + "expected_primary_signal": "missing_eos", + "fault_id": "template.missing_eos", + "kind": "fault", + "observed_primary_signal": "missing_eos", + "observed_signal_ids": [ + "missing_eos" + ], + "primary_expectation_met": true, + "seed": 4123 + }, + { + "audit_status": "FAIL", + "case_sha256": "6873d3a4ca6385ff12812af03fe126911aa1fa14e0ae17d184a8bd2a74ccd9a6", + "expected_primary_signal": "prompt_token_supervised", + "fault_id": "mask.prompt_supervised", + "kind": "fault", + "observed_primary_signal": "prompt_token_supervised", + "observed_signal_ids": [ + "prompt_token_supervised" + ], + "primary_expectation_met": true, + "seed": 4123 + }, + { + "audit_status": "FAIL", + "case_sha256": "27f70b80eb6ae96505a6762ba8b2b65785cccdcae231ffa5f63d5137140e1fb2", + "expected_primary_signal": "assistant_fully_truncated", + "fault_id": "truncation.answer_removed", + "kind": "fault", + "observed_primary_signal": "assistant_fully_truncated", + "observed_signal_ids": [ + "assistant_fully_truncated" + ], + "primary_expectation_met": true, + "seed": 4123 + }, + { + "audit_status": "PASS", + "case_sha256": "0dadafaf18d0a35bb5ba6ac9bd893de8ee9b07bfe9b7664b8553833b73b403e4", + "expected_primary_signal": "exact_duplicate", + "fault_id": "data.exact_duplicate", + "kind": "fault", + "observed_primary_signal": "exact_duplicate", + "observed_signal_ids": [ + "exact_duplicate" + ], + "primary_expectation_met": true, + "seed": 4123 + }, + { + "audit_status": "FAIL", + "case_sha256": "1896027123bc3c71daaf579952bc43f74b1a9adcf8c06c7c6d81f0484e28c408", + "expected_primary_signal": "prompt_conflict", + "fault_id": "data.prompt_conflict", + "kind": "fault", + "observed_primary_signal": "prompt_conflict", + "observed_signal_ids": [ + "prompt_conflict" + ], + "primary_expectation_met": true, + "seed": 4123 + }, + { + "audit_status": "FAIL", + "case_sha256": "26d786e5bff4924acf6a01c4dea6a838e52ebc085b244eeaba9bec64fd01ef3a", + "expected_primary_signal": "train_eval_leakage", + "fault_id": "data.train_eval_leakage", + "kind": "fault", + "observed_primary_signal": "train_eval_leakage", + "observed_signal_ids": [ + "train_eval_leakage" + ], + "primary_expectation_met": true, + "seed": 4123 + }, + { + "audit_status": "FAIL", + "case_sha256": "8bf5be2d60c97115f3122b6b54a0c34db8db3c1f289a5ff8a63ad9a41d6bd7d8", + "expected_primary_signal": "secret_pattern", + "fault_id": "privacy.secret_pattern", + "kind": "fault", + "observed_primary_signal": "secret_pattern", + "observed_signal_ids": [ + "secret_pattern" + ], + "primary_expectation_met": true, + "seed": 4123 + }, + { + "audit_status": "PASS", + "case_sha256": "0e732215ed158974025756a085608e5b0366d01daa2cc2ab777526123e171cdb", + "expected_primary_signal": null, + "fault_id": null, + "kind": "clean_control", + "observed_primary_signal": null, + "observed_signal_ids": [], + "primary_expectation_met": true, + "seed": 4124 + }, + { + "audit_status": "FAIL", + "case_sha256": "9da5a47e584420e62d34db505129dbe1e015c0af68507dbdae475543ba95fabb", + "expected_primary_signal": "invalid_role", + "fault_id": "schema.invalid_role", + "kind": "fault", + "observed_primary_signal": "invalid_role", + "observed_signal_ids": [ + "invalid_role", + "role_order" + ], + "primary_expectation_met": true, + "seed": 4124 + }, + { + "audit_status": "FAIL", + "case_sha256": "1da487165c1ff68558b35a2c160f87c5f419400493b849c20e829a1ade0c8c4c", + "expected_primary_signal": "role_order", + "fault_id": "schema.role_order", + "kind": "fault", + "observed_primary_signal": "role_order", + "observed_signal_ids": [ + "role_order" + ], + "primary_expectation_met": true, + "seed": 4124 + }, + { + "audit_status": "FAIL", + "case_sha256": "807ee62284217dde4564c6f01138254b8c6a05d3aff7de9be88ed9b4c38e5b91", + "expected_primary_signal": "missing_eos", + "fault_id": "template.missing_eos", + "kind": "fault", + "observed_primary_signal": "missing_eos", + "observed_signal_ids": [ + "missing_eos" + ], + "primary_expectation_met": true, + "seed": 4124 + }, + { + "audit_status": "FAIL", + "case_sha256": "ef9197610c8b70abfeb05e1e6b51188c5cbbc3a9ea1cf076a29ee9a3ec1c17f8", + "expected_primary_signal": "prompt_token_supervised", + "fault_id": "mask.prompt_supervised", + "kind": "fault", + "observed_primary_signal": "prompt_token_supervised", + "observed_signal_ids": [ + "prompt_token_supervised" + ], + "primary_expectation_met": true, + "seed": 4124 + }, + { + "audit_status": "FAIL", + "case_sha256": "8366bed91b48aa74b6394a9cd45193b30a415512f4b74c9614a974c1065c7501", + "expected_primary_signal": "assistant_fully_truncated", + "fault_id": "truncation.answer_removed", + "kind": "fault", + "observed_primary_signal": "assistant_fully_truncated", + "observed_signal_ids": [ + "assistant_fully_truncated" + ], + "primary_expectation_met": true, + "seed": 4124 + }, + { + "audit_status": "PASS", + "case_sha256": "dd87856c254b69aa6b3a0fcf348e48acd5b27b761440ad7449719a4c9495e803", + "expected_primary_signal": "exact_duplicate", + "fault_id": "data.exact_duplicate", + "kind": "fault", + "observed_primary_signal": "exact_duplicate", + "observed_signal_ids": [ + "exact_duplicate" + ], + "primary_expectation_met": true, + "seed": 4124 + }, + { + "audit_status": "FAIL", + "case_sha256": "5e846efac574b9980587f38bae8365217cf8d243a116f099771298da03fd7470", + "expected_primary_signal": "prompt_conflict", + "fault_id": "data.prompt_conflict", + "kind": "fault", + "observed_primary_signal": "prompt_conflict", + "observed_signal_ids": [ + "prompt_conflict" + ], + "primary_expectation_met": true, + "seed": 4124 + }, + { + "audit_status": "FAIL", + "case_sha256": "66521150c766c7a89380e135d37c7cf1e8407eea97b744bd4aa5a28b73197f2d", + "expected_primary_signal": "train_eval_leakage", + "fault_id": "data.train_eval_leakage", + "kind": "fault", + "observed_primary_signal": "train_eval_leakage", + "observed_signal_ids": [ + "train_eval_leakage" + ], + "primary_expectation_met": true, + "seed": 4124 + }, + { + "audit_status": "FAIL", + "case_sha256": "33554dcacf18b1dfb30f7a890f81857fa0bb325b99509304dce565b509457bde", + "expected_primary_signal": "secret_pattern", + "fault_id": "privacy.secret_pattern", + "kind": "fault", + "observed_primary_signal": "secret_pattern", + "observed_signal_ids": [ + "secret_pattern" + ], + "primary_expectation_met": true, + "seed": 4124 + }, + { + "audit_status": "PASS", + "case_sha256": "ace9cc5fc923ef4aba52567657243163556880685e5f325a58d5c9b99601cead", + "expected_primary_signal": null, + "fault_id": null, + "kind": "clean_control", + "observed_primary_signal": null, + "observed_signal_ids": [], + "primary_expectation_met": true, + "seed": 4125 + }, + { + "audit_status": "FAIL", + "case_sha256": "553e11b53c6a7557bbe3fa51ec8709fb7cba2ffe8433e6c8e5c2e2c9ebc0db4f", + "expected_primary_signal": "invalid_role", + "fault_id": "schema.invalid_role", + "kind": "fault", + "observed_primary_signal": "invalid_role", + "observed_signal_ids": [ + "invalid_role", + "role_order" + ], + "primary_expectation_met": true, + "seed": 4125 + }, + { + "audit_status": "FAIL", + "case_sha256": "14253fb73cd8cfd707ef19085cdce71069a08384d497dad464b71010e5326bd0", + "expected_primary_signal": "role_order", + "fault_id": "schema.role_order", + "kind": "fault", + "observed_primary_signal": "role_order", + "observed_signal_ids": [ + "role_order" + ], + "primary_expectation_met": true, + "seed": 4125 + }, + { + "audit_status": "FAIL", + "case_sha256": "f2538bd621b5ebac17f84979c03f1e8fc801ff1e73cb91401d95e2c2ef6348b9", + "expected_primary_signal": "missing_eos", + "fault_id": "template.missing_eos", + "kind": "fault", + "observed_primary_signal": "missing_eos", + "observed_signal_ids": [ + "missing_eos" + ], + "primary_expectation_met": true, + "seed": 4125 + }, + { + "audit_status": "FAIL", + "case_sha256": "a5dd4f010bc624d9b5a42f6353c7689b4956dec905c0e84780c389f7575030b9", + "expected_primary_signal": "prompt_token_supervised", + "fault_id": "mask.prompt_supervised", + "kind": "fault", + "observed_primary_signal": "prompt_token_supervised", + "observed_signal_ids": [ + "prompt_token_supervised" + ], + "primary_expectation_met": true, + "seed": 4125 + }, + { + "audit_status": "FAIL", + "case_sha256": "209d24608f720906feb78176794b9e65befb79f207f4a19aefd29d5db6eed22e", + "expected_primary_signal": "assistant_fully_truncated", + "fault_id": "truncation.answer_removed", + "kind": "fault", + "observed_primary_signal": "assistant_fully_truncated", + "observed_signal_ids": [ + "assistant_fully_truncated" + ], + "primary_expectation_met": true, + "seed": 4125 + }, + { + "audit_status": "PASS", + "case_sha256": "915213d0822e9e4ec6d2f7cc5a3a9c4ca468b6c24552ffb2da9f705d8134f778", + "expected_primary_signal": "exact_duplicate", + "fault_id": "data.exact_duplicate", + "kind": "fault", + "observed_primary_signal": "exact_duplicate", + "observed_signal_ids": [ + "exact_duplicate" + ], + "primary_expectation_met": true, + "seed": 4125 + }, + { + "audit_status": "FAIL", + "case_sha256": "fe528d268c95566c4c5c8a95dc50e8d3e3f9b4e3aa2e8d93e32ced13f3c6a8c9", + "expected_primary_signal": "prompt_conflict", + "fault_id": "data.prompt_conflict", + "kind": "fault", + "observed_primary_signal": "prompt_conflict", + "observed_signal_ids": [ + "prompt_conflict" + ], + "primary_expectation_met": true, + "seed": 4125 + }, + { + "audit_status": "FAIL", + "case_sha256": "785c17546e76870cfd6ae94f8f6590d0c568c78cebaf2483bba0608e9a842589", + "expected_primary_signal": "train_eval_leakage", + "fault_id": "data.train_eval_leakage", + "kind": "fault", + "observed_primary_signal": "train_eval_leakage", + "observed_signal_ids": [ + "train_eval_leakage" + ], + "primary_expectation_met": true, + "seed": 4125 + }, + { + "audit_status": "FAIL", + "case_sha256": "773337be8c79756471b1b682f76cf8de910b7e1a8d2e8e2e20199cf299d02279", + "expected_primary_signal": "secret_pattern", + "fault_id": "privacy.secret_pattern", + "kind": "fault", + "observed_primary_signal": "secret_pattern", + "observed_signal_ids": [ + "secret_pattern" + ], + "primary_expectation_met": true, + "seed": 4125 + }, + { + "audit_status": "PASS", + "case_sha256": "9da9642c4ad79e0a58bae384083b6124b0d91116060ad898e7a41981f7dedf01", + "expected_primary_signal": null, + "fault_id": null, + "kind": "clean_control", + "observed_primary_signal": null, + "observed_signal_ids": [], + "primary_expectation_met": true, + "seed": 4126 + }, + { + "audit_status": "FAIL", + "case_sha256": "ecadf36b9421e676893b7ad6a9cea00b75264a242b9b3012c0587f83bef3461b", + "expected_primary_signal": "invalid_role", + "fault_id": "schema.invalid_role", + "kind": "fault", + "observed_primary_signal": "invalid_role", + "observed_signal_ids": [ + "invalid_role", + "role_order" + ], + "primary_expectation_met": true, + "seed": 4126 + }, + { + "audit_status": "FAIL", + "case_sha256": "45b7685fe0cc4f18ebd7f1329d53af30a39c86eb3efbe15a7dbb3908959d2d2f", + "expected_primary_signal": "role_order", + "fault_id": "schema.role_order", + "kind": "fault", + "observed_primary_signal": "role_order", + "observed_signal_ids": [ + "role_order" + ], + "primary_expectation_met": true, + "seed": 4126 + }, + { + "audit_status": "FAIL", + "case_sha256": "c7dc803be7016e987b3acb087bbc0a386f019f24a16194511b7d3b017e3c302f", + "expected_primary_signal": "missing_eos", + "fault_id": "template.missing_eos", + "kind": "fault", + "observed_primary_signal": "missing_eos", + "observed_signal_ids": [ + "missing_eos" + ], + "primary_expectation_met": true, + "seed": 4126 + }, + { + "audit_status": "FAIL", + "case_sha256": "10746fd1b74a5d41f7b033be72a8af5485fe0894356d6338f60c677094a14037", + "expected_primary_signal": "prompt_token_supervised", + "fault_id": "mask.prompt_supervised", + "kind": "fault", + "observed_primary_signal": "prompt_token_supervised", + "observed_signal_ids": [ + "prompt_token_supervised" + ], + "primary_expectation_met": true, + "seed": 4126 + }, + { + "audit_status": "FAIL", + "case_sha256": "f08d878405300c29302418b5f7733b8ec67c59c73665f37338b95c0db9a1af44", + "expected_primary_signal": "assistant_fully_truncated", + "fault_id": "truncation.answer_removed", + "kind": "fault", + "observed_primary_signal": "assistant_fully_truncated", + "observed_signal_ids": [ + "assistant_fully_truncated" + ], + "primary_expectation_met": true, + "seed": 4126 + }, + { + "audit_status": "PASS", + "case_sha256": "23078ab97b51def96a5987736b0481e6b793bb566d5365c8c8a18c59fb10cdd9", + "expected_primary_signal": "exact_duplicate", + "fault_id": "data.exact_duplicate", + "kind": "fault", + "observed_primary_signal": "exact_duplicate", + "observed_signal_ids": [ + "exact_duplicate" + ], + "primary_expectation_met": true, + "seed": 4126 + }, + { + "audit_status": "FAIL", + "case_sha256": "960c6f8ae94e6f923e0c721d773fc5848ee30b3ccdb7bda446d86eb15d581d8c", + "expected_primary_signal": "prompt_conflict", + "fault_id": "data.prompt_conflict", + "kind": "fault", + "observed_primary_signal": "prompt_conflict", + "observed_signal_ids": [ + "prompt_conflict" + ], + "primary_expectation_met": true, + "seed": 4126 + }, + { + "audit_status": "FAIL", + "case_sha256": "4415b1cc4f4978c9b92215595e47d8ac5e2c9f4dbcf6e588a3bf8f2a6d336129", + "expected_primary_signal": "train_eval_leakage", + "fault_id": "data.train_eval_leakage", + "kind": "fault", + "observed_primary_signal": "train_eval_leakage", + "observed_signal_ids": [ + "train_eval_leakage" + ], + "primary_expectation_met": true, + "seed": 4126 + }, + { + "audit_status": "FAIL", + "case_sha256": "4aa4ca025171bbf93dc618b5e1131de9ef7abcf9131c1a09cfca4037b2903b1f", + "expected_primary_signal": "secret_pattern", + "fault_id": "privacy.secret_pattern", + "kind": "fault", + "observed_primary_signal": "secret_pattern", + "observed_signal_ids": [ + "secret_pattern" + ], + "primary_expectation_met": true, + "seed": 4126 + }, + { + "audit_status": "PASS", + "case_sha256": "5e7619217bacbe800be15894fdf3ad12349b56d141e49ef144bcdf6cb50225be", + "expected_primary_signal": null, + "fault_id": null, + "kind": "clean_control", + "observed_primary_signal": null, + "observed_signal_ids": [], + "primary_expectation_met": true, + "seed": 4127 + }, + { + "audit_status": "FAIL", + "case_sha256": "63817644c1806993b5dc199b4d44b7c3ab2b4e461cc48813f2ea75c390fbe46f", + "expected_primary_signal": "invalid_role", + "fault_id": "schema.invalid_role", + "kind": "fault", + "observed_primary_signal": "invalid_role", + "observed_signal_ids": [ + "invalid_role", + "role_order" + ], + "primary_expectation_met": true, + "seed": 4127 + }, + { + "audit_status": "FAIL", + "case_sha256": "daae4347d57097c8d85acd647ce66ef4f78b34c74a9fe42897061fb42131186d", + "expected_primary_signal": "role_order", + "fault_id": "schema.role_order", + "kind": "fault", + "observed_primary_signal": "role_order", + "observed_signal_ids": [ + "role_order" + ], + "primary_expectation_met": true, + "seed": 4127 + }, + { + "audit_status": "FAIL", + "case_sha256": "4baaad4dadfda8644d706b6e00f2a63ae34938453c6407723f4e7f61ce0b4215", + "expected_primary_signal": "missing_eos", + "fault_id": "template.missing_eos", + "kind": "fault", + "observed_primary_signal": "missing_eos", + "observed_signal_ids": [ + "missing_eos" + ], + "primary_expectation_met": true, + "seed": 4127 + }, + { + "audit_status": "FAIL", + "case_sha256": "6cef96304628e0b8d323b973f2ed16f8c10847236d7967304aea9d24f6deafb0", + "expected_primary_signal": "prompt_token_supervised", + "fault_id": "mask.prompt_supervised", + "kind": "fault", + "observed_primary_signal": "prompt_token_supervised", + "observed_signal_ids": [ + "prompt_token_supervised" + ], + "primary_expectation_met": true, + "seed": 4127 + }, + { + "audit_status": "FAIL", + "case_sha256": "824850b3d6d24275733316e6c951de783edff63a4b2491af16e95a2c1c423c40", + "expected_primary_signal": "assistant_fully_truncated", + "fault_id": "truncation.answer_removed", + "kind": "fault", + "observed_primary_signal": "assistant_fully_truncated", + "observed_signal_ids": [ + "assistant_fully_truncated" + ], + "primary_expectation_met": true, + "seed": 4127 + }, + { + "audit_status": "PASS", + "case_sha256": "cbf93ed6573a5a1460bb7244b8cdd161a79fa551b97d607e4ae7ab9d61623f0f", + "expected_primary_signal": "exact_duplicate", + "fault_id": "data.exact_duplicate", + "kind": "fault", + "observed_primary_signal": "exact_duplicate", + "observed_signal_ids": [ + "exact_duplicate" + ], + "primary_expectation_met": true, + "seed": 4127 + }, + { + "audit_status": "FAIL", + "case_sha256": "e1aa015ebdea2f62b0860894478b8e8e361de2be22015e03ef6a5317e7251978", + "expected_primary_signal": "prompt_conflict", + "fault_id": "data.prompt_conflict", + "kind": "fault", + "observed_primary_signal": "prompt_conflict", + "observed_signal_ids": [ + "prompt_conflict" + ], + "primary_expectation_met": true, + "seed": 4127 + }, + { + "audit_status": "FAIL", + "case_sha256": "70027ac367e4429645b880be15be4275d85941606f1470af9d2569bda75aade7", + "expected_primary_signal": "train_eval_leakage", + "fault_id": "data.train_eval_leakage", + "kind": "fault", + "observed_primary_signal": "train_eval_leakage", + "observed_signal_ids": [ + "train_eval_leakage" + ], + "primary_expectation_met": true, + "seed": 4127 + }, + { + "audit_status": "FAIL", + "case_sha256": "c426fd8580ee5c0310ebb04b9baa069dd75d9ffa79861548e5f09d387e05841f", + "expected_primary_signal": "secret_pattern", + "fault_id": "privacy.secret_pattern", + "kind": "fault", + "observed_primary_signal": "secret_pattern", + "observed_signal_ids": [ + "secret_pattern" + ], + "primary_expectation_met": true, + "seed": 4127 + }, + { + "audit_status": "PASS", + "case_sha256": "475557e6e5b5df96038968ea2ab8b172d8d49a1aa512bb9a17abcb367ca7706d", + "expected_primary_signal": null, + "fault_id": null, + "kind": "clean_control", + "observed_primary_signal": null, + "observed_signal_ids": [], + "primary_expectation_met": true, + "seed": 4128 + }, + { + "audit_status": "FAIL", + "case_sha256": "1ce4a2d128a6db9d2ac617a7b1982efa8039cf0c0ec5e380549fbd8c2e241c01", + "expected_primary_signal": "invalid_role", + "fault_id": "schema.invalid_role", + "kind": "fault", + "observed_primary_signal": "invalid_role", + "observed_signal_ids": [ + "invalid_role", + "role_order" + ], + "primary_expectation_met": true, + "seed": 4128 + }, + { + "audit_status": "FAIL", + "case_sha256": "6de06e32e00ad58457e5bc1eb4730fd100220389bdd27ed657bc0e24e569f29b", + "expected_primary_signal": "role_order", + "fault_id": "schema.role_order", + "kind": "fault", + "observed_primary_signal": "role_order", + "observed_signal_ids": [ + "role_order" + ], + "primary_expectation_met": true, + "seed": 4128 + }, + { + "audit_status": "FAIL", + "case_sha256": "f3965d909f2d7c07c8fd8b3a8d12108fadbfe4cd9aa2e851f6fbdb19733d6bf8", + "expected_primary_signal": "missing_eos", + "fault_id": "template.missing_eos", + "kind": "fault", + "observed_primary_signal": "missing_eos", + "observed_signal_ids": [ + "missing_eos" + ], + "primary_expectation_met": true, + "seed": 4128 + }, + { + "audit_status": "FAIL", + "case_sha256": "4bc2f2dd8fcc9ac78ff1b0376448cc431c79d0575b3150c4b4962367db3b3985", + "expected_primary_signal": "prompt_token_supervised", + "fault_id": "mask.prompt_supervised", + "kind": "fault", + "observed_primary_signal": "prompt_token_supervised", + "observed_signal_ids": [ + "prompt_token_supervised" + ], + "primary_expectation_met": true, + "seed": 4128 + }, + { + "audit_status": "FAIL", + "case_sha256": "19bab18fcca10c08100c5062529bb3ba8bcc4a57d326ef73650b7dc04fb1275f", + "expected_primary_signal": "assistant_fully_truncated", + "fault_id": "truncation.answer_removed", + "kind": "fault", + "observed_primary_signal": "assistant_fully_truncated", + "observed_signal_ids": [ + "assistant_fully_truncated" + ], + "primary_expectation_met": true, + "seed": 4128 + }, + { + "audit_status": "PASS", + "case_sha256": "2b04dcadb2ad9c50b0ffb6e520007b34bddba9058ebd5ecc8c663c8682fd3d40", + "expected_primary_signal": "exact_duplicate", + "fault_id": "data.exact_duplicate", + "kind": "fault", + "observed_primary_signal": "exact_duplicate", + "observed_signal_ids": [ + "exact_duplicate" + ], + "primary_expectation_met": true, + "seed": 4128 + }, + { + "audit_status": "FAIL", + "case_sha256": "b2c2213bd18a1ef9041b4e947ac9475157e35ab565fec770d3387376b5c1b7d2", + "expected_primary_signal": "prompt_conflict", + "fault_id": "data.prompt_conflict", + "kind": "fault", + "observed_primary_signal": "prompt_conflict", + "observed_signal_ids": [ + "prompt_conflict" + ], + "primary_expectation_met": true, + "seed": 4128 + }, + { + "audit_status": "FAIL", + "case_sha256": "0dbf42390ad6d9794a99a0eec77c477d59cbbffd9f8fff778e4615dbf0f5503a", + "expected_primary_signal": "train_eval_leakage", + "fault_id": "data.train_eval_leakage", + "kind": "fault", + "observed_primary_signal": "train_eval_leakage", + "observed_signal_ids": [ + "train_eval_leakage" + ], + "primary_expectation_met": true, + "seed": 4128 + }, + { + "audit_status": "FAIL", + "case_sha256": "897eb6d79aae29b5abc0ac4ecb501a80a382197dbb3fddfd738b53b915188108", + "expected_primary_signal": "secret_pattern", + "fault_id": "privacy.secret_pattern", + "kind": "fault", + "observed_primary_signal": "secret_pattern", + "observed_signal_ids": [ + "secret_pattern" + ], + "primary_expectation_met": true, + "seed": 4128 + }, + { + "audit_status": "PASS", + "case_sha256": "f35dc4fe626aeea7bd1c6310cccc2b8ece5eb3334550ae9f4bca91cb9179c039", + "expected_primary_signal": null, + "fault_id": null, + "kind": "clean_control", + "observed_primary_signal": null, + "observed_signal_ids": [], + "primary_expectation_met": true, + "seed": 4129 + }, + { + "audit_status": "FAIL", + "case_sha256": "d8282f6d967d6c102301b3a28ae74115602da73dfdba04219ca185c42a2674da", + "expected_primary_signal": "invalid_role", + "fault_id": "schema.invalid_role", + "kind": "fault", + "observed_primary_signal": "invalid_role", + "observed_signal_ids": [ + "invalid_role", + "role_order" + ], + "primary_expectation_met": true, + "seed": 4129 + }, + { + "audit_status": "FAIL", + "case_sha256": "17e0f95ecef38e00a38defb0d746c11a559c76878503d2f8039f7e1908150286", + "expected_primary_signal": "role_order", + "fault_id": "schema.role_order", + "kind": "fault", + "observed_primary_signal": "role_order", + "observed_signal_ids": [ + "role_order" + ], + "primary_expectation_met": true, + "seed": 4129 + }, + { + "audit_status": "FAIL", + "case_sha256": "05dc1d34b33f4785c3a8897d0802fcb1e399ee9b775c68e4b79680619d50d1dd", + "expected_primary_signal": "missing_eos", + "fault_id": "template.missing_eos", + "kind": "fault", + "observed_primary_signal": "missing_eos", + "observed_signal_ids": [ + "missing_eos" + ], + "primary_expectation_met": true, + "seed": 4129 + }, + { + "audit_status": "FAIL", + "case_sha256": "68da4319e9632b2a92423b2c329b8e0dde098195603b1e11a5ce838401978ad4", + "expected_primary_signal": "prompt_token_supervised", + "fault_id": "mask.prompt_supervised", + "kind": "fault", + "observed_primary_signal": "prompt_token_supervised", + "observed_signal_ids": [ + "prompt_token_supervised" + ], + "primary_expectation_met": true, + "seed": 4129 + }, + { + "audit_status": "FAIL", + "case_sha256": "b25884df6ff0293203fb96d29d45830d52b34a01461bd9c745f16435dfb2cc70", + "expected_primary_signal": "assistant_fully_truncated", + "fault_id": "truncation.answer_removed", + "kind": "fault", + "observed_primary_signal": "assistant_fully_truncated", + "observed_signal_ids": [ + "assistant_fully_truncated" + ], + "primary_expectation_met": true, + "seed": 4129 + }, + { + "audit_status": "PASS", + "case_sha256": "93bfdf6bfe90cc4be235df021647054e5b23cc62e36287d4ca3f8ad03603f86a", + "expected_primary_signal": "exact_duplicate", + "fault_id": "data.exact_duplicate", + "kind": "fault", + "observed_primary_signal": "exact_duplicate", + "observed_signal_ids": [ + "exact_duplicate" + ], + "primary_expectation_met": true, + "seed": 4129 + }, + { + "audit_status": "FAIL", + "case_sha256": "607cfaf4697510a4104b4c7b2647e5e6d31a528ff4bacebcbbf717c915ffd7e5", + "expected_primary_signal": "prompt_conflict", + "fault_id": "data.prompt_conflict", + "kind": "fault", + "observed_primary_signal": "prompt_conflict", + "observed_signal_ids": [ + "prompt_conflict" + ], + "primary_expectation_met": true, + "seed": 4129 + }, + { + "audit_status": "FAIL", + "case_sha256": "a173586ab768cbb3f554912647a6238bb2b2ce715b69ffa183ea8628ea219471", + "expected_primary_signal": "train_eval_leakage", + "fault_id": "data.train_eval_leakage", + "kind": "fault", + "observed_primary_signal": "train_eval_leakage", + "observed_signal_ids": [ + "train_eval_leakage" + ], + "primary_expectation_met": true, + "seed": 4129 + }, + { + "audit_status": "FAIL", + "case_sha256": "1e7d276589dea9ec13d9e741913e6e3e55c305deda7cdcebaf6152ea76bf7ee2", + "expected_primary_signal": "secret_pattern", + "fault_id": "privacy.secret_pattern", + "kind": "fault", + "observed_primary_signal": "secret_pattern", + "observed_signal_ids": [ + "secret_pattern" + ], + "primary_expectation_met": true, + "seed": 4129 + } + ], + "gates": { + "clean_false_positive_rate_at_most_10_percent": true, + "critical_fault_recall_100_percent": true, + "expected_and_observed_primary_recorded": true, + "fresh_process_byte_reproducible": true, + "macro_recall_at_least_90_percent": true, + "raw_synthetic_text_absent": true + }, + "integrity": { + "algorithm": "sha256", + "canonical_payload_sha256": "eb55eaa16a492881ff1bcec8812b2ef37bb2ed6055b6b90a1da408386f632e3a" + }, + "per_fault": [ + { + "case_count": 30, + "detected_count": 30, + "expected_signal": "invalid_role", + "fault_id": "schema.invalid_role", + "recall": 1.0, + "severity": "error" + }, + { + "case_count": 30, + "detected_count": 30, + "expected_signal": "role_order", + "fault_id": "schema.role_order", + "recall": 1.0, + "severity": "error" + }, + { + "case_count": 30, + "detected_count": 30, + "expected_signal": "missing_eos", + "fault_id": "template.missing_eos", + "recall": 1.0, + "severity": "error" + }, + { + "case_count": 30, + "detected_count": 30, + "expected_signal": "prompt_token_supervised", + "fault_id": "mask.prompt_supervised", + "recall": 1.0, + "severity": "critical" + }, + { + "case_count": 30, + "detected_count": 30, + "expected_signal": "assistant_fully_truncated", + "fault_id": "truncation.answer_removed", + "recall": 1.0, + "severity": "critical" + }, + { + "case_count": 30, + "detected_count": 30, + "expected_signal": "exact_duplicate", + "fault_id": "data.exact_duplicate", + "recall": 1.0, + "severity": "warning" + }, + { + "case_count": 30, + "detected_count": 30, + "expected_signal": "prompt_conflict", + "fault_id": "data.prompt_conflict", + "recall": 1.0, + "severity": "error" + }, + { + "case_count": 30, + "detected_count": 30, + "expected_signal": "train_eval_leakage", + "fault_id": "data.train_eval_leakage", + "recall": 1.0, + "severity": "critical" + }, + { + "case_count": 30, + "detected_count": 30, + "expected_signal": "secret_pattern", + "fault_id": "privacy.secret_pattern", + "recall": 1.0, + "severity": "critical" + } + ], + "protocol": { + "audit_config": { + "max_file_bytes": 16777216, + "max_line_bytes": 1048576, + "max_records": 100000, + "require_assistant_mask": true, + "scan_privacy": true + }, + "faults": [ + { + "expected_signal": "invalid_role", + "fault_id": "schema.invalid_role", + "severity": "error" + }, + { + "expected_signal": "role_order", + "fault_id": "schema.role_order", + "severity": "error" + }, + { + "expected_signal": "missing_eos", + "fault_id": "template.missing_eos", + "severity": "error" + }, + { + "expected_signal": "prompt_token_supervised", + "fault_id": "mask.prompt_supervised", + "severity": "critical" + }, + { + "expected_signal": "assistant_fully_truncated", + "fault_id": "truncation.answer_removed", + "severity": "critical" + }, + { + "expected_signal": "exact_duplicate", + "fault_id": "data.exact_duplicate", + "severity": "warning" + }, + { + "expected_signal": "prompt_conflict", + "fault_id": "data.prompt_conflict", + "severity": "error" + }, + { + "expected_signal": "train_eval_leakage", + "fault_id": "data.train_eval_leakage", + "severity": "critical" + }, + { + "expected_signal": "secret_pattern", + "fault_id": "privacy.secret_pattern", + "severity": "critical" + } + ], + "seed_range": { + "count": 30, + "first": 4100, + "last": 4129 + } + }, + "provenance": { + "implementation_git_commit": "b0f168ab4d6a959d4e28611e345a250c14ca2ea9", + "protocol_git_commit": "d4c74cefca6729124142635dc8893b5e3ddec2d3", + "python_implementation": "CPython", + "python_version": "3.11" + }, + "schema_version": "sftguard.confirmatory-fault-matrix.v1", + "status": "PASS", + "summary": { + "case_count": 300, + "clean_control_count": 30, + "clean_false_positive_count": 0, + "clean_false_positive_rate": 0.0, + "fault_case_count": 270, + "macro_recall": 1.0 + }, + "tool": { + "name": "sftguard", + "version": "0.1.0" + } +} diff --git a/evidence/v0.1.0-confirmatory.manifest.json b/evidence/v0.1.0-confirmatory.manifest.json new file mode 100644 index 0000000..8111161 --- /dev/null +++ b/evidence/v0.1.0-confirmatory.manifest.json @@ -0,0 +1,12 @@ +{ + "artifact_file_sha256": "961b2842d7791f6f3af7bec5bce6105e433b68d41bf6f7b82a9f3a98e1f695ab", + "artifact_integrity_sha256": "eb55eaa16a492881ff1bcec8812b2ef37bb2ed6055b6b90a1da408386f632e3a", + "artifact_path": "evidence/v0.1.0-confirmatory.json", + "implementation_git_commit": "b0f168ab4d6a959d4e28611e345a250c14ca2ea9", + "integrity": { + "algorithm": "sha256", + "canonical_payload_sha256": "39badf5b3dc862852b890b347f6015127ee33bfb295bd1d6a022cfd028bd2131" + }, + "protocol_git_commit": "d4c74cefca6729124142635dc8893b5e3ddec2d3", + "schema_version": "sftguard.confirmatory-manifest.v1" +} diff --git a/examples/adapter-eval.jsonl b/examples/adapter-eval.jsonl new file mode 100644 index 0000000..52c7752 --- /dev/null +++ b/examples/adapter-eval.jsonl @@ -0,0 +1,6 @@ +{"seed":0,"sample_id":"target-0","group":"target","metric":"task_score","value":0.62} +{"seed":0,"sample_id":"retention-0","group":"retention","metric":"general_score","value":0.795} +{"seed":1,"sample_id":"target-1","group":"target","metric":"task_score","value":0.63} +{"seed":1,"sample_id":"retention-1","group":"retention","metric":"general_score","value":0.785} +{"seed":2,"sample_id":"target-2","group":"target","metric":"task_score","value":0.63} +{"seed":2,"sample_id":"retention-2","group":"retention","metric":"general_score","value":0.805} diff --git a/examples/base-eval.jsonl b/examples/base-eval.jsonl new file mode 100644 index 0000000..2aebafb --- /dev/null +++ b/examples/base-eval.jsonl @@ -0,0 +1,6 @@ +{"seed":0,"sample_id":"target-0","group":"target","metric":"task_score","value":0.50} +{"seed":0,"sample_id":"retention-0","group":"retention","metric":"general_score","value":0.80} +{"seed":1,"sample_id":"target-1","group":"target","metric":"task_score","value":0.52} +{"seed":1,"sample_id":"retention-1","group":"retention","metric":"general_score","value":0.79} +{"seed":2,"sample_id":"target-2","group":"target","metric":"task_score","value":0.51} +{"seed":2,"sample_id":"retention-2","group":"retention","metric":"general_score","value":0.81} diff --git a/examples/clean-eval.jsonl b/examples/clean-eval.jsonl new file mode 100644 index 0000000..8f281c6 --- /dev/null +++ b/examples/clean-eval.jsonl @@ -0,0 +1 @@ +{"messages":[{"role":"system","content":"Return one class label."},{"role":"user","content":"Classify the held-out development fixture."},{"role":"assistant","content":"class_a"}],"_sftguard":{"token_ids":[12,22,32,2],"token_roles":["system","user","assistant","assistant"],"loss_mask":[false,false,true,true],"eos_token_id":2,"original_assistant_tokens":2,"truncated":false}} diff --git a/examples/clean-train.jsonl b/examples/clean-train.jsonl new file mode 100644 index 0000000..fa7fdd1 --- /dev/null +++ b/examples/clean-train.jsonl @@ -0,0 +1,2 @@ +{"messages":[{"role":"system","content":"Return one class label."},{"role":"user","content":"Classify the first development fixture."},{"role":"assistant","content":"class_a"}],"_sftguard":{"token_ids":[10,20,30,2],"token_roles":["system","user","assistant","assistant"],"loss_mask":[false,false,true,true],"eos_token_id":2,"original_assistant_tokens":2,"truncated":false}} +{"messages":[{"role":"system","content":"Return one class label."},{"role":"user","content":"Classify the second development fixture."},{"role":"assistant","content":"class_b"}],"_sftguard":{"token_ids":[11,21,31,2],"token_roles":["system","user","assistant","assistant"],"loss_mask":[false,false,true,true],"eos_token_id":2,"original_assistant_tokens":2,"truncated":false}} diff --git a/examples/regression-contract.json b/examples/regression-contract.json new file mode 100644 index 0000000..7c85d0c --- /dev/null +++ b/examples/regression-contract.json @@ -0,0 +1,17 @@ +{ + "bootstrap": {"draws": 10000, "confidence": 0.95, "seed": 2339}, + "metrics": [ + { + "group": "target", + "metric": "task_score", + "higher_is_better": true, + "min_improvement": 0.05 + }, + { + "group": "retention", + "metric": "general_score", + "higher_is_better": true, + "max_regression": 0.02 + } + ] +} diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..d37ca64 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,57 @@ +[build-system] +requires = ["setuptools>=77"] +build-backend = "setuptools.build_meta" + +[project] +name = "sftguard" +version = "0.1.0" +description = "Local-first correctness and regression gates for supervised fine-tuning pipelines" +readme = "README.md" +requires-python = ">=3.11" +license = "Apache-2.0" +license-files = ["LICENSE"] +authors = [{ name = "Muhammad Labeeb Aryan" }] +keywords = ["fine-tuning", "llm", "regression-testing", "sft"] +classifiers = [ + "Development Status :: 3 - Alpha", + "Environment :: Console", + "Operating System :: OS Independent", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Topic :: Scientific/Engineering :: Artificial Intelligence", +] +dependencies = [] + +[project.urls] +Homepage = "https://github.com/Labeeb2339/sftguard" +Documentation = "https://github.com/Labeeb2339/sftguard#readme" +Issues = "https://github.com/Labeeb2339/sftguard/issues" +Source = "https://github.com/Labeeb2339/sftguard" + +[project.optional-dependencies] +dev = [ + "build>=1.2,<2", + "pytest>=8,<9", + "ruff>=0.9,<1", +] + +[project.scripts] +sftguard = "sftguard.cli:main" + +[tool.setuptools] +package-dir = { "" = "src" } + +[tool.setuptools.packages.find] +where = ["src"] + +[tool.pytest.ini_options] +testpaths = ["tests/python"] +addopts = "-q --basetemp=.pytest-tmp-basetemp" + +[tool.ruff] +target-version = "py311" +line-length = 100 + +[tool.ruff.lint] +select = ["E", "F", "I", "UP", "B"] diff --git a/schemas/audit-report.schema.json b/schemas/audit-report.schema.json new file mode 100644 index 0000000..f25a22c --- /dev/null +++ b/schemas/audit-report.schema.json @@ -0,0 +1,48 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "urn:sftguard:schema:audit-report:v1", + "title": "SFTGuard audit report", + "type": "object", + "required": [ + "schema_version", + "tool", + "status", + "inputs", + "checks", + "summary", + "findings", + "integrity" + ], + "properties": { + "schema_version": {"const": "sftguard.audit.v1"}, + "status": {"enum": ["PASS", "FAIL", "ABSTAIN"]}, + "inputs": {"type": "array"}, + "checks": {"type": "object"}, + "summary": {"type": "object"}, + "findings": { + "type": "array", + "items": { + "type": "object", + "required": ["signal", "severity", "location"], + "properties": { + "signal": {"type": "string"}, + "severity": {"enum": ["info", "warning", "error", "critical"]}, + "location": { + "type": "object", + "required": ["split", "line"], + "properties": { + "split": {"enum": ["train", "evaluation"]}, + "line": {"type": "integer", "minimum": 1}, + "field": {"type": "string"} + } + }, + "redacted": {"type": "boolean"} + } + } + }, + "integrity": { + "type": "object", + "required": ["algorithm", "canonical_payload_sha256"] + } + } +} diff --git a/schemas/confirmatory-fault-matrix.schema.json b/schemas/confirmatory-fault-matrix.schema.json new file mode 100644 index 0000000..9facef1 --- /dev/null +++ b/schemas/confirmatory-fault-matrix.schema.json @@ -0,0 +1,32 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "urn:sftguard:schema:confirmatory-fault-matrix:v1", + "title": "SFTGuard confirmatory fault-matrix evidence", + "type": "object", + "required": [ + "schema_version", + "tool", + "provenance", + "protocol", + "summary", + "per_fault", + "cases", + "gates", + "carry_forward", + "status", + "integrity" + ], + "properties": { + "schema_version": {"const": "sftguard.confirmatory-fault-matrix.v1"}, + "status": {"enum": ["PASS", "FAIL"]}, + "carry_forward": {"type": "boolean"}, + "tool": {"type": "object"}, + "provenance": {"type": "object"}, + "protocol": {"type": "object"}, + "summary": {"type": "object"}, + "per_fault": {"type": "array", "minItems": 9, "maxItems": 9}, + "cases": {"type": "array", "minItems": 300, "maxItems": 300}, + "gates": {"type": "object"}, + "integrity": {"type": "object"} + } +} diff --git a/schemas/confirmatory-manifest.schema.json b/schemas/confirmatory-manifest.schema.json new file mode 100644 index 0000000..6f04eaf --- /dev/null +++ b/schemas/confirmatory-manifest.schema.json @@ -0,0 +1,33 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "urn:sftguard:schema:confirmatory-manifest:v1", + "title": "SFTGuard confirmatory evidence manifest", + "type": "object", + "additionalProperties": false, + "required": [ + "schema_version", + "artifact_file_sha256", + "artifact_integrity_sha256", + "artifact_path", + "implementation_git_commit", + "protocol_git_commit", + "integrity" + ], + "properties": { + "schema_version": {"const": "sftguard.confirmatory-manifest.v1"}, + "artifact_file_sha256": {"type": "string", "pattern": "^[0-9a-f]{64}$"}, + "artifact_integrity_sha256": {"type": "string", "pattern": "^[0-9a-f]{64}$"}, + "artifact_path": {"const": "evidence/v0.1.0-confirmatory.json"}, + "implementation_git_commit": {"type": "string", "pattern": "^[0-9a-f]{40}$"}, + "protocol_git_commit": {"type": "string", "pattern": "^[0-9a-f]{40}$"}, + "integrity": { + "type": "object", + "additionalProperties": false, + "required": ["algorithm", "canonical_payload_sha256"], + "properties": { + "algorithm": {"const": "sha256"}, + "canonical_payload_sha256": {"type": "string", "pattern": "^[0-9a-f]{64}$"} + } + } + } +} diff --git a/schemas/regression-card.schema.json b/schemas/regression-card.schema.json new file mode 100644 index 0000000..87239d9 --- /dev/null +++ b/schemas/regression-card.schema.json @@ -0,0 +1,47 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "urn:sftguard:schema:regression-card:v1", + "title": "SFTGuard Regression Card", + "type": "object", + "required": [ + "schema_version", + "tool", + "status", + "inputs", + "metrics", + "summary", + "integrity" + ], + "properties": { + "schema_version": {"const": "sftguard.regression-card.v1"}, + "status": {"enum": ["PASS", "FAIL", "ABSTAIN"]}, + "inputs": {"type": "array"}, + "bootstrap": {"type": "object"}, + "metrics": { + "type": "array", + "items": { + "type": "object", + "required": [ + "group", + "metric", + "status", + "pair_count", + "mean_normalized_improvement", + "confidence_interval", + "threshold", + "decision_rule" + ], + "properties": { + "group": {"enum": ["target", "retention"]}, + "metric": {"type": "string"}, + "status": {"enum": ["PASS", "FAIL"]} + } + } + }, + "summary": {"type": "object"}, + "integrity": { + "type": "object", + "required": ["algorithm", "canonical_payload_sha256"] + } + } +} diff --git a/schemas/regression-contract.schema.json b/schemas/regression-contract.schema.json new file mode 100644 index 0000000..1fb19c3 --- /dev/null +++ b/schemas/regression-contract.schema.json @@ -0,0 +1,32 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "urn:sftguard:schema:regression-contract:v1", + "title": "SFTGuard Regression Card contract", + "type": "object", + "required": ["metrics"], + "properties": { + "bootstrap": { + "type": "object", + "properties": { + "draws": {"type": "integer", "minimum": 1, "maximum": 100000}, + "confidence": {"type": "number", "exclusiveMinimum": 0, "exclusiveMaximum": 1}, + "seed": {"type": "integer"} + } + }, + "metrics": { + "type": "array", + "minItems": 2, + "items": { + "type": "object", + "required": ["group", "metric", "higher_is_better"], + "properties": { + "group": {"enum": ["target", "retention"]}, + "metric": {"type": "string", "pattern": "^[A-Za-z0-9_.-]{1,64}$"}, + "higher_is_better": {"type": "boolean"}, + "min_improvement": {"type": "number"}, + "max_regression": {"type": "number", "minimum": 0} + } + } + } + } +} diff --git a/schemas/regression-evaluation-record.schema.json b/schemas/regression-evaluation-record.schema.json new file mode 100644 index 0000000..f466a9c --- /dev/null +++ b/schemas/regression-evaluation-record.schema.json @@ -0,0 +1,14 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "urn:sftguard:schema:regression-evaluation-record:v1", + "title": "SFTGuard paired evaluation record", + "type": "object", + "required": ["seed", "sample_id", "group", "metric", "value"], + "properties": { + "seed": {"type": "integer"}, + "sample_id": {"type": "string", "minLength": 1, "maxLength": 512}, + "group": {"type": "string", "pattern": "^[A-Za-z0-9_.-]{1,64}$"}, + "metric": {"type": "string", "pattern": "^[A-Za-z0-9_.-]{1,64}$"}, + "value": {"type": "number"} + } +} diff --git a/schemas/sft-record.schema.json b/schemas/sft-record.schema.json new file mode 100644 index 0000000..a0eb932 --- /dev/null +++ b/schemas/sft-record.schema.json @@ -0,0 +1,42 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "urn:sftguard:schema:sft-record:v1", + "title": "SFTGuard chat record", + "type": "object", + "required": ["messages"], + "properties": { + "messages": { + "type": "array", + "minItems": 1, + "items": { + "type": "object", + "required": ["role", "content"], + "properties": { + "role": {"enum": ["system", "user", "assistant", "tool"]}, + "content": {"type": "string"} + } + } + }, + "_sftguard": { + "type": "object", + "required": [ + "token_ids", + "token_roles", + "loss_mask", + "eos_token_id", + "original_assistant_tokens" + ], + "properties": { + "token_ids": {"type": "array", "items": {"type": "integer"}}, + "token_roles": { + "type": "array", + "items": {"enum": ["system", "user", "assistant", "tool"]} + }, + "loss_mask": {"type": "array", "items": {"type": "boolean"}}, + "eos_token_id": {"type": "integer"}, + "original_assistant_tokens": {"type": "integer", "minimum": 0}, + "truncated": {"type": "boolean", "default": false} + } + } + } +} diff --git a/scripts/confirmatory_evidence.py b/scripts/confirmatory_evidence.py new file mode 100644 index 0000000..daf5242 --- /dev/null +++ b/scripts/confirmatory_evidence.py @@ -0,0 +1,296 @@ +"""Generate once or independently verify the frozen SFTGuard v0.1 evidence.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import subprocess +import sys +import tempfile +from pathlib import Path +from typing import Any + +PROTOCOL_COMMIT = "d4c74cefca6729124142635dc8893b5e3ddec2d3" +REPOSITORY_ROOT = Path(__file__).resolve().parents[1] +DEFAULT_ARTIFACT = Path("evidence/v0.1.0-confirmatory.json") +DEFAULT_MANIFEST = Path("evidence/v0.1.0-confirmatory.manifest.json") +MAX_EVIDENCE_BYTES = 16 * 1024 * 1024 + + +class EvidenceError(RuntimeError): + """Raised when an evidence safety or verification condition fails.""" + + +def _git(*arguments: str, cwd: Path = REPOSITORY_ROOT) -> str: + completed = subprocess.run( + ["git", *arguments], + cwd=cwd, + check=False, + capture_output=True, + text=True, + encoding="utf-8", + ) + if completed.returncode != 0: + detail = completed.stderr.strip() or completed.stdout.strip() or "git command failed" + raise EvidenceError(detail) + return completed.stdout.strip() + + +def _sha256(value: bytes) -> str: + return hashlib.sha256(value).hexdigest() + + +def _canonical_json_bytes(value: Any) -> bytes: + return json.dumps( + value, + ensure_ascii=False, + allow_nan=False, + separators=(",", ":"), + sort_keys=True, + ).encode("utf-8") + + +def _seal(payload: dict[str, Any]) -> dict[str, Any]: + sealed = dict(payload) + sealed["integrity"] = { + "algorithm": "sha256", + "canonical_payload_sha256": _sha256(_canonical_json_bytes(sealed)), + } + return sealed + + +def _pretty_json_bytes(value: Any) -> bytes: + return ( + json.dumps( + value, + ensure_ascii=False, + allow_nan=False, + indent=2, + sort_keys=True, + ) + + "\n" + ).encode("utf-8") + + +def _verify_integrity(value: dict[str, Any]) -> bool: + integrity = value.get("integrity") + if not isinstance(integrity, dict): + return False + expected = integrity.get("canonical_payload_sha256") + payload = dict(value) + payload.pop("integrity", None) + return isinstance(expected, str) and expected == _sha256(_canonical_json_bytes(payload)) + + +def _reject_duplicate_keys(pairs: list[tuple[str, Any]]) -> dict[str, Any]: + result: dict[str, Any] = {} + for key, value in pairs: + if key in result: + raise EvidenceError("JSON object contains a duplicate key") + result[key] = value + return result + + +def _parse_object(raw: bytes, label: str) -> dict[str, Any]: + if len(raw) > MAX_EVIDENCE_BYTES: + raise EvidenceError(f"{label} exceeds the evidence byte limit") + try: + value = json.loads(raw.decode("utf-8"), object_pairs_hook=_reject_duplicate_keys) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise EvidenceError(f"{label} is not valid UTF-8 JSON") from exc + if not isinstance(value, dict): + raise EvidenceError(f"{label} must contain a JSON object") + return value + + +def _worker_bytes(source_root: Path, implementation_commit: str) -> bytes: + environment = os.environ.copy() + existing_path = environment.get("PYTHONPATH") + source_path = str(source_root / "src") + environment["PYTHONPATH"] = ( + source_path if not existing_path else os.pathsep.join((source_path, existing_path)) + ) + environment["PYTHONHASHSEED"] = "0" + completed = subprocess.run( + [ + sys.executable, + "-m", + "sftguard.confirmatory", + "--protocol-commit", + PROTOCOL_COMMIT, + "--implementation-commit", + implementation_commit, + ], + cwd=source_root, + env=environment, + check=False, + capture_output=True, + ) + if completed.returncode not in {0, 1}: + detail = completed.stderr.decode("utf-8", errors="replace").strip() + raise EvidenceError(detail or "confirmatory worker failed") + artifact = _parse_object(completed.stdout, "worker output") + if not _verify_integrity(artifact): + raise EvidenceError("worker produced an invalid artifact integrity hash") + return completed.stdout + + +def _resolve_output(path: Path) -> Path: + resolved = (REPOSITORY_ROOT / path).resolve() if not path.is_absolute() else path.resolve() + try: + resolved.relative_to(REPOSITORY_ROOT) + except ValueError as exc: + raise EvidenceError("evidence outputs must remain inside the repository") from exc + return resolved + + +def _write_exclusive(path: Path, raw: bytes) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + try: + with path.open("xb") as stream: + stream.write(raw) + stream.flush() + os.fsync(stream.fileno()) + except FileExistsError as exc: + raise EvidenceError(f"refusing to overwrite existing evidence: {path}") from exc + + +def generate(artifact_path: Path, manifest_path: Path) -> None: + artifact = _resolve_output(artifact_path) + manifest = _resolve_output(manifest_path) + if artifact.exists() or manifest.exists(): + raise EvidenceError("refusing to overwrite an existing evidence bundle") + if _git("status", "--porcelain", "--untracked-files=all"): + raise EvidenceError("generation requires a completely clean Git worktree") + + implementation_commit = _git("rev-parse", "HEAD") + _git("merge-base", "--is-ancestor", PROTOCOL_COMMIT, implementation_commit) + remote_branches = _git("branch", "-r", "--contains", implementation_commit) + if not any(line.strip().startswith("origin/") for line in remote_branches.splitlines()): + raise EvidenceError("implementation commit must be pushed to origin before generation") + + first = _worker_bytes(REPOSITORY_ROOT, implementation_commit) + second = _worker_bytes(REPOSITORY_ROOT, implementation_commit) + if first != second: + raise EvidenceError("fresh worker processes did not reproduce byte-identical evidence") + + evidence = _parse_object(first, "confirmatory artifact") + manifest_value = _seal( + { + "schema_version": "sftguard.confirmatory-manifest.v1", + "artifact_file_sha256": _sha256(first), + "artifact_integrity_sha256": evidence["integrity"]["canonical_payload_sha256"], + "artifact_path": artifact.relative_to(REPOSITORY_ROOT).as_posix(), + "implementation_git_commit": implementation_commit, + "protocol_git_commit": PROTOCOL_COMMIT, + } + ) + _write_exclusive(artifact, first) + try: + _write_exclusive(manifest, _pretty_json_bytes(manifest_value)) + except BaseException: + artifact.unlink(missing_ok=True) + raise + print(f"wrote {artifact.relative_to(REPOSITORY_ROOT)}") + print(f"artifact sha256: {_sha256(first)}") + print(f"carry forward: {str(evidence.get('carry_forward')).lower()}") + + +def _verify_bundle(artifact_path: Path, manifest_path: Path) -> tuple[bytes, dict[str, Any]]: + artifact = _resolve_output(artifact_path) + manifest = _resolve_output(manifest_path) + try: + if artifact.stat().st_size > MAX_EVIDENCE_BYTES: + raise EvidenceError("artifact exceeds the evidence byte limit") + if manifest.stat().st_size > MAX_EVIDENCE_BYTES: + raise EvidenceError("manifest exceeds the evidence byte limit") + artifact_raw = artifact.read_bytes() + manifest_raw = manifest.read_bytes() + except OSError as exc: + raise EvidenceError("evidence bundle cannot be read") from exc + artifact_value = _parse_object(artifact_raw, "artifact") + manifest_value = _parse_object(manifest_raw, "manifest") + if not _verify_integrity(artifact_value) or not _verify_integrity(manifest_value): + raise EvidenceError("artifact or manifest integrity verification failed") + if manifest_value.get("artifact_file_sha256") != _sha256(artifact_raw): + raise EvidenceError("raw artifact SHA-256 does not match the manifest") + if ( + manifest_value.get("artifact_integrity_sha256") + != artifact_value["integrity"]["canonical_payload_sha256"] + ): + raise EvidenceError("canonical artifact SHA-256 does not match the manifest") + if manifest_value.get("protocol_git_commit") != PROTOCOL_COMMIT: + raise EvidenceError("manifest protocol commit does not match the frozen runner") + if manifest_value.get("artifact_path") != artifact.relative_to(REPOSITORY_ROOT).as_posix(): + raise EvidenceError("manifest artifact path does not match the selected artifact") + if artifact_value.get("schema_version") != "sftguard.confirmatory-fault-matrix.v1": + raise EvidenceError("artifact schema version is not the frozen v0.1 schema") + provenance = artifact_value.get("provenance") + if not isinstance(provenance, dict): + raise EvidenceError("artifact provenance is missing") + if provenance.get("protocol_git_commit") != PROTOCOL_COMMIT: + raise EvidenceError("artifact protocol commit does not match the frozen runner") + implementation_commit = provenance.get("implementation_git_commit") + if ( + not isinstance(implementation_commit, str) + or len(implementation_commit) != 40 + or any(character not in "0123456789abcdef" for character in implementation_commit) + ): + raise EvidenceError("artifact implementation commit is invalid") + if implementation_commit != manifest_value.get("implementation_git_commit"): + raise EvidenceError("implementation commit differs between artifact and manifest") + return artifact_raw, manifest_value + + +def verify(artifact_path: Path, manifest_path: Path, *, replay: bool) -> None: + artifact_raw, manifest = _verify_bundle(artifact_path, manifest_path) + implementation_commit = manifest["implementation_git_commit"] + if replay: + _git("cat-file", "-e", f"{implementation_commit}^{{commit}}") + with tempfile.TemporaryDirectory(prefix="sftguard-evidence-") as temporary: + worktree = Path(temporary) / "source" + _git("worktree", "add", "--detach", str(worktree), implementation_commit) + try: + replayed = _worker_bytes(worktree, implementation_commit) + finally: + _git("worktree", "remove", "--force", str(worktree)) + if replayed != artifact_raw: + raise EvidenceError("exact implementation replay did not match checked-in evidence") + print("evidence integrity verified") + if replay: + print("exact implementation replay matched") + + +def _parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + commands = parser.add_subparsers(dest="command", required=True) + generate_command = commands.add_parser("generate") + verify_command = commands.add_parser("verify") + for command in (generate_command, verify_command): + command.add_argument("--artifact", type=Path, default=DEFAULT_ARTIFACT) + command.add_argument("--manifest", type=Path, default=DEFAULT_MANIFEST) + verify_command.add_argument( + "--integrity-only", + action="store_true", + help="verify hashes without recreating the implementation worktree", + ) + return parser + + +def main() -> int: + args = _parser().parse_args() + try: + if args.command == "generate": + generate(args.artifact, args.manifest) + else: + verify(args.artifact, args.manifest, replay=not args.integrity_only) + except EvidenceError as exc: + print(f"error: {exc}", file=sys.stderr) + return 2 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/sftguard/__init__.py b/src/sftguard/__init__.py new file mode 100644 index 0000000..a547413 --- /dev/null +++ b/src/sftguard/__init__.py @@ -0,0 +1,8 @@ +"""SFTGuard public package surface.""" + +__version__ = "0.1.0" + +from .audit import AuditConfig, audit_files, audit_records +from .regression import build_regression_card + +__all__ = ["AuditConfig", "audit_files", "audit_records", "build_regression_card"] diff --git a/src/sftguard/__main__.py b/src/sftguard/__main__.py new file mode 100644 index 0000000..eb53e2f --- /dev/null +++ b/src/sftguard/__main__.py @@ -0,0 +1,3 @@ +from .cli import main + +raise SystemExit(main()) diff --git a/src/sftguard/artifacts.py b/src/sftguard/artifacts.py new file mode 100644 index 0000000..caa2b72 --- /dev/null +++ b/src/sftguard/artifacts.py @@ -0,0 +1,88 @@ +"""Deterministic, content-addressed JSON artifact helpers.""" + +from __future__ import annotations + +import copy +import hashlib +import json +import os +import tempfile +from collections.abc import Mapping +from pathlib import Path +from typing import Any + + +def canonical_json_bytes(value: Any) -> bytes: + """Return the repository's canonical UTF-8 JSON representation.""" + + return json.dumps( + value, + ensure_ascii=False, + allow_nan=False, + separators=(",", ":"), + sort_keys=True, + ).encode("utf-8") + + +def sha256_bytes(value: bytes) -> str: + return hashlib.sha256(value).hexdigest() + + +def seal_artifact(payload: Mapping[str, Any]) -> dict[str, Any]: + """Attach a hash of the payload excluding the integrity field itself.""" + + sealed = copy.deepcopy(dict(payload)) + sealed.pop("integrity", None) + sealed["integrity"] = { + "algorithm": "sha256", + "canonical_payload_sha256": sha256_bytes(canonical_json_bytes(sealed)), + } + return sealed + + +def verify_artifact(artifact: Mapping[str, Any]) -> bool: + integrity = artifact.get("integrity") + if not isinstance(integrity, Mapping): + return False + expected = integrity.get("canonical_payload_sha256") + if not isinstance(expected, str): + return False + payload = copy.deepcopy(dict(artifact)) + payload.pop("integrity", None) + return expected == sha256_bytes(canonical_json_bytes(payload)) + + +def write_artifact(path: str | Path | None, artifact: Mapping[str, Any]) -> None: + """Write an artifact atomically, or print it when path is ``None``.""" + + rendered = ( + json.dumps( + artifact, + ensure_ascii=False, + allow_nan=False, + indent=2, + sort_keys=True, + ) + + "\n" + ) + if path is None: + print(rendered, end="") + return + + destination = Path(path) + destination.parent.mkdir(parents=True, exist_ok=True) + handle, temporary_name = tempfile.mkstemp( + prefix=f".{destination.name}.", suffix=".tmp", dir=destination.parent + ) + try: + with os.fdopen(handle, "w", encoding="utf-8", newline="\n") as stream: + stream.write(rendered) + stream.flush() + os.fsync(stream.fileno()) + os.replace(temporary_name, destination) + except BaseException: + try: + os.unlink(temporary_name) + except FileNotFoundError: + pass + raise diff --git a/src/sftguard/audit.py b/src/sftguard/audit.py new file mode 100644 index 0000000..cfee05f --- /dev/null +++ b/src/sftguard/audit.py @@ -0,0 +1,396 @@ +"""Deterministic SFT dataset and rendered-mask audits.""" + +from __future__ import annotations + +from collections import Counter +from collections.abc import Iterable +from dataclasses import dataclass +from typing import Any + +from . import __version__ +from .artifacts import canonical_json_bytes, seal_artifact, sha256_bytes +from .io import ( + DEFAULT_MAX_FILE_BYTES, + DEFAULT_MAX_LINE_BYTES, + DEFAULT_MAX_RECORDS, + InputError, + InputSummary, + read_jsonl_bounded, + summarize_records, +) +from .models import SEVERITY_ORDER, Finding, Status, combine_status +from .privacy import scan_record +from .tokenizer import FixtureMaskInspector, MaskInspector + +ALLOWED_MESSAGE_ROLES = {"system", "user", "assistant", "tool"} + + +@dataclass(frozen=True) +class AuditConfig: + max_file_bytes: int = DEFAULT_MAX_FILE_BYTES + max_line_bytes: int = DEFAULT_MAX_LINE_BYTES + max_records: int = DEFAULT_MAX_RECORDS + require_assistant_mask: bool = True + scan_privacy: bool = True + + def as_dict(self) -> dict[str, Any]: + return { + "max_file_bytes": self.max_file_bytes, + "max_line_bytes": self.max_line_bytes, + "max_records": self.max_records, + "require_assistant_mask": self.require_assistant_mask, + "scan_privacy": self.scan_privacy, + } + + +@dataclass(frozen=True) +class _RecordIdentity: + split: str + line: int + record_sha256: str + prompt_sha256: str | None + answer_sha256: str | None + + +def _record_hash(record: dict[str, Any], *, split: str, line: int) -> tuple[str, bool]: + try: + return sha256_bytes(canonical_json_bytes(record)), True + except (TypeError, ValueError, RecursionError): + marker = canonical_json_bytes({"invalid_record": True, "split": split, "line": line}) + return sha256_bytes(marker), False + + +def _message_identity( + record: dict[str, Any], + *, + split: str, + line: int, + record_sha256: str, +) -> tuple[str | None, str | None, list[Finding]]: + findings: list[Finding] = [] + messages = record.get("messages") + if not isinstance(messages, list) or not messages: + findings.append( + Finding("invalid_messages", "error", split, line, record_sha256=record_sha256) + ) + return None, None, findings + + valid_messages: list[dict[str, str]] = [] + awaiting_assistant = False + for message in messages: + if not isinstance(message, dict): + findings.append( + Finding("invalid_messages", "error", split, line, record_sha256=record_sha256) + ) + continue + role = message.get("role") + content = message.get("content") + if not isinstance(role, str) or role not in ALLOWED_MESSAGE_ROLES: + findings.append( + Finding( + "invalid_role", + "error", + split, + line, + field="role", + record_sha256=record_sha256, + ) + ) + continue + if not isinstance(content, str): + findings.append( + Finding( + "invalid_messages", + "error", + split, + line, + field="content", + record_sha256=record_sha256, + ) + ) + continue + valid_messages.append({"role": role, "content": content}) + if role == "user": + awaiting_assistant = True + elif role == "assistant": + if not awaiting_assistant: + findings.append( + Finding("role_order", "error", split, line, record_sha256=record_sha256) + ) + awaiting_assistant = False + + if len(valid_messages) != len(messages): + return None, None, findings + + assistant_indexes = [ + index for index, message in enumerate(valid_messages) if message["role"] == "assistant" + ] + if not assistant_indexes: + findings.append( + Finding("missing_assistant", "error", split, line, record_sha256=record_sha256) + ) + return None, None, findings + final_index = assistant_indexes[-1] + prompt = valid_messages[:final_index] + answer = valid_messages[final_index]["content"] + prompt_hash = sha256_bytes(canonical_json_bytes(prompt)) + answer_hash = sha256_bytes(canonical_json_bytes(answer)) + return prompt_hash, answer_hash, findings + + +def _check_status(findings: Iterable[Finding], signals: set[str]) -> Status: + relevant = [finding for finding in findings if finding.signal in signals] + if any(SEVERITY_ORDER[finding.severity] >= SEVERITY_ORDER["error"] for finding in relevant): + return Status.FAIL + return Status.PASS + + +def _abstain_report(config: AuditConfig, reason: str) -> dict[str, Any]: + return seal_artifact( + { + "schema_version": "sftguard.audit.v1", + "tool": {"name": "sftguard", "version": __version__}, + "status": Status.ABSTAIN, + "config": config.as_dict(), + "inputs": [], + "checks": {"input": {"status": Status.ABSTAIN, "reason": reason}}, + "summary": {"finding_count": 0, "signal_counts": {}}, + "findings": [], + } + ) + + +def audit_files( + train_path: str, + evaluation_path: str, + *, + config: AuditConfig | None = None, + mask_inspector: MaskInspector | None = None, +) -> dict[str, Any]: + selected = config or AuditConfig() + try: + train = read_jsonl_bounded( + train_path, + max_file_bytes=selected.max_file_bytes, + max_line_bytes=selected.max_line_bytes, + max_records=selected.max_records, + ) + evaluation = read_jsonl_bounded( + evaluation_path, + max_file_bytes=selected.max_file_bytes, + max_line_bytes=selected.max_line_bytes, + max_records=selected.max_records, + ) + except InputError: + return _abstain_report(selected, "invalid_unreadable_or_oversized_input") + return audit_records( + train.records, + evaluation.records, + train_summary=train.summary, + evaluation_summary=evaluation.summary, + config=selected, + mask_inspector=mask_inspector, + ) + + +def audit_records( + train_records: Iterable[tuple[int, dict[str, Any]]], + evaluation_records: Iterable[tuple[int, dict[str, Any]]], + *, + train_summary: InputSummary | None = None, + evaluation_summary: InputSummary | None = None, + config: AuditConfig | None = None, + mask_inspector: MaskInspector | None = None, +) -> dict[str, Any]: + selected = config or AuditConfig() + inspector = mask_inspector or FixtureMaskInspector() + train = tuple(train_records) + evaluation = tuple(evaluation_records) + train_summary = train_summary or summarize_records(train) + evaluation_summary = evaluation_summary or summarize_records(evaluation) + + if not train or not evaluation: + return _abstain_report(selected, "train_and_evaluation_records_required") + + findings: list[Finding] = [] + identities: list[_RecordIdentity] = [] + mask_abstentions = 0 + seen_records: dict[tuple[str, str], int] = {} + + for split, records in (("train", train), ("evaluation", evaluation)): + for line, record in records: + record_sha, record_serializable = _record_hash(record, split=split, line=line) + if not record_serializable: + findings.append( + Finding( + "invalid_record", + "error", + split, + line, + record_sha256=record_sha, + ) + ) + prompt_sha, answer_sha, schema_findings = _message_identity( + record, + split=split, + line=line, + record_sha256=record_sha, + ) + findings.extend(schema_findings) + identities.append(_RecordIdentity(split, line, record_sha, prompt_sha, answer_sha)) + + duplicate_key = (split, record_sha) + if duplicate_key in seen_records: + findings.append( + Finding( + "exact_duplicate", + "warning", + split, + line, + related_line=seen_records[duplicate_key], + record_sha256=record_sha, + ) + ) + else: + seen_records[duplicate_key] = line + + if selected.scan_privacy: + for signal, category, field in scan_record(record): + findings.append( + Finding( + signal, + "critical" if signal == "secret_pattern" else "warning", + split, + line, + field=field, + category=category, + record_sha256=record_sha, + redacted=True, + ) + ) + + inspection = inspector.inspect(record) + if not inspection.available: + if selected.require_assistant_mask: + mask_abstentions += 1 + findings.append( + Finding( + inspection.reason or "assistant_mask_unavailable", + "info", + split, + line, + record_sha256=record_sha, + redacted=True, + ) + ) + else: + for signal in inspection.signals: + findings.append( + Finding( + signal.signal, + signal.severity, + split, + line, + record_sha256=record_sha, + ) + ) + + prompt_answers: dict[str, dict[str, int]] = {} + for identity in identities: + if ( + identity.split != "train" + or identity.prompt_sha256 is None + or identity.answer_sha256 is None + ): + continue + prior = prompt_answers.setdefault(identity.prompt_sha256, {}) + if prior and identity.answer_sha256 not in prior: + findings.append( + Finding( + "prompt_conflict", + "error", + identity.split, + identity.line, + related_line=min(prior.values()), + record_sha256=identity.record_sha256, + ) + ) + prior.setdefault(identity.answer_sha256, identity.line) + + prompts_by_split: dict[str, dict[str, int]] = {"train": {}, "evaluation": {}} + for identity in identities: + if identity.prompt_sha256 is not None: + prompts_by_split[identity.split].setdefault(identity.prompt_sha256, identity.line) + for prompt_sha in sorted(set(prompts_by_split["train"]) & set(prompts_by_split["evaluation"])): + findings.append( + Finding( + "train_eval_leakage", + "critical", + "train", + prompts_by_split["train"][prompt_sha], + related_line=prompts_by_split["evaluation"][prompt_sha], + record_sha256=prompt_sha, + ) + ) + + findings.sort( + key=lambda finding: ( + finding.split, + finding.line, + finding.signal, + finding.related_line or 0, + finding.category or "", + ) + ) + signal_counts = Counter(finding.signal for finding in findings) + + schema_signals = { + "invalid_record", + "invalid_messages", + "invalid_role", + "role_order", + "missing_assistant", + } + exact_signals = {"exact_duplicate", "prompt_conflict", "train_eval_leakage"} + privacy_signals = {"secret_pattern", "pii_like"} + mask_signals = { + "prompt_token_supervised", + "assistant_fully_truncated", + "missing_eos", + } + checks = { + "schema": {"status": _check_status(findings, schema_signals)}, + "duplicates_conflicts_and_leakage": {"status": _check_status(findings, exact_signals)}, + "privacy_patterns": {"status": _check_status(findings, privacy_signals)}, + "assistant_mask": { + "status": ( + Status.ABSTAIN if mask_abstentions else _check_status(findings, mask_signals) + ), + "abstained_record_count": mask_abstentions, + }, + } + overall = combine_status(*(entry["status"] for entry in checks.values())) + + return seal_artifact( + { + "schema_version": "sftguard.audit.v1", + "tool": {"name": "sftguard", "version": __version__}, + "status": overall, + "config": selected.as_dict(), + "inputs": [ + train_summary.as_dict("train"), + evaluation_summary.as_dict("evaluation"), + ], + "checks": checks, + "summary": { + "finding_count": len(findings), + "signal_counts": dict(sorted(signal_counts.items())), + "error_or_critical_count": sum( + SEVERITY_ORDER[finding.severity] >= SEVERITY_ORDER["error"] + for finding in findings + ), + "redacted_finding_count": sum(finding.redacted for finding in findings), + }, + "findings": [finding.as_dict() for finding in findings], + } + ) diff --git a/src/sftguard/benchmark.py b/src/sftguard/benchmark.py new file mode 100644 index 0000000..a506879 --- /dev/null +++ b/src/sftguard/benchmark.py @@ -0,0 +1,124 @@ +"""Development-only execution of the preregistered synthetic fault matrix.""" + +from __future__ import annotations + +import statistics +from collections import defaultdict +from collections.abc import Iterable +from typing import Any + +from . import __version__ +from .artifacts import canonical_json_bytes, seal_artifact +from .audit import audit_records +from .models import Status +from .synthetic import DEVELOPMENT_SEEDS, FAULT_SPECS, generate_case + + +def validate_development_seeds(seeds: Iterable[int]) -> tuple[int, ...]: + selected = tuple(sorted(set(seeds))) + if not selected: + raise ValueError("at least one development seed is required") + if any(seed not in DEVELOPMENT_SEEDS for seed in selected): + raise ValueError("benchmark execution is restricted to development seeds 0 through 9") + return selected + + +def _run_once(seeds: tuple[int, ...]) -> tuple[dict[str, Any], tuple[str, ...]]: + cases: list[dict[str, Any]] = [] + raw_markers: list[str] = [] + recall_by_fault: dict[str, list[bool]] = defaultdict(list) + clean_false_positives = 0 + clean_count = 0 + + for seed in seeds: + for fault_id in (None, *(spec.fault_id for spec in FAULT_SPECS)): + case = generate_case(seed, fault_id) + raw_markers.extend(case.private_markers) + report = audit_records(case.train, case.evaluation) + observed_signals = sorted(report["summary"]["signal_counts"]) + expected = case.fault.expected_signal if case.fault else None + observed_primary = expected in observed_signals if expected else not observed_signals + if case.fault is None: + clean_count += 1 + if observed_signals: + clean_false_positives += 1 + else: + recall_by_fault[case.fault.fault_id].append(observed_primary) + cases.append( + { + "seed": seed, + "kind": "fault" if case.fault else "clean_control", + "fault_id": case.fault.fault_id if case.fault else None, + "expected_primary_signal": expected, + "observed_primary_signal": observed_primary, + "observed_signals": observed_signals, + "audit_status": report["status"], + "case_sha256": case.case_sha256, + } + ) + + per_fault: list[dict[str, Any]] = [] + fault_recalls: list[float] = [] + for spec in FAULT_SPECS: + observations = recall_by_fault[spec.fault_id] + detected = sum(observations) + recall = detected / len(observations) + fault_recalls.append(recall) + per_fault.append( + { + "fault_id": spec.fault_id, + "expected_signal": spec.expected_signal, + "severity": spec.severity, + "case_count": len(observations), + "detected_count": detected, + "recall": round(recall, 12), + } + ) + + critical = [item for item in per_fault if item["severity"] == "critical"] + macro_recall = statistics.fmean(fault_recalls) + clean_false_positive_rate = clean_false_positives / clean_count + base_gates = { + "critical_fault_recall_100_percent": all(item["recall"] == 1.0 for item in critical), + "macro_recall_at_least_90_percent": macro_recall >= 0.9, + "clean_false_positive_rate_at_most_10_percent": clean_false_positive_rate <= 0.1, + "expected_and_observed_primary_recorded": all( + case["expected_primary_signal"] is None + or isinstance(case["observed_primary_signal"], bool) + for case in cases + ), + } + payload = { + "schema_version": "sftguard.development-benchmark.v1", + "tool": {"name": "sftguard", "version": __version__}, + "development_only": True, + "seed_range": {"first": seeds[0], "last": seeds[-1], "count": len(seeds)}, + "fault_count": len(FAULT_SPECS), + "summary": { + "case_count": len(cases), + "clean_control_count": clean_count, + "clean_false_positive_count": clean_false_positives, + "clean_false_positive_rate": round(clean_false_positive_rate, 12), + "macro_recall": round(macro_recall, 12), + }, + "per_fault": per_fault, + "cases": cases, + "gates": base_gates, + } + return payload, tuple(raw_markers) + + +def run_development_benchmark( + seeds: Iterable[int] = DEVELOPMENT_SEEDS, +) -> dict[str, Any]: + selected = validate_development_seeds(seeds) + first, markers = _run_once(selected) + second, _ = _run_once(selected) + reproducible = canonical_json_bytes(first) == canonical_json_bytes(second) + rendered = canonical_json_bytes(first).decode("utf-8") + redacted = all(marker not in rendered for marker in markers) + first["gates"]["in_process_byte_reproducible"] = reproducible + first["gates"]["raw_synthetic_text_absent"] = redacted + first["all_development_gates_passed"] = all(first["gates"].values()) + first["status"] = Status.PASS if first["all_development_gates_passed"] else Status.FAIL + return seal_artifact(first) diff --git a/src/sftguard/cli.py b/src/sftguard/cli.py new file mode 100644 index 0000000..c488e3a --- /dev/null +++ b/src/sftguard/cli.py @@ -0,0 +1,146 @@ +"""Command-line interface for audit, gate, benchmark, and demo workflows.""" + +from __future__ import annotations + +import argparse +import re +from collections.abc import Sequence +from typing import Any + +from . import __version__ +from .artifacts import seal_artifact, write_artifact +from .audit import AuditConfig, audit_files +from .benchmark import run_development_benchmark, validate_development_seeds +from .demo import run_demo +from .io import InputError, read_json_bounded, read_jsonl_bounded +from .models import Status +from .regression import build_regression_card + +EXIT_CODES = {Status.PASS: 0, Status.FAIL: 1, Status.ABSTAIN: 2} + + +def _bounded_positive_int(value: str) -> int: + parsed = int(value) + if parsed <= 0: + raise argparse.ArgumentTypeError("value must be positive") + return parsed + + +def _parse_seeds(value: str) -> tuple[int, ...]: + try: + selected: list[int] = [] + for part in value.split(","): + part = part.strip() + range_match = re.fullmatch(r"([0-9])-([0-9])", part) + if range_match: + start, end = (int(component) for component in range_match.groups()) + if end < start: + raise ValueError + selected.extend(range(start, end + 1)) + elif re.fullmatch(r"[0-9]", part): + selected.append(int(part)) + else: + raise ValueError + return validate_development_seeds(selected) + except ValueError as exc: + raise argparse.ArgumentTypeError( + "seeds must be a subset of development seeds 0 through 9" + ) from exc + + +def _input_abstention(schema_version: str) -> dict[str, Any]: + return seal_artifact( + { + "schema_version": schema_version, + "tool": {"name": "sftguard", "version": __version__}, + "status": Status.ABSTAIN, + "reason_codes": ["invalid_unreadable_or_oversized_input"], + } + ) + + +def _emit(artifact: dict[str, Any], output: str | None) -> int: + write_artifact(output, artifact) + status = Status(artifact["status"]) + return EXIT_CODES[status] + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + prog="sftguard", + description="Local-first correctness and regression gates for SFT pipelines.", + ) + parser.add_argument("--version", action="version", version=__version__) + commands = parser.add_subparsers(dest="command", required=True) + + audit = commands.add_parser("audit", help="audit training and evaluation JSONL files") + audit.add_argument("--train", required=True) + audit.add_argument("--eval", required=True, dest="evaluation") + audit.add_argument("--output") + audit.add_argument("--max-file-bytes", type=_bounded_positive_int, default=16 * 1024 * 1024) + audit.add_argument("--max-line-bytes", type=_bounded_positive_int, default=1024 * 1024) + audit.add_argument("--max-records", type=_bounded_positive_int, default=100_000) + + gate = commands.add_parser("gate", help="build a paired target/retention Regression Card") + gate.add_argument("--base", required=True) + gate.add_argument("--adapter", required=True) + gate.add_argument("--contract", required=True) + gate.add_argument("--output") + gate.add_argument("--max-file-bytes", type=_bounded_positive_int, default=16 * 1024 * 1024) + gate.add_argument("--max-line-bytes", type=_bounded_positive_int, default=1024 * 1024) + gate.add_argument("--max-records", type=_bounded_positive_int, default=100_000) + + benchmark = commands.add_parser( + "benchmark", help="run the deterministic development-only synthetic matrix" + ) + benchmark.add_argument("--seeds", type=_parse_seeds, default=tuple(range(10))) + benchmark.add_argument("--output") + + demo = commands.add_parser("demo", help="run a redacted deterministic end-to-end demo") + demo.add_argument("--output") + return parser + + +def main(argv: Sequence[str] | None = None) -> int: + args = build_parser().parse_args(argv) + if args.command == "audit": + config = AuditConfig( + max_file_bytes=args.max_file_bytes, + max_line_bytes=args.max_line_bytes, + max_records=args.max_records, + ) + return _emit(audit_files(args.train, args.evaluation, config=config), args.output) + + if args.command == "gate": + try: + base = read_jsonl_bounded( + args.base, + max_file_bytes=args.max_file_bytes, + max_line_bytes=args.max_line_bytes, + max_records=args.max_records, + ) + adapter = read_jsonl_bounded( + args.adapter, + max_file_bytes=args.max_file_bytes, + max_line_bytes=args.max_line_bytes, + max_records=args.max_records, + ) + contract, _, _ = read_json_bounded(args.contract, max_file_bytes=args.max_file_bytes) + artifact = build_regression_card( + base.records, + adapter.records, + contract, + base_summary=base.summary, + adapter_summary=adapter.summary, + ) + except InputError: + artifact = _input_abstention("sftguard.regression-card.v1") + return _emit(artifact, args.output) + + if args.command == "benchmark": + return _emit(run_development_benchmark(args.seeds), args.output) + + if args.command == "demo": + return _emit(run_demo(), args.output) + + raise AssertionError("argparse accepted an unknown command") diff --git a/src/sftguard/confirmatory.py b/src/sftguard/confirmatory.py new file mode 100644 index 0000000..8bda5fc --- /dev/null +++ b/src/sftguard/confirmatory.py @@ -0,0 +1,231 @@ +"""Frozen v0.1 confirmatory fault-matrix evidence construction. + +The public orchestration script invokes this module only from a clean, pushed +implementation commit. Keeping construction separate from orchestration makes +the payload testable on development seeds without touching the confirmatory +range before the implementation is frozen. +""" + +from __future__ import annotations + +import argparse +import json +import platform +import re +import statistics +import sys +from collections import defaultdict +from collections.abc import Callable, Iterable, Sequence +from typing import Any + +from . import __version__ +from .artifacts import canonical_json_bytes, seal_artifact +from .audit import AuditConfig, audit_records +from .models import Status +from .synthetic import ( + CONFIRMATORY_SEEDS, + FAULT_SPECS, + SyntheticCase, + generate_confirmatory_case, +) + +_FULL_COMMIT = re.compile(r"^[0-9a-f]{40}$") + + +def pretty_json_bytes(value: Any) -> bytes: + """Render stable human-readable JSON for the checked-in evidence file.""" + + return ( + json.dumps( + value, + ensure_ascii=False, + allow_nan=False, + indent=2, + sort_keys=True, + ) + + "\n" + ).encode("utf-8") + + +def _validate_commit(value: str, label: str) -> None: + if _FULL_COMMIT.fullmatch(value) is None: + raise ValueError(f"{label} must be a full lowercase Git commit") + + +def _build_artifact( + seeds: Iterable[int], + case_factory: Callable[[int, str | None], SyntheticCase], + *, + protocol_commit: str, + implementation_commit: str, + fresh_process_byte_reproducible: bool, +) -> dict[str, Any]: + """Build a sealed matrix; tests exercise this with development seeds.""" + + _validate_commit(protocol_commit, "protocol_commit") + _validate_commit(implementation_commit, "implementation_commit") + selected = tuple(seeds) + if not selected or tuple(sorted(set(selected))) != selected: + raise ValueError("seeds must be a non-empty sorted unique sequence") + + cases: list[dict[str, Any]] = [] + recall_by_fault: dict[str, list[bool]] = defaultdict(list) + private_text: set[str] = set() + clean_false_positives = 0 + + for seed in selected: + for fault_id in (None, *(spec.fault_id for spec in FAULT_SPECS)): + case = case_factory(seed, fault_id) + private_text.update(case.private_markers) + for _, record in (*case.train, *case.evaluation): + messages = record.get("messages") + if isinstance(messages, list): + private_text.update( + message["content"] + for message in messages + if isinstance(message, dict) and isinstance(message.get("content"), str) + ) + + report = audit_records(case.train, case.evaluation) + observed_signals = tuple(sorted(report["summary"]["signal_counts"])) + expected = case.fault.expected_signal if case.fault else None + expectation_met = expected in observed_signals if expected else not observed_signals + observed_primary = expected if expected in observed_signals else None + + if case.fault is None: + if observed_signals: + clean_false_positives += 1 + else: + recall_by_fault[case.fault.fault_id].append(expectation_met) + + cases.append( + { + "audit_status": report["status"], + "case_sha256": case.case_sha256, + "expected_primary_signal": expected, + "fault_id": case.fault.fault_id if case.fault else None, + "kind": "fault" if case.fault else "clean_control", + "observed_primary_signal": observed_primary, + "observed_signal_ids": list(observed_signals), + "primary_expectation_met": expectation_met, + "seed": seed, + } + ) + + per_fault: list[dict[str, Any]] = [] + recalls: list[float] = [] + for spec in FAULT_SPECS: + observations = recall_by_fault[spec.fault_id] + detected_count = sum(observations) + recall = detected_count / len(observations) + recalls.append(recall) + per_fault.append( + { + "case_count": len(observations), + "detected_count": detected_count, + "expected_signal": spec.expected_signal, + "fault_id": spec.fault_id, + "recall": round(recall, 12), + "severity": spec.severity, + } + ) + + clean_count = len(selected) + clean_false_positive_rate = clean_false_positives / clean_count + macro_recall = statistics.fmean(recalls) + critical_faults = [item for item in per_fault if item["severity"] == "critical"] + expected_observed_recorded = all( + "expected_primary_signal" in case and "observed_primary_signal" in case for case in cases + ) + + payload_without_redaction_gate: dict[str, Any] = { + "schema_version": "sftguard.confirmatory-fault-matrix.v1", + "tool": {"name": "sftguard", "version": __version__}, + "provenance": { + "implementation_git_commit": implementation_commit, + "protocol_git_commit": protocol_commit, + "python_implementation": platform.python_implementation(), + "python_version": f"{sys.version_info.major}.{sys.version_info.minor}", + }, + "protocol": { + "audit_config": AuditConfig().as_dict(), + "faults": [ + { + "expected_signal": spec.expected_signal, + "fault_id": spec.fault_id, + "severity": spec.severity, + } + for spec in FAULT_SPECS + ], + "seed_range": { + "count": len(selected), + "first": selected[0], + "last": selected[-1], + }, + }, + "summary": { + "case_count": len(cases), + "clean_control_count": clean_count, + "clean_false_positive_count": clean_false_positives, + "clean_false_positive_rate": round(clean_false_positive_rate, 12), + "fault_case_count": len(cases) - clean_count, + "macro_recall": round(macro_recall, 12), + }, + "per_fault": per_fault, + "cases": cases, + "gates": { + "clean_false_positive_rate_at_most_10_percent": (clean_false_positive_rate <= 0.10), + "critical_fault_recall_100_percent": all( + item["recall"] == 1.0 for item in critical_faults + ), + "expected_and_observed_primary_recorded": expected_observed_recorded, + "fresh_process_byte_reproducible": fresh_process_byte_reproducible, + "macro_recall_at_least_90_percent": macro_recall >= 0.90, + }, + } + rendered = canonical_json_bytes(payload_without_redaction_gate).decode("utf-8") + payload_without_redaction_gate["gates"]["raw_synthetic_text_absent"] = all( + value not in rendered for value in private_text + ) + carry_forward = all(payload_without_redaction_gate["gates"].values()) + payload_without_redaction_gate["carry_forward"] = carry_forward + payload_without_redaction_gate["status"] = Status.PASS if carry_forward else Status.FAIL + return seal_artifact(payload_without_redaction_gate) + + +def build_confirmatory_artifact( + *, + protocol_commit: str, + implementation_commit: str, + fresh_process_byte_reproducible: bool = True, +) -> dict[str, Any]: + """Execute the exact preregistered seed range and fault matrix.""" + + return _build_artifact( + CONFIRMATORY_SEEDS, + generate_confirmatory_case, + protocol_commit=protocol_commit, + implementation_commit=implementation_commit, + fresh_process_byte_reproducible=fresh_process_byte_reproducible, + ) + + +def _parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description="Build frozen SFTGuard evidence bytes.") + parser.add_argument("--protocol-commit", required=True) + parser.add_argument("--implementation-commit", required=True) + return parser + + +def main(argv: Sequence[str] | None = None) -> int: + args = _parser().parse_args(argv) + artifact = build_confirmatory_artifact( + protocol_commit=args.protocol_commit, + implementation_commit=args.implementation_commit, + ) + sys.stdout.buffer.write(pretty_json_bytes(artifact)) + return 0 if artifact["status"] == Status.PASS else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/sftguard/demo.py b/src/sftguard/demo.py new file mode 100644 index 0000000..fdf6e6e --- /dev/null +++ b/src/sftguard/demo.py @@ -0,0 +1,77 @@ +"""Small deterministic, redacted end-to-end demonstration.""" + +from __future__ import annotations + +from typing import Any + +from . import __version__ +from .artifacts import seal_artifact +from .audit import audit_records +from .models import Status +from .regression import build_regression_card +from .synthetic import generate_case + + +def _evaluation_records() -> tuple[ + tuple[tuple[int, dict[str, Any]], ...], tuple[tuple[int, dict[str, Any]], ...] +]: + base: list[tuple[int, dict[str, Any]]] = [] + adapter: list[tuple[int, dict[str, Any]]] = [] + line = 1 + for seed in range(4): + for group, metric, base_value, adapter_value in ( + ("target", "task_score", 0.50 + seed * 0.01, 0.65 + seed * 0.01), + ("retention", "general_score", 0.80 - seed * 0.01, 0.795 - seed * 0.01), + ): + common = { + "seed": seed, + "sample_id": f"development-{group}-{seed}", + "group": group, + "metric": metric, + } + base.append((line, {**common, "value": base_value})) + adapter.append((line, {**common, "value": adapter_value})) + line += 1 + return tuple(base), tuple(adapter) + + +def run_demo() -> dict[str, Any]: + case = generate_case(0) + audit = audit_records(case.train, case.evaluation) + base, adapter = _evaluation_records() + card = build_regression_card( + base, + adapter, + { + "bootstrap": {"draws": 10_000, "confidence": 0.95, "seed": 2339}, + "metrics": [ + { + "group": "target", + "metric": "task_score", + "higher_is_better": True, + "min_improvement": 0.10, + }, + { + "group": "retention", + "metric": "general_score", + "higher_is_better": True, + "max_regression": 0.02, + }, + ], + }, + ) + status = ( + Status.PASS + if audit["status"] == Status.PASS and card["status"] == Status.PASS + else Status.FAIL + ) + return seal_artifact( + { + "schema_version": "sftguard.demo.v1", + "tool": {"name": "sftguard", "version": __version__}, + "status": status, + "note": "Deterministic synthetic development demonstration; not external evidence.", + "audit": audit, + "regression_card": card, + } + ) diff --git a/src/sftguard/io.py b/src/sftguard/io.py new file mode 100644 index 0000000..cb1a21f --- /dev/null +++ b/src/sftguard/io.py @@ -0,0 +1,153 @@ +"""Bounded JSON and JSONL readers used by every untrusted input path.""" + +from __future__ import annotations + +import hashlib +import json +from collections.abc import Iterable +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +DEFAULT_MAX_FILE_BYTES = 16 * 1024 * 1024 +DEFAULT_MAX_LINE_BYTES = 1024 * 1024 +DEFAULT_MAX_RECORDS = 100_000 + + +class InputError(ValueError): + """Raised when an input cannot be safely or unambiguously parsed.""" + + +@dataclass(frozen=True) +class InputSummary: + sha256: str + byte_count: int + record_count: int + + def as_dict(self, split: str) -> dict[str, Any]: + return { + "split": split, + "sha256": self.sha256, + "byte_count": self.byte_count, + "record_count": self.record_count, + } + + +@dataclass(frozen=True) +class ParsedJSONL: + records: tuple[tuple[int, dict[str, Any]], ...] + summary: InputSummary + + +def _reject_duplicate_keys(pairs: list[tuple[str, Any]]) -> dict[str, Any]: + result: dict[str, Any] = {} + for key, value in pairs: + if key in result: + raise InputError("JSON object contains a duplicate key") + result[key] = value + return result + + +def _decode_json(raw: bytes, *, context: str) -> Any: + try: + text = raw.decode("utf-8") + except UnicodeDecodeError as exc: + raise InputError(f"{context} is not valid UTF-8") from exc + try: + return json.loads( + text, + object_pairs_hook=_reject_duplicate_keys, + parse_constant=lambda _value: (_ for _ in ()).throw( + InputError(f"{context} contains a non-finite number") + ), + ) + except InputError: + raise + except (json.JSONDecodeError, RecursionError) as exc: + raise InputError(f"{context} is not valid JSON") from exc + + +def read_jsonl_bounded( + path: str | Path, + *, + max_file_bytes: int = DEFAULT_MAX_FILE_BYTES, + max_line_bytes: int = DEFAULT_MAX_LINE_BYTES, + max_records: int = DEFAULT_MAX_RECORDS, +) -> ParsedJSONL: + source = Path(path) + try: + size = source.stat().st_size + except OSError as exc: + raise InputError("input file cannot be read") from exc + if size > max_file_bytes: + raise InputError("input file exceeds the configured byte limit") + + digest = hashlib.sha256() + records: list[tuple[int, dict[str, Any]]] = [] + bytes_seen = 0 + try: + with source.open("rb") as stream: + for line_number, raw_line in enumerate(stream, start=1): + bytes_seen += len(raw_line) + digest.update(raw_line) + if bytes_seen > max_file_bytes: + raise InputError("input file grew beyond the configured byte limit") + if len(raw_line) > max_line_bytes: + raise InputError(f"JSONL line {line_number} exceeds the byte limit") + stripped = raw_line.strip() + if not stripped: + continue + if len(records) >= max_records: + raise InputError("input file exceeds the configured record limit") + decoded = _decode_json(stripped, context=f"JSONL line {line_number}") + if not isinstance(decoded, dict): + raise InputError(f"JSONL line {line_number} must be an object") + records.append((line_number, decoded)) + except InputError: + raise + except OSError as exc: + raise InputError("input file cannot be read") from exc + + return ParsedJSONL( + records=tuple(records), + summary=InputSummary( + sha256=digest.hexdigest(), + byte_count=bytes_seen, + record_count=len(records), + ), + ) + + +def read_json_bounded( + path: str | Path, *, max_file_bytes: int = DEFAULT_MAX_FILE_BYTES +) -> tuple[Any, str, int]: + source = Path(path) + try: + size = source.stat().st_size + if size > max_file_bytes: + raise InputError("JSON file exceeds the configured byte limit") + raw = source.read_bytes() + except InputError: + raise + except OSError as exc: + raise InputError("JSON file cannot be read") from exc + if len(raw) > max_file_bytes: + raise InputError("JSON file grew beyond the configured byte limit") + return _decode_json(raw, context="JSON file"), hashlib.sha256(raw).hexdigest(), len(raw) + + +def summarize_records(records: Iterable[tuple[int, dict[str, Any]]]) -> InputSummary: + """Create a deterministic input summary for generated in-memory records.""" + + from .artifacts import canonical_json_bytes + + materialized = [record for _, record in records] + try: + raw = canonical_json_bytes(materialized) + except (TypeError, ValueError, RecursionError): + raw = canonical_json_bytes({"invalid_input": True, "record_count": len(materialized)}) + return InputSummary( + sha256=hashlib.sha256(raw).hexdigest(), + byte_count=len(raw), + record_count=len(materialized), + ) diff --git a/src/sftguard/models.py b/src/sftguard/models.py new file mode 100644 index 0000000..f31699b --- /dev/null +++ b/src/sftguard/models.py @@ -0,0 +1,59 @@ +"""Shared status and finding types.""" + +from __future__ import annotations + +from dataclasses import dataclass +from dataclasses import field as dataclass_field +from enum import StrEnum +from typing import Any + + +class Status(StrEnum): + PASS = "PASS" + FAIL = "FAIL" + ABSTAIN = "ABSTAIN" + + +SEVERITY_ORDER = {"info": 0, "warning": 1, "error": 2, "critical": 3} + + +@dataclass(frozen=True) +class Finding: + signal: str + severity: str + split: str + line: int + field: str | None = None + category: str | None = None + related_line: int | None = None + record_sha256: str | None = None + redacted: bool = False + details: dict[str, Any] = dataclass_field(default_factory=dict) + + def as_dict(self) -> dict[str, Any]: + result: dict[str, Any] = { + "signal": self.signal, + "severity": self.severity, + "location": {"split": self.split, "line": self.line}, + } + if self.field is not None: + result["location"]["field"] = self.field + if self.category is not None: + result["category"] = self.category + if self.related_line is not None: + result["related_line"] = self.related_line + if self.record_sha256 is not None: + result["record_sha256"] = self.record_sha256 + if self.redacted: + result["redacted"] = True + if self.details: + result["details"] = dict(sorted(self.details.items())) + return result + + +def combine_status(*statuses: Status) -> Status: + if Status.FAIL in statuses: + return Status.FAIL + if Status.ABSTAIN in statuses: + return Status.ABSTAIN + return Status.PASS diff --git a/src/sftguard/privacy.py b/src/sftguard/privacy.py new file mode 100644 index 0000000..a8bb9c7 --- /dev/null +++ b/src/sftguard/privacy.py @@ -0,0 +1,63 @@ +"""Conservative secret and PII-like pattern location without value retention.""" + +from __future__ import annotations + +import re +from collections.abc import Iterator +from typing import Any + +_SECRET_PATTERNS = ( + ("openai_style_key", re.compile(r"\bsk-(?:test-)?[A-Za-z0-9_-]{12,}\b")), + ("aws_access_key", re.compile(r"\bAKIA[0-9A-Z]{16}\b")), + ( + "assigned_credential", + re.compile( + r"(?i)\b(?:api[_-]?key|access[_-]?token|auth[_-]?token|password|secret)" + r"\s*[:=]\s*[\"']?[A-Za-z0-9_./+=-]{12,}" + ), + ), + ("bearer_token", re.compile(r"(?i)\bbearer\s+[A-Za-z0-9._~+/=-]{12,}")), +) + +_PII_PATTERNS = ( + ( + "email_address", + re.compile(r"(?i)(? str: + for component in reversed(path): + if component in {"content", "arguments", "name", "role"}: + return str(component) + return "other" + + +def iter_strings(value: Any, path: tuple[str | int, ...] = ()) -> Iterator[tuple[str, str]]: + if isinstance(value, str): + yield _field_label(path), value + elif isinstance(value, list): + for index, item in enumerate(value): + yield from iter_strings(item, path + (index,)) + elif isinstance(value, dict): + for key in sorted(value): + yield from iter_strings(value[key], path + (key,)) + + +def scan_record(value: dict[str, Any]) -> tuple[tuple[str, str, str], ...]: + """Return unique ``(signal, category, safe-field)`` tuples, never matches.""" + + findings: set[tuple[str, str, str]] = set() + for field, text in iter_strings(value): + for category, pattern in _SECRET_PATTERNS: + if pattern.search(text): + findings.add(("secret_pattern", category, field)) + for category, pattern in _PII_PATTERNS: + if pattern.search(text): + findings.add(("pii_like", category, field)) + return tuple(sorted(findings)) diff --git a/src/sftguard/regression.py b/src/sftguard/regression.py new file mode 100644 index 0000000..032f419 --- /dev/null +++ b/src/sftguard/regression.py @@ -0,0 +1,387 @@ +"""Paired target/retention Regression Cards with deterministic bootstrap CIs.""" + +from __future__ import annotations + +import math +import re +import statistics +from collections.abc import Iterable, Mapping +from dataclasses import dataclass +from typing import Any + +from . import __version__ +from .artifacts import canonical_json_bytes, seal_artifact, sha256_bytes +from .io import InputSummary, summarize_records +from .models import Status + +_IDENTIFIER = re.compile(r"^[A-Za-z0-9_.-]{1,64}$") +MAX_BOOTSTRAP_WORK = 10_000_000 + + +@dataclass(frozen=True) +class BootstrapConfig: + draws: int = 10_000 + confidence: float = 0.95 + seed: int = 2339 + + +class _XorShift32: + def __init__(self, seed: int): + self.state = seed & 0xFFFFFFFF or 0x6D2B79F5 + + def next_u32(self) -> int: + value = self.state + value ^= (value << 13) & 0xFFFFFFFF + value ^= value >> 17 + value ^= (value << 5) & 0xFFFFFFFF + self.state = value & 0xFFFFFFFF + return self.state + + def randbelow(self, stop: int) -> int: + if stop <= 0: + raise ValueError("stop must be positive") + return self.next_u32() % stop + + +def _round(value: float) -> float: + return round(value, 12) + + +def _percentile(sorted_values: list[float], probability: float) -> float: + if not sorted_values: + raise ValueError("cannot calculate a percentile of no values") + position = (len(sorted_values) - 1) * probability + lower = math.floor(position) + upper = math.ceil(position) + if lower == upper: + return sorted_values[lower] + fraction = position - lower + return sorted_values[lower] * (1.0 - fraction) + sorted_values[upper] * fraction + + +def seed_cluster_bootstrap( + values_by_seed: Mapping[int, tuple[float, ...]], config: BootstrapConfig +) -> tuple[float, float]: + """Cluster-resample seeds, retaining all pairs inside each selected seed.""" + + seeds = sorted(values_by_seed) + if not seeds: + raise ValueError("at least one seed cluster is required") + if config.draws <= 0 or config.draws > 100_000: + raise ValueError("bootstrap draws must be between 1 and 100000") + if not 0.0 < config.confidence < 1.0: + raise ValueError("bootstrap confidence must be between 0 and 1") + + generator = _XorShift32(config.seed) + estimates: list[float] = [] + for _ in range(config.draws): + sampled_values: list[float] = [] + for _ in seeds: + selected_seed = seeds[generator.randbelow(len(seeds))] + sampled_values.extend(values_by_seed[selected_seed]) + estimates.append(statistics.fmean(sampled_values)) + estimates.sort() + tail = (1.0 - config.confidence) / 2.0 + return _round(_percentile(estimates, tail)), _round(_percentile(estimates, 1.0 - tail)) + + +@dataclass(frozen=True) +class _Evaluation: + seed: int + sample_id: str + group: str + metric: str + value: float + + @property + def key(self) -> tuple[int, str, str, str]: + return self.seed, self.sample_id, self.group, self.metric + + +def _parse_evaluations( + records: Iterable[tuple[int, dict[str, Any]]], +) -> tuple[dict[tuple[int, str, str, str], _Evaluation], set[str]]: + parsed: dict[tuple[int, str, str, str], _Evaluation] = {} + errors: set[str] = set() + for _, record in records: + seed = record.get("seed") + sample_id = record.get("sample_id") + group = record.get("group") + metric = record.get("metric") + value = record.get("value") + valid = ( + isinstance(seed, int) + and not isinstance(seed, bool) + and isinstance(sample_id, str) + and 0 < len(sample_id) <= 512 + and isinstance(group, str) + and _IDENTIFIER.fullmatch(group) is not None + and isinstance(metric, str) + and _IDENTIFIER.fullmatch(metric) is not None + and isinstance(value, (int, float)) + and not isinstance(value, bool) + and math.isfinite(float(value)) + ) + if not valid: + errors.add("invalid_evaluation_record") + continue + evaluation = _Evaluation(seed, sample_id, group, metric, float(value)) + if evaluation.key in parsed: + errors.add("duplicate_pair_key") + else: + parsed[evaluation.key] = evaluation + return parsed, errors + + +def _parse_contract(contract: Any) -> tuple[list[dict[str, Any]], BootstrapConfig, set[str]]: + errors: set[str] = set() + if not isinstance(contract, dict): + return [], BootstrapConfig(), {"invalid_contract"} + raw_metrics = contract.get("metrics") + raw_bootstrap = contract.get("bootstrap", {}) + if not isinstance(raw_metrics, list) or not raw_metrics: + errors.add("invalid_contract_metrics") + raw_metrics = [] + if not isinstance(raw_bootstrap, dict): + errors.add("invalid_bootstrap_config") + raw_bootstrap = {} + + draws = raw_bootstrap.get("draws", 10_000) + confidence = raw_bootstrap.get("confidence", 0.95) + seed = raw_bootstrap.get("seed", 2339) + if ( + not isinstance(draws, int) + or isinstance(draws, bool) + or not 1 <= draws <= 100_000 + or not isinstance(confidence, (int, float)) + or isinstance(confidence, bool) + or not 0.0 < float(confidence) < 1.0 + or not isinstance(seed, int) + or isinstance(seed, bool) + ): + errors.add("invalid_bootstrap_config") + bootstrap = BootstrapConfig() + else: + bootstrap = BootstrapConfig(draws, float(confidence), seed) + + parsed: list[dict[str, Any]] = [] + seen: set[tuple[str, str]] = set() + groups: set[str] = set() + for raw in raw_metrics: + if not isinstance(raw, dict): + errors.add("invalid_metric_contract") + continue + group = raw.get("group") + metric = raw.get("metric") + higher = raw.get("higher_is_better") + if ( + group not in {"target", "retention"} + or not isinstance(metric, str) + or _IDENTIFIER.fullmatch(metric) is None + or not isinstance(higher, bool) + ): + errors.add("invalid_metric_contract") + continue + key = (group, metric) + if key in seen: + errors.add("duplicate_metric_contract") + continue + seen.add(key) + groups.add(group) + + if group == "target": + threshold = raw.get("min_improvement", 0.0) + threshold_name = "min_improvement" + if not isinstance(threshold, (int, float)) or isinstance(threshold, bool): + errors.add("invalid_metric_threshold") + continue + else: + threshold = raw.get("max_regression") + threshold_name = "max_regression" + if ( + not isinstance(threshold, (int, float)) + or isinstance(threshold, bool) + or float(threshold) < 0.0 + ): + errors.add("invalid_metric_threshold") + continue + if not math.isfinite(float(threshold)): + errors.add("invalid_metric_threshold") + continue + parsed.append( + { + "group": group, + "metric": metric, + "higher_is_better": higher, + threshold_name: float(threshold), + } + ) + + if groups != {"target", "retention"}: + errors.add("target_and_retention_contracts_required") + return parsed, bootstrap, errors + + +def _abstain_card( + reasons: set[str], + *, + base_summary: InputSummary, + adapter_summary: InputSummary, + contract_hash: str, +) -> dict[str, Any]: + return seal_artifact( + { + "schema_version": "sftguard.regression-card.v1", + "tool": {"name": "sftguard", "version": __version__}, + "status": Status.ABSTAIN, + "inputs": [ + base_summary.as_dict("base"), + adapter_summary.as_dict("adapter"), + {"split": "contract", "sha256": contract_hash}, + ], + "reason_codes": sorted(reasons), + "metrics": [], + "summary": {"metric_count": 0, "pair_count": 0, "seed_cluster_count": 0}, + } + ) + + +def _safe_summary(records: tuple[tuple[int, dict[str, Any]], ...]) -> InputSummary: + try: + return summarize_records(records) + except (TypeError, ValueError, RecursionError): + marker = canonical_json_bytes({"invalid_input": True, "record_count": len(records)}) + return InputSummary( + sha256=sha256_bytes(marker), + byte_count=len(marker), + record_count=len(records), + ) + + +def _safe_contract_hash(contract: Any) -> str: + try: + material = canonical_json_bytes(contract) + except (TypeError, ValueError, RecursionError): + material = canonical_json_bytes({"invalid_contract": True}) + return sha256_bytes(material) + + +def build_regression_card( + base_records: Iterable[tuple[int, dict[str, Any]]], + adapter_records: Iterable[tuple[int, dict[str, Any]]], + contract: Any, + *, + base_summary: InputSummary | None = None, + adapter_summary: InputSummary | None = None, +) -> dict[str, Any]: + base_materialized = tuple(base_records) + adapter_materialized = tuple(adapter_records) + base_summary = base_summary or _safe_summary(base_materialized) + adapter_summary = adapter_summary or _safe_summary(adapter_materialized) + contract_hash = _safe_contract_hash(contract) + + base, base_errors = _parse_evaluations(base_materialized) + adapter, adapter_errors = _parse_evaluations(adapter_materialized) + metric_contracts, bootstrap, contract_errors = _parse_contract(contract) + errors = base_errors | adapter_errors | contract_errors + if set(base) != set(adapter): + errors.add("unpaired_evaluation_records") + if not base: + errors.add("paired_evaluation_records_required") + + declared = {(item["group"], item["metric"]) for item in metric_contracts} + observed = {(item.group, item.metric) for item in base.values()} + if observed - declared: + errors.add("undeclared_metric_records") + if declared - observed: + errors.add("contract_metric_without_pairs") + if bootstrap.draws * len(base) > MAX_BOOTSTRAP_WORK: + errors.add("bootstrap_work_limit_exceeded") + if errors: + return _abstain_card( + errors, + base_summary=base_summary, + adapter_summary=adapter_summary, + contract_hash=contract_hash, + ) + + metric_results: list[dict[str, Any]] = [] + all_seeds: set[int] = set() + for metric_contract in sorted( + metric_contracts, key=lambda item: (item["group"], item["metric"]) + ): + group = metric_contract["group"] + metric = metric_contract["metric"] + sign = 1.0 if metric_contract["higher_is_better"] else -1.0 + values_by_seed: dict[int, list[float]] = {} + for key, base_value in base.items(): + if base_value.group != group or base_value.metric != metric: + continue + delta = sign * (adapter[key].value - base_value.value) + values_by_seed.setdefault(base_value.seed, []).append(delta) + all_seeds.add(base_value.seed) + frozen_clusters = {seed: tuple(values) for seed, values in sorted(values_by_seed.items())} + all_values = [value for values in frozen_clusters.values() for value in values] + estimate = _round(statistics.fmean(all_values)) + ci_low, ci_high = seed_cluster_bootstrap(frozen_clusters, bootstrap) + + if group == "target": + threshold_name = "min_improvement" + threshold = metric_contract[threshold_name] + passed = ci_low >= threshold + decision = "lower_confidence_bound_at_or_above_minimum" + else: + threshold_name = "max_regression" + threshold = metric_contract[threshold_name] + passed = ci_low >= -threshold + decision = "lower_confidence_bound_within_regression_budget" + metric_results.append( + { + "group": group, + "metric": metric, + "higher_is_better": metric_contract["higher_is_better"], + "status": Status.PASS if passed else Status.FAIL, + "pair_count": len(all_values), + "seed_cluster_count": len(frozen_clusters), + "mean_normalized_improvement": estimate, + "confidence_interval": {"low": ci_low, "high": ci_high}, + "threshold": {threshold_name: _round(threshold)}, + "decision_rule": decision, + } + ) + + overall = ( + Status.PASS + if all(result["status"] == Status.PASS for result in metric_results) + else Status.FAIL + ) + return seal_artifact( + { + "schema_version": "sftguard.regression-card.v1", + "tool": {"name": "sftguard", "version": __version__}, + "status": overall, + "inputs": [ + base_summary.as_dict("base"), + adapter_summary.as_dict("adapter"), + {"split": "contract", "sha256": contract_hash}, + ], + "bootstrap": { + "method": "paired_seed_cluster_xorshift32", + "draws": bootstrap.draws, + "confidence": bootstrap.confidence, + "seed": bootstrap.seed, + }, + "metrics": metric_results, + "summary": { + "metric_count": len(metric_results), + "pair_count": len(base), + "seed_cluster_count": len(all_seeds), + "passed_metric_count": sum( + result["status"] == Status.PASS for result in metric_results + ), + "failed_metric_count": sum( + result["status"] == Status.FAIL for result in metric_results + ), + }, + } + ) diff --git a/src/sftguard/synthetic.py b/src/sftguard/synthetic.py new file mode 100644 index 0000000..8f31207 --- /dev/null +++ b/src/sftguard/synthetic.py @@ -0,0 +1,156 @@ +"""Deterministic synthetic fault generation with phase-separated seeds.""" + +from __future__ import annotations + +import copy +import hashlib +from dataclasses import dataclass +from typing import Any + +from .artifacts import canonical_json_bytes, sha256_bytes + +DEVELOPMENT_SEEDS = tuple(range(10)) +CONFIRMATORY_SEEDS = tuple(range(4100, 4130)) + + +@dataclass(frozen=True) +class FaultSpec: + fault_id: str + expected_signal: str + severity: str + + +FAULT_SPECS = ( + FaultSpec("schema.invalid_role", "invalid_role", "error"), + FaultSpec("schema.role_order", "role_order", "error"), + FaultSpec("template.missing_eos", "missing_eos", "error"), + FaultSpec("mask.prompt_supervised", "prompt_token_supervised", "critical"), + FaultSpec("truncation.answer_removed", "assistant_fully_truncated", "critical"), + FaultSpec("data.exact_duplicate", "exact_duplicate", "warning"), + FaultSpec("data.prompt_conflict", "prompt_conflict", "error"), + FaultSpec("data.train_eval_leakage", "train_eval_leakage", "critical"), + FaultSpec("privacy.secret_pattern", "secret_pattern", "critical"), +) + + +@dataclass(frozen=True) +class SyntheticCase: + seed: int + fault: FaultSpec | None + train: tuple[tuple[int, dict[str, Any]], ...] + evaluation: tuple[tuple[int, dict[str, Any]], ...] + case_sha256: str + private_markers: tuple[str, ...] + + +def _marker(seed: int, split: str, index: int) -> str: + hexadecimal = hashlib.sha256(f"sftguard:{seed}:{split}:{index}".encode()).hexdigest()[:16] + return hexadecimal.translate(str.maketrans("0123456789abcdef", "abcdefghijklmnop")) + + +def _record(marker: str, answer: str, offset: int) -> dict[str, Any]: + return { + "messages": [ + {"role": "system", "content": "Return the requested class label."}, + {"role": "user", "content": f"Classify marker_{marker}."}, + {"role": "assistant", "content": answer}, + ], + "_sftguard": { + "token_ids": [10 + offset, 20 + offset, 30 + offset, 2], + "token_roles": ["system", "user", "assistant", "assistant"], + "loss_mask": [False, False, True, True], + "eos_token_id": 2, + "original_assistant_tokens": 2, + "truncated": False, + }, + } + + +def _clean(seed: int) -> tuple[list[dict[str, Any]], list[dict[str, Any]], list[str]]: + train_markers = [_marker(seed, "train", index) for index in range(3)] + evaluation_markers = [_marker(seed, "evaluation", index) for index in range(2)] + train = [ + _record(marker, "class_a" if index % 2 == 0 else "class_b", index) + for index, marker in enumerate(train_markers) + ] + evaluation = [ + _record(marker, "class_b" if index % 2 == 0 else "class_a", index + 10) + for index, marker in enumerate(evaluation_markers) + ] + return train, evaluation, train_markers + evaluation_markers + + +def _generate_case(seed: int, fault_id: str | None = None) -> SyntheticCase: + spec = next((item for item in FAULT_SPECS if item.fault_id == fault_id), None) + if fault_id is not None and spec is None: + raise ValueError("unknown synthetic fault identifier") + + train, evaluation, markers = _clean(seed) + private_markers = list(markers) + if spec is not None: + if spec.fault_id == "schema.invalid_role": + train[0]["messages"][1]["role"] = "invalid_development_role" + elif spec.fault_id == "schema.role_order": + train[0]["messages"].insert(0, {"role": "assistant", "content": "class_a"}) + elif spec.fault_id == "template.missing_eos": + train[0]["_sftguard"]["token_ids"][-1] = 99 + elif spec.fault_id == "mask.prompt_supervised": + train[0]["_sftguard"]["loss_mask"][1] = True + elif spec.fault_id == "truncation.answer_removed": + train[0]["_sftguard"] = { + "token_ids": [10, 20], + "token_roles": ["system", "user"], + "loss_mask": [False, False], + "eos_token_id": 2, + "original_assistant_tokens": 2, + "truncated": True, + } + elif spec.fault_id == "data.exact_duplicate": + train.append(copy.deepcopy(train[0])) + elif spec.fault_id == "data.prompt_conflict": + conflicting = copy.deepcopy(train[0]) + conflicting["messages"][-1]["content"] = "class_conflict" + train.append(conflicting) + elif spec.fault_id == "data.train_eval_leakage": + evaluation.append(copy.deepcopy(train[0])) + elif spec.fault_id == "privacy.secret_pattern": + secret = f"sk-test-{hashlib.sha256(f'secret:{seed}'.encode()).hexdigest()[:20]}" + train[0]["messages"][1]["content"] += f" Credential {secret}." + private_markers.append(secret) + + numbered_train = tuple((index + 1, record) for index, record in enumerate(train)) + numbered_evaluation = tuple((index + 1, record) for index, record in enumerate(evaluation)) + case_hash = sha256_bytes( + canonical_json_bytes( + { + "seed": seed, + "fault_id": fault_id, + "train": train, + "evaluation": evaluation, + } + ) + ) + return SyntheticCase( + seed=seed, + fault=spec, + train=numbered_train, + evaluation=numbered_evaluation, + case_sha256=case_hash, + private_markers=tuple(private_markers), + ) + + +def generate_case(seed: int, fault_id: str | None = None) -> SyntheticCase: + """Generate one development case without exposing confirmatory seeds.""" + + if seed not in DEVELOPMENT_SEEDS: + raise ValueError("synthetic generation is restricted to development seeds 0 through 9") + return _generate_case(seed, fault_id) + + +def generate_confirmatory_case(seed: int, fault_id: str | None = None) -> SyntheticCase: + """Generate a preregistered confirmatory case for the evidence runner only.""" + + if seed not in CONFIRMATORY_SEEDS: + raise ValueError("seed is outside the frozen confirmatory range") + return _generate_case(seed, fault_id) diff --git a/src/sftguard/tokenizer.py b/src/sftguard/tokenizer.py new file mode 100644 index 0000000..c1a2399 --- /dev/null +++ b/src/sftguard/tokenizer.py @@ -0,0 +1,94 @@ +"""Tokenizer-independent assistant-mask inspection contracts and fixtures.""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any, Protocol + +FIXTURE_FIELD = "_sftguard" + + +@dataclass(frozen=True) +class InspectionSignal: + signal: str + severity: str + + +@dataclass(frozen=True) +class MaskInspection: + available: bool + signals: tuple[InspectionSignal, ...] = () + reason: str | None = None + + +class MaskInspector(Protocol): + def inspect(self, record: dict[str, Any]) -> MaskInspection: + """Inspect rendered tokens without returning their text.""" + + +class FixtureMaskInspector: + """Inspect a deterministic token/mask fixture embedded in a JSONL record. + + Real tokenizer integrations can implement :class:`MaskInspector` and supply + the same semantic output. The fixture deliberately contains token IDs and + role labels only; audit artifacts never copy either. + """ + + _ROLES = {"system", "user", "assistant", "tool"} + + def inspect(self, record: dict[str, Any]) -> MaskInspection: + raw = record.get(FIXTURE_FIELD) + if raw is None: + return MaskInspection(available=False, reason="assistant_mask_unavailable") + if not isinstance(raw, dict): + return MaskInspection(available=False, reason="mask_fixture_invalid") + + token_ids = raw.get("token_ids") + token_roles = raw.get("token_roles") + loss_mask = raw.get("loss_mask") + eos_token_id = raw.get("eos_token_id") + original_assistant_tokens = raw.get("original_assistant_tokens") + truncated = raw.get("truncated", False) + + valid = ( + isinstance(token_ids, list) + and isinstance(token_roles, list) + and isinstance(loss_mask, list) + and len(token_ids) == len(token_roles) == len(loss_mask) + and all(isinstance(token, int) and not isinstance(token, bool) for token in token_ids) + and all(isinstance(role, str) and role in self._ROLES for role in token_roles) + and all(isinstance(masked, bool) for masked in loss_mask) + and isinstance(eos_token_id, int) + and not isinstance(eos_token_id, bool) + and isinstance(original_assistant_tokens, int) + and not isinstance(original_assistant_tokens, bool) + and original_assistant_tokens >= 0 + and isinstance(truncated, bool) + ) + if not valid: + return MaskInspection(available=False, reason="mask_fixture_invalid") + + signals: set[InspectionSignal] = set() + if any( + masked and role != "assistant" + for masked, role in zip(loss_mask, token_roles, strict=True) + ): + signals.add(InspectionSignal("prompt_token_supervised", "critical")) + + supervised_assistant = [ + index + for index, (masked, role) in enumerate(zip(loss_mask, token_roles, strict=True)) + if masked and role == "assistant" + ] + assistant_positions = [ + index for index, role in enumerate(token_roles) if role == "assistant" + ] + if truncated and original_assistant_tokens > 0 and not supervised_assistant: + signals.add(InspectionSignal("assistant_fully_truncated", "critical")) + elif assistant_positions and token_ids[assistant_positions[-1]] != eos_token_id: + signals.add(InspectionSignal("missing_eos", "error")) + + return MaskInspection( + available=True, + signals=tuple(sorted(signals, key=lambda item: (item.signal, item.severity))), + ) diff --git a/tests/python/test_artifacts_io.py b/tests/python/test_artifacts_io.py new file mode 100644 index 0000000..b8dbb03 --- /dev/null +++ b/tests/python/test_artifacts_io.py @@ -0,0 +1,70 @@ +from __future__ import annotations + +import json + +import pytest + +from sftguard.artifacts import ( + canonical_json_bytes, + seal_artifact, + verify_artifact, + write_artifact, +) +from sftguard.io import InputError, read_json_bounded, read_jsonl_bounded + + +def test_canonical_json_is_sorted_compact_and_utf8() -> None: + assert canonical_json_bytes({"z": 1, "a": "Sarawak"}) == b'{"a":"Sarawak","z":1}' + + +def test_sealed_artifact_verifies_and_tampering_fails() -> None: + artifact = seal_artifact({"status": "PASS", "count": 2}) + assert verify_artifact(artifact) + artifact["count"] = 3 + assert not verify_artifact(artifact) + + +def test_write_artifact_round_trips(tmp_path) -> None: + artifact = seal_artifact({"status": "PASS"}) + destination = tmp_path / "nested" / "report.json" + write_artifact(destination, artifact) + assert json.loads(destination.read_text(encoding="utf-8")) == artifact + + +def test_jsonl_reader_streams_objects_and_hashes_bytes(tmp_path) -> None: + source = tmp_path / "records.jsonl" + source.write_bytes(b'{"a":1}\n\n{"b":2}\n') + parsed = read_jsonl_bounded(source) + assert parsed.records == ((1, {"a": 1}), (3, {"b": 2})) + assert parsed.summary.byte_count == source.stat().st_size + assert parsed.summary.record_count == 2 + assert len(parsed.summary.sha256) == 64 + + +@pytest.mark.parametrize( + ("contents", "kwargs"), + [ + (b'{"a":1}\n', {"max_file_bytes": 2}), + (b'{"long":"value"}\n', {"max_line_bytes": 4}), + (b'{"a":1}\n{"b":2}\n', {"max_records": 1}), + (b'{"a":1,"a":2}\n', {}), + (b"[1,2,3]\n", {}), + (b'{"score":NaN}\n', {}), + (b"\xff\xfe\n", {}), + ], +) +def test_jsonl_reader_rejects_malformed_or_bounded_input(tmp_path, contents, kwargs) -> None: + source = tmp_path / "bad.jsonl" + source.write_bytes(contents) + with pytest.raises(InputError): + read_jsonl_bounded(source, **kwargs) + + +def test_bounded_json_rejects_duplicate_keys_and_large_file(tmp_path) -> None: + source = tmp_path / "contract.json" + source.write_text('{"x":1,"x":2}', encoding="utf-8") + with pytest.raises(InputError): + read_json_bounded(source) + source.write_text('{"x":1}', encoding="utf-8") + with pytest.raises(InputError): + read_json_bounded(source, max_file_bytes=2) diff --git a/tests/python/test_audit.py b/tests/python/test_audit.py new file mode 100644 index 0000000..3742798 --- /dev/null +++ b/tests/python/test_audit.py @@ -0,0 +1,121 @@ +from __future__ import annotations + +import copy +import json + +import pytest + +from sftguard.audit import audit_files, audit_records +from sftguard.models import Status +from sftguard.synthetic import FAULT_SPECS, generate_case +from sftguard.tokenizer import FixtureMaskInspector + + +def test_clean_control_passes_without_findings() -> None: + case = generate_case(0) + report = audit_records(case.train, case.evaluation) + assert report["status"] == Status.PASS + assert report["findings"] == [] + assert report["summary"]["signal_counts"] == {} + + +@pytest.mark.parametrize("fault", FAULT_SPECS, ids=lambda fault: fault.fault_id) +def test_each_registered_fault_emits_its_primary_signal(fault) -> None: + case = generate_case(1, fault.fault_id) + report = audit_records(case.train, case.evaluation) + assert fault.expected_signal in report["summary"]["signal_counts"] + + +def test_warning_duplicate_is_detected_without_forcing_failure() -> None: + case = generate_case(2, "data.exact_duplicate") + report = audit_records(case.train, case.evaluation) + assert report["status"] == Status.PASS + assert report["summary"]["signal_counts"] == {"exact_duplicate": 1} + + +def test_missing_mask_abstains_and_is_not_silently_passed() -> None: + case = generate_case(2) + train = copy.deepcopy(case.train) + train[0][1].pop("_sftguard") + report = audit_records(train, case.evaluation) + assert report["status"] == Status.ABSTAIN + assert report["checks"]["assistant_mask"]["status"] == Status.ABSTAIN + assert report["summary"]["signal_counts"]["assistant_mask_unavailable"] == 1 + + +def test_known_failure_dominates_mask_abstention() -> None: + case = generate_case(3, "schema.invalid_role") + train = copy.deepcopy(case.train) + train[0][1].pop("_sftguard") + report = audit_records(train, case.evaluation) + assert report["status"] == Status.FAIL + + +def test_invalid_mask_fixture_abstains() -> None: + case = generate_case(4) + train = copy.deepcopy(case.train) + train[0][1]["_sftguard"]["loss_mask"] = [True] + report = audit_records(train, case.evaluation) + assert report["status"] == Status.ABSTAIN + assert "mask_fixture_invalid" in report["summary"]["signal_counts"] + + +def test_direct_nonfinite_record_fails_closed() -> None: + case = generate_case(4) + train = copy.deepcopy(case.train) + train[0][1]["score"] = float("nan") + report = audit_records(train, case.evaluation) + assert report["status"] == Status.FAIL + assert report["summary"]["signal_counts"]["invalid_record"] == 1 + + +def test_secret_and_pii_findings_are_redacted_and_values_absent() -> None: + case = generate_case(5) + secret = "sk-test-abcdefghijklmnopqrst" + email = "person@example.invalid" + train = copy.deepcopy(case.train) + train[0][1]["messages"][1]["content"] += f" {secret} {email}" + report = audit_records(train, case.evaluation) + rendered = json.dumps(report, sort_keys=True) + assert secret not in rendered + assert email not in rendered + findings = [ + finding + for finding in report["findings"] + if finding["signal"] in {"secret_pattern", "pii_like"} + ] + assert {finding["signal"] for finding in findings} == {"secret_pattern", "pii_like"} + assert all(finding["redacted"] is True for finding in findings) + assert all("category" in finding and "location" in finding for finding in findings) + + +def test_artifact_never_contains_prompt_or_response_text() -> None: + case = generate_case(6) + report = audit_records(case.train, case.evaluation) + rendered = json.dumps(report, sort_keys=True) + for _, record in (*case.train, *case.evaluation): + for message in record["messages"]: + assert message["content"] not in rendered + + +def test_mask_inspector_flags_each_semantic_fault() -> None: + inspector = FixtureMaskInspector() + for fault_id, signal in ( + ("template.missing_eos", "missing_eos"), + ("mask.prompt_supervised", "prompt_token_supervised"), + ("truncation.answer_removed", "assistant_fully_truncated"), + ): + case = generate_case(7, fault_id) + inspection = inspector.inspect(case.train[0][1]) + assert inspection.available + assert signal in {item.signal for item in inspection.signals} + + +def test_audit_files_returns_abstention_for_bad_input(tmp_path) -> None: + train = tmp_path / "train.jsonl" + evaluation = tmp_path / "eval.jsonl" + train.write_text("not-json\n", encoding="utf-8") + evaluation.write_text("{}\n", encoding="utf-8") + report = audit_files(str(train), str(evaluation)) + assert report["status"] == Status.ABSTAIN + assert report["inputs"] == [] diff --git a/tests/python/test_benchmark.py b/tests/python/test_benchmark.py new file mode 100644 index 0000000..755522a --- /dev/null +++ b/tests/python/test_benchmark.py @@ -0,0 +1,53 @@ +from __future__ import annotations + +import json + +import pytest + +from sftguard.artifacts import canonical_json_bytes, verify_artifact +from sftguard.benchmark import run_development_benchmark, validate_development_seeds +from sftguard.models import Status +from sftguard.synthetic import DEVELOPMENT_SEEDS, FAULT_SPECS, generate_case + + +def test_generator_accepts_every_development_seed() -> None: + for seed in DEVELOPMENT_SEEDS: + assert generate_case(seed).seed == seed + + +@pytest.mark.parametrize("seed", [-1, 10, 99]) +def test_generator_rejects_non_development_seeds(seed) -> None: + with pytest.raises(ValueError, match="development seeds"): + generate_case(seed) + + +def test_seed_validator_deduplicates_and_sorts() -> None: + assert validate_development_seeds([3, 1, 3, 2]) == (1, 2, 3) + + +def test_benchmark_passes_all_registered_faults_on_development_matrix() -> None: + result = run_development_benchmark() + assert result["status"] == Status.PASS + assert result["development_only"] is True + assert result["summary"]["case_count"] == len(DEVELOPMENT_SEEDS) * (len(FAULT_SPECS) + 1) + assert result["summary"]["macro_recall"] == 1.0 + assert result["summary"]["clean_false_positive_rate"] == 0.0 + assert all(result["gates"].values()) + assert verify_artifact(result) + + +def test_benchmark_is_byte_deterministic() -> None: + first = run_development_benchmark([0, 1, 2]) + second = run_development_benchmark([0, 1, 2]) + assert canonical_json_bytes(first) == canonical_json_bytes(second) + + +def test_benchmark_artifact_excludes_synthetic_record_text_and_secret() -> None: + case = generate_case(0, "privacy.secret_pattern") + result = run_development_benchmark([0]) + rendered = json.dumps(result, sort_keys=True) + for marker in case.private_markers: + assert marker not in rendered + for _, record in (*case.train, *case.evaluation): + for message in record["messages"]: + assert message["content"] not in rendered diff --git a/tests/python/test_cli.py b/tests/python/test_cli.py new file mode 100644 index 0000000..f55fb74 --- /dev/null +++ b/tests/python/test_cli.py @@ -0,0 +1,119 @@ +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from sftguard.cli import build_parser, main + +REPOSITORY = Path(__file__).resolve().parents[2] + + +def _read(path: Path): + return json.loads(path.read_text(encoding="utf-8")) + + +def test_audit_cli_pass_exit_and_artifact(tmp_path) -> None: + output = tmp_path / "audit.json" + code = main( + [ + "audit", + "--train", + str(REPOSITORY / "examples" / "clean-train.jsonl"), + "--eval", + str(REPOSITORY / "examples" / "clean-eval.jsonl"), + "--output", + str(output), + ] + ) + assert code == 0 + assert _read(output)["status"] == "PASS" + + +def test_gate_cli_pass_exit_and_artifact(tmp_path) -> None: + output = tmp_path / "card.json" + code = main( + [ + "gate", + "--base", + str(REPOSITORY / "examples" / "base-eval.jsonl"), + "--adapter", + str(REPOSITORY / "examples" / "adapter-eval.jsonl"), + "--contract", + str(REPOSITORY / "examples" / "regression-contract.json"), + "--output", + str(output), + ] + ) + assert code == 0 + assert _read(output)["status"] == "PASS" + + +def test_demo_and_benchmark_cli_pass(tmp_path) -> None: + demo = tmp_path / "demo.json" + benchmark = tmp_path / "benchmark.json" + assert main(["demo", "--output", str(demo)]) == 0 + assert main(["benchmark", "--seeds", "0-2", "--output", str(benchmark)]) == 0 + assert _read(demo)["status"] == "PASS" + assert _read(benchmark)["development_only"] is True + + +def test_audit_cli_abstains_on_unreadable_input(tmp_path) -> None: + output = tmp_path / "audit.json" + code = main( + [ + "audit", + "--train", + str(tmp_path / "missing.jsonl"), + "--eval", + str(tmp_path / "also-missing.jsonl"), + "--output", + str(output), + ] + ) + assert code == 2 + assert _read(output)["status"] == "ABSTAIN" + + +def test_gate_cli_fail_exit(tmp_path) -> None: + base = REPOSITORY / "examples" / "base-eval.jsonl" + adapter = tmp_path / "adapter.jsonl" + rows = [json.loads(line) for line in base.read_text(encoding="utf-8").splitlines()] + for row in rows: + if row["group"] == "target": + row["value"] -= 0.1 + adapter.write_text( + "".join(json.dumps(row, separators=(",", ":")) + "\n" for row in rows), + encoding="utf-8", + ) + output = tmp_path / "card.json" + code = main( + [ + "gate", + "--base", + str(base), + "--adapter", + str(adapter), + "--contract", + str(REPOSITORY / "examples" / "regression-contract.json"), + "--output", + str(output), + ] + ) + assert code == 1 + assert _read(output)["status"] == "FAIL" + + +def test_cli_rejects_out_of_scope_benchmark_seed() -> None: + parser = build_parser() + with pytest.raises(SystemExit) as exc: + parser.parse_args(["benchmark", "--seeds", "10"]) + assert exc.value.code == 2 + + +def test_cli_rejects_huge_benchmark_range_without_expanding_it() -> None: + parser = build_parser() + with pytest.raises(SystemExit) as exc: + parser.parse_args(["benchmark", "--seeds", "0-1000000000"]) + assert exc.value.code == 2 diff --git a/tests/python/test_confirmatory_contract.py b/tests/python/test_confirmatory_contract.py new file mode 100644 index 0000000..14d80b6 --- /dev/null +++ b/tests/python/test_confirmatory_contract.py @@ -0,0 +1,57 @@ +from __future__ import annotations + +import json + +import pytest + +from sftguard.artifacts import verify_artifact +from sftguard.confirmatory import _build_artifact, pretty_json_bytes +from sftguard.synthetic import ( + CONFIRMATORY_SEEDS, + FAULT_SPECS, + generate_case, + generate_confirmatory_case, +) + +FAKE_COMMIT = "a" * 40 + + +def test_frozen_confirmatory_range_is_exact_without_generating_it() -> None: + assert CONFIRMATORY_SEEDS == tuple(range(4100, 4130)) + assert len(CONFIRMATORY_SEEDS) == 30 + + +def test_phase_separation_rejects_the_other_phase_before_generation() -> None: + with pytest.raises(ValueError, match="development seeds"): + generate_case(CONFIRMATORY_SEEDS[0]) + with pytest.raises(ValueError, match="confirmatory range"): + generate_confirmatory_case(0) + + +def test_confirmatory_payload_logic_is_covered_only_with_development_seed() -> None: + artifact = _build_artifact( + (0,), + generate_case, + protocol_commit=FAKE_COMMIT, + implementation_commit=FAKE_COMMIT, + fresh_process_byte_reproducible=True, + ) + assert artifact["status"] == "PASS" + assert artifact["carry_forward"] is True + assert artifact["summary"]["case_count"] == len(FAULT_SPECS) + 1 + assert artifact["gates"]["raw_synthetic_text_absent"] is True + assert all(artifact["gates"].values()) + assert verify_artifact(artifact) + assert json.loads(pretty_json_bytes(artifact)) == artifact + + +@pytest.mark.parametrize("commit", ["short", "G" * 40, "a" * 39]) +def test_evidence_payload_rejects_noncanonical_commits(commit: str) -> None: + with pytest.raises(ValueError, match="full lowercase Git commit"): + _build_artifact( + (0,), + generate_case, + protocol_commit=commit, + implementation_commit=FAKE_COMMIT, + fresh_process_byte_reproducible=True, + ) diff --git a/tests/python/test_regression.py b/tests/python/test_regression.py new file mode 100644 index 0000000..887acb3 --- /dev/null +++ b/tests/python/test_regression.py @@ -0,0 +1,153 @@ +from __future__ import annotations + +import json + +import pytest + +from sftguard.artifacts import canonical_json_bytes, verify_artifact +from sftguard.models import Status +from sftguard.regression import ( + MAX_BOOTSTRAP_WORK, + BootstrapConfig, + build_regression_card, + seed_cluster_bootstrap, +) + + +def _records( + target_delta: float = 0.12, + retention_delta: float = -0.005, + seed_count: int = 4, +): + base = [] + adapter = [] + line = 1 + for seed in range(seed_count): + for group, metric, baseline, delta in ( + ("target", "task_score", 0.5, target_delta), + ("retention", "general_score", 0.8, retention_delta), + ): + common = { + "seed": seed, + "sample_id": f"private-sample-{group}-{seed}", + "group": group, + "metric": metric, + } + base.append((line, {**common, "value": baseline})) + adapter.append((line, {**common, "value": baseline + delta})) + line += 1 + return tuple(base), tuple(adapter) + + +def _contract(): + return { + "bootstrap": {"draws": 1000, "confidence": 0.95, "seed": 2339}, + "metrics": [ + { + "group": "target", + "metric": "task_score", + "higher_is_better": True, + "min_improvement": 0.05, + }, + { + "group": "retention", + "metric": "general_score", + "higher_is_better": True, + "max_regression": 0.02, + }, + ], + } + + +def test_regression_card_passes_target_and_retention_contract() -> None: + base, adapter = _records() + card = build_regression_card(base, adapter, _contract()) + assert card["status"] == Status.PASS + assert all(metric["status"] == Status.PASS for metric in card["metrics"]) + assert card["summary"]["pair_count"] == 8 + assert verify_artifact(card) + + +@pytest.mark.parametrize( + ("target_delta", "retention_delta", "failed_group"), + [(0.01, -0.005, "target"), (0.12, -0.05, "retention")], +) +def test_regression_card_fails_violated_gate(target_delta, retention_delta, failed_group) -> None: + base, adapter = _records(target_delta, retention_delta) + card = build_regression_card(base, adapter, _contract()) + assert card["status"] == Status.FAIL + assert ( + next(item for item in card["metrics"] if item["group"] == failed_group)["status"] + == Status.FAIL + ) + + +def test_unpaired_records_abstain() -> None: + base, adapter = _records() + card = build_regression_card(base, adapter[:-1], _contract()) + assert card["status"] == Status.ABSTAIN + assert "unpaired_evaluation_records" in card["reason_codes"] + + +def test_duplicate_pair_key_abstains() -> None: + base, adapter = _records() + card = build_regression_card((*base, base[0]), adapter, _contract()) + assert card["status"] == Status.ABSTAIN + assert "duplicate_pair_key" in card["reason_codes"] + + +def test_contract_requires_both_target_and_retention() -> None: + base, adapter = _records() + contract = _contract() + contract["metrics"] = contract["metrics"][:1] + card = build_regression_card(base, adapter, contract) + assert card["status"] == Status.ABSTAIN + assert "target_and_retention_contracts_required" in card["reason_codes"] + + +def test_nonfinite_value_abstains_without_serializing_it() -> None: + base, adapter = _records() + broken = list(adapter) + broken[0] = (broken[0][0], {**broken[0][1], "value": float("nan")}) + card = build_regression_card(base, broken, _contract()) + assert card["status"] == Status.ABSTAIN + assert b"NaN" not in canonical_json_bytes(card) + + +def test_nonfinite_contract_abstains_without_serializing_it() -> None: + base, adapter = _records() + contract = _contract() + contract["metrics"][0]["min_improvement"] = float("nan") + card = build_regression_card(base, adapter, contract) + assert card["status"] == Status.ABSTAIN + assert "invalid_metric_threshold" in card["reason_codes"] + assert b"NaN" not in canonical_json_bytes(card) + + +def test_seed_cluster_bootstrap_is_deterministic() -> None: + clusters = {0: (0.1, 0.2), 1: (0.3,), 2: (0.4, 0.5)} + config = BootstrapConfig(draws=2000, confidence=0.9, seed=2339) + assert seed_cluster_bootstrap(clusters, config) == seed_cluster_bootstrap(clusters, config) + + +def test_constant_cluster_bootstrap_has_exact_interval() -> None: + interval = seed_cluster_bootstrap( + {0: (0.25,), 1: (0.25,), 2: (0.25,)}, BootstrapConfig(draws=100) + ) + assert interval == (0.25, 0.25) + + +def test_regression_card_contains_no_sample_ids() -> None: + base, adapter = _records() + rendered = json.dumps(build_regression_card(base, adapter, _contract()), sort_keys=True) + assert "private-sample" not in rendered + + +def test_bootstrap_joint_work_limit_abstains_before_resampling() -> None: + base, adapter = _records(seed_count=51) + contract = _contract() + contract["bootstrap"]["draws"] = 100_000 + assert contract["bootstrap"]["draws"] * len(base) > MAX_BOOTSTRAP_WORK + card = build_regression_card(base, adapter, contract) + assert card["status"] == Status.ABSTAIN + assert "bootstrap_work_limit_exceeded" in card["reason_codes"] diff --git a/web/README.md b/web/README.md new file mode 100644 index 0000000..f44e85b --- /dev/null +++ b/web/README.md @@ -0,0 +1,42 @@ +# SFTGuard report viewer + +Static, local-first browser UI for SFTGuard aggregate report artifacts. + +```bash +npm ci +npm run dev +``` + +`npm run check` performs the TypeScript, unit-test, and production-build checks. + +## Supported artifacts + +The viewer validates and adapts these checked-in core artifact formats: + +- `sftguard.audit.v1`; +- `sftguard.regression-card.v1`; +- `sftguard.development-benchmark.v1`; and +- `sftguard.confirmatory-fault-matrix.v1`. + +The illustrative UI fixtures use the additional `sftguard.report.v1` render +envelope documented in `src/reportSchema.ts`. The allow-listed render model +contains: + +- a `PASS`, `FAIL`, or `ABSTAIN` decision; +- one audit, regression, or fault-matrix summary; +- aggregate issue categories and counts; +- content-addressed provenance; and +- claim and privacy boundaries. + +Unknown fields are not copied into the render model. Audit record locations, +record hashes and details, evaluation rows, benchmark `cases`, and confirmatory +protocol internals are intentionally discarded by the adapters. Missing +timestamps, commits, runtime details, or replay evidence are shown as +unavailable rather than inferred. Artifact digests are displayed as declared +and explicitly marked unverified because this static viewer does not reproduce +the Python canonical-JSON hash calculation. + +Imports are read into the current browser tab only and are neither uploaded nor +stored. The default entry is adapted from the checked-in v0.1 confirmatory +artifact; the regression, failing fault-matrix, and abstaining audit entries are +clearly labelled illustrative UI fixtures. diff --git a/web/index.html b/web/index.html new file mode 100644 index 0000000..ed33e74 --- /dev/null +++ b/web/index.html @@ -0,0 +1,17 @@ + + + + + + + + SFTGuard — Local report viewer + + +
+ + + diff --git a/web/package-lock.json b/web/package-lock.json new file mode 100644 index 0000000..8bf6d35 --- /dev/null +++ b/web/package-lock.json @@ -0,0 +1,1373 @@ +{ + "name": "sftguard-report-viewer", + "version": "0.1.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "sftguard-report-viewer", + "version": "0.1.0", + "dependencies": { + "react": "19.2.7", + "react-dom": "19.2.7" + }, + "devDependencies": { + "@types/react": "19.2.17", + "@types/react-dom": "19.2.3", + "@vitejs/plugin-react": "6.0.3", + "typescript": "5.9.3", + "vite": "8.1.5", + "vitest": "4.1.10" + } + }, + "node_modules/@emnapi/core": { + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/@emnapi/core/-/core-1.11.1.tgz", + "integrity": "sha512-RSvbQmHzdKzNsLYa/wHrbc3KN4sYLKAdPZxqiM2HATqv/SBk2/ENSHpvXGaLOMcsAyz0poEGqkmmKYG3OWiJEQ==", + "dev": true, + "license": "MIT", + "optional": true, + "peer": true, + "dependencies": { + "@emnapi/wasi-threads": "1.2.2", + "tslib": "^2.4.0" + } + }, + "node_modules/@emnapi/runtime": { + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.11.1.tgz", + "integrity": "sha512-vgj7R3y3Wgx24IQaGPA/R6YFXLHVMOZ0uVEyIQPaWs+rd1AzfEMXlAC22FYwO1XkKR6NPsq7mUandH8oIRdZFw==", + "dev": true, + "license": "MIT", + "optional": true, + "peer": true, + "dependencies": { + "tslib": "^2.4.0" + } + }, + "node_modules/@emnapi/wasi-threads": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/@emnapi/wasi-threads/-/wasi-threads-1.2.2.tgz", + "integrity": "sha512-c95qOXkHdydNKhscBTebqEC1CVAZpyqOfVfBzQ1qgzyl3gfeldUjIggDbIZgDKsHLgnsM+igH7TJ/eAasaVuMA==", + "dev": true, + "license": "MIT", + "optional": true, + "dependencies": { + "tslib": "^2.4.0" + } + }, + "node_modules/@jridgewell/sourcemap-codec": { + "version": "1.5.5", + "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.5.5.tgz", + "integrity": "sha512-cYQ9310grqxueWbl+WuIUIaiUaDcj7WOq5fVhEljNVgRfOUhY9fy2zTvfoqWsnebh8Sl70VScFbICvJnLKB0Og==", + "dev": true, + "license": "MIT" + }, + "node_modules/@napi-rs/wasm-runtime": { + "version": "1.1.6", + "resolved": "https://registry.npmjs.org/@napi-rs/wasm-runtime/-/wasm-runtime-1.1.6.tgz", + "integrity": "sha512-ZLv/JdUfkvOy9eCnnBaGfiO+XimbjebAeO+MRQqD/B+FR1tnRN0tpKSJHRbE8sFfS6aqsXZ67TQjfwfsxULVbg==", + "dev": true, + "license": "MIT", + "optional": true, + "dependencies": { + "@tybys/wasm-util": "^0.10.3" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Brooooooklyn" + }, + "peerDependencies": { + "@emnapi/core": "^1.7.1", + "@emnapi/runtime": "^1.7.1" + } + }, + "node_modules/@oxc-project/types": { + "version": "0.139.0", + "resolved": "https://registry.npmjs.org/@oxc-project/types/-/types-0.139.0.tgz", + "integrity": "sha512-r9gHphtCs+1M7J0pw6Sn/hh/Wpa/iQrOOkrNAlVLF/gHq+/CJmHIWKKUUhdWjcD6CIa8idarspCsASiXCXvFUw==", + "dev": true, + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/Boshen" + } + }, + "node_modules/@rolldown/binding-android-arm64": { + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/@rolldown/binding-android-arm64/-/binding-android-arm64-1.1.5.tgz", + "integrity": "sha512-lZg8fqIv2v7FF237bwMgzGZEJvGL79/s5knJ/i6FmsGF4XXlzccZ4jb+TrFIxtSSxFtIpdsgrPZeMk1I9AFcyQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-darwin-arm64": { + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/@rolldown/binding-darwin-arm64/-/binding-darwin-arm64-1.1.5.tgz", + "integrity": "sha512-51Bnx9pNiMRKSUNtBfySkNJ9vMU9Hh3I1ozDd6gyPPYzaXCfnptUcEZxXGYFn+ul2dtcMUiqGR1Yai2K10uoTw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-darwin-x64": { + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/@rolldown/binding-darwin-x64/-/binding-darwin-x64-1.1.5.tgz", + "integrity": "sha512-Tm+gbfC0aHu1tBA/JvKQh32S0K6YgCHkiAF4/W6xX0K0RmNuc94VeK419dJoE65R5aRxmo+noZQSWrAMF6yb6g==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-freebsd-x64": { + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/@rolldown/binding-freebsd-x64/-/binding-freebsd-x64-1.1.5.tgz", + "integrity": "sha512-JMzDKCCXq93YccG5gz3hvOs1oXRKAf0XYpfOS88e+wZrC8Iugj6j68867vrYZkvpDDpKn/KoKORThmchMpF6TA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-arm-gnueabihf": { + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm-gnueabihf/-/binding-linux-arm-gnueabihf-1.1.5.tgz", + "integrity": "sha512-uML21j2K5TfPGutKxub+M+nLjZIrWjXQ5Grx4lCe/nimTj9B4L63zHpjXLl4y0L3mcm2htEQIb06oCG/szerNw==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-arm64-gnu": { + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm64-gnu/-/binding-linux-arm64-gnu-1.1.5.tgz", + "integrity": "sha512-navSiuTMogvnQoZoM/v+l3ZWo50/NTwSHSzheABx/RCnmUPaKwq9qSo4Br2OYRs21+Fz8uFqITZM3H4opOB0/Q==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-arm64-musl": { + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm64-musl/-/binding-linux-arm64-musl-1.1.5.tgz", + "integrity": "sha512-lAryqH7IteztmCXQXk0etKj4wBQ7Gx5S6LjKhsgp9zb8I5bsuvU/2llH1hDQcjsFeqIsovMVN339/8pUDDBXxA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-ppc64-gnu": { + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-ppc64-gnu/-/binding-linux-ppc64-gnu-1.1.5.tgz", + "integrity": "sha512-fsK/sNBnxzBlL4O1JNrZakVQxPspqpED5dLtNsZS9oOKmtSpdNIzxH2kkol5HYTWJN47sE20ztMJPxfZ89qGOg==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-s390x-gnu": { + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-s390x-gnu/-/binding-linux-s390x-gnu-1.1.5.tgz", + "integrity": "sha512-gLYb4BIadlfTOYT5gO503n8zQjXflgzpD0FcyKh0Mzx3rqCZKnHoJWV9xe1KXUJ5lx2JfcSHr/mhzS0PC/McAA==", + "cpu": [ + "s390x" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-x64-gnu": { + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-x64-gnu/-/binding-linux-x64-gnu-1.1.5.tgz", + "integrity": "sha512-FjcpEKUyJygHgs1o50VYNvkt5+7Le/VEdYt0AkRpkL33MnyQfwr8l5mXwMmfmTbyMPr5vJLC+8/Gd9gXnwU1QQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-x64-musl": { + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-x64-musl/-/binding-linux-x64-musl-1.1.5.tgz", + "integrity": "sha512-Me+PfPI2TMeOQk0gYWfLQZtTktrmzbr8cDboqX83XKc7UrgAi55gF+2dUkWdxd19n55Essp2yeca+O9N5rBxHg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-openharmony-arm64": { + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/@rolldown/binding-openharmony-arm64/-/binding-openharmony-arm64-1.1.5.tgz", + "integrity": "sha512-yc5WrLzXks6zCQfn9Oxr8pORKyl/pF+QjHmW/Qx3qu0oyrrNC+y2JLTU1E2rcWYAmzlnqngWXHQjy51VzW70Vw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openharmony" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-wasm32-wasi": { + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/@rolldown/binding-wasm32-wasi/-/binding-wasm32-wasi-1.1.5.tgz", + "integrity": "sha512-VbQGPX2b4r48TAMIM2cjgluIM1HYutm4pcTEJsle7iEP7sB1dFqtPLBVbdLAZCxy1txCcPxf4QFf4v8uvltPqA==", + "cpu": [ + "wasm32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "dependencies": { + "@emnapi/core": "1.11.1", + "@emnapi/runtime": "1.11.1", + "@napi-rs/wasm-runtime": "^1.1.6" + }, + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-win32-arm64-msvc": { + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/@rolldown/binding-win32-arm64-msvc/-/binding-win32-arm64-msvc-1.1.5.tgz", + "integrity": "sha512-gHv82k63z4qpV5+Q1y/12KrK0ltWBukVDI8nZcbT7Tt/ZlOIVwppazneq0F93oDxTo3IgAMEDIoQh3E2n6mVsw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-win32-x64-msvc": { + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/@rolldown/binding-win32-x64-msvc/-/binding-win32-x64-msvc-1.1.5.tgz", + "integrity": "sha512-tTZuDBPw85tEN5PQi1pnEBzDy0Z49HtScLAbD5t6hyeU92A95pRWaSMw1GZZi/RwgSgUIl0xrSlXIT/9QzvYSA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/pluginutils": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/@rolldown/pluginutils/-/pluginutils-1.0.1.tgz", + "integrity": "sha512-2j9bGt5Jh8hj+vPtgzPtl72j0yRxHAyumoo6TNfAjsLB04UtpSvPbPcDcBMxz7n+9CYB0c1GxQFxYRg2jimqGw==", + "dev": true, + "license": "MIT" + }, + "node_modules/@standard-schema/spec": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@standard-schema/spec/-/spec-1.1.0.tgz", + "integrity": "sha512-l2aFy5jALhniG5HgqrD6jXLi/rUWrKvqN/qJx6yoJsgKhblVd+iqqU4RCXavm/jPityDo5TCvKMnpjKnOriy0w==", + "dev": true, + "license": "MIT" + }, + "node_modules/@tybys/wasm-util": { + "version": "0.10.3", + "resolved": "https://registry.npmjs.org/@tybys/wasm-util/-/wasm-util-0.10.3.tgz", + "integrity": "sha512-F3fo1MYrRJYL3zER0OUOmkutjr1Vp23m7OsSgp7nq4SP6OqX6C/56XFIPAl5bt3zaBRjmW7SGz3u/6LwFpYcOg==", + "dev": true, + "license": "MIT", + "optional": true, + "dependencies": { + "tslib": "^2.4.0" + } + }, + "node_modules/@types/chai": { + "version": "5.2.3", + "resolved": "https://registry.npmjs.org/@types/chai/-/chai-5.2.3.tgz", + "integrity": "sha512-Mw558oeA9fFbv65/y4mHtXDs9bPnFMZAL/jxdPFUpOHHIXX91mcgEHbS5Lahr+pwZFR8A7GQleRWeI6cGFC2UA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/deep-eql": "*", + "assertion-error": "^2.0.1" + } + }, + "node_modules/@types/deep-eql": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/@types/deep-eql/-/deep-eql-4.0.2.tgz", + "integrity": "sha512-c9h9dVVMigMPc4bwTvC5dxqtqJZwQPePsWjPlpSOnojbor6pGqdk541lfA7AqFQr5pB1BRdq0juY9db81BwyFw==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/estree": { + "version": "1.0.9", + "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.9.tgz", + "integrity": "sha512-GhdPgy1el4/ImP05X05Uw4cw2/M93BCUmnEvWZNStlCzEKME4Fkk+YpoA5OiHNQmoS7Cafb8Xa3Pya8m1Qrzeg==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/react": { + "version": "19.2.17", + "resolved": "https://registry.npmjs.org/@types/react/-/react-19.2.17.tgz", + "integrity": "sha512-MXfmqaVPEVgkBT/aY0aGCkRWWtByiYQXo3xdQ8r5RzuFrPiRn8Gar2tQdXSUQ2GKV3bkXckek89V8wQBY2Q/Aw==", + "dev": true, + "license": "MIT", + "peer": true, + "dependencies": { + "csstype": "^3.2.2" + } + }, + "node_modules/@types/react-dom": { + "version": "19.2.3", + "resolved": "https://registry.npmjs.org/@types/react-dom/-/react-dom-19.2.3.tgz", + "integrity": "sha512-jp2L/eY6fn+KgVVQAOqYItbF0VY/YApe5Mz2F0aykSO8gx31bYCZyvSeYxCHKvzHG5eZjc+zyaS5BrBWya2+kQ==", + "dev": true, + "license": "MIT", + "peerDependencies": { + "@types/react": "^19.2.0" + } + }, + "node_modules/@vitejs/plugin-react": { + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/@vitejs/plugin-react/-/plugin-react-6.0.3.tgz", + "integrity": "sha512-vmFvco5/QuC2f9Oj+wTk0+9XeDFkHxSamwZKYc7MxYwKICfvUvlMhqKI0VuICPltGqh1neqBKDvO4kes1ya8vg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@rolldown/pluginutils": "^1.0.1" + }, + "engines": { + "node": "^20.19.0 || >=22.12.0" + }, + "peerDependencies": { + "@rolldown/plugin-babel": "^0.1.7 || ^0.2.0", + "babel-plugin-react-compiler": "^1.0.0", + "vite": "^8.0.0" + }, + "peerDependenciesMeta": { + "@rolldown/plugin-babel": { + "optional": true + }, + "babel-plugin-react-compiler": { + "optional": true + } + } + }, + "node_modules/@vitest/expect": { + "version": "4.1.10", + "resolved": "https://registry.npmjs.org/@vitest/expect/-/expect-4.1.10.tgz", + "integrity": "sha512-YsCn+qAk1GWjQOWFEsEcL2gNQ0zmVmQu3T03qP6UyjhtmdtwtbuI+DASn/7iQB3HGTXkdBwGddzxPlmiql5vlA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@standard-schema/spec": "^1.1.0", + "@types/chai": "^5.2.2", + "@vitest/spy": "4.1.10", + "@vitest/utils": "4.1.10", + "chai": "^6.2.2", + "tinyrainbow": "^3.1.0" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/@vitest/mocker": { + "version": "4.1.10", + "resolved": "https://registry.npmjs.org/@vitest/mocker/-/mocker-4.1.10.tgz", + "integrity": "sha512-v0xaezt+DKEmKfaxg133ldzADrwLGd7Ze1MfQQTYfvs8OqZIwbxyxaYURivwV7sWy5fqn3rH5uOrSp07bp44Ow==", + "dev": true, + "license": "MIT", + "dependencies": { + "@vitest/spy": "4.1.10", + "estree-walker": "^3.0.3", + "magic-string": "^0.30.21" + }, + "funding": { + "url": "https://opencollective.com/vitest" + }, + "peerDependencies": { + "msw": "^2.4.9", + "vite": "^6.0.0 || ^7.0.0 || ^8.0.0" + }, + "peerDependenciesMeta": { + "msw": { + "optional": true + }, + "vite": { + "optional": true + } + } + }, + "node_modules/@vitest/pretty-format": { + "version": "4.1.10", + "resolved": "https://registry.npmjs.org/@vitest/pretty-format/-/pretty-format-4.1.10.tgz", + "integrity": "sha512-W1HsjSH4MXQ9YfmmhLAoIYf1HRfekQCGngeIgcei6MP5QQGWUe0gkopdZQaVCFO+JDJMrAJGwa5pRpNpvy4P8Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "tinyrainbow": "^3.1.0" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/@vitest/runner": { + "version": "4.1.10", + "resolved": "https://registry.npmjs.org/@vitest/runner/-/runner-4.1.10.tgz", + "integrity": "sha512-IKI6kpIH+LmpROplyLwBBaCfMgOZOMsygVa6BARD6ahA04VRuJSa6OaVG7kRvSEMD870Vd91rSSw0eegtWyLGg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@vitest/utils": "4.1.10", + "pathe": "^2.0.3" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/@vitest/snapshot": { + "version": "4.1.10", + "resolved": "https://registry.npmjs.org/@vitest/snapshot/-/snapshot-4.1.10.tgz", + "integrity": "sha512-xRkfOT1qpTAi/Ti4Y1LtfRc3kEuqxGw59eN2jN9pRWMtS/XDevekhcFSqvQqjUNGksfjMJu3Y+oJ+4Ypn2OaJw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@vitest/pretty-format": "4.1.10", + "@vitest/utils": "4.1.10", + "magic-string": "^0.30.21", + "pathe": "^2.0.3" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/@vitest/spy": { + "version": "4.1.10", + "resolved": "https://registry.npmjs.org/@vitest/spy/-/spy-4.1.10.tgz", + "integrity": "sha512-PLf/Ugvoq5wO/b4rwYCR1h2PSIdXz7wnkQFMiUpLdtM7l6pqVFcQIBEHyT1+l+cj7mNwAfZHzqXqDyjvOuwbDw==", + "dev": true, + "license": "MIT", + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/@vitest/utils": { + "version": "4.1.10", + "resolved": "https://registry.npmjs.org/@vitest/utils/-/utils-4.1.10.tgz", + "integrity": "sha512-fy9am/HWxbaGt/Sawrp90vt6Y6jQwf1RX77cz3uwoJwJVMli/e1IEwRPnMNJ7vKfPTwo0diXifkpPvwH9v7nGA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@vitest/pretty-format": "4.1.10", + "convert-source-map": "^2.0.0", + "tinyrainbow": "^3.1.0" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/assertion-error": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/assertion-error/-/assertion-error-2.0.1.tgz", + "integrity": "sha512-Izi8RQcffqCeNVgFigKli1ssklIbpHnCYc6AknXGYoB6grJqyeby7jv12JUQgmTAnIDnbck1uxksT4dzN3PWBA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + } + }, + "node_modules/chai": { + "version": "6.2.2", + "resolved": "https://registry.npmjs.org/chai/-/chai-6.2.2.tgz", + "integrity": "sha512-NUPRluOfOiTKBKvWPtSD4PhFvWCqOi0BGStNWs57X9js7XGTprSmFoz5F0tWhR4WPjNeR9jXqdC7/UpSJTnlRg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + } + }, + "node_modules/convert-source-map": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/convert-source-map/-/convert-source-map-2.0.0.tgz", + "integrity": "sha512-Kvp459HrV2FEJ1CAsi1Ku+MY3kasH19TFykTz2xWmMeq6bk2NU3XXvfJ+Q61m0xktWwt+1HSYf3JZsTms3aRJg==", + "dev": true, + "license": "MIT" + }, + "node_modules/csstype": { + "version": "3.2.3", + "resolved": "https://registry.npmjs.org/csstype/-/csstype-3.2.3.tgz", + "integrity": "sha512-z1HGKcYy2xA8AGQfwrn0PAy+PB7X/GSj3UVJW9qKyn43xWa+gl5nXmU4qqLMRzWVLFC8KusUX8T/0kCiOYpAIQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/detect-libc": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.1.2.tgz", + "integrity": "sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">=8" + } + }, + "node_modules/es-module-lexer": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/es-module-lexer/-/es-module-lexer-2.3.1.tgz", + "integrity": "sha512-shc1dbU90Yl/xq1QrC7QRtfcwURZuVRfPhZbDoldJ1cn1gzDvBaBWlv0eFolj5+0znnPJz5TXLxsN77X/12KTA==", + "dev": true, + "license": "MIT" + }, + "node_modules/estree-walker": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/estree-walker/-/estree-walker-3.0.3.tgz", + "integrity": "sha512-7RUKfXgSMMkzt6ZuXmqapOurLGPPfgj6l9uRZ7lRGolvk0y2yocc35LdcxKC5PQZdn2DMqioAQ2NoWcrTKmm6g==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/estree": "^1.0.0" + } + }, + "node_modules/expect-type": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/expect-type/-/expect-type-1.4.0.tgz", + "integrity": "sha512-KfYbmpRm0VbLjEvVa9yGwCi9GI34xvi7A/HXYWQO65CSD2u3MczUJSuwXKFIxlGsgBQizV9q5J9NHj4VG0n+pA==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">=12.0.0" + } + }, + "node_modules/fdir": { + "version": "6.5.0", + "resolved": "https://registry.npmjs.org/fdir/-/fdir-6.5.0.tgz", + "integrity": "sha512-tIbYtZbucOs0BRGqPJkshJUYdL+SDH7dVM8gjy+ERp3WAUjLEFJE+02kanyHtwjWOnwrKYBiwAmM0p4kLJAnXg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12.0.0" + }, + "peerDependencies": { + "picomatch": "^3 || ^4" + }, + "peerDependenciesMeta": { + "picomatch": { + "optional": true + } + } + }, + "node_modules/fsevents": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", + "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, + "node_modules/lightningcss": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss/-/lightningcss-1.32.0.tgz", + "integrity": "sha512-NXYBzinNrblfraPGyrbPoD19C1h9lfI/1mzgWYvXUTe414Gz/X1FD2XBZSZM7rRTrMA8JL3OtAaGifrIKhQ5yQ==", + "dev": true, + "license": "MPL-2.0", + "dependencies": { + "detect-libc": "^2.0.3" + }, + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + }, + "optionalDependencies": { + "lightningcss-android-arm64": "1.32.0", + "lightningcss-darwin-arm64": "1.32.0", + "lightningcss-darwin-x64": "1.32.0", + "lightningcss-freebsd-x64": "1.32.0", + "lightningcss-linux-arm-gnueabihf": "1.32.0", + "lightningcss-linux-arm64-gnu": "1.32.0", + "lightningcss-linux-arm64-musl": "1.32.0", + "lightningcss-linux-x64-gnu": "1.32.0", + "lightningcss-linux-x64-musl": "1.32.0", + "lightningcss-win32-arm64-msvc": "1.32.0", + "lightningcss-win32-x64-msvc": "1.32.0" + } + }, + "node_modules/lightningcss-android-arm64": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-android-arm64/-/lightningcss-android-arm64-1.32.0.tgz", + "integrity": "sha512-YK7/ClTt4kAK0vo6w3X+Pnm0D2cf2vPHbhOXdoNti1Ga0al1P4TBZhwjATvjNwLEBCnKvjJc2jQgHXH0NEwlAg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-darwin-arm64": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-darwin-arm64/-/lightningcss-darwin-arm64-1.32.0.tgz", + "integrity": "sha512-RzeG9Ju5bag2Bv1/lwlVJvBE3q6TtXskdZLLCyfg5pt+HLz9BqlICO7LZM7VHNTTn/5PRhHFBSjk5lc4cmscPQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-darwin-x64": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-darwin-x64/-/lightningcss-darwin-x64-1.32.0.tgz", + "integrity": "sha512-U+QsBp2m/s2wqpUYT/6wnlagdZbtZdndSmut/NJqlCcMLTWp5muCrID+K5UJ6jqD2BFshejCYXniPDbNh73V8w==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-freebsd-x64": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-freebsd-x64/-/lightningcss-freebsd-x64-1.32.0.tgz", + "integrity": "sha512-JCTigedEksZk3tHTTthnMdVfGf61Fky8Ji2E4YjUTEQX14xiy/lTzXnu1vwiZe3bYe0q+SpsSH/CTeDXK6WHig==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-linux-arm-gnueabihf": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-linux-arm-gnueabihf/-/lightningcss-linux-arm-gnueabihf-1.32.0.tgz", + "integrity": "sha512-x6rnnpRa2GL0zQOkt6rts3YDPzduLpWvwAF6EMhXFVZXD4tPrBkEFqzGowzCsIWsPjqSK+tyNEODUBXeeVHSkw==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-linux-arm64-gnu": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-linux-arm64-gnu/-/lightningcss-linux-arm64-gnu-1.32.0.tgz", + "integrity": "sha512-0nnMyoyOLRJXfbMOilaSRcLH3Jw5z9HDNGfT/gwCPgaDjnx0i8w7vBzFLFR1f6CMLKF8gVbebmkUN3fa/kQJpQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-linux-arm64-musl": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-linux-arm64-musl/-/lightningcss-linux-arm64-musl-1.32.0.tgz", + "integrity": "sha512-UpQkoenr4UJEzgVIYpI80lDFvRmPVg6oqboNHfoH4CQIfNA+HOrZ7Mo7KZP02dC6LjghPQJeBsvXhJod/wnIBg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-linux-x64-gnu": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-linux-x64-gnu/-/lightningcss-linux-x64-gnu-1.32.0.tgz", + "integrity": "sha512-V7Qr52IhZmdKPVr+Vtw8o+WLsQJYCTd8loIfpDaMRWGUZfBOYEJeyJIkqGIDMZPwPx24pUMfwSxxI8phr/MbOA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-linux-x64-musl": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-linux-x64-musl/-/lightningcss-linux-x64-musl-1.32.0.tgz", + "integrity": "sha512-bYcLp+Vb0awsiXg/80uCRezCYHNg1/l3mt0gzHnWV9XP1W5sKa5/TCdGWaR/zBM2PeF/HbsQv/j2URNOiVuxWg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-win32-arm64-msvc": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-win32-arm64-msvc/-/lightningcss-win32-arm64-msvc-1.32.0.tgz", + "integrity": "sha512-8SbC8BR40pS6baCM8sbtYDSwEVQd4JlFTOlaD3gWGHfThTcABnNDBda6eTZeqbofalIJhFx0qKzgHJmcPTnGdw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-win32-x64-msvc": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-win32-x64-msvc/-/lightningcss-win32-x64-msvc-1.32.0.tgz", + "integrity": "sha512-Amq9B/SoZYdDi1kFrojnoqPLxYhQ4Wo5XiL8EVJrVsB8ARoC1PWW6VGtT0WKCemjy8aC+louJnjS7U18x3b06Q==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/magic-string": { + "version": "0.30.21", + "resolved": "https://registry.npmjs.org/magic-string/-/magic-string-0.30.21.tgz", + "integrity": "sha512-vd2F4YUyEXKGcLHoq+TEyCjxueSeHnFxyyjNp80yg0XV4vUhnDer/lvvlqM/arB5bXQN5K2/3oinyCRyx8T2CQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/sourcemap-codec": "^1.5.5" + } + }, + "node_modules/nanoid": { + "version": "3.3.16", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.16.tgz", + "integrity": "sha512-bzlKTyNJ7+LdGIIwy8ijFpIqEQIvafahV7eYykJ8Cvh42EdJeODoJ6gUJXpQJvej1BddH8OqTXZNE/KfbWAu8Q==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "bin": { + "nanoid": "bin/nanoid.cjs" + }, + "engines": { + "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1" + } + }, + "node_modules/obug": { + "version": "2.1.4", + "resolved": "https://registry.npmjs.org/obug/-/obug-2.1.4.tgz", + "integrity": "sha512-4a+OsYv9UktOJKE+l1A4OufDgdRF9PifWj+tJnHURo/P+WOxpG4GzUFL9qCalmWauao6ogiG+QvnCovwPoyAWA==", + "dev": true, + "funding": [ + "https://github.com/sponsors/sxzz", + "https://opencollective.com/debug" + ], + "license": "MIT", + "engines": { + "node": ">=12.20.0" + } + }, + "node_modules/pathe": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/pathe/-/pathe-2.0.3.tgz", + "integrity": "sha512-WUjGcAqP1gQacoQe+OBJsFA7Ld4DyXuUIjZ5cc75cLHvJ7dtNsTugphxIADwspS+AraAUePCKrSVtPLFj/F88w==", + "dev": true, + "license": "MIT" + }, + "node_modules/picocolors": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", + "integrity": "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==", + "dev": true, + "license": "ISC" + }, + "node_modules/picomatch": { + "version": "4.0.5", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.5.tgz", + "integrity": "sha512-RvwwcruNjI1ncT5xRakeyS9Lf8lcItv34KD+aif+VH9kduAyfYBipGh12274xtenIPZ119/R9BdTBa8gAwSh0A==", + "dev": true, + "license": "MIT", + "peer": true, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" + } + }, + "node_modules/postcss": { + "version": "8.5.20", + "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.20.tgz", + "integrity": "sha512-lW616l85ucIQL+FocMmL7pQFPqBmwejrCMg+iPxyImlrANNJG9NHq/RkyCZopDhd8C3LA03PHRJDjkbGu8vvug==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/postcss/" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/postcss" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "dependencies": { + "nanoid": "^3.3.16", + "picocolors": "^1.1.1", + "source-map-js": "^1.2.1" + }, + "engines": { + "node": "^10 || ^12 || >=14" + } + }, + "node_modules/react": { + "version": "19.2.7", + "resolved": "https://registry.npmjs.org/react/-/react-19.2.7.tgz", + "integrity": "sha512-HNe9WslTbXmFK8o8cmwgAeJFSBvt1bPdHCVKtaaV+WlAN36mpT4hcRpwbf3fY56ar2oIXzsBpOAiIRHAdY0OlQ==", + "license": "MIT", + "peer": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/react-dom": { + "version": "19.2.7", + "resolved": "https://registry.npmjs.org/react-dom/-/react-dom-19.2.7.tgz", + "integrity": "sha512-t0BRVXvbiE/o20Hfw669rLbMCDWtYZLvmJigy2f0MxsXF+71pxhR3xOkspmsO8h3ZlNzyibAmtCa3l4lYKk6gQ==", + "license": "MIT", + "dependencies": { + "scheduler": "^0.27.0" + }, + "peerDependencies": { + "react": "^19.2.7" + } + }, + "node_modules/rolldown": { + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/rolldown/-/rolldown-1.1.5.tgz", + "integrity": "sha512-t9z29cJjXf/vxQ8dyhCSpt6H6aSwHTk8cT5I3iy6SMXuFpk5mB6PL6XfC8PCwrPTx93udwKUm9HRteAlTGBLiA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@oxc-project/types": "=0.139.0", + "@rolldown/pluginutils": "^1.0.0" + }, + "bin": { + "rolldown": "bin/cli.mjs" + }, + "engines": { + "node": "^20.19.0 || >=22.12.0" + }, + "optionalDependencies": { + "@rolldown/binding-android-arm64": "1.1.5", + "@rolldown/binding-darwin-arm64": "1.1.5", + "@rolldown/binding-darwin-x64": "1.1.5", + "@rolldown/binding-freebsd-x64": "1.1.5", + "@rolldown/binding-linux-arm-gnueabihf": "1.1.5", + "@rolldown/binding-linux-arm64-gnu": "1.1.5", + "@rolldown/binding-linux-arm64-musl": "1.1.5", + "@rolldown/binding-linux-ppc64-gnu": "1.1.5", + "@rolldown/binding-linux-s390x-gnu": "1.1.5", + "@rolldown/binding-linux-x64-gnu": "1.1.5", + "@rolldown/binding-linux-x64-musl": "1.1.5", + "@rolldown/binding-openharmony-arm64": "1.1.5", + "@rolldown/binding-wasm32-wasi": "1.1.5", + "@rolldown/binding-win32-arm64-msvc": "1.1.5", + "@rolldown/binding-win32-x64-msvc": "1.1.5" + } + }, + "node_modules/scheduler": { + "version": "0.27.0", + "resolved": "https://registry.npmjs.org/scheduler/-/scheduler-0.27.0.tgz", + "integrity": "sha512-eNv+WrVbKu1f3vbYJT/xtiF5syA5HPIMtf9IgY/nKg0sWqzAUEvqY/xm7OcZc/qafLx/iO9FgOmeSAp4v5ti/Q==", + "license": "MIT" + }, + "node_modules/siginfo": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/siginfo/-/siginfo-2.0.0.tgz", + "integrity": "sha512-ybx0WO1/8bSBLEWXZvEd7gMW3Sn3JFlW3TvX1nREbDLRNQNaeNN8WK0meBwPdAaOI7TtRRRJn/Es1zhrrCHu7g==", + "dev": true, + "license": "ISC" + }, + "node_modules/source-map-js": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.1.tgz", + "integrity": "sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA==", + "dev": true, + "license": "BSD-3-Clause", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/stackback": { + "version": "0.0.2", + "resolved": "https://registry.npmjs.org/stackback/-/stackback-0.0.2.tgz", + "integrity": "sha512-1XMJE5fQo1jGH6Y/7ebnwPOBEkIEnT4QF32d5R1+VXdXveM0IBMJt8zfaxX1P3QhVwrYe+576+jkANtSS2mBbw==", + "dev": true, + "license": "MIT" + }, + "node_modules/std-env": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/std-env/-/std-env-4.2.0.tgz", + "integrity": "sha512-oCUKSupKTHX53EyjDtuZQ64pjLJ6yYCtpmEw0goYxtjG9KpbRe8KAsl2tBUGU9DyMcJ0RwJ8GqJAFzMXcXW1Rw==", + "dev": true, + "license": "MIT" + }, + "node_modules/tinybench": { + "version": "2.9.0", + "resolved": "https://registry.npmjs.org/tinybench/-/tinybench-2.9.0.tgz", + "integrity": "sha512-0+DUvqWMValLmha6lr4kD8iAMK1HzV0/aKnCtWb9v9641TnP/MFb7Pc2bxoxQjTXAErryXVgUOfv2YqNllqGeg==", + "dev": true, + "license": "MIT" + }, + "node_modules/tinyexec": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/tinyexec/-/tinyexec-1.2.4.tgz", + "integrity": "sha512-SHf/r48b7vOrjve9PxJo3MN5v5yuyjHvdUcrQffT3WXMUfnGmHDVbC4k3sHJaJTgZCwpUplIaAo5ANtMyp3YHg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + } + }, + "node_modules/tinyglobby": { + "version": "0.2.17", + "resolved": "https://registry.npmjs.org/tinyglobby/-/tinyglobby-0.2.17.tgz", + "integrity": "sha512-wXR/dYpcqKmfWpEdZjiKJOwCNFndD0DMnrW/cYjVGttEkBfVgcLFHoNrlj47mjOVic9yyNu65alsgF4NQyTa2g==", + "dev": true, + "license": "MIT", + "dependencies": { + "fdir": "^6.5.0", + "picomatch": "^4.0.4" + }, + "engines": { + "node": ">=12.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/SuperchupuDev" + } + }, + "node_modules/tinyrainbow": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/tinyrainbow/-/tinyrainbow-3.1.0.tgz", + "integrity": "sha512-Bf+ILmBgretUrdJxzXM0SgXLZ3XfiaUuOj/IKQHuTXip+05Xn+uyEYdVg0kYDipTBcLrCVyUzAPz7QmArb0mmw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/tslib": { + "version": "2.8.1", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", + "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", + "dev": true, + "license": "0BSD", + "optional": true + }, + "node_modules/typescript": { + "version": "5.9.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", + "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + }, + "node_modules/vite": { + "version": "8.1.5", + "resolved": "https://registry.npmjs.org/vite/-/vite-8.1.5.tgz", + "integrity": "sha512-7ULLwsCdYx/nRyrpiEwvqb5TFHrMVZyBt+rg/OAXT7rgj/z+DtTDyKFeLAdDkubDVDKD8jOsndmy7m55XcfUsw==", + "dev": true, + "license": "MIT", + "peer": true, + "dependencies": { + "lightningcss": "^1.32.0", + "picomatch": "^4.0.5", + "postcss": "^8.5.17", + "rolldown": "~1.1.5", + "tinyglobby": "^0.2.17" + }, + "bin": { + "vite": "bin/vite.js" + }, + "engines": { + "node": "^20.19.0 || >=22.12.0" + }, + "funding": { + "url": "https://github.com/vitejs/vite?sponsor=1" + }, + "optionalDependencies": { + "fsevents": "~2.3.3" + }, + "peerDependencies": { + "@types/node": "^20.19.0 || >=22.12.0", + "@vitejs/devtools": "^0.3.0", + "esbuild": "^0.27.0 || ^0.28.0", + "jiti": ">=1.21.0", + "less": "^4.0.0", + "sass": "^1.70.0", + "sass-embedded": "^1.70.0", + "stylus": ">=0.54.8", + "sugarss": "^5.0.0", + "terser": "^5.16.0", + "tsx": "^4.8.1", + "yaml": "^2.4.2" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + }, + "@vitejs/devtools": { + "optional": true + }, + "esbuild": { + "optional": true + }, + "jiti": { + "optional": true + }, + "less": { + "optional": true + }, + "sass": { + "optional": true + }, + "sass-embedded": { + "optional": true + }, + "stylus": { + "optional": true + }, + "sugarss": { + "optional": true + }, + "terser": { + "optional": true + }, + "tsx": { + "optional": true + }, + "yaml": { + "optional": true + } + } + }, + "node_modules/vitest": { + "version": "4.1.10", + "resolved": "https://registry.npmjs.org/vitest/-/vitest-4.1.10.tgz", + "integrity": "sha512-R9jUTe5S4Qb0HCd4TNqpC7oGcrMssMRGXLW80ubjWsW9VH5GF8y1Y0SFLY9AbqSk6nt0PnOx4H4WNJYZ13GUPw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@vitest/expect": "4.1.10", + "@vitest/mocker": "4.1.10", + "@vitest/pretty-format": "4.1.10", + "@vitest/runner": "4.1.10", + "@vitest/snapshot": "4.1.10", + "@vitest/spy": "4.1.10", + "@vitest/utils": "4.1.10", + "es-module-lexer": "^2.0.0", + "expect-type": "^1.3.0", + "magic-string": "^0.30.21", + "obug": "^2.1.1", + "pathe": "^2.0.3", + "picomatch": "^4.0.3", + "std-env": "^4.0.0-rc.1", + "tinybench": "^2.9.0", + "tinyexec": "^1.0.2", + "tinyglobby": "^0.2.15", + "tinyrainbow": "^3.1.0", + "vite": "^6.0.0 || ^7.0.0 || ^8.0.0", + "why-is-node-running": "^2.3.0" + }, + "bin": { + "vitest": "vitest.mjs" + }, + "engines": { + "node": "^20.0.0 || ^22.0.0 || >=24.0.0" + }, + "funding": { + "url": "https://opencollective.com/vitest" + }, + "peerDependencies": { + "@edge-runtime/vm": "*", + "@opentelemetry/api": "^1.9.0", + "@types/node": "^20.0.0 || ^22.0.0 || >=24.0.0", + "@vitest/browser-playwright": "4.1.10", + "@vitest/browser-preview": "4.1.10", + "@vitest/browser-webdriverio": "4.1.10", + "@vitest/coverage-istanbul": "4.1.10", + "@vitest/coverage-v8": "4.1.10", + "@vitest/ui": "4.1.10", + "happy-dom": "*", + "jsdom": "*", + "vite": "^6.0.0 || ^7.0.0 || ^8.0.0" + }, + "peerDependenciesMeta": { + "@edge-runtime/vm": { + "optional": true + }, + "@opentelemetry/api": { + "optional": true + }, + "@types/node": { + "optional": true + }, + "@vitest/browser-playwright": { + "optional": true + }, + "@vitest/browser-preview": { + "optional": true + }, + "@vitest/browser-webdriverio": { + "optional": true + }, + "@vitest/coverage-istanbul": { + "optional": true + }, + "@vitest/coverage-v8": { + "optional": true + }, + "@vitest/ui": { + "optional": true + }, + "happy-dom": { + "optional": true + }, + "jsdom": { + "optional": true + }, + "vite": { + "optional": false + } + } + }, + "node_modules/why-is-node-running": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/why-is-node-running/-/why-is-node-running-2.3.0.tgz", + "integrity": "sha512-hUrmaWBdVDcxvYqnyh09zunKzROWjbZTiNy8dBEjkS7ehEDQibXJ7XvlmtbwuTclUiIyN+CyXQD4Vmko8fNm8w==", + "dev": true, + "license": "MIT", + "dependencies": { + "siginfo": "^2.0.0", + "stackback": "0.0.2" + }, + "bin": { + "why-is-node-running": "cli.js" + }, + "engines": { + "node": ">=8" + } + } + } +} diff --git a/web/package.json b/web/package.json new file mode 100644 index 0000000..d987643 --- /dev/null +++ b/web/package.json @@ -0,0 +1,26 @@ +{ + "name": "sftguard-report-viewer", + "version": "0.1.0", + "private": true, + "type": "module", + "scripts": { + "dev": "vite", + "build": "tsc --noEmit && vite build", + "preview": "vite preview", + "typecheck": "tsc --noEmit --pretty false", + "test": "vitest run", + "check": "npm run typecheck && npm run test && npm run build" + }, + "dependencies": { + "react": "19.2.7", + "react-dom": "19.2.7" + }, + "devDependencies": { + "@types/react": "19.2.17", + "@types/react-dom": "19.2.3", + "@vitejs/plugin-react": "6.0.3", + "typescript": "5.9.3", + "vite": "8.1.5", + "vitest": "4.1.10" + } +} diff --git a/web/src/App.test.tsx b/web/src/App.test.tsx new file mode 100644 index 0000000..b27bfb6 --- /dev/null +++ b/web/src/App.test.tsx @@ -0,0 +1,31 @@ +import { renderToStaticMarkup } from "react-dom/server"; +import { describe, expect, it } from "vitest"; +import App from "./App"; + +describe("SFTGuard report viewer", () => { + it("renders all decision states and the local-only import boundary", () => { + const html = renderToStaticMarkup(); + + expect(html).toContain("PASS"); + expect(html).toContain("FAIL"); + expect(html).toContain("ABSTAIN"); + expect(html).toContain("Choose local file"); + expect(html).toContain("Nothing is uploaded"); + expect(html).toContain("Claim boundary"); + expect(html).toContain("Privacy boundary"); + expect(html).toContain("Unverified — digest not recomputed"); + expect(html).not.toContain("dangerouslySetInnerHTML"); + expect(html).not.toMatch(/Ã|Â|â€|�/u); + }); + + it("uses accessible landmarks and explicit progress semantics", () => { + const html = renderToStaticMarkup(); + + expect(html).toContain('href="#report-main"'); + expect(html).toContain('
= { + PASS: { + eyebrow: "Release gate cleared", + action: "Proceed within this evidence contract", + explanation: "Every required check represented by this report passed.", + }, + FAIL: { + eyebrow: "Release gate blocked", + action: "Resolve failures before release", + explanation: "At least one required check violated its declared gate.", + }, + ABSTAIN: { + eyebrow: "Decision withheld", + action: "Supply the missing evidence", + explanation: "The harness could not make a trustworthy pass/fail decision.", + }, +}; + +function reportSourceKey(source: ReportSource): string { + return `${source.origin}:${source.report.reportId}`; +} + +function signedPoints(value: number): string { + const sign = value > 0 ? "+" : value < 0 ? "\u2212" : ""; + return `${sign}${Math.abs(value).toFixed(1)} pp`; +} + +function signedMetric( + value: number, + unit: ScoreContract["unit"], +): string { + if (unit === "percentage-points") return signedPoints(value); + const sign = value > 0 ? "+" : value < 0 ? "\u2212" : ""; + return `${sign}${Math.abs(value).toFixed(3)} normalized`; +} + +function compactNumber(value: number | null): string { + if (value === null) return "Not verified"; + return new Intl.NumberFormat("en", { notation: "compact" }).format(value); +} + +function displayDate(value: string): string { + return new Intl.DateTimeFormat("en", { + year: "numeric", + month: "short", + day: "2-digit", + hour: "2-digit", + minute: "2-digit", + hour12: false, + timeZone: "UTC", + timeZoneName: "short", + }).format(new Date(value)); +} + +function shortHash(hash: string): string { + return `${hash.slice(0, 10)}\u2026${hash.slice(-8)}`; +} + +function StatusBadge({ status, compact = false }: { status: ReportStatus; compact?: boolean }) { + return ( + + + ); +} + +function SectionHeading({ + number, + title, + detail, +}: { + number: string; + title: string; + detail: string; +}) { + return ( +
+ +
+

{title}

+

{detail}

+
+
+ ); +} + +function MetricCard({ + label, + value, + context, + tone = "neutral", + children, +}: { + label: string; + value: string; + context: string; + tone?: "positive" | "negative" | "caution" | "neutral"; + children?: ReactNode; +}) { + return ( +
+

{label}

+

{value}

+

{context}

+ {children} +
+ ); +} + +function ScoreEvidence({ score, label }: { score: ScoreContract; label: string }) { + if (score.baseScore !== null && score.adapterScore !== null) { + return ( +
+ Base {score.baseScore.toFixed(1)} +
+ ); + } + if (score.confidenceInterval) { + return ( +
+ Pairs {score.pairCount ?? "unavailable"} +
+ ); + } + return null; +} + +function RegressionContract({ report }: { report: SftGuardReport }) { + const regression = report.regression; + if (!regression) { + return ( +
+ + +
+ ); + } + + const targetPassed = regression.target.status === "PASS"; + const retentionPassed = regression.retention.status === "PASS"; + + return ( +
+ + + + + + +
+ ); +} + +function AuditContract({ report }: { report: SftGuardReport }) { + if (!report.audit) return null; + const audit = report.audit; + return ( +
+ + + +
+ ); +} + +function FaultCoveragePanel({ report }: { report: SftGuardReport }) { + const coverage = report.faultCoverage; + if (!coverage) { + return ( +
+

Not part of this artifact

+

No fault matrix attached

+

+ Open the checked-in fault-matrix sample or import a compatible report + to inspect detection coverage and clean-control behavior. +

+
+ ); + } + + const coveragePercent = + coverage.expectedFaultClasses === 0 + ? 0 + : (coverage.detectedFaultClasses / coverage.expectedFaultClasses) * 100; + return ( +
+
+
+ {coverage.detectedFaultClasses} + of {coverage.expectedFaultClasses} +
+
+
+

+ {coverage.carryForward ? "Carry-forward allowed" : "Carry-forward blocked"} +

+

+ This reports deterministic synthetic coverage of SFTGuard’s own + checks—not coverage of unknown real-world faults. +

+
+ + + +
+
+
+ ); +} + +function CoverageBar({ + label, + value, + target, + inverse = false, +}: { + label: string; + value: number; + target: string; + inverse?: boolean; +}) { + return ( +
+
+ {label} + {target} +
+
{value.toFixed(1)}%
+
+ +
+
+ ); +} + +function IssueSummary({ issues }: { issues: ReportIssue[] }) { + const categoryCounts = useMemo(() => { + const counts = new Map(); + for (const issue of issues) { + counts.set(issue.category, (counts.get(issue.category) ?? 0) + issue.count); + } + return [...counts.entries()].sort((left, right) => right[1] - left[1]); + }, [issues]); + + if (issues.length === 0) { + return ( +
+

No reported issues

+

The artifact contains no issue records.

+
+ ); + } + + return ( + <> +
+ {categoryCounts.map(([category, count]) => ( + + {category} {count} + + ))} +
+
+ + + + + + + + + + + + {issues.map((issue) => ( + + + + + + + ))} + +
Reported SFTGuard issues
SignalSeverityOutcomeCount
+
{issue.title}
+ {issue.id} +

{issue.detail}

+
+ + {issue.severity} + + + + {issue.count}
+
+ + ); +} + +function ProvenancePanel({ report }: { report: SftGuardReport }) { + const provenance = report.provenance; + const values: Array<{ + label: string; + value: string | null; + isHash?: boolean; + }> = [ + { label: "Tool version", value: provenance.toolVersion }, + { label: "Protocol commit", value: provenance.protocolCommit, isHash: true }, + { + label: "Implementation", + value: provenance.implementationCommit, + isHash: true, + }, + { label: "Input SHA-256", value: provenance.inputSha256, isHash: true }, + { + label: "Declared artifact SHA-256", + value: provenance.artifactSha256, + isHash: true, + }, + { + label: "Digest verification", + value: + provenance.integrityStatus === "verified" + ? "Verified" + : provenance.integrityStatus === "mismatch" + ? "Mismatch" + : "Unverified — digest not recomputed", + }, + { label: "Runtime", value: provenance.runtime }, + ]; + const replayTone = + provenance.deterministicReplay === null + ? "replay-unknown" + : provenance.deterministicReplay + ? "replay-pass" + : "replay-fail"; + const replayTitle = + provenance.deterministicReplay === null + ? "Unavailable" + : provenance.deterministicReplay + ? "Matched" + : "Not matched"; + const replayDetail = + provenance.deterministicReplay === null + ? "The artifact does not declare a replay result" + : provenance.deterministicReplay + ? "Canonical payload reproduced" + : "Reproduction claim is blocked"; + return ( +
+
+

Deterministic replay

+ {replayTitle} + {replayDetail} +
+
+ {values.map(({ label, value, isHash = false }) => { + return ( +
+
{label}
+
+ {value === null + ? "Unavailable in artifact" + : isHash + ? shortHash(value) + : value} +
+
+ ); + })} +
+
+ ); +} + +function BoundaryPanel({ report }: { report: SftGuardReport }) { + return ( +
+
+

Claim boundary

+

Evidence, not certification

+
    + {report.boundaries.claim.map((item) => ( +
  • {item}
  • + ))} +
+
+
+

Privacy boundary

+

Local viewing, narrow display

+
    + {report.boundaries.privacy.map((item) => ( +
  • {item}
  • + ))} +
+
+
+ ); +} + +export default function App() { + const [selectedKey, setSelectedKey] = useState( + reportSourceKey(SAMPLE_REPORTS[0]), + ); + const [importedReport, setImportedReport] = useState(null); + const [importMessage, setImportMessage] = useState( + "Choose a compatible JSON report. The file never leaves this browser tab.", + ); + const [importError, setImportError] = useState(false); + const fileInputRef = useRef(null); + + const reports = importedReport + ? [importedReport, ...SAMPLE_REPORTS] + : SAMPLE_REPORTS; + const activeSource = + reports.find((source) => reportSourceKey(source) === selectedKey) ?? reports[0]; + const report = activeSource.report; + const statusCopy = STATUS_COPY[report.status]; + + async function handleImport(event: ChangeEvent) { + const file = event.target.files?.[0]; + if (!file) return; + try { + if (file.size > MAX_REPORT_BYTES) { + throw new RangeError("Report exceeds the 2 MB local import limit."); + } + const parsed = parseReportText(await file.text()); + const source: ReportSource = { + report: parsed, + origin: "local import", + sourceLabel: file.name, + }; + setImportedReport(source); + setSelectedKey(reportSourceKey(source)); + setImportError(false); + setImportMessage(`${file.name} loaded locally. Nothing was uploaded or stored.`); + } catch (error) { + setImportError(true); + setImportMessage( + error instanceof Error ? error.message : "The report could not be read.", + ); + } finally { + event.target.value = ""; + } + } + + function removeImportedReport() { + setImportedReport(null); + setSelectedKey(reportSourceKey(SAMPLE_REPORTS[0])); + setImportError(false); + setImportMessage("Local import cleared from memory. Checked-in samples remain available."); + fileInputRef.current?.focus(); + } + + return ( + <> + + Skip to report + +
+
+ + + + SFTGuard + Local report viewer + + +
+
+
+ +
+ + +
+
+
+
+ + {reportTypeLabel(report.reportType)} + {activeSource.origin} + report-declared decision +
+

{statusCopy.eyebrow}

+

{report.title}

+

{report.summary}

+

{statusCopy.action}

+
+
+ {report.status.slice(0, 1)} +
+ {report.status} + {statusCopy.explanation} +
+
+
+ Report {report.reportId} + + {report.generatedAt + ? `Generated ${displayDate(report.generatedAt)}` + : "Generated time unavailable in artifact"} + +
+
+ +
+ +
Evaluation contract results
+ + +
+ +
+ +
Signals and issue categories
+ +
+ +
+ +
Fault coverage results
+ +
+ +
+ +
Artifact provenance
+ +
+ +
+ +
Claim and privacy boundaries
+ +
+ +
+ SFTGuard v0.1 viewer +

+ This viewer displays supplied evidence. It does not train a + model, certify safety, prove dataset quality, or detect unknown + failures. It validates the display schema but does not + recompute or independently attest declared hashes. Imported JSON remains in + this tab and is discarded on refresh. +

+
+
+
+
+ + ); +} diff --git a/web/src/data/sample-audit.json b/web/src/data/sample-audit.json new file mode 100644 index 0000000..9b39d28 --- /dev/null +++ b/web/src/data/sample-audit.json @@ -0,0 +1,54 @@ +{ + "schemaVersion": "sftguard.report.v1", + "reportType": "audit", + "reportId": "sample-audit-mask-abstention", + "title": "Training data and mask audit", + "generatedAt": "2026-07-19T09:30:00.000Z", + "status": "ABSTAIN", + "summary": "Structural checks completed, but the tokenizer could not expose a trustworthy assistant-token mask. Release approval is withheld.", + "audit": { + "recordsScanned": 1280, + "checksPassed": 7, + "checksTotal": 8, + "supervisedTokens": null + }, + "issues": [ + { + "id": "template.mask_unavailable", + "category": "Template & mask", + "severity": "error", + "outcome": "ABSTAIN", + "count": 1, + "title": "Assistant-token mask unavailable", + "detail": "The selected tokenizer template did not expose a verifiable assistant-token mask. SFTGuard did not infer one." + }, + { + "id": "data.exact_duplicate", + "category": "Data integrity", + "severity": "warning", + "outcome": "FAIL", + "count": 3, + "title": "Exact duplicate records", + "detail": "Three content hashes occurred more than once. Report data contains counts and hashes only." + } + ], + "provenance": { + "toolVersion": "0.1.0-sample", + "protocolCommit": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "implementationCommit": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "inputSha256": "1111111111111111111111111111111111111111111111111111111111111111", + "artifactSha256": "2222222222222222222222222222222222222222222222222222222222222222", + "runtime": "Python 3.12 · local illustrative fixture", + "deterministicReplay": true + }, + "boundaries": { + "claim": [ + "Illustrative UI fixture, not checked confirmatory evidence.", + "An abstention means the evidence contract was incomplete; it is not a pass." + ], + "privacy": [ + "No prompts, responses, model weights, or detected secret values are included.", + "Counts and content-addressed identifiers are illustrative placeholders." + ] + } +} diff --git a/web/src/data/sample-fault.json b/web/src/data/sample-fault.json new file mode 100644 index 0000000..96df531 --- /dev/null +++ b/web/src/data/sample-fault.json @@ -0,0 +1,65 @@ +{ + "schemaVersion": "sftguard.report.v1", + "reportType": "fault-matrix", + "reportId": "sample-fault-matrix-fail", + "title": "Synthetic fault-injection matrix", + "generatedAt": "2026-07-19T10:00:00.000Z", + "status": "FAIL", + "summary": "Eight of nine illustrative fault classes were detected. The macro-recall gate failed, so carry-forward remains blocked.", + "faultCoverage": { + "expectedFaultClasses": 9, + "detectedFaultClasses": 8, + "criticalRecallPercent": 100, + "macroRecallPercent": 88.9, + "cleanFalsePositiveRatePercent": 6.7, + "carryForward": false + }, + "issues": [ + { + "id": "data.exact_duplicate", + "category": "Data integrity", + "severity": "warning", + "outcome": "FAIL", + "count": 1, + "title": "Duplicate fault missed", + "detail": "The illustrative exact-duplicate case did not emit its required primary signal." + }, + { + "id": "gate.macro_recall", + "category": "Fault coverage", + "severity": "error", + "outcome": "FAIL", + "count": 1, + "title": "Macro recall below gate", + "detail": "Observed 88.9% against the frozen 90% minimum." + }, + { + "id": "gate.critical_recall", + "category": "Fault coverage", + "severity": "info", + "outcome": "PASS", + "count": 1, + "title": "Critical recall met", + "detail": "All illustrative critical fault classes emitted their expected primary signal." + } + ], + "provenance": { + "toolVersion": "0.1.0-sample", + "protocolCommit": "dddddddddddddddddddddddddddddddddddddddd", + "implementationCommit": "eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee", + "inputSha256": "5555555555555555555555555555555555555555555555555555555555555555", + "artifactSha256": "6666666666666666666666666666666666666666666666666666666666666666", + "runtime": "Python 3.12 · local illustrative fixture", + "deterministicReplay": false + }, + "boundaries": { + "claim": [ + "Illustrative UI fixture; confirmatory seeds have not been represented here.", + "Synthetic fault detection does not establish coverage of unknown real-world failures." + ], + "privacy": [ + "Injected strings and raw records are intentionally omitted.", + "Only expected and observed aggregate signals would appear in a real artifact." + ] + } +} diff --git a/web/src/data/sample-regression.json b/web/src/data/sample-regression.json new file mode 100644 index 0000000..5af9c67 --- /dev/null +++ b/web/src/data/sample-regression.json @@ -0,0 +1,64 @@ +{ + "schemaVersion": "sftguard.report.v1", + "reportType": "regression", + "reportId": "sample-regression-card-pass", + "title": "Adapter release regression card", + "generatedAt": "2026-07-19T09:45:00.000Z", + "status": "PASS", + "summary": "The adapter cleared the declared target gain and stayed inside the retention regression budget for this supplied evaluation contract.", + "regression": { + "target": { + "label": "Domain task accuracy", + "baseScore": 61.2, + "adapterScore": 73.6, + "deltaPercentagePoints": 12.4, + "minimumGainPercentagePoints": 5 + }, + "retention": { + "label": "General capability score", + "baseScore": 78.8, + "adapterScore": 78.2, + "deltaPercentagePoints": -0.6, + "maximumRegressionPercentagePoints": 2 + } + }, + "issues": [ + { + "id": "regression.target_gain", + "category": "Target behavior", + "severity": "info", + "outcome": "PASS", + "count": 1, + "title": "Target gain cleared", + "detail": "Observed +12.4 percentage points against a preregistered +5.0 point minimum." + }, + { + "id": "regression.retention_budget", + "category": "Retention", + "severity": "info", + "outcome": "PASS", + "count": 1, + "title": "Retention stayed in budget", + "detail": "Observed −0.6 percentage points against a maximum permitted regression of −2.0 points." + } + ], + "provenance": { + "toolVersion": "0.1.0-sample", + "protocolCommit": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "implementationCommit": "cccccccccccccccccccccccccccccccccccccccc", + "inputSha256": "3333333333333333333333333333333333333333333333333333333333333333", + "artifactSha256": "4444444444444444444444444444444444444444444444444444444444444444", + "runtime": "Python 3.12 · local illustrative fixture", + "deterministicReplay": true + }, + "boundaries": { + "claim": [ + "Illustrative UI fixture, not a measured adapter result.", + "A pass applies only to the supplied target and retention evaluation records." + ], + "privacy": [ + "No evaluation examples or model outputs are embedded.", + "The report contains only aggregate scores and placeholder hashes." + ] + } +} diff --git a/web/src/main.tsx b/web/src/main.tsx new file mode 100644 index 0000000..d172db6 --- /dev/null +++ b/web/src/main.tsx @@ -0,0 +1,13 @@ +import { StrictMode } from "react"; +import { createRoot } from "react-dom/client"; +import App from "./App"; +import "./styles.css"; + +const root = document.getElementById("root"); +if (!root) throw new Error("SFTGuard viewer root element is missing."); + +createRoot(root).render( + + + , +); diff --git a/web/src/reportSchema.test.ts b/web/src/reportSchema.test.ts new file mode 100644 index 0000000..fe14c31 --- /dev/null +++ b/web/src/reportSchema.test.ts @@ -0,0 +1,330 @@ +import { describe, expect, it } from "vitest"; +import auditJson from "./data/sample-audit.json"; +import faultJson from "./data/sample-fault.json"; +import regressionJson from "./data/sample-regression.json"; +import { + MAX_REPORT_BYTES, + parseReport, + parseReportText, +} from "./reportSchema"; + +const HASH_A = "a".repeat(64); +const HASH_B = "b".repeat(64); +const HASH_C = "c".repeat(64); + +const coreAuditArtifact = { + schema_version: "sftguard.audit.v1", + tool: { name: "sftguard", version: "0.1.0" }, + status: "ABSTAIN", + inputs: [ + { + split: "train", + sha256: HASH_A, + byte_count: 120, + record_count: 2, + }, + { + split: "evaluation", + sha256: HASH_B, + byte_count: 80, + record_count: 1, + }, + ], + checks: { + schema: { status: "PASS" }, + assistant_mask: { + status: "ABSTAIN", + reason: "private-record-marker", + }, + }, + summary: { + finding_count: 1, + signal_counts: { pii_like: 1 }, + error_or_critical_count: 0, + redacted_finding_count: 1, + }, + findings: [ + { + signal: "pii_like", + severity: "warning", + category: "Privacy pattern", + location: { split: "train", line: 2, field: "messages" }, + record_sha256: HASH_C, + details: { raw_match: "private-record-marker" }, + }, + ], + integrity: { algorithm: "sha256", canonical_payload_sha256: HASH_C }, +}; + +const coreRegressionArtifact = { + schema_version: "sftguard.regression-card.v1", + tool: { name: "sftguard", version: "0.1.0" }, + status: "FAIL", + inputs: [ + { + split: "base", + sha256: HASH_A, + byte_count: 400, + record_count: 8, + }, + { + split: "adapter", + sha256: HASH_B, + byte_count: 400, + record_count: 8, + }, + { split: "contract", sha256: HASH_C }, + ], + bootstrap: { + method: "paired_seed_cluster_xorshift32", + draws: 1000, + confidence: 0.95, + seed: 7, + }, + metrics: [ + { + group: "target", + metric: "answer_accuracy", + higher_is_better: true, + status: "PASS", + pair_count: 8, + seed_cluster_count: 2, + mean_normalized_improvement: 0.12, + confidence_interval: { low: 0.08, high: 0.16 }, + threshold: { min_improvement: 0.05 }, + decision_rule: "lower bound meets minimum improvement", + }, + { + group: "retention", + metric: "general_accuracy", + higher_is_better: true, + status: "FAIL", + pair_count: 8, + seed_cluster_count: 2, + mean_normalized_improvement: -0.04, + confidence_interval: { low: -0.06, high: -0.02 }, + threshold: { max_regression: 0.03 }, + decision_rule: "lower bound stays above retention floor", + }, + ], + summary: { metric_count: 2, pair_count: 8, seed_cluster_count: 2 }, + integrity: { algorithm: "sha256", canonical_payload_sha256: HASH_B }, +}; + +const developmentBenchmarkArtifact = { + schema_version: "sftguard.development-benchmark.v1", + tool: { name: "sftguard", version: "0.1.0" }, + development_only: true, + seed_range: { first: 1, last: 2, count: 2 }, + fault_count: 2, + summary: { + case_count: 6, + clean_control_count: 2, + clean_false_positive_count: 0, + clean_false_positive_rate: 0, + macro_recall: 0.75, + }, + per_fault: [ + { + fault_id: "train_eval_leakage", + expected_signal: "train_eval_leakage", + severity: "critical", + case_count: 2, + detected_count: 2, + recall: 1, + }, + { + fault_id: "missing_eos", + expected_signal: "missing_eos", + severity: "error", + case_count: 2, + detected_count: 1, + recall: 0.5, + }, + ], + cases: [{ raw_synthetic_text: "raw-case-marker" }], + gates: { + critical_fault_recall_100_percent: true, + macro_recall_at_least_90_percent: false, + clean_false_positive_rate_at_most_10_percent: true, + expected_and_observed_primary_recorded: true, + in_process_byte_reproducible: true, + raw_synthetic_text_absent: true, + }, + all_development_gates_passed: false, + status: "FAIL", + integrity: { algorithm: "sha256", canonical_payload_sha256: HASH_A }, +}; + +const confirmatoryFaultMatrixArtifact = { + schema_version: "sftguard.confirmatory-fault-matrix.v1", + tool: { name: "sftguard", version: "0.1.0" }, + provenance: { + implementation_git_commit: "1".repeat(40), + protocol_git_commit: "2".repeat(40), + python_implementation: "CPython", + python_version: "3.13", + }, + protocol: { private_fixture_note: "protocol-private-marker" }, + summary: { + case_count: 6, + clean_control_count: 2, + clean_false_positive_count: 0, + clean_false_positive_rate: 0, + fault_case_count: 4, + macro_recall: 0.75, + }, + per_fault: [ + { + fault_id: "train_eval_leakage", + expected_signal: "train_eval_leakage", + severity: "critical", + case_count: 2, + detected_count: 2, + recall: 1, + }, + { + fault_id: "missing_eos", + expected_signal: "missing_eos", + severity: "error", + case_count: 2, + detected_count: 1, + recall: 0.5, + }, + ], + cases: [{ raw_private_text: "confirmatory-case-marker" }], + gates: { + clean_false_positive_rate_at_most_10_percent: true, + critical_fault_recall_100_percent: true, + expected_and_observed_primary_recorded: true, + fresh_process_byte_reproducible: true, + macro_recall_at_least_90_percent: false, + raw_synthetic_text_absent: true, + }, + carry_forward: false, + status: "FAIL", + integrity: { algorithm: "sha256", canonical_payload_sha256: HASH_C }, +}; + +describe("SFTGuard report schema", () => { + it("loads the three checked-in status and report variants", () => { + const audit = parseReport(auditJson); + const regression = parseReport(regressionJson); + const fault = parseReport(faultJson); + + expect(audit.status).toBe("ABSTAIN"); + expect(audit.audit?.supervisedTokens).toBeNull(); + expect(regression.status).toBe("PASS"); + expect(regression.regression?.target.delta).toBe(12.4); + expect(fault.status).toBe("FAIL"); + expect(fault.faultCoverage?.detectedFaultClasses).toBe(8); + }); + + it("allow-lists aggregate fields and ignores unknown raw content", () => { + const report = parseReport({ + ...regressionJson, + rawPrompt: "this must never be rendered", + messages: [{ role: "user", content: "private" }], + }); + expect(report).not.toHaveProperty("rawPrompt"); + expect(report).not.toHaveProperty("messages"); + }); + + it("adapts the core audit without retaining record-level context", () => { + const report = parseReport(coreAuditArtifact); + const serialized = JSON.stringify(report); + + expect(report.reportType).toBe("audit"); + expect(report.status).toBe("ABSTAIN"); + expect(report.audit?.recordsScanned).toBe(3); + expect(report.audit?.checksPassed).toBe(1); + expect(report.generatedAt).toBeNull(); + expect(report.provenance.protocolCommit).toBeNull(); + expect(report.provenance.integrityStatus).toBe("unverified"); + expect(serialized).not.toContain("private-record-marker"); + expect(serialized).not.toContain("record_sha256"); + expect(report.issues[0]).not.toHaveProperty("location"); + }); + + it("adapts normalized regression metrics and preserves metric decisions", () => { + const report = parseReport(coreRegressionArtifact); + + expect(report.reportType).toBe("regression"); + expect(report.status).toBe("FAIL"); + expect(report.regression?.target.unit).toBe("normalized-delta"); + expect(report.regression?.target.delta).toBe(0.12); + expect(report.regression?.target.baseScore).toBeNull(); + expect(report.regression?.target.status).toBe("PASS"); + expect(report.regression?.retention.status).toBe("FAIL"); + expect(report.provenance.runtime).toBeNull(); + }); + + it("adapts development fault aggregates and drops the raw cases array", () => { + const report = parseReport(developmentBenchmarkArtifact); + const serialized = JSON.stringify(report); + + expect(report.reportType).toBe("fault-matrix"); + expect(report.faultCoverage?.detectedFaultClasses).toBe(1); + expect(report.faultCoverage?.criticalRecallPercent).toBe(100); + expect(report.faultCoverage?.macroRecallPercent).toBe(75); + expect(report.provenance.deterministicReplay).toBe(true); + expect(serialized).not.toContain("raw-case-marker"); + expect(report).not.toHaveProperty("cases"); + }); + + it("adapts confirmatory aggregates, commits, gates, and fresh-process replay", () => { + const report = parseReport(confirmatoryFaultMatrixArtifact); + const serialized = JSON.stringify(report); + + expect(report.title).toBe("Confirmatory fault-injection matrix"); + expect(report.status).toBe("FAIL"); + expect(report.faultCoverage?.carryForward).toBe(false); + expect(report.faultCoverage?.macroRecallPercent).toBe(75); + expect(report.provenance.protocolCommit).toBe("2".repeat(40)); + expect(report.provenance.implementationCommit).toBe("1".repeat(40)); + expect(report.provenance.runtime).toBe("CPython 3.13"); + expect(report.provenance.deterministicReplay).toBe(true); + expect(report.provenance.integrityStatus).toBe("unverified"); + expect(serialized).not.toContain("confirmatory-case-marker"); + expect(serialized).not.toContain("protocol-private-marker"); + expect(report).not.toHaveProperty("cases"); + expect(report).not.toHaveProperty("protocol"); + }); + + it("fails closed on unsupported status and malformed provenance", () => { + expect(() => + parseReport({ ...regressionJson, status: "UNKNOWN" }), + ).toThrow(/status must be one of/i); + expect(() => + parseReport({ + ...regressionJson, + provenance: { ...regressionJson.provenance, artifactSha256: "short" }, + }), + ).toThrow(/sha-256/i); + expect(() => + parseReport({ + ...regressionJson, + regression: { + ...regressionJson.regression, + target: { + ...regressionJson.regression.target, + deltaPercentagePoints: 99, + }, + }, + }), + ).toThrow(/adapterScore minus baseScore/i); + }); + + it("rejects invalid JSON and files beyond the local limit", () => { + expect(() => parseReportText("not json")).toThrow(/not valid json/i); + expect(() => parseReportText(" ".repeat(MAX_REPORT_BYTES + 1))).toThrow( + /2 MB/i, + ); + }); + + it("rejects unsupported artifact schemas", () => { + expect(() => + parseReport({ schema_version: "sftguard.unknown.v9" }), + ).toThrow(/unsupported sftguard report schema/i); + }); +}); diff --git a/web/src/reportSchema.ts b/web/src/reportSchema.ts new file mode 100644 index 0000000..d3b8862 --- /dev/null +++ b/web/src/reportSchema.ts @@ -0,0 +1,1325 @@ +/** + * SFTGuard browser render model and artifact adapters. + * + * The parser is deliberately allow-list based: it copies only documented + * aggregate fields into the render model. Unknown fields are ignored and are + * never rendered. Core artifact adapters intentionally discard record-level + * locations, hashes, details, and synthetic cases before React sees them. + */ +export const REPORT_SCHEMA_VERSION = "sftguard.report.v1" as const; +export const AUDIT_SCHEMA_VERSION = "sftguard.audit.v1" as const; +export const REGRESSION_SCHEMA_VERSION = + "sftguard.regression-card.v1" as const; +export const DEVELOPMENT_BENCHMARK_SCHEMA_VERSION = + "sftguard.development-benchmark.v1" as const; +export const CONFIRMATORY_FAULT_MATRIX_SCHEMA_VERSION = + "sftguard.confirmatory-fault-matrix.v1" as const; +export const MAX_REPORT_BYTES = 2 * 1024 * 1024; + +export type ReportStatus = "PASS" | "FAIL" | "ABSTAIN"; +export type ReportType = "audit" | "regression" | "fault-matrix"; +export type IssueSeverity = "critical" | "error" | "warning" | "info"; + +export interface ReportIssue { + id: string; + category: string; + severity: IssueSeverity; + outcome: ReportStatus; + count: number; + title: string; + detail: string; +} + +export interface AuditSummary { + recordsScanned: number; + checksPassed: number; + checksTotal: number; + supervisedTokens: number | null; +} + +export interface ScoreContract { + label: string; + status: "PASS" | "FAIL"; + baseScore: number | null; + adapterScore: number | null; + delta: number; + unit: "percentage-points" | "normalized-delta"; + pairCount?: number; + confidenceInterval?: { low: number; high: number }; +} + +export interface TargetScoreContract extends ScoreContract { + minimumGain: number; +} + +export interface RetentionScoreContract extends ScoreContract { + maximumRegression: number; +} + +export interface RegressionSummary { + target: TargetScoreContract; + retention: RetentionScoreContract; +} + +export interface FaultCoverage { + expectedFaultClasses: number; + detectedFaultClasses: number; + criticalRecallPercent: number; + macroRecallPercent: number; + cleanFalsePositiveRatePercent: number; + carryForward: boolean; +} + +export interface ReportProvenance { + toolVersion: string; + protocolCommit: string | null; + implementationCommit: string | null; + inputSha256: string | null; + artifactSha256: string; + integrityStatus: "verified" | "mismatch" | "unverified"; + runtime: string | null; + deterministicReplay: boolean | null; +} + +export interface ReportBoundaries { + claim: string[]; + privacy: string[]; +} + +export interface SftGuardReport { + schemaVersion: typeof REPORT_SCHEMA_VERSION; + reportType: ReportType; + reportId: string; + title: string; + generatedAt: string | null; + status: ReportStatus; + summary: string; + audit?: AuditSummary; + regression?: RegressionSummary; + faultCoverage?: FaultCoverage; + issues: ReportIssue[]; + provenance: ReportProvenance; + boundaries: ReportBoundaries; +} + +function asRecord(value: unknown, label: string): Record { + if (typeof value !== "object" || value === null || Array.isArray(value)) { + throw new TypeError(`${label} must be an object.`); + } + return value as Record; +} + +function asString( + value: unknown, + label: string, + maximumLength = 500, +): string { + if (typeof value !== "string" || value.trim().length === 0) { + throw new TypeError(`${label} must be a non-empty string.`); + } + if (value.length > maximumLength) { + throw new RangeError(`${label} is longer than ${maximumLength} characters.`); + } + return value; +} + +function asFiniteNumber(value: unknown, label: string): number { + if (typeof value !== "number" || !Number.isFinite(value)) { + throw new TypeError(`${label} must be a finite number.`); + } + return value; +} + +function asNonNegativeInteger(value: unknown, label: string): number { + const number = asFiniteNumber(value, label); + if (!Number.isSafeInteger(number) || number < 0) { + throw new RangeError(`${label} must be a non-negative safe integer.`); + } + return number; +} + +function asBoolean(value: unknown, label: string): boolean { + if (typeof value !== "boolean") { + throw new TypeError(`${label} must be true or false.`); + } + return value; +} + +function asEnum( + value: unknown, + label: string, + choices: readonly T[], +): T { + if (typeof value !== "string" || !choices.includes(value as T)) { + throw new RangeError(`${label} must be one of ${choices.join(", ")}.`); + } + return value as T; +} + +function asSha256(value: unknown, label: string): string { + const hash = asString(value, label, 64); + if (!/^[0-9a-f]{64}$/.test(hash)) { + throw new RangeError(`${label} must be a lowercase SHA-256 digest.`); + } + return hash; +} + +function asCommit(value: unknown, label: string): string { + const commit = asString(value, label, 40); + if (!/^[0-9a-f]{40}$/.test(commit)) { + throw new RangeError(`${label} must be a full lowercase Git commit.`); + } + return commit; +} + +function asStringArray(value: unknown, label: string): string[] { + if (!Array.isArray(value) || value.length === 0 || value.length > 20) { + throw new RangeError(`${label} must contain between 1 and 20 entries.`); + } + return value.map((entry, index) => + asString(entry, `${label}[${index}]`, 300), + ); +} + +function parseIssue(value: unknown, index: number): ReportIssue { + const issue = asRecord(value, `issues[${index}]`); + return { + id: asString(issue.id, `issues[${index}].id`, 120), + category: asString(issue.category, `issues[${index}].category`, 80), + severity: asEnum(issue.severity, `issues[${index}].severity`, [ + "critical", + "error", + "warning", + "info", + ] as const), + outcome: asEnum(issue.outcome, `issues[${index}].outcome`, [ + "PASS", + "FAIL", + "ABSTAIN", + ] as const), + count: asNonNegativeInteger(issue.count, `issues[${index}].count`), + title: asString(issue.title, `issues[${index}].title`, 140), + detail: asString(issue.detail, `issues[${index}].detail`, 500), + }; +} + +function parseAudit(value: unknown): AuditSummary { + const audit = asRecord(value, "audit"); + const supervisedTokens = + audit.supervisedTokens === null + ? null + : asNonNegativeInteger(audit.supervisedTokens, "audit.supervisedTokens"); + const parsed = { + recordsScanned: asNonNegativeInteger( + audit.recordsScanned, + "audit.recordsScanned", + ), + checksPassed: asNonNegativeInteger( + audit.checksPassed, + "audit.checksPassed", + ), + checksTotal: asNonNegativeInteger(audit.checksTotal, "audit.checksTotal"), + supervisedTokens, + }; + if (parsed.checksPassed > parsed.checksTotal) { + throw new RangeError("audit.checksPassed cannot exceed checksTotal."); + } + return parsed; +} + +function parseScore( + value: unknown, + label: string, +): Omit { + const score = asRecord(value, label); + const parsed = { + label: asString(score.label, `${label}.label`, 100), + baseScore: asFiniteNumber(score.baseScore, `${label}.baseScore`), + adapterScore: asFiniteNumber(score.adapterScore, `${label}.adapterScore`), + delta: asFiniteNumber( + score.deltaPercentagePoints, + `${label}.deltaPercentagePoints`, + ), + unit: "percentage-points" as const, + }; + const calculatedDelta = parsed.adapterScore - parsed.baseScore; + if (Math.abs(calculatedDelta - parsed.delta) > 1e-9) { + throw new RangeError( + `${label}.deltaPercentagePoints must equal adapterScore minus baseScore.`, + ); + } + return parsed; +} + +function parseRegression(value: unknown): RegressionSummary { + const regression = asRecord(value, "regression"); + const target = asRecord(regression.target, "regression.target"); + const retention = asRecord(regression.retention, "regression.retention"); + const targetScore = parseScore(target, "regression.target"); + const retentionScore = parseScore(retention, "regression.retention"); + const minimumGain = asFiniteNumber( + target.minimumGainPercentagePoints, + "regression.target.minimumGainPercentagePoints", + ); + const maximumRegression = asFiniteNumber( + retention.maximumRegressionPercentagePoints, + "regression.retention.maximumRegressionPercentagePoints", + ); + return { + target: { + ...targetScore, + status: targetScore.delta >= minimumGain ? "PASS" : "FAIL", + minimumGain, + }, + retention: { + ...retentionScore, + status: + retentionScore.delta >= -maximumRegression ? "PASS" : "FAIL", + maximumRegression, + }, + }; +} + +function asPercent(value: unknown, label: string): number { + const percent = asFiniteNumber(value, label); + if (percent < 0 || percent > 100) { + throw new RangeError(`${label} must be between 0 and 100.`); + } + return percent; +} + +function parseFaultCoverage(value: unknown): FaultCoverage { + const coverage = asRecord(value, "faultCoverage"); + const expected = asNonNegativeInteger( + coverage.expectedFaultClasses, + "faultCoverage.expectedFaultClasses", + ); + const detected = asNonNegativeInteger( + coverage.detectedFaultClasses, + "faultCoverage.detectedFaultClasses", + ); + if (detected > expected) { + throw new RangeError( + "faultCoverage.detectedFaultClasses cannot exceed expectedFaultClasses.", + ); + } + return { + expectedFaultClasses: expected, + detectedFaultClasses: detected, + criticalRecallPercent: asPercent( + coverage.criticalRecallPercent, + "faultCoverage.criticalRecallPercent", + ), + macroRecallPercent: asPercent( + coverage.macroRecallPercent, + "faultCoverage.macroRecallPercent", + ), + cleanFalsePositiveRatePercent: asPercent( + coverage.cleanFalsePositiveRatePercent, + "faultCoverage.cleanFalsePositiveRatePercent", + ), + carryForward: asBoolean( + coverage.carryForward, + "faultCoverage.carryForward", + ), + }; +} + +function parseProvenance(value: unknown): ReportProvenance { + const provenance = asRecord(value, "provenance"); + return { + toolVersion: asString(provenance.toolVersion, "provenance.toolVersion", 80), + protocolCommit: asCommit( + provenance.protocolCommit, + "provenance.protocolCommit", + ), + implementationCommit: asCommit( + provenance.implementationCommit, + "provenance.implementationCommit", + ), + inputSha256: asSha256(provenance.inputSha256, "provenance.inputSha256"), + artifactSha256: asSha256( + provenance.artifactSha256, + "provenance.artifactSha256", + ), + integrityStatus: "unverified", + runtime: asString(provenance.runtime, "provenance.runtime", 160), + deterministicReplay: asBoolean( + provenance.deterministicReplay, + "provenance.deterministicReplay", + ), + }; +} + +function parseEnvelopeReport(value: unknown): SftGuardReport { + const report = asRecord(value, "report"); + const schemaVersion = asEnum(report.schemaVersion, "schemaVersion", [ + REPORT_SCHEMA_VERSION, + ] as const); + const reportType = asEnum(report.reportType, "reportType", [ + "audit", + "regression", + "fault-matrix", + ] as const); + const generatedAt = asString(report.generatedAt, "generatedAt", 80); + if (!Number.isFinite(Date.parse(generatedAt))) { + throw new RangeError("generatedAt must be an ISO-compatible date string."); + } + if (!Array.isArray(report.issues) || report.issues.length > 100) { + throw new RangeError("issues must be an array with at most 100 entries."); + } + const boundaries = asRecord(report.boundaries, "boundaries"); + + const parsed: SftGuardReport = { + schemaVersion, + reportType, + reportId: asString(report.reportId, "reportId", 120), + title: asString(report.title, "title", 160), + generatedAt, + status: asEnum(report.status, "status", [ + "PASS", + "FAIL", + "ABSTAIN", + ] as const), + summary: asString(report.summary, "summary", 800), + issues: report.issues.map(parseIssue), + provenance: parseProvenance(report.provenance), + boundaries: { + claim: asStringArray(boundaries.claim, "boundaries.claim"), + privacy: asStringArray(boundaries.privacy, "boundaries.privacy"), + }, + }; + + if (reportType === "audit") parsed.audit = parseAudit(report.audit); + if (reportType === "regression") { + parsed.regression = parseRegression(report.regression); + } + if (reportType === "fault-matrix") { + parsed.faultCoverage = parseFaultCoverage(report.faultCoverage); + } + return parsed; +} + +const REPORT_STATUSES = ["PASS", "FAIL", "ABSTAIN"] as const; +const ISSUE_SEVERITIES = ["critical", "error", "warning", "info"] as const; + +function asBoundedArray( + value: unknown, + label: string, + maximumLength: number, +): unknown[] { + if (!Array.isArray(value) || value.length > maximumLength) { + throw new RangeError( + `${label} must be an array with at most ${maximumLength} entries.`, + ); + } + return value; +} + +function asFraction(value: unknown, label: string): number { + const fraction = asFiniteNumber(value, label); + if (fraction < 0 || fraction > 1) { + throw new RangeError(`${label} must be between 0 and 1.`); + } + return fraction; +} + +function asNonNegativeNumber(value: unknown, label: string): number { + const number = asFiniteNumber(value, label); + if (number < 0) { + throw new RangeError(`${label} must be non-negative.`); + } + return number; +} + +function humanizeIdentifier(value: string): string { + const words = value.replace(/[_-]+/g, " ").replace(/\s+/g, " ").trim(); + return `${words.slice(0, 1).toUpperCase()}${words.slice(1)}`; +} + +function formatDecimal(value: number): string { + const normalized = Object.is(value, -0) ? 0 : value; + return normalized.toFixed(4).replace(/\.0+$|(?<=\.[0-9]*?)0+$/g, ""); +} + +function parseCoreMetadata( + report: Record, + label: string, + deterministicReplay: boolean | null = null, +): ReportProvenance { + const tool = asRecord(report.tool, `${label}.tool`); + if (asString(tool.name, `${label}.tool.name`, 80) !== "sftguard") { + throw new RangeError(`${label}.tool.name must be sftguard.`); + } + const integrity = asRecord(report.integrity, `${label}.integrity`); + if ( + asString(integrity.algorithm, `${label}.integrity.algorithm`, 20) !== + "sha256" + ) { + throw new RangeError(`${label}.integrity.algorithm must be sha256.`); + } + return { + toolVersion: asString(tool.version, `${label}.tool.version`, 80), + protocolCommit: null, + implementationCommit: null, + inputSha256: null, + artifactSha256: asSha256( + integrity.canonical_payload_sha256, + `${label}.integrity.canonical_payload_sha256`, + ), + integrityStatus: "unverified", + runtime: null, + deterministicReplay, + }; +} + +function validateCoreInputs( + value: unknown, + label: string, + requireRecordCount = false, +): number { + const inputs = asBoundedArray(value, label, 100); + let records = 0; + inputs.forEach((entry, index) => { + const input = asRecord(entry, `${label}[${index}]`); + asString(input.split, `${label}[${index}].split`, 80); + asSha256(input.sha256, `${label}[${index}].sha256`); + if (input.byte_count !== undefined) { + asNonNegativeInteger(input.byte_count, `${label}[${index}].byte_count`); + } + if (input.record_count === undefined) { + if (requireRecordCount) { + throw new TypeError(`${label}[${index}].record_count is required.`); + } + } else { + records += asNonNegativeInteger( + input.record_count, + `${label}[${index}].record_count`, + ); + } + }); + return records; +} + +function parseAuditArtifact(report: Record): SftGuardReport { + const label = "audit artifact"; + const status = asEnum(report.status, `${label}.status`, REPORT_STATUSES); + const provenance = parseCoreMetadata(report, label); + const recordsScanned = validateCoreInputs(report.inputs, `${label}.inputs`, true); + + const checks = asRecord(report.checks, `${label}.checks`); + const checkEntries = Object.entries(checks); + if (checkEntries.length === 0 || checkEntries.length > 100) { + throw new RangeError(`${label}.checks must contain between 1 and 100 entries.`); + } + const parsedChecks = checkEntries.map(([name, value]) => { + asString(name, `${label}.checks key`, 100); + const check = asRecord(value, `${label}.checks.${name}`); + return { + name, + status: asEnum( + check.status, + `${label}.checks.${name}.status`, + REPORT_STATUSES, + ), + }; + }); + + const findings = asBoundedArray(report.findings, `${label}.findings`, 100_000); + type AuditAggregate = { + signal: string; + category: string; + severity: IssueSeverity; + count: number; + }; + const aggregates = new Map(); + const derivedSignalCounts = new Map(); + findings.forEach((entry, index) => { + const finding = asRecord(entry, `${label}.findings[${index}]`); + const signal = asString( + finding.signal, + `${label}.findings[${index}].signal`, + 120, + ); + const severity = asEnum( + finding.severity, + `${label}.findings[${index}].severity`, + ISSUE_SEVERITIES, + ); + const location = asRecord( + finding.location, + `${label}.findings[${index}].location`, + ); + asEnum( + location.split, + `${label}.findings[${index}].location.split`, + ["train", "evaluation"] as const, + ); + const line = asNonNegativeInteger( + location.line, + `${label}.findings[${index}].location.line`, + ); + if (line < 1) { + throw new RangeError( + `${label}.findings[${index}].location.line must be at least 1.`, + ); + } + if (location.field !== undefined) { + asString( + location.field, + `${label}.findings[${index}].location.field`, + 160, + ); + } + const category = + finding.category === undefined + ? "Audit finding" + : asString( + finding.category, + `${label}.findings[${index}].category`, + 80, + ); + const key = `${category}\u0000${signal}\u0000${severity}`; + const aggregate = aggregates.get(key); + if (aggregate) { + aggregate.count += 1; + } else { + if (aggregates.size >= 100) { + throw new RangeError( + `${label}.findings contain more than 100 aggregate categories.`, + ); + } + aggregates.set(key, { signal, category, severity, count: 1 }); + } + derivedSignalCounts.set( + signal, + (derivedSignalCounts.get(signal) ?? 0) + 1, + ); + }); + + const summary = asRecord(report.summary, `${label}.summary`); + const findingCount = asNonNegativeInteger( + summary.finding_count, + `${label}.summary.finding_count`, + ); + if (findingCount !== findings.length) { + throw new RangeError( + `${label}.summary.finding_count must match findings length.`, + ); + } + const signalCounts = asRecord( + summary.signal_counts, + `${label}.summary.signal_counts`, + ); + if (Object.keys(signalCounts).length > 100) { + throw new RangeError(`${label}.summary.signal_counts has too many entries.`); + } + for (const [signal, count] of Object.entries(signalCounts)) { + const parsedCount = asNonNegativeInteger( + count, + `${label}.summary.signal_counts.${signal}`, + ); + if (derivedSignalCounts.get(signal) !== parsedCount) { + throw new RangeError( + `${label}.summary.signal_counts must match aggregate findings.`, + ); + } + } + if (Object.keys(signalCounts).length !== derivedSignalCounts.size) { + throw new RangeError( + `${label}.summary.signal_counts must cover every finding signal.`, + ); + } + + const issues: ReportIssue[] = [...aggregates.values()].map( + (aggregate, index) => ({ + id: `audit-signal-${index + 1}`, + category: aggregate.category, + severity: aggregate.severity, + outcome: + status === "ABSTAIN" + ? "ABSTAIN" + : aggregate.severity === "critical" || aggregate.severity === "error" + ? "FAIL" + : "PASS", + count: aggregate.count, + title: humanizeIdentifier(aggregate.signal), + detail: `${aggregate.count} aggregate finding${aggregate.count === 1 ? "" : "s"} reported. Record locations and record-level details are not copied into this viewer.`, + }), + ); + for (const check of parsedChecks.filter((entry) => entry.status === "ABSTAIN")) { + issues.push({ + id: `audit-check-${check.name}`, + category: "Audit check", + severity: "warning", + outcome: "ABSTAIN", + count: 1, + title: humanizeIdentifier(check.name), + detail: + "This check abstained. Its raw reason and record context remain outside the aggregate render model.", + }); + } + + return { + schemaVersion: REPORT_SCHEMA_VERSION, + reportType: "audit", + reportId: `core-audit-${provenance.artifactSha256.slice(0, 12)}`, + title: "Dataset and mask audit", + generatedAt: null, + status, + summary: `The audit declared ${status} across ${recordsScanned} record${recordsScanned === 1 ? "" : "s"}, ${findingCount} finding${findingCount === 1 ? "" : "s"}, and ${parsedChecks.length} check${parsedChecks.length === 1 ? "" : "s"}.`, + audit: { + recordsScanned, + checksPassed: parsedChecks.filter((entry) => entry.status === "PASS").length, + checksTotal: parsedChecks.length, + supervisedTokens: null, + }, + issues, + provenance, + boundaries: { + claim: [ + "The decision reflects the supplied local audit artifact only.", + "A passing audit does not prove dataset quality, model quality, or model safety.", + ], + privacy: [ + "Only aggregate signals, severities, and counts are rendered.", + "Record locations, record hashes, finding details, and source records are discarded by the adapter.", + ], + }, + }; +} + +interface ParsedCoreMetric { + group: "target" | "retention"; + metric: string; + status: "PASS" | "FAIL"; + pairCount: number; + mean: number; + interval: { low: number; high: number }; + threshold: number; +} + +function parseCoreMetric(value: unknown, index: number): ParsedCoreMetric { + const label = `regression artifact.metrics[${index}]`; + const metric = asRecord(value, label); + const group = asEnum(metric.group, `${label}.group`, [ + "target", + "retention", + ] as const); + if (metric.higher_is_better !== undefined) { + asBoolean(metric.higher_is_better, `${label}.higher_is_better`); + } + if (metric.seed_cluster_count !== undefined) { + asNonNegativeInteger( + metric.seed_cluster_count, + `${label}.seed_cluster_count`, + ); + } + const interval = asRecord( + metric.confidence_interval, + `${label}.confidence_interval`, + ); + const low = asFiniteNumber(interval.low, `${label}.confidence_interval.low`); + const high = asFiniteNumber( + interval.high, + `${label}.confidence_interval.high`, + ); + if (low > high) { + throw new RangeError(`${label}.confidence_interval.low cannot exceed high.`); + } + const threshold = asRecord(metric.threshold, `${label}.threshold`); + const thresholdKey = + group === "target" ? "min_improvement" : "max_regression"; + asString(metric.decision_rule, `${label}.decision_rule`, 300); + return { + group, + metric: asString(metric.metric, `${label}.metric`, 120), + status: asEnum(metric.status, `${label}.status`, ["PASS", "FAIL"] as const), + pairCount: asNonNegativeInteger(metric.pair_count, `${label}.pair_count`), + mean: asFiniteNumber( + metric.mean_normalized_improvement, + `${label}.mean_normalized_improvement`, + ), + interval: { low, high }, + threshold: asNonNegativeNumber( + threshold[thresholdKey], + `${label}.threshold.${thresholdKey}`, + ), + }; +} + +function parseRegressionArtifact( + report: Record, +): SftGuardReport { + const label = "regression artifact"; + const status = asEnum(report.status, `${label}.status`, REPORT_STATUSES); + const provenance = parseCoreMetadata(report, label); + validateCoreInputs(report.inputs, `${label}.inputs`); + const rawMetrics = asBoundedArray(report.metrics, `${label}.metrics`, 100); + const metrics = rawMetrics + .map(parseCoreMetric) + .sort((left, right) => + `${left.group}:${left.metric}`.localeCompare(`${right.group}:${right.metric}`), + ); + const summary = asRecord(report.summary, `${label}.summary`); + const metricCount = asNonNegativeInteger( + summary.metric_count, + `${label}.summary.metric_count`, + ); + if (metricCount !== metrics.length) { + throw new RangeError(`${label}.summary.metric_count must match metrics length.`); + } + asNonNegativeInteger(summary.pair_count, `${label}.summary.pair_count`); + asNonNegativeInteger( + summary.seed_cluster_count, + `${label}.summary.seed_cluster_count`, + ); + + const reasonCodes = + report.reason_codes === undefined + ? [] + : asBoundedArray(report.reason_codes, `${label}.reason_codes`, 100).map( + (reason, index) => + asString(reason, `${label}.reason_codes[${index}]`, 160), + ); + const issues: ReportIssue[] = metrics.map((metric, index) => ({ + id: `regression-metric-${index + 1}`, + category: + metric.group === "target" ? "Target behavior" : "Retention behavior", + severity: metric.status === "FAIL" ? "error" : "info", + outcome: metric.status, + count: 1, + title: metric.metric, + detail: `Mean normalized improvement ${formatDecimal(metric.mean)} with reported interval [${formatDecimal(metric.interval.low)}, ${formatDecimal(metric.interval.high)}] across ${metric.pairCount} pairs; ${metric.group === "target" ? "minimum improvement" : "maximum regression"} ${formatDecimal(metric.threshold)}.`, + })); + reasonCodes.forEach((reason, index) => { + issues.push({ + id: `regression-reason-${index + 1}`, + category: "Decision evidence", + severity: "warning", + outcome: "ABSTAIN", + count: 1, + title: humanizeIdentifier(reason), + detail: + "The regression card reported this aggregate reason code without exposing evaluation examples.", + }); + }); + if (status === "ABSTAIN" && reasonCodes.length === 0) { + issues.push({ + id: "regression-reason-unavailable", + category: "Decision evidence", + severity: "warning", + outcome: "ABSTAIN", + count: 1, + title: "Reason unavailable", + detail: "The artifact abstained without a reason code in its aggregate fields.", + }); + } + + const target = metrics.find((metric) => metric.group === "target"); + const retention = metrics.find((metric) => metric.group === "retention"); + const adapted: SftGuardReport = { + schemaVersion: REPORT_SCHEMA_VERSION, + reportType: "regression", + reportId: `core-regression-${provenance.artifactSha256.slice(0, 12)}`, + title: "Target and retention regression card", + generatedAt: null, + status, + summary: `The regression card declared ${status} across ${metricCount} aggregate metric${metricCount === 1 ? "" : "s"}. Estimates are normalized paired improvements, not raw percentage-point scores.`, + issues, + provenance, + boundaries: { + claim: [ + "The decision applies only to the supplied metric contract and paired evaluation artifact.", + "Passing declared gates does not establish broad model quality or safety.", + ], + privacy: [ + "Only aggregate estimates, intervals, thresholds, pair counts, and reason codes are rendered.", + "Evaluation rows, prompts, responses, and per-example results are not copied into the viewer.", + ], + }, + }; + if (target && retention) { + adapted.regression = { + target: { + label: target.metric, + status: target.status, + baseScore: null, + adapterScore: null, + delta: target.mean, + unit: "normalized-delta", + pairCount: target.pairCount, + confidenceInterval: target.interval, + minimumGain: target.threshold, + }, + retention: { + label: retention.metric, + status: retention.status, + baseScore: null, + adapterScore: null, + delta: retention.mean, + unit: "normalized-delta", + pairCount: retention.pairCount, + confidenceInterval: retention.interval, + maximumRegression: retention.threshold, + }, + }; + } + return adapted; +} + +interface ParsedFaultClass { + faultId: string; + expectedSignal: string; + severity: IssueSeverity; + caseCount: number; + detectedCount: number; + recall: number; +} + +function parseFaultClasses(value: unknown, label: string): ParsedFaultClass[] { + return asBoundedArray(value, label, 90).map((entry, index) => { + const faultLabel = `${label}[${index}]`; + const fault = asRecord(entry, faultLabel); + const caseCount = asNonNegativeInteger( + fault.case_count, + `${faultLabel}.case_count`, + ); + if (caseCount < 1) { + throw new RangeError(`${faultLabel}.case_count must be at least 1.`); + } + const detectedCount = asNonNegativeInteger( + fault.detected_count, + `${faultLabel}.detected_count`, + ); + if (detectedCount > caseCount) { + throw new RangeError( + `${faultLabel}.detected_count cannot exceed case_count.`, + ); + } + const recall = asFraction(fault.recall, `${faultLabel}.recall`); + if (Math.abs(recall - detectedCount / caseCount) > 1e-9) { + throw new RangeError(`${faultLabel}.recall must match aggregate counts.`); + } + return { + faultId: asString(fault.fault_id, `${faultLabel}.fault_id`, 120), + expectedSignal: asString( + fault.expected_signal, + `${faultLabel}.expected_signal`, + 120, + ), + severity: asEnum( + fault.severity, + `${faultLabel}.severity`, + ISSUE_SEVERITIES, + ), + caseCount, + detectedCount, + recall, + }; + }); +} + +function parseDevelopmentBenchmarkArtifact( + report: Record, +): SftGuardReport { + const label = "development benchmark artifact"; + if (asBoolean(report.development_only, `${label}.development_only`) !== true) { + throw new RangeError(`${label}.development_only must be true.`); + } + const status = asEnum(report.status, `${label}.status`, ["PASS", "FAIL"] as const); + const gates = asRecord(report.gates, `${label}.gates`); + const gateKeys = [ + "critical_fault_recall_100_percent", + "macro_recall_at_least_90_percent", + "clean_false_positive_rate_at_most_10_percent", + "expected_and_observed_primary_recorded", + "in_process_byte_reproducible", + "raw_synthetic_text_absent", + ] as const; + const parsedGates = new Map( + gateKeys.map((key) => [ + key, + asBoolean(gates[key], `${label}.gates.${key}`), + ]), + ); + const allGatesPassed = asBoolean( + report.all_development_gates_passed, + `${label}.all_development_gates_passed`, + ); + if (allGatesPassed !== [...parsedGates.values()].every(Boolean)) { + throw new RangeError( + `${label}.all_development_gates_passed must match the declared gates.`, + ); + } + if ((status === "PASS") !== allGatesPassed) { + throw new RangeError(`${label}.status must match the development gates.`); + } + const provenance = parseCoreMetadata( + report, + label, + parsedGates.get("in_process_byte_reproducible") ?? null, + ); + + const seedRange = asRecord(report.seed_range, `${label}.seed_range`); + asNonNegativeInteger(seedRange.first, `${label}.seed_range.first`); + asNonNegativeInteger(seedRange.last, `${label}.seed_range.last`); + const seedCount = asNonNegativeInteger( + seedRange.count, + `${label}.seed_range.count`, + ); + if (seedCount < 1) { + throw new RangeError(`${label}.seed_range.count must be at least 1.`); + } + + const faults = parseFaultClasses(report.per_fault, `${label}.per_fault`); + const faultCount = asNonNegativeInteger( + report.fault_count, + `${label}.fault_count`, + ); + if (faultCount !== faults.length) { + throw new RangeError(`${label}.fault_count must match per_fault length.`); + } + + const summary = asRecord(report.summary, `${label}.summary`); + const caseCount = asNonNegativeInteger( + summary.case_count, + `${label}.summary.case_count`, + ); + const cleanControlCount = asNonNegativeInteger( + summary.clean_control_count, + `${label}.summary.clean_control_count`, + ); + const cleanFalsePositiveCount = asNonNegativeInteger( + summary.clean_false_positive_count, + `${label}.summary.clean_false_positive_count`, + ); + if (cleanFalsePositiveCount > cleanControlCount) { + throw new RangeError( + `${label}.summary.clean_false_positive_count cannot exceed clean_control_count.`, + ); + } + if ( + caseCount !== + cleanControlCount + faults.reduce((total, fault) => total + fault.caseCount, 0) + ) { + throw new RangeError( + `${label}.summary.case_count must match clean and injected aggregate counts.`, + ); + } + const cleanFalsePositiveRate = asFraction( + summary.clean_false_positive_rate, + `${label}.summary.clean_false_positive_rate`, + ); + if ( + cleanControlCount > 0 && + Math.abs(cleanFalsePositiveRate - cleanFalsePositiveCount / cleanControlCount) > + 1e-9 + ) { + throw new RangeError( + `${label}.summary.clean_false_positive_rate must match aggregate counts.`, + ); + } + const macroRecall = asFraction( + summary.macro_recall, + `${label}.summary.macro_recall`, + ); + const calculatedMacro = + faults.length === 0 + ? 0 + : faults.reduce((total, fault) => total + fault.recall, 0) / faults.length; + if (Math.abs(macroRecall - calculatedMacro) > 1e-9) { + throw new RangeError(`${label}.summary.macro_recall must match per_fault recall.`); + } + + const criticalFaults = faults.filter((fault) => fault.severity === "critical"); + const criticalRecall = + criticalFaults.length === 0 + ? 0 + : Math.min(...criticalFaults.map((fault) => fault.recall)); + const issues: ReportIssue[] = faults.map((fault, index) => ({ + id: `fault-class-${index + 1}`, + category: "Fault detection", + severity: fault.severity, + outcome: fault.detectedCount === fault.caseCount ? "PASS" : "FAIL", + count: fault.caseCount, + title: humanizeIdentifier(fault.faultId), + detail: `Expected signal ${fault.expectedSignal}; detected ${fault.detectedCount} of ${fault.caseCount} aggregate synthetic cases (${(fault.recall * 100).toFixed(1)}% recall).`, + })); + for (const [key, passed] of parsedGates) { + if (!passed) { + issues.push({ + id: `development-gate-${key}`, + category: "Development gate", + severity: key === "critical_fault_recall_100_percent" ? "critical" : "error", + outcome: "FAIL", + count: 1, + title: humanizeIdentifier(key), + detail: "This aggregate development-only gate was reported as false.", + }); + } + } + + return { + schemaVersion: REPORT_SCHEMA_VERSION, + reportType: "fault-matrix", + reportId: `core-benchmark-${provenance.artifactSha256.slice(0, 12)}`, + title: "Development fault-injection matrix", + generatedAt: null, + status, + summary: `The development-only benchmark declared ${status} across ${caseCount} synthetic cases and ${faultCount} known fault classes.`, + faultCoverage: { + expectedFaultClasses: faultCount, + detectedFaultClasses: faults.filter( + (fault) => fault.detectedCount === fault.caseCount, + ).length, + criticalRecallPercent: criticalRecall * 100, + macroRecallPercent: macroRecall * 100, + cleanFalsePositiveRatePercent: cleanFalsePositiveRate * 100, + carryForward: allGatesPassed, + }, + issues, + provenance, + boundaries: { + claim: [ + "This is a development-only synthetic benchmark of known SFTGuard checks.", + "It is not a confirmatory benchmark and does not demonstrate coverage of unknown real-world faults.", + ], + privacy: [ + "Only per-fault aggregate counts, recall, and gate results are rendered.", + "The cases array and any raw synthetic case content are discarded by the adapter.", + ], + }, + }; +} + +function parseConfirmatoryFaultMatrixArtifact( + report: Record, +): SftGuardReport { + const label = "confirmatory fault-matrix artifact"; + const status = asEnum(report.status, `${label}.status`, ["PASS", "FAIL"] as const); + asRecord(report.protocol, `${label}.protocol`); + asBoundedArray(report.cases, `${label}.cases`, 100_000); + + const gates = asRecord(report.gates, `${label}.gates`); + const gateKeys = [ + "clean_false_positive_rate_at_most_10_percent", + "critical_fault_recall_100_percent", + "expected_and_observed_primary_recorded", + "fresh_process_byte_reproducible", + "macro_recall_at_least_90_percent", + "raw_synthetic_text_absent", + ] as const; + const parsedGates = new Map( + gateKeys.map((key) => [ + key, + asBoolean(gates[key], `${label}.gates.${key}`), + ]), + ); + const carryForward = asBoolean( + report.carry_forward, + `${label}.carry_forward`, + ); + if (carryForward !== [...parsedGates.values()].every(Boolean)) { + throw new RangeError(`${label}.carry_forward must match the declared gates.`); + } + if ((status === "PASS") !== carryForward) { + throw new RangeError(`${label}.status must match carry_forward.`); + } + + const artifactProvenance = asRecord( + report.provenance, + `${label}.provenance`, + ); + const pythonImplementation = asString( + artifactProvenance.python_implementation, + `${label}.provenance.python_implementation`, + 80, + ); + const pythonVersion = asString( + artifactProvenance.python_version, + `${label}.provenance.python_version`, + 40, + ); + const baseProvenance = parseCoreMetadata( + report, + label, + parsedGates.get("fresh_process_byte_reproducible") ?? null, + ); + const provenance: ReportProvenance = { + ...baseProvenance, + protocolCommit: asCommit( + artifactProvenance.protocol_git_commit, + `${label}.provenance.protocol_git_commit`, + ), + implementationCommit: asCommit( + artifactProvenance.implementation_git_commit, + `${label}.provenance.implementation_git_commit`, + ), + runtime: `${pythonImplementation} ${pythonVersion}`, + }; + + const faults = parseFaultClasses(report.per_fault, `${label}.per_fault`); + if (faults.length === 0) { + throw new RangeError(`${label}.per_fault must contain at least one entry.`); + } + const summary = asRecord(report.summary, `${label}.summary`); + const caseCount = asNonNegativeInteger( + summary.case_count, + `${label}.summary.case_count`, + ); + const cleanControlCount = asNonNegativeInteger( + summary.clean_control_count, + `${label}.summary.clean_control_count`, + ); + const faultCaseCount = asNonNegativeInteger( + summary.fault_case_count, + `${label}.summary.fault_case_count`, + ); + const cleanFalsePositiveCount = asNonNegativeInteger( + summary.clean_false_positive_count, + `${label}.summary.clean_false_positive_count`, + ); + if (cleanFalsePositiveCount > cleanControlCount) { + throw new RangeError( + `${label}.summary.clean_false_positive_count cannot exceed clean_control_count.`, + ); + } + if (caseCount !== cleanControlCount + faultCaseCount) { + throw new RangeError( + `${label}.summary.case_count must equal clean and fault case counts.`, + ); + } + if ( + faultCaseCount !== + faults.reduce((total, fault) => total + fault.caseCount, 0) + ) { + throw new RangeError( + `${label}.summary.fault_case_count must match per_fault aggregate counts.`, + ); + } + const cleanFalsePositiveRate = asFraction( + summary.clean_false_positive_rate, + `${label}.summary.clean_false_positive_rate`, + ); + if ( + cleanControlCount > 0 && + Math.abs(cleanFalsePositiveRate - cleanFalsePositiveCount / cleanControlCount) > + 1e-9 + ) { + throw new RangeError( + `${label}.summary.clean_false_positive_rate must match aggregate counts.`, + ); + } + const macroRecall = asFraction( + summary.macro_recall, + `${label}.summary.macro_recall`, + ); + const calculatedMacro = + faults.reduce((total, fault) => total + fault.recall, 0) / faults.length; + if (Math.abs(macroRecall - calculatedMacro) > 1e-9) { + throw new RangeError(`${label}.summary.macro_recall must match per_fault recall.`); + } + + const criticalFaults = faults.filter((fault) => fault.severity === "critical"); + const criticalRecall = + criticalFaults.length === 0 + ? 0 + : Math.min(...criticalFaults.map((fault) => fault.recall)); + const issues: ReportIssue[] = faults.map((fault, index) => ({ + id: `confirmatory-fault-class-${index + 1}`, + category: "Confirmatory fault detection", + severity: fault.severity, + outcome: fault.detectedCount === fault.caseCount ? "PASS" : "FAIL", + count: fault.caseCount, + title: humanizeIdentifier(fault.faultId), + detail: `Expected signal ${fault.expectedSignal}; detected ${fault.detectedCount} of ${fault.caseCount} aggregate confirmatory cases (${(fault.recall * 100).toFixed(1)}% recall).`, + })); + for (const [key, passed] of parsedGates) { + if (!passed) { + issues.push({ + id: `confirmatory-gate-${key}`, + category: "Confirmatory gate", + severity: key === "critical_fault_recall_100_percent" ? "critical" : "error", + outcome: "FAIL", + count: 1, + title: humanizeIdentifier(key), + detail: "This aggregate confirmatory gate was reported as false.", + }); + } + } + + return { + schemaVersion: REPORT_SCHEMA_VERSION, + reportType: "fault-matrix", + reportId: `confirmatory-matrix-${provenance.artifactSha256.slice(0, 12)}`, + title: "Confirmatory fault-injection matrix", + generatedAt: null, + status, + summary: `The confirmatory matrix declared ${status} across ${caseCount} synthetic cases and ${faults.length} known fault classes.`, + faultCoverage: { + expectedFaultClasses: faults.length, + detectedFaultClasses: faults.filter( + (fault) => fault.detectedCount === fault.caseCount, + ).length, + criticalRecallPercent: criticalRecall * 100, + macroRecallPercent: macroRecall * 100, + cleanFalsePositiveRatePercent: cleanFalsePositiveRate * 100, + carryForward, + }, + issues, + provenance, + boundaries: { + claim: [ + "The result is scoped to the declared frozen protocol, provenance, and known fault matrix.", + "It does not establish coverage of unknown faults, model quality, or model safety.", + ], + privacy: [ + "Only summary, per-fault aggregates, gates, and declared provenance are rendered.", + "All per-case fields and any private synthetic content are discarded by the adapter.", + ], + }, + }; +} + +export function parseReport(value: unknown): SftGuardReport { + const report = asRecord(value, "report"); + if ("schemaVersion" in report) { + return parseEnvelopeReport(value); + } + const schemaVersion = asString(report.schema_version, "schema_version", 100); + if (schemaVersion === AUDIT_SCHEMA_VERSION) { + return parseAuditArtifact(report); + } + if (schemaVersion === REGRESSION_SCHEMA_VERSION) { + return parseRegressionArtifact(report); + } + if (schemaVersion === DEVELOPMENT_BENCHMARK_SCHEMA_VERSION) { + return parseDevelopmentBenchmarkArtifact(report); + } + if (schemaVersion === CONFIRMATORY_FAULT_MATRIX_SCHEMA_VERSION) { + return parseConfirmatoryFaultMatrixArtifact(report); + } + throw new RangeError( + `Unsupported SFTGuard report schema: ${schemaVersion}.`, + ); +} + +export function parseReportText(text: string): SftGuardReport { + if (new TextEncoder().encode(text).byteLength > MAX_REPORT_BYTES) { + throw new RangeError("Report exceeds the 2 MB local import limit."); + } + let value: unknown; + try { + value = JSON.parse(text) as unknown; + } catch { + throw new SyntaxError("The selected file is not valid JSON."); + } + return parseReport(value); +} + +export function reportTypeLabel(reportType: ReportType): string { + if (reportType === "fault-matrix") return "Fault matrix"; + if (reportType === "regression") return "Regression card"; + return "Dataset audit"; +} diff --git a/web/src/sampleReports.ts b/web/src/sampleReports.ts new file mode 100644 index 0000000..af3237b --- /dev/null +++ b/web/src/sampleReports.ts @@ -0,0 +1,34 @@ +import auditJson from "./data/sample-audit.json"; +import faultJson from "./data/sample-fault.json"; +import regressionJson from "./data/sample-regression.json"; +import confirmatoryJson from "../../evidence/v0.1.0-confirmatory.json"; +import { parseReport, type SftGuardReport } from "./reportSchema"; + +export interface ReportSource { + report: SftGuardReport; + origin: "checked-in evidence" | "checked-in sample" | "local import"; + sourceLabel: string; +} + +export const SAMPLE_REPORTS: ReportSource[] = [ + { + report: parseReport(confirmatoryJson), + origin: "checked-in evidence", + sourceLabel: "v0.1 confirmatory evidence", + }, + { + report: parseReport(regressionJson), + origin: "checked-in sample", + sourceLabel: "Regression sample", + }, + { + report: parseReport(faultJson), + origin: "checked-in sample", + sourceLabel: "Fault-matrix sample", + }, + { + report: parseReport(auditJson), + origin: "checked-in sample", + sourceLabel: "Audit sample", + }, +]; diff --git a/web/src/styles.css b/web/src/styles.css new file mode 100644 index 0000000..2c44f57 --- /dev/null +++ b/web/src/styles.css @@ -0,0 +1,1411 @@ +:root { + color: #13221f; + background: #ecece5; + font-family: + Inter, ui-sans-serif, -apple-system, BlinkMacSystemFont, "Segoe UI", + sans-serif; + font-synthesis: none; + text-rendering: optimizeLegibility; + --ink: #13221f; + --muted: #60706c; + --line: #d7dad1; + --canvas: #f6f6f0; + --paper: #fdfdf9; + --navy: #152c2a; + --teal: #126d62; + --teal-soft: #dcece6; + --pass: #16745f; + --pass-soft: #dcefe7; + --fail: #b84238; + --fail-soft: #f5dfda; + --abstain: #9a6317; + --abstain-soft: #f5e9cf; + --shadow: 0 24px 60px rgba(24, 41, 37, 0.09); +} + +* { + box-sizing: border-box; +} + +html { + scroll-behavior: smooth; +} + +body { + margin: 0; + min-width: 320px; + min-height: 100vh; + background: + radial-gradient(circle at 12% 8%, rgba(25, 112, 98, 0.08), transparent 30rem), + linear-gradient(135deg, #efefe8 0%, #e8e9e2 100%); +} + +button, +input { + font: inherit; +} + +button, +a, +label[for] { + -webkit-tap-highlight-color: transparent; +} + +button { + color: inherit; +} + +code { + font-family: "SFMono-Regular", Consolas, "Liberation Mono", monospace; +} + +.skip-link { + position: fixed; + top: 0.75rem; + left: 0.75rem; + z-index: 100; + transform: translateY(-160%); + border-radius: 0.45rem; + background: var(--ink); + color: #fff; + padding: 0.7rem 1rem; + text-decoration: none; + transition: transform 160ms ease; +} + +.skip-link:focus { + transform: translateY(0); +} + +:focus-visible { + outline: 3px solid #1c8072; + outline-offset: 3px; +} + +.sr-only { + position: absolute; + width: 1px; + height: 1px; + padding: 0; + margin: -1px; + overflow: hidden; + clip: rect(0, 0, 0, 0); + white-space: nowrap; + border: 0; +} + +.app-shell { + width: min(1540px, calc(100% - 2rem)); + margin: 1rem auto; + overflow: hidden; + border: 1px solid rgba(75, 93, 88, 0.2); + border-radius: 1.4rem; + background: var(--canvas); + box-shadow: var(--shadow); +} + +.topbar { + min-height: 76px; + display: flex; + align-items: center; + justify-content: space-between; + gap: 1rem; + padding: 0.95rem 1.3rem; + border-bottom: 1px solid var(--line); + background: rgba(253, 253, 249, 0.92); +} + +.brand { + display: inline-flex; + align-items: center; + gap: 0.8rem; + color: inherit; + text-decoration: none; +} + +.brand-mark { + display: grid; + place-items: center; + width: 2.5rem; + height: 2.5rem; + border-radius: 0.75rem; + background: var(--navy); + color: #f6fffb; + font-family: "SFMono-Regular", Consolas, monospace; + font-weight: 750; + font-size: 0.78rem; + letter-spacing: 0.08em; + box-shadow: inset 0 0 0 1px rgba(255, 255, 255, 0.12); +} + +.brand > span:last-child { + display: flex; + flex-direction: column; + gap: 0.08rem; +} + +.brand strong { + font-size: 1rem; + letter-spacing: -0.02em; +} + +.brand small { + color: var(--muted); + font-size: 0.7rem; + letter-spacing: 0.06em; + text-transform: uppercase; +} + +.local-indicator { + display: inline-flex; + align-items: center; + gap: 0.55rem; + color: #3e5a53; + font-size: 0.76rem; + font-weight: 650; +} + +.local-indicator > span { + width: 0.55rem; + height: 0.55rem; + border-radius: 50%; + background: #2a9b82; + box-shadow: 0 0 0 5px rgba(42, 155, 130, 0.1); +} + +.workspace { + display: grid; + grid-template-columns: 292px minmax(0, 1fr); + align-items: start; +} + +.report-rail { + position: sticky; + top: 0; + min-height: calc(100vh - 108px); + display: flex; + flex-direction: column; + gap: 1.15rem; + padding: 1.3rem 1.05rem; + border-right: 1px solid var(--line); + background: #f0f1ea; +} + +.rail-heading { + display: flex; + align-items: center; + justify-content: space-between; + padding: 0 0.3rem; +} + +.rail-heading p { + margin: 0; + color: #60706c; + font-size: 0.69rem; + font-weight: 750; + letter-spacing: 0.12em; + text-transform: uppercase; +} + +.rail-heading span { + min-width: 1.8rem; + border-radius: 999px; + background: #dfe3da; + padding: 0.24rem 0.45rem; + color: #4b5d58; + text-align: center; + font-family: "SFMono-Regular", Consolas, monospace; + font-size: 0.66rem; +} + +.report-list { + display: grid; + gap: 0.52rem; +} + +.report-list button { + appearance: none; + width: 100%; + display: grid; + gap: 0.5rem; + border: 1px solid transparent; + border-radius: 0.85rem; + background: transparent; + padding: 0.85rem; + text-align: left; + cursor: pointer; + transition: + border-color 150ms ease, + background 150ms ease, + transform 150ms ease; +} + +.report-list button:hover { + border-color: #d3d8ce; + background: rgba(255, 255, 255, 0.6); +} + +.report-list button.is-active { + border-color: #cad4cc; + background: var(--paper); + box-shadow: 0 8px 24px rgba(31, 52, 47, 0.06); +} + +.report-list button.is-active::before { + content: ""; + position: absolute; +} + +.report-list-topline { + display: flex; + align-items: center; + justify-content: space-between; + gap: 0.4rem; + color: #61716d; + font-size: 0.66rem; + font-weight: 700; + letter-spacing: 0.06em; + text-transform: uppercase; +} + +.report-list button > strong { + color: #1b2b28; + font-size: 0.86rem; + line-height: 1.3; +} + +.report-list button > small { + color: #7a8884; + font-size: 0.68rem; +} + +.status-badge { + width: fit-content; + display: inline-flex; + align-items: center; + gap: 0.5rem; + border: 1px solid currentColor; + border-radius: 999px; + padding: 0.4rem 0.68rem; + font-size: 0.72rem; + font-weight: 800; + letter-spacing: 0.08em; +} + +.status-badge.is-compact { + gap: 0.3rem; + border-color: transparent; + padding: 0.12rem 0; + background: transparent; + font-size: 0.59rem; +} + +.status-dot { + width: 0.45rem; + height: 0.45rem; + border-radius: 50%; + background: currentColor; +} + +.status-pass { + color: var(--pass); + background: var(--pass-soft); +} + +.status-fail { + color: var(--fail); + background: var(--fail-soft); +} + +.status-abstain { + color: var(--abstain); + background: var(--abstain-soft); +} + +.import-card { + margin-top: auto; + border: 1px solid #d5dad0; + border-radius: 1rem; + background: #e7e9e1; + padding: 1rem; +} + +.import-eyebrow { + margin: 0 0 0.45rem; + color: var(--teal); + font-size: 0.65rem; + font-weight: 800; + letter-spacing: 0.11em; + text-transform: uppercase; +} + +.import-card h2 { + margin: 0; + font-size: 1.02rem; + letter-spacing: -0.025em; +} + +.import-card > p:not(.import-eyebrow, .import-message) { + margin: 0.6rem 0 0.9rem; + color: #61706c; + font-size: 0.73rem; + line-height: 1.55; +} + +.import-button { + appearance: none; + min-height: 42px; + display: inline-flex; + align-items: center; + justify-content: center; + width: 100%; + border-radius: 0.65rem; + border: 0; + background: var(--navy); + color: #f7fffb; + padding: 0.65rem 0.8rem; + font-size: 0.76rem; + font-weight: 750; + cursor: pointer; + transition: + background 150ms ease, + transform 150ms ease; +} + +.import-button:hover { + background: #1c413c; + transform: translateY(-1px); +} + +.visually-hidden-input { + position: absolute; + width: 1px; + height: 1px; + opacity: 0; + pointer-events: none; +} + +.import-message { + min-height: 2.5em; + margin: 0.7rem 0 0; + color: #677570; + font-size: 0.65rem; + line-height: 1.4; +} + +.import-message.is-error { + color: var(--fail); +} + +.clear-import { + appearance: none; + border: 0; + background: none; + padding: 0.4rem 0 0; + color: #476760; + font-size: 0.68rem; + font-weight: 700; + text-decoration: underline; + text-underline-offset: 0.2rem; + cursor: pointer; +} + +.report-main { + min-width: 0; + padding: 1.35rem; +} + +.decision-hero { + position: relative; + overflow: hidden; + display: grid; + grid-template-columns: minmax(0, 1fr) 240px; + gap: 2rem; + border-radius: 1.25rem; + background: var(--navy); + color: #f5faf7; + padding: clamp(1.5rem, 4vw, 3.2rem); + isolation: isolate; +} + +.decision-hero::before, +.decision-hero::after { + content: ""; + position: absolute; + z-index: -1; + border-radius: 50%; + pointer-events: none; +} + +.decision-hero::before { + width: 28rem; + height: 28rem; + right: -11rem; + top: -15rem; + background: radial-gradient(circle, rgba(103, 211, 183, 0.18), transparent 66%); +} + +.decision-hero::after { + width: 15rem; + height: 15rem; + left: 34%; + bottom: -11rem; + border: 1px solid rgba(255, 255, 255, 0.1); + box-shadow: + 0 0 0 2.5rem rgba(255, 255, 255, 0.018), + 0 0 0 5rem rgba(255, 255, 255, 0.012); +} + +.hero-pass { + --hero-accent: #73d8b9; +} + +.hero-fail { + --hero-accent: #ff988b; +} + +.hero-abstain { + --hero-accent: #edc173; +} + +.hero-meta { + display: flex; + flex-wrap: wrap; + align-items: center; + gap: 0.7rem; + margin-bottom: 2.8rem; +} + +.hero-meta > span:not(.status-badge) { + color: #9fb3ad; + font-size: 0.7rem; + font-weight: 650; + letter-spacing: 0.05em; + text-transform: capitalize; +} + +.hero-meta > span:not(.status-badge) + span:not(.status-badge)::before { + content: "/"; + margin-right: 0.7rem; + color: #687d77; +} + +.hero-eyebrow { + margin: 0 0 0.7rem; + color: var(--hero-accent); + font-size: 0.74rem; + font-weight: 800; + letter-spacing: 0.11em; + text-transform: uppercase; +} + +.decision-copy h1 { + max-width: 760px; + margin: 0; + font-size: clamp(2rem, 5vw, 4.2rem); + line-height: 0.98; + letter-spacing: -0.055em; + text-wrap: balance; +} + +.hero-summary { + max-width: 720px; + margin: 1.35rem 0 0; + color: #c6d3cf; + font-size: clamp(0.92rem, 1.5vw, 1.07rem); + line-height: 1.68; +} + +.hero-action { + width: fit-content; + margin: 1.45rem 0 0; + border-left: 2px solid var(--hero-accent); + padding: 0.2rem 0 0.2rem 0.8rem; + color: #f0f7f4; + font-size: 0.8rem; + font-weight: 700; +} + +.decision-seal { + align-self: center; + display: grid; + justify-items: center; + gap: 1rem; + text-align: center; +} + +.decision-seal > span { + display: grid; + place-items: center; + width: 8rem; + height: 8rem; + border: 1px solid color-mix(in srgb, var(--hero-accent) 70%, transparent); + border-radius: 50%; + background: color-mix(in srgb, var(--hero-accent) 8%, transparent); + color: var(--hero-accent); + font-family: "SFMono-Regular", Consolas, monospace; + font-size: 3.7rem; + font-weight: 300; + box-shadow: + inset 0 0 0 0.75rem rgba(255, 255, 255, 0.025), + 0 0 0 0.65rem rgba(255, 255, 255, 0.025); +} + +.decision-seal div { + display: grid; + gap: 0.35rem; +} + +.decision-seal strong { + color: var(--hero-accent); + font-size: 0.78rem; + letter-spacing: 0.14em; +} + +.decision-seal small { + max-width: 200px; + color: #9fb1ac; + font-size: 0.66rem; + line-height: 1.45; +} + +.hero-footer { + grid-column: 1 / -1; + display: flex; + align-items: center; + justify-content: space-between; + gap: 1rem; + border-top: 1px solid rgba(255, 255, 255, 0.1); + margin-top: 0.5rem; + padding-top: 1.1rem; + color: #91a69f; + font-size: 0.68rem; +} + +.hero-footer code { + color: #c9d7d3; +} + +.report-section { + scroll-margin-top: 1rem; + padding: clamp(2.3rem, 5vw, 4.5rem) clamp(0.1rem, 2vw, 1rem); + border-bottom: 1px solid var(--line); +} + +.section-heading { + display: grid; + grid-template-columns: 3rem minmax(0, 1fr); + gap: 1rem; + align-items: start; + margin-bottom: 1.5rem; +} + +.section-number { + padding-top: 0.34rem; + color: #8b9894; + font-family: "SFMono-Regular", Consolas, monospace; + font-size: 0.72rem; +} + +.section-heading h2 { + margin: 0; + color: #182824; + font-size: clamp(1.45rem, 2.6vw, 2.2rem); + letter-spacing: -0.04em; +} + +.section-heading p { + margin: 0.45rem 0 0; + color: #6a7975; + font-size: 0.83rem; + line-height: 1.55; +} + +.metric-grid { + display: grid; + gap: 0.8rem; +} + +.metric-grid.two-up { + grid-template-columns: repeat(2, minmax(0, 1fr)); +} + +.metric-grid.three-up { + grid-template-columns: repeat(3, minmax(0, 1fr)); +} + +.supplemental-metrics { + margin-top: 0.8rem; +} + +.metric-card { + position: relative; + overflow: hidden; + min-height: 190px; + border: 1px solid var(--line); + border-radius: 1rem; + background: var(--paper); + padding: 1.25rem; +} + +.metric-card::after { + content: ""; + position: absolute; + right: 0; + top: 0; + width: 3rem; + height: 3rem; + border-radius: 0 0 0 3rem; + background: #e8ebe4; +} + +.metric-positive::after { + background: var(--pass-soft); +} + +.metric-negative::after { + background: var(--fail-soft); +} + +.metric-caution::after { + background: var(--abstain-soft); +} + +.metric-label { + margin: 0; + color: #687772; + font-size: 0.69rem; + font-weight: 800; + letter-spacing: 0.09em; + text-transform: uppercase; +} + +.metric-value { + margin: 1.05rem 0 0; + color: #172723; + font-size: clamp(2rem, 4vw, 3.25rem); + font-weight: 600; + letter-spacing: -0.055em; +} + +.metric-positive .metric-value { + color: var(--pass); +} + +.metric-negative .metric-value { + color: var(--fail); +} + +.metric-caution .metric-value { + color: var(--abstain); +} + +.metric-context { + max-width: 440px; + margin: 0.35rem 0 0; + color: #71807c; + font-size: 0.73rem; + line-height: 1.45; +} + +.score-pair { + display: grid; + grid-template-columns: auto minmax(2rem, 1fr) auto; + align-items: center; + gap: 0.65rem; + margin-top: 1.2rem; + color: #687772; + font-family: "SFMono-Regular", Consolas, monospace; + font-size: 0.64rem; +} + +.score-line { + height: 1px; + background: linear-gradient(90deg, #d8ded7, #7eb5a6, #d8ded7); +} + +.category-strip { + display: flex; + flex-wrap: wrap; + gap: 0.5rem; + margin-bottom: 0.8rem; +} + +.category-strip span { + display: inline-flex; + align-items: center; + gap: 0.55rem; + border: 1px solid #d8ddd4; + border-radius: 999px; + background: #f8f8f3; + padding: 0.45rem 0.7rem; + color: #586863; + font-size: 0.69rem; +} + +.category-strip strong { + min-width: 1.3rem; + border-radius: 999px; + background: #e5e9e1; + padding: 0.12rem 0.32rem; + text-align: center; + color: #2b423c; + font-size: 0.63rem; +} + +.table-shell { + overflow-x: auto; + border: 1px solid var(--line); + border-radius: 1rem; + background: var(--paper); +} + +table { + width: 100%; + border-collapse: collapse; + text-align: left; +} + +th { + background: #f0f2eb; + padding: 0.78rem 1rem; + color: #687772; + font-size: 0.64rem; + font-weight: 800; + letter-spacing: 0.08em; + text-transform: uppercase; +} + +td { + padding: 1rem; + border-top: 1px solid #e1e4dd; + color: #3a4c47; + font-size: 0.76rem; + vertical-align: top; +} + +td:first-child { + width: 62%; +} + +.issue-title { + color: #1a2a26; + font-size: 0.85rem; + font-weight: 750; +} + +td code { + display: inline-block; + margin-top: 0.35rem; + color: #697a75; + font-size: 0.64rem; +} + +td p { + max-width: 680px; + margin: 0.55rem 0 0; + color: #778580; + font-size: 0.69rem; + line-height: 1.52; +} + +.number-cell { + text-align: right; + font-variant-numeric: tabular-nums; +} + +.severity { + display: inline-block; + border-radius: 999px; + padding: 0.25rem 0.48rem; + font-size: 0.61rem; + font-weight: 750; + text-transform: capitalize; +} + +.severity-critical, +.severity-error { + background: var(--fail-soft); + color: var(--fail); +} + +.severity-warning { + background: var(--abstain-soft); + color: var(--abstain); +} + +.severity-info { + background: var(--teal-soft); + color: var(--teal); +} + +.empty-panel { + border: 1px dashed #cbd1c8; + border-radius: 1rem; + background: rgba(255, 255, 255, 0.45); + padding: clamp(1.4rem, 4vw, 2.4rem); +} + +.empty-panel h3 { + margin: 0; + font-size: 1.2rem; + letter-spacing: -0.03em; +} + +.empty-panel p { + max-width: 650px; + margin: 0.55rem 0 0; + color: #71807c; + font-size: 0.76rem; + line-height: 1.6; +} + +.empty-panel .empty-kicker { + margin: 0 0 0.5rem; + color: var(--teal); + font-size: 0.63rem; + font-weight: 800; + letter-spacing: 0.09em; + text-transform: uppercase; +} + +.compact-empty { + padding: 1.4rem; +} + +.coverage-layout { + display: grid; + grid-template-columns: 230px minmax(0, 1fr); + align-items: center; + gap: clamp(1.5rem, 5vw, 4rem); + border: 1px solid var(--line); + border-radius: 1rem; + background: var(--paper); + padding: clamp(1.5rem, 4vw, 2.5rem); +} + +.coverage-ring { + --coverage: 0%; + width: min(100%, 210px); + aspect-ratio: 1; + display: grid; + place-items: center; + border-radius: 50%; + background: conic-gradient(var(--teal) var(--coverage), #e4e8e0 0); + position: relative; +} + +.coverage-ring::after { + content: ""; + position: absolute; + inset: 0.85rem; + border-radius: 50%; + background: var(--paper); +} + +.coverage-ring > div { + position: relative; + z-index: 1; + display: grid; + justify-items: center; +} + +.coverage-ring strong { + color: #1c342e; + font-size: 3.2rem; + font-weight: 550; + letter-spacing: -0.06em; +} + +.coverage-ring span { + color: #74827e; + font-size: 0.72rem; +} + +.coverage-result { + margin: 0; + color: #263d37; + font-size: 1.12rem; + font-weight: 750; + letter-spacing: -0.025em; +} + +.coverage-copy > p:nth-child(2) { + max-width: 670px; + margin: 0.55rem 0 1.4rem; + color: #71807c; + font-size: 0.76rem; + line-height: 1.55; +} + +.coverage-bars { + display: grid; + gap: 1rem; +} + +.coverage-row { + display: grid; + grid-template-columns: minmax(150px, 1fr) auto; + gap: 0.4rem 1rem; +} + +.coverage-row > div:first-child { + display: flex; + align-items: baseline; + gap: 0.5rem; + color: #43554f; + font-size: 0.72rem; + font-weight: 700; +} + +.coverage-target { + color: #8a9692; + font-size: 0.62rem; + font-weight: 500; +} + +.coverage-value { + color: #263b35; + font-family: "SFMono-Regular", Consolas, monospace; + font-size: 0.7rem; +} + +.bar-track { + grid-column: 1 / -1; + height: 0.38rem; + overflow: hidden; + border-radius: 999px; + background: #e2e6de; +} + +.bar-track span { + display: block; + height: 100%; + border-radius: inherit; + background: var(--teal); +} + +.bar-track.is-inverse span { + background: var(--abstain); +} + +.provenance-grid { + display: grid; + grid-template-columns: minmax(190px, 0.65fr) minmax(0, 1.35fr); + gap: 0.8rem; +} + +.replay-card, +.provenance-list { + border: 1px solid var(--line); + border-radius: 1rem; + background: var(--paper); +} + +.replay-card { + min-height: 220px; + display: flex; + flex-direction: column; + justify-content: flex-end; + padding: 1.3rem; +} + +.replay-card::before { + content: ""; + width: 2.5rem; + height: 2.5rem; + margin-bottom: auto; + border: 1px solid currentColor; + border-radius: 50%; + background: radial-gradient(circle, currentColor 0 19%, transparent 21%); +} + +.replay-pass { + color: var(--pass); +} + +.replay-fail { + color: var(--fail); +} + +.replay-unknown { + color: var(--muted); +} + +.replay-card p { + margin: 0; + color: #72817c; + font-size: 0.68rem; + font-weight: 750; + letter-spacing: 0.07em; + text-transform: uppercase; +} + +.replay-card strong { + margin-top: 0.45rem; + font-size: 1.55rem; + letter-spacing: -0.04em; +} + +.replay-card span { + margin-top: 0.25rem; + color: #7a8884; + font-size: 0.68rem; +} + +.provenance-list { + margin: 0; + overflow: hidden; +} + +.provenance-list > div { + display: grid; + grid-template-columns: minmax(130px, 0.8fr) minmax(0, 1.2fr); + gap: 1rem; + padding: 0.84rem 1rem; + border-bottom: 1px solid #e2e5de; +} + +.provenance-list > div:last-child { + border-bottom: 0; +} + +.provenance-list dt { + color: #74827e; + font-size: 0.68rem; +} + +.provenance-list dd { + min-width: 0; + margin: 0; + color: #30443e; + font-size: 0.7rem; + font-weight: 650; + overflow-wrap: anywhere; +} + +.provenance-list .hash-value { + font-family: "SFMono-Regular", Consolas, monospace; + font-size: 0.66rem; +} + +.boundaries-section { + border-bottom: 0; +} + +.boundary-grid { + display: grid; + grid-template-columns: repeat(2, minmax(0, 1fr)); + gap: 0.8rem; +} + +.boundary-card { + min-height: 260px; + border-radius: 1rem; + padding: clamp(1.3rem, 3vw, 2rem); +} + +.claim-boundary { + background: #1b302d; + color: #ecf5f1; +} + +.privacy-boundary { + border: 1px solid #cddbd4; + background: var(--teal-soft); + color: #1c3932; +} + +.boundary-label { + margin: 0; + color: #79c9b3; + font-size: 0.65rem; + font-weight: 800; + letter-spacing: 0.1em; + text-transform: uppercase; +} + +.privacy-boundary .boundary-label { + color: var(--teal); +} + +.boundary-card h3 { + margin: 0.75rem 0 1.2rem; + font-size: 1.35rem; + letter-spacing: -0.035em; +} + +.boundary-card ul { + display: grid; + gap: 0.8rem; + margin: 0; + padding: 0; + list-style: none; +} + +.boundary-card li { + position: relative; + padding-left: 1rem; + color: #b9cbc5; + font-size: 0.73rem; + line-height: 1.55; +} + +.privacy-boundary li { + color: #4e6b63; +} + +.boundary-card li::before { + content: ""; + position: absolute; + left: 0; + top: 0.56rem; + width: 0.35rem; + height: 0.35rem; + border-radius: 50%; + background: #73cbb3; +} + +.report-footer { + display: grid; + grid-template-columns: auto minmax(0, 720px); + justify-content: space-between; + gap: 2rem; + border-top: 1px solid var(--line); + padding: 1.6rem 1rem 0.7rem; + color: #75837f; + font-size: 0.67rem; +} + +.report-footer span { + color: #40554f; + font-weight: 750; +} + +.report-footer p { + margin: 0; + line-height: 1.6; + text-align: right; +} + +@media (max-width: 1080px) { + .workspace { + grid-template-columns: 240px minmax(0, 1fr); + } + + .report-rail { + padding-inline: 0.75rem; + } + + .decision-hero { + grid-template-columns: minmax(0, 1fr) 170px; + } + + .decision-seal > span { + width: 6.5rem; + height: 6.5rem; + font-size: 3rem; + } + + .coverage-layout { + grid-template-columns: 180px minmax(0, 1fr); + } +} + +@media (max-width: 820px) { + .app-shell { + width: 100%; + margin: 0; + border: 0; + border-radius: 0; + } + + .topbar { + position: sticky; + top: 0; + z-index: 20; + } + + .workspace { + display: block; + } + + .report-rail { + position: static; + min-height: 0; + border-right: 0; + border-bottom: 1px solid var(--line); + padding: 1rem; + } + + .report-list { + display: grid; + grid-auto-flow: column; + grid-auto-columns: minmax(230px, 72vw); + overflow-x: auto; + padding-bottom: 0.4rem; + scroll-snap-type: x proximity; + } + + .report-list button { + scroll-snap-align: start; + } + + .import-card { + margin-top: 0; + } + + .decision-hero { + grid-template-columns: 1fr; + } + + .decision-seal { + grid-template-columns: auto minmax(0, 1fr); + justify-items: start; + text-align: left; + } + + .decision-seal > span { + width: 4.5rem; + height: 4.5rem; + font-size: 2.1rem; + } + + .hero-footer { + flex-direction: column; + align-items: flex-start; + } + + .metric-grid.three-up { + grid-template-columns: 1fr; + } + + .coverage-layout { + grid-template-columns: 1fr; + } + + .coverage-ring { + width: 170px; + } + + .provenance-grid { + grid-template-columns: 1fr; + } +} + +@media (max-width: 600px) { + .topbar { + min-height: 68px; + padding-inline: 1rem; + } + + .local-indicator { + max-width: 9rem; + font-size: 0.65rem; + line-height: 1.3; + text-align: right; + } + + .report-main { + padding: 0.75rem; + } + + .decision-hero { + border-radius: 1rem; + padding: 1.25rem; + } + + .hero-meta { + margin-bottom: 2rem; + } + + .decision-copy h1 { + font-size: clamp(2rem, 11vw, 3rem); + } + + .metric-grid.two-up, + .boundary-grid { + grid-template-columns: 1fr; + } + + .section-heading { + grid-template-columns: 2rem minmax(0, 1fr); + gap: 0.5rem; + } + + .table-shell { + border: 0; + background: transparent; + overflow: visible; + } + + table, + thead, + tbody, + tr, + th, + td { + display: block; + } + + thead { + position: absolute; + width: 1px; + height: 1px; + overflow: hidden; + clip: rect(0, 0, 0, 0); + } + + tbody { + display: grid; + gap: 0.7rem; + } + + tr { + border: 1px solid var(--line); + border-radius: 0.85rem; + background: var(--paper); + padding: 0.9rem; + } + + td, + td:first-child { + width: auto; + border: 0; + padding: 0.35rem 0; + } + + .number-cell { + text-align: left; + } + + .number-cell::before { + content: "Count: "; + color: #7b8985; + } + + .provenance-list > div { + grid-template-columns: 1fr; + gap: 0.3rem; + } + + .report-footer { + grid-template-columns: 1fr; + gap: 0.8rem; + } + + .report-footer p { + text-align: left; + } +} + +@media (prefers-reduced-motion: reduce) { + *, + *::before, + *::after { + scroll-behavior: auto !important; + transition-duration: 0.01ms !important; + animation-duration: 0.01ms !important; + animation-iteration-count: 1 !important; + } +} diff --git a/web/src/vite-env.d.ts b/web/src/vite-env.d.ts new file mode 100644 index 0000000..11f02fe --- /dev/null +++ b/web/src/vite-env.d.ts @@ -0,0 +1 @@ +/// diff --git a/web/tsconfig.json b/web/tsconfig.json new file mode 100644 index 0000000..220f11d --- /dev/null +++ b/web/tsconfig.json @@ -0,0 +1,21 @@ +{ + "compilerOptions": { + "target": "ES2022", + "useDefineForClassFields": true, + "lib": ["ES2022", "DOM", "DOM.Iterable"], + "allowJs": false, + "skipLibCheck": true, + "esModuleInterop": true, + "allowSyntheticDefaultImports": true, + "strict": true, + "forceConsistentCasingInFileNames": true, + "module": "ESNext", + "moduleResolution": "Bundler", + "resolveJsonModule": true, + "isolatedModules": true, + "noEmit": true, + "jsx": "react-jsx", + "types": ["vite/client", "vitest/globals"] + }, + "include": ["src", "vite.config.ts"] +} diff --git a/web/vite.config.ts b/web/vite.config.ts new file mode 100644 index 0000000..2fb6b7b --- /dev/null +++ b/web/vite.config.ts @@ -0,0 +1,13 @@ +import react from "@vitejs/plugin-react"; +import { defineConfig } from "vitest/config"; + +export default defineConfig({ + plugins: [react()], + build: { + target: "es2022", + sourcemap: true, + }, + test: { + environment: "node", + }, +});