From 14c91822d4f900bd7973a13b9fef2d4ffd99cac3 Mon Sep 17 00:00:00 2001 From: coreytshaffer <78175888+coreytshaffer@users.noreply.github.com> Date: Sat, 11 Jul 2026 21:07:29 -0700 Subject: [PATCH 1/3] Add CR-123 eval scoring scope --- docs/change/change_log.md | 1 + .../requests/CR-123-eval-scoring-scope.md | 42 +++++++++++++++ docs/current_backlog.md | 9 +++- docs/research/eval_fixture_schema.md | 8 ++- docs/research/eval_scoring_scope.md | 53 +++++++++++++++++++ tests/fixtures/evals/README.md | 6 ++- 6 files changed, 115 insertions(+), 4 deletions(-) create mode 100644 docs/change/requests/CR-123-eval-scoring-scope.md create mode 100644 docs/research/eval_scoring_scope.md diff --git a/docs/change/change_log.md b/docs/change/change_log.md index 71558ba..bcc314d 100644 --- a/docs/change/change_log.md +++ b/docs/change/change_log.md @@ -5,6 +5,7 @@ This file provides a chronological, human-readable record of applied codebase an *Note: For operational task and run history, consult `.triagecore/ledger.jsonl`.* ## [Unreleased] +- Implemented CR-123 (Eval Scoring Scope Boundary): Added a docs-only scope boundary for the first future safety-boundary scoring slice, limiting it to deterministic explicit-file comparison between validated `eval_case_v0` fixtures and already-exported actual-outcome records. Preserved the external evaluator boundary and kept model/backend calls, runtime probing, routing, admission, ledger writes, approval state, adversarial expansion, and any self-certification claims out of scope. - Implemented CR-122 (Eval Fixture Validation CLI): Added `tc eval validate-fixtures --input ` as a narrow read-only wrapper around the CR-121 JSONL validator, printing a bounded pass count on valid fixtures and fail-closed, line-aware diagnostics on invalid input. Added synthetic CLI tests only and kept scoring, observed-behavior comparison, model/backend calls, routing/admission integration, ledger writes, runtime behavior, and adversarial/tampering expansion out of scope. - Implemented CR-121 (Eval Fixture Validator): Added a pure deterministic JSONL validator for the CR-077 safety-boundary eval fixture contract, with line-aware diagnostics and fail-closed handling for malformed JSON, non-object/empty lines, missing required fields, empty or duplicate `case_id`, closed-vocabulary violations, and required nested fixture shape. Added synthetic unit tests only and kept `tc eval`, scoring, model/backend calls, routing/admission integration, ledger writes, and adversarial tampering tests out of scope. - Implemented CR-120 (Telemetry Lane Release Hygiene): Added a docs-only post-merge hygiene note for the completed CR-117 through CR-119 lane, marked the reviewer checkpoint/release-hygiene candidate complete, and corrected stale telemetry brief wording that could confuse the CR-114 reviewer checkpoint with the CR-119 probe validation gate. No runtime code, schemas, tests, CLI behavior, routing, ledger writes, probe execution, endpoint calls, tags, or model/backend calls were changed. diff --git a/docs/change/requests/CR-123-eval-scoring-scope.md b/docs/change/requests/CR-123-eval-scoring-scope.md new file mode 100644 index 0000000..313f802 --- /dev/null +++ b/docs/change/requests/CR-123-eval-scoring-scope.md @@ -0,0 +1,42 @@ +# CR-123: Eval Scoring Scope Boundary + +## Status + +Implemented (docs-only scope) + +## Summary + +Define the allowed shape of a future safety-boundary eval scoring slice after +CR-121 fixture validation and CR-122 fixture validation CLI stabilization. + +This slice is intentionally documentation-only. It clarifies that a future +TriageCore scoring surface may compare already-exported actual-outcome files +against already-validated fixture expectations, but must not execute model +work, probe runtimes, mutate ledgers, approve actions, or replace the +independent `agent-control-evals` harness. + +## Scope + +- Add an eval scoring scope note for the CR-077 through CR-122 fixture lane. +- Update backlog and fixture docs so the next implementation slice is bounded + before code is added. +- Preserve the distinction between fixture expectation validation, static + actual-outcome comparison, and independent evaluator scoring. +- Keep the first future implementation candidate limited to explicit input + files and deterministic summary output. + +## Non-Goals + +- No scoring CLI in this slice. +- No observed-behavior execution. +- No model, completion, chat, embedding, backend, endpoint, or network calls. +- No routing, admission, ledger, identity, approval, or runtime integration. +- No mutation of actual-outcome files or fixture files. +- No adversarial tampering suite or new fixture families. +- No claim that TriageCore can certify itself. + +## Validation + +- `git diff --check` + +Docs-only validation is sufficient because this slice edits documentation only. diff --git a/docs/current_backlog.md b/docs/current_backlog.md index 7aa9ac2..6738866 100644 --- a/docs/current_backlog.md +++ b/docs/current_backlog.md @@ -6,6 +6,10 @@ This document summarizes the active TriageCore backlog after CR-115. ## Active GitHub Backlog +- CR-123: Eval Scoring Scope Boundary + - Status: complete via CR-123 (docs-only scope) + - Purpose: Define the first safe scoring-capable follow-on after CR-121 and CR-122: deterministic explicit-file comparison between validated `eval_case_v0` fixtures and already-exported actual-outcome contract records, while keeping model/backend calls, endpoint probing, routing, admission, ledger writes, approval state, adversarial expansion, and self-certification claims out of scope. + - CR-122: Eval Fixture Validation CLI - Status: complete via CR-122 (validator CLI only) - Purpose: Expose the CR-121 safety-boundary eval fixture validator through `tc eval validate-fixtures --input `, with bounded pass output and fail-closed line-aware diagnostics, while keeping scoring, observed-behavior comparison, model/backend calls, routing/admission integration, ledger writes, runtime behavior, and adversarial/tampering expansion out of scope. @@ -120,7 +124,7 @@ This document summarizes the active TriageCore backlog after CR-115. - Empirical AI safety evaluation track - Source: CR-076 and CR-077 research framing/eval taxonomy docs - - Status: research question, threat model, eval taxonomy, fixture schema, toy boundary fixtures, **TC-EVAL-001 (Export Actual Outcome Contract Files)**, **TC-EVAL-002 (Actual Outcome Export CLI Smoke)**, **[x] TC-EVAL-003 (Map One Real Internal Decision Path Into the Export Contract)**, **[x] TC-EVAL-004 (Export One Real Privacy Scanner Actual)**, **[x] TC-EVAL-005 / 006 / 007 (Privacy Reason Normalization)** documented; fixture validator complete via CR-121; fixture validation CLI complete via CR-122; scoring/evaluator execution, adversarial tests, toy audit tampering eval, behavioral route diffing, **[x] TC-EVAL-008 (Structured Privacy Scanner Finding Codes)**, **[x] TC-EVAL-009 (Shared Internal Reason-Code Constants for Privacy Findings)**, **[x] TC-EVAL-010 (Export One Forbidden Tool-Call Actual)** and technical report remain future slices + - Status: research question, threat model, eval taxonomy, fixture schema, toy boundary fixtures, **TC-EVAL-001 (Export Actual Outcome Contract Files)**, **TC-EVAL-002 (Actual Outcome Export CLI Smoke)**, **[x] TC-EVAL-003 (Map One Real Internal Decision Path Into the Export Contract)**, **[x] TC-EVAL-004 (Export One Real Privacy Scanner Actual)**, **[x] TC-EVAL-005 / 006 / 007 (Privacy Reason Normalization)** documented; fixture validator complete via CR-121; fixture validation CLI complete via CR-122; first scoring scope boundary complete via CR-123; static explicit-file scoring comparison, adversarial tests, toy audit tampering eval, behavioral route diffing, **[x] TC-EVAL-008 (Structured Privacy Scanner Finding Codes)**, **[x] TC-EVAL-009 (Shared Internal Reason-Code Constants for Privacy Findings)**, **[x] TC-EVAL-010 (Export One Forbidden Tool-Call Actual)** and technical report remain future slices - Purpose: make TriageCore legible as a reproducible local-first AI control and evaluation harness for testing privacy, routing, identity, provenance, audit, and human-approval boundaries under controlled adversarial pressure. - Operator UX implementation path @@ -212,7 +216,7 @@ Keep three work lanes distinct: For signed ledger coverage, the reviewer-facing `validation_result` path and the signed `route_decision` path are now in place, including a smoke example, a capability-targeted doctor check, and a consolidated reviewer checkpoint for the latter. The current safe lane is packaging/stabilization, reviewer entrypoint maintenance, smoke-runbook clarity, video-first submission packaging, and release-readiness documentation. Deeper signing, cryptographic lifecycle work, and Issue #73 runtime key rotation should remain separate CRs. Do not treat a valid signature as approval, safety, or correctness. -For the empirical AI safety evaluation track, CR-121 completes fixture validation and CR-122 exposes it through a narrow CLI. Keep the next slices sequential: scoring/evaluator execution only after the validation CLI remains stable, and broader adversarial/tampering studies only after scoring is separately reviewed. +For the empirical AI safety evaluation track, CR-121 completes fixture validation, CR-122 exposes it through a narrow CLI, and CR-123 scopes the first safe scoring follow-on. Keep the next slices sequential: the next scoring implementation should be deterministic explicit-file comparison only, and broader adversarial/tampering studies should wait until that static comparison path is separately reviewed. For external runtime interoperability, the next approved slice should be policy tests or execution-path validation for the bounded adapter path. @@ -224,4 +228,5 @@ For operator UX, future slices should focus on reviewability, export polish, and - **[done] Telemetry schema and synthetic-fixture validation (CR-118)**: Hardened the existing local backend probe's serialized record contract before any further probe work — strict schema, pure validator, synthetic fixtures only, no endpoint calls, no routing integration, no ledger writes, and no CLI behavior changes. - **[done] Local backend telemetry probe validation gate (CR-119)**: Every emitted local backend probe result now validates against the CR-118 record contract before it is treated as an observation; validation failure raises a fail-closed `ProbeInputError`, with no generation calls, routing integration, or ledger writes. - **[done] Reviewer checkpoint or release-hygiene slice (CR-120, docs-only)**: Froze the completed CR-117 through CR-119 lane in a concise operations note, corrected stale CR-114 telemetry wording, and kept future telemetry work behind a new explicit scope pass instead of adding more features. +- **[done] Eval scoring scope boundary (CR-123, docs-only)**: Scoped the first future scoring-capable eval slice as deterministic explicit-file comparison between validated fixtures and already-exported actual-outcome records, with no model/backend calls, routing, admission, ledger writes, approval state changes, adversarial expansion, or self-certification claims. - **Next slice TBD by scope pass**: Do not start routing integration, ledger integration, circuit breakers, automatic discovery, background polling, daily-driver enforcement, or additional telemetry behavior without a new approved CR. diff --git a/docs/research/eval_fixture_schema.md b/docs/research/eval_fixture_schema.md index 484eea0..eb174dd 100644 --- a/docs/research/eval_fixture_schema.md +++ b/docs/research/eval_fixture_schema.md @@ -140,4 +140,10 @@ CR-121 treats this document as the contract for a narrow validator. The validato The validator checks fixture integrity only, and CR-122 exposes that validation through `tc eval validate-fixtures --input `. These fixtures still are not -scored or executed until a separate scoring/evaluator slice exists. +scored or executed by the validator path. + +CR-123 scopes the first safe scoring-capable follow-on as deterministic +explicit-file comparison between validated fixtures and already-exported +actual-outcome records. That future implementation must remain offline, +read-only, and separate from model/backend execution, routing, ledger writes, +approval state, and external evaluator authority. diff --git a/docs/research/eval_scoring_scope.md b/docs/research/eval_scoring_scope.md new file mode 100644 index 0000000..3791dae --- /dev/null +++ b/docs/research/eval_scoring_scope.md @@ -0,0 +1,53 @@ +# Eval Scoring Scope Boundary + +This note scopes the first scoring-capable follow-on after the CR-121 fixture +validator and CR-122 `tc eval validate-fixtures` command. + +## Allowed First Scoring Slice + +A future scoring slice may be a deterministic, explicit-file comparison between: + +- a validated `eval_case_v0` JSONL fixture file +- a directory or file of already-exported actual-outcome contract JSON records + +The comparison may report: + +- total fixture cases considered +- missing actuals by `case_id` +- unexpected actuals not present in the fixture +- decision matches or mismatches against `expected_control_plane_decision` +- boundary-family counts +- expected eval outcome counts from the fixture +- a bounded pass/fail/block summary for the static comparison + +The comparison must treat invalid fixtures or invalid actual-outcome records as +fail-closed input errors, not as softer test failures. + +## Required Boundaries + +The scoring slice must remain offline and read-only. It must not call model +backends, execute prompts, probe endpoints, mutate source files, write ledger +events, update approval state, or route work. + +TriageCore may compare static files that already exist. It must not claim that +the comparison certifies production safety, approves execution, or replaces an +external evaluator. + +## Relationship To External Evals + +The independent `agent-control-evals` harness remains the authoritative place +for cross-repository evaluation runs. TriageCore can provide local smoke +comparison for exported actuals only so operators can catch obvious contract +drift before handing artifacts to the external harness. + +## Out Of Scope For The First Slice + +- adversarial or tampering fixture expansion +- behavioral route diffing +- benchmark ranking +- model quality scoring +- automatic discovery of actuals +- default output locations +- ledger persistence +- policy enforcement +- human approval decisions diff --git a/tests/fixtures/evals/README.md b/tests/fixtures/evals/README.md index d83a6db..ad082c5 100644 --- a/tests/fixtures/evals/README.md +++ b/tests/fixtures/evals/README.md @@ -21,4 +21,8 @@ These fixtures exist to define what a future evaluator will consume. They do not - CR-121 adds a narrow validator for required fields and deterministic labels. - CR-122 exposes that validator through `tc eval validate-fixtures --input `. -- A future CR can add scoring only after the validation CLI is stable. +- CR-123 scopes the first safe scoring-capable follow-on as explicit-file + comparison between validated fixtures and already-exported actual-outcome + records. +- A future implementation CR can add that static comparison only after the + boundary remains stable. From 0588fc13af32e30836f147683417f8c18d4b3783 Mon Sep 17 00:00:00 2001 From: coreytshaffer <78175888+coreytshaffer@users.noreply.github.com> Date: Sat, 11 Jul 2026 22:23:41 -0700 Subject: [PATCH 2/3] Honor terminal resilience routes --- .../daily_driver_orchestrator_spec.md | 15 +++--- docs/change/change_log.md | 1 + ...CR-124-honor-terminal-resilience-routes.md | 35 ++++++++++++ docs/current_backlog.md | 11 ++-- tests/test_client.py | 3 +- tests/test_tc_run_cli.py | 53 +++++++++++++++++++ triage_core/client.py | 33 +++++++++++- 7 files changed, 139 insertions(+), 12 deletions(-) create mode 100644 docs/change/requests/CR-124-honor-terminal-resilience-routes.md diff --git a/docs/architecture/daily_driver_orchestrator_spec.md b/docs/architecture/daily_driver_orchestrator_spec.md index d0acad3..39d2cd3 100644 --- a/docs/architecture/daily_driver_orchestrator_spec.md +++ b/docs/architecture/daily_driver_orchestrator_spec.md @@ -31,7 +31,7 @@ privacy, routing, and evidence layers. | Dimension | Readiness | Basis | |---|---|---| -| Local-first daily driver | ~60% | Governed loop exists in the library (`TriageClient.run_task`); no operator run surface, live probe, or enforced budgets. | +| Local-first daily driver | ~60% | Governed loop is available through `tc run`; live capability signals and enforced budgets remain absent. | | Frontier-cloud orchestrator | ~35% | Only live cloud backend is `QwenCloudBackend`; Claude/GPT/Gemini exist solely as after-the-fact file handoffs (Codex/Antigravity). | | Token efficiency | Measured, not enforced | `context_budget.py` / context packs record usage; budgets are advisory, no pre-send compaction is bound into the run path. | @@ -40,11 +40,12 @@ after M0 (below) produces daily-use evidence. See **Evidence Requirements**. ## What Already Works (verified in-repo) -- `TriageClient.run_task` runs a full governed loop: verify packet → privacy scan +- `TriageClient.run_task` runs a governed loop: verify packet → privacy scan (fail-closed) → external-safe packet gate → classify → specialist route → resilience - route (6-way) → local execute, with a real cloud branch (`_execute_cloud_task`) that - executes `QwenCloudBackend` when the route is `cloud_primary/secondary` and the packet is - external-safe. + route. `local_heavy` and `local_fast` execute locally; `cloud_primary/secondary` use the + real cloud branch (`_execute_cloud_task`) only for external-safe packets; `human_handoff` + and the currently unimplemented `deterministic` route return `handoff_required` without + model execution. - `routing/resilience_router.py` encodes local-first ordering: deterministic → local_fast / local_heavy → cloud, with high-sensitivity forced to human handoff and cloud blocked for local-only privacy. @@ -54,8 +55,8 @@ after M0 (below) produces daily-use evidence. See **Evidence Requirements**. ## Gaps (planning targets) -- **G1 — No unified operator run surface.** `run_task` is library-only; `tc` has no - `run`/`orchestrate`; `triagecore run-pipeline` is local-only and bypasses the router. +- **G1 — No unified operator run surface.** `tc run` exposes the governed loop, but + `triagecore run-pipeline` remains local-only and bypasses the router. - **G2 — Cloud is Qwen, not frontier.** No live Claude/GPT/Gemini backends, no provider abstraction beyond OpenAI-compatible, no per-provider cost/credit model. - **G3 — Route decisions outrun execution bindings.** `local_heavy`/`local_fast` and diff --git a/docs/change/change_log.md b/docs/change/change_log.md index bcc314d..1d39fe2 100644 --- a/docs/change/change_log.md +++ b/docs/change/change_log.md @@ -5,6 +5,7 @@ This file provides a chronological, human-readable record of applied codebase an *Note: For operational task and run history, consult `.triagecore/ledger.jsonl`.* ## [Unreleased] +- Implemented CR-124 (Honor Terminal Resilience Routes): Made `human_handoff` and the currently unimplemented `deterministic` route terminal at the governed `TriageClient.run_task` boundary. Each now records the route decision plus a `worker_result` with `worker_result_status=not_attempted`, returns `handoff_required`, and reaches the existing `tc run` exit-3 contract without invoking a local or cloud backend. This slice does not add approval-and-resume behavior, change the broader `human_review_required` semantics, or alter cloud gating, signatures, reducer behavior, or other execution surfaces. - Implemented CR-123 (Eval Scoring Scope Boundary): Added a docs-only scope boundary for the first future safety-boundary scoring slice, limiting it to deterministic explicit-file comparison between validated `eval_case_v0` fixtures and already-exported actual-outcome records. Preserved the external evaluator boundary and kept model/backend calls, runtime probing, routing, admission, ledger writes, approval state, adversarial expansion, and any self-certification claims out of scope. - Implemented CR-122 (Eval Fixture Validation CLI): Added `tc eval validate-fixtures --input ` as a narrow read-only wrapper around the CR-121 JSONL validator, printing a bounded pass count on valid fixtures and fail-closed, line-aware diagnostics on invalid input. Added synthetic CLI tests only and kept scoring, observed-behavior comparison, model/backend calls, routing/admission integration, ledger writes, runtime behavior, and adversarial/tampering expansion out of scope. - Implemented CR-121 (Eval Fixture Validator): Added a pure deterministic JSONL validator for the CR-077 safety-boundary eval fixture contract, with line-aware diagnostics and fail-closed handling for malformed JSON, non-object/empty lines, missing required fields, empty or duplicate `case_id`, closed-vocabulary violations, and required nested fixture shape. Added synthetic unit tests only and kept `tc eval`, scoring, model/backend calls, routing/admission integration, ledger writes, and adversarial tampering tests out of scope. diff --git a/docs/change/requests/CR-124-honor-terminal-resilience-routes.md b/docs/change/requests/CR-124-honor-terminal-resilience-routes.md new file mode 100644 index 0000000..4ba92c2 --- /dev/null +++ b/docs/change/requests/CR-124-honor-terminal-resilience-routes.md @@ -0,0 +1,35 @@ +# CR-124: Honor Terminal Resilience Routes + +## Status + +Implemented + +## Summary + +Make the terminal resilience routes selected by `TriageClient.run_task` control +execution. A `human_handoff` route and the currently unimplemented +`deterministic` route now stop before any backend call, record a metadata-only +`worker_result` with `worker_result_status=not_attempted`, and return the +existing governed handoff result consumed by `tc run` as exit code 3. + +## Scope + +- Treat `human_handoff` as terminal at the governed execution boundary. +- Treat `deterministic` as terminal until a real deterministic executor exists. +- Preserve route-decision and worker-result evidence for terminal routes. +- Add offline sentinel-backend coverage proving terminal routes do not execute. +- Correct the daily-driver architecture note to match the enforced behavior. + +## Non-Goals + +- No approval-and-resume workflow. +- No change making every `human_review_required` flag pre-execution blocking. +- No deterministic executor implementation. +- No changes to cloud gating, privacy scanning, signatures, reducer behavior, + other execution surfaces, or historical ledger records. + +## Validation + +- `python -m pytest -q tests/test_tc_run_cli.py tests/test_local_only_routing.py` +- `python -m pytest -q` +- `git diff --check` diff --git a/docs/current_backlog.md b/docs/current_backlog.md index 6738866..3f5cbb8 100644 --- a/docs/current_backlog.md +++ b/docs/current_backlog.md @@ -2,10 +2,14 @@ ## Status -This document summarizes the active TriageCore backlog after CR-115. +This document summarizes the active TriageCore backlog after CR-124. ## Active GitHub Backlog +- CR-124: Honor Terminal Resilience Routes + - Status: complete via CR-124 + - Purpose: Make `human_handoff` and the currently unimplemented `deterministic` resilience routes terminal at `TriageClient.run_task`: record the route decision and a `worker_result` with `not_attempted`, return `handoff_required`, and ensure `tc run` exits 3 without invoking any backend. This slice does not add an approval-and-resume workflow or make every `human_review_required` flag pre-execution blocking. + - CR-123: Eval Scoring Scope Boundary - Status: complete via CR-123 (docs-only scope) - Purpose: Define the first safe scoring-capable follow-on after CR-121 and CR-122: deterministic explicit-file comparison between validated `eval_case_v0` fixtures and already-exported actual-outcome contract records, while keeping model/backend calls, endpoint probing, routing, admission, ledger writes, approval state, adversarial expansion, and self-certification claims out of scope. @@ -216,7 +220,7 @@ Keep three work lanes distinct: For signed ledger coverage, the reviewer-facing `validation_result` path and the signed `route_decision` path are now in place, including a smoke example, a capability-targeted doctor check, and a consolidated reviewer checkpoint for the latter. The current safe lane is packaging/stabilization, reviewer entrypoint maintenance, smoke-runbook clarity, video-first submission packaging, and release-readiness documentation. Deeper signing, cryptographic lifecycle work, and Issue #73 runtime key rotation should remain separate CRs. Do not treat a valid signature as approval, safety, or correctness. -For the empirical AI safety evaluation track, CR-121 completes fixture validation, CR-122 exposes it through a narrow CLI, and CR-123 scopes the first safe scoring follow-on. Keep the next slices sequential: the next scoring implementation should be deterministic explicit-file comparison only, and broader adversarial/tampering studies should wait until that static comparison path is separately reviewed. +For the empirical AI safety evaluation track, CR-121 completes fixture validation, CR-122 exposes it through a narrow CLI, and CR-123 scopes a future safe scoring follow-on. Defer any in-repository comparison implementation until the governed path produces real exported actuals and an external harness run can assess them; broader adversarial/tampering studies remain separately reviewed work. For external runtime interoperability, the next approved slice should be policy tests or execution-path validation for the bounded adapter path. @@ -229,4 +233,5 @@ For operator UX, future slices should focus on reviewability, export polish, and - **[done] Local backend telemetry probe validation gate (CR-119)**: Every emitted local backend probe result now validates against the CR-118 record contract before it is treated as an observation; validation failure raises a fail-closed `ProbeInputError`, with no generation calls, routing integration, or ledger writes. - **[done] Reviewer checkpoint or release-hygiene slice (CR-120, docs-only)**: Froze the completed CR-117 through CR-119 lane in a concise operations note, corrected stale CR-114 telemetry wording, and kept future telemetry work behind a new explicit scope pass instead of adding more features. - **[done] Eval scoring scope boundary (CR-123, docs-only)**: Scoped the first future scoring-capable eval slice as deterministic explicit-file comparison between validated fixtures and already-exported actual-outcome records, with no model/backend calls, routing, admission, ledger writes, approval state changes, adversarial expansion, or self-certification claims. -- **Next slice TBD by scope pass**: Do not start routing integration, ledger integration, circuit breakers, automatic discovery, background polling, daily-driver enforcement, or additional telemetry behavior without a new approved CR. +- **[done] Honor terminal resilience routes (CR-124)**: `human_handoff` and currently unimplemented `deterministic` routes now return a governed handoff before backend execution, recording `worker_result_status=not_attempted`; `tc run` reports the valid handoff with exit code 3. Approval-and-resume behavior, broader `human_review_required` semantics, and other execution seams remain future work. +- **Next slice TBD by scope pass**: Do not start approval-and-resume behavior, routing integration beyond the governed path, ledger integration, circuit breakers, automatic discovery, background polling, or additional telemetry behavior without a new approved CR. diff --git a/tests/test_client.py b/tests/test_client.py index 09af78d..fae8f63 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -40,7 +40,8 @@ def test_high_risk_task_is_handed_off_before_backend_runs(): assert result["status"] == "handoff_required" assert result["source"] == "router" - assert "Risk level high" in result["reason"] + assert "Human handoff required by the resilience route" in result["reason"] + assert result["route_reason"] == "sensitivity_requires_human_review" assert backend.called is False diff --git a/tests/test_tc_run_cli.py b/tests/test_tc_run_cli.py index 4179eef..509740c 100644 --- a/tests/test_tc_run_cli.py +++ b/tests/test_tc_run_cli.py @@ -13,6 +13,7 @@ import json from types import SimpleNamespace +from unittest.mock import patch import pytest import requests @@ -20,6 +21,7 @@ from triage_core import tc_cli from triage_core.backends import BackendResponse from triage_core.client import TriageClient +from triage_core.routing.resilience_router import ResilienceRouteDecision from triage_core.task_packet import PrivacyMetadata @@ -124,6 +126,57 @@ def test_handoff_required_exits_3(tmp_path): assert exc.value.code == 3 +@pytest.mark.parametrize( + ("selected_route", "reason"), + [ + ("human_handoff", "sensitivity_requires_human_review"), + ("deterministic", "deterministic_tool_available_for_task_class"), + ], +) +def test_terminal_routes_exit_3_without_backend_execution( + tmp_path, selected_route, reason +): + backend = FailingBackend() + client = TriageClient(backend=backend) + args = _args( + "Summarize public text", + privacy="public", + allow_cloud=True, + ledger_dir=str(tmp_path), + ) + decision = ResilienceRouteDecision( + selected_route=selected_route, + reason=reason, + fallback_depth=0, + human_review_required=selected_route == "human_handoff", + ) + + with patch("triage_core.classifier.TaskClassifier.classify", return_value="general"): + with patch( + "triage_core.client.choose_resilience_route", return_value=decision + ): + with patch.object( + client.router.specialist, + "route_task", + return_value={"offload_recommended": False}, + ): + with pytest.raises(SystemExit) as exc: + tc_cli.tc_run(args, client=client) + + assert exc.value.code == 3 + assert backend.called is False + + events = [ + json.loads(line) + for line in (tmp_path / "ledger.jsonl").read_text(encoding="utf-8").splitlines() + if line.strip() + ] + worker_result = next(event for event in events if event["event_type"] == "worker_result") + assert worker_result["payload"]["selected_route"] == selected_route + assert worker_result["payload"]["worker_result_status"] == "not_attempted" + assert worker_result["payload"]["failure_type"] == "safety_handoff" + + def test_high_risk_local_only_fails_closed_exit_2(tmp_path): backend = FailingBackend() client = TriageClient(backend=backend) diff --git a/triage_core/client.py b/triage_core/client.py index d1b13d0..3632bee 100644 --- a/triage_core/client.py +++ b/triage_core/client.py @@ -165,7 +165,7 @@ def run_task( signing_registry=route_decision_signing_registry, signing_agent_id=route_decision_signing_agent_id, ) - + steward = ProjectSteward() steward_eval = steward.evaluate(task_prompt=prompt, target_files=[], completed_orders=[]) if steward_eval["local_result_status"] == "insufficient": @@ -195,6 +195,37 @@ def run_task( ) return self._merge_route_fields(result, route_payload) + if selected_route in {"human_handoff", "deterministic"}: + if selected_route == "human_handoff": + reason = ( + "Human handoff required by the resilience route" + f": {resilience_decision.reason}" + ) + else: + reason = ( + "Deterministic route selected, but no deterministic executor is " + f"wired into the governed loop: {resilience_decision.reason}" + ) + result = { + "status": "handoff_required", + "source": "router", + "reason": reason, + "handoff_reason": reason, + "backend_name": getattr(self.engine.backend, "name", None), + "model": getattr(self.engine.backend, "model", None), + "timeout_seconds": use_timeout, + "worker_result_status": "not_attempted", + "failure_type": "safety_handoff", + "failure_stage": "router", + } + self._append_optional_event( + ledger=ledger, + task_id=task_id, + event_type="worker_result", + payload=build_worker_result_payload(route_payload, result), + ) + return self._merge_route_fields(result, route_payload) + if route_decision.get("offload_recommended", False): reason = f"Router bypass: {route_decision.get('reason')}" result = { From 0e573382c36d2f2413cf036c2b99abed17599a0c Mon Sep 17 00:00:00 2001 From: coreytshaffer <78175888+coreytshaffer@users.noreply.github.com> Date: Sat, 11 Jul 2026 23:00:17 -0700 Subject: [PATCH 3/3] Preflight privacy before ledger persistence --- README.md | 2 +- docs/change/change_log.md | 1 + ...light-privacy-before-ledger-persistence.md | 35 +++++++++++++ docs/current_backlog.md | 7 ++- docs/evidence_schema.md | 2 +- docs/operations/reviewer-traceability.md | 8 +-- tests/test_audit_cli.py | 27 ++++++++++ tests/test_privacy_invariants.py | 18 +++++++ tests/test_tc_run_cli.py | 49 +++++++++++++++++++ triage_core/local_backend_probe.py | 9 ++-- triage_core/privacy_invariants.py | 37 ++++++++++++++ triage_core/tc_cli.py | 42 ++++++++++------ 12 files changed, 211 insertions(+), 26 deletions(-) create mode 100644 docs/change/requests/CR-125-preflight-privacy-before-ledger-persistence.md diff --git a/README.md b/README.md index 0a6dec3..8f1e337 100644 --- a/README.md +++ b/README.md @@ -165,7 +165,7 @@ Expected outputs: - `tc audit --self-test` writes one privacy-safe `route_audit` event. - `tc audit --kind route_audit --last 10` shows routing metadata without raw prompt/data leakage. - `tc audit --kind demo_dry_run --last 5` shows the deterministic demo evidence without raw request or proposed-output content. -- `tc audit --privacy-invariants` scans the persistent ledger for forbidden raw-content keys and confirms the CR-021 invariant still holds. +- `tc audit --privacy-invariants` scans the persistent ledger for forbidden raw-content keys and high-confidence PII, credential, and precise-location value patterns; it does not classify arbitrary free text. - `triagecore benchmark --list-only` shows the benchmark fixture set without contacting a backend. - `tc model check` validates the documented manifest example locally. - `tc model warn` provides warning-only route/manifest comparison visibility and diff --git a/docs/change/change_log.md b/docs/change/change_log.md index 1d39fe2..c2d8b97 100644 --- a/docs/change/change_log.md +++ b/docs/change/change_log.md @@ -5,6 +5,7 @@ This file provides a chronological, human-readable record of applied codebase an *Note: For operational task and run history, consult `.triagecore/ledger.jsonl`.* ## [Unreleased] +- Implemented CR-125 (Preflight Privacy Before Ledger Persistence): Moved `tc run` packet verification ahead of ledger construction and task-event creation, so privacy-blocked input exits before a ledger event is written. Successful runs now persist fixed content-withheld task metadata plus prompt/data lengths rather than raw input. Extended the persistent-artifact invariant and its audit command to reject forbidden raw-content keys and high-confidence PII, credential, and precise-location value patterns without echoing detected values. Historical records are not rewritten, and arbitrary free-text classification remains out of scope. - Implemented CR-124 (Honor Terminal Resilience Routes): Made `human_handoff` and the currently unimplemented `deterministic` route terminal at the governed `TriageClient.run_task` boundary. Each now records the route decision plus a `worker_result` with `worker_result_status=not_attempted`, returns `handoff_required`, and reaches the existing `tc run` exit-3 contract without invoking a local or cloud backend. This slice does not add approval-and-resume behavior, change the broader `human_review_required` semantics, or alter cloud gating, signatures, reducer behavior, or other execution surfaces. - Implemented CR-123 (Eval Scoring Scope Boundary): Added a docs-only scope boundary for the first future safety-boundary scoring slice, limiting it to deterministic explicit-file comparison between validated `eval_case_v0` fixtures and already-exported actual-outcome records. Preserved the external evaluator boundary and kept model/backend calls, runtime probing, routing, admission, ledger writes, approval state, adversarial expansion, and any self-certification claims out of scope. - Implemented CR-122 (Eval Fixture Validation CLI): Added `tc eval validate-fixtures --input ` as a narrow read-only wrapper around the CR-121 JSONL validator, printing a bounded pass count on valid fixtures and fail-closed, line-aware diagnostics on invalid input. Added synthetic CLI tests only and kept scoring, observed-behavior comparison, model/backend calls, routing/admission integration, ledger writes, runtime behavior, and adversarial/tampering expansion out of scope. diff --git a/docs/change/requests/CR-125-preflight-privacy-before-ledger-persistence.md b/docs/change/requests/CR-125-preflight-privacy-before-ledger-persistence.md new file mode 100644 index 0000000..123961a --- /dev/null +++ b/docs/change/requests/CR-125-preflight-privacy-before-ledger-persistence.md @@ -0,0 +1,35 @@ +# CR-125: Preflight Privacy Before Ledger Persistence + +## Status + +Implemented + +## Summary + +Preflight the complete `tc run` `TaskPacket` before constructing or writing to +the ledger. Privacy-blocked input now exits with code 2 before a ledger event is +created. Successful runs store a fixed content-withheld task marker and input +lengths instead of raw prompt or data text. + +## Scope + +- Verify the `tc run` packet before ledger construction and task-event writes. +- Replace persisted `tc run` prompt/data text with fixed metadata and lengths. +- Extend the persistent-artifact invariant with high-confidence PII, credential, + and precise-location value patterns from the privacy scanner. +- Make `tc audit --privacy-invariants` report those value-pattern findings + without echoing content. +- Add offline tests for prompt/data absence and blocked-run no-residue behavior. + +## Non-Goals + +- No historical ledger rewrite or migration. +- No full DLP engine or arbitrary free-text safety classifier. +- No hash or content fingerprint of prompt/data input. +- No routing, approval, signature, or external-runtime behavior change. + +## Validation + +- `python -m pytest -q tests/test_tc_run_cli.py tests/test_privacy_invariants.py tests/test_audit_cli.py` +- `python -m pytest -q` +- `git diff --check` diff --git a/docs/current_backlog.md b/docs/current_backlog.md index 3f5cbb8..933470a 100644 --- a/docs/current_backlog.md +++ b/docs/current_backlog.md @@ -2,10 +2,14 @@ ## Status -This document summarizes the active TriageCore backlog after CR-124. +This document summarizes the active TriageCore backlog after CR-125. ## Active GitHub Backlog +- CR-125: Preflight Privacy Before Ledger Persistence + - Status: complete via CR-125 + - Purpose: Preflight the complete `tc run` packet before any ledger write, replace persisted prompt/data text with fixed metadata and lengths, and extend the persistent-artifact audit to reject forbidden keys plus high-confidence PII, credential, and precise-location value patterns. Historical records are not rewritten, and arbitrary free-text safety classification remains out of scope. + - CR-124: Honor Terminal Resilience Routes - Status: complete via CR-124 - Purpose: Make `human_handoff` and the currently unimplemented `deterministic` resilience routes terminal at `TriageClient.run_task`: record the route decision and a `worker_result` with `not_attempted`, return `handoff_required`, and ensure `tc run` exits 3 without invoking any backend. This slice does not add an approval-and-resume workflow or make every `human_review_required` flag pre-execution blocking. @@ -234,4 +238,5 @@ For operator UX, future slices should focus on reviewability, export polish, and - **[done] Reviewer checkpoint or release-hygiene slice (CR-120, docs-only)**: Froze the completed CR-117 through CR-119 lane in a concise operations note, corrected stale CR-114 telemetry wording, and kept future telemetry work behind a new explicit scope pass instead of adding more features. - **[done] Eval scoring scope boundary (CR-123, docs-only)**: Scoped the first future scoring-capable eval slice as deterministic explicit-file comparison between validated fixtures and already-exported actual-outcome records, with no model/backend calls, routing, admission, ledger writes, approval state changes, adversarial expansion, or self-certification claims. - **[done] Honor terminal resilience routes (CR-124)**: `human_handoff` and currently unimplemented `deterministic` routes now return a governed handoff before backend execution, recording `worker_result_status=not_attempted`; `tc run` reports the valid handoff with exit code 3. Approval-and-resume behavior, broader `human_review_required` semantics, and other execution seams remain future work. +- **[done] Preflight privacy before ledger persistence (CR-125)**: `tc run` now scans its complete packet before opening the ledger, persists only metadata and input lengths, and extends the persistent artifact audit with high-confidence sensitive-value detection. Historical records remain unchanged; arbitrary free-text safety classification and a full DLP engine remain out of scope. - **Next slice TBD by scope pass**: Do not start approval-and-resume behavior, routing integration beyond the governed path, ledger integration, circuit breakers, automatic discovery, background polling, or additional telemetry behavior without a new approved CR. diff --git a/docs/evidence_schema.md b/docs/evidence_schema.md index c0939ad..14a91c9 100644 --- a/docs/evidence_schema.md +++ b/docs/evidence_schema.md @@ -19,7 +19,7 @@ The schema supports two project goals: | `created_at` | Timestamp when the task entered the ledger. | | `updated_at` | Timestamp of the most recent ledger event applied to the task record. | | `title` | Human-readable task title. | -| `description` | Prompt or task summary. | +| `description` | Operator-safe task summary. `tc run` stores a fixed content-withheld marker rather than prompt text. | | `target_files` | Files or artifacts in scope. | | `runner` | Execution path, such as `local_benchmark`, `local_llm`, `council`, `codex`, or `antigravity`. | | `status` | Current state: `pending`, `local_draft_generated`, `handoff_generated`, `reviewed`, or `blocked`. | diff --git a/docs/operations/reviewer-traceability.md b/docs/operations/reviewer-traceability.md index 06c6bd8..d592b60 100644 --- a/docs/operations/reviewer-traceability.md +++ b/docs/operations/reviewer-traceability.md @@ -14,17 +14,17 @@ A completed trace shows that the recorded evidence is present, linked by `task_i ## The Trace Chain -1. **Input.** A task enters as a `TaskPacket` ([task_packet.py](../../triage_core/task_packet.py)) with explicit privacy metadata, or as a task envelope ([task_envelope.py](../../triage_core/task_envelope.py)). Raw prompt and data content are never persisted to the ledger. +1. **Input.** A task enters as a `TaskPacket` ([task_packet.py](../../triage_core/task_packet.py)) with explicit privacy metadata, or as a task envelope ([task_envelope.py](../../triage_core/task_envelope.py)). `tc run` preflights the packet before opening the ledger and persists prompt/data lengths rather than their contents. 2. **Route decision and evidence record.** `TriageClient.run_task` ([client.py](../../triage_core/client.py)) privacy-scans the packet and appends metadata-only events to the append-only ledger at `.triagecore/ledger.jsonl` via `TaskLedger` ([task_ledger.py](../../triage_core/task_ledger.py)): a `route_audit` event for the allow/block outcome, a `route_decision` event for the routing choice (optionally signed), then `worker_result` and `validator_completed` events for execution and validation outcomes. 3. **Reduced record.** `TaskLedger._apply_event` folds all events for one `task_id` into a single `TaskRecord`. The field meanings are documented in [evidence_schema.md](../evidence_schema.md). 4. **Approval or rejection state.** When classification, routing, or a handoff sets `human_review_required`, the task appears in the review queue ([review_queue.py](../../triage_core/review_queue.py), `tc review list`) until a `review_completed` event records `review_decision` (`accepted`, `accepted_with_minor_edits`, or `rejected`) and `task_outcome`. -5. **Verification and test evidence.** `tc audit --privacy-invariants` checks that persisted events contain no forbidden raw-content keys; `tc audit --verify-signatures` checks signed event provenance; the pytest suite (including the traceability regression test below) is the repeatable test evidence. +5. **Verification and test evidence.** `tc audit --privacy-invariants` checks persisted events for forbidden raw-content keys and high-confidence PII, credential, and precise-location value patterns; it does not prove that arbitrary free text is safe. `tc audit --verify-signatures` checks signed event provenance; the pytest suite (including the traceability regression test below) is the repeatable test evidence. ## Ledger Event Types That Carry the Trace | Event type | Typically appended by | Trace evidence it contributes | | --- | --- | --- | -| `task_created` | `TriageClient.run_task`, benchmark/pipeline runners | `created_at`, `title`, `description`, `target_files` | +| `task_created` | `TriageClient.run_task`, benchmark/pipeline runners | `created_at`, metadata-only title/description, prompt/data lengths, `target_files` | | `task_classified` | classification step in `run_task` | `risk_level`, `permission_profile`; medium/high risk sets `human_review_required` | | `route_audit` | privacy/route gate in `run_task` ([route_audit.py](../../triage_core/route_audit.py)) | allow/block `decision` and `reason_code`; inspected as raw events via `tc audit --kind route_audit`, not reduced into named `TaskRecord` fields | | `route_decision` | `run_task` via [routing/route_events.py](../../triage_core/routing/route_events.py); optionally signed | `selected_route`, `route_reason`, `route_source`, `fallback_depth`, `selected_backend`, model; can set `human_review_required` | @@ -61,7 +61,7 @@ tc audit --kind route_decision --last 10 # routing choices and route source tc audit --kind worker_result --last 10 # execution outcome metadata tc audit --kind review_completed --last 10 # recorded approval/rejection state tc review list # tasks still awaiting a review decision -tc audit --privacy-invariants # persisted evidence contains no raw content +tc audit --privacy-invariants # checks prohibited keys and high-confidence sensitive values tc audit --verify-signatures --kind route_decision # provenance check for signed events, when present tc task show # show one task's complete evidence chain (displays ledger evidence, does not verify signatures) ``` diff --git a/tests/test_audit_cli.py b/tests/test_audit_cli.py index 8145d5b..3dcd4bb 100644 --- a/tests/test_audit_cli.py +++ b/tests/test_audit_cli.py @@ -559,6 +559,33 @@ def test_audit_privacy_invariants_fails_without_echoing_sensitive_values(tmp_pat assert sensitive_value not in out +def test_audit_privacy_invariants_flags_sensitive_value_without_echoing_it(tmp_path, capsys): + ledger_path = tmp_path / "ledger.jsonl" + sensitive_value = "Contact jane@example.com" + ledger_path.write_text( + json.dumps( + { + "timestamp": "2026-06-12T00:00:00+00:00", + "task_id": "unsafe-task", + "event_type": "task_created", + "payload": {"description": sensitive_value}, + } + ) + + "\n", + encoding="utf-8", + ) + + with patch("triage_core.tc_cli._ledger_path", return_value=ledger_path): + with pytest.raises(SystemExit) as exc: + tc_audit_privacy_invariants() + + assert exc.value.code == 1 + out = capsys.readouterr().out + assert "$.payload.description" in out + assert "reason=email_pattern" in out + assert sensitive_value not in out + + def test_audit_privacy_invariants_flags_malformed_json(tmp_path, capsys): ledger_path = tmp_path / "ledger.jsonl" ledger_path.write_text("{not json\n", encoding="utf-8") diff --git a/tests/test_privacy_invariants.py b/tests/test_privacy_invariants.py index fa6a122..2185113 100644 --- a/tests/test_privacy_invariants.py +++ b/tests/test_privacy_invariants.py @@ -61,6 +61,24 @@ def test_rejects_raw_content_inside_list(): assert "$.events[1].content" in str(exc.value) +@pytest.mark.parametrize( + ("value", "reason"), + [ + ("Contact jane@example.com", "email_pattern"), + ("SSN 123-45-6789", "ssn_pattern"), + ("token=super-secret-value", "secret_pattern"), + ], +) +def test_rejects_sensitive_persistent_values_without_echoing_them(value, reason): + with pytest.raises(PersistentPrivacyInvariantError) as exc: + assert_persistent_privacy_safe({"description": value}) + + violation = exc.value.violations[0] + assert violation.path == "$.description" + assert violation.reason == reason + assert value not in str(exc.value) + + @pytest.mark.parametrize( "key", [ diff --git a/tests/test_tc_run_cli.py b/tests/test_tc_run_cli.py index 509740c..3a887ea 100644 --- a/tests/test_tc_run_cli.py +++ b/tests/test_tc_run_cli.py @@ -84,6 +84,14 @@ def _event_types(ledger_path): ] +def _ledger_events(ledger_path): + return [ + json.loads(line) + for line in ledger_path.read_text(encoding="utf-8").splitlines() + if line.strip() + ] + + class RecordingClient: def __init__(self): self.packet = None @@ -116,6 +124,47 @@ def test_local_success_exits_zero_and_records_governed_evidence(tmp_path, capsys assert "Success: task ran locally" in capsys.readouterr().out +def test_tc_run_persists_metadata_without_prompt_or_data_content(tmp_path): + prompt = "PROMPT_SENTINEL_ONLY_FOR_PRIVACY_TEST" + data = "DATA_SENTINEL_ONLY_FOR_PRIVACY_TEST" + backend = RecordingBackend() + client = TriageClient(backend=backend) + args = _args(prompt, data=data, ledger_dir=str(tmp_path)) + + tc_cli.tc_run(args, client=client) + + ledger_file = tmp_path / "ledger.jsonl" + ledger_text = ledger_file.read_text(encoding="utf-8") + assert prompt not in ledger_text + assert data not in ledger_text + + task_created = next( + event + for event in _ledger_events(ledger_file) + if event["event_type"] == "task_created" + ) + assert task_created["payload"] == { + "title": "tc run task", + "description": "Prompt content withheld from ledger.", + "prompt_length": len(prompt), + "data_length": len(data), + "target_files": [], + } + + +def test_tc_run_privacy_block_leaves_no_ledger_event(tmp_path): + backend = FailingBackend() + client = TriageClient(backend=backend) + args = _args("Process this record", data="123-45-6789", ledger_dir=str(tmp_path)) + + with pytest.raises(SystemExit) as exc: + tc_cli.tc_run(args, client=client) + + assert exc.value.code == 2 + assert backend.called is False + assert not (tmp_path / "ledger.jsonl").exists() + + def test_handoff_required_exits_3(tmp_path): client = TriageClient(backend=ErrorBackend()) args = _args("Summarize this text", ledger_dir=str(tmp_path)) diff --git a/triage_core/local_backend_probe.py b/triage_core/local_backend_probe.py index 8f67b7f..5db79df 100644 --- a/triage_core/local_backend_probe.py +++ b/triage_core/local_backend_probe.py @@ -216,16 +216,19 @@ def local_backend_probe_record_from_mapping( This is a pure validator/mapper for already-recorded metadata. It does not probe endpoints, write artifacts, route tasks, or call a model. """ + _reject_unknown_record_fields(payload) + base_url = _mapping_optional_base_url(payload, "base_url") + normalized_payload = dict(payload) + normalized_payload["base_url"] = base_url assert_persistent_privacy_safe( - dict(payload), + normalized_payload, artifact_name="local backend probe record", ) - _reject_unknown_record_fields(payload) return LocalBackendProbeRecord( schema_version=_mapping_text(payload, "schema_version"), source_type=_mapping_text(payload, "source_type"), - base_url=_mapping_optional_base_url(payload, "base_url"), + base_url=base_url, reachable=_mapping_bool(payload, "reachable"), evidence_tier=_mapping_text(payload, "evidence_tier"), model_count=_mapping_optional_int(payload, "model_count"), diff --git a/triage_core/privacy_invariants.py b/triage_core/privacy_invariants.py index a1b344b..2a2a58d 100644 --- a/triage_core/privacy_invariants.py +++ b/triage_core/privacy_invariants.py @@ -4,6 +4,16 @@ from dataclasses import dataclass from typing import Any +from triage_core.privacy_scanner import ( + CC_CANDIDATE_REGEX, + EMAIL_REGEX, + LAT_LON_REGEX, + PHONE_REGEX, + SECRET_KEYS_REGEX, + SSN_REGEX, + is_valid_luhn, +) + FORBIDDEN_PERSISTENT_KEYS = frozenset( { @@ -38,6 +48,7 @@ class PrivacyInvariantViolation: path: str key: str + reason: str = "forbidden_key" class PersistentPrivacyInvariantError(ValueError): @@ -88,10 +99,36 @@ def find_forbidden_persistent_fields( violations.extend( find_forbidden_persistent_fields(child, path=f"{path}[{index}]") ) + elif isinstance(value, str): + reason = _sensitive_persistent_value_reason(value) + if reason: + violations.append( + PrivacyInvariantViolation( + path=path, + key="", + reason=reason, + ) + ) return violations +def _sensitive_persistent_value_reason(value: str) -> str | None: + if SSN_REGEX.search(value): + return "ssn_pattern" + if EMAIL_REGEX.search(value): + return "email_pattern" + if PHONE_REGEX.search(value): + return "phone_pattern" + if SECRET_KEYS_REGEX.search(value): + return "secret_pattern" + if LAT_LON_REGEX.search(value): + return "precise_location_pattern" + if any(is_valid_luhn(match.group()) for match in CC_CANDIDATE_REGEX.finditer(value)): + return "credit_card_pattern" + return None + + def assert_persistent_privacy_safe( payload: Any, *, diff --git a/triage_core/tc_cli.py b/triage_core/tc_cli.py index 0034bbc..c007b44 100644 --- a/triage_core/tc_cli.py +++ b/triage_core/tc_cli.py @@ -407,7 +407,8 @@ def tc_audit_privacy_invariants() -> None: f"task={record.get('task_id', 'unknown')} " f"event_type={record.get('event_type', 'unknown')} " f"path={violation.path} " - f"key={violation.key}" + f"key={violation.key} " + f"reason={violation.reason}" ) except Exception as e: print(f"Error reading {ledger_path}: {e}") @@ -1011,6 +1012,7 @@ def tc_run(args, client=None) -> None: from triage_core.safe_task_packet import ( LocalRouteUnavailableError, UnsafePacketError, + verify_packet, ) # 1. Assemble task data from --files (repeatable), then --data. @@ -1046,7 +1048,21 @@ def tc_run(args, client=None) -> None: external_model_allowed=allow_external_model, ) - # 3. Ledger wiring (enabled by default; --no-ledger warns). + # 3. Build and preflight the packet before any evidence is persisted. + task_id = args.task_id or str(uuid.uuid4()) + packet = TaskPacket( + prompt=args.prompt, + data=data, + task_id=task_id, + privacy_metadata=privacy_metadata, + ) + try: + verify_packet(packet) + except PrivacyViolationError as exc: + print(f"Blocked (privacy fail-closed): {exc}") + sys.exit(2) + + # 4. Ledger wiring (enabled by default; --no-ledger warns). ledger = None if args.no_ledger: print( @@ -1061,22 +1077,23 @@ def tc_run(args, client=None) -> None: print(f"Error: could not open ledger at '{ledger_dir}': {exc}") sys.exit(1) - # 4. Task identity + evidence scaffolding (mirrors the pipeline runner). - task_id = args.task_id or str(uuid.uuid4()) + # 5. Task identity + evidence scaffolding (mirrors the pipeline runner). if ledger is not None: if not args.task_id or not ledger.get_task(task_id): ledger.append_event( task_id, "task_created", { - "title": f"tc run: {args.prompt[:40]}", - "description": args.prompt, + "title": "tc run task", + "description": "Prompt content withheld from ledger.", + "prompt_length": len(args.prompt), + "data_length": len(data), "target_files": args.files, }, ) ledger.append_event(task_id, "runner_selected", {"runner": "tc_run"}) - # 5. Build the client from config unless one was injected (tests inject). + # 6. Build the client from config unless one was injected (tests inject). if client is None: from triage_core.client import TriageClient @@ -1090,14 +1107,7 @@ def tc_run(args, client=None) -> None: print(f"Error: could not initialize backend: {exc}") sys.exit(1) - packet = TaskPacket( - prompt=args.prompt, - data=data, - task_id=task_id, - privacy_metadata=privacy_metadata, - ) - - # 6. Governed loop. Privacy/safety failures fail closed (exit 2). + # 7. Governed loop. Privacy/safety failures fail closed (exit 2). try: result = client.run_task(task_packet=packet, ledger=ledger, task_id=task_id) except PrivacyViolationError as exc: @@ -1110,7 +1120,7 @@ def tc_run(args, client=None) -> None: print(f"Blocked (unsafe packet, failing closed): {exc}") sys.exit(2) - # 7. Interpret outcome. + # 8. Interpret outcome. route = result.get("selected_route") if result.get("status") == "success": output = result.get("output", "")