From 0aa257a2cdcdb5dd63cf80f7b7f07c0db61abedc Mon Sep 17 00:00:00 2001 From: coreytshaffer <78175888+coreytshaffer@users.noreply.github.com> Date: Thu, 9 Jul 2026 22:58:22 -0700 Subject: [PATCH] Implement CR-119 telemetry probe validation gate --- docs/change/change_log.md | 1 + ...ocal-backend-telemetry-probe-validation.md | 51 ++++++++++++++ docs/current_backlog.md | 6 +- docs/operations/local-backend-telemetry.md | 6 ++ tests/test_local_backend_probe.py | 70 +++++++++++++++++++ triage_core/local_backend_probe.py | 15 +++- 6 files changed, 145 insertions(+), 4 deletions(-) create mode 100644 docs/change/requests/CR-119-local-backend-telemetry-probe-validation.md diff --git a/docs/change/change_log.md b/docs/change/change_log.md index 5f40cb4..9287fe3 100644 --- a/docs/change/change_log.md +++ b/docs/change/change_log.md @@ -5,6 +5,7 @@ This file provides a chronological, human-readable record of applied codebase an *Note: For operational task and run history, consult `.triagecore/ledger.jsonl`.* ## [Unreleased] +- Implemented CR-119 (Local Backend Telemetry Probe Validation Gate): Added a fail-closed emission gate for local backend telemetry probe results so every `LocalBackendProbeRecord` round-trips through the CR-118 `local_backend_probe_record.v1` validator before the probe returns, renders, or writes it as an observation. The gate applies to both reachable records and fail-closed probe observations, and contract-validation failure raises `ProbeInputError` instead of producing a successful probe result. No new backend types, probe endpoints, model calls, routing integration, circuit breakers, degraded modes, ledger writes, or CLI behavior were added. - Implemented CR-117 (Task-Show Opt-In Signature Verification): Added an opt-in `tc task show --verify-signatures` path that verifies the signed ledger events (`route_audit`, `validation_result`, `route_decision`) belonging to the shown task via a new task-scoped `verify_task_event_signatures` helper, reusing the CR-097 fail-closed identity-registry-load categories. Fail-closed: invalid or malformed signatures — and identity-registry load failures — exit 1; unsigned signed-type events remain informational and exit 0; there is no `--strict` in this slice. Flag-off output is byte-for-byte unchanged, and the whole-ledger `tc audit --verify-signatures` behavior is untouched. Telemetry lane references shift to CR-118+ (schema candidate CR-118, probe candidate CR-119+) in active planning and handoff docs. Read-only and evidence-only: no changes to approval, signing, routing, or execution semantics, and no new egress, backends, or ledger writes. - Implemented CR-115 (Final Control-Plane Extraction Package): Added four docs-only doctrine/handoff documents — a control-plane invariant checklist mapping each invariant to its enforcement locus and reviewer verification command, a repeatable outer-loop control review recipe with risk classification and stop conditions, a final capability note recording that the assistant model's role was drafting/review/evidence collection under plan-gated human control with no runtime dependency on any model, and a future agent/maintainer handoff entry point with ordered next slices and binding conventions. Telemetry lane references renumbered from CR-115+ to CR-116+. All documents preserve the distinction that signatures, manifests, evaluator verdicts, and model recommendations are evidence, not approval. No runtime code, tests, tags, or approval/signature/identity/routing/privacy/audit semantics were changed. - Implemented CR-114 (Reviewer Checkpoint 2026-07-07 and Tag Reconciliation): Added a docs-only dated reviewer checkpoint consolidating the CR-100 through CR-113 arc at HEAD `f8bf33c` — fresh local validation evidence (803 passed / 2 skipped full regression, privacy invariant pass over 698 ledger records, route_decision signature verification, identity listing, backend-free benchmark fixture listing, doctor runtime-safety postures) — and recorded the finding that the `v0.1.0-reviewer-checkpoint-2026-07-02` tag recommended by the previous checkpoint was never created, with the `355c521` anchor preserved in-document and exact operator-run tag commands recommended but deliberately not executed. No runtime code, tests, tags, or approval/signature/identity/routing/privacy/audit semantics were changed. diff --git a/docs/change/requests/CR-119-local-backend-telemetry-probe-validation.md b/docs/change/requests/CR-119-local-backend-telemetry-probe-validation.md new file mode 100644 index 0000000..743007b --- /dev/null +++ b/docs/change/requests/CR-119-local-backend-telemetry-probe-validation.md @@ -0,0 +1,51 @@ +# CR-119: Local Backend Telemetry Probe Validation Gate + +## Status + +Implemented (probe-observability hardening) + +## Summary + +Require every local backend telemetry probe result to satisfy the CR-118 +`local_backend_probe_record.v1` contract before the probe treats it as an +emitted observation. The probe still contacts only explicit local metadata +endpoints, never invokes a model, never writes the ledger, and does not feed +routing or execution. + +## Scope + +- Add a shared emitted-record validation gate in `triage_core/local_backend_probe.py`. +- Validate each `LocalBackendProbeRecord.to_dict()` through + `local_backend_probe_record_from_mapping(...)` before returning it from the + probe path. +- Apply the gate to both reachable and fail-closed records such as + `timeout`, `endpoint_unreachable`, `malformed_response`, + `permission_or_policy_blocked`, `probe_disabled`, and + `unsupported_backend`. +- If record-contract validation fails, raise `ProbeInputError` so CLI callers + exit 1 instead of rendering or writing an invalid observation. +- Add offline tests using injected transports and monkeypatched validators; + no CR-119 test contacts a real backend or model endpoint. + +## Non-Goals + +- No new backend source types. +- No model generation calls, chat calls, completion calls, or embeddings. +- No routing integration, route-input population, circuit breakers, degraded + modes, daily-driver enforcement, or ledger writes. +- No new serialized fields or changes to the CR-118 schema. +- No background polling or automatic endpoint discovery. + +## Validation + +- `tests/test_local_backend_probe.py` proves reachable and failed probe + results pass through the CR-118 validator before being emitted. +- A validator failure raises `ProbeInputError`, preserving fail-closed CLI + behavior. + +## Notes + +This slice keeps telemetry as explicit-endpoint observability only. A valid +probe record says that a metadata endpoint produced a contract-valid +observation at one moment; it is not a benchmark, routing authorization, +availability guarantee, model-quality claim, or safety claim. diff --git a/docs/current_backlog.md b/docs/current_backlog.md index 871c9a6..70df523 100644 --- a/docs/current_backlog.md +++ b/docs/current_backlog.md @@ -15,7 +15,7 @@ This document summarizes the active TriageCore backlog after CR-115. - Purpose: Consolidate the CR-100 through CR-113 route-worker telemetry and runtime strategy lanes into a dated reviewer checkpoint at HEAD `f8bf33c` with fresh validation evidence (803 passed / 2 skipped, privacy invariants, signature verification, identity list, backend-free benchmark listing), and reconcile the never-created `v0.1.0-reviewer-checkpoint-2026-07-02` tag by preserving the `355c521` anchor in-document and recommending exact tag commands without running them. Note: CR-114 was claimed by this checkpoint slice at commit time, shifting the telemetry probe implementation candidate previously referenced as "CR-114+"; after CR-115 was claimed by the extraction package and CR-117 by the task-show signature-verification slice, the telemetry lane is now CR-118+. - CR-113: Local Backend Telemetry Design Brief - - Status: complete via CR-113 (docs-only); telemetry probe implementation remains a gated future candidate (CR-118+) + - Status: complete via CR-113 (docs-only); CR-118 pinned the serialized record contract, and CR-119 gates emitted probe results through that contract before treating them as observations - Purpose: Define the boundaries for a future read-only local backend telemetry slice — metadata-only availability and model/runtime identity observations for Ollama, LM Studio, and llama.cpp local endpoints — including candidate fields with privacy considerations, a closed failure vocabulary, evidence-tier provenance, an opt-in probe posture, and a reviewer path that requires no model execution, before any probe code is written. - CR-112: Recorded Runtime Strategy Report Export @@ -209,6 +209,6 @@ For operator UX, future slices should focus on reviewability, export polish, and ## Next Candidate Slices - **[done] Signature verification on task show (CR-117, runtime-safe)**: Opt-in `tc task show --verify-signatures` verifies the shown task's signed ledger events via a task-scoped helper that reuses the CR-097 fail-closed categories; fail-closed (exits 1) on invalid or malformed signatures and registry-load failure, while unsigned signed-type events stay informational (exit 0). Whole-ledger `tc audit --verify-signatures` behavior is unchanged ([task-show-signature-verification.md](operations/task-show-signature-verification.md)). -- **Telemetry schema and synthetic-fixture validation (CR-118 candidate)**: Harden the existing local backend probe's serialized record contract before any further probe work — strict schema, pure validator, synthetic fixtures only, no endpoint calls, no routing integration, no ledger writes, and no CLI behavior changes. -- **Local backend telemetry probe follow-up (CR-119+ candidate)**: Keep any probe behavior changes bounded by the CR-113 design brief ([local-backend-telemetry.md](operations/local-backend-telemetry.md)) — opt-in, explicit-endpoint, fail-closed, no generation calls, no ledger writes. +- **[done] Telemetry schema and synthetic-fixture validation (CR-118)**: Hardened the existing local backend probe's serialized record contract before any further probe work — strict schema, pure validator, synthetic fixtures only, no endpoint calls, no routing integration, no ledger writes, and no CLI behavior changes. +- **[done] Local backend telemetry probe validation gate (CR-119)**: Every emitted local backend probe result now validates against the CR-118 record contract before it is treated as an observation; validation failure raises a fail-closed `ProbeInputError`, with no generation calls, routing integration, or ledger writes. - **Reviewer checkpoint or release-hygiene slice**: Freeze the completed route-worker reviewer lane into a concise checkpoint, packaging note, or release-hygiene update instead of adding more telemetry features. diff --git a/docs/operations/local-backend-telemetry.md b/docs/operations/local-backend-telemetry.md index 93ce063..4b274b5 100644 --- a/docs/operations/local-backend-telemetry.md +++ b/docs/operations/local-backend-telemetry.md @@ -23,6 +23,9 @@ failure vocabulary, tiered evidence, and `base_url` redaction described here. **CR-118 pins the serialized record contract** with `schemas/local_backend_probe_record.schema.json` and a pure strict mapping validator for synthetic fixtures and recorded metadata. +**CR-119 gates probe emission through that contract**: every probe result is +validated against the CR-118 mapper before the probe returns, renders, or +writes it as an observation. Routing wiring, route-input population (`ResilienceRouteInput`), circuit breakers, degraded modes, and any daily-driver enforcement **remain future @@ -132,6 +135,9 @@ What a future reviewer should be able to verify without running a model: - **Record shape.** Deterministic `synthetic_fixture`-tier examples validate through the strict mapping path and schema contract, with no backend present at all. +- **Probe emission gate.** Every emitted probe result, including fail-closed + observations, round-trips through the CR-118 validator before it is treated + as a successful observation. - **Fail-closed behavior.** Each failure category is reachable in tests without a live backend (unsupported source type, disabled probe, and malformed fixture responses need no network; unreachable/timeout can use a diff --git a/tests/test_local_backend_probe.py b/tests/test_local_backend_probe.py index 4b0cf71..75d8d33 100644 --- a/tests/test_local_backend_probe.py +++ b/tests/test_local_backend_probe.py @@ -10,10 +10,12 @@ import pytest import requests +import triage_core.local_backend_probe as local_backend_probe from triage_core.local_backend_probe import ( ERROR_CATEGORIES, LocalBackendProbeRecord, ProbeInputError, + local_backend_probe_record_from_mapping, probe_local_backend, redact_base_url, render_probe_record, @@ -71,6 +73,31 @@ def test_reachable_ollama_counts_models_no_names_by_default(): assert rec.evidence_tier == "local_metadata_probe" +def test_successful_probe_result_is_validated_against_record_contract(monkeypatch): + observed_payloads = [] + + def spy(payload): + observed_payloads.append(dict(payload)) + return local_backend_probe_record_from_mapping(payload) + + monkeypatch.setattr( + local_backend_probe, + "local_backend_probe_record_from_mapping", + spy, + ) + + rec = probe_local_backend( + source_type="ollama", + base_url="http://localhost:11434", + transport=_ollama_ok, + ) + + assert rec.reachable is True + assert observed_payloads + assert observed_payloads[0]["schema_version"] == "local_backend_probe_record.v1" + assert observed_payloads[0]["source_type"] == "ollama" + + def test_openai_shape_backends_use_v1_models(): rec = probe_local_backend( source_type="lm_studio", @@ -125,6 +152,49 @@ def test_timeout_category(): assert rec.error_category == "timeout" +def test_failed_probe_result_is_validated_against_record_contract(monkeypatch): + observed_payloads = [] + + def spy(payload): + observed_payloads.append(dict(payload)) + return local_backend_probe_record_from_mapping(payload) + + monkeypatch.setattr( + local_backend_probe, + "local_backend_probe_record_from_mapping", + spy, + ) + + rec = probe_local_backend( + source_type="ollama", + base_url="http://localhost:11434", + transport=_timeout, + ) + + assert rec.reachable is False + assert rec.error_category == "timeout" + assert observed_payloads + assert observed_payloads[0]["error_category"] == "timeout" + + +def test_probe_result_contract_validation_failure_fails_closed(monkeypatch): + def reject(payload): + raise ValueError("schema drift") + + monkeypatch.setattr( + local_backend_probe, + "local_backend_probe_record_from_mapping", + reject, + ) + + with pytest.raises(ProbeInputError, match="CR-118 record-contract"): + probe_local_backend( + source_type="ollama", + base_url="http://localhost:11434", + transport=_ollama_ok, + ) + + def test_endpoint_unreachable_category(): rec = probe_local_backend( source_type="ollama", base_url="http://localhost:11434", transport=_refused diff --git a/triage_core/local_backend_probe.py b/triage_core/local_backend_probe.py index 7c47ec1..8f67b7f 100644 --- a/triage_core/local_backend_probe.py +++ b/triage_core/local_backend_probe.py @@ -321,7 +321,7 @@ def _record( response_latency_ms: Optional[int] = None, observed_at: Optional[str] = None, ) -> LocalBackendProbeRecord: - return LocalBackendProbeRecord( + record = LocalBackendProbeRecord( source_type=source_type, base_url=base_url, reachable=reachable, @@ -332,6 +332,19 @@ def _record( error_category=error_category, observed_at=observed_at, ) + return _validate_emitted_probe_record(record) + + +def _validate_emitted_probe_record( + record: LocalBackendProbeRecord, +) -> LocalBackendProbeRecord: + """Fail closed unless the emitted probe record satisfies the CR-118 contract.""" + try: + return local_backend_probe_record_from_mapping(record.to_dict()) + except (ProbeInputError, ValueError) as exc: + raise ProbeInputError( + "probe result failed CR-118 record-contract validation" + ) from exc def probe_local_backend(