From 1c30ccf00b6420b2168b748e6c795ac945b3ad40 Mon Sep 17 00:00:00 2001 From: pcvantol Date: Thu, 17 Sep 2026 10:43:59 +0200 Subject: [PATCH 1/6] fix: complete canonical telemetry exports --- docs/engineering/PROVIDER_USAGE_SEMANTICS.md | 39 +- .../telemetry-detail-example.json | 91 +++++ .../telemetry-detail-example.md | 51 +++ .../telemetry-overview-example.json | 48 +++ .../telemetry-overview-example.md | 32 ++ package-lock.json | 4 +- package.json | 2 +- pyproject.toml | 2 +- .../ENGINEERING_PLATFORM_CONFIG.json | 2 +- .../ENGINEERING_PLATFORM_VERSION.json | 8 +- src/engineering_platform/assets/dashboard.css | 2 + src/engineering_platform/assets/dashboard.js | 260 +++++++------ .../assets/dashboard_locales.mjs | 5 + .../execution_reporting.py | 26 +- src/engineering_platform/execution_timing.py | 89 ++++- src/engineering_platform/platform_version.py | 2 +- src/engineering_platform/provider_usage.py | 174 +++++++-- src/engineering_platform/server.py | 222 +++++++++-- .../telemetry_contract.py | 95 +++-- src/engineering_platform/telemetry_export.py | 363 ++++++++++++++++++ src/engineering_platform/telemetry_metrics.py | 185 +++++++++ .../templates/workspace-config.json | 2 +- tests/engineering/dashboard.spec.mjs | 161 +++++++- tests/engineering/test_server_foundation.py | 117 +++++- .../engineering/test_telemetry_contract_v2.py | 212 +++++++++- 25 files changed, 1907 insertions(+), 287 deletions(-) create mode 100644 docs/engineering/examples/telemetry-followup/telemetry-detail-example.json create mode 100644 docs/engineering/examples/telemetry-followup/telemetry-detail-example.md create mode 100644 docs/engineering/examples/telemetry-followup/telemetry-overview-example.json create mode 100644 docs/engineering/examples/telemetry-followup/telemetry-overview-example.md create mode 100644 src/engineering_platform/telemetry_export.py create mode 100644 src/engineering_platform/telemetry_metrics.py diff --git a/docs/engineering/PROVIDER_USAGE_SEMANTICS.md b/docs/engineering/PROVIDER_USAGE_SEMANTICS.md index 7830974d..8f4f80eb 100644 --- a/docs/engineering/PROVIDER_USAGE_SEMANTICS.md +++ b/docs/engineering/PROVIDER_USAGE_SEMANTICS.md @@ -3,7 +3,7 @@ ## Owning contract Engineering Platform owns the read model defined by -`telemetry-contract@2.0`. `provider_usage.py`, `execution_timing.py` and +`telemetry-contract@2.1`. `provider_usage.py`, `execution_timing.py` and `telemetry_contract.py` are the calculation authority. The API, dashboard, Engineering Report, Markdown download and JSON export project that contract; they must not independently calculate totals or bottlenecks. @@ -14,6 +14,13 @@ observations and coverage. Coverage is `COMPLETE`, `PARTIAL`, `UNAVAILABLE` or `CONFLICT`. `AUTHORITATIVE` describes provenance, not completeness. A measured zero, an unknown value and an unavailable observation are distinct. +Coverage aggregation retains expected, present, valid, missing and conflicting +observations. A source `CONFLICT` remains `CONFLICT` at run, UTC-day and chain +scope even when its numeric present count equals the expected count. Unknown +expected populations remain unknown. Independent metrics propagate only their +own dependencies; one invalid cached-input observation does not invalidate an +otherwise valid output observation. + The scopes are: - `EP_RUN_ATTEMPT`: exactly one EP execution attempt; @@ -100,6 +107,13 @@ segment; simultaneous independent categories become `PARALLEL_OVERLAP`; gaps become `UNASSIGNED`. No proportional rescaling is used. Complete compatible intervals close exactly on elapsed time except presentation rounding. +The exclusive distribution uses the wall-clock interval envelope for both its +positions and durations. The independently measured monotonic process duration +remains `total_monotonic_duration_ms`. Their signed difference is reported as +`clock_difference_ms`; it is never inserted into `UNASSIGNED`. Category +durations are accumulated at timestamp precision and rounded together only for +the integer-millisecond presentation, so no category can become negative. + Provider process duration is cumulative monotonic process lifetime. Provider coverage is the wall-clock interval union inside the run envelope. Neither is model inference time. Conflicting clocks or boundaries produce `CONFLICT` @@ -113,3 +127,26 @@ autonomy acceptance remain separate states. Immutable historical reports and qualification evidence are never rewritten. Read-time reprojection retains its calculation version and source reference; missing historical event metadata is never fabricated. + +## Export contract + +`telemetry-export@1.0` is the read-only export envelope for the overview and +detail Markdown/JSON downloads. Each envelope carries a snapshot digest, +as-of timestamp, project, UTC selection, scope, source references, displayed +and full population, export completeness and metric coverage. Markdown and +JSON serialize that one model; neither recalculates totals. + +The overview export contains every retained row in the active project/filter +scope rather than only the visible page. Detail export can select the UTC day, +one attempt, or its verified execution chain. Full exports disable the UI +preview limits for runs, invocations and spans. JSON retains numeric machine +values and `null`; Markdown localizes human headings and explicitly renders +unavailable values. Neither export includes prompts, replies, commands, +secrets, raw tool output, or span metadata outside the telemetry allow-list. + +Synthetic, secret-free examples are retained with this contract: + +- [overview Markdown](examples/telemetry-followup/telemetry-overview-example.md) +- [overview JSON](examples/telemetry-followup/telemetry-overview-example.json) +- [detail Markdown](examples/telemetry-followup/telemetry-detail-example.md) +- [detail JSON](examples/telemetry-followup/telemetry-detail-example.json) diff --git a/docs/engineering/examples/telemetry-followup/telemetry-detail-example.json b/docs/engineering/examples/telemetry-followup/telemetry-detail-example.json new file mode 100644 index 00000000..9511a99c --- /dev/null +++ b/docs/engineering/examples/telemetry-followup/telemetry-detail-example.json @@ -0,0 +1,91 @@ +{ + "as_of": "2026-09-17T10:00:00+00:00", + "completeness": { + "displayed_population": 1, + "export": "COMPLETE", + "full_population": 1, + "metrics_may_be_partial": true + }, + "contract_version": "telemetry-contract@2.1", + "data": { + "attempt": { + "run_id": "synthetic-run-1", + "status": "COMPLETE", + "telemetry_snapshot": { + "attempt": { + "run_id": "synthetic-run-1", + "scope": "EP_RUN_ATTEMPT", + "timing": { + "clock_difference_ms": 0, + "coverage": {"state": "COMPLETE"}, + "exclusive_distribution": [ + {"category": "PROVIDER_EXECUTION", "duration_ms": 6000, "share_percent": 60.0}, + {"category": "UNASSIGNED", "duration_ms": 4000, "share_percent": 40.0} + ], + "exclusive_distribution_closes": true, + "exclusive_envelope_duration_ms": 10000, + "exclusive_measurement_basis": "WALL_CLOCK_INTERVAL_ENVELOPE", + "timeline": [ + { + "completed_at": "2026-09-17T10:00:06+00:00", + "duration_ms": 6000, + "measurement_basis": "MONOTONIC", + "outcome": "COMPLETE", + "parent_phase_id": "total", + "phase_id": "provider", + "phase_name": "PROVIDER_EXECUTION", + "relative_end_ms": 6000, + "relative_start_ms": 0, + "started_at": "2026-09-17T10:00:00+00:00" + } + ], + "total_monotonic_duration_ms": 10000 + }, + "usage": { + "cache_ratio_percent": 80.0, + "invocations": [ + { + "cached_input_tokens": 1000, + "duration_ms": 6000, + "input_tokens": 1250, + "invocation_id": "synthetic-invocation-1", + "model": "observed-model", + "model_provenance": "AUTHORITATIVE", + "output_tokens": 75, + "phase": "PROVIDER_EXECUTION", + "role": "IMPLEMENTATION", + "timing_correlation": "UNAVAILABLE", + "uncached_input_tokens": 250, + "usage_coverage": "COMPLETE" + } + ], + "metrics": { + "input_tokens": { + "coverage": "COMPLETE", + "expected_observations": 1, + "observed_observations": 1, + "provenance": "AUTHORITATIVE", + "unit": "tokens", + "value": 1250 + } + } + } + }, + "contract_version": "telemetry-contract@2.1", + "source_snapshot_reference": "synthetic-run-1" + } + } + }, + "export_schema_version": "telemetry-export@1.0", + "locale": "nl", + "selection": { + "date": "2026-09-17", + "filters": {"selected_utc_date": "2026-09-17"}, + "project_id": "synthetic-project", + "run_id": "synthetic-run-1", + "scope": "EP_RUN_ATTEMPT", + "timezone": "UTC" + }, + "snapshot_id": "sha256:synthetic-detail-example", + "source_snapshot_references": ["synthetic-run-1"] +} diff --git a/docs/engineering/examples/telemetry-followup/telemetry-detail-example.md b/docs/engineering/examples/telemetry-followup/telemetry-detail-example.md new file mode 100644 index 00000000..5d43c80f --- /dev/null +++ b/docs/engineering/examples/telemetry-followup/telemetry-detail-example.md @@ -0,0 +1,51 @@ +# Telemetriedetail + +## Selectie + +| Veld | Waarde | +| --- | --- | +| project_id | synthetic-project | +| scope | EP_RUN_ATTEMPT | +| date | 2026-09-17 | +| timezone | UTC | +| run_id | synthetic-run-1 | + +Snapshot: `sha256:synthetic-detail-example` +As-of: `2026-09-17T10:00:00+00:00` +Contract: `telemetry-contract@2.1` +Exportschema: `telemetry-export@1.0` + +## Samenvatting + +| Veld | Waarde | Dekking | +| --- | ---: | --- | +| Doorlooptijd (monotoon) | 10000 ms | COMPLETE | +| Exclusieve wall-clock-envelope | 10000 ms | COMPLETE | +| Waargenomen input | 1250 tokens | COMPLETE (1/1) | +| Cached input | 1000 tokens | COMPLETE (1/1) | +| Uncached input | 250 tokens | COMPLETE (1/1) | +| Waargenomen output | 75 tokens | COMPLETE (1/1) | +| Cacheratio | 80,0% | COMPLETE (1/1) | + +## Exclusieve doorlooptijdverdeling + +| Categorie | Duur | Aandeel | +| --- | ---: | ---: | +| PROVIDER_EXECUTION | 6000 ms | 60,0% | +| UNASSIGNED | 4000 ms | 40,0% | + +Meetbasis: `WALL_CLOCK_INTERVAL_ENVELOPE`. De verdeling sluit exact op 10000 ms. + +## Providerinvocations + +| Invocation | Rol | Model (herkomst) | Duur | Input | Cached | Uncached | Output | Dekking | Timingkoppeling | +| --- | --- | --- | ---: | ---: | ---: | ---: | ---: | --- | --- | +| synthetic-invocation-1 | IMPLEMENTATION | observed-model (AUTHORITATIVE) | 6000 ms | 1250 | 1000 | 250 | 75 | COMPLETE | UNAVAILABLE | + +## Tijdlijn + +| Span | Parent | Fase | Start | Einde | Duur | Meetbasis | Uitkomst | +| --- | --- | --- | ---: | ---: | ---: | --- | --- | +| provider | total | PROVIDER_EXECUTION | 0 ms | 6000 ms | 6000 ms | MONOTONIC | COMPLETE | + +Ontbrekende timingkoppeling is expliciet `UNAVAILABLE`; zij wordt niet uit namen of tijdstippen afgeleid. diff --git a/docs/engineering/examples/telemetry-followup/telemetry-overview-example.json b/docs/engineering/examples/telemetry-followup/telemetry-overview-example.json new file mode 100644 index 00000000..649a7c68 --- /dev/null +++ b/docs/engineering/examples/telemetry-followup/telemetry-overview-example.json @@ -0,0 +1,48 @@ +{ + "as_of": "2026-09-17T10:00:00+00:00", + "completeness": { + "displayed_population": 1, + "export": "COMPLETE", + "full_population": 1, + "metrics_may_be_partial": true + }, + "contract_version": "telemetry-contract@2.1", + "data": { + "overview": { + "row_count": 1, + "rows": [ + { + "blocked_count": 0, + "complete_count": 1, + "contract_version": "telemetry-contract@2.1", + "date": "2026-09-17", + "failed_count": 0, + "input_tokens": 1250, + "measurement_coverage": "PARTIAL", + "output_tokens": 75, + "prompt_count": 1, + "total_tokens": 1325 + } + ], + "summary": { + "blocked": 0, + "completed": 1, + "day_count": 1, + "failed": 0, + "run_count": 1 + } + } + }, + "export_schema_version": "telemetry-export@1.0", + "locale": "nl", + "selection": { + "aggregation_level": "UTC_DAY", + "filters": {"retention_days": 90}, + "project_id": "synthetic-project", + "scope": "TELEMETRY_OVERVIEW", + "sort": {"direction": "desc", "key": "date"}, + "timezone": "UTC" + }, + "snapshot_id": "sha256:synthetic-overview-example", + "source_snapshot_references": ["2026-09-17"] +} diff --git a/docs/engineering/examples/telemetry-followup/telemetry-overview-example.md b/docs/engineering/examples/telemetry-followup/telemetry-overview-example.md new file mode 100644 index 00000000..e011a671 --- /dev/null +++ b/docs/engineering/examples/telemetry-followup/telemetry-overview-example.md @@ -0,0 +1,32 @@ +# Telemetrieoverzicht + +## Selectie + +| Veld | Waarde | +| --- | --- | +| project_id | synthetic-project | +| scope | TELEMETRY_OVERVIEW | +| timezone | UTC | +| aggregation_level | UTC_DAY | +| sort | date, aflopend | + +Snapshot: `sha256:synthetic-overview-example` +As-of: `2026-09-17T10:00:00+00:00` +Contract: `telemetry-contract@2.1` +Exportschema: `telemetry-export@1.0` + +## Dekking + +De export bevat 1 van 1 geselecteerde dagrijen. De export is compleet; de inputmeting is gedeeltelijk en blijft daarom `PARTIAL`. + +## Samenvatting + +| Dagen | Uitvoeringen | Voltooid | Geblokkeerd | Mislukt | +| ---: | ---: | ---: | ---: | ---: | +| 1 | 1 | 1 | 0 | 0 | + +## Uitvoeringen + +| Datum | Uitvoeringen | Waargenomen input | Waargenomen output | Meetdekking | +| --- | ---: | ---: | ---: | --- | +| 2026-09-17 | 1 | 1250 | 75 | PARTIAL | diff --git a/package-lock.json b/package-lock.json index d0d6ce11..d6d798f1 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "engineering-platform-browser-validation", - "version": "2.3.79", + "version": "2.3.80", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "engineering-platform-browser-validation", - "version": "2.3.79", + "version": "2.3.80", "devDependencies": { "@playwright/test": "1.63.0" } diff --git a/package.json b/package.json index 3f4387bd..eddb8e0a 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "engineering-platform-browser-validation", "private": true, - "version": "2.3.79", + "version": "2.3.80", "scripts": { "test:engineering-dashboard": "PYTHONPATH=src python3 -m engineering_platform.dashboard_browser_validation", "test:engineering-dashboard-logic": "node --test tests/engineering/dashboard_status_store.test.mjs tests/engineering/ui_localization_contract.test.mjs tests/engineering/dashboard_translation.test.mjs", diff --git a/pyproject.toml b/pyproject.toml index 597ed0e9..83bf7cf9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "engineering-platform" -version = "2.3.79" +version = "2.3.80" description = "Local-first Engineering Platform execution operations runtime" readme = "README.md" requires-python = ">=3.14,<3.15" diff --git a/src/engineering_platform/ENGINEERING_PLATFORM_CONFIG.json b/src/engineering_platform/ENGINEERING_PLATFORM_CONFIG.json index 849b5421..42ac3009 100644 --- a/src/engineering_platform/ENGINEERING_PLATFORM_CONFIG.json +++ b/src/engineering_platform/ENGINEERING_PLATFORM_CONFIG.json @@ -3,7 +3,7 @@ "platform": { "id": "engineering-platform", "name": "Engineering Platform", - "version": "2.3.79", + "version": "2.3.80", "generation": 2, "documentation_namespace": "engineering-platform", "capability_registry_version": 1 diff --git a/src/engineering_platform/ENGINEERING_PLATFORM_VERSION.json b/src/engineering_platform/ENGINEERING_PLATFORM_VERSION.json index 321279fb..3acd5867 100644 --- a/src/engineering_platform/ENGINEERING_PLATFORM_VERSION.json +++ b/src/engineering_platform/ENGINEERING_PLATFORM_VERSION.json @@ -1,15 +1,15 @@ { "bootstrap_contract": "2026.12", "checkpoint_format": 1, - "dashboard_version": "2.3.79", + "dashboard_version": "2.3.80", "handoff_protocol": 1, "memory_format": 2, "minimum_codex_cli": "0.146.0", "inbox_protocol": 1, - "platform_version": "2.3.79", + "platform_version": "2.3.80", "report_format": 2, - "runner_version": "2.3.79", + "runner_version": "2.3.80", "status_model": 1, "storage_schema": 45, - "watcher_version": "2.3.79" + "watcher_version": "2.3.80" } diff --git a/src/engineering_platform/assets/dashboard.css b/src/engineering_platform/assets/dashboard.css index ed92a8d8..b9b991e4 100644 --- a/src/engineering_platform/assets/dashboard.css +++ b/src/engineering_platform/assets/dashboard.css @@ -117,6 +117,7 @@ button:not(.theme-toggle):not(.section-state-toggle):hover:not(:disabled){backgr #dashboardSplash{align-items:center;background:#101015;display:flex;inset:0;justify-content:center;padding:24px;position:fixed;text-align:center;transition:opacity .25s ease,visibility .25s ease;z-index:100}#dashboardSplash[hidden]{display:none}body.dashboard-ready #dashboardSplash{opacity:0;pointer-events:none;visibility:hidden}.dashboard-splash__content{align-items:center;display:flex;flex-direction:column;gap:12px;max-width:360px}.dashboard-splash__title{color:#f7f3ee;font:700 clamp(28px,8vw,42px)/1.1 system-ui;margin:0}.dashboard-splash__version{color:#b79aff;font:600 14px/1.3 system-ui;letter-spacing:.04em;text-transform:uppercase}.dashboard-splash__loading{color:#c8c4cc;font:14px system-ui}.dashboard-splash__spinner{animation:dashboard-splash-spin .85s linear infinite;border:3px solid #332a44;border-radius:50%;border-top-color:#b79aff;height:34px;width:34px}@keyframes dashboard-splash-spin{to{transform:rotate(360deg)}} .dashboard-splash__icon{border-radius:18px;display:block;height:72px;width:72px} #copyToast{align-items:center;background:var(--house-style-surface);border:1px solid var(--house-style);border-radius:999px;box-shadow:0 8px 24px #0009;box-sizing:border-box;color:var(--house-style-contrast);display:flex;font:600 13px system-ui;gap:8px;inset:auto;inset-block-end:max(20px,env(safe-area-inset-bottom));inset-inline-start:50%;margin:0;max-width:calc(100vw - 32px);opacity:0;padding:10px 15px;pointer-events:none;position:fixed;transform:translate(-50%,16px);transition:opacity .16s ease,transform .16s ease;z-index:110}#copyToast::before{content:attr(data-toast-glyph);font:700 17px/1 system-ui}#copyToast.copy-toast--visible{opacity:1;transform:translate(-50%,0)} +#copyToast[hidden]{display:none} body{overflow-x:hidden}.dashboard-grid,.dashboard-grid>*{min-width:0}.telemetry,.platform-health{min-width:0}.telemetry-scroll{max-width:100%;-webkit-overflow-scrolling:touch}.platform-health__component{min-width:0}.platform-health__component-detail{overflow-wrap:anywhere}.dashboard-titlebar{align-items:center;background:#121217;box-shadow:0 10px 18px #121217;box-sizing:border-box;display:flex;gap:14px;justify-content:space-between;margin:0 0 18px;max-width:100%;padding:8px 16px 12px;position:sticky;top:0;width:100%;z-index:15}.dashboard-titlebar:has(+.dashboard-status-banner:not([hidden])){margin-bottom:0}.dashboard-status-banner{align-items:flex-start;box-sizing:border-box;display:flex;gap:8px;margin:0 0 18px;padding:10px 16px;position:sticky;top:0;width:100%;z-index:14}.dashboard-status-banner[hidden]{display:none}.dashboard-status-banner--usage-limit,.dashboard-status-banner--usage-warning,.dashboard-status-banner--usage-critical{box-shadow:0 10px 18px #121217;font:600 13px/1.45 system-ui}.dashboard-status-banner--usage-limit{background:color-mix(in srgb,var(--house-style) 24%,#241b16);border-bottom:1px solid var(--house-style);color:#fff0dc}.dashboard-status-banner--usage-limit strong{color:var(--house-style)}.dashboard-status-banner--usage-warning{background:#5b4316;border-bottom:1px solid #f4c34f;color:#fff4d6}.dashboard-status-banner--usage-warning strong{color:#ffd768}.dashboard-status-banner--usage-critical{background:#5b1d27;border-bottom:1px solid #ff6b7d;color:#ffe9ed}.dashboard-status-banner--usage-critical strong{color:#ff99a7}.dashboard-status-banner strong{flex:none}.dashboard-status-banner span{min-width:0}.dashboard-titlebar__brand{align-items:center;display:flex;flex:1 1 260px;gap:10px;min-width:0}.dashboard-titlebar h1{font-size:28px;line-height:1.1;margin:0;overflow-wrap:anywhere}.dashboard-titlebar__actions{align-items:center;display:flex;flex:none;gap:12px}.auto-refresh-toggle{align-items:center;color:#b9b6c0;cursor:pointer;display:flex;flex:none;font-size:12px;gap:7px;white-space:nowrap}.auto-refresh-toggle input{accent-color:#54d6a0;height:18px;margin:0;width:18px}.section-state-toggle{align-items:center;background:transparent;border:0;color:#b9b6c0;cursor:pointer;display:flex;font:12px system-ui;gap:7px;margin:0;min-height:44px;min-width:44px;padding:0;position:relative}.section-state-toggle::before{background:#4a4a55;border-radius:999px;content:"";height:24px;transition:background .18s ease;width:42px}.section-state-toggle::after{background:#f7f3ee;border-radius:50%;box-shadow:0 1px 3px #0008;content:"";height:18px;left:3px;position:absolute;top:13px;transform:translateX(0);transition:transform .18s ease;width:18px}.section-state-toggle[aria-checked="true"]::before{background:#8dc7ff}.section-state-toggle[aria-checked="true"]::after{transform:translateX(18px)}.section-state-toggle:focus-visible{box-shadow:none;outline:2px solid #8dc7ff;outline-offset:3px}@media (max-width:1400px){.dashboard-titlebar{align-items:flex-start;flex-wrap:wrap}.dashboard-titlebar__actions{flex:1 1 100%;flex-wrap:wrap;justify-content:flex-end}}@media (max-width:760px){.dashboard-titlebar__actions{justify-content:flex-start}}@media (max-width:620px){.dashboard-status-banner{align-items:stretch;flex-direction:column;margin-bottom:12px;padding:10px 14px}.dashboard-titlebar h1{font-size:25px}.auto-refresh-toggle span,.section-state-toggle__label{display:none}.dashboard-titlebar__actions{gap:8px}}html[data-theme="light"] .dashboard-status-banner--usage-limit{background:color-mix(in srgb,var(--house-style) 18%,#fff);color:#643a13}html[data-theme="light"] .dashboard-status-banner--usage-warning{background:#fff4d6;color:#654600}html[data-theme="light"] .dashboard-status-banner--usage-critical{background:#ffe8eb;color:#7d1726} .dashboard-titlebar__brand{align-items:center;display:flex;gap:10px;min-width:0}.dashboard-app-icon{border-radius:9px;display:block;flex:none;height:32px;width:32px}@media (max-width:620px){.dashboard-app-icon{height:28px;width:28px}} .inbox-queue{--category-color:#818cf8;background:#24242d;border:2px solid var(--category-color);border-radius:18px;box-shadow:0 5px 24px #0006;cursor:pointer;padding:14px}.inbox-queue>summary{border-bottom:1px solid var(--category-color);box-sizing:border-box;cursor:pointer;display:block;list-style:none;margin-bottom:8px;min-height:35px;padding:0 40px 10px 0;position:relative}.inbox-queue>summary::-webkit-details-marker{display:none}.inbox-queue>summary::before{color:var(--category-color);content:"▸ ";font-size:24px;line-height:1;margin:0;padding:0;position:absolute;right:0;top:0;vertical-align:-2px}.inbox-queue[open]>summary::before{content:"▾ "}.inbox-queue>summary>strong{color:var(--category-color);font-size:17px;font-weight:700;letter-spacing:.04em;line-height:1.25;text-transform:uppercase}.inbox-queue .category-description{color:#c9c7e2;font-size:14px;line-height:1.4;margin:12px 0}.inbox-queue .estimate-meta{color:#c9c7e2}.inbox-queue .queue-item{border-top-color:#48446f} @@ -1329,3 +1330,4 @@ html[data-theme="light"] :is(.dashboard-status-banner--provider-auth_required .p background-color:color-mix(in srgb,#b89025 14%,#fff)!important; } .telemetry-note{color:var(--dashboard-muted,#b9b6c0);font-size:12px;line-height:1.45;margin:0 0 10px}.telemetry-run-detail{border-top:1px solid color-mix(in srgb,var(--category-color,#fb7185) 45%,transparent);display:grid;gap:16px;margin-top:16px;padding-top:16px}.telemetry-run-detail h4{color:var(--category-color,#fb7185);font-size:14px;margin:14px 0 8px}.telemetry-scope-switcher{display:flex;flex-wrap:wrap;gap:8px}.telemetry-scope-switcher button{background:var(--dashboard-expanded-content-surface,#24242d);border:1px solid color-mix(in srgb,var(--category-color,#fb7185) 72%,transparent);border-radius:7px;color:inherit;cursor:pointer;font:400 12px/1.2 system-ui;min-height:32px;padding:5px 9px}.telemetry-scope-switcher button:hover{background:color-mix(in srgb,var(--category-color,#fb7185) 14%,var(--dashboard-expanded-content-surface,#24242d));border-color:var(--category-color,#fb7185)}.telemetry-scope-switcher button[aria-selected="true"]{background:color-mix(in srgb,var(--category-color,#fb7185) 24%,var(--dashboard-expanded-content-surface,#24242d));border-color:var(--category-color,#fb7185);box-shadow:inset 3px 0 0 var(--category-color,#fb7185)}.telemetry-scope-switcher button:focus:not(:focus-visible){box-shadow:none!important;outline:0!important}.telemetry-scope-switcher button[aria-selected="true"]:focus:not(:focus-visible){box-shadow:inset 3px 0 0 var(--category-color,#fb7185)!important}.telemetry-timeline{display:grid;gap:5px;list-style:none;margin:0;padding:0}.telemetry-timeline li{border-left:2px solid color-mix(in srgb,var(--category-color,#fb7185) 60%,transparent);margin-left:calc(var(--timeline-depth,0) * 14px);padding:6px 8px;word-break:break-word}@media(max-width:620px){.telemetry-detail-modal__content .technical-grid{grid-template-columns:minmax(0,1fr)}.telemetry-scope-switcher{position:sticky;top:0;background:var(--dashboard-expanded-content-surface,#24242d);padding-block:6px;z-index:2}.telemetry-timeline li{margin-left:min(calc(var(--timeline-depth,0) * 9px),45px)}} +.telemetry-timeline-axis{color:var(--dashboard-muted,#b9b6c0);display:flex;font:11px ui-monospace,monospace;justify-content:space-between;margin:4px 8px}.telemetry-timeline__label{font-size:12px;line-height:1.35}.telemetry-timeline__track{background:color-mix(in srgb,var(--category-color,#fb7185) 10%,transparent);border:1px solid color-mix(in srgb,var(--category-color,#fb7185) 24%,transparent);border-radius:4px;height:14px;margin:5px 0;overflow:hidden;position:relative}.telemetry-timeline__bar{background:var(--category-color,#fb7185);border-radius:3px;display:block;height:100%;min-width:2px;position:absolute}.telemetry-timeline__bar:focus-visible{outline:3px solid var(--house-style);outline-offset:2px}.telemetry-timeline__bar--unpositioned{background:repeating-linear-gradient(135deg,var(--category-color,#fb7185) 0 4px,transparent 4px 8px);inset:0;width:100%}.telemetry-timeline__identity{color:var(--dashboard-muted,#b9b6c0);display:block;font-size:10px;overflow-wrap:anywhere}.telemetry-export{min-width:38px}@media(max-width:620px){.telemetry-timeline-axis{margin-inline:0}.telemetry-timeline__identity{font-size:9px}.telemetry-actions{flex-wrap:wrap}} diff --git a/src/engineering_platform/assets/dashboard.js b/src/engineering_platform/assets/dashboard.js index 4c170c0a..940d7cc8 100644 --- a/src/engineering_platform/assets/dashboard.js +++ b/src/engineering_platform/assets/dashboard.js @@ -4358,13 +4358,19 @@ const EXECUTION_TELEMETRY_PAGE_SIZE = 7; let executionTelemetryRows = [], executionTelemetryPage = 1, executionTelemetrySort = { key: "date", direction: "desc" }; function telemetryComparableValue(row, key) { const value = row?.[key]; - return key === "date" ? String(value || "") : Number.isFinite(Number(value)) ? Number(value) : -1; + return key === "date" ? String(value || "") : typeof value === "number" && Number.isFinite(value) ? value : null; } function sortedExecutionTelemetryRows() { const { key, direction } = executionTelemetrySort, multiplier = direction === "asc" ? 1 : -1; return [...executionTelemetryRows].sort((left, right) => { const leftValue = telemetryComparableValue(left, key), rightValue = telemetryComparableValue(right, key); - return typeof leftValue === "number" + if (leftValue == null && rightValue == null) { + const leftDate = String(left?.date || ""), rightDate = String(right?.date || ""); + return leftDate < rightDate ? -1 : leftDate > rightDate ? 1 : 0; + } + if (leftValue == null) return 1; + if (rightValue == null) return -1; + return typeof leftValue === "number" && typeof rightValue === "number" ? (leftValue - rightValue) * multiplier : (leftValue < rightValue ? -1 : leftValue > rightValue ? 1 : 0) * multiplier; }); @@ -4402,15 +4408,61 @@ function executionTelemetryText(rows = sortedExecutionTelemetryRows()) { ]); return [headings, ...values].map((line) => line.map((value) => String(value ?? "—").replaceAll("\t", " ").replaceAll("\n", " ")).join("\t")).join("\n"); } -function downloadExecutionTelemetry() { +function downloadTelemetryResponse(url, fallbackName) { + // Start the server-owned response while the user activation is still live. + // Fetching first and creating a Blob after a network await causes Chromium + // to suppress the download on touch and other activation-gated surfaces. + // Content-Disposition supplies the authoritative filename. A same-origin + // hidden target keeps an HTTP error inspectable without navigating the + // Console; successful attachment responses remain native downloads. + let frame = $("telemetryExportDownloadTarget"); + if (!frame) { + frame = document.createElement("iframe"); + frame.id = "telemetryExportDownloadTarget"; + frame.name = "telemetry-export-download-target"; + frame.hidden = true; + frame.title = t("telemetry.export_download_target"); + document.body.append(frame); + } + const inspectFailure = () => { + try { + if (!frame.contentWindow || frame.contentWindow.location.href === "about:blank") return; + const text = frame.contentDocument?.body?.textContent?.trim(); + if (text) showDashboardToast(t("telemetry.export_failed"), DASHBOARD_TOAST_GLYPHS.error); + } catch { + showDashboardToast(t("telemetry.export_failed"), DASHBOARD_TOAST_GLYPHS.error); + } + }; + if (!frame.dataset.failureListenerInstalled) { + frame.dataset.failureListenerInstalled = "true"; + frame.addEventListener("load", inspectFailure); + } + const link = document.createElement("a"); + link.href = url; link.target = frame.name; link.hidden = true; + link.setAttribute("aria-label", fallbackName); + document.body.append(link); link.click(); link.remove(); + return Promise.resolve(); +} +async function downloadExecutionTelemetry(format) { if (!executionTelemetryRows.length) return; - const blob = new Blob([executionTelemetryText()], { type: "text/tab-separated-values;charset=utf-8" }); - const url = URL.createObjectURL(blob), link = document.createElement("a"); - link.href = url; - link.download = "execution-host-telemetry.tsv"; - link.click(); - URL.revokeObjectURL(url); - void recordUserAction("telemetry_downloaded"); + const markdown = format === "markdown", project = document.body.dataset.projectId || ""; + const parameters = new URLSearchParams({ + project, format: markdown ? "markdown" : "json", locale: dashboardLocale, + sort: executionTelemetrySort.key, direction: executionTelemetrySort.direction, + }); + const buttons = document.querySelectorAll("#executionTelemetry .telemetry-export"); + buttons.forEach((button) => button.disabled = true); + try { + await downloadTelemetryResponse( + `/api/telemetry/export?${parameters}`, + `telemetry-overview-${project || "project"}-utc.${markdown ? "md" : "json"}`, + ); + void recordUserAction("telemetry_downloaded"); + } catch (error) { + showDashboardToast(error instanceof Error ? error.message : t("telemetry.export_failed"), DASHBOARD_TOAST_GLYPHS.error); + } finally { + buttons.forEach((button) => button.disabled = !executionTelemetryRows.length); + } } async function copyExecutionTelemetry() { if (!executionTelemetryRows.length) return; @@ -4458,7 +4510,8 @@ function executionTelemetry(rows) { tableBody = document.createElement("tbody"), navigation = document.createElement("nav"), actions = document.createElement("div"), - download = document.createElement("button"), + downloadMarkdown = document.createElement("button"), + downloadJson = document.createElement("button"), copy = document.createElement("button"), clear = document.createElement("button"), retention = document.createElement("div"), @@ -4518,7 +4571,8 @@ function executionTelemetry(rows) { navigation.setAttribute("aria-label", t("telemetry.pagination_label")); actions.className = "log-card-actions telemetry-actions"; for (const [button, className, glyph, label, handler] of [ - [download, "dashboard-action dashboard-action--download telemetry-download", "↓", "telemetry.download", downloadExecutionTelemetry], + [downloadMarkdown, "dashboard-action dashboard-action--download telemetry-export telemetry-export--markdown", "MD", "telemetry.download_overview_markdown", () => downloadExecutionTelemetry("markdown")], + [downloadJson, "dashboard-action dashboard-action--download telemetry-export telemetry-export--json", "{}", "telemetry.download_overview_json", () => downloadExecutionTelemetry("json")], [copy, "dashboard-action dashboard-action--copy telemetry-copy", "⧉", "telemetry.copy", copyExecutionTelemetry], ...(!CENTRAL_CONSOLE ? [[clear, "dashboard-action dashboard-action--destructive telemetry-clear", "⊠", "telemetry.clear_title", clearExecutionTelemetry]] : []), ]) { @@ -4618,7 +4672,8 @@ function executionTelemetry(rows) { } updateExecutionTelemetrySortHeaders(); } -let telemetryDetailTrigger = null, telemetryDetailPayload = null, telemetryDetailDate = null, telemetryDetailRequestId = 0; +let telemetryDetailTrigger = null, telemetryDetailPayload = null, telemetryDetailDate = null, telemetryDetailRequestId = 0, + telemetryDetailSelection = { scope: "UTC_DAY_DETAIL", runId: null }; function telemetryMs(value) { return typeof value === "number" && value >= 0 ? telemetryDuration(value / 1000) : t("format.unavailable"); } function telemetryNumber(value) { return typeof value === "number" && Number.isFinite(value) @@ -4638,9 +4693,8 @@ function telemetryMetricValue(metric) { return `${telemetryNumber(metric.value)} · ${telemetryCoverage(metric)}`; } function telemetryPercent(value) { - const percent = Number(value); - return Number.isFinite(percent) - ? locale.number(percent, { minimumFractionDigits: 1, maximumFractionDigits: 1 }) + "%" + return typeof value === "number" && Number.isFinite(value) + ? locale.number(value, { minimumFractionDigits: 1, maximumFractionDigits: 1 }) + "%" : t("format.unavailable"); } function telemetryMetric(label, value) { @@ -4648,10 +4702,6 @@ function telemetryMetric(label, value) { field.append(Object.assign(document.createElement("span"), { className: "label", textContent: label }), Object.assign(document.createElement("strong"), { textContent: telemetryMs(value) })); return field; } -function telemetryRunMetric(value, phaseTelemetry) { - if (typeof value === "number" && value >= 0) return telemetryMs(value); - return phaseTelemetry === "RECORDED" ? t("telemetry.not_executed") : t("telemetry.not_recorded_short"); -} function telemetryLabel(phase) { return t("telemetry.phase." + String(phase || "").toLowerCase(), {}, String(phase || t("format.unavailable"))); } function telemetryDetailSortableTable(columns, rows, initialSort, appendRow) { let sort = initialSort; @@ -4715,100 +4765,28 @@ function setTelemetryDetailDownloads(payload, date) { button.setAttribute("aria-label", t(label, { date: telemetryDate(date) })); } } -function telemetryMarkdownCell(value) { - return String(value ?? t("format.unavailable")).replaceAll("|", "\\|").replaceAll(/[\r\n]+/g, " ").trim(); -} -function telemetryMarkdownTable(headings, rows) { - return [ - `| ${headings.map(telemetryMarkdownCell).join(" | ")} |`, - `| ${headings.map(() => "---").join(" | ")} |`, - ...rows.map((row) => `| ${row.map(telemetryMarkdownCell).join(" | ")} |`), - ].join("\n"); -} -function telemetryDetailMarkdown(detail, date) { - const summary = detail?.summary || {}, phases = Array.isArray(detail?.inclusive_phases) ? detail.inclusive_phases : Array.isArray(detail?.phases) ? detail.phases : []; - const exclusive = Array.isArray(detail?.exclusive_distribution) ? detail.exclusive_distribution : []; - const runs = Array.isArray(detail?.runs) ? detail.runs : [], bottlenecks = detail?.bottlenecks || {}; - const usage = summary.usage || {}; - const summaryRows = [ - [t("telemetry.executions"), summary.executions ?? 0], [t("telemetry.complete"), summary.completed ?? 0], - [t("telemetry.blocked"), summary.blocked ?? 0], [t("telemetry.failed"), summary.failed ?? 0], - [t("telemetry.population"), `N=${summary.population ?? summary.executions ?? 0}`], - [t("telemetry.elapsed"), telemetryMs(summary.total_wall_time?.average_ms)], - [t("telemetry.provider_unique_coverage"), telemetryMs(summary.provider_unique_coverage?.average_ms)], - [t("telemetry.provider_cumulative_process"), telemetryMs(summary.provider_cumulative_process?.average_ms)], - [t("telemetry.external_wait"), telemetryMs(summary.external_wait?.average_ms)], - [t("telemetry.unassigned"), telemetryMs(summary.unassigned?.average_ms)], - [t("telemetry.observed_input"), telemetryMetricValue(usage.input_tokens)], - [t("telemetry.observed_output"), telemetryMetricValue(usage.output_tokens)], - [t("telemetry.cache_ratio"), telemetryPercent(summary.cache_ratio_percent)], - [t("telemetry.contract_version"), detail?.contract_version || t("format.unavailable")], - ]; - const phaseRows = phases.map((phase) => [telemetryLabel(phase.phase), telemetryMs(phase.average_ms), telemetryMs(phase.median_ms), telemetryMs(phase.total_ms), telemetryPercent(phase.share_percent), phase.runs]); - const exclusiveRows = exclusive.map((row) => [telemetryLabel(row.category), telemetryMs(row.duration_ms), telemetryPercent(row.share_percent)]); - const runRows = runs.map((run) => [ - run.run_id, run.started_at ? locale.dateTime(new Date(run.started_at)) : t("format.unavailable"), translate(run.status), - telemetryMs(run.total_duration_ms), telemetryRunMetric(run.provider_duration_ms, run.phase_telemetry), - telemetryRunMetric(run.external_wait_ms, run.phase_telemetry), telemetryRunMetric(run.unassigned_ms, run.phase_telemetry), - telemetryNumber(run.input_tokens), telemetryNumber(run.output_tokens), telemetryPercent(run.cache_ratio_percent), - run.largest_phase ? telemetryLabel(run.largest_phase) : telemetryRunMetric(null, run.phase_telemetry), run.producer_type, - run.repository, run.model, - ]); - const bottleneckLines = [ - `${t("telemetry.longest_average_phase")}: ${bottlenecks.longest_average_phase ? telemetryLabel(bottlenecks.longest_average_phase.phase) : t("format.unavailable")}`, - `${t("telemetry.largest_accumulated_phase")}: ${bottlenecks.largest_accumulated_phase ? telemetryLabel(bottlenecks.largest_accumulated_phase.phase) : t("format.unavailable")}`, - `${t("telemetry.longest_individual_span")}: ${bottlenecks.longest_individual_span ? bottlenecks.longest_individual_span.label || telemetryLabel(bottlenecks.longest_individual_span.phase) : t("format.unavailable")}`, - ...(Array.isArray(bottlenecks.top_time_consumers) ? bottlenecks.top_time_consumers.map((item) => `${telemetryLabel(item.phase)} — ${telemetryPercent(item.share_percent)}`) : []), - ]; - const runDetails = []; - for (const run of runs) { - const snapshot = run.telemetry_snapshot || {}, attempt = snapshot.attempt || {}, chain = snapshot.chain || {}; - const invocations = Array.isArray(attempt.usage?.invocations) ? attempt.usage.invocations : []; - runDetails.push(`## ${t("telemetry.scope_attempt")} — ${run.run_id}`, "", - `${t("telemetry.measurement_coverage")}: ${attempt.timing?.coverage?.state || t("format.unavailable")}`, "", - invocations.length ? telemetryMarkdownTable( - [t("telemetry.phase"), t("telemetry.role"), t("telemetry.model"), t("telemetry.duration"), t("telemetry.input"), t("telemetry.cached_input"), t("telemetry.uncached_input"), t("telemetry.output"), t("telemetry.usage_coverage"), t("telemetry.invocation_id")], - invocations.map((item) => [telemetryLabel(item.phase), item.role, `${item.model || t("format.unavailable")} (${item.model_provenance || t("format.unavailable")})`, telemetryMs(item.duration_ms), telemetryNumber(item.input_tokens), telemetryNumber(item.cached_input_tokens), telemetryNumber(item.uncached_input_tokens), telemetryNumber(item.output_tokens), item.usage_coverage, item.invocation_id]), - ) : t("format.unavailable"), "", - `### ${t("telemetry.scope_chain")}`, "", - telemetryMarkdownTable([t("table.details"), t("history.markdown_value")], [ - [t("telemetry.chain_coverage"), chain.coverage || t("format.unavailable")], - [t("telemetry.chain_attempts"), chain.attempt_count ?? t("format.unavailable")], - [t("telemetry.original_attempts"), chain.original_attempt_count ?? t("format.unavailable")], - [t("telemetry.retries"), chain.retry_count ?? t("format.unavailable")], - [t("telemetry.resumes"), chain.resume_count ?? t("format.unavailable")], - [t("telemetry.chain_elapsed"), telemetryMs(chain.elapsed_ms)], - [t("telemetry.chain_processing"), telemetryMs(chain.processing_time_ms)], - [t("telemetry.inter_attempt_gaps"), telemetryMs(chain.inter_attempt_gap_ms)], - [t("telemetry.outside_window"), chain.outside_selected_window_count ?? 0], - ]), ""); - } - return [ - `# ${t("telemetry.detail_title", { date: telemetryDate(date) })}`, "", t("telemetry.detail_description"), "", - `## ${t("telemetry.summary")}`, "", telemetryMarkdownTable([t("table.details"), t("history.markdown_value")], summaryRows), "", - `## ${t("telemetry.inclusive_title")}`, "", t("telemetry.inclusive_help"), "", phaseRows.length ? telemetryMarkdownTable([t("telemetry.phase"), t("telemetry.average"), t("telemetry.median"), t("telemetry.accumulated"), t("telemetry.share"), t("telemetry.runs")], phaseRows) : t("telemetry.not_recorded"), "", - `## ${t("telemetry.exclusive_title")}`, "", t("telemetry.exclusive_help"), "", exclusiveRows.length ? telemetryMarkdownTable([t("telemetry.category"), t("telemetry.duration"), t("telemetry.share")], exclusiveRows) : t("telemetry.not_recorded"), "", - `## ${t("telemetry.bottlenecks")}`, "", ...bottleneckLines.map((line) => `- ${line}`), "", - `## ${t("telemetry.runs")}`, "", runRows.length ? telemetryMarkdownTable([t("telemetry.run_id"), t("telemetry.start_time"), t("telemetry.status"), t("telemetry.duration"), t("telemetry.provider_unique_coverage"), t("telemetry.external_wait"), t("telemetry.unassigned"), t("telemetry.input"), t("telemetry.output"), t("telemetry.cache_ratio"), t("telemetry.largest_phase"), t("telemetry.producer_type"), t("telemetry.target_repository"), t("telemetry.model")], runRows) : t("format.unavailable"), "", - ...runDetails, - ].join("\n"); -} -function downloadTelemetryDetail(format) { +async function downloadTelemetryDetail(format) { if (!telemetryDetailPayload || !telemetryDetailDate) return; - const markdown = format === "markdown"; - const content = markdown - ? telemetryDetailMarkdown(telemetryDetailPayload, telemetryDetailDate) - : JSON.stringify(telemetryDetailPayload, null, 2) + "\n"; - const url = URL.createObjectURL(new Blob([content], { type: markdown ? "text/markdown;charset=utf-8" : "application/json;charset=utf-8" })); - const link = document.createElement("a"); - link.href = url; - link.download = `execution-telemetry-${telemetryDetailDate}.${markdown ? "md" : "json"}`; - link.hidden = true; - document.body.append(link); - link.click(); - link.remove(); - setTimeout(() => URL.revokeObjectURL(url), 0); - void recordUserAction(markdown ? "telemetry_detail_markdown_downloaded" : "telemetry_detail_json_downloaded"); + const markdown = format === "markdown", project = document.body.dataset.projectId || ""; + const captured = { date: telemetryDetailDate, ...telemetryDetailSelection }; + const parameters = new URLSearchParams({ + project, format: markdown ? "markdown" : "json", locale: dashboardLocale, + scope: captured.scope, + }); + if (captured.runId) parameters.set("run_id", captured.runId); + const buttons = [$('telemetryDetailDownloadMarkdown'), $('telemetryDetailDownloadJson')]; + buttons.forEach((button) => button.disabled = true); + try { + await downloadTelemetryResponse( + `/api/telemetry/${encodeURIComponent(captured.date)}/export?${parameters}`, + `telemetry-detail-${project || "project"}-${captured.runId || captured.date}.${markdown ? "md" : "json"}`, + ); + void recordUserAction(markdown ? "telemetry_detail_markdown_downloaded" : "telemetry_detail_json_downloaded"); + } catch (error) { + showDashboardToast(error instanceof Error ? error.message : t("telemetry.export_failed"), DASHBOARD_TOAST_GLYPHS.error); + } finally { + buttons.forEach((button) => button.disabled = false); + } } function closeTelemetryDetail() { const modal = $("telemetryDetailModal"); if (modal.open) modal.close(); } function telemetryDetailTableScroll(table, label) { @@ -4825,6 +4803,7 @@ function openTelemetryDetail(date, trigger) { void recordUserAction("telemetry_detail_opened"); const requestId = ++telemetryDetailRequestId; telemetryDetailTrigger = trigger || document.activeElement; + telemetryDetailSelection = { scope: "UTC_DAY_DETAIL", runId: null }; const modal = $("telemetryDetailModal"), content = $("telemetryDetailContent"); $("telemetryDetailTitle").textContent = t("telemetry.detail_title", { date: telemetryDate(date) }); $("telemetryDetailDescription").textContent = t("telemetry.detail_description"); @@ -4847,7 +4826,7 @@ function telemetryTextField(label, value, title) { if (title) { strong.title = title; strong.tabIndex = 0; } field.append(name, strong); return field; } -function renderCanonicalRunDetail(run, host) { +function renderCanonicalRunDetail(run, host, selectForExport = false) { host.replaceChildren(); const snapshot = run?.telemetry_snapshot || {}, attempt = snapshot.attempt || {}, chain = snapshot.chain || {}; const timing = attempt.timing || {}, usage = attempt.usage || {}, invocations = Array.isArray(usage.invocations) ? usage.invocations : []; @@ -4856,11 +4835,14 @@ function renderCanonicalRunDetail(run, host) { const attemptButton = Object.assign(document.createElement("button"), { type: "button", textContent: t("telemetry.scope_attempt") }); const chainButton = Object.assign(document.createElement("button"), { type: "button", textContent: t("telemetry.scope_chain") }); attemptButton.setAttribute("role", "tab"); chainButton.setAttribute("role", "tab"); - const select = (showChain) => { + const select = (showChain, updateExportSelection = selectForExport) => { attemptPanel.hidden = showChain; chainPanel.hidden = !showChain; attemptButton.setAttribute("aria-selected", String(!showChain)); chainButton.setAttribute("aria-selected", String(showChain)); + if (updateExportSelection) telemetryDetailSelection = { + scope: showChain ? "EXECUTION_CHAIN" : "EP_RUN_ATTEMPT", runId: run.run_id, + }; }; - attemptButton.addEventListener("click", () => select(false)); chainButton.addEventListener("click", () => select(true)); + attemptButton.addEventListener("click", () => select(false, true)); chainButton.addEventListener("click", () => select(true, true)); switcher.append(attemptButton, chainButton); select(false); host.append(switcher); const attemptGrid = document.createElement("div"); attemptGrid.className = "technical-grid"; @@ -4889,37 +4871,57 @@ function renderCanonicalRunDetail(run, host) { if (timeline.length) { const byId = new Map(timeline.map((span) => [span.phase_id, span])); const depth = (span) => { let value = 0, parent = span.parent_phase_id, seen = new Set(); while (parent && byId.has(parent) && !seen.has(parent)) { seen.add(parent); value += 1; parent = byId.get(parent).parent_phase_id; } return value; }; + const axisDuration = timing.timeline_axis?.duration_ms; + const axis = document.createElement("div"); axis.className = "telemetry-timeline-axis"; + axis.append( + Object.assign(document.createElement("span"), { textContent: telemetryMs(0) }), + Object.assign(document.createElement("span"), { textContent: telemetryMs(axisDuration) }), + ); + timelineSection.append(axis); const list = document.createElement("ol"); list.className = "telemetry-timeline"; timeline.forEach((span) => { const item = document.createElement("li"); item.style.setProperty("--timeline-depth", String(depth(span))); - item.textContent = `${telemetryLabel(span.phase_name)} · ${telemetryMs(span.duration_ms)} · ${translate(span.outcome)}`; - item.title = String(span.phase_id || ""); list.append(item); + const label = document.createElement("div"); label.className = "telemetry-timeline__label"; + label.textContent = `${telemetryLabel(span.phase_name)} · ${telemetryMs(span.duration_ms)} · ${translate(span.outcome)} · ${span.measurement_basis || t("format.unavailable")}`; + const track = document.createElement("div"); track.className = "telemetry-timeline__track"; + const bar = document.createElement("span"); bar.className = "telemetry-timeline__bar"; + if (typeof axisDuration === "number" && axisDuration > 0 && typeof span.relative_start_ms === "number" && typeof span.relative_end_ms === "number") { + bar.style.left = `${Math.max(0, Math.min(100, span.relative_start_ms * 100 / axisDuration))}%`; + bar.style.width = `${Math.max(.25, Math.min(100, (span.relative_end_ms - span.relative_start_ms) * 100 / axisDuration))}%`; + } else bar.classList.add("telemetry-timeline__bar--unpositioned"); + bar.title = `${span.phase_id || t("format.unavailable")} · ${span.started_at || t("format.unavailable")} — ${span.completed_at || t("format.unavailable")}`; + bar.tabIndex = 0; track.append(bar); + const identity = document.createElement("code"); identity.className = "telemetry-timeline__identity"; + identity.textContent = `${span.phase_id || t("format.unavailable")}${span.parent_phase_id ? ` ← ${span.parent_phase_id}` : ""}`; + item.append(label, track, identity); list.append(item); }); timelineSection.append(list); - if (timing.timeline_truncated) timelineSection.append(Object.assign(document.createElement("p"), { className: "telemetry-note", textContent: t("telemetry.timeline_truncated", { count: timing.timeline_limit }) })); + if (timing.timeline_truncated) timelineSection.append(Object.assign(document.createElement("p"), { className: "telemetry-note", textContent: t("telemetry.timeline_truncated_full_export", { shown: timeline.length, total: timing.timeline_observation_count }) })); } else timelineSection.append(Object.assign(document.createElement("p"), { textContent: t("telemetry.not_recorded") })); attemptPanel.append(timelineSection); const invocationSection = document.createElement("section"); - invocationSection.append(Object.assign(document.createElement("h4"), { textContent: `${t("telemetry.invocations")} (${invocations.length})` })); + invocationSection.append(Object.assign(document.createElement("h4"), { textContent: `${t("telemetry.invocations")} (${usage.invocation_observation_count ?? invocations.length})` })); if (invocations.length) { const columns = [ ["phase", "telemetry.phase"], ["role", "telemetry.role"], ["model", "telemetry.model"], ["duration_ms", "telemetry.duration", "desc"], ["input_tokens", "telemetry.input", "desc"], ["cached_input_tokens", "telemetry.cached_input"], ["uncached_input_tokens", "telemetry.uncached_input"], - ["output_tokens", "telemetry.output"], ["usage_coverage", "telemetry.usage_coverage"], ["invocation_id", "telemetry.invocation_id"], ["retry_ordinal", "telemetry.retry_identity"], - ].map(([key, label, defaultDirection]) => ({ key, label, defaultDirection, value: (item) => key === "phase" ? telemetryLabel(item.phase) : Number.isFinite(Number(item[key])) ? Number(item[key]) : String(item[key] || "") })); + ["output_tokens", "telemetry.output"], ["usage_coverage", "telemetry.usage_coverage"], ["timing_correlation", "telemetry.timing_correlation"], ["invocation_id", "telemetry.invocation_id"], ["retry_ordinal", "telemetry.retry_identity"], + ].map(([key, label, defaultDirection]) => ({ key, label, defaultDirection, value: (item) => key === "phase" ? telemetryLabel(item.phase) : typeof item[key] === "number" && Number.isFinite(item[key]) ? item[key] : String(item[key] || "") })); const table = telemetryDetailSortableTable(columns, invocations, { key: "input_tokens", direction: "desc" }, (item, body) => { const row = document.createElement("tr"); const model = item.model ? `${item.model} (${item.model_provenance || t("format.unavailable")})` : t("format.unavailable"); - [telemetryLabel(item.phase), item.role || t("format.unavailable"), model, telemetryMs(item.duration_ms), telemetryNumber(item.input_tokens), telemetryNumber(item.cached_input_tokens), telemetryNumber(item.uncached_input_tokens), telemetryNumber(item.output_tokens), item.usage_coverage || t("format.unavailable"), item.invocation_id || t("format.unavailable"), item.retry_ordinal ?? t("format.unavailable")] + [telemetryLabel(item.phase), item.role || t("format.unavailable"), model, telemetryMs(item.duration_ms), telemetryNumber(item.input_tokens), telemetryNumber(item.cached_input_tokens), telemetryNumber(item.uncached_input_tokens), telemetryNumber(item.output_tokens), item.usage_coverage || t("format.unavailable"), item.timing_correlation || t("format.unavailable"), item.invocation_id || t("format.unavailable"), item.retry_ordinal ?? t("format.unavailable")] .forEach((value) => row.append(Object.assign(document.createElement("td"), { textContent: String(value) }))); body.append(row); }); invocationSection.append(telemetryDetailTableScroll(table, t("telemetry.invocations"))); + if (usage.invocation_table_truncated) invocationSection.append(Object.assign(document.createElement("p"), { className: "telemetry-note", textContent: t("telemetry.invocations_truncated_full_export", { shown: invocations.length, total: usage.invocation_observation_count }) })); } else invocationSection.append(Object.assign(document.createElement("p"), { textContent: t("telemetry.not_recorded") })); attemptPanel.append(invocationSection); const chainGrid = document.createElement("div"); chainGrid.className = "technical-grid"; + const chainMetric = (name) => chain.usage_metrics?.[name]; chainGrid.append( telemetryTextField(t("telemetry.chain_coverage"), chain.coverage, Array.isArray(chain.reasons) ? chain.reasons.join("; ") : chain.reason), telemetryTextField(t("telemetry.chain_attempts"), chain.attempt_count), @@ -4931,6 +4933,12 @@ function renderCanonicalRunDetail(run, host) { telemetryTextField(t("telemetry.inter_attempt_gaps"), telemetryMs(chain.inter_attempt_gap_ms)), telemetryTextField(t("telemetry.outside_window"), chain.outside_selected_window_count ?? 0), telemetryTextField(t("telemetry.mission_scope"), chain.mission_scope_label || t("format.unavailable")), + telemetryTextField(t("telemetry.invocations"), chain.provider_invocation_count ?? t("format.unavailable")), + telemetryTextField(t("telemetry.observed_input"), telemetryMetricValue(chainMetric("input_tokens")), chainMetric("input_tokens")?.missing_reason), + telemetryTextField(t("telemetry.cached_input"), telemetryMetricValue(chainMetric("cached_input_tokens")), chainMetric("cached_input_tokens")?.missing_reason), + telemetryTextField(t("telemetry.uncached_input"), telemetryMetricValue(chainMetric("uncached_input_tokens")), chainMetric("uncached_input_tokens")?.missing_reason), + telemetryTextField(t("telemetry.observed_output"), telemetryMetricValue(chainMetric("output_tokens")), chainMetric("output_tokens")?.missing_reason), + telemetryTextField(t("telemetry.cache_ratio"), telemetryPercent(chain.cache_ratio_percent), telemetryCoverage(chain.cache_ratio_population)), ); chainPanel.append(chainGrid); const chainRuns = Array.isArray(chain.runs) ? chain.runs : []; @@ -4975,7 +4983,7 @@ function renderTelemetryDetail(detail, content) { : [categoryKey, "duration_ms", "share_percent"]; const table = telemetryDetailSortableTable(columns.map((label, index) => ({ key: columnKeys[index], label, - value: (item) => index === 0 ? telemetryLabel(item[categoryKey]) : Number(item[columnKeys[index]]) || 0, + value: (item) => index === 0 ? telemetryLabel(item[categoryKey]) : typeof item[columnKeys[index]] === "number" && Number.isFinite(item[columnKeys[index]]) ? item[columnKeys[index]] : null, })), values, { key: categoryKey, direction: "asc" }, (item, body) => { const row = document.createElement("tr"), category = telemetryLabel(item[categoryKey]); const rendered = durationKey === "total_ms" @@ -5012,11 +5020,11 @@ function renderTelemetryDetail(detail, content) { ["provider_duration_ms", "telemetry.provider_unique_coverage"], ["external_wait_ms", "telemetry.external_wait"], ["unassigned_ms", "telemetry.unassigned"], ["input_tokens", "telemetry.input"], ["output_tokens", "telemetry.output"], ["cache_ratio_percent", "telemetry.cache_ratio"], ["largest_phase", "telemetry.largest_phase"], ["model", "telemetry.model"], - ].map(([key, label, defaultDirection]) => ({ key, label, defaultDirection, value: (run) => key === "largest_phase" ? telemetryLabel(run[key]) : key === "started_at" ? Date.parse(run[key]) || 0 : Number.isFinite(Number(run[key])) ? Number(run[key]) : String(run[key] || "") })); + ].map(([key, label, defaultDirection]) => ({ key, label, defaultDirection, value: (run) => key === "largest_phase" ? telemetryLabel(run[key]) : key === "started_at" ? Date.parse(run[key]) || null : typeof run[key] === "number" && Number.isFinite(run[key]) ? run[key] : String(run[key] || "") })); const runTable = telemetryDetailSortableTable(runColumns, runs, { key: "started_at", direction: "desc" }, (run, runBody) => { const row = document.createElement("tr"), id = document.createElement("button"); row.className = "telemetry-row"; row.tabIndex = 0; row.setAttribute("role", "button"); row.setAttribute("aria-label", String(run.run_id || t("format.unavailable"))); id.type = "button"; id.className = "telemetry-run-link"; id.textContent = run.run_id; - const selectRun = () => { runBody.querySelectorAll('.telemetry-row[data-selected="true"]').forEach((candidate) => { candidate.dataset.selected = "false"; }); row.dataset.selected = "true"; renderCanonicalRunDetail(run, runDetailHost); }; + const selectRun = () => { runBody.querySelectorAll('.telemetry-row[data-selected="true"]').forEach((candidate) => { candidate.dataset.selected = "false"; }); row.dataset.selected = "true"; renderCanonicalRunDetail(run, runDetailHost, true); }; row.addEventListener("click", selectRun); row.addEventListener("keydown", (event) => { if (event.key === "Enter" || event.key === " ") { event.preventDefault(); selectRun(); } }); id.addEventListener("click", (event) => { event.stopPropagation(); selectRun(); openPromptHistoryDetail({ run_id: run.run_id, title: run.run_id }); }); [id, run.started_at ? locale.dateTime(new Date(run.started_at)) : t("format.unavailable"), translate(run.status), telemetryMs(run.total_duration_ms), telemetryMs(run.provider_duration_ms), telemetryMs(run.external_wait_ms), telemetryMs(run.unassigned_ms), telemetryNumber(run.input_tokens), telemetryNumber(run.output_tokens), telemetryPercent(run.cache_ratio_percent), run.largest_phase ? telemetryLabel(run.largest_phase) : t("format.unavailable"), run.model || t("format.unavailable")].forEach((value) => { const cell = document.createElement("td"); if (value instanceof Element) cell.append(value); else cell.textContent = String(value); row.append(cell); }); diff --git a/src/engineering_platform/assets/dashboard_locales.mjs b/src/engineering_platform/assets/dashboard_locales.mjs index 6156b9f7..ed4a7796 100644 --- a/src/engineering_platform/assets/dashboard_locales.mjs +++ b/src/engineering_platform/assets/dashboard_locales.mjs @@ -3150,6 +3150,11 @@ Object.assign(DASHBOARD_MESSAGES.es, { "log_event.terminal_report_corrected":"Informe final corregido", "log_event.active_run_lease_reconciled":"Arrendamiento de ejecución activa conciliado", "log_event.detached_runner_job_missing":"Falta la tarea de ejecución desconectada", "log_event.inbox_scan":"Bandeja de entrada analizada", "log_event.active_transaction_queue_refreshed":"Cola de ejecución activa actualizada", "log_event.inbox_log_unavailable":"Registro del monitor de bandeja de entrada no disponible", "log_event.dashboard_log_unavailable":"Registro del panel de estado no disponible" }); +Object.assign(DASHBOARD_MESSAGES.en, {"telemetry.download_overview_markdown":"Download complete overview as Markdown","telemetry.download_overview_json":"Download complete overview as JSON","telemetry.export_failed":"The telemetry export could not be created. Try again.","telemetry.export_download_target":"Telemetry export download response","telemetry.timeline_truncated_full_export":"Showing {shown} of {total} spans. The complete detail export contains all spans.","telemetry.invocations_truncated_full_export":"Showing {shown} of {total} invocations. The complete detail export contains all invocations.","telemetry.timing_correlation":"Timing correlation"}); +Object.assign(DASHBOARD_MESSAGES.nl, {"telemetry.download_overview_markdown":"Volledig overzicht als Markdown downloaden","telemetry.download_overview_json":"Volledig overzicht als JSON downloaden","telemetry.export_failed":"De telemetrie-export kon niet worden gemaakt. Probeer opnieuw.","telemetry.export_download_target":"Downloadrespons voor telemetrie-export","telemetry.timeline_truncated_full_export":"{shown} van {total} spans worden getoond. De volledige detailexport bevat alle spans.","telemetry.invocations_truncated_full_export":"{shown} van {total} invocations worden getoond. De volledige detailexport bevat alle invocations.","telemetry.timing_correlation":"Timingkoppeling"}); +Object.assign(DASHBOARD_MESSAGES.de, {"telemetry.download_overview_markdown":"Vollständige Übersicht als Markdown herunterladen","telemetry.download_overview_json":"Vollständige Übersicht als JSON herunterladen","telemetry.export_failed":"Der Telemetrieexport konnte nicht erstellt werden. Versuchen Sie es erneut.","telemetry.export_download_target":"Downloadantwort des Telemetrieexports","telemetry.timeline_truncated_full_export":"{shown} von {total} Zeitspannen werden angezeigt. Der vollständige Detailexport enthält alle Zeitspannen.","telemetry.invocations_truncated_full_export":"{shown} von {total} Aufrufen werden angezeigt. Der vollständige Detailexport enthält alle Aufrufe.","telemetry.timing_correlation":"Zeitkorrelation"}); +Object.assign(DASHBOARD_MESSAGES.fr, {"telemetry.download_overview_markdown":"Télécharger la vue complète en Markdown","telemetry.download_overview_json":"Télécharger la vue complète en JSON","telemetry.export_failed":"L’export de télémétrie n’a pas pu être créé. Réessayez.","telemetry.export_download_target":"Réponse de téléchargement de l’export de télémétrie","telemetry.timeline_truncated_full_export":"{shown} intervalles sur {total} sont affichés. L’export détaillé complet contient tous les intervalles.","telemetry.invocations_truncated_full_export":"{shown} invocations sur {total} sont affichées. L’export détaillé complet contient toutes les invocations.","telemetry.timing_correlation":"Corrélation temporelle"}); +Object.assign(DASHBOARD_MESSAGES.es, {"telemetry.download_overview_markdown":"Descargar el resumen completo como Markdown","telemetry.download_overview_json":"Descargar el resumen completo como JSON","telemetry.export_failed":"No se pudo crear la exportación de telemetría. Inténtalo de nuevo.","telemetry.export_download_target":"Respuesta de descarga de la exportación de telemetría","telemetry.timeline_truncated_full_export":"Se muestran {shown} de {total} tramos. La exportación detallada completa contiene todos los tramos.","telemetry.invocations_truncated_full_export":"Se muestran {shown} de {total} invocaciones. La exportación detallada completa contiene todas las invocaciones.","telemetry.timing_correlation":"Correlación temporal"}); Object.assign(DASHBOARD_MESSAGES.en, { "action.recover": "Recover", "preflight.sync": "Preflight failed: {reason}\nRecovery: {recovery}", diff --git a/src/engineering_platform/execution_reporting.py b/src/engineering_platform/execution_reporting.py index 927d1722..959bf769 100644 --- a/src/engineering_platform/execution_reporting.py +++ b/src/engineering_platform/execution_reporting.py @@ -1357,18 +1357,21 @@ def generate_terminal_report( provider_usage = attempt_telemetry.get("usage", {}) if isinstance(attempt_telemetry, dict) else {} churn = provider_usage.get("context_churn") if isinstance(provider_usage.get("context_churn"), dict) else {} usage_metrics = provider_usage.get("metrics") if isinstance(provider_usage.get("metrics"), dict) else {} - def usage_metric(name: str) -> str: - metric = usage_metrics.get(name) + def usage_metric(name: str, metrics: object = usage_metrics) -> str: + metric = metrics.get(name) if isinstance(metrics, dict) else None if not isinstance(metric, dict): return "UNAVAILABLE" value = metric.get("value", "UNAVAILABLE") - observed = metric.get("observed_observations", 0) - expected = metric.get("expected_observations", 0) + observed = metric.get("observed_observations") + expected = metric.get("expected_observations") coverage = metric.get("coverage", "UNAVAILABLE") reason = f"; {metric['missing_reason']}" if metric.get("missing_reason") else "" - return f"{value} ({coverage}; {observed}/{expected} invocations{reason})" + population = f"{observed if isinstance(observed, int) else 'UNKNOWN'}/{expected if isinstance(expected, int) else 'UNKNOWN'}" + return f"{value} ({coverage}; {population} invocations{reason})" def observed(value: object) -> object: return "UNAVAILABLE" if value is None else value + def percentage(value: object) -> str: + return f"{value}%" if isinstance(value, (int, float)) and not isinstance(value, bool) else "UNAVAILABLE" invocation_lines = [ "### Invocation Detail", "| Phase | Role | Provider | Model (provenance) | Duration | Input | Cached | Uncached | Output | Usage coverage | Invocation |", @@ -1403,7 +1406,7 @@ def observed(value: object) -> object: f"- Observed Cached Input: `{usage_metric('cached_input_tokens')}` (component of input, not additional volume)", f"- Derived Uncached Input: `{usage_metric('uncached_input_tokens')}`", f"- Observed Cumulative Invocation Output: `{usage_metric('output_tokens')}`", - f"- Cache Ratio: `{observed(provider_usage.get('cache_ratio_percent'))}`% over compatible observations; coverage `{(provider_usage.get('cache_ratio_population') or {}).get('coverage', 'UNAVAILABLE')}`", + f"- Cache Ratio: `{percentage(provider_usage.get('cache_ratio_percent'))}` over compatible observations; coverage `{(provider_usage.get('cache_ratio_population') or {}).get('coverage', 'UNAVAILABLE')}`", f"- Largest Cumulative Invocation Input: `{observed(provider_usage.get('max_input_tokens_per_invocation'))}` (not a context-window measurement)", f"- Observed Final Usage Snapshots: `{provider_usage.get('usage_snapshot_count') or 'UNAVAILABLE'}`", "- Actual Single-Request Context Size: `UNAVAILABLE` (not emitted by Codex CLI JSONL).", @@ -1450,7 +1453,9 @@ def duration_value(name: str) -> str: value = timing.get(name) return f"{value / 1000:.3f} s" if isinstance(value, int) else "UNAVAILABLE" timing_lines.extend(( - f"- Total Wall Time: `{duration_value('total_wall_time_ms')}`", + f"- Monotonic Process Duration: `{duration_value('total_monotonic_duration_ms')}`", + f"- Exclusive Wall-Clock Envelope: `{duration_value('exclusive_envelope_duration_ms')}`; basis `{timing.get('exclusive_measurement_basis') or 'UNAVAILABLE'}`", + f"- Wall/Monotonic Clock Difference: `{duration_value('clock_difference_ms')}`; diagnostic `{timing.get('clock_difference_diagnostic') or 'UNAVAILABLE'}`", f"- Provider Process Duration (cumulative): `{duration_value('provider_cumulative_process_duration_ms')}`", f"- Provider Coverage in Elapsed Time (interval union): `{duration_value('provider_unique_coverage_ms')}`", "- Model Inference Time: `UNAVAILABLE` (provider process lifetime may include tool and I/O waits)", @@ -1483,6 +1488,7 @@ def duration_value(name: str) -> str: f"- Historical Total Wall Time: `{timing['total_wall_time_ms'] / 1000:.3f}` s (phase telemetry incomplete)." ) chain = telemetry_snapshot.get("chain") if isinstance(telemetry_snapshot.get("chain"), dict) else {} + chain_usage = chain.get("usage_metrics") if isinstance(chain.get("usage_metrics"), dict) else {} timing_lines.extend(( "", "## Execution Chain Scope", @@ -1493,6 +1499,12 @@ def duration_value(name: str) -> str: f"- Measured Attempt Processing Time: `{chain.get('processing_time_ms') / 1000:.3f} s`" if isinstance(chain.get("processing_time_ms"), int) else "- Measured Attempt Processing Time: `UNAVAILABLE`", f"- Inter-Attempt Gaps: `{chain.get('inter_attempt_gap_ms') / 1000:.3f} s`" if isinstance(chain.get("inter_attempt_gap_ms"), int) else "- Inter-Attempt Gaps: `UNAVAILABLE`", f"- Related Attempts Outside Selected Window: `{chain.get('outside_selected_window_count', 0)}`", + f"- Provider Invocations: `{chain.get('provider_invocation_count', 'UNAVAILABLE')}`", + f"- Observed Cumulative Chain Input: `{usage_metric('input_tokens', chain_usage)}`", + f"- Observed Cached Chain Input: `{usage_metric('cached_input_tokens', chain_usage)}`", + f"- Derived Uncached Chain Input: `{usage_metric('uncached_input_tokens', chain_usage)}`", + f"- Observed Cumulative Chain Output: `{usage_metric('output_tokens', chain_usage)}`", + f"- Chain Cache Ratio: `{percentage(chain.get('cache_ratio_percent'))}` over compatible observations; coverage `{(chain.get('cache_ratio_population') or {}).get('coverage', 'UNAVAILABLE')}`", )) qualification_status = qualification.get("qualification") if qualification else "not recorded" qualification_summary_line = ( diff --git a/src/engineering_platform/execution_timing.py b/src/engineering_platform/execution_timing.py index a5e17ff9..9e14679f 100644 --- a/src/engineering_platform/execution_timing.py +++ b/src/engineering_platform/execution_timing.py @@ -20,6 +20,7 @@ import uuid from .storage import EngineeringStorageError, open_storage +from .telemetry_metrics import TELEMETRY_CALCULATION_VERSION PHASES = frozenset({ @@ -257,7 +258,7 @@ def phase_spans(root: Path, run_id: str, *, central_database: Path | None = None return result -TIMING_CALCULATION_VERSION = "telemetry-contract@2.0" +TIMING_CALCULATION_VERSION = TELEMETRY_CALCULATION_VERSION TIMING_COMPLETE, TIMING_PARTIAL, TIMING_UNAVAILABLE, TIMING_CONFLICT = ( "COMPLETE", "PARTIAL", "UNAVAILABLE", "CONFLICT" ) @@ -288,10 +289,29 @@ def _union_duration(intervals: list[tuple[datetime, datetime]]) -> int: return round(sum((end - start).total_seconds() * 1000 for start, end in merged)) +def _duration_us(start: datetime, end: datetime) -> int: + delta = end - start + return delta.days * 86_400_000_000 + delta.seconds * 1_000_000 + delta.microseconds + + +def _rounded_category_milliseconds(values_us: Mapping[str, int]) -> dict[str, int]: + """Round one microsecond partition without fabricating an UNASSIGNED rest.""" + if not values_us: + return {} + result = {key: value // 1000 for key, value in values_us.items()} + target = round(sum(values_us.values()) / 1000) + remainder = target - sum(result.values()) + ranked = sorted(values_us, key=lambda key: (-(values_us[key] % 1000), key)) + for key in ranked[:remainder]: + result[key] += 1 + return result + + def timing_summary( root: Path, run_id: str, *, central_database: Path | None = None, _spans: list[dict[str, object]] | None = None, _historical_total: object = _NO_HISTORICAL_TOTAL, + timeline_limit: int | None = 500, ) -> dict[str, object]: """Return inclusive workload and an interval-derived elapsed-time partition. @@ -380,6 +400,7 @@ def same_phase_ancestor(span: Mapping[str, object]) -> bool: boundary_conflicts: list[str] = [] total = None wall_clock_total = None + exclusive_envelope_total = None if len(total_spans) == 1: start, end = _timestamp(total_spans[0].get("started_at")), _timestamp(total_spans[0].get("completed_at")) if start is not None and end is not None and end >= start: @@ -388,6 +409,7 @@ def same_phase_ancestor(span: Mapping[str, object]) -> bool: measured_duration = int(total_spans[0]["duration_ms"]) if abs(wall_duration - measured_duration) <= max(250, round(measured_duration * .01)): envelope, total = (start, end), measured_duration + exclusive_envelope_total = wall_duration else: boundary_conflicts.append("TOTAL_EXECUTION wall-clock and monotonic duration conflict") total = measured_duration @@ -419,15 +441,15 @@ def same_phase_ancestor(span: Mapping[str, object]) -> bool: if clipped_end > clipped_start: interval_rows.append((clipped_start, clipped_end, span)) - exclusive: dict[str, int] = {} + exclusive_us: dict[str, int] = {} if envelope is not None and total is not None and not boundary_conflicts: boundaries = {envelope[0], envelope[1]} for start, end, _ in interval_rows: boundaries.update((start, end)) ordered_boundaries = sorted(boundaries) for start, end in zip(ordered_boundaries, ordered_boundaries[1:]): - duration = round((end - start).total_seconds() * 1000) - if duration <= 0: + duration_us = _duration_us(start, end) + if duration_us <= 0: continue active = [span for span_start, span_end, span in interval_rows if span_start < end and span_end > start] if not active: @@ -440,10 +462,8 @@ def same_phase_ancestor(span: Mapping[str, object]) -> bool: ] categories = {str(span["phase_name"]) for span in leaves} category = next(iter(categories)) if len(categories) == 1 else "PARALLEL_OVERLAP" - exclusive[category] = exclusive.get(category, 0) + duration - rounding_delta = total - sum(exclusive.values()) - if rounding_delta: - exclusive["UNASSIGNED"] = exclusive.get("UNASSIGNED", 0) + rounding_delta + exclusive_us[category] = exclusive_us.get(category, 0) + duration_us + exclusive = _rounded_category_milliseconds(exclusive_us) interval_union_by_phase: dict[str, int] = {} for phase in inclusive: @@ -459,6 +479,7 @@ def same_phase_ancestor(span: Mapping[str, object]) -> bool: ) reasons = list(dict.fromkeys(boundary_conflicts + parent_conflicts)) share = lambda value: round(value * 100 / total, 3) if isinstance(total, int) and total else 0.0 + exclusive_share = lambda value: round(value * 100 / exclusive_envelope_total, 3) if isinstance(exclusive_envelope_total, int) and exclusive_envelope_total else 0.0 inclusive_rows = [ { "phase": phase, "duration_ms": duration, "share_percent": share(duration), @@ -469,7 +490,7 @@ def same_phase_ancestor(span: Mapping[str, object]) -> bool: for phase, duration in sorted(inclusive.items(), key=lambda item: (-item[1], item[0])) ] exclusive_rows = [ - {"category": category, "duration_ms": duration, "share_percent": share(duration)} + {"category": category, "duration_ms": duration, "share_percent": exclusive_share(duration)} for category, duration in sorted(exclusive.items(), key=lambda item: (-item[1], item[0])) ] def span_label(span: Mapping[str, object]) -> str: @@ -523,26 +544,55 @@ def processing_ancestor(span: Mapping[str, object]) -> bool: legacy_active = max(0, total - external) if isinstance(total, int) else None legacy_overhead = max(0, legacy_active - processing_coverage) if isinstance(legacy_active, int) else None timeline_rows = [] - for span in spans[:500]: + selected_timeline = spans if timeline_limit is None else spans[:timeline_limit] + for span in selected_timeline: projected = dict(span) metadata = span.get("metadata") basis = metadata.get("measurement_basis") if isinstance(metadata, Mapping) else None projected["measurement_basis"] = basis or ( "RECONCILED_WALL_CLOCK" if span.get("outcome") == "STALE" else "UNKNOWN_HISTORICAL" ) + start, end = _timestamp(span.get("started_at")), _timestamp(span.get("completed_at")) + projected["relative_start_ms"] = ( + round(_duration_us(envelope[0], start) / 1000) + if envelope is not None and start is not None else None + ) + projected["relative_end_ms"] = ( + round(_duration_us(envelope[0], end) / 1000) + if envelope is not None and end is not None else None + ) timeline_rows.append(projected) return { "contract_version": TIMING_CALCULATION_VERSION, "scope": "EP_RUN_ATTEMPT", "coverage": { - "state": state, "reason": "; ".join(reasons) if reasons else None, + "state": state, + "coverage": state, + "reason": "; ".join(reasons) if reasons else None, + "missing_reason": "; ".join(reasons) if reasons else None, "expected_observations": len(completed), - "observed_observations": len(completed) - len(boundary_conflicts), + "present_observations": len(completed), + "valid_observations": max(0, len(completed) - len(boundary_conflicts)), + "observed_observations": max(0, len(completed) - len(boundary_conflicts)), + "missing_observations": 0, + "conflicting_observations": len(boundary_conflicts), }, "phase_telemetry_available": True, "historical_total_available": False, "total_wall_time_ms": total, "total_monotonic_duration_ms": total, "total_wall_clock_observed_ms": wall_clock_total, + "exclusive_envelope_duration_ms": exclusive_envelope_total, + "exclusive_measurement_basis": "WALL_CLOCK_INTERVAL_ENVELOPE" if envelope is not None else None, + "clock_difference_ms": ( + wall_clock_total - total + if isinstance(total, int) and isinstance(wall_clock_total, int) else None + ), + "clock_difference_diagnostic": ( + "CLOCK_DIFFERENCE_WITHIN_TOLERANCE" + if isinstance(total, int) and isinstance(wall_clock_total, int) and total != wall_clock_total and not boundary_conflicts + else "CLOCKS_EQUAL" if total == wall_clock_total and total is not None else + "CLOCKS_NOT_COMPARABLE" if boundary_conflicts else None + ), "boundary_reconciliation_ms": ( total - wall_clock_total if isinstance(total, int) and isinstance(wall_clock_total, int) else None @@ -576,12 +626,19 @@ def processing_ancestor(span: Mapping[str, object]) -> bool: "exclusive_distribution": exclusive_rows, "exclusive_distribution_total_ms": sum(exclusive.values()) if exclusive else None, "exclusive_distribution_closes": bool( - state == TIMING_COMPLETE and exclusive and sum(exclusive.values()) == total + state == TIMING_COMPLETE and exclusive + and sum(exclusive.values()) == exclusive_envelope_total ), "timeline": timeline_rows, "timeline_observation_count": len(spans), - "timeline_limit": 500, - "timeline_truncated": len(spans) > 500, + "timeline_limit": timeline_limit, + "timeline_truncated": timeline_limit is not None and len(spans) > timeline_limit, + "timeline_axis": { + "started_at": envelope[0].isoformat() if envelope is not None else None, + "completed_at": envelope[1].isoformat() if envelope is not None else None, + "duration_ms": exclusive_envelope_total, + "measurement_basis": "WALL_CLOCK_INTERVAL_ENVELOPE" if envelope is not None else None, + }, "provider_execution_time_ms": provider, "provider_cumulative_process_duration_ms": provider, "provider_unique_coverage_ms": provider_unique if envelope is not None else None, @@ -625,6 +682,7 @@ def processing_ancestor(span: Mapping[str, object]) -> bool: def timing_summaries( root: Path, run_ids: list[str], *, central_database: Path | None = None, + timeline_limit: int | None = 500, ) -> dict[str, dict[str, object]]: """Load a bounded run population without one timing query per row.""" identifiers = list(dict.fromkeys(value for value in run_ids if isinstance(value, str) and value))[:1000] @@ -663,6 +721,7 @@ def timing_summaries( run_id: timing_summary( root, run_id, central_database=central_database, _spans=spans_by_run[run_id], _historical_total=historical.get(run_id), + timeline_limit=timeline_limit, ) for run_id in identifiers } diff --git a/src/engineering_platform/platform_version.py b/src/engineering_platform/platform_version.py index b6949eca..efe5705f 100644 --- a/src/engineering_platform/platform_version.py +++ b/src/engineering_platform/platform_version.py @@ -12,7 +12,7 @@ SEMVER = re.compile(r"^(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)$") CONTRACT = re.compile(r"^(\d{4})\.(0[1-9]|1[0-2])$") -CURRENT_PLATFORM_VERSION = "2.3.79" +CURRENT_PLATFORM_VERSION = "2.3.80" MANIFEST_FIELDS = frozenset( { "platform_version", diff --git a/src/engineering_platform/provider_usage.py b/src/engineering_platform/provider_usage.py index 51ffa0ac..096ce321 100644 --- a/src/engineering_platform/provider_usage.py +++ b/src/engineering_platform/provider_usage.py @@ -19,6 +19,10 @@ from .agent_state import redact_diagnostic from .storage import EngineeringStorageError, open_storage +from .telemetry_metrics import ( + COMPLETE, CONFLICT, PARTIAL, TELEMETRY_CALCULATION_VERSION, UNAVAILABLE, + aggregate_coverage, metric_coverage, +) RATE_TABLE_VERSION = "2026-08-18" @@ -28,9 +32,7 @@ "gpt-5.6-terra": {"uncached_input": 50.0, "cached_input": 5.0, "output": 300.0}, "gpt-5.6-luna": {"uncached_input": 5.0, "cached_input": 0.5, "output": 30.0}, } -AUTHORITATIVE, DERIVED, UNAVAILABLE = "AUTHORITATIVE", "DERIVED", "UNAVAILABLE" -COMPLETE, PARTIAL, CONFLICT = "COMPLETE", "PARTIAL", "CONFLICT" -TELEMETRY_CALCULATION_VERSION = "telemetry-contract@2.0" +AUTHORITATIVE, DERIVED = "AUTHORITATIVE", "DERIVED" _SPEED_STATES = frozenset({"FAST", "NORMAL_DEFAULT", "OTHER", "UNKNOWN"}) _SAFE_CHURN_TEXT_FIELDS = frozenset({ "interruption_classification", @@ -47,6 +49,13 @@ "file_read_observation_coverage", "tool_output_coverage", }) +_SAFE_CHURN_IDENTITY_FIELDS = frozenset({"historical_pr_identity_hashes"}) +_CHURN_MAX_FIELDS = frozenset({"maximum_tool_output_bytes"}) +_CHURN_UNIQUE_FIELDS = frozenset({"historical_unique_pr_results"}) +_CHURN_COVERAGE_FIELDS = frozenset({ + "event_identity_coverage", "historical_pr_metrics_coverage", + "file_read_observation_coverage", "tool_output_coverage", +}) _MODEL_NORMALIZATION = { "gpt-5.6-sol": "gpt-5.6-sol", "gpt-5.6-terra": "gpt-5.6-terra", @@ -189,7 +198,9 @@ def _structured_pr_id(value: object) -> str | None: repository, number = match.group(1), int(match.group(2)) if number is None: return None - repository = repository.casefold().strip() if isinstance(repository, str) and repository.strip() else "current-repository" + if not isinstance(repository, str) or not repository.strip(): + return None + repository = repository.casefold().strip() return hashlib.sha256(f"{repository}#{number}".encode()).hexdigest() @@ -208,7 +219,7 @@ def _structured_pr_results(raw: object) -> tuple[int, set[str]] | None: return len(values), {str(value) for value in identities} -def churn_from_jsonl(*outputs: str) -> dict[str, int | str]: +def churn_from_jsonl(*outputs: str) -> dict[str, object]: """Measure bounded churn after invocation/item lifecycle deduplication. Codex item ``started``/``updated``/``completed`` records describe one item, @@ -380,6 +391,7 @@ def churn_from_jsonl(*outputs: str) -> dict[str, int | str]: result["distinct_files_read"] = len(reads) result["unique_read_commands"] = len(reads) result["historical_unique_pr_results"] = len(pr_identities) + result["historical_pr_identity_hashes"] = sorted(pr_identities)[:250] result["event_identity_coverage"] = ( CONFLICT if result["conflicting_terminal_events"] else PARTIAL if missing_identity else COMPLETE ) @@ -481,7 +493,7 @@ def persist_provider_invocation(root: Path, invocation: ProviderInvocation, *, c # diagnostic to let the watcher recover the same terminal outcome after a # host interruption. Keep this allow-list deliberately narrow: arbitrary # provider output is never retained here. - churn: dict[str, int | str] = {} + churn: dict[str, object] = {} for key, value in (invocation.churn or {}).items(): number = _number(value) if number is not None: @@ -490,6 +502,13 @@ def persist_provider_invocation(root: Path, invocation: ProviderInvocation, *, c compact = redact_diagnostic(value, limit=120) if compact: churn[key] = compact + elif key in _SAFE_CHURN_IDENTITY_FIELDS and isinstance(value, (list, tuple)): + identities = sorted({ + item.casefold() for item in value + if isinstance(item, str) and re.fullmatch(r"[0-9a-f]{64}", item.casefold()) + })[:250] + if identities: + churn[key] = identities snapshots = tuple( { key: _number(snapshot.get(key)) @@ -576,6 +595,7 @@ def provider_usage_summary( root: Path, run_id: str, *, central_database: Path | None = None, _rows: list[sqlite3.Row] | None = None, _snapshot_rows: list[sqlite3.Row] | None = None, + invocation_limit: int | None = 250, ) -> dict[str, object]: """Derive the canonical run-level usage projection. @@ -619,7 +639,7 @@ def provider_usage_summary( rows, snapshot_rows = _rows, _snapshot_rows if not rows: return {"invocation_detail": UNAVAILABLE} - inputs = [row["input_tokens"] for row in rows if isinstance(row["input_tokens"], int)] + inputs: list[int] = [] snapshot_conflicts: dict[str, set[str]] = {} previous_snapshot: dict[str, sqlite3.Row] = {} for snapshot in snapshot_rows: @@ -633,7 +653,19 @@ def provider_usage_summary( if key in {"input_tokens", "cached_input_tokens"}: snapshot_conflicts[invocation_id].add("uncached_input_tokens") previous_snapshot[invocation_id] = snapshot - churn: dict[str, int | str] = {} + inputs = [ + int(row["input_tokens"]) for row in rows + if isinstance(row["input_tokens"], int) + and "input_tokens" not in snapshot_conflicts.get(str(row["invocation_id"]), set()) + ] + churn: dict[str, object] = {} + pr_identity_hashes: set[str] = set() + coverage_evidence: dict[str, list[Mapping[str, object]]] = { + key: [] for key in _CHURN_COVERAGE_FIELDS + } + text_evidence: dict[str, set[str]] = { + key: set() for key in _SAFE_CHURN_TEXT_FIELDS.difference(_CHURN_COVERAGE_FIELDS) + } for row in rows: try: values = json.loads(row["churn"]) @@ -641,36 +673,70 @@ def provider_usage_summary( values = {} if isinstance(values, dict): for key, value in values.items(): - if isinstance(value, int): + if key in _SAFE_CHURN_IDENTITY_FIELDS and isinstance(value, list): + pr_identity_hashes.update( + item for item in value + if isinstance(item, str) and re.fullmatch(r"[0-9a-f]{64}", item) + ) + elif key in _CHURN_UNIQUE_FIELDS: + # Exact scope-wide uniqueness is derived from the retained + # opaque identity union below, never by summing counts. + continue + elif isinstance(value, int): previous = churn.get(key, 0) - churn[key] = (previous if isinstance(previous, int) else 0) + value + if key in _CHURN_MAX_FIELDS: + churn[key] = max(previous if isinstance(previous, int) else 0, value) + else: + churn[key] = (previous if isinstance(previous, int) else 0) + value elif key in _SAFE_CHURN_TEXT_FIELDS and isinstance(value, str): - # Scope is invocation evidence, not an aggregate. The - # last invocation is the effective run projection. - churn[key] = value + if key in _CHURN_COVERAGE_FIELDS: + coverage_evidence[key].append({ + "coverage": value, + "expected_observations": 1, + "present_observations": 0 if value == UNAVAILABLE else 1, + "valid_observations": 1 if value == COMPLETE else 0, + "conflicting_observations": 1 if value == CONFLICT else 0, + }) + else: + text_evidence[key].add(value) + if pr_identity_hashes: + churn["historical_unique_pr_results"] = len(pr_identity_hashes) + for key, evidence in coverage_evidence.items(): + if evidence: + churn[key] = aggregate_coverage(evidence)["coverage"] + for key, evidence in text_evidence.items(): + if evidence: + churn[key] = next(iter(evidence)) if len(evidence) == 1 else "MIXED" def total(key: str) -> int | float | None: - values = [row[key] for row in rows if isinstance(row[key], (int, float))] + values = [ + row[key] for row in rows + if isinstance(row[key], (int, float)) and not isinstance(row[key], bool) + and key not in snapshot_conflicts.get(str(row["invocation_id"]), set()) + ] return sum(values) if values else None def coverage(key: str, *, compatible: Callable[[sqlite3.Row], bool] | None = None) -> dict[str, object]: - observed = sum( + present = sum( isinstance(row[key], (int, float)) and not isinstance(row[key], bool) - and (compatible(row) if compatible is not None else True) for row in rows ) conflicts = sum(key in values for values in snapshot_conflicts.values()) - state = CONFLICT if conflicts else COMPLETE if observed == len(rows) else PARTIAL if observed else UNAVAILABLE - return { - "coverage": state, - "expected_observations": len(rows), - "observed_observations": observed, - "missing_reason": ( - f"Non-monotone {key} snapshots for {conflicts} invocation(s)" if conflicts - else None if state == COMPLETE - else f"{key} observed for {observed} of {len(rows)} invocations" - ), - } + valid = sum( + isinstance(row[key], (int, float)) and not isinstance(row[key], bool) + and key not in snapshot_conflicts.get(str(row["invocation_id"]), set()) + and (compatible(row) if compatible is not None else True) + for row in rows + ) + reason = ( + f"Non-monotone {key} snapshots for {conflicts} invocation(s)" if conflicts + else None if valid == len(rows) + else f"{key} valid for {valid} of {len(rows)} invocations" + ) + return metric_coverage( + expected=len(rows), present=present, valid=valid, + conflicting=conflicts, reason=reason, + ) def metric( key: str, value: object, *, meaning: str, unit: str, provenance: str = AUTHORITATIVE, @@ -705,6 +771,8 @@ def metric( if isinstance(row["input_tokens"], int) and isinstance(row["cached_input_tokens"], int) and row["cached_input_tokens"] <= row["input_tokens"] + and "input_tokens" not in snapshot_conflicts.get(str(row["invocation_id"]), set()) + and "cached_input_tokens" not in snapshot_conflicts.get(str(row["invocation_id"]), set()) ] compatible_input = sum(row["input_tokens"] for row in compatible_cache_rows) compatible_cached = sum(row["cached_input_tokens"] for row in compatible_cache_rows) @@ -768,18 +836,34 @@ def metric( "usage_coverage": CONFLICT if conflicts else COMPLETE if not missing else PARTIAL if len(missing) < 3 else UNAVAILABLE, "missing_usage_fields": missing, "conflicting_usage_fields": sorted(conflicts), + "usage_metrics": { + key: { + "value": row[key] if key not in conflicts and isinstance(row[key], int) else None, + **metric_coverage( + expected=1, + present=int(isinstance(row[key], int)), + valid=int(isinstance(row[key], int) and key not in conflicts), + conflicting=int(key in conflicts), + ), + } + for key in ("input_tokens", "cached_input_tokens", "uncached_input_tokens", "output_tokens") + }, + "timing_correlation": UNAVAILABLE, }) exact_pr_coverage = churn.get("historical_pr_metrics_coverage", UNAVAILABLE) + unique_pr_coverage = exact_pr_coverage + if exact_pr_coverage == COMPLETE and len(rows) > 1 and not pr_identity_hashes: + unique_pr_coverage = PARTIAL exact_file_coverage = churn.get("file_read_observation_coverage", UNAVAILABLE) result = { "contract_version": TELEMETRY_CALCULATION_VERSION, "scope": "EP_RUN_ATTEMPT", "invocation_detail": AUTHORITATIVE, "provider_invocation_count": len(rows), - "invocations": invocation_rows[:250], + "invocations": invocation_rows if invocation_limit is None else invocation_rows[:invocation_limit], "invocation_observation_count": len(invocation_rows), - "invocation_table_limit": 250, - "invocation_table_truncated": len(invocation_rows) > 250, + "invocation_table_limit": invocation_limit, + "invocation_table_truncated": invocation_limit is not None and len(invocation_rows) > invocation_limit, "metrics": usage_metrics, "provider_invocations_by_role": calls_by_role, "uncached_input_by_role": uncached_input_by_role or None, @@ -791,8 +875,20 @@ def metric( "total_provider_execution_ms": total("duration_ms"), "cache_ratio_percent": cache_ratio, "cache_ratio_population": { - "coverage": COMPLETE if len(compatible_cache_rows) == len(rows) else PARTIAL if compatible_cache_rows else UNAVAILABLE, - "expected_observations": len(rows), "observed_observations": len(compatible_cache_rows), + **metric_coverage( + expected=len(rows), + present=sum( + isinstance(row["input_tokens"], int) and isinstance(row["cached_input_tokens"], int) + for row in rows + ), + valid=len(compatible_cache_rows), + conflicting=sum( + bool({"input_tokens", "cached_input_tokens"}.intersection( + snapshot_conflicts.get(str(row["invocation_id"]), set()) + )) + for row in rows + ), + ), "input_tokens": compatible_input or None, "cached_input_tokens": compatible_cached or None, }, "max_input_tokens_per_invocation": max(inputs) if inputs else None, @@ -814,9 +910,17 @@ def metric( "historical_pr_queries": churn.get("historical_pr_queries") if observed_history else None, "historical_pr_results": churn.get("historical_pr_results") if observed_history else None, "historical_pr_result_occurrences": churn.get("historical_pr_result_occurrences") if exact_pr_coverage != UNAVAILABLE else None, - "historical_unique_pr_results": churn.get("historical_unique_pr_results") if exact_pr_coverage != UNAVAILABLE else None, + "historical_unique_pr_results": ( + churn.get("historical_unique_pr_results") + if pr_identity_hashes or len(rows) == 1 else None + ) if exact_pr_coverage != UNAVAILABLE else None, "historical_pr_details_fetched": churn.get("historical_pr_details_fetched") if exact_pr_coverage != UNAVAILABLE else None, - "historical_pr_metrics_coverage": exact_pr_coverage, + "historical_pr_metrics_coverage": unique_pr_coverage, + "historical_unique_pr_coverage": unique_pr_coverage, + "historical_unique_pr_missing_reason": ( + "Invocation-level unique counts lack scope-wide opaque identities" + if unique_pr_coverage == PARTIAL and not pr_identity_hashes else None + ), "legacy_historical_pr_output_lines": churn.get("historical_pr_results_legacy_lines") or ( churn.get("historical_pr_results") if observed_history and exact_pr_coverage == UNAVAILABLE else None ), @@ -842,6 +946,7 @@ def metric( def provider_usage_summaries( root: Path, run_ids: list[str], *, central_database: Path | None = None, + invocation_limit: int | None = 250, ) -> dict[str, dict[str, object]]: """Load a bounded run population in two queries, then reuse the canonical reducer.""" identifiers = list(dict.fromkeys(value for value in run_ids if isinstance(value, str) and value))[:1000] @@ -889,6 +994,7 @@ def provider_usage_summaries( run_id: provider_usage_summary( root, run_id, central_database=central_database, _rows=rows_by_run[run_id], _snapshot_rows=snapshots_by_run[run_id], + invocation_limit=invocation_limit, ) for run_id in identifiers } diff --git a/src/engineering_platform/server.py b/src/engineering_platform/server.py index 750c5e84..5e19dda0 100644 --- a/src/engineering_platform/server.py +++ b/src/engineering_platform/server.py @@ -71,6 +71,7 @@ from . import server_service from . import system_server_service from . import storage +from . import telemetry_export from . import managed_codex_runtime from . import provider_readiness from .platform_components import ( @@ -104,6 +105,11 @@ from .execution_timing import timing_summaries, timing_summary from .provider_usage import provider_usage_summaries, provider_usage_summary from .telemetry_contract import load_lineage_graph, run_telemetry_snapshot +from .telemetry_metrics import ( + aggregate_coverage as aggregate_telemetry_coverage, + aggregate_numeric_metric, + metric_coverage, +) from .parity_context import ParityProjectStore, project_context from .platform_version import CURRENT_PLATFORM_VERSION, EngineeringPlatformManifest from .providers import ( @@ -3932,28 +3938,32 @@ def average(key: str) -> float | None: for timing in (projection(snapshot, "timing"),) if isinstance(timing.get(key), int) and timing[key] >= 0] return round(sum(values) / len(values), 3) if values else None - usage_metrics: dict[str, list[Mapping[str, object]]] = {} + usage_metrics: dict[str, list[Mapping[str, object]]] = { + name: [] for name in ("input_tokens", "cached_input_tokens", "uncached_input_tokens", "output_tokens") + } for _, snapshot in rows: usage = projection(snapshot, "usage") metrics = usage.get("metrics", {}) - if isinstance(metrics, Mapping): - for name, metric in metrics.items(): - if isinstance(metric, Mapping): - usage_metrics.setdefault(str(name), []).append(metric) - def observed_tokens(name: str) -> int | None: - values = [int(metric["value"]) for metric in usage_metrics.get(name, []) - if isinstance(metric.get("value"), int)] - return sum(values) if values else None - input_tokens, output_tokens = observed_tokens("input_tokens"), observed_tokens("output_tokens") - coverage_states = { - str(metric.get("coverage")) - for metrics in usage_metrics.values() for metric in metrics - if metric.get("coverage") + for name in usage_metrics: + metric = metrics.get(name) if isinstance(metrics, Mapping) else None + usage_metrics[name].append(metric if isinstance(metric, Mapping) else { + "value": None, "unit": "tokens", "provenance": "UNAVAILABLE", + **metric_coverage( + expected=None, present=0, valid=0, + reason=f"Usage projection unavailable for {snapshot.get('run_id', 'run')}", + ), + }) + daily_usage = { + name: aggregate_numeric_metric(values, aggregation_level="UTC_DAY", unit="tokens") + for name, values in usage_metrics.items() } + input_tokens = daily_usage["input_tokens"]["value"] + output_tokens = daily_usage["output_tokens"]["value"] + coverage_states = {str(metric["coverage"]) for metric in daily_usage.values()} measurement_coverage = ( "CONFLICT" if "CONFLICT" in coverage_states else "PARTIAL" if "PARTIAL" in coverage_states or "UNAVAILABLE" in coverage_states else - "COMPLETE" if coverage_states else "UNAVAILABLE" + "COMPLETE" ) entries.append({ "date": date, @@ -3972,6 +3982,7 @@ def observed_tokens(name: str) -> int | None: "total_tokens": input_tokens + output_tokens if input_tokens is not None and output_tokens is not None else None, "measurement_coverage": measurement_coverage, + "usage_metrics": daily_usage, "contract_version": selected_version if ( selected_version := next((snapshot.get("contract_version") for _, snapshot in rows if snapshot.get("contract_version")), None) @@ -4055,7 +4066,9 @@ def timing_aggregate(key: str) -> dict[str, int] | None: } -def _central_console_telemetry_detail(data_root: Path, project_id: str, execution_date: str) -> dict[str, object] | None: +def _central_console_telemetry_detail( + data_root: Path, project_id: str, execution_date: str, *, full: bool = False, +) -> dict[str, object] | None: """Return the canonical contract used by UI, Markdown and JSON export.""" if not re.fullmatch(r"[0-9]{4}-[0-9]{2}-[0-9]{2}", execution_date): return None @@ -4075,7 +4088,7 @@ def _central_console_telemetry_detail(data_root: Path, project_id: str, executio matching.append(record) if not matching: return None - selected_records = matching[:MAX_TELEMETRY_DAY_RUNS] + selected_records = matching if full else matching[:MAX_TELEMETRY_DAY_RUNS] identifiers = [str(record["run_id"]) for record in selected_records] lineage_graph = load_lineage_graph( data_root, central_database=data_root / SERVER_DATABASE_FILENAME, @@ -4100,9 +4113,11 @@ def _central_console_telemetry_detail(data_root: Path, project_id: str, executio try: usage_cache = provider_usage_summaries( data_root, telemetry_identifiers, central_database=data_root / SERVER_DATABASE_FILENAME, + invocation_limit=None if full else 250, ) timing_cache = timing_summaries( data_root, telemetry_identifiers, central_database=data_root / SERVER_DATABASE_FILENAME, + timeline_limit=None if full else 500, ) except (storage.EngineeringStorageError, sqlite3.DatabaseError): usage_cache, timing_cache = {}, {} @@ -4175,6 +4190,10 @@ def aggregate(values: list[int]) -> dict[str, int] | None: exclusive_values[str(row["category"])] = exclusive_values.get(str(row["category"]), 0) + int(row["duration_ms"]) longest_spans.extend(row for row in timing.get("longest_individual_spans", []) if isinstance(row, Mapping)) wall_total = sum(int(value) for value in (timing.get("total_wall_time_ms") for timing in timings) if isinstance(value, int)) + exclusive_envelope_total = sum( + int(value) for value in (timing.get("exclusive_envelope_duration_ms") for timing in timings) + if isinstance(value, int) + ) phases = [ {"phase": phase, **aggregate(values), "share_percent": round(sum(values) * 100 / wall_total, 3) if wall_total else None, @@ -4183,7 +4202,7 @@ def aggregate(values: list[int]) -> dict[str, int] | None: ] exclusive = [ {"category": category, "duration_ms": value, - "share_percent": round(value * 100 / wall_total, 3) if wall_total else None} + "share_percent": round(value * 100 / exclusive_envelope_total, 3) if exclusive_envelope_total else None} for category, value in sorted(exclusive_values.items(), key=lambda item: (-item[1], item[0])) ] longest_average = next(iter(sorted( @@ -4200,17 +4219,50 @@ def values(key: str) -> list[int]: return [int(timing[key]) for timing in timings if isinstance(timing.get(key), int)] observed_usage: dict[str, dict[str, object]] = {} for name in ("input_tokens", "cached_input_tokens", "uncached_input_tokens", "output_tokens"): - metric_rows = [usage.get("metrics", {}).get(name) for usage in usages if isinstance(usage.get("metrics"), Mapping)] - numeric = [int(metric["value"]) for metric in metric_rows if isinstance(metric, Mapping) and isinstance(metric.get("value"), int)] - expected = sum(int(metric.get("expected_observations", 0)) for metric in metric_rows if isinstance(metric, Mapping)) - observed = sum(int(metric.get("observed_observations", 0)) for metric in metric_rows if isinstance(metric, Mapping)) - observed_usage[name] = { - "value": sum(numeric) if numeric else None, - "coverage": "COMPLETE" if expected and observed == expected else "PARTIAL" if observed else "UNAVAILABLE", - "expected_observations": expected, "observed_observations": observed, + metric_rows: list[Mapping[str, object]] = [] + for index, usage in enumerate(usages): + metrics = usage.get("metrics") + metric = metrics.get(name) if isinstance(metrics, Mapping) else None + metric_rows.append(metric if isinstance(metric, Mapping) else { + "value": None, "unit": "tokens", "provenance": "UNAVAILABLE", + **metric_coverage( + expected=None, present=0, valid=0, + reason=f"Usage projection unavailable for {identifiers[index]}", + ), + "source_snapshot_reference": identifiers[index], + }) + observed_usage[name] = aggregate_numeric_metric( + metric_rows, aggregation_level="UTC_DAY", unit="tokens", + ) + cache_sources: list[Mapping[str, object]] = [] + cache_inputs = 0 + cache_cached = 0 + for index, usage in enumerate(usages): + population = usage.get("cache_ratio_population") + if isinstance(population, Mapping): + cache_sources.append(population) + if population.get("coverage") != "CONFLICT": + if isinstance(population.get("input_tokens"), int): + cache_inputs += int(population["input_tokens"]) + if isinstance(population.get("cached_input_tokens"), int): + cache_cached += int(population["cached_input_tokens"]) + else: + cache_sources.append(metric_coverage( + expected=None, present=0, valid=0, + reason=f"Cache-ratio population unavailable for {identifiers[index]}", + )) + cache_coverage = aggregate_telemetry_coverage(cache_sources) + timing_coverage_sources = [ + { + "coverage": timing.get("coverage", {}).get("state"), + **{key: timing.get("coverage", {}).get(key) for key in ( + "expected_observations", "present_observations", "valid_observations", + "observed_observations", "conflicting_observations", "missing_reason", + )}, } - cache_inputs = sum(int(usage.get("cache_ratio_population", {}).get("input_tokens", 0) or 0) for usage in usages if isinstance(usage.get("cache_ratio_population"), Mapping)) - cache_cached = sum(int(usage.get("cache_ratio_population", {}).get("cached_input_tokens", 0) or 0) for usage in usages if isinstance(usage.get("cache_ratio_population"), Mapping)) + for timing in timings if isinstance(timing.get("coverage"), Mapping) + ] + timing_coverage = aggregate_telemetry_coverage(timing_coverage_sources) summary = { "executions": len(run_rows), "population": len(run_rows), "completed": sum(row["status"] == "COMPLETE" for row in run_rows), @@ -4224,19 +4276,34 @@ def values(key: str) -> list[int]: "external_wait": aggregate(values("external_wait_time_ms")), "unassigned": aggregate(values("unassigned_time_ms")), "usage": observed_usage, - "cache_ratio_percent": round(cache_cached * 100 / cache_inputs, 3) if cache_inputs else None, + "cache_ratio_percent": ( + round(cache_cached * 100 / cache_inputs, 3) + if cache_inputs and cache_coverage["coverage"] != "CONFLICT" else None + ), + "cache_ratio_population": { + **cache_coverage, + "input_tokens": cache_inputs or None, + "cached_input_tokens": cache_cached or None, + }, + "timing_coverage": timing_coverage, } return { "contract_version": selected[0][1].get("contract_version"), "source_snapshot_references": [str(record["run_id"]) for record, _ in selected], "date": execution_date, "timezone": "UTC", "scope": "EP_RUN_ATTEMPTS_IN_UTC_DAY", "matching_run_count": len(matching), "returned_run_count": len(selected), - "runs_truncated": len(matching) > len(selected), "run_limit": MAX_TELEMETRY_DAY_RUNS, + "runs_truncated": len(matching) > len(selected), "run_limit": None if full else MAX_TELEMETRY_DAY_RUNS, "summary": summary, "runs": run_rows, "inclusive_phases": phases, "phases": phases, "inclusive_shares_additive": False, "exclusive_distribution": exclusive, - "exclusive_distribution_closes": bool(wall_total and sum(exclusive_values.values()) == wall_total), + "exclusive_envelope_duration_ms": exclusive_envelope_total or None, + "exclusive_distribution_closes": bool( + exclusive_envelope_total + and timing_coverage["coverage"] == "COMPLETE" + and all(bool(timing.get("exclusive_distribution_closes")) for timing in timings) + and sum(exclusive_values.values()) == exclusive_envelope_total + ), "phase_telemetry_available": bool(phases), "bottlenecks": { "longest_average_phase": longest_average, @@ -5205,6 +5272,23 @@ def _send_text(self, status_code: int, payload: str) -> None: except (BrokenPipeError, ConnectionResetError): return + def _send_download(self, payload: bytes, *, content_type: str, filename: str) -> None: + """Return one bounded read-only export with explicit browser metadata.""" + self.send_response(200) + self.send_header("Content-Type", content_type) + self.send_header("Content-Disposition", f'attachment; filename="{filename}"') + self.send_header("Content-Length", str(len(payload))) + self.send_header("Cache-Control", "no-store") + self.send_header("X-Content-Type-Options", "nosniff") + route = getattr(self, "_console_route", None) + if route is not None: + self.send_header("EP-Console-Route-Owner", route.owner) + self.end_headers() + try: + self.wfile.write(payload) + except (BrokenPipeError, ConnectionResetError): + return + def _send_artifact_bytes(self, payload: bytes, instance_id: str) -> None: """Return the verified immutable artifact bytes without JSON re-encoding.""" self.send_response(200) @@ -6034,6 +6118,82 @@ def _delegate_dashboard(self, method: str) -> None: "source": "CENTRAL", }) return + if request.path == "/api/telemetry/export": + parameters = parse_qs(request.query) + export_format = (parameters.get("format") or [""])[0] + locale = (parameters.get("locale") or ["en"])[0] + sort_key = (parameters.get("sort") or ["date"])[0] + direction = (parameters.get("direction") or ["desc"])[0] + allowed_sort = { + "date", "prompt_count", "average_total_execution_seconds", + "average_queue_wait_seconds", "input_tokens", "output_tokens", + "total_tokens", "complete_count", "blocked_count", "failed_count", + } + if ( + export_format not in {"markdown", "json"} + or locale not in telemetry_export.SUPPORTED_LOCALES + or sort_key not in allowed_sort or direction not in {"asc", "desc"} + ): + self._send(400, {"error": "TELEMETRY_EXPORT_SELECTION_INVALID"}) + return + configuration = _central_console_configuration(self.server.data_root) # type: ignore[attr-defined] + model = telemetry_export.overview_model( + project_id=selected, + rows=_central_console_telemetry(self.server.data_root, selected), # type: ignore[attr-defined] + sort_key=sort_key, sort_direction=direction, locale=locale, + retention_days=configuration.get("telemetry_retention_days") + if isinstance(configuration.get("telemetry_retention_days"), int) else None, + ) + markdown = export_format == "markdown" + payload = telemetry_export.serialize_markdown(model) if markdown else telemetry_export.serialize_json(model) + self._send_download( + payload, + content_type="text/markdown; charset=utf-8" if markdown else "application/json; charset=utf-8", + filename=f"telemetry-overview-{selected}-utc.{('md' if markdown else 'json')}", + ) + return + telemetry_export_match = re.fullmatch( + r"/api/telemetry/([0-9]{4}-[0-9]{2}-[0-9]{2})/export", request.path, + ) + if telemetry_export_match: + parameters = parse_qs(request.query) + export_format = (parameters.get("format") or [""])[0] + locale = (parameters.get("locale") or ["en"])[0] + scope = (parameters.get("scope") or ["UTC_DAY_DETAIL"])[0] + run_id = (parameters.get("run_id") or [None])[0] + if ( + export_format not in {"markdown", "json"} + or locale not in telemetry_export.SUPPORTED_LOCALES + or scope not in {"UTC_DAY_DETAIL", "EP_RUN_ATTEMPT", "EXECUTION_CHAIN"} + or (run_id is not None and not _SAFE_REPORT_ID.fullmatch(run_id)) + or (scope != "UTC_DAY_DETAIL" and run_id is None) + ): + self._send(400, {"error": "TELEMETRY_EXPORT_SELECTION_INVALID"}) + return + export_date = telemetry_export_match.group(1) + detail = _central_console_telemetry_detail( + self.server.data_root, selected, export_date, full=True, # type: ignore[attr-defined] + ) + if detail is None: + self._send(404, {"error": "TELEMETRY_NOT_FOUND"}) + return + try: + model = telemetry_export.detail_model( + project_id=selected, execution_date=export_date, detail=detail, + scope=scope, run_id=run_id, locale=locale, + ) + except ValueError: + self._send(404, {"error": "TELEMETRY_EXPORT_RUN_NOT_FOUND"}) + return + markdown = export_format == "markdown" + payload = telemetry_export.serialize_markdown(model) if markdown else telemetry_export.serialize_json(model) + context = run_id if run_id is not None else export_date + self._send_download( + payload, + content_type="text/markdown; charset=utf-8" if markdown else "application/json; charset=utf-8", + filename=f"telemetry-detail-{selected}-{scope.casefold().replace('_', '-')}-{context}.{('md' if markdown else 'json')}", + ) + return telemetry_match = re.fullmatch(r"/api/telemetry/([0-9]{4}-[0-9]{2}-[0-9]{2})", request.path) if telemetry_match: detail = _central_console_telemetry_detail( diff --git a/src/engineering_platform/telemetry_contract.py b/src/engineering_platform/telemetry_contract.py index e7782b53..5c663b43 100644 --- a/src/engineering_platform/telemetry_contract.py +++ b/src/engineering_platform/telemetry_contract.py @@ -17,6 +17,7 @@ COMPLETE, CONFLICT, PARTIAL, TELEMETRY_CALCULATION_VERSION, UNAVAILABLE, provider_usage_summary, ) +from .telemetry_metrics import aggregate_coverage, aggregate_numeric_metric, metric_coverage from .storage import open_storage @@ -167,11 +168,13 @@ def execution_chain_summary( issues.append(f"chain-limit:{MAX_CHAIN_RUNS}") chain_rows: list[dict[str, object]] = [] - invocation_ids: set[str] = set() duplicate_invocations = 0 - usage_totals = {key: 0 for key in ("input_tokens", "cached_input_tokens", "uncached_input_tokens", "output_tokens")} - usage_observed = {key: 0 for key in usage_totals} - usage_expected = 0 + metric_names = ("input_tokens", "cached_input_tokens", "uncached_input_tokens", "output_tokens") + usage_sources: dict[str, list[Mapping[str, object]]] = {key: [] for key in metric_names} + provider_invocation_count = 0 + cache_ratio_sources: list[Mapping[str, object]] = [] + cache_ratio_input = 0 + cache_ratio_cached = 0 intervals: list[tuple[datetime, datetime]] = [] processing_ms = 0 mission_values: set[str] = set() @@ -204,28 +207,35 @@ def execution_chain_summary( if usage is None: usage = provider_usage_summary(root, member, central_database=central_database) usage_cache[member] = usage - invocations = usage.get("invocations", []) if isinstance(usage, Mapping) else [] - if isinstance(invocations, list): - for invocation in invocations: - identity_value = invocation.get("invocation_id") if isinstance(invocation, Mapping) else None - if not isinstance(identity_value, str): - issues.append(f"missing-invocation-identity:{member}") - continue - if identity_value in invocation_ids: - duplicate_invocations += 1 - continue - invocation_ids.add(identity_value) - usage_expected += 1 - conflicts = invocation.get("conflicting_usage_fields", []) - if isinstance(conflicts, list) and conflicts: - issues.append(f"conflicting-usage:{identity_value}") - for key in usage_totals: - value = invocation.get(key) - if isinstance(value, int) and not isinstance(value, bool): - usage_totals[key] += value - usage_observed[key] += 1 - if isinstance(usage, Mapping) and usage.get("invocation_table_truncated"): - issues.append(f"invocation-projection-truncated:{member}") + if isinstance(usage, Mapping): + count = usage.get("provider_invocation_count") + if isinstance(count, int) and not isinstance(count, bool) and count >= 0: + provider_invocation_count += count + metrics = usage.get("metrics") + for key in metric_names: + metric = metrics.get(key) if isinstance(metrics, Mapping) else None + if isinstance(metric, Mapping): + usage_sources[key].append(metric) + else: + usage_sources[key].append({ + "value": None, "provenance": UNAVAILABLE, "unit": "tokens", + **metric_coverage(expected=None, present=0, valid=0, + reason=f"Usage projection unavailable for {member}"), + "source_snapshot_reference": member, + }) + population = usage.get("cache_ratio_population") + if isinstance(population, Mapping): + cache_ratio_sources.append(population) + if population.get("coverage") != CONFLICT: + if isinstance(population.get("input_tokens"), int): + cache_ratio_input += int(population["input_tokens"]) + if isinstance(population.get("cached_input_tokens"), int): + cache_ratio_cached += int(population["cached_input_tokens"]) + else: + cache_ratio_sources.append(metric_coverage( + expected=None, present=0, valid=0, + reason=f"Cache-ratio population unavailable for {member}", + )) timing = timing_cache.get(member) if timing is None: timing = timing_summary(root, member, central_database=central_database) @@ -272,17 +282,17 @@ def execution_chain_summary( if root_context is None or not bool(root_context["fresh_submission"]): issues.append("original-attempt-not-proven") coverage = CONFLICT if any(value.startswith(("cycle", "conflicting", "duplicate")) for value in issues) else PARTIAL if issues else COMPLETE - usage_metrics = {} - for key, value in usage_totals.items(): - observed = usage_observed[key] - usage_metrics[key] = { - "value": value if observed else None, - "coverage": COMPLETE if observed == usage_expected and usage_expected else PARTIAL if observed else UNAVAILABLE, - "expected_observations": usage_expected, "observed_observations": observed, - "provenance": "AUTHORITATIVE", "unit": "tokens", - "aggregation_level": "EXECUTION_CHAIN", "calculation_version": TELEMETRY_CALCULATION_VERSION, - "missing_reason": None if observed == usage_expected and usage_expected else f"{key} observed for {observed} of {usage_expected} invocations", - } + usage_metrics = { + key: aggregate_numeric_metric( + values, aggregation_level="EXECUTION_CHAIN", aggregation="sum", unit="tokens", + ) + for key, values in usage_sources.items() + } + cache_ratio_coverage = aggregate_coverage(cache_ratio_sources) + cache_ratio = ( + round(cache_ratio_cached * 100 / cache_ratio_input, 3) + if cache_ratio_input and cache_ratio_coverage["coverage"] != CONFLICT else None + ) return { "contract_version": TELEMETRY_CALCULATION_VERSION, "scope": "EXECUTION_CHAIN", "selected_run_id": run_id, "root_run_id": root_run, @@ -298,9 +308,16 @@ def execution_chain_summary( "last_completed_at": last_end.isoformat() if last_end else None, "elapsed_ms": elapsed_ms, "processing_time_ms": processing_ms, "covered_elapsed_ms": covered_ms if intervals else None, "inter_attempt_gap_ms": gaps_ms, - "provider_invocation_count": usage_expected, + "provider_invocation_count": provider_invocation_count, "duplicate_invocation_count": duplicate_invocations, - "usage_metrics": usage_metrics, "outside_selected_window_count": outside_window, + "usage_metrics": usage_metrics, + "cache_ratio_percent": cache_ratio, + "cache_ratio_population": { + **cache_ratio_coverage, + "input_tokens": cache_ratio_input or None, + "cached_input_tokens": cache_ratio_cached or None, + }, + "outside_selected_window_count": outside_window, "runs": sorted(chain_rows, key=lambda row: str(row.get("started_at") or "")), } diff --git a/src/engineering_platform/telemetry_export.py b/src/engineering_platform/telemetry_export.py new file mode 100644 index 00000000..b5e41cfa --- /dev/null +++ b/src/engineering_platform/telemetry_export.py @@ -0,0 +1,363 @@ +"""Canonical telemetry export models and Markdown/JSON serializers. + +Exports are read-only projections. Both formats serialize the same model; +neither serializer recalculates telemetry metrics. +""" + +from __future__ import annotations + +from datetime import datetime, timezone +import hashlib +import json +from typing import Mapping, Sequence + +from .telemetry_metrics import aggregate_numeric_metric, metric_coverage + + +EXPORT_SCHEMA_VERSION = "telemetry-export@1.0" +SUPPORTED_LOCALES = frozenset({"en", "nl", "de", "fr", "es"}) + +_LABELS = { + "en": {"overview": "Telemetry overview", "detail": "Telemetry detail", "selection": "Selection", "summary": "Summary", "coverage": "Coverage", "runs": "Runs", "inclusive": "Inclusive phase workload", "inclusive_note": "Inclusive shares may overlap and are not additive.", "exclusive": "Exclusive elapsed-time distribution", "bottlenecks": "Bottlenecks", "invocations": "Provider invocations", "timeline": "Timeline", "chain": "Execution chain", "limitations": "Limitations and conflicts", "field": "Field", "value": "Value", "unavailable": "Unavailable"}, + "nl": {"overview": "Telemetrieoverzicht", "detail": "Telemetriedetail", "selection": "Selectie", "summary": "Samenvatting", "coverage": "Dekking", "runs": "Uitvoeringen", "inclusive": "Inclusieve fasewerklast", "inclusive_note": "Inclusieve aandelen mogen overlappen en zijn niet optelbaar.", "exclusive": "Exclusieve doorlooptijdverdeling", "bottlenecks": "Knelpunten", "invocations": "Providerinvocations", "timeline": "Tijdlijn", "chain": "Uitvoeringsketen", "limitations": "Beperkingen en conflicten", "field": "Veld", "value": "Waarde", "unavailable": "Niet beschikbaar"}, + "de": {"overview": "Telemetrieübersicht", "detail": "Telemetriedetail", "selection": "Auswahl", "summary": "Zusammenfassung", "coverage": "Abdeckung", "runs": "Ausführungen", "inclusive": "Inklusive Phasenarbeitslast", "inclusive_note": "Inklusive Anteile dürfen sich überlappen und sind nicht addierbar.", "exclusive": "Exklusive Laufzeitverteilung", "bottlenecks": "Engpässe", "invocations": "Provider-Aufrufe", "timeline": "Zeitachse", "chain": "Ausführungskette", "limitations": "Einschränkungen und Konflikte", "field": "Feld", "value": "Wert", "unavailable": "Nicht verfügbar"}, + "fr": {"overview": "Vue d’ensemble de la télémétrie", "detail": "Détail de télémétrie", "selection": "Sélection", "summary": "Résumé", "coverage": "Couverture", "runs": "Exécutions", "inclusive": "Charge de phase inclusive", "inclusive_note": "Les parts inclusives peuvent se chevaucher et ne sont pas additionnables.", "exclusive": "Répartition exclusive du temps", "bottlenecks": "Goulets d’étranglement", "invocations": "Invocations fournisseur", "timeline": "Chronologie", "chain": "Chaîne d’exécution", "limitations": "Limites et conflits", "field": "Champ", "value": "Valeur", "unavailable": "Indisponible"}, + "es": {"overview": "Resumen de telemetría", "detail": "Detalle de telemetría", "selection": "Selección", "summary": "Resumen", "coverage": "Cobertura", "runs": "Ejecuciones", "inclusive": "Carga de fase inclusiva", "inclusive_note": "Las proporciones inclusivas pueden solaparse y no son sumables.", "exclusive": "Distribución exclusiva del tiempo", "bottlenecks": "Cuellos de botella", "invocations": "Invocaciones del proveedor", "timeline": "Cronología", "chain": "Cadena de ejecución", "limitations": "Limitaciones y conflictos", "field": "Campo", "value": "Valor", "unavailable": "No disponible"}, +} + + +def overview_model( + *, project_id: str, rows: Sequence[Mapping[str, object]], sort_key: str, + sort_direction: str, locale: str, retention_days: int | None = None, +) -> dict[str, object]: + comparable = [row for row in rows if _sort_value(row, sort_key)[0] == 0] + unavailable = [row for row in rows if _sort_value(row, sort_key)[0] != 0] + ordered = sorted( + comparable, key=lambda row: _sort_value(row, sort_key)[1], + reverse=sort_direction == "desc", + ) + sorted(unavailable, key=lambda row: str(row.get("date") or ""), reverse=True) + selection = { + "project_id": project_id, "scope": "TELEMETRY_OVERVIEW", + "timezone": "UTC", "filters": {"retention_days": retention_days}, + "aggregation_level": "UTC_DAY", "sort": {"key": sort_key, "direction": sort_direction}, + } + names = ("input_tokens", "cached_input_tokens", "uncached_input_tokens", "output_tokens") + usage_summary: dict[str, object] = {} + for name in names: + metrics = [] + for row in ordered: + usage = row.get("usage_metrics") + metric = usage.get(name) if isinstance(usage, Mapping) else None + metrics.append(metric if isinstance(metric, Mapping) else { + "value": None, + **metric_coverage( + expected=None, present=0, valid=0, + reason=f"{name} coverage is unavailable for {row.get('date', 'day')}", + ), + }) + usage_summary[name] = aggregate_numeric_metric( + metrics, aggregation_level="SELECTED_OVERVIEW_POPULATION", unit="tokens", + ) + data = { + "summary": { + "day_count": len(ordered), + "run_count": sum(int(row.get("prompt_count", 0)) for row in ordered if isinstance(row.get("prompt_count"), int)), + "completed": sum(int(row.get("complete_count", 0)) for row in ordered if isinstance(row.get("complete_count"), int)), + "blocked": sum(int(row.get("blocked_count", 0)) for row in ordered if isinstance(row.get("blocked_count"), int)), + "failed": sum(int(row.get("failed_count", 0)) for row in ordered if isinstance(row.get("failed_count"), int)), + "usage": usage_summary, + }, + "rows": [dict(row) for row in ordered], "row_count": len(ordered), + } + return _envelope( + locale=locale, selection=selection, data={"overview": data}, + references=[str(row.get("date")) for row in ordered if row.get("date")], + displayed_population=len(ordered), full_population=len(ordered), export_complete=True, + contract_version=_contract_version(ordered), + ) + + +def detail_model( + *, project_id: str, execution_date: str, detail: Mapping[str, object], + scope: str, run_id: str | None, locale: str, +) -> dict[str, object]: + runs = [row for row in detail.get("runs", []) if isinstance(row, Mapping)] + selected = next((row for row in runs if row.get("run_id") == run_id), None) + if scope in {"EP_RUN_ATTEMPT", "EXECUTION_CHAIN"} and selected is None: + raise ValueError("TELEMETRY_EXPORT_RUN_NOT_FOUND") + if scope == "EP_RUN_ATTEMPT": + data: dict[str, object] = {"attempt": _safe_run(selected or {})} + references = [str(run_id)] + displayed_population = full_population = 1 + export_complete = True + elif scope == "EXECUTION_CHAIN": + snapshot = (selected or {}).get("telemetry_snapshot", {}) + chain = snapshot.get("chain", {}) if isinstance(snapshot, Mapping) else {} + data = {"selected_attempt": _safe_run(selected or {}), "chain": _safe_chain(chain)} + references = [str(row.get("run_id")) for row in chain.get("runs", []) if isinstance(row, Mapping)] if isinstance(chain, Mapping) else [str(run_id)] + chain_runs = chain.get("runs", []) if isinstance(chain, Mapping) else [] + displayed_population = full_population = len(chain_runs) if isinstance(chain_runs, list) else 0 + reasons = chain.get("reasons", []) if isinstance(chain, Mapping) else [] + export_complete = not any( + isinstance(reason, str) and reason.startswith("chain-limit:") + for reason in (reasons if isinstance(reasons, list) else []) + ) + else: + scope = "UTC_DAY_DETAIL" + data = {"day_detail": _safe_day_detail(detail)} + references = [str(value) for value in detail.get("source_snapshot_references", []) if isinstance(value, str)] + matching = detail.get("matching_run_count") + returned = detail.get("returned_run_count") + displayed_population = int(returned) if isinstance(returned, int) else len(runs) + full_population = int(matching) if isinstance(matching, int) else len(runs) + export_complete = not bool(detail.get("runs_truncated")) + selection = { + "project_id": project_id, "scope": scope, "date": execution_date, + "timezone": str(detail.get("timezone") or "UTC"), "run_id": run_id, + "filters": {"selected_utc_date": execution_date}, + } + return _envelope( + locale=locale, selection=selection, data=data, references=references, + displayed_population=displayed_population, + full_population=full_population, + export_complete=export_complete, + contract_version=str(detail.get("contract_version") or "UNAVAILABLE"), + ) + + +def serialize_json(model: Mapping[str, object]) -> bytes: + return (json.dumps(model, ensure_ascii=False, indent=2, sort_keys=True, allow_nan=False) + "\n").encode("utf-8") + + +def serialize_markdown(model: Mapping[str, object]) -> bytes: + locale = str(model.get("locale") or "en") + labels = _LABELS.get(locale, _LABELS["en"]) + selection = model.get("selection", {}) + data = model.get("data", {}) + title = labels["overview"] if isinstance(data, Mapping) and "overview" in data else labels["detail"] + lines = [f"# {_md(title)}", "", f"## {_md(labels['selection'])}", "", _table( + [labels["field"], labels["value"]], + [[key, _display(value, labels)] for key, value in selection.items()] if isinstance(selection, Mapping) else [], + ), "", f"Snapshot: `{_md(model.get('snapshot_id'))}` ", f"As-of: `{_md(model.get('as_of'))}` ", f"Contract: `{_md(model.get('contract_version'))}` ", f"Export schema: `{_md(model.get('export_schema_version'))}`", ""] + completeness = model.get("completeness", {}) + lines += [f"## {_md(labels['coverage'])}", "", _table( + [labels["field"], labels["value"]], + [[key, _display(value, labels)] for key, value in completeness.items()] if isinstance(completeness, Mapping) else [], + ), ""] + if isinstance(data, Mapping) and isinstance(data.get("overview"), Mapping): + summary = data["overview"].get("summary", {}) + if isinstance(summary, Mapping): + lines += [f"## {_md(labels['summary'])}", "", _table( + [labels["field"], labels["value"]], + [[key, _display(value, labels)] for key, value in summary.items()], + ), ""] + rows = data["overview"].get("rows", []) + columns = sorted({str(key) for row in rows if isinstance(row, Mapping) for key in row}) + lines += [f"## {_md(labels['runs'])}", "", _mapping_table(columns, rows, labels), ""] + elif isinstance(data, Mapping): + detail = data.get("day_detail") or data.get("attempt") or data.get("selected_attempt") or {} + if isinstance(detail, Mapping): + lines += _markdown_detail(detail, labels) + chain = data.get("chain") + if isinstance(chain, Mapping): + lines += [f"## {_md(labels['chain'])}", "", _table( + [labels["field"], labels["value"]], + [[key, _display(value, labels)] for key, value in chain.items() if key != "runs"], + ), ""] + chain_rows = chain.get("runs", []) + if isinstance(chain_rows, list) and chain_rows: + columns = sorted({str(key) for row in chain_rows if isinstance(row, Mapping) for key in row}) + lines += [_mapping_table(columns, chain_rows, labels), ""] + limitations = _limitations(model) + if limitations: + lines += [f"## {_md(labels['limitations'])}", "", *[f"- {_md(reason)}" for reason in limitations], ""] + return ("\n".join(lines).rstrip() + "\n").encode("utf-8") + + +def _envelope(*, locale: str, selection: Mapping[str, object], data: Mapping[str, object], references: Sequence[str], displayed_population: int, full_population: int, export_complete: bool, contract_version: str) -> dict[str, object]: + locale = locale if locale in SUPPORTED_LOCALES else "en" + as_of = datetime.now(timezone.utc).isoformat() + core = {"contract_version": contract_version, "selection": selection, "data": data, "source_snapshot_references": list(dict.fromkeys(references))} + digest = hashlib.sha256(json.dumps(core, sort_keys=True, separators=(",", ":"), ensure_ascii=False, allow_nan=False).encode()).hexdigest() + return { + "export_schema_version": EXPORT_SCHEMA_VERSION, + "contract_version": contract_version, + "snapshot_id": f"sha256:{digest}", "as_of": as_of, "locale": locale, + "selection": dict(selection), "source_snapshot_references": core["source_snapshot_references"], + "completeness": { + "export": "COMPLETE" if export_complete and displayed_population == full_population else "PARTIAL", + "displayed_population": displayed_population, "full_population": full_population, + "metrics_may_be_partial": True, + }, + "data": data, + } + + +def _safe_day_detail(detail: Mapping[str, object]) -> dict[str, object]: + return { + key: value for key, value in detail.items() + if key not in {"runs", "phases"} + } | {"runs": [_safe_run(row) for row in detail.get("runs", []) if isinstance(row, Mapping)]} + + +def _safe_run(run: Mapping[str, object]) -> dict[str, object]: + result = {key: value for key, value in run.items() if key != "telemetry_snapshot"} + snapshot = run.get("telemetry_snapshot", {}) + if isinstance(snapshot, Mapping): + attempt = snapshot.get("attempt", {}) + result["telemetry_snapshot"] = { + "contract_version": snapshot.get("contract_version"), + "source_snapshot_reference": snapshot.get("source_snapshot_reference"), + "presentation_semantics": snapshot.get("presentation_semantics"), + "attempt": _safe_attempt(attempt) if isinstance(attempt, Mapping) else {}, + "chain": _safe_chain(snapshot.get("chain", {})), + } + return result + + +def _safe_attempt(attempt: Mapping[str, object]) -> dict[str, object]: + timing = attempt.get("timing", {}) + usage = attempt.get("usage", {}) + safe_timing = dict(timing) if isinstance(timing, Mapping) else {} + if isinstance(safe_timing.get("timeline"), list): + allowed = {"phase_id", "phase_name", "phase_category", "parent_phase_id", "attempt", "ordinal", "started_at", "completed_at", "duration_ms", "outcome", "measurement_basis", "relative_start_ms", "relative_end_ms"} + safe_timing["timeline"] = [ + {key: value for key, value in row.items() if key in allowed} + for row in safe_timing["timeline"] if isinstance(row, Mapping) + ] + safe_usage = dict(usage) if isinstance(usage, Mapping) else {} + safe_usage.pop("context_churn", None) + return {"scope": attempt.get("scope"), "run_id": attempt.get("run_id"), "timing": safe_timing, "usage": safe_usage} + + +def _safe_chain(chain: object) -> dict[str, object]: + return dict(chain) if isinstance(chain, Mapping) else {} + + +def _markdown_detail(detail: Mapping[str, object], labels: Mapping[str, str]) -> list[str]: + lines: list[str] = [] + scalar_detail = [ + [key, _display(value, labels)] for key, value in detail.items() + if not isinstance(value, (Mapping, list)) + ] + if scalar_detail: + lines += [f"## {_md(labels['summary'])}", "", _table( + [labels["field"], labels["value"]], scalar_detail, + ), ""] + summary = detail.get("summary") + if isinstance(summary, Mapping): + lines += [f"## {_md(labels['summary'])}", "", _table([labels["field"], labels["value"]], [[key, _display(value, labels)] for key, value in summary.items()]), ""] + phases = detail.get("inclusive_phases") or detail.get("phase_aggregates") + if isinstance(phases, list): + lines += [f"## {_md(labels['inclusive'])}", "", labels["inclusive_note"], "", _records_table(phases, labels), ""] + exclusive = detail.get("exclusive_distribution") + if isinstance(exclusive, list): + lines += [f"## {_md(labels['exclusive'])}", "", _records_table(exclusive, labels), ""] + lines += _markdown_bottlenecks(detail.get("bottlenecks"), labels) + snapshots = [detail.get("telemetry_snapshot")] if isinstance(detail.get("telemetry_snapshot"), Mapping) else [] + snapshots += [run.get("telemetry_snapshot") for run in detail.get("runs", []) if isinstance(run, Mapping) and isinstance(run.get("telemetry_snapshot"), Mapping)] if isinstance(detail.get("runs"), list) else [] + for snapshot in snapshots: + attempt = snapshot.get("attempt", {}) if isinstance(snapshot, Mapping) else {} + usage = attempt.get("usage", {}) if isinstance(attempt, Mapping) else {} + timing = attempt.get("timing", {}) if isinstance(attempt, Mapping) else {} + if isinstance(timing, Mapping): + timing_summary = [ + [key, _display(value, labels)] for key, value in timing.items() + if key not in {"timeline", "inclusive_phase_rows", "phase_aggregates", "exclusive_distribution"} + and not isinstance(value, (Mapping, list)) + ] + lines += [f"## {_md(labels['exclusive'])}", "", _table( + [labels["field"], labels["value"]], timing_summary, + ), ""] + phase_rows = timing.get("inclusive_phase_rows") or timing.get("phase_aggregates") + if isinstance(phase_rows, list): + lines += [f"## {_md(labels['inclusive'])}", "", labels["inclusive_note"], "", _records_table(phase_rows, labels), ""] + exclusive_rows = timing.get("exclusive_distribution") + if isinstance(exclusive_rows, list): + lines += [f"## {_md(labels['exclusive'])}", "", _records_table(exclusive_rows, labels), ""] + lines += _markdown_bottlenecks(timing.get("bottlenecks"), labels) + metrics = usage.get("metrics", {}) if isinstance(usage, Mapping) else {} + if isinstance(metrics, Mapping): + lines += [f"## {_md(labels['coverage'])}", "", _table( + [labels["field"], labels["value"]], + [[key, _display(value, labels)] for key, value in metrics.items()], + ), ""] + invocations = usage.get("invocations", []) if isinstance(usage, Mapping) else [] + timeline = timing.get("timeline", []) if isinstance(timing, Mapping) else [] + if isinstance(invocations, list): + lines += [f"## {_md(labels['invocations'])}", "", _records_table(invocations, labels), ""] + if isinstance(timeline, list): + lines += [f"## {_md(labels['timeline'])}", "", _records_table(timeline, labels), ""] + return lines + + +def _markdown_bottlenecks(value: object, labels: Mapping[str, str]) -> list[str]: + if not isinstance(value, Mapping) or not value: + return [] + rows = [[key, _display(observation, labels)] for key, observation in value.items()] + return [ + f"## {_md(labels['bottlenecks'])}", "", + _table([labels["field"], labels["value"]], rows), "", + ] + + +def _limitations(value: object) -> list[str]: + found: list[str] = [] + + def visit(item: object) -> None: + if isinstance(item, Mapping): + for key, nested in item.items(): + if key in {"missing_reason", "reason"} and isinstance(nested, str) and nested.strip(): + found.append(nested.strip()) + elif key == "reasons" and isinstance(nested, list): + found.extend(reason.strip() for reason in nested if isinstance(reason, str) and reason.strip()) + else: + visit(nested) + elif isinstance(item, list): + for nested in item: + visit(nested) + + visit(value) + return list(dict.fromkeys(found)) + + +def _records_table(rows: Sequence[object], labels: Mapping[str, str]) -> str: + mappings = [row for row in rows if isinstance(row, Mapping)] + columns = sorted({str(key) for row in mappings for key in row if not isinstance(row.get(key), (dict, list))}) + return _mapping_table(columns, mappings, labels) if columns else labels["unavailable"] + + +def _mapping_table(columns: Sequence[str], rows: Sequence[object], labels: Mapping[str, str]) -> str: + return _table(list(columns), [[_display(row.get(column), labels) for column in columns] for row in rows if isinstance(row, Mapping)]) + + +def _table(headings: Sequence[object], rows: Sequence[Sequence[object]]) -> str: + return "\n".join([ + "| " + " | ".join(_md(value) for value in headings) + " |", + "| " + " | ".join("---" for _ in headings) + " |", + *("| " + " | ".join(_md(value) for value in row) + " |" for row in rows), + ]) + + +def _md(value: object) -> str: + return str(value if value is not None else "").replace("&", "&").replace("<", "<").replace(">", ">").replace("|", "\\|").replace("\r", " ").replace("\n", " ").strip() + + +def _display(value: object, labels: Mapping[str, str]) -> str: + if value is None: + return labels["unavailable"] + if isinstance(value, (dict, list)): + return json.dumps(value, ensure_ascii=False, sort_keys=True, allow_nan=False) + return str(value) + + +def _sort_value(row: Mapping[str, object], key: str) -> tuple[int, object]: + value = row.get(key) + if value is None or isinstance(value, bool): + return (1, "") + return (0, value if isinstance(value, (int, float)) else str(value)) + + +def _contract_version(rows: Sequence[Mapping[str, object]]) -> str: + values = {str(row.get("contract_version")) for row in rows if row.get("contract_version")} + return next(iter(values)) if len(values) == 1 else "MIXED" if values else "UNAVAILABLE" diff --git a/src/engineering_platform/telemetry_metrics.py b/src/engineering_platform/telemetry_metrics.py new file mode 100644 index 00000000..629fbfd1 --- /dev/null +++ b/src/engineering_platform/telemetry_metrics.py @@ -0,0 +1,185 @@ +"""Shared metric coverage and aggregation semantics for telemetry projections. + +The helpers in this module deliberately know nothing about UI presentation. +They preserve the difference between a value being present and that value +being valid evidence. Every higher-level projection (run, day and chain) +uses the same state transition rules. +""" + +from __future__ import annotations + +from typing import Iterable, Mapping + + +COMPLETE = "COMPLETE" +PARTIAL = "PARTIAL" +UNAVAILABLE = "UNAVAILABLE" +CONFLICT = "CONFLICT" +COVERAGE_STATES = frozenset({COMPLETE, PARTIAL, UNAVAILABLE, CONFLICT}) +TELEMETRY_CALCULATION_VERSION = "telemetry-contract@2.1" + + +def metric_coverage( + *, + expected: int | None, + present: int, + valid: int, + conflicting: int = 0, + reason: str | None = None, +) -> dict[str, object]: + """Return one explainable coverage record without inventing population. + + ``observed_observations`` remains as a compatibility alias for valid + observations. New consumers use the explicit present/valid/conflicting + fields. + """ + if expected is not None and expected < 0: + expected = None + present = max(0, int(present)) + valid = max(0, min(int(valid), present)) + conflicting = max(0, min(int(conflicting), present)) + missing = max(0, expected - present) if expected is not None else None + if conflicting: + state = CONFLICT + elif expected is None: + state = PARTIAL if present else UNAVAILABLE + elif expected == 0: + # A known empty population is complete; this is distinct from an + # unknown expected count represented by ``None``. + state = COMPLETE if present == 0 else CONFLICT + elif valid == expected and present == expected: + state = COMPLETE + elif present or valid: + state = PARTIAL + else: + state = UNAVAILABLE + if reason is None and state != COMPLETE: + if conflicting: + reason = f"{conflicting} conflicting observation(s)" + elif expected is None: + reason = "Expected observation population is unknown" + else: + reason = f"{valid} valid of {expected} expected observation(s)" + return { + "coverage": state, + "expected_observations": expected, + "present_observations": present, + "valid_observations": valid, + "observed_observations": valid, + "missing_observations": missing, + "conflicting_observations": conflicting, + "missing_reason": reason, + } + + +def aggregate_coverage(metrics: Iterable[Mapping[str, object]]) -> dict[str, object]: + """Merge source coverage while retaining conflicts and unknown totals.""" + rows = list(metrics) + if not rows: + return metric_coverage(expected=None, present=0, valid=0) + expected_known = all( + isinstance(row.get("expected_observations"), int) + and not isinstance(row.get("expected_observations"), bool) + for row in rows + ) + expected = sum(int(row["expected_observations"]) for row in rows) if expected_known else None + present = sum(_count(row, "present_observations", fallback="observed_observations") for row in rows) + valid = sum(_valid_count(row) for row in rows) + conflicting = sum(_conflict_count(row) for row in rows) + reasons = [ + str(row.get("missing_reason")) + for row in rows + if row.get("coverage") != COMPLETE and row.get("missing_reason") + ] + source_states = {row.get("coverage") for row in rows} + if CONFLICT in source_states and conflicting == 0: + conflicting = 1 + present = max(present, 1) + result = metric_coverage( + expected=expected, present=present, valid=valid, conflicting=conflicting, + reason="; ".join(dict.fromkeys(reasons)) or None, + ) + # Counts explain population, but they cannot upgrade an upstream semantic + # state. A timing projection can have all numeric durations present while + # remaining PARTIAL because its parent relation is missing, for example. + if result["coverage"] != CONFLICT and ({PARTIAL, UNAVAILABLE} & source_states): + result["coverage"] = PARTIAL if present or valid or COMPLETE in source_states or PARTIAL in source_states else UNAVAILABLE + result["missing_reason"] = ( + "; ".join(dict.fromkeys(reasons)) + or "One or more source projections are partial or unavailable" + ) + return result + + +def aggregate_numeric_metric( + metrics: Iterable[Mapping[str, object]], *, aggregation_level: str, + aggregation: str = "sum", meaning: str | None = None, + unit: str | None = None, provenance: str | None = None, +) -> dict[str, object]: + """Aggregate valid metric values according to their declared meaning.""" + rows = list(metrics) + coverage = aggregate_coverage(rows) + values = [ + row.get("value") for row in rows + if row.get("coverage") != CONFLICT + and isinstance(row.get("value"), (int, float)) + and not isinstance(row.get("value"), bool) + ] + value: int | float | None + if not values: + value = None + elif aggregation == "max": + value = max(values) + elif aggregation == "sum": + value = sum(values) + else: + raise ValueError(f"Unsupported telemetry aggregation: {aggregation}") + return { + "value": value, + "meaning": meaning or next((str(row["meaning"]) for row in rows if row.get("meaning")), None), + "unit": unit or next((str(row["unit"]) for row in rows if row.get("unit")), None), + "aggregation": aggregation, + "aggregation_level": aggregation_level, + "provenance": provenance or _merged_provenance(rows), + **coverage, + "calculation_version": TELEMETRY_CALCULATION_VERSION, + "source_snapshot_references": sorted({ + str(reference) + for row in rows + for reference in _references(row) + if reference + }), + } + + +def _count(row: Mapping[str, object], key: str, *, fallback: str) -> int: + value = row.get(key, row.get(fallback, 0)) + return int(value) if isinstance(value, int) and not isinstance(value, bool) and value >= 0 else 0 + + +def _valid_count(row: Mapping[str, object]) -> int: + value = row.get("valid_observations") + if isinstance(value, int) and not isinstance(value, bool) and value >= 0: + return value + observed = _count(row, "observed_observations", fallback="observed_observations") + return 0 if row.get("coverage") == CONFLICT else observed + + +def _conflict_count(row: Mapping[str, object]) -> int: + value = row.get("conflicting_observations") + if isinstance(value, int) and not isinstance(value, bool) and value >= 0: + return value + return 1 if row.get("coverage") == CONFLICT else 0 + + +def _merged_provenance(rows: list[Mapping[str, object]]) -> str: + values = {str(row.get("provenance")) for row in rows if row.get("provenance")} + return next(iter(values)) if len(values) == 1 else "MIXED" if values else "UNAVAILABLE" + + +def _references(row: Mapping[str, object]) -> list[object]: + plural = row.get("source_snapshot_references") + if isinstance(plural, list): + return plural + reference = row.get("source_snapshot_reference") + return [reference] if reference is not None else [] diff --git a/src/engineering_platform/templates/workspace-config.json b/src/engineering_platform/templates/workspace-config.json index e406d78c..c2313b8c 100644 --- a/src/engineering_platform/templates/workspace-config.json +++ b/src/engineering_platform/templates/workspace-config.json @@ -3,7 +3,7 @@ "platform": { "id": "engineering-platform", "name": "Engineering Platform", - "version": "2.3.79", + "version": "2.3.80", "generation": 2, "documentation_namespace": "engineering-platform", "capability_registry_version": 1 diff --git a/tests/engineering/dashboard.spec.mjs b/tests/engineering/dashboard.spec.mjs index 43a64503..b4644fa8 100644 --- a/tests/engineering/dashboard.spec.mjs +++ b/tests/engineering/dashboard.spec.mjs @@ -1,4 +1,4 @@ -import { spawn } from "node:child_process"; +import { execFileSync, spawn } from "node:child_process"; import { mkdirSync, mkdtempSync, readFileSync, rmSync } from "node:fs"; import { tmpdir } from "node:os"; import { fileURLToPath } from "node:url"; @@ -17,6 +17,16 @@ let dashboardRoot; let installationRoot; let dashboardUrl; +function setSyntheticTelemetryRun(runId, date, present = true) { + const script = present + ? "import sqlite3,sys; c=sqlite3.connect(sys.argv[1] + '/epdata.sqlite'); c.execute(\"INSERT OR REPLACE INTO ep_execution_runs(run_id,project_id,state,created_at,updated_at,execution_mode) VALUES(?, 'dashboard-fixture', 'COMPLETE', ?, ?, 'MANAGED')\", (sys.argv[2], sys.argv[3] + 'T12:00:00+00:00', sys.argv[3] + 'T12:01:00+00:00')); c.commit(); c.close()" + : "import sqlite3,sys; c=sqlite3.connect(sys.argv[1] + '/epdata.sqlite'); c.execute('DELETE FROM ep_execution_runs WHERE run_id=?', (sys.argv[2],)); c.commit(); c.close()"; + execFileSync("python3", ["-c", script, dashboardRoot, runId, date], { + cwd: repository, + env: { ...process.env, PYTHONPATH: path.join(repository, "src") }, + }); +} + const TELEMETRY_PHASES = [ "QUEUE_WAIT", "SUBMISSION_CLAIM", "INITIALIZATION", "HOST_PREFLIGHT", "WORKSPACE_PREFLIGHT", "CAPABILITY_PREFLIGHT", "DETERMINISTIC_ADMISSION", @@ -2429,11 +2439,15 @@ test.describe("Engineering Status browser smoke", () => { await expect(page.locator("#promptHistoryDetailContent .prompt-detail-card--pull-requests")).toContainText("Gewijzigde bestanden: 5"); const markdown = page.locator("#promptHistoryDetailDownloadMarkdown"); const json = page.locator("#promptHistoryDetailDownloadJson"); + const requestedExports = []; + page.on("download", (download) => requestedExports.push(download)); await expect(markdown).toHaveAttribute("aria-label", "Uitvoeringsdetails als Markdown downloaden voor Modal prompt"); await expect(json).toHaveAttribute("aria-label", "Uitvoeringsdetails als JSON downloaden voor Modal prompt"); await expect(json).toHaveText("{}"); const markdownDownload = page.waitForEvent("download"); await markdown.click(); + await expect.poll(() => requestedExports.length).toBe(1); + await expect(page.locator("#copyToast")).toBeHidden(); const downloadedMarkdown = await markdownDownload; expect(downloadedMarkdown.suggestedFilename()).toBe("execution-details-inbox-modal.md"); const markdownContent = readFileSync(await downloadedMarkdown.path(), "utf8"); @@ -4861,6 +4875,7 @@ test.describe("Engineering Status browser smoke", () => { }); test("exports one loaded telemetry day as Markdown and JSON", async ({ page }) => { + setSyntheticTelemetryRun("inbox-day-export", "2026-08-24"); const detail = { summary: { executions: 1, completed: 1, blocked: 0, failed: 0, @@ -4871,7 +4886,18 @@ test.describe("Engineering Status browser smoke", () => { bottlenecks: { longest_average_phase: { phase: "VALIDATION" }, largest_accumulated_phase: { phase: "VALIDATION" }, top_time_consumers: [{ phase: "VALIDATION", share_percent: 20 }] }, runs: [{ run_id: "inbox-day-export", started_at: "2026-08-24T12:00:00Z", status: "COMPLETE", total_duration_ms: 60000, queue_wait_ms: 5000, provider_duration_ms: 12000, validation_duration_ms: 12000, external_wait_ms: 0, largest_phase: "VALIDATION", producer_type: "HUMAN", repository: "pcvantol/djconnect", model: "gpt-5.6", phase_telemetry: "RECORDED" }], }; - await page.route("**/api/telemetry/2026-08-24", (route) => route.fulfill({ json: detail })); + await page.route("**/api/telemetry/2026-08-24", (route) => { + const requestUrl = new URL(route.request().url()); + if (!requestUrl.pathname.endsWith("/export")) return route.fulfill({ json: detail }); + const markdown = requestUrl.searchParams.get("format") === "markdown"; + return route.fulfill({ + contentType: markdown ? "text/markdown; charset=utf-8" : "application/json; charset=utf-8", + headers: { "Content-Disposition": `attachment; filename="telemetry-detail-dashboard-fixture-utc-day-detail-2026-08-24.${markdown ? "md" : "json"}"` }, + body: markdown + ? "# Telemetriedetail\n\nSnapshot: `sha256:detail-snapshot`\n\n## Samenvatting\n\n| run_id | coverage |\n| --- | --- |\n| inbox-day-export | COMPLETE |\n" + : "{}", + }); + }); await page.goto(dashboardUrl, { waitUntil: "domcontentloaded" }); await page.locator("#autoRefresh").uncheck(); await page.evaluate(() => window.executionTelemetry([{ @@ -4892,16 +4918,50 @@ test.describe("Engineering Status browser smoke", () => { const markdownDownload = page.waitForEvent("download"); await markdown.click(); const downloadedMarkdown = await markdownDownload; - expect(downloadedMarkdown.suggestedFilename()).toBe("execution-telemetry-2026-08-24.md"); + expect(downloadedMarkdown.suggestedFilename()).toBe("telemetry-detail-dashboard-fixture-utc-day-detail-2026-08-24.md"); const markdownContent = readFileSync(await downloadedMarkdown.path(), "utf8"); - expect(markdownContent).toContain("# Uitvoeringstelemetrie — 24-08-2026"); + expect(markdownContent).toContain("# Telemetriedetail"); expect(markdownContent).toContain("## Samenvatting"); expect(markdownContent).toContain("inbox-day-export"); const jsonDownload = page.waitForEvent("download"); await json.click(); const downloadedJson = await jsonDownload; - expect(downloadedJson.suggestedFilename()).toBe("execution-telemetry-2026-08-24.json"); - expect(JSON.parse(readFileSync(await downloadedJson.path(), "utf8"))).toEqual(detail); + expect(downloadedJson.suggestedFilename()).toBe("telemetry-detail-dashboard-fixture-utc-day-detail-2026-08-24.json"); + const jsonContent = JSON.parse(readFileSync(await downloadedJson.path(), "utf8")); + expect(jsonContent.export_schema_version).toBe("telemetry-export@1.0"); + expect(jsonContent.selection).toMatchObject({ project_id: "dashboard-fixture", scope: "UTC_DAY_DETAIL", date: "2026-08-24" }); + expect(jsonContent.data.day_detail.runs.map((run) => run.run_id)).toContain("inbox-day-export"); + expect(markdownContent).toContain(jsonContent.snapshot_id); + setSyntheticTelemetryRun("inbox-day-export", "2026-08-24", false); + }); + + test("shows an export error and allows the same captured detail selection to retry", async ({ page }) => { + const date = "2026-08-23", runId = "retry-detail-export"; + const detail = { + summary: { executions: 1 }, phases: [], bottlenecks: { top_time_consumers: [] }, + runs: [{ run_id: runId, status: "COMPLETE", telemetry_snapshot: { attempt: { timing: {}, usage: {} }, chain: {} } }], + }; + await page.route(`**/api/telemetry/${date}`, (route) => { + const requestUrl = new URL(route.request().url()); + return requestUrl.pathname.endsWith("/export") ? route.continue() : route.fulfill({ json: detail }); + }); + await page.goto(dashboardUrl, { waitUntil: "domcontentloaded" }); + await page.evaluate((selectedDate) => window.executionTelemetry([{ + date: selectedDate, prompt_count: 1, complete_count: 1, blocked_count: 0, failed_count: 0, + }]), date); + await page.locator("#executionTelemetry").evaluate((element) => { element.open = true; }); + await dispatchDashboardPointerClick(page.locator("#executionTelemetryRows .telemetry-row")); + await page.locator("#telemetryDetailDownloadJson").click(); + await expect(page.locator("#copyToast")).toContainText(DASHBOARD_MESSAGES.nl["telemetry.export_failed"]); + + setSyntheticTelemetryRun(runId, date); + const retryDownload = page.waitForEvent("download"); + await page.locator("#telemetryDetailDownloadJson").click(); + const downloaded = await retryDownload; + expect(downloaded.suggestedFilename()).toBe(`telemetry-detail-dashboard-fixture-utc-day-detail-${date}.json`); + const content = JSON.parse(readFileSync(await downloaded.path(), "utf8")); + expect(content.selection).toMatchObject({ scope: "UTC_DAY_DETAIL", date }); + setSyntheticTelemetryRun(runId, date, false); }); test("gives every table a coloured first column and sorts telemetry columns", async ({ page }) => { @@ -4987,6 +5047,7 @@ test.describe("Engineering Status browser smoke", () => { }); test("offers read-only download and copy actions for telemetry in CENTRAL", async ({ page }) => { + setSyntheticTelemetryRun("overview-export-run", "2026-08-24"); let clearRequests = 0; await page.route("**/api/telemetry/clear", (route) => { clearRequests += 1; @@ -5002,10 +5063,26 @@ test.describe("Engineering Status browser smoke", () => { await page.locator("#executionTelemetry").evaluate((element) => { element.open = true; }); const actions = page.locator("#executionTelemetry .telemetry-actions"); await expect(actions).toHaveCSS("justify-content", "flex-end"); - await expect(actions.getByRole("button", { name: "Telemetrie downloaden" })).toBeEnabled(); + await expect(actions.getByRole("button", { name: "Volledig overzicht als Markdown downloaden" })).toBeEnabled(); + await expect(actions.getByRole("button", { name: "Volledig overzicht als JSON downloaden" })).toBeEnabled(); await expect(actions.getByRole("button", { name: "Telemetrie kopiëren" })).toBeEnabled(); await expect(actions.getByRole("button", { name: "Telemetrie wissen" })).toHaveCount(0); + const markdownDownload = page.waitForEvent("download"); + await actions.getByRole("button", { name: "Volledig overzicht als Markdown downloaden" }).click(); + const markdown = await markdownDownload; + expect(markdown.suggestedFilename()).toBe("telemetry-overview-dashboard-fixture-utc.md"); + const markdownContent = readFileSync(await markdown.path(), "utf8"); + expect(markdownContent).toContain("# Telemetrieoverzicht"); + const jsonDownload = page.waitForEvent("download"); + await actions.getByRole("button", { name: "Volledig overzicht als JSON downloaden" }).click(); + const json = await jsonDownload; + expect(json.suggestedFilename()).toBe("telemetry-overview-dashboard-fixture-utc.json"); + const jsonContent = JSON.parse(readFileSync(await json.path(), "utf8")); + expect(jsonContent.selection.sort).toEqual({ key: "date", direction: "desc" }); + expect(jsonContent.data.overview.summary.run_count).toBe(1); + expect(markdownContent).toContain(jsonContent.snapshot_id); expect(clearRequests).toBe(0); + setSyntheticTelemetryRun("overview-export-run", "2026-08-24", false); }); test("sorts telemetry detail tables with the same header treatment as logs", async ({ page }) => { @@ -5271,8 +5348,9 @@ test.describe("Engineering Status browser smoke", () => { test("formats telemetry percentages with one localized decimal place", async ({ page }) => { await page.route("**/api/events", (route) => route.abort()); await page.route("**/api/telemetry/2026-08-16", (route) => route.fulfill({ json: { - summary: {}, + summary: { cache_ratio_percent: null }, phases: [], + exclusive_distribution: [{ category: "UNASSIGNED", duration_ms: 0, share_percent: 0 }], bottlenecks: { top_time_consumers: [{ phase: "PROVIDER_EXECUTION", share_percent: 61.848 }] }, runs: [], } })); @@ -5285,6 +5363,10 @@ test.describe("Engineering Status browser smoke", () => { await dispatchDashboardPointerClick(page.locator("#executionTelemetry > summary")); await dispatchDashboardPointerClick(page.locator("#executionTelemetryRows tr")); await expect(page.locator("#telemetryDetailContent")).toContainText("61,8%"); + await expect(page.locator("#telemetryDetailContent")).toContainText("0,0%"); + const cacheField = page.locator("#telemetryDetailContent .field").filter({ hasText: DASHBOARD_MESSAGES.nl["telemetry.cache_ratio"] }).first(); + await expect(cacheField).toContainText(DASHBOARD_MESSAGES.nl["format.unavailable"]); + await expect(cacheField).not.toContainText("0,0%"); }); test("projects canonical bottlenecks and complete per-run telemetry detail", async ({ page }) => { @@ -5317,6 +5399,7 @@ test.describe("Engineering Status browser smoke", () => { test("qualifies canonical telemetry coverage, timing, invocation and chain scopes", async ({ page }) => { await page.setViewportSize({ width: 1440, height: 1000 }); const runId = "inbox-cd4ba8cb829a4ee1b852369c602c3653"; + setSyntheticTelemetryRun(runId, "2026-09-17"); const invocations = Array.from({ length: 9 }, (_, index) => ({ invocation_id: `invocation-${index + 1}`, phase: index < 6 ? "PROVIDER_EXECUTION" : "CAPABILITY_REVIEW", role: index ? "REVIEWER" : "PRIMARY", provider: "codex-cli", state: "COMPLETE", @@ -5330,10 +5413,10 @@ test.describe("Engineering Status browser smoke", () => { missing_usage_fields: index === 8 ? ["input_tokens", "cached_input_tokens", "output_tokens"] : [], })); const timeline = [ - { phase_id: "total", parent_phase_id: null, phase_name: "TOTAL_EXECUTION", duration_ms: 2721545, outcome: "COMPLETE" }, - { phase_id: "provider", parent_phase_id: "total", phase_name: "PROVIDER_EXECUTION", duration_ms: 1233062, outcome: "COMPLETE" }, - { phase_id: "validation", parent_phase_id: "total", phase_name: "VALIDATION", duration_ms: 148244, outcome: "COMPLETE" }, - { phase_id: "nested", parent_phase_id: "validation", phase_name: "VALIDATION", duration_ms: 4200, outcome: "COMPLETE" }, + { phase_id: "total", parent_phase_id: null, phase_name: "TOTAL_EXECUTION", duration_ms: 2721545, outcome: "COMPLETE", measurement_basis: "MONOTONIC", started_at: "2026-09-17T10:00:00Z", completed_at: "2026-09-17T10:45:21.545Z", relative_start_ms: 0, relative_end_ms: 2721545 }, + { phase_id: "provider", parent_phase_id: "total", phase_name: "PROVIDER_EXECUTION", duration_ms: 1233062, outcome: "COMPLETE", measurement_basis: "MONOTONIC", started_at: "2026-09-17T10:02:00Z", completed_at: "2026-09-17T10:22:33.062Z", relative_start_ms: 120000, relative_end_ms: 1353062 }, + { phase_id: "validation", parent_phase_id: "total", phase_name: "VALIDATION", duration_ms: 148244, outcome: "COMPLETE", measurement_basis: "MONOTONIC", started_at: "2026-09-17T10:18:00Z", completed_at: "2026-09-17T10:20:28.244Z", relative_start_ms: 1080000, relative_end_ms: 1228244 }, + { phase_id: "nested", parent_phase_id: "validation", phase_name: "VALIDATION", duration_ms: 4200, outcome: "COMPLETE", measurement_basis: "MONOTONIC", started_at: "2026-09-17T10:18:10Z", completed_at: "2026-09-17T10:18:14.200Z", relative_start_ms: 1090000, relative_end_ms: 1094200 }, ]; const chainRuns = Array.from({ length: 7 }, (_, index) => ({ run_id: index === 6 ? runId : `inbox-chain-${index + 1}`, relation: index ? "RETRY" : "ORIGINAL", @@ -5341,7 +5424,7 @@ test.describe("Engineering Status browser smoke", () => { provider_invocation_count: index === 6 ? 9 : 5, })); const detail = { - contract_version: "telemetry-contract@2.0", source_snapshot_references: [runId], scope: "EP_RUN_ATTEMPTS_IN_UTC_DAY", + contract_version: "telemetry-contract@2.1", source_snapshot_references: [runId], scope: "EP_RUN_ATTEMPTS_IN_UTC_DAY", summary: { executions: 1, population: 1, completed: 1, blocked: 0, failed: 0, total_wall_time: { average_ms: 2721545, median_ms: 2721545, population: 1 }, @@ -5377,10 +5460,11 @@ test.describe("Engineering Status browser smoke", () => { model: "gpt-6-codex-experimental", phase_telemetry: "RECORDED", usage_coverage: { input_tokens: "PARTIAL", output_tokens: "PARTIAL" }, timing_coverage: { state: "COMPLETE" }, telemetry_snapshot: { - contract_version: "telemetry-contract@2.0", + contract_version: "telemetry-contract@2.1", attempt: { scope: "EP_RUN_ATTEMPT", timing: { total_wall_time_ms: 2721545, provider_unique_coverage_ms: 1233072, provider_cumulative_process_duration_ms: 1233062, external_wait_time_ms: 364132, unassigned_time_ms: 648751, coverage: { state: "COMPLETE" }, timeline, + timeline_axis: { duration_ms: 2721545, started_at: "2026-09-17T10:00:00Z", completed_at: "2026-09-17T10:45:21.545Z", measurement_basis: "WALL_CLOCK_INTERVAL_ENVELOPE" }, }, usage: { invocations, cache_ratio_percent: 92.557, cache_ratio_population: { coverage: "PARTIAL", observed_observations: 8, expected_observations: 9 }, metrics: { @@ -5391,13 +5475,32 @@ test.describe("Engineering Status browser smoke", () => { chain: { scope: "EXECUTION_CHAIN", coverage: "COMPLETE", mission_scope_label: "EP execution within this Mission", attempt_count: 7, original_attempt_count: 1, retry_count: 6, resume_count: 0, elapsed_ms: 30281903, - processing_time_ms: 14040669, inter_attempt_gap_ms: 16217525, outside_selected_window_count: 6, runs: chainRuns, + processing_time_ms: 14040669, inter_attempt_gap_ms: 16217525, outside_selected_window_count: 6, + provider_invocation_count: 39, usage_metrics: { + input_tokens: { value: 18858573, coverage: "PARTIAL", observed_observations: 38, expected_observations: 39 }, + cached_input_tokens: { value: 16921536, coverage: "PARTIAL", observed_observations: 38, expected_observations: 39 }, + uncached_input_tokens: { value: 1937037, coverage: "PARTIAL", observed_observations: 38, expected_observations: 39 }, + output_tokens: { value: 165758, coverage: "PARTIAL", observed_observations: 38, expected_observations: 39 }, + }, cache_ratio_percent: 89.728, cache_ratio_population: { coverage: "PARTIAL", observed_observations: 38, expected_observations: 39 }, + runs: chainRuns, }, }, }], }; await page.route("**/api/events", (route) => route.abort()); - await page.route("**/api/telemetry/2026-09-17", (route) => route.fulfill({ json: detail })); + await page.route("**/api/telemetry/2026-09-17", (route) => { + const requestUrl = new URL(route.request().url()); + if (!requestUrl.pathname.endsWith("/export")) return route.fulfill({ json: detail }); + return route.fulfill({ + contentType: "application/json; charset=utf-8", + headers: { "Content-Disposition": `attachment; filename="telemetry-detail-dashboard-fixture-execution-chain-${runId}.json"` }, + body: JSON.stringify({ + export_schema_version: "telemetry-export@1.0", contract_version: "telemetry-contract@2.1", + snapshot_id: "sha256:chain-snapshot", selection: { scope: "EXECUTION_CHAIN", run_id: runId }, + data: { chain: detail.runs[0].telemetry_snapshot.chain }, + }), + }); + }); await page.goto(dashboardUrl, { waitUntil: "domcontentloaded" }); await selectDashboardLocale(page, "nl"); await page.evaluate(() => window.executionTelemetry([{ date: "2026-09-17", prompt_count: 1, average_total_execution_seconds: 2721.545, input_tokens: 8558573, output_tokens: 65758, complete_count: 1, blocked_count: 0, failed_count: 0 }])); @@ -5412,6 +5515,10 @@ test.describe("Engineering Status browser smoke", () => { await expect(content).toContainText("gpt-6-codex-experimental (AUTHORITATIVE)"); await expect(content).toContainText("3.847.785"); await expect(content).not.toContainText("Actieve context"); + await expect(content).toContainText("Timingkoppeling"); + await expect(content.locator(".telemetry-timeline__bar--unpositioned")).toHaveCount(0); + await expect(content.locator(".telemetry-timeline__bar")).toHaveCount(4); + await expect(content.locator(".telemetry-timeline__bar").nth(1)).toHaveAttribute("title", /provider.*2026-09-17T10:02:00Z/); const scopeButtons = content.locator(".telemetry-scope-switcher button"); await expect(scopeButtons).toHaveCount(2); for (const button of await scopeButtons.all()) { @@ -5426,20 +5533,34 @@ test.describe("Engineering Status browser smoke", () => { await expect(chainScopeButton).toHaveCSS("box-shadow", /3px 0px 0px/); await expect(content).toContainText("Pogingen in keten"); await expect(content).toContainText("6"); + await expect(content).toContainText("18.858.573 · Gedeeltelijk (38/39)"); + await expect(content).toContainText("39"); await expect(content).toContainText("EP execution within this Mission"); + const chainDownloadPromise = page.waitForEvent("download"); + await page.locator("#telemetryDetailDownloadJson").click(); + const chainDownload = await chainDownloadPromise; + expect(chainDownload.suggestedFilename()).toBe(`telemetry-detail-dashboard-fixture-execution-chain-${runId}.json`); + const chainExport = JSON.parse(readFileSync(await chainDownload.path(), "utf8")); + expect(chainExport.export_schema_version).toBe("telemetry-export@1.0"); + expect(chainExport.selection).toMatchObject({ scope: "EXECUTION_CHAIN", run_id: runId }); + expect(chainExport.data.selected_attempt.run_id).toBe(runId); const overflow = await page.evaluate(() => document.documentElement.scrollWidth > document.documentElement.clientWidth); expect(overflow).toBe(false); const screenshotDirectory = process.env.TELEMETRY_SCREENSHOT_DIR; if (screenshotDirectory) { mkdirSync(screenshotDirectory, { recursive: true }); await page.screenshot({ path: path.join(screenshotDirectory, "telemetry-desktop.png"), fullPage: true }); - await page.setViewportSize({ width: 390, height: 844 }); - await expect(page.locator("#telemetryDetailModal")).toBeVisible(); - const mobileOverflow = await page.evaluate(() => document.documentElement.scrollWidth > document.documentElement.clientWidth); - expect(mobileOverflow).toBe(false); + } + await page.setViewportSize({ width: 390, height: 844 }); + await expect(page.locator("#telemetryDetailModal")).toBeVisible(); + const mobileOverflow = await page.evaluate(() => document.documentElement.scrollWidth > document.documentElement.clientWidth); + expect(mobileOverflow).toBe(false); + await expect(content.locator(".telemetry-timeline__bar")).toHaveCount(4); + if (screenshotDirectory) { await content.evaluate((element) => { element.scrollTop = element.scrollHeight; }); await page.screenshot({ path: path.join(screenshotDirectory, "telemetry-mobile.png"), fullPage: true }); } + setSyntheticTelemetryRun(runId, "2026-09-17", false); }); test("uses one uninterrupted selected-row treatment for telemetry rows", async ({ page }) => { diff --git a/tests/engineering/test_server_foundation.py b/tests/engineering/test_server_foundation.py index c2035c9c..30d92e48 100644 --- a/tests/engineering/test_server_foundation.py +++ b/tests/engineering/test_server_foundation.py @@ -3,6 +3,7 @@ from engineering_platform.storage import sqlite_connection import json +import http.server import inspect import io import logging @@ -11,11 +12,12 @@ import re import subprocess import sys -from datetime import datetime, timezone +from datetime import datetime, timedelta, timezone from pathlib import Path import socket import sqlite3 import tempfile +import threading import unittest from contextlib import redirect_stderr, redirect_stdout from unittest.mock import call, patch @@ -23,7 +25,9 @@ from urllib.request import Request, urlopen from engineering_platform import file_inbox, local_repository_binding, project_topology, providers, server, submission_service +from engineering_platform.execution_timing import complete_phase, start_phase from engineering_platform.platform_components import PLATFORM_COMPONENT_IDS +from engineering_platform.provider_usage import ProviderInvocation, persist_provider_invocation from engineering_platform.providers import LaunchdRuntimeDetails, ProviderStatus @@ -2517,3 +2521,114 @@ def test_root_reuses_historical_console_with_request_scoped_project_selection(se with urlopen(f"http://127.0.0.1:{port}/api/logs/operations_console") as response: remaining = json.loads(response.read()) self.assertNotIn("central_console_test", [entry["event"] for entry in remaining["entries"]]) + + def test_four_telemetry_exports_use_one_full_server_side_snapshot(self) -> None: + """Exercise all product download routes against a real isolated Server.""" + with socket.socket() as probe: + probe.bind(("127.0.0.1", 0)); port = probe.getsockname()[1] + server.initialize(self.root, bind_port=port) + declaration = json.loads( + (Path(__file__).parent / "fixtures" / "repository_attachment" / "python-authority.json").read_text(encoding="utf-8") + ) + declaration["project"]["id"] = "telemetry-export" + declaration["project"]["authority_repository_id"] = "telemetry-export" + declaration["repository"]["id"] = "telemetry-export" + with sqlite_connection(self.root / server.SERVER_DATABASE_FILENAME) as connection: + project_topology.register_server_local_topology(connection, declaration=declaration) + for index in range(105): + run_id = f"export-run-{index:03d}" + connection.execute( + "INSERT INTO ep_execution_runs(run_id,project_id,state,created_at,updated_at,execution_mode) VALUES(?,?,?,?,?,'MANAGED')", + (run_id, "telemetry-export", "COMPLETE", "2026-09-17T10:00:00+00:00", "2026-09-17T10:00:10+00:00"), + ) + started = datetime(2026, 9, 17, 10, tzinfo=timezone.utc) + total = start_phase( + self.root, "export-run-000", "TOTAL_EXECUTION", started_at=started, + monotonic_clock=0, central_database=self.root / server.SERVER_DATABASE_FILENAME, + ) + provider = start_phase( + self.root, "export-run-000", "PROVIDER_EXECUTION", started_at=started, + parent_phase_id=total.phase_id, monotonic_clock=0, + central_database=self.root / server.SERVER_DATABASE_FILENAME, + ) + complete_phase(self.root, provider, completed_at=started + timedelta(seconds=6), monotonic_clock=6) + complete_phase(self.root, total, completed_at=started + timedelta(seconds=10), monotonic_clock=10) + persist_provider_invocation( + self.root, + ProviderInvocation( + "export-run-000", 1, "codex_cli", "observed-model", "PROVIDER_EXECUTION", "IMPLEMENTATION", + started.isoformat(), (started + timedelta(seconds=6)).isoformat(), 6000, + {"input_tokens": 100, "cached_input_tokens": 80, "output_tokens": 10}, + invocation_id="export-invocation", + ), + central_database=self.root / server.SERVER_DATABASE_FILENAME, + ) + # Exercise the exact in-process read models as well as the installed + # HTTP boundary below. The Server itself runs in a child process, so + # this assertion keeps per-module coverage tied to the canonical + # aggregation implementation rather than to a copied fixture. + projected_overview = server._central_console_telemetry(self.root, "telemetry-export") + self.assertEqual(projected_overview[0]["prompt_count"], 105) + projected_detail = server._central_console_telemetry_detail( + self.root, "telemetry-export", "2026-09-17", full=True, + ) + self.assertIsNotNone(projected_detail) + self.assertEqual(projected_detail["matching_run_count"], 105) + self.assertEqual(len(projected_detail["runs"]), 105) + # Run the same production request handler in-process so route, + # serializer and response-header coverage belongs to this exact + # candidate. Separate lifecycle tests retain the installed child- + # process startup qualification. + httpd = http.server.ThreadingHTTPServer(("127.0.0.1", port), server._HealthHandler) + httpd.data_root = self.root.resolve() + httpd.central_data_transfer_lock = threading.RLock() + httpd.central_data_transfer_active = False + httpd.restart_after_shutdown = False + thread = threading.Thread(target=httpd.serve_forever, daemon=True) + thread.start() + + def stop_httpd() -> None: + httpd.shutdown() + httpd.server_close() + thread.join(timeout=2) + + self.addCleanup(stop_httpd) + + base = f"http://127.0.0.1:{port}" + urls = { + "overview_json": f"{base}/api/telemetry/export?project=telemetry-export&format=json&locale=nl&sort=date&direction=desc", + "overview_markdown": f"{base}/api/telemetry/export?project=telemetry-export&format=markdown&locale=nl&sort=date&direction=desc", + "detail_json": f"{base}/api/telemetry/2026-09-17/export?project=telemetry-export&format=json&locale=nl&scope=UTC_DAY_DETAIL", + "detail_markdown": f"{base}/api/telemetry/2026-09-17/export?project=telemetry-export&format=markdown&locale=nl&scope=UTC_DAY_DETAIL", + } + downloads: dict[str, tuple[object, bytes]] = {} + for name, url in urls.items(): + with urlopen(url) as response: + downloads[name] = (response.headers, response.read()) + self.assertIn("attachment; filename=\"telemetry-", response.headers["Content-Disposition"]) + self.assertEqual(response.headers["Cache-Control"], "no-store") + if name.endswith("json"): + self.assertEqual(response.headers["Content-Type"], "application/json; charset=utf-8") + self.assertTrue(response.headers["Content-Disposition"].endswith('.json"')) + else: + self.assertEqual(response.headers["Content-Type"], "text/markdown; charset=utf-8") + self.assertTrue(response.headers["Content-Disposition"].endswith('.md"')) + + overview = json.loads(downloads["overview_json"][1]) + detail = json.loads(downloads["detail_json"][1]) + self.assertEqual(overview["data"]["overview"]["summary"]["run_count"], 105) + self.assertEqual(overview["completeness"]["export"], "COMPLETE") + self.assertEqual(len(detail["data"]["day_detail"]["runs"]), 105) + self.assertEqual(detail["completeness"]["full_population"], 105) + self.assertEqual(detail["completeness"]["displayed_population"], 105) + self.assertEqual(detail["completeness"]["export"], "COMPLETE") + self.assertNotIn("NaN", downloads["detail_json"][1].decode("utf-8")) + overview_markdown = downloads["overview_markdown"][1].decode("utf-8") + detail_markdown = downloads["detail_markdown"][1].decode("utf-8") + self.assertIn("# Telemetrieoverzicht", overview_markdown) + self.assertIn("## Samenvatting", overview_markdown) + self.assertIn("# Telemetriedetail", detail_markdown) + self.assertIn("export-run-104", detail_markdown) + self.assertIn(str(overview["snapshot_id"]), overview_markdown) + self.assertIn(str(detail["snapshot_id"]), detail_markdown) + self.assertEqual(detail["contract_version"], overview["contract_version"]) diff --git a/tests/engineering/test_telemetry_contract_v2.py b/tests/engineering/test_telemetry_contract_v2.py index 44ad802a..b65df9cc 100644 --- a/tests/engineering/test_telemetry_contract_v2.py +++ b/tests/engineering/test_telemetry_contract_v2.py @@ -17,6 +17,8 @@ from engineering_platform.storage import open_storage from engineering_platform.storage import EngineeringStorageError from engineering_platform.telemetry_contract import execution_chain_summary +from engineering_platform.telemetry_export import detail_model, overview_model, serialize_json, serialize_markdown +from engineering_platform.telemetry_metrics import aggregate_coverage def command_event(kind: str, item_id: str | None, command: str, output: str = "", exit_code: object = 0) -> str: @@ -167,6 +169,26 @@ def test_cache_ratio_uses_only_compatible_pairs(self) -> None: self.assertEqual(summary["cache_ratio_population"]["observed_observations"], 1) self.assertEqual(summary["metrics"]["cached_input_tokens"]["coverage"], "PARTIAL") + def test_conflicting_snapshot_is_not_promoted_by_matching_counts(self) -> None: + persist_provider_invocation(self.root, ProviderInvocation( + "conflict", 1, "codex_cli", None, "PROVIDER_EXECUTION", "IMPLEMENTATION", + "2026-09-17T00:00:00+00:00", "2026-09-17T00:00:01+00:00", 1000, + {"input_tokens": 90, "cached_input_tokens": 40, "output_tokens": 5}, + invocation_id="conflicting-invocation", + usage_snapshots=( + {"input_tokens": 100, "cached_input_tokens": 50, "output_tokens": 6}, + {"input_tokens": 90, "cached_input_tokens": 40, "output_tokens": 5}, + ), + )) + summary = provider_usage_summary(self.root, "conflict") + metric = summary["metrics"]["input_tokens"] + self.assertEqual(metric["coverage"], "CONFLICT") + self.assertEqual(metric["present_observations"], 1) + self.assertEqual(metric["valid_observations"], 0) + self.assertEqual(metric["conflicting_observations"], 1) + self.assertIsNone(metric["value"]) + self.assertIsNone(summary["cache_ratio_percent"]) + def test_invocation_receipt_replay_is_idempotent_and_conflict_fails_closed(self) -> None: invocation = ProviderInvocation( "receipt", 1, "codex_cli", None, "PROVIDER_EXECUTION", "IMPLEMENTATION", @@ -182,6 +204,43 @@ def test_invocation_receipt_replay_is_idempotent_and_conflict_fails_closed(self) **{**invocation.__dict__, "usage": {"input_tokens": 101, "cached_input_tokens": 80, "output_tokens": 1}} )) + def test_pr_uniqueness_is_union_across_invocations_and_repositories(self) -> None: + def pr_churn(item: str, repository: str) -> dict[str, object]: + payload = json.dumps([{"number": 17, "repository": {"nameWithOwner": repository}}]) + return churn_from_jsonl(command_event("item.completed", item, "gh pr list --json number,repository", payload)) + for ordinal, repository in enumerate(("owner/repo", "owner/repo", "other/repo"), 1): + persist_provider_invocation(self.root, ProviderInvocation( + "prs", ordinal, "codex_cli", None, "PROVIDER_EXECUTION", "IMPLEMENTATION", + "2026-09-17T00:00:00+00:00", "2026-09-17T00:00:01+00:00", 1000, + {}, invocation_id=f"pr-invocation-{ordinal}", churn=pr_churn(f"pr-{ordinal}", repository), + )) + summary = provider_usage_summary(self.root, "prs") + self.assertEqual(summary["historical_pr_result_occurrences"], 3) + self.assertEqual(summary["historical_unique_pr_results"], 2) + self.assertEqual(summary["historical_pr_metrics_coverage"], "COMPLETE") + + def test_churn_maximum_and_partial_coverage_keep_their_metric_semantics(self) -> None: + for ordinal, coverage, maximum in ((1, "PARTIAL", 900), (2, "COMPLETE", 400)): + persist_provider_invocation(self.root, ProviderInvocation( + "churn-semantics", ordinal, "codex_cli", None, "PROVIDER_EXECUTION", "IMPLEMENTATION", + "2026-09-17T00:00:00+00:00", "2026-09-17T00:00:01+00:00", 1000, + {}, invocation_id=f"churn-{ordinal}", churn={ + "historical_pr_metrics_coverage": coverage, + "maximum_tool_output_bytes": maximum, + }, + )) + summary = provider_usage_summary(self.root, "churn-semantics") + self.assertEqual(summary["context_churn"]["maximum_tool_output_bytes"], 900) + self.assertEqual(summary["historical_pr_metrics_coverage"], "PARTIAL") + + def test_structured_pr_without_repository_is_not_fabricated(self) -> None: + payload = json.dumps([{"number": 17}]) + churn = churn_from_jsonl(command_event( + "item.completed", "pr-missing-repository", "gh pr list --json number", payload, + )) + self.assertEqual(churn["historical_pr_metrics_coverage"], "PARTIAL") + self.assertEqual(churn["historical_unique_pr_results"], 0) + def test_exclusive_timing_closes_and_parallel_is_visible(self) -> None: start = datetime(2026, 9, 17, tzinfo=timezone.utc) record_phase(self.root, "timing", "TOTAL_EXECUTION", started_at=start, completed_at=start + timedelta(seconds=10)) @@ -207,6 +266,41 @@ def test_incomparable_boundaries_are_conflict_not_forced_to_close(self) -> None: self.assertEqual(summary["coverage"]["state"], "CONFLICT") self.assertFalse(summary["exclusive_distribution_closes"]) + def test_clock_difference_uses_wall_envelope_without_negative_rest(self) -> None: + start = datetime(2026, 9, 17, tzinfo=timezone.utc) + record_phase(self.root, "clock", "TOTAL_EXECUTION", started_at=start, completed_at=start + timedelta(milliseconds=10_100)) + record_phase(self.root, "clock", "PROVIDER_EXECUTION", started_at=start, completed_at=start + timedelta(milliseconds=10_100)) + with open_storage(self.root) as connection: + connection.execute("UPDATE execution_phase_spans SET duration_ms=10000 WHERE run_id='clock' AND phase_name='TOTAL_EXECUTION'") + summary = timing_summary(self.root, "clock") + distribution = {row["category"]: row["duration_ms"] for row in summary["exclusive_distribution"]} + self.assertEqual(summary["coverage"]["state"], "COMPLETE") + self.assertEqual(summary["total_monotonic_duration_ms"], 10_000) + self.assertEqual(summary["exclusive_envelope_duration_ms"], 10_100) + self.assertEqual(summary["clock_difference_ms"], 100) + self.assertEqual(distribution, {"PROVIDER_EXECUTION": 10_100}) + self.assertTrue(summary["exclusive_distribution_closes"]) + self.assertTrue(all(value >= 0 for value in distribution.values())) + + def test_opposite_clock_difference_and_small_segments_close_own_envelope(self) -> None: + start = datetime(2026, 9, 17, tzinfo=timezone.utc) + record_phase(self.root, "clock-opposite", "TOTAL_EXECUTION", started_at=start, completed_at=start + timedelta(seconds=10)) + with open_storage(self.root) as connection: + connection.execute("UPDATE execution_phase_spans SET duration_ms=10100 WHERE run_id='clock-opposite' AND phase_name='TOTAL_EXECUTION'") + for index in range(20): + segment_start = start + timedelta(microseconds=index * 500_000) + connection.execute( + """INSERT INTO execution_phase_spans(phase_id,run_id,phase_name,phase_category,parent_phase_id,attempt,ordinal,started_at,completed_at,duration_ms,outcome,metadata) + VALUES(?,?,?,?,?,?,?,?,?,?,?,?)""", + (f"small-{index}", "clock-opposite", "VALIDATION", "VALIDATION", None, 1, index + 2, + segment_start.isoformat(), (segment_start + timedelta(microseconds=500_000)).isoformat(), 500, "COMPLETE", '{"measurement_basis":"MONOTONIC"}'), + ) + summary = timing_summary(self.root, "clock-opposite") + self.assertEqual(summary["clock_difference_ms"], -100) + self.assertEqual(summary["exclusive_distribution_total_ms"], 10_000) + self.assertTrue(summary["exclusive_distribution_closes"]) + self.assertTrue(all(row["duration_ms"] >= 0 for row in summary["exclusive_distribution"])) + def test_missing_parent_is_partial_and_recovery_basis_is_explicit(self) -> None: start = datetime(2026, 9, 17, tzinfo=timezone.utc) record_phase(self.root, "partial", "TOTAL_EXECUTION", started_at=start, completed_at=start + timedelta(seconds=10)) @@ -300,7 +394,7 @@ def test_daily_api_overview_and_embedded_snapshot_share_canonical_values(self) - run = detail["runs"][0] snapshot = run["telemetry_snapshot"] metrics = snapshot["attempt"]["usage"]["metrics"] - self.assertEqual(detail["contract_version"], "telemetry-contract@2.0") + self.assertEqual(detail["contract_version"], "telemetry-contract@2.1") self.assertEqual(detail["summary"]["usage"]["input_tokens"]["value"], metrics["input_tokens"]["value"]) self.assertEqual(run["input_tokens"], metrics["input_tokens"]["value"]) self.assertEqual(detail["summary"]["provider_unique_coverage"]["average_ms"], snapshot["attempt"]["timing"]["provider_unique_coverage_ms"]) @@ -310,6 +404,33 @@ def test_daily_api_overview_and_embedded_snapshot_share_canonical_values(self) - self.assertEqual(overview[0]["input_tokens"], metrics["input_tokens"]["value"]) self.assertEqual(overview[0]["contract_version"], detail["contract_version"]) + def test_conflict_survives_run_day_and_chain_aggregation(self) -> None: + self._run("conflict-run", "sub-conflict", "2026-09-17T11:00:00+00:00", "2026-09-17T11:00:10+00:00") + persist_provider_invocation(self.root, ProviderInvocation( + "conflict-run", 1, "codex_cli", None, "PROVIDER_EXECUTION", "IMPLEMENTATION", + "2026-09-17T11:00:00+00:00", "2026-09-17T11:00:01+00:00", 1000, + {"input_tokens": 90, "cached_input_tokens": 40, "output_tokens": 5}, + invocation_id="conflict-chain-invocation", + usage_snapshots=( + {"input_tokens": 100, "cached_input_tokens": 50, "output_tokens": 6}, + {"input_tokens": 90, "cached_input_tokens": 40, "output_tokens": 5}, + ), + ), central_database=self.database) + run = provider_usage_summary(self.root, "conflict-run", central_database=self.database) + day = server._central_console_telemetry_detail(self.root, "forge", "2026-09-17") + chain = execution_chain_summary(self.root, "conflict-run", central_database=self.database) + self.assertEqual(run["metrics"]["input_tokens"]["coverage"], "CONFLICT") + assert day is not None + self.assertEqual(day["summary"]["usage"]["input_tokens"]["coverage"], "CONFLICT") + self.assertEqual(chain["usage_metrics"]["input_tokens"]["coverage"], "CONFLICT") + self.assertIsNone(chain["cache_ratio_percent"]) + + def test_missing_usage_projection_keeps_day_metric_incomplete(self) -> None: + self._run("missing-usage", "sub-missing", "2026-09-17T12:00:00+00:00", "2026-09-17T12:00:10+00:00") + detail = server._central_console_telemetry_detail(self.root, "forge", "2026-09-17") + assert detail is not None + self.assertNotEqual(detail["summary"]["usage"]["input_tokens"]["coverage"], "COMPLETE") + class CanonicalTelemetryBatchTests(unittest.TestCase): def test_large_day_is_bounded_and_attempt_observations_are_batch_loaded(self) -> None: @@ -319,7 +440,7 @@ def test_large_day_is_bounded_and_attempt_observations_are_batch_loaded(self) -> "updated_at": "2026-09-17T10:00:01+00:00", } for index in range(150)] snapshot = { - "contract_version": "telemetry-contract@2.0", + "contract_version": "telemetry-contract@2.1", "attempt": {"usage": {"metrics": {}, "invocations": []}, "timing": {}}, "chain": {"coverage": "PARTIAL", "runs": []}, } @@ -343,5 +464,92 @@ def test_large_day_is_bounded_and_attempt_observations_are_batch_loaded(self) -> self.assertEqual(composer.call_count, server.MAX_TELEMETRY_DAY_RUNS) +class CanonicalTelemetryExportTests(unittest.TestCase): + def test_equal_observation_counts_do_not_upgrade_partial_or_conflict(self) -> None: + partial = aggregate_coverage([{ + "coverage": "PARTIAL", "expected_observations": 1, + "present_observations": 1, "valid_observations": 1, + "missing_reason": "Parent relation unavailable", + }]) + conflict = aggregate_coverage([{ + "coverage": "CONFLICT", "expected_observations": 1, + "present_observations": 1, "valid_observations": 1, + "missing_reason": "Clock basis conflict", + }]) + self.assertEqual(partial["coverage"], "PARTIAL") + self.assertEqual(conflict["coverage"], "CONFLICT") + + def test_overview_and_detail_json_markdown_share_snapshot_model(self) -> None: + rows = [{ + "date": "2026-09-17", "prompt_count": 1, "input_tokens": None, + "measurement_coverage": "PARTIAL", "contract_version": "telemetry-contract@2.1", + }] + overview = overview_model( + project_id="forge", rows=rows, sort_key="date", sort_direction="desc", locale="nl", + ) + overview_json = json.loads(serialize_json(overview)) + overview_markdown = serialize_markdown(overview).decode() + self.assertEqual(overview_json["snapshot_id"], overview["snapshot_id"]) + self.assertIsNone(overview_json["data"]["overview"]["rows"][0]["input_tokens"]) + self.assertIn("Niet beschikbaar", overview_markdown) + + detail = { + "contract_version": "telemetry-contract@2.1", "timezone": "UTC", + "source_snapshot_references": ["run-1"], "matching_run_count": 1, + "returned_run_count": 1, "runs_truncated": False, + "summary": { + "cache_ratio_percent": None, + "usage": {"input_tokens": {"value": None, "coverage": "PARTIAL", "missing_reason": "One invocation lacks input usage"}}, + }, + "inclusive_phases": [], "exclusive_distribution": [], + "bottlenecks": {"longest_average_phase": {"phase": "VALIDATION", "average_ms": 1250}}, + "runs": [{"run_id": "run-1", "telemetry_snapshot": { + "contract_version": "telemetry-contract@2.1", "source_snapshot_reference": "run-1", + "attempt": {"scope": "EP_RUN_ATTEMPT", "run_id": "run-1", "usage": { + "invocations": [{"invocation_id": f"inv-{index}", "input_tokens": index} for index in range(251)], + }, "timing": {"timeline": [{"phase_id": f"span-{index}", "duration_ms": 1} for index in range(501)]}}, + "chain": {"runs": [{"run_id": "run-1"}]}, + }}], + } + exported = detail_model( + project_id="forge", execution_date="2026-09-17", detail=detail, + scope="UTC_DAY_DETAIL", run_id=None, locale="en", + ) + parsed = json.loads(serialize_json(exported)) + snapshot = parsed["data"]["day_detail"]["runs"][0]["telemetry_snapshot"]["attempt"] + self.assertEqual(len(snapshot["usage"]["invocations"]), 251) + self.assertEqual(len(snapshot["timing"]["timeline"]), 501) + detail_markdown = serialize_markdown(exported).decode() + self.assertIn("Provider invocations", detail_markdown) + self.assertIn("## Bottlenecks", detail_markdown) + self.assertIn("## Limitations and conflicts", detail_markdown) + self.assertIn("One invocation lacks input usage", detail_markdown) + chain_export = detail_model( + project_id="forge", execution_date="2026-09-17", detail=detail, + scope="EXECUTION_CHAIN", run_id="run-1", locale="en", + ) + self.assertEqual(chain_export["selection"]["scope"], "EXECUTION_CHAIN") + self.assertEqual(chain_export["completeness"]["displayed_population"], 1) + self.assertEqual(chain_export["completeness"]["full_population"], 1) + + def test_markdown_escapes_table_and_html_text_while_json_preserves_data(self) -> None: + model = overview_model( + project_id="forge", locale="fr", sort_key="date", sort_direction="desc", + rows=[{ + "date": "2026-09-17", "prompt_count": 1, + "measurement_coverage": "PARTIAL | \nnext", + "contract_version": "telemetry-contract@2.1", + }], + ) + markdown = serialize_markdown(model).decode("utf-8") + parsed = json.loads(serialize_json(model)) + self.assertIn("PARTIAL \\| <conflict> next", markdown) + self.assertNotIn("", markdown) + self.assertEqual( + parsed["data"]["overview"]["rows"][0]["measurement_coverage"], + "PARTIAL | \nnext", + ) + + if __name__ == "__main__": unittest.main() From d7b96d1b4e914236b18bd6100eb119a5e9c82ab7 Mon Sep 17 00:00:00 2001 From: pcvantol Date: Thu, 17 Sep 2026 12:16:07 +0200 Subject: [PATCH 2/6] fix: harden telemetry export download headers --- src/engineering_platform/server.py | 21 ++++++--- tests/engineering/test_server_foundation.py | 48 +++++++++++++++++++++ 2 files changed, 64 insertions(+), 5 deletions(-) diff --git a/src/engineering_platform/server.py b/src/engineering_platform/server.py index 5e19dda0..6a8ffe9d 100644 --- a/src/engineering_platform/server.py +++ b/src/engineering_platform/server.py @@ -322,6 +322,15 @@ def _attachment_content_disposition(filename: object) -> str: return f'attachment; filename="{sanitized}"' +def _telemetry_export_content_type(export_format: object) -> str: + """Return a fixed MIME type for one supported telemetry export format.""" + if export_format == "markdown": + return "text/markdown; charset=utf-8" + if export_format == "json": + return "application/json; charset=utf-8" + raise ValueError("telemetry export format is invalid") + + def _report_content_disposition(report_id: object) -> str: """Compose the report filename only after independently validating its id.""" if not isinstance(report_id, str) or not _SAFE_REPORT_ID.fullmatch(report_id): @@ -5272,11 +5281,13 @@ def _send_text(self, status_code: int, payload: str) -> None: except (BrokenPipeError, ConnectionResetError): return - def _send_download(self, payload: bytes, *, content_type: str, filename: str) -> None: - """Return one bounded read-only export with explicit browser metadata.""" + def _send_download(self, payload: bytes, *, export_format: str, filename: str) -> None: + """Return one bounded read-only export with fail-closed browser metadata.""" + content_type = _telemetry_export_content_type(export_format) + content_disposition = _attachment_content_disposition(filename) self.send_response(200) self.send_header("Content-Type", content_type) - self.send_header("Content-Disposition", f'attachment; filename="{filename}"') + self.send_header("Content-Disposition", content_disposition) self.send_header("Content-Length", str(len(payload))) self.send_header("Cache-Control", "no-store") self.send_header("X-Content-Type-Options", "nosniff") @@ -6148,7 +6159,7 @@ def _delegate_dashboard(self, method: str) -> None: payload = telemetry_export.serialize_markdown(model) if markdown else telemetry_export.serialize_json(model) self._send_download( payload, - content_type="text/markdown; charset=utf-8" if markdown else "application/json; charset=utf-8", + export_format=export_format, filename=f"telemetry-overview-{selected}-utc.{('md' if markdown else 'json')}", ) return @@ -6190,7 +6201,7 @@ def _delegate_dashboard(self, method: str) -> None: context = run_id if run_id is not None else export_date self._send_download( payload, - content_type="text/markdown; charset=utf-8" if markdown else "application/json; charset=utf-8", + export_format=export_format, filename=f"telemetry-detail-{selected}-{scope.casefold().replace('_', '-')}-{context}.{('md' if markdown else 'json')}", ) return diff --git a/tests/engineering/test_server_foundation.py b/tests/engineering/test_server_foundation.py index 30d92e48..c0e4f05b 100644 --- a/tests/engineering/test_server_foundation.py +++ b/tests/engineering/test_server_foundation.py @@ -149,6 +149,32 @@ def end_headers(self) -> None: pass def test_server_presentation_boundaries_reject_unsafe_headers_and_normalize_quota_data(self) -> None: """Console-only helpers remain fail-closed for unsafe or malformed inputs.""" + class DetachedDownloadHandler: + def __init__(self) -> None: + self.responses: list[int] = [] + self.headers: list[tuple[str, str]] = [] + self.wfile = io.BytesIO() + + def send_response(self, status: int) -> None: + self.responses.append(status) + + def send_header(self, name: str, value: str) -> None: + self.headers.append((name, value)) + + def end_headers(self) -> None: + pass + + self.assertEqual( + server._telemetry_export_content_type("markdown"), + "text/markdown; charset=utf-8", + ) + self.assertEqual( + server._telemetry_export_content_type("json"), + "application/json; charset=utf-8", + ) + for unsafe_format in (None, "", "text/plain", "json\r\nInjected: value"): + with self.assertRaises(ValueError): + server._telemetry_export_content_type(unsafe_format) self.assertEqual( server._attachment_content_disposition("qualification-report.md"), 'attachment; filename="qualification-report.md"', @@ -164,6 +190,28 @@ def test_server_presentation_boundaries_reject_unsafe_headers_and_normalize_quot with self.assertRaises(ValueError): server._report_content_disposition(unsafe_id) + download = DetachedDownloadHandler() + server._HealthHandler._send_download( + download, b"{}", export_format="json", filename="telemetry-overview-project-utc.json", + ) + self.assertEqual(download.responses, [200]) + self.assertIn(("Content-Type", "application/json; charset=utf-8"), download.headers) + self.assertIn( + ("Content-Disposition", 'attachment; filename="telemetry-overview-project-utc.json"'), + download.headers, + ) + for export_format, filename in ( + ("text/plain", "telemetry.txt"), + ("json", "telemetry.json\r\nInjected: value"), + ): + rejected = DetachedDownloadHandler() + with self.assertRaises(ValueError): + server._HealthHandler._send_download( + rejected, b"{}", export_format=export_format, filename=filename, + ) + self.assertEqual(rejected.responses, []) + self.assertEqual(rejected.headers, []) + self.assertIsNone(server._remaining_rate_limit_capacity({})) self.assertIsNone(server._remaining_rate_limit_capacity({"windows": ["malformed", {"used_percent": True}]})) self.assertEqual(server._remaining_rate_limit_capacity({"windows": [{"used_percent": -4}, {"used_percent": 125}]}), 0.0) From 0cc3a0b8f5e159f592244997ffcf788d49154ff6 Mon Sep 17 00:00:00 2001 From: pcvantol Date: Thu, 17 Sep 2026 14:54:48 +0200 Subject: [PATCH 3/6] fix: close telemetry export review findings --- docs/engineering/PROVIDER_USAGE_SEMANTICS.md | 35 +- .../telemetry-detail-example.json | 13 +- .../telemetry-detail-example.md | 7 +- .../telemetry-overview-example.json | 13 +- .../telemetry-overview-example.md | 7 +- src/engineering_platform/assets/dashboard.js | 40 +- .../execution_reporting.py | 3 +- src/engineering_platform/execution_timing.py | 7 +- src/engineering_platform/provider_usage.py | 141 +++++- src/engineering_platform/server.py | 251 +++++++++-- .../telemetry_contract.py | 47 +- src/engineering_platform/telemetry_export.py | 141 +++++- src/engineering_platform/telemetry_metrics.py | 13 +- tests/engineering/dashboard.spec.mjs | 18 +- tests/engineering/test_server_foundation.py | 55 ++- .../engineering/test_telemetry_contract_v2.py | 402 +++++++++++++++++- 16 files changed, 1064 insertions(+), 129 deletions(-) diff --git a/docs/engineering/PROVIDER_USAGE_SEMANTICS.md b/docs/engineering/PROVIDER_USAGE_SEMANTICS.md index 8f4f80eb..a4b5b832 100644 --- a/docs/engineering/PROVIDER_USAGE_SEMANTICS.md +++ b/docs/engineering/PROVIDER_USAGE_SEMANTICS.md @@ -3,7 +3,7 @@ ## Owning contract Engineering Platform owns the read model defined by -`telemetry-contract@2.1`. `provider_usage.py`, `execution_timing.py` and +`telemetry-contract@2.2`. `provider_usage.py`, `execution_timing.py` and `telemetry_contract.py` are the calculation authority. The API, dashboard, Engineering Report, Markdown download and JSON export project that contract; they must not independently calculate totals or bottlenecks. @@ -21,6 +21,14 @@ expected populations remain unknown. Independent metrics propagate only their own dependencies; one invalid cached-input observation does not invalidate an otherwise valid output observation. +A numeric aggregate marked `VALID_OBSERVATIONS_SUBTOTAL` contains only the +observations that remain valid for that metric. Higher scopes retain that +subtotal while independently retaining `PARTIAL` or `CONFLICT` coverage and +its observation counts. An unmarked numeric value from a conflicted legacy +source is not admitted. This makes sums and maxima independent of whether the +same population is grouped first by run, day, chain or overview. A compatible +cache-ratio population follows the same rule and preserves a measured zero. + The scopes are: - `EP_RUN_ATTEMPT`: exactly one EP execution attempt; @@ -88,6 +96,15 @@ observations. Output lines are never PRs inspected. When only old unstructured output-line totals survive, the value is named `legacy_historical_pr_output_lines` and exact PR fields are unavailable. +Opaque PR identities are retained up to a fixed privacy/storage bound. Every +query-bearing invocation records whether its retained identity set is complete +and whether it was truncated. Scope-wide uniqueness is exact only when every +contributing set is explicitly complete. Otherwise the exact value is `null`, +coverage is `PARTIAL`/`UNAVAILABLE`/`CONFLICT` as applicable, and a safe lower +bound is the greater of the retained union and each source's own lower bound. +Legacy rows without an explicit set-completeness marker are never promoted to +an exact complete union. + Read-command counters are derived command observations. Exact file-read observations require reliable tool metadata for opaque file identity, revision and (when relevant) range. Repeating a shell command is not proof that a file @@ -130,16 +147,26 @@ missing historical event metadata is never fabricated. ## Export contract -`telemetry-export@1.0` is the read-only export envelope for the overview and +`telemetry-export@1.1` is the read-only export envelope for the overview and detail Markdown/JSON downloads. Each envelope carries a snapshot digest, -as-of timestamp, project, UTC selection, scope, source references, displayed +source-as-of timestamp, separate download timestamp, project, UTC selection, scope, source references, displayed and full population, export completeness and metric coverage. Markdown and JSON serialize that one model; neither recalculates totals. +All CENTRAL reads used to build one model share one short read-only SQLite +transaction. The transaction is released before serialization or download. +The already projected, privacy-safe model is retained in a bounded in-memory +cache for ten minutes so Markdown and JSON can read back the same snapshot ID. +A missing, expired, project-foreign or selection-foreign ID fails explicitly; +it is never replaced silently by a newer read. + The overview export contains every retained row in the active project/filter scope rather than only the visible page. Detail export can select the UTC day, one attempt, or its verified execution chain. Full exports disable the UI -preview limits for runs, invocations and spans. JSON retains numeric machine +preview limits for runs, invocations and spans. A chain detail includes a safe +full attempt record, invocations, spans, timing distribution and coverage for +every retained verified member, including members outside the selected UTC +day. Export completeness is separate from measurement coverage. JSON retains numeric machine values and `null`; Markdown localizes human headings and explicitly renders unavailable values. Neither export includes prompts, replies, commands, secrets, raw tool output, or span metadata outside the telemetry allow-list. diff --git a/docs/engineering/examples/telemetry-followup/telemetry-detail-example.json b/docs/engineering/examples/telemetry-followup/telemetry-detail-example.json index 9511a99c..18c4b9e7 100644 --- a/docs/engineering/examples/telemetry-followup/telemetry-detail-example.json +++ b/docs/engineering/examples/telemetry-followup/telemetry-detail-example.json @@ -6,7 +6,8 @@ "full_population": 1, "metrics_may_be_partial": true }, - "contract_version": "telemetry-contract@2.1", + "contract_version": "telemetry-contract@2.2", + "downloaded_at": "2026-09-17T10:00:05+00:00", "data": { "attempt": { "run_id": "synthetic-run-1", @@ -71,12 +72,12 @@ } } }, - "contract_version": "telemetry-contract@2.1", + "contract_version": "telemetry-contract@2.2", "source_snapshot_reference": "synthetic-run-1" } } }, - "export_schema_version": "telemetry-export@1.0", + "export_schema_version": "telemetry-export@1.1", "locale": "nl", "selection": { "date": "2026-09-17", @@ -86,6 +87,12 @@ "scope": "EP_RUN_ATTEMPT", "timezone": "UTC" }, + "source": { + "as_of": "2026-09-17T10:00:00+00:00", + "kind": "CENTRAL_READ_TRANSACTION", + "reference": "central-schema:45:data-version:4" + }, + "source_as_of": "2026-09-17T10:00:00+00:00", "snapshot_id": "sha256:synthetic-detail-example", "source_snapshot_references": ["synthetic-run-1"] } diff --git a/docs/engineering/examples/telemetry-followup/telemetry-detail-example.md b/docs/engineering/examples/telemetry-followup/telemetry-detail-example.md index 5d43c80f..bc0f64db 100644 --- a/docs/engineering/examples/telemetry-followup/telemetry-detail-example.md +++ b/docs/engineering/examples/telemetry-followup/telemetry-detail-example.md @@ -11,9 +11,10 @@ | run_id | synthetic-run-1 | Snapshot: `sha256:synthetic-detail-example` -As-of: `2026-09-17T10:00:00+00:00` -Contract: `telemetry-contract@2.1` -Exportschema: `telemetry-export@1.0` +Source as-of: `2026-09-17T10:00:00+00:00` +Downloaded at: `2026-09-17T10:00:05+00:00` +Contract: `telemetry-contract@2.2` +Exportschema: `telemetry-export@1.1` ## Samenvatting diff --git a/docs/engineering/examples/telemetry-followup/telemetry-overview-example.json b/docs/engineering/examples/telemetry-followup/telemetry-overview-example.json index 649a7c68..3bb71107 100644 --- a/docs/engineering/examples/telemetry-followup/telemetry-overview-example.json +++ b/docs/engineering/examples/telemetry-followup/telemetry-overview-example.json @@ -6,7 +6,8 @@ "full_population": 1, "metrics_may_be_partial": true }, - "contract_version": "telemetry-contract@2.1", + "contract_version": "telemetry-contract@2.2", + "downloaded_at": "2026-09-17T10:00:05+00:00", "data": { "overview": { "row_count": 1, @@ -14,7 +15,7 @@ { "blocked_count": 0, "complete_count": 1, - "contract_version": "telemetry-contract@2.1", + "contract_version": "telemetry-contract@2.2", "date": "2026-09-17", "failed_count": 0, "input_tokens": 1250, @@ -33,7 +34,7 @@ } } }, - "export_schema_version": "telemetry-export@1.0", + "export_schema_version": "telemetry-export@1.1", "locale": "nl", "selection": { "aggregation_level": "UTC_DAY", @@ -43,6 +44,12 @@ "sort": {"direction": "desc", "key": "date"}, "timezone": "UTC" }, + "source": { + "as_of": "2026-09-17T10:00:00+00:00", + "kind": "CENTRAL_READ_TRANSACTION", + "reference": "central-schema:45:data-version:4" + }, + "source_as_of": "2026-09-17T10:00:00+00:00", "snapshot_id": "sha256:synthetic-overview-example", "source_snapshot_references": ["2026-09-17"] } diff --git a/docs/engineering/examples/telemetry-followup/telemetry-overview-example.md b/docs/engineering/examples/telemetry-followup/telemetry-overview-example.md index e011a671..dc809053 100644 --- a/docs/engineering/examples/telemetry-followup/telemetry-overview-example.md +++ b/docs/engineering/examples/telemetry-followup/telemetry-overview-example.md @@ -11,9 +11,10 @@ | sort | date, aflopend | Snapshot: `sha256:synthetic-overview-example` -As-of: `2026-09-17T10:00:00+00:00` -Contract: `telemetry-contract@2.1` -Exportschema: `telemetry-export@1.0` +Source as-of: `2026-09-17T10:00:00+00:00` +Downloaded at: `2026-09-17T10:00:05+00:00` +Contract: `telemetry-contract@2.2` +Exportschema: `telemetry-export@1.1` ## Dekking diff --git a/src/engineering_platform/assets/dashboard.js b/src/engineering_platform/assets/dashboard.js index 940d7cc8..00af2fd8 100644 --- a/src/engineering_platform/assets/dashboard.js +++ b/src/engineering_platform/assets/dashboard.js @@ -4355,7 +4355,8 @@ const executionTelemetryColumns = [ ["blocked_count", "telemetry.blocked"], ["failed_count", "telemetry.failed"], ]; const EXECUTION_TELEMETRY_PAGE_SIZE = 7; -let executionTelemetryRows = [], executionTelemetryPage = 1, executionTelemetrySort = { key: "date", direction: "desc" }; +let executionTelemetryRows = [], executionTelemetryPage = 1, executionTelemetrySort = { key: "date", direction: "desc" }, + executionTelemetryExportSnapshot = null; function telemetryComparableValue(row, key) { const value = row?.[key]; return key === "date" ? String(value || "") : typeof value === "number" && Number.isFinite(value) ? value : null; @@ -4397,6 +4398,7 @@ function setExecutionTelemetrySort(key) { ? { key, direction: executionTelemetrySort.direction === "asc" ? "desc" : "asc" } : { key, direction: key === "date" ? "desc" : "asc" }; executionTelemetryPage = 1; + executionTelemetryExportSnapshot = null; executionTelemetry(executionTelemetryRows); } function executionTelemetryText(rows = sortedExecutionTelemetryRows()) { @@ -4453,13 +4455,26 @@ async function downloadExecutionTelemetry(format) { const buttons = document.querySelectorAll("#executionTelemetry .telemetry-export"); buttons.forEach((button) => button.disabled = true); try { + const bindingParameters = new URLSearchParams(parameters); + bindingParameters.delete("format"); + const binding = `/api/telemetry/export?${bindingParameters}`; + if (!executionTelemetryExportSnapshot || executionTelemetryExportSnapshot.binding !== binding) { + const prepareParameters = new URLSearchParams(parameters); + prepareParameters.set("prepare", "1"); + prepareParameters.delete("format"); + const response = await fetch(`/api/telemetry/export?${prepareParameters}`, { cache: "no-store" }); + const prepared = await response.json().catch(() => ({})); + if (!response.ok || typeof prepared.snapshot_id !== "string") throw new Error(t("telemetry.export_failed")); + executionTelemetryExportSnapshot = { binding, id: prepared.snapshot_id }; + } + parameters.set("snapshot_id", executionTelemetryExportSnapshot.id); await downloadTelemetryResponse( `/api/telemetry/export?${parameters}`, `telemetry-overview-${project || "project"}-utc.${markdown ? "md" : "json"}`, ); void recordUserAction("telemetry_downloaded"); } catch (error) { - showDashboardToast(error instanceof Error ? error.message : t("telemetry.export_failed"), DASHBOARD_TOAST_GLYPHS.error); + showDashboardToast(t("telemetry.export_failed"), DASHBOARD_TOAST_GLYPHS.error); } finally { buttons.forEach((button) => button.disabled = !executionTelemetryRows.length); } @@ -4493,6 +4508,7 @@ async function clearExecutionTelemetry() { } } function executionTelemetry(rows) { + if (rows !== executionTelemetryRows) executionTelemetryExportSnapshot = null; let panel = $("executionTelemetry"), body = $("executionTelemetryRows"), pagination = $("executionTelemetryPagination"); @@ -4673,7 +4689,7 @@ function executionTelemetry(rows) { updateExecutionTelemetrySortHeaders(); } let telemetryDetailTrigger = null, telemetryDetailPayload = null, telemetryDetailDate = null, telemetryDetailRequestId = 0, - telemetryDetailSelection = { scope: "UTC_DAY_DETAIL", runId: null }; + telemetryDetailSelection = { scope: "UTC_DAY_DETAIL", runId: null }, telemetryDetailExportSnapshot = null; function telemetryMs(value) { return typeof value === "number" && value >= 0 ? telemetryDuration(value / 1000) : t("format.unavailable"); } function telemetryNumber(value) { return typeof value === "number" && Number.isFinite(value) @@ -4777,13 +4793,27 @@ async function downloadTelemetryDetail(format) { const buttons = [$('telemetryDetailDownloadMarkdown'), $('telemetryDetailDownloadJson')]; buttons.forEach((button) => button.disabled = true); try { + const binding = `${captured.date}:${captured.scope}:${captured.runId || ""}:${project}:${dashboardLocale}`; + if (!telemetryDetailExportSnapshot || telemetryDetailExportSnapshot.binding !== binding) { + const prepareParameters = new URLSearchParams(parameters); + prepareParameters.set("prepare", "1"); + prepareParameters.delete("format"); + const response = await fetch( + `/api/telemetry/${encodeURIComponent(captured.date)}/export?${prepareParameters}`, + { cache: "no-store" }, + ); + const prepared = await response.json().catch(() => ({})); + if (!response.ok || typeof prepared.snapshot_id !== "string") throw new Error(t("telemetry.export_failed")); + telemetryDetailExportSnapshot = { binding, id: prepared.snapshot_id }; + } + parameters.set("snapshot_id", telemetryDetailExportSnapshot.id); await downloadTelemetryResponse( `/api/telemetry/${encodeURIComponent(captured.date)}/export?${parameters}`, `telemetry-detail-${project || "project"}-${captured.runId || captured.date}.${markdown ? "md" : "json"}`, ); void recordUserAction(markdown ? "telemetry_detail_markdown_downloaded" : "telemetry_detail_json_downloaded"); } catch (error) { - showDashboardToast(error instanceof Error ? error.message : t("telemetry.export_failed"), DASHBOARD_TOAST_GLYPHS.error); + showDashboardToast(t("telemetry.export_failed"), DASHBOARD_TOAST_GLYPHS.error); } finally { buttons.forEach((button) => button.disabled = false); } @@ -4804,6 +4834,7 @@ function openTelemetryDetail(date, trigger) { const requestId = ++telemetryDetailRequestId; telemetryDetailTrigger = trigger || document.activeElement; telemetryDetailSelection = { scope: "UTC_DAY_DETAIL", runId: null }; + telemetryDetailExportSnapshot = null; const modal = $("telemetryDetailModal"), content = $("telemetryDetailContent"); $("telemetryDetailTitle").textContent = t("telemetry.detail_title", { date: telemetryDate(date) }); $("telemetryDetailDescription").textContent = t("telemetry.detail_description"); @@ -4841,6 +4872,7 @@ function renderCanonicalRunDetail(run, host, selectForExport = false) { if (updateExportSelection) telemetryDetailSelection = { scope: showChain ? "EXECUTION_CHAIN" : "EP_RUN_ATTEMPT", runId: run.run_id, }; + if (updateExportSelection) telemetryDetailExportSnapshot = null; }; attemptButton.addEventListener("click", () => select(false, true)); chainButton.addEventListener("click", () => select(true, true)); switcher.append(attemptButton, chainButton); select(false); host.append(switcher); diff --git a/src/engineering_platform/execution_reporting.py b/src/engineering_platform/execution_reporting.py index 959bf769..7a78f868 100644 --- a/src/engineering_platform/execution_reporting.py +++ b/src/engineering_platform/execution_reporting.py @@ -1437,7 +1437,8 @@ def percentage(value: object) -> str: f"- Escalation Reasons: `{churn.get('context_escalation_reasons', 'NONE')}`", f"- Historical PR Search/List Queries: `{observed(provider_usage.get('historical_pr_queries'))}`", f"- Structured PR Result Occurrences: `{observed(provider_usage.get('historical_pr_result_occurrences'))}`; coverage `{provider_usage.get('historical_pr_metrics_coverage', 'UNAVAILABLE')}`", - f"- Unique Structured PR Results: `{observed(provider_usage.get('historical_unique_pr_results'))}`", + f"- Unique Structured PR Results: `{observed(provider_usage.get('historical_unique_pr_results'))}`; coverage `{provider_usage.get('historical_unique_pr_coverage', 'UNAVAILABLE')}`", + f"- Unique Structured PR Results Lower Bound: `{observed(provider_usage.get('historical_unique_pr_results_lower_bound'))}` (shown only when exact scope-wide uniqueness is unavailable)", f"- PR Details/Diffs Actually Fetched: `{observed(provider_usage.get('historical_pr_details_fetched'))}`", f"- Legacy PR Output Lines: `{observed(provider_usage.get('legacy_historical_pr_output_lines'))}` (legacy derived counter; not PRs inspected)", f"- Historical Commits Inspected: `{provider_usage.get('historical_commit_results') if provider_usage.get('historical_context_metrics_authority') != 'UNAVAILABLE' else 'UNAVAILABLE'}`", diff --git a/src/engineering_platform/execution_timing.py b/src/engineering_platform/execution_timing.py index 9e14679f..d9dba206 100644 --- a/src/engineering_platform/execution_timing.py +++ b/src/engineering_platform/execution_timing.py @@ -683,12 +683,14 @@ def processing_ancestor(span: Mapping[str, object]) -> bool: def timing_summaries( root: Path, run_ids: list[str], *, central_database: Path | None = None, timeline_limit: int | None = 500, + _read_connection: sqlite3.Connection | None = None, ) -> dict[str, dict[str, object]]: """Load a bounded run population without one timing query per row.""" identifiers = list(dict.fromkeys(value for value in run_ids if isinstance(value, str) and value))[:1000] if not identifiers: return {} - connection = _connection(root, central_database) + owns_connection = _read_connection is None + connection = _read_connection if _read_connection is not None else _connection(root, central_database) placeholders = ",".join("?" for _ in identifiers) keys = ( "phase_id", "run_id", "phase_name", "phase_category", "parent_phase_id", @@ -707,7 +709,8 @@ def timing_summaries( identifiers, ).fetchall() finally: - connection.close() + if owns_connection: + connection.close() spans_by_run: dict[str, list[dict[str, object]]] = {run_id: [] for run_id in identifiers} for row in rows: item = dict(zip(keys, row, strict=True)) diff --git a/src/engineering_platform/provider_usage.py b/src/engineering_platform/provider_usage.py index 096ce321..bb34051d 100644 --- a/src/engineering_platform/provider_usage.py +++ b/src/engineering_platform/provider_usage.py @@ -21,7 +21,7 @@ from .storage import EngineeringStorageError, open_storage from .telemetry_metrics import ( COMPLETE, CONFLICT, PARTIAL, TELEMETRY_CALCULATION_VERSION, UNAVAILABLE, - aggregate_coverage, metric_coverage, + VALID_SUBTOTAL, aggregate_coverage, metric_coverage, ) @@ -46,16 +46,23 @@ "context_escalation_diagnostic", "event_identity_coverage", "historical_pr_metrics_coverage", + "historical_pr_identity_coverage", "file_read_observation_coverage", "tool_output_coverage", }) _SAFE_CHURN_IDENTITY_FIELDS = frozenset({"historical_pr_identity_hashes"}) +_SAFE_CHURN_BOOLEAN_FIELDS = frozenset({ + "historical_pr_identity_set_complete", "historical_pr_identity_set_truncated", +}) _CHURN_MAX_FIELDS = frozenset({"maximum_tool_output_bytes"}) -_CHURN_UNIQUE_FIELDS = frozenset({"historical_unique_pr_results"}) +_CHURN_UNIQUE_FIELDS = frozenset({ + "historical_unique_pr_results", "historical_pr_unique_lower_bound", +}) _CHURN_COVERAGE_FIELDS = frozenset({ "event_identity_coverage", "historical_pr_metrics_coverage", - "file_read_observation_coverage", "tool_output_coverage", + "historical_pr_identity_coverage", "file_read_observation_coverage", "tool_output_coverage", }) +_MAX_PR_IDENTITY_HASHES = 250 _MODEL_NORMALIZATION = { "gpt-5.6-sol": "gpt-5.6-sol", "gpt-5.6-terra": "gpt-5.6-terra", @@ -212,11 +219,12 @@ def _structured_pr_results(raw: object) -> tuple[int, set[str]] | None: except json.JSONDecodeError: return None values = payload if isinstance(payload, list) else [payload] - identities = {_structured_pr_id(value) for value in values} - identities.discard(None) - if len(identities) != len(values): + observed = [_structured_pr_id(value) for value in values] + if any(identity is None for identity in observed): return None - return len(values), {str(value) for value in identities} + # Repeated result occurrences are valid structured output. They remain in + # the occurrence count while the opaque identity set deduplicates them. + return len(values), {str(identity) for identity in observed} def churn_from_jsonl(*outputs: str) -> dict[str, object]: @@ -391,7 +399,14 @@ def churn_from_jsonl(*outputs: str) -> dict[str, object]: result["distinct_files_read"] = len(reads) result["unique_read_commands"] = len(reads) result["historical_unique_pr_results"] = len(pr_identities) - result["historical_pr_identity_hashes"] = sorted(pr_identities)[:250] + result["historical_pr_identity_hashes"] = sorted(pr_identities)[:_MAX_PR_IDENTITY_HASHES] + # The exact invocation-local count is retained, but the opaque identities + # stay bounded. Cross-invocation uniqueness is exact only when every + # contributing identity set is explicitly complete. + identity_set_complete = ( + result["historical_pr_unstructured_queries"] == 0 + and len(pr_identities) <= _MAX_PR_IDENTITY_HASHES + ) result["event_identity_coverage"] = ( CONFLICT if result["conflicting_terminal_events"] else PARTIAL if missing_identity else COMPLETE ) @@ -400,9 +415,18 @@ def churn_from_jsonl(*outputs: str) -> dict[str, object]: CONFLICT if result["conflicting_terminal_events"] else PARTIAL if missing_identity else COMPLETE ) if result["historical_pr_queries"]: + result["historical_pr_identity_set_complete"] = identity_set_complete + result["historical_pr_identity_set_truncated"] = len(pr_identities) > _MAX_PR_IDENTITY_HASHES + result["historical_pr_identity_retained_count"] = min( + len(pr_identities), _MAX_PR_IDENTITY_HASHES, + ) + result["historical_pr_unique_lower_bound"] = len(pr_identities) result["historical_pr_metrics_coverage"] = ( PARTIAL if result["historical_pr_unstructured_queries"] else COMPLETE ) + result["historical_pr_identity_coverage"] = ( + COMPLETE if identity_set_complete else PARTIAL + ) if not historical_observed: for key in ( "historical_commit_queries", "historical_commit_results", "historical_pr_queries", @@ -498,6 +522,8 @@ def persist_provider_invocation(root: Path, invocation: ProviderInvocation, *, c number = _number(value) if number is not None: churn[key] = number + elif key in _SAFE_CHURN_BOOLEAN_FIELDS and isinstance(value, bool): + churn[key] = value elif key in _SAFE_CHURN_TEXT_FIELDS and isinstance(value, str): compact = redact_diagnostic(value, limit=120) if compact: @@ -506,9 +532,19 @@ def persist_provider_invocation(root: Path, invocation: ProviderInvocation, *, c identities = sorted({ item.casefold() for item in value if isinstance(item, str) and re.fullmatch(r"[0-9a-f]{64}", item.casefold()) - })[:250] + })[:_MAX_PR_IDENTITY_HASHES] if identities: churn[key] = identities + raw_identities = (invocation.churn or {}).get("historical_pr_identity_hashes") + if isinstance(raw_identities, (list, tuple)): + supplied_identity_count = len({ + item.casefold() for item in raw_identities + if isinstance(item, str) and re.fullmatch(r"[0-9a-f]{64}", item.casefold()) + }) + if supplied_identity_count > _MAX_PR_IDENTITY_HASHES: + churn["historical_pr_identity_set_complete"] = False + churn["historical_pr_identity_set_truncated"] = True + churn["historical_pr_identity_retained_count"] = _MAX_PR_IDENTITY_HASHES snapshots = tuple( { key: _number(snapshot.get(key)) @@ -660,6 +696,7 @@ def provider_usage_summary( ] churn: dict[str, object] = {} pr_identity_hashes: set[str] = set() + pr_identity_sources: list[dict[str, object]] = [] coverage_evidence: dict[str, list[Mapping[str, object]]] = { key: [] for key in _CHURN_COVERAGE_FIELDS } @@ -672,6 +709,18 @@ def provider_usage_summary( except (TypeError, json.JSONDecodeError): values = {} if isinstance(values, dict): + query_count = _number(values.get("historical_pr_queries")) or 0 + if query_count: + retained = { + item for item in values.get("historical_pr_identity_hashes", []) + if isinstance(item, str) and re.fullmatch(r"[0-9a-f]{64}", item) + } if isinstance(values.get("historical_pr_identity_hashes"), list) else set() + pr_identity_sources.append({ + "retained": retained, + "complete": values.get("historical_pr_identity_set_complete") is True, + "invocation_unique_count": _number(values.get("historical_unique_pr_results")), + "coverage": values.get("historical_pr_identity_coverage", UNAVAILABLE), + }) for key, value in values.items(): if key in _SAFE_CHURN_IDENTITY_FIELDS and isinstance(value, list): pr_identity_hashes.update( @@ -682,7 +731,9 @@ def provider_usage_summary( # Exact scope-wide uniqueness is derived from the retained # opaque identity union below, never by summing counts. continue - elif isinstance(value, int): + elif key in _SAFE_CHURN_BOOLEAN_FIELDS and isinstance(value, bool): + continue + elif isinstance(value, int) and not isinstance(value, bool): previous = churn.get(key, 0) if key in _CHURN_MAX_FIELDS: churn[key] = max(previous if isinstance(previous, int) else 0, value) @@ -699,8 +750,19 @@ def provider_usage_summary( }) else: text_evidence[key].add(value) - if pr_identity_hashes: - churn["historical_unique_pr_results"] = len(pr_identity_hashes) + if pr_identity_sources: + identity_sets_complete = all( + source["complete"] is True and source["coverage"] == COMPLETE + for source in pr_identity_sources + ) + lower_bound = max( + len(pr_identity_hashes), + *(int(source["invocation_unique_count"] or 0) for source in pr_identity_sources), + ) + churn["historical_pr_identity_set_complete"] = identity_sets_complete + churn["historical_pr_unique_lower_bound"] = lower_bound + if identity_sets_complete: + churn["historical_unique_pr_results"] = len(pr_identity_hashes) for key, evidence in coverage_evidence.items(): if evidence: churn[key] = aggregate_coverage(evidence)["coverage"] @@ -742,13 +804,20 @@ def metric( key: str, value: object, *, meaning: str, unit: str, provenance: str = AUTHORITATIVE, compatible: Callable[[sqlite3.Row], bool] | None = None, ) -> dict[str, object]: + coverage_record = coverage(key, compatible=compatible) return { "value": value, "meaning": meaning, "unit": unit, "aggregation_level": "EP_RUN_ATTEMPT", "provenance": provenance, - **coverage(key, compatible=compatible), + "value_semantics": ( + VALID_SUBTOTAL + if isinstance(value, (int, float)) and not isinstance(value, bool) + and int(coverage_record["valid_observations"]) > 0 + else None + ), + **coverage_record, "calculation_version": TELEMETRY_CALCULATION_VERSION, "source_snapshot_reference": run_id, } @@ -851,8 +920,16 @@ def metric( "timing_correlation": UNAVAILABLE, }) exact_pr_coverage = churn.get("historical_pr_metrics_coverage", UNAVAILABLE) - unique_pr_coverage = exact_pr_coverage - if exact_pr_coverage == COMPLETE and len(rows) > 1 and not pr_identity_hashes: + if not pr_identity_sources: + unique_pr_coverage = UNAVAILABLE + elif any(source["coverage"] == CONFLICT for source in pr_identity_sources): + unique_pr_coverage = CONFLICT + elif all( + source["complete"] is True and source["coverage"] == COMPLETE + for source in pr_identity_sources + ): + unique_pr_coverage = COMPLETE + else: unique_pr_coverage = PARTIAL exact_file_coverage = churn.get("file_read_observation_coverage", UNAVAILABLE) result = { @@ -889,7 +966,9 @@ def metric( for row in rows ), ), - "input_tokens": compatible_input or None, "cached_input_tokens": compatible_cached or None, + "value_semantics": VALID_SUBTOTAL if compatible_cache_rows else None, + "input_tokens": compatible_input if compatible_cache_rows else None, + "cached_input_tokens": compatible_cached if compatible_cache_rows else None, }, "max_input_tokens_per_invocation": max(inputs) if inputs else None, "median_input_tokens_per_invocation": median(inputs) if inputs else None, @@ -912,14 +991,25 @@ def metric( "historical_pr_result_occurrences": churn.get("historical_pr_result_occurrences") if exact_pr_coverage != UNAVAILABLE else None, "historical_unique_pr_results": ( churn.get("historical_unique_pr_results") - if pr_identity_hashes or len(rows) == 1 else None - ) if exact_pr_coverage != UNAVAILABLE else None, + if unique_pr_coverage == COMPLETE else None + ), + "historical_unique_pr_results_lower_bound": ( + churn.get("historical_pr_unique_lower_bound") + if pr_identity_sources else None + ), + "historical_pr_identity_retained_count": ( + len(pr_identity_hashes) if pr_identity_sources else None + ), + "historical_pr_identity_set_complete": ( + unique_pr_coverage == COMPLETE if pr_identity_sources else None + ), "historical_pr_details_fetched": churn.get("historical_pr_details_fetched") if exact_pr_coverage != UNAVAILABLE else None, - "historical_pr_metrics_coverage": unique_pr_coverage, + "historical_pr_metrics_coverage": exact_pr_coverage, "historical_unique_pr_coverage": unique_pr_coverage, "historical_unique_pr_missing_reason": ( - "Invocation-level unique counts lack scope-wide opaque identities" - if unique_pr_coverage == PARTIAL and not pr_identity_hashes else None + "One or more query-bearing invocations have a truncated, legacy, or incomplete identity set" + if unique_pr_coverage == PARTIAL else + "Conflicting PR identity evidence" if unique_pr_coverage == CONFLICT else None ), "legacy_historical_pr_output_lines": churn.get("historical_pr_results_legacy_lines") or ( churn.get("historical_pr_results") if observed_history and exact_pr_coverage == UNAVAILABLE else None @@ -947,12 +1037,16 @@ def metric( def provider_usage_summaries( root: Path, run_ids: list[str], *, central_database: Path | None = None, invocation_limit: int | None = 250, + _read_connection: sqlite3.Connection | None = None, ) -> dict[str, dict[str, object]]: """Load a bounded run population in two queries, then reuse the canonical reducer.""" identifiers = list(dict.fromkeys(value for value in run_ids if isinstance(value, str) and value))[:1000] if not identifiers: return {} - if central_database is None: + owns_connection = _read_connection is None + if _read_connection is not None: + connection = _read_connection + elif central_database is None: connection = open_storage(root) else: database = central_database.resolve() @@ -983,7 +1077,8 @@ def provider_usage_summaries( identifiers, ).fetchall() finally: - connection.close() + if owns_connection: + connection.close() rows_by_run: dict[str, list[sqlite3.Row]] = {run_id: [] for run_id in identifiers} snapshots_by_run: dict[str, list[sqlite3.Row]] = {run_id: [] for run_id in identifiers} for row in rows: diff --git a/src/engineering_platform/server.py b/src/engineering_platform/server.py index 6a8ffe9d..4b475d5c 100644 --- a/src/engineering_platform/server.py +++ b/src/engineering_platform/server.py @@ -7,6 +7,8 @@ from __future__ import annotations import argparse +from collections.abc import Iterator +from contextlib import contextmanager from dataclasses import asdict, dataclass from datetime import datetime, timedelta, timezone import hashlib @@ -106,6 +108,7 @@ from .provider_usage import provider_usage_summaries, provider_usage_summary from .telemetry_contract import load_lineage_graph, run_telemetry_snapshot from .telemetry_metrics import ( + VALID_SUBTOTAL, aggregate_coverage as aggregate_telemetry_coverage, aggregate_numeric_metric, metric_coverage, @@ -3277,9 +3280,13 @@ def _central_run_record(row: sqlite3.Row, project_id: str) -> dict[str, object]: } -def _central_console_run_records(data_root: Path, project_id: str) -> list[dict[str, object]]: +def _central_console_run_records( + data_root: Path, project_id: str, *, _read_connection: sqlite3.Connection | None = None, +) -> list[dict[str, object]]: """Read all project runs with their admitted CENTRAL submission lineage.""" - with storage.sqlite_connection(data_root / SERVER_DATABASE_FILENAME) as connection: + owns_connection = _read_connection is None + connection = _read_connection or sqlite3.connect(data_root / SERVER_DATABASE_FILENAME) + try: connection.row_factory = sqlite3.Row rows = connection.execute( """SELECT r.run_id,r.state AS run_state,r.created_at,r.updated_at,r.execution_mode, @@ -3317,6 +3324,9 @@ def _central_console_run_records(data_root: Path, project_id: str) -> list[dict[ ORDER BY r.created_at DESC,r.run_id DESC LIMIT 1000""", (project_id,), ).fetchall() + finally: + if owns_connection: + connection.close() return [_central_run_record(row, project_id) for row in rows] @@ -3900,7 +3910,68 @@ def _central_console_terminal_execution_diagnostic(data_root: Path, run_id: str) return redact_diagnostic(diagnostic, limit=500) if diagnostic else None -def _central_console_telemetry(data_root: Path, project_id: str) -> list[dict[str, object]]: +@contextmanager +def _telemetry_read_snapshot( + data_root: Path, +) -> Iterator[tuple[sqlite3.Connection, str, str]]: + """Hold one short, read-only SQLite snapshot for a canonical export model.""" + database = (data_root / SERVER_DATABASE_FILENAME).resolve() + connection = sqlite3.connect( + f"file:{database}?mode=ro", uri=True, isolation_level=None, timeout=10, + ) + connection.row_factory = sqlite3.Row + try: + connection.execute("PRAGMA query_only=ON") + connection.execute("PRAGMA foreign_keys=ON") + connection.execute("PRAGMA busy_timeout=10000") + connection.execute("BEGIN") + # The first read establishes the SQLite snapshot before any helper is + # allowed to inspect runs, lineage, usage or timing. + schema_row = connection.execute( + "SELECT COALESCE(MAX(version),0) FROM engineering_schema_migrations" + ).fetchone() + data_version = int(connection.execute("PRAGMA data_version").fetchone()[0]) + source_as_of = datetime.now(timezone.utc).isoformat() + source_reference = f"central-schema:{int(schema_row[0])}:data-version:{data_version}" + yield connection, source_as_of, source_reference + finally: + if connection.in_transaction: + connection.rollback() + connection.close() + + +def _telemetry_retention_days(connection: sqlite3.Connection) -> int: + row = connection.execute( + "SELECT value FROM engineering_metadata WHERE key='console.telemetry_retention_days'" + ).fetchone() + try: + value = json.loads(str(row[0])) if row is not None else None + except json.JSONDecodeError: + value = None + allowed = central_database.CONSOLE_CONFIGURATION_OPTIONS["telemetry_retention_days"] + return int(value) if value in allowed else int( + central_database.CONSOLE_CONFIGURATION_DEFAULTS["telemetry_retention_days"] + ) + + +_TELEMETRY_EXPORT_STORE_LOCK = RLock() + + +def _telemetry_export_store(server_instance: object) -> telemetry_export.ExportSnapshotStore: + store = getattr(server_instance, "telemetry_export_snapshots", None) + if isinstance(store, telemetry_export.ExportSnapshotStore): + return store + with _TELEMETRY_EXPORT_STORE_LOCK: + store = getattr(server_instance, "telemetry_export_snapshots", None) + if not isinstance(store, telemetry_export.ExportSnapshotStore): + store = telemetry_export.ExportSnapshotStore() + setattr(server_instance, "telemetry_export_snapshots", store) + return store + + +def _central_console_telemetry( + data_root: Path, project_id: str, *, _read_connection: sqlite3.Connection | None = None, +) -> list[dict[str, object]]: """Aggregate canonical run snapshots by UTC day. The retired ``execution_runs`` telemetry projection is intentionally not @@ -3908,16 +3979,20 @@ def _central_console_telemetry(data_root: Path, project_id: str) -> list[dict[st """ grouped: dict[str, list[tuple[Mapping[str, object], dict[str, object]]]] = {} terminal_records = [ - record for record in _central_console_run_records(data_root, project_id) + record for record in _central_console_run_records( + data_root, project_id, _read_connection=_read_connection, + ) if record.get("state") in {"COMPLETE", "BLOCKED", "FAILED"} ] identifiers = [str(record["run_id"]) for record in terminal_records] try: timing_by_run = timing_summaries( data_root, identifiers, central_database=data_root / SERVER_DATABASE_FILENAME, + _read_connection=_read_connection, ) usage_by_run = provider_usage_summaries( data_root, identifiers, central_database=data_root / SERVER_DATABASE_FILENAME, + _read_connection=_read_connection, ) except (storage.EngineeringStorageError, sqlite3.DatabaseError): timing_by_run, usage_by_run = {}, {} @@ -4077,6 +4152,7 @@ def timing_aggregate(key: str) -> dict[str, int] | None: def _central_console_telemetry_detail( data_root: Path, project_id: str, execution_date: str, *, full: bool = False, + _read_connection: sqlite3.Connection | None = None, ) -> dict[str, object] | None: """Return the canonical contract used by UI, Markdown and JSON export.""" if not re.fullmatch(r"[0-9]{4}-[0-9]{2}-[0-9]{2}", execution_date): @@ -4084,7 +4160,10 @@ def _central_console_telemetry_detail( day = datetime.strptime(execution_date, "%Y-%m-%d").replace(tzinfo=timezone.utc) day_end = day + timedelta(days=1) matching: list[Mapping[str, object]] = [] - for record in _central_console_run_records(data_root, project_id): + all_records = _central_console_run_records( + data_root, project_id, _read_connection=_read_connection, + ) + for record in all_records: if record.get("state") not in {"COMPLETE", "BLOCKED", "FAILED"}: continue timestamp = record.get("updated_at") @@ -4101,6 +4180,7 @@ def _central_console_telemetry_detail( identifiers = [str(record["run_id"]) for record in selected_records] lineage_graph = load_lineage_graph( data_root, central_database=data_root / SERVER_DATABASE_FILENAME, + _read_connection=_read_connection, ) contexts = lineage_graph[0] related = set(identifiers) @@ -4123,10 +4203,12 @@ def _central_console_telemetry_detail( usage_cache = provider_usage_summaries( data_root, telemetry_identifiers, central_database=data_root / SERVER_DATABASE_FILENAME, invocation_limit=None if full else 250, + _read_connection=_read_connection, ) timing_cache = timing_summaries( data_root, telemetry_identifiers, central_database=data_root / SERVER_DATABASE_FILENAME, timeline_limit=None if full else 500, + _read_connection=_read_connection, ) except (storage.EngineeringStorageError, sqlite3.DatabaseError): usage_cache, timing_cache = {}, {} @@ -4146,15 +4228,20 @@ def attempt(snapshot: Mapping[str, object], key: str) -> Mapping[str, object]: nested = value.get(key, {}) if isinstance(value, Mapping) else {} return nested if isinstance(nested, Mapping) else {} - run_rows: list[dict[str, object]] = [] - for record, snapshot in selected: + def run_row(record: Mapping[str, object], snapshot: Mapping[str, object]) -> dict[str, object]: timing, usage = attempt(snapshot, "timing"), attempt(snapshot, "usage") metrics = usage.get("metrics", {}) if isinstance(usage.get("metrics"), Mapping) else {} def metric_value(name: str) -> object: metric = metrics.get(name) return metric.get("value") if isinstance(metric, Mapping) else None - run_rows.append({ + try: + observed = datetime.fromisoformat(str(record.get("updated_at")).replace("Z", "+00:00")).astimezone(timezone.utc) + outside_selected_window = not day <= observed < day_end + except ValueError: + outside_selected_window = None + return { "run_id": str(record["run_id"]), "started_at": record.get("created_at"), + "completed_at": record.get("updated_at"), "status": record.get("state"), "duration_label": "duration", "total_duration_ms": timing.get("total_wall_time_ms"), "queue_wait_ms": timing.get("queue_wait_time_ms"), @@ -4173,8 +4260,36 @@ def metric_value(name: str) -> object: "usage_coverage": {name: value.get("coverage") for name, value in metrics.items() if isinstance(value, Mapping)}, "timing_coverage": timing.get("coverage"), "phase_telemetry": "RECORDED" if timing.get("phase_telemetry_available") else "NOT_RECORDED", + "outside_selected_window": outside_selected_window, "chain": snapshot.get("chain"), "telemetry_snapshot": snapshot, - }) + } + + run_rows = [run_row(record, snapshot) for record, snapshot in selected] + chain_attempts: dict[str, list[dict[str, object]]] = {} + if full: + records_by_run = {str(record["run_id"]): record for record in all_records} + snapshots_by_run = {str(record["run_id"]): snapshot for record, snapshot in selected} + for record, snapshot in selected: + selected_run_id = str(record["run_id"]) + chain = snapshot.get("chain", {}) + chain_rows = chain.get("runs", []) if isinstance(chain, Mapping) else [] + details: list[dict[str, object]] = [] + for chain_row in chain_rows if isinstance(chain_rows, list) else []: + member = chain_row.get("run_id") if isinstance(chain_row, Mapping) else None + if not isinstance(member, str) or member not in records_by_run: + continue + member_snapshot = snapshots_by_run.get(member) + if member_snapshot is None: + member_snapshot = run_telemetry_snapshot( + data_root, member, + central_database=data_root / SERVER_DATABASE_FILENAME, + window_start=day, window_end=day_end, + _usage_cache=usage_cache, _timing_cache=timing_cache, + _lineage_graph=lineage_graph, + ) + snapshots_by_run[member] = member_snapshot + details.append(run_row(records_by_run[member], member_snapshot)) + chain_attempts[selected_run_id] = details def aggregate(values: list[int]) -> dict[str, int] | None: if not values: @@ -4246,15 +4361,23 @@ def values(key: str) -> list[int]: cache_sources: list[Mapping[str, object]] = [] cache_inputs = 0 cache_cached = 0 + cache_population_observed = False for index, usage in enumerate(usages): population = usage.get("cache_ratio_population") if isinstance(population, Mapping): cache_sources.append(population) - if population.get("coverage") != "CONFLICT": + if ( + population.get("coverage") != "CONFLICT" + or population.get("value_semantics") == VALID_SUBTOTAL + ): + source_observed = False if isinstance(population.get("input_tokens"), int): cache_inputs += int(population["input_tokens"]) + source_observed = True if isinstance(population.get("cached_input_tokens"), int): cache_cached += int(population["cached_input_tokens"]) + source_observed = True + cache_population_observed = cache_population_observed or source_observed else: cache_sources.append(metric_coverage( expected=None, present=0, valid=0, @@ -4287,12 +4410,13 @@ def values(key: str) -> list[int]: "usage": observed_usage, "cache_ratio_percent": ( round(cache_cached * 100 / cache_inputs, 3) - if cache_inputs and cache_coverage["coverage"] != "CONFLICT" else None + if cache_inputs and cache_population_observed else None ), "cache_ratio_population": { **cache_coverage, - "input_tokens": cache_inputs or None, - "cached_input_tokens": cache_cached or None, + "value_semantics": VALID_SUBTOTAL if cache_population_observed else None, + "input_tokens": cache_inputs if cache_population_observed else None, + "cached_input_tokens": cache_cached if cache_population_observed else None, }, "timing_coverage": timing_coverage, } @@ -4303,6 +4427,7 @@ def values(key: str) -> list[int]: "matching_run_count": len(matching), "returned_run_count": len(selected), "runs_truncated": len(matching) > len(selected), "run_limit": None if full else MAX_TELEMETRY_DAY_RUNS, "summary": summary, "runs": run_rows, + "chain_attempts": chain_attempts if full else {}, "inclusive_phases": phases, "phases": phases, "inclusive_shares_additive": False, "exclusive_distribution": exclusive, @@ -6132,6 +6257,8 @@ def _delegate_dashboard(self, method: str) -> None: if request.path == "/api/telemetry/export": parameters = parse_qs(request.query) export_format = (parameters.get("format") or [""])[0] + prepare = (parameters.get("prepare") or [""])[0] == "1" + snapshot_id = (parameters.get("snapshot_id") or [None])[0] locale = (parameters.get("locale") or ["en"])[0] sort_key = (parameters.get("sort") or ["date"])[0] direction = (parameters.get("direction") or ["desc"])[0] @@ -6141,20 +6268,46 @@ def _delegate_dashboard(self, method: str) -> None: "total_tokens", "complete_count", "blocked_count", "failed_count", } if ( - export_format not in {"markdown", "json"} + (not prepare and export_format not in {"markdown", "json"}) + or (prepare and export_format not in {"", "markdown", "json"}) or locale not in telemetry_export.SUPPORTED_LOCALES or sort_key not in allowed_sort or direction not in {"asc", "desc"} + or (snapshot_id is not None and not re.fullmatch(r"sha256:[0-9a-f]{64}", snapshot_id)) ): self._send(400, {"error": "TELEMETRY_EXPORT_SELECTION_INVALID"}) return - configuration = _central_console_configuration(self.server.data_root) # type: ignore[attr-defined] - model = telemetry_export.overview_model( - project_id=selected, - rows=_central_console_telemetry(self.server.data_root, selected), # type: ignore[attr-defined] - sort_key=sort_key, sort_direction=direction, locale=locale, - retention_days=configuration.get("telemetry_retention_days") - if isinstance(configuration.get("telemetry_retention_days"), int) else None, - ) + binding = json.dumps({ + "project_id": selected, "scope": "TELEMETRY_OVERVIEW", + "locale": locale, "sort": sort_key, "direction": direction, + }, sort_keys=True, separators=(",", ":")) + store = _telemetry_export_store(self.server) + model = store.read(snapshot_id, binding=binding) if snapshot_id else None + if snapshot_id and model is None: + self._send(409, {"error": "TELEMETRY_EXPORT_SNAPSHOT_UNAVAILABLE"}) + return + if model is None: + with _telemetry_read_snapshot(self.server.data_root) as ( # type: ignore[attr-defined] + read_connection, source_as_of, source_reference, + ): + model = telemetry_export.overview_model( + project_id=selected, + rows=_central_console_telemetry( + self.server.data_root, selected, # type: ignore[attr-defined] + _read_connection=read_connection, + ), + sort_key=sort_key, sort_direction=direction, locale=locale, + retention_days=_telemetry_retention_days(read_connection), + source_as_of=source_as_of, source_reference=source_reference, + ) + snapshot_id = store.retain(model, binding=binding) + if prepare: + self._send(200, { + "snapshot_id": snapshot_id, + "expires_in_seconds": telemetry_export.SNAPSHOT_TTL_SECONDS, + "selection": model["selection"], + }) + return + model = telemetry_export.download_model(model) markdown = export_format == "markdown" payload = telemetry_export.serialize_markdown(model) if markdown else telemetry_export.serialize_json(model) self._send_download( @@ -6169,33 +6322,61 @@ def _delegate_dashboard(self, method: str) -> None: if telemetry_export_match: parameters = parse_qs(request.query) export_format = (parameters.get("format") or [""])[0] + prepare = (parameters.get("prepare") or [""])[0] == "1" + snapshot_id = (parameters.get("snapshot_id") or [None])[0] locale = (parameters.get("locale") or ["en"])[0] scope = (parameters.get("scope") or ["UTC_DAY_DETAIL"])[0] run_id = (parameters.get("run_id") or [None])[0] if ( - export_format not in {"markdown", "json"} + (not prepare and export_format not in {"markdown", "json"}) + or (prepare and export_format not in {"", "markdown", "json"}) or locale not in telemetry_export.SUPPORTED_LOCALES or scope not in {"UTC_DAY_DETAIL", "EP_RUN_ATTEMPT", "EXECUTION_CHAIN"} or (run_id is not None and not _SAFE_REPORT_ID.fullmatch(run_id)) or (scope != "UTC_DAY_DETAIL" and run_id is None) + or (snapshot_id is not None and not re.fullmatch(r"sha256:[0-9a-f]{64}", snapshot_id)) ): self._send(400, {"error": "TELEMETRY_EXPORT_SELECTION_INVALID"}) return export_date = telemetry_export_match.group(1) - detail = _central_console_telemetry_detail( - self.server.data_root, selected, export_date, full=True, # type: ignore[attr-defined] - ) - if detail is None: - self._send(404, {"error": "TELEMETRY_NOT_FOUND"}) + binding = json.dumps({ + "project_id": selected, "scope": scope, "date": export_date, + "run_id": run_id, "locale": locale, + }, sort_keys=True, separators=(",", ":")) + store = _telemetry_export_store(self.server) + model = store.read(snapshot_id, binding=binding) if snapshot_id else None + if snapshot_id and model is None: + self._send(409, {"error": "TELEMETRY_EXPORT_SNAPSHOT_UNAVAILABLE"}) return - try: - model = telemetry_export.detail_model( - project_id=selected, execution_date=export_date, detail=detail, - scope=scope, run_id=run_id, locale=locale, - ) - except ValueError: - self._send(404, {"error": "TELEMETRY_EXPORT_RUN_NOT_FOUND"}) + if model is None: + with _telemetry_read_snapshot(self.server.data_root) as ( # type: ignore[attr-defined] + read_connection, source_as_of, source_reference, + ): + detail = _central_console_telemetry_detail( + self.server.data_root, selected, export_date, full=True, # type: ignore[attr-defined] + _read_connection=read_connection, + ) + if detail is None: + self._send(404, {"error": "TELEMETRY_NOT_FOUND"}) + return + try: + model = telemetry_export.detail_model( + project_id=selected, execution_date=export_date, detail=detail, + scope=scope, run_id=run_id, locale=locale, + source_as_of=source_as_of, source_reference=source_reference, + ) + except ValueError: + self._send(404, {"error": "TELEMETRY_EXPORT_RUN_NOT_FOUND"}) + return + snapshot_id = store.retain(model, binding=binding) + if prepare: + self._send(200, { + "snapshot_id": snapshot_id, + "expires_in_seconds": telemetry_export.SNAPSHOT_TTL_SECONDS, + "selection": model["selection"], + }) return + model = telemetry_export.download_model(model) markdown = export_format == "markdown" payload = telemetry_export.serialize_markdown(model) if markdown else telemetry_export.serialize_json(model) context = run_id if run_id is not None else export_date diff --git a/src/engineering_platform/telemetry_contract.py b/src/engineering_platform/telemetry_contract.py index 5c663b43..0d48a623 100644 --- a/src/engineering_platform/telemetry_contract.py +++ b/src/engineering_platform/telemetry_contract.py @@ -18,6 +18,7 @@ provider_usage_summary, ) from .telemetry_metrics import aggregate_coverage, aggregate_numeric_metric, metric_coverage +from .telemetry_metrics import VALID_SUBTOTAL from .storage import open_storage @@ -66,9 +67,11 @@ def _run_records(connection: sqlite3.Connection) -> dict[str, sqlite3.Row]: def load_lineage_graph( root: Path, *, central_database: Path | None = None, + _read_connection: sqlite3.Connection | None = None, ) -> tuple[list[sqlite3.Row], dict[str, sqlite3.Row], dict[str, dict[str, object]]]: """Load the bounded explicit identity graph once for a telemetry request.""" - connection = _connection(root, central_database) + owns_connection = _read_connection is None + connection = _read_connection if _read_connection is not None else _connection(root, central_database) try: contexts = connection.execute( """SELECT run_id,submission_id,fresh_submission,retry_parent_run_id, @@ -91,7 +94,8 @@ def load_lineage_graph( ): identities[str(row["run_id"])] = dict(row) finally: - connection.close() + if owns_connection: + connection.close() return contexts, runs, identities @@ -119,6 +123,9 @@ def execution_chain_summary( } # Walk to the one explicit root. Missing and cyclic parents stay visible. + # A proven different Forge Action is a hard lineage boundary: neither its + # own observations nor descendants behind it belong to this chain scope. + selected_action = identities.get(run_id, {}).get("engineering_action_id") cursor = run_id seen: set[str] = set() while cursor in by_run: @@ -133,6 +140,14 @@ def execution_chain_summary( if str(parent) not in runs: issues.append(f"missing-parent:{parent}") break + parent_action = identities.get(str(parent), {}).get("engineering_action_id") + if ( + isinstance(selected_action, str) and selected_action + and isinstance(parent_action, str) and parent_action + and parent_action != selected_action + ): + issues.append(f"conflicting-action-identity:{parent}") + break cursor = str(parent) root_run = cursor @@ -163,7 +178,16 @@ def execution_chain_summary( continue visited.add(current) component.append(current) - pending.extend(sorted(children.get(current, ()))) + for child in sorted(children.get(current, ())): + child_action = identities.get(child, {}).get("engineering_action_id") + if ( + isinstance(selected_action, str) and selected_action + and isinstance(child_action, str) and child_action + and child_action != selected_action + ): + issues.append(f"conflicting-action-identity:{child}") + continue + pending.append(child) if pending: issues.append(f"chain-limit:{MAX_CHAIN_RUNS}") @@ -175,6 +199,7 @@ def execution_chain_summary( cache_ratio_sources: list[Mapping[str, object]] = [] cache_ratio_input = 0 cache_ratio_cached = 0 + cache_population_observed = False intervals: list[tuple[datetime, datetime]] = [] processing_ms = 0 mission_values: set[str] = set() @@ -226,11 +251,18 @@ def execution_chain_summary( population = usage.get("cache_ratio_population") if isinstance(population, Mapping): cache_ratio_sources.append(population) - if population.get("coverage") != CONFLICT: + if ( + population.get("coverage") != CONFLICT + or population.get("value_semantics") == VALID_SUBTOTAL + ): + source_observed = False if isinstance(population.get("input_tokens"), int): cache_ratio_input += int(population["input_tokens"]) + source_observed = True if isinstance(population.get("cached_input_tokens"), int): cache_ratio_cached += int(population["cached_input_tokens"]) + source_observed = True + cache_population_observed = cache_population_observed or source_observed else: cache_ratio_sources.append(metric_coverage( expected=None, present=0, valid=0, @@ -291,7 +323,7 @@ def execution_chain_summary( cache_ratio_coverage = aggregate_coverage(cache_ratio_sources) cache_ratio = ( round(cache_ratio_cached * 100 / cache_ratio_input, 3) - if cache_ratio_input and cache_ratio_coverage["coverage"] != CONFLICT else None + if cache_ratio_input and cache_population_observed else None ) return { "contract_version": TELEMETRY_CALCULATION_VERSION, @@ -314,8 +346,9 @@ def execution_chain_summary( "cache_ratio_percent": cache_ratio, "cache_ratio_population": { **cache_ratio_coverage, - "input_tokens": cache_ratio_input or None, - "cached_input_tokens": cache_ratio_cached or None, + "value_semantics": VALID_SUBTOTAL if cache_population_observed else None, + "input_tokens": cache_ratio_input if cache_population_observed else None, + "cached_input_tokens": cache_ratio_cached if cache_population_observed else None, }, "outside_selected_window_count": outside_window, "runs": sorted(chain_rows, key=lambda row: str(row.get("started_at") or "")), diff --git a/src/engineering_platform/telemetry_export.py b/src/engineering_platform/telemetry_export.py index b5e41cfa..9ca25eb0 100644 --- a/src/engineering_platform/telemetry_export.py +++ b/src/engineering_platform/telemetry_export.py @@ -6,29 +6,81 @@ from __future__ import annotations +from copy import deepcopy from datetime import datetime, timezone import hashlib import json +from threading import RLock +from time import monotonic from typing import Mapping, Sequence from .telemetry_metrics import aggregate_numeric_metric, metric_coverage -EXPORT_SCHEMA_VERSION = "telemetry-export@1.0" +EXPORT_SCHEMA_VERSION = "telemetry-export@1.1" SUPPORTED_LOCALES = frozenset({"en", "nl", "de", "fr", "es"}) +SNAPSHOT_TTL_SECONDS = 600 +MAX_RETAINED_SNAPSHOTS = 32 +MAX_SNAPSHOT_BYTES = 32 * 1024 * 1024 + + +class ExportSnapshotStore: + """Bounded in-memory readback for already projected, privacy-safe models.""" + + def __init__(self) -> None: + self._lock = RLock() + self._models: dict[str, tuple[float, str, dict[str, object]]] = {} + + def retain(self, model: Mapping[str, object], *, binding: str) -> str: + snapshot_id = model.get("snapshot_id") + if not isinstance(snapshot_id, str) or not snapshot_id.startswith("sha256:"): + raise ValueError("TELEMETRY_EXPORT_SNAPSHOT_INVALID") + encoded = json.dumps(model, sort_keys=True, separators=(",", ":"), allow_nan=False).encode("utf-8") + if len(encoded) > MAX_SNAPSHOT_BYTES: + raise ValueError("TELEMETRY_EXPORT_SNAPSHOT_TOO_LARGE") + now = monotonic() + with self._lock: + self._expire(now) + if len(self._models) >= MAX_RETAINED_SNAPSHOTS and snapshot_id not in self._models: + oldest = min(self._models, key=lambda key: self._models[key][0]) + self._models.pop(oldest, None) + self._models[snapshot_id] = (now + SNAPSHOT_TTL_SECONDS, binding, deepcopy(dict(model))) + return snapshot_id + + def read(self, snapshot_id: str, *, binding: str) -> dict[str, object] | None: + now = monotonic() + with self._lock: + self._expire(now) + retained = self._models.get(snapshot_id) + if retained is None or retained[1] != binding: + return None + return deepcopy(retained[2]) + + def _expire(self, now: float) -> None: + for key, (expires_at, _, _) in list(self._models.items()): + if expires_at <= now: + self._models.pop(key, None) + + +def download_model(model: Mapping[str, object]) -> dict[str, object]: + """Stamp download time without changing the retained source snapshot.""" + result = deepcopy(dict(model)) + result["downloaded_at"] = datetime.now(timezone.utc).isoformat() + return result _LABELS = { - "en": {"overview": "Telemetry overview", "detail": "Telemetry detail", "selection": "Selection", "summary": "Summary", "coverage": "Coverage", "runs": "Runs", "inclusive": "Inclusive phase workload", "inclusive_note": "Inclusive shares may overlap and are not additive.", "exclusive": "Exclusive elapsed-time distribution", "bottlenecks": "Bottlenecks", "invocations": "Provider invocations", "timeline": "Timeline", "chain": "Execution chain", "limitations": "Limitations and conflicts", "field": "Field", "value": "Value", "unavailable": "Unavailable"}, - "nl": {"overview": "Telemetrieoverzicht", "detail": "Telemetriedetail", "selection": "Selectie", "summary": "Samenvatting", "coverage": "Dekking", "runs": "Uitvoeringen", "inclusive": "Inclusieve fasewerklast", "inclusive_note": "Inclusieve aandelen mogen overlappen en zijn niet optelbaar.", "exclusive": "Exclusieve doorlooptijdverdeling", "bottlenecks": "Knelpunten", "invocations": "Providerinvocations", "timeline": "Tijdlijn", "chain": "Uitvoeringsketen", "limitations": "Beperkingen en conflicten", "field": "Veld", "value": "Waarde", "unavailable": "Niet beschikbaar"}, - "de": {"overview": "Telemetrieübersicht", "detail": "Telemetriedetail", "selection": "Auswahl", "summary": "Zusammenfassung", "coverage": "Abdeckung", "runs": "Ausführungen", "inclusive": "Inklusive Phasenarbeitslast", "inclusive_note": "Inklusive Anteile dürfen sich überlappen und sind nicht addierbar.", "exclusive": "Exklusive Laufzeitverteilung", "bottlenecks": "Engpässe", "invocations": "Provider-Aufrufe", "timeline": "Zeitachse", "chain": "Ausführungskette", "limitations": "Einschränkungen und Konflikte", "field": "Feld", "value": "Wert", "unavailable": "Nicht verfügbar"}, - "fr": {"overview": "Vue d’ensemble de la télémétrie", "detail": "Détail de télémétrie", "selection": "Sélection", "summary": "Résumé", "coverage": "Couverture", "runs": "Exécutions", "inclusive": "Charge de phase inclusive", "inclusive_note": "Les parts inclusives peuvent se chevaucher et ne sont pas additionnables.", "exclusive": "Répartition exclusive du temps", "bottlenecks": "Goulets d’étranglement", "invocations": "Invocations fournisseur", "timeline": "Chronologie", "chain": "Chaîne d’exécution", "limitations": "Limites et conflits", "field": "Champ", "value": "Valeur", "unavailable": "Indisponible"}, - "es": {"overview": "Resumen de telemetría", "detail": "Detalle de telemetría", "selection": "Selección", "summary": "Resumen", "coverage": "Cobertura", "runs": "Ejecuciones", "inclusive": "Carga de fase inclusiva", "inclusive_note": "Las proporciones inclusivas pueden solaparse y no son sumables.", "exclusive": "Distribución exclusiva del tiempo", "bottlenecks": "Cuellos de botella", "invocations": "Invocaciones del proveedor", "timeline": "Cronología", "chain": "Cadena de ejecución", "limitations": "Limitaciones y conflictos", "field": "Campo", "value": "Valor", "unavailable": "No disponible"}, + "en": {"overview": "Telemetry overview", "detail": "Telemetry detail", "selection": "Selection", "summary": "Summary", "coverage": "Coverage", "runs": "Runs", "attempt": "Attempt", "inclusive": "Inclusive phase workload", "inclusive_note": "Inclusive shares may overlap and are not additive.", "exclusive": "Exclusive elapsed-time distribution", "bottlenecks": "Bottlenecks", "invocations": "Provider invocations", "timeline": "Timeline", "chain": "Execution chain", "limitations": "Limitations and conflicts", "field": "Field", "value": "Value", "unavailable": "Unavailable"}, + "nl": {"overview": "Telemetrieoverzicht", "detail": "Telemetriedetail", "selection": "Selectie", "summary": "Samenvatting", "coverage": "Dekking", "runs": "Uitvoeringen", "attempt": "Poging", "inclusive": "Inclusieve fasewerklast", "inclusive_note": "Inclusieve aandelen mogen overlappen en zijn niet optelbaar.", "exclusive": "Exclusieve doorlooptijdverdeling", "bottlenecks": "Knelpunten", "invocations": "Providerinvocations", "timeline": "Tijdlijn", "chain": "Uitvoeringsketen", "limitations": "Beperkingen en conflicten", "field": "Veld", "value": "Waarde", "unavailable": "Niet beschikbaar"}, + "de": {"overview": "Telemetrieübersicht", "detail": "Telemetriedetail", "selection": "Auswahl", "summary": "Zusammenfassung", "coverage": "Abdeckung", "runs": "Ausführungen", "attempt": "Versuch", "inclusive": "Inklusive Phasenarbeitslast", "inclusive_note": "Inklusive Anteile dürfen sich überlappen und sind nicht addierbar.", "exclusive": "Exklusive Laufzeitverteilung", "bottlenecks": "Engpässe", "invocations": "Provider-Aufrufe", "timeline": "Zeitachse", "chain": "Ausführungskette", "limitations": "Einschränkungen und Konflikte", "field": "Feld", "value": "Wert", "unavailable": "Nicht verfügbar"}, + "fr": {"overview": "Vue d’ensemble de la télémétrie", "detail": "Détail de télémétrie", "selection": "Sélection", "summary": "Résumé", "coverage": "Couverture", "runs": "Exécutions", "attempt": "Tentative", "inclusive": "Charge de phase inclusive", "inclusive_note": "Les parts inclusives peuvent se chevaucher et ne sont pas additionnables.", "exclusive": "Répartition exclusive du temps", "bottlenecks": "Goulets d’étranglement", "invocations": "Invocations fournisseur", "timeline": "Chronologie", "chain": "Chaîne d’exécution", "limitations": "Limites et conflits", "field": "Champ", "value": "Valeur", "unavailable": "Indisponible"}, + "es": {"overview": "Resumen de telemetría", "detail": "Detalle de telemetría", "selection": "Selección", "summary": "Resumen", "coverage": "Cobertura", "runs": "Ejecuciones", "attempt": "Intento", "inclusive": "Carga de fase inclusiva", "inclusive_note": "Las proporciones inclusivas pueden solaparse y no son sumables.", "exclusive": "Distribución exclusiva del tiempo", "bottlenecks": "Cuellos de botella", "invocations": "Invocaciones del proveedor", "timeline": "Cronología", "chain": "Cadena de ejecución", "limitations": "Limitaciones y conflictos", "field": "Campo", "value": "Valor", "unavailable": "No disponible"}, } def overview_model( *, project_id: str, rows: Sequence[Mapping[str, object]], sort_key: str, sort_direction: str, locale: str, retention_days: int | None = None, + source_as_of: str | None = None, source_reference: str | None = None, ) -> dict[str, object]: comparable = [row for row in rows if _sort_value(row, sort_key)[0] == 0] unavailable = [row for row in rows if _sort_value(row, sort_key)[0] != 0] @@ -74,12 +126,14 @@ def overview_model( references=[str(row.get("date")) for row in ordered if row.get("date")], displayed_population=len(ordered), full_population=len(ordered), export_complete=True, contract_version=_contract_version(ordered), + source_as_of=source_as_of, source_reference=source_reference, ) def detail_model( *, project_id: str, execution_date: str, detail: Mapping[str, object], scope: str, run_id: str | None, locale: str, + source_as_of: str | None = None, source_reference: str | None = None, ) -> dict[str, object]: runs = [row for row in detail.get("runs", []) if isinstance(row, Mapping)] selected = next((row for row in runs if row.get("run_id") == run_id), None) @@ -93,12 +147,32 @@ def detail_model( elif scope == "EXECUTION_CHAIN": snapshot = (selected or {}).get("telemetry_snapshot", {}) chain = snapshot.get("chain", {}) if isinstance(snapshot, Mapping) else {} - data = {"selected_attempt": _safe_run(selected or {}), "chain": _safe_chain(chain)} + attempts_by_selected = detail.get("chain_attempts", {}) + raw_attempts = ( + attempts_by_selected.get(str(run_id), []) + if isinstance(attempts_by_selected, Mapping) else [] + ) + chain_attempts = [ + _safe_run(row) for row in raw_attempts if isinstance(row, Mapping) + ] if isinstance(raw_attempts, list) else [] + data = { + "selected_attempt_reference": str(run_id), + "selected_attempt": _safe_run(selected or {}), + "chain": _safe_chain(chain), + "chain_attempts": chain_attempts, + } references = [str(row.get("run_id")) for row in chain.get("runs", []) if isinstance(row, Mapping)] if isinstance(chain, Mapping) else [str(run_id)] chain_runs = chain.get("runs", []) if isinstance(chain, Mapping) else [] - displayed_population = full_population = len(chain_runs) if isinstance(chain_runs, list) else 0 + full_population = len(chain_runs) if isinstance(chain_runs, list) else 0 + displayed_population = len(chain_attempts) reasons = chain.get("reasons", []) if isinstance(chain, Mapping) else [] - export_complete = not any( + expected_ids = { + str(row.get("run_id")) for row in chain_runs if isinstance(row, Mapping) and row.get("run_id") + } if isinstance(chain_runs, list) else set() + delivered_ids = { + str(row.get("run_id")) for row in chain_attempts if row.get("run_id") + } + export_complete = expected_ids == delivered_ids and not any( isinstance(reason, str) and reason.startswith("chain-limit:") for reason in (reasons if isinstance(reasons, list) else []) ) @@ -122,6 +196,7 @@ def detail_model( full_population=full_population, export_complete=export_complete, contract_version=str(detail.get("contract_version") or "UNAVAILABLE"), + source_as_of=source_as_of, source_reference=source_reference, ) @@ -138,7 +213,7 @@ def serialize_markdown(model: Mapping[str, object]) -> bytes: lines = [f"# {_md(title)}", "", f"## {_md(labels['selection'])}", "", _table( [labels["field"], labels["value"]], [[key, _display(value, labels)] for key, value in selection.items()] if isinstance(selection, Mapping) else [], - ), "", f"Snapshot: `{_md(model.get('snapshot_id'))}` ", f"As-of: `{_md(model.get('as_of'))}` ", f"Contract: `{_md(model.get('contract_version'))}` ", f"Export schema: `{_md(model.get('export_schema_version'))}`", ""] + ), "", f"Snapshot: `{_md(model.get('snapshot_id'))}` ", f"Source as-of: `{_md(model.get('source_as_of') or model.get('as_of'))}` ", f"Downloaded at: `{_md(model.get('downloaded_at'))}` ", f"Contract: `{_md(model.get('contract_version'))}` ", f"Export schema: `{_md(model.get('export_schema_version'))}`", ""] completeness = model.get("completeness", {}) lines += [f"## {_md(labels['coverage'])}", "", _table( [labels["field"], labels["value"]], @@ -155,8 +230,11 @@ def serialize_markdown(model: Mapping[str, object]) -> bytes: columns = sorted({str(key) for row in rows if isinstance(row, Mapping) for key in row}) lines += [f"## {_md(labels['runs'])}", "", _mapping_table(columns, rows, labels), ""] elif isinstance(data, Mapping): - detail = data.get("day_detail") or data.get("attempt") or data.get("selected_attempt") or {} - if isinstance(detail, Mapping): + chain_attempts = data.get("chain_attempts") + detail = data.get("day_detail") or data.get("attempt") or ( + {} if isinstance(chain_attempts, list) else data.get("selected_attempt") + ) or {} + if isinstance(detail, Mapping) and detail: lines += _markdown_detail(detail, labels) chain = data.get("chain") if isinstance(chain, Mapping): @@ -168,21 +246,38 @@ def serialize_markdown(model: Mapping[str, object]) -> bytes: if isinstance(chain_rows, list) and chain_rows: columns = sorted({str(key) for row in chain_rows if isinstance(row, Mapping) for key in row}) lines += [_mapping_table(columns, chain_rows, labels), ""] + if isinstance(chain_attempts, list): + for attempt in chain_attempts: + if not isinstance(attempt, Mapping): + continue + lines += [ + f"## {_md(labels['attempt'])}: `{_md(attempt.get('run_id'))}`", "", + *_markdown_detail(attempt, labels), + ] limitations = _limitations(model) if limitations: lines += [f"## {_md(labels['limitations'])}", "", *[f"- {_md(reason)}" for reason in limitations], ""] return ("\n".join(lines).rstrip() + "\n").encode("utf-8") -def _envelope(*, locale: str, selection: Mapping[str, object], data: Mapping[str, object], references: Sequence[str], displayed_population: int, full_population: int, export_complete: bool, contract_version: str) -> dict[str, object]: +def _envelope(*, locale: str, selection: Mapping[str, object], data: Mapping[str, object], references: Sequence[str], displayed_population: int, full_population: int, export_complete: bool, contract_version: str, source_as_of: str | None = None, source_reference: str | None = None) -> dict[str, object]: locale = locale if locale in SUPPORTED_LOCALES else "en" - as_of = datetime.now(timezone.utc).isoformat() - core = {"contract_version": contract_version, "selection": selection, "data": data, "source_snapshot_references": list(dict.fromkeys(references))} + as_of = source_as_of or datetime.now(timezone.utc).isoformat() + core = { + "contract_version": contract_version, "selection": selection, "data": data, + "source_snapshot_references": list(dict.fromkeys(references)), + "source": { + "kind": "CENTRAL_READ_TRANSACTION", + "as_of": as_of, + "reference": source_reference, + }, + } digest = hashlib.sha256(json.dumps(core, sort_keys=True, separators=(",", ":"), ensure_ascii=False, allow_nan=False).encode()).hexdigest() return { "export_schema_version": EXPORT_SCHEMA_VERSION, "contract_version": contract_version, - "snapshot_id": f"sha256:{digest}", "as_of": as_of, "locale": locale, + "snapshot_id": f"sha256:{digest}", "as_of": as_of, "source_as_of": as_of, + "downloaded_at": None, "source": core["source"], "locale": locale, "selection": dict(selection), "source_snapshot_references": core["source_snapshot_references"], "completeness": { "export": "COMPLETE" if export_complete and displayed_population == full_population else "PARTIAL", @@ -231,7 +326,19 @@ def _safe_attempt(attempt: Mapping[str, object]) -> dict[str, object]: def _safe_chain(chain: object) -> dict[str, object]: - return dict(chain) if isinstance(chain, Mapping) else {} + if not isinstance(chain, Mapping): + return {} + allowed = { + "contract_version", "scope", "selected_run_id", "root_run_id", + "coverage", "reason", "reasons", "mission_scope_label", "mission_id", + "engineering_action_id", "attempt_count", "original_attempt_count", + "retry_count", "resume_count", "first_started_at", "last_completed_at", + "elapsed_ms", "processing_time_ms", "covered_elapsed_ms", + "inter_attempt_gap_ms", "provider_invocation_count", + "duplicate_invocation_count", "usage_metrics", "cache_ratio_percent", + "cache_ratio_population", "outside_selected_window_count", "runs", + } + return {key: value for key, value in chain.items() if key in allowed} def _markdown_detail(detail: Mapping[str, object], labels: Mapping[str, str]) -> list[str]: diff --git a/src/engineering_platform/telemetry_metrics.py b/src/engineering_platform/telemetry_metrics.py index 629fbfd1..8933a935 100644 --- a/src/engineering_platform/telemetry_metrics.py +++ b/src/engineering_platform/telemetry_metrics.py @@ -16,7 +16,8 @@ UNAVAILABLE = "UNAVAILABLE" CONFLICT = "CONFLICT" COVERAGE_STATES = frozenset({COMPLETE, PARTIAL, UNAVAILABLE, CONFLICT}) -TELEMETRY_CALCULATION_VERSION = "telemetry-contract@2.1" +TELEMETRY_CALCULATION_VERSION = "telemetry-contract@2.2" +VALID_SUBTOTAL = "VALID_OBSERVATIONS_SUBTOTAL" def metric_coverage( @@ -121,7 +122,10 @@ def aggregate_numeric_metric( coverage = aggregate_coverage(rows) values = [ row.get("value") for row in rows - if row.get("coverage") != CONFLICT + if ( + row.get("coverage") != CONFLICT + or row.get("value_semantics") == VALID_SUBTOTAL + ) and isinstance(row.get("value"), (int, float)) and not isinstance(row.get("value"), bool) ] @@ -141,6 +145,11 @@ def aggregate_numeric_metric( "aggregation": aggregation, "aggregation_level": aggregation_level, "provenance": provenance or _merged_provenance(rows), + # The value is the subtotal over observations that remain valid for + # this metric. Coverage remains independent and can still be PARTIAL + # or CONFLICT. Higher-level reducers may retain only values carrying + # this marker when a source population is conflicted. + "value_semantics": VALID_SUBTOTAL if value is not None else None, **coverage, "calculation_version": TELEMETRY_CALCULATION_VERSION, "source_snapshot_references": sorted({ diff --git a/tests/engineering/dashboard.spec.mjs b/tests/engineering/dashboard.spec.mjs index b4644fa8..49a038a4 100644 --- a/tests/engineering/dashboard.spec.mjs +++ b/tests/engineering/dashboard.spec.mjs @@ -4889,6 +4889,9 @@ test.describe("Engineering Status browser smoke", () => { await page.route("**/api/telemetry/2026-08-24", (route) => { const requestUrl = new URL(route.request().url()); if (!requestUrl.pathname.endsWith("/export")) return route.fulfill({ json: detail }); + if (requestUrl.searchParams.get("prepare") === "1") return route.fulfill({ json: { + snapshot_id: `sha256:${"d".repeat(64)}`, selection: { scope: "UTC_DAY_DETAIL", date: "2026-08-24" }, + } }); const markdown = requestUrl.searchParams.get("format") === "markdown"; return route.fulfill({ contentType: markdown ? "text/markdown; charset=utf-8" : "application/json; charset=utf-8", @@ -4928,7 +4931,7 @@ test.describe("Engineering Status browser smoke", () => { const downloadedJson = await jsonDownload; expect(downloadedJson.suggestedFilename()).toBe("telemetry-detail-dashboard-fixture-utc-day-detail-2026-08-24.json"); const jsonContent = JSON.parse(readFileSync(await downloadedJson.path(), "utf8")); - expect(jsonContent.export_schema_version).toBe("telemetry-export@1.0"); + expect(jsonContent.export_schema_version).toBe("telemetry-export@1.1"); expect(jsonContent.selection).toMatchObject({ project_id: "dashboard-fixture", scope: "UTC_DAY_DETAIL", date: "2026-08-24" }); expect(jsonContent.data.day_detail.runs.map((run) => run.run_id)).toContain("inbox-day-export"); expect(markdownContent).toContain(jsonContent.snapshot_id); @@ -5424,7 +5427,7 @@ test.describe("Engineering Status browser smoke", () => { provider_invocation_count: index === 6 ? 9 : 5, })); const detail = { - contract_version: "telemetry-contract@2.1", source_snapshot_references: [runId], scope: "EP_RUN_ATTEMPTS_IN_UTC_DAY", + contract_version: "telemetry-contract@2.2", source_snapshot_references: [runId], scope: "EP_RUN_ATTEMPTS_IN_UTC_DAY", summary: { executions: 1, population: 1, completed: 1, blocked: 0, failed: 0, total_wall_time: { average_ms: 2721545, median_ms: 2721545, population: 1 }, @@ -5460,7 +5463,7 @@ test.describe("Engineering Status browser smoke", () => { model: "gpt-6-codex-experimental", phase_telemetry: "RECORDED", usage_coverage: { input_tokens: "PARTIAL", output_tokens: "PARTIAL" }, timing_coverage: { state: "COMPLETE" }, telemetry_snapshot: { - contract_version: "telemetry-contract@2.1", + contract_version: "telemetry-contract@2.2", attempt: { scope: "EP_RUN_ATTEMPT", timing: { total_wall_time_ms: 2721545, provider_unique_coverage_ms: 1233072, provider_cumulative_process_duration_ms: 1233062, external_wait_time_ms: 364132, unassigned_time_ms: 648751, coverage: { state: "COMPLETE" }, timeline, @@ -5491,13 +5494,16 @@ test.describe("Engineering Status browser smoke", () => { await page.route("**/api/telemetry/2026-09-17", (route) => { const requestUrl = new URL(route.request().url()); if (!requestUrl.pathname.endsWith("/export")) return route.fulfill({ json: detail }); + if (requestUrl.searchParams.get("prepare") === "1") return route.fulfill({ json: { + snapshot_id: `sha256:${"c".repeat(64)}`, selection: { scope: "EXECUTION_CHAIN", run_id: runId }, + } }); return route.fulfill({ contentType: "application/json; charset=utf-8", headers: { "Content-Disposition": `attachment; filename="telemetry-detail-dashboard-fixture-execution-chain-${runId}.json"` }, body: JSON.stringify({ - export_schema_version: "telemetry-export@1.0", contract_version: "telemetry-contract@2.1", + export_schema_version: "telemetry-export@1.1", contract_version: "telemetry-contract@2.2", snapshot_id: "sha256:chain-snapshot", selection: { scope: "EXECUTION_CHAIN", run_id: runId }, - data: { chain: detail.runs[0].telemetry_snapshot.chain }, + data: { selected_attempt: detail.runs[0], chain: detail.runs[0].telemetry_snapshot.chain }, }), }); }); @@ -5541,7 +5547,7 @@ test.describe("Engineering Status browser smoke", () => { const chainDownload = await chainDownloadPromise; expect(chainDownload.suggestedFilename()).toBe(`telemetry-detail-dashboard-fixture-execution-chain-${runId}.json`); const chainExport = JSON.parse(readFileSync(await chainDownload.path(), "utf8")); - expect(chainExport.export_schema_version).toBe("telemetry-export@1.0"); + expect(chainExport.export_schema_version).toBe("telemetry-export@1.1"); expect(chainExport.selection).toMatchObject({ scope: "EXECUTION_CHAIN", run_id: runId }); expect(chainExport.data.selected_attempt.run_id).toBe(runId); const overflow = await page.evaluate(() => document.documentElement.scrollWidth > document.documentElement.clientWidth); diff --git a/tests/engineering/test_server_foundation.py b/tests/engineering/test_server_foundation.py index c0e4f05b..18ab2eb4 100644 --- a/tests/engineering/test_server_foundation.py +++ b/tests/engineering/test_server_foundation.py @@ -2611,12 +2611,28 @@ def test_four_telemetry_exports_use_one_full_server_side_snapshot(self) -> None: ), central_database=self.root / server.SERVER_DATABASE_FILENAME, ) + persist_provider_invocation( + self.root, + ProviderInvocation( + "export-run-000", 2, "codex_cli", "observed-model", "PROVIDER_EXECUTION", "IMPLEMENTATION", + started.isoformat(), (started + timedelta(seconds=6)).isoformat(), 6000, + {"input_tokens": 190, "cached_input_tokens": 90, "output_tokens": 9}, + invocation_id="export-conflicting-invocation", + usage_snapshots=( + {"input_tokens": 200, "cached_input_tokens": 100, "output_tokens": 10}, + {"input_tokens": 190, "cached_input_tokens": 90, "output_tokens": 9}, + ), + ), + central_database=self.root / server.SERVER_DATABASE_FILENAME, + ) # Exercise the exact in-process read models as well as the installed # HTTP boundary below. The Server itself runs in a child process, so # this assertion keeps per-module coverage tied to the canonical # aggregation implementation rather than to a copied fixture. projected_overview = server._central_console_telemetry(self.root, "telemetry-export") self.assertEqual(projected_overview[0]["prompt_count"], 105) + self.assertEqual(projected_overview[0]["usage_metrics"]["input_tokens"]["value"], 100) + self.assertEqual(projected_overview[0]["usage_metrics"]["input_tokens"]["coverage"], "CONFLICT") projected_detail = server._central_console_telemetry_detail( self.root, "telemetry-export", "2026-09-17", full=True, ) @@ -2643,11 +2659,23 @@ def stop_httpd() -> None: self.addCleanup(stop_httpd) base = f"http://127.0.0.1:{port}" + overview_selection = ( + f"{base}/api/telemetry/export?project=telemetry-export&locale=nl" + "&sort=date&direction=desc" + ) + with urlopen(overview_selection + "&prepare=1") as response: + overview_snapshot = json.loads(response.read())["snapshot_id"] + detail_selection = ( + f"{base}/api/telemetry/2026-09-17/export?project=telemetry-export" + "&locale=nl&scope=UTC_DAY_DETAIL" + ) + with urlopen(detail_selection + "&prepare=1") as response: + detail_snapshot = json.loads(response.read())["snapshot_id"] urls = { - "overview_json": f"{base}/api/telemetry/export?project=telemetry-export&format=json&locale=nl&sort=date&direction=desc", - "overview_markdown": f"{base}/api/telemetry/export?project=telemetry-export&format=markdown&locale=nl&sort=date&direction=desc", - "detail_json": f"{base}/api/telemetry/2026-09-17/export?project=telemetry-export&format=json&locale=nl&scope=UTC_DAY_DETAIL", - "detail_markdown": f"{base}/api/telemetry/2026-09-17/export?project=telemetry-export&format=markdown&locale=nl&scope=UTC_DAY_DETAIL", + "overview_json": overview_selection + f"&format=json&snapshot_id={overview_snapshot}", + "overview_markdown": overview_selection + f"&format=markdown&snapshot_id={overview_snapshot}", + "detail_json": detail_selection + f"&format=json&snapshot_id={detail_snapshot}", + "detail_markdown": detail_selection + f"&format=markdown&snapshot_id={detail_snapshot}", } downloads: dict[str, tuple[object, bytes]] = {} for name, url in urls.items(): @@ -2665,11 +2693,19 @@ def stop_httpd() -> None: overview = json.loads(downloads["overview_json"][1]) detail = json.loads(downloads["detail_json"][1]) self.assertEqual(overview["data"]["overview"]["summary"]["run_count"], 105) + overview_input = overview["data"]["overview"]["summary"]["usage"]["input_tokens"] + detail_input = detail["data"]["day_detail"]["summary"]["usage"]["input_tokens"] + self.assertEqual(overview_input["value"], 100) + self.assertEqual(detail_input["value"], 100) + self.assertEqual(overview_input["coverage"], "CONFLICT") + self.assertEqual(detail_input["coverage"], "CONFLICT") + self.assertEqual(overview["snapshot_id"], overview_snapshot) self.assertEqual(overview["completeness"]["export"], "COMPLETE") self.assertEqual(len(detail["data"]["day_detail"]["runs"]), 105) self.assertEqual(detail["completeness"]["full_population"], 105) self.assertEqual(detail["completeness"]["displayed_population"], 105) self.assertEqual(detail["completeness"]["export"], "COMPLETE") + self.assertEqual(detail["snapshot_id"], detail_snapshot) self.assertNotIn("NaN", downloads["detail_json"][1].decode("utf-8")) overview_markdown = downloads["overview_markdown"][1].decode("utf-8") detail_markdown = downloads["detail_markdown"][1].decode("utf-8") @@ -2680,3 +2716,14 @@ def stop_httpd() -> None: self.assertIn(str(overview["snapshot_id"]), overview_markdown) self.assertIn(str(detail["snapshot_id"]), detail_markdown) self.assertEqual(detail["contract_version"], overview["contract_version"]) + with self.assertRaises(HTTPError) as mismatch: + urlopen( + f"{base}/api/telemetry/export?project=telemetry-export&format=json&locale=nl" + f"&sort=prompt_count&direction=desc&snapshot_id={overview_snapshot}" + ) + self.assertEqual(mismatch.exception.code, 409) + self.assertEqual( + json.loads(mismatch.exception.read())["error"], + "TELEMETRY_EXPORT_SNAPSHOT_UNAVAILABLE", + ) + mismatch.exception.close() diff --git a/tests/engineering/test_telemetry_contract_v2.py b/tests/engineering/test_telemetry_contract_v2.py index b65df9cc..4a79ee36 100644 --- a/tests/engineering/test_telemetry_contract_v2.py +++ b/tests/engineering/test_telemetry_contract_v2.py @@ -3,7 +3,6 @@ from datetime import datetime, timedelta, timezone import json from pathlib import Path -import sqlite3 from tempfile import TemporaryDirectory import unittest from unittest.mock import patch @@ -14,11 +13,15 @@ AUTHORITATIVE, ProviderInvocation, churn_from_jsonl, persist_provider_invocation, provider_usage_summary, usage_from_jsonl, usage_snapshots_from_jsonl, ) -from engineering_platform.storage import open_storage +from engineering_platform.storage import open_storage, sqlite_connection from engineering_platform.storage import EngineeringStorageError from engineering_platform.telemetry_contract import execution_chain_summary -from engineering_platform.telemetry_export import detail_model, overview_model, serialize_json, serialize_markdown +from engineering_platform.telemetry_export import ( + ExportSnapshotStore, detail_model, download_model, overview_model, + serialize_json, serialize_markdown, +) from engineering_platform.telemetry_metrics import aggregate_coverage +from engineering_platform.telemetry_metrics import VALID_SUBTOTAL, aggregate_numeric_metric, metric_coverage def command_event(kind: str, item_id: str | None, command: str, output: str = "", exit_code: object = 0) -> str: @@ -85,6 +88,36 @@ def test_structured_pr_results_count_objects_not_json_lines(self) -> None: self.assertEqual(result["historical_pr_results"], 6) self.assertEqual(result["historical_pr_metrics_coverage"], "COMPLETE") + def test_pr_identity_retention_boundaries_and_duplicate_occurrences(self) -> None: + for count, complete in ((249, True), (250, True), (251, False), (300, False)): + payload = json.dumps([ + {"number": number, "repository": {"nameWithOwner": "owner/repo"}} + for number in range(count) + ]) + result = churn_from_jsonl(command_event( + "item.completed", f"pr-{count}", + "gh pr list --json number,repository", payload, + )) + self.assertEqual(result["historical_unique_pr_results"], count) + self.assertEqual(len(result["historical_pr_identity_hashes"]), min(count, 250)) + self.assertEqual(result["historical_pr_identity_set_complete"], complete) + self.assertEqual(result["historical_pr_identity_set_truncated"], not complete) + self.assertEqual( + result["historical_pr_identity_coverage"], + "COMPLETE" if complete else "PARTIAL", + ) + duplicate = json.dumps([ + {"number": 17, "repository": {"nameWithOwner": "owner/repo"}}, + {"number": 17, "repository": {"nameWithOwner": "owner/repo"}}, + ]) + repeated = churn_from_jsonl(command_event( + "item.completed", "duplicate-pr", + "gh pr list --json number,repository", duplicate, + )) + self.assertEqual(repeated["historical_pr_result_occurrences"], 2) + self.assertEqual(repeated["historical_unique_pr_results"], 1) + self.assertEqual(repeated["historical_pr_identity_coverage"], "COMPLETE") + def test_unstructured_pr_and_read_commands_never_claim_exact_observations(self) -> None: result = churn_from_jsonl("\n".join(( command_event("item.completed", "pr", "gh pr list", "row one\nrow two"), @@ -218,6 +251,56 @@ def pr_churn(item: str, repository: str) -> dict[str, object]: self.assertEqual(summary["historical_pr_result_occurrences"], 3) self.assertEqual(summary["historical_unique_pr_results"], 2) self.assertEqual(summary["historical_pr_metrics_coverage"], "COMPLETE") + self.assertEqual(summary["historical_unique_pr_coverage"], "COMPLETE") + + def test_truncated_and_legacy_pr_identity_sets_never_claim_exact_scope_uniqueness(self) -> None: + def structured(count: int, *, offset: int = 0) -> dict[str, object]: + payload = json.dumps([ + {"number": number, "repository": {"nameWithOwner": "owner/repo"}} + for number in range(offset, offset + count) + ]) + return churn_from_jsonl(command_event( + "item.completed", f"pr-{offset}-{count}", + "gh pr list --json number,repository", payload, + )) + + churn_sources = (structured(200), structured(300, offset=100)) + expected_retained = len({ + identity + for churn in churn_sources + for identity in churn["historical_pr_identity_hashes"] + }) + for ordinal, churn in enumerate(churn_sources, 1): + persist_provider_invocation(self.root, ProviderInvocation( + "bounded-prs", ordinal, "codex_cli", None, "PROVIDER_EXECUTION", "IMPLEMENTATION", + "2026-09-17T00:00:00+00:00", "2026-09-17T00:00:01+00:00", 1000, + {}, invocation_id=f"bounded-pr-{ordinal}", churn=churn, + )) + bounded = provider_usage_summary(self.root, "bounded-prs") + self.assertIsNone(bounded["historical_unique_pr_results"]) + self.assertEqual(bounded["historical_unique_pr_results_lower_bound"], expected_retained) + self.assertEqual(bounded["historical_unique_pr_coverage"], "PARTIAL") + self.assertEqual(bounded["historical_pr_identity_retained_count"], expected_retained) + with open_storage(self.root) as connection: + stored = json.loads(connection.execute( + "SELECT churn FROM provider_invocations WHERE invocation_id='bounded-pr-2'" + ).fetchone()[0]) + self.assertEqual(len(stored["historical_pr_identity_hashes"]), 250) + self.assertTrue(stored["historical_pr_identity_set_truncated"]) + + persist_provider_invocation(self.root, ProviderInvocation( + "legacy-prs", 1, "codex_cli", None, "PROVIDER_EXECUTION", "IMPLEMENTATION", + "2026-09-17T00:00:00+00:00", "2026-09-17T00:00:01+00:00", 1000, + {}, invocation_id="legacy-pr", churn={ + "historical_pr_queries": 1, + "historical_unique_pr_results": 4, + "historical_pr_metrics_coverage": "COMPLETE", + }, + )) + legacy = provider_usage_summary(self.root, "legacy-prs") + self.assertIsNone(legacy["historical_unique_pr_results"]) + self.assertEqual(legacy["historical_unique_pr_results_lower_bound"], 4) + self.assertEqual(legacy["historical_unique_pr_coverage"], "PARTIAL") def test_churn_maximum_and_partial_coverage_keep_their_metric_semantics(self) -> None: for ordinal, coverage, maximum in ((1, "PARTIAL", 900), (2, "COMPLETE", 400)): @@ -324,7 +407,7 @@ def setUp(self) -> None: self.root = Path(self.temporary.name) server.initialize(self.root) self.database = self.root / server.SERVER_DATABASE_FILENAME - with sqlite3.connect(self.database) as connection: + with sqlite_connection(self.database) as connection: connection.execute("INSERT INTO ep_project_registrations VALUES('forge','{}','ACTIVE','2026-09-17T00:00:00+00:00','2026-09-17T00:00:00+00:00')") connection.execute("INSERT INTO ep_repository_registrations VALUES('forge','forge','forge','authority','{}','2026-09-17T00:00:00+00:00','2026-09-17T00:00:00+00:00')") @@ -332,7 +415,7 @@ def tearDown(self) -> None: self.temporary.cleanup() def _run(self, run_id: str, submission: str, started: str, ended: str, *, parent: str | None = None, resume: bool = False) -> None: - with sqlite3.connect(self.database) as connection: + with sqlite_connection(self.database) as connection: connection.execute("INSERT INTO ep_execution_runs VALUES(?,?,?,?,?,?)", (run_id, "forge", "COMPLETE", started, ended, "MANAGED")) connection.execute( "INSERT INTO execution_run_qualification_context VALUES(?,?,?,?,?,?)", @@ -361,13 +444,65 @@ def test_original_retry_chain_aggregates_without_double_invocations(self) -> Non def test_missing_parent_and_cycle_never_claim_complete(self) -> None: self._run("missing", "sub-m", "2026-09-17T00:00:00+00:00", "2026-09-17T00:00:01+00:00", parent="absent") self.assertEqual(execution_chain_summary(self.root, "missing", central_database=self.database)["coverage"], "PARTIAL") - with sqlite3.connect(self.database) as connection: + with sqlite_connection(self.database) as connection: connection.execute("INSERT INTO ep_execution_runs VALUES('a','forge','COMPLETE','2026-09-17T00:00:00+00:00','2026-09-17T00:00:01+00:00','MANAGED')") connection.execute("INSERT INTO ep_execution_runs VALUES('b','forge','COMPLETE','2026-09-17T00:00:00+00:00','2026-09-17T00:00:01+00:00','MANAGED')") connection.execute("INSERT INTO execution_run_qualification_context VALUES('a','sub-a',0,'b',NULL,'2026-09-17T00:00:00+00:00')") connection.execute("INSERT INTO execution_run_qualification_context VALUES('b','sub-b',0,'a',NULL,'2026-09-17T00:00:00+00:00')") self.assertEqual(execution_chain_summary(self.root, "a", central_database=self.database)["coverage"], "CONFLICT") + def test_verified_different_forge_actions_are_not_aggregated_as_one_chain(self) -> None: + contexts = [ + {"run_id": "action-a", "fresh_submission": 1, "retry_parent_run_id": None, "resume_parent_run_id": None}, + {"run_id": "action-b", "fresh_submission": 0, "retry_parent_run_id": "action-a", "resume_parent_run_id": None}, + # Even though C has the selected Action again, it sits behind a + # proven different-Action edge and must not re-enter the chain. + {"run_id": "action-c", "fresh_submission": 0, "retry_parent_run_id": "action-b", "resume_parent_run_id": None}, + ] + runs = { + run_id: { + "run_id": run_id, "state": "COMPLETE", + "created_at": f"2026-09-17T00:00:{offset:02d}+00:00", + "updated_at": f"2026-09-17T00:00:{offset + 1:02d}+00:00", + } + for run_id, offset in (("action-a", 0), ("action-b", 2), ("action-c", 4)) + } + identities = { + "action-a": {"engineering_action_id": "ACTION-A"}, + "action-b": {"engineering_action_id": "ACTION-B"}, + "action-c": {"engineering_action_id": "ACTION-A"}, + } + def metric(run_id: str, value: int) -> dict[str, object]: + return { + "contract_version": "telemetry-contract@2.2", "provider_invocation_count": 1, + "metrics": {name: { + "value": value, "value_semantics": VALID_SUBTOTAL, "coverage": "COMPLETE", + "expected_observations": 1, "present_observations": 1, + "valid_observations": 1, "conflicting_observations": 0, + "source_snapshot_reference": run_id, + } for name in ("input_tokens", "cached_input_tokens", "uncached_input_tokens", "output_tokens")}, + } + chain = execution_chain_summary( + self.root, "action-b", central_database=self.database, + _lineage_graph=(contexts, runs, identities), + _usage_cache={ + "action-a": metric("action-a", 100), + "action-b": metric("action-b", 50), + "action-c": metric("action-c", 900), + }, + _timing_cache={ + "action-a": {"total_wall_time_ms": 1000}, + "action-b": {"total_wall_time_ms": 1000}, + "action-c": {"total_wall_time_ms": 1000}, + }, + ) + self.assertEqual(chain["coverage"], "CONFLICT") + self.assertEqual(chain["attempt_count"], 1) + self.assertEqual(chain["runs"][0]["run_id"], "action-b") + self.assertEqual(chain["usage_metrics"]["input_tokens"]["value"], 50) + self.assertIn("conflicting-action-identity:action-a", chain["reasons"]) + self.assertIn("conflicting-action-identity:action-c", chain["reasons"]) + def test_daily_api_overview_and_embedded_snapshot_share_canonical_values(self) -> None: self._run("api-run", "sub-api", "2026-09-17T10:00:00+00:00", "2026-09-17T10:00:10+00:00") started = datetime(2026, 9, 17, 10, tzinfo=timezone.utc) @@ -394,7 +529,7 @@ def test_daily_api_overview_and_embedded_snapshot_share_canonical_values(self) - run = detail["runs"][0] snapshot = run["telemetry_snapshot"] metrics = snapshot["attempt"]["usage"]["metrics"] - self.assertEqual(detail["contract_version"], "telemetry-contract@2.1") + self.assertEqual(detail["contract_version"], "telemetry-contract@2.2") self.assertEqual(detail["summary"]["usage"]["input_tokens"]["value"], metrics["input_tokens"]["value"]) self.assertEqual(run["input_tokens"], metrics["input_tokens"]["value"]) self.assertEqual(detail["summary"]["provider_unique_coverage"]["average_ms"], snapshot["attempt"]["timing"]["provider_unique_coverage_ms"]) @@ -425,12 +560,206 @@ def test_conflict_survives_run_day_and_chain_aggregation(self) -> None: self.assertEqual(chain["usage_metrics"]["input_tokens"]["coverage"], "CONFLICT") self.assertIsNone(chain["cache_ratio_percent"]) + def test_valid_subtotals_survive_conflict_across_run_day_chain_and_grouping(self) -> None: + self._run("mixed-a", "sub-a", "2026-09-17T11:00:00+00:00", "2026-09-17T11:00:10+00:00") + self._run( + "mixed-b", "sub-b", "2026-09-17T11:00:20+00:00", + "2026-09-17T11:00:30+00:00", parent="mixed-a", + ) + persist_provider_invocation(self.root, ProviderInvocation( + "mixed-a", 1, "codex_cli", None, "PROVIDER_EXECUTION", "IMPLEMENTATION", + "2026-09-17T11:00:00+00:00", "2026-09-17T11:00:01+00:00", 1000, + {"input_tokens": 100, "cached_input_tokens": 80, "output_tokens": 10}, + invocation_id="mixed-valid", + ), central_database=self.database) + persist_provider_invocation(self.root, ProviderInvocation( + "mixed-a", 2, "codex_cli", None, "PROVIDER_EXECUTION", "IMPLEMENTATION", + "2026-09-17T11:00:01+00:00", "2026-09-17T11:00:02+00:00", 1000, + {"input_tokens": 190, "cached_input_tokens": 90, "output_tokens": 8}, + invocation_id="mixed-conflict", usage_snapshots=( + {"input_tokens": 200, "cached_input_tokens": 100, "output_tokens": 9}, + {"input_tokens": 190, "cached_input_tokens": 90, "output_tokens": 8}, + ), + ), central_database=self.database) + persist_provider_invocation(self.root, ProviderInvocation( + "mixed-b", 1, "codex_cli", None, "PROVIDER_EXECUTION", "IMPLEMENTATION", + "2026-09-17T11:00:20+00:00", "2026-09-17T11:00:21+00:00", 1000, + {"input_tokens": 50, "cached_input_tokens": 25, "output_tokens": 5}, + invocation_id="mixed-b-valid", + ), central_database=self.database) + + run_a = provider_usage_summary(self.root, "mixed-a", central_database=self.database) + run_b = provider_usage_summary(self.root, "mixed-b", central_database=self.database) + metric_a = run_a["metrics"]["input_tokens"] + metric_b = run_b["metrics"]["input_tokens"] + self.assertEqual(metric_a["value"], 100) + self.assertEqual(metric_a["coverage"], "CONFLICT") + self.assertEqual(metric_a["value_semantics"], VALID_SUBTOTAL) + direct = aggregate_numeric_metric( + [metric_a, metric_b], aggregation_level="TEST_POPULATION", unit="tokens", + ) + grouped = aggregate_numeric_metric([ + aggregate_numeric_metric([metric_a], aggregation_level="GROUP", unit="tokens"), + aggregate_numeric_metric([metric_b], aggregation_level="GROUP", unit="tokens"), + ], aggregation_level="TEST_POPULATION", unit="tokens") + for result in (direct, grouped): + self.assertEqual(result["value"], 150) + self.assertEqual(result["coverage"], "CONFLICT") + self.assertEqual(result["valid_observations"], 2) + self.assertEqual(result["conflicting_observations"], 1) + + day = server._central_console_telemetry_detail(self.root, "forge", "2026-09-17") + assert day is not None + self.assertEqual(day["summary"]["usage"]["input_tokens"]["value"], 150) + self.assertEqual(day["summary"]["usage"]["input_tokens"]["coverage"], "CONFLICT") + self.assertEqual(day["summary"]["cache_ratio_percent"], 70.0) + self.assertEqual(day["summary"]["cache_ratio_population"]["coverage"], "CONFLICT") + chain = execution_chain_summary(self.root, "mixed-b", central_database=self.database) + self.assertEqual(chain["usage_metrics"]["input_tokens"]["value"], 150) + self.assertEqual(chain["usage_metrics"]["input_tokens"]["coverage"], "CONFLICT") + self.assertEqual(chain["cache_ratio_percent"], 70.0) + + conflict_without_subtotal = { + "value": 999, "coverage": "CONFLICT", "expected_observations": 1, + "present_observations": 1, "valid_observations": 0, + "conflicting_observations": 1, + } + all_conflict = aggregate_numeric_metric( + [conflict_without_subtotal], aggregation_level="TEST", unit="tokens", + ) + self.assertIsNone(all_conflict["value"]) + real_zero = aggregate_numeric_metric([{ + "value": 0, "value_semantics": VALID_SUBTOTAL, + **metric_coverage(expected=2, present=2, valid=1, conflicting=1), + }], aggregation_level="TEST", unit="tokens") + self.assertEqual(real_zero["value"], 0) + self.assertEqual(real_zero["coverage"], "CONFLICT") + def test_missing_usage_projection_keeps_day_metric_incomplete(self) -> None: self._run("missing-usage", "sub-missing", "2026-09-17T12:00:00+00:00", "2026-09-17T12:00:10+00:00") detail = server._central_console_telemetry_detail(self.root, "forge", "2026-09-17") assert detail is not None self.assertNotEqual(detail["summary"]["usage"]["input_tokens"]["coverage"], "COMPLETE") + def test_chain_export_contains_full_details_for_attempts_on_different_days(self) -> None: + self._run("day-one", "sub-one", "2026-09-16T23:59:40+00:00", "2026-09-16T23:59:50+00:00") + self._run( + "day-two", "sub-two", "2026-09-17T00:00:10+00:00", + "2026-09-17T00:00:20+00:00", parent="day-one", + ) + for index, run_id in enumerate(("day-one", "day-two"), 1): + started = datetime(2026, 9, 16 + index - 1, 23 if index == 1 else 0, 59 if index == 1 else 0, 40 if index == 1 else 10, tzinfo=timezone.utc) + total = start_phase( + self.root, run_id, "TOTAL_EXECUTION", started_at=started, + monotonic_clock=0, central_database=self.database, + ) + provider = start_phase( + self.root, run_id, "PROVIDER_EXECUTION", started_at=started, + parent_phase_id=total.phase_id, monotonic_clock=0, + central_database=self.database, + ) + complete_phase( + self.root, provider, completed_at=started + timedelta(seconds=index), + monotonic_clock=index, + ) + complete_phase( + self.root, total, completed_at=started + timedelta(seconds=10), + monotonic_clock=10, + ) + persist_provider_invocation(self.root, ProviderInvocation( + run_id, 1, "codex_cli", f"model-{index}", "PROVIDER_EXECUTION", "IMPLEMENTATION", + started.isoformat(), (started + timedelta(seconds=index)).isoformat(), index * 1000, + {"input_tokens": index * 100, "cached_input_tokens": index * 50, "output_tokens": index * 10}, + model_authority=AUTHORITATIVE, invocation_id=f"invocation-{run_id}", + ), central_database=self.database) + + with sqlite_connection(self.database) as connection: + connection.executemany( + """INSERT INTO provider_invocations( + invocation_id,run_id,ordinal,provider,phase,role,started_at, + completed_at,duration_ms,usage_authority,speed_state,retry_ordinal, + rate_table_version,churn + ) VALUES(?,'day-one',?,'codex_cli','PROVIDER_EXECUTION','IMPLEMENTATION', + '2026-09-16T23:59:40+00:00','2026-09-16T23:59:41+00:00',1, + 'UNAVAILABLE','UNKNOWN',0,'2026-08-18','{}')""", + [(f"invocation-day-one-{ordinal}", ordinal) for ordinal in range(2, 252)], + ) + connection.executemany( + """INSERT INTO execution_phase_spans( + phase_id,run_id,phase_name,phase_category,parent_phase_id,attempt, + ordinal,started_at,completed_at,duration_ms,outcome,metadata + ) VALUES(?,'day-one','VALIDATION','VALIDATION',NULL,1,?, + '2026-09-16T23:59:40+00:00','2026-09-16T23:59:40.001+00:00',1, + 'COMPLETE','{"measurement_basis":"MONOTONIC"}')""", + [(f"span-day-one-{ordinal}", ordinal) for ordinal in range(3, 502)], + ) + + detail = server._central_console_telemetry_detail( + self.root, "forge", "2026-09-17", full=True, + ) + assert detail is not None + self.assertEqual(len(detail["runs"]), 1) + attempts = detail["chain_attempts"]["day-two"] + self.assertEqual([row["run_id"] for row in attempts], ["day-one", "day-two"]) + self.assertTrue(attempts[0]["outside_selected_window"]) + self.assertFalse(attempts[1]["outside_selected_window"]) + self.assertEqual( + [row["telemetry_snapshot"]["attempt"]["usage"]["invocations"][0]["invocation_id"] for row in attempts], + ["invocation-day-one", "invocation-day-two"], + ) + self.assertEqual(len(attempts[0]["telemetry_snapshot"]["attempt"]["usage"]["invocations"]), 251) + self.assertEqual(len(attempts[0]["telemetry_snapshot"]["attempt"]["timing"]["timeline"]), 501) + self.assertTrue(all(row["telemetry_snapshot"]["attempt"]["timing"]["timeline"] for row in attempts)) + + model = detail_model( + project_id="forge", execution_date="2026-09-17", detail=detail, + scope="EXECUTION_CHAIN", run_id="day-two", locale="nl", + ) + parsed = json.loads(serialize_json(model)) + exported_attempts = parsed["data"]["chain_attempts"] + self.assertEqual([row["run_id"] for row in exported_attempts], ["day-one", "day-two"]) + self.assertEqual(parsed["data"]["chain"]["usage_metrics"]["input_tokens"]["value"], 300) + self.assertEqual(parsed["completeness"]["export"], "COMPLETE") + markdown = serialize_markdown(model).decode("utf-8") + self.assertIn("## Poging: `day-one`", markdown) + self.assertIn("## Poging: `day-two`", markdown) + self.assertIn("invocation-day-one", markdown) + self.assertIn("invocation-day-two", markdown) + self.assertIn("invocation-day-one-251", markdown) + self.assertIn("span-day-one-501", markdown) + + def test_export_read_transaction_never_mixes_writer_commits_between_loaders(self) -> None: + self._run("snapshot-run", "sub-snapshot", "2026-09-17T10:00:00+00:00", "2026-09-17T10:00:10+00:00") + with sqlite_connection(self.database) as connection: + connection.execute("PRAGMA journal_mode=WAL") + with server._telemetry_read_snapshot(self.root) as (read_connection, source_as_of, source_reference): + before = server._central_console_telemetry_detail( + self.root, "forge", "2026-09-17", full=True, + _read_connection=read_connection, + ) + persist_provider_invocation(self.root, ProviderInvocation( + "snapshot-run", 1, "codex_cli", None, "PROVIDER_EXECUTION", "IMPLEMENTATION", + "2026-09-17T10:00:00+00:00", "2026-09-17T10:00:01+00:00", 1000, + {"input_tokens": 100, "cached_input_tokens": 80, "output_tokens": 10}, + invocation_id="snapshot-invocation", + ), central_database=self.database) + during = server._central_console_telemetry_detail( + self.root, "forge", "2026-09-17", full=True, + _read_connection=read_connection, + ) + self.assertTrue(source_as_of.endswith("+00:00")) + self.assertIn("central-schema:", source_reference) + after = server._central_console_telemetry_detail( + self.root, "forge", "2026-09-17", full=True, + ) + assert before is not None and during is not None and after is not None + for projection in (before, during): + usage = projection["runs"][0]["telemetry_snapshot"]["attempt"]["usage"] + self.assertEqual(usage.get("invocation_detail"), "UNAVAILABLE") + usage_after = after["runs"][0]["telemetry_snapshot"]["attempt"]["usage"] + self.assertEqual(usage_after["provider_invocation_count"], 1) + self.assertEqual(usage_after["metrics"]["input_tokens"]["value"], 100) + class CanonicalTelemetryBatchTests(unittest.TestCase): def test_large_day_is_bounded_and_attempt_observations_are_batch_loaded(self) -> None: @@ -440,7 +769,7 @@ def test_large_day_is_bounded_and_attempt_observations_are_batch_loaded(self) -> "updated_at": "2026-09-17T10:00:01+00:00", } for index in range(150)] snapshot = { - "contract_version": "telemetry-contract@2.1", + "contract_version": "telemetry-contract@2.2", "attempt": {"usage": {"metrics": {}, "invocations": []}, "timing": {}}, "chain": {"coverage": "PARTIAL", "runs": []}, } @@ -465,6 +794,31 @@ def test_large_day_is_bounded_and_attempt_observations_are_batch_loaded(self) -> class CanonicalTelemetryExportTests(unittest.TestCase): + def test_retained_export_snapshot_is_bound_immutable_and_expires_explicitly(self) -> None: + model = overview_model( + project_id="forge", rows=[], sort_key="date", sort_direction="desc", locale="en", + source_as_of="2026-09-17T10:00:00+00:00", source_reference="central-schema:45:data-version:4", + ) + store = ExportSnapshotStore() + with patch("engineering_platform.telemetry_export.monotonic", return_value=100.0): + snapshot_id = store.retain(model, binding="project=forge") + retained = store.read(snapshot_id, binding="project=forge") + self.assertEqual(retained, model) + assert retained is not None + retained["selection"]["project_id"] = "mutated" + with patch("engineering_platform.telemetry_export.monotonic", return_value=101.0): + self.assertEqual( + store.read(snapshot_id, binding="project=forge")["selection"]["project_id"], + "forge", + ) + self.assertIsNone(store.read(snapshot_id, binding="project=other")) + downloaded = download_model(model) + self.assertEqual(downloaded["snapshot_id"], snapshot_id) + self.assertIsNotNone(downloaded["downloaded_at"]) + self.assertIsNone(model["downloaded_at"]) + with patch("engineering_platform.telemetry_export.monotonic", return_value=701.0): + self.assertIsNone(store.read(snapshot_id, binding="project=forge")) + def test_equal_observation_counts_do_not_upgrade_partial_or_conflict(self) -> None: partial = aggregate_coverage([{ "coverage": "PARTIAL", "expected_observations": 1, @@ -482,7 +836,7 @@ def test_equal_observation_counts_do_not_upgrade_partial_or_conflict(self) -> No def test_overview_and_detail_json_markdown_share_snapshot_model(self) -> None: rows = [{ "date": "2026-09-17", "prompt_count": 1, "input_tokens": None, - "measurement_coverage": "PARTIAL", "contract_version": "telemetry-contract@2.1", + "measurement_coverage": "PARTIAL", "contract_version": "telemetry-contract@2.2", }] overview = overview_model( project_id="forge", rows=rows, sort_key="date", sort_direction="desc", locale="nl", @@ -494,7 +848,7 @@ def test_overview_and_detail_json_markdown_share_snapshot_model(self) -> None: self.assertIn("Niet beschikbaar", overview_markdown) detail = { - "contract_version": "telemetry-contract@2.1", "timezone": "UTC", + "contract_version": "telemetry-contract@2.2", "timezone": "UTC", "source_snapshot_references": ["run-1"], "matching_run_count": 1, "returned_run_count": 1, "runs_truncated": False, "summary": { @@ -504,13 +858,14 @@ def test_overview_and_detail_json_markdown_share_snapshot_model(self) -> None: "inclusive_phases": [], "exclusive_distribution": [], "bottlenecks": {"longest_average_phase": {"phase": "VALIDATION", "average_ms": 1250}}, "runs": [{"run_id": "run-1", "telemetry_snapshot": { - "contract_version": "telemetry-contract@2.1", "source_snapshot_reference": "run-1", + "contract_version": "telemetry-contract@2.2", "source_snapshot_reference": "run-1", "attempt": {"scope": "EP_RUN_ATTEMPT", "run_id": "run-1", "usage": { "invocations": [{"invocation_id": f"inv-{index}", "input_tokens": index} for index in range(251)], }, "timing": {"timeline": [{"phase_id": f"span-{index}", "duration_ms": 1} for index in range(501)]}}, "chain": {"runs": [{"run_id": "run-1"}]}, }}], } + detail["chain_attempts"] = {"run-1": list(detail["runs"])} exported = detail_model( project_id="forge", execution_date="2026-09-17", detail=detail, scope="UTC_DAY_DETAIL", run_id=None, locale="en", @@ -538,7 +893,7 @@ def test_markdown_escapes_table_and_html_text_while_json_preserves_data(self) -> rows=[{ "date": "2026-09-17", "prompt_count": 1, "measurement_coverage": "PARTIAL | \nnext", - "contract_version": "telemetry-contract@2.1", + "contract_version": "telemetry-contract@2.2", }], ) markdown = serialize_markdown(model).decode("utf-8") @@ -550,6 +905,29 @@ def test_markdown_escapes_table_and_html_text_while_json_preserves_data(self) -> "PARTIAL | \nnext", ) + def test_chain_attempt_sections_are_localized_in_all_supported_export_locales(self) -> None: + expected = { + "en": "## Attempt: `run-1`", + "nl": "## Poging: `run-1`", + "de": "## Versuch: `run-1`", + "fr": "## Tentative: `run-1`", + "es": "## Intento: `run-1`", + } + for locale, heading in expected.items(): + with self.subTest(locale=locale): + detail = { + "contract_version": "telemetry-contract@2.2", "timezone": "UTC", + "runs": [{"run_id": "run-1", "telemetry_snapshot": { + "chain": {"runs": [{"run_id": "run-1"}]}, + }}], + "chain_attempts": {"run-1": [{"run_id": "run-1", "status": "COMPLETE"}]}, + } + model = detail_model( + project_id="forge", execution_date="2026-09-17", detail=detail, + scope="EXECUTION_CHAIN", run_id="run-1", locale=locale, + ) + self.assertIn(heading, serialize_markdown(model).decode("utf-8")) + if __name__ == "__main__": unittest.main() From dcdc4fe05327ee3cd83755d3f0dd2c2bda107c93 Mon Sep 17 00:00:00 2001 From: pcvantol Date: Thu, 17 Sep 2026 15:11:59 +0200 Subject: [PATCH 4/6] fix: harden complete telemetry exports --- docs/engineering/PROVIDER_USAGE_SEMANTICS.md | 20 +++- src/engineering_platform/provider_usage.py | 23 ++++ src/engineering_platform/server.py | 66 +++++++++-- src/engineering_platform/telemetry_export.py | 45 ++++++-- tests/engineering/test_server_foundation.py | 40 +++++++ .../engineering/test_telemetry_contract_v2.py | 103 ++++++++++++++---- 6 files changed, 252 insertions(+), 45 deletions(-) diff --git a/docs/engineering/PROVIDER_USAGE_SEMANTICS.md b/docs/engineering/PROVIDER_USAGE_SEMANTICS.md index a4b5b832..80b495aa 100644 --- a/docs/engineering/PROVIDER_USAGE_SEMANTICS.md +++ b/docs/engineering/PROVIDER_USAGE_SEMANTICS.md @@ -103,7 +103,10 @@ contributing set is explicitly complete. Otherwise the exact value is `null`, coverage is `PARTIAL`/`UNAVAILABLE`/`CONFLICT` as applicable, and a safe lower bound is the greater of the retained union and each source's own lower bound. Legacy rows without an explicit set-completeness marker are never promoted to -an exact complete union. +an exact complete union. A modern record that claims completeness while its +declared unique count, retained count, opaque identities, truncation marker or +coverage disagree fails closed as `CONFLICT`; its safe lower bound may remain +visible, but it is not projected as an exact unique total. Read-command counters are derived command observations. Exact file-read observations require reliable tool metadata for opaque file identity, revision @@ -158,10 +161,17 @@ transaction. The transaction is released before serialization or download. The already projected, privacy-safe model is retained in a bounded in-memory cache for ten minutes so Markdown and JSON can read back the same snapshot ID. A missing, expired, project-foreign or selection-foreign ID fails explicitly; -it is never replaced silently by a newer read. - -The overview export contains every retained row in the active project/filter -scope rather than only the visible page. Detail export can select the UTC day, +it is never replaced silently by a newer read. The cache has independent +per-model, item-count and total encoded-byte limits. It evicts oldest snapshots +within that process-wide budget; a single model that cannot fit is rejected as +an explicit `TELEMETRY_EXPORT_SNAPSHOT_TOO_LARGE` product response rather than +being truncated or retained without a bound. + +The overview export reads the entire active project/filter population through +bounded 500-record database pages inside the same read transaction; the +1,000-run Console preview limit and 360-day UI limit are not export limits. +It therefore contains every retained row rather than only the visible page. +Detail export can select the UTC day, one attempt, or its verified execution chain. Full exports disable the UI preview limits for runs, invocations and spans. A chain detail includes a safe full attempt record, invocations, spans, timing distribution and coverage for diff --git a/src/engineering_platform/provider_usage.py b/src/engineering_platform/provider_usage.py index bb34051d..2808dff9 100644 --- a/src/engineering_platform/provider_usage.py +++ b/src/engineering_platform/provider_usage.py @@ -545,6 +545,29 @@ def persist_provider_invocation(root: Path, invocation: ProviderInvocation, *, c churn["historical_pr_identity_set_complete"] = False churn["historical_pr_identity_set_truncated"] = True churn["historical_pr_identity_retained_count"] = _MAX_PR_IDENTITY_HASHES + retained_identity_count = len(churn.get("historical_pr_identity_hashes", [])) + declared_unique_count = _number( + (invocation.churn or {}).get("historical_unique_pr_results") + ) + declared_retained_count = _number( + (invocation.churn or {}).get("historical_pr_identity_retained_count") + ) + claims_complete = churn.get("historical_pr_identity_set_complete") is True + identity_metadata_conflict = claims_complete and ( + churn.get("historical_pr_identity_coverage") != COMPLETE + or churn.get("historical_pr_identity_set_truncated") is True + or declared_unique_count is None + or declared_unique_count != supplied_identity_count + or retained_identity_count != supplied_identity_count + or declared_retained_count not in {None, retained_identity_count} + ) + churn["historical_pr_identity_retained_count"] = retained_identity_count + churn["historical_pr_unique_lower_bound"] = max( + retained_identity_count, declared_unique_count or 0, + ) + if identity_metadata_conflict: + churn["historical_pr_identity_set_complete"] = False + churn["historical_pr_identity_coverage"] = CONFLICT snapshots = tuple( { key: _number(snapshot.get(key)) diff --git a/src/engineering_platform/server.py b/src/engineering_platform/server.py index 4b475d5c..7c1237e3 100644 --- a/src/engineering_platform/server.py +++ b/src/engineering_platform/server.py @@ -3282,14 +3282,16 @@ def _central_run_record(row: sqlite3.Row, project_id: str) -> dict[str, object]: def _central_console_run_records( data_root: Path, project_id: str, *, _read_connection: sqlite3.Connection | None = None, + record_limit: int | None = 1000, ) -> list[dict[str, object]]: - """Read all project runs with their admitted CENTRAL submission lineage.""" + """Read project runs with bounded pages from one consistent source view.""" owns_connection = _read_connection is None connection = _read_connection or sqlite3.connect(data_root / SERVER_DATABASE_FILENAME) try: connection.row_factory = sqlite3.Row - rows = connection.execute( - """SELECT r.run_id,r.state AS run_state,r.created_at,r.updated_at,r.execution_mode, + if owns_connection and record_limit is None: + connection.execute("BEGIN") + query = """SELECT r.run_id,r.state AS run_state,r.created_at,r.updated_at,r.execution_mode, d.submission_id,d.state AS dispatch_state,d.operator_resolution, d.resolution_submission_id, retry.run_id AS retry_child_run_id, @@ -3319,12 +3321,22 @@ def _central_console_run_records( LEFT JOIN ep_forge_action_context_envelopes AS a ON a.submission_id=s.submission_id LEFT JOIN ep_forge_planning_context_envelopes AS p ON p.submission_id=s.submission_id LEFT JOIN ep_execution_host_evidence AS h ON h.run_id=r.run_id - LEFT JOIN execution_activity_summaries AS activity_summary ON activity_summary.run_id=r.run_id + LEFT JOIN execution_activity_summaries AS activity_summary ON activity_summary.run_id=r.run_id WHERE r.project_id=? - ORDER BY r.created_at DESC,r.run_id DESC LIMIT 1000""", - (project_id,), - ).fetchall() + ORDER BY r.created_at DESC,r.run_id DESC LIMIT ? OFFSET ?""" + page_size = 500 + rows: list[sqlite3.Row] = [] + offset = 0 + while record_limit is None or offset < record_limit: + batch_size = page_size if record_limit is None else min(page_size, record_limit - offset) + page = connection.execute(query, (project_id, batch_size, offset)).fetchall() + rows.extend(page) + if len(page) < batch_size: + break + offset += len(page) finally: + if owns_connection and connection.in_transaction: + connection.rollback() if owns_connection: connection.close() return [_central_run_record(row, project_id) for row in rows] @@ -3969,8 +3981,26 @@ def _telemetry_export_store(server_instance: object) -> telemetry_export.ExportS return store +def _retain_telemetry_export_snapshot( + store: telemetry_export.ExportSnapshotStore, + model: Mapping[str, object], *, binding: str, +) -> tuple[str | None, tuple[int, str] | None]: + """Retain one model or return a safe HTTP status/diagnostic pair.""" + try: + return store.retain(model, binding=binding), None + except ValueError as error: + code = str(error) + safe_code = ( + code if code.startswith("TELEMETRY_EXPORT_") + else "TELEMETRY_EXPORT_SNAPSHOT_INVALID" + ) + return None, ( + 413 if safe_code == "TELEMETRY_EXPORT_SNAPSHOT_TOO_LARGE" else 500, + safe_code, + ) def _central_console_telemetry( - data_root: Path, project_id: str, *, _read_connection: sqlite3.Connection | None = None, + data_root: Path, project_id: str, *, full: bool = False, + _read_connection: sqlite3.Connection | None = None, ) -> list[dict[str, object]]: """Aggregate canonical run snapshots by UTC day. @@ -3981,6 +4011,7 @@ def _central_console_telemetry( terminal_records = [ record for record in _central_console_run_records( data_root, project_id, _read_connection=_read_connection, + record_limit=None if full else 1000, ) if record.get("state") in {"COMPLETE", "BLOCKED", "FAILED"} ] @@ -4072,7 +4103,8 @@ def average(key: str) -> float | None: if snapshot.get("contract_version")), None) ) else None, }) - return sorted(entries, key=lambda entry: str(entry["date"]), reverse=True)[:360] + ordered = sorted(entries, key=lambda entry: str(entry["date"]), reverse=True) + return ordered if full else ordered[:360] def _legacy_central_console_telemetry_detail(data_root: Path, project_id: str, execution_date: str) -> dict[str, object] | None: @@ -4162,6 +4194,7 @@ def _central_console_telemetry_detail( matching: list[Mapping[str, object]] = [] all_records = _central_console_run_records( data_root, project_id, _read_connection=_read_connection, + record_limit=None if full else 1000, ) for record in all_records: if record.get("state") not in {"COMPLETE", "BLOCKED", "FAILED"}: @@ -6293,13 +6326,19 @@ def _delegate_dashboard(self, method: str) -> None: project_id=selected, rows=_central_console_telemetry( self.server.data_root, selected, # type: ignore[attr-defined] + full=True, _read_connection=read_connection, ), sort_key=sort_key, sort_direction=direction, locale=locale, retention_days=_telemetry_retention_days(read_connection), source_as_of=source_as_of, source_reference=source_reference, ) - snapshot_id = store.retain(model, binding=binding) + snapshot_id, snapshot_error = _retain_telemetry_export_snapshot( + store, model, binding=binding, + ) + if snapshot_error is not None: + self._send(snapshot_error[0], {"error": snapshot_error[1]}) + return if prepare: self._send(200, { "snapshot_id": snapshot_id, @@ -6368,7 +6407,12 @@ def _delegate_dashboard(self, method: str) -> None: except ValueError: self._send(404, {"error": "TELEMETRY_EXPORT_RUN_NOT_FOUND"}) return - snapshot_id = store.retain(model, binding=binding) + snapshot_id, snapshot_error = _retain_telemetry_export_snapshot( + store, model, binding=binding, + ) + if snapshot_error is not None: + self._send(snapshot_error[0], {"error": snapshot_error[1]}) + return if prepare: self._send(200, { "snapshot_id": snapshot_id, diff --git a/src/engineering_platform/telemetry_export.py b/src/engineering_platform/telemetry_export.py index 9ca25eb0..0ad207cb 100644 --- a/src/engineering_platform/telemetry_export.py +++ b/src/engineering_platform/telemetry_export.py @@ -20,31 +20,55 @@ EXPORT_SCHEMA_VERSION = "telemetry-export@1.1" SUPPORTED_LOCALES = frozenset({"en", "nl", "de", "fr", "es"}) SNAPSHOT_TTL_SECONDS = 600 -MAX_RETAINED_SNAPSHOTS = 32 -MAX_SNAPSHOT_BYTES = 32 * 1024 * 1024 +MAX_RETAINED_SNAPSHOTS = 16 +MAX_SNAPSHOT_BYTES = 16 * 1024 * 1024 +MAX_RETAINED_SNAPSHOT_BYTES = 64 * 1024 * 1024 class ExportSnapshotStore: """Bounded in-memory readback for already projected, privacy-safe models.""" - def __init__(self) -> None: + def __init__( + self, *, max_snapshots: int = MAX_RETAINED_SNAPSHOTS, + max_snapshot_bytes: int = MAX_SNAPSHOT_BYTES, + max_retained_bytes: int = MAX_RETAINED_SNAPSHOT_BYTES, + ) -> None: + if not 0 < max_snapshot_bytes <= max_retained_bytes or max_snapshots < 1: + raise ValueError("TELEMETRY_EXPORT_SNAPSHOT_LIMIT_INVALID") self._lock = RLock() - self._models: dict[str, tuple[float, str, dict[str, object]]] = {} + self._max_snapshots = max_snapshots + self._max_snapshot_bytes = max_snapshot_bytes + self._max_retained_bytes = max_retained_bytes + self._retained_bytes = 0 + self._models: dict[str, tuple[float, str, int, dict[str, object]]] = {} def retain(self, model: Mapping[str, object], *, binding: str) -> str: snapshot_id = model.get("snapshot_id") if not isinstance(snapshot_id, str) or not snapshot_id.startswith("sha256:"): raise ValueError("TELEMETRY_EXPORT_SNAPSHOT_INVALID") encoded = json.dumps(model, sort_keys=True, separators=(",", ":"), allow_nan=False).encode("utf-8") - if len(encoded) > MAX_SNAPSHOT_BYTES: + encoded_size = len(encoded) + if encoded_size > self._max_snapshot_bytes: raise ValueError("TELEMETRY_EXPORT_SNAPSHOT_TOO_LARGE") now = monotonic() with self._lock: self._expire(now) - if len(self._models) >= MAX_RETAINED_SNAPSHOTS and snapshot_id not in self._models: + previous = self._models.pop(snapshot_id, None) + if previous is not None: + self._retained_bytes -= previous[2] + while self._models and ( + len(self._models) >= self._max_snapshots + or self._retained_bytes + encoded_size > self._max_retained_bytes + ): oldest = min(self._models, key=lambda key: self._models[key][0]) - self._models.pop(oldest, None) - self._models[snapshot_id] = (now + SNAPSHOT_TTL_SECONDS, binding, deepcopy(dict(model))) + evicted = self._models.pop(oldest) + self._retained_bytes -= evicted[2] + if self._retained_bytes + encoded_size > self._max_retained_bytes: + raise ValueError("TELEMETRY_EXPORT_SNAPSHOT_TOO_LARGE") + self._models[snapshot_id] = ( + now + SNAPSHOT_TTL_SECONDS, binding, encoded_size, deepcopy(dict(model)), + ) + self._retained_bytes += encoded_size return snapshot_id def read(self, snapshot_id: str, *, binding: str) -> dict[str, object] | None: @@ -54,12 +78,13 @@ def read(self, snapshot_id: str, *, binding: str) -> dict[str, object] | None: retained = self._models.get(snapshot_id) if retained is None or retained[1] != binding: return None - return deepcopy(retained[2]) + return deepcopy(retained[3]) def _expire(self, now: float) -> None: - for key, (expires_at, _, _) in list(self._models.items()): + for key, (expires_at, _, encoded_size, _) in list(self._models.items()): if expires_at <= now: self._models.pop(key, None) + self._retained_bytes -= encoded_size def download_model(model: Mapping[str, object]) -> dict[str, object]: diff --git a/tests/engineering/test_server_foundation.py b/tests/engineering/test_server_foundation.py index 18ab2eb4..f94b70cc 100644 --- a/tests/engineering/test_server_foundation.py +++ b/tests/engineering/test_server_foundation.py @@ -74,6 +74,35 @@ def _system_inventory(self, selected: Path | None = None) -> dict[str, object]: "single_operational_installation_status": "UNVERIFIED_SCOPE", } + def test_full_telemetry_overview_pages_beyond_console_preview_limit(self) -> None: + server.initialize(self.root) + with sqlite_connection(self.root / server.SERVER_DATABASE_FILENAME) as connection: + connection.executemany( + "INSERT INTO ep_execution_runs(run_id,project_id,state,created_at,updated_at,execution_mode) " + "VALUES(?,?,?,?,?,'MANAGED')", + [ + ( + f"full-overview-{index:04d}", "full-overview", "COMPLETE", + "2026-09-17T10:00:00+00:00", "2026-09-17T10:00:10+00:00", + ) + for index in range(1001) + ], + ) + with patch("engineering_platform.server.timing_summaries", return_value={}), patch( + "engineering_platform.server.provider_usage_summaries", return_value={}, + ): + rows = server._central_console_telemetry( + self.root, "full-overview", full=True, + ) + self.assertEqual(len(rows), 1) + self.assertEqual(rows[0]["prompt_count"], 1001) + model = server.telemetry_export.overview_model( + project_id="full-overview", rows=rows, sort_key="date", + sort_direction="desc", locale="en", + ) + self.assertEqual(model["data"]["overview"]["summary"]["run_count"], 1001) + self.assertEqual(model["completeness"]["export"], "COMPLETE") + def _resolved_system_service(self, selected: Path) -> tuple[object, dict[str, object], None]: return ( server.system_server_service.SystemServerService( @@ -2727,3 +2756,14 @@ def stop_httpd() -> None: "TELEMETRY_EXPORT_SNAPSHOT_UNAVAILABLE", ) mismatch.exception.close() + httpd.telemetry_export_snapshots = server.telemetry_export.ExportSnapshotStore( + max_snapshots=1, max_snapshot_bytes=1, max_retained_bytes=1, + ) + with self.assertRaises(HTTPError) as oversized: + urlopen(overview_selection.replace("locale=nl", "locale=es") + "&prepare=1") + self.assertEqual(oversized.exception.code, 413) + self.assertEqual( + json.loads(oversized.exception.read())["error"], + "TELEMETRY_EXPORT_SNAPSHOT_TOO_LARGE", + ) + oversized.exception.close() diff --git a/tests/engineering/test_telemetry_contract_v2.py b/tests/engineering/test_telemetry_contract_v2.py index 4a79ee36..3a0283af 100644 --- a/tests/engineering/test_telemetry_contract_v2.py +++ b/tests/engineering/test_telemetry_contract_v2.py @@ -3,6 +3,7 @@ from datetime import datetime, timedelta, timezone import json from pathlib import Path +import sqlite3 from tempfile import TemporaryDirectory import unittest from unittest.mock import patch @@ -302,6 +303,35 @@ def structured(count: int, *, offset: int = 0) -> dict[str, object]: self.assertEqual(legacy["historical_unique_pr_results_lower_bound"], 4) self.assertEqual(legacy["historical_unique_pr_coverage"], "PARTIAL") + def test_inconsistent_complete_pr_identity_metadata_fails_closed(self) -> None: + persist_provider_invocation(self.root, ProviderInvocation( + "conflicting-pr-identities", 1, "codex_cli", None, + "PROVIDER_EXECUTION", "IMPLEMENTATION", + "2026-09-17T00:00:00+00:00", "2026-09-17T00:00:01+00:00", 1000, + {}, invocation_id="conflicting-pr-identity", churn={ + "historical_pr_queries": 1, + "historical_unique_pr_results": 2, + "historical_pr_identity_hashes": ["a" * 64], + "historical_pr_identity_set_complete": True, + "historical_pr_identity_set_truncated": False, + "historical_pr_identity_retained_count": 1, + "historical_pr_metrics_coverage": "COMPLETE", + "historical_pr_identity_coverage": "COMPLETE", + }, + )) + with open_storage(self.root) as connection: + stored = json.loads(connection.execute( + "SELECT churn FROM provider_invocations WHERE invocation_id=?", + ("conflicting-pr-identity",), + ).fetchone()[0]) + self.assertFalse(stored["historical_pr_identity_set_complete"]) + self.assertEqual(stored["historical_pr_identity_coverage"], "CONFLICT") + self.assertEqual(stored["historical_pr_unique_lower_bound"], 2) + summary = provider_usage_summary(self.root, "conflicting-pr-identities") + self.assertIsNone(summary["historical_unique_pr_results"]) + self.assertEqual(summary["historical_unique_pr_results_lower_bound"], 2) + self.assertEqual(summary["historical_unique_pr_coverage"], "CONFLICT") + def test_churn_maximum_and_partial_coverage_keep_their_metric_semantics(self) -> None: for ordinal, coverage, maximum in ((1, "PARTIAL", 900), (2, "COMPLETE", 400)): persist_provider_invocation(self.root, ProviderInvocation( @@ -730,25 +760,34 @@ def test_chain_export_contains_full_details_for_attempts_on_different_days(self) def test_export_read_transaction_never_mixes_writer_commits_between_loaders(self) -> None: self._run("snapshot-run", "sub-snapshot", "2026-09-17T10:00:00+00:00", "2026-09-17T10:00:10+00:00") - with sqlite_connection(self.database) as connection: - connection.execute("PRAGMA journal_mode=WAL") - with server._telemetry_read_snapshot(self.root) as (read_connection, source_as_of, source_reference): - before = server._central_console_telemetry_detail( - self.root, "forge", "2026-09-17", full=True, - _read_connection=read_connection, - ) - persist_provider_invocation(self.root, ProviderInvocation( - "snapshot-run", 1, "codex_cli", None, "PROVIDER_EXECUTION", "IMPLEMENTATION", - "2026-09-17T10:00:00+00:00", "2026-09-17T10:00:01+00:00", 1000, - {"input_tokens": 100, "cached_input_tokens": 80, "output_tokens": 10}, - invocation_id="snapshot-invocation", - ), central_database=self.database) - during = server._central_console_telemetry_detail( - self.root, "forge", "2026-09-17", full=True, - _read_connection=read_connection, - ) - self.assertTrue(source_as_of.endswith("+00:00")) - self.assertIn("central-schema:", source_reference) + # Keep one WAL-capable connection alive. SQLite on macOS cannot open a + # WAL database read-only after the final WAL owner has removed -shm, + # whereas the installed CENTRAL writer remains alive in production. + wal_keeper = sqlite3.connect(self.database) + try: + wal_keeper.execute("PRAGMA journal_mode=WAL") + wal_keeper.execute("SELECT COUNT(*) FROM engineering_schema_migrations").fetchone() + with server._telemetry_read_snapshot(self.root) as ( + read_connection, source_as_of, source_reference, + ): + before = server._central_console_telemetry_detail( + self.root, "forge", "2026-09-17", full=True, + _read_connection=read_connection, + ) + persist_provider_invocation(self.root, ProviderInvocation( + "snapshot-run", 1, "codex_cli", None, "PROVIDER_EXECUTION", "IMPLEMENTATION", + "2026-09-17T10:00:00+00:00", "2026-09-17T10:00:01+00:00", 1000, + {"input_tokens": 100, "cached_input_tokens": 80, "output_tokens": 10}, + invocation_id="snapshot-invocation", + ), central_database=self.database) + during = server._central_console_telemetry_detail( + self.root, "forge", "2026-09-17", full=True, + _read_connection=read_connection, + ) + self.assertTrue(source_as_of.endswith("+00:00")) + self.assertIn("central-schema:", source_reference) + finally: + wal_keeper.close() after = server._central_console_telemetry_detail( self.root, "forge", "2026-09-17", full=True, ) @@ -819,6 +858,32 @@ def test_retained_export_snapshot_is_bound_immutable_and_expires_explicitly(self with patch("engineering_platform.telemetry_export.monotonic", return_value=701.0): self.assertIsNone(store.read(snapshot_id, binding="project=forge")) + def test_retained_export_snapshots_enforce_item_and_process_byte_budgets(self) -> None: + first = overview_model( + project_id="first", rows=[], sort_key="date", sort_direction="desc", locale="en", + ) + second = overview_model( + project_id="second", rows=[], sort_key="date", sort_direction="desc", locale="en", + ) + encoded_sizes = [ + len(json.dumps(model, sort_keys=True, separators=(",", ":"), allow_nan=False).encode()) + for model in (first, second) + ] + store = ExportSnapshotStore( + max_snapshots=8, + max_snapshot_bytes=max(encoded_sizes), + max_retained_bytes=max(encoded_sizes) + 1, + ) + first_id = store.retain(first, binding="first") + second_id = store.retain(second, binding="second") + self.assertIsNone(store.read(first_id, binding="first")) + self.assertEqual(store.read(second_id, binding="second"), second) + too_small = ExportSnapshotStore( + max_snapshots=1, max_snapshot_bytes=1, max_retained_bytes=1, + ) + with self.assertRaisesRegex(ValueError, "TELEMETRY_EXPORT_SNAPSHOT_TOO_LARGE"): + too_small.retain(first, binding="first") + def test_equal_observation_counts_do_not_upgrade_partial_or_conflict(self) -> None: partial = aggregate_coverage([{ "coverage": "PARTIAL", "expected_observations": 1, From 33f8f6b72ed275210a1f7dda8f1710c1b07a8870 Mon Sep 17 00:00:00 2001 From: pcvantol Date: Thu, 17 Sep 2026 15:19:47 +0200 Subject: [PATCH 5/6] fix: page complete telemetry evidence --- docs/engineering/PROVIDER_USAGE_SEMANTICS.md | 8 +- src/engineering_platform/execution_timing.py | 40 +++++--- src/engineering_platform/provider_usage.py | 97 ++++++++++++------- tests/engineering/test_server_foundation.py | 32 ++++-- .../engineering/test_telemetry_contract_v2.py | 10 ++ 5 files changed, 130 insertions(+), 57 deletions(-) diff --git a/docs/engineering/PROVIDER_USAGE_SEMANTICS.md b/docs/engineering/PROVIDER_USAGE_SEMANTICS.md index 80b495aa..e06d1383 100644 --- a/docs/engineering/PROVIDER_USAGE_SEMANTICS.md +++ b/docs/engineering/PROVIDER_USAGE_SEMANTICS.md @@ -106,7 +106,9 @@ Legacy rows without an explicit set-completeness marker are never promoted to an exact complete union. A modern record that claims completeness while its declared unique count, retained count, opaque identities, truncation marker or coverage disagree fails closed as `CONFLICT`; its safe lower bound may remain -visible, but it is not projected as an exact unique total. +visible, but it is not projected as an exact unique total. The same invariant +is checked again during read projection, so a pre-existing corrupt or +inconsistent stored row cannot bypass the owning writer's validation. Read-command counters are derived command observations. Exact file-read observations require reliable tool metadata for opaque file identity, revision @@ -170,7 +172,9 @@ being truncated or retained without a bound. The overview export reads the entire active project/filter population through bounded 500-record database pages inside the same read transaction; the 1,000-run Console preview limit and 360-day UI limit are not export limits. -It therefore contains every retained row rather than only the visible page. +Usage and timing reducers page that identical run population in 500-identifier +batches as well. It therefore contains every retained row and its available +measurements rather than only the visible page. Detail export can select the UTC day, one attempt, or its verified execution chain. Full exports disable the UI preview limits for runs, invocations and spans. A chain detail includes a safe diff --git a/src/engineering_platform/execution_timing.py b/src/engineering_platform/execution_timing.py index d9dba206..4f3b5170 100644 --- a/src/engineering_platform/execution_timing.py +++ b/src/engineering_platform/execution_timing.py @@ -685,30 +685,42 @@ def timing_summaries( timeline_limit: int | None = 500, _read_connection: sqlite3.Connection | None = None, ) -> dict[str, dict[str, object]]: - """Load a bounded run population without one timing query per row.""" - identifiers = list(dict.fromkeys(value for value in run_ids if isinstance(value, str) and value))[:1000] + """Load a run population in bounded query pages without an N+1 query.""" + identifiers = list(dict.fromkeys( + value for value in run_ids if isinstance(value, str) and value + )) if not identifiers: return {} owns_connection = _read_connection is None connection = _read_connection if _read_connection is not None else _connection(root, central_database) - placeholders = ",".join("?" for _ in identifiers) keys = ( "phase_id", "run_id", "phase_name", "phase_category", "parent_phase_id", "attempt", "ordinal", "started_at", "completed_at", "duration_ms", "outcome", "metadata", ) + rows: list[tuple[object, ...]] = [] + historical_rows: list[tuple[object, ...]] = [] + started_read_transaction = owns_connection and not connection.in_transaction try: - rows = connection.execute( - f"""SELECT phase_id,run_id,phase_name,phase_category,parent_phase_id, - attempt,ordinal,started_at,completed_at,duration_ms,outcome,metadata - FROM execution_phase_spans WHERE run_id IN ({placeholders}) - ORDER BY run_id,ordinal""", - identifiers, - ).fetchall() - historical_rows = connection.execute( - f"SELECT run_id,total_execution_seconds FROM execution_runs WHERE run_id IN ({placeholders})", - identifiers, - ).fetchall() + if started_read_transaction: + connection.execute("BEGIN") + for offset in range(0, len(identifiers), 500): + batch = identifiers[offset:offset + 500] + placeholders = ",".join("?" for _ in batch) + rows.extend(connection.execute( + f"""SELECT phase_id,run_id,phase_name,phase_category,parent_phase_id, + attempt,ordinal,started_at,completed_at,duration_ms,outcome,metadata + FROM execution_phase_spans WHERE run_id IN ({placeholders}) + ORDER BY run_id,ordinal""", + batch, + ).fetchall()) + historical_rows.extend(connection.execute( + f"""SELECT run_id,total_execution_seconds FROM execution_runs + WHERE run_id IN ({placeholders})""", + batch, + ).fetchall()) finally: + if started_read_transaction and connection.in_transaction: + connection.rollback() if owns_connection: connection.close() spans_by_run: dict[str, list[dict[str, object]]] = {run_id: [] for run_id in identifiers} diff --git a/src/engineering_platform/provider_usage.py b/src/engineering_platform/provider_usage.py index 2808dff9..9c862b89 100644 --- a/src/engineering_platform/provider_usage.py +++ b/src/engineering_platform/provider_usage.py @@ -74,6 +74,29 @@ def _number(value: object) -> int | None: return value if isinstance(value, int) and not isinstance(value, bool) and value >= 0 else None +def _pr_identity_metadata_conflicts(values: Mapping[str, object]) -> bool: + """Reject an exactness claim unless every bounded identity invariant agrees.""" + if values.get("historical_pr_identity_set_complete") is not True: + return False + raw_identities = values.get("historical_pr_identity_hashes") + if not isinstance(raw_identities, (list, tuple)): + return True + identities = { + item.casefold() for item in raw_identities + if isinstance(item, str) and re.fullmatch(r"[0-9a-f]{64}", item.casefold()) + } + unique_count = _number(values.get("historical_unique_pr_results")) + retained_count = _number(values.get("historical_pr_identity_retained_count")) + return ( + values.get("historical_pr_identity_coverage") != COMPLETE + or values.get("historical_pr_identity_set_truncated") is not False + or unique_count is None + or unique_count != len(identities) + or retained_count != len(identities) + or len(raw_identities) != len(identities) + ) + + def speed_state(metadata: Mapping[str, object] | None) -> str: """Return only a runtime-observed speed state; UI preferences are irrelevant.""" known_fields = { @@ -549,23 +572,11 @@ def persist_provider_invocation(root: Path, invocation: ProviderInvocation, *, c declared_unique_count = _number( (invocation.churn or {}).get("historical_unique_pr_results") ) - declared_retained_count = _number( - (invocation.churn or {}).get("historical_pr_identity_retained_count") - ) - claims_complete = churn.get("historical_pr_identity_set_complete") is True - identity_metadata_conflict = claims_complete and ( - churn.get("historical_pr_identity_coverage") != COMPLETE - or churn.get("historical_pr_identity_set_truncated") is True - or declared_unique_count is None - or declared_unique_count != supplied_identity_count - or retained_identity_count != supplied_identity_count - or declared_retained_count not in {None, retained_identity_count} - ) churn["historical_pr_identity_retained_count"] = retained_identity_count churn["historical_pr_unique_lower_bound"] = max( retained_identity_count, declared_unique_count or 0, ) - if identity_metadata_conflict: + if _pr_identity_metadata_conflicts(churn): churn["historical_pr_identity_set_complete"] = False churn["historical_pr_identity_coverage"] = CONFLICT snapshots = tuple( @@ -732,6 +743,10 @@ def provider_usage_summary( except (TypeError, json.JSONDecodeError): values = {} if isinstance(values, dict): + values = dict(values) + if _pr_identity_metadata_conflicts(values): + values["historical_pr_identity_set_complete"] = False + values["historical_pr_identity_coverage"] = CONFLICT query_count = _number(values.get("historical_pr_queries")) or 0 if query_count: retained = { @@ -1062,8 +1077,10 @@ def provider_usage_summaries( invocation_limit: int | None = 250, _read_connection: sqlite3.Connection | None = None, ) -> dict[str, dict[str, object]]: - """Load a bounded run population in two queries, then reuse the canonical reducer.""" - identifiers = list(dict.fromkeys(value for value in run_ids if isinstance(value, str) and value))[:1000] + """Load a run population in bounded query pages, then reuse the canonical reducer.""" + identifiers = list(dict.fromkeys( + value for value in run_ids if isinstance(value, str) and value + )) if not identifiers: return {} owns_connection = _read_connection is None @@ -1078,28 +1095,38 @@ def provider_usage_summaries( connection = sqlite3.connect(database, isolation_level=None) connection.execute("PRAGMA foreign_keys=ON") connection.row_factory = sqlite3.Row - placeholders = ",".join("?" for _ in identifiers) + rows: list[sqlite3.Row] = [] + snapshot_rows: list[sqlite3.Row] = [] + started_read_transaction = owns_connection and not connection.in_transaction try: - rows = connection.execute( - f"""SELECT invocation_id,run_id,ordinal,provider,model,model_authority, - raw_provider_model,phase,role,started_at,completed_at,duration_ms, - input_tokens,cached_input_tokens,uncached_input_tokens,output_tokens, - reasoning_tokens,total_tokens,estimated_credits,estimated_eur, - speed_state,usage_authority,churn,retry_ordinal - FROM provider_invocations WHERE run_id IN ({placeholders}) - ORDER BY run_id,ordinal""", - identifiers, - ).fetchall() - snapshot_rows = connection.execute( - f"""SELECT p.run_id,s.invocation_id,s.ordinal,s.input_tokens,s.cached_input_tokens, - s.uncached_input_tokens,s.output_tokens,s.input_delta,s.cached_input_delta, - s.uncached_input_delta,s.output_delta - FROM provider_usage_snapshots AS s - JOIN provider_invocations AS p ON p.invocation_id=s.invocation_id - WHERE p.run_id IN ({placeholders}) ORDER BY p.run_id,s.invocation_id,s.ordinal""", - identifiers, - ).fetchall() + if started_read_transaction: + connection.execute("BEGIN") + for offset in range(0, len(identifiers), 500): + batch = identifiers[offset:offset + 500] + placeholders = ",".join("?" for _ in batch) + rows.extend(connection.execute( + f"""SELECT invocation_id,run_id,ordinal,provider,model,model_authority, + raw_provider_model,phase,role,started_at,completed_at,duration_ms, + input_tokens,cached_input_tokens,uncached_input_tokens,output_tokens, + reasoning_tokens,total_tokens,estimated_credits,estimated_eur, + speed_state,usage_authority,churn,retry_ordinal + FROM provider_invocations WHERE run_id IN ({placeholders}) + ORDER BY run_id,ordinal""", + batch, + ).fetchall()) + snapshot_rows.extend(connection.execute( + f"""SELECT p.run_id,s.invocation_id,s.ordinal,s.input_tokens,s.cached_input_tokens, + s.uncached_input_tokens,s.output_tokens,s.input_delta,s.cached_input_delta, + s.uncached_input_delta,s.output_delta + FROM provider_usage_snapshots AS s + JOIN provider_invocations AS p ON p.invocation_id=s.invocation_id + WHERE p.run_id IN ({placeholders}) + ORDER BY p.run_id,s.invocation_id,s.ordinal""", + batch, + ).fetchall()) finally: + if started_read_transaction and connection.in_transaction: + connection.rollback() if owns_connection: connection.close() rows_by_run: dict[str, list[sqlite3.Row]] = {run_id: [] for run_id in identifiers} diff --git a/tests/engineering/test_server_foundation.py b/tests/engineering/test_server_foundation.py index f94b70cc..13c45ec8 100644 --- a/tests/engineering/test_server_foundation.py +++ b/tests/engineering/test_server_foundation.py @@ -88,14 +88,34 @@ def test_full_telemetry_overview_pages_beyond_console_preview_limit(self) -> Non for index in range(1001) ], ) - with patch("engineering_platform.server.timing_summaries", return_value={}), patch( - "engineering_platform.server.provider_usage_summaries", return_value={}, - ): - rows = server._central_console_telemetry( - self.root, "full-overview", full=True, - ) + measured_run = "full-overview-0000" + started = datetime(2026, 9, 17, 10, tzinfo=timezone.utc) + total = start_phase( + self.root, measured_run, "TOTAL_EXECUTION", started_at=started, + monotonic_clock=0, central_database=self.root / server.SERVER_DATABASE_FILENAME, + ) + complete_phase( + self.root, total, completed_at=started + timedelta(seconds=1), + monotonic_clock=1, + ) + persist_provider_invocation( + self.root, + ProviderInvocation( + measured_run, 1, "codex_cli", "observed-model", "PROVIDER_EXECUTION", + "IMPLEMENTATION", started.isoformat(), + (started + timedelta(seconds=1)).isoformat(), 1000, + {"input_tokens": 100, "cached_input_tokens": 80, "output_tokens": 10}, + invocation_id="full-overview-final-page-invocation", + ), + central_database=self.root / server.SERVER_DATABASE_FILENAME, + ) + rows = server._central_console_telemetry( + self.root, "full-overview", full=True, + ) self.assertEqual(len(rows), 1) self.assertEqual(rows[0]["prompt_count"], 1001) + self.assertEqual(rows[0]["usage_metrics"]["input_tokens"]["value"], 100) + self.assertEqual(rows[0]["average_total_execution_seconds"], 1.0) model = server.telemetry_export.overview_model( project_id="full-overview", rows=rows, sort_key="date", sort_direction="desc", locale="en", diff --git a/tests/engineering/test_telemetry_contract_v2.py b/tests/engineering/test_telemetry_contract_v2.py index 3a0283af..bb7ee8cd 100644 --- a/tests/engineering/test_telemetry_contract_v2.py +++ b/tests/engineering/test_telemetry_contract_v2.py @@ -327,6 +327,16 @@ def test_inconsistent_complete_pr_identity_metadata_fails_closed(self) -> None: self.assertFalse(stored["historical_pr_identity_set_complete"]) self.assertEqual(stored["historical_pr_identity_coverage"], "CONFLICT") self.assertEqual(stored["historical_pr_unique_lower_bound"], 2) + corrupt_stored = { + **stored, + "historical_pr_identity_set_complete": True, + "historical_pr_identity_coverage": "COMPLETE", + } + with open_storage(self.root) as connection: + connection.execute( + "UPDATE provider_invocations SET churn=? WHERE invocation_id=?", + (json.dumps(corrupt_stored, sort_keys=True), "conflicting-pr-identity"), + ) summary = provider_usage_summary(self.root, "conflicting-pr-identities") self.assertIsNone(summary["historical_unique_pr_results"]) self.assertEqual(summary["historical_unique_pr_results_lower_bound"], 2) From 5231e4a44106456a0bd695922c5921c70d0f3c0a Mon Sep 17 00:00:00 2001 From: pcvantol Date: Thu, 17 Sep 2026 15:26:01 +0200 Subject: [PATCH 6/6] fix: enforce bounded identity readback --- docs/engineering/PROVIDER_USAGE_SEMANTICS.md | 3 ++- src/engineering_platform/provider_usage.py | 2 ++ .../engineering/test_telemetry_contract_v2.py | 27 +++++++++++++++++++ 3 files changed, 31 insertions(+), 1 deletion(-) diff --git a/docs/engineering/PROVIDER_USAGE_SEMANTICS.md b/docs/engineering/PROVIDER_USAGE_SEMANTICS.md index e06d1383..687bab32 100644 --- a/docs/engineering/PROVIDER_USAGE_SEMANTICS.md +++ b/docs/engineering/PROVIDER_USAGE_SEMANTICS.md @@ -108,7 +108,8 @@ declared unique count, retained count, opaque identities, truncation marker or coverage disagree fails closed as `CONFLICT`; its safe lower bound may remain visible, but it is not projected as an exact unique total. The same invariant is checked again during read projection, so a pre-existing corrupt or -inconsistent stored row cannot bypass the owning writer's validation. +inconsistent stored row cannot bypass the owning writer's validation. Readback +also enforces the 250-identity storage bound and canonical lowercase hash form. Read-command counters are derived command observations. Exact file-read observations require reliable tool metadata for opaque file identity, revision diff --git a/src/engineering_platform/provider_usage.py b/src/engineering_platform/provider_usage.py index 9c862b89..14440c3a 100644 --- a/src/engineering_platform/provider_usage.py +++ b/src/engineering_platform/provider_usage.py @@ -91,9 +91,11 @@ def _pr_identity_metadata_conflicts(values: Mapping[str, object]) -> bool: values.get("historical_pr_identity_coverage") != COMPLETE or values.get("historical_pr_identity_set_truncated") is not False or unique_count is None + or len(identities) > _MAX_PR_IDENTITY_HASHES or unique_count != len(identities) or retained_count != len(identities) or len(raw_identities) != len(identities) + or any(item != item.casefold() for item in raw_identities if isinstance(item, str)) ) diff --git a/tests/engineering/test_telemetry_contract_v2.py b/tests/engineering/test_telemetry_contract_v2.py index bb7ee8cd..e26a0962 100644 --- a/tests/engineering/test_telemetry_contract_v2.py +++ b/tests/engineering/test_telemetry_contract_v2.py @@ -342,6 +342,33 @@ def test_inconsistent_complete_pr_identity_metadata_fails_closed(self) -> None: self.assertEqual(summary["historical_unique_pr_results_lower_bound"], 2) self.assertEqual(summary["historical_unique_pr_coverage"], "CONFLICT") + for identities in ( + [f"{index:064x}" for index in range(251)], + ["A" * 64], + ): + corrupt_stored = { + **stored, + "historical_unique_pr_results": len(identities), + "historical_pr_identity_hashes": identities, + "historical_pr_identity_set_complete": True, + "historical_pr_identity_set_truncated": False, + "historical_pr_identity_retained_count": len(identities), + "historical_pr_unique_lower_bound": len(identities), + "historical_pr_identity_coverage": "COMPLETE", + } + with open_storage(self.root) as connection: + connection.execute( + "UPDATE provider_invocations SET churn=? WHERE invocation_id=?", + (json.dumps(corrupt_stored, sort_keys=True), "conflicting-pr-identity"), + ) + summary = provider_usage_summary(self.root, "conflicting-pr-identities") + self.assertIsNone(summary["historical_unique_pr_results"]) + self.assertEqual( + summary["historical_unique_pr_results_lower_bound"], len(identities), + ) + self.assertFalse(summary["historical_pr_identity_set_complete"]) + self.assertEqual(summary["historical_unique_pr_coverage"], "CONFLICT") + def test_churn_maximum_and_partial_coverage_keep_their_metric_semantics(self) -> None: for ordinal, coverage, maximum in ((1, "PARTIAL", 900), (2, "COMPLETE", 400)): persist_provider_invocation(self.root, ProviderInvocation(