From c6c72cb94551f5537b1e9d2ea6c4d646da48b902 Mon Sep 17 00:00:00 2001 From: Pavilion-devs Date: Tue, 11 Aug 2026 02:01:48 +0100 Subject: [PATCH 1/2] feat: ship the GlassBox RC1 evidence plane --- .dockerignore | 15 + .env.example | 23 + .gitignore | 7 + README.md | 826 +- apps/console/.nvmrc | 1 + apps/console/README.md | 91 +- .../(console)/campaigns/[campaignId]/page.tsx | 23 + apps/console/app/(console)/campaigns/page.tsx | 24 + .../investigations/[receiptId]/page.tsx | 112 + .../app/(console)/investigations/page.tsx | 47 + apps/console/app/(console)/layout.tsx | 28 + apps/console/app/(console)/page.tsx | 58 + apps/console/app/(console)/receipts/page.tsx | 22 + apps/console/app/(console)/recovery/page.tsx | 20 + .../(console)/settings/connection-center.tsx | 149 + apps/console/app/(console)/settings/page.tsx | 14 + apps/console/app/(console)/trust/page.tsx | 15 + .../app/api/control/[...path]/route.ts | 13 + apps/console/app/architecture/page.tsx | 5 + apps/console/app/components/console-shell.tsx | 208 + .../app/components/identifier-control.tsx | 43 + apps/console/app/components/ui.tsx | 214 + apps/console/app/docs/architecture/layout.tsx | 11 + apps/console/app/docs/architecture/page.mdx | 97 + .../app/docs/assessment/completeness/page.mdx | 97 + .../docs/assessment/deterministic/page.mdx | 97 + apps/console/app/docs/cli/page.mdx | 118 + apps/console/app/docs/how-it-works/page.mdx | 93 + .../app/docs/integrations/datahub/page.mdx | 106 + .../app/docs/integrations/mcp/page.mdx | 86 + .../app/docs/integrations/transport/page.mdx | 114 + .../app/docs/invalidation-loop/page.mdx | 110 + apps/console/app/docs/layout.tsx | 29 + apps/console/app/docs/page.mdx | 137 + apps/console/app/docs/policy/page.mdx | 90 + apps/console/app/docs/quickstart/layout.tsx | 11 + apps/console/app/docs/quickstart/page.mdx | 145 + apps/console/app/docs/receipts/page.mdx | 109 + apps/console/app/docs/recovery/page.mdx | 112 + apps/console/app/docs/schemas/page.mdx | 87 + apps/console/app/docs/setup/page.mdx | 123 + apps/console/app/globals.css | 1621 +-- apps/console/app/home/page.tsx | 291 + apps/console/app/layout.tsx | 30 +- apps/console/app/lib/control-api.ts | 344 + apps/console/app/lib/forensics-api.ts | 366 + apps/console/app/page.tsx | 426 - .../components/docs/ArchitectureDiagram.tsx | 48 + apps/console/components/docs/Callout.tsx | 54 + apps/console/components/docs/Card.tsx | 60 + apps/console/components/docs/CodeBlock.tsx | 77 + apps/console/components/docs/CodeGroup.tsx | 64 + apps/console/components/docs/DocsSidebar.tsx | 41 + apps/console/components/docs/DocsTopbar.tsx | 68 + apps/console/components/docs/GlassBoxMark.tsx | 33 + apps/console/components/docs/Icon.tsx | 26 + apps/console/components/docs/PrevNext.tsx | 45 + .../console/components/docs/SearchPalette.tsx | 128 + apps/console/components/docs/Steps.tsx | 35 + apps/console/components/docs/ThemeToggle.tsx | 24 + apps/console/components/docs/Toc.tsx | 72 + apps/console/components/docs/solar-icons.ts | 46 + apps/console/components/site/CopyCommand.tsx | 52 + apps/console/lib/docs-nav.ts | 136 + apps/console/lib/remark-code-meta.ts | 27 + apps/console/lib/sites-vite-plugin.ts | 45 + apps/console/lib/theme.ts | 39 + apps/console/mdx-components.tsx | 95 + apps/console/next.config.ts | 32 +- apps/console/package-lock.json | 10486 ++++++++++------ apps/console/package.json | 24 +- apps/console/proxy.ts | 69 + apps/console/public/fonts/plex-mono-400.woff2 | Bin 0 -> 10052 bytes apps/console/public/fonts/plex-mono-600.woff2 | Bin 0 -> 10120 bytes apps/console/public/fonts/plex-sans-var.woff2 | Bin 0 -> 40240 bytes .../media/glassbox-architecture-diagram.svg | 452 + apps/console/tests/rendered-html.test.mjs | 129 +- apps/console/tsconfig.json | 1 + apps/console/vite.config.ts | 51 +- benchmarks/__init__.py | 1 + benchmarks/agent_invalidation/__init__.py | 5 + benchmarks/agent_invalidation/__main__.py | 5 + .../agent_invalidation/fixtures.v1.json | 447 + benchmarks/agent_invalidation/runner.py | 1019 ++ deploy/production/.env.production.example | 33 + deploy/production/Caddyfile | 96 + deploy/production/Dockerfile.console | 25 + deploy/production/Dockerfile.python | 30 + deploy/production/README.md | 120 + deploy/production/compose.yml | 237 + ...d-flagship-estate-and-evidence-ablation.md | 128 + docs/adr/0026-loopback-console-read-model.md | 58 + ...ontrol-plane-and-self-hosted-deployment.md | 135 + docs/adr/README.md | 7 + .../glassbox-architecture-diagram.png | Bin 0 -> 954766 bytes .../glassbox-architecture-diagram.svg | 452 + docs/assets/glassbox-readme-banner.png | Bin 0 -> 580157 bytes docs/assets/glassbox-readme-banner.svg | 110 + docs/benchmarks/README.md | 82 + .../agent-invalidation-ablation.v1.json | 1231 ++ ...hub-1.6.0-hosted-production-otlp.live.json | 108 + ...tahub-1.6.0-one-command-flagship.live.json | 397 + docs/replay.md | 24 +- examples/end_to_end_flagship.py | 16 +- examples/flagship-estate.compose.yml | 88 + examples/flagship_demo.py | 572 + plan.md | 137 +- pyproject.toml | 13 + release-evidence/SHA256SUMS | 4 +- release-evidence/glassbox_core-0.1.0.cdx.json | 4 +- release-evidence/release-report.json | 12 +- .../upstream/upstream-packet.json | 8 +- schemas/benchmark-report/0.1.0/schema.json | 419 + scripts/forensics_mcp_capture.py | 242 + scripts/guarded_schema_change.py | 316 + scripts/remote_forensics_mcp_capture.py | 46 + scripts/remote_guarded_schema_change.py | 57 + scripts/remote_otlp_capture.py | 402 + .../src/glassbox_compiler/receiver.py | 123 +- .../src/glassbox_control/__init__.py | 6 + .../src/glassbox_control/crypto.py | 93 + .../src/glassbox_control/datahub.py | 148 + .../src/glassbox_control/server.py | 425 + .../src/glassbox_control/store.py | 412 + .../src/glassbox_forensics/__init__.py | 4 + .../src/glassbox_forensics/live_state.py | 45 +- .../src/glassbox_forensics/server.py | 205 +- .../src/glassbox_forensics/service.py | 264 + .../glassbox_invalidation/postgres_store.py | 8 +- .../glassbox_invalidation/receipt_store.py | 8 +- .../transactional_store.py | 8 +- .../integration/test_postgres_invalidation.py | 29 +- .../unit/test_agent_invalidation_benchmark.py | 101 + tests/unit/test_control_plane.py | 271 + tests/unit/test_flagship_demo.py | 259 + tests/unit/test_forensics_mcp.py | 66 +- tests/unit/test_forensics_mcp_capture.py | 56 + tests/unit/test_forensics_service.py | 83 +- tests/unit/test_guarded_schema_change.py | 117 + tests/unit/test_production_edge.py | 68 + tests/unit/test_receiver_entrypoint.py | 66 + tests/unit/test_remote_otlp_capture.py | 196 + tests/unit/test_trusted_receipt_state.py | 29 +- uv.lock | 8 +- 144 files changed, 23712 insertions(+), 5739 deletions(-) create mode 100644 .dockerignore create mode 100644 apps/console/.nvmrc create mode 100644 apps/console/app/(console)/campaigns/[campaignId]/page.tsx create mode 100644 apps/console/app/(console)/campaigns/page.tsx create mode 100644 apps/console/app/(console)/investigations/[receiptId]/page.tsx create mode 100644 apps/console/app/(console)/investigations/page.tsx create mode 100644 apps/console/app/(console)/layout.tsx create mode 100644 apps/console/app/(console)/page.tsx create mode 100644 apps/console/app/(console)/receipts/page.tsx create mode 100644 apps/console/app/(console)/recovery/page.tsx create mode 100644 apps/console/app/(console)/settings/connection-center.tsx create mode 100644 apps/console/app/(console)/settings/page.tsx create mode 100644 apps/console/app/(console)/trust/page.tsx create mode 100644 apps/console/app/api/control/[...path]/route.ts create mode 100644 apps/console/app/architecture/page.tsx create mode 100644 apps/console/app/components/console-shell.tsx create mode 100644 apps/console/app/components/identifier-control.tsx create mode 100644 apps/console/app/components/ui.tsx create mode 100644 apps/console/app/docs/architecture/layout.tsx create mode 100644 apps/console/app/docs/architecture/page.mdx create mode 100644 apps/console/app/docs/assessment/completeness/page.mdx create mode 100644 apps/console/app/docs/assessment/deterministic/page.mdx create mode 100644 apps/console/app/docs/cli/page.mdx create mode 100644 apps/console/app/docs/how-it-works/page.mdx create mode 100644 apps/console/app/docs/integrations/datahub/page.mdx create mode 100644 apps/console/app/docs/integrations/mcp/page.mdx create mode 100644 apps/console/app/docs/integrations/transport/page.mdx create mode 100644 apps/console/app/docs/invalidation-loop/page.mdx create mode 100644 apps/console/app/docs/layout.tsx create mode 100644 apps/console/app/docs/page.mdx create mode 100644 apps/console/app/docs/policy/page.mdx create mode 100644 apps/console/app/docs/quickstart/layout.tsx create mode 100644 apps/console/app/docs/quickstart/page.mdx create mode 100644 apps/console/app/docs/receipts/page.mdx create mode 100644 apps/console/app/docs/recovery/page.mdx create mode 100644 apps/console/app/docs/schemas/page.mdx create mode 100644 apps/console/app/docs/setup/page.mdx create mode 100644 apps/console/app/home/page.tsx create mode 100644 apps/console/app/lib/control-api.ts create mode 100644 apps/console/app/lib/forensics-api.ts delete mode 100644 apps/console/app/page.tsx create mode 100644 apps/console/components/docs/ArchitectureDiagram.tsx create mode 100644 apps/console/components/docs/Callout.tsx create mode 100644 apps/console/components/docs/Card.tsx create mode 100644 apps/console/components/docs/CodeBlock.tsx create mode 100644 apps/console/components/docs/CodeGroup.tsx create mode 100644 apps/console/components/docs/DocsSidebar.tsx create mode 100644 apps/console/components/docs/DocsTopbar.tsx create mode 100644 apps/console/components/docs/GlassBoxMark.tsx create mode 100644 apps/console/components/docs/Icon.tsx create mode 100644 apps/console/components/docs/PrevNext.tsx create mode 100644 apps/console/components/docs/SearchPalette.tsx create mode 100644 apps/console/components/docs/Steps.tsx create mode 100644 apps/console/components/docs/ThemeToggle.tsx create mode 100644 apps/console/components/docs/Toc.tsx create mode 100644 apps/console/components/docs/solar-icons.ts create mode 100644 apps/console/components/site/CopyCommand.tsx create mode 100644 apps/console/lib/docs-nav.ts create mode 100644 apps/console/lib/remark-code-meta.ts create mode 100644 apps/console/lib/sites-vite-plugin.ts create mode 100644 apps/console/lib/theme.ts create mode 100644 apps/console/mdx-components.tsx create mode 100644 apps/console/proxy.ts create mode 100644 apps/console/public/fonts/plex-mono-400.woff2 create mode 100644 apps/console/public/fonts/plex-mono-600.woff2 create mode 100644 apps/console/public/fonts/plex-sans-var.woff2 create mode 100644 apps/console/public/media/glassbox-architecture-diagram.svg create mode 100644 benchmarks/__init__.py create mode 100644 benchmarks/agent_invalidation/__init__.py create mode 100644 benchmarks/agent_invalidation/__main__.py create mode 100644 benchmarks/agent_invalidation/fixtures.v1.json create mode 100644 benchmarks/agent_invalidation/runner.py create mode 100644 deploy/production/.env.production.example create mode 100644 deploy/production/Caddyfile create mode 100644 deploy/production/Dockerfile.console create mode 100644 deploy/production/Dockerfile.python create mode 100644 deploy/production/README.md create mode 100644 deploy/production/compose.yml create mode 100644 docs/adr/0025-pinned-flagship-estate-and-evidence-ablation.md create mode 100644 docs/adr/0026-loopback-console-read-model.md create mode 100644 docs/adr/0027-authenticated-control-plane-and-self-hosted-deployment.md create mode 100644 docs/architecture/glassbox-architecture-diagram.png create mode 100644 docs/architecture/glassbox-architecture-diagram.svg create mode 100644 docs/assets/glassbox-readme-banner.png create mode 100644 docs/assets/glassbox-readme-banner.svg create mode 100644 docs/benchmarks/README.md create mode 100644 docs/benchmarks/agent-invalidation-ablation.v1.json create mode 100644 docs/compatibility/datahub-1.6.0-hosted-production-otlp.live.json create mode 100644 docs/compatibility/datahub-1.6.0-one-command-flagship.live.json create mode 100644 examples/flagship-estate.compose.yml create mode 100644 examples/flagship_demo.py create mode 100644 schemas/benchmark-report/0.1.0/schema.json create mode 100644 scripts/forensics_mcp_capture.py create mode 100644 scripts/guarded_schema_change.py create mode 100644 scripts/remote_forensics_mcp_capture.py create mode 100644 scripts/remote_guarded_schema_change.py create mode 100644 scripts/remote_otlp_capture.py create mode 100644 services/control-plane/src/glassbox_control/__init__.py create mode 100644 services/control-plane/src/glassbox_control/crypto.py create mode 100644 services/control-plane/src/glassbox_control/datahub.py create mode 100644 services/control-plane/src/glassbox_control/server.py create mode 100644 services/control-plane/src/glassbox_control/store.py create mode 100644 tests/unit/test_agent_invalidation_benchmark.py create mode 100644 tests/unit/test_control_plane.py create mode 100644 tests/unit/test_flagship_demo.py create mode 100644 tests/unit/test_forensics_mcp_capture.py create mode 100644 tests/unit/test_guarded_schema_change.py create mode 100644 tests/unit/test_production_edge.py create mode 100644 tests/unit/test_remote_otlp_capture.py diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..14b7735 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,15 @@ +.git +.github +.glassbox +.venv +**/.env* +**/trusted-signers.json +**/.next +**/.open-next +**/.wrangler +**/dist +**/node_modules +**/tsconfig.tsbuildinfo +benchmarks/results +docs/compatibility/*.live.json +videos diff --git a/.env.example b/.env.example index 1f00d9f..12abbd5 100644 --- a/.env.example +++ b/.env.example @@ -15,3 +15,26 @@ GLASSBOX_OTLP_BEARER_TOKEN= # Public-key trust policy path. The file is not a private key, but modification is a # security-sensitive operator action. GLASSBOX_SIGNER_TRUST_POLICY_PATH= + +# Read-only loopback API exposed by `glassbox-forensics-mcp --transport streamable-http`. +# The console never receives database credentials or signing keys. +GLASSBOX_FORENSICS_API_URL=http://127.0.0.1:8788 +GLASSBOX_FORENSICS_API_TOKEN= + +# Authenticated deployment control plane. The master key must be an unpadded +# base64url-encoded 32-byte value and must come from a secret manager. +GLASSBOX_CONTROL_API_URL=http://127.0.0.1:8790 +GLASSBOX_CONTROL_API_TOKEN= +GLASSBOX_CONTROL_MASTER_KEY= +GLASSBOX_CONTROL_MASTER_KEY_ID=control-v1 +GLASSBOX_CONTROL_DB_PATH=.glassbox/control.sqlite3 +GLASSBOX_ORGANIZATION=default + +# Human authorization is derived from an upstream OIDC identity. This explicit +# local bypass must remain false in an internet deployment. +GLASSBOX_ADMIN_EMAILS= +GLASSBOX_ALLOW_LOCAL_OPERATOR=false + +# Optional public-site hosts. Hosts listed here receive the landing page at `/` +# and never expose operator routes. Leave empty for a console-only self-hosted install. +GLASSBOX_PUBLIC_HOSTS= diff --git a/.gitignore b/.gitignore index e46968d..0faed05 100644 --- a/.gitignore +++ b/.gitignore @@ -17,12 +17,19 @@ reproducibility-dist/ # Local runtime state .env +deploy/production/.env.production +deploy/production/trusted-signers.json +.glassbox/ *.db *.sqlite* artifacts/private/ traces/ keys/ +# Local demo-video production workspace. The public demo is linked from README; +# captures, voice tracks, render caches, and encoded masters stay out of Git. +videos/ + # Editors and operating systems .DS_Store .idea/ diff --git a/README.md b/README.md index 232511c..7625ff6 100644 --- a/README.md +++ b/README.md @@ -1,567 +1,483 @@ -# GlassBox - -GlassBox is a DataHub-native execution-provenance and control layer for AI agents. -It records a verifiable **Decision Bill of Materials (DBOM)** for consequential -agent outputs, connects curated runtime evidence to DataHub's governed metadata -graph, and provides the foundation for deterministic invalidation and safe replay. - -> Data lineage tells you where data went. GlassBox tells you what your agents -> believed and did because of it. - -## Current status - -GlassBox is in the live vertical-slice implementation phase. The repository -currently contains: - -- the DBOM 0.1 specification, canonicalizer, verifier, and tamper tests; -- a guarded DataHub Core capability probe; -- a framework-neutral Python runtime with nested-run correlation, evidence capture, - risk-typed tool execution, privacy-safe callback tokens, and direct MCP middleware; -- OpenTelemetry GenAI span mapping plus tested LangChain/LangGraph and Google ADK - adapters; -- a strict OTLP/HTTP JSON provenance compiler with verified DataHub URN resolution, - append-only operational event persistence, signed DBOM generation, and fail-safe - replay classification; -- a fail-closed live receipt pipeline that automatically registers each compiled - signed DBOM and a durable publication obligation in the same transactional state - used by the Action and MCP, verifies canonical state readback, then publishes and - directly verifies its governed DataHub projection; -- an authenticated, bounded OTLP/HTTP JSON receiver that returns 200 only after - publication evidence is sealed, plus an independent recovery worker and - zero-write direct verification for completed redelivery; -- an idempotent DataHub receipt publisher proven through double-write and direct - entity readback against Core 1.6.0; -- a deterministic materiality engine and installable DataHub Actions plugin with - signed-receipt reverse indexing, content-addressed campaigns, native incident - writeback, receipt quarantine, append-only audit, feedback-loop protection, and a - live-proven Kafka retry/commit-failure/restart path plus an independently proven - PostgreSQL Queue lease/visibility/acknowledgement path; -- a transactional SQLite WAL profile with atomic receipt indexing, competing-process - campaign leases, expired-work recovery, sealed verification evidence, and - zero-write verified redelivery on one host; -- a PostgreSQL 14+ profile with the same receipt and three-outbox contract, row-locked - multi-worker claims, database-clock leases, operator-only bootstrap, and a real - eight-connection PostgreSQL 16 concurrency proof; -- a signed, content-addressed State Transfer 0.1 contract that moves trusted receipt, - field-lineage, and supersession state atomically between SQLite and PostgreSQL - while refusing to reactivate old leases, campaigns, or notification side effects; -- a second durable owner-routing outbox plus a native DataHub ownership webhook - adapter with environment-loaded credentials, stable idempotency keys, and - privacy-minimized acceptance evidence; -- a signed, content-addressed Replay Bundle 0.1 contract with exact historical - resource matching, deterministic policy decisions, digest-bound expiring - approvals, and a structurally non-executing dry-run renderer; -- a policy-rechecked read-only replay executor plus a digest-pinned OCI profile with - network denial, read-only root, dropped capabilities, resource ceilings, bounded - transport, host-created attestations, and exact tool source/schema label checks; -- a new signed receipt, raw-free structural diff, history-preserving supersession, - and verified idempotent DataHub incident closure that leaves both receipt - Documents unchanged; -- a closed Semantic Policy 0.1 contract with content-addressed declarative rule - packs, an explicit operator trust registry, complete change coverage, numeric - tolerance and unordered-multiset primitives, raw-free durable assessments, and a - live Core 1.6.0 non-exact-equivalence proof; -- a separately versioned PostgreSQL recovery state machine linked to the exact - completed invalidation campaign and source receipt, with server-clock leases, - atomic raw-free artifact sealing, persisted replay/supersession/closure IDs, - append-only checkpoint events, per-attempt physical-write evidence, restart-safe - DataHub effects, and exact prior closure recovery with zero writes, now live-proven - across both committed-checkpoint and nine-process uncertain-completion campaigns - against DataHub Core 1.6.0, PostgreSQL 16.14, and the exact digest-pinned OCI - capability; -- the portable `datahub-agent-forensics` Skill with signed-DBOM inspection, - projection-only safeguards, canonical impact classification, report templates, and - adversarial evaluations; -- a protocol-neutral read-only forensics service plus an MCP v2 adapter with six - proof-carrying tools for receipt verification, recorded influence, deterministic - prospective impact, complete reverse scans, and actual persisted Action findings - from the same PostgreSQL state authority; -- a responsive, judge-facing forensic console that explains receipt integrity, - exact field influence, stale/unaffected outcomes, governed response, and replay - boundaries without exposing raw agent content; -- architecture and compatibility decision records; -- a maintainer-facing native metadata RFC for immutable agent decisions, qualified - runtime influence, verification, incidents, and append-only supersession, grounded - in the working compatibility implementation; -- a release-evidence gate that verifies archive safety, wheel RECORD integrity, - entry points, packaged contracts, byte-identical rebuilds, SHA-256 checksums, a - CycloneDX lockfile SBOM, and clean Python 3.11–3.13 installs; -- the complete product and delivery plan in [`plan.md`](plan.md). - -No UI screenshot or mock response is considered proof of integration. DataHub -capabilities are marked proven only after a live probe writes metadata and reads it -back directly from the configured server. - -## Development - -Requirements: - -- Python 3.11–3.13 (3.12 recommended); -- [`uv`](https://docs.astral.sh/uv/); -- Docker Desktop only for disposable live DataHub Core and PostgreSQL checks. +![GlassBox product banner](docs/assets/glassbox-readme-banner.png) -```bash -uv sync --all-extras -uv run ruff format --check . -uv run ruff check . -uv run mypy -uv run pytest -``` +# GlassBox β€” Signed Decision Evidence and Deterministic Invalidation for AI Agents -Before publishing or handing off a source tree, run the repository preflight. It -evaluates the exact tracked and prospective source set, rejects generated files, -unsafe links, oversized files, private-key material, common credential formats, -personal machine paths, malformed JSON, and non-canonical project URLs, then emits -a content-addressed raw-free inventory: +> A DataHub-native execution-provenance and control layer for AI agents β€” +> recording a signed **Decision Bill of Materials** for consequential agent +> outputs, binding runtime evidence to **DataHub's** governed metadata graph, +> turning a metadata change into a deterministic invalidation campaign, and +> closing the loop through authorized replay and append-only supersession. -```bash -uv run python -m scripts.repository_preflight --root . -``` +Built as an open-source **DataHub ecosystem project**, not a disposable +prototype: every capability claim traces to a committed, sanitized live evidence +report. -Validate a receipt without DataHub: +--- -```bash -uv run glassbox-dbom verify tests/fixtures/dbom/valid-read-only.json -``` +## Live -For production authority, validate the operator registry and verify a signed receipt -against it. A self-contained signature without this policy proves key possession, -not authorization: +- 🌐 **Product** β€” https://glassboxhq.xyz +- 🎬 **Two-minute demo** β€” https://youtu.be/g-j9zD5cxLk +- πŸ“– **Documentation** β€” https://glassboxhq.xyz/docs +- πŸ—ΊοΈ **Architecture** β€” https://glassboxhq.xyz/docs/architecture +- πŸ† **Devpost submission** β€” https://devpost.com/software/glassbox-yr49mu +- 🀝 **DataHub contributions** β€” [Agent Forensics Skill PR #120](https://github.com/datahub-project/datahub-skills/pull/120) Β· [Core pgQueue proof PR #19004](https://github.com/datahub-project/datahub/pull/19004) +- 🧾 **Live evidence reports** β€” [`docs/compatibility/`](docs/compatibility/) -```bash -uv run glassbox-dbom verify-policy /etc/glassbox/trusted-signers.json -uv run glassbox-dbom verify receipt.json \ - --signer-trust-policy /etc/glassbox/trusted-signers.json \ - --json -``` +The hosted release candidate is connected to an isolated, least-privilege +DataHub Core `v1.6.0` reference estate. Its public OTLP path has admitted a real +two-span agent run, persisted the signed receipt, published five curated DataHub +Document aspects, directly read them back, and proven idempotent redelivery. The +sanitized proof is +[`datahub-1.6.0-hosted-production-otlp.live.json`](docs/compatibility/datahub-1.6.0-hosted-production-otlp.live.json). -Raw receipt files default to current-time `ADMISSION`. Use `--trust-mode HISTORICAL` -only when checksummed state or equivalent independent evidence proves the receipt was -admitted before the signer retired; a signer-authored timestamp is not that proof. +--- -Use `glassbox-dbom signer-entry` to derive a policy entry from an -environment-indirect private key without returning its private bytes. The complete -procedure is the [signer rotation runbook](docs/operations/signing-key-rotation.md). +## Table of Contents -Inspect the DataHub probe without performing network writes: +- [Quick Path](#quick-path) +- [Why It Stands Out](#why-it-stands-out) +- [What it does](#what-it-does) +- [Architecture](#architecture) +- [Requirements](#requirements) +- [Setup](#setup) +- [Usage](#usage) +- [GlassBox at the decision boundary](#glassbox-at-the-decision-boundary) + - [Instrument a run](#instrument-a-run) + - [Install the DataHub Action](#install-the-datahub-action) + - [Query decision evidence over MCP](#query-decision-evidence-over-mcp) + - [Open the operator console](#open-the-operator-console) +- [Trust model](#trust-model) +- [Project layout](#project-layout) +- [License](#license) -```bash -uv run glassbox-datahub-probe plan -``` +--- -A live probe is deliberately gated. See -[`docs/compatibility/datahub-1.6.0.md`](docs/compatibility/datahub-1.6.0.md) -before running it. +## Quick Path -Compile, sign, emit, and directly verify the deterministic live receipt proof: +If you want the shortest path through the project: ```bash -export GLASSBOX_STATE_POSTGRES_DSN='postgresql://...' -uv run python -m examples.end_to_end_receipt --allow-live +uv sync --all-extras +uv run --all-extras python -m examples.flagship_demo \ + --allow-live \ + --output .glassbox/flagship/one-command-report.json ``` -When `GLASSBOX_STATE_POSTGRES_DSN` is present, the proof automatically opens the -already initialized `glassbox` Action schema in runtime mode, registers and rereads -the signed receipt, and only then publishes DataHub. Without that environment -variable, it runs the legacy DataHub-only compatibility proof. See the -[provenance compiler contract](docs/provenance-compiler.md) for explicit schema and -DSN-environment-name options, and the committed -[combined DataHub/PostgreSQL live report](docs/compatibility/datahub-1.6.0-postgresql-live-receipt-pipeline.live.json). +That single command downloads the commit-pinned official DataHub Core `v1.6.0` +quickstart, starts it on isolated ports with PostgreSQL 16, waits for health, +builds and inspects the replay sandbox, runs the real causal chain, validates +every proof boundary, writes a raw-free report, and removes only its own estate. + +It is one connected chain, not a replay fixture: the exact receipt quarantined by +the live Action becomes the source of a fingerprint-authorized corrected bundle, +the corrected evidence digest replaces the affected action input, the new +decision is produced inside a source/schema-bound hardened container, and DataHub +directly reads back both receipts plus their immutable supersession relation +before the incident resolves. -Run the used-field invalidation proof plus unrelated-field negative control: +For the docs site: ```bash -uv run python -m examples.end_to_end_invalidation --allow-live +cd apps/console +npm install +GLASSBOX_PUBLIC_HOSTS=glassbox.localhost npm run dev ``` -See the [invalidation action contract](docs/invalidation-action.md) and its -[live evidence report](docs/compatibility/datahub-1.6.0-invalidation.live.json). +Then open: -Run the genuine GMS-to-Kafka-to-Actions transport proof, including one injected -action failure, exhaustion of the synchronous commit retry window, exact -same-offset redelivery, and a same-group recovery commit: +- `http://glassbox.localhost:3000` for the public landing page; +- `http://glassbox.localhost:3000/docs` for the documentation; +- `http://glassbox.localhost:3000/docs/architecture` for the interactive architecture; +- `http://localhost:3000` for the disconnected operator console. -```bash -uv run python -m examples.end_to_end_broker_invalidation --allow-live +For the implementation details: + +- Quickstart β€” [`apps/console/app/docs/quickstart`](apps/console/app/docs/quickstart/page.mdx) +- Architecture β€” [`apps/console/app/docs/architecture`](apps/console/app/docs/architecture/page.mdx) +- Decision records β€” [`docs/adr/`](docs/adr/) + +## Why It Stands Out + +GlassBox is strong for AI-agent governance because it does three things +together: + +- **It records what a decision actually depended on.** Every consequential + output gets a canonicalised, Ed25519-signed Decision Bill of Materials whose + dependency set is resolved to real DataHub URNs β€” so a later question about + impact is a lookup, not an investigation. +- **It decides materiality with a rule, not a model.** A metadata change becomes + a content-addressed campaign, and a versioned, deterministic policy pack + produces a state plus a durable reason code. The same receipt and the same + change always produce the same verdict. +- **It closes the loop without rewriting history.** Quarantine is explicit and + reversible. Recovery requires a digest-bound approval, executes inside a + digest-pinned container, and lands as an append-only supersession that leaves + both receipt Documents byte-unchanged. + +The differentiator is not provenance in isolation. It is the complete chain: + +```text +runtime evidence β†’ governed projection β†’ deterministic assessment + β†’ durable quarantine β†’ authorized recovery β†’ verified closure ``` -Its sanitized [Kafka evidence report](docs/compatibility/datahub-1.6.0-kafka-invalidation.live.json) -records the broker partition, unchanged failed-ack offset, exact recovered delivery, -recovery commit, retry counts, and all schema offsets observed on the third-process -negative control. The same proof exercises the transactional state profile after -automatic signed-receipt registration and canonical readback, resolves the native -DataHub owner, accepts exactly one loopback webhook, and re-verifies a completed -material campaign without another DataHub write or webhook call. +## What it does + +- **Signed decision receipts** β€” canonicalises payloads with RFC 8785, digests + with SHA-256, commits evidence sets as Merkle trees, and signs with Ed25519. + `receipt_id` and the whole `integrity` object are excluded from digest + material so the identity stays independently verifiable. +- **Operator-scoped signer trust** β€” a signature proves key possession, not + authorization. The closed `glassbox.signer-trust.v1` policy binds every trusted + signer to both its key ID and the SHA-256 fingerprint of its raw public key. +- **Framework-neutral runtime capture** β€” normalises every instrumentation mode + into immutable `RuntimeEvent` records, correlates nested agent runs through + explicit parent run and span IDs, and keeps raw tool arguments and results on + the application call stack rather than on a span. +- **Strict provenance compilation** β€” accepts normalised events, exporter-neutral + OpenTelemetry spans, or strict OTLP/HTTP protobuf-JSON. Pins the supported + GenAI semantic schema URL and rejects dropped attributes, dropped events, + duplicate span identities, and ambiguous agent-span selection instead of + guessing. +- **Verified governed publication** β€” resolves every dependency to a real + DataHub URN, then verifies the projection by direct entity readback rather than + by write acknowledgement. +- **Durable publication obligations** β€” the signed receipt, its dependency index, + and a `READY` publication row are inserted in one transaction. HTTP 200 means + publication evidence is sealed; a 503 means retry, and the obligation survives + the sender disappearing. +- **Deterministic invalidation** β€” normalises supported `MetadataChangeLogEvent_v1` + payloads into a closed change model, evaluates a pure versioned materiality + engine, and records raw-free assessments carrying reason codes and policy + versions. +- **Honest completeness** β€” dependency resolution, field-lineage coverage, and + wildcard queries are tracked separately, because proving a decision *was* + affected needs one match while proving it *was not* needs the complete set. +- **Two state profiles, one protocol** β€” SQLite WAL for multi-process + coordination on one host, PostgreSQL 14+ for workers across hosts, with + row-locked claims and database-clock leases. A state transition cannot exist in + one profile and silently not the other. +- **Independent transport proofs** β€” acknowledged at-least-once Kafka delivery + and the official PostgreSQL Queue source are each proven against their own live + estate. A success in one never marks the other proven. +- **Content-addressed replay bundles** β€” derived from a verified source receipt, + independently signed, with digest-bound expiring approvals and a structurally + non-executing dry-run renderer. +- **Isolated execution** β€” replay runs in one exact OCI image ID (never a mutable + tag), with network denial, read-only root, dropped capabilities, resource + ceilings, and a host-created content-addressed isolation attestation. +- **Domain-semantic policies** β€” exact equality is the default; widening it + requires the caller to name an exact content-addressed `policy_id` *and* an + operator registry that already trusts it. +- **Read-only forensics surface** β€” six proof-carrying MCP tools over the same + PostgreSQL state authority the Action writes to, with prospective + classifications kept visibly separate from actually persisted findings. +- **Raw-free by construction** β€” digests, governed URNs, reason codes, and + verification results cross the boundary. Prompts, model outputs, tool + arguments, field values, credentials, and signing keys never do. + +## Architecture + +![GlassBox architecture](docs/architecture/glassbox-architecture-diagram.png) + +In short: an instrumented agent run emits OTLP, and the compiler turns it into a +canonical signed receipt whose dependencies resolve to real DataHub URNs. That +receipt registers in transactional state alongside a durable publication +obligation, then publishes a governed DataHub projection verified by direct +readback. When DataHub emits a metadata change, the Action builds a +content-addressed campaign, a deterministic rule pack decides materiality, and +findings write back as incidents with optional receipt quarantine. Recovery is a +separate authorized path: a signed replay bundle, a digest-bound approval, +execution in a digest-pinned container, and an append-only supersession that +DataHub reads back before the incident closes. + +**Explore it:** + +- πŸ—ΊοΈ Interactive architecture β€” https://glassboxhq.xyz/docs/architecture +- πŸ“– Documentation β€” https://glassboxhq.xyz/docs +- 🧾 Live evidence reports β€” [`docs/compatibility/`](docs/compatibility/) +- 🧱 Decision records β€” [`docs/adr/`](docs/adr/) + +## Requirements + +- macOS or Linux +- Python 3.11–3.13 (3.12 recommended) and [`uv`](https://docs.astral.sh/uv/) +- Docker with Compose `!override` support, and enough memory for the DataHub + quickstart profile +- Free host ports `13306`, `14319`, `15432`, `18080`, `19002`, `19092`, `19200` + for the flagship estate β€” every one has a CLI override +- Optional: PostgreSQL 14+ for the multi-worker state profile + +## Setup + +### 1. Install -Run the separate PostgreSQL Queue source proof against a fresh disposable -PostgreSQL 16 database: +```bash +uv sync --all-extras +uv run ruff format --check . +uv run ruff check . +uv run mypy +uv run pytest +``` + +Install only the extras a given process needs: ```bash -export GLASSBOX_PGQUEUE_PASSWORD='a-disposable-local-password' -uv run python -m examples.end_to_end_pgqueue_invalidation \ - --server http://localhost:18080 \ - --schema-registry-url http://localhost:18080/schema-registry/api/ \ - --pg-host-port 127.0.0.1:55434 \ - --initialize-schema \ - --allow-live +uv sync --extra langchain +uv sync --extra google-adk +uv sync --extra mcp +uv sync --extra actions --extra datahub --extra postgres ``` -Its sanitized -[pgQueue evidence report](docs/compatibility/datahub-1.6.0-pgqueue-invalidation.live.json) -proves the official Actions source against real PostgreSQL leases and contiguous -offsets: the offset stays behind after failed acknowledgement, a restart cannot -steal the live lease, the exact handle returns after visibility expiry, completed -work is freshly verified with zero duplicate emissions, the ack marker advances the -offset, and a third restart is empty. Kafka and pgQueue claims remain deliberately -independent. +### 2. Establish signing authority -Initialize the SQLite state database before starting the reference Actions pipeline. -`register-receipt` remains available for historical imports or explicit repair; live -compiler paths register automatically: +Signature integrity and operator trust are separate concerns. `signer-entry` +derives a policy-ready public entry from an environment-indirect private key +without returning its private bytes. ```bash -mkdir -p .glassbox -export GLASSBOX_SIGNER_TRUST_POLICY_PATH=/etc/glassbox/trusted-signers.json -uv run glassbox-datahub-action inspect-install -uv run glassbox-datahub-action validate-config examples/datahub-actions-invalidation.yml -uv run glassbox-invalidation-state init .glassbox/invalidation.sqlite3 -uv run glassbox-invalidation-state register-receipt \ - .glassbox/invalidation.sqlite3 receipt.json \ - --field-coverage COMPLETE \ - --field-rule glassbox.sql-column-lineage.v1 \ - --wildcard-query false +uv run glassbox-dbom signer-entry +uv run glassbox-dbom verify-policy /etc/glassbox/trusted-signers.json ``` -This SQLite profile coordinates processes on one host. It is deliberately not -presented as a multi-node or network-filesystem deployment. +See the [signer rotation runbook](docs/operations/signing-key-rotation.md) for +enrollment, overlap, retirement, revocation, and rollback. -The [ecosystem packaging note](docs/upstream/datahub-actions-contribution.md) -documents why this ships through DataHub's public external-plugin contract before -seeking core inclusion, along with the clean-wheel and release checklist. -The complete maintainer-facing submission sequence and ready-to-adapt discussion -text live in the [upstream contribution packet](docs/upstream/README.md). +### 3. Initialize a state profile -For workers that may run on different hosts, initialize the PostgreSQL profile once -with a privileged bootstrap identity, then run compilers and Action workers with an -environment-injected DSN and narrower runtime privileges. The explicit registration -shown here is the manual import/repair path: +SQLite coordinates processes on one host. PostgreSQL coordinates workers that +may run on different hosts; `postgres-init` is an operator-only bootstrap, after +which compilers and Action workers run with narrower runtime privileges. ```bash -uv sync --extra actions --extra datahub --extra postgres +# Single host +mkdir -p .glassbox +uv run glassbox-invalidation-state init .glassbox/invalidation.sqlite3 + +# Multiple workers export GLASSBOX_STATE_POSTGRES_DSN='postgresql://...' -export GLASSBOX_SIGNER_TRUST_POLICY_PATH=/etc/glassbox/trusted-signers.json uv run glassbox-invalidation-state postgres-init \ --dsn-env GLASSBOX_STATE_POSTGRES_DSN \ --schema glassbox -uv run glassbox-invalidation-state postgres-register-receipt receipt.json \ - --dsn-env GLASSBOX_STATE_POSTGRES_DSN \ - --schema glassbox \ - --field-coverage COMPLETE \ - --field-rule glassbox.sql-column-lineage.v1 \ - --wildcard-query false ``` -The DSN value is never placed in Actions configuration or status output. See the -[PostgreSQL pipeline example](examples/datahub-actions-invalidation-postgres.yml), -[ADR-0010](docs/adr/0010-postgresql-multi-worker-invalidation-state.md), and the -sanitized [PostgreSQL 16 trusted-signer proof report](docs/compatibility/postgresql-16-trusted-signer-state.live.json). -That proof establishes real server, multi-connection coordination; it deliberately -does not claim a physical multi-host deployment, managed failover, or network-partition -recovery. +The DSN value is never placed in Actions configuration or status output. -The combined -[trusted receipt publication report](docs/compatibility/datahub-1.6.0-postgresql-trusted-receipt-pipeline.live.json) -proves current-time signer admission, checksummed PostgreSQL admission evidence, two -real DataHub upserts plus direct aspect readback, and completed redelivery with zero -additional DataHub writes. +### 4. Verify the installation -Run the live OTLP receiver after bootstrapping the PostgreSQL state schema. Inject a -base64url-encoded 32-byte Ed25519 private key and receiver bearer token through your -secret manager; never place either value on the command line: +```bash +uv run glassbox-dbom verify tests/fixtures/dbom/valid-read-only.json +uv run glassbox-datahub-probe plan +uv run glassbox-datahub-action inspect-install +``` + +`verify` needs no network and no database. `probe plan` shows what a live probe +would do without performing any network write. `inspect-install` verifies the +Actions entry point is installed exactly once. + +## Usage ```bash -export GLASSBOX_STATE_POSTGRES_DSN='postgresql://...' -export GLASSBOX_RECEIPT_SIGNING_KEY='...' -export GLASSBOX_OTLP_BEARER_TOKEN='...' -export GLASSBOX_SIGNER_TRUST_POLICY_PATH=/etc/glassbox/trusted-signers.json -export DATAHUB_GMS_TOKEN='...' +# Verify a receipt against operator authority +uv run glassbox-dbom verify receipt.json \ + --signer-trust-policy /etc/glassbox/trusted-signers.json --json + +# Bounded receipt and outbox status +uv run glassbox-invalidation-state status .glassbox/invalidation.sqlite3 + +# Receive OTLP traces and publish receipts uv run glassbox-otlp-receiver serve \ --signing-key-id glassbox-prod-2026-08 \ --environment PROD \ --output-kind agent-decision \ --output-mime-type application/json -``` - -Exporters send OTLP protobuf JSON to `POST /v1/traces` with -`Authorization: Bearer …`. HTTP 200 means the signed receipt, dependency index, and -publication obligation exist and DataHub direct-read evidence has been sealed. A -503 means the sender should retry; the durable obligation can also be repaired after -the sender disappears: -```bash +# Recover stranded publication obligations uv run glassbox-otlp-receiver drain --limit 100 -``` -Completed redelivery performs a fresh DataHub readback with zero writes. The -reference server is single-flight and expects TLS termination and rate limiting in a -production proxy. Non-loopback binds require bearer authentication unless the -operator supplies the explicit unsafe override. See -[ADR-0016](docs/adr/0016-durable-receipt-publication-and-otlp-acknowledgement.md) and -the combined -[DataHub/PostgreSQL/HTTP live report](docs/compatibility/datahub-1.6.0-postgresql-otlp-receiver.live.json). - -The newer -[trusted OTLP live report](docs/compatibility/datahub-1.6.0-postgresql-trusted-otlp-receiver.live.json) -also proves that the receiver signer passed the active policy gate before binding, -that PostgreSQL verified its sealed admission evidence, and that authenticated 200 -redelivery performed fresh DataHub reads with zero writes. - -Trusted-signer admission evidence uses SQLite schema version 4 and PostgreSQL schema -version 3. Runtime processes do not perform implicit migration. The explicit -[state-transfer runbook](docs/operations/state-transfer.md) exports only from a -runtime that understands the source schema, independently verifies migration and -receipt authority, and atomically activates receipts under the destination's current -trust policy. Unsupported pre-policy schemas still require the last compatible -exporter or deliberate rebuild; changing a schema-version marker is never safe. - -Build and inspect a complete read-only replay plan without calling any tool or -external service: +# Build, verify, and render a replay without invoking any tool +uv run glassbox-replay bundle --help +uv run glassbox-replay verify-bundle bundle.json +uv run glassbox-replay dry-run bundle.json -```bash -uv run python -m examples.replay_dry_run +# Read-only decision-evidence MCP server +uv run glassbox-forensics-mcp + +# The complete one-command causal proof +uv run --all-extras python -m examples.flagship_demo --allow-live ``` -The report commits the exact bundle and action set while asserting zero external -calls and history mutations. See the [replay contract](docs/replay.md), -[Replay Bundle 0.1 integrity profile](schemas/replay-bundle/0.1.0/README.md), and -[ADR-0011](docs/adr/0011-content-addressed-replay-and-approval-binding.md). +Proof-oriented and inspection switches: -Execute the complete offline read-only chain and produce a new signed receipt, a -raw-free diff, and a history-preserving supersession record: +- `--allow-live` β€” required gate before anything touches a live estate. +- `--keep-estate` β€” leave the flagship estate up to inspect DataHub or attach + the console; the default disposable run removes the schema and the estate. +- `--pricing-semantic-policy` β€” run the supersession boundary with the versioned + domain policy instead of exact equality. +- `--trust-mode HISTORICAL` β€” admit a receipt signed by a since-retired signer, + only when independent evidence proves the admission time. -```bash -uv run python -m examples.replay_read_only -``` +Every command that needs a signing key, bearer token, or DSN reads it from a +named environment variable. No secret is accepted as a positional argument. -The in-process capability boundary is intentionally not described as an OS sandbox; -see [ADR-0012](docs/adr/0012-capability-scoped-read-only-replay-and-supersession.md). +## GlassBox at the decision boundary -Run the guarded live DataHub Core proof: +GlassBox is both an instrumentation layer and a control boundary: -```bash -uv run python -m examples.end_to_end_replay_supersession --allow-live +```text +agent run + β†’ GlassBox runtime: capture Β· correlate Β· redact + β†’ provenance compiler: canonicalise Β· resolve URNs Β· sign + β†’ transactional state + governed DataHub projection + β”‚ + β””β†’ metadata change β†’ campaign β†’ assessment β†’ writeback + β”‚ + quarantine β†’ authorized replay β†’ supersession β†’ closure ``` -The sanitized [live evidence report](docs/compatibility/datahub-1.6.0-replay-supersession.live.json) -records exact double-write/readback and unchanged before/after hashes for both receipt -Documents. +It fails **safe** rather than open: when something cannot be proven, it is +recorded as unproven instead of inferred. An ambiguous agent span is a compile +error, an unresolvable dependency is recorded as unresolved, and coverage that is +not `COMPLETE` produces `UNKNOWN` rather than a false `UNAFFECTED`. -Run the same live boundary with the versioned pricing semantic policy: +### Instrument a run -```bash -uv run python -m examples.end_to_end_replay_supersession \ - --pricing-semantic-policy \ - --allow-live -``` - -The [semantic-policy evidence](docs/compatibility/datahub-1.6.0-semantic-policy.live.json) -records a real structural price change as policy-proven `EQUIVALENT` while retaining -`exact_match=false`, verifies all 19 DataHub relation properties, stores no compared -values, and leaves both receipt Documents unchanged. Exact equality is still the -default; domain policy use requires the exact content ID in an operator-trusted -registry. See [ADR-0023](docs/adr/0023-content-addressed-domain-semantic-policies.md). +```python +from glassbox import ActionEffect, EvidenceRole, EvidenceState, GlassBox -Run the guarded one-command causal flagship against initialized PostgreSQL state and -local DataHub Core: +runtime = GlassBox() -```bash -export GLASSBOX_STATE_POSTGRES_DSN='postgresql://...' -uv run python -m scripts.build_replay_sandbox -# Copy the printed image_digest into this environment variable. -export GLASSBOX_REPLAY_SANDBOX_IMAGE='sha256:...' -uv run python -m examples.end_to_end_flagship \ - --server http://localhost:8080 \ - --sandbox-image-digest "$GLASSBOX_REPLAY_SANDBOX_IMAGE" \ - --allow-live +@runtime.consequential(agent_id="pricing-agent", workflow_id="recommend-price") +def recommend_price(customer_id: str) -> dict[str, int]: + runtime.observe_evidence( + entity_type="dataset", + datahub_urn="urn:li:dataset:(urn:li:dataPlatform:postgres,commerce.orders,PROD)", + state=EvidenceState.OBSERVED, + role=EvidenceRole.INPUT, + representation={"customer_id": customer_id}, + ) + return runtime.call_tool( + "pricing.lookup", + lambda: {"recommended_price": 42}, + effect=ActionEffect.READ_ONLY, + ) ``` -This is one connected chain, not a replay fixture: the exact receipt quarantined by -the live Action becomes the source of a fingerprint-authorized corrected bundle; -the corrected evidence digest replaces the affected action input; the new decision -changes inside a source/schema-bound hardened container; and DataHub directly reads -back both receipts plus their immutable supersession relation before resolving the -exact incident. The command proves the receipt Documents did not change, runs the -unrelated-field zero-write control, and cross-checks the incident through official -DataHub MCP and GlassBox MCP. See -[ADR-0020](docs/adr/0020-signed-invalidation-to-recovery-handoff.md), -[ADR-0021](docs/adr/0021-oci-isolated-replay-and-verified-incident-closure.md), and the -[raw-free live report](docs/compatibility/datahub-1.6.0-flagship-causal-recovery.live.json). -[ADR-0022](docs/adr/0022-postgresql-durable-recovery-orchestration.md) defines the -new durable coordinator. The guarded crash proof then runs that causal chain through -four fresh checkpoint workers plus one closed-redelivery worker; every process exits -abruptly with the configured fault code after PostgreSQL readback. The committed -[raw-free crash report](docs/compatibility/datahub-1.6.0-durable-recovery-crash.live.json) -proves ordered restart recovery, real OCI isolation, DataHub direct readback, exact -zero-write closure reuse, and unchanged source and receipt history. - -The complementary -[uncertain-completion crash report](docs/compatibility/datahub-1.6.0-durable-recovery-uncertain-crash.live.json) -injects process death after each successful OCI or DataHub operation but before its -PostgreSQL completion call. Four fresh workers recover only after the server-clock -lease expires, and a ninth proves closed redelivery. The report distinguishes -historical emission evidence from physical writes on the retry: replay-receipt and -exact-closure retries perform zero writes, while immutable supersession safely -repeats its verified double-write. - -Install the forensic skill in an Agent Skills-compatible project: +Arguments, results, and evidence representations are committed by digest; they +are not retained in runtime events. See the +[runtime instrumentation contract](docs/runtime-instrumentation.md). + +### Install the DataHub Action + +GlassBox ships through DataHub's public external-plugin contract. Both checks +below are offline and never connect to DataHub: ```bash -mkdir -p .agents/skills -cp -R skills/datahub-agent-forensics .agents/skills/ +uv run glassbox-datahub-action inspect-install +uv run glassbox-datahub-action validate-config examples/datahub-actions-invalidation.yml ``` -Then ask: `Use $datahub-agent-forensics to explain why this agent decision is -stale.` The skill composes with DataHub Search, Lineage, and Quality for discovery, -but keeps integrity and impact decisions deterministic. See the -[skill contribution guide](docs/datahub-agent-forensics-skill.md). +The Action consumes `MetadataChangeLog_Versioned_v1` through the pinned Actions +`kafka` source or the official PostgreSQL Queue source. The pipeline name is the +consumer-group identity: it must be stable and unique. See the +[invalidation action contract](docs/invalidation-action.md). -Run the decision-evidence MCP companion over stdio against the same PostgreSQL -schema as the DataHub Action: +### Query decision evidence over MCP ```bash uv sync --extra mcp --extra postgres export GLASSBOX_STATE_POSTGRES_DSN='postgresql://...' -export GLASSBOX_SIGNER_TRUST_POLICY_PATH=/etc/glassbox/trusted-signers.json uv run glassbox-forensics-mcp \ --state-postgres-dsn-env GLASSBOX_STATE_POSTGRES_DSN \ --state-postgres-schema glassbox \ - --signer-trust-policy "$GLASSBOX_SIGNER_TRUST_POLICY_PATH" + --signer-trust-policy /etc/glassbox/trusted-signers.json ``` It complements DataHub's official MCP server: DataHub owns catalog discovery and -generic lineage, while GlassBox owns signed run-specific decision evidence. All six -tools are read-only. Prospective classifications remain visibly separate from -campaigns actually persisted and writeback-verified by the Action. There is no -quarantine, approval, replay-execution, resolution, or supersession tool. See the -[MCP contract](docs/forensics-mcp.md), -[ADR-0013](docs/adr/0013-read-only-forensics-mcp.md), and -[ADR-0014](docs/adr/0014-shared-live-decision-state.md). Signer authority and key -rotation are defined by -[ADR-0017](docs/adr/0017-operator-trusted-receipt-signers-and-rotation.md). - -This boundary is now live-proven, not fixture-replayed. The guarded -[dual-MCP evidence report](docs/compatibility/datahub-1.6.0-dual-mcp-forensics.live.json) -runs the official DataHub MCP server `0.6.0` beside GlassBox MCP against one real -Action-completed incident on Core `v1.6.0` with PostgreSQL-backed state. It -cross-binds the catalog dataset, exact field type, incident health, receipt, -observed influence, `STALE` finding, quarantine, completed campaign, and verified -DataHub writeback. It also records the measured official-server limitation that the -exact Incident entity body was unavailable through `get_entities`; no missing -evidence is fabricated. - -For audit or CI, bind natural-language answers to that proof with the installable -claim-ledger evaluator: +generic lineage, GlassBox owns signed run-specific decision evidence. All six +tools are read-only, and there is no quarantine, approval, replay-execution, +resolution, or supersession tool. Prospective classifications stay visibly +separate from campaigns actually persisted and writeback-verified by the Action. -```bash -glassbox-agent-narration brief dual-mcp-evidence.json --pretty -glassbox-agent-narration evaluate dual-mcp-evidence.json agent-response.json --pretty -``` - -The committed -[agent narration evaluation](docs/compatibility/datahub-1.6.0-dual-mcp-agent-narration.eval.json) -records both an ordinary independent agent run and a pressure case demanding -invented Incident details, global scope, and mutation authority. Both preserved all -18 evidence facts and required limitations. Free-prose review remains visibly -model-based rather than being promoted to deterministic proof; see -[ADR-0019](docs/adr/0019-machine-auditable-agent-narration.md). - -Read the proposed native model in -[Agent decision receipts and runtime influence](docs/rfcs/000-agent-decision-receipts-and-runtime-influence.md). -The draft directly extends DataHub Agent Registry RFC #16012 and is explicitly not -claimed as submitted or accepted. - -Run its guarded live Core proof: +The forensic Skill installs into any Agent Skills-compatible project: ```bash -uv run python -m examples.end_to_end_forensics_skill --allow-live +mkdir -p .agents/skills +cp -R skills/datahub-agent-forensics .agents/skills/ ``` -The sanitized [forensics evidence report](docs/compatibility/datahub-1.6.0-agent-forensics.live.json) -records the official CLI direct read, verified signed artifact, projection-only -boundary, and used-field/unrelated-field policy controls. - -Open the local forensic console: +### Open the operator console ```bash +glassbox-forensics-mcp \ + --transport streamable-http \ + --state-postgres-dsn-env GLASSBOX_STATE_POSTGRES_DSN + cd apps/console npm install -npm run dev +GLASSBOX_FORENSICS_API_URL=http://127.0.0.1:8788 npm run dev ``` -The console's default investigation is backed by the same guarded Core proof and -includes a material `revenue` change plus an `internal_note` negative control. Its -own build, rendered-content tests, lint, and strict type-check commands are documented -in [`apps/console/README.md`](apps/console/README.md). It is not described as hosted -until a separate public deployment is explicitly performed. - -Production owner routing is optional and requires an HTTPS receiver that honors the -campaign `Idempotency-Key`; see the -[invalidation action contract](docs/invalidation-action.md). Webhook acceptance is -not presented as proof that a human read the notification. - -Install only the framework adapter you need: - -```bash -uv sync --extra langchain -uv sync --extra google-adk -uv sync --extra mcp -uv sync --extra actions --extra datahub -uv sync --extra actions --extra datahub --extra postgres -``` - -Instrument a minimal run: - -```python -from glassbox import ActionEffect, EvidenceRole, EvidenceState, GlassBox - -runtime = GlassBox() - -@runtime.consequential(agent_id="pricing-agent", workflow_id="recommend-price") -def recommend_price(customer_id: str) -> dict[str, int]: - runtime.observe_evidence( - entity_type="dataset", - datahub_urn="urn:li:dataset:(urn:li:dataPlatform:postgres,commerce.orders,PROD)", - state=EvidenceState.OBSERVED, - role=EvidenceRole.INPUT, - representation={"customer_id": customer_id}, - ) - return runtime.call_tool( - "pricing.lookup", - lambda: {"recommended_price": 42}, - effect=ActionEffect.READ_ONLY, - ) -``` - -Arguments, results, and evidence representations are committed by digest; they are -not retained in runtime events. See the -[`runtime instrumentation contract`](docs/runtime-instrumentation.md) for correlation, -redaction, callback, and action-policy semantics. +Overview, investigations, receipts, campaigns, recovery, trust, and connections +are independent application routes reading the configured verified receipt and +campaign stores. When the service is absent the console shows an explicit +connection state rather than hard-coded proof content. See +[`apps/console/README.md`](apps/console/README.md). ## Trust model - Evidence is always `OBSERVED`, `DECLARED`, `INFERRED`, or `UNKNOWN`. - Raw high-cardinality traces remain outside DataHub. -- Receipts are append-only; replays create new receipts. -- A signature proves integrity and key possession, not operator trust or factual - truth. Production admission also requires an operator policy bound to the key ID - and public-key fingerprint. +- Receipts are append-only; replays create new receipts and an immutable + supersession relation. Both Documents remain byte-unchanged. +- A signature proves integrity and key possession β€” not operator trust, and not + factual truth. Production admission also requires an operator policy bound to + the key ID and public-key fingerprint. - Unknown-effect and irreversible actions are never automatically replayed. +Stated boundaries, kept in the open: + +- The SQLite profile coordinates processes on **one host**. It is not a + multi-node or network-filesystem deployment. +- The PostgreSQL proof establishes real server, multi-connection coordination. It + does not claim physical multi-host deployment, managed failover, or + network-partition recovery. +- The OCI replay profile is a strong, verifiable sandbox β€” not a formal isolation + guarantee. +- The reference OTLP receiver is single-flight and expects TLS termination and + rate limiting in a production proxy. + See [`AGENTS.md`](AGENTS.md) for binding engineering rules and [`SECURITY.md`](SECURITY.md) for vulnerability reporting and data-handling rules. -See the [signer rotation runbook](docs/operations/signing-key-rotation.md) for -enrollment, overlap, retirement, revocation, and rollback. + +## Project layout + +```text +packages/ + sdk/ # framework-neutral runtime, evidence capture, OTel mapping, adapters + dbom/ # DBOM 0.1 canonicalizer, verifier, signer trust, CLI + datahub-adapter/# capability probe, compatibility layer, governed publication + policy/ # semantic policy contract, registry, equivalence primitives +services/ + compiler/ # provenance compiler and authenticated OTLP receiver + invalidation-action/ # DataHub Actions plugin, materiality, campaigns, state CLI + replay-worker/ # replay bundles, capability execution, OCI isolation + forensics-mcp/ # protocol-neutral ForensicsService + read-only MCP adapter + control-plane/ # authenticated self-hosted control plane +apps/console/ # operator console, MDX documentation site, landing page +schemas/ # seven closed JSON contracts (DBOM, replay, policy, transfer, …) +examples/ # live proofs, the flagship demo, and pinned estate compose +benchmarks/ # evidence-ablation harness and published report schema +skills/ # portable datahub-agent-forensics Agent Skill +docs/ + adr/ # architecture decision records + compatibility/ # sanitized live evidence reports + architecture/ # diagram source and rendered assets +deploy/ # self-hosted production deployment profile +tests/ # unit, contract, tamper, and live-gated coverage +``` ## License -Apache License 2.0. See [`LICENSE`](LICENSE). +Apache License 2.0 β€” see [`LICENSE`](LICENSE). diff --git a/apps/console/.nvmrc b/apps/console/.nvmrc new file mode 100644 index 0000000..f9e7451 --- /dev/null +++ b/apps/console/.nvmrc @@ -0,0 +1 @@ +22.23.1 diff --git a/apps/console/README.md b/apps/console/README.md index c5f8aee..d848ced 100644 --- a/apps/console/README.md +++ b/apps/console/README.md @@ -1,29 +1,73 @@ # GlassBox Forensics Console -The judge-facing investigation surface for GlassBox. It turns signed decision -receipts and DataHub metadata changes into a compact causal explanation: +The read-only operator application for signed agent decisions, DataHub impact +campaigns, and recovery review. The console has independent routes for overview, +investigations, receipts, campaigns, recovery, trust, and settings. -- whether the receipt itself is trustworthy; -- which observed dependency changed; -- why the change is or is not material to the decision; -- which governed response is allowed next; -- and where DataHub projections stop being cryptographic evidence. +It does not bundle a successful demo response. Every record is read from the +configured GlassBox Forensics API. An absent or unreachable service produces an +explicit connection state rather than invented application data. -The default case is backed by the repository's live DataHub Core 1.6.0 proof. -Its negative-control view demonstrates that GlassBox can prove a decision was -unaffected when complete field lineage excludes the changed field. +## Evidence connection + +Run the existing forensics service over its loopback-only HTTP surface: + +```bash +glassbox-forensics-mcp \ + --transport streamable-http \ + --state-postgres-dsn-env GLASSBOX_STATE_POSTGRES_DSN \ + --signer-trust-policy "$GLASSBOX_SIGNER_TRUST_POLICY_PATH" +``` + +Configure the console server: + +```bash +GLASSBOX_FORENSICS_API_URL=http://127.0.0.1:8788 +GLASSBOX_FORENSICS_API_TOKEN=local-private-service-token +GLASSBOX_CONTROL_API_URL=http://127.0.0.1:8790 +GLASSBOX_CONTROL_API_TOKEN=local-private-control-token +GLASSBOX_ALLOW_LOCAL_OPERATOR=true +``` + +The HTTP surface is loopback-only unless a private service bearer token is supplied. +Remote access belongs behind the authenticated operator boundary in +[`deploy/production`](../../deploy/production/README.md); the browser never receives +the private API credential, PostgreSQL DSN, signing material, control master key, or +DataHub service-account token. + +## Connection Center + +The Connections route talks to the private `glassbox-control` service through a +server-side allowlisted proxy. An administrator can run a real DataHub SDK test, +prove write permission through a deterministic synthetic Document and direct +readback, save the token encrypted, and create or revoke named agent ingestion keys. +The clear ingestion key is displayed once. A configured DataHub UI origin activates +top-level and entity-specific deep links without putting the service-account token +in the link or browser. + +For the live flagship, run `examples.flagship_demo` with `--keep-estate`. The +raw-free report records the retained schema as `estate.state_postgres_schema`; +pass that exact value to the service with `--state-postgres-schema`. The flagship +uses an ephemeral demo signer, so local inspection may use +`--allow-untrusted-signers`; production must supply an operator trust policy. ## Local development -Requires Node.js 22.13 or newer. +Node.js 22.13 or newer is required. ```bash npm install -npm run dev +GLASSBOX_PUBLIC_HOSTS=glassbox.localhost npm run dev ``` -The development server prints the exact local URL. The preferred quality gate -is: +One build intentionally has two faces. `http://glassbox.localhost:3000` serves +the public landing page and documentation, while `http://localhost:3000` keeps +the operator console at `/`. A self-hosted organization that leaves +`GLASSBOX_PUBLIC_HOSTS` empty receives only the console. The production reference +uses `glassboxhq.xyz` for the public surface and `app.glassboxhq.xyz` for the +OAuth-protected console; public hosts cannot reach operator or control API routes. + +Quality gates: ```bash npm test @@ -31,18 +75,9 @@ npm run lint npm run typecheck ``` -`npm test` creates a production build and checks the rendered investigation for -the core forensic claims and the absence of starter or secret material. The console -does not declare database or object-storage bindings because the current experience -is a read-only projection of verified GlassBox evidence. - -## Safety posture - -This console is deliberately raw-free. It renders receipt digests, governed -metadata, reason codes, evidence identifiers, and proof-gate outcomes. It does -not render model prompts, raw tool payloads, credentials, query results, or -arbitrary DataHub custom properties. +## Trust boundary -The console is a presentation layer. Receipt verification, materiality -classification, quarantine, and replay authorization remain enforced by the -GlassBox backend and the `datahub-agent-forensics` skill. +The console displays only bounded receipt projections, governed URNs, +deterministic findings, workflow state, and fresh verification results. It never +receives prompts, outputs, tool bodies, credentials, raw query results, or signing +keys. Policy, quarantine, and recovery authority remain in the GlassBox services. diff --git a/apps/console/app/(console)/campaigns/[campaignId]/page.tsx b/apps/console/app/(console)/campaigns/[campaignId]/page.tsx new file mode 100644 index 0000000..8dbf509 --- /dev/null +++ b/apps/console/app/(console)/campaigns/[campaignId]/page.tsx @@ -0,0 +1,23 @@ +import Link from "next/link"; +import { IdentifierControl } from "../../../components/identifier-control"; +import { DataHubEntityLink } from "../../../components/console-shell"; +import { ConnectionEmpty, DateValue, PageHeader, StatusPill, formatUrn, humanize, shortDigest, shortId } from "../../../components/ui"; +import { getCampaign } from "../../../lib/forensics-api"; + +export default async function CampaignDetailPage({ params }: { params: Promise<{ campaignId: string }> }) { + const campaignId = decodeURIComponent((await params).campaignId); + const result = await getCampaign(campaignId); + const campaign = result.data?.campaign; + return <> + } action={
{campaign && }← Back to campaigns
} /> + {!campaign ? : ( +
+
+
C

DataHub metadata change

{humanize(campaign.change.kind)}

{formatUrn(campaign.change.entity_urn)}
OccurredAspect{campaign.change.aspect_name}Assessments{campaign.assessments.length}Writeback{humanize(campaign.processing.datahub_writeback_state)}
+

Deterministic policy

Decision findings

{campaign.policy_version}
{campaign.assessments.map((assessment) => D
{shortDigest(assessment.receipt_id)}{humanize(assessment.reason_code)}
β†’)}
+
+ +
+ )} + ; +} diff --git a/apps/console/app/(console)/campaigns/page.tsx b/apps/console/app/(console)/campaigns/page.tsx new file mode 100644 index 0000000..9760967 --- /dev/null +++ b/apps/console/app/(console)/campaigns/page.tsx @@ -0,0 +1,24 @@ +import Link from "next/link"; +import { ConnectionEmpty, EmptyRecords, PageHeader, StatusPill, formatDate, formatUrn, humanize } from "../../components/ui"; +import { getCampaigns } from "../../lib/forensics-api"; + +export default async function CampaignsPage() { + const result = await getCampaigns(); + return <> + + {result.data === null ? : ( +
+
Invalidation historyDirect Action state, not projected UI state
{result.data.total} campaigns
+ {result.data.campaigns.length ?
+
DataHub changeFindingsWritebackProcessed
+ {result.data.campaigns.map((campaign) => + C{humanize(campaign.change.kind)}{formatUrn(campaign.change.entity_urn)} + {campaign.assessments.slice(0, 2).map((item) => )} + + {formatDate(campaign.change.occurred_at)}β†’ + )} +
: } +
+ )} + ; +} diff --git a/apps/console/app/(console)/investigations/[receiptId]/page.tsx b/apps/console/app/(console)/investigations/[receiptId]/page.tsx new file mode 100644 index 0000000..6ada2a3 --- /dev/null +++ b/apps/console/app/(console)/investigations/[receiptId]/page.tsx @@ -0,0 +1,112 @@ +import Link from "next/link"; +import { IdentifierControl } from "../../../components/identifier-control"; +import { DataHubEntityLink } from "../../../components/console-shell"; +import { ConnectionEmpty, DateValue, EmptyRecords, Icon, PageHeader, StatusPill, formatDate, formatUrn, humanize, shortDigest } from "../../../components/ui"; +import { getPublicationReadback, type PublicationReadback } from "../../../lib/control-api"; +import { getFindings, getReceipt } from "../../../lib/forensics-api"; + +export default async function InvestigationDetailPage({ params }: { params: Promise<{ receiptId: string }> }) { + const receiptId = decodeURIComponent((await params).receiptId); + const [receipt, findings, readback] = await Promise.all([getReceipt(receiptId), getFindings(receiptId), getPublicationReadback(receiptId)]); + const proof = receipt.data ? publicationProof(receipt.data.publication, receipt.data.influence.document_urn, readback.data) : null; + return <> + } + action={
{receipt.data && }← Back to investigations
} + /> + {receipt.data === null ? : ( +
+
+
+
D

Verification

{humanize(receipt.data.verification.verification_state)}

{formatUrn(receipt.data.influence.document_urn)}
+
EndedDependencies{receipt.data.influence.completeness.recorded_dependencies}Resolution{humanize(receipt.data.influence.completeness.dependency_resolution)}Field coverage{humanize(receipt.data.influence.completeness.field_lineage_coverage)}
+
+ {proof &&
+

Publication proof

DataHub durability

+
+

{proof.verified ? "Cross-plane proof complete" : "Proof requires attention"}

{proof.verified ? "Persisted and verified now" : humanize(proof.state)}

{proof.documentUrn ? formatUrn(proof.documentUrn) : "No sealed document URN"}
+
Durable state
{proof.workflowStatus}
{proof.authority}
Fresh readback
{proof.readbackState}
Direct from DataHub
Sealed aspects
{proof.matchedAspectCount}/{proof.sealedAspectCount}
Present now
Idempotent writes
{proof.emissionCount ?? "β€”"}
Recorded emissions
+ {proof.sealedAspects.length > 0 &&
{proof.sealedAspects.map((aspect) => {aspect})}
} + {!proof.verified &&

{readback.message ?? "The durable record and current DataHub state do not agree."}

} +
+
} +
+

Runtime influence

Dependencies

{receipt.data.influence.dependencies.length} records
+ {receipt.data.influence.dependencies.length ?
{receipt.data.influence.dependencies.map((dependency) =>
)}
: } +
+
+

Action history

Findings

{findings.data?.findings_total ?? 0} findings
+ {findings.data?.findings.length ?
{findings.data.findings.map((finding) =>
: } +
+
+ +
+ )} + ; +} + +type PublicationState = { + receipt_id: string; + availability: string; + durability?: { + authority: string; + workflow_status: string; + attempt_count: number; + last_error_recorded: boolean; + sealed_evidence: boolean; + }; + datahub?: { + document_urn: string | null; + aspect_names: string[]; + aspect_count: number; + emission_count: number | null; + }; +}; + +function publicationProof( + publication: PublicationState, + influenceDocumentUrn: string, + readback: PublicationReadback | null, +) { + const sealedAspects = publication.datahub?.aspect_names ?? []; + const currentAspects = new Set(readback?.aspect_names ?? []); + const matchedAspectCount = sealedAspects.filter((aspect) => currentAspects.has(aspect)).length; + const identifiersMatch = Boolean( + readback && + readback.receipt_id === publication.receipt_id && + readback.document_urn === publication.datahub?.document_urn && + readback.document_urn === influenceDocumentUrn, + ); + const durable = Boolean( + publication.availability === "AVAILABLE" && + publication.durability?.authority === "POSTGRESQL" && + publication.durability.workflow_status === "COMPLETED" && + publication.durability.sealed_evidence && + !publication.durability.last_error_recorded && + publication.datahub?.emission_count === 2, + ); + const verified = Boolean( + durable && + identifiersMatch && + readback?.verification_state === "VERIFIED_NOW" && + sealedAspects.length > 0 && + matchedAspectCount === sealedAspects.length, + ); + return { + verified, + state: verified ? "VERIFIED_NOW" : readback ? "MISMATCH" : "UNAVAILABLE", + documentUrn: publication.datahub?.document_urn ?? null, + workflowStatus: publication.durability?.workflow_status ?? humanize(publication.availability), + authority: publication.durability?.authority ?? "No durable authority", + readbackState: readback?.verification_state ?? "UNAVAILABLE", + sealedAspects, + sealedAspectCount: sealedAspects.length, + matchedAspectCount, + emissionCount: publication.datahub?.emission_count ?? null, + }; +} diff --git a/apps/console/app/(console)/investigations/page.tsx b/apps/console/app/(console)/investigations/page.tsx new file mode 100644 index 0000000..005183b --- /dev/null +++ b/apps/console/app/(console)/investigations/page.tsx @@ -0,0 +1,47 @@ +import Link from "next/link"; +import { ConnectionEmpty, EmptyRecords, PageHeader, StatusPill, byRisk, formatDate, formatUrn, needsReview, shortDigest } from "../../components/ui"; +import { getDecisions } from "../../lib/forensics-api"; + +export default async function InvestigationsPage({ searchParams }: { searchParams: Promise<{ query?: string; show?: string }> }) { + const { query = "", show } = await searchParams; + const result = await getDecisions(query); + const showAll = show === "all"; + + // `/receipts` is the complete append-only index. This page is the work + // queue, so it defaults to the states that actually need an operator. + const all = result.data?.decisions ?? []; + const review = all.filter((decision) => needsReview(decision.state)); + const shown = [...(showAll ? all : review)].sort(byRisk); + const suffix = query ? `&query=${encodeURIComponent(query)}` : ""; + + return ( + <> + + {result.data === null ? : ( +
+
+
+ {shown.length ?
+
Decision receiptStateDependenciesEnded
+ {shown.map((decision) => + D{shortDigest(decision.receipt_id)}{formatUrn(decision.document_urn)} + + {decision.resolved_dependency_count}/{decision.dependency_count}{decision.field_lineage_coverage.toLowerCase()} field coverage + {formatDate(decision.ended_at)} + )} +
: } +
+ )} + + ); +} diff --git a/apps/console/app/(console)/layout.tsx b/apps/console/app/(console)/layout.tsx new file mode 100644 index 0000000..ffcefe2 --- /dev/null +++ b/apps/console/app/(console)/layout.tsx @@ -0,0 +1,28 @@ +import { ConsoleShell } from "../components/console-shell"; +import { getConnectionState } from "../lib/forensics-api"; +import { getDataHubUiOrigin } from "../lib/control-api"; + +// Operator routes depend on authenticated request headers and runtime service +// configuration. Rendering this route group at build time would freeze an +// anonymous, unconfigured snapshot into the production image. +export const dynamic = "force-dynamic"; + +/** + * Wraps every operator route in the console shell. Scoped to this route group + * so `/docs` and `/architecture` render without the sidebar and topbar. + */ +export default async function ConsoleLayout({ + children, +}: Readonly<{ + children: React.ReactNode; +}>) { + const [connection, datahubUiOrigin] = await Promise.all([ + getConnectionState(), + getDataHubUiOrigin(), + ]); + return ( + + {children} + + ); +} diff --git a/apps/console/app/(console)/page.tsx b/apps/console/app/(console)/page.tsx new file mode 100644 index 0000000..700dcdc --- /dev/null +++ b/apps/console/app/(console)/page.tsx @@ -0,0 +1,58 @@ +import Link from "next/link"; +import { ConnectionEmpty, EmptyRecords, Icon, PageHeader, StatusPill, byRisk, compactChangeLabel, compactUrn, formatDate, shortDigest } from "../components/ui"; +import { getCampaigns, getDecisions, getOverview } from "../lib/forensics-api"; + +export default async function OverviewPage() { + const [overview, decisions, campaigns] = await Promise.all([ + getOverview(), + getDecisions(), + getCampaigns(), + ]); + return ( + <> + + {overview.data === null ? ( + + ) : ( + <> +
+

Needs review

{overview.data.counts.review_required}Stale, at risk, or unknown
+

Decision receipts

{overview.data.counts.receipts}In the configured index
+

Dependencies

{overview.data.counts.dependencies}{overview.data.counts.unresolved_dependencies} unresolved
+

Campaigns

{overview.data.counts.campaigns}Persisted Action history
+
+
+
+

Highest risk first

Investigation queue

View all
+ {decisions.data && decisions.data.decisions.length ? ( +
+ {[...decisions.data.decisions].sort(byRisk).slice(0, 5).map((decision) => ( + + D + {shortDigest(decision.receipt_id)}{decision.dependency_count} dependencies Β· {formatDate(decision.ended_at)} + β†’ + + ))} +
+ ) : } +
+
+

DataHub changes

Latest campaigns

View all β†’
+ {campaigns.data && campaigns.data.campaigns.length ? ( +
+ {campaigns.data.campaigns.slice(0, 4).map((campaign) => ( + +
+ ) : } +
+
+ + )} + + ); +} diff --git a/apps/console/app/(console)/receipts/page.tsx b/apps/console/app/(console)/receipts/page.tsx new file mode 100644 index 0000000..b9705f5 --- /dev/null +++ b/apps/console/app/(console)/receipts/page.tsx @@ -0,0 +1,22 @@ +import Link from "next/link"; +import { ConnectionEmpty, DateValue, EmptyRecords, PageHeader, StatusPill, formatUrn, shortDigest } from "../../components/ui"; +import { getDecisions } from "../../lib/forensics-api"; + +export default async function ReceiptsPage() { + const result = await getDecisions(); + return <> + + {result.data === null ? : ( +
+
Receipt indexRaw content is never returned
{result.data.total} receipts
+ {result.data.decisions.length ?
+ {result.data.decisions.map((receipt) => +
R
+ {shortDigest(receipt.receipt_id)}{formatUrn(receipt.document_urn)} +
Dependencies
{receipt.dependency_count}
Resolved
{receipt.resolved_dependency_count}
Ended
+ )} +
: } +
+ )} + ; +} diff --git a/apps/console/app/(console)/recovery/page.tsx b/apps/console/app/(console)/recovery/page.tsx new file mode 100644 index 0000000..f93da70 --- /dev/null +++ b/apps/console/app/(console)/recovery/page.tsx @@ -0,0 +1,20 @@ +import Link from "next/link"; +import { ConnectionEmpty, EmptyRecords, PageHeader, StatusPill, formatUrn, humanize } from "../../components/ui"; +import { getCampaigns } from "../../lib/forensics-api"; + +export default async function RecoveryPage() { + const result = await getCampaigns(); + const recoverable = result.data?.campaigns.filter((campaign) => campaign.assessments.some((item) => item.quarantine_required)) ?? []; + return <> + + {result.data === null ? : recoverable.length ? ( +
+ {recoverable.map((campaign) =>
+
R

{humanize(campaign.change.kind)}

{formatUrn(campaign.change.entity_urn)}
+
1Impact classified2Decision quarantined3Recovery authorization
+
Awaiting authorization Β· {campaign.assessments.filter((item) => item.quarantine_required).length} decision{campaign.assessments.filter((item) => item.quarantine_required).length === 1 ? "" : "s"}Review campaign β†’
+
)} +
+ ) : } + ; +} diff --git a/apps/console/app/(console)/settings/connection-center.tsx b/apps/console/app/(console)/settings/connection-center.tsx new file mode 100644 index 0000000..5310cda --- /dev/null +++ b/apps/console/app/(console)/settings/connection-center.tsx @@ -0,0 +1,149 @@ +"use client"; + +import { FormEvent, useState, useTransition } from "react"; +import type { + ConnectionProof, + ControlSnapshot, + DataHubConnectionSummary, + IngestionKeySummary, +} from "../../lib/control-api"; +import { DateValue, StatusPill } from "../../components/ui"; + +type Notice = { tone: "success" | "error" | "neutral"; text: string } | null; + +export function ConnectionCenter({ snapshot }: { snapshot: ControlSnapshot }) { + const [tab, setTab] = useState<"datahub" | "agents">("datahub"); + const [connection, setConnection] = useState(snapshot.connection); + const [keys, setKeys] = useState(snapshot.keys); + const [report, setReport] = useState(null); + const [oneTimeSecret, setOneTimeSecret] = useState(null); + const [notice, setNotice] = useState( + snapshot.message ? { tone: "neutral", text: snapshot.message } : null, + ); + const [pending, startTransition] = useTransition(); + const canManage = snapshot.role === "admin"; + + async function connectionAction(form: HTMLFormElement, persist: boolean) { + const values = new FormData(form); + const payload = { + server_url: String(values.get("server_url") ?? ""), + ui_url: String(values.get("ui_url") ?? ""), + token: String(values.get("token") ?? ""), + write_proof: values.get("write_proof") === "on", + }; + setNotice(null); + startTransition(async () => { + const response = await fetch(`/api/control/${persist ? "connection" : "connection/test"}`, { + method: persist ? "PUT" : "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify(payload), + }); + const body = await response.json(); + if (!response.ok) { + setNotice({ tone: "error", text: errorMessage(body) }); + return; + } + const nextReport = (body.report ?? body.connection?.probe) as ConnectionProof; + setReport(nextReport); + if (persist) { + setConnection(body.connection as DataHubConnectionSummary); + form.reset(); + setNotice({ + tone: "success", + text: "DataHub is verified and encrypted. Runtime services can now reload this connection.", + }); + } else { + setNotice({ tone: "success", text: "The live DataHub verification completed." }); + } + }); + } + + async function createKey(event: FormEvent) { + event.preventDefault(); + const form = event.currentTarget; + const name = String(new FormData(form).get("name") ?? ""); + setNotice(null); + startTransition(async () => { + const response = await fetch("/api/control/ingestion-keys", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ name }), + }); + const body = await response.json(); + if (!response.ok) { + setNotice({ tone: "error", text: errorMessage(body) }); + return; + } + setKeys((current) => [body.key as IngestionKeySummary, ...current]); + setOneTimeSecret(body.secret as string); + setNotice({ tone: "success", text: "Ingestion key created. Copy it now; it is shown once." }); + form.reset(); + }); + } + + function revokeKey(keyId: string) { + setNotice(null); + startTransition(async () => { + const response = await fetch(`/api/control/ingestion-keys/${keyId}`, { method: "DELETE" }); + const body = await response.json(); + if (!response.ok) { + setNotice({ tone: "error", text: errorMessage(body) }); + return; + } + setKeys((current) => current.map((key) => + key.key_id === keyId ? { ...key, state: "REVOKED" as const } : key, + )); + setNotice({ tone: "success", text: "The ingestion key is revoked." }); + }); + } + + return ( +
+
+ + +
+ + {notice &&
{notice.text}
} + + {tab === "datahub" ?
+
+

Organization connection

{connection ? "Update DataHub" : "Connect DataHub"}

{connection && }
+

Use a scoped DataHub service account. The token travels to the private control plane, is verified against DataHub, and is stored encrypted.

+
{ event.preventDefault(); void connectionAction(event.currentTarget, true); }}> + + + + +
+
+
+ +
:
+

Agent authentication

Create an ingestion key

Give each agent or environment its own key. GlassBox stores only a keyed digest, and the receiver checks revocation on every request.
+ {oneTimeSecret &&

Copy this key now

{oneTimeSecret}It cannot be recovered after you leave this page.
} +

Issued credentials

Ingestion keys

{keys.filter((key) => key.state === "ACTIVE").length} active
{keys.length ?
{keys.map((key) =>
)}
:
No ingestion keys have been issued.
}
+
} +
+ ); +} + +function ConnectionProofCard({ connection, report }: { connection: DataHubConnectionSummary | null; report: ConnectionProof | null }) { + const proof = report ?? connection?.probe ?? null; + return ; +} + +function ProofRow({ label, value }: { label: string; value: string }) { + return
{label}
; +} + +function errorMessage(body: unknown) { + if (body && typeof body === "object") { + const error = (body as { error?: unknown }).error; + if (error && typeof error === "object") { + const message = (error as { message?: unknown }).message; + if (typeof message === "string") return message; + } + } + return "The operation did not complete."; +} diff --git a/apps/console/app/(console)/settings/page.tsx b/apps/console/app/(console)/settings/page.tsx new file mode 100644 index 0000000..66c6f5b --- /dev/null +++ b/apps/console/app/(console)/settings/page.tsx @@ -0,0 +1,14 @@ +import { PageHeader, StatusPill } from "../../components/ui"; +import { configuredApiUrl, getConnectionState } from "../../lib/forensics-api"; +import { getControlSnapshot } from "../../lib/control-api"; +import { ConnectionCenter } from "./connection-center"; + +export default async function SettingsPage() { + const [connection, control] = await Promise.all([getConnectionState(), getControlSnapshot()]); + const endpoint = configuredApiUrl(); + return <> + + +
G

GlassBox evidence plane

{connection === "connected" ? "Connected" : "Unavailable"}

Endpoint
{endpoint ? "Private service network" : "Not configured"}
Policy
Read only Β· raw free
+ ; +} diff --git a/apps/console/app/(console)/trust/page.tsx b/apps/console/app/(console)/trust/page.tsx new file mode 100644 index 0000000..3312bc1 --- /dev/null +++ b/apps/console/app/(console)/trust/page.tsx @@ -0,0 +1,15 @@ +import { ConnectionEmpty, Icon, PageHeader, StatusPill } from "../../components/ui"; +import { getOverview } from "../../lib/forensics-api"; + +export default async function TrustPage() { + const result = await getOverview(); + return <> + + {result.data === null ? : ( +
+

Evidence boundary

Raw-free operator surface

The console receives receipt projections, deterministic findings, and verified processing state. It does not receive prompts, outputs, tool bodies, credentials, or signing keys.
+
Receipt index
Campaign store
Unresolved dependencies{result.data.counts.unresolved_dependencies}
Raw content returnedNever
+
+ )} + ; +} diff --git a/apps/console/app/api/control/[...path]/route.ts b/apps/console/app/api/control/[...path]/route.ts new file mode 100644 index 0000000..0040c04 --- /dev/null +++ b/apps/console/app/api/control/[...path]/route.ts @@ -0,0 +1,13 @@ +import { proxyControlRequest } from "../../../lib/control-api"; + +type Context = { params: Promise<{ path: string[] }> }; + +async function handle(request: Request, context: Context) { + const { path } = await context.params; + return proxyControlRequest(path.join("/"), request); +} + +export const GET = handle; +export const POST = handle; +export const PUT = handle; +export const DELETE = handle; diff --git a/apps/console/app/architecture/page.tsx b/apps/console/app/architecture/page.tsx new file mode 100644 index 0000000..da40a73 --- /dev/null +++ b/apps/console/app/architecture/page.tsx @@ -0,0 +1,5 @@ +import { redirect } from "next/navigation"; + +export default function ArchitectureAlias() { + redirect("/docs/architecture"); +} diff --git a/apps/console/app/components/console-shell.tsx b/apps/console/app/components/console-shell.tsx new file mode 100644 index 0000000..549a4de --- /dev/null +++ b/apps/console/app/components/console-shell.tsx @@ -0,0 +1,208 @@ +"use client"; + +import Link from "next/link"; +import { usePathname, useRouter } from "next/navigation"; +import { createContext, FormEvent, ReactNode, useContext, useState, useSyncExternalStore, useTransition } from "react"; +import { Icon, type IconName } from "./ui"; +import type { ConnectionState } from "../lib/forensics-api"; + +const primary: ReadonlyArray<{ href: string; label: string; icon: IconName }> = [ + { href: "/", label: "Overview", icon: "grid" }, + { href: "/investigations", label: "Investigations", icon: "search" }, + { href: "/receipts", label: "Receipts", icon: "receipt" }, + { href: "/campaigns", label: "Campaigns", icon: "pulse" }, + { href: "/recovery", label: "Recovery", icon: "replay" }, +]; + +const system: ReadonlyArray<{ href: string; label: string; icon: IconName }> = [ + { href: "/trust", label: "Trust & privacy", icon: "shield" }, + { href: "/settings", label: "Connections", icon: "settings" }, +]; + +const EXACT_RECEIPT_ID = /^gbx:receipt:sha256:[0-9a-f]{64}$/i; +const DataHubUiContext = createContext(null); + +/* + * The theme lives on , written by the pre-paint script in the layout + * before React exists. Reading it through an external store keeps the server + * snapshot at the light default while the client reads the real attribute, so + * a stored dark preference resolves without a hydration mismatch. + */ +const themeListeners = new Set<() => void>(); + +function subscribeTheme(onChange: () => void) { + themeListeners.add(onChange); + return () => void themeListeners.delete(onChange); +} + +function readTheme(): "light" | "dark" { + return document.documentElement.dataset.theme === "dark" ? "dark" : "light"; +} + +function applyTheme(next: "light" | "dark") { + if (next === "dark") document.documentElement.dataset.theme = "dark"; + else delete document.documentElement.dataset.theme; + try { + localStorage.setItem("gbx-theme", next); + } catch { + // A blocked storage backend still gets the toggle for this session. + } + themeListeners.forEach((notify) => notify()); +} + +export function ConsoleShell({ + connection, + datahubUiOrigin, + children, +}: { + connection: ConnectionState; + datahubUiOrigin: string | null; + children: ReactNode; +}) { + const pathname = usePathname(); + const router = useRouter(); + const [menuOpen, setMenuOpen] = useState(false); + const [lookup, setLookup] = useState(""); + const [refreshing, startRefresh] = useTransition(); + const theme = useSyncExternalStore(subscribeTheme, readTheme, () => "light" as const); + + function active(href: string) { + return href === "/" ? pathname === "/" : pathname.startsWith(href); + } + + // Exact receipt IDs open directly. Partial IDs and URNs use the filterable + // list so an inexact operator search never turns into a detail-page 404. + function navigateLookup(rawTerm: string) { + const term = rawTerm.trim(); + if (!term) return; + setLookup(""); + router.push( + EXACT_RECEIPT_ID.test(term) + ? `/investigations/${encodeURIComponent(term)}` + : `/investigations?show=all&query=${encodeURIComponent(term)}`, + ); + } + + function submitLookup(event: FormEvent) { + event.preventDefault(); + // Read the submitted DOM value instead of relying on the latest React + // render. This keeps rapid type-then-Enter lookup deterministic. + navigateLookup(String(new FormData(event.currentTarget).get("lookup") ?? "")); + } + + function toggleTheme() { + applyTheme(theme === "dark" ? "light" : "dark"); + } + + return +
+ + +
+
+ +
+ + setLookup(event.target.value)} + onKeyDown={(event) => { + if (event.key === "Enter") { + event.preventDefault(); + navigateLookup(event.currentTarget.value); + } + }} + /> + +
+
+ {datahubUiOrigin && DataHub} + Raw-free + + +
+
+
{children}
+
+ {menuOpen &&
+ ; +} + +export function DataHubEntityLink({ urn, label = "Open in DataHub" }: { urn: string; label?: string }) { + const origin = useContext(DataHubUiContext); + if (!origin) return null; + const linkedUrn = schemaFieldDatasetUrn(urn) ?? urn; + const entityType = linkedUrn.split(":", 4)[2]; + if (!entityType) return null; + return {label}; +} + +function schemaFieldDatasetUrn(urn: string) { + if (!urn.startsWith("urn:li:schemaField:(urn:li:dataset:")) return null; + const closing = urn.lastIndexOf("),"); + return closing > 0 ? urn.slice("urn:li:schemaField:(".length, closing + 1) : null; +} diff --git a/apps/console/app/components/identifier-control.tsx b/apps/console/app/components/identifier-control.tsx new file mode 100644 index 0000000..5193273 --- /dev/null +++ b/apps/console/app/components/identifier-control.tsx @@ -0,0 +1,43 @@ +"use client"; + +import { useState } from "react"; + +export function IdentifierControl({ + value, + compact, + label, +}: { + value: string; + compact: string; + label: string; +}) { + const [expanded, setExpanded] = useState(false); + const [copied, setCopied] = useState(false); + + async function copy() { + try { + await navigator.clipboard.writeText(value); + setCopied(true); + window.setTimeout(() => setCopied(false), 1600); + } catch { + setCopied(false); + } + } + + return ( + + {expanded ? value : compact} + + + + + + ); +} diff --git a/apps/console/app/components/ui.tsx b/apps/console/app/components/ui.tsx new file mode 100644 index 0000000..aac78de --- /dev/null +++ b/apps/console/app/components/ui.tsx @@ -0,0 +1,214 @@ +import Link from "next/link"; +import type { ReactNode } from "react"; +import type { ConnectionState } from "../lib/forensics-api"; + +export function PageHeader({ + eyebrow, + title, + description, + action, +}: { + eyebrow: string; + title: string; + description?: ReactNode; + action?: ReactNode; +}) { + return ( +
+

{eyebrow}

{title}

{description && {description}}
+ {action} +
+ ); +} + +export function ConnectionEmpty({ + connection, + message, +}: { + connection: ConnectionState; + message: string | null; +}) { + return ( +
+ +
+

{connection === "not-configured" ? "Connection required" : "Connection unavailable"}

+

{connection === "not-configured" ? "Connect your evidence plane" : "GlassBox cannot reach the evidence plane"}

+ {message ?? "No application data was loaded."} +
+ Open settings +
+ ); +} + +export function EmptyRecords({ title, description }: { title: string; description: string }) { + return

{title}

{description}

; +} + +/* + * Service enums carry three different axes β€” decision trust, processing + * progress, and store availability β€” through one pill. Mapping them + * explicitly keeps an unrecognised enum visibly neutral rather than letting a + * near-miss (FAILED vs FAIL) silently render as an unremarkable grey chip. + */ +const STATE_META: Record = { + // Decision trust + STALE: { label: "Stale", tone: "danger" }, + AT_RISK: { label: "At risk", tone: "danger" }, + UNKNOWN: { label: "Unknown", tone: "warn" }, + NO_RECORDED_FINDING: { label: "No findings", tone: "neutral" }, + UNAFFECTED: { label: "Unaffected", tone: "safe" }, + SUPERSEDED: { label: "Superseded", tone: "muted" }, + VERIFIED: { label: "Verified", tone: "safe" }, + VERIFIED_NOW: { label: "Verified", tone: "safe" }, + PROVEN: { label: "Proven", tone: "safe" }, + UNVERIFIED: { label: "Unverified", tone: "warn" }, + ACTIVE: { label: "Active", tone: "safe" }, + REVOKED: { label: "Revoked", tone: "danger" }, + // Processing + COMPLETED: { label: "Completed", tone: "safe" }, + PENDING: { label: "Pending", tone: "warn" }, + AWAITING_AUTHORIZATION: { label: "Awaiting authorization", tone: "warn" }, + FAILED: { label: "Failed", tone: "danger" }, + NOT_REQUIRED: { label: "Not required", tone: "muted" }, + // Completeness and availability + COMPLETE: { label: "Complete", tone: "safe" }, + INCOMPLETE: { label: "Incomplete", tone: "warn" }, + AVAILABLE: { label: "Available", tone: "safe" }, + UNAVAILABLE: { label: "Unavailable", tone: "danger" }, + OBSERVED: { label: "Observed", tone: "neutral" }, +}; + +/** Ordering for triage surfaces: the least trustworthy decision sorts first. */ +const RISK_RANK: Record = { + STALE: 4, + AT_RISK: 4, + UNKNOWN: 3, + NO_RECORDED_FINDING: 2, + UNAFFECTED: 1, + SUPERSEDED: 0, +}; + +/** States that put a decision in the operator's work queue. */ +export const REVIEW_STATES = ["STALE", "AT_RISK", "UNKNOWN"]; + +export function needsReview(state: string) { + return REVIEW_STATES.includes(state); +} + +export function byRisk(a: T, b: T) { + const rank = (RISK_RANK[b.state] ?? 2) - (RISK_RANK[a.state] ?? 2); + return rank !== 0 ? rank : (b.ended_at ?? "").localeCompare(a.ended_at ?? ""); +} + +export function StatusPill({ state }: { state: string }) { + const meta = STATE_META[state] ?? { label: humanize(state), tone: "neutral" }; + return