diff --git a/docs/enterprise-document-evidence-envelope.md b/docs/enterprise-document-evidence-envelope.md new file mode 100644 index 0000000..60ccc45 --- /dev/null +++ b/docs/enterprise-document-evidence-envelope.md @@ -0,0 +1,199 @@ +--- +type: Concept +title: Enterprise-document evidence envelope +okf_status: active +tags: + - public + - developer +freshness: "2026-09-15" +resource: https://github.com/alphazede/bran +public_boundary: public +--- + +# Enterprise-document evidence envelope + +This is the V1 contract for [issue #20](https://github.com/alphazede/bran/issues/20). +A managed parser (Google Document AI or another owner-approved tool) reads +native DOCX, XLSX, PPTX, or PDF. BRAN does not. BRAN validates a JSON envelope +around that output so the evidence can enter provenance, DLP, public-boundary, +query, and packet workflows without executing the original file. + +The schema is +[`schemas/enterprise-document-evidence-envelope.schema.json`](../schemas/enterprise-document-evidence-envelope.schema.json). +JSON Schema cannot bind digest equality, asset-path safety, family/media +pairing, or relation acyclicity. Those rules live in +[`tools/ci/enterprise_contract_check.py`](../tools/ci/enterprise_contract_check.py), +the schema's `x-semantic-oracle`. Default tests and the checker make no +provider or network calls. + +## Product boundary + +The parser owns native extraction and parser-specific limits. BRAN owns: + +- original media type, byte length, and SHA-256 +- parser identity, processor, version, and attestation state +- source locator plus revision state (`attested` with a value, or + `unavailable` with a null value) +- deterministic evidence and anchor identities +- hashes of normalized content and referenced assets +- fidelity, truncation, malformed-input, and unavailable receipts +- containment, byte budgets, DLP, classification, and public-boundary outcomes +- admission into query and packet paths + +An unavailable parser, revision, classification, or public-boundary claim +stays unavailable. BRAN does not invent completeness, permission, fidelity, +or revision evidence. + +Never store or return credentials, OAuth tokens, signed URLs, cookies, or +raw authentication state in any envelope field — including `source.locator`, +parser strings, asset paths/roles, receipts, reasons, and relation ids. The +oracle rejects envelopes whose strings contain a secret marker +(`X-Goog-Signature=`, `X-Goog-Credential=`, `access_token=`, +`refresh_token=`, `ya29.`, `AIza`, `private_key`, or `-----BEGIN `) as +`secret-reflection`, before digest checks. That list is substring matching, +not a DLP product. + +## Envelope + +Required keys, no extras: `schema_version` (`1.0.0`), `evidence_id`, +`envelope_digest`, `original`, `parser`, `source`, `normalized`, `anchors`, +`assets`, `relations`, `fidelity`, `receipts`, `hazards`, `policy`, +`admission`. + +Fail-closed budgets: canonical envelope 1 MiB, original 20 MiB, 4096 +anchors, 256 assets, 8192 relations, 8192 UTF-8 bytes per text field. +Overrun is `oversized`. + +`original.media_type` is one of PDF, DOCX, PPTX, or XLSX. That media type +selects exactly one content family: + +| Media type | Family | +|---|---| +| `application/pdf` | `fixed-layout` | +| WordprocessingML DOCX | `flow` | +| PresentationML PPTX | `presentation` | +| SpreadsheetML XLSX | `grid` | + +`normalized.content.family` and every anchor family must match that pairing. +A mismatch is `unsupported-evidence`. + +## Canonical bytes and digests + +Canonical bytes are UTF-8 +`json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=False)`. + +- `envelope_digest` is SHA-256 of those bytes after omitting `envelope_digest`. +- `normalized.digest` is SHA-256 of the canonical `normalized.content` object. +- Each `anchors[].text_digest` is SHA-256 of the UTF-8 anchor text. + +Equivalent logical input therefore serializes byte-identically. Object key +order must not change a digest. The oracle proves this on the first admitted +positive fixture by reversing object keys; it does not reorder arrays or +repeat the check on the other three positives. + +## Anchors, assets, and relations + +Anchors are the citable units. Each has a stable `id`, family-specific +`role`, text, `text_digest`, and a locator: + +- `fixed-layout`: page, block, bbox +- `flow`: section, ordinal +- `presentation`: slide, shape, z_index +- `grid`: sheet, row, column + +Allowed roles by family: + +| Family | Roles | +|---|---| +| `fixed-layout` | heading, paragraph, table, figure, header, footer, title, list | +| `flow` | heading, paragraph, table, list, header, footer, title | +| `presentation` | heading, paragraph, table, list, title, notes, shape | +| `grid` | sheet, cell, range, table, header, chart | + +The schema `role` enum is the union of those sets. The oracle requires the +family subset. + +Anchor ids are unique and sorted. Asset ids share that namespace, are unique +against anchors, and must themselves be sorted. Assets are content-addressed. +Asset `path` is a relative POSIX path of `[A-Za-z0-9._-]` segments separated +by `/`, at most 256 bytes, with no `.` or `..` segment, no leading slash, no +backslash, and no NUL. Spaces fail as `unsafe-asset-path`. + +Relations (`derived-from`, `cites`, `extracts`, `summarizes`) must name +existing ids, must not be self-loops, and must be unique sorted tuples. +`derived-from` and `extracts` share one acyclicity graph: a cycle that +alternates those kinds still fails. + +## Fidelity, receipts, hazards, and policy + +`fidelity` keys must be exactly the family set. Values are `exact`, +`normalized`, `approximated`, or `unsupported`. + +| Family | Features | +|---|---| +| `fixed-layout` | text, reading_order, bounding_boxes, tables, figures, ocr, javascript | +| `flow` | text, paragraphs, headings, lists, tables, headers_footers, macros | +| `presentation` | text, slides, shapes, speaker_notes, z_order, macros, animations | +| `grid` | text, sheets, cells, formulas, charts, macros | + +Of those keys, `macros`, `formulas`, `javascript`, and `animations` must +never be `exact`. `embedded_objects`, `external_relationships`, and +`round_trip` are never-exact in the oracle but are not family features; +using them as fidelity keys is `malformed-structure`. + +`receipts.unavailable.features` is exactly the features marked +`unsupported`. `receipts.unavailable.revision` / `parser_attestation` must +match `source.revision.state` and `parser.attestation`. Truncation and +malformed-input receipts carry a reason only when the condition is present. +When `truncated` is false, `omitted_bytes` and `omitted_anchor_count` must +be 0. + +Active content or external references fail closed +(`active-content`, `external-reference`). They never become a valid envelope. + +`policy.dlp.status` is `not-evaluated`, `passed`, or `findings`. +`public_boundary.outcome` `admit-export` is allowed only when the boundary +value is `public`. + +## Admission into query and packet + +`admission` is the DLP/refusal gate on this envelope. It is not an export +verdict and not a trust verdict. The oracle blocks `admitted` only when +`policy.dlp.status` is `findings`. Classification, public-boundary outcome, +parser attestation, truncation, and malformed-input do not flip admission. +Shipped positives may be `admitted` while `classification` is `internal` +and `public_boundary.outcome` is `reject-export`. + +| Condition | `status` | `packet` / `query` | `reasons` | +|---|---|---|---| +| DLP `findings` | `rejected` | `ineligible` | includes `dlp-findings` | +| Otherwise accepted | `admitted` | `eligible` | empty | +| Otherwise refused | `rejected` | `ineligible` | non-empty | + +A later ingest path may consider only `admitted` envelopes, and only through +their anchors (id, family, role, locator, text digest) plus original digest, +parser identity/version, source locator/revision, derivation relations, and +fidelity/truncation receipts. `rejected` envelopes stay out of query and +packet. `admitted` does not authorize export: ingest must still honor +`policy.classification` and `policy.public_boundary` separately, and must +not treat `unavailable` parser or revision claims as attested. + +Today `bran query` and `bran packet` still rank repository files. They do not +read this envelope. That ingest is follow-up work, not this issue. Native +Office/PDF adapters (#21, #22, #23, #26) stay conditional on a proven gap in +managed parser output. The shared conformance suite is #25. + +## Fixtures + +Synthetic normalized outputs, not native round-trips, under +`fixtures/enterprise-documents/`: + +- positive: `pdf-fixed-layout.json`, `docx-flow.json`, + `pptx-presentation.json`, `xlsx-grid.json` +- negative: `digest-mismatch`, `unsafe-asset-path`, `active-content`, + `external-reference`, `malformed-structure`, `oversized`, + `secret-reflection`, `unsupported-evidence` + +The Google attestation profile records a Document AI `output.digest` as this +envelope's `envelope_digest`. See +[`schemas/google-source-attestation.schema.json`](../schemas/google-source-attestation.schema.json). diff --git a/fixtures/enterprise-documents/negative/secret-reflection.json b/fixtures/enterprise-documents/negative/secret-reflection.json new file mode 100644 index 0000000..9bc89ce --- /dev/null +++ b/fixtures/enterprise-documents/negative/secret-reflection.json @@ -0,0 +1,147 @@ +{ + "admission": { + "packet": "ineligible", + "query": "ineligible", + "reasons": [ + "secret-reflection" + ], + "status": "rejected" + }, + "anchors": [ + { + "family": "fixed-layout", + "id": "anc:pdf:page-1:heading-1", + "locator": { + "bbox": { + "x0": 72, + "x1": 540, + "y0": 720, + "y1": 748 + }, + "block": 1, + "family": "fixed-layout", + "page": 1 + }, + "role": "heading", + "text": "Synthetic PDF layout brief", + "text_digest": "0eccf5ccb2f7dc05bbef6656ae3a0a34f04c449f3b0c4b15a70de42d3cc7aa44" + }, + { + "family": "fixed-layout", + "id": "anc:pdf:page-1:paragraph-1", + "locator": { + "bbox": { + "x0": 72, + "x1": 540, + "y0": 680, + "y1": 710 + }, + "block": 2, + "family": "fixed-layout", + "page": 1 + }, + "role": "paragraph", + "text": "This fixture is a synthetic normalized layout excerpt.", + "text_digest": "0a97350151e16b94d4ad25704a4b0005c2a09dff16ab46678e0d2d208da8aa6b" + } + ], + "assets": [ + { + "byte_length": 128, + "id": "asset:pdf:cover", + "media_type": "image/png", + "path": "assets/cover.png", + "role": "figure", + "sha256": "6ad05dd80aa7f0c315369d5a01db350e129439131bea7ee090aba2c034be8fe4" + } + ], + "envelope_digest": "543e5d2935ba5c28998d4a30b591cc8ca92150ffc50c8c904b6203a77f6ee7af", + "evidence_id": "evd:pdf:synthetic-layout-brief", + "fidelity": { + "bounding_boxes": "approximated", + "figures": "normalized", + "javascript": "unsupported", + "ocr": "approximated", + "reading_order": "approximated", + "tables": "unsupported", + "text": "normalized" + }, + "hazards": { + "active_content": { + "kinds": [], + "present": false + }, + "external_references": { + "count": 0, + "present": false + } + }, + "normalized": { + "content": { + "family": "fixed-layout", + "language": "en", + "text": "Synthetic PDF layout brief This fixture is a synthetic normalized layout excerpt." + }, + "digest": "036844ad7c5f538411f3d65d407e408cdfb58ced4592149d767c358fc31a98ca" + }, + "original": { + "byte_length": 4096, + "media_type": "application/pdf", + "sha256": "863392864fb6f736d8492844d61ce61678c38f63bc06db1b45e4283b769912ab" + }, + "parser": { + "attestation": "unavailable", + "identity": "google-document-ai", + "processor": "pretrained-layout-parser-v1.0-2024-06-03", + "version": "1.0" + }, + "policy": { + "classification": { + "status": "evaluated", + "value": "internal" + }, + "dlp": { + "findings": [], + "status": "passed" + }, + "public_boundary": { + "outcome": "reject-export", + "value": "internal" + } + }, + "receipts": { + "malformed_input": { + "present": false, + "reason": null + }, + "truncation": { + "omitted_anchor_count": 0, + "omitted_bytes": 0, + "reason": null, + "truncated": false + }, + "unavailable": { + "features": [ + "javascript", + "tables" + ], + "parser_attestation": true, + "revision": true + } + }, + "relations": [ + { + "from": "anc:pdf:page-1:paragraph-1", + "kind": "derived-from", + "to": "anc:pdf:page-1:heading-1" + } + ], + "schema_version": "1.0.0", + "source": { + "locator": "synthetic:enterprise-documents/pdf-fixed-layout?X-Goog-Signature=00", + "revision": { + "state": "unavailable", + "value": null + } + } +} diff --git a/tools/ci/enterprise_contract_check.py b/tools/ci/enterprise_contract_check.py index 490ea65..b642751 100644 --- a/tools/ci/enterprise_contract_check.py +++ b/tools/ci/enterprise_contract_check.py @@ -131,8 +131,19 @@ "external-reference.json": "external-reference", "malformed-structure.json": "malformed-structure", "oversized.json": "oversized", + "secret-reflection.json": "secret-reflection", "unsupported-evidence.json": "unsupported-evidence", } +SECRET_MARKERS = ( + "-----BEGIN ", + "AIza", + "X-Goog-Credential=", + "X-Goog-Signature=", + "access_token=", + "private_key", + "refresh_token=", + "ya29.", +) def bran_root() -> Path: @@ -153,6 +164,26 @@ def text_digest(text: str) -> str: return sha256_hex(text.encode("utf-8")) +def walk_strings(value: object) -> list[str]: + found: list[str] = [] + pending: list[object] = [value] + while pending: + current = pending.pop() + if isinstance(current, str): + found.append(current) + elif isinstance(current, dict): + pending.extend(current.values()) + elif isinstance(current, list): + pending.extend(current) + return found + + +def contains_secret(value: object) -> bool: + return any( + marker in text for text in walk_strings(value) for marker in SECRET_MARKERS + ) + + def envelope_digest(envelope: dict[str, Any]) -> str: body = {key: value for key, value in envelope.items() if key != "envelope_digest"} return sha256_hex(canonical_bytes(body)) @@ -640,6 +671,8 @@ def classify(value: object) -> str | None: else: if admission["packet"] != "ineligible" or admission["query"] != "ineligible" or not reasons: return "malformed-structure" + if contains_secret(value): + return "secret-reflection" for anchor in anchors: if anchor["text_digest"] != text_digest(anchor["text"]): diff --git a/tools/ci/test-budget.json b/tools/ci/test-budget.json index da92370..86b9ad1 100644 --- a/tools/ci/test-budget.json +++ b/tools/ci/test-budget.json @@ -278,6 +278,7 @@ "fixtures/enterprise-documents/negative/external-reference.json", "fixtures/enterprise-documents/negative/malformed-structure.json", "fixtures/enterprise-documents/negative/oversized.json", + "fixtures/enterprise-documents/negative/secret-reflection.json", "fixtures/enterprise-documents/negative/unsafe-asset-path.json", "fixtures/enterprise-documents/negative/unsupported-evidence.json", "fixtures/enterprise-documents/positive/docx-flow.json",