From 12bd0447a9161f1122db96ab4590d3bafdf2f258 Mon Sep 17 00:00:00 2001 From: Dylan Roy <249851642+d6g8k5htny-coder@users.noreply.github.com> Date: Thu, 22 Oct 2026 18:03:44 -0500 Subject: [PATCH] Add bounded public lexical retrieval and advisory coordination audits Eight new paths only. Source pickup main#475/6353027897. Lexical retrieval is not semantic embeddings; audit concurrency grants no task lease. No credentials, artifact/cache uploads, schedules, required-workflow edits or status authority. --- .github/workflows/coordination-lock.yml | 49 +++ .github/workflows/public-vector-index.yml | 38 ++ docs/COORDINATION_LOCK.md | 135 ++++++++ docs/PUBLIC_VECTOR_INDEX.md | 185 ++++++++++ tests/test_coordination_lock_check.py | 284 +++++++++++++++ tests/test_public_vector_index.py | 386 +++++++++++++++++++++ tools/coordination_lock_check.py | 281 +++++++++++++++ tools/public_vector_index.py | 404 ++++++++++++++++++++++ 8 files changed, 1762 insertions(+) create mode 100644 .github/workflows/coordination-lock.yml create mode 100644 .github/workflows/public-vector-index.yml create mode 100644 docs/COORDINATION_LOCK.md create mode 100644 docs/PUBLIC_VECTOR_INDEX.md create mode 100644 tests/test_coordination_lock_check.py create mode 100644 tests/test_public_vector_index.py create mode 100644 tools/coordination_lock_check.py create mode 100644 tools/public_vector_index.py diff --git a/.github/workflows/coordination-lock.yml b/.github/workflows/coordination-lock.yml new file mode 100644 index 000000000..93a374061 --- /dev/null +++ b/.github/workflows/coordination-lock.yml @@ -0,0 +1,49 @@ +name: Bounded coordination audit gate +'on': + workflow_dispatch: + inputs: + node_id: + description: Existing node ID to inspect; this audit grants no ownership + required: true + type: string + claimant_id: + description: Actual agent ID to compare with the recorded owner (optional) + required: false + default: '' + type: string +permissions: + contents: read +jobs: + audit: + if: github.repository == 'd6g8k5htny-coder/main' && toJSON(github.event.repository.private) == 'false' && github.ref == 'refs/heads/main' + # Serializes only this participating audit job. Pending replacement, timeout, + # or cancellation never releases an existing task-local PICKUP. + concurrency: + group: ul-coordination-node-${{ inputs.node_id }} + cancel-in-progress: false + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + persist-credentials: false + lfs: false + submodules: false + - name: Test the report-only guard in both Python modes + shell: bash + run: | + set -euo pipefail + python3 -B -S -m unittest discover -s tests -p test_coordination_lock_check.py -v 2>&1 | tee "$RUNNER_TEMP/coordination-normal.log" + grep -Eq '^Ran [1-9][0-9]* tests? in ' "$RUNNER_TEMP/coordination-normal.log" + python3 -B -O -S -m unittest discover -s tests -p test_coordination_lock_check.py -v 2>&1 | tee "$RUNNER_TEMP/coordination-optimized.log" + grep -Eq '^Ran [1-9][0-9]* tests? in ' "$RUNNER_TEMP/coordination-optimized.log" + - name: Inspect the pinned public owner view without granting a lease + shell: bash + env: + SOURCE_COMMIT: ${{ github.sha }} + NODE_ID: ${{ inputs.node_id }} + CLAIMANT_ID: ${{ inputs.claimant_id }} + run: | + set -euo pipefail + test "$(git rev-parse HEAD)" = "$SOURCE_COMMIT" + python3 -B -S tools/coordination_lock_check.py report --commit "$SOURCE_COMMIT" --node "$NODE_ID" --claimant "$CLAIMANT_ID" diff --git a/.github/workflows/public-vector-index.yml b/.github/workflows/public-vector-index.yml new file mode 100644 index 000000000..38c216aa7 --- /dev/null +++ b/.github/workflows/public-vector-index.yml @@ -0,0 +1,38 @@ +name: Public record lexical vector index +'on': + workflow_dispatch: +permissions: + contents: read +concurrency: + group: public-vector-index + cancel-in-progress: false +jobs: + rebuild: + if: github.repository == 'd6g8k5htny-coder/main' && toJSON(github.event.repository.private) == 'false' && github.ref == 'refs/heads/main' + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + persist-credentials: false + lfs: false + submodules: false + - name: Test the index controls in both Python modes + shell: bash + run: | + set -euo pipefail + python3 -B -S -m unittest discover -s tests -p test_public_vector_index.py -v 2>&1 | tee "$RUNNER_TEMP/index-normal.log" + grep -Eq '^Ran [1-9][0-9]* tests? in ' "$RUNNER_TEMP/index-normal.log" + python3 -B -O -S -m unittest discover -s tests -p test_public_vector_index.py -v 2>&1 | tee "$RUNNER_TEMP/index-optimized.log" + grep -Eq '^Ran [1-9][0-9]* tests? in ' "$RUNNER_TEMP/index-optimized.log" + - name: Rebuild pinned public blobs and emit complete bounded log delivery + shell: bash + env: + SOURCE_COMMIT: ${{ github.sha }} + run: | + set -euo pipefail + test "$(git rev-parse HEAD)" = "$SOURCE_COMMIT" + python3 -B -S tools/public_vector_index.py build --commit "$SOURCE_COMMIT" --output "$RUNNER_TEMP/public-vector-index.json" + python3 -B -S tools/public_vector_index.py check --index "$RUNNER_TEMP/public-vector-index.json" --commit "$SOURCE_COMMIT" + python3 -B -S tools/public_vector_index.py log --index "$RUNNER_TEMP/public-vector-index.json" + printf 'Rebuilt lexical vectors at `%s`. Complete bounded payload is in the job log; no artifact or cache was uploaded. Retrieval does not confer scientific acceptance.\n' "$SOURCE_COMMIT" >> "$GITHUB_STEP_SUMMARY" diff --git a/docs/COORDINATION_LOCK.md b/docs/COORDINATION_LOCK.md new file mode 100644 index 000000000..66ec9fc12 --- /dev/null +++ b/docs/COORDINATION_LOCK.md @@ -0,0 +1,135 @@ +# Coordination audit: bounded report, no pickup authority + +`tools/coordination_lock_check.py` reads the existing public node-status view at +one exact Git commit and reports its owner, freshness, source identities, stops +and missing inputs. It writes no ownership record and grants no authority. Every +JSON result includes `advisory_only: true`, `authority_granted: false`, +`universal_lock: false`, `native_reconciliation_required: true` and +`scientific_effect: "NONE"`. + +This is the bounded, disjoint contribution claimed in +[475/6353027897](https://github.com/d6g8k5htny-coder/main/issues/475#issuecomment-6353027897). +It does not implement the genuine distributed lease or cryptographic fence under +[351/6294319918](https://github.com/d6g8k5htny-coder/main/issues/351#issuecomment-6294319918): +R01 governs, product LOCKED, source HOLD, writer UNKNOWN and runtime NOT RUN +remain that source's recorded disposition. The existing semantic work's owner +and stops remain with +[333/6206436926](https://github.com/d6g8k5htny-coder/main/issues/333#issuecomment-6206436926) +and its +[current continuation](https://github.com/d6g8k5htny-coder/main/issues/333#issuecomment-6351948287). +No report, queue, clock or absence of a record transfers those scopes or removes +permission holds. + +## Run the report + +Run from the repository root, with the pinned commit already available locally: + +```sh +python3 -B -S tools/coordination_lock_check.py report \ + --commit FULL_40_CHARACTER_LOWERCASE_COMMIT \ + --node eng.hard-gate --claimant actual-session-id +``` + +Omit `--claimant`, or pass an empty value, for observation without a claimant +comparison. `--now 2026-10-22T22:00:00Z` supplies an explicit clock for a +reproducible audit or test; otherwise the process reads the actual current UTC +time. An explicit clock is an input to this report, never evidence of authority. +The timestamp format is strictly `YYYY-MM-DDTHH:MM:SSZ`, with a real calendar +date. Naive times and alternate offsets are refused. + +The command emits one ASCII JSON line. Strings are JSON-escaped so record text +cannot add multiline workflow commands. Exit 0 means a usable `OBSERVED` +report without recorded blockers; it never means PICKUP, a passing lease, +permission to write, or successful native reconciliation. Exit 2 accompanies +`HOLD`, `CONFLICT` or `EXPLICIT_HANDOFF_REQUIRED`. + +| Observation | Reported consequence | +| --- | --- | +| Missing record, null owner or malformed/unknown schema | `HOLD` | +| Requested claimant differs from recorded owner | `CONFLICT`; recorded owner retained | +| Owner has `released_url`, with the same or no claimant | `EXPLICIT_HANDOFF_REQUIRED`; recorded owner retained | +| Observation older than one hour, recorded stale/unknown, or a future observation | Blocking freshness finding; recorded owner retained | +| Stops, missing inputs, refresh errors or open carried-forward findings | Blocking findings retained | +| Fresh, valid owner with no recorded blockers | `OBSERVED`; authority still false | + +When findings coexist, a valid different-owner comparison reports `CONFLICT`, +and a valid released-owner comparison reports `EXPLICIT_HANDOFF_REQUIRED`; +additional freshness, stop and missing-input blockers remain in the same report. +Malformed records report `HOLD`. Available raw owner, source, stop and +missing-input fields can still appear on an invalid record and must be read +alongside its validation blockers. + +`status` is `fresh`, `stale` or `unknown` for this observation, and +`recorded_status` preserves the recorded freshness flag. The signed +`observed_age_seconds` makes a future observation visible as a negative age; +such a report has `status: "unknown"`. One hour is a fixed checkpoint for +observational freshness, not an ownership expiry or a permission rule. At +exactly 3,600 seconds, a recorded fresh observation remains fresh. A stale owner +requires current task-local reconciliation and any required explicit handoff. +Repeated audits, including repeated matching claimant IDs, cannot manufacture +authority or resolve a duplicate pickup. + +## Source and input bounds + +The only data path is `docs/site/node-status.json` as a regular Git blob in the +supplied commit. The working-tree copy is ignored, even if replaced or dirty. +The reader checks the exact commit resolution, tree path and regular-file mode, +Git blob digest and byte count. A usable source identity includes the repository, +commit, path, blob ID, SHA-256 and pinned public source URL, together with the +record's `sources` and `source_rev`. Record source revisions must agree with +their pinned URLs and the primary source revision. Referenced sources are +reported, not downloaded or revalidated by this command. + +The repository root and configured `origin` must identify +`d6g8k5htny-coder/main` in one of the tool's closed HTTPS/SSH forms. This is a +local configuration guard; it does not authenticate a remote or establish that +the pinned record is the latest native task state. Ambient `GIT_*` variables, +replacement objects, lazy fetching and transport protocols cannot substitute +another source. The command needs no token, performs no network reads, executes +no mathematical payloads, and mutates no Git objects or records. + +Limits are a 1 MiB blob, 256 map records, 256 entries per record list, 16,384 +characters per validated text field, 20 characters per JSON integer token and +ten seconds per local Git command. Floating-point tokens are unsupported by +the v0 record contract. Duplicate JSON keys, nonstandard numeric constants, invalid UTF-8, parse +anomalies, unsupported fields and unavailable source objects produce a hold. +The tool supports a stricter bounded profile of v0 of +[the existing schema](../architecture/node-status.schema.json), including source +URL/revision equality and a resolution URL for resolved carried-forward findings. +Those checks are observational only: they amend no schema, accept no status and +create no additional authority register. + +## What workflow concurrency guarantees + +The `audit` job in `.github/workflows/coordination-lock.yml` uses the stable +repository-local group `ul-coordination-node-${{ inputs.node_id }}` with +`cancel-in-progress: false` and a five-minute job timeout. GitHub treats group +names case-insensitively. Participating audit jobs +for the same node share that group; a newly queued audit does not cancel its +running predecessor. Default concurrency retains at most one running and one +pending job, and a later pending job replaces an earlier pending one. Ordering +is not guaranteed. See +[GitHub's concurrency documentation](https://docs.github.com/en/actions/how-tos/write-workflows/choose-when-workflows-run/control-workflow-concurrency). + +This job serializes its own participating audits within this repository. It +does not serialize or fence writers, other tasks, external sessions, other +repositories or jobs using another group. Duplicate pickups outside this job +are not prevented. GitHub cancellation is asynchronous: server condition +checks, runner messages, signals and forced termination are separate stages, +and some conditions can keep work running. See +[GitHub's cancellation documentation](https://docs.github.com/en/actions/reference/workflows-and-actions/workflow-cancellation). +Cancellation, failure, replacement of a pending audit and timeout release no +owner or PICKUP. An audit is a report each time it runs. + +The unresolved +[node-status freshness follow-up #490](https://github.com/d6g8k5htny-coder/main/issues/490) +means this derived view cannot substitute for fresh native task-local +reconciliation. Claims, reviews, handoffs and releases continue in the current +[agent message board](https://github.com/d6g8k5htny-coder/main/issues/475) and the +existing task discussion under the +[current workflow](../governance/OP-WORKFLOW-20260930.md), +[owner mandate](../governance/OWNER_MANDATE_20261015.md) and +[standing delegation](../governance/OP-AUTONOMY-20260923-v2.1.md). +Current required checks, protection rules and deployment requirements retain +their existing meaning. This audit deploys nothing and grants no scientific +acceptance. diff --git a/docs/PUBLIC_VECTOR_INDEX.md b/docs/PUBLIC_VECTOR_INDEX.md new file mode 100644 index 000000000..2899aaa85 --- /dev/null +++ b/docs/PUBLIC_VECTOR_INDEX.md @@ -0,0 +1,185 @@ +# Public lexical vector index + +This is a small retrieval baseline for an agent reading the public +`d6g8k5htny-coder/main` repository. It builds deterministic vectors from word +counts and returns matching source excerpts with pinned GitHub links. It uses +Python 3.9 or newer, Git and the Python standard library. There is no trained +encoder, learned embedding, model download, external model call, service, +credential, package installation or paid storage dependency. Lexical similarity +does not establish semantic understanding or mathematical correctness. + +The separately owned semantic-memory work remains a dependency for any learned +semantic capability. [The retained owner record](https://github.com/d6g8k5htny-coder/main/issues/333#issuecomment-6206436926) +preserves Root `01a11d37`; [the current disposition](https://github.com/d6g8k5htny-coder/main/issues/333#issuecomment-6351948287) +keeps Task 2 `REQUEST_CHANGES`, P1/P2 unaccepted, R01/R02 `STOPPED` and the old +author's closeout `UNKNOWN`. This tool does not retry those namespaces, transfer +their ownership or complete that work. Existing #351 and #460 owner scopes and +holds are unchanged. The bounded implementation pickup is +[main #475/6353027897](https://github.com/d6g8k5htny-coder/main/issues/475#issuecomment-6353027897). +Read the [owner mandate](../governance/OWNER_MANDATE_20261015.md) alongside the +[standing delegation](../governance/OP-AUTONOMY-20260923-v2.1.md). + +## Exact source boundary + +Only these five already-public paths are selected, in this fixed order: + +1. `AGENTS.md` +2. `architecture/node-status.schema.json` +3. `docs/WORKSPACE.md` +4. `docs/site/node-status.json` +5. `governance/OP-WORKFLOW-20260930.md` + +There are no user path arguments, globs, recursive document discovery or private +connectors. All private records, local files outside this list, dirty changes +and untracked files are excluded. A source being readable somewhere else does +not authorize adding it. Changing this allowlist requires a separate source and +publication review. The public node file and its existing schema are read as +source text; this tool does not write or reinterpret node status, ownership, +stops, source revisions or scientific effects. It creates no status authority. + +`origin` must identify the fixed public repository. The caller supplies a full +lowercase 40-character commit SHA1. The tool reads the commit, each traversed +tree and selected blob through local Git plumbing with replacement objects and +lazy fetch disabled, and verifies each object's SHA1 over its actual Git object +header and bytes. Git subprocesses use argument arrays and `shell=False`. +No source is read from the working tree. Only regular `100644` blob entries +under regular tree parents are accepted; symlinks, executables and submodules +at selected paths fail closed. Missing selected paths are omitted in a full +rebuild, including their former chunks; no previous index is merged or retained. + +The remote URL is a scope guard, not proof that a supplied commit was published +or is the current public branch. Obtain the expected commit through the public +repository and its applicable review process. The tool performs no network +requests and does not follow links contained in records. A partial clone must +already contain the selected commit, trees and blobs; missing objects fail +instead of silently fetching. + +## Build and query from a pinned public source + +A fresh agent can clone the public repository and use a reviewed implementation +commit containing this tool. Set `TOOL_COMMIT` to that exact published commit, +and `SOURCE_COMMIT` to the exact public source revision to consume. The checkout +command below pins the implementation; `--commit` independently pins the data. +Use complete 40-character SHA1 values obtained from the public repository. + +```sh +git clone https://github.com/d6g8k5htny-coder/main.git public-main +cd public-main +TOOL_COMMIT='' +SOURCE_COMMIT='' +git checkout --detach "$TOOL_COMMIT" +INDEX_DIR="$(python3 -c 'from pathlib import Path; import tempfile; print(Path(tempfile.mkdtemp(prefix="public-vector-")).resolve())')" +python3 -B -S tools/public_vector_index.py build --commit "$SOURCE_COMMIT" --output "$INDEX_DIR/index.json" +python3 -B -S tools/public_vector_index.py check --index "$INDEX_DIR/index.json" --commit "$SOURCE_COMMIT" +python3 -B -S tools/public_vector_index.py search --index "$INDEX_DIR/index.json" --commit "$SOURCE_COMMIT" --query 'eng.hard-gate' --limit 5 +``` + +`search` returns JSON with cosine scores, exact text, character ranges, stable +chunk IDs and source links. Read the returned pinned source and its linked +evidence before acting. A match is a navigation aid, never an ownership claim, +review disposition or scientific acceptance. Queries are limited to 4,096 +characters and result counts to 1–20. Ties sort by path, start character and ID. +Hash collisions can return unrelated matches; tokenization does not understand +synonyms, mathematical meaning or non-ASCII words. Chunks can split a sentence +or JSON node. Adjacent chunks and the full pinned source remain available. + +Both `check` and CLI `search` rebuild from the expected source commit and require +byte-exact equality with the provided index. Changed text, vectors, metadata, +unknown fields, different whitespace and stale source commits are refused. +An old commit can still be checked explicitly as a historical source; callers +must choose the expected current revision themselves. Refreshing to a new +revision requires a full rebuild, so deletions cannot survive through an old +index. The in-process `search_index` function expects an index already returned +by `check_index`; direct use of an unchecked dictionary is not a trust boundary. + +Output is an explicitly selected file outside the repository. It is written to +a newly created mode-0600 temporary file and atomically replaces the requested +regular destination. Destination symlinks and symlink parent directories are +refused, and a failed exclusive temporary-file creation preserves preexisting +files. Use a canonical directory path: on macOS `/tmp` and `/var` are symlink +aliases, so resolve the directory as above (or use `/private/tmp`). Parent +directories must already exist. The tool does not delete old index files at +other paths or purge historical Git commits and logs. + +## Deterministic format and capacity + +The artifact is canonical ASCII JSON: sorted keys, compact separators and one +terminal newline. Schema version 1 contains `source`, `algorithm`, `documents` +and `chunks`. `source` identifies the repository and commit; each document +identifies its path, blob SHA1, full raw-byte SHA256 and byte count. Each chunk +references that document path and contains text, a pinned URL, a stable ID, +zero-based half-open Unicode character offsets and one-based inclusive lines. +An unchanged document's ID remains stable across unrelated commits: it is the +SHA256 of repository, path, blob SHA1, start and end joined with NUL characters. +No build timestamp, run ID or mutable source URL is added. Existing timestamps +inside original source records remain source text. + +Algorithm `lexical-feature-hashing`, version 1, lowercases text, extracts +`[a-z0-9]+` tokens, hashes each token with SHA256 and maps the first four digest +bytes, interpreted big-endian, modulo 256. Sparse vectors contain sorted +`[dimension, positive-count]` pairs. Search computes cosine similarity using the +same query transform; the index stores integers rather than floating vectors. + +| Bound | Limit | +| --- | --- | +| One selected UTF-8 source | 32,768 raw bytes | +| All selected sources combined | 65,536 raw bytes | +| One chunk | 2,048 Unicode characters | +| All chunks | 40 | +| Serialized index | 131,072 bytes (128 KiB) | +| Framed-log input | 262,144 UTF-8 bytes | +| One log payload chunk | 768 raw bytes / 1,024 base64 characters | + +Commit objects are bounded to 65,536 bytes and each traversed tree to 2 MiB. +An invalid object, invalid UTF-8 source or exceeded bound rejects the entire +build. It never truncates a source or silently publishes a partial result. +ASCII JSON escaping can exceed the payload cap even when raw inputs fit. +Capacity changes require review along with source-boundary changes. + +## Temporary workflow log transport + +The manual workflow builds and checks the index in the runner's temporary +directory, then calls `log --index`. That command checks the complete index +against the commit embedded in it before printing any payload. Base64 avoids +interpreting source text as workflow commands. The protocol emits one `BEGIN` +record with raw byte count, SHA256 and chunk count, numbered `CHUNK` records, +and an `END` record repeating the SHA256. The `reconstruct_index` parser requires +the complete ordered sequence; missing, duplicate, reordered, corrupted and +oversized payloads are refused. Standard GitHub UTC timestamp prefixes and +unrelated log lines are accepted. Use the downloaded log for the index step, +not an arbitrarily large archive or combined history. + +An agent can consume a saved log using the same expected source SHA1: + +```sh +python3 -B -S tools/public_vector_index.py log --index "$INDEX_DIR/index.json" > "$INDEX_DIR/index-step.log" +python3 -B -S tools/public_vector_index.py reconstruct --log "$INDEX_DIR/index-step.log" --commit "$SOURCE_COMMIT" --output "$INDEX_DIR/restored.json" +python3 -B -S tools/public_vector_index.py search --index "$INDEX_DIR/restored.json" --commit "$SOURCE_COMMIT" --query 'eng.hard-gate' +``` + +Framing hashes establish completeness and byte identity, not authenticity. +Reconstruction also rebuilds from the selected Git commit before writing the +index. Logs are temporary and can expire or be truncated. A fresh consumer can +always rebuild from its pinned public source instead. Logs are no durable +research archive, delivery receipt, mathematical proof or scientific evidence. + +The workflow uses a standard public-repository Ubuntu runner, with no artifact +upload, cache action, external model or storage service. GitHub documents +[standard hosted runners for public repositories](https://docs.github.com/en/actions/reference/runners/github-hosted-runners#standard-github-hosted-runners-for-public-repositories) +and [Actions billing, including logs and job summaries](https://docs.github.com/en/billing/concepts/product-billing/github-actions). +It requests no larger runner or paid feature; these policy links remain the +source of truth if GitHub changes its terms. + +## Verification + +```sh +python3 -B -S -m unittest discover -s tests -p test_public_vector_index.py -v +python3 -B -S -O -m unittest discover -s tests -p test_public_vector_index.py -v +``` + +Tests use local disposable Git repositories and real blobs, rebuilds, CLI +queries and output files. They cover dirty/untracked exclusions, original +source identities, deletion, stale and forged inputs, nonregular entries, +corrupt and replaced objects, size limits, ranking and tie order, safe output, +and complete/corrupt/missing/duplicate/reordered/oversized log transport. +They perform no mathematical experiment and establish no scientific status. diff --git a/tests/test_coordination_lock_check.py b/tests/test_coordination_lock_check.py new file mode 100644 index 000000000..e20ad8926 --- /dev/null +++ b/tests/test_coordination_lock_check.py @@ -0,0 +1,284 @@ +"""CLI regressions using real pinned Git objects; no mathematical code runs.""" +from datetime import datetime, timedelta, timezone +import hashlib +import json +import os +from pathlib import Path +import subprocess +import sys +import tempfile +import unittest + +SCRIPT = Path(__file__).resolve().parents[1] / "tools/coordination_lock_check.py" +PATH = "docs/site/node-status.json" +ORIGIN = "https://github.com/d6g8k5htny-coder/main.git" +NOW = "2026-10-22T22:00:00Z" +EVIDENCE = "https://github.com/d6g8k5htny-coder/main/issues/475#issuecomment-6353027897" +RELEASE = "https://github.com/d6g8k5htny-coder/main/issues/475#issuecomment-6353027898" + + +def record(): + return { + "schema_version": 0, "node_id": "eng.audit", "owner": {"id": "alice", "evidence_url": EVIDENCE}, + "next_action": "Refresh the native task discussion.", "missing_input": [], "stops": [], + "sources": [{"url": EVIDENCE, "rev": "2026-10-22T21:59:00Z"}], + "source_rev": "2026-10-22T21:59:00Z", "observed_at": "2026-10-22T21:59:00Z", + "status": "fresh", "scientific_effect": "NONE", + } + + +class CoordinationLockTests(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + self.addCleanup(self.tmp.cleanup) + self.root = Path(self.tmp.name) + self.git("init", "-q") + self.git("config", "user.name", "Fixture") + self.git("config", "user.email", "fixture@example.invalid") + self.git("remote", "add", "origin", ORIGIN) + self.value = record() + self.commit = self.pin({"eng.audit": self.value}) + + def git(self, *args): + result = subprocess.run(["git", "-C", str(self.root), *args], capture_output=True, check=True) + return result.stdout.decode().strip() + + def pin(self, data=None, raw=None): + path = self.root / PATH + path.parent.mkdir(parents=True, exist_ok=True) + if path.is_symlink(): + path.unlink() + path.write_bytes(raw if raw is not None else json.dumps(data, ensure_ascii=True).encode()) + self.git("add", "--", PATH) + self.git("commit", "-qm", "fixture", "--allow-empty") + self.raw = path.read_bytes() + return self.git("rev-parse", "HEAD") + + def run_report(self, *, claimant="alice", commit=None, now=NOW, node="eng.audit", env=None): + args = [sys.executable, "-B", "-S", str(SCRIPT), "report", "--commit", commit or self.commit, "--node", node] + if claimant is not None: + args += ["--claimant", claimant] + if now is not None: + args += ["--now", now] + run_env = os.environ.copy() + if env: + run_env.update(env) + process = subprocess.run(args, cwd=self.root, capture_output=True, env=run_env, timeout=10) + self.assertIn(process.returncode, (0, 2), process.stderr.decode()) + self.assertEqual(len(process.stdout.splitlines()), 1, process.stderr.decode()) + self.assertTrue(process.stdout.isascii()) + result = json.loads(process.stdout) + self.assertIs(result["authority_granted"], False) + self.assertIs(result["universal_lock"], False) + self.assertIs(result["advisory_only"], True) + self.assertIs(result["native_reconciliation_required"], True) + self.assertEqual(result["scientific_effect"], "NONE") + return process, result + + def update(self, **changes): + self.value.update(changes) + self.commit = self.pin({"eng.audit": self.value}) + + def hold(self, **args): + process, result = self.run_report(**args) + self.assertEqual(process.returncode, 2) + self.assertEqual(result["disposition"], "HOLD") + self.assertTrue(result["blockers"]) + return result + + def test_matching_owner_reports_exact_pinned_source_and_age_without_authority(self): + process, result = self.run_report() + self.assertEqual(process.returncode, 0) + self.assertEqual(result["disposition"], "OBSERVED") + self.assertEqual(result["owner"], self.value["owner"]) + self.assertEqual(result["status"], "fresh") + self.assertEqual(result["observed_age_seconds"], 60) + self.assertEqual(result["sources"], self.value["sources"]) + self.assertEqual(result["source_rev"], self.value["source_rev"]) + self.assertEqual(result["source"]["commit"], self.commit) + self.assertEqual(result["source"]["blob"], self.git("rev-parse", self.commit + ":" + PATH)) + self.assertEqual(result["source"]["sha256"], hashlib.sha256(self.raw).hexdigest()) + self.assertEqual(result["source"]["url"], "https://github.com/d6g8k5htny-coder/main/blob/" + self.commit + "/" + PATH) + + def test_dirty_tree_replacement_is_ignored(self): + (self.root / PATH).write_text('{"eng.audit":{"owner":null}}') + process, result = self.run_report() + self.assertEqual(process.returncode, 0) + self.assertEqual(result["owner"]["id"], "alice") + self.assertEqual(result["source"]["sha256"], hashlib.sha256(self.raw).hexdigest()) + + def test_different_owner_is_conflict(self): + process, result = self.run_report(claimant="bob") + self.assertEqual(process.returncode, 2) + self.assertEqual(result["disposition"], "CONFLICT") + self.assertEqual(result["owner"]["id"], "alice") + + def test_release_url_preserves_owner_and_requires_explicit_handoff(self): + owner = dict(self.value["owner"], released_url=RELEASE) + self.update(owner=owner) + process, result = self.run_report() + self.assertEqual(process.returncode, 2) + self.assertEqual(result["disposition"], "EXPLICIT_HANDOFF_REQUIRED") + self.assertEqual(result["owner"], owner) + _, other = self.run_report(claimant="bob") + self.assertEqual(other["disposition"], "CONFLICT") + + def test_missing_or_blank_claimant_is_observe_only(self): + for claimant in (None, ""): + with self.subTest(claimant=claimant): + process, result = self.run_report(claimant=claimant) + self.assertEqual(process.returncode, 0) + self.assertIsNone(result["claimant_id"]) + + def test_null_owner_is_hold(self): + self.update(owner=None) + result = self.hold() + self.assertIsNone(result["owner"]) + + def test_missing_record_is_hold(self): + self.hold(node="eng.missing") + + def test_missing_input_and_stops_are_preserved_and_hold(self): + self.update(stops=["Permission HOLD remains.\n::error::untrusted"], missing_input=["Fresh native task reconciliation."]) + result = self.hold() + self.assertEqual(result["stops"], self.value["stops"]) + self.assertEqual(result["missing_input"], self.value["missing_input"]) + + def test_stale_age_holds_without_expiring_owner(self): + self.update(observed_at="2026-10-22T20:59:59Z") + result = self.hold() + self.assertEqual(result["status"], "stale") + self.assertEqual(result["observed_age_seconds"], 3601) + self.assertEqual(result["owner"]["id"], "alice") + + def test_freshness_window_boundary_is_observation_only(self): + self.update(observed_at="2026-10-22T21:00:00Z") + process, result = self.run_report() + self.assertEqual(process.returncode, 0) + self.assertEqual(result["observed_age_seconds"], 3600) + + def test_recorded_stale_and_unknown_hold_even_when_recent(self): + for status in ("stale", "unknown"): + with self.subTest(status=status): + self.update(status=status) + result = self.hold() + self.assertEqual(result["status"], status) + + def test_future_observation_is_unknown_and_hold(self): + self.update(observed_at="2026-10-22T22:00:01Z") + result = self.hold() + self.assertEqual(result["status"], "unknown") + self.assertEqual(result["observed_age_seconds"], -1) + + def test_times_require_real_strict_utc(self): + for value in ("2026-10-22T21:59:00+00:00", "2026-10-22T21:59:00", "2026-02-30T21:59:00Z"): + with self.subTest(value=value): + self.update(observed_at=value) + self.hold() + self.hold(now=value) + + def test_unknown_schema_and_extra_fields_hold(self): + for change in ({"schema_version": 1}, {"schema_version": True}, {"unrecognized": "value"}): + with self.subTest(change=change): + self.value = record() + self.update(**change) + self.hold() + + def test_duplicate_keys_and_invalid_json_hold(self): + for raw in (b'{"eng.audit":{},"eng.audit":{}}', b'{"eng.audit":{"owner":null,"owner":{}}}', b'{', b'NaN', b'{} trailing'): + with self.subTest(raw=raw): + self.commit = self.pin(raw=raw) + self.hold() + + def test_source_revision_identity_mismatch_holds(self): + self.update(source_rev="2026-10-22T21:58:00Z") + self.hold() + self.value = record() + self.update(sources=[{"url": "https://github.com/d6g8k5htny-coder/main/blob/" + "a" * 40 + "/record.json", "rev": "b" * 40}], source_rev="b" * 40) + self.hold() + + def test_numeric_parse_anomalies_are_blocked_before_raw_fields_are_reported(self): + for raw_value in (b"1e9999", b"-1e9999", b"1" * 10000): + with self.subTest(raw_value=raw_value[:20]): + self.commit = self.pin(raw=b'{"eng.audit":{"owner":' + raw_value + b'}}') + self.hold() + + def test_wrong_remote_holds_before_reading_record(self): + self.git("remote", "set-url", "origin", "https://github.com/foreign/private.git") + result = self.hold() + self.assertIsNone(result["owner"]) + + def test_commit_must_be_exact_full_existing_commit(self): + for value in ("HEAD", self.commit[:12], "f" * 40, "--help"): + with self.subTest(value=value): + self.hold(commit=value) + + def test_record_node_identity_must_match_map_key(self): + self.update(node_id="eng.other") + self.hold() + + def test_nonregular_blob_is_refused(self): + path = self.root / PATH + path.unlink() + path.symlink_to("some-other-file") + self.git("add", "--", PATH) + self.git("commit", "-qm", "symlink") + self.commit = self.git("rev-parse", "HEAD") + self.hold() + + def test_missing_blob_is_hold(self): + self.git("rm", "-q", "--", PATH) + self.git("commit", "-qm", "missing") + self.commit = self.git("rev-parse", "HEAD") + self.hold() + + def test_repeated_matching_pickups_never_manufacture_authority_or_write(self): + before = self.git("status", "--porcelain=v1", "--untracked-files=all") + first = self.run_report()[1] + second = self.run_report()[1] + self.assertEqual(first, second) + self.assertEqual(self.git("status", "--porcelain=v1", "--untracked-files=all"), before) + + def test_oversize_blob_is_hold(self): + self.commit = self.pin(raw=b" " * 1_048_577) + self.hold() + + def test_git_environment_cannot_redirect_source(self): + process, result = self.run_report(env={"GIT_DIR": "/nonexistent", "GIT_WORK_TREE": "/nonexistent", "GIT_CONFIG_COUNT": "1", "GIT_CONFIG_KEY_0": "remote.origin.url", "GIT_CONFIG_VALUE_0": "https://foreign.invalid/repo"}) + self.assertEqual(process.returncode, 0) + self.assertEqual(result["owner"]["id"], "alice") + + def test_git_blob_replacement_cannot_substitute_null_owner(self): + original = self.git("rev-parse", self.commit + ":" + PATH) + replacement = self.root / "replacement.json" + replacement.write_text('{"eng.audit":{"owner":null}}') + replacement_blob = self.git("hash-object", "-w", str(replacement)) + self.git("replace", original, replacement_blob) + process, result = self.run_report() + self.assertEqual(process.returncode, 0) + self.assertEqual(result["owner"]["id"], "alice") + self.assertEqual(result["source"]["blob"], original) + self.assertEqual(result["source"]["sha256"], hashlib.sha256(self.raw).hexdigest()) + + def test_runtime_clock_is_current_utc_when_now_is_omitted(self): + observed = datetime.now(timezone.utc).replace(microsecond=0) - timedelta(seconds=60) + self.update(observed_at=observed.strftime("%Y-%m-%dT%H:%M:%SZ")) + process, result = self.run_report(now=None) + self.assertEqual(process.returncode, 0) + self.assertLess(abs(result["observed_age_seconds"] - 60), 10) + self.assertRegex(result["evaluated_at"], r"^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$") + + def test_malformed_record_fields_and_refresh_errors_hold(self): + for change in ({"owner": {}}, {"stops": "hold"}, {"sources": []}, + {"status": {}}, {"scientific_effect": "PROVED"}, + {"refresh_error": "native refresh failed"}, + {"carried_forward": [{"id": "finding", "text": "Unresolved", + "source_url": EVIDENCE, "status": "open"}]}): + with self.subTest(change=change): + self.value = record() + self.update(**change) + self.hold() + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_public_vector_index.py b/tests/test_public_vector_index.py new file mode 100644 index 000000000..e1c85e25c --- /dev/null +++ b/tests/test_public_vector_index.py @@ -0,0 +1,386 @@ +"""Behavioral controls for the bounded public lexical index; all Git data is local.""" +import base64 +import copy +import hashlib +import json +import os +from pathlib import Path +import subprocess +import sys +import tempfile +import time +import unittest +from unittest.mock import patch +import zlib + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tools")) +try: + import public_vector_index as indexer +except ImportError: + indexer = None + + +class PublicVectorIndexTests(unittest.TestCase): + def setUp(self): + self.assertIsNotNone(indexer, "bounded public vector index implementation is missing") + self.tmp = tempfile.TemporaryDirectory() + self.addCleanup(self.tmp.cleanup) + # macOS /var and /tmp are symlinks; output deliberately requires a direct path. + self.base = Path(self.tmp.name).resolve() + self.repo = self.base / "repo" + self.repo.mkdir() + self.git("init", "-q") + self.git("config", "user.name", "Local test fixture") + self.git("config", "user.email", "fixture@example.invalid") + self.git("remote", "add", "origin", "https://github.com/d6g8k5htny-coder/main.git") + self.source = b"alpha alpha beta\npublic workflow\n" + (self.repo / "AGENTS.md").write_bytes(self.source) + (self.repo / "private-record.txt").write_text("excluded private sentinel", encoding="utf-8") + self.commit = self.save() + + def git(self, *args): + return subprocess.run(["git", "-C", str(self.repo), *args], check=True, + stdout=subprocess.PIPE, stderr=subprocess.PIPE).stdout.decode().strip() + + def save(self): + self.git("add", "-A") + self.git("commit", "-qm", "fixture snapshot") + return self.git("rev-parse", "HEAD") + + def build(self, commit=None): + return indexer.build_index(self.repo, commit or self.commit) + + def test_pinned_git_bytes_exclude_dirty_untracked_and_nonallowlisted_records(self): + # Reading the working tree or broadening the path selection leaks the sentinels. + (self.repo / "AGENTS.md").write_text("dirty sentinel", encoding="utf-8") + (self.repo / "untracked.txt").write_text("untracked sentinel", encoding="utf-8") + raw = self.build() + data = json.loads(raw) + self.assertEqual(data["source"], {"repository": "d6g8k5htny-coder/main", "commit": self.commit}) + self.assertEqual(data["algorithm"], {"name": "lexical-feature-hashing", "version": 1, + "dimensions": 256, "tokenizer": "ascii-alnum-lower"}) + self.assertEqual([d["path"] for d in data["documents"]], ["AGENTS.md"]) + doc = data["documents"][0] + self.assertEqual(doc["sha256"], hashlib.sha256(self.source).hexdigest()) + expected_blob = hashlib.sha1(b"blob 33\0" + self.source).hexdigest() + self.assertEqual(doc["blob_sha1"], expected_blob) + self.assertEqual(doc["bytes"], 33) + self.assertEqual(data["chunks"][0]["text"], self.source.decode()) + self.assertIn("/blob/" + self.commit + "/AGENTS.md#L1-L2", data["chunks"][0]["source_url"]) + for sentinel in (b"dirty sentinel", b"untracked sentinel", b"excluded private sentinel"): + self.assertNotIn(sentinel, raw) + + def test_repeated_build_is_byte_exact_ascii_json_without_run_metadata(self): + raw = self.build() + self.assertEqual(raw, self.build()) + raw.decode("ascii") + data = json.loads(raw) + self.assertEqual(set(data), {"schema_version", "source", "algorithm", "documents", "chunks"}) + self.assertEqual(data["schema_version"], 1) + self.assertEqual(indexer.check_index(self.repo, raw, self.commit), data) + + def test_full_rebuild_drops_deleted_documents_without_carryover(self): + (self.repo / "docs").mkdir() + (self.repo / "docs/WORKSPACE.md").write_text("workspace", encoding="utf-8") + first = self.save() + (self.repo / "AGENTS.md").unlink() + later = self.save() + self.assertEqual(len(json.loads(self.build(first))["documents"]), 2) + current = json.loads(self.build(later)) + self.assertEqual([d["path"] for d in current["documents"]], ["docs/WORKSPACE.md"]) + self.assertNotIn("AGENTS.md", {c["path"] for c in current["chunks"]}) + + def test_no_remaining_allowlisted_records_produces_empty_index(self): + (self.repo / "AGENTS.md").unlink() + data = json.loads(self.build(self.save())) + self.assertEqual(data["documents"], []) + self.assertEqual(data["chunks"], []) + + def test_source_commit_requires_full_lowercase_sha1_and_expected_commit(self): + for commit in ("HEAD", self.commit[:12], self.commit.upper(), "f" * 40): + with self.subTest(commit=commit), self.assertRaises(ValueError): + self.build(commit) + raw = self.build() + (self.repo / "AGENTS.md").write_text("new public source", encoding="utf-8") + later = self.save() + with self.assertRaisesRegex(ValueError, "stale|match"): + indexer.check_index(self.repo, raw, later) + + def test_other_repository_origin_is_refused(self): + self.git("remote", "set-url", "origin", "https://github.com/other/private.git") + with self.assertRaisesRegex(ValueError, "repository|origin"): + self.build() + + def test_symlink_executable_and_submodule_allowlist_entries_are_refused(self): + for mode in ("120000", "100755", "160000"): + with self.subTest(mode=mode): + oid = self.commit if mode == "160000" else self.git("rev-parse", self.commit + ":AGENTS.md") + self.git("update-index", "--cacheinfo", mode + "," + oid + ",AGENTS.md") + self.git("commit", "-qm", "nonregular allowlisted fixture") + with self.assertRaisesRegex(ValueError, "regular|mode"): + self.build(self.git("rev-parse", "HEAD")) + + def test_nonregular_parent_of_allowlisted_path_is_refused(self): + os.symlink("elsewhere", self.repo / "docs") + with self.assertRaisesRegex(ValueError, "tree|mode|regular"): + self.build(self.save()) + + def test_source_byte_and_utf8_limits_fail_closed(self): + for source in (b"x" * 32769, b"\xff\xfe"): + with self.subTest(length=len(source)): + (self.repo / "AGENTS.md").write_bytes(source) + with self.assertRaises(ValueError): + self.build(self.save()) + + def test_combined_source_limit_fails_instead_of_partial_index(self): + (self.repo / "AGENTS.md").write_bytes(b"a" * 24000) + (self.repo / "docs").mkdir() + (self.repo / "docs/WORKSPACE.md").write_bytes(b"b" * 24000) + (self.repo / "governance").mkdir() + (self.repo / "governance/OP-WORKFLOW-20260930.md").write_bytes(b"c" * 24000) + with self.assertRaisesRegex(ValueError, "total|combined"): + self.build(self.save()) + + def test_ascii_json_expansion_cannot_exceed_payload_limit(self): + (self.repo / "AGENTS.md").write_text("\u00e9" * 16384, encoding="utf-8") + (self.repo / "docs").mkdir() + (self.repo / "docs/WORKSPACE.md").write_text("\u00e9" * 16384, encoding="utf-8") + with self.assertRaisesRegex(ValueError, "payload|index.*limit"): + self.build(self.save()) + + def test_corrupt_git_object_cannot_become_trusted_source(self): + oid = self.git("rev-parse", self.commit + ":AGENTS.md") + path = self.repo / ".git/objects" / oid[:2] / oid[2:] + path.chmod(0o600) + path.write_bytes(zlib.compress(b"blob 33\0" + b"z" * 33)) + with self.assertRaises(ValueError): + self.build() + + def test_git_replace_cannot_substitute_source_bytes(self): + original = self.git("rev-parse", self.commit + ":AGENTS.md") + (self.repo / "replacement.txt").write_text("replacement sentinel", encoding="utf-8") + replacement = self.git("hash-object", "-w", "replacement.txt") + self.git("replace", original, replacement) + self.assertEqual(json.loads(self.build())["chunks"][0]["text"], self.source.decode()) + + def test_chunk_ranges_cover_source_once_and_identity_survives_unrelated_commit(self): + text = "alpha\n" * 500 + (self.repo / "AGENTS.md").write_text(text, encoding="utf-8") + first = self.save() + data = json.loads(self.build(first)) + chunks = data["chunks"] + self.assertEqual([(c["start_char"], c["end_char"]) for c in chunks], [(0, 2048), (2048, 3000)]) + self.assertEqual("".join(c["text"] for c in chunks), text) + self.assertEqual((chunks[0]["start_line"], chunks[0]["end_line"]), (1, 342)) + (self.repo / "private-record.txt").write_text("unrelated change", encoding="utf-8") + later = json.loads(self.build(self.save())) + self.assertEqual([c["id"] for c in chunks], [c["id"] for c in later["chunks"]]) + + def test_sparse_vectors_count_lexical_tokens_and_search_returns_pinned_source(self): + raw = self.build() + data = indexer.check_index(self.repo, raw, self.commit) + # SHA256('alpha') -> bin173; beta->231; public->117; workflow->159. + self.assertEqual(data["chunks"][0]["vector"], [[117, 1], [159, 1], [173, 2], [231, 1]]) + results = indexer.search_index(data, "ALPHA", limit=1) + self.assertEqual(len(results), 1) + self.assertEqual(results[0]["path"], "AGENTS.md") + self.assertAlmostEqual(results[0]["score"], 2 / (7 ** 0.5)) + self.assertEqual(indexer.search_index(data, "!!!"), []) + with self.assertRaises(ValueError): + indexer.search_index(data, "x" * 4097) + with self.assertRaises(ValueError): + indexer.search_index(data, "alpha", limit=0) + + def test_public_node_record_is_retrievable_with_original_owner_stops_and_links(self): + (self.repo / "docs/site").mkdir(parents=True) + node = {"eng.hard-gate": {"schema_version": 0, "owner": None, + "stops": ["source review remains open"], + "scientific_effect": "NONE", + "sources": [{"url": "https://github.com/d6g8k5htny-coder/main/issues/333"}]}} + text = json.dumps(node) + (self.repo / "docs/site/node-status.json").write_text(text, encoding="utf-8") + data = json.loads(self.build(self.save())) + result = indexer.search_index(data, "eng.hard-gate")[0] + self.assertEqual(result["path"], "docs/site/node-status.json") + self.assertEqual(result["text"], text) + self.assertIn('"owner": null', result["text"]) + self.assertIn('"source review remains open"', result["text"]) + self.assertIn('/issues/333', result["text"]) + + def test_search_ties_are_sorted_by_path_and_character_range(self): + (self.repo / "docs").mkdir() + (self.repo / "docs/WORKSPACE.md").write_bytes(self.source) + data = json.loads(self.build(self.save())) + self.assertEqual([r["path"] for r in indexer.search_index(data, "alpha")], + ["AGENTS.md", "docs/WORKSPACE.md"]) + + def test_checker_refuses_modified_metadata_text_vectors_or_noncanonical_json(self): + data = json.loads(self.build()) + mutations = [lambda d: d["documents"][0].update(sha256="0" * 64), + lambda d: d["chunks"][0].update(text="forged"), + lambda d: d["chunks"][0].update(vector=[[0, 1]]), + lambda d: d.update(extra="unapproved")] + for mutate in mutations: + forged = copy.deepcopy(data) + mutate(forged) + with self.assertRaises(ValueError): + raw = (json.dumps(forged, ensure_ascii=True, sort_keys=True, + separators=(",", ":")) + "\n").encode("ascii") + indexer.check_index(self.repo, raw, self.commit) + with self.assertRaises(ValueError): + indexer.check_index(self.repo, json.dumps(data, indent=2).encode(), self.commit) + + def test_output_write_is_external_atomic_and_refuses_symlinks_or_checkout(self): + raw = self.build() + output = self.base / "index.json" + indexer.write_output(self.repo, output, raw) + self.assertEqual(output.read_bytes(), raw) + indexer.write_output(self.repo, output, raw) + output.unlink() + victim = self.base / "victim" + victim.write_text("keep", encoding="utf-8") + output.symlink_to(victim) + with self.assertRaises(ValueError): + indexer.write_output(self.repo, output, raw) + self.assertEqual(victim.read_text(), "keep") + with self.assertRaises(ValueError): + indexer.write_output(self.repo, self.repo / "generated-index.json", raw) + self.assertFalse((self.repo / "generated-index.json").exists()) + + def test_output_refuses_symlink_parent_directory(self): + real = self.base / "real" + real.mkdir() + alias = self.base / "alias" + alias.symlink_to(real, target_is_directory=True) + with self.assertRaises(ValueError): + indexer.write_output(self.repo, alias / "index.json", self.build()) + self.assertEqual(list(real.iterdir()), []) + + def test_temporary_filename_collision_preserves_preexisting_file(self): + collision = self.base / ".public-vector-index-collision" + collision.write_bytes(b"preexisting unrelated bytes") + output = self.base / "index.json" + with patch.object(indexer.secrets, "token_hex", return_value="collision"): + with self.assertRaises(FileExistsError): + indexer.write_output(self.repo, output, self.build()) + self.assertTrue(collision.exists(), "failed exclusive create deleted a file it did not own") + self.assertEqual(collision.read_bytes(), b"preexisting unrelated bytes") + self.assertFalse(output.exists()) + + def test_git_timeout_covers_a_child_that_produces_no_stdout_or_eof(self): + fake_bin = self.base / "fake-bin" + fake_bin.mkdir() + fake_git = fake_bin / "git" + fake_git.write_text("#!" + sys.executable + "\nimport time\ntime.sleep(3)\n", encoding="utf-8") + fake_git.chmod(0o755) + started = time.monotonic() + with patch.dict(os.environ, {"PATH": str(fake_bin) + os.pathsep + os.defpath}): + with patch.object(indexer, "GIT_TIMEOUT_SECONDS", 0.1, create=True): + with self.assertRaisesRegex(ValueError, "timed out"): + indexer._git(self.repo, ["cat-file", "-t", self.commit], 100) + self.assertLess(time.monotonic() - started, 2) + + def test_log_roundtrip_accepts_timestamped_github_lines_and_checks_raw_digest(self): + raw = self.build() + log = indexer.frame_index(raw) + self.assertEqual(indexer.reconstruct_index(log), raw) + timestamped = "unrelated job line\n" + "\n".join( + "2026-10-22T23:00:00.1234567Z " + line for line in log.splitlines()) + "\n" + self.assertEqual(indexer.reconstruct_index(timestamped), raw) + self.assertIn(hashlib.sha256(raw).hexdigest(), log) + self.assertIn(str(len(raw)), log.splitlines()[0]) + + def test_log_missing_duplicate_reordered_or_corrupt_chunks_fail_closed(self): + raw = self.build() + lines = indexer.frame_index(raw).splitlines() + self.assertGreaterEqual(len(lines), 4) + candidates = [lines[:-1], lines[1:], lines[:1] + lines[2:], + lines[:2] + [lines[1]] + lines[2:], + [lines[0], lines[2], lines[1]] + lines[3:], + lines + lines] + altered = list(lines) + fields = altered[1].split(" ") + payload = bytearray(base64.b64decode(fields[-1])) + payload[0] ^= 1 + fields[-1] = base64.b64encode(payload).decode() + altered[1] = " ".join(fields) + candidates.append(altered) + for candidate in candidates: + with self.subTest(lines=len(candidate)), self.assertRaises(ValueError): + indexer.reconstruct_index("\n".join(candidate) + "\n") + + def test_log_oversize_declared_or_actual_payload_is_refused(self): + with self.assertRaises(ValueError): + indexer.frame_index(b"x" * 131073) + log = indexer.frame_index(self.build()) + first, rest = log.split("\n", 1) + fields = first.split(" ") + fields[2] = "131073" + with self.assertRaises(ValueError): + indexer.reconstruct_index(" ".join(fields) + "\n" + rest) + with self.assertRaises(ValueError): + indexer.reconstruct_index("x" * 262145) + + def test_cli_build_check_log_reconstruct_search_and_stale_failure(self): + tool = ROOT / "tools/public_vector_index.py" + output = self.base / "index.json" + command = [sys.executable, "-B", "-S", str(tool)] + def run(*args): + return subprocess.run(command + list(args), cwd=self.repo, + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + built = run("build", "--commit", self.commit, "--output", str(output)) + self.assertEqual(built.returncode, 0, built.stderr.decode()) + checked = run("check", "--index", str(output), "--commit", self.commit) + self.assertEqual(checked.returncode, 0, checked.stderr.decode()) + logged = run("log", "--index", str(output)) + self.assertEqual(logged.returncode, 0, logged.stderr.decode()) + self.assertEqual(indexer.reconstruct_index(logged.stdout.decode()), output.read_bytes()) + logpath = self.base / "job.log" + logpath.write_bytes(logged.stdout) + restored = self.base / "restored.json" + reconstructed = run("reconstruct", "--log", str(logpath), "--commit", self.commit, + "--output", str(restored)) + self.assertEqual(reconstructed.returncode, 0, reconstructed.stderr.decode()) + self.assertEqual(restored.read_bytes(), output.read_bytes()) + searched = run("search", "--index", str(output), "--commit", self.commit, "--query", "alpha") + self.assertEqual(searched.returncode, 0, searched.stderr.decode()) + self.assertEqual(json.loads(searched.stdout)[0]["path"], "AGENTS.md") + stale = run("check", "--index", str(output), "--commit", "f" * 40) + self.assertNotEqual(stale.returncode, 0) + + def test_cli_log_cannot_export_forged_local_payload(self): + path = self.base / "forged.json" + data = json.loads(self.build()) + data["chunks"][0]["text"] = "private forged sentinel" + path.write_text(json.dumps(data, ensure_ascii=True, sort_keys=True, + separators=(",", ":")) + "\n", encoding="ascii") + result = subprocess.run([sys.executable, "-B", "-S", str(ROOT / "tools/public_vector_index.py"), + "log", "--index", str(path)], cwd=self.repo, + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + self.assertNotEqual(result.returncode, 0) + self.assertEqual(result.stdout, b"") + + def test_cli_search_refuses_stale_or_canonically_serialized_tampering(self): + path = self.base / "index.json" + original = self.build() + path.write_bytes(original) + (self.repo / "AGENTS.md").write_text("successor public source", encoding="utf-8") + later = self.save() + forged = json.loads(original) + forged["chunks"][0]["vector"] = [[0, 999]] + forged_raw = (json.dumps(forged, ensure_ascii=True, sort_keys=True, + separators=(",", ":")) + "\n").encode("ascii") + for expected_commit, raw in ((later, original), (self.commit, forged_raw)): + with self.subTest(expected_commit=expected_commit): + path.write_bytes(raw) + result = subprocess.run([sys.executable, "-B", "-S", + str(ROOT / "tools/public_vector_index.py"), "search", "--index", str(path), + "--commit", expected_commit, "--query", "alpha"], cwd=self.repo, + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + self.assertNotEqual(result.returncode, 0) + self.assertEqual(result.stdout, b"") + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/coordination_lock_check.py b/tools/coordination_lock_check.py new file mode 100644 index 000000000..2d34667d9 --- /dev/null +++ b/tools/coordination_lock_check.py @@ -0,0 +1,281 @@ +#!/usr/bin/env python3 +"""Read a pinned public node-status view. Every result is advisory, never a lease.""" +from __future__ import annotations + +import argparse +from datetime import datetime, timezone +import hashlib +import json +import os +from pathlib import Path +import re +import subprocess +import sys + +REPOSITORY = "d6g8k5htny-coder/main" +SOURCE_PATH = "docs/site/node-status.json" +ORIGINS = frozenset({ + "https://github.com/d6g8k5htny-coder/main.git", + "https://github.com/d6g8k5htny-coder/main", + "git@github.com:d6g8k5htny-coder/main.git", + "ssh://git@github.com/d6g8k5htny-coder/main.git", +}) +SHA = re.compile(r"[0-9a-f]{40}\Z") +NODE = re.compile(r"[A-Za-z0-9][A-Za-z0-9_.-]{0,199}\Z") +UTC = re.compile(r"[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z\Z") +COMMENT = re.compile(r"https://github\.com/d6g8k5htny-coder/(?:main|Math-)/(?:issues|pull)/[1-9][0-9]*#(?:issuecomment|pullrequestreview)-[1-9][0-9]*\Z") +OBJECT = re.compile(r"https://github\.com/d6g8k5htny-coder/(?:main|Math-)/(?:issues|pull)/[1-9][0-9]*(?:#(?:issuecomment|pullrequestreview)-[1-9][0-9]*)?\Z") +PINNED = re.compile(r"https://github\.com/d6g8k5htny-coder/(?:main|Math-)/(?:blob|tree)/([0-9a-f]{40})/[A-Za-z0-9_./-]+\Z") +REQUIRED = frozenset({"schema_version", "node_id", "owner", "next_action", "missing_input", "stops", "sources", "source_rev", "observed_at", "status", "scientific_effect"}) +OPTIONAL = frozenset({"predecessor", "successor", "carried_forward", "refresh_error"}) +MAX_BYTES = 1_048_576 +MAX_RECORDS = 256 +MAX_ITEMS = 256 +MAX_TEXT = 16_384 +OBSERVATION_WINDOW_SECONDS = 3600 + + +class Refused(ValueError): + """An input cannot support a usable advisory observation.""" + + +class Parser(argparse.ArgumentParser): + def error(self, message): + raise Refused("invalid command arguments") + + +def utc_time(value): + if not isinstance(value, str) or UTC.fullmatch(value) is None: + raise Refused("timestamp must be a real UTC YYYY-MM-DDTHH:MM:SSZ value") + try: + return datetime.strptime(value, "%Y-%m-%dT%H:%M:%SZ").replace(tzinfo=timezone.utc) + except ValueError as error: + raise Refused("timestamp must be a real UTC YYYY-MM-DDTHH:MM:SSZ value") from error + + +def text(value): + return isinstance(value, str) and 0 < len(value) <= MAX_TEXT + + +def unique_pairs(pairs): + result = {} + for key, value in pairs: + if key in result: + raise Refused("duplicate JSON key") + result[key] = value + return result + + +def invalid_constant(value): + raise Refused("unsupported JSON numeric value") + + +def bounded_integer(value): + if len(value) > 20: + raise Refused("JSON integer exceeds read bound") + return int(value) + + +def git(root, *args): + # No credential lookup, lazy network fetch, object replacement, or ambient + # GIT_DIR/config injection. Only fixed read-only plumbing is called below. + env = {key: value for key, value in os.environ.items() if not key.startswith("GIT_")} + env.update(GIT_CONFIG_NOSYSTEM="1", GIT_CONFIG_GLOBAL=os.devnull, + GIT_TERMINAL_PROMPT="0", GIT_NO_LAZY_FETCH="1", GIT_OPTIONAL_LOCKS="0") + command = ["git", "--no-replace-objects", "-c", "protocol.allow=never", + "-c", "core.hooksPath=" + os.devnull, "-c", "core.fsmonitor=false", + "-c", "gc.auto=0", "-C", str(root), *args] + result = subprocess.run(command, env=env, capture_output=True, timeout=10, check=False) + if result.returncode: + # Git diagnostics can contain local paths or configuration; do not copy them. + raise Refused("required local Git object or repository metadata unavailable") + if len(result.stdout) > MAX_BYTES: + raise Refused("Git output exceeds read bound") + return result.stdout + + +def pinned_source(root, commit, report): + if not isinstance(commit, str) or SHA.fullmatch(commit) is None: + raise Refused("commit must be an exact lowercase 40-character commit ID") + if git(root, "rev-parse", "--show-toplevel").decode().strip() != str(root): + raise Refused("run from the repository root") + origins = git(root, "config", "--local", "--get-all", "remote.origin.url").decode().splitlines() + if len(origins) != 1 or origins[0] not in ORIGINS: + raise Refused("origin is not the expected public repository") + if git(root, "rev-parse", "--verify", commit + "^{commit}").decode().strip() != commit: + raise Refused("commit identity mismatch") + listing = git(root, "ls-tree", "-z", "--full-tree", commit, "--", SOURCE_PATH) + entries = listing.split(b"\0") + if len(entries) != 2 or entries[1] != b"": + raise Refused("fixed node-status path is missing or ambiguous") + metadata, separator, path = entries[0].partition(b"\t") + parts = metadata.split(b" ") + if (separator != b"\t" or path != SOURCE_PATH.encode() or len(parts) != 3 + or parts[0] not in (b"100644", b"100755") or parts[1] != b"blob"): + raise Refused("node-status must be a regular Git blob") + blob = parts[2].decode("ascii") + if SHA.fullmatch(blob) is None: + raise Refused("invalid Git blob identity") + size = int(git(root, "cat-file", "-s", blob).decode().strip()) + if size < 0 or size > MAX_BYTES: + raise Refused("node-status blob exceeds read bound") + raw = git(root, "cat-file", "blob", blob) + if len(raw) != size or hashlib.sha1(b"blob " + str(size).encode() + b"\0" + raw).hexdigest() != blob: + raise Refused("Git blob identity mismatch") + report["source"] = {"repository": REPOSITORY, "commit": commit, "path": SOURCE_PATH, + "blob": blob, "sha256": hashlib.sha256(raw).hexdigest(), + "url": "https://github.com/" + REPOSITORY + "/blob/" + commit + "/" + SOURCE_PATH} + return raw + + +def string_list(value): + return isinstance(value, list) and len(value) <= MAX_ITEMS and all(text(item) for item in value) + + +def check_record(record, node): + if not isinstance(record, dict) or not REQUIRED <= set(record) or set(record) - REQUIRED - OPTIONAL: + raise Refused("record does not match the closed v0 schema") + if type(record["schema_version"]) is not int or record["schema_version"] != 0: + raise Refused("unknown record schema version") + if not isinstance(record["node_id"], str) or NODE.fullmatch(record["node_id"]) is None or record["node_id"] != node: + raise Refused("record node identity mismatch") + if not text(record["next_action"]): + raise Refused("invalid next_action") + for field in ("missing_input", "stops"): + if not string_list(record[field]): + raise Refused("invalid " + field) + if record["status"] not in ("fresh", "stale", "unknown") or record["scientific_effect"] != "NONE": + raise Refused("invalid record status or scientific effect") + utc_time(record["observed_at"]) + owner = record["owner"] + if owner is not None: + if (not isinstance(owner, dict) or not {"id", "evidence_url"} <= set(owner) + or set(owner) - {"id", "evidence_url", "released_url"} or not text(owner["id"]) + or not isinstance(owner["evidence_url"], str) or COMMENT.fullmatch(owner["evidence_url"]) is None): + raise Refused("invalid recorded owner") + if "released_url" in owner and (not isinstance(owner["released_url"], str) or COMMENT.fullmatch(owner["released_url"]) is None): + raise Refused("invalid recorded owner release URL") + sources = record["sources"] + if not isinstance(sources, list) or not 0 < len(sources) <= MAX_ITEMS: + raise Refused("invalid record sources") + for source in sources: + if not isinstance(source, dict) or set(source) != {"url", "rev"} or not text(source["url"]) or not text(source["rev"]): + raise Refused("invalid source identity") + pinned = PINNED.fullmatch(source["url"]) + if pinned: + if source["rev"] != pinned.group(1): + raise Refused("pinned source revision identity mismatch") + elif OBJECT.fullmatch(source["url"]): + utc_time(source["rev"]) + else: + raise Refused("unsupported source URL") + if record["source_rev"] != sources[0]["rev"]: + raise Refused("source_rev must equal the primary source revision") + for field in ("predecessor", "successor"): + link = record.get(field) + if link is not None and (not isinstance(link, dict) or set(link) != {"node_id", "url"} + or not text(link["node_id"]) or not text(link["url"]) + or re.fullmatch(r"https?://\S+", link["url"]) is None): + raise Refused("invalid " + field) + if "refresh_error" in record and record["refresh_error"] is not None and not isinstance(record["refresh_error"], str): + raise Refused("invalid refresh_error") + if "carried_forward" in record: + carried = record["carried_forward"] + if not isinstance(carried, list) or len(carried) > MAX_ITEMS: + raise Refused("invalid carried_forward") + seen = set() + for item in carried: + if (not isinstance(item, dict) or not {"id", "text", "source_url", "status"} <= set(item) + or set(item) - {"id", "text", "source_url", "status", "resolution_url"} + or any(not text(item[field]) for field in ("id", "text", "source_url")) + or item["id"] in seen or item["status"] not in ("open", "resolved")): + raise Refused("invalid carried_forward item") + seen.add(item["id"]) + if item["status"] == "resolved" or "resolution_url" in item: + if not isinstance(item.get("resolution_url"), str) or COMMENT.fullmatch(item["resolution_url"]) is None: + raise Refused("invalid carried_forward resolution") + + +def base_report(): + return {"report_version": 1, "disposition": "HOLD", "authority_granted": False, + "universal_lock": False, "advisory_only": True, "native_reconciliation_required": True, + "scientific_effect": "NONE", "node_id": None, "claimant_id": None, + "owner": None, "status": "unknown", "recorded_status": None, "observed_at": None, + "observed_age_seconds": None, "observation_window_seconds": OBSERVATION_WINDOW_SECONDS, + "stops": [], "missing_input": [], "blockers": [], "sources": [], "source_rev": None, + "source": None, "evaluated_at": None} + + +def observe(args, report): + report["node_id"] = args.node + report["claimant_id"] = args.claimant or None + if NODE.fullmatch(args.node) is None: + raise Refused("invalid requested node ID") + if args.claimant and (not text(args.claimant) or any(ord(char) < 32 or ord(char) == 127 for char in args.claimant)): + raise Refused("invalid claimant ID") + now = utc_time(args.now) if args.now is not None else datetime.now(timezone.utc).replace(microsecond=0) + report["evaluated_at"] = now.strftime("%Y-%m-%dT%H:%M:%SZ") + raw = pinned_source(Path.cwd().resolve(), args.commit, report) + data = json.loads(raw.decode("utf-8"), object_pairs_hook=unique_pairs, + parse_constant=invalid_constant, parse_float=invalid_constant, + parse_int=bounded_integer) + if not isinstance(data, dict) or len(data) > MAX_RECORDS: + raise Refused("node-status must be a bounded record map") + if args.node not in data: + raise Refused("requested node has no recorded status") + record = data[args.node] + # Keep available owner/stop/source fields visible even when schema validation + # refuses the record. Their presence does not make them validated evidence. + if isinstance(record, dict): + for field in ("owner", "stops", "missing_input", "sources", "source_rev", "observed_at"): + if field in record: + report[field] = record[field] + check_record(record, args.node) + report["recorded_status"] = record["status"] + report["observed_age_seconds"] = age = int((now - utc_time(record["observed_at"])).total_seconds()) + report["status"] = ("unknown" if age < 0 else "stale" if age > OBSERVATION_WINDOW_SECONDS else record["status"]) + if age < 0: + report["blockers"].append("observation is in the future") + elif report["status"] != "fresh": + report["blockers"].append("record observation is stale or unknown; owner is not released") + if record["owner"] is None: + report["blockers"].append("no recorded owner; fresh native pickup reconciliation is required") + if record["stops"]: + report["blockers"].append("recorded stops remain") + if record["missing_input"]: + report["blockers"].append("recorded missing inputs remain") + if record.get("refresh_error"): + report["blockers"].append("recorded refresh error remains") + if any(item["status"] == "open" for item in record.get("carried_forward", [])): + report["blockers"].append("open carried-forward findings remain") + owner = record["owner"] + if owner is not None and args.claimant and owner["id"] != args.claimant: + report["disposition"] = "CONFLICT" + report["blockers"].append("claimant differs from the recorded owner") + elif owner is not None and "released_url" in owner: + report["disposition"] = "EXPLICIT_HANDOFF_REQUIRED" + report["blockers"].append("release URL preserves the owner; explicit native handoff is required") + elif not report["blockers"]: + report["disposition"] = "OBSERVED" + + +def main(argv=None): + report = base_report() + try: + parser = Parser(description=__doc__, allow_abbrev=False) + subparsers = parser.add_subparsers(dest="command", required=True, parser_class=Parser) + command = subparsers.add_parser("report", allow_abbrev=False) + command.add_argument("--commit", required=True) + command.add_argument("--node", required=True) + command.add_argument("--claimant") + command.add_argument("--now") + observe(parser.parse_args(argv), report) + except (OSError, ValueError, UnicodeError, RecursionError, subprocess.TimeoutExpired) as error: + # Do not copy arbitrary parser/Git diagnostics or input bytes into logs. + report["blockers"].append(str(error) if isinstance(error, Refused) else "input parse or local read anomaly") + print(json.dumps(report, ensure_ascii=True, sort_keys=True, separators=(",", ":"), allow_nan=False)) + return 0 if report["disposition"] == "OBSERVED" else 2 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/public_vector_index.py b/tools/public_vector_index.py new file mode 100644 index 000000000..6ae6c9f7c --- /dev/null +++ b/tools/public_vector_index.py @@ -0,0 +1,404 @@ +#!/usr/bin/env python3 +"""Bounded, reproducible lexical retrieval over a fixed set of public Git blobs. + +No learned model, remote service, working-tree ingestion, or scientific inference. +Run from the public repository checkout; see docs/PUBLIC_VECTOR_INDEX.md. +""" +import argparse +import base64 +from collections import Counter +import hashlib +import json +import math +import os +from pathlib import Path +import re +import secrets +import selectors +import stat +import subprocess +import sys +import time + +REPOSITORY = "d6g8k5htny-coder/main" +ALLOWLIST = ( + "AGENTS.md", + "architecture/node-status.schema.json", + "docs/WORKSPACE.md", + "docs/site/node-status.json", + "governance/OP-WORKFLOW-20260930.md", +) +MAX_SOURCE_BYTES = 32768 +MAX_TOTAL_SOURCE_BYTES = 65536 +CHUNK_CHARS = 2048 +MAX_CHUNKS = 40 +MAX_INDEX_BYTES = 131072 +MAX_LOG_BYTES = 262144 +FRAME_BYTES = 768 +GIT_TIMEOUT_SECONDS = 15 +MARKER = "PUBLIC_VECTOR_INDEX_V1" +ALGORITHM = {"name": "lexical-feature-hashing", "version": 1, + "dimensions": 256, "tokenizer": "ascii-alnum-lower"} +SHA1 = re.compile(r"[0-9a-f]{40}\Z") +TIMESTAMP = re.compile(r"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?Z ") + + +def _git(repo, args, limit): + """Read local plumbing only; prohibit lazy fetch, replacements and shell use.""" + env = {k: v for k, v in os.environ.items() if not k.startswith("GIT_")} + env.update(GIT_NO_REPLACE_OBJECTS="1", GIT_NO_LAZY_FETCH="1", GIT_TERMINAL_PROMPT="0", + GIT_CONFIG_NOSYSTEM="1", GIT_CONFIG_GLOBAL=os.devnull) + command = ["git", "--no-replace-objects", "-c", "protocol.allow=never", "-C", str(repo)] + args + with subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, + env=env, shell=False) as process: + deadline = time.monotonic() + GIT_TIMEOUT_SECONDS + selector = selectors.DefaultSelector() + selector.register(process.stdout, selectors.EVENT_READ) + raw = bytearray() + try: + while True: + remaining = deadline - time.monotonic() + if remaining <= 0 or not selector.select(remaining): + raise subprocess.TimeoutExpired(command, GIT_TIMEOUT_SECONDS) + piece = os.read(process.stdout.fileno(), min(65536, limit + 1 - len(raw))) + if not piece: + break + raw.extend(piece) + if len(raw) > limit: + raise ValueError("Git object or response exceeds byte limit") + code = process.wait(timeout=max(0, deadline - time.monotonic())) + except subprocess.TimeoutExpired as exc: + raise ValueError("local Git operation timed out") from exc + finally: + selector.close() + if process.poll() is None: + process.kill() + process.wait() + if code: + raise ValueError("required local Git object or repository configuration is unavailable") + return bytes(raw) + + +def _validate_commit(commit): + if not isinstance(commit, str) or not SHA1.fullmatch(commit): + raise ValueError("commit must be a full lowercase 40-character Git SHA1") + + +def _repository_root(repo): + root = Path(_git(repo, ["rev-parse", "--show-toplevel"], 4096).decode().strip()).resolve() + origins = _git(repo, ["config", "--local", "--get-all", "remote.origin.url"], 4096).decode().splitlines() + permitted = {"https://github.com/" + REPOSITORY, "https://github.com/" + REPOSITORY + ".git", + "git@github.com:" + REPOSITORY, "git@github.com:" + REPOSITORY + ".git"} + if len(origins) != 1 or origins[0] not in permitted: + raise ValueError("origin must identify the fixed public source repository") + return root + + +def _object(repo, oid, kind, limit): + if not SHA1.fullmatch(oid): + raise ValueError("invalid Git object identity") + actual_kind = _git(repo, ["cat-file", "-t", oid], 32).decode().strip() + if actual_kind != kind: + raise ValueError("Git object type does not match source identity") + size_raw = _git(repo, ["cat-file", "-s", oid], 32).decode().strip() + if not re.fullmatch(r"0|[1-9][0-9]*", size_raw) or int(size_raw) > limit: + raise ValueError("Git source object exceeds byte limit") + raw = _git(repo, ["cat-file", kind, oid], limit) + header = (kind + " " + str(len(raw)) + "\0").encode("ascii") + if len(raw) != int(size_raw) or hashlib.sha1(header + raw).hexdigest() != oid: + raise ValueError("Git object bytes do not match their source SHA1") + return raw + + +def _tree(repo, oid, cache): + if oid not in cache: + raw = _object(repo, oid, "tree", 2097152) + entries = {} + offset = 0 + while offset < len(raw): + space = raw.find(b" ", offset) + nul = raw.find(b"\0", space + 1) + if space < offset or nul < space or nul + 21 > len(raw): + raise ValueError("malformed Git source tree") + mode, name = raw[offset:space], raw[space + 1:nul] + if not name or b"/" in name or name in entries or name in (b".", b".."): + raise ValueError("ambiguous Git source tree entry") + entries[name] = (mode, raw[nul + 1:nul + 21].hex()) + offset = nul + 21 + cache[oid] = entries + return cache[oid] + + +def _source_blob(repo, root_tree, path, cache): + tree_oid = root_tree + parts = path.encode("ascii").split(b"/") + for position, part in enumerate(parts): + entry = _tree(repo, tree_oid, cache).get(part) + if entry is None: + return None + mode, oid = entry + if position < len(parts) - 1: + if mode != b"40000": + raise ValueError("allowlisted parent must have regular Git tree mode") + tree_oid = oid + else: + if mode != b"100644": + raise ValueError("allowlisted source must have regular Git mode 100644") + return oid, _object(repo, oid, "blob", MAX_SOURCE_BYTES) + + +def _vector(text): + counts = Counter() + for token in re.findall(r"[a-z0-9]+", text.lower(), flags=re.ASCII): + dimension = int.from_bytes(hashlib.sha256(token.encode("ascii")).digest()[:4], "big") % 256 + counts[dimension] += 1 + return [[dimension, counts[dimension]] for dimension in sorted(counts)] + + +def _json_bytes(data): + return (json.dumps(data, ensure_ascii=True, sort_keys=True, separators=(",", ":"), + allow_nan=False) + "\n").encode("ascii") + + +def build_index(repo, commit): + """Fully rebuild from verified commit/tree/blob bytes; absent paths disappear.""" + _validate_commit(commit) + repo = _repository_root(repo) + commit_raw = _object(repo, commit, "commit", 65536) + first_line = commit_raw.split(b"\n", 1)[0] + if not re.fullmatch(rb"tree [0-9a-f]{40}", first_line): + raise ValueError("commit has no valid source tree") + root_tree = first_line[5:].decode("ascii") + documents, chunks, cache = [], [], {} + total = 0 + for path in ALLOWLIST: + source = _source_blob(repo, root_tree, path, cache) + if source is None: + continue + blob, raw = source + total += len(raw) + if total > MAX_TOTAL_SOURCE_BYTES: + raise ValueError("combined source total exceeds byte limit") + try: + text = raw.decode("utf-8", errors="strict") + except UnicodeDecodeError as exc: + raise ValueError("allowlisted source must be UTF-8") from exc + documents.append({"path": path, "blob_sha1": blob, + "sha256": hashlib.sha256(raw).hexdigest(), "bytes": len(raw)}) + for start in range(0, len(text), CHUNK_CHARS): + end = min(start + CHUNK_CHARS, len(text)) + excerpt = text[start:end] + start_line = text.count("\n", 0, start) + 1 + end_line = text.count("\n", 0, end - 1) + 1 + identity = "\0".join((REPOSITORY, path, blob, str(start), str(end))) + chunks.append({"id": hashlib.sha256(identity.encode("ascii")).hexdigest(), + "path": path, "start_char": start, "end_char": end, + "start_line": start_line, "end_line": end_line, "text": excerpt, + "source_url": "https://github.com/" + REPOSITORY + "/blob/" + commit + "/" + + path + "#L" + str(start_line) + "-L" + str(end_line), + "vector": _vector(excerpt)}) + if len(chunks) > MAX_CHUNKS: + raise ValueError("index chunk count exceeds limit") + raw = _json_bytes({"schema_version": 1, "source": {"repository": REPOSITORY, "commit": commit}, + "algorithm": ALGORITHM, "documents": documents, "chunks": chunks}) + if len(raw) > MAX_INDEX_BYTES: + raise ValueError("serialized index payload exceeds byte limit") + return raw + + +def check_index(repo, raw, commit): + """Require exact current-source reconstruction, including every text/vector field.""" + _validate_commit(commit) + if not isinstance(raw, bytes) or len(raw) > MAX_INDEX_BYTES: + raise ValueError("index payload exceeds byte limit or is not bytes") + if raw != build_index(repo, commit): + raise ValueError("index is stale, noncanonical or does not match the expected source commit") + return json.loads(raw) + + +def search_index(index, query, limit=5): + """Cosine lexical retrieval; callers must first check_index at an expected commit.""" + if not isinstance(query, str) or len(query) > 4096: + raise ValueError("query exceeds 4096-character limit") + if type(limit) is not int or not 1 <= limit <= 20: + raise ValueError("search result limit must be 1..20") + query_vector = dict(_vector(query)) + query_norm = math.sqrt(sum(v * v for v in query_vector.values())) + if not query_norm: + return [] + results = [] + for chunk in index["chunks"]: + vector = dict(chunk["vector"]) + dot = sum(query_vector.get(dimension, 0) * count for dimension, count in vector.items()) + if dot: + score = dot / (query_norm * math.sqrt(sum(v * v for v in vector.values()))) + results.append({"id": chunk["id"], "path": chunk["path"], "start_char": chunk["start_char"], + "end_char": chunk["end_char"], "source_url": chunk["source_url"], + "score": score, "text": chunk["text"]}) + results.sort(key=lambda result: (-result["score"], result["path"], result["start_char"], result["id"])) + return results[:limit] + + +def _open_parent(path): + """Open each directory with O_NOFOLLOW, using directory descriptors throughout.""" + path = Path(os.path.abspath(path)) + descriptor = os.open(path.anchor, os.O_RDONLY | os.O_DIRECTORY) + try: + for part in path.parent.parts[1:]: + next_descriptor = os.open(part, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW, + dir_fd=descriptor) + os.close(descriptor) + descriptor = next_descriptor + return path, descriptor + except OSError as exc: + os.close(descriptor) + raise ValueError("output parent must be existing directories without symlinks") from exc + + +def write_output(repo, output, raw): + """Atomic external-file replacement; never follow a destination symlink.""" + root = _repository_root(repo) + path = Path(os.path.abspath(output)) + resolved = path.resolve() + if resolved == root or root in resolved.parents: + raise ValueError("index output must be outside the source repository") + if len(raw) > MAX_INDEX_BYTES: + raise ValueError("index payload exceeds byte limit") + path, directory = _open_parent(path) + temporary = ".public-vector-index-" + secrets.token_hex(12) + created = False + try: + try: + existing = os.stat(path.name, dir_fd=directory, follow_symlinks=False) + except FileNotFoundError: + existing = None + if existing is not None and not stat.S_ISREG(existing.st_mode): + raise ValueError("output must be a regular file, never a symlink") + descriptor = os.open(temporary, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, + 0o600, dir_fd=directory) + created = True + with os.fdopen(descriptor, "wb") as stream: + stream.write(raw) + stream.flush() + os.fsync(stream.fileno()) + os.replace(temporary, path.name, src_dir_fd=directory, dst_dir_fd=directory) + finally: + if created: + try: + os.unlink(temporary, dir_fd=directory) + except FileNotFoundError: + pass + os.close(directory) + + +def frame_index(raw): + """Frame checked index bytes for temporary logs; this protocol is not authentication.""" + if not isinstance(raw, bytes) or not raw or len(raw) > MAX_INDEX_BYTES: + raise ValueError("framed payload must be 1..131072 bytes") + digest = hashlib.sha256(raw).hexdigest() + count = (len(raw) + FRAME_BYTES - 1) // FRAME_BYTES + lines = ["{} BEGIN {} {} {}".format(MARKER, len(raw), digest, count)] + for number, offset in enumerate(range(0, len(raw), FRAME_BYTES), 1): + lines.append("{} CHUNK {} {}".format(MARKER, number, + base64.b64encode(raw[offset:offset + FRAME_BYTES]).decode("ascii"))) + lines.append("{} END {}".format(MARKER, digest)) + return "\n".join(lines) + "\n" + + +def reconstruct_index(log): + """Reject incomplete, duplicated, reordered, corrupt or oversized log framing.""" + if not isinstance(log, str) or len(log.encode("utf-8")) > MAX_LOG_BYTES: + raise ValueError("log exceeds byte limit") + lines = [] + for line in log.splitlines(): + line = TIMESTAMP.sub("", line, count=1) if TIMESTAMP.match(line) else line + if line.startswith(MARKER): + lines.append(line) + if len(lines) < 3: + raise ValueError("missing complete index log frame") + begin = re.fullmatch(MARKER + r" BEGIN ([1-9][0-9]{0,5}) ([0-9a-f]{64}) ([1-9][0-9]{0,2})", lines[0]) + if begin is None: + raise ValueError("invalid or missing log frame header") + size, digest, count = int(begin[1]), begin[2], int(begin[3]) + if size > MAX_INDEX_BYTES or count != (size + FRAME_BYTES - 1) // FRAME_BYTES: + raise ValueError("declared log payload exceeds limit or has inconsistent chunk count") + if len(lines) != count + 2 or lines[-1] != MARKER + " END " + digest: + raise ValueError("incomplete, duplicate or missing terminal log frame") + payload = bytearray() + for number, line in enumerate(lines[1:-1], 1): + match = re.fullmatch(MARKER + r" CHUNK ([1-9][0-9]{0,2}) ([A-Za-z0-9+/=]{1,1024})", line) + if match is None or int(match[1]) != number: + raise ValueError("missing, duplicated or reordered log chunk") + try: + piece = base64.b64decode(match[2], validate=True) + except ValueError as exc: + raise ValueError("invalid base64 log chunk") from exc + expected = FRAME_BYTES if number < count else size - FRAME_BYTES * (count - 1) + if len(piece) != expected or base64.b64encode(piece).decode("ascii") != match[2]: + raise ValueError("invalid log chunk length or encoding") + payload.extend(piece) + raw = bytes(payload) + if len(raw) != size or hashlib.sha256(raw).hexdigest() != digest: + raise ValueError("reconstructed log payload digest or byte count mismatch") + return raw + + +def _read_bounded(path, limit): + with open(path, "rb") as stream: + raw = stream.read(limit + 1) + if len(raw) > limit: + raise ValueError("input file exceeds byte limit") + return raw + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__) + commands = parser.add_subparsers(dest="command", required=True) + build = commands.add_parser("build", help="rebuild only allowlisted committed public blobs") + build.add_argument("--commit", required=True) + build.add_argument("--output", required=True) + for command in ("check", "search", "log"): + item = commands.add_parser(command) + item.add_argument("--index", required=True) + if command != "log": + item.add_argument("--commit", required=True) + if command == "search": + item.add_argument("--query", required=True) + item.add_argument("--limit", type=int, default=5) + reconstruct = commands.add_parser("reconstruct") + reconstruct.add_argument("--log", required=True) + reconstruct.add_argument("--commit", required=True) + reconstruct.add_argument("--output", required=True) + args = parser.parse_args(argv) + repo = Path.cwd() + try: + if args.command == "build": + raw = build_index(repo, args.commit) + write_output(repo, args.output, raw) + print("public lexical index: {} bytes; SHA256 {}".format(len(raw), hashlib.sha256(raw).hexdigest())) + elif args.command == "reconstruct": + raw = reconstruct_index(_read_bounded(args.log, MAX_LOG_BYTES).decode("utf-8")) + check_index(repo, raw, args.commit) + write_output(repo, args.output, raw) + print("public lexical index reconstructed and source-checked") + else: + raw = _read_bounded(args.index, MAX_INDEX_BYTES) + if args.command == "log": + data = json.loads(raw) + commit = data["source"]["commit"] + check_index(repo, raw, commit) + sys.stdout.write(frame_index(raw)) + else: + data = check_index(repo, raw, args.commit) + if args.command == "check": + print("public lexical index matches expected source commit " + args.commit) + else: + sys.stdout.buffer.write(_json_bytes(search_index(data, args.query, args.limit))) + except (ValueError, OSError, KeyError, TypeError, UnicodeError) as exc: + print("public lexical index refused: " + str(exc), file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main())