diff --git a/scripts/publish-ctxpack-redact.py b/scripts/publish-ctxpack-redact.py old mode 100644 new mode 100755 index 56db1a2..3191bb9 --- a/scripts/publish-ctxpack-redact.py +++ b/scripts/publish-ctxpack-redact.py @@ -5,7 +5,15 @@ Rewrites every ``*.jsonl`` file under IN PLACE, replacing secret-shaped values with ``***REDACTED***`` while keeping field names (and, -inside free text, variable names) intact for debuggability. +inside free text, variable names) intact for debuggability. ``tombstones.jsonl`` +(added by ctxpack format v2) is treated like any other table file; unknown +future table files are picked up by the same ``*.jsonl`` glob. + +After the JSONL pass, stamps ``manifest.json`` with ``"redacted": true`` +(publication-artifact marker per the ctxpack v2 contract) without touching any +other manifest field -- in particular ``counts`` is never modified, so count +consistency and the round-trip self-test still hold. Re-running is a no-op: +an already-``true`` stamp is reported and the file is left byte-identical. What is redacted (fail-closed heuristics, JSONL-aware -- never blind sed): 1. Key-name rule: any JSON object field whose name is secret-shaped (see @@ -45,7 +53,10 @@ the DB keeps the original values and the next export redacts them again. - Row counts and file sets are unchanged, so manifest counts and the round-trip self-test still hold on redacted output. - - Re-running is a no-op (``***REDACTED***`` carries no secret shape). + - ``manifest.json`` must exist and be a JSON object; its ``redacted`` stamp + is set to ``true`` (atomic replace) and no other manifest content changes. + - Re-running is a no-op (``***REDACTED***`` carries no secret shape and an + already-true stamp is left byte-identical). - Files are replaced atomically (tmp + rename); rows that do not parse as JSON are left byte-identical and reported (the self-test then fails closed on them as malformed rows, as before). @@ -251,6 +262,35 @@ def redact_file(path): return rows, count, skipped +def stamp_manifest(snap): + """Stamp ``manifest.json`` with ``redacted: true`` (publication marker). + + Returns ``"stamped"`` when the flag was added, ``"already"`` when the + manifest already carried ``redacted: true`` (left byte-identical). + Raises ``ValueError`` on a missing/invalid manifest or a non-bool stamp; + callers fail closed. + """ + path = os.path.join(snap, "manifest.json") + if not os.path.isfile(path): + raise ValueError("manifest.json is missing (not a ctxpack dir?)") + with open(path, "r", encoding="utf-8") as handle: + manifest = json.load(handle) + if not isinstance(manifest, dict): + raise ValueError("manifest.json is not a JSON object") + current = manifest.get("redacted") + if current is True: + return "already" + if current not in (None, False): + raise ValueError("manifest.json 'redacted' is not a boolean: %r" % (current,)) + manifest["redacted"] = True + tmp_path = path + ".redact-tmp" + with open(tmp_path, "w", encoding="utf-8") as handle: + json.dump(manifest, handle, ensure_ascii=False, indent=2) + handle.write("\n") + os.replace(tmp_path, path) + return "stamped" + + def main(argv): if len(argv) != 2 or argv[1] in ("-h", "--help"): sys.stderr.write("usage: publish-ctxpack-redact.py \n") @@ -281,6 +321,12 @@ def main(argv): total_skipped += skipped if count or skipped: print(" %-28s rows=%d redactions=%d nonjson_skipped=%d" % (name, rows, count, skipped)) + try: + stamp = stamp_manifest(snap) + except ValueError as error: + sys.stderr.write("publish-ctxpack-redact: FAIL: %s\n" % error) + return 1 + print(" manifest.json redacted=true (%s; counts untouched)" % stamp) print( "publish-ctxpack-redact: %d files, %d rows, %d replacements, %d non-JSON lines left untouched" % (len(files), total_rows, total_count, total_skipped) diff --git a/scripts/publish-ctxpack-selftest.sh b/scripts/publish-ctxpack-selftest.sh index a9c5776..3da2b3e 100755 --- a/scripts/publish-ctxpack-selftest.sh +++ b/scripts/publish-ctxpack-selftest.sh @@ -4,29 +4,42 @@ # Usage: scripts/publish-ctxpack-selftest.sh [--no-round-trip] # # Checks (fail-closed, exit 1): -# 1. manifest.json exists with format=carryctx-pack-dir, format_version=1, -# a non-empty counts map and a project_id; project.json exists. +# 1. manifest.json exists with format=carryctx-pack-dir and a supported +# format_version (1 or 2), a non-empty counts map and a project_id; +# project.json exists. v2 shape is enforced too: counts.tombstones, +# non-empty unique parents that never name the manifest's own export_id, +# watermark rows agreeing with counts, sequences/source shape, and the +# redacted:true publication stamp. v1 keeps its legacy shape (no +# watermarks, no tombstone counts) and needs no redacted stamp. # 2. Every counts entry has a matching .jsonl whose line count -# equals the manifest count. +# equals the manifest count; a table with rows but no counts entry fails +# closed (undeclared row set), matching carryctx check_counts. # 3. Every line of every *.jsonl parses as JSON. -# 4. Round-trip (unless --no-round-trip): fresh git repo + fresh +# 4. Validator fixture matrix: synthetic v1/v2 packs prove the checks above +# accept valid packs and refuse newer format versions, tombstone/count +# skew, malformed parents/watermarks, unredacted v2 packs, and unknown +# or undeclared tables. +# 5. Round-trip (unless --no-round-trip): fresh git repo + fresh # `carryctx init --minimal` scratch project, `carryctx import # --mode replace`, then the imported table counts must match the # manifest (worktrees rows may be pruned by re-anchoring, so # worktrees <= manifest is accepted and reported). -# 5. Redaction fixture: a synthetic two-file pack holding FAKE secret -# values (never real credentials) is copied to a staging dir and run -# through scripts/publish-ctxpack-redact.py; the staging copy must come -# out redacted (names kept, JSON valid, row counts unchanged) while the -# source copy still holds the fakes byte-identical (local DB untouched). +# 6. Redaction fixture: synthetic v1/v2 packs holding FAKE secret values +# (never real credentials) are copied to a staging dir and run through +# scripts/publish-ctxpack-redact.py; the staging copy must come out +# redacted (names kept, JSON valid, row counts unchanged, tombstones +# rows handled, manifest.redacted=true stamped, counts untouched) while +# the source copy still holds the fakes byte-identical (local DB +# untouched). # -# Known carryctx v1 limitation (loud WARN, still exit 0): import loads +# Known carryctx import limitation (loud WARN, still exit 0): import loads # sessions while worktree rows missing at the import target are pruned, so -# a pack containing worktree-bound sessions fails with -# "Failed to load pack table 'sessions': FOREIGN KEY constraint failed". +# a pack containing worktree-bound sessions can fail with +# "Failed to load pack table 'sessions': FOREIGN KEY constraint failed" on +# carryctx versions predating the nulling of pruned worktree references. # The export itself is intact (checks 1-3 prove it); only the re-import -# probe cannot pass until carryctx nulls or re-anchors those refs on -# import. Do not "fix" this by editing the snapshot. +# probe cannot pass until the import target handles those refs. Do not +# "fix" this by editing the snapshot. set -euo pipefail PACK="${1:?usage: publish-ctxpack-selftest.sh [--no-round-trip]}" @@ -81,80 +94,466 @@ trap cleanup EXIT log "checking manifest + per-table counts in $PACK" timeout "$TIMEOUT_SECS" python3 - "$PACK" <<'PYEOF' -import json, sys +import copy +import json +import os +import re +import shutil +import sys +import tempfile pack = sys.argv[1] -with open(pack + "/manifest.json") as f: - manifest = json.load(f) -if manifest.get("format") != "carryctx-pack-dir": - sys.exit("manifest format != carryctx-pack-dir: %r" % manifest.get("format")) -if manifest.get("format_version") != 1: - sys.exit("manifest format_version != 1: %r" % manifest.get("format_version")) -counts = manifest.get("counts") -if not isinstance(counts, dict) or not counts: - sys.exit("manifest counts missing or empty") -if not manifest.get("project_id"): - sys.exit("manifest project_id missing") - -with open(pack + "/project.json") as f: - project = json.load(f) -if project.get("id") != manifest["project_id"]: - sys.exit("project.json id != manifest project_id") - -worktree_bound_sessions = 0 -for table, expected in sorted(counts.items()): - path = "%s/%s.jsonl" % (pack, table) +PACK_FORMAT = "carryctx-pack-dir" +V1_TABLES = ( + "agents", + "tasks", + "task_dependencies", + "progress_items", + "sessions", + "worktrees", + "checkpoints", + "checkpoint_corrections", + "scopes", + "decisions", + "handoffs", + "teams", + "team_members", + "graph_nodes", + "graph_edges", + "events", + "sequences", +) +V2_TABLES = V1_TABLES + ("tombstones",) +CURRENT_FORMAT_VERSION = 2 +_RFC3339_RE = re.compile(r"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?(?:Z|[+-]\d{2}:\d{2})\Z") + + +class PackError(Exception): + pass + + +def require(condition, message): + if not condition: + raise PackError(message) + + +def is_int(value): + return isinstance(value, int) and not isinstance(value, bool) + + +def read_json(path, what): try: - with open(path) as f: - lines = f.read().splitlines() - except FileNotFoundError: - sys.exit("missing table file %s.jsonl (manifest wants %d rows)" % (table, expected)) - for i, line in enumerate(lines, 1): + with open(path) as handle: + return json.load(handle) + except FileNotFoundError as error: + raise PackError("%s is missing" % what) from error + except ValueError as error: + raise PackError("%s is not valid JSON (%s)" % (what, error)) from error + + +def validate_pack(directory, verbose=True): + """Validate a ctxpack directory (format v1 or v2), fail-closed. + + Mirrors the carryctx-pack manifest/check_counts contract; returns + (format_version, counts, worktree_bound_sessions). + """ + manifest = read_json(os.path.join(directory, "manifest.json"), "manifest.json") + require(isinstance(manifest, dict), "manifest.json is not a JSON object") + require( + manifest.get("format") == PACK_FORMAT, + "manifest format != %s: %r" % (PACK_FORMAT, manifest.get("format")), + ) + version = manifest.get("format_version") + require(is_int(version), "manifest format_version is not an integer: %r" % (version,)) + require( + version <= CURRENT_FORMAT_VERSION, + "manifest format_version %d is newer than supported %d" % (version, CURRENT_FORMAT_VERSION), + ) + require( + version >= 1, + "manifest format_version %d is older than the oldest supported 1" % version, + ) + for field in ("carryctx_version", "project_id", "export_id"): + value = manifest.get(field) + require( + isinstance(value, str) and value.strip(), + "manifest %s is missing or empty" % field, + ) + require(is_int(manifest.get("schema_version")), "manifest schema_version is not an integer") + created_at = manifest.get("created_at") + require( + isinstance(created_at, str) and _RFC3339_RE.match(created_at), + "manifest created_at is not RFC3339: %r" % (created_at,), + ) + require(isinstance(manifest.get("source"), dict), "manifest source is not an object") + for key, value in manifest["source"].items(): + require( + value is None or isinstance(value, str), + "manifest source.%s is not a string" % key, + ) + + counts = manifest.get("counts") + require(isinstance(counts, dict) and counts, "manifest counts missing or empty") + for table, expected in counts.items(): + require(isinstance(table, str) and table, "manifest counts has an empty table name") + require( + is_int(expected) and expected >= 0, + "manifest counts.%s is not a non-negative integer" % table, + ) + + redacted = manifest.get("redacted", False) + require(isinstance(redacted, bool), "manifest redacted is not a boolean: %r" % (redacted,)) + + allowed = V1_TABLES if version == 1 else V2_TABLES + unknown = sorted(set(counts) - set(allowed)) + require(not unknown, "manifest counts unknown table(s): %s" % ", ".join(unknown)) + + if version == 1: + # Legacy shape: v2-only fields must not appear in a v1 manifest. + require("tombstones" not in counts, "manifest v1 may not declare counts.tombstones") + require(not manifest.get("watermarks"), "manifest v1 may not carry watermarks") + else: + # v2 shape (carryctx-pack manifest.rs): tombstone counts are + # explicit, parents are unique non-empty ids that never self-reference, + # watermarks must agree with counts, and the publication stamp must be + # present -- this self-test runs after the redaction pass. + require("tombstones" in counts, "manifest v2 omits counts.tombstones") + require( + redacted is True, + "manifest v2 is not marked redacted: true (run the export-time redaction pass first)", + ) + parents = manifest.get("parents", []) + require(isinstance(parents, list), "manifest parents is not a list") + seen = set() + for parent in parents: + require( + isinstance(parent, str) and parent.strip(), + "manifest v2 has an empty parents entry", + ) + require( + parent != manifest["export_id"], + "manifest v2 lists its own export_id as a parent", + ) + require(parent not in seen, "manifest v2 lists parent %s more than once" % parent) + seen.add(parent) + watermarks = manifest.get("watermarks", {}) + require(isinstance(watermarks, dict), "manifest watermarks is not an object") + for table, watermark in sorted(watermarks.items()): + require(table in V2_TABLES, "manifest watermarks unknown table %s" % table) + require(isinstance(watermark, dict), "manifest watermark %s is not an object" % table) + rows = watermark.get("rows") + require( + is_int(rows) and rows >= 0, + "manifest watermark %s rows is not a non-negative integer" % table, + ) + declared = counts.get(table, 0) + require( + rows == declared, + "manifest watermark %s declares %d rows but counts declare %d" + % (table, rows, declared), + ) + max_updated = watermark.get("max_updated_at") + require( + max_updated is None or isinstance(max_updated, str), + "manifest watermark %s max_updated_at is not a string" % table, + ) + + sequences = manifest.get("sequences", {}) + require(isinstance(sequences, dict), "manifest sequences is not an object") + for kind, value in sequences.items(): + require( + is_int(value) and value >= 0, + "manifest sequences.%s is not a non-negative integer" % kind, + ) + + project = read_json(os.path.join(directory, "project.json"), "project.json") + require(isinstance(project, dict), "project.json is not a JSON object") + require( + project.get("id") == manifest["project_id"], + "project.json id != manifest project_id", + ) + + worktree_bound_sessions = 0 + for table in allowed: + path = os.path.join(directory, "%s.jsonl" % table) try: - row = json.loads(line) - except json.JSONDecodeError as e: - sys.exit("%s.jsonl line %d: invalid JSON (%s)" % (table, i, e)) - if table == "sessions" and row.get("worktree_id"): - worktree_bound_sessions += 1 - if len(lines) != expected: - sys.exit("%s.jsonl has %d rows, manifest wants %d" % (table, len(lines), expected)) - print(" table %-22s rows=%d ok" % (table, expected)) - -print("manifest + counts + JSON: PASS (%d tables)" % len(counts)) + with open(path) as handle: + lines = handle.read().splitlines() + except FileNotFoundError as error: + raise PackError("missing table file %s.jsonl" % table) from error + rows = 0 + for number, line in enumerate(lines, 1): + if not line.strip(): + continue + try: + row = json.loads(line) + except ValueError as error: + raise PackError( + "%s.jsonl line %d: invalid JSON (%s)" % (table, number, error) + ) from error + require( + isinstance(row, dict), + "%s.jsonl line %d is not a JSON object" % (table, number), + ) + rows += 1 + if table == "sessions" and row.get("worktree_id"): + worktree_bound_sessions += 1 + if table in counts: + require( + rows == counts[table], + "%s.jsonl has %d rows, manifest wants %d" % (table, rows, counts[table]), + ) + else: + require( + rows == 0, + "%s.jsonl has %d undeclared rows (manifest omits counts)" % (table, rows), + ) + if verbose: + print(" table %-22s rows=%d ok" % (table, rows)) + return version, counts, worktree_bound_sessions + + +def fixture_manifest(version): + return { + "format": PACK_FORMAT, + "format_version": version, + "carryctx_version": "0.9.1", + "schema_version": 18, + "project_id": "01FAKEFIXTURE00000000000000", + "export_id": "01FAKEFIXTURE00000000000001", + "created_at": "2026-09-10T00:00:00Z", + "parents": [], + "sequences": {"display_id_CTX": 1}, + "source": {"git_commit": "0" * 40, "hostname": "fixture"}, + "counts": {"tasks": 1, "tombstones": 1}, + "redacted": True, + } + + +def v1_manifest(): + manifest = fixture_manifest(1) + manifest["counts"].pop("tombstones", None) + manifest.pop("redacted", None) + return manifest + + +TASK_ROW = {"id": "01FAKEFIXTURE00000000000002", "title": "fixture"} +TOMBSTONE_ROW = { + "project_id": "01FAKEFIXTURE00000000000000", + "table_name": "worktrees", + "row_id": "01FAKEFIXTURE00000000000003", + "deleted_at": "2026-09-10T00:00:00Z", + "deleted_by": None, + "reason": "fixture", +} + + +def write_fixture(root, name, manifest, tables, raw_files=None, drop_files=(), project_id=None): + directory = os.path.join(root, name) + os.makedirs(directory) + with open(os.path.join(directory, "manifest.json"), "w") as handle: + json.dump(manifest, handle, indent=2) + with open(os.path.join(directory, "project.json"), "w") as handle: + json.dump({"id": manifest.get("project_id") if project_id is None else project_id}, handle) + table_set = set(V2_TABLES) + table_set.update(manifest.get("counts", {})) + table_set.update(tables) + for table in sorted(table_set): + with open(os.path.join(directory, "%s.jsonl" % table), "w") as handle: + for row in tables.get(table, []): + handle.write(json.dumps(row) + "\n") + for table, text in (raw_files or {}).items(): + with open(os.path.join(directory, "%s.jsonl" % table), "w") as handle: + handle.write(text) + for table in drop_files: + os.remove(os.path.join(directory, "%s.jsonl" % table)) + return directory + + +def build_cases(): + cases = [] + + def add(name, manifest, tables, expect, raw_files=None, drop_files=(), project_id=None): + cases.append( + { + "name": name, + "manifest": manifest, + "tables": tables, + "expect": expect, + "raw_files": raw_files, + "drop_files": drop_files, + "project_id": project_id, + } + ) + + add("v2-minimal", fixture_manifest(2), {"tasks": [TASK_ROW], "tombstones": [TOMBSTONE_ROW]}, None) + + manifest = fixture_manifest(2) + manifest["parents"] = ["01FAKEPARENT0000000000001"] + manifest["watermarks"] = {"tasks": {"rows": 1, "max_updated_at": "2026-09-10T00:00:00Z"}} + manifest["sequences"]["display_id_decision"] = 2 + add("v2-full", manifest, {"tasks": [TASK_ROW], "tombstones": [TOMBSTONE_ROW]}, None) + + add("v1-legacy", v1_manifest(), {"tasks": [TASK_ROW]}, None) + manifest = v1_manifest() + manifest["redacted"] = True + add("v1-redacted", manifest, {"tasks": [TASK_ROW]}, None) + + manifest = v1_manifest() + manifest["format_version"] = 3 + add("v3-future", manifest, {"tasks": [TASK_ROW]}, "newer than supported") + manifest = v1_manifest() + manifest["format_version"] = 0 + add("v0-ancient", manifest, {"tasks": [TASK_ROW]}, "older than the oldest") + + manifest = fixture_manifest(2) + manifest["counts"].pop("tombstones") + add("v2-missing-tombstone-count", manifest, {"tasks": [TASK_ROW], "tombstones": [TOMBSTONE_ROW]}, "tombstones") + manifest = fixture_manifest(2) + manifest["counts"]["tombstones"] = 2 + add("v2-tombstone-count-skew", manifest, {"tasks": [TASK_ROW], "tombstones": [TOMBSTONE_ROW]}, "tombstones") + manifest = fixture_manifest(2) + manifest["parents"] = [""] + add("v2-parent-empty", manifest, {"tasks": [TASK_ROW], "tombstones": [TOMBSTONE_ROW]}, "parents") + manifest = fixture_manifest(2) + manifest["parents"] = ["01FAKEPARENT0000000000001", "01FAKEPARENT0000000000001"] + add("v2-parent-duplicate", manifest, {"tasks": [TASK_ROW], "tombstones": [TOMBSTONE_ROW]}, "more than once") + manifest = fixture_manifest(2) + manifest["parents"] = [manifest["export_id"]] + add("v2-parent-self", manifest, {"tasks": [TASK_ROW], "tombstones": [TOMBSTONE_ROW]}, "own export_id") + manifest = fixture_manifest(2) + manifest["watermarks"] = {"bogus_table": {"rows": 0}} + add("v2-watermark-unknown-table", manifest, {"tasks": [TASK_ROW], "tombstones": [TOMBSTONE_ROW]}, "unknown table") + manifest = fixture_manifest(2) + manifest["watermarks"] = {"tasks": {"rows": 2}} + add("v2-watermark-skew", manifest, {"tasks": [TASK_ROW], "tombstones": [TOMBSTONE_ROW]}, "watermark") + manifest = fixture_manifest(2) + manifest["watermarks"] = {"tasks": {"rows": "1"}} + add("v2-watermark-bad-shape", manifest, {"tasks": [TASK_ROW], "tombstones": [TOMBSTONE_ROW]}, "watermark") + manifest = fixture_manifest(2) + manifest["redacted"] = False + add("v2-unredacted", manifest, {"tasks": [TASK_ROW], "tombstones": [TOMBSTONE_ROW]}, "redacted") + manifest = fixture_manifest(2) + manifest["redacted"] = "true" + add("v2-redacted-string", manifest, {"tasks": [TASK_ROW], "tombstones": [TOMBSTONE_ROW]}, "redacted") + + manifest = v1_manifest() + manifest["watermarks"] = {"tasks": {"rows": 1}} + add("v1-watermarks", manifest, {"tasks": [TASK_ROW]}, "watermarks") + manifest = v1_manifest() + manifest["counts"]["tombstones"] = 0 + add("v1-tombstone-count", manifest, {"tasks": [TASK_ROW]}, "tombstones") + + manifest = fixture_manifest(2) + manifest["counts"]["tasks"] = -1 + add("counts-negative", manifest, {"tasks": [TASK_ROW], "tombstones": [TOMBSTONE_ROW]}, "tasks") + manifest = fixture_manifest(2) + manifest["counts"]["bogus_table"] = 1 + add("counts-unknown-table", manifest, {"tasks": [TASK_ROW], "tombstones": [TOMBSTONE_ROW]}, "unknown table") + manifest = fixture_manifest(2) + add("declared-count-skew", manifest, {"tasks": [], "tombstones": [TOMBSTONE_ROW]}, "tasks.jsonl has 0 rows") + manifest = fixture_manifest(2) + add( + "tombstone-file-missing", + manifest, + {"tasks": [TASK_ROW], "tombstones": [TOMBSTONE_ROW]}, + "missing table file tombstones.jsonl", + drop_files=("tombstones",), + ) + manifest = fixture_manifest(2) + manifest["counts"].pop("tasks") + add("undeclared-rows", manifest, {"tasks": [TASK_ROW], "tombstones": [TOMBSTONE_ROW]}, "undeclared") + manifest = fixture_manifest(2) + manifest["project_id"] = "01FAKEFIXTURE00000000000099" + add( + "project-id-mismatch", + manifest, + {"tasks": [TASK_ROW], "tombstones": [TOMBSTONE_ROW]}, + "project.json", + project_id="01FAKEFIXTURE00000000000000", + ) + manifest = fixture_manifest(2) + add( + "bad-jsonl", + manifest, + {"tasks": [TASK_ROW], "tombstones": [TOMBSTONE_ROW]}, + "invalid JSON", + raw_files={"tasks": "{not json\n"}, + ) + return cases + + +try: + version, counts, worktree_bound_sessions = validate_pack(pack) +except PackError as failure: + sys.exit("publish-ctxpack-selftest: FAIL: %s" % failure) + +print("manifest v%d + counts + JSON: PASS (%d tables)" % (version, len(counts))) print("WORKTREE_BOUND_SESSIONS=%d" % worktree_bound_sessions) + +# Validator fixture matrix (check 4): the same validate_pack used on the real +# snapshot must accept valid v1/v2 packs and refuse the documented failure +# modes. +matrix_root = tempfile.mkdtemp(prefix="ctxpack-selftest-matrix.") +failures = [] +case_count = 0 +try: + for case in build_cases(): + directory = write_fixture( + matrix_root, + case["name"], + copy.deepcopy(case["manifest"]), + copy.deepcopy(case["tables"]), + raw_files=case["raw_files"], + drop_files=case["drop_files"], + project_id=case["project_id"], + ) + try: + validate_pack(directory, verbose=False) + error = None + except PackError as failure: + error = str(failure) + case_count += 1 + name = case["name"] + expect = case["expect"] + if expect is None: + if error is not None: + failures.append("fixture %s: expected accept, got: %s" % (name, error)) + elif error is None: + failures.append("fixture %s: expected reject (%s), got accept" % (name, expect)) + elif expect not in error: + failures.append("fixture %s: expected error containing %r, got: %s" % (name, expect, error)) +finally: + shutil.rmtree(matrix_root, ignore_errors=True) +if failures: + sys.exit("publish-ctxpack-selftest: FAIL: validator fixture matrix: " + "; ".join(failures)) +print("validator fixture matrix: PASS (%d cases)" % case_count) PYEOF -# Check 5 — planted-fake-secret redaction fixture (CTX-0281). Synthetic pack -# only; FAKE marker values, never real credentials. Models the pipeline -# guarantee: the staging copy is redacted, the source ("local DB") copy is -# byte-identical afterwards. +# Check 6 — planted-fake-secret redaction fixture (CTX-0281, extended for +# ctxpack v2 in CTX-0314). Synthetic v1 and v2 packs only; FAKE marker +# values, never real credentials. Models the pipeline guarantee: the staging +# copy is redacted and stamped redacted=true (counts untouched, tombstones +# rows redacted like any other table), the source ("local DB") copy stays +# byte-identical. REDACT_PY="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/publish-ctxpack-redact.py" [[ -f "$REDACT_PY" ]] || fail "redactor $REDACT_PY not found" -log "checking export-time redaction (planted-fake-secret fixture)" +log "checking export-time redaction (planted-fake-secret v1+v2 fixtures)" FIXT_ROOT="$(mktemp -d "${TMPDIR:-/tmp}/ctxpack-redact-fixture.XXXXXX")" -FIXT_LOCAL="$FIXT_ROOT/local" -FIXT_STAGING="$FIXT_ROOT/staging" -mkdir -p "$FIXT_LOCAL" "$FIXT_STAGING" -FIXT_LOCAL="$FIXT_LOCAL" FIXT_STAGING="$FIXT_STAGING" timeout "$TIMEOUT_SECS" python3 - <<'PYEOF' -import json, os +FIXT_ROOT="$FIXT_ROOT" timeout "$TIMEOUT_SECS" python3 - <<'PYEOF' +import json, os, shutil -local = os.environ["FIXT_LOCAL"] +root = os.environ["FIXT_ROOT"] # Fake-but-shaped secret values: each carries a FAKE marker so a hit can # never be mistaken for a real credential, while still tripping every # redactor rule (secret field names, NAME=value env-dump lines, 40+ runs). fakes = [ - "sk-FAKE-0123456789abcdef0123456789abcdef01", # OPENAI_API_KEY value + "sk-FAKE-0123456789abcdef0123456789abcdef01", # OPENAI_API_KEY / API_KEY value "FAKECLOUDFLARETOKENFAKECLOUDFLARE01", # CLOUDFLARE_API_TOKEN value "ghp_FAKEFAKEFAKEFAKEFAKEFAKEFAKEFAKE00", # GH_PAT-shaped value ] sha = "da39a3ee5e6b4b0d3255bfef95601890afd80709" # must SURVIVE (SHA-1 exempt) -manifest = { - "format": "carryctx-pack-dir", - "format_version": 1, - "project_id": "01FAKEFIXTURE00000000000000", - "counts": {"progress_items": 1, "events": 1}, -} progress_row = { "id": "01FAKEFIXTURE00000000000001", "task_id": "01FAKEFIXTURE00000000000002", @@ -169,64 +568,146 @@ event_row = { separators=(",", ":"), ), } -with open(local + "/manifest.json", "w") as f: - json.dump(manifest, f) -with open(local + "/project.json", "w") as f: - json.dump({"id": manifest["project_id"]}, f) -with open(local + "/progress_items.jsonl", "w") as f: - f.write(json.dumps(progress_row) + "\n") -with open(local + "/events.jsonl", "w") as f: - f.write(json.dumps(event_row) + "\n") -with open(local + "/fakes.txt", "w") as f: - f.write("\n".join(fakes + [sha]) + "\n") -print("fixture pack written (source copy)") +tombstone_row = { + "project_id": "01FAKEFIXTURE00000000000000", + "table_name": "worktrees", + "row_id": "01FAKEFIXTURE00000000000004", + "deleted_at": "2026-09-10T00:00:00Z", + "deleted_by": "ctxpack-selftest", + "reason": "cleanup API_KEY=%s" % fakes[0], +} +v2_manifest = { + "format": "carryctx-pack-dir", + "format_version": 2, + "carryctx_version": "0.9.1", + "schema_version": 18, + "project_id": "01FAKEFIXTURE00000000000000", + "export_id": "01FAKEFIXTURE00000000000005", + "created_at": "2026-09-10T00:00:00Z", + "parents": [], + "sequences": {"display_id_CTX": 1}, + "source": {"git_commit": "0" * 40, "hostname": "fixture"}, + "counts": {"progress_items": 1, "events": 1, "tombstones": 1}, + "watermarks": {"tombstones": {"rows": 1}}, +} +v1_manifest = { + "format": "carryctx-pack-dir", + "format_version": 1, + "project_id": "01FAKEFIXTURE00000000000000", + "counts": {"progress_items": 1, "events": 1}, +} +for name, manifest, tables in ( + ("local-v1", v1_manifest, {"progress_items": [progress_row], "events": [event_row]}), + ( + "local-v2", + v2_manifest, + {"progress_items": [progress_row], "events": [event_row], "tombstones": [tombstone_row]}, + ), +): + directory = os.path.join(root, name) + os.makedirs(directory) + with open(os.path.join(directory, "manifest.json"), "w") as handle: + json.dump(manifest, handle, indent=2) + handle.write("\n") + shutil.copyfile( + os.path.join(directory, "manifest.json"), + os.path.join(directory, "manifest.json.orig"), + ) + with open(os.path.join(directory, "project.json"), "w") as handle: + json.dump({"id": manifest["project_id"]}, handle) + for table, rows in tables.items(): + with open(os.path.join(directory, "%s.jsonl" % table), "w") as handle: + for row in rows: + handle.write(json.dumps(row) + "\n") +with open(os.path.join(root, "fakes.txt"), "w") as handle: + handle.write("\n".join(fakes + [sha]) + "\n") +print("fixture packs written (v1 + v2 source copies)") PYEOF -cp "$FIXT_LOCAL/progress_items.jsonl" "$FIXT_STAGING/progress_items.jsonl" -cp "$FIXT_LOCAL/events.jsonl" "$FIXT_STAGING/events.jsonl" -timeout "$TIMEOUT_SECS" python3 "$REDACT_PY" "$FIXT_STAGING" >/dev/null || - fail "redactor failed on the planted-secret fixture" -FIXT_LOCAL="$FIXT_LOCAL" FIXT_STAGING="$FIXT_STAGING" timeout "$TIMEOUT_SECS" python3 - <<'PYEOF' +for variant in v1 v2; do + cp -a "$FIXT_ROOT/local-$variant" "$FIXT_ROOT/staging-$variant" + timeout "$TIMEOUT_SECS" python3 "$REDACT_PY" "$FIXT_ROOT/staging-$variant" >/dev/null || + fail "redactor failed on the $variant planted-secret fixture" +done +FIXT_ROOT="$FIXT_ROOT" timeout "$TIMEOUT_SECS" python3 - <<'PYEOF' import json, os, sys -local = os.environ["FIXT_LOCAL"] -staging = os.environ["FIXT_STAGING"] -with open(local + "/fakes.txt") as f: - fakes_and_sha = f.read().split() +root = os.environ["FIXT_ROOT"] +with open(os.path.join(root, "fakes.txt")) as handle: + fakes_and_sha = handle.read().split() fakes, sha = fakes_and_sha[:3], fakes_and_sha[3] -staged_text = "" -for table in ("progress_items", "events"): - path = "%s/%s.jsonl" % (staging, table) - with open(path) as f: - lines = f.read().splitlines() - if len(lines) != 1: - sys.exit("staging %s.jsonl has %d rows, want 1" % (table, len(lines))) - for i, line in enumerate(lines, 1): - try: - json.loads(line) - except ValueError as e: - sys.exit("staging %s.jsonl line %d: invalid JSON (%s)" % (table, i, e)) - staged_text += "\n".join(lines) + "\n" - -for fake in fakes: - if fake in staged_text: - sys.exit("staging copy still contains planted fake value %s..." % fake[:12]) -if "***REDACTED***" not in staged_text: - sys.exit("staging copy has no ***REDACTED*** markers") -for name in ("OPENAI_API_KEY", "CLOUDFLARE_API_TOKEN", "gh_pat"): - if name not in staged_text: - sys.exit("staging copy lost field/variable name %s (names must survive)" % name) -if sha not in staged_text: - sys.exit("staging copy lost the SHA-1 control string (over-redaction)") - -local_text = "" -for table in ("progress_items", "events"): - with open("%s/%s.jsonl" % (local, table)) as f: - local_text += f.read() -for fake in fakes: - if fake not in local_text: - sys.exit("source copy lost planted fake value (local DB must stay untouched)") -print("redaction fixture: staging redacted, source untouched: PASS") +for variant, tables in ( + ("v1", ("progress_items", "events")), + ("v2", ("progress_items", "events", "tombstones")), +): + local = os.path.join(root, "local-" + variant) + staging = os.path.join(root, "staging-" + variant) + + # The source copy is the local DB stand-in: byte-identical manifest and + # the planted fakes still present. + with open(os.path.join(local, "manifest.json"), "rb") as handle: + source_manifest_bytes = handle.read() + with open(os.path.join(local, "manifest.json.orig"), "rb") as handle: + original_manifest_bytes = handle.read() + if source_manifest_bytes != original_manifest_bytes: + sys.exit("%s source manifest was rewritten (local DB must stay untouched)" % variant) + + with open(os.path.join(staging, "manifest.json")) as handle: + staged_manifest = json.load(handle) + with open(os.path.join(local, "manifest.json")) as handle: + source_manifest = json.load(handle) + if staged_manifest.get("redacted") is not True: + sys.exit("%s staging manifest: redacted is not true (%r)" % (variant, staged_manifest.get("redacted"))) + if staged_manifest.get("counts") != source_manifest.get("counts"): + sys.exit( + "%s staging manifest: counts changed (%r -> %r)" + % (variant, source_manifest.get("counts"), staged_manifest.get("counts")) + ) + + staged_text = "" + local_text = "" + for table in tables: + with open(os.path.join(local, "%s.jsonl" % table)) as handle: + local_lines = [line for line in handle.read().splitlines() if line.strip()] + local_text += "\n".join(local_lines) + "\n" + with open(os.path.join(staging, "%s.jsonl" % table)) as handle: + staged_lines = [line for line in handle.read().splitlines() if line.strip()] + if len(staged_lines) != len(local_lines): + sys.exit( + "%s staging %s.jsonl has %d rows, want %d" + % (variant, table, len(staged_lines), len(local_lines)) + ) + for number, line in enumerate(staged_lines, 1): + try: + json.loads(line) + except ValueError as error: + sys.exit("%s staging %s.jsonl line %d: invalid JSON (%s)" % (variant, table, number, error)) + staged_text += "\n".join(staged_lines) + "\n" + + for fake in fakes: + if fake in staged_text: + sys.exit("%s staging copy still contains planted fake value %s..." % (variant, fake[:12])) + if fake not in local_text: + sys.exit("%s source copy lost planted fake value (local DB must stay untouched)" % variant) + if "***REDACTED***" not in staged_text: + sys.exit("%s staging copy has no ***REDACTED*** markers" % variant) + for name in ("OPENAI_API_KEY", "CLOUDFLARE_API_TOKEN", "gh_pat"): + if name not in staged_text: + sys.exit("%s staging copy lost field/variable name %s (names must survive)" % (variant, name)) + if sha not in staged_text: + sys.exit("%s staging copy lost the SHA-1 control string (over-redaction)" % variant) + if variant == "v2": + with open(os.path.join(staging, "tombstones.jsonl")) as handle: + tombstone_rows = [json.loads(line) for line in handle if line.strip()] + if len(tombstone_rows) != 1: + sys.exit("v2 staging tombstones.jsonl has %d rows, want 1" % len(tombstone_rows)) + reason = tombstone_rows[0].get("reason") or "" + if tombstone_rows[0].get("table_name") != "worktrees" or "cleanup" not in reason: + sys.exit("v2 staging tombstone row content changed unexpectedly: %r" % tombstone_rows[0]) + if "API_KEY" not in reason: + sys.exit("v2 staging copy lost the tombstone reason variable name") + print(" redaction fixture %s: staging redacted + stamped, source untouched: PASS" % variant) +print("redaction fixture: PASS") PYEOF if [[ "$ROUND_TRIP" == 0 ]]; then diff --git a/scripts/publish-ctxpack.sh b/scripts/publish-ctxpack.sh index fe62d30..75ce74c 100755 --- a/scripts/publish-ctxpack.sh +++ b/scripts/publish-ctxpack.sh @@ -20,11 +20,15 @@ # secret-named fields (*_KEY/*_TOKEN/*_SECRET/*_PASSWORD, GH_PAT, # CLOUDFLARE_*, AWS_*) and 40+-char token-like runs become # `***REDACTED***`. JSONL-aware (parse per line, redact, re-serialize), -# row counts unchanged, re-runnable no-op. The local carryctx DB is +# row counts unchanged, re-runnable no-op, covers every table file +# including format-v2 `tombstones.jsonl`; stamps `redacted: true` into +# manifest.json (counts untouched). The local carryctx DB is # never modified -- it keeps the originals and the next export redacts # them again. -# 3. Validates (round-trip self-test, incl. a planted-fake-secret fixture -# proving staging is redacted while the source copy is untouched). +# 3. Validates (round-trip self-test for format v1 and v2 snapshots, incl. +# a planted-fake-secret fixture proving staging is redacted while the +# source copy is untouched, and a validator accept/reject fixture +# matrix). # 4. Refreshes the mirror README.md pointer + LATEST file. # 5. Commits and pushes to the mirror repo main branch. # @@ -214,12 +218,13 @@ not just the code. - Each `-/` directory is one `carryctx export --pack-format dir` snapshot (manifest.json + project.json - + per-table `*.jsonl`). + + per-table `*.jsonl`; format v2 snapshots also carry `tombstones.jsonl`). - `LATEST` names the newest snapshot directory. - Each snapshot carries a `source.json` with the source commit/branch it was taken from (`repo`, `repo_commit`, `repo_branch` keys). -- The mirror is publish-only: CarryCtx merge mode is unsupported (v1), so - snapshots are never merged back; re-import is a manual, replace-mode affair. +- Every snapshot is a redacted publication artifact (`redacted: true` in + manifest.json); CarryCtx refuses redacted bundles as merge sources, so + snapshots are never merged back. Re-import is a manual, replace-mode affair. - Trigger: the commander's merge closeout runs `just workflow-publish` in the __SOURCE_REPO__ repo. No git hook drives this (squash-merges never fire local hooks). @@ -228,8 +233,9 @@ not just the code. Snapshots intentionally contain agent display names and absolute workspace paths. Secret-shaped values never reach the mirror: every export passes -through an automatic JSONL-aware redaction step before validation and -commit (secret-named fields such as `*_KEY` / `*_TOKEN` / `*_SECRET` / +through an automatic JSONL-aware redaction step (all table files, including +`tombstones.jsonl`) before validation and commit (secret-named fields such +as `*_KEY` / `*_TOKEN` / `*_SECRET` / `*_PASSWORD`, `GH_PAT`, `CLOUDFLARE_*`, `AWS_*`, plus `NAME=value` pairs and 40+-char token-like runs in free text, become `***REDACTED***`; field/variable names are kept for debuggability; the local database is