From 35751dba4acd9b8587bbba52eb04e723115636d4 Mon Sep 17 00:00:00 2001 From: mcp-governance-curator Date: Thu, 10 Sep 2026 16:59:05 +0800 Subject: [PATCH] [CTX-0012] chore(devtools): publish workflow mirror (CTX-0012) --- justfile | 8 + scripts/publish-ctxpack-redact.py | 293 ++++++++++++++++++++++++ scripts/publish-ctxpack-selftest.sh | 300 +++++++++++++++++++++++++ scripts/publish-ctxpack.sh | 331 ++++++++++++++++++++++++++++ 4 files changed, 932 insertions(+) create mode 100644 scripts/publish-ctxpack-redact.py create mode 100755 scripts/publish-ctxpack-selftest.sh create mode 100755 scripts/publish-ctxpack.sh diff --git a/justfile b/justfile index cb5df45..cecf2d4 100644 --- a/justfile +++ b/justfile @@ -45,3 +45,11 @@ hooks-uninstall: # Run every stage gate (commit messages are gated separately per commit) check: lint fmt-check + +# Publish a ctxpack snapshot to the bitty-mcp-workflow mirror (commander +# merge closeout only; never a git hook). Dry run exports + validates without push. +workflow-publish *args: + bash scripts/publish-ctxpack.sh {{args}} + +workflow-publish-dry *args: + bash scripts/publish-ctxpack.sh --dry-run {{args}} diff --git a/scripts/publish-ctxpack-redact.py b/scripts/publish-ctxpack-redact.py new file mode 100644 index 0000000..56db1a2 --- /dev/null +++ b/scripts/publish-ctxpack-redact.py @@ -0,0 +1,293 @@ +#!/usr/bin/env python3 +"""publish-ctxpack-redact.py -- export-time secret redaction for ctxpack snapshots. + +Usage: publish-ctxpack-redact.py + +Rewrites every ``*.jsonl`` file under IN PLACE, replacing +secret-shaped values with ``***REDACTED***`` while keeping field names (and, +inside free text, variable names) intact for debuggability. + +What is redacted (fail-closed heuristics, JSONL-aware -- never blind sed): + 1. Key-name rule: any JSON object field whose name is secret-shaped (see + is_secret_name: a _-separated or camelCase segment equals KEY / TOKEN / + SECRET / PASSWORD / PAT, or the name contains GH_PAT, or it starts with + CLOUDFLARE_ / AWS_) has a string value replaced wholesale. + (AWS_/CLOUDFLARE_ prefixes intentionally over-match, e.g. AWS_REGION: + a lost region string is cheaper than a leaked credential. A bare KEY + segment such as keyboard-event handlers also redacts: fail-closed.) + 2. Free-text rule, applied to every remaining string value: + a. ``NAME=value`` / ``NAME: value`` pairs (shell env dumps, YAML-ish + prose, pasted kwargs) where NAME is secret-shaped per rule 1 -- the + name is kept, the value is replaced (quoted values may contain + spaces). Substring lookalikes do NOT match: ``monkey=``, ``keyboard=``, + ``DispatchPriority:``, ``GOPATH_BIN=`` survive because KEY / PAT is + not a standalone segment of those names. + b. Standalone 40+-char ``[A-Za-z0-9_\\-+]`` runs (plus up to two ``=`` + padding chars) that look token-ish: exactly-40 hex runs are EXEMPT + (git SHA-1s are ubiquitous in workflow prose and must survive; + longer hex still redacts), single-class runs (all-lowercase + test/branch names like ``soak_read_...``) are EXEMPT, and ``/`` and + mid-run ``=`` are deliberately NOT in the run alphabet -- so absolute + workspace paths and URLs (which the mirror intentionally publishes) + are never mangled, and ``NAME=`` never merges into one run + that would eat the field name. A run must mix at least two of + {lowercase, UPPERCASE, digits} or carry ``+``/``=`` base64 punctuation + to count as token-ish. Slash-bearing tokens are still caught by + rules 1 and 2a. + +Nested JSON-encoded strings (carryctx ``payload_json`` / ``metadata_json`` +columns) are decoded, redacted recursively, and re-encoded -- but only when +a redaction actually fired inside, so untouched rows stay byte-identical. + +Scope and safety: + - Operates ONLY on the snapshot directory argument (the staging/mirror + copy or a --dry-run tmp dir). It NEVER touches the local carryctx DB; + the DB keeps the original values and the next export redacts them again. + - Row counts and file sets are unchanged, so manifest counts and the + round-trip self-test still hold on redacted output. + - Re-running is a no-op (``***REDACTED***`` carries no secret shape). + - Files are replaced atomically (tmp + rename); rows that do not parse + as JSON are left byte-identical and reported (the self-test then fails + closed on them as malformed rows, as before). + +Exit 0 on success (prints a per-run summary plus a ``REDACTIONS=`` line +for the publish script to record in source.json). Exit 1 on structural +failure, 2 on usage error. +""" + +import json +import os +import re +import sys + +REDACTED = "***REDACTED***" + +# Candidate ["]NAME["]=value | NAME: value | NAME = "quoted value", all on ONE +# line (separator whitespace is [ \t]* so `word:` never swallows the next +# line as its value). The name may itself be quoted (prose-embedded JSON +# snippets); the callback re-validates the name via is_secret_name, so this +# stays deliberately broad (any word around : or =) and non-secret pairs +# pass through byte-identical. +_ASSIGN_RE = re.compile( + r"""(?P["']?)\b(?P[A-Za-z_][A-Za-z0-9_.\-]*)(?P=nq)""" + r"""[ \t]*(?P[:=])[ \t]*""" + r"""(?:(?P["'])(?P.*?)(?P=q)|(?P[^\s"'`,;]+))""", +) + +# `=` is padding-only, never a run joiner: otherwise NAME= would merge +# into one run and eat the field name. Trailing `={0,2}` still catches +# base64 padding without gluing across `=`. +_RUN_RE = re.compile(r"[A-Za-z0-9_\-+]{40,}={0,2}") +_SHA1_RE = re.compile(r"[0-9a-fA-F]{40}\Z") + +_SEGMENT_RE = re.compile(r"[A-Z]+(?![a-z])|[A-Z][a-z0-9]*|[a-z]+|[0-9]+") +_SECRET_SEGMENTS = frozenset(["KEY", "TOKEN", "SECRET", "PASSWORD", "PAT"]) + + +def _segments(name): + """Split a field/variable name into UPPER-cased word segments.""" + found = [] + for part in re.split(r"[_\-.]+", name): + found.extend(_SEGMENT_RE.findall(part)) + return [seg.upper() for seg in found] + + +def is_secret_name(name): + """True when a field/variable name is secret-shaped (spec list). + + Segment-based, so ``monkey``, ``keyboard``, ``tokenizer``, + ``DispatchPriority`` and ``GOPATH_BIN`` do NOT match while ``api_key``, + ``OPENAI_API_KEY``, ``gh_pat`` and ``handle_key_event`` do. + """ + upper = name.upper() + if "GH_PAT" in upper: + return True + if upper.startswith("CLOUDFLARE_") or upper.startswith("AWS_"): + return True + return any(seg in _SECRET_SEGMENTS for seg in _segments(name)) + + +def _looks_tokenish(run): + """True when a 40+ run mixes classes like a real token (not a slug).""" + if "+" in run or run.endswith("="): + return True + classes = 0 + if re.search(r"[a-z]", run): + classes += 1 + if re.search(r"[A-Z]", run): + classes += 1 + if re.search(r"[0-9]", run): + classes += 1 + return classes >= 2 + + +def _already_redacted(value): + return value == REDACTED or value.startswith("***") + + +def redact_text(text): + """Apply free-text rules to one string. Returns (new_text, count).""" + count = 0 + + # NOTE: the span rebuild below preserves the exact original separator + # whitespace (quoted values may contain spaces; bare values end at + # whitespace or quote/comma/semicolon boundaries). Names that are not + # secret-shaped pass through byte-identical (no count). + def _assign_simple(match): + nonlocal count + if not is_secret_name(match.group("name")): + return match.group(0) + qval = match.group("qval") + if qval is not None: + if not qval or _already_redacted(qval): + return match.group(0) + count += 1 + start, end = match.span("qval") + return match.string[match.start():start] + REDACTED + match.string[end:match.end()] + value = match.group("val") + if not value or _already_redacted(value): + return match.group(0) + count += 1 + start, end = match.span("val") + return match.string[match.start():start] + REDACTED + match.string[end:match.end()] + + text = _ASSIGN_RE.sub(_assign_simple, text) + + def _run(match): + nonlocal count + run = match.group(0) + if _already_redacted(run): + return run + if _SHA1_RE.match(run): + return run # git SHA-1 exemption (documented above) + if not _looks_tokenish(run): + return run # single-class slug (branch/test/flag name), not a token + count += 1 + return REDACTED + + text = _RUN_RE.sub(_run, text) + return text, count + + +def redact_value(node): + """Recursively redact decoded JSON. Returns (new_node, count).""" + if isinstance(node, dict): + count = 0 + out = {} + for key, value in node.items(): + if isinstance(key, str) and isinstance(value, str): + if is_secret_name(key): + if value and not _already_redacted(value): + out[key] = REDACTED + count += 1 + else: + out[key] = value + continue + new_value, sub = redact_value(value) + out[key] = new_value + count += sub + return out, count + if isinstance(node, list): + count = 0 + out = [] + for item in node: + new_item, sub = redact_value(item) + out.append(new_item) + count += sub + return out, count + if isinstance(node, str): + stripped = node.strip() + if len(stripped) >= 2 and stripped[0] in "{[": + try: + decoded = json.loads(stripped) + except ValueError: + decoded = None + if isinstance(decoded, (dict, list)): + new_decoded, sub = redact_value(decoded) + if sub: + return ( + json.dumps(new_decoded, ensure_ascii=False, separators=(",", ":")), + sub, + ) + return node, 0 + return redact_text(node) + return node, 0 + + +def redact_file(path): + """Redact one .jsonl file atomically. Returns (rows, count, skipped).""" + with open(path, "r", encoding="utf-8") as handle: + raw_lines = handle.read().splitlines() + out_lines = [] + rows = 0 + count = 0 + skipped = 0 + changed = False + for line in raw_lines: + if not line.strip(): + out_lines.append(line) + continue + try: + row = json.loads(line) + except ValueError: + skipped += 1 + out_lines.append(line) # byte-identical; self-test fails closed + continue + rows += 1 + new_row, sub = redact_value(row) + count += sub + if sub: + out_lines.append(json.dumps(new_row, ensure_ascii=False)) + changed = True + else: + out_lines.append(line) # byte-identical fast path + if changed: + tmp_path = path + ".redact-tmp" + with open(tmp_path, "w", encoding="utf-8") as handle: + handle.write("\n".join(out_lines)) + if out_lines: + handle.write("\n") + os.replace(tmp_path, path) + return rows, count, skipped + + +def main(argv): + if len(argv) != 2 or argv[1] in ("-h", "--help"): + sys.stderr.write("usage: publish-ctxpack-redact.py \n") + return 2 + snap = argv[1] + if not os.path.isdir(snap): + sys.stderr.write("publish-ctxpack-redact: FAIL: not a directory: %s\n" % snap) + return 1 + files = sorted( + name + for name in os.listdir(snap) + if name.endswith(".jsonl") + and os.path.isfile(os.path.join(snap, name)) + and not os.path.islink(os.path.join(snap, name)) + ) + if not files: + sys.stderr.write( + "publish-ctxpack-redact: FAIL: no *.jsonl files in %s (not a ctxpack dir?)\n" % snap + ) + return 1 + total_rows = 0 + total_count = 0 + total_skipped = 0 + for name in files: + rows, count, skipped = redact_file(os.path.join(snap, name)) + total_rows += rows + total_count += count + total_skipped += skipped + if count or skipped: + print(" %-28s rows=%d redactions=%d nonjson_skipped=%d" % (name, rows, count, skipped)) + print( + "publish-ctxpack-redact: %d files, %d rows, %d replacements, %d non-JSON lines left untouched" + % (len(files), total_rows, total_count, total_skipped) + ) + print("REDACTIONS=%d" % total_count) + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv)) diff --git a/scripts/publish-ctxpack-selftest.sh b/scripts/publish-ctxpack-selftest.sh new file mode 100755 index 0000000..a9c5776 --- /dev/null +++ b/scripts/publish-ctxpack-selftest.sh @@ -0,0 +1,300 @@ +#!/usr/bin/env bash +# publish-ctxpack-selftest.sh — round-trip self-test for a ctxpack snapshot. +# +# Usage: scripts/publish-ctxpack-selftest.sh [--no-round-trip] +# +# Checks (fail-closed, exit 1): +# 1. manifest.json exists with format=carryctx-pack-dir, format_version=1, +# a non-empty counts map and a project_id; project.json exists. +# 2. Every counts entry has a matching .jsonl whose line count +# equals the manifest count. +# 3. Every line of every *.jsonl parses as JSON. +# 4. Round-trip (unless --no-round-trip): fresh git repo + fresh +# `carryctx init --minimal` scratch project, `carryctx import +# --mode replace`, then the imported table counts must match the +# manifest (worktrees rows may be pruned by re-anchoring, so +# worktrees <= manifest is accepted and reported). +# 5. Redaction fixture: a synthetic two-file pack holding FAKE secret +# values (never real credentials) is copied to a staging dir and run +# through scripts/publish-ctxpack-redact.py; the staging copy must come +# out redacted (names kept, JSON valid, row counts unchanged) while the +# source copy still holds the fakes byte-identical (local DB untouched). +# +# Known carryctx v1 limitation (loud WARN, still exit 0): import loads +# sessions while worktree rows missing at the import target are pruned, so +# a pack containing worktree-bound sessions fails with +# "Failed to load pack table 'sessions': FOREIGN KEY constraint failed". +# The export itself is intact (checks 1-3 prove it); only the re-import +# probe cannot pass until carryctx nulls or re-anchors those refs on +# import. Do not "fix" this by editing the snapshot. +set -euo pipefail + +PACK="${1:?usage: publish-ctxpack-selftest.sh [--no-round-trip]}" +ROUND_TRIP=1 +if [[ "${2:-}" == "--no-round-trip" ]]; then + ROUND_TRIP=0 +elif [[ -n "${2:-}" ]]; then + echo "publish-ctxpack-selftest: FAIL: unknown flag $2" >&2 + exit 2 +fi + +TIMEOUT_SECS="${SELFTEST_TIMEOUT:-120}" +KEEP_SCRATCH=0 +if [[ "${SELFTEST_KEEP_SCRATCH:-0}" == 1 ]]; then + KEEP_SCRATCH=1 +fi + +fail() { + echo "publish-ctxpack-selftest: FAIL: $1" >&2 + exit 1 +} + +warn() { + echo "publish-ctxpack-selftest: WARN: $1" >&2 +} + +log() { + echo "publish-ctxpack-selftest: $1" +} + +command -v python3 >/dev/null 2>&1 || fail "python3 not on PATH" +command -v carryctx >/dev/null 2>&1 || fail "carryctx not on PATH" +command -v git >/dev/null 2>&1 || fail "git not on PATH" +command -v timeout >/dev/null 2>&1 || fail "timeout not on PATH" +[[ -d "$PACK" ]] || fail "export dir $PACK not found" + +SCRATCH="" +FIXT_ROOT="" +cleanup() { + if [[ "$KEEP_SCRATCH" == 0 && -n "$SCRATCH" && -d "$SCRATCH" ]]; then + rm -rf "$SCRATCH" + elif [[ -n "$SCRATCH" ]]; then + log "keeping scratch dir $SCRATCH (SELFTEST_KEEP_SCRATCH=1)" + fi + if [[ "$KEEP_SCRATCH" == 0 && -n "$FIXT_ROOT" && -d "$FIXT_ROOT" ]]; then + rm -rf "$FIXT_ROOT" + elif [[ -n "$FIXT_ROOT" ]]; then + log "keeping fixture dir $FIXT_ROOT (SELFTEST_KEEP_SCRATCH=1)" + fi +} +trap cleanup EXIT + +log "checking manifest + per-table counts in $PACK" +timeout "$TIMEOUT_SECS" python3 - "$PACK" <<'PYEOF' +import json, sys + +pack = sys.argv[1] + +with open(pack + "/manifest.json") as f: + manifest = json.load(f) +if manifest.get("format") != "carryctx-pack-dir": + sys.exit("manifest format != carryctx-pack-dir: %r" % manifest.get("format")) +if manifest.get("format_version") != 1: + sys.exit("manifest format_version != 1: %r" % manifest.get("format_version")) +counts = manifest.get("counts") +if not isinstance(counts, dict) or not counts: + sys.exit("manifest counts missing or empty") +if not manifest.get("project_id"): + sys.exit("manifest project_id missing") + +with open(pack + "/project.json") as f: + project = json.load(f) +if project.get("id") != manifest["project_id"]: + sys.exit("project.json id != manifest project_id") + +worktree_bound_sessions = 0 +for table, expected in sorted(counts.items()): + path = "%s/%s.jsonl" % (pack, table) + try: + with open(path) as f: + lines = f.read().splitlines() + except FileNotFoundError: + sys.exit("missing table file %s.jsonl (manifest wants %d rows)" % (table, expected)) + for i, line in enumerate(lines, 1): + try: + row = json.loads(line) + except json.JSONDecodeError as e: + sys.exit("%s.jsonl line %d: invalid JSON (%s)" % (table, i, e)) + if table == "sessions" and row.get("worktree_id"): + worktree_bound_sessions += 1 + if len(lines) != expected: + sys.exit("%s.jsonl has %d rows, manifest wants %d" % (table, len(lines), expected)) + print(" table %-22s rows=%d ok" % (table, expected)) + +print("manifest + counts + JSON: PASS (%d tables)" % len(counts)) +print("WORKTREE_BOUND_SESSIONS=%d" % worktree_bound_sessions) +PYEOF + +# Check 5 — planted-fake-secret redaction fixture (CTX-0281). Synthetic pack +# only; FAKE marker values, never real credentials. Models the pipeline +# guarantee: the staging copy is redacted, the source ("local DB") copy is +# byte-identical afterwards. +REDACT_PY="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/publish-ctxpack-redact.py" +[[ -f "$REDACT_PY" ]] || fail "redactor $REDACT_PY not found" +log "checking export-time redaction (planted-fake-secret fixture)" +FIXT_ROOT="$(mktemp -d "${TMPDIR:-/tmp}/ctxpack-redact-fixture.XXXXXX")" +FIXT_LOCAL="$FIXT_ROOT/local" +FIXT_STAGING="$FIXT_ROOT/staging" +mkdir -p "$FIXT_LOCAL" "$FIXT_STAGING" +FIXT_LOCAL="$FIXT_LOCAL" FIXT_STAGING="$FIXT_STAGING" timeout "$TIMEOUT_SECS" python3 - <<'PYEOF' +import json, os + +local = os.environ["FIXT_LOCAL"] +# Fake-but-shaped secret values: each carries a FAKE marker so a hit can +# never be mistaken for a real credential, while still tripping every +# redactor rule (secret field names, NAME=value env-dump lines, 40+ runs). +fakes = [ + "sk-FAKE-0123456789abcdef0123456789abcdef01", # OPENAI_API_KEY value + "FAKECLOUDFLARETOKENFAKECLOUDFLARE01", # CLOUDFLARE_API_TOKEN value + "ghp_FAKEFAKEFAKEFAKEFAKEFAKEFAKEFAKE00", # GH_PAT-shaped value +] +sha = "da39a3ee5e6b4b0d3255bfef95601890afd80709" # must SURVIVE (SHA-1 exempt) +manifest = { + "format": "carryctx-pack-dir", + "format_version": 1, + "project_id": "01FAKEFIXTURE00000000000000", + "counts": {"progress_items": 1, "events": 1}, +} +progress_row = { + "id": "01FAKEFIXTURE00000000000001", + "task_id": "01FAKEFIXTURE00000000000002", + "type": "note", + "content": "shell env:\nOPENAI_API_KEY=%s\nCLOUDFLARE_API_TOKEN=%s\nplain prose stays" % (fakes[0], fakes[1]), +} +event_row = { + "id": "01FAKEFIXTURE00000000000003", + "type": "progress.created", + "payload_json": json.dumps( + {"gh_pat": fakes[2], "detail": "session done", "sha": sha}, + separators=(",", ":"), + ), +} +with open(local + "/manifest.json", "w") as f: + json.dump(manifest, f) +with open(local + "/project.json", "w") as f: + json.dump({"id": manifest["project_id"]}, f) +with open(local + "/progress_items.jsonl", "w") as f: + f.write(json.dumps(progress_row) + "\n") +with open(local + "/events.jsonl", "w") as f: + f.write(json.dumps(event_row) + "\n") +with open(local + "/fakes.txt", "w") as f: + f.write("\n".join(fakes + [sha]) + "\n") +print("fixture pack written (source copy)") +PYEOF +cp "$FIXT_LOCAL/progress_items.jsonl" "$FIXT_STAGING/progress_items.jsonl" +cp "$FIXT_LOCAL/events.jsonl" "$FIXT_STAGING/events.jsonl" +timeout "$TIMEOUT_SECS" python3 "$REDACT_PY" "$FIXT_STAGING" >/dev/null || + fail "redactor failed on the planted-secret fixture" +FIXT_LOCAL="$FIXT_LOCAL" FIXT_STAGING="$FIXT_STAGING" timeout "$TIMEOUT_SECS" python3 - <<'PYEOF' +import json, os, sys + +local = os.environ["FIXT_LOCAL"] +staging = os.environ["FIXT_STAGING"] +with open(local + "/fakes.txt") as f: + fakes_and_sha = f.read().split() +fakes, sha = fakes_and_sha[:3], fakes_and_sha[3] + +staged_text = "" +for table in ("progress_items", "events"): + path = "%s/%s.jsonl" % (staging, table) + with open(path) as f: + lines = f.read().splitlines() + if len(lines) != 1: + sys.exit("staging %s.jsonl has %d rows, want 1" % (table, len(lines))) + for i, line in enumerate(lines, 1): + try: + json.loads(line) + except ValueError as e: + sys.exit("staging %s.jsonl line %d: invalid JSON (%s)" % (table, i, e)) + staged_text += "\n".join(lines) + "\n" + +for fake in fakes: + if fake in staged_text: + sys.exit("staging copy still contains planted fake value %s..." % fake[:12]) +if "***REDACTED***" not in staged_text: + sys.exit("staging copy has no ***REDACTED*** markers") +for name in ("OPENAI_API_KEY", "CLOUDFLARE_API_TOKEN", "gh_pat"): + if name not in staged_text: + sys.exit("staging copy lost field/variable name %s (names must survive)" % name) +if sha not in staged_text: + sys.exit("staging copy lost the SHA-1 control string (over-redaction)") + +local_text = "" +for table in ("progress_items", "events"): + with open("%s/%s.jsonl" % (local, table)) as f: + local_text += f.read() +for fake in fakes: + if fake not in local_text: + sys.exit("source copy lost planted fake value (local DB must stay untouched)") +print("redaction fixture: staging redacted, source untouched: PASS") +PYEOF + +if [[ "$ROUND_TRIP" == 0 ]]; then + log "round-trip skipped (--no-round-trip); static checks passed" + exit 0 +fi + +# Round-trip probe: the export must re-import into a scratch project with +# matching table counts. Capture the worktree-bound session count first so +# the known-import-limitation WARN below can key off pack content, not +# import stderr text alone. +BOUND="$(timeout "$TIMEOUT_SECS" python3 -c " +import json +n = 0 +with open('$PACK/sessions.jsonl') as f: + for line in f: + if json.loads(line).get('worktree_id'): + n += 1 +print(n) +")" + +SCRATCH="$(mktemp -d "${TMPDIR:-/tmp}/ctxpack-roundtrip.XXXXXX")" +log "round-trip scratch: $SCRATCH" +timeout "$TIMEOUT_SECS" git -C "$SCRATCH" init -q || + fail "git init failed in scratch" +timeout "$TIMEOUT_SECS" carryctx init --project "$SCRATCH" --name ctxpack-roundtrip --minimal --agent ctxpack-selftest >/dev/null || + fail "carryctx init failed in scratch" + +IMPORT_OUT="$SCRATCH/import.json" +set +e +timeout "$TIMEOUT_SECS" carryctx import "$PACK" --project "$SCRATCH" --mode replace --yes --agent ctxpack-selftest >"$IMPORT_OUT" 2>"$SCRATCH/import.err" +IMPORT_RC=$? +set -e + +if [[ "$IMPORT_RC" -ne 0 ]]; then + if grep -q "FOREIGN KEY constraint failed" "$SCRATCH/import.err" "$IMPORT_OUT" 2>/dev/null && + [[ "$BOUND" -gt 0 ]]; then + warn "import probe hit the known carryctx v1 limitation: pack holds $BOUND worktree-bound session(s) whose worktree rows are pruned at the import target, so sessions fail FK (export itself validated above; mirror stays publish-only)" + warn "round-trip: KNOWN-ISSUE (see script header); static checks passed" + exit 0 + fi + cat "$SCRATCH/import.err" >&2 2>/dev/null || true + fail "carryctx import failed (rc=$IMPORT_RC); see above" +fi + +log "comparing imported counts with manifest" +timeout "$TIMEOUT_SECS" python3 - "$PACK" "$IMPORT_OUT" <<'PYEOF' +import json, sys + +with open(sys.argv[1] + "/manifest.json") as f: + expected = json.load(f)["counts"] +with open(sys.argv[2]) as f: + got = json.load(f)["counts"] + +for table in sorted(expected): + if table not in got: + sys.exit("imported counts missing table %s" % table) + if table == "worktrees": + # Re-anchoring prunes worktree rows whose dirs are absent at the + # import target; fewer-or-equal is the honest expectation here. + if got[table] > expected[table]: + sys.exit("worktrees grew on import (%d > %d)" % (got[table], expected[table])) + print(" table %-22s manifest=%d imported=%d (prune-tolerant) ok" % (table, expected[table], got[table])) + elif got[table] != expected[table]: + sys.exit("table %s: manifest=%d imported=%d" % (table, expected[table], got[table])) + else: + print(" table %-22s rows=%d ok" % (table, got[table])) +print("round-trip counts: PASS") +PYEOF + +log "self-test PASS" diff --git a/scripts/publish-ctxpack.sh b/scripts/publish-ctxpack.sh new file mode 100755 index 0000000..fe62d30 --- /dev/null +++ b/scripts/publish-ctxpack.sh @@ -0,0 +1,331 @@ +#!/usr/bin/env bash +# publish-ctxpack.sh — publish a ctxpack snapshot to the workflow-mirror repo. +# +# On every PR merge the commander runs `just workflow-publish` from a +# bitty-mcp checkout (the trigger is the merge closeout, NOT a git hook: +# GitHub squash-merges never fire local git hooks, and `carryctx hooks +# install` behavior is intentionally left untouched). +# +# Repo-specific bits are env defaults, not hardcodes: override +# WORKFLOW_SOURCE_REPO / WORKFLOW_MIRROR_URL / WORKFLOW_MIRROR_DIR to reuse +# this script verbatim in another repo. Defaults target +# bitty-terminal/bitty-mcp-workflow. +# +# What it does: +# 1. Exports `carryctx export --pack-format dir` from this repo into a +# staging clone of bitty-terminal/bitty-mcp-workflow as +# -/. +# 2. Redacts secret-shaped values inside the STAGING copy only (export-time +# redaction pass, scripts/publish-ctxpack-redact.py): values of +# secret-named fields (*_KEY/*_TOKEN/*_SECRET/*_PASSWORD, GH_PAT, +# CLOUDFLARE_*, AWS_*) and 40+-char token-like runs become +# `***REDACTED***`. JSONL-aware (parse per line, redact, re-serialize), +# row counts unchanged, re-runnable no-op. The local carryctx DB is +# never modified -- it keeps the originals and the next export redacts +# them again. +# 3. Validates (round-trip self-test, incl. a planted-fake-secret fixture +# proving staging is redacted while the source copy is untouched). +# 4. Refreshes the mirror README.md pointer + LATEST file. +# 5. Commits and pushes to the mirror repo main branch. +# +# Privacy: the pack intentionally contains agent display names and absolute +# workspace paths, but secret-shaped values can NEVER reach the mirror: the +# redaction pass (step 2) runs on every export, dry-run or publish, before +# validation and commit, so even a live key accidentally pasted into a note +# or session payload is replaced with `***REDACTED***` (names kept for +# debuggability). A leaked-then-rotated secret still needs rotation at the +# source -- redaction limits mirror exposure, it does not un-leak anything. +# +# Usage: +# scripts/publish-ctxpack.sh [--dry-run] [--mirror DIR] [--url URL] +# [--git-timeout SECS] [--keep-tmp] +# --dry-run export + validate (incl. round-trip self-test) only; +# no clone, no commit, no push. +# --mirror DIR staging clone location +# (default: $WORKFLOW_MIRROR_DIR or +# /../bitty-mcp-workflow). +# --url URL mirror remote for a fresh clone +# (default: $WORKFLOW_MIRROR_URL or +# https://github.com/bitty-terminal/bitty-mcp-workflow.git). +# --git-timeout SECS timeout for every git/carryctx network or heavy op +# (default: $GIT_TIMEOUT or 120). +# --keep-tmp keep temp dirs on exit (debug aid; prints paths). +# +# Guarantees: idempotent (re-running for the same source SHA reuses the +# existing snapshot dir and no-ops when the mirror is already current); +# fail-closed with clear errors; never writes to the source repo's own git +# state (only read-only rev-parse/branch queries there). +set -euo pipefail + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +SOURCE_REPO="${WORKFLOW_SOURCE_REPO:-bitty-mcp}" +MIRROR_NAME="${SOURCE_REPO}-workflow" +MIRROR_DEFAULT="$REPO_ROOT/../$MIRROR_NAME" +MIRROR_URL_DEFAULT="https://github.com/bitty-terminal/$MIRROR_NAME.git" + +DRY_RUN=0 +MIRROR="${WORKFLOW_MIRROR_DIR:-$MIRROR_DEFAULT}" +MIRROR_URL="${WORKFLOW_MIRROR_URL:-$MIRROR_URL_DEFAULT}" +GIT_TIMEOUT="${GIT_TIMEOUT:-120}" +KEEP_TMP=0 +REDACTIONS=0 + +while [[ $# -gt 0 ]]; do + case "$1" in + --dry-run) + DRY_RUN=1 + shift + ;; + --mirror) + MIRROR="${2:?--mirror requires a directory}" + shift 2 + ;; + --mirror=*) + MIRROR="${1#--mirror=}" + shift + ;; + --url) + MIRROR_URL="${2:?--url requires a remote URL}" + shift 2 + ;; + --url=*) + MIRROR_URL="${1#--url=}" + shift + ;; + --git-timeout) + GIT_TIMEOUT="${2:?--git-timeout requires seconds}" + shift 2 + ;; + --git-timeout=*) + GIT_TIMEOUT="${1#--git-timeout=}" + shift + ;; + --keep-tmp) + KEEP_TMP=1 + shift + ;; + --help | -h) + sed -n '2,/^set -euo/p' "${BASH_SOURCE[0]}" + exit 0 + ;; + *) + echo "publish-ctxpack: FAIL: unknown flag $1 (see --help)" >&2 + exit 2 + ;; + esac +done + +fail() { + echo "publish-ctxpack: FAIL: $1" >&2 + exit 1 +} + +log() { + echo "publish-ctxpack: $1" +} + +have() { command -v "$1" >/dev/null 2>&1; } + +have git || fail "git not on PATH" +have carryctx || fail "carryctx not on PATH" +have timeout || fail "timeout not on PATH" +have python3 || fail "python3 not on PATH (needed by the redaction pass)" + +# Read-only introspection of the source checkout. Never a write op here: +# no checkout, reset, commit, stash, or index/worktree mutation. +REPO_SHA="$(timeout "$GIT_TIMEOUT" git -C "$REPO_ROOT" rev-parse HEAD 2>/dev/null)" || + fail "cannot read $SOURCE_REPO HEAD (not a git checkout?)" +REPO_SHORT="$(timeout "$GIT_TIMEOUT" git -C "$REPO_ROOT" rev-parse --short HEAD 2>/dev/null)" || + fail "cannot read $SOURCE_REPO short SHA" +REPO_BRANCH="$(timeout "$GIT_TIMEOUT" git -C "$REPO_ROOT" branch --show-current 2>/dev/null || true)" +if [[ "${REPO_BRANCH:-}" != "main" ]]; then + log "WARN: $SOURCE_REPO checkout is on branch '${REPO_BRANCH:-detached}', snapshotting HEAD anyway" +fi +STAMP="$(date -u +%Y%m%dT%H%M%SZ)" +SNAPSHOT="${STAMP}-${REPO_SHORT}" + +TMP_ROOT="" +cleanup() { + if [[ "$KEEP_TMP" == 0 && -n "$TMP_ROOT" && -d "$TMP_ROOT" ]]; then + rm -rf "$TMP_ROOT" + elif [[ -n "$TMP_ROOT" ]]; then + log "keeping tmp dir $TMP_ROOT (--keep-tmp)" + fi +} +trap cleanup EXIT + +# Mirror git ops always go through this wrapper: timeout-wrapped and +# confined to the staging clone. $REPO_ROOT is never a write target. +mirror_git() { + timeout "$GIT_TIMEOUT" git -C "$MIRROR" "$@" +} + +ensure_mirror() { + if [[ -d "$MIRROR/.git" ]]; then + local remote + remote="$(mirror_git remote get-url origin 2>/dev/null)" || + fail "staging clone $MIRROR has no origin remote" + case "$remote" in + *"$MIRROR_NAME"*) + ;; + *) + fail "staging clone origin ($remote) does not look like $MIRROR_NAME; refusing to push (override with --mirror/--url)" + ;; + esac + if [[ -n "$(mirror_git status --porcelain 2>/dev/null)" ]]; then + fail "staging clone $MIRROR has uncommitted changes; clean it up and re-run" + fi + mirror_git fetch origin --prune || + fail "git fetch failed in staging clone (network/auth?)" + if mirror_git rev-parse --verify HEAD >/dev/null 2>&1; then + mirror_git checkout -q main 2>/dev/null || + mirror_git checkout -q -b main --track origin/main || + fail "cannot check out main in staging clone" + mirror_git reset -q --hard origin/main || + fail "cannot fast-forward staging clone to origin/main" + else + # Freshly cloned empty repo (or brand-new mirror): start main here. + mirror_git checkout -q -b main || + fail "cannot create main branch in empty staging clone" + fi + return 0 + fi + if [[ -e "$MIRROR" ]]; then + fail "$MIRROR exists but is not a git clone; move it aside or pass --mirror" + fi + log "cloning mirror $MIRROR_URL -> $MIRROR" + timeout "$GIT_TIMEOUT" git clone -q "$MIRROR_URL" "$MIRROR" || + fail "git clone failed (network/auth?); set --url or pre-clone --mirror yourself" + ensure_mirror +} + +write_mirror_readme() { + # Refresh the mirror-root pointer only when the template changes + # (keeps re-publishes idempotent: no churn commit for identical text). + local readme="$MIRROR/README.md" tmp + tmp="$(mktemp)" + cat >"$tmp" <<'EOF' +# __MIRROR_NAME__ — engineering workflow mirror + +Transparent companion to [__SOURCE_REPO__](https://github.com/bitty-terminal/__SOURCE_REPO__): +every PR merge publishes a CarryCtx ctxpack snapshot here, so the whole +engineering workflow (tasks, sessions, decisions, checkpoints) is reviewable, +not just the code. + +- Each `-/` directory is one + `carryctx export --pack-format dir` snapshot (manifest.json + project.json + + per-table `*.jsonl`). +- `LATEST` names the newest snapshot directory. +- Each snapshot carries a `source.json` with the source commit/branch it was + taken from (`repo`, `repo_commit`, `repo_branch` keys). +- The mirror is publish-only: CarryCtx merge mode is unsupported (v1), so + snapshots are never merged back; re-import is a manual, replace-mode affair. +- Trigger: the commander's merge closeout runs `just workflow-publish` in + the __SOURCE_REPO__ repo. No git hook drives this (squash-merges never fire local + hooks). + +## Privacy notice + +Snapshots intentionally contain agent display names and absolute workspace +paths. Secret-shaped values never reach the mirror: every export passes +through an automatic JSONL-aware redaction step before validation and +commit (secret-named fields such as `*_KEY` / `*_TOKEN` / `*_SECRET` / +`*_PASSWORD`, `GH_PAT`, `CLOUDFLARE_*`, `AWS_*`, plus `NAME=value` pairs +and 40+-char token-like runs in free text, become `***REDACTED***`; +field/variable names are kept for debuggability; the local database is +never modified). Redaction limits mirror exposure -- it does not un-leak a +secret that was already pushed anywhere: rotate at the source and report +suspected leaks to the repository owner immediately. +EOF + sed -i "s/__MIRROR_NAME__/$MIRROR_NAME/g; s/__SOURCE_REPO__/$SOURCE_REPO/g" "$tmp" + if [[ -f "$readme" ]] && cmp -s "$tmp" "$readme"; then + rm -f "$tmp" + else + mv "$tmp" "$readme" + fi +} + +run_export() { + local dest="$1" + log "exporting ctxpack -> $dest" + timeout "$GIT_TIMEOUT" carryctx export --pack-format dir -o "$dest" --project "$REPO_ROOT" || + fail "carryctx export failed" + [[ -f "$dest/manifest.json" && -f "$dest/project.json" ]] || + fail "export incomplete (manifest.json/project.json missing)" + redact_snapshot "$dest" +} + +# Export-time redaction seam (post-export, pre-validate/commit). Operates +# ONLY on the snapshot copy ($dest: the staging clone or a --dry-run tmp +# dir); the local carryctx DB is strictly read-only throughout this script. +# Row counts are unchanged, so manifest counts and the self-test still hold. +redact_snapshot() { + local dest="$1" redact_out + log "redacting secret-shaped values in staging copy $dest (local DB untouched)" + redact_out="$(timeout "$GIT_TIMEOUT" python3 "$REPO_ROOT/scripts/publish-ctxpack-redact.py" "$dest")" || + fail "secret redaction failed; mirror left untouched" + log "$redact_out" + REDACTIONS="$(printf '%s\n' "$redact_out" | sed -n 's/^REDACTIONS=//p')" + REDACTIONS="${REDACTIONS:-0}" +} + +run_selftest() { + local dest="$1" + log "validating snapshot (self-test)" + bash "$REPO_ROOT/scripts/publish-ctxpack-selftest.sh" "$dest" || + fail "snapshot self-test failed; mirror left untouched" +} + +if [[ "$DRY_RUN" == 1 ]]; then + TMP_ROOT="$(mktemp -d "${TMPDIR:-/tmp}/ctxpack-dry.XXXXXX")" + SNAP_DIR="$TMP_ROOT/$SNAPSHOT" + run_export "$SNAP_DIR" + run_selftest "$SNAP_DIR" + log "dry-run PASS: snapshot $SNAPSHOT exports + validates; nothing pushed" + exit 0 +fi + +ensure_mirror + +# Idempotency: a snapshot for this source SHA already published? Reuse it so +# a re-run converges instead of duplicating history. +EXISTING="" +shopt -s nullglob +candidates=("$MIRROR"/*-"$REPO_SHORT") +if ((${#candidates[@]} > 0)) && [[ -d "${candidates[0]}" ]]; then + EXISTING="$(basename "${candidates[0]}")" +fi +shopt -u nullglob + +if [[ -n "$EXISTING" ]]; then + log "snapshot for $REPO_SHORT already exists ($EXISTING); refreshing pointers" + SNAP_DIR="$MIRROR/$EXISTING" +else + SNAP_DIR="$MIRROR/$SNAPSHOT" + run_export "$SNAP_DIR" +fi + +run_selftest "$SNAP_DIR" + +SOURCE_JSON="$SNAP_DIR/source.json" +if ! [[ -f "$SOURCE_JSON" ]] || ! grep -q "\"repo_commit\":\"$REPO_SHA\"" "$SOURCE_JSON"; then + printf '{"snapshot":"%s","repo":"%s","repo_commit":"%s","repo_branch":"%s","exported_at":"%s","redactions":%d,"tool":"scripts/publish-ctxpack.sh"}\n' \ + "$(basename "$SNAP_DIR")" "$SOURCE_REPO" "$REPO_SHA" "${REPO_BRANCH:-detached}" "$STAMP" "$REDACTIONS" >"$SOURCE_JSON" +fi + +write_mirror_readme +printf '%s\n' "$(basename "$SNAP_DIR")" >"$MIRROR/LATEST" + +mirror_git add -A || + fail "git add failed in staging clone" +if mirror_git diff --cached --quiet; then + log "mirror already current at $(cat "$MIRROR/LATEST"); nothing to push" + exit 0 +fi +mirror_git -c user.name="${GIT_AUTHOR_NAME:-$SOURCE_REPO-publisher}" \ + -c user.email="${GIT_AUTHOR_EMAIL:-$SOURCE_REPO-publisher@users.noreply.github.com}" \ + commit -q -m "chore(ctxpack): snapshot $(basename "$SNAP_DIR")" || + fail "git commit failed in staging clone" +mirror_git push origin main || + fail "git push failed (network/auth/permissions?)" +log "published $(basename "$SNAP_DIR") ($SOURCE_REPO $REPO_SHORT); LATEST updated"