diff --git a/.github/workflows/snapshot-source.yml b/.github/workflows/snapshot-source.yml new file mode 100644 index 0000000..488603c --- /dev/null +++ b/.github/workflows/snapshot-source.yml @@ -0,0 +1,77 @@ +name: Snapshot source + +# Verifies that the CarryCtx publication at refs/heads/carryctx-snapshots was +# taken from the current main HEAD. In-repo publication retired the separate +# *-workflow mirror: the commander's merge closeout runs `just workflow-publish` +# (native `carryctx export --publication` + `git push` of the snapshot branch). +# +# Triggers: +# - push to main: fails while a newer main HEAD is unpublished, so a red run +# is the closeout signal to publish. +# - push to carryctx-snapshots: re-checks and turns green once the closeout +# publishes the snapshot for the pending main HEAD. +# Pull requests are unaffected: neither trigger is a pull_request event, so PR +# builds and their required checks are untouched. Before the first publication +# the branch does not exist yet; the check then passes with a notice so main is +# not blocked while in-repo publication rolls out. + +on: + push: + branches: + - main + - carryctx-snapshots + +permissions: + contents: read + +concurrency: + group: snapshot-source-${{ github.ref }} + cancel-in-progress: true + +jobs: + snapshot-source: + name: Snapshot source matches main + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - name: Check out repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 1 + + - name: Fetch main and snapshot refs + run: | + set -euo pipefail + git fetch --no-tags origin +refs/heads/main:refs/remotes/origin/main + git fetch --no-tags origin +refs/heads/carryctx-snapshots:refs/remotes/origin/carryctx-snapshots || true + + - name: Verify CarryCtx-Source trailer matches main HEAD + run: | + set -euo pipefail + snapshot_ref=refs/remotes/origin/carryctx-snapshots + if ! git rev-parse --verify -q "$snapshot_ref" >/dev/null; then + echo "::notice title=CarryCtx snapshot::refs/heads/carryctx-snapshots is not published yet; skipping source check." + exit 0 + fi + main_sha="$(git rev-parse refs/remotes/origin/main)" + source_line="$(git log -1 --format=%B "$snapshot_ref" | sed -n 's/^CarryCtx-Source: //p' | head -n1)" + if [ -z "$source_line" ]; then + echo "::error::snapshot tip has no CarryCtx-Source trailer" + exit 1 + fi + # Trailer shape: `@ ()`. + rev="${source_line#*@}" + rev="${rev%% *}" + if [ -z "$rev" ] || [ "$rev" = "$source_line" ]; then + echo "::error::cannot parse CarryCtx-Source trailer: $source_line" + exit 1 + fi + case "$main_sha" in + "$rev"*) + echo "snapshot source $rev matches main HEAD $main_sha" + ;; + *) + echo "::error::snapshot refs/heads/carryctx-snapshots source ($rev) != main HEAD ($main_sha); run 'just workflow-publish' during merge closeout" + exit 1 + ;; + esac diff --git a/AGENTS.md b/AGENTS.md index ca19aaa..5249391 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -41,12 +41,12 @@ - Preserve unrelated changes. Do not commit, push, release, or mutate remote state without explicit authorization. - Fresh clones have no CarryCtx state DB. Restore the local DB from the - workflow mirror with `just workflow-import` (validate-only: - `just workflow-import-dry`). It validates the snapshot before any write, - refuses to replace a non-empty local DB without `--force`, preserves the - committed `.carryctx/config.toml`, and prints provenance and restored - counts. Mirror snapshots are redacted publication artifacts: never merge - them back, and rotate at the source any secret that leaked before rotation. + in-repo snapshot branch with `just workflow-import` (validate-only: + `just workflow-import-dry`). It fetches `refs/heads/carryctx-snapshots`, + refuses to replace a non-empty local DB without `--force`, and prints + provenance and restored counts. Snapshots are redacted publication artifacts + from `carryctx export --publication`: never merge them back, and rotate at + the source any secret that leaked before rotation. ## Delivery lifecycle diff --git a/README.md b/README.md index 7f99cea..248d768 100644 --- a/README.md +++ b/README.md @@ -32,27 +32,25 @@ be treated as instructions or authority. Exact transports, schemas, authentication mechanisms, supported clients, and write-capable operations remain open until accepted through reviewable contracts. -## Workflow mirror restore +## Workflow snapshot restore CarryCtx runtime state (`.git/carryctx/state.sqlite`) is never cloned. The -engineering workflow is mirrored to -[bitty-mcp-workflow](https://github.com/bitty-terminal/bitty-mcp-workflow) as -redacted ctxpack snapshots, with `LATEST` naming the newest snapshot. A fresh -clone can restore its local CarryCtx DB from that mirror: +redacted engineering snapshot lives in this repository on the branch +`refs/heads/carryctx-snapshots`, one commit per publication. The commander's +merge closeout publishes it with `just workflow-publish`; a fresh clone +restores its local CarryCtx DB from that branch: ```sh -just workflow-import-dry # fetch + validate the LATEST snapshot; no DB writes +just workflow-import-dry # fetch + validate the snapshot; no DB writes just workflow-import # initialize CarryCtx state if needed, then import ``` -The import validates snapshot shape, per-table row counts, and the v2 redacted -stamp before any write, refuses to replace a non-empty local DB without -`--force` (`just workflow-import --force`, or pass flags directly to -`scripts/fetch-ctxpack.sh`), and prints provenance (snapshot id + source -commit) plus restored counts. Mirror snapshots are redacted publication -artifacts: CarryCtx refuses them as merge sources, so restore always uses -replace mode, and a secret that leaked before rotation must still be rotated -at the source. +The import fetches `refs/heads/carryctx-snapshots`, refuses to replace a +non-empty local DB without `--force` (`just workflow-import --force`), and +prints provenance (snapshot commit + source). Snapshots are redacted +publication artifacts produced by `carryctx export --publication`: CarryCtx +refuses them as merge sources, so restore always uses replace mode, and a +secret that leaked before rotation must still be rotated at the source. ## Current status diff --git a/justfile b/justfile index 33378a3..9bdd6d7 100644 --- a/justfile +++ b/justfile @@ -46,19 +46,27 @@ hooks-uninstall: # Run every stage gate (commit messages are gated separately per commit) check: lint fmt-check -# Publish a ctxpack snapshot to the bitty-mcp-workflow mirror (commander -# merge closeout only; never a git hook). Dry run exports + validates without push. +# Publish a redacted CarryCtx snapshot inside this repo (commander merge +# closeout only; never a git hook). `carryctx export --publication` redacts the +# bundle, stamps manifest.redacted, and commits one snapshot to the fixed ref +# `refs/heads/carryctx-snapshots`; the target pushes that branch only when the +# local ref advanced (native carryctx commits one snapshot per export, so a +# re-run publishes again rather than no-opping). Canonical closeout runs from +# the primary checkout on branch main +# (`cd "$BITTY_WORKSPACE/bitty-plugins/bitty-mcp" && just workflow-publish`); a +# detached or feature worktree records that branch as the snapshot source. Dry +# run validates the export and writes neither the ref nor the remote. workflow-publish *args: - bash scripts/publish-ctxpack.sh {{args}} + bash scripts/workflow-publish.sh {{args}} workflow-publish-dry *args: - bash scripts/publish-ctxpack.sh --dry-run {{args}} + bash scripts/workflow-publish.sh --dry-run {{args}} -# Restore the local CarryCtx DB from the bitty-mcp-workflow mirror LATEST -# snapshot (fresh-clone recipe). Refuses to replace a non-empty local DB -# without --force, e.g. `just workflow-import --force`. +# Restore the local CarryCtx DB from the in-repo snapshot branch +# `refs/heads/carryctx-snapshots` (fresh-clone recipe). Refuses to replace a +# non-empty local DB without --force, e.g. `just workflow-import --force`. workflow-import *args: - bash scripts/fetch-ctxpack.sh {{args}} + bash scripts/workflow-import.sh {{args}} workflow-import-dry *args: - bash scripts/fetch-ctxpack.sh --dry-run {{args}} + bash scripts/workflow-import.sh --dry-run {{args}} diff --git a/scripts/fetch-ctxpack-selftest.sh b/scripts/fetch-ctxpack-selftest.sh deleted file mode 100755 index 300ff50..0000000 --- a/scripts/fetch-ctxpack-selftest.sh +++ /dev/null @@ -1,265 +0,0 @@ -#!/usr/bin/env bash -# fetch-ctxpack-selftest.sh — self-test for scripts/fetch-ctxpack.sh. -# -# Builds an isolated lab (temp HOME/XDG/TMPDIR + a local fixture mirror repo -# holding one redacted v2 ctxpack snapshot) and exercises, against throwaway -# target repos only: -# 1. empty-DB import: a fresh target with no state DB imports the LATEST -# snapshot; restored task count and provenance match the fixture; the -# pre-existing .carryctx/config.toml is left byte-identical. -# 2. non-empty refusal: a target with local rows is refused without -# --force and its DB is unchanged. -# 3. --force path: the same target is replaced with --force; the local-only -# task disappears and the fixture counts are restored. -# 4. --dry-run no-write: fetch + validate only; no state DB is created. -# 5. validation refusal: a tampered mirror (counts <-> *.jsonl skew) is -# rejected before any DB write. -# -# Isolation guarantee: every case runs with HOME/XDG_*/TMPDIR redirected into -# the scratch dir and with --project pointing at a throwaway repository, so -# the real bitty-mcp CarryCtx DB is never opened, created, or mutated. -# -# Usage: scripts/fetch-ctxpack-selftest.sh -# SELFTEST_KEEP_SCRATCH=1 keep the scratch dir and print its path. -set -euo pipefail - -REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -FETCH="$REPO_ROOT/scripts/fetch-ctxpack.sh" -REDACT="$REPO_ROOT/scripts/publish-ctxpack-redact.py" -TIMEOUT_SECS="${SELFTEST_TIMEOUT:-180}" -KEEP_SCRATCH="${SELFTEST_KEEP_SCRATCH:-0}" - -fail() { - echo "fetch-ctxpack-selftest: FAIL: $1" >&2 - exit 1 -} - -pass() { - echo "fetch-ctxpack-selftest: PASS: $1" -} - -log() { - echo "fetch-ctxpack-selftest: $1" -} - -have() { command -v "$1" >/dev/null 2>&1; } - -have git || fail "git not on PATH" -have carryctx || fail "carryctx not on PATH" -have python3 || fail "python3 not on PATH" -have timeout || fail "timeout not on PATH" -[[ -x "$FETCH" || -f "$FETCH" ]] || fail "fetch script $FETCH not found" -[[ -f "$REDACT" ]] || fail "redaction helper $REDACT not found" - -SCRATCH="$(mktemp -d "${TMPDIR:-/tmp}/fetch-ctxpack-selftest.XXXXXX")" -cleanup() { - if [[ "$KEEP_SCRATCH" == 0 && -d "$SCRATCH" ]]; then - rm -rf "$SCRATCH" - elif [[ -n "$SCRATCH" ]]; then - log "keeping scratch dir $SCRATCH (SELFTEST_KEEP_SCRATCH=1)" - fi -} -trap cleanup EXIT - -# Isolate every carryctx/git invocation: temp HOME, XDG dirs, and TMPDIR so -# no global registry, config, or temp artifact can touch the real user state. -export HOME="$SCRATCH/home" -export XDG_CONFIG_HOME="$HOME/.config" -export XDG_DATA_HOME="$HOME/.local/share" -export XDG_CACHE_HOME="$HOME/.cache" -export XDG_STATE_HOME="$HOME/.local/state" -export TMPDIR="$SCRATCH/tmp" -mkdir -p "$HOME" "$TMPDIR" -unset CARRYCTX_AGENT CARRYCTX_SESSION CARRYCTX_TASK 2>/dev/null || true - -gitq() { timeout "$TIMEOUT_SECS" git -C "$1" "${@:2}"; } -cctx() { timeout "$TIMEOUT_SECS" carryctx --project "$1" "${@:2}"; } - -tasks_total() { - cctx "$1" stats --json | python3 -c \ - 'import json,sys; print(json.load(sys.stdin)["data"]["tasks_total"])' -} - -has_task_title() { - cctx "$1" task list --json | python3 -c ' -import json, sys -title = sys.argv[1] -data = json.load(sys.stdin)["data"] -tasks = data if isinstance(data, list) else data.get("tasks", []) -print("yes" if any(t.get("title") == title for t in tasks) else "no") -' "$2" -} - -git_init() { - timeout "$TIMEOUT_SECS" git init -q -b main "$1" - gitq "$1" config user.email "selftest@example.invalid" - gitq "$1" config user.name "fetch-ctxpack-selftest" -} - -# --- fixture: seed project + redacted v2 snapshot + local mirror repo ------- -log "building fixture project + mirror in $SCRATCH" -SEED="$SCRATCH/seed" -mkdir -p "$SEED" -git_init "$SEED" -cctx "$SEED" init --name fetch-ctxpack-selftest --minimal --agent fetch-ctxpack-selftest >/dev/null || - fail "fixture carryctx init failed" -cctx "$SEED" agent register --name fetch-ctxpack-selftest --kind subagent \ - --role implementer --provider selftest >/dev/null || fail "fixture agent register failed" -cctx "$SEED" task create --title "fixture task one" --priority high \ - --agent fetch-ctxpack-selftest >/dev/null || fail "fixture task create failed" -cctx "$SEED" task create --title "fixture task two" --priority low \ - --agent fetch-ctxpack-selftest >/dev/null || fail "fixture task create failed" - -PACK="$SCRATCH/pack" -timeout "$TIMEOUT_SECS" carryctx export --project "$SEED" --pack-format dir -o "$PACK" >/dev/null || - fail "fixture export failed" -timeout "$TIMEOUT_SECS" python3 "$REDACT" "$PACK" >/dev/null || - fail "fixture redaction failed" - -# Drop [worktree.cleanup] from the fixture config so `carryctx init` at the -# import target rewrites it (current defaults add the section); case 1 then -# proves the script restores the committed config file byte-identically. -timeout "$TIMEOUT_SECS" python3 - "$SEED/.carryctx/config.toml" <<'PYEOF' || -import re -import sys - -path = sys.argv[1] -with open(path) as fh: - text = fh.read() -stripped = re.sub(r"\n\[worktree\.cleanup\]\n(?:[a-z_]+ = .*\n)+", "\n", text) -if stripped == text or "[worktree.cleanup]" in stripped: - sys.exit("fixture config strip failed") -with open(path, "w") as fh: - fh.write(stripped) -PYEOF - fail "fixture config strip failed" - -SNAP_NAME="20260101T000000Z-fixture" -FIXTURE_COMMIT="cafebabecafebabecafebabecafebabecafebabe" -MIRROR_SRC="$SCRATCH/mirror-src" -mkdir -p "$MIRROR_SRC/$SNAP_NAME" -cp -a "$PACK/." "$MIRROR_SRC/$SNAP_NAME/" -printf '{"snapshot":"%s","repo_commit":"%s","repo_branch":"fixture","exported_at":"20260101T000000Z","redactions":0,"tool":"fetch-ctxpack-selftest"}\n' \ - "$SNAP_NAME" "$FIXTURE_COMMIT" >"$MIRROR_SRC/$SNAP_NAME/source.json" -printf '%s\n' "$SNAP_NAME" >"$MIRROR_SRC/LATEST" -git_init "$MIRROR_SRC" -gitq "$MIRROR_SRC" add -A -gitq "$MIRROR_SRC" commit -qm "fixture snapshot" - -# --- helpers for per-case throwaway targets --------------------------------- -make_target() { - local dir="$1" - mkdir -p "$dir" - git_init "$dir" - cp -a "$SEED/.carryctx" "$dir/.carryctx" - gitq "$dir" add -A - gitq "$dir" commit -qm "base" -} - -run_fetch() { - local out="$1" - shift - set +e - timeout "$TIMEOUT_SECS" "$FETCH" "$@" >"$out" 2>&1 - local rc=$? - set -e - echo "$rc" -} - -MIRROR_CLONE="$SCRATCH/mirror-clone" -MIRROR_BAD_CLONE="$SCRATCH/mirror-bad-clone" - -# --- case 1: empty-DB import ------------------------------------------------ -log "case 1: empty-DB import" -TARGET1="$SCRATCH/target1" -make_target "$TARGET1" -RC="$(run_fetch "$SCRATCH/case1.log" --project "$TARGET1" --url "$MIRROR_SRC" --mirror "$MIRROR_CLONE")" -[[ "$RC" == 0 ]] || { - cat "$SCRATCH/case1.log" >&2 - fail "case 1: fetch-ctxpack exit $RC (expected 0)" -} -[[ "$(tasks_total "$TARGET1")" == 2 ]] || fail "case 1: expected 2 restored tasks" -grep -q "snapshot=$SNAP_NAME" "$SCRATCH/case1.log" || - fail "case 1: provenance snapshot id missing" -grep -q "source_commit=$FIXTURE_COMMIT" "$SCRATCH/case1.log" || - fail "case 1: provenance source commit missing" -cmp -s "$SEED/.carryctx/config.toml" "$TARGET1/.carryctx/config.toml" || - fail "case 1: .carryctx/config.toml was not preserved" -pass "empty-DB import restored 2 tasks with provenance and preserved config" - -# --- case 2: non-empty refusal ---------------------------------------------- -log "case 2: non-empty refusal" -cctx "$TARGET1" task create --title "local-only task" --priority medium \ - --agent fetch-ctxpack-selftest >/dev/null || fail "case 2: local task create failed" -[[ "$(tasks_total "$TARGET1")" == 3 ]] || fail "case 2: expected 3 local tasks before refusal" -RC="$(run_fetch "$SCRATCH/case2.log" --project "$TARGET1" --url "$MIRROR_SRC" --mirror "$MIRROR_CLONE")" -[[ "$RC" != 0 ]] || fail "case 2: expected non-zero exit without --force" -grep -q "refusing to overwrite non-empty state" "$SCRATCH/case2.log" || - fail "case 2: refusal message missing" -[[ "$(tasks_total "$TARGET1")" == 3 ]] || fail "case 2: local DB changed during refusal" -[[ "$(has_task_title "$TARGET1" "local-only task")" == "yes" ]] || - fail "case 2: local-only task vanished during refusal" -pass "non-empty DB refused without --force and left unchanged" - -# --- case 3: --force replacement -------------------------------------------- -log "case 3: --force replacement" -RC="$(run_fetch "$SCRATCH/case3.log" --project "$TARGET1" --url "$MIRROR_SRC" --mirror "$MIRROR_CLONE" --force)" -[[ "$RC" == 0 ]] || { - cat "$SCRATCH/case3.log" >&2 - fail "case 3: fetch-ctxpack exit $RC (expected 0 with --force)" -} -[[ "$(tasks_total "$TARGET1")" == 2 ]] || fail "case 3: expected fixture count 2 after --force" -[[ "$(has_task_title "$TARGET1" "local-only task")" == "no" ]] || - fail "case 3: local-only task survived --force replacement" -pass "--force replaced non-empty DB with the snapshot" - -# --- case 4: --dry-run writes nothing --------------------------------------- -log "case 4: --dry-run no-write" -TARGET2="$SCRATCH/target2" -make_target "$TARGET2" -RC="$(run_fetch "$SCRATCH/case4.log" --project "$TARGET2" --url "$MIRROR_SRC" --mirror "$MIRROR_CLONE" --dry-run)" -[[ "$RC" == 0 ]] || { - cat "$SCRATCH/case4.log" >&2 - fail "case 4: dry-run exit $RC (expected 0)" -} -grep -q "dry-run PASS" "$SCRATCH/case4.log" || fail "case 4: dry-run PASS line missing" -[[ ! -e "$TARGET2/.git/carryctx/state.sqlite" ]] || - fail "case 4: dry-run created a state DB" -pass "dry-run validated without creating or writing the local DB" - -# --- case 5: tampered snapshot refused before any write --------------------- -log "case 5: tampered snapshot refusal" -MIRROR_BAD="$SCRATCH/mirror-bad" -cp -a "$MIRROR_SRC" "$MIRROR_BAD" -printf '%s\n' '{"display_id":"CTX-9999","row":"tampered"}' >>"$MIRROR_BAD/$SNAP_NAME/tasks.jsonl" -gitq "$MIRROR_BAD" add -A -gitq "$MIRROR_BAD" commit -qm "tamper: counts <-> jsonl skew" -TARGET3="$SCRATCH/target3" -make_target "$TARGET3" -RC="$(run_fetch "$SCRATCH/case5.log" --project "$TARGET3" --url "$MIRROR_BAD" --mirror "$MIRROR_BAD_CLONE" --dry-run)" -[[ "$RC" != 0 ]] || fail "case 5: tampered snapshot was not refused" -grep -q "snapshot validation failed" "$SCRATCH/case5.log" || - fail "case 5: validation failure message missing" -[[ ! -e "$TARGET3/.git/carryctx/state.sqlite" ]] || - fail "case 5: tampered snapshot wrote a state DB" -pass "tampered snapshot refused before any local DB write" - -# --- case 6: empty DB file (read-command side effect) still restores ------- -log "case 6: empty DB file with no project row" -TARGET4="$SCRATCH/target4" -make_target "$TARGET4" -cctx "$TARGET4" project show >/dev/null 2>&1 || - fail "case 6: project show failed" -[[ -e "$TARGET4/.git/carryctx/state.sqlite" ]] || - fail "case 6: project show did not create the expected empty DB" -RC="$(run_fetch "$SCRATCH/case6.log" --project "$TARGET4" --url "$MIRROR_SRC" --mirror "$MIRROR_CLONE")" -[[ "$RC" == 0 ]] || { - cat "$SCRATCH/case6.log" >&2 - fail "case 6: fetch-ctxpack exit $RC (expected 0)" -} -grep -q "no project row in local DB" "$SCRATCH/case6.log" || - fail "case 6: init path was not taken" -[[ "$(tasks_total "$TARGET4")" == 2 ]] || fail "case 6: expected 2 restored tasks" -pass "empty DB with no project row was initialized and restored" - -log "all cases PASS (empty import, refusal, --force, dry-run, tamper refusal, empty-DB init)" diff --git a/scripts/fetch-ctxpack.sh b/scripts/fetch-ctxpack.sh deleted file mode 100755 index 1efdb20..0000000 --- a/scripts/fetch-ctxpack.sh +++ /dev/null @@ -1,420 +0,0 @@ -#!/usr/bin/env bash -# fetch-ctxpack.sh — restore this repo's local CarryCtx DB from the -# workflow-mirror LATEST snapshot (fresh-clone recipe). -# -# Fresh clones have no .git/carryctx/state.sqlite: CarryCtx runtime state is -# never cloned. The engineering history lives in the `-workflow` mirror -# as redacted ctxpack snapshots (format v1, current format v2). This script: -# 1. ensures a staging clone of the mirror (default -# bitty-terminal/bitty-mcp-workflow, override with --url / --mirror), -# 2. reads LATEST (or --snapshot ID) and copies that snapshot directory to -# a local temp staging path, -# 3. validates the staged copy with the publish validator's static checks -# (scripts/publish-ctxpack-selftest.sh --no-round-trip: format v1/v2 -# shape, counts <-> *.jsonl agreement, JSON parse, v2 redacted stamp, -# fixture matrix) BEFORE anything touches the local DB, -# 4. initializes the local CarryCtx state if it has no project row, then -# imports the staged snapshot with `carryctx import --mode replace`, -# 5. prints provenance (snapshot id + source commit) and `carryctx stats`. -# -# Safety: -# - a NON-EMPTY local state DB is never replaced without --force; -# - --dry-run fetches + validates only; it never initializes or imports; -# - the pre-existing .carryctx/config.toml is conserved: `carryctx init` -# rewrites local config defaults, so the script restores the committed -# file after init and leaves the checkout clean; -# - mirror fetch is read-only (fetch/reset of the staging clone only); -# nothing is ever pushed. -# -# Mirrors hold redacted publication artifacts (manifest.redacted=true for v2) -# and redaction is one-way; CarryCtx refuses redacted bundles as merge -# sources, so restoring always uses replace mode. A leaked-then-rotated -# secret is already in the mirror: rotate at the source, redaction only -# limits exposure. -# -# Usage: -# scripts/fetch-ctxpack.sh [--dry-run] [--force] [--mirror DIR] [--url URL] -# [--snapshot ID] [--project DIR] -# [--git-timeout SECS] [--keep-tmp] -# --dry-run fetch + validate only; no init, no import, no DB writes. -# --force replace a non-empty local DB (default: refuse). -# --mirror DIR staging clone location -# (default: $WORKFLOW_MIRROR_DIR or /../bitty-mcp-workflow). -# --url URL mirror remote for a fresh clone -# (default: $WORKFLOW_MIRROR_URL or -# https://github.com/bitty-terminal/bitty-mcp-workflow.git). -# --snapshot ID restore this snapshot directory instead of LATEST. -# --project DIR target repo root (default: the repo containing this -# script; useful for throwaway clones and testing). -# --git-timeout SECS timeout for every git/carryctx/python op -# (default: $GIT_TIMEOUT or 120). -# --keep-tmp keep temp dirs on exit (debug aid; prints paths). -set -euo pipefail - -REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -MIRROR_DEFAULT="$REPO_ROOT/../bitty-mcp-workflow" -MIRROR_URL_DEFAULT="https://github.com/bitty-terminal/bitty-mcp-workflow.git" - -DRY_RUN=0 -FORCE=0 -KEEP_TMP=0 -MIRROR="${WORKFLOW_MIRROR_DIR:-$MIRROR_DEFAULT}" -MIRROR_URL="${WORKFLOW_MIRROR_URL:-$MIRROR_URL_DEFAULT}" -GIT_TIMEOUT="${GIT_TIMEOUT:-120}" -SNAPSHOT_OVERRIDE="" -PROJECT="${WORKFLOW_IMPORT_PROJECT:-$REPO_ROOT}" - -while [[ $# -gt 0 ]]; do - case "$1" in - --dry-run) - DRY_RUN=1 - shift - ;; - --force) - FORCE=1 - shift - ;; - --mirror) - MIRROR="${2:?--mirror requires a directory}" - shift 2 - ;; - --mirror=*) - MIRROR="${1#--mirror=}" - shift - ;; - --url) - MIRROR_URL="${2:?--url requires a remote URL}" - shift 2 - ;; - --url=*) - MIRROR_URL="${1#--url=}" - shift - ;; - --snapshot) - SNAPSHOT_OVERRIDE="${2:?--snapshot requires a snapshot directory name}" - shift 2 - ;; - --snapshot=*) - SNAPSHOT_OVERRIDE="${1#--snapshot=}" - shift - ;; - --project) - PROJECT="${2:?--project requires a directory}" - shift 2 - ;; - --project=*) - PROJECT="${1#--project=}" - shift - ;; - --git-timeout) - GIT_TIMEOUT="${2:?--git-timeout requires seconds}" - shift 2 - ;; - --git-timeout=*) - GIT_TIMEOUT="${1#--git-timeout=}" - shift - ;; - --keep-tmp) - KEEP_TMP=1 - shift - ;; - --help | -h) - sed -n '2,/^set -euo/p' "${BASH_SOURCE[0]}" - exit 0 - ;; - *) - echo "fetch-ctxpack: FAIL: unknown flag $1 (see --help)" >&2 - exit 2 - ;; - esac -done - -fail() { - echo "fetch-ctxpack: FAIL: $1" >&2 - exit 1 -} - -log() { - echo "fetch-ctxpack: $1" -} - -warn() { - echo "fetch-ctxpack: WARN: $1" >&2 -} - -have() { command -v "$1" >/dev/null 2>&1; } - -have git || fail "git not on PATH" -have carryctx || fail "carryctx not on PATH" -have timeout || fail "timeout not on PATH" -have python3 || fail "python3 not on PATH (state probe + snapshot provenance)" - -[[ -d "$PROJECT" ]] || fail "project dir $PROJECT not found" -[[ -f "$PROJECT/.carryctx/config.toml" ]] || - fail "$PROJECT has no .carryctx/config.toml (not a CarryCtx project?); refusing to guess a project identity" - -TMP_ROOT="" -cleanup() { - if [[ "$KEEP_TMP" == 0 && -n "$TMP_ROOT" && -d "$TMP_ROOT" ]]; then - rm -rf "$TMP_ROOT" - elif [[ -n "$TMP_ROOT" ]]; then - log "keeping tmp dir $TMP_ROOT (--keep-tmp)" - fi -} -trap cleanup EXIT - -# Mirror git ops always go through this wrapper: timeout-wrapped and -# confined to the staging clone. $PROJECT is never a write target. -mirror_git() { - timeout "$GIT_TIMEOUT" git -C "$MIRROR" "$@" -} - -ensure_mirror() { - if [[ -d "$MIRROR/.git" ]]; then - if [[ -n "$(mirror_git status --porcelain 2>/dev/null)" ]]; then - fail "staging clone $MIRROR has uncommitted changes; clean it up, pass --mirror, or remove it" - fi - mirror_git fetch -q origin --prune || - fail "git fetch failed in staging clone (network/auth?)" - mirror_git checkout -q main 2>/dev/null || - mirror_git checkout -q -b main --track origin/main || - fail "cannot check out main in staging clone" - mirror_git reset -q --hard origin/main || - fail "cannot fast-forward staging clone to origin/main" - return 0 - fi - if [[ -e "$MIRROR" ]]; then - fail "$MIRROR exists but is not a git clone; move it aside or pass --mirror" - fi - log "cloning mirror $MIRROR_URL -> $MIRROR" - timeout "$GIT_TIMEOUT" git clone -q "$MIRROR_URL" "$MIRROR" || - fail "git clone failed (network/auth?); set --url or pre-clone --mirror yourself" - ensure_mirror -} - -# Resolve + stage the snapshot. LATEST is a single line; snapshot ids are -# validated before use so a hostile mirror cannot escape the mirror dir. -resolve_snapshot() { - local latest - if [[ -n "$SNAPSHOT_OVERRIDE" ]]; then - SNAP="$SNAPSHOT_OVERRIDE" - else - [[ -f "$MIRROR/LATEST" ]] || fail "mirror $MIRROR has no LATEST file" - latest="$(head -n1 "$MIRROR/LATEST" | tr -d '[:space:]')" - [[ -n "$latest" ]] || fail "mirror $MIRROR LATEST is empty" - SNAP="$latest" - fi - [[ "$SNAP" =~ ^[A-Za-z0-9][A-Za-z0-9._-]*$ ]] || - fail "snapshot id '$SNAP' is not a plain directory name; refusing" - [[ -d "$MIRROR/$SNAP" ]] || fail "snapshot dir $MIRROR/$SNAP not found" -} - -stage_snapshot() { - local dest="$TMP_ROOT/$SNAP" - log "staging snapshot $SNAP -> $dest" - cp -a "$MIRROR/$SNAP" "$dest" || - fail "cannot copy snapshot $SNAP to staging dir" -} - -# Structural integrity + mirror honesty (v2 must carry the redacted stamp) -# via the publish validator's static checks. Import happens only after this -# passes; the check itself never touches the local CarryCtx DB. -validate_snapshot() { - log "validating staged snapshot (publish-ctxpack-selftest --no-round-trip)" - timeout "$GIT_TIMEOUT" bash "$REPO_ROOT/scripts/publish-ctxpack-selftest.sh" \ - "$STAGE" --no-round-trip || fail "snapshot validation failed; nothing imported" -} - -# Legacy v1 mirror snapshots predate the export-time redaction pass (the v2 -# validator already refuses unstamped v2 packs). Import is unaffected; the -# warning keeps the provenance honest. -warn_if_unstamped() { - local stamped - stamped="$(timeout "$GIT_TIMEOUT" python3 -c \ - 'import json,sys; print("true" if json.load(open(sys.argv[1])).get("redacted") is True else "false")' \ - "$STAGE/manifest.json")" || fail "cannot read snapshot manifest" - if [[ "$stamped" != "true" ]]; then - warn "snapshot $SNAP has no redacted stamp (legacy v1 artifact): treat mirror contents as unredacted historical data" - fi -} - -# Read-only probe of the local CarryCtx state. Never creates or mutates the -# DB; exits 0 with "exists projects rows" or a clear failure. Data tables are -# the schema-17/18 row sets; unknown/missing tables are skipped. -probe_state() { - timeout "$GIT_TIMEOUT" python3 - "$PROJECT" <<'PYEOF' -import os -import sqlite3 -import subprocess -import sys - -repo = sys.argv[1] -try: - out = subprocess.run( - ["git", "-C", repo, "rev-parse", "--git-common-dir"], - capture_output=True, text=True, check=True, timeout=60, - ).stdout.strip() -except Exception as exc: # noqa: BLE001 - report, never traceback - sys.exit("fetch-ctxpack: FAIL: cannot resolve git common dir for %s (%s)" % (repo, exc)) -if not out: - sys.exit("fetch-ctxpack: FAIL: empty git common dir for %s" % repo) -common = out if os.path.isabs(out) else os.path.join(repo, out) -db = os.path.join(common, "carryctx", "state.sqlite") - -exists = os.path.exists(db) -projects = 0 -rows = 0 -if exists: - tables = ( - "agents", "tasks", "task_dependencies", "progress_items", "sessions", - "worktrees", "checkpoints", "checkpoint_corrections", "scopes", - "decisions", "handoffs", "events", "graph_nodes", "graph_edges", - "teams", "team_members", - ) - try: - con = sqlite3.connect("file:%s?mode=ro" % db, uri=True) - try: - names = {r[0] for r in con.execute( - "SELECT name FROM sqlite_master WHERE type='table'")} - if "projects" in names: - projects = con.execute("SELECT COUNT(*) FROM projects").fetchone()[0] - rows = sum( - con.execute("SELECT COUNT(*) FROM %s" % t).fetchone()[0] - for t in tables if t in names - ) - finally: - con.close() - except sqlite3.Error as exc: - sys.exit("fetch-ctxpack: FAIL: cannot read local state DB %s (%s)" % (db, exc)) -print("%d %d %d" % (1 if exists else 0, projects, rows)) -PYEOF -} - -config_project_id() { - timeout "$GIT_TIMEOUT" python3 - "$PROJECT/.carryctx/config.toml" <<'PYEOF' -import sys -import tomllib - -try: - with open(sys.argv[1], "rb") as fh: - cfg = tomllib.load(fh) -except (OSError, tomllib.TOMLDecodeError): - print("") - sys.exit(0) -print(cfg.get("project", {}).get("id", "")) -PYEOF -} - -bundle_project_id() { - timeout "$GIT_TIMEOUT" python3 - "$STAGE/manifest.json" <<'PYEOF' -import json -import sys - -with open(sys.argv[1]) as fh: - print(json.load(fh).get("project_id", "")) -PYEOF -} - -# `carryctx init` rewrites .carryctx/config.toml with current defaults (it -# adds new sections and resets e.g. [verification] commands). The committed -# config is project data, not runtime state: keep a copy and restore it so -# the checkout stays clean. -ensure_project_row() { - local cfg="$PROJECT/.carryctx/config.toml" backup="" - if [[ -f "$cfg" ]]; then - backup="$TMP_ROOT/config.toml.before" - cp -p "$cfg" "$backup" || fail "cannot back up $cfg" - fi - log "no project row in local DB: initializing CarryCtx state (carryctx init --non-interactive)" - if ! timeout "$GIT_TIMEOUT" carryctx init --non-interactive --project "$PROJECT" \ - >"$TMP_ROOT/init.log" 2>&1; then - cat "$TMP_ROOT/init.log" >&2 || true - fail "carryctx init failed" - fi - if [[ -n "$backup" && -f "$cfg" ]] && ! cmp -s "$backup" "$cfg"; then - cp -p "$backup" "$cfg" || fail "cannot restore $cfg after carryctx init" - log "restored pre-existing .carryctx/config.toml (carryctx init rewrites local config defaults)" - fi -} - -print_provenance() { - timeout "$GIT_TIMEOUT" python3 - "$STAGE" "$SNAP" <<'PYEOF' -import json -import os -import sys - -stage, snap = sys.argv[1], sys.argv[2] -with open(os.path.join(stage, "manifest.json")) as fh: - manifest = json.load(fh) -source = {} -source_path = os.path.join(stage, "source.json") -if os.path.exists(source_path): - with open(source_path) as fh: - source = json.load(fh) -print("fetch-ctxpack: provenance: snapshot=%s export_id=%s format_version=v%s carryctx=%s schema=%s redacted=%s" % ( - snap, - manifest.get("export_id", "?"), - manifest.get("format_version", "?"), - manifest.get("carryctx_version", "?"), - manifest.get("schema_version", "?"), - "true" if manifest.get("redacted") is True else "false", -)) -print("fetch-ctxpack: provenance: source_commit=%s exported_at=%s" % ( - source.get("repo_commit", "unknown"), - source.get("exported_at", "unknown"), -)) -PYEOF -} - -# --- pre-flight: local state ------------------------------------------------- -STATE_LINE="$(probe_state)" || fail "cannot inspect local CarryCtx state" -read -r STATE_EXISTS STATE_PROJECTS STATE_ROWS <<<"$STATE_LINE" -log "local state: db_exists=$STATE_EXISTS project_rows=$STATE_PROJECTS data_rows=$STATE_ROWS" -if [[ "$STATE_PROJECTS" -gt 1 ]]; then - fail "local DB has $STATE_PROJECTS project rows; refusing to replace an ambiguous state DB" -fi -if [[ "$STATE_PROJECTS" -eq 1 && "$STATE_ROWS" -gt 0 && "$FORCE" == 0 && "$DRY_RUN" == 0 ]]; then - fail "local DB already holds $STATE_ROWS row(s); refusing to overwrite non-empty state without --force (local DB untouched)" -fi - -# --- fetch + validate -------------------------------------------------------- -TMP_ROOT="$(mktemp -d "${TMPDIR:-/tmp}/ctxpack-import.XXXXXX")" -ensure_mirror -resolve_snapshot -STAGE="$TMP_ROOT/$SNAP" -stage_snapshot -validate_snapshot -warn_if_unstamped - -BUNDLE_ID="$(bundle_project_id)" || fail "cannot read snapshot project id" -CONFIG_ID="$(config_project_id)" || fail "cannot read local .carryctx project id" -if [[ -n "$BUNDLE_ID" && -n "$CONFIG_ID" && "$BUNDLE_ID" != "$CONFIG_ID" ]]; then - fail "snapshot project $BUNDLE_ID does not match local project $CONFIG_ID; refusing to fork project identity" -fi - -if [[ "$DRY_RUN" == 1 ]]; then - print_provenance - if [[ "$STATE_PROJECTS" -eq 1 && "$STATE_ROWS" -gt 0 ]]; then - log "dry-run PASS: snapshot $SNAP fetched + validated; local DB has $STATE_ROWS row(s), a real run needs --force" - else - log "dry-run PASS: snapshot $SNAP fetched + validated; nothing imported" - fi - exit 0 -fi - -# --- restore ----------------------------------------------------------------- -if [[ "$STATE_PROJECTS" -eq 0 ]]; then - ensure_project_row -elif [[ "$STATE_ROWS" -gt 0 ]]; then - warn "--force: replacing $STATE_ROWS existing local row(s) in $PROJECT" -fi - -log "importing snapshot $SNAP into $PROJECT (replace mode)" -if ! timeout "$GIT_TIMEOUT" carryctx import "$STAGE" --project "$PROJECT" --mode replace --yes; then - fail "carryctx import failed; local DB is left in the state carryctx reports above" -fi - -print_provenance -log "restore complete; local counts:" -timeout "$GIT_TIMEOUT" carryctx stats --project "$PROJECT" || fail "carryctx stats failed" -log "restored from mirror snapshot $SNAP; mirror artifacts are redacted, never merge them back" diff --git a/scripts/publish-ctxpack-redact.py b/scripts/publish-ctxpack-redact.py deleted file mode 100755 index 3191bb9..0000000 --- a/scripts/publish-ctxpack-redact.py +++ /dev/null @@ -1,339 +0,0 @@ -#!/usr/bin/env python3 -"""publish-ctxpack-redact.py -- export-time secret redaction for ctxpack snapshots. - -Usage: publish-ctxpack-redact.py - -Rewrites every ``*.jsonl`` file under IN PLACE, replacing -secret-shaped values with ``***REDACTED***`` while keeping field names (and, -inside free text, variable names) intact for debuggability. ``tombstones.jsonl`` -(added by ctxpack format v2) is treated like any other table file; unknown -future table files are picked up by the same ``*.jsonl`` glob. - -After the JSONL pass, stamps ``manifest.json`` with ``"redacted": true`` -(publication-artifact marker per the ctxpack v2 contract) without touching any -other manifest field -- in particular ``counts`` is never modified, so count -consistency and the round-trip self-test still hold. Re-running is a no-op: -an already-``true`` stamp is reported and the file is left byte-identical. - -What is redacted (fail-closed heuristics, JSONL-aware -- never blind sed): - 1. Key-name rule: any JSON object field whose name is secret-shaped (see - is_secret_name: a _-separated or camelCase segment equals KEY / TOKEN / - SECRET / PASSWORD / PAT, or the name contains GH_PAT, or it starts with - CLOUDFLARE_ / AWS_) has a string value replaced wholesale. - (AWS_/CLOUDFLARE_ prefixes intentionally over-match, e.g. AWS_REGION: - a lost region string is cheaper than a leaked credential. A bare KEY - segment such as keyboard-event handlers also redacts: fail-closed.) - 2. Free-text rule, applied to every remaining string value: - a. ``NAME=value`` / ``NAME: value`` pairs (shell env dumps, YAML-ish - prose, pasted kwargs) where NAME is secret-shaped per rule 1 -- the - name is kept, the value is replaced (quoted values may contain - spaces). Substring lookalikes do NOT match: ``monkey=``, ``keyboard=``, - ``DispatchPriority:``, ``GOPATH_BIN=`` survive because KEY / PAT is - not a standalone segment of those names. - b. Standalone 40+-char ``[A-Za-z0-9_\\-+]`` runs (plus up to two ``=`` - padding chars) that look token-ish: exactly-40 hex runs are EXEMPT - (git SHA-1s are ubiquitous in workflow prose and must survive; - longer hex still redacts), single-class runs (all-lowercase - test/branch names like ``soak_read_...``) are EXEMPT, and ``/`` and - mid-run ``=`` are deliberately NOT in the run alphabet -- so absolute - workspace paths and URLs (which the mirror intentionally publishes) - are never mangled, and ``NAME=`` never merges into one run - that would eat the field name. A run must mix at least two of - {lowercase, UPPERCASE, digits} or carry ``+``/``=`` base64 punctuation - to count as token-ish. Slash-bearing tokens are still caught by - rules 1 and 2a. - -Nested JSON-encoded strings (carryctx ``payload_json`` / ``metadata_json`` -columns) are decoded, redacted recursively, and re-encoded -- but only when -a redaction actually fired inside, so untouched rows stay byte-identical. - -Scope and safety: - - Operates ONLY on the snapshot directory argument (the staging/mirror - copy or a --dry-run tmp dir). It NEVER touches the local carryctx DB; - the DB keeps the original values and the next export redacts them again. - - Row counts and file sets are unchanged, so manifest counts and the - round-trip self-test still hold on redacted output. - - ``manifest.json`` must exist and be a JSON object; its ``redacted`` stamp - is set to ``true`` (atomic replace) and no other manifest content changes. - - Re-running is a no-op (``***REDACTED***`` carries no secret shape and an - already-true stamp is left byte-identical). - - Files are replaced atomically (tmp + rename); rows that do not parse - as JSON are left byte-identical and reported (the self-test then fails - closed on them as malformed rows, as before). - -Exit 0 on success (prints a per-run summary plus a ``REDACTIONS=`` line -for the publish script to record in source.json). Exit 1 on structural -failure, 2 on usage error. -""" - -import json -import os -import re -import sys - -REDACTED = "***REDACTED***" - -# Candidate ["]NAME["]=value | NAME: value | NAME = "quoted value", all on ONE -# line (separator whitespace is [ \t]* so `word:` never swallows the next -# line as its value). The name may itself be quoted (prose-embedded JSON -# snippets); the callback re-validates the name via is_secret_name, so this -# stays deliberately broad (any word around : or =) and non-secret pairs -# pass through byte-identical. -_ASSIGN_RE = re.compile( - r"""(?P["']?)\b(?P[A-Za-z_][A-Za-z0-9_.\-]*)(?P=nq)""" - r"""[ \t]*(?P[:=])[ \t]*""" - r"""(?:(?P["'])(?P.*?)(?P=q)|(?P[^\s"'`,;]+))""", -) - -# `=` is padding-only, never a run joiner: otherwise NAME= would merge -# into one run and eat the field name. Trailing `={0,2}` still catches -# base64 padding without gluing across `=`. -_RUN_RE = re.compile(r"[A-Za-z0-9_\-+]{40,}={0,2}") -_SHA1_RE = re.compile(r"[0-9a-fA-F]{40}\Z") - -_SEGMENT_RE = re.compile(r"[A-Z]+(?![a-z])|[A-Z][a-z0-9]*|[a-z]+|[0-9]+") -_SECRET_SEGMENTS = frozenset(["KEY", "TOKEN", "SECRET", "PASSWORD", "PAT"]) - - -def _segments(name): - """Split a field/variable name into UPPER-cased word segments.""" - found = [] - for part in re.split(r"[_\-.]+", name): - found.extend(_SEGMENT_RE.findall(part)) - return [seg.upper() for seg in found] - - -def is_secret_name(name): - """True when a field/variable name is secret-shaped (spec list). - - Segment-based, so ``monkey``, ``keyboard``, ``tokenizer``, - ``DispatchPriority`` and ``GOPATH_BIN`` do NOT match while ``api_key``, - ``OPENAI_API_KEY``, ``gh_pat`` and ``handle_key_event`` do. - """ - upper = name.upper() - if "GH_PAT" in upper: - return True - if upper.startswith("CLOUDFLARE_") or upper.startswith("AWS_"): - return True - return any(seg in _SECRET_SEGMENTS for seg in _segments(name)) - - -def _looks_tokenish(run): - """True when a 40+ run mixes classes like a real token (not a slug).""" - if "+" in run or run.endswith("="): - return True - classes = 0 - if re.search(r"[a-z]", run): - classes += 1 - if re.search(r"[A-Z]", run): - classes += 1 - if re.search(r"[0-9]", run): - classes += 1 - return classes >= 2 - - -def _already_redacted(value): - return value == REDACTED or value.startswith("***") - - -def redact_text(text): - """Apply free-text rules to one string. Returns (new_text, count).""" - count = 0 - - # NOTE: the span rebuild below preserves the exact original separator - # whitespace (quoted values may contain spaces; bare values end at - # whitespace or quote/comma/semicolon boundaries). Names that are not - # secret-shaped pass through byte-identical (no count). - def _assign_simple(match): - nonlocal count - if not is_secret_name(match.group("name")): - return match.group(0) - qval = match.group("qval") - if qval is not None: - if not qval or _already_redacted(qval): - return match.group(0) - count += 1 - start, end = match.span("qval") - return match.string[match.start():start] + REDACTED + match.string[end:match.end()] - value = match.group("val") - if not value or _already_redacted(value): - return match.group(0) - count += 1 - start, end = match.span("val") - return match.string[match.start():start] + REDACTED + match.string[end:match.end()] - - text = _ASSIGN_RE.sub(_assign_simple, text) - - def _run(match): - nonlocal count - run = match.group(0) - if _already_redacted(run): - return run - if _SHA1_RE.match(run): - return run # git SHA-1 exemption (documented above) - if not _looks_tokenish(run): - return run # single-class slug (branch/test/flag name), not a token - count += 1 - return REDACTED - - text = _RUN_RE.sub(_run, text) - return text, count - - -def redact_value(node): - """Recursively redact decoded JSON. Returns (new_node, count).""" - if isinstance(node, dict): - count = 0 - out = {} - for key, value in node.items(): - if isinstance(key, str) and isinstance(value, str): - if is_secret_name(key): - if value and not _already_redacted(value): - out[key] = REDACTED - count += 1 - else: - out[key] = value - continue - new_value, sub = redact_value(value) - out[key] = new_value - count += sub - return out, count - if isinstance(node, list): - count = 0 - out = [] - for item in node: - new_item, sub = redact_value(item) - out.append(new_item) - count += sub - return out, count - if isinstance(node, str): - stripped = node.strip() - if len(stripped) >= 2 and stripped[0] in "{[": - try: - decoded = json.loads(stripped) - except ValueError: - decoded = None - if isinstance(decoded, (dict, list)): - new_decoded, sub = redact_value(decoded) - if sub: - return ( - json.dumps(new_decoded, ensure_ascii=False, separators=(",", ":")), - sub, - ) - return node, 0 - return redact_text(node) - return node, 0 - - -def redact_file(path): - """Redact one .jsonl file atomically. Returns (rows, count, skipped).""" - with open(path, "r", encoding="utf-8") as handle: - raw_lines = handle.read().splitlines() - out_lines = [] - rows = 0 - count = 0 - skipped = 0 - changed = False - for line in raw_lines: - if not line.strip(): - out_lines.append(line) - continue - try: - row = json.loads(line) - except ValueError: - skipped += 1 - out_lines.append(line) # byte-identical; self-test fails closed - continue - rows += 1 - new_row, sub = redact_value(row) - count += sub - if sub: - out_lines.append(json.dumps(new_row, ensure_ascii=False)) - changed = True - else: - out_lines.append(line) # byte-identical fast path - if changed: - tmp_path = path + ".redact-tmp" - with open(tmp_path, "w", encoding="utf-8") as handle: - handle.write("\n".join(out_lines)) - if out_lines: - handle.write("\n") - os.replace(tmp_path, path) - return rows, count, skipped - - -def stamp_manifest(snap): - """Stamp ``manifest.json`` with ``redacted: true`` (publication marker). - - Returns ``"stamped"`` when the flag was added, ``"already"`` when the - manifest already carried ``redacted: true`` (left byte-identical). - Raises ``ValueError`` on a missing/invalid manifest or a non-bool stamp; - callers fail closed. - """ - path = os.path.join(snap, "manifest.json") - if not os.path.isfile(path): - raise ValueError("manifest.json is missing (not a ctxpack dir?)") - with open(path, "r", encoding="utf-8") as handle: - manifest = json.load(handle) - if not isinstance(manifest, dict): - raise ValueError("manifest.json is not a JSON object") - current = manifest.get("redacted") - if current is True: - return "already" - if current not in (None, False): - raise ValueError("manifest.json 'redacted' is not a boolean: %r" % (current,)) - manifest["redacted"] = True - tmp_path = path + ".redact-tmp" - with open(tmp_path, "w", encoding="utf-8") as handle: - json.dump(manifest, handle, ensure_ascii=False, indent=2) - handle.write("\n") - os.replace(tmp_path, path) - return "stamped" - - -def main(argv): - if len(argv) != 2 or argv[1] in ("-h", "--help"): - sys.stderr.write("usage: publish-ctxpack-redact.py \n") - return 2 - snap = argv[1] - if not os.path.isdir(snap): - sys.stderr.write("publish-ctxpack-redact: FAIL: not a directory: %s\n" % snap) - return 1 - files = sorted( - name - for name in os.listdir(snap) - if name.endswith(".jsonl") - and os.path.isfile(os.path.join(snap, name)) - and not os.path.islink(os.path.join(snap, name)) - ) - if not files: - sys.stderr.write( - "publish-ctxpack-redact: FAIL: no *.jsonl files in %s (not a ctxpack dir?)\n" % snap - ) - return 1 - total_rows = 0 - total_count = 0 - total_skipped = 0 - for name in files: - rows, count, skipped = redact_file(os.path.join(snap, name)) - total_rows += rows - total_count += count - total_skipped += skipped - if count or skipped: - print(" %-28s rows=%d redactions=%d nonjson_skipped=%d" % (name, rows, count, skipped)) - try: - stamp = stamp_manifest(snap) - except ValueError as error: - sys.stderr.write("publish-ctxpack-redact: FAIL: %s\n" % error) - return 1 - print(" manifest.json redacted=true (%s; counts untouched)" % stamp) - print( - "publish-ctxpack-redact: %d files, %d rows, %d replacements, %d non-JSON lines left untouched" - % (len(files), total_rows, total_count, total_skipped) - ) - print("REDACTIONS=%d" % total_count) - return 0 - - -if __name__ == "__main__": - sys.exit(main(sys.argv)) diff --git a/scripts/publish-ctxpack-selftest.sh b/scripts/publish-ctxpack-selftest.sh deleted file mode 100755 index 3da2b3e..0000000 --- a/scripts/publish-ctxpack-selftest.sh +++ /dev/null @@ -1,781 +0,0 @@ -#!/usr/bin/env bash -# publish-ctxpack-selftest.sh — round-trip self-test for a ctxpack snapshot. -# -# Usage: scripts/publish-ctxpack-selftest.sh [--no-round-trip] -# -# Checks (fail-closed, exit 1): -# 1. manifest.json exists with format=carryctx-pack-dir and a supported -# format_version (1 or 2), a non-empty counts map and a project_id; -# project.json exists. v2 shape is enforced too: counts.tombstones, -# non-empty unique parents that never name the manifest's own export_id, -# watermark rows agreeing with counts, sequences/source shape, and the -# redacted:true publication stamp. v1 keeps its legacy shape (no -# watermarks, no tombstone counts) and needs no redacted stamp. -# 2. Every counts entry has a matching .jsonl whose line count -# equals the manifest count; a table with rows but no counts entry fails -# closed (undeclared row set), matching carryctx check_counts. -# 3. Every line of every *.jsonl parses as JSON. -# 4. Validator fixture matrix: synthetic v1/v2 packs prove the checks above -# accept valid packs and refuse newer format versions, tombstone/count -# skew, malformed parents/watermarks, unredacted v2 packs, and unknown -# or undeclared tables. -# 5. Round-trip (unless --no-round-trip): fresh git repo + fresh -# `carryctx init --minimal` scratch project, `carryctx import -# --mode replace`, then the imported table counts must match the -# manifest (worktrees rows may be pruned by re-anchoring, so -# worktrees <= manifest is accepted and reported). -# 6. Redaction fixture: synthetic v1/v2 packs holding FAKE secret values -# (never real credentials) are copied to a staging dir and run through -# scripts/publish-ctxpack-redact.py; the staging copy must come out -# redacted (names kept, JSON valid, row counts unchanged, tombstones -# rows handled, manifest.redacted=true stamped, counts untouched) while -# the source copy still holds the fakes byte-identical (local DB -# untouched). -# -# Known carryctx import limitation (loud WARN, still exit 0): import loads -# sessions while worktree rows missing at the import target are pruned, so -# a pack containing worktree-bound sessions can fail with -# "Failed to load pack table 'sessions': FOREIGN KEY constraint failed" on -# carryctx versions predating the nulling of pruned worktree references. -# The export itself is intact (checks 1-3 prove it); only the re-import -# probe cannot pass until the import target handles those refs. Do not -# "fix" this by editing the snapshot. -set -euo pipefail - -PACK="${1:?usage: publish-ctxpack-selftest.sh [--no-round-trip]}" -ROUND_TRIP=1 -if [[ "${2:-}" == "--no-round-trip" ]]; then - ROUND_TRIP=0 -elif [[ -n "${2:-}" ]]; then - echo "publish-ctxpack-selftest: FAIL: unknown flag $2" >&2 - exit 2 -fi - -TIMEOUT_SECS="${SELFTEST_TIMEOUT:-120}" -KEEP_SCRATCH=0 -if [[ "${SELFTEST_KEEP_SCRATCH:-0}" == 1 ]]; then - KEEP_SCRATCH=1 -fi - -fail() { - echo "publish-ctxpack-selftest: FAIL: $1" >&2 - exit 1 -} - -warn() { - echo "publish-ctxpack-selftest: WARN: $1" >&2 -} - -log() { - echo "publish-ctxpack-selftest: $1" -} - -command -v python3 >/dev/null 2>&1 || fail "python3 not on PATH" -command -v carryctx >/dev/null 2>&1 || fail "carryctx not on PATH" -command -v git >/dev/null 2>&1 || fail "git not on PATH" -command -v timeout >/dev/null 2>&1 || fail "timeout not on PATH" -[[ -d "$PACK" ]] || fail "export dir $PACK not found" - -SCRATCH="" -FIXT_ROOT="" -cleanup() { - if [[ "$KEEP_SCRATCH" == 0 && -n "$SCRATCH" && -d "$SCRATCH" ]]; then - rm -rf "$SCRATCH" - elif [[ -n "$SCRATCH" ]]; then - log "keeping scratch dir $SCRATCH (SELFTEST_KEEP_SCRATCH=1)" - fi - if [[ "$KEEP_SCRATCH" == 0 && -n "$FIXT_ROOT" && -d "$FIXT_ROOT" ]]; then - rm -rf "$FIXT_ROOT" - elif [[ -n "$FIXT_ROOT" ]]; then - log "keeping fixture dir $FIXT_ROOT (SELFTEST_KEEP_SCRATCH=1)" - fi -} -trap cleanup EXIT - -log "checking manifest + per-table counts in $PACK" -timeout "$TIMEOUT_SECS" python3 - "$PACK" <<'PYEOF' -import copy -import json -import os -import re -import shutil -import sys -import tempfile - -pack = sys.argv[1] - -PACK_FORMAT = "carryctx-pack-dir" -V1_TABLES = ( - "agents", - "tasks", - "task_dependencies", - "progress_items", - "sessions", - "worktrees", - "checkpoints", - "checkpoint_corrections", - "scopes", - "decisions", - "handoffs", - "teams", - "team_members", - "graph_nodes", - "graph_edges", - "events", - "sequences", -) -V2_TABLES = V1_TABLES + ("tombstones",) -CURRENT_FORMAT_VERSION = 2 -_RFC3339_RE = re.compile(r"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?(?:Z|[+-]\d{2}:\d{2})\Z") - - -class PackError(Exception): - pass - - -def require(condition, message): - if not condition: - raise PackError(message) - - -def is_int(value): - return isinstance(value, int) and not isinstance(value, bool) - - -def read_json(path, what): - try: - with open(path) as handle: - return json.load(handle) - except FileNotFoundError as error: - raise PackError("%s is missing" % what) from error - except ValueError as error: - raise PackError("%s is not valid JSON (%s)" % (what, error)) from error - - -def validate_pack(directory, verbose=True): - """Validate a ctxpack directory (format v1 or v2), fail-closed. - - Mirrors the carryctx-pack manifest/check_counts contract; returns - (format_version, counts, worktree_bound_sessions). - """ - manifest = read_json(os.path.join(directory, "manifest.json"), "manifest.json") - require(isinstance(manifest, dict), "manifest.json is not a JSON object") - require( - manifest.get("format") == PACK_FORMAT, - "manifest format != %s: %r" % (PACK_FORMAT, manifest.get("format")), - ) - version = manifest.get("format_version") - require(is_int(version), "manifest format_version is not an integer: %r" % (version,)) - require( - version <= CURRENT_FORMAT_VERSION, - "manifest format_version %d is newer than supported %d" % (version, CURRENT_FORMAT_VERSION), - ) - require( - version >= 1, - "manifest format_version %d is older than the oldest supported 1" % version, - ) - for field in ("carryctx_version", "project_id", "export_id"): - value = manifest.get(field) - require( - isinstance(value, str) and value.strip(), - "manifest %s is missing or empty" % field, - ) - require(is_int(manifest.get("schema_version")), "manifest schema_version is not an integer") - created_at = manifest.get("created_at") - require( - isinstance(created_at, str) and _RFC3339_RE.match(created_at), - "manifest created_at is not RFC3339: %r" % (created_at,), - ) - require(isinstance(manifest.get("source"), dict), "manifest source is not an object") - for key, value in manifest["source"].items(): - require( - value is None or isinstance(value, str), - "manifest source.%s is not a string" % key, - ) - - counts = manifest.get("counts") - require(isinstance(counts, dict) and counts, "manifest counts missing or empty") - for table, expected in counts.items(): - require(isinstance(table, str) and table, "manifest counts has an empty table name") - require( - is_int(expected) and expected >= 0, - "manifest counts.%s is not a non-negative integer" % table, - ) - - redacted = manifest.get("redacted", False) - require(isinstance(redacted, bool), "manifest redacted is not a boolean: %r" % (redacted,)) - - allowed = V1_TABLES if version == 1 else V2_TABLES - unknown = sorted(set(counts) - set(allowed)) - require(not unknown, "manifest counts unknown table(s): %s" % ", ".join(unknown)) - - if version == 1: - # Legacy shape: v2-only fields must not appear in a v1 manifest. - require("tombstones" not in counts, "manifest v1 may not declare counts.tombstones") - require(not manifest.get("watermarks"), "manifest v1 may not carry watermarks") - else: - # v2 shape (carryctx-pack manifest.rs): tombstone counts are - # explicit, parents are unique non-empty ids that never self-reference, - # watermarks must agree with counts, and the publication stamp must be - # present -- this self-test runs after the redaction pass. - require("tombstones" in counts, "manifest v2 omits counts.tombstones") - require( - redacted is True, - "manifest v2 is not marked redacted: true (run the export-time redaction pass first)", - ) - parents = manifest.get("parents", []) - require(isinstance(parents, list), "manifest parents is not a list") - seen = set() - for parent in parents: - require( - isinstance(parent, str) and parent.strip(), - "manifest v2 has an empty parents entry", - ) - require( - parent != manifest["export_id"], - "manifest v2 lists its own export_id as a parent", - ) - require(parent not in seen, "manifest v2 lists parent %s more than once" % parent) - seen.add(parent) - watermarks = manifest.get("watermarks", {}) - require(isinstance(watermarks, dict), "manifest watermarks is not an object") - for table, watermark in sorted(watermarks.items()): - require(table in V2_TABLES, "manifest watermarks unknown table %s" % table) - require(isinstance(watermark, dict), "manifest watermark %s is not an object" % table) - rows = watermark.get("rows") - require( - is_int(rows) and rows >= 0, - "manifest watermark %s rows is not a non-negative integer" % table, - ) - declared = counts.get(table, 0) - require( - rows == declared, - "manifest watermark %s declares %d rows but counts declare %d" - % (table, rows, declared), - ) - max_updated = watermark.get("max_updated_at") - require( - max_updated is None or isinstance(max_updated, str), - "manifest watermark %s max_updated_at is not a string" % table, - ) - - sequences = manifest.get("sequences", {}) - require(isinstance(sequences, dict), "manifest sequences is not an object") - for kind, value in sequences.items(): - require( - is_int(value) and value >= 0, - "manifest sequences.%s is not a non-negative integer" % kind, - ) - - project = read_json(os.path.join(directory, "project.json"), "project.json") - require(isinstance(project, dict), "project.json is not a JSON object") - require( - project.get("id") == manifest["project_id"], - "project.json id != manifest project_id", - ) - - worktree_bound_sessions = 0 - for table in allowed: - path = os.path.join(directory, "%s.jsonl" % table) - try: - with open(path) as handle: - lines = handle.read().splitlines() - except FileNotFoundError as error: - raise PackError("missing table file %s.jsonl" % table) from error - rows = 0 - for number, line in enumerate(lines, 1): - if not line.strip(): - continue - try: - row = json.loads(line) - except ValueError as error: - raise PackError( - "%s.jsonl line %d: invalid JSON (%s)" % (table, number, error) - ) from error - require( - isinstance(row, dict), - "%s.jsonl line %d is not a JSON object" % (table, number), - ) - rows += 1 - if table == "sessions" and row.get("worktree_id"): - worktree_bound_sessions += 1 - if table in counts: - require( - rows == counts[table], - "%s.jsonl has %d rows, manifest wants %d" % (table, rows, counts[table]), - ) - else: - require( - rows == 0, - "%s.jsonl has %d undeclared rows (manifest omits counts)" % (table, rows), - ) - if verbose: - print(" table %-22s rows=%d ok" % (table, rows)) - return version, counts, worktree_bound_sessions - - -def fixture_manifest(version): - return { - "format": PACK_FORMAT, - "format_version": version, - "carryctx_version": "0.9.1", - "schema_version": 18, - "project_id": "01FAKEFIXTURE00000000000000", - "export_id": "01FAKEFIXTURE00000000000001", - "created_at": "2026-09-10T00:00:00Z", - "parents": [], - "sequences": {"display_id_CTX": 1}, - "source": {"git_commit": "0" * 40, "hostname": "fixture"}, - "counts": {"tasks": 1, "tombstones": 1}, - "redacted": True, - } - - -def v1_manifest(): - manifest = fixture_manifest(1) - manifest["counts"].pop("tombstones", None) - manifest.pop("redacted", None) - return manifest - - -TASK_ROW = {"id": "01FAKEFIXTURE00000000000002", "title": "fixture"} -TOMBSTONE_ROW = { - "project_id": "01FAKEFIXTURE00000000000000", - "table_name": "worktrees", - "row_id": "01FAKEFIXTURE00000000000003", - "deleted_at": "2026-09-10T00:00:00Z", - "deleted_by": None, - "reason": "fixture", -} - - -def write_fixture(root, name, manifest, tables, raw_files=None, drop_files=(), project_id=None): - directory = os.path.join(root, name) - os.makedirs(directory) - with open(os.path.join(directory, "manifest.json"), "w") as handle: - json.dump(manifest, handle, indent=2) - with open(os.path.join(directory, "project.json"), "w") as handle: - json.dump({"id": manifest.get("project_id") if project_id is None else project_id}, handle) - table_set = set(V2_TABLES) - table_set.update(manifest.get("counts", {})) - table_set.update(tables) - for table in sorted(table_set): - with open(os.path.join(directory, "%s.jsonl" % table), "w") as handle: - for row in tables.get(table, []): - handle.write(json.dumps(row) + "\n") - for table, text in (raw_files or {}).items(): - with open(os.path.join(directory, "%s.jsonl" % table), "w") as handle: - handle.write(text) - for table in drop_files: - os.remove(os.path.join(directory, "%s.jsonl" % table)) - return directory - - -def build_cases(): - cases = [] - - def add(name, manifest, tables, expect, raw_files=None, drop_files=(), project_id=None): - cases.append( - { - "name": name, - "manifest": manifest, - "tables": tables, - "expect": expect, - "raw_files": raw_files, - "drop_files": drop_files, - "project_id": project_id, - } - ) - - add("v2-minimal", fixture_manifest(2), {"tasks": [TASK_ROW], "tombstones": [TOMBSTONE_ROW]}, None) - - manifest = fixture_manifest(2) - manifest["parents"] = ["01FAKEPARENT0000000000001"] - manifest["watermarks"] = {"tasks": {"rows": 1, "max_updated_at": "2026-09-10T00:00:00Z"}} - manifest["sequences"]["display_id_decision"] = 2 - add("v2-full", manifest, {"tasks": [TASK_ROW], "tombstones": [TOMBSTONE_ROW]}, None) - - add("v1-legacy", v1_manifest(), {"tasks": [TASK_ROW]}, None) - manifest = v1_manifest() - manifest["redacted"] = True - add("v1-redacted", manifest, {"tasks": [TASK_ROW]}, None) - - manifest = v1_manifest() - manifest["format_version"] = 3 - add("v3-future", manifest, {"tasks": [TASK_ROW]}, "newer than supported") - manifest = v1_manifest() - manifest["format_version"] = 0 - add("v0-ancient", manifest, {"tasks": [TASK_ROW]}, "older than the oldest") - - manifest = fixture_manifest(2) - manifest["counts"].pop("tombstones") - add("v2-missing-tombstone-count", manifest, {"tasks": [TASK_ROW], "tombstones": [TOMBSTONE_ROW]}, "tombstones") - manifest = fixture_manifest(2) - manifest["counts"]["tombstones"] = 2 - add("v2-tombstone-count-skew", manifest, {"tasks": [TASK_ROW], "tombstones": [TOMBSTONE_ROW]}, "tombstones") - manifest = fixture_manifest(2) - manifest["parents"] = [""] - add("v2-parent-empty", manifest, {"tasks": [TASK_ROW], "tombstones": [TOMBSTONE_ROW]}, "parents") - manifest = fixture_manifest(2) - manifest["parents"] = ["01FAKEPARENT0000000000001", "01FAKEPARENT0000000000001"] - add("v2-parent-duplicate", manifest, {"tasks": [TASK_ROW], "tombstones": [TOMBSTONE_ROW]}, "more than once") - manifest = fixture_manifest(2) - manifest["parents"] = [manifest["export_id"]] - add("v2-parent-self", manifest, {"tasks": [TASK_ROW], "tombstones": [TOMBSTONE_ROW]}, "own export_id") - manifest = fixture_manifest(2) - manifest["watermarks"] = {"bogus_table": {"rows": 0}} - add("v2-watermark-unknown-table", manifest, {"tasks": [TASK_ROW], "tombstones": [TOMBSTONE_ROW]}, "unknown table") - manifest = fixture_manifest(2) - manifest["watermarks"] = {"tasks": {"rows": 2}} - add("v2-watermark-skew", manifest, {"tasks": [TASK_ROW], "tombstones": [TOMBSTONE_ROW]}, "watermark") - manifest = fixture_manifest(2) - manifest["watermarks"] = {"tasks": {"rows": "1"}} - add("v2-watermark-bad-shape", manifest, {"tasks": [TASK_ROW], "tombstones": [TOMBSTONE_ROW]}, "watermark") - manifest = fixture_manifest(2) - manifest["redacted"] = False - add("v2-unredacted", manifest, {"tasks": [TASK_ROW], "tombstones": [TOMBSTONE_ROW]}, "redacted") - manifest = fixture_manifest(2) - manifest["redacted"] = "true" - add("v2-redacted-string", manifest, {"tasks": [TASK_ROW], "tombstones": [TOMBSTONE_ROW]}, "redacted") - - manifest = v1_manifest() - manifest["watermarks"] = {"tasks": {"rows": 1}} - add("v1-watermarks", manifest, {"tasks": [TASK_ROW]}, "watermarks") - manifest = v1_manifest() - manifest["counts"]["tombstones"] = 0 - add("v1-tombstone-count", manifest, {"tasks": [TASK_ROW]}, "tombstones") - - manifest = fixture_manifest(2) - manifest["counts"]["tasks"] = -1 - add("counts-negative", manifest, {"tasks": [TASK_ROW], "tombstones": [TOMBSTONE_ROW]}, "tasks") - manifest = fixture_manifest(2) - manifest["counts"]["bogus_table"] = 1 - add("counts-unknown-table", manifest, {"tasks": [TASK_ROW], "tombstones": [TOMBSTONE_ROW]}, "unknown table") - manifest = fixture_manifest(2) - add("declared-count-skew", manifest, {"tasks": [], "tombstones": [TOMBSTONE_ROW]}, "tasks.jsonl has 0 rows") - manifest = fixture_manifest(2) - add( - "tombstone-file-missing", - manifest, - {"tasks": [TASK_ROW], "tombstones": [TOMBSTONE_ROW]}, - "missing table file tombstones.jsonl", - drop_files=("tombstones",), - ) - manifest = fixture_manifest(2) - manifest["counts"].pop("tasks") - add("undeclared-rows", manifest, {"tasks": [TASK_ROW], "tombstones": [TOMBSTONE_ROW]}, "undeclared") - manifest = fixture_manifest(2) - manifest["project_id"] = "01FAKEFIXTURE00000000000099" - add( - "project-id-mismatch", - manifest, - {"tasks": [TASK_ROW], "tombstones": [TOMBSTONE_ROW]}, - "project.json", - project_id="01FAKEFIXTURE00000000000000", - ) - manifest = fixture_manifest(2) - add( - "bad-jsonl", - manifest, - {"tasks": [TASK_ROW], "tombstones": [TOMBSTONE_ROW]}, - "invalid JSON", - raw_files={"tasks": "{not json\n"}, - ) - return cases - - -try: - version, counts, worktree_bound_sessions = validate_pack(pack) -except PackError as failure: - sys.exit("publish-ctxpack-selftest: FAIL: %s" % failure) - -print("manifest v%d + counts + JSON: PASS (%d tables)" % (version, len(counts))) -print("WORKTREE_BOUND_SESSIONS=%d" % worktree_bound_sessions) - -# Validator fixture matrix (check 4): the same validate_pack used on the real -# snapshot must accept valid v1/v2 packs and refuse the documented failure -# modes. -matrix_root = tempfile.mkdtemp(prefix="ctxpack-selftest-matrix.") -failures = [] -case_count = 0 -try: - for case in build_cases(): - directory = write_fixture( - matrix_root, - case["name"], - copy.deepcopy(case["manifest"]), - copy.deepcopy(case["tables"]), - raw_files=case["raw_files"], - drop_files=case["drop_files"], - project_id=case["project_id"], - ) - try: - validate_pack(directory, verbose=False) - error = None - except PackError as failure: - error = str(failure) - case_count += 1 - name = case["name"] - expect = case["expect"] - if expect is None: - if error is not None: - failures.append("fixture %s: expected accept, got: %s" % (name, error)) - elif error is None: - failures.append("fixture %s: expected reject (%s), got accept" % (name, expect)) - elif expect not in error: - failures.append("fixture %s: expected error containing %r, got: %s" % (name, expect, error)) -finally: - shutil.rmtree(matrix_root, ignore_errors=True) -if failures: - sys.exit("publish-ctxpack-selftest: FAIL: validator fixture matrix: " + "; ".join(failures)) -print("validator fixture matrix: PASS (%d cases)" % case_count) -PYEOF - -# Check 6 — planted-fake-secret redaction fixture (CTX-0281, extended for -# ctxpack v2 in CTX-0314). Synthetic v1 and v2 packs only; FAKE marker -# values, never real credentials. Models the pipeline guarantee: the staging -# copy is redacted and stamped redacted=true (counts untouched, tombstones -# rows redacted like any other table), the source ("local DB") copy stays -# byte-identical. -REDACT_PY="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/publish-ctxpack-redact.py" -[[ -f "$REDACT_PY" ]] || fail "redactor $REDACT_PY not found" -log "checking export-time redaction (planted-fake-secret v1+v2 fixtures)" -FIXT_ROOT="$(mktemp -d "${TMPDIR:-/tmp}/ctxpack-redact-fixture.XXXXXX")" -FIXT_ROOT="$FIXT_ROOT" timeout "$TIMEOUT_SECS" python3 - <<'PYEOF' -import json, os, shutil - -root = os.environ["FIXT_ROOT"] -# Fake-but-shaped secret values: each carries a FAKE marker so a hit can -# never be mistaken for a real credential, while still tripping every -# redactor rule (secret field names, NAME=value env-dump lines, 40+ runs). -fakes = [ - "sk-FAKE-0123456789abcdef0123456789abcdef01", # OPENAI_API_KEY / API_KEY value - "FAKECLOUDFLARETOKENFAKECLOUDFLARE01", # CLOUDFLARE_API_TOKEN value - "ghp_FAKEFAKEFAKEFAKEFAKEFAKEFAKEFAKE00", # GH_PAT-shaped value -] -sha = "da39a3ee5e6b4b0d3255bfef95601890afd80709" # must SURVIVE (SHA-1 exempt) -progress_row = { - "id": "01FAKEFIXTURE00000000000001", - "task_id": "01FAKEFIXTURE00000000000002", - "type": "note", - "content": "shell env:\nOPENAI_API_KEY=%s\nCLOUDFLARE_API_TOKEN=%s\nplain prose stays" % (fakes[0], fakes[1]), -} -event_row = { - "id": "01FAKEFIXTURE00000000000003", - "type": "progress.created", - "payload_json": json.dumps( - {"gh_pat": fakes[2], "detail": "session done", "sha": sha}, - separators=(",", ":"), - ), -} -tombstone_row = { - "project_id": "01FAKEFIXTURE00000000000000", - "table_name": "worktrees", - "row_id": "01FAKEFIXTURE00000000000004", - "deleted_at": "2026-09-10T00:00:00Z", - "deleted_by": "ctxpack-selftest", - "reason": "cleanup API_KEY=%s" % fakes[0], -} -v2_manifest = { - "format": "carryctx-pack-dir", - "format_version": 2, - "carryctx_version": "0.9.1", - "schema_version": 18, - "project_id": "01FAKEFIXTURE00000000000000", - "export_id": "01FAKEFIXTURE00000000000005", - "created_at": "2026-09-10T00:00:00Z", - "parents": [], - "sequences": {"display_id_CTX": 1}, - "source": {"git_commit": "0" * 40, "hostname": "fixture"}, - "counts": {"progress_items": 1, "events": 1, "tombstones": 1}, - "watermarks": {"tombstones": {"rows": 1}}, -} -v1_manifest = { - "format": "carryctx-pack-dir", - "format_version": 1, - "project_id": "01FAKEFIXTURE00000000000000", - "counts": {"progress_items": 1, "events": 1}, -} -for name, manifest, tables in ( - ("local-v1", v1_manifest, {"progress_items": [progress_row], "events": [event_row]}), - ( - "local-v2", - v2_manifest, - {"progress_items": [progress_row], "events": [event_row], "tombstones": [tombstone_row]}, - ), -): - directory = os.path.join(root, name) - os.makedirs(directory) - with open(os.path.join(directory, "manifest.json"), "w") as handle: - json.dump(manifest, handle, indent=2) - handle.write("\n") - shutil.copyfile( - os.path.join(directory, "manifest.json"), - os.path.join(directory, "manifest.json.orig"), - ) - with open(os.path.join(directory, "project.json"), "w") as handle: - json.dump({"id": manifest["project_id"]}, handle) - for table, rows in tables.items(): - with open(os.path.join(directory, "%s.jsonl" % table), "w") as handle: - for row in rows: - handle.write(json.dumps(row) + "\n") -with open(os.path.join(root, "fakes.txt"), "w") as handle: - handle.write("\n".join(fakes + [sha]) + "\n") -print("fixture packs written (v1 + v2 source copies)") -PYEOF -for variant in v1 v2; do - cp -a "$FIXT_ROOT/local-$variant" "$FIXT_ROOT/staging-$variant" - timeout "$TIMEOUT_SECS" python3 "$REDACT_PY" "$FIXT_ROOT/staging-$variant" >/dev/null || - fail "redactor failed on the $variant planted-secret fixture" -done -FIXT_ROOT="$FIXT_ROOT" timeout "$TIMEOUT_SECS" python3 - <<'PYEOF' -import json, os, sys - -root = os.environ["FIXT_ROOT"] -with open(os.path.join(root, "fakes.txt")) as handle: - fakes_and_sha = handle.read().split() -fakes, sha = fakes_and_sha[:3], fakes_and_sha[3] - -for variant, tables in ( - ("v1", ("progress_items", "events")), - ("v2", ("progress_items", "events", "tombstones")), -): - local = os.path.join(root, "local-" + variant) - staging = os.path.join(root, "staging-" + variant) - - # The source copy is the local DB stand-in: byte-identical manifest and - # the planted fakes still present. - with open(os.path.join(local, "manifest.json"), "rb") as handle: - source_manifest_bytes = handle.read() - with open(os.path.join(local, "manifest.json.orig"), "rb") as handle: - original_manifest_bytes = handle.read() - if source_manifest_bytes != original_manifest_bytes: - sys.exit("%s source manifest was rewritten (local DB must stay untouched)" % variant) - - with open(os.path.join(staging, "manifest.json")) as handle: - staged_manifest = json.load(handle) - with open(os.path.join(local, "manifest.json")) as handle: - source_manifest = json.load(handle) - if staged_manifest.get("redacted") is not True: - sys.exit("%s staging manifest: redacted is not true (%r)" % (variant, staged_manifest.get("redacted"))) - if staged_manifest.get("counts") != source_manifest.get("counts"): - sys.exit( - "%s staging manifest: counts changed (%r -> %r)" - % (variant, source_manifest.get("counts"), staged_manifest.get("counts")) - ) - - staged_text = "" - local_text = "" - for table in tables: - with open(os.path.join(local, "%s.jsonl" % table)) as handle: - local_lines = [line for line in handle.read().splitlines() if line.strip()] - local_text += "\n".join(local_lines) + "\n" - with open(os.path.join(staging, "%s.jsonl" % table)) as handle: - staged_lines = [line for line in handle.read().splitlines() if line.strip()] - if len(staged_lines) != len(local_lines): - sys.exit( - "%s staging %s.jsonl has %d rows, want %d" - % (variant, table, len(staged_lines), len(local_lines)) - ) - for number, line in enumerate(staged_lines, 1): - try: - json.loads(line) - except ValueError as error: - sys.exit("%s staging %s.jsonl line %d: invalid JSON (%s)" % (variant, table, number, error)) - staged_text += "\n".join(staged_lines) + "\n" - - for fake in fakes: - if fake in staged_text: - sys.exit("%s staging copy still contains planted fake value %s..." % (variant, fake[:12])) - if fake not in local_text: - sys.exit("%s source copy lost planted fake value (local DB must stay untouched)" % variant) - if "***REDACTED***" not in staged_text: - sys.exit("%s staging copy has no ***REDACTED*** markers" % variant) - for name in ("OPENAI_API_KEY", "CLOUDFLARE_API_TOKEN", "gh_pat"): - if name not in staged_text: - sys.exit("%s staging copy lost field/variable name %s (names must survive)" % (variant, name)) - if sha not in staged_text: - sys.exit("%s staging copy lost the SHA-1 control string (over-redaction)" % variant) - if variant == "v2": - with open(os.path.join(staging, "tombstones.jsonl")) as handle: - tombstone_rows = [json.loads(line) for line in handle if line.strip()] - if len(tombstone_rows) != 1: - sys.exit("v2 staging tombstones.jsonl has %d rows, want 1" % len(tombstone_rows)) - reason = tombstone_rows[0].get("reason") or "" - if tombstone_rows[0].get("table_name") != "worktrees" or "cleanup" not in reason: - sys.exit("v2 staging tombstone row content changed unexpectedly: %r" % tombstone_rows[0]) - if "API_KEY" not in reason: - sys.exit("v2 staging copy lost the tombstone reason variable name") - print(" redaction fixture %s: staging redacted + stamped, source untouched: PASS" % variant) -print("redaction fixture: PASS") -PYEOF - -if [[ "$ROUND_TRIP" == 0 ]]; then - log "round-trip skipped (--no-round-trip); static checks passed" - exit 0 -fi - -# Round-trip probe: the export must re-import into a scratch project with -# matching table counts. Capture the worktree-bound session count first so -# the known-import-limitation WARN below can key off pack content, not -# import stderr text alone. -BOUND="$(timeout "$TIMEOUT_SECS" python3 -c " -import json -n = 0 -with open('$PACK/sessions.jsonl') as f: - for line in f: - if json.loads(line).get('worktree_id'): - n += 1 -print(n) -")" - -SCRATCH="$(mktemp -d "${TMPDIR:-/tmp}/ctxpack-roundtrip.XXXXXX")" -log "round-trip scratch: $SCRATCH" -timeout "$TIMEOUT_SECS" git -C "$SCRATCH" init -q || - fail "git init failed in scratch" -timeout "$TIMEOUT_SECS" carryctx init --project "$SCRATCH" --name ctxpack-roundtrip --minimal --agent ctxpack-selftest >/dev/null || - fail "carryctx init failed in scratch" - -IMPORT_OUT="$SCRATCH/import.json" -set +e -timeout "$TIMEOUT_SECS" carryctx import "$PACK" --project "$SCRATCH" --mode replace --yes --agent ctxpack-selftest >"$IMPORT_OUT" 2>"$SCRATCH/import.err" -IMPORT_RC=$? -set -e - -if [[ "$IMPORT_RC" -ne 0 ]]; then - if grep -q "FOREIGN KEY constraint failed" "$SCRATCH/import.err" "$IMPORT_OUT" 2>/dev/null && - [[ "$BOUND" -gt 0 ]]; then - warn "import probe hit the known carryctx v1 limitation: pack holds $BOUND worktree-bound session(s) whose worktree rows are pruned at the import target, so sessions fail FK (export itself validated above; mirror stays publish-only)" - warn "round-trip: KNOWN-ISSUE (see script header); static checks passed" - exit 0 - fi - cat "$SCRATCH/import.err" >&2 2>/dev/null || true - fail "carryctx import failed (rc=$IMPORT_RC); see above" -fi - -log "comparing imported counts with manifest" -timeout "$TIMEOUT_SECS" python3 - "$PACK" "$IMPORT_OUT" <<'PYEOF' -import json, sys - -with open(sys.argv[1] + "/manifest.json") as f: - expected = json.load(f)["counts"] -with open(sys.argv[2]) as f: - got = json.load(f)["counts"] - -for table in sorted(expected): - if table not in got: - sys.exit("imported counts missing table %s" % table) - if table == "worktrees": - # Re-anchoring prunes worktree rows whose dirs are absent at the - # import target; fewer-or-equal is the honest expectation here. - if got[table] > expected[table]: - sys.exit("worktrees grew on import (%d > %d)" % (got[table], expected[table])) - print(" table %-22s manifest=%d imported=%d (prune-tolerant) ok" % (table, expected[table], got[table])) - elif got[table] != expected[table]: - sys.exit("table %s: manifest=%d imported=%d" % (table, expected[table], got[table])) - else: - print(" table %-22s rows=%d ok" % (table, got[table])) -print("round-trip counts: PASS") -PYEOF - -log "self-test PASS" diff --git a/scripts/publish-ctxpack.sh b/scripts/publish-ctxpack.sh deleted file mode 100755 index d5aa8be..0000000 --- a/scripts/publish-ctxpack.sh +++ /dev/null @@ -1,340 +0,0 @@ -#!/usr/bin/env bash -# publish-ctxpack.sh — publish a ctxpack snapshot to the workflow-mirror repo. -# -# On every PR merge the commander runs `just workflow-publish` from a -# bitty-mcp checkout (the trigger is the merge closeout, NOT a git hook: -# GitHub squash-merges never fire local git hooks, and `carryctx hooks -# install` behavior is intentionally left untouched). -# -# Repo-specific bits are env defaults, not hardcodes: override -# WORKFLOW_SOURCE_REPO / WORKFLOW_MIRROR_URL / WORKFLOW_MIRROR_DIR to reuse -# this script verbatim in another repo. Defaults target -# bitty-terminal/bitty-mcp-workflow. -# -# What it does: -# 1. Exports `carryctx export --pack-format dir` from this repo into a -# staging clone of bitty-terminal/bitty-mcp-workflow as -# -/. -# 2. Redacts secret-shaped values inside the STAGING copy only (export-time -# redaction pass, scripts/publish-ctxpack-redact.py): values of -# secret-named fields (*_KEY/*_TOKEN/*_SECRET/*_PASSWORD, GH_PAT, -# CLOUDFLARE_*, AWS_*) and 40+-char token-like runs become -# `***REDACTED***`. JSONL-aware (parse per line, redact, re-serialize), -# row counts unchanged, re-runnable no-op, covers every table file -# including format-v2 `tombstones.jsonl`; stamps `redacted: true` into -# manifest.json (counts untouched). The local carryctx DB is -# never modified -- it keeps the originals and the next export redacts -# them again. -# 3. Validates (round-trip self-test for format v1 and v2 snapshots, incl. -# a planted-fake-secret fixture proving staging is redacted while the -# source copy is untouched, and a validator accept/reject fixture -# matrix). -# 4. Refreshes the mirror README.md pointer + LATEST file. -# 5. Commits and pushes to the mirror repo main branch. -# -# Privacy: the pack intentionally contains agent display names and absolute -# workspace paths, but secret-shaped values can NEVER reach the mirror: the -# redaction pass (step 2) runs on every export, dry-run or publish, before -# validation and commit, so even a live key accidentally pasted into a note -# or session payload is replaced with `***REDACTED***` (names kept for -# debuggability). A leaked-then-rotated secret still needs rotation at the -# source -- redaction limits mirror exposure, it does not un-leak anything. -# -# Usage: -# scripts/publish-ctxpack.sh [--dry-run] [--mirror DIR] [--url URL] -# [--git-timeout SECS] [--keep-tmp] -# --dry-run export + validate (incl. round-trip self-test) only; -# no clone, no commit, no push. -# --mirror DIR staging clone location -# (default: $WORKFLOW_MIRROR_DIR or -# /../bitty-mcp-workflow). -# --url URL mirror remote for a fresh clone -# (default: $WORKFLOW_MIRROR_URL or -# https://github.com/bitty-terminal/bitty-mcp-workflow.git). -# --git-timeout SECS timeout for every git/carryctx network or heavy op -# (default: $GIT_TIMEOUT or 120). -# --keep-tmp keep temp dirs on exit (debug aid; prints paths). -# -# Guarantees: idempotent (re-running for the same source SHA reuses the -# existing snapshot dir and no-ops when the mirror is already current); -# fail-closed with clear errors; never writes to the source repo's own git -# state (only read-only rev-parse/branch queries there). -set -euo pipefail - -REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -SOURCE_REPO="${WORKFLOW_SOURCE_REPO:-bitty-mcp}" -MIRROR_NAME="${SOURCE_REPO}-workflow" -MIRROR_DEFAULT="$REPO_ROOT/../$MIRROR_NAME" -MIRROR_URL_DEFAULT="https://github.com/bitty-terminal/$MIRROR_NAME.git" - -DRY_RUN=0 -MIRROR="${WORKFLOW_MIRROR_DIR:-$MIRROR_DEFAULT}" -MIRROR_URL="${WORKFLOW_MIRROR_URL:-$MIRROR_URL_DEFAULT}" -GIT_TIMEOUT="${GIT_TIMEOUT:-120}" -KEEP_TMP=0 -REDACTIONS=0 - -while [[ $# -gt 0 ]]; do - case "$1" in - --dry-run) - DRY_RUN=1 - shift - ;; - --mirror) - MIRROR="${2:?--mirror requires a directory}" - shift 2 - ;; - --mirror=*) - MIRROR="${1#--mirror=}" - shift - ;; - --url) - MIRROR_URL="${2:?--url requires a remote URL}" - shift 2 - ;; - --url=*) - MIRROR_URL="${1#--url=}" - shift - ;; - --git-timeout) - GIT_TIMEOUT="${2:?--git-timeout requires seconds}" - shift 2 - ;; - --git-timeout=*) - GIT_TIMEOUT="${1#--git-timeout=}" - shift - ;; - --keep-tmp) - KEEP_TMP=1 - shift - ;; - --help | -h) - sed -n '2,/^set -euo/p' "${BASH_SOURCE[0]}" - exit 0 - ;; - *) - echo "publish-ctxpack: FAIL: unknown flag $1 (see --help)" >&2 - exit 2 - ;; - esac -done - -fail() { - echo "publish-ctxpack: FAIL: $1" >&2 - exit 1 -} - -log() { - echo "publish-ctxpack: $1" -} - -have() { command -v "$1" >/dev/null 2>&1; } - -have git || fail "git not on PATH" -have carryctx || fail "carryctx not on PATH" -have timeout || fail "timeout not on PATH" -have python3 || fail "python3 not on PATH (needed by the redaction pass)" - -# Read-only introspection of the source checkout. Never a write op here: -# no checkout, reset, commit, stash, or index/worktree mutation. -REPO_SHA="$(timeout "$GIT_TIMEOUT" git -C "$REPO_ROOT" rev-parse HEAD 2>/dev/null)" || - fail "cannot read $SOURCE_REPO HEAD (not a git checkout?)" -REPO_SHORT="$(timeout "$GIT_TIMEOUT" git -C "$REPO_ROOT" rev-parse --short HEAD 2>/dev/null)" || - fail "cannot read $SOURCE_REPO short SHA" -REPO_BRANCH="$(timeout "$GIT_TIMEOUT" git -C "$REPO_ROOT" branch --show-current 2>/dev/null || true)" -if [[ "${REPO_BRANCH:-}" != "main" ]]; then - log "WARN: $SOURCE_REPO checkout is on branch '${REPO_BRANCH:-detached}', snapshotting HEAD anyway" -fi -STAMP="$(date -u +%Y%m%dT%H%M%SZ)" -SNAPSHOT="${STAMP}-${REPO_SHORT}" - -TMP_ROOT="" -cleanup() { - if [[ "$KEEP_TMP" == 0 && -n "$TMP_ROOT" && -d "$TMP_ROOT" ]]; then - rm -rf "$TMP_ROOT" - elif [[ -n "$TMP_ROOT" ]]; then - log "keeping tmp dir $TMP_ROOT (--keep-tmp)" - fi -} -trap cleanup EXIT - -# Mirror git ops always go through this wrapper: timeout-wrapped and -# confined to the staging clone. $REPO_ROOT is never a write target. -mirror_git() { - timeout "$GIT_TIMEOUT" git -C "$MIRROR" "$@" -} - -ensure_mirror() { - if [[ -d "$MIRROR/.git" ]]; then - local remote - remote="$(mirror_git remote get-url origin 2>/dev/null)" || - fail "staging clone $MIRROR has no origin remote" - case "$remote" in - *"$MIRROR_NAME"*) - ;; - *) - fail "staging clone origin ($remote) does not look like $MIRROR_NAME; refusing to push (override with --mirror/--url)" - ;; - esac - if [[ -n "$(mirror_git status --porcelain 2>/dev/null)" ]]; then - fail "staging clone $MIRROR has uncommitted changes; clean it up and re-run" - fi - mirror_git fetch origin --prune || - fail "git fetch failed in staging clone (network/auth?)" - if mirror_git rev-parse --verify HEAD >/dev/null 2>&1; then - mirror_git checkout -q main 2>/dev/null || - mirror_git checkout -q -b main --track origin/main || - fail "cannot check out main in staging clone" - mirror_git reset -q --hard origin/main || - fail "cannot fast-forward staging clone to origin/main" - else - # Freshly cloned empty repo (or brand-new mirror): start main here. - mirror_git checkout -q -b main || - fail "cannot create main branch in empty staging clone" - fi - return 0 - fi - if [[ -e "$MIRROR" ]]; then - fail "$MIRROR exists but is not a git clone; move it aside or pass --mirror" - fi - log "cloning mirror $MIRROR_URL -> $MIRROR" - timeout "$GIT_TIMEOUT" git clone -q "$MIRROR_URL" "$MIRROR" || - fail "git clone failed (network/auth?); set --url or pre-clone --mirror yourself" - ensure_mirror -} - -write_mirror_readme() { - # Refresh the mirror-root pointer only when the template changes - # (keeps re-publishes idempotent: no churn commit for identical text). - local readme="$MIRROR/README.md" tmp - tmp="$(mktemp)" - cat >"$tmp" <<'EOF' -# __MIRROR_NAME__ — engineering workflow mirror - -Transparent companion to [__SOURCE_REPO__](https://github.com/bitty-terminal/__SOURCE_REPO__): -every PR merge publishes a CarryCtx ctxpack snapshot here, so the whole -engineering workflow (tasks, sessions, decisions, checkpoints) is reviewable, -not just the code. - -- Each `-/` directory is one - `carryctx export --pack-format dir` snapshot (manifest.json + project.json - + per-table `*.jsonl`; format v2 snapshots also carry `tombstones.jsonl`). -- `LATEST` names the newest snapshot directory. -- Each snapshot carries a `source.json` with the source commit/branch it was - taken from (`repo`, `repo_commit`, `repo_branch` keys). -- Every snapshot is a redacted publication artifact (`redacted: true` in - manifest.json); CarryCtx refuses redacted bundles as merge sources, so - snapshots are never merged back. To restore a fresh clone's local CarryCtx - DB from `LATEST`, run `just workflow-import` (`just workflow-import-dry` - validates only); restore always uses replace mode and refuses to replace a - non-empty local DB without `--force`. -- Trigger: the commander's merge closeout runs `just workflow-publish` in - the __SOURCE_REPO__ repo. No git hook drives this (squash-merges never fire local - hooks). - -## Privacy notice - -Snapshots intentionally contain agent display names and absolute workspace -paths. Secret-shaped values never reach the mirror: every export passes -through an automatic JSONL-aware redaction step (all table files, including -`tombstones.jsonl`) before validation and commit (secret-named fields such -as `*_KEY` / `*_TOKEN` / `*_SECRET` / -`*_PASSWORD`, `GH_PAT`, `CLOUDFLARE_*`, `AWS_*`, plus `NAME=value` pairs -and 40+-char token-like runs in free text, become `***REDACTED***`; -field/variable names are kept for debuggability; the local database is -never modified). Redaction limits mirror exposure -- it does not un-leak a -secret that was already pushed anywhere: rotate at the source and report -suspected leaks to the repository owner immediately. -EOF - sed -i "s/__MIRROR_NAME__/$MIRROR_NAME/g; s/__SOURCE_REPO__/$SOURCE_REPO/g" "$tmp" - if [[ -f "$readme" ]] && cmp -s "$tmp" "$readme"; then - rm -f "$tmp" - else - mv "$tmp" "$readme" - fi -} - -run_export() { - local dest="$1" - log "exporting ctxpack -> $dest" - timeout "$GIT_TIMEOUT" carryctx export --pack-format dir -o "$dest" --project "$REPO_ROOT" || - fail "carryctx export failed" - [[ -f "$dest/manifest.json" && -f "$dest/project.json" ]] || - fail "export incomplete (manifest.json/project.json missing)" - redact_snapshot "$dest" -} - -# Export-time redaction seam (post-export, pre-validate/commit). Operates -# ONLY on the snapshot copy ($dest: the staging clone or a --dry-run tmp -# dir); the local carryctx DB is strictly read-only throughout this script. -# Row counts are unchanged, so manifest counts and the self-test still hold. -redact_snapshot() { - local dest="$1" redact_out - log "redacting secret-shaped values in staging copy $dest (local DB untouched)" - redact_out="$(timeout "$GIT_TIMEOUT" python3 "$REPO_ROOT/scripts/publish-ctxpack-redact.py" "$dest")" || - fail "secret redaction failed; mirror left untouched" - log "$redact_out" - REDACTIONS="$(printf '%s\n' "$redact_out" | sed -n 's/^REDACTIONS=//p')" - REDACTIONS="${REDACTIONS:-0}" -} - -run_selftest() { - local dest="$1" - log "validating snapshot (self-test)" - bash "$REPO_ROOT/scripts/publish-ctxpack-selftest.sh" "$dest" || - fail "snapshot self-test failed; mirror left untouched" -} - -if [[ "$DRY_RUN" == 1 ]]; then - TMP_ROOT="$(mktemp -d "${TMPDIR:-/tmp}/ctxpack-dry.XXXXXX")" - SNAP_DIR="$TMP_ROOT/$SNAPSHOT" - run_export "$SNAP_DIR" - run_selftest "$SNAP_DIR" - log "dry-run PASS: snapshot $SNAPSHOT exports + validates; nothing pushed" - exit 0 -fi - -ensure_mirror - -# Idempotency: a snapshot for this source SHA already published? Reuse it so -# a re-run converges instead of duplicating history. -EXISTING="" -shopt -s nullglob -candidates=("$MIRROR"/*-"$REPO_SHORT") -if ((${#candidates[@]} > 0)) && [[ -d "${candidates[0]}" ]]; then - EXISTING="$(basename "${candidates[0]}")" -fi -shopt -u nullglob - -if [[ -n "$EXISTING" ]]; then - log "snapshot for $REPO_SHORT already exists ($EXISTING); refreshing pointers" - SNAP_DIR="$MIRROR/$EXISTING" -else - SNAP_DIR="$MIRROR/$SNAPSHOT" - run_export "$SNAP_DIR" -fi - -run_selftest "$SNAP_DIR" - -SOURCE_JSON="$SNAP_DIR/source.json" -if ! [[ -f "$SOURCE_JSON" ]] || ! grep -q "\"repo_commit\":\"$REPO_SHA\"" "$SOURCE_JSON"; then - printf '{"snapshot":"%s","repo":"%s","repo_commit":"%s","repo_branch":"%s","exported_at":"%s","redactions":%d,"tool":"scripts/publish-ctxpack.sh"}\n' \ - "$(basename "$SNAP_DIR")" "$SOURCE_REPO" "$REPO_SHA" "${REPO_BRANCH:-detached}" "$STAMP" "$REDACTIONS" >"$SOURCE_JSON" -fi - -write_mirror_readme -printf '%s\n' "$(basename "$SNAP_DIR")" >"$MIRROR/LATEST" - -mirror_git add -A || - fail "git add failed in staging clone" -if mirror_git diff --cached --quiet; then - log "mirror already current at $(cat "$MIRROR/LATEST"); nothing to push" - exit 0 -fi -mirror_git -c user.name="${GIT_AUTHOR_NAME:-$SOURCE_REPO-publisher}" \ - -c user.email="${GIT_AUTHOR_EMAIL:-$SOURCE_REPO-publisher@users.noreply.github.com}" \ - commit -q -m "chore(ctxpack): snapshot $(basename "$SNAP_DIR")" || - fail "git commit failed in staging clone" -mirror_git push origin main || - fail "git push failed (network/auth/permissions?)" -log "published $(basename "$SNAP_DIR") ($SOURCE_REPO $REPO_SHORT); LATEST updated" diff --git a/scripts/workflow-import.sh b/scripts/workflow-import.sh new file mode 100755 index 0000000..d355ed3 --- /dev/null +++ b/scripts/workflow-import.sh @@ -0,0 +1,220 @@ +#!/usr/bin/env bash +# workflow-import.sh — restore the local CarryCtx DB from the in-repo snapshot. +# +# In-repo restore (no mirror repository): fetches the publication branch +# `refs/heads/carryctx-snapshots` from the remote, then runs the native +# `carryctx import --from-git --mode replace --yes`. CarryCtx never +# touches the network, so this target fetches the branch itself. +# +# Safety (thin, no Python): +# - refuses to replace a non-empty local DB (one project row with data rows) +# without --force, leaving the DB untouched; +# - initializes a fresh clone's CarryCtx state (no project row) before import; +# - --dry-run validates the snapshot and writes nothing; +# - restores the committed `.carryctx/config.toml` byte-identically, because +# `carryctx import` rewrites it with current config defaults; +# - prints snapshot provenance (commit, export id, CarryCtx-Source trailer). +# +# Usage: +# scripts/workflow-import.sh [--force] [--dry-run] [--project DIR] +# [--remote NAME] [--git-timeout SECS] [--help] +# --force replace a non-empty local DB. +# --dry-run fetch + validate only; no DB write. +# --project DIR repository to restore (default: this checkout root). +# --remote NAME git remote to fetch from (default: origin). +# --git-timeout SECS timeout for every carryctx/git op (default: 120). +set -euo pipefail + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +SNAP_BRANCH="refs/heads/carryctx-snapshots" +PROJECT="$REPO_ROOT" +REMOTE="${WORKFLOW_REMOTE:-origin}" +GIT_TIMEOUT="${GIT_TIMEOUT:-120}" +FORCE=0 +DRY_RUN=0 + +usage() { + sed -n '2,/^set -euo/p' "${BASH_SOURCE[0]}" | sed '$d' +} + +while [[ $# -gt 0 ]]; do + case "$1" in + --force) + FORCE=1 + shift + ;; + --dry-run) + DRY_RUN=1 + shift + ;; + --project) + PROJECT="${2:?--project requires a directory}" + shift 2 + ;; + --project=*) + PROJECT="${1#--project=}" + shift + ;; + --remote) + REMOTE="${2:?--remote requires a name}" + shift 2 + ;; + --remote=*) + REMOTE="${1#--remote=}" + shift + ;; + --git-timeout) + GIT_TIMEOUT="${2:?--git-timeout requires seconds}" + shift 2 + ;; + --git-timeout=*) + GIT_TIMEOUT="${1#--git-timeout=}" + shift + ;; + --help | -h) + usage + exit 0 + ;; + *) + echo "workflow-import: FAIL: unknown flag $1 (see --help)" >&2 + exit 2 + ;; + esac +done + +fail() { + echo "workflow-import: FAIL: $1" >&2 + exit 1 +} + +log() { + echo "workflow-import: $1" +} + +warn() { + echo "workflow-import: WARN: $1" >&2 +} + +have() { command -v "$1" >/dev/null 2>&1; } + +have git || fail "git not on PATH" +have carryctx || fail "carryctx not on PATH" +have timeout || fail "timeout not on PATH" + +PROJECT="$(cd "$PROJECT" && pwd)" || fail "project directory $PROJECT not found" +TRACK_REF="refs/remotes/$REMOTE/carryctx-snapshots" +TMP_ROOT="$(mktemp -d "${TMPDIR:-/tmp}/workflow-import.XXXXXX")" +cleanup() { + rm -rf "$TMP_ROOT" +} +trap cleanup EXIT + +# Read-only probe of the local CarryCtx DB. Never creates or mutates it. Prints +# " "; project_rows is 1 when a projects row exists. +probe_state() { + local db="$1" projects=0 rows=0 expr="" table present + [[ -f "$db" ]] || { + printf '0 0\n' + return 0 + } + have sqlite3 || fail "sqlite3 not on PATH (needed to inspect the local state DB); install it or pass a fresh --project" + present="$(timeout "$GIT_TIMEOUT" sqlite3 -readonly "$db" \ + "SELECT name FROM sqlite_master WHERE type='table';" 2>/dev/null || true)" + if grep -qx 'projects' <<<"$present"; then + projects="$(timeout "$GIT_TIMEOUT" sqlite3 -readonly "$db" 'SELECT COUNT(*) FROM projects;' 2>/dev/null || echo 0)" + fi + for table in agents tasks task_dependencies progress_items sessions worktrees \ + checkpoints checkpoint_corrections scopes decisions handoffs events \ + graph_nodes graph_edges teams team_members; do + if grep -qx "$table" <<<"$present"; then + expr+="+(SELECT COUNT(*) FROM \"$table\")" + fi + done + if [[ -n "$expr" ]]; then + rows="$(timeout "$GIT_TIMEOUT" sqlite3 -readonly "$db" "SELECT 0${expr};" 2>/dev/null || echo 0)" + fi + printf '%s %s\n' "$projects" "$rows" +} + +GIT_COMMON="$(timeout "$GIT_TIMEOUT" git -C "$PROJECT" rev-parse --path-format=absolute --git-common-dir 2>/dev/null)" || + fail "cannot resolve the git common dir for $PROJECT" +DB="$GIT_COMMON/carryctx/state.sqlite" +STATE_LINE="$(probe_state "$DB")" || fail "cannot inspect local CarryCtx state" +read -r PROJECTS ROWS <<<"$STATE_LINE" +log "local state: db_exists=$([[ -f "$DB" ]] && echo 1 || echo 0) project_rows=$PROJECTS data_rows=$ROWS" + +if [[ "$PROJECTS" -gt 1 ]]; then + fail "local DB has $PROJECTS project rows; refusing to replace an ambiguous state DB" +fi +if [[ "$DRY_RUN" == 0 && "$PROJECTS" -ge 1 && "$ROWS" -gt 0 && "$FORCE" == 0 ]]; then + fail "local DB already holds $ROWS data row(s); refusing to overwrite non-empty state without --force (local DB untouched)" +fi + +log "fetching $REMOTE $SNAP_BRANCH -> $TRACK_REF" +if ! timeout "$GIT_TIMEOUT" git -C "$PROJECT" fetch "$REMOTE" \ + "refs/heads/carryctx-snapshots:refs/remotes/$REMOTE/carryctx-snapshots"; then + fail "git fetch failed (has refs/heads/carryctx-snapshots been published? network/auth?)" +fi + +print_provenance() { + local commit export_id source_line message + commit="$(timeout "$GIT_TIMEOUT" git -C "$PROJECT" rev-parse --short "$TRACK_REF" 2>/dev/null || echo '?')" + message="$(timeout "$GIT_TIMEOUT" git -C "$PROJECT" log -1 --format=%B "$TRACK_REF" 2>/dev/null || true)" + export_id="$(sed -n 's/^CarryCtx-Export-Id: //p' <<<"$message" | head -n1)" + source_line="$(sed -n 's/^CarryCtx-Source: //p' <<<"$message" | head -n1)" + log "provenance: snapshot=$commit export_id=${export_id:-?} source=${source_line:-?}" +} + +if [[ "$DRY_RUN" == 1 ]]; then + if ! timeout "$GIT_TIMEOUT" carryctx import --from-git "$TRACK_REF" \ + --mode replace --yes --dry-run --project "$PROJECT" >"$TMP_ROOT/import.json" 2>"$TMP_ROOT/import.err"; then + cat "$TMP_ROOT/import.err" >&2 2>/dev/null || true + fail "carryctx import --dry-run failed; local DB untouched" + fi + print_provenance + if [[ "$PROJECTS" -ge 1 && "$ROWS" -gt 0 ]]; then + log "dry-run PASS: snapshot fetched + validated; local DB has $ROWS data row(s), a real run needs --force" + else + log "dry-run PASS: snapshot fetched + validated; nothing imported" + fi + exit 0 +fi + +CFG="$PROJECT/.carryctx/config.toml" +CFG_BACKUP="" +if [[ -f "$CFG" ]]; then + CFG_BACKUP="$TMP_ROOT/config.toml.before" + cp -p "$CFG" "$CFG_BACKUP" || fail "cannot back up $CFG" +fi + +# Fresh clone / no project row: initialize CarryCtx state explicitly so the +# documented fresh-clone path never depends on importer auto-init. The +# committed `.carryctx/config.toml` (when present) is restored byte-identically +# below, because `carryctx init` rewrites it with current config defaults. +if [[ "$PROJECTS" -eq 0 ]]; then + log "no project row in local DB: initializing CarryCtx state (carryctx init --non-interactive)" + if ! timeout "$GIT_TIMEOUT" carryctx init --non-interactive --project "$PROJECT" >"$TMP_ROOT/init.log" 2>&1; then + cat "$TMP_ROOT/init.log" >&2 2>/dev/null || true + fail "carryctx init failed" + fi +fi + +if [[ "$PROJECTS" -ge 1 && "$ROWS" -gt 0 ]]; then + warn "--force: replacing $ROWS existing local data row(s) in $PROJECT" +fi + +log "importing snapshot $TRACK_REF into $PROJECT (replace mode)" +if ! timeout "$GIT_TIMEOUT" carryctx import --from-git "$TRACK_REF" \ + --mode replace --yes --project "$PROJECT" >"$TMP_ROOT/import.json" 2>"$TMP_ROOT/import.err"; then + cat "$TMP_ROOT/import.err" >&2 2>/dev/null || true + fail "carryctx import failed; local DB is left in the state carryctx reports above" +fi + +if [[ -n "$CFG_BACKUP" && -f "$CFG" ]] && ! cmp -s "$CFG_BACKUP" "$CFG"; then + cp -p "$CFG_BACKUP" "$CFG" || fail "cannot restore $CFG after import" + log "restored pre-existing .carryctx/config.toml (import rewrites local config defaults)" +fi + +print_provenance +log "restore complete; local counts:" +timeout "$GIT_TIMEOUT" carryctx stats --project "$PROJECT" || fail "carryctx stats failed" diff --git a/scripts/workflow-publish.sh b/scripts/workflow-publish.sh new file mode 100755 index 0000000..e3aa346 --- /dev/null +++ b/scripts/workflow-publish.sh @@ -0,0 +1,155 @@ +#!/usr/bin/env bash +# workflow-publish.sh — publish a redacted CarryCtx snapshot inside this repo. +# +# In-repo publication (no mirror repository): `carryctx export --publication` +# redacts every table row and project.json, stamps manifest.redacted, and +# commits exactly one snapshot to the fixed public ref +# `refs/heads/carryctx-snapshots`. CarryCtx never touches the network, so this +# target pushes that local ref itself — but only when the ref actually +# advanced. Native carryctx commits one snapshot per export, so a re-run +# publishes again rather than no-opping; the guard skips only a ref that did +# not advance. +# +# Trigger: the commander's merge closeout runs `just workflow-publish` from the +# primary checkout (NOT a git hook: GitHub squash-merges never fire local +# hooks, and `carryctx hooks install` behavior is intentionally untouched). +# The companion CI gate (`snapshot-source`) verifies the pushed snapshot's +# `CarryCtx-Source` trailer matches main HEAD. +# +# What it does: +# 1. `carryctx export --pack-format dir -o --publication`, which +# redacts the bundle and commits it to `refs/heads/carryctx-snapshots`. +# 2. Refuses to push unless the committed `manifest.json` is stamped +# `redacted: true`. +# 3. Pushes `refs/heads/carryctx-snapshots` to the remote only when the local +# ref changed; otherwise prints a no-op message. +# +# The local CarryCtx DB is never modified. `--dry-run` validates the export and +# writes neither the ref nor the remote. +# +# Usage: +# scripts/workflow-publish.sh [--dry-run] [--project DIR] [--remote NAME] +# [--git-timeout SECS] [--help] +# --dry-run export + report only; no ref write, no push. +# --project DIR repository to publish (default: this checkout root). +# --remote NAME git remote to push to (default: origin). +# --git-timeout SECS timeout for every carryctx/git op (default: 120). +set -euo pipefail + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +PUB_REF="refs/heads/carryctx-snapshots" +PROJECT="$REPO_ROOT" +REMOTE="${WORKFLOW_REMOTE:-origin}" +GIT_TIMEOUT="${GIT_TIMEOUT:-120}" +DRY_RUN=0 + +usage() { + sed -n '2,/^set -euo/p' "${BASH_SOURCE[0]}" | sed '$d' +} + +while [[ $# -gt 0 ]]; do + case "$1" in + --dry-run) + DRY_RUN=1 + shift + ;; + --project) + PROJECT="${2:?--project requires a directory}" + shift 2 + ;; + --project=*) + PROJECT="${1#--project=}" + shift + ;; + --remote) + REMOTE="${2:?--remote requires a name}" + shift 2 + ;; + --remote=*) + REMOTE="${1#--remote=}" + shift + ;; + --git-timeout) + GIT_TIMEOUT="${2:?--git-timeout requires seconds}" + shift 2 + ;; + --git-timeout=*) + GIT_TIMEOUT="${1#--git-timeout=}" + shift + ;; + --help | -h) + usage + exit 0 + ;; + *) + echo "workflow-publish: FAIL: unknown flag $1 (see --help)" >&2 + exit 2 + ;; + esac +done + +fail() { + echo "workflow-publish: FAIL: $1" >&2 + exit 1 +} + +log() { + echo "workflow-publish: $1" +} + +have() { command -v "$1" >/dev/null 2>&1; } + +have git || fail "git not on PATH" +have carryctx || fail "carryctx not on PATH" +have timeout || fail "timeout not on PATH" + +PROJECT="$(cd "$PROJECT" && pwd)" || fail "project directory $PROJECT not found" + +BRANCH="$(timeout "$GIT_TIMEOUT" git -C "$PROJECT" branch --show-current 2>/dev/null || true)" +if [[ "${BRANCH:-}" != "main" ]]; then + log "WARN: checkout is on branch '${BRANCH:-detached}', not main; snapshot provenance will record that branch" +fi + +TMP_ROOT="$(mktemp -d "${TMPDIR:-/tmp}/workflow-publish.XXXXXX")" +cleanup() { + rm -rf "$TMP_ROOT" +} +trap cleanup EXIT + +BEFORE="$(timeout "$GIT_TIMEOUT" git -C "$PROJECT" rev-parse -q --verify "$PUB_REF" 2>/dev/null || true)" + +EXPORT_ARGS=(export --pack-format dir -o "$TMP_ROOT/pack" --publication --project "$PROJECT") +if [[ "$DRY_RUN" == 1 ]]; then + EXPORT_ARGS+=(--dry-run) +fi + +log "exporting redacted publication (ref $PUB_REF)" +if ! timeout "$GIT_TIMEOUT" carryctx "${EXPORT_ARGS[@]}" >"$TMP_ROOT/export.json" 2>"$TMP_ROOT/export.err"; then + cat "$TMP_ROOT/export.err" >&2 2>/dev/null || true + fail "carryctx export --publication failed" +fi + +if [[ "$DRY_RUN" == 1 ]]; then + log "dry-run PASS: export validated; no ref written, nothing pushed" + exit 0 +fi + +AFTER="$(timeout "$GIT_TIMEOUT" git -C "$PROJECT" rev-parse "$PUB_REF" 2>/dev/null)" || + fail "publication ref $PUB_REF was not created by carryctx export --publication" + +MANIFEST="$(timeout "$GIT_TIMEOUT" git -C "$PROJECT" show "$PUB_REF:manifest.json" 2>/dev/null)" || + fail "cannot read $PUB_REF:manifest.json" +if ! grep -Eq '"redacted"[[:space:]]*:[[:space:]]*true' <<<"$MANIFEST"; then + fail "publication $PUB_REF is not stamped redacted:true; refusing to push" +fi + +if [[ -n "$BEFORE" && "$BEFORE" == "$AFTER" ]]; then + log "publication already current at ${AFTER:0:12}; nothing to push" + exit 0 +fi + +log "pushing $PUB_REF (${BEFORE:0:12} -> ${AFTER:0:12}) to $REMOTE" +if ! timeout "$GIT_TIMEOUT" git -C "$PROJECT" push "$REMOTE" "refs/heads/carryctx-snapshots:refs/heads/carryctx-snapshots"; then + fail "git push failed (network/auth/permissions?); the local publication commit is preserved" +fi +log "published $PUB_REF at ${AFTER:0:12}"