From 0b0ebd2f1e61c7d51b91e6f84902c4a35db7d5be Mon Sep 17 00:00:00 2001 From: nathan nelson Date: Mon, 7 Sep 2026 16:09:43 -0600 Subject: [PATCH 1/2] =?UTF-8?q?feat:=20one=20archive=20for=20Claude=20Code?= =?UTF-8?q?=20and=20Codex=20=E2=80=94=20codex-sync,=20shared-mcp,=20reconc?= =?UTF-8?q?ile=20capture?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Longhand now captures Codex Desktop / CLI threads (the rollouts under ~/.codex/sessions) into the same ~/.longhand archive the Claude Code hooks write, and holds the second client to the same rules as the first: exact records, stable IDs, nothing stored twice, drift never silent. - `longhand codex-sync`: bounded, exact-record capture by default (no vector model); `--semantic` runs the full pipeline. Defaults 50 sessions / 16 MiB / 20k events; `--dry-run`, `--watch`, `--include-subagents`. - `reconcile --fix` captures Codex too, so the scheduled reconciler keeps both clients current with no extra setup. - `longhand shared-mcp`: four read-only keyword tools over the SQLite archive (no Chroma) — the server Codex talks to, and the way Claude searches Codex history by phrase. - Codex adapter with skip sets for UI mirrors / bookkeeping (Claude-parser precedent), subagent (guardian) threads skipped by default, git ops and error detection on the archive path — including `cmd:` literals inside Codex `exec` scripts — and a regression-gated shapes fixture. - doctor: "Codex capture" row, archived-session remedy (`codex-sync --semantic`; `analyze` never embeds events), Codex drift named by kind. - Fixed: CoreML ONNX ingest failures on macOS (shared CPU-pinned model); `frame-link` (Artifact publish link) triaged; `mcp>=1.2.0` floor (FastMCP). Originally drafted in Codex; reviewed, hardened for CI (mypy, ruff format), dogfooded on a live archive and a fresh-venv wheel install. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01LzrMzN4dfuJJKc4UZP6SFb --- CHANGELOG.md | 76 +++ CLAUDE.md | 4 + README.md | 25 +- docs/codex.md | 159 ++++++ longhand/cli/_commands.py | 169 +++++- longhand/codex.py | 504 ++++++++++++++++++ longhand/lightweight_mcp.py | 154 ++++++ longhand/parser.py | 22 +- longhand/recall/reconcile.py | 119 +++-- longhand/setup_commands.py | 107 +++- longhand/storage/store.py | 4 +- longhand/storage/vector_store.py | 17 +- pyproject.toml | 3 +- .../com.longhand.codex-sync.plist.template | 48 ++ tests/conftest.py | 4 + tests/fixtures/codex_shapes/entries.jsonl | 26 + .../fixtures/transcript_shapes/entries.jsonl | 1 + tests/test_codex.py | 463 ++++++++++++++++ tests/test_codex_shapes.py | 176 ++++++ 19 files changed, 2016 insertions(+), 65 deletions(-) create mode 100644 docs/codex.md create mode 100644 longhand/codex.py create mode 100644 longhand/lightweight_mcp.py create mode 100644 scripts/com.longhand.codex-sync.plist.template create mode 100644 tests/fixtures/codex_shapes/entries.jsonl create mode 100644 tests/test_codex.py create mode 100644 tests/test_codex_shapes.py diff --git a/CHANGELOG.md b/CHANGELOG.md index b9aaebd..3d509f5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,82 @@ commits and tag annotations of those releases. --- +## [Unreleased] + +Longhand now keeps one archive for Claude Code **and** Codex. This is the +first release that captures a second client, and it holds that client to the +same rules as the first: exact records, stable IDs, nothing stored twice, +drift never silent. + +### Added + +- **`longhand codex-sync`** captures Codex Desktop and Codex CLI threads (the + rollouts under `~/.codex/sessions`) into the same `~/.longhand` archive the + Claude Code hooks write, under the same ingest lock, with `codex:`-namespaced + session and tool IDs. Exact-record capture by default — every message, + reasoning summary, tool call, and output, verbatim and searchable, without + loading the embedding model; `--semantic` runs the full pipeline so `recall` + and semantic `search` see the sessions. Per run it imports up to 50 sessions + of up to 16 MiB and 20,000 events each; larger ones are reported as + `deferred`, never partially imported. `--dry-run` lists rollouts without + opening the archive; `--watch` polls every 60 seconds. +- **`reconcile --fix` captures Codex too.** New or changed rollouts are picked + up alongside the Claude buckets, so an installed reconciler keeps both + clients current with no extra setup. The report and the `reconcile` MCP + tool gained `codex_*` fields. +- **`longhand shared-mcp`** — a four-tool, read-only MCP server + (`list_sessions` with `source` and `project` filters, `search`, + `get_session_timeline`, `get_event_text`) over the SQLite archive: no + Chroma, no model. Keyword matching, not semantic. It is the server Codex + talks to, and the way Claude searches Codex history by phrase. + `python -m longhand.lightweight_mcp` runs the same server. +- **Commits made from Codex show up in `find_commits` and `git-log`** on the + default capture path — including commands run inside Codex's `exec` + scripts, whose `cmd:` literals are decoded for git and error detection. +- **A Codex shapes gate.** `tests/fixtures/codex_shapes/` holds every rollout + record shape seen in the wild (both Codex CLI generations), and + `tests/test_codex_shapes.py` fails the moment one is undispositioned. Unknown + shapes are preserved as `unknown` events with raw JSON intact, and `doctor`'s + "Transcript format" row names Codex drift by its nested kind + (`event_msg/`). +- **`doctor` knows about Codex.** A "Codex capture" row — only on machines + that have Codex — shows rollouts captured vs. changed since. Archived Codex + sessions are split out of "Sessions needing analysis" with the remedy that + actually works for them: `codex-sync --semantic`. (`analyze --all` was the + old advice; it never embeds events, so it could not have made them + searchable — a Promise 5 defect caught before it shipped.) +- `scripts/com.longhand.codex-sync.plist.template` — a macOS LaunchAgent for + 60-second capture, for anyone who wants faster than the reconciler. + +### Changed + +- **What Codex capture stores.** Codex writes every message twice — a + canonical `response_item` and a UI `event_msg` mirror — plus token + accounting and turn bookkeeping. Only the canonical items are stored; the + mirrors and bookkeeping are skipped (the same rule `KNOWN_SKIP_ENTRY_TYPES` + applies to Claude transcripts), and reasoning without a readable summary is + skipped rather than stored as an empty row. On a real corpus this removed + six of every ten rows without losing a searchable character. +- **Subagent threads are skipped by default.** Codex spawns threads for + itself (its "guardian" approval reviewer) that re-quote the parent + conversation, so capturing them returned every search hit twice. + `--include-subagents` captures them. +- The `mcp` dependency floor is now `>=1.2.0`, the first release with FastMCP. + +### Fixed + +- **Embedding no longer takes the CoreML path on macOS.** Every Chroma + collection now shares one `ONNXMiniLM_L6_V2` pinned to the CPU execution + provider, instead of each collection loading its own default model. The + CoreML build is the `Failed to create MLModel … .mlmodelc` ingest failure + that `doctor` had been surfacing from `hook-errors-*.log`. +- **`frame-link` is dispositioned.** Claude Code's new entry for a published + page — the Artifact URL plus the local `.html` it came from — is preserved + as a triaged unknown (recallable, so a promotion candidate beside `pr-link`) + instead of tripping the drift row on every `doctor` run. + +--- + ## [1.0.1] — 2026-08-12 Three `doctor` rows told you the wrong thing. Found by dogfooding 1.0.0 on a diff --git a/CLAUDE.md b/CLAUDE.md index 1479c7a..58e8d19 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -52,3 +52,7 @@ When a user asks about past work: ## Deeper Tools (less common starting points) Beyond the decision tree above: `get_session_timeline` with `tail` (the last N events, replaces get_latest_events), `find_episodes` with `episode_id` (full detail: referenced events, diff, post-fix file state), `list_projects` with `match` (fuzzy candidates with scored reasons — "which project did you mean?"), `list_plans` (browse plan-file writes), `get_stats` (store health), and `reconcile` (re-ingest drift) — **`reconcile` defaults to a dry run; pass `fix=true` to actually heal.** + +## Codex sessions (1.1.0+) + +Codex Desktop / CLI threads live in the same archive with `codex:`-prefixed session ids. This server lists and pages them (`list_sessions`, `get_session_timeline`) and `find_commits` sees commits made from Codex — but `recall` and semantic `search` only see a Codex session after `longhand codex-sync --semantic` has indexed it. For "what did I do in Codex" questions, use the `longhand-shared` server's keyword search (literal phrases; its session listing takes source="codex"), or tell the user to run the semantic sync. diff --git a/README.md b/README.md index 973cf5e..2d83122 100644 --- a/README.md +++ b/README.md @@ -2,11 +2,13 @@ # Longhand +**Using Codex too?** [Share one memory archive between Claude Code and Codex](docs/codex.md). + [![Longhand MCP server](https://glama.ai/mcp/servers/Wynelson94/longhand/badges/score.svg)](https://glama.ai/mcp/servers/Wynelson94/longhand) [![PyPI version](https://img.shields.io/pypi/v/longhand?label=PyPI&color=blue)](https://pypi.org/project/longhand/) ![Python](https://img.shields.io/badge/python-3.10+-blue) ![License](https://img.shields.io/badge/license-MIT-green) -![Tests](https://img.shields.io/badge/tests-546%20passing-brightgreen) +![Tests](https://img.shields.io/badge/tests-580%20passing-brightgreen) ![Local](https://img.shields.io/badge/100%25-local-informational) [![SafeSkill 93/100](https://img.shields.io/badge/SafeSkill-93%2F100_Verified%20Safe-brightgreen)](https://safeskill.dev/scan/wynelson94-longhand) @@ -85,7 +87,7 @@ longhand analyze --all # fill in episodes + vectors whenever, safe to Exact-text search, timelines, file history, and commit lookup all work after `--skip-analysis`. Semantic `recall` needs the `analyze --all` pass to complete. Typical throughput on an M-class Mac is ~1–2 sessions/sec for full analysis. -> *Status: v1.0.1 — stable, daily-driver tested, security-audited (zero critical findings), on PyPI, available as a Claude Code plugin. Validated against 433 real Claude Code sessions across 37 inferred projects (measured 2026-08-12). 546 unit tests passing.* +> *Status: v1.0.1 — stable, daily-driver tested, security-audited (zero critical findings), on PyPI, available as a Claude Code plugin. Validated against 433 real Claude Code sessions across 37 inferred projects (measured 2026-08-12). 580 unit tests passing.* **Full docs:** [Longhand Wiki](https://github.com/Wynelson94/longhand/wiki) — getting started, CLI reference, MCP tools reference, architecture, and troubleshooting. @@ -127,6 +129,8 @@ Longhand pins `chromadb<1.0` for **every** Python version, not just 3.14. The pi **Windows: CI-tested, best-effort.** A `windows-latest × py3.12` leg runs on every PR and has gone green on every run since v0.13.0, but it is non-blocking and covers one Python version on GitHub's runners. That is honest evidence, not a support tier — Linux and macOS are the tested platforms. Windows bugs are welcome as issues; they just aren't release-blocking. +**Codex Desktop and Codex CLI** threads are captured into the same archive from 1.1.0 — see [Works with Codex](#works-with-codex). + --- ## Compatibility @@ -433,6 +437,21 @@ Both are non-blocking and run in one to two seconds. You don't have to think abo --- +## Works with Codex + +Use Codex Desktop or the Codex CLI too? Longhand captures those threads into the same archive, so a question asked in Claude Code can be answered from work done in Codex, and the other way round. + +```sh +pip install -U longhand +longhand codex-sync # capture every Codex thread on this machine +claude mcp add --scope user longhand-shared -- longhand shared-mcp # keyword search across both clients, from Claude +codex mcp add longhand -- longhand shared-mcp # the same server from Codex (Desktop: config.toml, see the docs) +``` + +From then on `reconcile --fix` captures new Codex threads too, so the scheduled reconciler keeps both clients current. Codex sessions are stored exact-record-only — verbatim and searchable, no model loaded — until `longhand codex-sync --semantic` makes them recallable. Threads Codex spawns for itself are skipped, UI mirrors are never stored twice, and unknown record shapes surface in `doctor` like any other drift. Setup, bounds, and the macOS launchd template: **[docs/codex.md](docs/codex.md)**. + +--- + ## Architecture ``` @@ -528,7 +547,7 @@ Longhand is flat-cost: the cap is per-call, not per-corpus. Recalling across 10 --- -546 unit tests passing. All 13 MCP tools stress-tested. Full security audit: zero critical findings, zero high findings. `~/.longhand/` created with 0700 permissions, all SQL parameterized, all inputs bounded. Dependencies: chromadb, typer, rich, pydantic, mcp. +580 unit tests passing. All 13 MCP tools stress-tested. Full security audit: zero critical findings, zero high findings. `~/.longhand/` created with 0700 permissions, all SQL parameterized, all inputs bounded. Dependencies: chromadb, typer, rich, pydantic, mcp. --- diff --git a/docs/codex.md b/docs/codex.md new file mode 100644 index 0000000..d190497 --- /dev/null +++ b/docs/codex.md @@ -0,0 +1,159 @@ +# Shared memory for Claude Code and Codex + +Longhand keeps one archive for both clients. Claude Code sessions arrive +through the hooks you already have; Codex Desktop and Codex CLI threads (the +"rollouts" under `~/.codex/sessions`) arrive through `longhand codex-sync`. +Both land in `~/.longhand`, with Codex sessions namespaced `codex:` +so nothing collides. A question asked in either client can be answered from +work done in the other. + +Requires Longhand 1.1.0 or newer. + +## Set up in three commands + +```sh +pip install -U longhand +longhand codex-sync # capture every Codex thread on this machine +longhand doctor # the "Codex capture" row confirms it +``` + +Then connect the shared keyword server to each client: + +```sh +# Claude Code — alongside the existing `longhand` server +claude mcp add --scope user longhand-shared -- longhand shared-mcp + +# Codex CLI +codex mcp add longhand -- longhand shared-mcp +``` + +Codex Desktop does not put `codex` on your PATH. Add the server to +`~/.codex/config.toml` instead and restart the app: + +```toml +[mcp_servers.longhand] +command = "longhand" +args = ["shared-mcp"] + +# Only if your archive is not at ~/.longhand: +[mcp_servers.longhand.env] +LONGHAND_DATA_DIR = "/Users/you/.longhand" +``` + +Use an absolute path to `longhand` (`which longhand`) when the desktop app's +PATH differs from your terminal's. If you relocated the archive with +`LONGHAND_DATA_DIR`, set the same value for both clients and for the capture +commands — one archive is the whole point. + +## What each client sees + +| | Claude Code `longhand` server | `longhand shared-mcp` | +| ------------------------------------------------- | ----------------------------- | -------------------------- | +| Lists and pages Codex sessions | yes | yes | +| Keyword search across both clients | no | yes (`search`, literal) | +| `recall` and semantic search over Codex sessions | after `codex-sync --semantic` | no | +| Commits made in Codex | yes (`find_commits`) | through `search` | +| Loads the embedding model | yes | never | + +The shared server exposes four read-only tools: `list_sessions` (with +`source="codex"|"claude"` and `project` substring filters), `search`, +`get_session_timeline`, and `get_event_text`. Search matches literal phrases, +not meaning. It reads the same SQLite rows both clients write, never opens +Chroma, and never loads a model; long texts and raw records page in +8,000-character slices. A broad query on a large archive can hit the server's +instruction budget — narrow it to a session. + +Codex sessions are captured exact-record-only by default (ingestion stage +`archived`): every message, reasoning summary, tool call, and output is stored +verbatim and searchable, but not embedded. `longhand codex-sync --semantic` +runs the full pipeline — embeddings, episodes, project inference — on captured +sessions so `recall` and semantic `search` see them. It is the remedy `doctor` +names for archived sessions; `analyze` is not, because it never embeds events. + +## Keeping capture current + +`longhand reconcile --fix` captures new or changed Codex rollouts along with +everything it already does for Claude transcripts, so the scheduled reconciler +(`longhand schedule install-reconciler`, every 30 minutes on macOS) keeps Codex +current with no extra setup. For an immediate capture run `longhand codex-sync`; +for a foreground loop run `longhand codex-sync --watch` (every 60 seconds until +interrupted). + +### Faster capture with launchd (macOS) + +`scripts/com.longhand.codex-sync.plist.template` is a ready-made user +LaunchAgent that runs `codex-sync` at login and every 60 seconds. Fill in the +interpreter that has Longhand installed and your home directory, then load it: + +```sh +PY="$(command -v python3)" # must be the Python that has longhand installed +mkdir -p ~/.longhand/logs +sed "s|__PYTHON__|$PY|g; s|__HOME__|$HOME|g" \ + scripts/com.longhand.codex-sync.plist.template \ + > ~/Library/LaunchAgents/com.longhand.codex-sync.plist +launchctl bootstrap "gui/$(id -u)" ~/Library/LaunchAgents/com.longhand.codex-sync.plist +``` + +The job exits between scans, so `state = not running` with `last exit code = 0` +is normal. Inspect it and its capture reports with: + +```sh +launchctl print "gui/$(id -u)/com.longhand.codex-sync" +tail -n 5 ~/.longhand/logs/codex-sync.log +``` + +launchd does not read your shell profile, so a relocated archive needs +`LONGHAND_DATA_DIR` in the plist's `EnvironmentVariables`. To stop capture +without deleting any archived history, `launchctl bootout "gui/$(id -u)"` the +plist and remove it. + +On Linux, run `--watch` in a persistent terminal or schedule `longhand +codex-sync` with a systemd timer or cron. On Windows, `--watch` or Task +Scheduler. + +## What is captured, and what is not + +- **Threads you drove.** Codex also spawns threads for itself — its "guardian" + approval reviewer, for one — that re-quote the parent thread. `codex-sync` + skips them by default so every search hit appears once; `--include-subagents` + captures them. +- **Canonical items only.** Codex writes every message twice: once as a + canonical `response_item` and once as a UI `event_msg` mirror. The mirrors, + token accounting, and turn bookkeeping are skipped so nothing is stored + twice. Reasoning is stored when Codex provides a readable summary; encrypted + reasoning has no readable content and is skipped. +- **Bounds.** Per run: up to 50 sessions, each up to 16 MiB and 20,000 events. + Larger rollouts are reported as `deferred`, never partially imported. Raise + the bounds with `--limit`, `--max-file-kb`, and `--max-events`. These are + input bounds, not a memory ceiling — the default capture never loads a model. +- **Drift is never silent.** A record shape Longhand does not recognize is + preserved as an `unknown` event with its raw JSON intact, and surfaces in + `longhand doctor`'s "Transcript format" row as `response_item/` or + `event_msg/`. `tests/fixtures/codex_shapes/` regression-gates every + known shape. +- **Shell commands are understood; scripts are text.** Commands run through + Codex's shell tools, and the `cmd:` literals inside its `exec` scripts, feed + error detection and git extraction, so commits made from Codex show up in + `find_commits` and `longhand git-log`. Patches and the rest of a script stay + recorded text — they are not translated into Claude-style file replay. +- **Redaction and locks apply.** Opt-in secret redaction covers Codex records. + Capture runs under the same ingest lock as the hooks; a Claude hook that + fires while a capture holds the lock skips that turn, and the reconciler + heals it. + +This is shared, retrievable history — not a transfer of a model's live +context. Only locally saved rollouts are available. + +## Verify + +```sh +python3 -m pytest tests/test_codex.py tests/test_codex_shapes.py -q +longhand codex-sync --dry-run +``` + +The tests use synthetic records and a sanitized fixture of real rollout +shapes; they cover cross-client storage and keyword retrieval, stable IDs, +redaction, subagent skipping, the skip rules, git extraction, bounded capture, +and reconcile's capture path. + +Codex MCP configuration: [official documentation](https://learn.chatgpt.com/docs/extend/mcp). diff --git a/longhand/cli/_commands.py b/longhand/cli/_commands.py index ae34eb3..054a6bd 100644 --- a/longhand/cli/_commands.py +++ b/longhand/cli/_commands.py @@ -32,6 +32,7 @@ _resolve_prefix, console, ) +from longhand.codex import DEFAULT_MAX_EVENTS, DEFAULT_MAX_FILE_BYTES, DEFAULT_SESSION_LIMIT from longhand.parser import JSONLParser, discover_sessions from longhand.recall import recall as recall_pipeline from longhand.replay import ReplayEngine @@ -300,6 +301,98 @@ def demo( # ----------------------------------------------------------------------------- +@app.command("codex-sync", rich_help_panel="Data") +def codex_sync_cmd( + semantic: bool = typer.Option( + False, + "--semantic", + help="Run the full pipeline (embeddings, episodes, project inference) so recall and " + "search see these sessions; the default captures exact records only", + ), + limit: int = typer.Option( + DEFAULT_SESSION_LIMIT, "--limit", min=1, help="Maximum sessions imported per scan" + ), + max_file_kb: int = typer.Option( + DEFAULT_MAX_FILE_BYTES // 1024, + "--max-file-kb", + min=1, + help="Defer rollouts larger than this many KiB", + ), + max_events: int = typer.Option( + DEFAULT_MAX_EVENTS, "--max-events", min=1, help="Defer sessions with more events than this" + ), + include_subagents: bool = typer.Option( + False, + "--include-subagents", + help="Also capture threads Codex spawned for itself (e.g. its approval reviewer); " + "skipped by default because they re-quote the parent thread", + ), + dry_run: bool = typer.Option( + False, + "--dry-run", + help="List rollouts and sizes without opening the archive or loading embeddings", + ), + codex_home: str | None = typer.Option( + None, "--codex-home", help="Codex home (defaults to CODEX_HOME or ~/.codex)" + ), + data_dir: str | None = typer.Option(None, "--data-dir", help="Shared Longhand archive"), + watch: bool = typer.Option( + False, "--watch", help="Keep importing changed Codex sessions until interrupted" + ), + interval: int = typer.Option( + 60, "--interval", min=10, help="Seconds between scans with --watch" + ), +): + """Capture Codex history into the same archive Claude uses. + + Exact records only by default, with no vector model loaded. `reconcile + --fix` runs this capture too, so a scheduled reconciler keeps Codex + current on its own; run this directly for an immediate capture, or with + --watch for a 60-second loop. + """ + import json + import time + + from longhand.codex import ( + CodexArchiveStore, + discover_codex_sessions, + is_subagent_rollout, + read_session_meta, + sync_codex, + ) + + if dry_run: + for path in discover_codex_sessions(codex_home): + size = path.stat().st_size + console.print( + json.dumps( + { + "path": str(path), + "bytes": size, + "within_size_limit": size <= max_file_kb * 1024, + "subagent": is_subagent_rollout(read_session_meta(path)), + } + ) + ) + return + store = _get_store(data_dir) if semantic else CodexArchiveStore(data_dir) + while True: + report = sync_codex( + store, + codex_home, + limit=limit, + max_file_bytes=max_file_kb * 1024, + max_events=max_events, + include_subagents=include_subagents, + ) + console.print(json.dumps(report)) + if not watch: + if report["errors"]: + raise typer.Exit(1) + return + time.sleep(interval) + + @app.command(rich_help_panel="Data") def ingest( path: str | None = typer.Argument( @@ -481,26 +574,53 @@ def reconcile( store = _get_store(data_dir) report = run_reconcile(store, fix=fix) - if report.files_on_disk == 0: + if report.files_on_disk == 0 and report.codex_on_disk == 0: console.print("[yellow]No session files found on disk.[/yellow]") - console.print("Default location: ~/.claude/projects") + console.print("Default location: ~/.claude/projects (Codex: ~/.codex/sessions)") return - console.print(f"[bold]On disk:[/bold] {report.files_on_disk} JSONL files") - console.print(f" [green]{report.fully_indexed}[/green] fully indexed") - console.print( - f" [yellow]{len(report.partially_indexed)}[/yellow] partially indexed (ingest crashed mid-pipeline)" - ) - console.print(f" [yellow]{len(report.null_project)}[/yellow] ingested but project_id IS NULL") - console.print(f" [red]{len(report.missing)}[/red] missing from sessions") - if report.skipped_oversize: + if report.files_on_disk: + console.print(f"[bold]On disk:[/bold] {report.files_on_disk} JSONL files") + console.print(f" [green]{report.fully_indexed}[/green] fully indexed") console.print( - f" [dim]{len(report.skipped_oversize)} skipped — over the parser size cap " - "(not ingestable by any path)[/dim]" + f" [yellow]{len(report.partially_indexed)}[/yellow] partially indexed (ingest crashed mid-pipeline)" + ) + console.print( + f" [yellow]{len(report.null_project)}[/yellow] ingested but project_id IS NULL" + ) + console.print(f" [red]{len(report.missing)}[/red] missing from sessions") + if report.skipped_oversize: + console.print( + f" [dim]{len(report.skipped_oversize)} skipped — over the parser size cap " + "(not ingestable by any path)[/dim]" + ) + else: + console.print("[dim]No Claude Code session files on disk.[/dim]") + + if report.codex_on_disk: + current = ( + report.codex_on_disk + - report.codex_pending + - report.codex_skipped_subagents + - report.codex_oversize ) + console.print(f"[bold]Codex rollouts:[/bold] {report.codex_on_disk} on disk") + console.print(f" [green]{current}[/green] captured and current") + console.print(f" [yellow]{report.codex_pending}[/yellow] new or changed since capture") + if report.codex_skipped_subagents: + console.print( + f" [dim]{report.codex_skipped_subagents} subagent thread(s) skipped " + "(longhand codex-sync --include-subagents to capture them)[/dim]" + ) + if report.codex_oversize: + console.print( + f" [dim]{report.codex_oversize} over the capture size bound " + "(longhand codex-sync --max-file-kb to raise it)[/dim]" + ) + fixable_claude = bool(report.missing or report.null_project or report.partially_indexed) if not fix: - if report.missing or report.null_project or report.partially_indexed: + if fixable_claude or report.codex_pending: console.print("\n[dim]Run with --fix to re-ingest.[/dim]") return @@ -508,17 +628,23 @@ def reconcile( console.print("[yellow]Another ingest is running — aborting reconcile.[/yellow]") raise typer.Exit(1) - if not (report.missing or report.null_project or report.partially_indexed): + if not fixable_claude and not report.codex_pending: console.print("\n[green]Nothing to fix.[/green]") return n_fixable = len(report.missing) + len(report.null_project) + len(report.partially_indexed) - console.print(f"\n[cyan]Re-ingesting {n_fixable} file(s)...[/cyan]") + if n_fixable: + console.print(f"\n[cyan]Re-ingesting {n_fixable} file(s)...[/cyan]") for err in report.errors: console.print(f" [red]✗[/red] {Path(err['path']).name}: {err['error']}") console.print( f"\n[bold]Re-ingested {report.ingested}[/bold] ([dim]{len(report.errors)} errors[/dim])" ) + if report.codex_on_disk: + deferred = ( + f" ([dim]{len(report.codex_deferred)} deferred[/dim])" if report.codex_deferred else "" + ) + console.print(f"[bold]Captured {report.codex_ingested}[/bold] Codex rollout(s){deferred}") # ----------------------------------------------------------------------------- @@ -2544,6 +2670,19 @@ def mcp_server_cmd(): _run_mcp_server() +@app.command("shared-mcp", rich_help_panel="Plumbing") +def shared_mcp_cmd(): + """Run the shared-archive MCP server (stdio): keyword search over the archive + both Claude and Codex write, with no vector model loaded. + + Register it as `longhand-shared` in Claude Code and as `longhand` in Codex — + see docs/codex.md. + """ + from longhand.lightweight_mcp import mcp + + mcp.run() + + @app.command("ingest-session", hidden=True) def ingest_session_cmd( transcript: str | None = typer.Option( diff --git a/longhand/codex.py b/longhand/codex.py new file mode 100644 index 0000000..6dc1e67 --- /dev/null +++ b/longhand/codex.py @@ -0,0 +1,504 @@ +"""Codex rollout adapter: the same archive, the same rules, a second client. + +Codex Desktop writes one JSONL "rollout" per thread under ``~/.codex/sessions``. +This module turns those records into Longhand events the way ``parser.py`` +turns Claude Code transcripts into events — same Event model, same ID +discipline, same drift promise: + +* ``response_item`` records are the canonical conversation (messages, + reasoning summaries, tool calls and their outputs) and become events. +* ``event_msg`` records are Codex's UI stream. Every kind we know either + mirrors a canonical item or is turn bookkeeping, so known kinds are skipped + (a message is never stored twice) and UNKNOWN kinds are preserved as + ``unknown`` events so ``longhand doctor`` can surface them. +* ``session_meta`` / ``turn_context`` carry cwd, model, and branch; they are + stored as metadata ``system`` rows. +* Encrypted reasoning is opaque by design: a reasoning item with a summary + becomes a thinking event; one without a summary is skipped, because + ciphertext has no recall value. + +Every stored event keeps its raw record. Session and tool IDs live in a +``codex:`` namespace so nothing collides with Claude records. +""" + +from __future__ import annotations + +import json +import os +import re +from dataclasses import dataclass, field +from itertools import islice +from pathlib import Path +from typing import TYPE_CHECKING, Any + +from longhand.extractors.errors import detect_error +from longhand.extractors.git import extract_git_signal +from longhand.types import Event, EventType + +if TYPE_CHECKING: + from longhand.storage.sqlite_store import SQLiteStore + from longhand.types import Session + +# Capture bounds. These are input bounds, not a memory ceiling: the default +# SQLite-only capture never loads the embedding model. Sized from real Codex +# Desktop rollouts, where 1–4 MB and a few hundred items per thread is normal. +DEFAULT_SESSION_LIMIT = 50 +DEFAULT_MAX_FILE_BYTES = 16 * 1024 * 1024 +DEFAULT_MAX_EVENTS = 20_000 + +# Top-level record types with nothing recallable in them — usage accounting +# and world-state snapshots. Skipped, not stored: the Claude parser's +# KNOWN_SKIP_ENTRY_TYPES rule. Every member needs a fixture line in +# tests/fixtures/codex_shapes/entries.jsonl. +CODEX_SKIP_RECORD_TYPES = frozenset({"token_usage_record", "world_state"}) + +# `event_msg` kinds that mirror something the canonical `response_item` +# stream already provides (user_message / agent_message / agent_reasoning +# duplicate messages and reasoning summaries; exec_command_end and +# patch_apply_end duplicate tool outputs) or are turn bookkeeping with no +# content (task_started, task_complete, item_completed, token_count, +# thread_settings_applied). Skipped so nothing lands twice. A kind that is +# NOT in this set is preserved as an unknown event — drift is never silent. +CODEX_SKIP_EVENT_MSG_TYPES = frozenset( + { + "agent_message", + "agent_reasoning", + "exec_command_end", + "item_completed", + "patch_apply_end", + "task_complete", + "task_started", + "thread_settings_applied", + "token_count", + "user_message", + } +) + +# Record types the adapter routes: into events, into session metadata, or to +# a skip decision. +CODEX_HANDLED_RECORD_TYPES = frozenset( + {"session_meta", "turn_context", "response_item", "event_msg"} +) +# `response_item` kinds the adapter turns into events. +CODEX_HANDLED_ITEM_TYPES = frozenset( + { + "message", + "reasoning", + "function_call", + "function_call_output", + "custom_tool_call", + "custom_tool_call_output", + } +) + +# `exec` is Codex Desktop's scripting tool: a JS snippet that calls +# tools.exec_command({cmd: "..."}) one or more times. The command literals +# are pulled out so error suppression and git detection see the real command. +_JS_CMD_LITERAL = re.compile(r"""\bcmd\s*:\s*("(?:[^"\\]|\\.)*"|'(?:[^'\\]|\\.)*')""") + + +def discover_codex_sessions(codex_home: str | Path | None = None) -> list[Path]: + """Every rollout under the Codex home (`CODEX_HOME`, default `~/.codex`).""" + root = Path(codex_home or os.environ.get("CODEX_HOME") or Path.home() / ".codex").expanduser() + found: set[Path] = set() + for name in ("sessions", "archived_sessions"): + folder = root / name + if folder.is_dir(): + found.update(folder.rglob("*.jsonl")) + return sorted(found) + + +def read_session_meta(path: Path) -> dict[str, Any] | None: + """The `session_meta` payload from a rollout's first line, or None. + + Reads one bounded line, so it is cheap enough to call on every rollout + during discovery. + """ + try: + with path.open("r", encoding="utf-8", errors="replace") as f: + line = f.readline(1024 * 1024) + entry = json.loads(line) + except (OSError, ValueError): + return None + if not isinstance(entry, dict) or entry.get("type") != "session_meta": + return None + payload = entry.get("payload") + return payload if isinstance(payload, dict) else None + + +def is_subagent_rollout(meta: dict[str, Any] | None) -> bool: + """Whether a rollout is a thread Codex spawned for itself. + + Codex marks those — e.g. the "guardian" approval reviewer — with a + structured `source` ({"subagent": {...}}) where a user-driven thread has + the client name ("vscode", "cli"). They re-quote the parent thread, so + capturing them by default would store every conversation twice. + """ + if not meta: + return False + source = meta.get("source") + return isinstance(source, dict) and "subagent" in source + + +def shell_commands_from_tool_input(tool_input: dict[str, Any] | None) -> str: + """The shell command(s) a Codex tool call ran, newline-joined; "" if none. + + Direct shell tools carry `cmd` (a string or an argv list). The `exec` + scripting tool embeds `cmd: "..."` literals inside JS, which are decoded + with JSON escaping rules (the common subset of JS string escapes). + """ + if not tool_input: + return "" + command = tool_input.get("cmd", tool_input.get("command")) + if isinstance(command, list): + return " ".join(str(part) for part in command) + if isinstance(command, str): + return command + script = tool_input.get("input") + if not isinstance(script, str): + return "" + commands: list[str] = [] + for match in _JS_CMD_LITERAL.finditer(script): + literal = match.group(1) + inner = literal[1:-1] + try: + if literal[0] == '"': + commands.append(json.loads(literal)) + else: + rewrapped = inner.replace("\\'", "'").replace('"', '\\"') + commands.append(json.loads(f'"{rewrapped}"')) + except ValueError: + commands.append(inner) + return "\n".join(commands) + + +def _text(content: Any) -> str: + if isinstance(content, str): + return content + if not isinstance(content, list): + return "" + parts = [] + for block in content: + if not isinstance(block, dict): + continue + if block.get("type") in {"input_text", "output_text", "text", "summary_text"}: + if isinstance(block.get("text"), str): + parts.append(block["text"]) + elif block.get("type") in {"input_image", "image"}: + parts.append("[image]") + return "\n".join(parts) + + +def disposition(entry: dict[str, Any]) -> str: + """How the adapter treats one raw record: "handled", "skipped", or "unknown". + + tests/test_codex_shapes.py asserts every fixture line is handled or + skipped; "unknown" is what doctor's drift row reports on a live archive. + """ + record_type = entry.get("type") + payload = entry.get("payload") + kind = payload.get("type") if isinstance(payload, dict) else None + if record_type in CODEX_SKIP_RECORD_TYPES: + return "skipped" + if record_type == "event_msg": + return "skipped" if kind in CODEX_SKIP_EVENT_MSG_TYPES else "unknown" + if record_type == "response_item": + return "handled" if kind in CODEX_HANDLED_ITEM_TYPES else "unknown" + if record_type in CODEX_HANDLED_RECORD_TYPES: + return "handled" + return "unknown" + + +class CodexAdapter: + """Stateful conversion of one complete rollout, in append order.""" + + def __init__(self, metadata: dict[str, Any]): + identity = metadata.get("id") or metadata.get("session_id") + if not isinstance(identity, str) or not identity: + raise ValueError("Codex session_meta is missing its session id") + self.session_id = f"codex:{identity}" + self.cwd = metadata.get("cwd") + git = metadata.get("git") or {} + self.branch = git.get("branch") if isinstance(git, dict) else None + self.model = None + self.calls: dict[str, Event] = {} + + def convert(self, entry: dict[str, Any], sequence: int) -> list[Event]: + from longhand.parser import _parse_timestamp + + record_type = entry.get("type") + payload = entry.get("payload") + payload = payload if isinstance(payload, dict) else {} + kind = payload.get("type") + + if record_type in CODEX_SKIP_RECORD_TYPES: + return [] + if record_type == "event_msg" and kind in CODEX_SKIP_EVENT_MSG_TYPES: + return [] + if record_type == "turn_context": + self.cwd = payload.get("cwd", self.cwd) + self.model = payload.get("model", self.model) + + event = Event( + event_id=f"{self.session_id}:{sequence}", + session_id=self.session_id, + parent_event_id=None, + event_type=EventType.SYSTEM, + sequence=sequence, + timestamp=_parse_timestamp(entry.get("timestamp")), + cwd=self.cwd, + git_branch=self.branch, + model=self.model, + raw=entry, + ) + if record_type in ("session_meta", "turn_context"): + return [event] + if record_type != "response_item" or kind not in CODEX_HANDLED_ITEM_TYPES: + # An undispositioned shape: keep it, raw intact, where doctor's + # drift row can see it — the same path the Claude parser takes. + event.event_type = EventType.UNKNOWN + event.content = json.dumps(entry)[:500] + return [event] + + if kind == "message": + role = payload.get("role") + if role == "user": + event.event_type = EventType.USER_MESSAGE + elif role == "assistant": + event.event_type = EventType.ASSISTANT_TEXT + # Instructions (developer/system roles) stay in raw, rather than + # polluting recall results. + if event.event_type != EventType.SYSTEM: + event.content = _text(payload.get("content")) + elif kind == "reasoning": + summary = _text(payload.get("summary")) + if not summary: + return [] # encrypted-only reasoning: nothing readable to keep + event.event_type = EventType.ASSISTANT_THINKING + event.content = summary + elif kind in {"function_call", "custom_tool_call"}: + event.event_type = EventType.TOOL_CALL + event.tool_name = payload.get("name") + call_id = payload.get("call_id") + event.tool_use_id = f"{self.session_id}:{call_id}" if call_id else None + tool_input: dict[str, Any] + if kind == "function_call": + arguments = payload.get("arguments", {}) + try: + arguments = json.loads(arguments) if isinstance(arguments, str) else arguments + except ValueError: + arguments = {"arguments": arguments} + tool_input = arguments if isinstance(arguments, dict) else {"arguments": arguments} + else: + tool_input = {"input": payload.get("input", "")} + # When the shell command had to be dug out of an `exec` script, + # surface it under the key the rest of Longhand reads, so git-ops + # re-extraction and error suppression see it too. + command = shell_commands_from_tool_input(tool_input) + if command and "cmd" not in tool_input and "command" not in tool_input: + tool_input["command"] = command + event.tool_input = tool_input + event.content = f"Tool: {event.tool_name}\n" + json.dumps( + tool_input, ensure_ascii=False + ) + if event.tool_use_id: + self.calls[event.tool_use_id] = event + else: # function_call_output / custom_tool_call_output + event.event_type = EventType.TOOL_RESULT + call_id = payload.get("call_id") + event.tool_use_id = f"{self.session_id}:{call_id}" if call_id else None + output = payload.get("output", "") + event.content = ( + output if isinstance(output, str) else json.dumps(output, ensure_ascii=False) + ) + event.tool_output = event.content + call = self.calls.get(event.tool_use_id or "") + if call: + event.parent_event_id = call.event_id + event.tool_name = call.tool_name + # Same gate as the Claude parser: signals come from + # command-executing calls, with the command as context so + # probe/search noise is suppressed at the source. + command = shell_commands_from_tool_input(call.tool_input) + if command: + signal = detect_error(event.content, tool_name=call.tool_name, command=command) + if signal: + event.error_detected = True + event.error_snippet = signal.snippet + event.error_category = signal.category + event.error_severity = signal.severity + git = extract_git_signal(command, event.content) + if git: + event.git_operation = git.operation_type + event.git_commit_hash = git.commit_hash + event.git_commit_message = git.commit_message + return [event] + + +class CodexArchiveStore: + """Exact-record capture into the shared SQLite archive — no Chroma, no model. + + Stage `archived` marks a session whose raw capture is complete and whose + semantic analysis has deliberately not run; `longhand codex-sync + --semantic` upgrades it in place. + """ + + completed_stage = "archived" + + def __init__( + self, data_dir: str | Path | None = None, sqlite: SQLiteStore | None = None + ) -> None: + from longhand.storage.sqlite_store import SQLiteStore + from longhand.storage.store import resolve_data_dir + + self.data_dir = resolve_data_dir(data_dir) + self.data_dir.mkdir(parents=True, exist_ok=True, mode=0o700) + self.sqlite = sqlite if sqlite is not None else SQLiteStore(self.data_dir / "longhand.db") + + def ingest_session(self, session: Session, events: list[Event]) -> None: + from longhand.storage.store import LonghandStore + + self.sqlite.mark_ingest_started(session.transcript_path, session.session_id) + self.sqlite.upsert_session(session) + self.sqlite.insert_events(events) + self.sqlite.upsert_tool_pairs(self.sqlite.build_tool_pairs_from_events(events)) + # Commits made from Codex show up in find_commits / git-log on this + # path too: the extraction is pure text work, no vectors involved. + self.sqlite.insert_git_operations( + LonghandStore._extract_git_operations(session.session_id, events) + ) + self.sqlite.log_ingestion( + session.transcript_path, + session.session_id, + Path(session.transcript_path).stat().st_size, + len(events), + ) + self.sqlite.set_analysis_stage(session.transcript_path, self.completed_stage) + + +@dataclass +class CodexScan: + """Discovery result: which rollouts need capture, and why the rest don't.""" + + candidates: list[Path] = field(default_factory=list) # new or changed, within bounds + unchanged: list[Path] = field(default_factory=list) # captured at this exact size + subagents: list[Path] = field(default_factory=list) # skipped unless include_subagents + oversize: list[Path] = field(default_factory=list) # over max_file_bytes + + @property + def on_disk(self) -> int: + return len(self.candidates) + len(self.unchanged) + len(self.subagents) + len(self.oversize) + + +def scan_codex_sessions( + sqlite: SQLiteStore, + codex_home: str | Path | None = None, + *, + max_file_bytes: int = DEFAULT_MAX_FILE_BYTES, + include_subagents: bool = False, + complete_stages: tuple[str, ...] = ("archived", "analyzed"), +) -> CodexScan: + """Classify every rollout on disk against the archive without parsing any.""" + scan = CodexScan() + stages = sqlite.analysis_stages() + for path in discover_codex_sessions(codex_home): + try: + size = path.stat().st_size + except OSError: + continue + key = str(path) + if sqlite.already_ingested(key, size) and stages.get(key) in complete_stages: + scan.unchanged.append(path) + elif not include_subagents and is_subagent_rollout(read_session_meta(path)): + scan.subagents.append(path) + elif size > max_file_bytes: + scan.oversize.append(path) + else: + scan.candidates.append(path) + return scan + + +def sync_codex( + store: Any, + codex_home: str | Path | None = None, + *, + limit: int = DEFAULT_SESSION_LIMIT, + max_file_bytes: int = DEFAULT_MAX_FILE_BYTES, + max_events: int = DEFAULT_MAX_EVENTS, + include_subagents: bool = False, + claim_lock: bool = True, +) -> dict[str, Any]: + """Import new or changed rollouts into the archive. + + `store` is a CodexArchiveStore (exact records only) or a LonghandStore + (`--semantic`: the full pipeline, vectors included). Runs under Longhand's + ingest lock; pass `claim_lock=False` when the caller already holds it, as + reconcile does. This is deliberately a separate discovery path: Codex + support never changes which files Claude's hooks scan. + + Report keys: ingested, skipped (unchanged), skipped_subagents, deferred + (over a bound, or past `limit` for this run), errors, locked. + """ + from longhand.parser import JSONLParser + from longhand.recall.project_fallback import claim_ingest_lock, release_ingest_lock + + if min(limit, max_file_bytes, max_events) < 1: + raise ValueError("Sync limits must be positive") + report: dict[str, Any] = { + "ingested": 0, + "skipped": 0, + "skipped_subagents": 0, + "deferred": [], + "errors": [], + "locked": False, + } + if claim_lock and not claim_ingest_lock(store): + report["locked"] = True + return report + try: + complete: tuple[str, ...] = ( + ("archived", "analyzed") if isinstance(store, CodexArchiveStore) else ("analyzed",) + ) + scan = scan_codex_sessions( + store.sqlite, + codex_home, + max_file_bytes=max_file_bytes, + include_subagents=include_subagents, + complete_stages=complete, + ) + report["skipped"] = len(scan.unchanged) + report["skipped_subagents"] = len(scan.subagents) + report["deferred"].extend(str(p) for p in scan.oversize) + for index, path in enumerate(scan.candidates): + if index >= limit: + report["deferred"].append(str(path)) + continue + try: + size = path.stat().st_size + parser = JSONLParser(path) + events = list(islice(parser.parse_events(), max_events + 1)) + if len(events) > max_events or path.stat().st_size > max_file_bytes: + report["deferred"].append(str(path)) + continue + if not events: + continue + session = parser.build_session(events) + if not session.session_id.startswith("codex:"): + raise ValueError("Not a Codex rollout (missing session_meta)") + store.ingest_session(session, events) + # Record the size that was parsed, not whatever the file grew + # to while embedding ran, so appended bytes are seen next scan. + with store.sqlite.connect() as conn: + conn.execute( + "UPDATE ingestion_log SET file_size = ? WHERE transcript_path = ?", + (size, str(path)), + ) + report["ingested"] += 1 + except Exception as exc: + report["errors"].append( + {"path": str(path), "error": f"{type(exc).__name__}: {exc}"} + ) + finally: + if claim_lock: + release_ingest_lock(store) + return report diff --git a/longhand/lightweight_mcp.py b/longhand/lightweight_mcp.py new file mode 100644 index 0000000..f73603e --- /dev/null +++ b/longhand/lightweight_mcp.py @@ -0,0 +1,154 @@ +"""Bounded keyword retrieval from the shared archive, with no vector model. + +Run as ``python -m longhand.lightweight_mcp``. This deliberately exposes only +read tools. Existing Longhand semantic tools can still use the same archive. +""" + +from __future__ import annotations + +import os +import sqlite3 +import sys +from contextlib import contextmanager +from pathlib import Path + +try: + from mcp.server.fastmcp import FastMCP +except ImportError: + print( + "The `mcp` package (>= 1.2.0) is required for the shared MCP server. " + "It ships with Longhand — reinstall with:\n" + " pip install --upgrade longhand", + file=sys.stderr, + ) + sys.exit(1) + +mcp = FastMCP( + "longhand", + instructions=( + "Shared Claude and Codex history. Use search with a short literal phrase, " + "list_sessions to find sessions, then get_session_timeline for context. " + "Search is keyword matching, not semantic similarity. Records are exact; " + "display excerpts are bounded. Use get_event_text to page through longer records." + ), +) + + +@contextmanager +def connection(): + root = Path(os.environ.get("LONGHAND_DATA_DIR") or Path.home() / ".longhand") + conn = sqlite3.connect((root / "longhand.db").resolve().as_uri() + "?mode=ro", uri=True) + conn.row_factory = sqlite3.Row + conn.execute("PRAGMA cache_size = -2048") + # Bound scanning work as well as returned data (large archives may need + # a session filter). SQLite aborts the query after this instruction budget. + ticks = 0 + + def budget(): + nonlocal ticks + ticks += 1 + return int(ticks > 2000) + + conn.set_progress_handler(budget, 10000) + try: + yield conn + finally: + conn.close() + + +def bounded(limit): + return max(1, min(int(limit), 50)) + + +def _escape_like(text: str) -> str: + return text.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_") + + +@mcp.tool() +def list_sessions( + limit: int = 10, + offset: int = 0, + source: str | None = None, + project: str | None = None, +) -> list[dict]: + """List recent Claude and Codex sessions from the same local archive. + + `source` narrows to "codex" or "claude"; `project` matches a substring of + the session's project path. + """ + where: list[str] = [] + params: list[object] = [] + if source: + if source not in ("codex", "claude"): + raise ValueError('source must be "codex" or "claude"') + where.append( + "session_id LIKE 'codex:%'" if source == "codex" else "session_id NOT LIKE 'codex:%'" + ) + if project: + where.append("project_path LIKE ? ESCAPE '\\'") + params.append(f"%{_escape_like(project)}%") + clause = f"WHERE {' AND '.join(where)} " if where else "" + with connection() as conn: + return [ + dict(r) + for r in conn.execute( + "SELECT session_id, project_path, started_at, ended_at, event_count " + f"FROM sessions {clause}ORDER BY started_at DESC LIMIT ? OFFSET ?", + (*params, bounded(limit), max(0, offset)), + ) + ] + + +@mcp.tool() +def search(query: str, session_id: str | None = None, limit: int = 10) -> list[dict]: + """Find a literal phrase in stored text; narrow by session for large archives.""" + if not query.strip() or len(query) > 200: + raise ValueError("Use a nonempty phrase of at most 200 characters") + escaped = _escape_like(query) + where = "content LIKE ? ESCAPE '\\'" + params = [f"%{escaped}%"] + if session_id: + where += " AND session_id = ?" + params.append(session_id) + with connection() as conn: + return [ + dict(r) + for r in conn.execute( + "SELECT event_id, session_id, sequence, event_type, timestamp, " + "substr(content, 1, 2000) AS content, length(content) AS total_chars " + f"FROM events WHERE {where} ORDER BY timestamp DESC LIMIT ?", + (*params, bounded(limit)), + ) + ] + + +@mcp.tool() +def get_session_timeline(session_id: str, offset: int = 0, limit: int = 10) -> list[dict]: + """Read consecutive recorded events; paginate with offset.""" + with connection() as conn: + return [ + dict(r) + for r in conn.execute( + "SELECT event_id, sequence, event_type, timestamp, " + "substr(content, 1, 2000) AS content, length(content) AS total_chars " + "FROM events WHERE session_id = ? ORDER BY sequence LIMIT ? OFFSET ?", + (session_id, bounded(limit), max(0, offset)), + ) + ] + + +@mcp.tool() +def get_event_text(event_id: str, offset: int = 0, raw: bool = False) -> dict: + """Read an exact text slice (up to 8,000 characters), or the stored raw JSON.""" + column = "raw_json" if raw else "content" + with connection() as conn: + row = conn.execute( + f"SELECT substr({column}, ?, 8000) AS text, length({column}) AS total_chars " + "FROM events WHERE event_id = ?", + (max(0, offset) + 1, event_id), + ).fetchone() + return dict(row) if row else {} + + +if __name__ == "__main__": + mcp.run() diff --git a/longhand/parser.py b/longhand/parser.py index ea9aef5..f4ef353 100644 --- a/longhand/parser.py +++ b/longhand/parser.py @@ -13,13 +13,16 @@ from collections.abc import Iterator from datetime import datetime, timezone from pathlib import Path, PurePath -from typing import Any +from typing import TYPE_CHECKING, Any from longhand.extractors.errors import detect_error from longhand.extractors.git import extract_git_signal from longhand.redaction import redact_event, redaction_enabled from longhand.types import Event, EventType, FileOperation, Session +if TYPE_CHECKING: + from longhand.codex import CodexAdapter + # Hard limits — keep the parser bounded so a malicious or corrupted JSONL # can't crash or OOM the ingest pipeline. MAX_FILE_SIZE_BYTES = 500 * 1024 * 1024 # 500MB per session file @@ -63,7 +66,11 @@ # on the live corpus the day the row shipped). # - worktree-state: worktree orchestration snapshot; cwd/branch data already # lives on events. Triaged 2026-07-11 (same harvest as pr-link). -TRIAGED_UNKNOWN_ENTRY_TYPES = frozenset({"summary", "pr-link", "worktree-state"}) +# - frame-link: where a published page landed ({frameUrl: a claude.ai URL, +# path: the local .html, title}). Genuinely recallable ("where did that +# report go?") — a promotion candidate alongside pr-link — preserved until +# then. Triaged 2026-09-07 (doctor drift row on the live corpus, 30 in 30 days). +TRIAGED_UNKNOWN_ENTRY_TYPES = frozenset({"summary", "pr-link", "worktree-state", "frame-link"}) _MAX_UNIQUE_CWDS_SCANNED = 20 @@ -258,6 +265,9 @@ def __init__(self, file_path: str | Path): # the file sees the same setting (covers full, tail, and re-ingest # paths — they all construct events through this class). self._redact = redaction_enabled() + # Set on the first `session_meta` line: from then on the whole file is + # a Codex rollout and every entry routes through the adapter. + self._codex_adapter: CodexAdapter | None = None def parse_events(self) -> Iterator[Event]: """Yield Event objects from the session file, in file order. @@ -268,6 +278,7 @@ def parse_events(self) -> Iterator[Event]: """ sequence = 0 seen_ids: dict[str, int] = {} + self._codex_adapter = None with self.file_path.open("r", encoding="utf-8", errors="replace") as f: for _line_num, line in enumerate(f, start=1): # Skip lines that exceed the hard line-length limit @@ -389,6 +400,13 @@ def _entry_to_events(self, entry: dict[str, Any], base_sequence: int) -> list[Ev """ entry_type = entry.get("type", "unknown") + if entry_type == "session_meta" and self._codex_adapter is None: + from longhand.codex import CodexAdapter + + self._codex_adapter = CodexAdapter(entry.get("payload") or {}) + if self._codex_adapter is not None: + return self._codex_adapter.convert(entry, base_sequence) + # File history snapshots — create a minimal event if entry_type == "file-history-snapshot": return [self._file_snapshot_event(entry, base_sequence)] diff --git a/longhand/recall/reconcile.py b/longhand/recall/reconcile.py index 3b77720..451ea76 100644 --- a/longhand/recall/reconcile.py +++ b/longhand/recall/reconcile.py @@ -1,8 +1,12 @@ """Reconcile session transcripts on disk against the sessions table. Shared core for `longhand reconcile` (CLI) and the `reconcile` MCP tool. -Classifies every on-disk JSONL into one of three buckets and, when -`fix=True`, re-ingests the missing and null-project-id entries. +Classifies every Claude Code JSONL into buckets and, when `fix=True`, +re-ingests the missing, null-project, and partially-indexed entries. + +Codex rollouts ride along when a Codex home exists: new or changed ones are +captured exact-record-only under the same ingest lock, so the scheduled +reconciler keeps both clients' history current with no extra setup. """ from __future__ import annotations @@ -30,6 +34,13 @@ class ReconcileReport: errors: list[dict[str, str]] = field(default_factory=list) # [{path, error}] fix_applied: bool = False lock_unavailable: bool = False + # Codex rollouts — all zero on a machine without a Codex home. + codex_on_disk: int = 0 + codex_pending: int = 0 # new or changed since capture, within the size bound + codex_skipped_subagents: int = 0 + codex_oversize: int = 0 + codex_ingested: int = 0 + codex_deferred: list[str] = field(default_factory=list) def to_dict(self) -> dict[str, object]: return { @@ -47,57 +58,69 @@ def to_dict(self) -> dict[str, object]: "errors": self.errors, "fix_applied": self.fix_applied, "lock_unavailable": self.lock_unavailable, + "codex_on_disk": self.codex_on_disk, + "codex_pending": self.codex_pending, + "codex_skipped_subagents": self.codex_skipped_subagents, + "codex_oversize": self.codex_oversize, + "codex_ingested": self.codex_ingested, + "codex_deferred": self.codex_deferred, } -def run_reconcile(store: LonghandStore, fix: bool = False) -> ReconcileReport: +def run_reconcile( + store: LonghandStore, fix: bool = False, *, codex_home: str | Path | None = None +) -> ReconcileReport: """Classify on-disk JSONLs vs. indexed sessions; optionally re-ingest problem buckets. Without `fix`: returns counts only. - With `fix=True`: acquires the ingest lock and re-ingests missing, - null-project, and partially-indexed entries using current project - inference. If another ingest is running, returns with - `lock_unavailable=True` and zero ingested. + With `fix=True`: acquires the ingest lock, re-ingests missing, + null-project, and partially-indexed Claude entries using current project + inference, and captures new or changed Codex rollouts (exact records + only). If another ingest is running, returns with + `lock_unavailable=True` and nothing ingested. """ - files = discover_sessions() - if not files: - return ReconcileReport(files_on_disk=0, fully_indexed=0) + from longhand.codex import CodexArchiveStore, scan_codex_sessions, sync_codex - with store.sqlite.connect() as conn: - rows = conn.execute("SELECT transcript_path, project_id FROM sessions").fetchall() - indexed: dict[str, str | None] = {r[0]: r[1] for r in rows} - stages = store.sqlite.analysis_stages() + files = discover_sessions() missing: list[Path] = [] null_project: list[Path] = [] partial: list[Path] = [] oversize: list[Path] = [] fully_indexed = 0 - for f in files: - # Files past the parser's size cap can't be ingested by ANY path — - # report them instead of erroring on every --fix pass (or worse, - # sitting silently in the missing bucket forever). - try: - if f.stat().st_size > MAX_FILE_SIZE_BYTES: - oversize.append(f) - continue - except OSError: - pass - state = indexed.get(str(f), "__not_found__") - if state == "__not_found__": - missing.append(f) - elif state is None: - null_project.append(f) - elif stages.get(str(f)) == "pending": - # The ingest pipeline started but never finished — a crash left - # this session with events but possibly no analysis/vectors. - # Only 'pending' counts: NULL is a pre-v0.12 or live-tail row - # (unknown — treated as complete so upgrades don't stampede), - # and 'events' is a deliberate --skip-analysis defer owned by - # `longhand analyze --all`. - partial.append(f) - else: - fully_indexed += 1 + if files: + with store.sqlite.connect() as conn: + rows = conn.execute("SELECT transcript_path, project_id FROM sessions").fetchall() + indexed: dict[str, str | None] = {r[0]: r[1] for r in rows} + stages = store.sqlite.analysis_stages() + + for f in files: + # Files past the parser's size cap can't be ingested by ANY path — + # report them instead of erroring on every --fix pass (or worse, + # sitting silently in the missing bucket forever). + try: + if f.stat().st_size > MAX_FILE_SIZE_BYTES: + oversize.append(f) + continue + except OSError: + pass + state = indexed.get(str(f), "__not_found__") + if state == "__not_found__": + missing.append(f) + elif state is None: + null_project.append(f) + elif stages.get(str(f)) == "pending": + # The ingest pipeline started but never finished — a crash left + # this session with events but possibly no analysis/vectors. + # Only 'pending' counts: NULL is a pre-v0.12 or live-tail row + # (unknown — treated as complete so upgrades don't stampede), + # and 'events' is a deliberate --skip-analysis defer owned by + # `longhand analyze --all`. + partial.append(f) + else: + fully_indexed += 1 + + codex_scan = scan_codex_sessions(store.sqlite, codex_home) report = ReconcileReport( files_on_disk=len(files), @@ -106,13 +129,17 @@ def run_reconcile(store: LonghandStore, fix: bool = False) -> ReconcileReport: missing=[str(p) for p in missing], partially_indexed=[str(p) for p in partial], skipped_oversize=[str(p) for p in oversize], + codex_on_disk=codex_scan.on_disk, + codex_pending=len(codex_scan.candidates), + codex_skipped_subagents=len(codex_scan.subagents), + codex_oversize=len(codex_scan.oversize), ) if not fix: return report to_process = missing + null_project + partial - if not to_process: + if not to_process and not codex_scan.candidates: report.fix_applied = True return report @@ -132,6 +159,18 @@ def run_reconcile(store: LonghandStore, fix: bool = False) -> ReconcileReport: report.ingested += 1 except Exception as e: # noqa: BLE001 report.errors.append({"path": str(f), "error": str(e)}) + + if codex_scan.candidates: + # Exact-record capture through the store's own SQLite handle; the + # lock is already ours, so sync must not claim (or release) it. + codex_report = sync_codex( + CodexArchiveStore(store.data_dir, sqlite=store.sqlite), + codex_home, + claim_lock=False, + ) + report.codex_ingested = codex_report["ingested"] + report.codex_deferred = list(codex_report["deferred"]) + report.errors.extend(codex_report["errors"]) finally: release_ingest_lock(store) diff --git a/longhand/setup_commands.py b/longhand/setup_commands.py index cbc7241..49084c1 100644 --- a/longhand/setup_commands.py +++ b/longhand/setup_commands.py @@ -1071,7 +1071,15 @@ def _transcript_format_status(store: LonghandStore, days: int = 30) -> str: counter: Counter[str] = Counter() for row in rows: try: - entry_type = str(json.loads(row["raw_json"]).get("type", "?")) + raw = json.loads(row["raw_json"]) + entry_type = str(raw.get("type", "?")) + # Codex rollouts nest the informative kind one level down + # (response_item/, event_msg/) — name that, not + # the envelope. + if entry_type in ("response_item", "event_msg"): + payload = raw.get("payload") + kind = payload.get("type") if isinstance(payload, dict) else None + entry_type = f"{entry_type}/{kind or '?'}" except Exception: entry_type = "?" if entry_type not in dispositioned: @@ -1136,6 +1144,83 @@ class token is already there, so the split needs no new plumbing. return parts[2].removesuffix(":") +def _archived_session_count(store: LonghandStore) -> int: + """Sessions captured exact-record-only (Codex `archived` stage): no vectors yet.""" + try: + with store.sqlite.connect() as conn: + row = conn.execute( + "SELECT COUNT(*) FROM ingestion_log WHERE analysis_stage = 'archived'" + ).fetchone() + return int(row[0]) if row else 0 + except Exception: + return 0 + + +def _age_text(iso_timestamp: str) -> str: + """'3 min ago' style age for a stored ISO timestamp; '' if unparseable.""" + try: + then = datetime.fromisoformat(iso_timestamp) + if then.tzinfo is None: + then = then.replace(tzinfo=timezone.utc) + seconds = max(0, int((utcnow() - then).total_seconds())) + except Exception: + return "" + if seconds < 60: + return "just now" + if seconds < 3600: + return f"{seconds // 60} min ago" + if seconds < 86400: + return f"{seconds // 3600} h ago" + return f"{seconds // 86400} d ago" + + +def _codex_capture_status(store: LonghandStore) -> str | None: + """Codex rollouts on disk vs. captured in the archive. + + None when there is no Codex home: a machine without Codex should not see + a Codex row at all. Subagent threads are reported separately because + capture skips them by default. + """ + from longhand.codex import discover_codex_sessions, is_subagent_rollout, read_session_meta + + try: + files = discover_codex_sessions() + except Exception: + return None + if not files: + return None + primary = [f for f in files if not is_subagent_rollout(read_session_meta(f))] + try: + with store.sqlite.connect() as conn: + rows = conn.execute( + "SELECT transcript_path, file_size, ingested_at FROM ingestion_log " + "WHERE session_id LIKE 'codex:%'" + ).fetchall() + except Exception: + return "[dim]—[/dim] could not inspect the archive" + captured = {r["transcript_path"]: r for r in rows} + current = 0 + for f in primary: + row = captured.get(str(f)) + try: + if row is not None and row["file_size"] == f.stat().st_size: + current += 1 + except OSError: + pass + behind = len(primary) - current + latest = max((r["ingested_at"] for r in rows if r["ingested_at"]), default=None) + age = f", last capture {_age_text(latest)}" if latest and _age_text(latest) else "" + subagents = len(files) - len(primary) + note = f" [dim]({subagents} subagent thread(s) skipped by default)[/dim]" if subagents else "" + if behind == 0: + return f"[green]✓[/green] {current}/{len(primary)} rollouts captured{age}{note}" + return ( + f"[yellow]⚠[/yellow] {behind} of {len(primary)} rollouts new or changed since " + f"capture{age} — run [bold]longhand codex-sync[/bold] " + f"([bold]reconcile --fix[/bold] also captures them on its schedule){note}" + ) + + def _hook_errors_status(store: LonghandStore, days: int = 7) -> str: """Rich-formatted count of hook-error breadcrumbs in the last `days` days. @@ -1349,6 +1434,11 @@ def _row(label: str, value: str) -> None: # entries yet. _row("Transcript format", _transcript_format_status(store)) + # 5d. Codex capture — only on a machine that has Codex rollouts. + codex_row = _codex_capture_status(store) + if codex_row: + _row("Codex capture", codex_row) + # 6. Stats stats = store.stats() _row("Sessions ingested", f"{stats.get('sessions', 0):,}") @@ -1356,12 +1446,25 @@ def _row(label: str, value: str) -> None: _row("Projects inferred", f"{stats.get('projects', 0):,}") _row("Episodes extracted", f"{stats.get('episodes', 0):,}") - sessions_needing_analysis = max(0, stats.get("sessions", 0) - stats.get("outcomes", 0)) + # Archived Codex sessions have no outcome row either, but `analyze` is + # the wrong remedy for them: it rebuilds outcomes and episodes without + # ever embedding the events, so search would stay blind. Name the remedy + # that actually works for that class (Promise 5). + archived = _archived_session_count(store) + sessions_needing_analysis = max( + 0, stats.get("sessions", 0) - stats.get("outcomes", 0) - archived + ) if sessions_needing_analysis > 0: _row( "Sessions needing analysis", f"[yellow]{sessions_needing_analysis}[/yellow] (run [bold]longhand analyze --all[/bold])", ) + if archived > 0: + _row( + "Codex sessions archived", + f"[yellow]{archived}[/yellow] captured exact-record only — recall and search " + "skip them (run [bold]longhand codex-sync --semantic[/bold] to index them)", + ) # 7. Storage footprint — the corpus grows forever by design, but nothing # surfaced how big it had gotten (3 GB observed in the wild). diff --git a/longhand/storage/store.py b/longhand/storage/store.py index 86a5c25..bfce118 100644 --- a/longhand/storage/store.py +++ b/longhand/storage/store.py @@ -455,7 +455,9 @@ def _extract_git_operations(session_id: str, events: list[Event]) -> list[dict]: command = "" if e.tool_use_id: paired_input = tool_inputs.get(e.tool_use_id, {}) - command = paired_input.get("command", "") + command = paired_input.get("command", paired_input.get("cmd", "")) + if isinstance(command, list): + command = " ".join(str(part) for part in command) signal = extract_git_signal(command, e.tool_output or "") op_id = ( diff --git a/longhand/storage/vector_store.py b/longhand/storage/vector_store.py index 58a68fd..c6a3f56 100644 --- a/longhand/storage/vector_store.py +++ b/longhand/storage/vector_store.py @@ -16,6 +16,11 @@ import chromadb from chromadb.config import Settings +# The package-level name resolves on every supported chromadb (0.5.x ships it +# in a single module, 0.6.x re-exports it from a package), but 0.6.x's +# re-export is invisible to mypy — hence the targeted ignore. +from chromadb.utils.embedding_functions import ONNXMiniLM_L6_V2 # type: ignore[attr-defined] + from longhand.types import Event, EventType # Limit embedded text length to keep Chroma performant. @@ -24,7 +29,8 @@ # Per-upsert batch size for Chroma. Chroma tolerates larger batches but 500 # is the stable sweet spot: enough to amortize ONNX call overhead, small -# enough to avoid memory spikes on low-RAM systems. +# enough to avoid memory spikes on low-RAM systems. All collections in a +# VectorStore share one CPU embedding model (see VectorStore.__init__). CHROMA_BATCH_SIZE = 500 @@ -48,23 +54,31 @@ def __init__(self, persist_dir: str | Path): settings=Settings(anonymized_telemetry=False, allow_reset=True), ) + # Share one CPU model across collections; avoid CoreML compilation + # and loading a separate embedding model for every collection. + embedding = ONNXMiniLM_L6_V2(preferred_providers=["CPUExecutionProvider"]) + self.events_collection = self.client.get_or_create_collection( name="events", + embedding_function=embedding, metadata={"description": "All session events, embedded for semantic search"}, ) self.sessions_collection = self.client.get_or_create_collection( name="sessions", + embedding_function=embedding, metadata={"description": "One embedding per session for fuzzy recall (v0.2)"}, ) self.projects_collection = self.client.get_or_create_collection( name="projects", + embedding_function=embedding, metadata={"description": "One embedding per project for fuzzy project matching"}, ) self.segments_collection = self.client.get_or_create_collection( name="segments", + embedding_function=embedding, metadata={ "description": "One embedding per conversation segment for topic-level recall" }, @@ -72,6 +86,7 @@ def __init__(self, persist_dir: str | Path): self.episodes_collection = self.client.get_or_create_collection( name="episodes", + embedding_function=embedding, metadata={ "description": "One embedding per problem→fix episode for intent-framed recall" }, diff --git a/pyproject.toml b/pyproject.toml index 4cf5ab3..e839ad2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -36,7 +36,8 @@ dependencies = [ "typer>=0.12.0,<1.0", "rich>=13.7.0,<15.0", "pydantic>=2.6.0,<3.0", - "mcp>=1.0.0,<2.0", + # `longhand shared-mcp` uses mcp.server.fastmcp, which first shipped in 1.2.0. + "mcp>=1.2.0,<2.0", ] [project.optional-dependencies] diff --git a/scripts/com.longhand.codex-sync.plist.template b/scripts/com.longhand.codex-sync.plist.template new file mode 100644 index 0000000..2dc9803 --- /dev/null +++ b/scripts/com.longhand.codex-sync.plist.template @@ -0,0 +1,48 @@ + + + + + + Label + com.longhand.codex-sync + ProgramArguments + + __PYTHON__ + -m + longhand + codex-sync + --limit + 8 + --max-file-kb + 16384 + --max-events + 20000 + + EnvironmentVariables + + PYTHONUNBUFFERED + 1 + + ProcessType + Background + RunAtLoad + + StartInterval + 60 + StandardOutPath + __HOME__/.longhand/logs/codex-sync.log + StandardErrorPath + __HOME__/.longhand/logs/codex-sync.log + + diff --git a/tests/conftest.py b/tests/conftest.py index 70c28c9..02b10aa 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -18,6 +18,10 @@ def _no_update_check(monkeypatch): # A developer's real LONGHAND_DATA_DIR must never leak into tests; # tests that exercise the env var set it explicitly. monkeypatch.delenv("LONGHAND_DATA_DIR", raising=False) + # Nor may a developer's real ~/.codex: reconcile and doctor discover Codex + # rollouts there. Tests that exercise Codex capture point CODEX_HOME at a + # tmp directory explicitly. + monkeypatch.setenv("CODEX_HOME", "/nonexistent/longhand-test-codex-home") def _line(entry: dict[str, Any]) -> str: diff --git a/tests/fixtures/codex_shapes/entries.jsonl b/tests/fixtures/codex_shapes/entries.jsonl new file mode 100644 index 0000000..acd3ba1 --- /dev/null +++ b/tests/fixtures/codex_shapes/entries.jsonl @@ -0,0 +1,26 @@ +{"timestamp":"2026-04-14T19:45:20.948Z","type":"session_meta","payload":{"id":"shape-codex-1","timestamp":"2026-04-14T19:45:20.948Z","cwd":"/tmp/shapes-project","originator":"Codex Desktop","cli_version":"0.119.0","source":"vscode","model_provider":"openai","base_instructions":"(instructions omitted)","dynamic_tools":[],"git":{"branch":"main"}}} +{"timestamp":"2026-09-07T20:08:27.342Z","type":"session_meta","payload":{"id":"shape-codex-guardian","session_id":"shape-codex-guardian","timestamp":"2026-09-07T20:08:27.342Z","cwd":"/tmp/shapes-project","originator":"Codex Desktop","cli_version":"0.153.4","source":{"subagent":{"other":"guardian"}},"thread_source":"subagent","parent_thread_id":"shape-codex-1","multi_agent_version":1,"history_mode":"full","context_window":200000,"model_provider":"openai","base_instructions":"(instructions omitted)","git":{"branch":"main"}}} +{"timestamp":"2026-09-07T20:08:28.000Z","type":"turn_context","payload":{"turn_id":"shape-turn-1","cwd":"/tmp/shapes-project","model":"gpt-shape","approval_policy":"on-request","sandbox_policy":{"type":"workspace-write"},"effort":"medium","summary":"auto","current_date":"2026-09-07","timezone":"UTC","personality":"default","collaboration_mode":"default","realtime_active":false}} +{"timestamp":"2026-09-07T20:08:29.000Z","type":"event_msg","payload":{"type":"user_message","message":"run the tests and fix whatever breaks","images":[]}} +{"timestamp":"2026-09-07T20:08:30.000Z","type":"response_item","payload":{"type":"message","role":"user","content":[{"type":"input_text","text":"run the tests and fix whatever breaks"},{"type":"input_image","image_url":"data:image/png;base64,SHAPEIMAGE"}]}} +{"timestamp":"2026-09-07T20:08:31.000Z","type":"response_item","payload":{"type":"message","role":"developer","content":[{"type":"input_text","text":"(injected instructions omitted)"}]}} +{"timestamp":"2026-09-07T20:08:32.000Z","type":"event_msg","payload":{"type":"task_started","turn_id":"shape-turn-1","model_context_window":200000}} +{"timestamp":"2026-09-07T20:08:33.000Z","type":"response_item","payload":{"type":"reasoning","summary":[{"type":"summary_text","text":"Start with the failing module."}],"encrypted_content":"OPAQUE_CIPHERTEXT"}} +{"timestamp":"2026-09-07T20:08:34.000Z","type":"event_msg","payload":{"type":"agent_reasoning","text":"Start with the failing module."}} +{"timestamp":"2026-09-07T20:08:35.000Z","type":"response_item","payload":{"type":"reasoning","summary":[],"encrypted_content":"OPAQUE_CIPHERTEXT_ONLY"}} +{"timestamp":"2026-09-07T20:08:36.000Z","type":"response_item","payload":{"type":"function_call","name":"exec_command","call_id":"shape-call-1","arguments":"{\"cmd\":\"git commit -am 'Fix login'\",\"max_output_tokens\":1000}"}} +{"timestamp":"2026-09-07T20:08:37.000Z","type":"response_item","payload":{"type":"function_call_output","call_id":"shape-call-1","output":"[main abc1234] Fix login\n 1 file changed, 2 insertions(+)"}} +{"timestamp":"2026-09-07T20:08:38.000Z","type":"event_msg","payload":{"type":"exec_command_end","call_id":"shape-call-1","stdout":"[main abc1234] Fix login\n","stderr":"","exit_code":0,"duration":"0.2s","formatted_output":"[main abc1234] Fix login\n"}} +{"timestamp":"2026-09-07T20:08:39.000Z","type":"response_item","payload":{"type":"custom_tool_call","name":"exec","call_id":"shape-call-2","input":"text(await tools.exec_command({cmd:\"git push origin main\",\"max_output_tokens\":500}));"}} +{"timestamp":"2026-09-07T20:08:40.000Z","type":"response_item","payload":{"type":"custom_tool_call_output","call_id":"shape-call-2","output":"To github.com:example/repo.git\n abc1234..def5678 main -> main"}} +{"timestamp":"2026-09-07T20:08:41.000Z","type":"response_item","payload":{"type":"custom_tool_call","name":"apply_patch","call_id":"shape-call-3","input":"*** Begin Patch\n*** Update File: /tmp/shapes-project/main.py\n@@\n-old\n+new\n*** End Patch"}} +{"timestamp":"2026-09-07T20:08:42.000Z","type":"response_item","payload":{"type":"custom_tool_call_output","call_id":"shape-call-3","output":"Success. Updated the following files:\nM main.py"}} +{"timestamp":"2026-09-07T20:08:43.000Z","type":"event_msg","payload":{"type":"patch_apply_end","call_id":"shape-call-3","stdout":"Success.","stderr":"","success":true}} +{"timestamp":"2026-09-07T20:08:44.000Z","type":"event_msg","payload":{"type":"item_completed","item":{"id":"shape-item-1","type":"command_execution"}}} +{"timestamp":"2026-09-07T20:08:45.000Z","type":"response_item","payload":{"type":"message","role":"assistant","content":[{"type":"output_text","text":"Fixed the login test and pushed."}]}} +{"timestamp":"2026-09-07T20:08:46.000Z","type":"event_msg","payload":{"type":"agent_message","message":"Fixed the login test and pushed."}} +{"timestamp":"2026-09-07T20:08:47.000Z","type":"event_msg","payload":{"type":"token_count","info":{"total_token_usage":{"input_tokens":1200,"output_tokens":300}}}} +{"timestamp":"2026-09-07T20:08:48.000Z","type":"token_usage_record","payload":{"input_tokens":1200,"output_tokens":300,"total_tokens":1500}} +{"timestamp":"2026-09-07T20:08:49.000Z","type":"event_msg","payload":{"type":"task_complete","turn_id":"shape-turn-1","last_agent_message":"Fixed the login test and pushed."}} +{"timestamp":"2026-09-07T20:08:50.000Z","type":"event_msg","payload":{"type":"thread_settings_applied","thread_id":"shape-codex-1"}} +{"timestamp":"2026-09-07T20:08:51.000Z","type":"world_state","payload":{"full":true,"state":{}}} diff --git a/tests/fixtures/transcript_shapes/entries.jsonl b/tests/fixtures/transcript_shapes/entries.jsonl index e89d9f3..ea7245b 100644 --- a/tests/fixtures/transcript_shapes/entries.jsonl +++ b/tests/fixtures/transcript_shapes/entries.jsonl @@ -15,3 +15,4 @@ {"type":"pr-link","sessionId":"s-shapes","prNumber":42,"prUrl":"https://github.com/example/repo/pull/42","prRepository":"example/repo","timestamp":"2026-07-01T10:00:20.000Z"} {"type":"worktree-state","sessionId":"s-shapes","worktreeSession":{"originalCwd":"/tmp/shapes-project","worktreePath":"/tmp/shapes-project/.claude/worktrees/wt-1","worktreeName":"wt-1","worktreeBranch":"worktree-wt-1","originalBranch":"main","originalHeadCommit":"0000000000000000000000000000000000000000","sessionId":"s-shapes"}} {"type":"file-history-delta","messageId":"shape-fhd-1","snapshotMessageId":"shape-fhs-1","trackingPath":"/tmp/shapes-project/main.py","backup":{"backupFileName":null,"version":3,"backupTime":"2026-08-11T10:00:00.000Z","realParentDir":"/tmp/shapes-project"},"timestamp":"2026-08-11T10:00:00.000Z"} +{"type":"frame-link","frameUrl":"https://claude.ai/frames/shape-frame-1","path":"/tmp/shapes-project/report.html","sessionId":"s-shapes","timestamp":"2026-09-01T10:00:00.000Z","title":"Shapes report"} diff --git a/tests/test_codex.py b/tests/test_codex.py new file mode 100644 index 0000000..116f09e --- /dev/null +++ b/tests/test_codex.py @@ -0,0 +1,463 @@ +"""Cross-client archive regressions, using synthetic Codex rollout records.""" + +import json +from unittest.mock import MagicMock + +import pytest + +from longhand.codex import discover_codex_sessions, sync_codex +from longhand.parser import JSONLParser +from longhand.storage.sqlite_store import SQLiteStore + + +def record(kind, payload): + return {"timestamp": "2026-09-07T12:00:00Z", "type": kind, "payload": payload} + + +def write_rollout(path, identity="test-id", source=None): + path.parent.mkdir(parents=True, exist_ok=True) + meta = {"id": identity, "cwd": "/tmp/project", "git": {"branch": "main"}} + if source is not None: + meta["source"] = source + entries = [ + record("session_meta", meta), + record("turn_context", {"cwd": "/tmp/project", "model": "test-model"}), + record("event_msg", {"type": "user_message", "message": "Fix the login"}), + record( + "response_item", + { + "type": "message", + "role": "user", + "content": [ + {"type": "input_text", "text": "Fix the login"}, + {"type": "input_image", "image_url": "data:image/png;base64,SECRET_IMAGE"}, + ], + }, + ), + record( + "response_item", + { + "type": "reasoning", + "summary": [{"type": "summary_text", "text": "Recorded summary"}], + "encrypted_content": "OPAQUE_CIPHERTEXT", + }, + ), + record( + "response_item", + { + "type": "function_call", + "name": "exec_command", + "call_id": "call-1", + "arguments": json.dumps({"cmd": "git commit -m 'Fix login'"}), + }, + ), + record( + "response_item", + { + "type": "function_call_output", + "call_id": "call-1", + "output": "[main abc1234] Fix login\n 1 file changed", + }, + ), + record( + "response_item", + { + "type": "custom_tool_call", + "name": "exec", + "call_id": "call-2", + "input": "text(await tools.exec_command({cmd: 'pwd'}));", + }, + ), + record( + "response_item", + {"type": "custom_tool_call_output", "call_id": "call-2", "output": "project"}, + ), + record( + "response_item", + { + "type": "message", + "role": "assistant", + "content": [{"type": "output_text", "text": "Fixed login"}], + }, + ), + record("future_record", {"type": "new-kind", "new_data": True}), + ] + path.write_text("".join(json.dumps(e) + "\n" for e in entries)) + return entries + + +def test_codex_preserves_raw_and_avoids_duplicate_messages(tmp_path): + path = tmp_path / "rollout.jsonl" + entries = write_rollout(path) + parser = JSONLParser(path) + events = list(parser.parse_events()) + # The event_msg mirror of the user message is skipped, not stored twice; + # everything else keeps its raw record, including the undispositioned + # future_record, which lands as a preserved unknown event. + assert [e.raw for e in events] == [e for e in entries if e["type"] != "event_msg"] + assert sum(e.event_type == "user_message" for e in events) == 1 + assert events[2].content == "Fix the login\n[image]" + assert events[3].content == "Recorded summary" + assert "OPAQUE_CIPHERTEXT" not in "".join(e.content for e in events) + assert "SECRET_IMAGE" not in "".join(e.content for e in events) + session = parser.build_session(events) + assert session.session_id == "codex:test-id" + assert session.model == "test-model" + assert session.tool_call_count == 2 + assert events[5].git_commit_hash == "abc1234" + assert events[5].parent_event_id == events[4].event_id + assert events[5].tool_use_id == events[4].tool_use_id + assert events[6].tool_input["input"].startswith("text(await") + assert events[6].tool_input["command"] == "pwd" + assert events[-1].event_type == "unknown" + assert events[-1].raw["type"] == "future_record" + + +def test_append_and_reparse_ids_are_stable(tmp_path): + path = tmp_path / "rollout.jsonl" + write_rollout(path) + parser = JSONLParser(path) + before = list(parser.parse_events()) + with path.open("a") as f: + f.write( + json.dumps( + record("response_item", {"type": "message", "role": "user", "content": "Next task"}) + ) + + "\n" + ) + after = list(parser.parse_events()) + assert [e.event_id for e in before] == [e.event_id for e in after[:-1]] + other = tmp_path / "other.jsonl" + write_rollout(other, "other-id") + other_events = list(JSONLParser(other).parse_events()) + assert {e.event_id for e in before}.isdisjoint(e.event_id for e in other_events) + assert before[5].tool_use_id != other_events[5].tool_use_id + + +def test_codex_and_claude_share_sqlite_without_collisions(tmp_path, sample_session_file): + path = tmp_path / "rollout.jsonl" + write_rollout(path, "test-session-1") + store = SQLiteStore(tmp_path / "archive.db") + for source in (sample_session_file, path, path): + parser = JSONLParser(source) + events = list(parser.parse_events()) + store.upsert_session(parser.build_session(events)) + store.insert_events(events) + assert len(store.list_sessions()) == 2 + assert len(store.get_events(session_id="codex:test-session-1")) == 10 + assert store.get_events(session_id="test-session-1") + + +def test_discover_active_archived_and_custom_home(tmp_path, monkeypatch): + active = tmp_path / "sessions/2026/09/07/rollout.jsonl" + archived = tmp_path / "archived_sessions/archived.jsonl" + write_rollout(active) + write_rollout(archived, "archived") + (tmp_path / "unrelated.jsonl").write_text("{}\n") + monkeypatch.setenv("CODEX_HOME", str(tmp_path)) + assert set(discover_codex_sessions()) == {active, archived} + + +def test_sync_skips_unchanged_and_retries_append(tmp_path, monkeypatch): + source = tmp_path / "codex/sessions/rollout.jsonl" + write_rollout(source) + store = MagicMock() + store.data_dir = tmp_path / "archive" + store.data_dir.mkdir() + store.sqlite = SQLiteStore(store.data_dir / "longhand.db") + + def ingest(session, events): + store.sqlite.upsert_session(session) + store.sqlite.insert_events(events) + store.sqlite.log_ingestion( + session.transcript_path, session.session_id, source.stat().st_size, len(events) + ) + store.sqlite.set_analysis_stage(session.transcript_path, "analyzed") + + store.ingest_session.side_effect = ingest + assert sync_codex(store, tmp_path / "codex")["ingested"] == 1 + assert sync_codex(store, tmp_path / "codex")["skipped"] == 1 + assert store.sqlite.analysis_stages()[str(source)] == "analyzed" + # A failure after the ingestion log is written must be retried even + # when the transcript's byte count has not changed. + store.sqlite.set_analysis_stage(str(source), "pending") + assert sync_codex(store, tmp_path / "codex")["ingested"] == 1 + assert store.sqlite.analysis_stages()[str(source)] == "analyzed" + with source.open("a") as f: + f.write( + json.dumps( + record( + "response_item", + { + "type": "message", + "role": "assistant", + "content": [{"type": "output_text", "text": "Anything else?"}], + }, + ) + ) + + "\n" + ) + assert sync_codex(store, tmp_path / "codex")["ingested"] == 1 + assert len(store.sqlite.get_events(session_id="codex:test-id")) == 11 + + +def test_sync_honors_existing_writer_lock(tmp_path, monkeypatch): + monkeypatch.setattr("longhand.recall.project_fallback.claim_ingest_lock", lambda _: False) + store = MagicMock() + assert sync_codex(store, tmp_path)["locked"] + store.ingest_session.assert_not_called() + + +def test_codex_redaction_applies_to_raw_and_searchable_text(tmp_path, monkeypatch): + path = tmp_path / "rollout.jsonl" + write_rollout(path) + secret = "ghp_" + "a" * 36 + with path.open("a") as f: + f.write( + json.dumps( + record("response_item", {"type": "message", "role": "user", "content": secret}) + ) + + "\n" + ) + monkeypatch.setattr("longhand.parser.redaction_enabled", lambda: True) + events = list(JSONLParser(path).parse_events()) + assert secret not in events[-1].content + assert secret not in json.dumps(events[-1].raw) + + +def test_missing_metadata_identity_is_reported(tmp_path): + path = tmp_path / "bad.jsonl" + path.write_text(json.dumps(record("session_meta", {})) + "\n") + with pytest.raises(ValueError, match="session id"): + list(JSONLParser(path).parse_events()) + + +def test_codex_git_details_survive_storage_extraction(tmp_path): + from longhand.storage.store import LonghandStore + + path = tmp_path / "rollout.jsonl" + write_rollout(path) + events = list(JSONLParser(path).parse_events()) + operations = LonghandStore._extract_git_operations("codex:test-id", events) + assert len(operations) == 1 + assert operations[0]["files_changed_count"] == 1 + assert operations[0]["commit_hash"] == "abc1234" + + +def test_sync_bounds_work_before_ingestion(tmp_path): + source = tmp_path / "codex/sessions/one.jsonl" + write_rollout(source) + write_rollout(source.with_name("two.jsonl"), "two") + store = MagicMock() + store.data_dir = tmp_path / "archive" + store.data_dir.mkdir() + store.sqlite = SQLiteStore(store.data_dir / "longhand.db") + report = sync_codex(store, tmp_path / "codex", max_file_bytes=1) + assert len(report["deferred"]) == 2 + store.ingest_session.assert_not_called() + report = sync_codex(store, tmp_path / "codex", max_events=2) + assert len(report["deferred"]) == 2 + store.ingest_session.assert_not_called() + # The per-run limit defers the rest rather than dropping them. + report = sync_codex(store, tmp_path / "codex", limit=1) + assert report["ingested"] == 1 + assert len(report["deferred"]) == 1 + assert store.ingest_session.call_count == 1 + + +def test_vector_collections_share_cpu_model_without_loading_it(tmp_path, monkeypatch): + from longhand.storage import vector_store + + client = MagicMock() + factory = MagicMock(return_value=client) + embedding = MagicMock() + monkeypatch.setattr(vector_store.chromadb, "PersistentClient", factory) + monkeypatch.setattr(vector_store, "ONNXMiniLM_L6_V2", embedding) + vector_store.VectorStore(tmp_path) + embedding.assert_called_once_with(preferred_providers=["CPUExecutionProvider"]) + assert client.get_or_create_collection.call_count == 5 + for call in client.get_or_create_collection.call_args_list: + assert call.kwargs["embedding_function"] is embedding.return_value + assert vector_store.CHROMA_BATCH_SIZE == 500 + + +def test_archive_only_and_lightweight_search(tmp_path, sample_session_file, monkeypatch): + from longhand import lightweight_mcp + from longhand.codex import CodexArchiveStore + + source = tmp_path / "codex/sessions/one.jsonl" + write_rollout(source) + store = CodexArchiveStore(tmp_path / "archive") + # Embedding construction must never run on the archive-only path. + monkeypatch.setattr( + "longhand.storage.store.VectorStore", lambda *a, **k: pytest.fail("loaded vectors") + ) + assert sync_codex(store, tmp_path / "codex")["ingested"] == 1 + assert sync_codex(store, tmp_path / "codex")["skipped"] == 1 + parser = JSONLParser(sample_session_file) + events = list(parser.parse_events()) + store.ingest_session(parser.build_session(events), events) + monkeypatch.setenv("LONGHAND_DATA_DIR", str(store.data_dir)) + sessions = lightweight_mcp.list_sessions() + assert len(sessions) == 2 + assert [s["session_id"] for s in lightweight_mcp.list_sessions(source="codex")] == [ + "codex:test-id" + ] + assert len(lightweight_mcp.list_sessions(source="claude")) == 1 + assert all( + "/tmp/project" in s["project_path"] + for s in lightweight_mcp.list_sessions(project="/tmp/project") + ) + assert lightweight_mcp.list_sessions(project="no-such-project-anywhere") == [] + with pytest.raises(ValueError, match="codex"): + lightweight_mcp.list_sessions(source="gemini") + # Commits made from Codex reach the git_operations table on this path too. + with store.sqlite.connect() as conn: + commits = conn.execute( + "SELECT commit_hash FROM git_operations WHERE session_id = ?", ("codex:test-id",) + ).fetchall() + assert [row[0] for row in commits] == ["abc1234"] + hits = lightweight_mcp.search("Fix the login", session_id="codex:test-id") + assert len(hits) == 1 + assert lightweight_mcp.search("%", session_id="codex:test-id") == [] + first = lightweight_mcp.get_event_text(hits[0]["event_id"]) + assert first["text"] == "Fix the login\n[image]" + assert ( + lightweight_mcp.get_event_text(hits[0]["event_id"], offset=4)["text"] == first["text"][4:] + ) + raw = lightweight_mcp.get_event_text(hits[0]["event_id"], raw=True) + assert json.loads(raw["text"])["payload"]["role"] == "user" + assert len(lightweight_mcp.get_session_timeline("codex:test-id", limit=2)) == 2 + + +def test_sync_skips_subagent_threads_unless_asked(tmp_path): + from longhand.codex import CodexArchiveStore + + home = tmp_path / "codex" + write_rollout(home / "sessions/primary.jsonl", "primary") + write_rollout( + home / "sessions/guardian.jsonl", "guardian", source={"subagent": {"other": "guardian"}} + ) + store = CodexArchiveStore(tmp_path / "archive") + + report = sync_codex(store, home) + assert report["ingested"] == 1 + assert report["skipped_subagents"] == 1 + assert [s["session_id"] for s in store.sqlite.list_sessions()] == ["codex:primary"] + + report = sync_codex(store, home, include_subagents=True) + assert report["ingested"] == 1 + assert report["skipped_subagents"] == 0 + assert {s["session_id"] for s in store.sqlite.list_sessions()} == { + "codex:primary", + "codex:guardian", + } + + +def test_reconcile_captures_codex_rollouts_under_its_own_lock(temp_store, tmp_path, monkeypatch): + from longhand.recall import reconcile as reconcile_mod + from longhand.recall.project_fallback import claim_ingest_lock, release_ingest_lock + from longhand.recall.reconcile import run_reconcile + + # No Claude transcripts on "disk": this is the Codex-only path, and it + # must never walk a developer's real ~/.claude/projects. + monkeypatch.setattr(reconcile_mod, "discover_sessions", lambda: []) + home = tmp_path / "codex" + write_rollout(home / "sessions/primary.jsonl", "primary") + write_rollout( + home / "sessions/guardian.jsonl", "guardian", source={"subagent": {"other": "guardian"}} + ) + + report = run_reconcile(temp_store, fix=False, codex_home=home) + assert (report.codex_on_disk, report.codex_pending, report.codex_skipped_subagents) == ( + 2, + 1, + 1, + ) + assert report.codex_ingested == 0 + assert report.to_dict()["codex_pending"] == 1 + + fixed = run_reconcile(temp_store, fix=True, codex_home=home) + assert fixed.fix_applied + assert fixed.codex_ingested == 1 + assert fixed.errors == [] + assert temp_store.sqlite.get_events(session_id="codex:primary") + # The archive path stamps `archived`, never the semantic stages. + assert temp_store.sqlite.analysis_stages()[str(home / "sessions/primary.jsonl")] == "archived" + + # Reconcile owned the lock throughout and released it afterwards. + assert claim_ingest_lock(temp_store) + release_ingest_lock(temp_store) + + after = run_reconcile(temp_store, fix=False, codex_home=home) + assert (after.codex_pending, after.codex_ingested) == (0, 0) + + +def test_shared_mcp_and_codex_sync_are_registered_cli_commands(): + from typer.testing import CliRunner + + from longhand.cli import app + + runner = CliRunner() + shared = runner.invoke(app, ["shared-mcp", "--help"]) + assert shared.exit_code == 0, shared.output + assert "keyword search" in shared.output + sync = runner.invoke(app, ["codex-sync", "--help"]) + assert sync.exit_code == 0, sync.output + for default in ("16384", "20000", "50"): + assert default in sync.output, f"default {default} missing from --help" + assert "--include-subagents" in sync.output + + +def test_doctor_codex_capture_row(tmp_path, monkeypatch): + from longhand.codex import CodexArchiveStore + from longhand.setup_commands import _codex_capture_status + + store = CodexArchiveStore(tmp_path / "archive") + # No Codex home at all: no row, so a machine without Codex never sees one. + monkeypatch.setenv("CODEX_HOME", str(tmp_path / "absent")) + assert _codex_capture_status(store) is None + + home = tmp_path / "codex" + write_rollout(home / "sessions/primary.jsonl", "primary") + write_rollout( + home / "sessions/guardian.jsonl", "guardian", source={"subagent": {"other": "guardian"}} + ) + monkeypatch.setenv("CODEX_HOME", str(home)) + behind = _codex_capture_status(store) + assert "1 of 1 rollouts new or changed" in behind + assert "longhand codex-sync" in behind + assert "1 subagent thread(s) skipped" in behind + + sync_codex(store, home) + current = _codex_capture_status(store) + assert "1/1 rollouts captured" in current + assert "last capture" in current + + +def test_doctor_drift_row_names_the_codex_kind(tmp_path): + from unittest.mock import MagicMock + + from longhand.setup_commands import _transcript_format_status + from longhand.timeutil import utcnow + from longhand.types import Event, EventType + + store = MagicMock() + store.sqlite = SQLiteStore(tmp_path / "longhand.db") + entry = {"type": "event_msg", "payload": {"type": "brand_new_kind"}} + store.sqlite.insert_events( + [ + Event( + event_id="codex:drift:1", + session_id="codex:drift", + parent_event_id=None, + event_type=EventType.UNKNOWN, + sequence=1, + timestamp=utcnow(), + content=json.dumps(entry), + raw=entry, + ) + ] + ) + row = _transcript_format_status(store) + assert "event_msg/brand_new_kind ×1" in row diff --git a/tests/test_codex_shapes.py b/tests/test_codex_shapes.py new file mode 100644 index 0000000..1d1d5b1 --- /dev/null +++ b/tests/test_codex_shapes.py @@ -0,0 +1,176 @@ +"""Upstream-drift regression gate for Codex rollouts — the Codex twin of +test_transcript_shapes.py. + +Codex Desktop changes its rollout format without notice. Every record shape +we know about must be dispositioned — turned into events, consumed as session +metadata, or skipped by a CODEX_SKIP_* set with a written reason — and this +suite fails the moment a fixture line isn't. Shapes nobody has dispositioned +are preserved as `unknown` events (raw intact) so `longhand doctor` can +surface them, exactly as the Claude parser does. +""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from longhand import codex +from longhand.parser import JSONLParser +from longhand.types import EventType + +FIXTURE = Path(__file__).parent / "fixtures" / "codex_shapes" / "entries.jsonl" + + +def _fixture_entries() -> list[dict]: + return [json.loads(line) for line in FIXTURE.read_text().splitlines() if line.strip()] + + +def _describe(entry: dict) -> str: + record_type = entry.get("type") + kind = (entry.get("payload") or {}).get("type") + return f"{record_type}/{kind}" if kind else str(record_type) + + +def test_every_fixture_line_is_dispositioned(): + for entry in _fixture_entries(): + assert codex.disposition(entry) in {"handled", "skipped"}, ( + f"fixture shape {_describe(entry)!r} has no disposition — parse it in " + "CodexAdapter.convert, or add it to a CODEX_SKIP_* set with a written reason" + ) + + +def test_fixture_covers_every_skip_set_member(): + entries = _fixture_entries() + record_types = {e.get("type") for e in entries} + event_msg_kinds = { + (e.get("payload") or {}).get("type") for e in entries if e.get("type") == "event_msg" + } + missing_records = codex.CODEX_SKIP_RECORD_TYPES - record_types + assert not missing_records, ( + f"skip-set members without a fixture line: {sorted(missing_records)}" + ) + missing_kinds = codex.CODEX_SKIP_EVENT_MSG_TYPES - event_msg_kinds + assert not missing_kinds, ( + f"skipped event_msg kinds without a fixture line: {sorted(missing_kinds)}" + ) + + +def test_fixture_covers_every_handled_type(): + entries = _fixture_entries() + record_types = {e.get("type") for e in entries} + item_kinds = { + (e.get("payload") or {}).get("type") for e in entries if e.get("type") == "response_item" + } + missing_records = codex.CODEX_HANDLED_RECORD_TYPES - record_types + assert not missing_records, ( + f"handled record types without a fixture line: {sorted(missing_records)}" + ) + missing_kinds = codex.CODEX_HANDLED_ITEM_TYPES - item_kinds + assert not missing_kinds, ( + f"handled response_item kinds without a fixture line: {sorted(missing_kinds)}" + ) + + +def test_skip_and_handled_sets_are_disjoint(): + overlap = codex.CODEX_SKIP_RECORD_TYPES & codex.CODEX_HANDLED_RECORD_TYPES + assert not overlap, f"a record type cannot be both skipped and handled: {sorted(overlap)}" + + +def test_fixture_parses_with_expected_dispositions(tmp_path: Path): + """End-to-end through the real parser: canonical items become events, + mirrors and bookkeeping produce nothing, nothing lands as unknown.""" + target = tmp_path / "rollout.jsonl" + target.write_text(FIXTURE.read_text()) + + parser = JSONLParser(target) + events = list(parser.parse_events()) + + produced = {e.event_type for e in events} + assert { + EventType.USER_MESSAGE, + EventType.ASSISTANT_TEXT, + EventType.ASSISTANT_THINKING, + EventType.TOOL_CALL, + EventType.TOOL_RESULT, + EventType.SYSTEM, + } <= produced + assert EventType.UNKNOWN not in produced, "every fixture shape is dispositioned" + + # Skipped mirrors leave no blank rows behind: the only content-free rows + # are session metadata and injected instructions, which are `system`. + blank = [e for e in events if not e.content] + assert blank and all(e.event_type == EventType.SYSTEM for e in blank) + + # The event_msg copy of a message is not a second row; encrypted-only + # reasoning yields nothing; ciphertext never reaches searchable text. + assert sum(e.event_type == EventType.USER_MESSAGE for e in events) == 1 + assert sum(e.event_type == EventType.ASSISTANT_TEXT for e in events) == 1 + assert sum(e.event_type == EventType.ASSISTANT_THINKING for e in events) == 1 + assert "OPAQUE_CIPHERTEXT" not in "".join(e.content for e in events) + assert "SHAPEIMAGE" not in "".join(e.content for e in events) + + session = parser.build_session(events) + assert session.session_id == "codex:shape-codex-1" + assert session.model == "gpt-shape" + + # Git signals from a direct shell call and from an `exec` script alike. + assert {e.git_operation for e in events if e.git_operation} == {"commit", "push"} + commit = next(e for e in events if e.git_commit_hash) + assert commit.git_commit_hash == "abc1234" + push_call = next(e for e in events if e.tool_name == "exec") + assert push_call.tool_input["command"] == "git push origin main" + + +def test_unknown_shapes_are_preserved_not_dropped(): + adapter = codex.CodexAdapter({"id": "drift"}) + for entry in ( + {"type": "event_msg", "payload": {"type": "brand_new_kind", "detail": 1}}, + {"type": "response_item", "payload": {"type": "brand_new_item"}}, + {"type": "brand_new_record", "payload": {}}, + ): + assert codex.disposition(entry) == "unknown" + (event,) = adapter.convert(entry, 7) + assert event.event_type == EventType.UNKNOWN + assert event.raw == entry + assert event.content, "unknown events carry a searchable stub, like the Claude parser's" + + +def test_subagent_rollouts_are_recognized(tmp_path: Path): + primary, guardian = FIXTURE.read_text().splitlines()[:2] + (tmp_path / "primary.jsonl").write_text(primary + "\n") + (tmp_path / "guardian.jsonl").write_text(guardian + "\n") + (tmp_path / "empty.jsonl").write_text("") + + assert codex.is_subagent_rollout(codex.read_session_meta(tmp_path / "primary.jsonl")) is False + assert codex.is_subagent_rollout(codex.read_session_meta(tmp_path / "guardian.jsonl")) is True + assert codex.read_session_meta(tmp_path / "empty.jsonl") is None + assert codex.is_subagent_rollout(None) is False + + +@pytest.mark.parametrize( + ("tool_input", "expected"), + [ + ({"cmd": "git status"}, "git status"), + ({"cmd": ["git", "status", "--short"]}, "git status --short"), + ({"command": "ls"}, "ls"), + ( + { + "input": 'text(await tools.exec_command({cmd:"git status", max_output_tokens: 500}));' + }, + "git status", + ), + ({"input": "text(await tools.exec_command({cmd: 'pwd'}));"}, "pwd"), + ({"input": 'text(await tools.exec_command({cmd:"echo \\"hi\\""}));'}, 'echo "hi"'), + ( + {"input": 'a({cmd:"git add -A"}); b({cmd:"git commit -m x"});'}, + "git add -A\ngit commit -m x", + ), + ({"input": 'text(await tools.apply_patch("*** Begin Patch"));'}, ""), + ({}, ""), + (None, ""), + ], +) +def test_shell_commands_from_tool_input(tool_input, expected): + assert codex.shell_commands_from_tool_input(tool_input) == expected From b621d97a204e98946f6a6125a887015a336fe814 Mon Sep 17 00:00:00 2001 From: nathan nelson Date: Mon, 7 Sep 2026 16:21:26 -0600 Subject: [PATCH 2/2] test: assert codex-sync options by introspection, not rendered help Rich wraps and hyphenates long option names at the 80-column width CI runs with, so `--include-subagents` was not findable in the rendered text there (all five Linux legs, 578/579 otherwise green). Check the registered command's declared defaults and flags instead; keep the `--help` exit-code checks. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01LzrMzN4dfuJJKc4UZP6SFb --- tests/test_codex.py | 30 ++++++++++++++++++++++-------- 1 file changed, 22 insertions(+), 8 deletions(-) diff --git a/tests/test_codex.py b/tests/test_codex.py index 116f09e..569ef64 100644 --- a/tests/test_codex.py +++ b/tests/test_codex.py @@ -394,19 +394,33 @@ def test_reconcile_captures_codex_rollouts_under_its_own_lock(temp_store, tmp_pa def test_shared_mcp_and_codex_sync_are_registered_cli_commands(): + import inspect + from typer.testing import CliRunner from longhand.cli import app + commands = { + (info.name or info.callback.__name__.replace("_", "-")): info + for info in app.registered_commands + } + assert "shared-mcp" in commands + assert "codex-sync" in commands + runner = CliRunner() - shared = runner.invoke(app, ["shared-mcp", "--help"]) - assert shared.exit_code == 0, shared.output - assert "keyword search" in shared.output - sync = runner.invoke(app, ["codex-sync", "--help"]) - assert sync.exit_code == 0, sync.output - for default in ("16384", "20000", "50"): - assert default in sync.output, f"default {default} missing from --help" - assert "--include-subagents" in sync.output + for name in ("shared-mcp", "codex-sync"): + result = runner.invoke(app, [name, "--help"]) + assert result.exit_code == 0, result.output + + # Defaults and flags by introspection — rendered help wraps and hyphenates + # long option names at narrow terminal widths, so it is not a stable oracle. + signature = inspect.signature(commands["codex-sync"].callback) + options = {name: param.default for name, param in signature.parameters.items()} + assert options["limit"].default == 50 + assert options["max_file_kb"].default == 16384 + assert options["max_events"].default == 20000 + assert options["include_subagents"].default is False + assert "--include-subagents" in options["include_subagents"].param_decls def test_doctor_codex_capture_row(tmp_path, monkeypatch):