From fe49c495e96e9605838aa63169bc2ad6a2b96c86 Mon Sep 17 00:00:00 2001 From: leo Date: Sun, 20 Sep 2026 01:35:07 -0400 Subject: [PATCH 1/9] feat(viz): rebuild the dashboard as the project site with the agent record The dashboard is now a static site that reads one record layout from three sources: the live server next to a project, a snapshot written by automil viz export, or a local server reached from the hosted site through an SSH tunnel. The record is built from the project's own files (graph.json, the per-node archive, the orchestrator log, the activity journal) and the Claude Code session transcripts, which the framework now stores under automil/sessions// on SessionEnd and through automil activity store-sessions (the launcher's exit trap uses it; the shell script is gone). Views: lineage tree with the value chart and a node drawer that states the keep/discard verdict in words, the discovery timeline (agent actions, experiment runs, best value), the transcript reader with tool calls, results and subagents linked to the nodes they created, the nodes table, the notes, the runs page with the local-server connection, the home page and the remote-access page. Validation-only projection is one enforcement point; nothing under archive//certify is ever read. Co-Authored-By: Claude Fable 5.1 --- benchmarks/campaigns/preprint_130/README.md | 20 +- .../scripts/slurm/store_session_record.sh | 42 -- .../slurm/submit_discovery_campaign.sh | 5 +- benchmarks/src/autobench/campaign.py | 2 +- src/automil/cells/activity.py | 42 ++ src/automil/cli/activity.py | 68 ++ src/automil/cli/viz.py | 47 +- src/automil/session_record.py | 229 ++++++ src/automil/templates/.gitignore.j2 | 1 + src/automil/trajectory/redactor.py | 15 +- src/automil/viz/clock.py | 170 +++++ src/automil/viz/export.py | 214 ++++++ src/automil/viz/hostinfo.py | 47 ++ src/automil/viz/live.py | 175 +++++ src/automil/viz/record.py | 429 ++++++++++++ src/automil/viz/record_files.py | 260 +++++++ src/automil/viz/record_graph.py | 226 ++++++ src/automil/viz/record_routes.py | 144 ++++ src/automil/viz/record_sessions.py | 252 +++++++ src/automil/viz/record_timeline.py | 136 ++++ src/automil/viz/server.py | 88 ++- src/automil/viz/static/css/base.css | 371 ++++++++++ src/automil/viz/static/css/layout.css | 320 +++++++++ src/automil/viz/static/css/tokens.css | 156 +++++ src/automil/viz/static/css/views.css | 642 +++++++++++++++++ .../viz/static/fonts/IBMPlexMono-Medium.woff2 | Bin 0 -> 46724 bytes .../static/fonts/IBMPlexMono-Regular.woff2 | Bin 0 -> 45640 bytes .../viz/static/fonts/IBMPlexSans-Italic.woff2 | Bin 0 -> 67220 bytes .../viz/static/fonts/IBMPlexSans-Medium.woff2 | Bin 0 -> 66740 bytes .../static/fonts/IBMPlexSans-Regular.woff2 | Bin 0 -> 63020 bytes .../static/fonts/IBMPlexSans-SemiBold.woff2 | Bin 0 -> 67060 bytes src/automil/viz/static/img/fig1_overview.png | Bin 0 -> 633664 bytes src/automil/viz/static/index.html | 215 ++---- src/automil/viz/static/js/core.js | 248 +++++++ src/automil/viz/static/js/main.js | 142 ++++ src/automil/viz/static/js/markdown.js | 41 ++ src/automil/viz/static/js/source.js | 162 +++++ src/automil/viz/static/js/views/agent.js | 279 ++++++++ src/automil/viz/static/js/views/home.js | 114 +++ src/automil/viz/static/js/views/node.js | 179 +++++ src/automil/viz/static/js/views/nodes.js | 87 +++ src/automil/viz/static/js/views/notes.js | 38 + src/automil/viz/static/js/views/progress.js | 113 +++ src/automil/viz/static/js/views/remote.js | 41 ++ src/automil/viz/static/js/views/runs.js | 94 +++ src/automil/viz/static/js/views/timeline.js | 179 +++++ src/automil/viz/static/js/views/tree.js | 186 +++++ .../viz/static/vendor/3d-force-graph.min.js | 5 - src/automil/viz/static/vendor/marked.min.js | 69 ++ src/automil/viz/static/vendor/purify.min.js | 3 + .../viz/static/vendor/three-spritetext.min.js | 2 - src/automil/viz/static/vendor/three.min.js | 7 - src/automil/viz/subagents.py | 110 +++ src/automil/viz/transcript.py | 662 ++++++++++++++++++ src/automil/viz/transcript_commands.py | 97 +++ tests/cells/test_activity.py | 42 ++ tests/test_activity_cli.py | 72 ++ tests/test_session_record.py | 93 +++ tests/test_viz_export_cli.py | 42 ++ tests/trajectory/test_redactor.py | 13 + tests/viz/conftest.py | 393 +++++++++++ tests/viz/test_app_js_metrics_reader.py | 67 -- tests/viz/test_clock.py | 78 +++ tests/viz/test_export.py | 114 +++ tests/viz/test_firewall_forged.py | 116 +++ tests/viz/test_hostinfo.py | 22 + tests/viz/test_live_tail.py | 102 +++ tests/viz/test_record.py | 110 +++ tests/viz/test_record_graph.py | 142 ++++ tests/viz/test_record_routes.py | 134 ++++ tests/viz/test_record_sessions.py | 106 +++ tests/viz/test_record_timeline.py | 74 ++ tests/viz/test_subagents.py | 51 ++ tests/viz/test_transcript.py | 249 +++++++ tests/viz/test_transcript_commands.py | 61 ++ 75 files changed, 8939 insertions(+), 316 deletions(-) delete mode 100755 benchmarks/scripts/slurm/store_session_record.sh create mode 100644 src/automil/session_record.py create mode 100644 src/automil/viz/clock.py create mode 100644 src/automil/viz/export.py create mode 100644 src/automil/viz/hostinfo.py create mode 100644 src/automil/viz/live.py create mode 100644 src/automil/viz/record.py create mode 100644 src/automil/viz/record_files.py create mode 100644 src/automil/viz/record_graph.py create mode 100644 src/automil/viz/record_routes.py create mode 100644 src/automil/viz/record_sessions.py create mode 100644 src/automil/viz/record_timeline.py create mode 100644 src/automil/viz/static/css/base.css create mode 100644 src/automil/viz/static/css/layout.css create mode 100644 src/automil/viz/static/css/tokens.css create mode 100644 src/automil/viz/static/css/views.css create mode 100644 src/automil/viz/static/fonts/IBMPlexMono-Medium.woff2 create mode 100644 src/automil/viz/static/fonts/IBMPlexMono-Regular.woff2 create mode 100644 src/automil/viz/static/fonts/IBMPlexSans-Italic.woff2 create mode 100644 src/automil/viz/static/fonts/IBMPlexSans-Medium.woff2 create mode 100644 src/automil/viz/static/fonts/IBMPlexSans-Regular.woff2 create mode 100644 src/automil/viz/static/fonts/IBMPlexSans-SemiBold.woff2 create mode 100644 src/automil/viz/static/img/fig1_overview.png create mode 100644 src/automil/viz/static/js/core.js create mode 100644 src/automil/viz/static/js/main.js create mode 100644 src/automil/viz/static/js/markdown.js create mode 100644 src/automil/viz/static/js/source.js create mode 100644 src/automil/viz/static/js/views/agent.js create mode 100644 src/automil/viz/static/js/views/home.js create mode 100644 src/automil/viz/static/js/views/node.js create mode 100644 src/automil/viz/static/js/views/nodes.js create mode 100644 src/automil/viz/static/js/views/notes.js create mode 100644 src/automil/viz/static/js/views/progress.js create mode 100644 src/automil/viz/static/js/views/remote.js create mode 100644 src/automil/viz/static/js/views/runs.js create mode 100644 src/automil/viz/static/js/views/timeline.js create mode 100644 src/automil/viz/static/js/views/tree.js delete mode 100644 src/automil/viz/static/vendor/3d-force-graph.min.js create mode 100644 src/automil/viz/static/vendor/marked.min.js create mode 100644 src/automil/viz/static/vendor/purify.min.js delete mode 100644 src/automil/viz/static/vendor/three-spritetext.min.js delete mode 100644 src/automil/viz/static/vendor/three.min.js create mode 100644 src/automil/viz/subagents.py create mode 100644 src/automil/viz/transcript.py create mode 100644 src/automil/viz/transcript_commands.py create mode 100644 tests/test_session_record.py create mode 100644 tests/test_viz_export_cli.py create mode 100644 tests/viz/conftest.py delete mode 100644 tests/viz/test_app_js_metrics_reader.py create mode 100644 tests/viz/test_clock.py create mode 100644 tests/viz/test_export.py create mode 100644 tests/viz/test_firewall_forged.py create mode 100644 tests/viz/test_hostinfo.py create mode 100644 tests/viz/test_live_tail.py create mode 100644 tests/viz/test_record.py create mode 100644 tests/viz/test_record_graph.py create mode 100644 tests/viz/test_record_routes.py create mode 100644 tests/viz/test_record_sessions.py create mode 100644 tests/viz/test_record_timeline.py create mode 100644 tests/viz/test_subagents.py create mode 100644 tests/viz/test_transcript.py create mode 100644 tests/viz/test_transcript_commands.py diff --git a/benchmarks/campaigns/preprint_130/README.md b/benchmarks/campaigns/preprint_130/README.md index d6af56bc..0472d061 100644 --- a/benchmarks/campaigns/preprint_130/README.md +++ b/benchmarks/campaigns/preprint_130/README.md @@ -379,15 +379,21 @@ only the fresh folds); a baseline whose retry cached every fold has no timing and is submitted with `--cell --e5-hours `. Nothing from a rehearsal set is mirrored, frozen into the campaign selections, or certified. -**Session record.** Whatever the outcome, the job's exit copies the -runtime's own transcript of the cell's session, +**Session record.** The runtime's own transcript of the cell's session, `~/.claude/projects//.jsonl` (every user, assistant and tool message) with its sidecar directory (subagent transcripts, fetched -tool results), from the submitter's home into `/operator/session/` -(`store_session_record.sh `; every session the activity journal -saw, including one that failed before binding; idempotent; run it by hand -for a cell whose job predates it). The home copy is pruned by the runtime after -its cleanup period, so the cell root is the record, beside +tool results), is copied from the submitter's home into +`/automil/sessions//` (`transcript.jsonl`, +`subagents/`, `record.json`). The framework's `SessionEnd` hook stores it as +the session ends, and the job's exit runs +`automil activity store-sessions --root ` for every session the +activity journal saw (including one that failed before binding; idempotent; +run it by hand for a cell whose job predates it). Records made before this +layout sit at `/operator/session/.jsonl` and move with +`mkdir -p automil/sessions/ && mv operator/session/.jsonl automil/sessions//transcript.jsonl && mv operator/session//* automil/sessions//`. +`automil viz start` in the cell root shows the session beside the tree. The +home copy is pruned by the runtime after its cleanup period, so the cell root +is the record, beside `automil/.activity.jsonl` (session open, bind and end), `automil/graph.json`, `automil/results.tsv`, `automil/learnings.md`, `automil/plan.md`, `automil/orchestrator/orchestrator.log`, `operator/usage_before.txt`, diff --git a/benchmarks/scripts/slurm/store_session_record.sh b/benchmarks/scripts/slurm/store_session_record.sh deleted file mode 100755 index 9a3e4aa4..00000000 --- a/benchmarks/scripts/slurm/store_session_record.sh +++ /dev/null @@ -1,42 +0,0 @@ -#!/bin/bash -# Store the runtime's own record of a cell's agent session in the cell root. -# -# Claude Code writes every session's transcript (each user, assistant and -# tool message) to ~/.claude/projects//.jsonl, with a -# sidecar directory / holding subagent transcripts and fetched -# tool results, in the HOME of the user who ran it, and prunes them after -# its cleanup period (30 days by default). The cell root is the campaign's -# record, so this copies both into /operator/session/ for every -# session the cell's activity journal saw (opened, bound or ended: a session -# that failed before binding is still a session). Idempotent; group-readable. -# -# Usage: store_session_record.sh -# The discovery job runs it on exit (any outcome); run it by hand for a -# cell whose job predates it. -set -u -ROOT="${1:?usage: store_session_record.sh }" -JOURNAL="$ROOT/automil/.activity.jsonl" -[ -s "$JOURNAL" ] || { echo "store_session_record: no activity journal in $ROOT"; exit 1; } -DEST="$ROOT/operator/session" -rc=0 -found=0 -# The journal is written by campaign_operate, one JSON event per line. -for sid in $(grep -oE '"session_id":"[0-9a-f-]+"' "$JOURNAL" | cut -d'"' -f4 | awk '!seen[$0]++'); do - found=1 - src=$(find "$HOME/.claude/projects" -maxdepth 2 -name "$sid.jsonl" 2>/dev/null | head -1) - if [ -z "$src" ]; then - echo "store_session_record: transcript $sid.jsonl not found under $HOME/.claude/projects"; rc=1; continue - fi - mkdir -p "$DEST" || { rc=1; continue; } - cp -p "$src" "$DEST/" || { echo "store_session_record: copy of $sid.jsonl failed"; rc=1; continue; } - if [ -d "${src%.jsonl}" ]; then - cp -rp "${src%.jsonl}" "$DEST/" || { echo "store_session_record: copy of the $sid sidecar failed"; rc=1; } - fi - # The home copy carries the user's private group; the record takes the - # cell root's group so the team can read it and it bills the project. - chgrp -R "$(ls -ld "$ROOT" | awk '{print $4}')" "$DEST" 2>/dev/null || true - chmod -R g+rX "$DEST" - echo "store_session_record: $sid -> $DEST ($(wc -l < "$src") transcript lines, $(find "${src%.jsonl}" -type f 2>/dev/null | wc -l) sidecar files)" -done -[ "$found" = 1 ] || { echo "store_session_record: no session in $JOURNAL"; exit 1; } -exit $rc diff --git a/benchmarks/scripts/slurm/submit_discovery_campaign.sh b/benchmarks/scripts/slurm/submit_discovery_campaign.sh index 3119b7d2..2d369a9c 100644 --- a/benchmarks/scripts/slurm/submit_discovery_campaign.sh +++ b/benchmarks/scripts/slurm/submit_discovery_campaign.sh @@ -117,8 +117,11 @@ LOG="$LOG_DIR/${CELL}.log" OPDIR="$RUNTIME/$CELL/operator"; mkdir -p "$OPDIR" # The runtime's transcript of the session into the cell root, whatever the # outcome (the home copy is pruned by the runtime after its cleanup period). +# The SessionEnd hook stores it as the session ends; this covers a runtime +# that was killed first. Same implementation, one storage layout: +# /automil/sessions//. store_session_record() { # cell - "$PROJECT_DIR/benchmarks/scripts/slurm/store_session_record.sh" "$RUNTIME/$1" >> "$LOG" 2>&1 \ + uv run --frozen --no-sync automil activity store-sessions --root "$RUNTIME/$1" >> "$LOG" 2>&1 \ || echo "[$1] session record not stored (see $LOG)" } trap 'store_session_record "$CELL"; normalize_cell_modes "$CELL"' EXIT diff --git a/benchmarks/src/autobench/campaign.py b/benchmarks/src/autobench/campaign.py index 64013ea3..af2b6844 100644 --- a/benchmarks/src/autobench/campaign.py +++ b/benchmarks/src/autobench/campaign.py @@ -1221,7 +1221,7 @@ def materialize_discovery_cells( json.dumps(cell, indent=2, sort_keys=True) + "\n" ) (staging_adir / ".gitignore").write_text( - "graph.json\nresults.tsv\nresult.json\norchestrator/\ncells/\n" + "graph.json\nresults.tsv\nresult.json\norchestrator/\ncells/\nsessions/\n" ".activity.jsonl\n.activity.samples.json\n.activity.lock\n" ".automil_active\n.automil_worktrees/\n*.log\n*.pid\n" ) diff --git a/src/automil/cells/activity.py b/src/automil/cells/activity.py index 60205da4..f27bcfb7 100644 --- a/src/automil/cells/activity.py +++ b/src/automil/cells/activity.py @@ -418,6 +418,48 @@ def ingest_prometheus_metrics( return observed_sessions +@dataclass(frozen=True) +class JournalSession: + """One journaled runtime session, as the dashboard and the record store see it.""" + + session_id: str + cell_id: str | None + opened_at: float + ended_at: float | None + ended_by: str | None + binding_sha256: str | None + + +def journal_sessions(automil_dir: Path | str) -> tuple[JournalSession, ...]: + """Every session the journal knows, in the order they were opened. + + ``ended_by`` is ``None`` while a session is open, ``"hook"`` when the + runtime's own SessionEnd hook closed it, and the recorded ``finalized_by`` + marker for an attested close. A corrupt journal raises ``ActivityError``. + """ + root = Path(automil_dir) + if not (root / ACTIVITY_JOURNAL_FILENAME).exists(): + return () + with _activity_lock(root, exclusive=False): + replay = _read_journal_unlocked(root) + sessions = [] + for session in replay.sessions.values(): + ended_by = None + if session.end_event is not None: + ended_by = session.end_event.get("finalized_by") or "hook" + sessions.append( + JournalSession( + session_id=session.session_id, + cell_id=session.cell_id, + opened_at=session.opened_at, + ended_at=session.ended_at, + ended_by=ended_by, + binding_sha256=session.binding_sha256, + ) + ) + return tuple(sessions) + + def read_activity_report( automil_dir: Path | str, cell_id: str, diff --git a/src/automil/cli/activity.py b/src/automil/cli/activity.py index e1fbc117..a3b29264 100644 --- a/src/automil/cli/activity.py +++ b/src/automil/cli/activity.py @@ -38,6 +38,13 @@ def ingest() -> None: record_hook_event, ) + store_warning = None + if payload.get("hook_event_name") == "SessionEnd": + # The transcript is the only record of the agent's reasoning and the + # runtime prunes its own copy; copy it first, so accounting trouble + # below never costs the record. + store_warning = _store_transcript(automil_dir, payload) + try: if payload.get("hook_event_name") == "SessionEnd": # Scrape and record under one activity lock: the old two-step @@ -93,7 +100,68 @@ def ingest() -> None: if payload.get("hook_event_name") == "SessionStart": observe_activity_metrics(automil_dir) except ActivityError as exc: + if store_warning: + click.echo(store_warning, err=True) raise click.ClickException(str(exc)) from exc + if store_warning: + click.echo(store_warning, err=True) + + +def _store_transcript(automil_dir, payload: dict) -> str | None: + """Copy the session transcript into the project; a problem is a warning.""" + from pathlib import Path + + from automil.session_record import SessionRecordError, store_session_record + + transcript_path = payload.get("transcript_path") + if not isinstance(transcript_path, str) or not transcript_path: + return "session record not stored: the hook payload carried no transcript_path" + try: + outcome = store_session_record(automil_dir, payload.get("session_id"), Path(transcript_path)) + except (SessionRecordError, OSError) as exc: + return f"session record not stored: {exc}" + if outcome.action == "missing": + return f"session record not stored: {outcome.detail}" + return None + + +@activity.command("store-sessions", hidden=True) +@click.option( + "--root", "root", default=None, type=click.Path(), + help="Project root (or its automil/ dir). Default: discover from the working directory.", +) +@click.option( + "--claude-config-dir", "config_dir", default=None, type=click.Path(), + help="The runtime's config dir holding projects//.jsonl (default: CLAUDE_CONFIG_DIR or ~/.claude).", +) +def store_sessions(root: str | None, config_dir: str | None) -> None: + """Copy every journaled session's transcript into automil/sessions/. + + The SessionEnd hook stores a session as it ends; this covers sessions + whose runtime was killed first. Exit status 1 when any session has no + transcript left to copy. + """ + from pathlib import Path + + from automil.session_record import store_journaled_sessions + + if root is not None: + candidate = Path(root).resolve() + automil_dir = candidate if candidate.name == "automil" else candidate / "automil" + if not (automil_dir / "config.yaml").exists(): + raise click.ClickException(f"{root}: no automil/config.yaml found") + else: + automil_dir = _find_automil_dir() + outcomes = store_journaled_sessions(automil_dir, config_dir=Path(config_dir) if config_dir else None) + if not outcomes: + click.echo(f"no session in {automil_dir / '.activity.jsonl'}") + raise SystemExit(1) + missing = False + for outcome in outcomes: + click.echo(f"{outcome.session_id}: {outcome.action} ({outcome.detail})") + missing = missing or outcome.action == "missing" + if missing: + raise SystemExit(1) @activity.command("close", hidden=True) diff --git a/src/automil/cli/viz.py b/src/automil/cli/viz.py index 0fe321cb..263b588d 100644 --- a/src/automil/cli/viz.py +++ b/src/automil/cli/viz.py @@ -27,8 +27,12 @@ def viz_group(): "only on trusted networks — the dashboard exposes PIDs and node " "descriptions and has no auth.", ) -def viz_start(port: int | None, host: str | None): - """Start the 3D visualization dashboard.""" +@click.option( + "--tz", "tz_name", default=None, + help="IANA zone of the host that ran the project, for a record copied from elsewhere (default: this host's zone).", +) +def viz_start(port: int | None, host: str | None, tz_name: str | None): + """Start the dashboard: the site with this project as its run.""" from automil.viz.server import DEFAULT_PORT, cmd_start # noqa: PLC0415 adir = _find_automil_dir() # Port resolution order: explicit --port > viz.port in config > DEFAULT_PORT (8420). @@ -47,7 +51,44 @@ def viz_start(port: int | None, host: str | None): except Exception: # noqa: BLE001 cfg_port = None port = cfg_port if cfg_port is not None else DEFAULT_PORT - cmd_start(port=port, project_root=adir.parent, host=host) + cmd_start(port=port, project_root=adir.parent, host=host, tz_name=tz_name) + + +@viz_group.command("export") +@click.option("--out", "out", required=True, type=click.Path(), help="Directory for the site (index.html, static/, record/).") +@click.option("--run-id", "run_id", default=None, help="Run id in the record (default: the project name).") +@click.option("--title", default=None, help="Run title shown on the site (default: the project description).") +@click.option("--tz", "tz_name", default=None, help="IANA zone of the host that ran the project (default: this host's zone).") +@click.option("--force", is_flag=True, help="Write into a non-empty directory that holds no record.") +@click.option("--frontend/--no-frontend", "with_frontend", default=True, help="Copy the page and static assets next to record/ (default) or write only record/.") +@click.option("--single-file", "single", default=None, type=click.Path(), help="Also write one self-contained HTML file.") +def viz_export(out: str, run_id: str | None, title: str | None, tz_name: str | None, force: bool, with_frontend: bool, single: str | None): + """Write this project's record (and the site around it) to a directory.""" + from pathlib import Path # noqa: PLC0415 + + from automil.viz.clock import host_clock # noqa: PLC0415 + from automil.viz.export import ExportError, export_site, single_file # noqa: PLC0415 + from automil.viz.record import RunSource # noqa: PLC0415 + + adir = _find_automil_dir() + try: + clock = host_clock(tz_name=tz_name) + except ValueError as exc: + raise click.ClickException(str(exc)) from exc + source = RunSource(adir, clock, run_id=run_id, title=title, mode="static") + try: + summaries = export_site([source], Path(out), force=force, with_frontend=with_frontend) + except ExportError as exc: + raise click.ClickException(str(exc)) from exc + for summary in summaries: + click.echo( + f"exported {summary.run_id}: {summary.nodes} nodes, {summary.sessions} session(s), " + f"{summary.files} files -> {summary.out}" + ) + for warning in summary.warnings: + click.echo(f"warning: {warning}", err=True) + if single: + click.echo(f"single file -> {single_file(Path(single), Path(out))}") @viz_group.command("stop") diff --git a/src/automil/session_record.py b/src/automil/session_record.py new file mode 100644 index 00000000..aab792ff --- /dev/null +++ b/src/automil/session_record.py @@ -0,0 +1,229 @@ +"""Store the runtime's transcript of an agent session inside the project. + +Claude Code writes every session to ``/projects//.jsonl`` +with a sidecar directory ``/`` (subagent transcripts, fetched tool +results) in the user's home, and prunes them after its cleanup period. The +project is the record, so the session is copied to +``automil/sessions//{transcript.jsonl, subagents/..., record.json}``: +once by the ``SessionEnd`` hook, and again by ``automil activity +store-sessions`` for any session the hook missed. Both call the one function +here, and copying is idempotent. +""" +from __future__ import annotations + +import hashlib +import json +import os +import re +import shutil +import tempfile +from dataclasses import dataclass +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Literal, Mapping + +SESSIONS_DIRNAME = "sessions" +TRANSCRIPT_FILENAME = "transcript.jsonl" +RECORD_FILENAME = "record.json" +_SESSION_ID = re.compile(r"^[0-9a-f-]{8,64}$") + + +class SessionRecordError(ValueError): + """A session record request that cannot be honoured as stated.""" + + +@dataclass(frozen=True) +class StoredSession: + session_id: str + dir: Path + transcript: Path + sidecar: Path | None + manifest: Mapping[str, Any] + + +@dataclass(frozen=True) +class StoreOutcome: + session_id: str + action: Literal["stored", "unchanged", "missing"] + path: Path | None + detail: str + + +def claude_config_dir(env: Mapping[str, str] | None = None) -> Path: + """Where the runtime keeps its files: ``CLAUDE_CONFIG_DIR`` or ``~/.claude``.""" + environment = os.environ if env is None else env + configured = environment.get("CLAUDE_CONFIG_DIR") + if configured: + return Path(configured).expanduser() + return Path.home() / ".claude" + + +def validate_session_id(session_id: object) -> str: + if not isinstance(session_id, str) or not _SESSION_ID.match(session_id): + raise SessionRecordError(f"not a session id: {session_id!r}") + return session_id + + +def locate_home_transcript(session_id: str, *, config_dir: Path | None = None) -> Path | None: + """The runtime's own copy of a session, newest first when several exist.""" + validate_session_id(session_id) + projects = (config_dir or claude_config_dir()) / "projects" + if not projects.is_dir(): + return None + candidates = [] + for slug in projects.iterdir(): + candidate = slug / f"{session_id}.jsonl" + if candidate.is_file(): + candidates.append(candidate) + if not candidates: + return None + return max(candidates, key=lambda p: p.stat().st_mtime) + + +def session_dir(automil_dir: Path, session_id: str) -> Path: + return Path(automil_dir) / SESSIONS_DIRNAME / validate_session_id(session_id) + + +def _sha256(path: Path) -> tuple[str, int, int]: + digest = hashlib.sha256() + lines = 0 + size = 0 + with path.open("rb") as fh: + for chunk in iter(lambda: fh.read(1 << 20), b""): + digest.update(chunk) + lines += chunk.count(b"\n") + size += len(chunk) + return digest.hexdigest(), size, lines + + +def _ends_with_newline(path: Path) -> bool: + with path.open("rb") as fh: + try: + fh.seek(-1, os.SEEK_END) + except OSError: + return False + return fh.read(1) == b"\n" + + +def _read_manifest(path: Path) -> dict[str, Any] | None: + try: + payload = json.loads(path.read_text(encoding="utf-8")) + except (OSError, UnicodeError, json.JSONDecodeError): + return None + return payload if isinstance(payload, dict) else None + + +def _copy_file_atomic(source: Path, destination: Path) -> None: + destination.parent.mkdir(parents=True, exist_ok=True) + fd, tmp_name = tempfile.mkstemp(prefix=".tmp-", dir=str(destination.parent)) + os.close(fd) + tmp = Path(tmp_name) + try: + shutil.copyfile(source, tmp) + os.chmod(tmp, 0o666 & ~_current_umask()) + os.replace(tmp, destination) + finally: + if tmp.exists(): + tmp.unlink() + + +def _current_umask() -> int: + from automil.runtime_helpers import _current_umask as read_umask + + return read_umask() + + +def _copy_sidecar(source: Path, destination: Path) -> int: + """Replace ``destination``'s entries with ``source``'s; returns the file count.""" + destination.mkdir(parents=True, exist_ok=True) + count = 0 + for entry in sorted(source.iterdir()): + target = destination / entry.name + if entry.is_dir(): + tmp = destination / f".tmp-{entry.name}" + if tmp.exists(): + shutil.rmtree(tmp) + shutil.copytree(entry, tmp) + if target.exists(): + shutil.rmtree(target) + os.replace(tmp, target) + count += sum(1 for p in target.rglob("*") if p.is_file()) + elif entry.is_file(): + _copy_file_atomic(entry, target) + count += 1 + return count + + +def store_session_record(automil_dir: Path, session_id: str, transcript_path: Path) -> StoreOutcome: + """Copy one session's transcript and sidecar into the project. Idempotent.""" + session_id = validate_session_id(session_id) + source = Path(transcript_path) + if source.name != f"{session_id}.jsonl": + raise SessionRecordError(f"{source} is not the transcript of session {session_id}") + if not source.is_file(): + return StoreOutcome(session_id, "missing", None, f"transcript not found at {source}") + target_dir = session_dir(automil_dir, session_id) + manifest_path = target_dir / RECORD_FILENAME + digest, size, lines = _sha256(source) + previous = _read_manifest(manifest_path) + sidecar_source = source.with_suffix("") + sidecar_count = sum(1 for p in sidecar_source.rglob("*") if p.is_file()) if sidecar_source.is_dir() else 0 + if ( + previous is not None + and previous.get("sha256") == digest + and previous.get("bytes") == size + and previous.get("sidecar_files") == sidecar_count + and (target_dir / TRANSCRIPT_FILENAME).is_file() + ): + return StoreOutcome(session_id, "unchanged", target_dir, "record already current") + target_dir.mkdir(parents=True, exist_ok=True) + _copy_file_atomic(source, target_dir / TRANSCRIPT_FILENAME) + copied = _copy_sidecar(sidecar_source, target_dir) if sidecar_source.is_dir() else 0 + manifest = { + "schema": 1, + "session_id": session_id, + "source_path": str(source), + "stored_at": datetime.now(timezone.utc).isoformat(timespec="milliseconds").replace("+00:00", "Z"), + "bytes": size, + "lines": lines, + "sha256": digest, + "complete": _ends_with_newline(source), + "sidecar_files": copied, + } + fd, tmp_name = tempfile.mkstemp(prefix=".tmp-", dir=str(target_dir)) + with os.fdopen(fd, "w", encoding="utf-8") as fh: + json.dump(manifest, fh, indent=2, sort_keys=True) + fh.write("\n") + os.chmod(tmp_name, 0o666 & ~_current_umask()) + os.replace(tmp_name, manifest_path) + return StoreOutcome(session_id, "stored", target_dir, f"{lines} lines, {copied} sidecar files") + + +def store_journaled_sessions(automil_dir: Path, *, config_dir: Path | None = None) -> tuple[StoreOutcome, ...]: + """Store every session the activity journal knows; never raises per session.""" + from automil.cells.activity import journal_sessions + + outcomes = [] + for session in journal_sessions(automil_dir): + source = locate_home_transcript(session.session_id, config_dir=config_dir) + if source is None: + outcomes.append(StoreOutcome(session.session_id, "missing", None, "no transcript under the runtime's projects directory")) + continue + outcomes.append(store_session_record(automil_dir, session.session_id, source)) + return tuple(outcomes) + + +def read_stored_sessions(automil_dir: Path) -> tuple[StoredSession, ...]: + """Every stored session record under ``automil/sessions``.""" + folder = Path(automil_dir) / SESSIONS_DIRNAME + if not folder.is_dir(): + return () + records = [] + for entry in sorted(folder.iterdir()): + transcript = entry / TRANSCRIPT_FILENAME + if not entry.is_dir() or not _SESSION_ID.match(entry.name) or not transcript.is_file(): + continue + manifest = _read_manifest(entry / RECORD_FILENAME) or {} + sidecar = entry if (entry / "subagents").is_dir() else None + records.append(StoredSession(entry.name, entry, transcript, sidecar, manifest)) + return tuple(records) diff --git a/src/automil/templates/.gitignore.j2 b/src/automil/templates/.gitignore.j2 index c42ceade..f05cd3c7 100644 --- a/src/automil/templates/.gitignore.j2 +++ b/src/automil/templates/.gitignore.j2 @@ -3,6 +3,7 @@ graph.json results.tsv result.json orchestrator/ +sessions/ .automil_active .activity.jsonl .activity.samples.json diff --git a/src/automil/trajectory/redactor.py b/src/automil/trajectory/redactor.py index ca6b10e8..25fcf7b3 100644 --- a/src/automil/trajectory/redactor.py +++ b/src/automil/trajectory/redactor.py @@ -68,10 +68,21 @@ def _held_out_ids() -> frozenset: return _held_out_ids_cached(mtime) -def redact(s: str) -> str: - """Apply all compiled redaction patterns to a string. Returns a new string.""" +def redact_secrets(s: str) -> str: + """Mask credential-shaped substrings. Pure: no project lookup, no cwd. + + The dashboard record builders run outside any project cwd and must not + consult graph.json through ``_find_automil_dir``; they need only the + pattern half of :func:`redact`. + """ for pattern, replacement in _PATTERNS: s = pattern.sub(replacement, s) + return s + + +def redact(s: str) -> str: + """Apply all compiled redaction patterns to a string. Returns a new string.""" + s = redact_secrets(s) # D-139 dynamic: redact held-out node IDs to placeholder held_out = _held_out_ids() if held_out: diff --git a/src/automil/viz/clock.py b/src/automil/viz/clock.py new file mode 100644 index 00000000..72ad163a --- /dev/null +++ b/src/automil/viz/clock.py @@ -0,0 +1,170 @@ +"""The run host's clock: put naive host-local stamps and UTC stamps on one axis. + +Transcripts carry UTC timestamps. ``graph.json`` ``created_at``, +``completed/.json`` ``completed_at`` and the orchestrator log carry +naive host-local time. A record converts every stamp to UTC once, using the +run host's zone, so the frontend never has to guess. +""" +from __future__ import annotations + +import os +import re +from dataclasses import asdict, dataclass +from datetime import datetime, timedelta, timezone +from pathlib import Path +from statistics import median +from typing import Iterable, Literal, Mapping +from zoneinfo import ZoneInfo, ZoneInfoNotFoundError + +ClockSource = Literal["host", "explicit"] + +_QUARTER_HOUR_S = 15 * 60 +_LOG_ASCTIME = re.compile( + r"^(?P\d{4}-\d{2}-\d{2}) (?P