diff --git a/.gitignore b/.gitignore index ae9eaab..c5b05cb 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,4 @@ build/ dist-python/ .env .letta/ +token-efficiency-out/ diff --git a/scripts/token-efficiency/README.md b/scripts/token-efficiency/README.md new file mode 100644 index 0000000..a117a13 --- /dev/null +++ b/scripts/token-efficiency/README.md @@ -0,0 +1,46 @@ +# token-efficiency + +Compare token counts of a real agent session across three representations: +the harness's **native** session file, this repo's **trajectory** normalized +JSONL, and Harbor's **ATIF** (RFC 0001) as produced by Harbor's own +converters. + +```sh +export ANTHROPIC_API_KEY=... # token counting uses /v1/messages/count_tokens +bun scripts/token-efficiency/index.ts +``` + +`` is a Claude Code session +(`~/.claude/projects//.jsonl`) or a Codex rollout +(`~/.codex/sessions/.../rollout-*.jsonl`); the source is auto-detected. + +Example output: + +``` +format bytes tokens vs native file +native 168,592 77,281 — token-efficiency-out/rollout-.../native.jsonl +trajectory 46,763 18,438 4.2x token-efficiency-out/rollout-.../trajectory.jsonl +atif (harbor, minified) 88,410 35,847 2.2x token-efficiency-out/rollout-.../atif.min.json +atif (harbor, persisted) 92,293 36,841 2.1x token-efficiency-out/rollout-.../atif.json +``` + +Each representation is also written to `token-efficiency-out//` +(override with `--out-dir`). Other flags: `--source claude-code|codex`, +`--model ` (default `claude-opus-4-8`), `--untruncated` (adds a trajectory +row with tool-result truncation disabled). + +## ATIF details + +Harbor has no standalone conversion CLI — session → ATIF conversion lives in +its agent classes and normally runs inside a harness trial. +`harbor_atif_convert.py` drives the exact upstream +`ClaudeCode`/`Codex`.`_convert_events_to_trajectory` code from a harbor +checkout with the harness-only imports stubbed (requires `uv` and `git`; a +checkout is cloned to `~/.cache/trajectory/harbor-repo` on first use, override +with `HARBOR_REPO=`). The output validates against harbor's own ATIF +validator (`harbor-atif2otel`). + +Note on interpreting results: ATIF as Harbor produces it intentionally keeps +untruncated tool results, structured result payloads (in `extra`), and +per-step token metrics, so it carries more content than trajectory by design — +the comparison reflects each format's content policy, not just syntax. diff --git a/scripts/token-efficiency/harbor_atif_convert.py b/scripts/token-efficiency/harbor_atif_convert.py new file mode 100644 index 0000000..64fe4ed --- /dev/null +++ b/scripts/token-efficiency/harbor_atif_convert.py @@ -0,0 +1,141 @@ +"""Convert a Claude Code or Codex session file to ATIF using Harbor's own converters. + +Harbor (https://github.com/harbor-framework/harbor) has no standalone +conversion CLI — session -> ATIF conversion lives inside its agent classes +(`ClaudeCode._convert_events_to_trajectory`, `Codex._convert_events_to_trajectory`) +and normally runs as part of a harness trial. This driver imports those classes +from a Harbor checkout with the harness-only dependencies stubbed out, so the +exact upstream conversion code runs against a session file on disk. + +The full harbor package is not installable in isolation (heavy deps), so only +`pydantic` is required. Run via uv: + + uv run --python 3.12 --with pydantic scripts/harbor_atif_convert.py \ + --source claude-code --harbor --out atif.json + +Writes compact JSON to --out, and Harbor's as-persisted formatting +(`format_trajectory_json`, indent=2) to --out-pretty if given. +""" + +import argparse +import json +import logging +import shutil +import sys +import tempfile +import types +from pathlib import Path, PurePosixPath + + +def install_stubs(harbor_src: Path) -> None: + sys.path.insert(0, str(harbor_src)) + + # Bypass harbor/__init__.py (does importlib.metadata.version lookup). + pkg = types.ModuleType("harbor") + pkg.__path__ = [str(harbor_src / "harbor")] + pkg.__version__ = "stub" + sys.modules["harbor"] = pkg + + def stub(name: str, **attrs) -> None: + mod = types.ModuleType(name) + + class Placeholder: + def __init__(self, *args, **kwargs): + self.__dict__.update(kwargs) + + # Any name imported from a stubbed module that we don't explicitly + # provide resolves to a permissive placeholder class. + mod.__getattr__ = lambda item: Placeholder # type: ignore[method-assign] + for key, value in attrs.items(): + setattr(mod, key, value) + sys.modules[name] = mod + + class BaseInstalledAgent: # bare stand-in; __init__ is bypassed below + pass + + class Descriptor: + def __init__(self, *args, **kwargs): + self.__dict__.update(kwargs) + + def with_prompt_template(*args, **kwargs): + if len(args) == 1 and callable(args[0]) and not kwargs: + return args[0] + return lambda fn: fn + + stub( + "harbor.agents.installed.base", + BaseInstalledAgent=BaseInstalledAgent, + CliFlag=Descriptor, + EnvVar=Descriptor, + with_prompt_template=with_prompt_template, + ) + stub("harbor.agents.base") + stub("harbor.environments.base") + stub("harbor.models.agent.context") + stub( + "harbor.models.trial.paths", + EnvironmentPaths=types.SimpleNamespace(agent_dir=PurePosixPath("/agent")), + ) + stub( + "harbor.utils.env", + parse_bool_env_value=lambda value, default=False: ( + default if value is None else str(value).lower() in ("1", "true", "yes") + ), + ) + stub("harbor.utils.templating") + # harbor.models.agent.name (small enum), harbor.models.trajectories (pydantic + # ATIF models), and harbor.utils.trajectory_utils load for real. + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("session_file", type=Path) + parser.add_argument("--source", choices=["claude-code", "codex"], required=True) + parser.add_argument("--harbor", type=Path, required=True, help="Path to a harbor checkout") + parser.add_argument("--out", type=Path, required=True, help="Compact JSON output path") + parser.add_argument("--out-pretty", type=Path, help="As-persisted (indent=2) output path") + args = parser.parse_args() + + harbor_src = args.harbor / "src" + if not (harbor_src / "harbor").is_dir(): + sys.exit(f"not a harbor checkout: {args.harbor}") + + logging.basicConfig(level=logging.WARNING) + install_stubs(harbor_src) + + from harbor.agents.installed.claude_code import ClaudeCode # noqa: E402 + from harbor.agents.installed.codex import Codex # noqa: E402 + from harbor.utils.trajectory_utils import format_trajectory_json # noqa: E402 + + cls = ClaudeCode if args.source == "claude-code" else Codex + agent = object.__new__(cls) # skip harness-oriented __init__ + agent.logger = logging.getLogger(cls.__name__) + agent.model_name = None + agent.logs_dir = Path(".") + agent._version = None + + # The converters take a session *directory*; isolate the one session file so + # sibling sessions (and subagent transcripts) don't get merged in. + with tempfile.TemporaryDirectory() as tmp: + session_dir = Path(tmp) + shutil.copy(args.session_file, session_dir / args.session_file.name) + trajectory = agent._convert_events_to_trajectory(session_dir) + + if trajectory is None: + sys.exit("harbor converter returned no trajectory") + + data = ( + trajectory.to_json_dict() + if hasattr(trajectory, "to_json_dict") + else trajectory.model_dump(exclude_none=True) + ) + args.out.write_text(json.dumps(data, separators=(",", ":"))) + if args.out_pretty: + args.out_pretty.write_text(format_trajectory_json(data)) + print( + json.dumps({"steps": len(data.get("steps", [])), "schema_version": data.get("schema_version")}) + ) + + +if __name__ == "__main__": + main() diff --git a/scripts/token-efficiency/index.ts b/scripts/token-efficiency/index.ts new file mode 100644 index 0000000..50874a0 --- /dev/null +++ b/scripts/token-efficiency/index.ts @@ -0,0 +1,259 @@ +/** + * Compare token counts of an agent session across three representations: + * + * 1. native — the session file exactly as the harness wrote it + * 2. trajectory — this repo's normalized JSONL (default bounds, and + * optionally with tool-result truncation disabled) + * 3. atif — Harbor ATIF (RFC 0001) produced by Harbor's own + * converters (`ClaudeCode`/`Codex`._convert_events_to_trajectory), + * driven by scripts/harbor_atif_convert.py. Reported both + * minified and as Harbor persists it (indent=2). Note + * Harbor's converters keep untruncated tool results, + * structured result payloads, and per-step metrics, so + * ATIF carries more content than trajectory by design. + * + * Tokens are counted with the Anthropic count-tokens API + * (POST /v1/messages/count_tokens), chunked at 500K characters per request. + * Requires ANTHROPIC_API_KEY in the environment, plus `uv` and `git` on PATH + * (a harbor checkout is cloned to ~/.cache/trajectory/harbor-repo on first + * use; override with HARBOR_REPO=). + * + * Usage: + * bun scripts/token-efficiency/index.ts [options] + * + * Claude Code session (~/.claude/projects//.jsonl) + * or Codex rollout (~/.codex/sessions/.../rollout-*.jsonl) + * + * Options: + * --source Override source auto-detection + * --model Model for token counting (default claude-opus-4-8) + * --untruncated Also report trajectory with truncation disabled + * --out-dir Where to write each representation + * (default: token-efficiency-out//) + */ + +import { spawnSync } from "child_process"; +import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "fs"; +import { homedir, tmpdir } from "os"; +import { basename, dirname, join } from "path"; +import { fileURLToPath } from "url"; +import { normalizeTranscript } from "../../src/index.js"; +import type { NormalizeInput } from "../../src/index.js"; + +const CHUNK_CHARS = 500_000; +const API_URL = "https://api.anthropic.com/v1/messages/count_tokens"; + +interface CanonicalRecord { + role: string; + content?: string | null; + timestamp?: string; + tool_call_id?: string; + tool_calls?: { id: string; name: string; args: string }[]; + source?: string; + model?: string; +} + +function usage(): never { + console.error( + "usage: bun scripts/token-efficiency/index.ts [--source claude-code|codex] [--model ] [--untruncated] [--out-dir ]", + ); + process.exit(1); +} + +function parseArgs() { + const args = process.argv.slice(2); + let file: string | undefined; + let source: string | undefined; + let model = "claude-opus-4-8"; + let untruncated = false; + let outDir: string | undefined; + for (let i = 0; i < args.length; i++) { + const a = args[i]; + if (a === undefined) usage(); + else if (a === "--source") source = args[++i] ?? usage(); + else if (a === "--model") model = args[++i] ?? usage(); + else if (a === "--untruncated") untruncated = true; + else if (a === "--out-dir") outDir = args[++i] ?? usage(); + else if (!a.startsWith("--") && file === undefined) file = a; + else usage(); + } + if (!file) usage(); + return { file, source, model, untruncated, outDir }; +} + +function detectSource(transcript: string): string { + const firstLine = transcript.slice(0, transcript.indexOf("\n")); + try { + const obj = JSON.parse(firstLine); + if (obj?.type === "session_meta" && obj?.payload) return "codex"; + } catch { + /* fall through */ + } + return "claude-code"; +} + +const HARBOR_GIT_URL = "https://github.com/harbor-framework/harbor.git"; + +function ensureHarborCheckout(): string { + const repo = process.env.HARBOR_REPO ?? join(homedir(), ".cache", "trajectory", "harbor-repo"); + if (existsSync(join(repo, "src", "harbor"))) return repo; + if (process.env.HARBOR_REPO) { + console.error(`HARBOR_REPO=${repo} is not a harbor checkout`); + process.exit(1); + } + console.error(`cloning harbor into ${repo} ...`); + mkdirSync(dirname(repo), { recursive: true }); + const clone = spawnSync("git", ["clone", "--depth", "1", HARBOR_GIT_URL, repo], { + stdio: ["ignore", "inherit", "inherit"], + }); + if (clone.status !== 0) { + console.error("git clone of harbor failed"); + process.exit(1); + } + return repo; +} + +/** Run Harbor's own converter via scripts/harbor_atif_convert.py. */ +function harborAtif(sessionFile: string, source: string): { compact: string; pretty: string } { + const harborRepo = ensureHarborCheckout(); + const scriptsDir = dirname(fileURLToPath(import.meta.url)); + const outDir = mkdtempSync(join(tmpdir(), "harbor-atif-")); + const out = join(outDir, "atif.min.json"); + const outPretty = join(outDir, "atif.json"); + try { + const run = spawnSync( + "uv", + [ + "run", + "--no-project", + "--python", + "3.12", + "--with", + "pydantic", + join(scriptsDir, "harbor_atif_convert.py"), + sessionFile, + "--source", + source, + "--harbor", + harborRepo, + "--out", + out, + "--out-pretty", + outPretty, + ], + { stdio: ["ignore", "pipe", "inherit"], encoding: "utf8" }, + ); + if (run.status !== 0) { + console.error("harbor conversion failed (is `uv` installed?)"); + process.exit(1); + } + return { compact: readFileSync(out, "utf8"), pretty: readFileSync(outPretty, "utf8") }; + } finally { + rmSync(outDir, { recursive: true, force: true }); + } +} + +async function countTokens(text: string, model: string, apiKey: string, label: string): Promise { + let total = 0; + const chunkCount = Math.ceil(text.length / CHUNK_CHARS); + for (let i = 0; i < text.length; i += CHUNK_CHARS) { + const chunk = text.slice(i, i + CHUNK_CHARS); + let attempt = 0; + for (;;) { + const res = await fetch(API_URL, { + method: "POST", + headers: { + "x-api-key": apiKey, + "anthropic-version": "2023-06-01", + "content-type": "application/json", + }, + body: JSON.stringify({ model, messages: [{ role: "user", content: chunk }] }), + }); + if (res.ok) { + const body = (await res.json()) as { input_tokens: number }; + total += body.input_tokens; + break; + } + if ((res.status === 429 || res.status >= 500) && attempt < 5) { + attempt++; + const retryAfter = Number(res.headers.get("retry-after")) || 10 * attempt; + await new Promise((r) => setTimeout(r, retryAfter * 1000)); + continue; + } + throw new Error(`count_tokens failed for ${label}: ${res.status} ${await res.text()}`); + } + process.stderr.write(`\r${label}: chunk ${Math.floor(i / CHUNK_CHARS) + 1}/${chunkCount} (${total.toLocaleString()} tokens)`); + } + process.stderr.write("\n"); + return total; +} + +async function main() { + const { file, source: sourceArg, model, untruncated, outDir: outDirArg } = parseArgs(); + const apiKey = process.env.ANTHROPIC_API_KEY; + if (!apiKey) { + console.error("ANTHROPIC_API_KEY is not set"); + process.exit(1); + } + + const transcript = readFileSync(file, "utf8"); + const source = sourceArg ?? detectSource(transcript); + console.error(`source: ${source} | model: ${model} | file: ${file}`); + + const normalize = (full: boolean) => { + const input: NormalizeInput = { source, transcript } as NormalizeInput; + if (full) { + (input as any).bounds = { + toolResults: { maxCharacters: null }, + toolArguments: { maxCharacters: null }, + }; + } + const { records } = normalizeTranscript(input); + return records as unknown as CanonicalRecord[]; + }; + + const defaultRecords = normalize(false); + const trajectoryText = defaultRecords.map((r) => JSON.stringify(r)).join("\n") + "\n"; + const atif = harborAtif(file, source); + + const rows: { label: string; text: string; filename: string }[] = [ + { label: "native", text: transcript, filename: "native.jsonl" }, + { label: "trajectory", text: trajectoryText, filename: "trajectory.jsonl" }, + { label: "atif (harbor, minified)", text: atif.compact, filename: "atif.min.json" }, + { label: "atif (harbor, persisted)", text: atif.pretty, filename: "atif.json" }, + ]; + if (untruncated) { + const fullRecords = normalize(true); + rows.push({ + label: "trajectory-untruncated", + text: fullRecords.map((r) => JSON.stringify(r)).join("\n") + "\n", + filename: "trajectory-untruncated.jsonl", + }); + } + + const sessionStem = basename(file).replace(/\.[^.]+$/, ""); + const outDir = outDirArg ?? join("token-efficiency-out", sessionStem); + mkdirSync(outDir, { recursive: true }); + + const results: { label: string; bytes: number; tokens: number; path: string }[] = []; + for (const { label, text, filename } of rows) { + const path = join(outDir, filename); + writeFileSync(path, text); + const tokens = await countTokens(text, model, apiKey, label); + results.push({ label, bytes: Buffer.byteLength(text), tokens, path }); + } + + const native = results[0]!; + console.log(`\n${"format".padEnd(24)} ${"bytes".padStart(12)} ${"tokens".padStart(12)} ${"vs native".padStart(10)} file`); + for (const r of results) { + const factor = r === native ? "—" : `${(native.tokens / r.tokens).toFixed(1)}x`; + console.log( + `${r.label.padEnd(24)} ${r.bytes.toLocaleString().padStart(12)} ${r.tokens.toLocaleString().padStart(12)} ${factor.padStart(10)} ${r.path}`, + ); + } +} + +main().catch((err) => { + console.error(err); + process.exit(1); +});