From de6911af2cb1fbb67a12545f7e01f989337cf970 Mon Sep 17 00:00:00 2001 From: Sarah Wooders Date: Thu, 23 Jul 2026 12:49:15 -0700 Subject: [PATCH 1/4] Add token-efficiency comparison script Takes a Claude Code session or Codex rollout path and reports token counts (via the Anthropic count-tokens API) for the native file, the normalized trajectory JSONL, and a content-matched Harbor ATIF projection, with reduction factors vs native. Source is auto-detected; --untruncated adds a row with tool-result truncation disabled. --- scripts/token-efficiency.ts | 245 ++++++++++++++++++++++++++++++++++++ 1 file changed, 245 insertions(+) create mode 100644 scripts/token-efficiency.ts diff --git a/scripts/token-efficiency.ts b/scripts/token-efficiency.ts new file mode 100644 index 0000000..814d067 --- /dev/null +++ b/scripts/token-efficiency.ts @@ -0,0 +1,245 @@ +/** + * Compare token counts of an agent session across three representations: + * + * 1. native — the session file exactly as the harness wrote it + * 2. trajectory — this repo's normalized JSONL (default bounds, and + * optionally with tool-result truncation disabled) + * 3. atif — Harbor's ATIF (RFC 0001) built from the same normalized + * records, serialized compact. This is a content-matched + * projection: it measures ATIF *syntax* on trajectory's + * content selection. Harbor's own converters additionally + * keep untruncated results, structured result payloads, + * and per-step metrics, so their files are much larger. + * + * Tokens are counted with the Anthropic count-tokens API + * (POST /v1/messages/count_tokens), chunked at 500K characters per request. + * Requires ANTHROPIC_API_KEY in the environment. + * + * Usage: + * bun scripts/token-efficiency.ts [options] + * + * Claude Code session (~/.claude/projects//.jsonl) + * or Codex rollout (~/.codex/sessions/.../rollout-*.jsonl) + * + * Options: + * --source Override source auto-detection + * --model Model for token counting (default claude-opus-4-8) + * --untruncated Also report trajectory with truncation disabled + */ + +import { readFileSync } from "fs"; +import { normalizeTranscript } from "../src/index.js"; +import type { NormalizeInput } from "../src/index.js"; + +const CHUNK_CHARS = 500_000; +const API_URL = "https://api.anthropic.com/v1/messages/count_tokens"; + +interface CanonicalRecord { + role: string; + content?: string | null; + timestamp?: string; + tool_call_id?: string; + tool_calls?: { id: string; name: string; args: string }[]; + source?: string; + model?: string; +} + +function usage(): never { + console.error( + "usage: bun scripts/token-efficiency.ts [--source claude-code|codex] [--model ] [--untruncated]", + ); + process.exit(1); +} + +function parseArgs() { + const args = process.argv.slice(2); + let file: string | undefined; + let source: string | undefined; + let model = "claude-opus-4-8"; + let untruncated = false; + for (let i = 0; i < args.length; i++) { + const a = args[i]; + if (a === undefined) usage(); + else if (a === "--source") source = args[++i] ?? usage(); + else if (a === "--model") model = args[++i] ?? usage(); + else if (a === "--untruncated") untruncated = true; + else if (!a.startsWith("--") && file === undefined) file = a; + else usage(); + } + if (!file) usage(); + return { file, source, model, untruncated }; +} + +function detectSource(transcript: string): string { + const firstLine = transcript.slice(0, transcript.indexOf("\n")); + try { + const obj = JSON.parse(firstLine); + if (obj?.type === "session_meta" && obj?.payload) return "codex"; + } catch { + /* fall through */ + } + return "claude-code"; +} + +/** Canonical records -> Harbor ATIF (compact JSON). */ +function toAtif(records: CanonicalRecord[]): string { + const meta = records.find((r) => r.role === "meta"); + const steps: any[] = []; + const callOwner = new Map(); + let pendingReasoning: string[] = []; + + const newStep = (source: string, rec: CanonicalRecord) => { + const s: any = { step_id: steps.length + 1, source, message: rec.content ?? "" }; + if (rec.timestamp) s.timestamp = rec.timestamp; + steps.push(s); + return s; + }; + + for (const r of records) { + if (r.role === "meta") continue; + if (r.role === "user") { + newStep("user", r); + } else if (r.role === "reasoning") { + pendingReasoning.push(r.content ?? ""); + } else if (r.role === "assistant") { + const s = newStep("agent", r); + if (pendingReasoning.length > 0) { + s.reasoning_content = pendingReasoning.join("\n\n"); + pendingReasoning = []; + } + if (r.tool_calls) { + s.tool_calls = r.tool_calls.map((tc) => { + let args: unknown; + try { + args = JSON.parse(tc.args); + } catch { + args = { _raw: tc.args }; + } + callOwner.set(tc.id, s); + return { tool_call_id: tc.id, function_name: tc.name, arguments: args }; + }); + } + } else if (r.role === "tool") { + const owner = callOwner.get(r.tool_call_id ?? "") ?? steps.at(-1); + if (!owner) continue; + owner.observation ??= { results: [] }; + owner.observation.results.push({ + source_call_id: r.tool_call_id ?? null, + content: r.content ?? "", + }); + } + } + + if (pendingReasoning.length > 0) { + const tail = [...steps].reverse().find((s) => s.source === "agent"); + if (tail) { + tail.reasoning_content = [tail.reasoning_content, ...pendingReasoning] + .filter(Boolean) + .join("\n\n"); + } + } + + return JSON.stringify({ + schema_version: "ATIF-v1.7", + session_id: null, + agent: { name: meta?.source ?? "unknown", version: "unknown", model_name: meta?.model ?? null }, + steps, + }); +} + +async function countTokens(text: string, model: string, apiKey: string, label: string): Promise { + let total = 0; + const chunkCount = Math.ceil(text.length / CHUNK_CHARS); + for (let i = 0; i < text.length; i += CHUNK_CHARS) { + const chunk = text.slice(i, i + CHUNK_CHARS); + let attempt = 0; + for (;;) { + const res = await fetch(API_URL, { + method: "POST", + headers: { + "x-api-key": apiKey, + "anthropic-version": "2023-06-01", + "content-type": "application/json", + }, + body: JSON.stringify({ model, messages: [{ role: "user", content: chunk }] }), + }); + if (res.ok) { + const body = (await res.json()) as { input_tokens: number }; + total += body.input_tokens; + break; + } + if ((res.status === 429 || res.status >= 500) && attempt < 5) { + attempt++; + const retryAfter = Number(res.headers.get("retry-after")) || 10 * attempt; + await new Promise((r) => setTimeout(r, retryAfter * 1000)); + continue; + } + throw new Error(`count_tokens failed for ${label}: ${res.status} ${await res.text()}`); + } + process.stderr.write(`\r${label}: chunk ${Math.floor(i / CHUNK_CHARS) + 1}/${chunkCount} (${total.toLocaleString()} tokens)`); + } + process.stderr.write("\n"); + return total; +} + +async function main() { + const { file, source: sourceArg, model, untruncated } = parseArgs(); + const apiKey = process.env.ANTHROPIC_API_KEY; + if (!apiKey) { + console.error("ANTHROPIC_API_KEY is not set"); + process.exit(1); + } + + const transcript = readFileSync(file, "utf8"); + const source = sourceArg ?? detectSource(transcript); + console.error(`source: ${source} | model: ${model} | file: ${file}`); + + const normalize = (full: boolean) => { + const input: NormalizeInput = { source, transcript } as NormalizeInput; + if (full) { + (input as any).bounds = { + toolResults: { maxCharacters: null }, + toolArguments: { maxCharacters: null }, + }; + } + const { records } = normalizeTranscript(input); + return records as unknown as CanonicalRecord[]; + }; + + const defaultRecords = normalize(false); + const trajectoryText = defaultRecords.map((r) => JSON.stringify(r)).join("\n") + "\n"; + const atifText = toAtif(defaultRecords); + + const rows: { label: string; text: string }[] = [ + { label: "native", text: transcript }, + { label: "trajectory", text: trajectoryText }, + { label: "atif", text: atifText }, + ]; + if (untruncated) { + const fullRecords = normalize(true); + rows.push({ + label: "trajectory-untruncated", + text: fullRecords.map((r) => JSON.stringify(r)).join("\n") + "\n", + }); + } + + const results: { label: string; bytes: number; tokens: number }[] = []; + for (const { label, text } of rows) { + const tokens = await countTokens(text, model, apiKey, label); + results.push({ label, bytes: Buffer.byteLength(text), tokens }); + } + + const native = results[0]!; + console.log(`\n${"format".padEnd(24)} ${"bytes".padStart(12)} ${"tokens".padStart(12)} ${"vs native".padStart(10)}`); + for (const r of results) { + const factor = r === native ? "—" : `${(native.tokens / r.tokens).toFixed(1)}x`; + console.log( + `${r.label.padEnd(24)} ${r.bytes.toLocaleString().padStart(12)} ${r.tokens.toLocaleString().padStart(12)} ${factor.padStart(10)}`, + ); + } +} + +main().catch((err) => { + console.error(err); + process.exit(1); +}); From 9579fc51fa33696a7a8c5c34e056b1c0b7fd1531 Mon Sep 17 00:00:00 2001 From: Sarah Wooders Date: Thu, 23 Jul 2026 12:56:23 -0700 Subject: [PATCH 2/4] Use Harbor's own converters for the ATIF comparison Harbor has no standalone conversion CLI, so scripts/harbor_atif_convert.py drives its exact ClaudeCode/Codex _convert_events_to_trajectory code from a harbor checkout with the harness-only imports stubbed (only pydantic needed, run via uv). token-efficiency.ts now reports Harbor-produced ATIF (minified and as-persisted) instead of the content-matched projection, auto-cloning harbor to ~/.cache/trajectory/harbor-repo on first use (HARBOR_REPO overrides). --- scripts/harbor_atif_convert.py | 141 +++++++++++++++++++++++++++++++++ scripts/token-efficiency.ts | 141 +++++++++++++++++---------------- 2 files changed, 213 insertions(+), 69 deletions(-) create mode 100644 scripts/harbor_atif_convert.py diff --git a/scripts/harbor_atif_convert.py b/scripts/harbor_atif_convert.py new file mode 100644 index 0000000..64fe4ed --- /dev/null +++ b/scripts/harbor_atif_convert.py @@ -0,0 +1,141 @@ +"""Convert a Claude Code or Codex session file to ATIF using Harbor's own converters. + +Harbor (https://github.com/harbor-framework/harbor) has no standalone +conversion CLI — session -> ATIF conversion lives inside its agent classes +(`ClaudeCode._convert_events_to_trajectory`, `Codex._convert_events_to_trajectory`) +and normally runs as part of a harness trial. This driver imports those classes +from a Harbor checkout with the harness-only dependencies stubbed out, so the +exact upstream conversion code runs against a session file on disk. + +The full harbor package is not installable in isolation (heavy deps), so only +`pydantic` is required. Run via uv: + + uv run --python 3.12 --with pydantic scripts/harbor_atif_convert.py \ + --source claude-code --harbor --out atif.json + +Writes compact JSON to --out, and Harbor's as-persisted formatting +(`format_trajectory_json`, indent=2) to --out-pretty if given. +""" + +import argparse +import json +import logging +import shutil +import sys +import tempfile +import types +from pathlib import Path, PurePosixPath + + +def install_stubs(harbor_src: Path) -> None: + sys.path.insert(0, str(harbor_src)) + + # Bypass harbor/__init__.py (does importlib.metadata.version lookup). + pkg = types.ModuleType("harbor") + pkg.__path__ = [str(harbor_src / "harbor")] + pkg.__version__ = "stub" + sys.modules["harbor"] = pkg + + def stub(name: str, **attrs) -> None: + mod = types.ModuleType(name) + + class Placeholder: + def __init__(self, *args, **kwargs): + self.__dict__.update(kwargs) + + # Any name imported from a stubbed module that we don't explicitly + # provide resolves to a permissive placeholder class. + mod.__getattr__ = lambda item: Placeholder # type: ignore[method-assign] + for key, value in attrs.items(): + setattr(mod, key, value) + sys.modules[name] = mod + + class BaseInstalledAgent: # bare stand-in; __init__ is bypassed below + pass + + class Descriptor: + def __init__(self, *args, **kwargs): + self.__dict__.update(kwargs) + + def with_prompt_template(*args, **kwargs): + if len(args) == 1 and callable(args[0]) and not kwargs: + return args[0] + return lambda fn: fn + + stub( + "harbor.agents.installed.base", + BaseInstalledAgent=BaseInstalledAgent, + CliFlag=Descriptor, + EnvVar=Descriptor, + with_prompt_template=with_prompt_template, + ) + stub("harbor.agents.base") + stub("harbor.environments.base") + stub("harbor.models.agent.context") + stub( + "harbor.models.trial.paths", + EnvironmentPaths=types.SimpleNamespace(agent_dir=PurePosixPath("/agent")), + ) + stub( + "harbor.utils.env", + parse_bool_env_value=lambda value, default=False: ( + default if value is None else str(value).lower() in ("1", "true", "yes") + ), + ) + stub("harbor.utils.templating") + # harbor.models.agent.name (small enum), harbor.models.trajectories (pydantic + # ATIF models), and harbor.utils.trajectory_utils load for real. + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("session_file", type=Path) + parser.add_argument("--source", choices=["claude-code", "codex"], required=True) + parser.add_argument("--harbor", type=Path, required=True, help="Path to a harbor checkout") + parser.add_argument("--out", type=Path, required=True, help="Compact JSON output path") + parser.add_argument("--out-pretty", type=Path, help="As-persisted (indent=2) output path") + args = parser.parse_args() + + harbor_src = args.harbor / "src" + if not (harbor_src / "harbor").is_dir(): + sys.exit(f"not a harbor checkout: {args.harbor}") + + logging.basicConfig(level=logging.WARNING) + install_stubs(harbor_src) + + from harbor.agents.installed.claude_code import ClaudeCode # noqa: E402 + from harbor.agents.installed.codex import Codex # noqa: E402 + from harbor.utils.trajectory_utils import format_trajectory_json # noqa: E402 + + cls = ClaudeCode if args.source == "claude-code" else Codex + agent = object.__new__(cls) # skip harness-oriented __init__ + agent.logger = logging.getLogger(cls.__name__) + agent.model_name = None + agent.logs_dir = Path(".") + agent._version = None + + # The converters take a session *directory*; isolate the one session file so + # sibling sessions (and subagent transcripts) don't get merged in. + with tempfile.TemporaryDirectory() as tmp: + session_dir = Path(tmp) + shutil.copy(args.session_file, session_dir / args.session_file.name) + trajectory = agent._convert_events_to_trajectory(session_dir) + + if trajectory is None: + sys.exit("harbor converter returned no trajectory") + + data = ( + trajectory.to_json_dict() + if hasattr(trajectory, "to_json_dict") + else trajectory.model_dump(exclude_none=True) + ) + args.out.write_text(json.dumps(data, separators=(",", ":"))) + if args.out_pretty: + args.out_pretty.write_text(format_trajectory_json(data)) + print( + json.dumps({"steps": len(data.get("steps", [])), "schema_version": data.get("schema_version")}) + ) + + +if __name__ == "__main__": + main() diff --git a/scripts/token-efficiency.ts b/scripts/token-efficiency.ts index 814d067..3772725 100644 --- a/scripts/token-efficiency.ts +++ b/scripts/token-efficiency.ts @@ -4,16 +4,19 @@ * 1. native — the session file exactly as the harness wrote it * 2. trajectory — this repo's normalized JSONL (default bounds, and * optionally with tool-result truncation disabled) - * 3. atif — Harbor's ATIF (RFC 0001) built from the same normalized - * records, serialized compact. This is a content-matched - * projection: it measures ATIF *syntax* on trajectory's - * content selection. Harbor's own converters additionally - * keep untruncated results, structured result payloads, - * and per-step metrics, so their files are much larger. + * 3. atif — Harbor ATIF (RFC 0001) produced by Harbor's own + * converters (`ClaudeCode`/`Codex`._convert_events_to_trajectory), + * driven by scripts/harbor_atif_convert.py. Reported both + * minified and as Harbor persists it (indent=2). Note + * Harbor's converters keep untruncated tool results, + * structured result payloads, and per-step metrics, so + * ATIF carries more content than trajectory by design. * * Tokens are counted with the Anthropic count-tokens API * (POST /v1/messages/count_tokens), chunked at 500K characters per request. - * Requires ANTHROPIC_API_KEY in the environment. + * Requires ANTHROPIC_API_KEY in the environment, plus `uv` and `git` on PATH + * (a harbor checkout is cloned to ~/.cache/trajectory/harbor-repo on first + * use; override with HARBOR_REPO=). * * Usage: * bun scripts/token-efficiency.ts [options] @@ -27,7 +30,11 @@ * --untruncated Also report trajectory with truncation disabled */ -import { readFileSync } from "fs"; +import { spawnSync } from "child_process"; +import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync } from "fs"; +import { homedir, tmpdir } from "os"; +import { dirname, join } from "path"; +import { fileURLToPath } from "url"; import { normalizeTranscript } from "../src/index.js"; import type { NormalizeInput } from "../src/index.js"; @@ -81,70 +88,65 @@ function detectSource(transcript: string): string { return "claude-code"; } -/** Canonical records -> Harbor ATIF (compact JSON). */ -function toAtif(records: CanonicalRecord[]): string { - const meta = records.find((r) => r.role === "meta"); - const steps: any[] = []; - const callOwner = new Map(); - let pendingReasoning: string[] = []; - - const newStep = (source: string, rec: CanonicalRecord) => { - const s: any = { step_id: steps.length + 1, source, message: rec.content ?? "" }; - if (rec.timestamp) s.timestamp = rec.timestamp; - steps.push(s); - return s; - }; +const HARBOR_GIT_URL = "https://github.com/harbor-framework/harbor.git"; - for (const r of records) { - if (r.role === "meta") continue; - if (r.role === "user") { - newStep("user", r); - } else if (r.role === "reasoning") { - pendingReasoning.push(r.content ?? ""); - } else if (r.role === "assistant") { - const s = newStep("agent", r); - if (pendingReasoning.length > 0) { - s.reasoning_content = pendingReasoning.join("\n\n"); - pendingReasoning = []; - } - if (r.tool_calls) { - s.tool_calls = r.tool_calls.map((tc) => { - let args: unknown; - try { - args = JSON.parse(tc.args); - } catch { - args = { _raw: tc.args }; - } - callOwner.set(tc.id, s); - return { tool_call_id: tc.id, function_name: tc.name, arguments: args }; - }); - } - } else if (r.role === "tool") { - const owner = callOwner.get(r.tool_call_id ?? "") ?? steps.at(-1); - if (!owner) continue; - owner.observation ??= { results: [] }; - owner.observation.results.push({ - source_call_id: r.tool_call_id ?? null, - content: r.content ?? "", - }); - } +function ensureHarborCheckout(): string { + const repo = process.env.HARBOR_REPO ?? join(homedir(), ".cache", "trajectory", "harbor-repo"); + if (existsSync(join(repo, "src", "harbor"))) return repo; + if (process.env.HARBOR_REPO) { + console.error(`HARBOR_REPO=${repo} is not a harbor checkout`); + process.exit(1); + } + console.error(`cloning harbor into ${repo} ...`); + mkdirSync(dirname(repo), { recursive: true }); + const clone = spawnSync("git", ["clone", "--depth", "1", HARBOR_GIT_URL, repo], { + stdio: ["ignore", "inherit", "inherit"], + }); + if (clone.status !== 0) { + console.error("git clone of harbor failed"); + process.exit(1); } + return repo; +} - if (pendingReasoning.length > 0) { - const tail = [...steps].reverse().find((s) => s.source === "agent"); - if (tail) { - tail.reasoning_content = [tail.reasoning_content, ...pendingReasoning] - .filter(Boolean) - .join("\n\n"); +/** Run Harbor's own converter via scripts/harbor_atif_convert.py. */ +function harborAtif(sessionFile: string, source: string): { compact: string; pretty: string } { + const harborRepo = ensureHarborCheckout(); + const scriptsDir = dirname(fileURLToPath(import.meta.url)); + const outDir = mkdtempSync(join(tmpdir(), "harbor-atif-")); + const out = join(outDir, "atif.min.json"); + const outPretty = join(outDir, "atif.json"); + try { + const run = spawnSync( + "uv", + [ + "run", + "--no-project", + "--python", + "3.12", + "--with", + "pydantic", + join(scriptsDir, "harbor_atif_convert.py"), + sessionFile, + "--source", + source, + "--harbor", + harborRepo, + "--out", + out, + "--out-pretty", + outPretty, + ], + { stdio: ["ignore", "pipe", "inherit"], encoding: "utf8" }, + ); + if (run.status !== 0) { + console.error("harbor conversion failed (is `uv` installed?)"); + process.exit(1); } + return { compact: readFileSync(out, "utf8"), pretty: readFileSync(outPretty, "utf8") }; + } finally { + rmSync(outDir, { recursive: true, force: true }); } - - return JSON.stringify({ - schema_version: "ATIF-v1.7", - session_id: null, - agent: { name: meta?.source ?? "unknown", version: "unknown", model_name: meta?.model ?? null }, - steps, - }); } async function countTokens(text: string, model: string, apiKey: string, label: string): Promise { @@ -208,12 +210,13 @@ async function main() { const defaultRecords = normalize(false); const trajectoryText = defaultRecords.map((r) => JSON.stringify(r)).join("\n") + "\n"; - const atifText = toAtif(defaultRecords); + const atif = harborAtif(file, source); const rows: { label: string; text: string }[] = [ { label: "native", text: transcript }, { label: "trajectory", text: trajectoryText }, - { label: "atif", text: atifText }, + { label: "atif (harbor, minified)", text: atif.compact }, + { label: "atif (harbor, persisted)", text: atif.pretty }, ]; if (untruncated) { const fullRecords = normalize(true); From ec197f2da57ec27e8c77d13cb10a16de9ad7163c Mon Sep 17 00:00:00 2001 From: Sarah Wooders Date: Thu, 23 Jul 2026 13:00:32 -0700 Subject: [PATCH 3/4] Write each representation to disk alongside the token counts Outputs native/trajectory/atif files under token-efficiency-out// (override with --out-dir) and prints the path per row; directory gitignored. --- .gitignore | 1 + scripts/token-efficiency.ts | 41 +++++++++++++++++++++++-------------- 2 files changed, 27 insertions(+), 15 deletions(-) diff --git a/.gitignore b/.gitignore index ae9eaab..c5b05cb 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,4 @@ build/ dist-python/ .env .letta/ +token-efficiency-out/ diff --git a/scripts/token-efficiency.ts b/scripts/token-efficiency.ts index 3772725..ce3c18e 100644 --- a/scripts/token-efficiency.ts +++ b/scripts/token-efficiency.ts @@ -28,12 +28,14 @@ * --source Override source auto-detection * --model Model for token counting (default claude-opus-4-8) * --untruncated Also report trajectory with truncation disabled + * --out-dir Where to write each representation + * (default: token-efficiency-out//) */ import { spawnSync } from "child_process"; -import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync } from "fs"; +import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "fs"; import { homedir, tmpdir } from "os"; -import { dirname, join } from "path"; +import { basename, dirname, join } from "path"; import { fileURLToPath } from "url"; import { normalizeTranscript } from "../src/index.js"; import type { NormalizeInput } from "../src/index.js"; @@ -53,7 +55,7 @@ interface CanonicalRecord { function usage(): never { console.error( - "usage: bun scripts/token-efficiency.ts [--source claude-code|codex] [--model ] [--untruncated]", + "usage: bun scripts/token-efficiency.ts [--source claude-code|codex] [--model ] [--untruncated] [--out-dir ]", ); process.exit(1); } @@ -64,17 +66,19 @@ function parseArgs() { let source: string | undefined; let model = "claude-opus-4-8"; let untruncated = false; + let outDir: string | undefined; for (let i = 0; i < args.length; i++) { const a = args[i]; if (a === undefined) usage(); else if (a === "--source") source = args[++i] ?? usage(); else if (a === "--model") model = args[++i] ?? usage(); else if (a === "--untruncated") untruncated = true; + else if (a === "--out-dir") outDir = args[++i] ?? usage(); else if (!a.startsWith("--") && file === undefined) file = a; else usage(); } if (!file) usage(); - return { file, source, model, untruncated }; + return { file, source, model, untruncated, outDir }; } function detectSource(transcript: string): string { @@ -185,7 +189,7 @@ async function countTokens(text: string, model: string, apiKey: string, label: s } async function main() { - const { file, source: sourceArg, model, untruncated } = parseArgs(); + const { file, source: sourceArg, model, untruncated, outDir: outDirArg } = parseArgs(); const apiKey = process.env.ANTHROPIC_API_KEY; if (!apiKey) { console.error("ANTHROPIC_API_KEY is not set"); @@ -212,32 +216,39 @@ async function main() { const trajectoryText = defaultRecords.map((r) => JSON.stringify(r)).join("\n") + "\n"; const atif = harborAtif(file, source); - const rows: { label: string; text: string }[] = [ - { label: "native", text: transcript }, - { label: "trajectory", text: trajectoryText }, - { label: "atif (harbor, minified)", text: atif.compact }, - { label: "atif (harbor, persisted)", text: atif.pretty }, + const rows: { label: string; text: string; filename: string }[] = [ + { label: "native", text: transcript, filename: "native.jsonl" }, + { label: "trajectory", text: trajectoryText, filename: "trajectory.jsonl" }, + { label: "atif (harbor, minified)", text: atif.compact, filename: "atif.min.json" }, + { label: "atif (harbor, persisted)", text: atif.pretty, filename: "atif.json" }, ]; if (untruncated) { const fullRecords = normalize(true); rows.push({ label: "trajectory-untruncated", text: fullRecords.map((r) => JSON.stringify(r)).join("\n") + "\n", + filename: "trajectory-untruncated.jsonl", }); } - const results: { label: string; bytes: number; tokens: number }[] = []; - for (const { label, text } of rows) { + const sessionStem = basename(file).replace(/\.[^.]+$/, ""); + const outDir = outDirArg ?? join("token-efficiency-out", sessionStem); + mkdirSync(outDir, { recursive: true }); + + const results: { label: string; bytes: number; tokens: number; path: string }[] = []; + for (const { label, text, filename } of rows) { + const path = join(outDir, filename); + writeFileSync(path, text); const tokens = await countTokens(text, model, apiKey, label); - results.push({ label, bytes: Buffer.byteLength(text), tokens }); + results.push({ label, bytes: Buffer.byteLength(text), tokens, path }); } const native = results[0]!; - console.log(`\n${"format".padEnd(24)} ${"bytes".padStart(12)} ${"tokens".padStart(12)} ${"vs native".padStart(10)}`); + console.log(`\n${"format".padEnd(24)} ${"bytes".padStart(12)} ${"tokens".padStart(12)} ${"vs native".padStart(10)} file`); for (const r of results) { const factor = r === native ? "—" : `${(native.tokens / r.tokens).toFixed(1)}x`; console.log( - `${r.label.padEnd(24)} ${r.bytes.toLocaleString().padStart(12)} ${r.tokens.toLocaleString().padStart(12)} ${factor.padStart(10)}`, + `${r.label.padEnd(24)} ${r.bytes.toLocaleString().padStart(12)} ${r.tokens.toLocaleString().padStart(12)} ${factor.padStart(10)} ${r.path}`, ); } } From e33e70ada95162623ebdf1cf0c3cbe322dfdb4f3 Mon Sep 17 00:00:00 2001 From: Sarah Wooders Date: Thu, 23 Jul 2026 13:05:01 -0700 Subject: [PATCH 4/4] Move token-efficiency into its own scripts folder with README --- scripts/token-efficiency/README.md | 46 +++++++++++++++++++ .../harbor_atif_convert.py | 0 .../index.ts} | 8 ++-- 3 files changed, 50 insertions(+), 4 deletions(-) create mode 100644 scripts/token-efficiency/README.md rename scripts/{ => token-efficiency}/harbor_atif_convert.py (100%) rename scripts/{token-efficiency.ts => token-efficiency/index.ts} (96%) diff --git a/scripts/token-efficiency/README.md b/scripts/token-efficiency/README.md new file mode 100644 index 0000000..a117a13 --- /dev/null +++ b/scripts/token-efficiency/README.md @@ -0,0 +1,46 @@ +# token-efficiency + +Compare token counts of a real agent session across three representations: +the harness's **native** session file, this repo's **trajectory** normalized +JSONL, and Harbor's **ATIF** (RFC 0001) as produced by Harbor's own +converters. + +```sh +export ANTHROPIC_API_KEY=... # token counting uses /v1/messages/count_tokens +bun scripts/token-efficiency/index.ts +``` + +`` is a Claude Code session +(`~/.claude/projects//.jsonl`) or a Codex rollout +(`~/.codex/sessions/.../rollout-*.jsonl`); the source is auto-detected. + +Example output: + +``` +format bytes tokens vs native file +native 168,592 77,281 — token-efficiency-out/rollout-.../native.jsonl +trajectory 46,763 18,438 4.2x token-efficiency-out/rollout-.../trajectory.jsonl +atif (harbor, minified) 88,410 35,847 2.2x token-efficiency-out/rollout-.../atif.min.json +atif (harbor, persisted) 92,293 36,841 2.1x token-efficiency-out/rollout-.../atif.json +``` + +Each representation is also written to `token-efficiency-out//` +(override with `--out-dir`). Other flags: `--source claude-code|codex`, +`--model ` (default `claude-opus-4-8`), `--untruncated` (adds a trajectory +row with tool-result truncation disabled). + +## ATIF details + +Harbor has no standalone conversion CLI — session → ATIF conversion lives in +its agent classes and normally runs inside a harness trial. +`harbor_atif_convert.py` drives the exact upstream +`ClaudeCode`/`Codex`.`_convert_events_to_trajectory` code from a harbor +checkout with the harness-only imports stubbed (requires `uv` and `git`; a +checkout is cloned to `~/.cache/trajectory/harbor-repo` on first use, override +with `HARBOR_REPO=`). The output validates against harbor's own ATIF +validator (`harbor-atif2otel`). + +Note on interpreting results: ATIF as Harbor produces it intentionally keeps +untruncated tool results, structured result payloads (in `extra`), and +per-step token metrics, so it carries more content than trajectory by design — +the comparison reflects each format's content policy, not just syntax. diff --git a/scripts/harbor_atif_convert.py b/scripts/token-efficiency/harbor_atif_convert.py similarity index 100% rename from scripts/harbor_atif_convert.py rename to scripts/token-efficiency/harbor_atif_convert.py diff --git a/scripts/token-efficiency.ts b/scripts/token-efficiency/index.ts similarity index 96% rename from scripts/token-efficiency.ts rename to scripts/token-efficiency/index.ts index ce3c18e..50874a0 100644 --- a/scripts/token-efficiency.ts +++ b/scripts/token-efficiency/index.ts @@ -19,7 +19,7 @@ * use; override with HARBOR_REPO=). * * Usage: - * bun scripts/token-efficiency.ts [options] + * bun scripts/token-efficiency/index.ts [options] * * Claude Code session (~/.claude/projects//.jsonl) * or Codex rollout (~/.codex/sessions/.../rollout-*.jsonl) @@ -37,8 +37,8 @@ import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync import { homedir, tmpdir } from "os"; import { basename, dirname, join } from "path"; import { fileURLToPath } from "url"; -import { normalizeTranscript } from "../src/index.js"; -import type { NormalizeInput } from "../src/index.js"; +import { normalizeTranscript } from "../../src/index.js"; +import type { NormalizeInput } from "../../src/index.js"; const CHUNK_CHARS = 500_000; const API_URL = "https://api.anthropic.com/v1/messages/count_tokens"; @@ -55,7 +55,7 @@ interface CanonicalRecord { function usage(): never { console.error( - "usage: bun scripts/token-efficiency.ts [--source claude-code|codex] [--model ] [--untruncated] [--out-dir ]", + "usage: bun scripts/token-efficiency/index.ts [--source claude-code|codex] [--model ] [--untruncated] [--out-dir ]", ); process.exit(1); }