diff --git a/experiments/command_specialist/codex/README.md b/experiments/command_specialist/codex/README.md index e04eba8..cbd7956 100644 --- a/experiments/command_specialist/codex/README.md +++ b/experiments/command_specialist/codex/README.md @@ -86,3 +86,51 @@ its internal Python actions do not inherit all native shell resource checks. It is not general arbitrary-shell delegation or the actual PTY recorder workload. The old OpenCode2 inspection adapter is unrelated. The local 1.5B specialist can fail; keep those results and frontier recovery cost visible. + + +## Ten-stage continuous session + +Run `ten_step.py run --arm baseline`, then `--arm chained`, then `--arm grouped` +from the repository root with the MCP-enabled Python environment. Never run the +arms concurrently. `prepare` creates a fixture and prompt for manual use in a new +Codex chat instead. The same ten-stage synthetic incident packet covers inventory, +logs, configuration differences, tests, changed paths, command durations, failure +details, a combined summary and a SHA-256 manifest. Later stages consume earlier +outputs. No private transcript text is used in fixture data. + +Baseline may batch all work in one script and choose its own native call count. +Chained requests ten serial English handoffs, each with a new local process and +history. Grouped requests two handoffs of five stages. This tests ten mechanical +steps known at the start of one session; it does not force ten frontier decisions +or simulate discoveries that were unavailable in the original prompt. + +All arms get the installed interpreter path. The runner preserves ordinary host +startup, instructions, tool discovery and approval costs. The current loop uses +the nonquantized FP16 specialist, not a new model or trained adapter. Each local +worker has its existing action/time limits. Failures are retained without silent +native fallback. A success marker alone is insufficient: the external evaluator +reopens every saved JSON and checks the content and manifest, then reruns the +saved scripts on changed inputs in a separate directory. All twenty checks, input +integrity, expected handoff count, and local completion must pass. External +assessment is outside the timed chat. These scripts are trusted local code, not +sandboxed by the harness. + +Compare saved runs with: + +``` +python experiments/command_specialist/codex/compare_ten.py BASELINE/summary.json CHAINED/summary.json GROUPED/summary.json --out work/ten-stage-comparison.json +``` + +Keep each summary beside its original run.json and fixture.json. Savings require +successful runs with matching source hashes, input hashes, model and effort. +Token usage separates cached and uncached frontier input, frontier output, and +local model input/output. It does not estimate actual charges. event-times.jsonl +records arrival timestamps for raw events; the union of started/completed tool +spans avoids double counting overlap. Time outside those spans includes startup +and frontier work and must not be labeled entirely model inference. + +Subtracting observed serial local worker time gives a conditional instant-worker +floor with every other cost unchanged. It is not an achievable optimum or a +ceiling on redesigned workflows. Failed arms receive no savings claim. Repeat +successful matched AB/BA series before claiming typical savings; one run per arm +is a pilot that can reveal failure modes and batching effects. diff --git a/experiments/command_specialist/codex/TEN-STAGE-PILOT.md b/experiments/command_specialist/codex/TEN-STAGE-PILOT.md new file mode 100644 index 0000000..37fc931 --- /dev/null +++ b/experiments/command_specialist/codex/TEN-STAGE-PILOT.md @@ -0,0 +1,79 @@ +# Ten-stage incident packet pilot + +September 10, 2026. Three sequential fresh Codex CLI sessions used gpt-6-astra, +low effort, existing login/configuration and normal approval review. Local workers +used shell-specialist-f16 with the unchanged pilot adapter, 32768 context, 8192 +output allowance and one fresh bounded process/history per handoff. No quantized +model or retraining was introduced. This is one observation per arm, not a +statistical benchmark or a claim about general command work. + +Eight recent local chat transcripts contained 78 shell-tool events. Aggregate +patterns included reading files, inventory, log search, JSON processing, Git +state, validation, saved evidence, path handling and table data. Those patterns +informed ten synthetic stages; no transcript text or real project data appears +in the fixture. Private source hashes/provenance and raw events remain local. + +| Strategy | Handoffs | Native calls | Whole chat | Correct original stages | Correct alternate stages | +| --- | ---: | ---: | ---: | ---: | ---: | +| Normal Codex shell | 0 | 5 | 93.998 s | 10/10 | 10/10 | +| One handoff per stage | 10 | 0 | 200.381 s | 0/10 | 0/10 | +| Two groups of five | 2 | 0 | 156.575 s | 0/10 | 0/10 | + +The baseline could batch efficiently and implemented one script. All ten stages +were known at session start, with later stages consuming earlier outputs. This +is not ten independently arising requests, nor ten forced native commands. +Whole-chat timing includes startup, instruction/tool discovery, approval, model +work, execution and frontier completion. Independent original/alternate artifact +assessment runs afterward and is excluded from that timing. Input bytes stayed +unchanged in all three arms. Source and fixture hashes matched across the series. + +| Strategy | Frontier input | Cached input (included) | Uncached input | Frontier output | Local input / output | +| --- | ---: | ---: | ---: | ---: | ---: | +| Normal shell | 136130 | 123776 | 12354 | 2134 | 0 / 0 | +| Ten handoffs | 330385 | 310272 | 20113 | 2425 | 215216 / 4117 | +| Two handoffs | 254899 | 239360 | 15539 | 1588 | 92908 / 7417 | + +Input totals accumulate across model requests; they are not a single context +window size. Cached input is already included in input. These are observed token +counts, not actual dollar charges. Two handoffs used 25.6% fewer frontier output +tokens than baseline, but more input tokens and failed the task; that is not a +successful-task efficiency improvement. + +Ten handoffs made 151 local model calls and recorded ten failed executions. +Nine workers exhausted their limits. The one execution-verified worker only +printed its expected marker and created no requested JSON. The external oracle +rejected it. The first stage wrote a non-JSON inventory; later outputs were +missing. Two grouped workers made 32 model calls and seven failed executions; +one exhausted its limits and one failed. Detailed actions and stderr remain in +local result artifacts. Dependent failures are not independent model trials. + +| Strategy | Observed tool span union | Local worker time within spans | Outside observed tool spans | +| --- | ---: | ---: | ---: | +| Normal shell | 8.841 s | 0 s | 85.157 s | +| Ten handoffs | 98.306 s | 64.983 s | 102.075 s | +| Two handoffs | 100.688 s | 87.937 s | 55.887 s | + +Tool spans use event arrival timestamps. Outside-span time includes startup and +frontier work, not exclusively model inference. Removing all observed local +worker time, while holding every other cost fixed, leaves 135.398 s for ten +handoffs and 68.638 s for two. The latter is arithmetically 25.360 s below the +baseline, but both underlying runs failed. Neither number estimates an accurate +implementation or a global best possible saving. Correct repair, different +handoff boundaries and frontier behavior would change the timings. + +The experiment establishes that ten fresh lifecycles can be chained through the +real host, with saved results returned to the frontier. It does not establish +reliable task completion or time savings. Correctness and useful artifact-level +completion checks are the next bottlenecks. Grouping appears worth investigating, +but faster inference alone does not fix incorrect programs or redundant handoffs. + +An earlier 134.725 s baseline preflight also passed all twenty artifact checks. +Its original summary falsely flagged an input change because the collector used +Windows default decoding for a Unicode filename. Explicit UTF-8 decoding fixed +the collector; a separate correction artifact preserves the original evidence. +The preflight lacked the supplied interpreter context and is excluded above. + +Reproduce with the three ten_step.py arms and compare_ten.py documented in +README.md. Repeat successful matched runs in alternating order before estimating +typical latency or savings. No raw chat logs, generated worker programs, private +machine paths or model weights belong in this report's public source. diff --git a/experiments/command_specialist/codex/bench.py b/experiments/command_specialist/codex/bench.py index afbf22b..882be76 100644 --- a/experiments/command_specialist/codex/bench.py +++ b/experiments/command_specialist/codex/bench.py @@ -8,6 +8,7 @@ import subprocess import sys import time +import threading import uuid HERE = Path(__file__).resolve().parent @@ -77,10 +78,10 @@ def codex_command(): return [path] -def run(directory, model, effort): +def run(directory, model, effort, *, collector=None, timeout=600, extra_sources=()): if (directory / 'events.jsonl').exists(): raise ValueError('Run evidence already exists; prepare a fresh workspace') - sources = [HERE / 'server.py', HERE / 'bench.py', HERE.parent / 'delegate.py'] + sources = [HERE / 'server.py', HERE / 'bench.py', HERE.parent / 'delegate.py', *extra_sources] source_hashes = {p.name: hashlib.sha256(p.read_bytes()).hexdigest() for p in sources} root = directory / 'workspace' command = codex_command() + ['exec', '--json', '--approve-for-me', '-C', str(root), @@ -90,18 +91,30 @@ def run(directory, model, effort): start = time.perf_counter() timed_out = False with (directory / 'events.jsonl').open('wb') as output, (directory / 'stderr.txt').open('wb') as errors, (directory / 'prompt.txt').open('rb') as prompt: - process = subprocess.Popen(command, stdin=prompt, stdout=output, stderr=errors) + process = subprocess.Popen(command, stdin=prompt, stdout=subprocess.PIPE, stderr=errors) + def capture(): + with (directory / 'event-times.jsonl').open('w', encoding='utf-8') as times: + for index, line in enumerate(iter(process.stdout.readline, b'')): + output.write(line) + output.flush() + times.write(json.dumps({'line': index, 'elapsed_ms': round((time.perf_counter()-start)*1000)})+'\n') + times.flush() + reader = threading.Thread(target=capture, daemon=True) + reader.start() try: - process.wait(timeout=600) + process.wait(timeout=timeout) except (subprocess.TimeoutExpired, KeyboardInterrupt): timed_out = True subprocess.run(['taskkill', '/PID', str(process.pid), '/T', '/F'], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, timeout=15) process.wait(timeout=15) + reader.join(timeout=15) + if reader.is_alive(): + raise RuntimeError('Event capture did not close after Codex exit') write_json(directory / 'run.json', {'source_sha256': source_hashes, 'source_changed_during_run': any(hashlib.sha256(p.read_bytes()).hexdigest() != source_hashes[p.name] for p in sources), 'model': model, 'effort': effort, 'exit_code': process.returncode, 'timed_out_or_cancelled': timed_out, 'wall_ms': round((time.perf_counter()-start)*1000), 'command': command, 'billing': 'Existing Codex login; tokens are usage, not a dollar charge estimate'}) - return collect(directory) + return (collector or collect)(directory) def collect(directory): diff --git a/experiments/command_specialist/codex/compare_ten.py b/experiments/command_specialist/codex/compare_ten.py new file mode 100644 index 0000000..ea5ee7c --- /dev/null +++ b/experiments/command_specialist/codex/compare_ten.py @@ -0,0 +1,57 @@ +"""Compare whole-session ten-stage evidence without rewarding failed work.""" +import argparse +import json +from pathlib import Path + + +def compare(rows): + baseline = next(r for r in rows if r['arm'] == 'baseline') + results = [] + for row in rows: + usage = row['frontier_usage'] + result = {k: row[k] for k in ('arm', 'success', 'wall_ms', 'delegations', 'native_commands', + 'completed_stages', 'alternate_stages', 'local_worker_ms', 'outside_tool_span_ms')} + result['frontier_input_tokens'] = usage['input_tokens'] + result['frontier_cached_input_tokens'] = usage['cached_input_tokens'] + result['frontier_uncached_input_tokens'] = usage['input_tokens'] - usage['cached_input_tokens'] + result['frontier_output_tokens'] = usage['output_tokens'] + result['local_input_tokens'] = sum(r['input_tokens'] for r in row['local']) + result['local_output_tokens'] = sum(r['output_tokens'] for r in row['local']) + result['local_model_calls'] = sum(r['model_calls'] for r in row['local']) + result['failed_local_executions'] = sum(r['failed_executions'] for r in row['local']) + matched = all(row[k] == baseline[k] for k in ('model', 'effort', 'benchmark_identity')) + eligible = row['success'] and baseline['success'] and matched + result['savings_eligible'] = eligible + if eligible: + result['saved_ms'] = baseline['wall_ms'] - row['wall_ms'] + result['saved_percent'] = 100 * result['saved_ms'] / baseline['wall_ms'] + result['instant_worker_floor_ms'] = row['wall_ms'] - row['local_worker_ms'] + result['instant_worker_saved_ms'] = baseline['wall_ms'] - result['instant_worker_floor_ms'] + results.append(result) + return {'results': results, 'limits': [ + 'Each row is one whole Codex CLI session, not a latency distribution or billing estimate.', + 'Only successful matching-model runs receive savings. Verify matching source/config and fixtures before pooling.', + 'Instant-worker floor subtracts observed serial local worker wall time only; all other costs are held fixed.', + 'Outside-tool span is observed event timing, including startup and frontier work, not pure model inference.', + 'Independent changed-input evaluation happens after the timed chat.']} + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('summaries', nargs='+', type=Path) + parser.add_argument('--out', required=True, type=Path) + args = parser.parse_args() + rows = [] + for path in args.summaries: + row = json.loads(path.read_text(encoding='utf-8')) + run = json.loads((path.parent/'run.json').read_text(encoding='utf-8')) + fixture = json.loads((path.parent/'fixture.json').read_text(encoding='utf-8')) + row['benchmark_identity'] = {'source': run['source_sha256'], 'inputs': fixture['input_hashes'], 'case': fixture['case']} + rows.append(row) + report = compare(rows) + args.out.parent.mkdir(parents=True, exist_ok=True) + args.out.write_text(json.dumps(report, indent=2)+'\n', encoding='utf-8') + print(json.dumps(report, indent=2)) + + +if __name__ == '__main__': main() diff --git a/experiments/command_specialist/codex/ten_step.py b/experiments/command_specialist/codex/ten_step.py new file mode 100644 index 0000000..389d235 --- /dev/null +++ b/experiments/command_specialist/codex/ten_step.py @@ -0,0 +1,146 @@ +"""Run one ten-stage job in one fresh Codex session with three execution strategies.""" +import argparse +import hashlib +import json +from pathlib import Path +import shutil +import subprocess +import sys +import uuid + +from bench import run, write_json +from ten_step_workload import STEPS, inputs, assess + +HERE = Path(__file__).resolve().parent + + +def prepare(out, arm): + directory = out.resolve() / (arm + '-' + uuid.uuid4().hex[:8]) + root = directory / 'workspace' + root.mkdir(parents=True) + inputs(root) + subprocess.run(['git', 'init', '--quiet', str(root)], check=True) + scripts = ['build_packet.py'] if arm == 'baseline' else ([f'step{i:02}.py' for i in range(1, 11)] if arm == 'chained' else ['group01.py', 'group02.py']) + guidance = ('This is a synthetic benchmark fixture, not a source project. Do the requested incident investigation. ' + f'Installed Python 3.12 interpreter: {sys._base_executable}. Use it directly; runtime readiness is already checked. ' + 'Use Python 3.12 standard library only. Modify only the named Python scripts and out/*.json. ' + 'Do not edit inputs. No commits, PRs, installs or unrelated research. Inspect real outcomes; never fabricate success. ' + 'All work belongs to one session. Keep data references explicit.\n') + if arm == 'baseline': + guidance += 'Use normal native shell/file tools and build_packet.py. Batch work as efficiently as you judge useful; there is no required number of shell calls. Do not use a local model.\n' + else: + guidance += ('Use command_specialist.run_python_task with English intent, exact script target, relevant context and exact expected stdout. ' + 'Do not write source or command sequences yourself. Call serially; each call is a fresh worker. ' + 'After a failure, preserve it and continue later requested groups when possible; do not switch to native execution or silently retry in another worker.\n') + guidance += ('Make exactly ten handoffs, one per numbered stage.\n' if arm == 'chained' else 'Make exactly two handoffs: stages 1–5 together, then stages 6–10 together.\n') + (root / 'AGENTS.md').write_text(guidance, encoding='utf-8') + task = ('Build a reproducible incident evidence packet from synthetic files under inputs. ' + 'Complete these ten stages in order and report which saved outputs were actually verified. ' + 'Later stages consume earlier outputs. Each generated script must create out if missing. ' + 'Read the input files rather than hardcoding their data.\n\n' + + '\n'.join(f'{i}. {step}' for i, step in enumerate(STEPS, 1))) + if arm == 'baseline': + task += '\nUse build_packet.py to produce all ten outputs and print JOB OK followed by a newline only when complete. You may batch native work.' + elif arm == 'chained': + task += '\nFor stage N use target stepNN.py (01–10), save its requested output, and print exactly STEP NN OK followed by a newline. The expected_stdout field uses Windows CRLF. Supply the relevant stage and dependency filenames to each fresh worker.' + else: + task += '\nUse group01.py for stages 1–5 and group02.py for stages 6–10. Each group saves all five requested outputs and prints exactly GROUP 01 OK or GROUP 02 OK followed by a newline. expected_stdout uses Windows CRLF. Pass all stage requirements and dependency filenames for that group.' + (directory / 'prompt.txt').write_text(task, encoding='utf-8') + if arm != 'baseline': + (root / '.codex').mkdir() + (root / '.codex/config.toml').write_text('[mcp_servers.command_specialist]\ncommand = '+json.dumps(sys.executable)+'\nargs = '+json.dumps([str(HERE/'server.py'),'--root',str(root),'--artifacts',str(directory/'local'),'--allow-execute'])+'\nrequired = true\nstartup_timeout_sec = 30\ntool_timeout_sec = 330\ndefault_tools_approval_mode = "prompt"\n', encoding='utf-8') + write_json(directory/'fixture.json', {'case':'ten-stage-incident','arm':arm,'scripts':scripts, + 'input_hashes':{p.relative_to(root).as_posix():hashlib.sha256(p.read_bytes()).hexdigest() for p in (root/'inputs').rglob('*') if p.is_file()}}) + return directory + + +def replay(directory, scripts): + root = directory/'workspace' + target = directory/'alternate' + if target.exists(): + raise ValueError('Alternate evidence already exists; do not overwrite a measured run') + target.mkdir() + inputs(target, alternate=True) + executions=[] + for script in scripts: + source=root/script + if not source.exists(): + executions.append({'script':script,'exit_code':None,'error':'missing script'}) + continue + shutil.copyfile(source,target/script) + try: + result=subprocess.run([sys._base_executable,'-S',str(target/script)],cwd=target,capture_output=True,timeout=20) + executions.append({'script':script,'exit_code':result.returncode}) + (target/(script+'.stdout')).write_bytes(result.stdout) + (target/(script+'.stderr')).write_bytes(result.stderr) + except subprocess.TimeoutExpired: + executions.append({'script':script,'exit_code':124}) + return {'executions':executions,'checks':assess(target)} + + +def collect(directory): + root=directory/'workspace' + fixture=json.loads((directory/'fixture.json').read_text(encoding='utf-8')) + run_info=json.loads((directory/'run.json').read_text(encoding='utf-8')) + events=[json.loads(line) for line in (directory/'events.jsonl').read_text(encoding='utf-8').splitlines() if line.strip()] + times=[json.loads(line) for line in (directory/'event-times.jsonl').read_text(encoding='utf-8').splitlines()] + timestamps={r['line']:r['elapsed_ms'] for r in times} + turns=[e for e in events if e.get('type')=='turn.completed'] + completed=[e['item'] for e in events if e.get('type')=='item.completed'] + timings={} + for index,event in enumerate(events): + item=event.get('item',{}) + if item.get('type') not in ('command_execution','mcp_tool_call'): continue + row=timings.setdefault(item['id'],{'type':item['type']}) + if event['type']=='item.started': row['start_ms']=timestamps.get(index) + if event['type']=='item.completed': row['end_ms']=timestamps.get(index) + intervals=[v for v in timings.values() if v.get('start_ms') is not None and v.get('end_ms') is not None] + spans=sorted((v['start_ms'],v['end_ms']) for v in intervals) + merged=[] + for start,end in spans: + if merged and start<=merged[-1][1]: merged[-1][1]=max(merged[-1][1],end) + else: merged.append([start,end]) + tool_wall=sum(end-start for start,end in merged) + checks=assess(root) + alternate=replay(directory,fixture['scripts']) + local=[] + for path in (directory/'local').glob('*/result.json'): + r=json.loads(path.read_text(encoding='utf-8')) + local.append({'verified':r['verified'],'status':r['status'],'wall_ms':r['wall_ms'], + 'model_calls':len(r['model_calls']),'input_tokens':sum(c['response'].get('prompt_eval_count',0) for c in r['model_calls']), + 'output_tokens':sum(c['response'].get('eval_count',0) for c in r['model_calls']), + 'failed_executions':sum(a.get('result',{}).get('exit_code',0)!=0 for a in r['actions'])}) + expected_calls={'baseline':0,'chained':10,'grouped':2}[fixture['arm']] + mcp=[i for i in completed if i.get('type')=='mcp_tool_call' and i.get('server')=='command_specialist'] + inputs_unchanged=all((root/name).exists() and hashlib.sha256((root/name).read_bytes()).hexdigest()==value for name,value in fixture['input_hashes'].items()) + success=bool(turns and run_info['exit_code']==0 and not run_info['source_changed_during_run'] and inputs_unchanged + and all(c['passed'] for c in checks) and all(c['passed'] for c in alternate['checks']) + and all(e['exit_code']==0 for e in alternate['executions']) and len(mcp)==expected_calls + and len(local)==expected_calls and all(r['verified'] for r in local)) + summary={'arm':fixture['arm'],'success':success,'wall_ms':run_info['wall_ms'], + 'tool_span_ms':tool_wall,'outside_tool_span_ms':run_info['wall_ms']-tool_wall, + 'local_worker_ms':sum(r['wall_ms'] for r in local),'frontier_usage':{k:sum(t.get('usage',{}).get(k,0) for t in turns) for k in ('input_tokens','cached_input_tokens','output_tokens','reasoning_output_tokens')}, + 'native_commands':sum(i.get('type')=='command_execution' for i in completed),'delegations':len(mcp), + 'completed_stages':sum(c['passed'] for c in checks),'alternate_stages':sum(c['passed'] for c in alternate['checks']), + 'checks':checks,'alternate':alternate,'local':local,'tool_intervals':intervals,'input_unchanged':inputs_unchanged, + 'model':run_info['model'],'effort':run_info['effort'],'raw_events':str(directory/'events.jsonl')} + write_json(directory/'summary.json',summary) + return summary + + +def main(): + parser=argparse.ArgumentParser(description=__doc__) + parser.add_argument('operation',choices=['prepare','run']) + parser.add_argument('--arm',choices=['baseline','chained','grouped'],required=True) + parser.add_argument('--out',type=Path,default=Path('work/ten-stage')) + parser.add_argument('--model',default='gpt-6-astra') + parser.add_argument('--effort',default='low') + args=parser.parse_args() + directory=prepare(args.out,args.arm) + if args.operation=='prepare': print(directory); return + result=run(directory,args.model,args.effort,collector=collect,timeout=1800, + extra_sources=[Path(__file__),HERE/'ten_step_workload.py']) + print(json.dumps({k:v for k,v in result.items() if k not in ('checks','alternate','tool_intervals','local')},indent=2)) + + +if __name__=='__main__': main() diff --git a/experiments/command_specialist/codex/ten_step_workload.py b/experiments/command_specialist/codex/ten_step_workload.py new file mode 100644 index 0000000..7bca2bf --- /dev/null +++ b/experiments/command_specialist/codex/ten_step_workload.py @@ -0,0 +1,90 @@ +"""Synthetic ten-stage incident investigation derived from private command patterns.""" +import hashlib +import json +from pathlib import Path + + +def write_json(path, value): + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(value, indent=2), encoding='utf-8') + + +def inputs(root, alternate=False): + directory = root / 'inputs' + directory.mkdir(parents=True, exist_ok=True) + lines = ['INFO start', 'ERROR compiler: module missing', 'WARN retry', 'ERROR tests: failed', 'INFO stop'] + if alternate: + lines += ['WARN slow disk', 'ERROR packaging: failed'] + (directory / 'build.log').write_text('\n'.join(lines)+'\n', encoding='utf-8') + write_json(directory / 'desired.json', {'model': 'local-f16', 'context': 32768, 'output': 8192}) + write_json(directory / 'actual.json', {'model': 'local-f16' if not alternate else 'other-f16', 'context': 8192, 'output': 1024}) + write_json(directory / 'tests.json', [{'name': 'parser', 'status': 'failed'}, + {'name': 'paths', 'status': 'passed' if not alternate else 'failed'}, {'name': 'schema', 'status': 'passed'}]) + (directory / 'git-status.txt').write_text(' M src/parser.py\n?? run/check.py\n M docs/notes.md\n', encoding='utf-8') + commands = [{'name': 'compile', 'exit_code': 1, 'duration_ms': 1400, 'stderr': 'module missing'}, + {'name': 'tests', 'exit_code': 1, 'duration_ms': 2300, 'stderr': 'parser failed'}, + {'name': 'inspect', 'exit_code': 0, 'duration_ms': 300, 'stderr': ''}] + if alternate: + commands.append({'name': 'package', 'exit_code': 2, 'duration_ms': 800, 'stderr': 'archive failed'}) + write_json(directory / 'commands.json', commands) + (directory / "naïve team's notes.txt").write_text('Ticket SYNTH-42\nNo real customer data.\n', encoding='utf-8') + + +STEPS = [ + 'Inventory every regular file recursively under inputs. Save out/01.json as an object with files: a lexicographically sorted list of paths relative to inputs, using forward slashes.', + 'Read inputs/build.log. Count each first whitespace-delimited log level. Save out/02.json as an object mapping each observed level to its count.', + 'Read inputs/build.log. Save out/03.json as an object with errors: a list of the ERROR messages in original order, excluding the ERROR prefix and following space.', + 'Compare inputs/desired.json with inputs/actual.json. Save out/04.json as an object with mismatched_keys: a sorted list of keys whose values differ.', + 'Read inputs/tests.json. Save out/05.json with passed: the number whose status is passed, and failed: a sorted list of names whose status is failed.', + 'Read inputs/git-status.txt in Git porcelain format (first two characters are status, character three is a space, remaining text is the path). Save out/06.json with changed_paths: the sorted paths and untracked_paths: the sorted paths whose status is ??.', + 'Read inputs/commands.json. Save out/07.json with total_duration_ms: the sum of all duration_ms, and failed_commands: a sorted list of names with nonzero exit_code.', + 'Read out/07.json and inputs/commands.json. For each failed command named by out/07.json, look up its stderr. Save out/08.json as an object mapping those command names to their stderr strings.', + 'Read out/02.json, out/04.json, out/05.json and out/07.json. Save out/09.json with error_count: the ERROR count, config_mismatch_count: length of mismatched_keys, failed_test_count: length of failed, failed_command_count: length of failed_commands, and total_duration_ms: the value from out/07.json.', + 'Read all nine out/01.json through out/09.json files and confirm they parse as JSON. Save out/10.json as an object mapping each filename (01.json through 09.json, without directory) to the lowercase SHA-256 hex digest of its exact file bytes.' +] + + +def expected(root): + directory = root / 'inputs' + lines = (directory / 'build.log').read_text(encoding='utf-8').splitlines() + counts = {} + for line in lines: + level = line.split()[0] + counts[level] = counts.get(level, 0) + 1 + desired = json.loads((directory / 'desired.json').read_text(encoding='utf-8')) + actual = json.loads((directory / 'actual.json').read_text(encoding='utf-8')) + mismatches = sorted(k for k in desired if desired[k] != actual.get(k)) + tests = json.loads((directory / 'tests.json').read_text(encoding='utf-8')) + failed_tests = sorted(t['name'] for t in tests if t['status'] == 'failed') + status = (directory / 'git-status.txt').read_text(encoding='utf-8').splitlines() + commands = json.loads((directory / 'commands.json').read_text(encoding='utf-8')) + failed = sorted(c['name'] for c in commands if c['exit_code'] != 0) + total = sum(c['duration_ms'] for c in commands) + values = [ + {'files': sorted(p.relative_to(directory).as_posix() for p in directory.rglob('*') if p.is_file())}, + counts, {'errors': [line[6:] for line in lines if line.startswith('ERROR ')]}, + {'mismatched_keys': mismatches}, {'passed': sum(t['status'] == 'passed' for t in tests), 'failed': failed_tests}, + {'changed_paths': sorted(line[3:] for line in status), 'untracked_paths': sorted(line[3:] for line in status if line[:2] == '??')}, + {'total_duration_ms': total, 'failed_commands': failed}, + {c['name']: c['stderr'] for c in commands if c['name'] in failed}, + {'error_count': counts.get('ERROR', 0), 'config_mismatch_count': len(mismatches), + 'failed_test_count': len(failed_tests), 'failed_command_count': len(failed), 'total_duration_ms': total}] + return values + + +def assess(root): + checks = [] + for index, value in enumerate(expected(root), 1): + path = root / 'out' / f'{index:02}.json' + try: + actual = json.loads(path.read_text(encoding='utf-8-sig')) + checks.append({'step': index, 'passed': actual == value}) + except (OSError, ValueError) as error: + checks.append({'step': index, 'passed': False, 'error': str(error)}) + try: + actual = json.loads((root / 'out/10.json').read_text(encoding='utf-8-sig')) + hashes = {f'{index:02}.json': hashlib.sha256((root / 'out' / f'{index:02}.json').read_bytes()).hexdigest() for index in range(1, 10)} + checks.append({'step': 10, 'passed': actual == hashes}) + except (OSError, ValueError) as error: + checks.append({'step': 10, 'passed': False, 'error': str(error)}) + return checks diff --git a/experiments/command_specialist/codex/test_compare.py b/experiments/command_specialist/codex/test_compare.py index 581d71b..1168238 100644 --- a/experiments/command_specialist/codex/test_compare.py +++ b/experiments/command_specialist/codex/test_compare.py @@ -15,5 +15,19 @@ def test_different_frontier_settings_are_not_a_pair(self): {'case': 'csv', 'model': 'b', 'effort': 'low'}) +class TenStageComparisonInvariants(unittest.TestCase): + def test_failures_and_changed_sources_have_no_savings(self): + from compare_ten import compare as compare_ten + row = dict(arm='baseline', success=True, wall_ms=100, delegations=0, + native_commands=1, completed_stages=10, alternate_stages=10, + local_worker_ms=0, outside_tool_span_ms=90, local=[], + frontier_usage={'input_tokens':100, 'cached_input_tokens':80, 'output_tokens':10}, + model='same', effort='low', benchmark_identity={'source':'frozen'}) + for changes in ({'success':False}, {'benchmark_identity':{'source':'changed'}}): + result = compare_ten([row,dict(row,arm='chained',**changes)])['results'][1] + self.assertFalse(result['savings_eligible']) + self.assertNotIn('saved_ms',result) + + if __name__ == '__main__': unittest.main()