From ed90b4dffecaefc36b5ccfa743f3a663347b8f98 Mon Sep 17 00:00:00 2001 From: Noisemaker111 <139656120+Noisemaker111@users.noreply.github.com> Date: Thu, 10 Sep 2026 12:49:24 -0400 Subject: [PATCH] Connect English worker to separate Codex chats with paired harness evidence --- .github/workflows/agent-checks.yml | 2 + experiments/command_specialist/README.md | 3 + .../command_specialist/codex/README.md | 88 ++++++++ .../command_specialist/codex/RESULTS.md | 53 +++++ experiments/command_specialist/codex/bench.py | 191 ++++++++++++++++++ .../command_specialist/codex/compare.py | 34 ++++ .../command_specialist/codex/requirements.txt | 1 + .../command_specialist/codex/server.py | 80 ++++++++ .../command_specialist/codex/test_compare.py | 19 ++ experiments/command_specialist/delegate.py | 1 + 10 files changed, 472 insertions(+) create mode 100644 experiments/command_specialist/codex/README.md create mode 100644 experiments/command_specialist/codex/RESULTS.md create mode 100644 experiments/command_specialist/codex/bench.py create mode 100644 experiments/command_specialist/codex/compare.py create mode 100644 experiments/command_specialist/codex/requirements.txt create mode 100644 experiments/command_specialist/codex/server.py create mode 100644 experiments/command_specialist/codex/test_compare.py diff --git a/.github/workflows/agent-checks.yml b/.github/workflows/agent-checks.yml index fbc439a..602fcda 100644 --- a/.github/workflows/agent-checks.yml +++ b/.github/workflows/agent-checks.yml @@ -32,3 +32,5 @@ jobs: run: python -m unittest discover -s experiments/command_specialist -p test_delegate.py -v - name: Frozen main candidate and patch notes run: python scripts/check_release.py + - name: Codex comparison evidence invariants + run: python -m unittest discover -s experiments/command_specialist/codex -p test_compare.py -v diff --git a/experiments/command_specialist/README.md b/experiments/command_specialist/README.md index e228337..ffae69e 100644 --- a/experiments/command_specialist/README.md +++ b/experiments/command_specialist/README.md @@ -267,3 +267,6 @@ authorization. Keep full raw results retrievable when a compact packet is insuff See [the English handoff contract](ENGLISH_HANDOFF.md) for the fresh-worker local CLI, non-quantized runtime, configurable budgets and verification boundaries. This is separate from the earlier read-only inspection adapter. + +For the separately configured Codex hookup and paired normal-shell comparison, +see [Codex harness](codex/README.md) and its qualified [observations](codex/RESULTS.md). diff --git a/experiments/command_specialist/codex/README.md b/experiments/command_specialist/codex/README.md new file mode 100644 index 0000000..e04eba8 --- /dev/null +++ b/experiments/command_specialist/codex/README.md @@ -0,0 +1,88 @@ +# Codex English-delegation hookup + +This connects the local FP16 worker to a **fresh Codex chat**, and captures a +comparison against Codex using its normal shell/file tools. It does not compare +two local models. It is a Windows, trusted-workspace Python-task prototype. + +## Start a separate chat + +Install into an isolated Python 3.12 environment: + +``` +python -m pip install -r experiments/command_specialist/codex/requirements.txt +python experiments/command_specialist/codex/bench.py prepare --case csv --arm delegated +``` + +Open the printed directory's `workspace` folder as a project in Codex, start a +fresh chat, and send the text from the adjacent `prompt.txt`. The project-local +`.codex/config.toml` connects `command_specialist.run_python_task`. Trust that +specific project when Codex requests it. The tool executes generated Python with +the account's privileges, so approve only the intended trusted fixture operation. +No global configuration is edited. Desktop tool loading must be verified in the +new chat; CLI connection evidence is not a claim that the desktop UI was exercised. + +The tool takes English intent, an exact target, context, constraints and the +completion condition. It starts a new process/history per delegation and returns +verified observations and raw evidence to the originating Codex turn. Python +worker startup uses the base interpreter without site initialization, separate +from the MCP SDK's environment. Failed workers never change model or executor. +Cancellation terminates the worker process tree; an abruptly cancelled artifact +may remain marked running and must never be treated as success. + +## Measure through real Codex chats + +Run each arm separately (never concurrently) using the same model and effort: + +``` +python experiments/command_specialist/codex/bench.py run --case csv --arm baseline --model gpt-6-astra --effort low +python experiments/command_specialist/codex/bench.py run --case csv --arm delegated --model gpt-6-astra --effort low +``` + +`repair` is a second case with an actual broken program. Each run creates an +independent Git workspace and persisted Codex session, preserves normal user +configuration/rules, and uses automatic approval review with workspace-write. +It never disables sandboxing or hook trust. It uses the existing Codex login; +no API key is introduced and no dollar charge is inferred from token counts. + +The adjacent `events.jsonl` is the actual `codex exec --json` stream. +`final.txt` is the frontier's final response, `run.json` contains timing/config, +and `summary.json` contains observed usage and independent accuracy checks. +The first `thread.started` event identifies the saved chat for `codex resume`. +Opening/resuming it after measurement is allowed, but that interaction is not part +of the completed measured turn. A desktop conversation without captured events +must not be assigned CLI token/timing figures. + +Create a comparison from the two saved summaries: + +``` +python experiments/command_specialist/codex/compare.py BASELINE/summary.json DELEGATED/summary.json --out work/comparison.json +``` + +The runner checks the saved program on the original input and an alternate input, +then restores the original fixture. This detects hardcoded expected output. It +records failed local actions and requires the delegated worker to report verified +completion. The baseline writes/runs with native tools. The delegated arm submits +English once; fallback is not silently counted as local success. + +Measure whole process startup through frontier completion, including discovery, +permission handling, native commands, local inference/repair and persistence. +Report input, cached input, uncached input, output and local tokens separately. +Model generation time alone is not the user operation. Configured plugins and +existing instruction overhead remain part of this realistic first comparison. +The independent evaluator runs after the timed chat and is not in chat latency. + +One pair is a hookup smoke, **not evidence of general savings**. For benchmark +claims, freeze source/config/tasks, alternate AB/BA order, repeat matched pairs, +include failures/denials and multiple task sizes, separate model cold/warm state, +and report success plus p50/p95 with sufficient samples. Current fixtures are +development smoke tasks, not a frozen final test or a broad command benchmark. +Do not edit the harness mid-series and pool measurements as one experiment. + +## Boundary still open + +This is an explicit MCP operation with its own fixed workspace and trusted-code +execution permission. It does not intercept or replace Codex's native shell, and +its internal Python actions do not inherit all native shell resource checks. +It is not general arbitrary-shell delegation or the actual PTY recorder workload. +The old OpenCode2 inspection adapter is unrelated. The local 1.5B specialist can +fail; keep those results and frontier recovery cost visible. diff --git a/experiments/command_specialist/codex/RESULTS.md b/experiments/command_specialist/codex/RESULTS.md new file mode 100644 index 0000000..2583000 --- /dev/null +++ b/experiments/command_specialist/codex/RESULTS.md @@ -0,0 +1,53 @@ +# Configured Codex hookup observations + +September 10, 2026. These are development hookup smokes, not a frozen benchmark. +The frontier was the installed Codex CLI with its existing ChatGPT login, +`gpt-6-astra`, low reasoning, normal user configuration and workspace-write with +automatic approval review. Each operation ran in a separately persisted fresh chat +and fresh Git workspace. No global Codex configuration was changed. + +The successful delegated CSV operation was observed through the configured real +MCP host: Codex sent one English `run_python_task` call, the FP16 worker returned +verified output, and the originating frontier turn reported it. The saved script, +raw events, local evidence and evaluator outputs were reopened. Original and +alternate-input checks passed for both the normal-shell and delegated programs. + +| Observation | Normal shell | English delegation | +| --- | ---: | ---: | +| Whole Codex process through final result | 71.075 s | 43.118 s | +| Frontier input tokens | 154426 | 95506 | +| Cached frontier input tokens | 131456 | 82944 | +| Frontier output tokens | 995 | 486 | +| Local input / output tokens | 0 / 0 | 4454 / 181 | +| Native command calls | 6 | 1 | +| Local English handoffs | 0 | 1 | +| Original and changed-input correctness | pass | pass | + +One pair showed lower elapsed time and frontier usage for delegation. It does not +establish expected savings: startup, permission handling, cache state, repeated +runs and task diversity are not controlled sufficiently here. Native baseline +Python access needed sandbox recovery; both chats encountered an inaccessible +shell skill. Configured unrelated Cloudflare authentication also logged an error. +These are real host costs in this smoke, not costs to attribute to the local model. +Local inference/execution consumed 2.813 seconds inside the 43.118-second handoff. +Do not quote local duration as end-to-end latency or infer dollar charges. + +Earlier hookup failures are retained privately: the first worker stalled before +artifact initialization and was terminated; using the base stdlib interpreter +without SDK site startup and inherited protocol stdin allowed the next run to +execute. That next run returned an honest exhausted result because source generation +had not received the exact completion stdout. The worker printed `Total: 74.95` +and failed the required exact output. Passing the completion condition into the +source-generation context fixed that context omission. Multiple setup revisions +were involved, so failed and successful runs are not one pooled benchmark series. + +Raw artifacts remain in ignored `work/codex-bench`. A separate user workspace is +prepared for interactive testing; the Codex desktop UI itself was not operated. +General shell replacement, host-native per-command permissions, the real PTY +recorder, broad accuracy and statistically supported speedups remain unverified. + +A second configured-host operation repaired the existing NameError in one English +handoff with no native frontier commands. Original and alternate inputs passed; +whole Codex time was 32.075 seconds, frontier input/output 69912/348 tokens, local +input/output 4457/173 tokens, and local time 2.765 seconds. No paired repair baseline +was run, so this is additional hookup/repair evidence, not a savings comparison. diff --git a/experiments/command_specialist/codex/bench.py b/experiments/command_specialist/codex/bench.py new file mode 100644 index 0000000..afbf22b --- /dev/null +++ b/experiments/command_specialist/codex/bench.py @@ -0,0 +1,191 @@ +"""Prepare or run paired, fresh Codex chats; retain raw events and external checks.""" +import argparse +from datetime import datetime, timezone +import hashlib +import json +from pathlib import Path +import shutil +import subprocess +import sys +import time +import uuid + +HERE = Path(__file__).resolve().parent +CASES = { + 'csv': { + 'task': 'Create report.py that reads orders.csv, sums quantity times unit_price for rows whose status is paid, and prints the total as exactly two decimal places. Run it and verify the actual result. Use decimal arithmetic.', + 'files': {'orders.csv': 'status,quantity,unit_price\npaid,3,19.95\nrefunded,5,100.00\npaid,2,7.50\npaid,1,0.10\n'}, + 'stdout': '74.95\r\n', + 'alternate': {'file': 'orders.csv', 'content': 'status,quantity,unit_price\npaid,2,0.15\nrefunded,10,9.00\n', 'stdout': '0.30\r\n'}, + 'context': 'Python 3.12 standard library. orders.csv is UTF-8 CSV with status, quantity, unit_price columns. The file is in the working directory.'}, + 'repair': { + 'task': 'Run the existing report.py to observe its failure. Fix it so it sums the values in values.json and prints the sum. Rerun and verify the actual result.', + 'files': {'values.json': '[4, 9, 15]\n', 'report.py': 'import json\nfrom pathlib import Path\nvalues = json.loads(Path("values.json").read_text())\nprint(sum(valuez))\n'}, + 'stdout': '28\r\n', 'alternate': {'file': 'values.json', 'content': '[2, 7, -1]\n', 'stdout': '8\r\n'}, 'context': 'Python 3.12 standard library. values.json is a JSON array of integers in the working directory.'}, +} + + +def write_json(path, value): + path.write_text(json.dumps(value, ensure_ascii=False, indent=2), encoding='utf-8') + + +def prepare(out, case, arm): + directory = out.resolve() / (case + '-' + arm + '-' + uuid.uuid4().hex[:8]) + root = directory / 'workspace' + root.mkdir(parents=True) + data = CASES[case] + for name, content in data['files'].items(): + (root / name).write_text(content, encoding='utf-8', newline='') + # An independent repository prevents inheriting shell-forensics release chores. + subprocess.run(['git', 'init', '--quiet', str(root)], check=True) + instructions = ('This is an isolated command-specialist benchmark fixture, not an implementation project. ' + 'Do only the requested operation; no commits, PRs, dependency installs or unrelated browsing. ' + 'Modify only report.py. Read the provided input file. Use Python 3.12 standard library. ' + 'Report observed failures and any fallback honestly. Do not fabricate results.\n') + instructions += ('Use normal native shell/file tools. Do not invoke a local model or command-specialist.\n' + if arm == 'baseline' else + 'Call command_specialist.run_python_task once with English intent, exact target report.py, ' + 'known context and expected stdout. Do not write source or command sequences in that handoff. ' + 'If it fails, report failure; do not silently use another executor.\n') + (root / 'AGENTS.md').write_text(instructions, encoding='utf-8') + prompt = data['task'] + '\n' + data['context'] + '\nExpected stdout: ' + json.dumps(data['stdout']) + (directory / 'prompt.txt').write_text(prompt, encoding='utf-8') + if arm == 'delegated': + config = ('[mcp_servers.command_specialist]\n' + f'command = {json.dumps(sys.executable)}\n' + f'args = {json.dumps([str(HERE / "server.py"), "--root", str(root), "--artifacts", str(directory / "local"), "--allow-execute"])}\n' + 'required = true\nstartup_timeout_sec = 30\ntool_timeout_sec = 330\n' + 'default_tools_approval_mode = "prompt"\n') + (root / '.codex').mkdir() + (root / '.codex' / 'config.toml').write_text(config, encoding='utf-8') + write_json(directory / 'fixture.json', {'case': case, 'arm': arm, 'expected_stdout': data['stdout'], + 'created_at': datetime.now(timezone.utc).isoformat(), + 'input_hashes': {name: hashlib.sha256((root / name).read_bytes()).hexdigest() + for name in data['files'] if name != 'report.py'}}) + return directory + + +def codex_command(): + path = shutil.which('codex') + if not path: + raise RuntimeError('Codex CLI is not installed') + if Path(path).suffix.lower() in {'.cmd', '.ps1'}: + entry = Path(path).parent / 'node_modules' / '@openai' / 'codex' / 'bin' / 'codex.js' + if not entry.exists(): + raise RuntimeError('Cannot resolve installed Codex CLI entry point') + return [shutil.which('node'), str(entry)] + return [path] + + +def run(directory, model, effort): + if (directory / 'events.jsonl').exists(): + raise ValueError('Run evidence already exists; prepare a fresh workspace') + sources = [HERE / 'server.py', HERE / 'bench.py', HERE.parent / 'delegate.py'] + source_hashes = {p.name: hashlib.sha256(p.read_bytes()).hexdigest() for p in sources} + root = directory / 'workspace' + command = codex_command() + ['exec', '--json', '--approve-for-me', '-C', str(root), + '-m', model, '-c', 'model_reasoning_effort=' + json.dumps(effort), + '-c', 'projects.' + json.dumps(str(root)) + '.trust_level="trusted"', + '--output-last-message', str(directory / 'final.txt'), '-'] + start = time.perf_counter() + timed_out = False + with (directory / 'events.jsonl').open('wb') as output, (directory / 'stderr.txt').open('wb') as errors, (directory / 'prompt.txt').open('rb') as prompt: + process = subprocess.Popen(command, stdin=prompt, stdout=output, stderr=errors) + try: + process.wait(timeout=600) + except (subprocess.TimeoutExpired, KeyboardInterrupt): + timed_out = True + subprocess.run(['taskkill', '/PID', str(process.pid), '/T', '/F'], + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, timeout=15) + process.wait(timeout=15) + write_json(directory / 'run.json', {'source_sha256': source_hashes, 'source_changed_during_run': any(hashlib.sha256(p.read_bytes()).hexdigest() != source_hashes[p.name] for p in sources), 'model': model, 'effort': effort, 'exit_code': process.returncode, 'timed_out_or_cancelled': timed_out, + 'wall_ms': round((time.perf_counter()-start)*1000), 'command': command, + 'billing': 'Existing Codex login; tokens are usage, not a dollar charge estimate'}) + return collect(directory) + + +def collect(directory): + fixture = json.loads((directory / 'fixture.json').read_text(encoding='utf-8')) + run_info = json.loads((directory / 'run.json').read_text(encoding='utf-8')) + events = [] + for line in (directory / 'events.jsonl').read_text(encoding='utf-8').splitlines(): + try: + events.append(json.loads(line)) + except ValueError: + pass + turns = [event for event in events if event.get('type') == 'turn.completed'] + items = [event['item'] for event in events if event.get('type') == 'item.completed'] + usage = {key: sum(turn.get('usage', {}).get(key, 0) for turn in turns) + for key in ['input_tokens', 'cached_input_tokens', 'output_tokens', 'reasoning_output_tokens']} + root = directory / 'workspace' + checks = {'input_unchanged': all(hashlib.sha256((root / name).read_bytes()).hexdigest() == expected + for name, expected in fixture['input_hashes'].items())} + if (root / 'report.py').exists(): + try: + actual = subprocess.run([sys.executable, str(root / 'report.py')], cwd=root, + capture_output=True, timeout=20) + except subprocess.TimeoutExpired: + actual = subprocess.CompletedProcess([], 124, b'', b'External verification timed out') + (directory / 'verification-stdout.txt').write_bytes(actual.stdout) + (directory / 'verification-stderr.txt').write_bytes(actual.stderr) + checks.update(exit_code=actual.returncode, + stdout_matches=actual.stdout.decode('utf-8', errors='replace') == fixture['expected_stdout']) + else: + checks.update(exit_code=None, stdout_matches=False) + alternate = CASES[fixture['case']]['alternate'] + source = root / alternate['file'] + original = source.read_bytes() + checks['alternate_input_matches'] = False + if (root / 'report.py').exists(): + try: + source.write_text(alternate['content'], encoding='utf-8', newline='') + try: + probe = subprocess.run([sys.executable, str(root / 'report.py')], cwd=root, + capture_output=True, timeout=20) + except subprocess.TimeoutExpired: + probe = subprocess.CompletedProcess([], 124, b'', b'Alternate verification timed out') + checks['alternate_input_matches'] = probe.returncode == 0 and probe.stdout.decode('utf-8', errors='replace') == alternate['stdout'] + (directory / 'alternate-stdout.txt').write_bytes(probe.stdout) + finally: + source.write_bytes(original) + local = [] + for path in (directory / 'local').glob('*/result.json'): + record = json.loads(path.read_text(encoding='utf-8')) + local.append({'status': record['status'], 'verified': record['verified'], 'wall_ms': record['wall_ms'], + 'input_tokens': sum(c['response'].get('prompt_eval_count', 0) for c in record['model_calls']), + 'output_tokens': sum(c['response'].get('eval_count', 0) for c in record['model_calls']), + 'failed_executions': sum(a.get('result', {}).get('exit_code', 0) != 0 for a in record['actions'])}) + result = {'case': fixture['case'], 'arm': fixture['arm'], 'model': run_info['model'], + 'effort': run_info['effort'], 'wall_ms': run_info['wall_ms'], 'frontier_usage': usage, + 'turn_completed': bool(turns), 'codex_exit': run_info['exit_code'], 'local': local, 'checks': checks, + 'accurate': checks['input_unchanged'] and checks['exit_code'] == 0 and checks['stdout_matches'] and checks['alternate_input_matches'], + 'command_calls': sum(i.get('type') == 'command_execution' for i in items), + 'mcp_calls': sum(i.get('type') == 'mcp_tool_call' for i in items), + 'raw_events': str(directory / 'events.jsonl')} + result['success'] = bool(result['accurate'] and turns and run_info['exit_code'] == 0 and not run_info.get('source_changed_during_run', False) and + (fixture['arm'] == 'baseline' or (result['mcp_calls'] == 1 and len(local) == 1 and local[0]['verified']))) + write_json(directory / 'summary.json', result) + return result + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('operation', choices=['prepare', 'run', 'collect']) + parser.add_argument('--case', choices=list(CASES), default='csv') + parser.add_argument('--arm', choices=['baseline', 'delegated'], default='delegated') + parser.add_argument('--out', type=Path, default=Path('work/codex-bench')) + parser.add_argument('--directory', type=Path) + parser.add_argument('--model', default='gpt-6-astra') + parser.add_argument('--effort', default='low') + args = parser.parse_args() + directory = args.directory.resolve() if args.directory else prepare(args.out, args.case, args.arm) + if args.operation == 'prepare': + print(str(directory)) + elif args.operation == 'run': + print(json.dumps(run(directory, args.model, args.effort), indent=2)) + else: + print(json.dumps(collect(directory), indent=2)) + + +if __name__ == '__main__': + main() diff --git a/experiments/command_specialist/codex/compare.py b/experiments/command_specialist/codex/compare.py new file mode 100644 index 0000000..67042c4 --- /dev/null +++ b/experiments/command_specialist/codex/compare.py @@ -0,0 +1,34 @@ +"""Compare paired fresh-chat evidence without claiming savings for failed work.""" +import argparse +import json +from pathlib import Path + + +def compare(baseline, delegated): + if baseline['case'] != delegated['case'] or baseline['model'] != delegated['model'] or baseline['effort'] != delegated['effort']: + raise ValueError('Compare the same task and frontier settings') + if baseline['arm'] != 'baseline' or delegated['arm'] != 'delegated': + raise ValueError('Expected baseline then delegated results') + result = {'case': baseline['case'], 'sample_pairs': 1, + 'baseline': baseline, 'delegated': delegated, 'savings': None, + 'qualification': 'A single pair is a hookup smoke, not a speedup benchmark or billing estimate.'} + if baseline.get('success') and delegated.get('success'): + def reduction(a, b): + return round(100 * (a-b) / a, 2) if a else None + b, d = baseline['frontier_usage'], delegated['frontier_usage'] + result['savings'] = {'wall_time_percent': reduction(baseline['wall_ms'], delegated['wall_ms']), + 'frontier_input_percent': reduction(b['input_tokens'], d['input_tokens']), + 'frontier_uncached_input_percent': reduction(b['input_tokens']-b['cached_input_tokens'], d['input_tokens']-d['cached_input_tokens']), + 'frontier_output_percent': reduction(b['output_tokens'], d['output_tokens'])} + return result + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('baseline', type=Path) + parser.add_argument('delegated', type=Path) + parser.add_argument('--out', type=Path, required=True) + args = parser.parse_args() + result = compare(json.loads(args.baseline.read_text(encoding='utf-8')), json.loads(args.delegated.read_text(encoding='utf-8'))) + args.out.write_text(json.dumps(result, indent=2), encoding='utf-8') + print(json.dumps({'case': result['case'], 'savings': result['savings'], 'qualification': result['qualification']}, indent=2)) diff --git a/experiments/command_specialist/codex/requirements.txt b/experiments/command_specialist/codex/requirements.txt new file mode 100644 index 0000000..c2bd62f --- /dev/null +++ b/experiments/command_specialist/codex/requirements.txt @@ -0,0 +1 @@ +mcp==1.30.0 diff --git a/experiments/command_specialist/codex/server.py b/experiments/command_specialist/codex/server.py new file mode 100644 index 0000000..d87bf53 --- /dev/null +++ b/experiments/command_specialist/codex/server.py @@ -0,0 +1,80 @@ +"""Scoped stdio MCP bridge for trusted Python English handoffs.""" +import argparse +import asyncio +import json +from pathlib import Path +import sys + +from mcp.server.fastmcp import FastMCP +from mcp.types import ToolAnnotations + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from delegate import delegate + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('--root', type=Path, required=True) + parser.add_argument('--artifacts', type=Path, required=True) + parser.add_argument('--allow-execute', action='store_true') + args = parser.parse_args() + root = args.root.resolve(strict=True) + server = FastMCP('command-specialist', log_level='WARNING') + lock = asyncio.Lock() + + @server.tool(annotations=ToolAnnotations(readOnlyHint=False, destructiveHint=True, + idempotentHint=False, openWorldHint=True)) + async def run_python_task(task: str, target: str, expected_stdout: str, + context: str = '', constraints: list[str] | None = None) -> dict: + """Delegate English Python write/run/repair work to one fresh local FP16 worker. + + Supply intent, an exact relative .py target, relevant file/API context and + required stdout (including newlines). Do not supply source code or shell + commands. Executes generated Python with the caller's account privileges + in this server's fixed trusted workspace; this is not a sandbox. Returns + observed exits/output, verified status and reopenable raw evidence. Failed + work is not success; any frontier repair must be reported as fallback. + """ + async with lock: + handoff = {'task': task, 'targets': {'program': target}, 'context': context, + 'constraints': constraints or [], + 'completion': {'target': 'program', 'stdout': expected_stdout}} + # A separate process owns each lifecycle, including its model history. + import uuid + args.artifacts.mkdir(parents=True, exist_ok=True) + request = args.artifacts / (uuid.uuid4().hex + '-handoff.json') + request.write_text(json.dumps(handoff), encoding='utf-8') + command = [sys._base_executable, '-S', str(Path(__file__).resolve().parents[1] / 'delegate.py'), + '--task-file', str(request.resolve()), '--root', str(root), + '--artifacts', str(args.artifacts.resolve())] + if args.allow_execute: + command.append('--allow-execute') + process = await asyncio.create_subprocess_exec(*command, stdin=asyncio.subprocess.DEVNULL, stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE) + try: + stdout, stderr = await asyncio.wait_for(process.communicate(), timeout=310) + except (asyncio.CancelledError, TimeoutError): + # Terminate the worker tree, never start an alternative executor. + if process.returncode is None: + killer = await asyncio.create_subprocess_exec('taskkill', '/PID', str(process.pid), '/T', '/F', + stdout=asyncio.subprocess.DEVNULL, stderr=asyncio.subprocess.DEVNULL) + await killer.wait() + await process.wait() + raise + if not stdout: + return {'status': 'failed', 'verified': False, 'worker_exit': process.returncode, + 'error': stderr.decode('utf-8', errors='replace')[-2000:]} + packet = json.loads(stdout.decode('utf-8')) + raw = json.loads(Path(packet['raw_result']).read_text(encoding='utf-8')) + packet['local_usage'] = { + 'input_tokens': sum(c['response'].get('prompt_eval_count', 0) for c in raw['model_calls']), + 'output_tokens': sum(c['response'].get('eval_count', 0) for c in raw['model_calls']), + 'model_calls': len(raw['model_calls']), 'actions': len(raw['actions']), + 'failed_executions': sum(a.get('result', {}).get('exit_code', 0) != 0 for a in raw['actions'])} + return packet + + server.run(transport='stdio') + + +if __name__ == '__main__': + main() diff --git a/experiments/command_specialist/codex/test_compare.py b/experiments/command_specialist/codex/test_compare.py new file mode 100644 index 0000000..581d71b --- /dev/null +++ b/experiments/command_specialist/codex/test_compare.py @@ -0,0 +1,19 @@ +"""Comparison invariants: failed or mismatched runs cannot imply savings.""" +import unittest +from compare import compare + + +class ComparisonInvariants(unittest.TestCase): + def test_failed_work_has_no_savings_claim(self): + common = {'case': 'csv', 'model': 'same', 'effort': 'low', 'success': False} + result = compare(dict(common, arm='baseline'), dict(common, arm='delegated')) + self.assertIsNone(result['savings']) + + def test_different_frontier_settings_are_not_a_pair(self): + with self.assertRaises(ValueError): + compare({'case': 'csv', 'model': 'a', 'effort': 'low'}, + {'case': 'csv', 'model': 'b', 'effort': 'low'}) + + +if __name__ == '__main__': + unittest.main() diff --git a/experiments/command_specialist/delegate.py b/experiments/command_specialist/delegate.py index a128927..192e103 100644 --- a/experiments/command_specialist/delegate.py +++ b/experiments/command_specialist/delegate.py @@ -221,6 +221,7 @@ def infer(context, schema=None): generation_context = ( task['task'] + '\nRuntime/API context: ' + str(task.get('context', '')) + '\nConstraints: ' + str(task.get('constraints', [])) + + '\nRequired stdout exactly (JSON string): ' + json.dumps(task['completion']['stdout']) + '\nTarget binding: ' + name + ' = ' + str(path) + '\nCurrent saved source:\n' + (path.read_text(encoding='utf-8') if path.exists() else '(missing)') + '\nActual last execution result: ' + json.dumps(latest.get(name, {}), ensure_ascii=False) +