From db9c04c89eca3b592281a7a77e672b17334ead27 Mon Sep 17 00:00:00 2001 From: Noisemaker111 <139656120+Noisemaker111@users.noreply.github.com> Date: Thu, 10 Sep 2026 12:23:15 -0400 Subject: [PATCH] Add fresh English delegation loop with non-quantized local inference --- .github/workflows/agent-checks.yml | 2 + .../command_specialist/ENGLISH_HANDOFF.md | 114 +++++++ .../command_specialist/ENGLISH_RESULTS.md | 57 ++++ experiments/command_specialist/README.md | 6 + experiments/command_specialist/delegate.py | 303 ++++++++++++++++++ .../command_specialist/test_delegate.py | 46 +++ 6 files changed, 528 insertions(+) create mode 100644 experiments/command_specialist/ENGLISH_HANDOFF.md create mode 100644 experiments/command_specialist/ENGLISH_RESULTS.md create mode 100644 experiments/command_specialist/delegate.py create mode 100644 experiments/command_specialist/test_delegate.py diff --git a/.github/workflows/agent-checks.yml b/.github/workflows/agent-checks.yml index 58fdce6..fbc439a 100644 --- a/.github/workflows/agent-checks.yml +++ b/.github/workflows/agent-checks.yml @@ -28,5 +28,7 @@ jobs: run: python -m unittest discover -s experiments/command_specialist -p test_bindings.py -v - name: Native and PowerShell contract invariants run: python -m unittest discover -s experiments/command_specialist -p test_contract.py -v + - name: English delegation evidence and authorization invariants + run: python -m unittest discover -s experiments/command_specialist -p test_delegate.py -v - name: Frozen main candidate and patch notes run: python scripts/check_release.py diff --git a/experiments/command_specialist/ENGLISH_HANDOFF.md b/experiments/command_specialist/ENGLISH_HANDOFF.md new file mode 100644 index 0000000..c59dc18 --- /dev/null +++ b/experiments/command_specialist/ENGLISH_HANDOFF.md @@ -0,0 +1,114 @@ +# English handoff contract (prototype v1) + +A frontier supplies one English task, exact target bindings, relevant runtime/API +context, constraints, and an observable completion condition. The frontier does +not supply generated source, escaped shell strings, or a command sequence. +Missing target/API context is a context failure, not evidence of model incapacity. + +Each invocation creates a fresh worker message history and unique evidence directory. +It retains internal action/result history only for this delegation. It returns one +packet, never recursively delegates, and never resumes a previous local chat. +Ollama may keep model weights warm; that is not conversation persistence. + +The worker chooses compact typed actions. Premature finish is rejected with observed failure feedback. A write action requests raw file content +in a separate inference rather than putting a program inside a JSON string. Source generation receives an English view of the same task, saved source and actual last execution result; JSON action history stays in the selector. Native +write/read/process operations preserve exact target bindings. Actual execution +feedback goes back to the same worker for ordinary repair. This split is an +experimental baseline, not a measured superiority claim. + +The result distinguishes worker explanation from observed actions, saved files, +SHA-256 identities, exit codes, verbatim stdout/stderr and raw model responses. +Only a runtime-checked completion condition can set `verified`. A generated file, +`finish`, or exit zero alone cannot. A run must execute the same bytes that were +written and remain saved, exit zero without a limit violation, and match the +required stdout exactly, including platform newlines (Windows print uses `\r\n`). The worker must also finish within its action budget. +This verifies the stated stdout condition, not arbitrary semantic correctness. + +## Minimal local CLI + +From the repository root, invoke Python 3.12: + +``` +python experiments/command_specialist/delegate.py --task-file task.json --root scratch --allow-execute +``` + +Example handoff (scratch must already exist): + +```json +{ + "task": "Write a Python program that calculates the sum of integers 1 through 10. Run it, inspect the actual output, then finish.", + "targets": {"program": "sum.py"}, + "context": "Python 3.12; standard library only.", + "constraints": ["Modify only the bound program target"], + "completion": {"target": "program", "stdout": "55\r\n"} +} +``` + +This CLI supports trusted Python write/read/run tasks only. `--allow-execute` +explicitly authorizes native generated code with the caller's account privileges; +path binding constrains direct adapter operations, not Python's capabilities. +Natural-language constraints are model instructions, not a security sandbox. +Use disposable data. The CLI is not registered as a new host tool. A future host +adapter must use its native capabilities, session identity, hooks, instructions, +permissions and cancellation. CLI authorization does not substitute for those. +A missing authorization terminates as denied before model or native actions; no +alternate executor or model is attempted. + +Default bounds: 12 actions, 300 seconds for model/operation work, 20 seconds per +execution, 8192 output tokens per inference, 32768 context tokens, and 64 KiB +execution output. Execution output is spooled to per-action files and monitored; +limit detection can overshoot between polls, and raw files preserve that evidence. +Model HTTP timeout bounds the client's wait; it does not prove server inference +was cancelled. Process cleanup and final persistence can exceed the work deadline. +Ctrl-C terminates the running process tree and saves a cancelled result. Generated +programs that detach children are outside this trusted prototype's lifecycle +coverage. No concurrent benchmark or shared model setting change is implied. + +Raw artifacts are saved before execution and after each observation, with atomic +replacement of `result.json`. Abrupt host termination may leave status `running`; +never interpret that as success. The packet provides the artifact path. Reopen it +and the actual target to check persistence. Failure, exhausted budgets, unavailable +worker and denial preserve distinct statuses and do not silently fall back. + +## Representative PTY handoff (next stage) + +The full contract must accommodate this without the frontier writing code: + +> Write and run the bound checker using node-pty. Launch Bun on the exact recorder +> path with inherited environment and working directory, a 100x24 xterm-256color +> PTY, and a normalized absolute script path. Observe the diagnostic banner, send +> Escape once after startup, and report actual cancellation and child exit status. +> Scope: Startup/cancel only; no physical-key evidence generated. + +The frontier supplies checker and recorder bindings, the actual banner/cancel +markers and concise node-pty API context. Completion requires observations from +the real recorder and reopened saved report, not a replacement printing markers. +This Python-only prototype cannot yet perform that workload, host-native +permissions, or configured frontier return integration. The earlier file adapter +remains separate pending a verified replacement; its result is not proof of +English delegation. + +## Non-quantized runtime and capacity + +The English loop requires `shell-specialist-f16`: the original local +Qwen2.5-Coder-1.5B-Instruct weights exported as FP16 with the unchanged pilot LoRA +adapter. It checks Ollama model metadata before inference and rejects quantized +or unknown precision. FP16 is non-quantized 16-bit floating point, not FP32. +The old Q4 pilot remains installed for historical reproducibility but is not used +by this loop. No retraining or change to the trained adapter is involved. + +`--num-ctx 32768 --num-predict 8192 --seconds 300 --max-actions 12` are explicit +CLI controls and the defaults. The context window includes history and generated +output. A conservative UTF-8 byte upper bound plus per-message overhead reserves +the output allowance; oversized history fails instead of intentionally truncating +instructions. This can reject text before the model's token capacity is actually +full. Exact tokenizer budgeting is future work. Increasing output is permission +to generate more, not a minimum response length. Long-context reliability has not +been established by short task smokes. + +For a local runtime setup, use the existing llama.cpp `convert_hf_to_gguf.py` on +the original local base snapshot with `--outtype f16`. Import that GGUF with the +existing pilot's exact `ADAPTER` and chat template under `shell-specialist-f16` +using `ollama create`; do not pass a quantization option. Verify `ollama show` +reports F16 and compare the adapter identity. The tested machine's BF16 GGUF import +failed validation, while FP16 succeeded. Weights and setup logs stay private. diff --git a/experiments/command_specialist/ENGLISH_RESULTS.md b/experiments/command_specialist/ENGLISH_RESULTS.md new file mode 100644 index 0000000..0df1d56 --- /dev/null +++ b/experiments/command_specialist/ENGLISH_RESULTS.md @@ -0,0 +1,57 @@ +# English loop observations — September 10, 2026 + +The standalone public CLI now accepts one English handoff and returns a saved, +runtime-verified result from a fresh local worker. This is step 1/2 of the handoff +plan, not configured OpenCode2 delegation or the representative PTY recorder test. + +The active runtime is non-quantized FP16 Qwen2.5-Coder 1.5B with the original pilot +adapter. Original BF16 training weights were available; BF16 GGUF import failed +Ollama validation, while FP16 import succeeded. `ollama show` reported F16 and +`ollama ps` reported 32768 context, 4.2 GB and 100% GPU on the RTX 3070. These are +runtime observations, not peak memory measurements. The original Q4 model is not +used by the new loop. No retraining occurred. + +## Actual public operations + +All model calls used local Ollama, 32768 context and an 8192 output-token cap, +12 actions and a 300-second worker budget. Private task/result files and raw model +responses are retained under the owned worktree's ignored `work/` directory. + +- Fresh sum program: actual missing-file exit 2, local source generation/write, + rerun exit 0 with `55`, then finish. Saved source computes the sum. 2.235 seconds. +- Fresh sum-of-squares program with Unicode and an apostrophe in its exact path: + missing-file exit 2, write, rerun exit 0 with `140`, finish. 2.343 seconds. +- Existing broken program: actual `NameError` for `valuez`, local correction to + `values`, rerun exit 0 with `16`, finish. 1.594 seconds. +- Final-revision fresh squares target: exit 2, write, exit 0 with `140`, finish. + 2.843 seconds. This includes the corrected source newline handling. +- Without execution authorization: denied before any model/action call. +- One-action budget: exhausted and unverified even though the existing program + exited zero with the expected stdout. + +Times are individual whole CLI-worker measurements including inference, native +operations and persistence, not p50/p95, a speedup claim, or paid charges. They +exclude frontier conversation overhead. Every successful result and saved source +was reopened; recorded precision, action exits and source identity were checked. + +## Failures that informed the implementation + +The initial Q4 probe repeatedly ran a missing file. A first FP16 prompt generated +correct source but repeatedly copied a write example. Removing that example let +actions advance, but JSON action history contaminated source generation: one +output was a task-shaped dictionary that executed with empty stdout. The runtime +correctly refused success. English-only source-generation context containing the +same task, saved source and actual execution feedback yielded the successful runs. +Multiple variables changed; these observations do not isolate quantization effects. + +Action selection still wastes a run on absent files, and final worker explanations +are weak (often just the filename). Deterministic evidence verification is needed. +The model has not demonstrated broad English task reliability or long-context +reasoning. An 8K output cap is configured; these short samples did not use 8K tokens. + +Required compileall and both existing five-test suites passed. Three new concise +invariant tests passed for denial/fresh histories, exact target boundaries, and +verification against actual executed/saved bytes. They supplement the real runs. +Host hooks/permissions, PTY startup/cancel, detached child cleanup, and external +frontier delivery remain unverified. No host registration or global installation +was changed by this prototype. diff --git a/experiments/command_specialist/README.md b/experiments/command_specialist/README.md index 8fc9cc1..a8683d7 100644 --- a/experiments/command_specialist/README.md +++ b/experiments/command_specialist/README.md @@ -258,3 +258,9 @@ frontier endpoint. Report failures and fallback rate, not only answered cases. Integrate only once measured gains survive the whole host operation. Expensive or destructive actions retain the host's permission checks; model output never grants authorization. Keep full raw results retrievable when a compact packet is insufficient. + +## English delegation prototype + +See [the English handoff contract](ENGLISH_HANDOFF.md) for the fresh-worker local +CLI, non-quantized runtime, configurable budgets and verification boundaries. +This is separate from the earlier read-only inspection adapter. diff --git a/experiments/command_specialist/delegate.py b/experiments/command_specialist/delegate.py new file mode 100644 index 0000000..a128927 --- /dev/null +++ b/experiments/command_specialist/delegate.py @@ -0,0 +1,303 @@ +"""Single-use English delegation CLI for explicitly trusted local Python tasks. + +This prototype is not a sandbox or an OpenCode2 permission adapter. +""" +import argparse +import hashlib +import json +import os +from pathlib import Path +import subprocess +import sys +import time +import urllib.request +import uuid + +MODEL = 'shell-specialist-f16' +ACTION_SCHEMA = { + 'type': 'object', 'properties': { + 'action': {'type': 'string', 'enum': ['write', 'run', 'read', 'finish']}, + 'target': {'type': 'string'}, 'instruction': {'type': 'string'}}, + 'required': ['action', 'target', 'instruction'], 'additionalProperties': False} + + +def save(path, value): + temporary = path.with_suffix('.tmp') + temporary.write_text(json.dumps(value, ensure_ascii=False, indent=2), encoding='utf-8') + temporary.replace(path) + + +def digest(path): + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def validate(task, root): + if not isinstance(task, dict): + raise ValueError('handoff must be an object') + if not isinstance(task.get('task'), str) or not task['task'].strip(): + raise ValueError('task must contain English intent') + targets = task.get('targets') + if not isinstance(targets, dict) or not targets: + raise ValueError('targets must bind names to exact relative .py paths') + resolved = {} + for name, value in targets.items(): + if not isinstance(name, str) or not name or not isinstance(value, str): + raise ValueError('target names and paths must be strings') + path = (root / value).resolve() + if not path.is_relative_to(root) or Path(value).is_absolute() or path.suffix != '.py': + raise ValueError('target must be a relative .py file inside root') + resolved[name] = path + check = task.get('completion', {}) + if not isinstance(check, dict): + raise ValueError('completion must be an object') + if check.get('target') not in resolved or not isinstance(check.get('stdout'), str): + raise ValueError('completion requires a bound target and exact stdout') + return resolved + + +def completion_matches(status, path, written_hash, actual, expected): + return bool(status == 'unverified' and written_hash and path.exists() + and written_hash == digest(path) + and actual.get('sha256_before') == written_hash + and actual.get('exit_code') == 0 and not actual.get('limit') + and actual.get('stdout') == expected) + + +def kill_tree(process): + if os.name == 'nt': + subprocess.run(['taskkill', '/PID', str(process.pid), '/T', '/F'], + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, timeout=10) + else: + import signal + os.killpg(process.pid, signal.SIGKILL) + process.wait(timeout=10) + + +def execute(path, root, directory, remaining, output_limit): + stdout, stderr = directory / 'stdout.txt', directory / 'stderr.txt' + started = time.monotonic() + reason = None + with stdout.open('wb') as out, stderr.open('wb') as err: + process = subprocess.Popen([sys.executable, str(path)], cwd=root, stdout=out, + stderr=err, start_new_session=os.name != 'nt') + try: + while process.poll() is None: + if time.monotonic() - started >= remaining: + reason = 'execution_timeout' + if stdout.stat().st_size + stderr.stat().st_size > output_limit: + reason = 'output_limit' + if reason: + kill_tree(process) + break + time.sleep(.02) + except BaseException: + kill_tree(process) + raise + if stdout.stat().st_size + stderr.stat().st_size > output_limit: + reason = 'output_limit' + return {'exit_code': process.returncode, 'limit': reason, + 'stdout': stdout.read_bytes()[:output_limit].decode('utf-8', errors='replace'), + 'stderr': stderr.read_bytes()[:output_limit].decode('utf-8', errors='replace'), + 'raw_stdout': str(stdout), 'raw_stderr': str(stderr)} + + +def delegate(task, root, artifacts, *, max_actions=12, seconds=300, output_limit=65536, + num_ctx=32768, num_predict=8192, + base_url='http://127.0.0.1:11434', allow_execute=False): + if not 1 <= max_actions <= 20 or not 1 <= seconds <= 600 or not 1 <= output_limit <= 1048576: + raise ValueError('invalid lifecycle limits') + if not 1 <= num_predict < num_ctx <= 32768: + raise ValueError('require 1 <= output tokens < context <= 32768') + root = root.resolve(strict=True) + if not root.is_dir(): + raise ValueError('root must be a directory') + targets = validate(task, root) + run_dir = artifacts.resolve() / uuid.uuid4().hex + run_dir.mkdir(parents=True) + artifact = run_dir / 'result.json' + started = time.monotonic() + deadline = started + seconds + record = {'model': MODEL, 'task': task, 'root': str(root), 'actions': [], 'model_calls': [], + 'status': 'running', 'verified': False, 'limits': { + 'actions': max_actions, 'seconds': seconds, 'output_bytes': output_limit, + 'num_ctx': num_ctx, 'num_predict': num_predict}} + messages = [{'role': 'system', 'content': + 'You are a fresh local Python worker. Complete one English task. Choose one action: ' + 'write (instruction describes file content), run (execute a target with Python), ' + 'read (inspect a target), finish (explain outcome). Use only supplied target names. ' + 'After write, run; after a failure, repair using actual stderr. Do not repeat successful ' + 'work. Return action, target, instruction JSON. Never claim observations you lack. ' + 'A missing file requires write, not another run.'}, + {'role': 'user', 'content': json.dumps(task, ensure_ascii=False)}] + written = {} + latest = {} + + def persist(): + record['wall_ms'] = round((time.monotonic() - started) * 1000) + save(artifact, record) + + def infer(context, schema=None): + remaining = deadline - time.monotonic() + if remaining <= 0: + raise TimeoutError('worker time exhausted') + # Conservative UTF-8 byte upper bound reserves the output budget rather than + # silently allowing the inference server to discard earlier instructions. + prompt_upper_bound = sum(len(m['content'].encode('utf-8')) + 64 for m in context) + if prompt_upper_bound + num_predict > num_ctx: + raise ValueError('context budget exhausted; history was not silently truncated') + body = {'model': MODEL, 'messages': context, 'stream': False, 'think': False, + 'keep_alive': '5m', 'options': {'temperature': 0, 'seed': 20260909, + 'num_ctx': num_ctx, 'num_predict': num_predict}} + if schema: + body['format'] = schema + request = urllib.request.Request(base_url + '/api/chat', json.dumps(body).encode(), + {'Content-Type': 'application/json'}) + with urllib.request.urlopen(request, timeout=remaining) as response: + result = json.load(response) + record['model_calls'].append({'request': body, 'response': result}) + persist() + if result.get('done_reason') == 'length': + raise ValueError('model output limit reached') + if time.monotonic() >= deadline: + raise TimeoutError('worker time exhausted after inference') + return result['message']['content'] + + persist() + try: + if not allow_execute: + record['status'] = 'denied' + record['error'] = 'trusted local execution was not authorized; no alternative executor' + else: + request = urllib.request.Request(base_url + '/api/show', + json.dumps({'model': MODEL}).encode(), {'Content-Type': 'application/json'}) + with urllib.request.urlopen(request, timeout=min(10, seconds)) as response: + identity = json.load(response) + record['model_identity'] = identity + if identity.get('details', {}).get('quantization_level') not in {'BF16', 'F16', 'F32'}: + raise ValueError('non-quantized model required; refusing quantized or unknown weights') + for index in range(max_actions): + state = {name: {'exists': path.exists(), 'written': name in written, + 'last_exit': latest.get(name, {}).get('exit_code'), + 'stdout_matches': latest.get(name, {}).get('stdout') == task['completion']['stdout']} + for name, path in targets.items()} + raw = infer(messages + [{'role': 'user', 'content': + 'Current observed state: ' + json.dumps(state) + + '. Decide the NEXT action using the latest result. ' + 'A saved file that has not run needs action run. ' + 'A failed run needs action write to repair. ' + 'A successful matching run needs action finish. ' + 'Do not copy earlier actions.'}], ACTION_SCHEMA) + messages.append({'role': 'assistant', 'content': raw}) + action = json.loads(raw) + if (not isinstance(action, dict) or set(action) != {'action', 'target', 'instruction'} + or not all(isinstance(value, str) for value in action.values())): + raise ValueError('malformed worker action') + entry = {'selection': action} + record['actions'].append(entry) + persist() + if action.get('action') == 'finish': + check = task['completion'] + name = check['target'] + if completion_matches('unverified', targets[name], written.get(name), + latest.get(name, {}), check['stdout']): + record['worker_explanation'] = action.get('instruction', '') + record['status'] = 'unverified' + break + entry['result'] = {'error': 'Completion rejected: need a written file, a run of those bytes, ' + 'exit zero, and exact required stdout. Inspect the last result and repair.'} + messages.append({'role': 'user', 'content': json.dumps(entry['result'])}) + persist() + continue + name = action.get('target') + if name not in targets: + entry['result'] = {'error': 'unknown target; use a supplied binding name'} + else: + path = targets[name] + # Recheck exact binding immediately before every native operation. + if path.resolve() != path or not path.resolve().is_relative_to(root): + raise PermissionError('target binding changed') + operation = action.get('action') + if operation == 'write': + generation_context = ( + task['task'] + '\nRuntime/API context: ' + str(task.get('context', '')) + + '\nConstraints: ' + str(task.get('constraints', [])) + + '\nTarget binding: ' + name + ' = ' + str(path) + + '\nCurrent saved source:\n' + (path.read_text(encoding='utf-8') if path.exists() else '(missing)') + + '\nActual last execution result: ' + json.dumps(latest.get(name, {}), ensure_ascii=False) + + '\nReturn only the complete Python source to save. Do not return the task description or JSON.') + content = infer([{'role': 'system', 'content': + 'Write executable Python source only. No JSON wrapper, markdown fences or prose.'}, + {'role': 'user', 'content': generation_context}]) + if content.startswith('```'): + entry['result'] = {'error': 'raw file content required; markdown rejected'} + else: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(content, encoding='utf-8', newline='') + written[name] = digest(path) + latest.pop(name, None) + entry['result'] = {'saved_file': str(path), 'sha256': written[name], 'content': content} + elif operation == 'run': + directory = run_dir / str(index) + directory.mkdir() + before = digest(path) if path.exists() else None + entry['result'] = execute(path, root, directory, + max(.01, min(20, deadline-time.monotonic())), output_limit) + entry['result']['sha256_before'] = before + latest[name] = entry['result'] + elif operation == 'read': + entry['result'] = {'content': path.read_bytes()[:output_limit].decode('utf-8', errors='replace')} + else: + entry['result'] = {'error': 'unknown action'} + persist() + messages.append({'role': 'user', 'content': 'Observed native result: ' + + json.dumps(entry['result'], ensure_ascii=False)}) + else: + record['status'] = 'exhausted' + check = task['completion'] + name = check['target'] + actual = latest.get(name, {}) + record['verified'] = completion_matches(record['status'], targets[name], + written.get(name), actual, check['stdout']) + if record['verified']: + record['status'] = 'verified' + except KeyboardInterrupt: + record['status'] = 'cancelled' + except PermissionError as error: + record.update(status='denied', error=str(error)) + except (OSError, ValueError, KeyError, TypeError) as error: + record.update(status='failed', error=str(error)) + finally: + persist() + return {'status': record['status'], 'verified': record['verified'], + 'worker_explanation': (record.get('worker_explanation') or '')[:2000], + 'packet_limit_chars': 2000, + 'truncated': any(len(latest.get(task['completion']['target'], {}).get(k, '')) > 2000 + for k in ('stdout', 'stderr')), + 'observed': {key: value[:2000] if key in ('stdout', 'stderr') else value + for key, value in latest.get(task['completion']['target'], {}).items()}, + 'raw_result': str(artifact), 'wall_ms': record['wall_ms']} + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--task-file', type=Path, required=True) + parser.add_argument('--root', type=Path, required=True) + parser.add_argument('--artifacts', type=Path, default=Path('work/english-delegation')) + parser.add_argument('--allow-execute', action='store_true', + help='Authorize generated Python with your account privileges in a trusted task') + parser.add_argument('--max-actions', type=int, default=12) + parser.add_argument('--seconds', type=int, default=300) + parser.add_argument('--num-ctx', type=int, default=32768) + parser.add_argument('--num-predict', type=int, default=8192) + args = parser.parse_args() + task = json.loads(args.task_file.read_text(encoding='utf-8-sig')) + packet = delegate(task, args.root, args.artifacts, allow_execute=args.allow_execute, + max_actions=args.max_actions, seconds=args.seconds, + num_ctx=args.num_ctx, num_predict=args.num_predict) + sys.stdout.reconfigure(encoding='utf-8') + print(json.dumps(packet, ensure_ascii=False, indent=2)) + return 0 if packet['verified'] else 1 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/experiments/command_specialist/test_delegate.py b/experiments/command_specialist/test_delegate.py new file mode 100644 index 0000000..6ac252d --- /dev/null +++ b/experiments/command_specialist/test_delegate.py @@ -0,0 +1,46 @@ +"""Core evidence and authorization invariants; real model verification is separate.""" +import json +from pathlib import Path +import tempfile +import unittest + +from delegate import completion_matches, delegate, digest, execute, validate + + +class DelegationInvariants(unittest.TestCase): + def test_denial_is_terminal_and_each_handoff_is_fresh(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + task = {'task': 'Write and run a sum program', 'targets': {'p': 'sum.py'}, + 'completion': {'target': 'p', 'stdout': '55\n'}} + first = delegate(task, root, root / 'artifacts', base_url='http://127.0.0.1:1') + second = delegate(task, root, root / 'artifacts', base_url='http://127.0.0.1:1') + self.assertNotEqual(first['raw_result'], second['raw_result']) + saved = json.loads(Path(first['raw_result']).read_text()) + self.assertEqual(saved['status'], 'denied') + self.assertEqual(saved['model_calls'], []) + self.assertEqual(saved['actions'], []) + self.assertFalse((root / 'sum.py').exists()) + + def test_verification_requires_executed_saved_bytes_and_actual_output(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + path = root / 'sum.py' + path.write_text('print(sum(range(1, 11)))', encoding='utf-8') + identity = digest(path) + actual = execute(path, root, root, 5, 65536) + actual['sha256_before'] = identity + self.assertTrue(completion_matches('unverified', path, identity, actual, actual['stdout'])) + self.assertFalse(completion_matches('exhausted', path, identity, actual, actual['stdout'])) + self.assertFalse(completion_matches('unverified', path, identity, actual, 'wrong')) + path.write_text('print(0)', encoding='utf-8') + self.assertFalse(completion_matches('unverified', path, identity, actual, actual['stdout'])) + + def test_exact_target_cannot_escape_root(self): + with tempfile.TemporaryDirectory() as directory: + with self.assertRaises(ValueError): + validate({'task': 'write', 'targets': {'p': '../escape.py'}}, Path(directory)) + + +if __name__ == '__main__': + unittest.main()