From 78999db3c02774cc50821d60428cb8cb2620b2ce Mon Sep 17 00:00:00 2001 From: xormania <127287135+xormania@users.noreply.github.com> Date: Sat, 1 Aug 2026 22:54:59 -0400 Subject: [PATCH 001/158] test(experiment): define directory format surface --- tests/dev/security-gate-cases.sh | 4 +- tests/experiment/authorization-cases.sh | 634 +----------------- tests/experiment/contract-cases.sh | 462 +------------ tests/experiment/directory-intake-cases.sh | 53 ++ .../directories/minimal/experiment.cue | 12 + tests/security/fast.manifest | 4 +- 6 files changed, 91 insertions(+), 1078 deletions(-) create mode 100755 tests/experiment/directory-intake-cases.sh create mode 100644 tests/experiment/fixtures/directories/minimal/experiment.cue diff --git a/tests/dev/security-gate-cases.sh b/tests/dev/security-gate-cases.sh index 7a382b2..2dfb9ce 100644 --- a/tests/dev/security-gate-cases.sh +++ b/tests/dev/security-gate-cases.sh @@ -228,8 +228,8 @@ docker-harness-contract tests/dev/docker-harness-cases.sh SUMMARY failures=0 guard-command tests/guard/pretooluse-cases.sh SUMMARY failures=0 guard-mount tests/guard/cases.sh SUMMARY failures=0 config-authority tests/agent/config-guard.sh SUMMARY failures=0 -experiment-contract tests/experiment/contract-cases.sh SUMMARY failures=0 -experiment-authorization tests/experiment/authorization-cases.sh SUMMARY failures=0 +experiment-contract tests/experiment/contract-cases.sh EXPERIMENT CONTRACT PASS +experiment-authorization tests/experiment/authorization-cases.sh EXPERIMENT AUTHORIZATION PASS config-matrix tests/agent/config-matrix.sh SUMMARY failures=0 allowlist-schema tests/agent/allowlist-cases.sh SUMMARY failures=0 image-volume-policy tests/agent/image-volume-policy-cases.sh SUMMARY failures=0 diff --git a/tests/experiment/authorization-cases.sh b/tests/experiment/authorization-cases.sh index 73439a7..e518e2d 100755 --- a/tests/experiment/authorization-cases.sh +++ b/tests/experiment/authorization-cases.sh @@ -2,622 +2,24 @@ set -euo pipefail repo_root="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." >/dev/null 2>&1 && pwd)" -experiment="$repo_root/scripts/experiment" -cedar_tool="$repo_root/scripts/dev/cedar-tool" -fixture_root="$repo_root/tests/experiment/fixtures" - +agent_lab="$repo_root/scripts/agent-lab" +fixture="$repo_root/tests/experiment/fixtures/directories/minimal" work="$(mktemp -d)" -cleanup() { - find "$work" -type f -delete 2>/dev/null || true - find "$work" -type l -delete 2>/dev/null || true - find "$work" -depth -type d -exec rmdir {} + 2>/dev/null || true -} -trap cleanup EXIT -mkdir -p "$work/bin" "$work/home" "$work/runtime-tmp" - -failures=0 -pass() { printf 'PASS %s\n' "$1"; } -fail() { printf 'FAIL %s\n' "$1"; failures=$((failures + 1)); } - -spy_log="$work/tool-spy.log" -: > "$spy_log" -for tool in docker docker-compose podman curl wget; do - spy="$work/bin/$tool" - { - printf '#!/usr/bin/env bash\n' - printf 'printf "%%s\\n" "$0 $*" >> %q\n' "$spy_log" - printf 'exit 97\n' - } > "$spy" - chmod +x "$spy" -done - -capture() { - local name="$1" - shift - CAPTURE_STDOUT="$work/$name.stdout" - CAPTURE_STDERR="$work/$name.stderr" - CAPTURE_RC=0 - env -i \ - PATH="$work/bin:/usr/bin:/bin" \ - HOME="$work/home" \ - TMPDIR="$work/runtime-tmp" \ - LC_ALL=C \ - AGENT_LAB_CUE_TOOL_DIR="${AGENT_LAB_CUE_TOOL_DIR:-$repo_root/.cache/dev/tools/cue}" \ - AGENT_LAB_CEDAR_TOOL_DIR="${AGENT_LAB_CEDAR_TOOL_DIR:-$repo_root/.cache/dev/tools/cedar}" \ - "$@" > "$CAPTURE_STDOUT" 2> "$CAPTURE_STDERR" || CAPTURE_RC=$? -} - -capture usage-authorize "$experiment" authorize -if [ "$CAPTURE_RC" -eq 2 ] && [ ! -s "$CAPTURE_STDOUT" ] && - grep -Fq 'Usage: scripts/experiment authorize install [--] MANIFEST' "$CAPTURE_STDERR"; then - pass "authorize requires one bounded install request" -else - fail "authorize usage is not exposed exactly (rc=$CAPTURE_RC)" -fi - -capture usage-action "$experiment" authorize start -- "$fixture_root/valid.json" -if [ "$CAPTURE_RC" -eq 2 ] && [ ! -s "$CAPTURE_STDOUT" ]; then - pass "runtime lifecycle actions are outside the requested-plan seam" -else - fail "authorize accepted a runtime lifecycle action (rc=$CAPTURE_RC)" -fi - -capture usage-override "$experiment" authorize install --principal attacker "$fixture_root/valid.json" -if [ "$CAPTURE_RC" -eq 2 ] && [ ! -s "$CAPTURE_STDOUT" ]; then - pass "the public command rejects caller-selected Cedar inputs" -else - fail "authorize accepted a caller-selected Cedar input (rc=$CAPTURE_RC)" -fi - -for required in \ - "$cedar_tool" \ - "$repo_root/scripts/dev/cedar-tool.py" \ - "$repo_root/tools/cedar.lock" \ - "$repo_root/authorization/experiment/v0alpha1/schema.cedarschema" \ - "$repo_root/authorization/experiment/v0alpha1/operator.cedar"; do - if [ ! -f "$required" ]; then - printf 'INFRA Experiment authorization input is missing: %s\n' "$required" >&2 - exit 125 - fi -done -if [ ! -x "$experiment" ] || [ ! -x "$cedar_tool" ]; then - printf 'INFRA Experiment authorization entrypoints are not executable\n' >&2 - exit 125 -fi - -cedar_preflight_rc=0 -cedar_preflight_out="$($cedar_tool --version 2>&1)" || cedar_preflight_rc=$? -if [ "$cedar_preflight_rc" -ne 0 ]; then - printf 'INFRA Experiment authorization requires provisioned pinned Cedar: %s\n' \ - "$cedar_preflight_out" >&2 - exit 125 -fi - -repo_before="$work/repo.before" -git -C "$repo_root" status --porcelain=v1 --untracked-files=all > "$repo_before" - -authorization_sha256() { - python3 - "$repo_root" <<'PY' -from hashlib import sha256 -from pathlib import Path -import sys - -root = Path(sys.argv[1]) -names = sorted(( - "authorization/experiment/v0alpha1/operator.cedar", - "authorization/experiment/v0alpha1/schema.cedarschema", - "tools/cedar.lock", -)) -digest = sha256(b"agent-lab.authorization-contract.v1\0") -for name in names: - encoded = name.encode("utf-8") - data = (root / name).read_bytes() - digest.update(len(encoded).to_bytes(4, "big")) - digest.update(encoded) - digest.update(len(data).to_bytes(8, "big")) - digest.update(data) -print(f"sha256:{digest.hexdigest()}") -PY -} - -capture checked "$experiment" check -- "$fixture_root/valid.json" -if [ "$CAPTURE_RC" -ne 0 ] || [ -s "$CAPTURE_STDERR" ]; then - printf 'INFRA baseline CUE contract failed before authorization tests\n' >&2 - exit 125 -fi -capture permitted "$experiment" authorize install -- "$fixture_root/valid.json" - -checked_plan_digest="$(jq -er '.digest' "$work/checked.stdout" 2>/dev/null || true)" -checked_contract_digest="$(jq -er '.plan.contract.digest' "$work/checked.stdout" 2>/dev/null || true)" -expected_authorization_digest="$(authorization_sha256)" -if [ "$CAPTURE_RC" -eq 0 ] && [ ! -s "$CAPTURE_STDERR" ] && - jq -e \ - --arg plan "$checked_plan_digest" \ - --arg contract "$checked_contract_digest" \ - --arg authorization "$expected_authorization_digest" ' - .apiVersion == "agent-lab.authorization/v0alpha1" and - .kind == "ExperimentAuthorizationDecision" and - .verdict == "permit" and - .action == "experiment.install" and - .principal == { - assurance: "none", - authenticated: false, - id: "legacy-local-operator", - source: "fixed-local-cli", - type: "AgentLab::Principal" - } and - .binding == { - authorizationDigest: $authorization, - contractDigest: $contract, - planDigest: $plan - } and - .resource == { - id: $plan, - requestedName: "first-experiment", - type: "AgentLab::RequestedExperimentPlan" - } - ' "$CAPTURE_STDOUT" >/dev/null 2>&1; then - pass "Cedar permits the fixed operator for the exact CUE plan" -else - fail "Cedar did not bind its permit to the exact CUE plan (rc=$CAPTURE_RC)" -fi - -if [ "$(wc -l < "$work/permitted.stdout" | tr -d ' ')" -eq 1 ] && - [ "$(jq -cS . "$work/permitted.stdout" 2>/dev/null || true)" = "$(cat "$work/permitted.stdout")" ] && - LC_ALL=C grep -Eq '^[ -~]+$' "$work/permitted.stdout"; then - pass "the authorization decision is one canonical review-safe ASCII line" -else - fail "the authorization decision is not canonical ASCII" -fi - -capture reordered "$experiment" authorize install -- "$fixture_root/valid-reordered.json" -if [ "$CAPTURE_RC" -eq 0 ] && [ ! -s "$CAPTURE_STDERR" ] && - cmp -s "$work/permitted.stdout" "$CAPTURE_STDOUT"; then - pass "equivalent manifests authorize the same requested plan" -else - fail "equivalent manifests changed authorization identity (rc=$CAPTURE_RC)" -fi - -different="$work/different.json" -jq '.spec.members[1].resourceClass = "standard"' "$fixture_root/valid.json" > "$different" -capture different "$experiment" authorize install -- "$different" -different_plan_digest="$(jq -er '.binding.planDigest' "$CAPTURE_STDOUT" 2>/dev/null || true)" -if [ "$CAPTURE_RC" -eq 0 ] && [ ! -s "$CAPTURE_STDERR" ] && - [ "$(jq -r '.resource.requestedName' "$CAPTURE_STDOUT" 2>/dev/null || true)" = first-experiment ] && - [ -n "$different_plan_digest" ] && [ "$different_plan_digest" != "$checked_plan_digest" ] && - [ "$(jq -r '.resource.id' "$CAPTURE_STDOUT" 2>/dev/null || true)" = "$different_plan_digest" ]; then - pass "same-name different intent receives a different digest resource" -else - fail "requested name became authorization identity (rc=$CAPTURE_RC)" -fi - -capture envelope "$experiment" authorize install -- "$work/checked.stdout" -if [ "$CAPTURE_RC" -eq 1 ] && [ ! -s "$CAPTURE_STDOUT" ] && - grep -Fq 'FAIL Experiment manifest' "$CAPTURE_STDERR"; then - pass "authorize revalidates manifest intent instead of accepting an envelope" -else - fail "authorize accepted or mishandled a caller-supplied plan envelope (rc=$CAPTURE_RC)" -fi - -copy_root="$work/authorization-copy" -mkdir -p \ - "$copy_root/scripts/dev" \ - "$copy_root/contracts/experiment/v0alpha1/cue.mod" \ - "$copy_root/authorization/experiment/v0alpha1" \ - "$copy_root/tests/experiment/fixtures" \ - "$copy_root/tools" -cp "$repo_root/scripts/experiment" "$repo_root/scripts/experiment.py" \ - "$copy_root/scripts/" -cp "$repo_root/scripts/dev/cue-tool" "$repo_root/scripts/dev/cue-tool.py" \ - "$repo_root/scripts/dev/cedar-tool.py" "$copy_root/scripts/dev/" -cp "$repo_root/contracts/experiment/v0alpha1/schema.cue" \ - "$repo_root/contracts/experiment/v0alpha1/plan.cue" \ - "$copy_root/contracts/experiment/v0alpha1/" -cp "$repo_root/contracts/experiment/v0alpha1/cue.mod/module.cue" \ - "$copy_root/contracts/experiment/v0alpha1/cue.mod/" -cp "$repo_root/authorization/experiment/v0alpha1/schema.cedarschema" \ - "$repo_root/authorization/experiment/v0alpha1/operator.cedar" \ - "$copy_root/authorization/experiment/v0alpha1/" -cp "$repo_root/tools/cue.lock" "$repo_root/tools/cedar.lock" "$copy_root/tools/" -cp "$fixture_root/valid.json" "$copy_root/tests/experiment/fixtures/" -chmod +x "$copy_root/scripts/experiment" "$copy_root/scripts/dev/cue-tool" - -printf '%s\n' \ - '' \ - '@id("emergency-stop-v0")' \ - 'forbid (principal, action, resource);' >> \ - "$copy_root/authorization/experiment/v0alpha1/operator.cedar" -capture public-deny env \ - AGENT_LAB_CUE_TOOL_DIR="${AGENT_LAB_CUE_TOOL_DIR:-$repo_root/.cache/dev/tools/cue}" \ - AGENT_LAB_CEDAR_TOOL_DIR="${AGENT_LAB_CEDAR_TOOL_DIR:-$repo_root/.cache/dev/tools/cedar}" \ - "$copy_root/scripts/experiment" authorize install -- \ - "$copy_root/tests/experiment/fixtures/valid.json" -if [ "$CAPTURE_RC" -eq 1 ] && [ ! -s "$CAPTURE_STDERR" ] && - [ "$(jq -r '.verdict' "$CAPTURE_STDOUT" 2>/dev/null || true)" = deny ] && - [ "$(jq -r '.binding.planDigest' "$CAPTURE_STDOUT" 2>/dev/null || true)" = "$checked_plan_digest" ] && - [ "$(jq -cS . "$CAPTURE_STDOUT" 2>/dev/null || true)" = "$(cat "$CAPTURE_STDOUT")" ]; then - pass "a strict repository forbid produces one canonical ordinary deny" -else - fail "ordinary Cedar deny was not translated canonically (rc=$CAPTURE_RC)" -fi - -printf '\nthis is not Cedar\n' >> \ - "$copy_root/authorization/experiment/v0alpha1/operator.cedar" -capture corrupt-policy env \ +trap 'find "$work" -type f -delete 2>/dev/null || true; find "$work" -depth -type d -exec rmdir {} + 2>/dev/null || true' EXIT +mkdir -p "$work/home" "$work/tmp" +rc=0 +env -i PATH=/usr/bin:/bin HOME="$work/home" TMPDIR="$work/tmp" LC_ALL=C \ AGENT_LAB_CUE_TOOL_DIR="${AGENT_LAB_CUE_TOOL_DIR:-$repo_root/.cache/dev/tools/cue}" \ AGENT_LAB_CEDAR_TOOL_DIR="${AGENT_LAB_CEDAR_TOOL_DIR:-$repo_root/.cache/dev/tools/cedar}" \ - "$copy_root/scripts/experiment" authorize install -- \ - "$copy_root/tests/experiment/fixtures/valid.json" -if [ "$CAPTURE_RC" -eq 125 ] && [ ! -s "$CAPTURE_STDOUT" ] && - grep -Fq 'INFRA Experiment strict Cedar policy validation was not exact' "$CAPTURE_STDERR"; then - pass "invalid trusted policy is infrastructure uncertainty, never a deny" -else - fail "invalid trusted policy was translated as a decision (rc=$CAPTURE_RC)" -fi - -if python3 - "$repo_root" "$fixture_root/valid.json" <<'PY' -from importlib.util import module_from_spec, spec_from_file_location -from pathlib import Path -import json -import os -import signal -import subprocess -import sys - -root = Path(sys.argv[1]) -spec = spec_from_file_location("experiment_authorization", root / "scripts/experiment.py") -assert spec is not None and spec.loader is not None -module = module_from_spec(spec) -spec.loader.exec_module(module) - -manifest = json.loads(Path(sys.argv[2]).read_text(encoding="utf-8")) -plan = module.cue_plan(manifest) -binding = module.plan_binding(plan) -request, entities = module.cedar_documents(binding) -authorization_digest, snapshot = module.authorization_snapshot(root) - -assert binding.plan_digest == request["context"]["planDigest"] -assert request["resource"] == ( - f'AgentLab::RequestedExperimentPlan::"{binding.plan_digest}"' -) -assert request["principal"] == ( - 'AgentLab::Principal::"legacy-local-operator"' -) - -observed_helpers = [] -observed_arguments = [] -real_invoke = module.invoke_cedar -def observing_invoke(helper, arguments, repository): - observed_helpers.append(helper) - observed_arguments.append(arguments) - assert str(helper).startswith("/tmp/agent-lab-authorization-") - assert helper.read_bytes() == snapshot[module.CEDAR_HELPER] - return real_invoke(helper, arguments, repository) -module.invoke_cedar = observing_invoke -assert module.evaluate_cedar(snapshot, request, entities, root) == "permit" -assert len(observed_helpers) == 2 -assert observed_arguments[0][2] == "validate" -assert "--deny-warnings" in observed_arguments[0] -assert observed_arguments[0][observed_arguments[0].index("--validation-mode") + 1] == "strict" -assert observed_arguments[1][2] == "authorize" -assert observed_arguments[1][observed_arguments[1].index("--request-validation") + 1] == "true" -module.invoke_cedar = real_invoke - -alternate_request = dict(request) -alternate_request["principal"] = 'AgentLab::Principal::"intruder"' -alternate_entities = json.loads(json.dumps(entities)) -alternate_entities.append({ - "uid": {"type": "AgentLab::Principal", "id": "intruder"}, - "attrs": {"authenticated": False, "assurance": "none", "source": "fixed-local-cli"}, - "parents": [], -}) -assert module.evaluate_cedar(snapshot, alternate_request, alternate_entities, root) == "deny" - -mismatch_request = json.loads(json.dumps(request)) -mismatch_request["context"]["planDigest"] = "sha256:" + ("f" * 64) -assert module.evaluate_cedar(snapshot, mismatch_request, entities, root) == "deny" - -contract_request = json.loads(json.dumps(request)) -contract_request["context"]["contractDigest"] = "sha256:" + ("e" * 64) -assert module.evaluate_cedar(snapshot, contract_request, entities, root) == "deny" - -version_request = json.loads(json.dumps(request)) -version_request["context"]["bindingVersion"] = "v0alpha2" -assert module.evaluate_cedar(snapshot, version_request, entities, root) == "deny" - -empty = dict(snapshot) -empty["authorization/experiment/v0alpha1/operator.cedar"] = b"" -assert module.evaluate_cedar(empty, request, entities, root) == "deny" - -forbidden = dict(snapshot) -forbidden["authorization/experiment/v0alpha1/operator.cedar"] += b'''\n@id("emergency-stop-v0") -forbid (principal, action, resource);\n''' -assert module.evaluate_cedar(forbidden, request, entities, root) == "deny" - -completed = subprocess.CompletedProcess([], 0, b"\nALLOW\n", b"") -assert module.parse_cedar_authorization(completed) == "permit" -completed = subprocess.CompletedProcess([], 2, b"\nDENY\n", b"") -assert module.parse_cedar_authorization(completed) == "deny" -for outcome in ( - subprocess.CompletedProcess([], 0, b"ALLOW\n", b""), - subprocess.CompletedProcess([], 0, b"\nALLOW\nextra", b""), - subprocess.CompletedProcess([], 0, b"\nALLOW\n", b"warning"), - subprocess.CompletedProcess([], 1, b"\nDENY\n", b""), - subprocess.CompletedProcess([], 2, b"\nALLOW\n", b""), - subprocess.CompletedProcess([], -9, b"", b""), -): - try: - module.parse_cedar_authorization(outcome) - except module.InfrastructureError: - pass - else: - raise AssertionError(f"ambiguous evaluator tuple accepted: {outcome}") - -# The production runner bounds bytes while the evaluator is live and kills timeouts. -from tempfile import TemporaryDirectory -import time -control_signals = (signal.SIGHUP, signal.SIGINT, signal.SIGQUIT, signal.SIGTERM) -original_dispositions = {signum: signal.getsignal(signum) for signum in control_signals} -original_signal_mask = set(signal.pthread_sigmask(signal.SIG_BLOCK, set())) -with TemporaryDirectory(prefix="cedar-runner-cases-", dir="/tmp") as directory: - fake = Path(directory) / "fake.py" - original_limit = module.MAX_CEDAR_OUTPUT_BYTES - original_timeout = module.CEDAR_TIMEOUT_SECONDS - module.MAX_CEDAR_OUTPUT_BYTES = 1024 - fake.write_text( - "import sys, time\n" - "sys.stdout.buffer.write(b'x' * 2048)\n" - "sys.stdout.buffer.flush()\n" - "time.sleep(5)\n", - encoding="utf-8", - ) - started = time.monotonic() - try: - module.invoke_cedar(fake, (), root) - except module.InfrastructureError as error: - assert str(error) == "pinned Cedar emitted overlong output" - else: - raise AssertionError("overlong live evaluator output was accepted") - assert time.monotonic() - started < 2 - - module.MAX_CEDAR_OUTPUT_BYTES = original_limit - module.CEDAR_TIMEOUT_SECONDS = 0.1 - fake.write_text("import time\ntime.sleep(5)\n", encoding="utf-8") - started = time.monotonic() - try: - module.invoke_cedar(fake, (), root) - except module.InfrastructureError as error: - assert str(error) == "pinned Cedar evaluation timed out" - else: - raise AssertionError("evaluator timeout was accepted") - assert time.monotonic() - started < 2 - module.MAX_CEDAR_OUTPUT_BYTES = original_limit - module.CEDAR_TIMEOUT_SECONDS = original_timeout - - marker = Path(directory) / "cancelled.pid" - fake.write_text( - "from pathlib import Path\n" - "import os, sys, time\n" - "Path(sys.argv[1]).write_text(str(os.getpid()), encoding='ascii')\n" - "time.sleep(30)\n", - encoding="utf-8", - ) - driver = Path(directory) / "driver.py" - driver.write_text( - "from importlib.util import module_from_spec, spec_from_file_location\n" - "from pathlib import Path\n" - "import sys\n" - "spec = spec_from_file_location('cancel_contract', sys.argv[1])\n" - "assert spec is not None and spec.loader is not None\n" - "module = module_from_spec(spec)\n" - "spec.loader.exec_module(module)\n" - "module.invoke_cedar(Path(sys.argv[2]), (sys.argv[3],), Path(sys.argv[4]))\n", - encoding="utf-8", - ) - parent = subprocess.Popen( - [sys.executable, str(driver), str(root / "scripts/experiment.py"), - str(fake), str(marker), str(root)], - ) - deadline = time.monotonic() + 2 - while not marker.exists() and parent.poll() is None and time.monotonic() < deadline: - time.sleep(0.01) - assert marker.exists() - evaluator_pid = int(marker.read_text(encoding="ascii")) - parent.terminate() - assert parent.wait(timeout=2) == 143 - deadline = time.monotonic() + 2 - while time.monotonic() < deadline: - try: - os.kill(evaluator_pid, 0) - except ProcessLookupError: - break - time.sleep(0.01) - else: - raise AssertionError("cancelled Cedar evaluator survived its parent") - - signal_race_failures = [] - spawn_race_pid = Path(directory) / "spawn-race.pid" - race_driver = Path(directory) / "spawn-race-driver.py" - race_driver.write_text( - "from importlib.util import module_from_spec, spec_from_file_location\n" - "from pathlib import Path\n" - "import os, signal, sys\n" - "spec = spec_from_file_location('spawn_race_contract', sys.argv[1])\n" - "assert spec is not None and spec.loader is not None\n" - "module = module_from_spec(spec)\n" - "spec.loader.exec_module(module)\n" - "real_popen = module.subprocess.Popen\n" - "def signal_after_spawn(*args, **kwargs):\n" - " child = real_popen(*args, **kwargs)\n" - " Path(sys.argv[3]).write_text(str(child.pid), encoding='ascii')\n" - " os.kill(os.getpid(), signal.SIGTERM)\n" - " return child\n" - "module.subprocess.Popen = signal_after_spawn\n" - "module.invoke_cedar(Path(sys.argv[2]), ('unused',), Path(sys.argv[4]))\n", - encoding="utf-8", - ) - raced = subprocess.Popen( - [sys.executable, str(race_driver), str(root / "scripts/experiment.py"), - str(fake), str(spawn_race_pid), str(root)], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - ) - raced_stdout, raced_stderr = raced.communicate(timeout=2) - assert spawn_race_pid.exists() - raced_child = int(spawn_race_pid.read_text(encoding="ascii")) - try: - os.kill(raced_child, 0) - except ProcessLookupError: - spawn_race_leaked = False - else: - spawn_race_leaked = True - try: - os.killpg(raced_child, signal.SIGKILL) - except ProcessLookupError: - pass - if raced.returncode != 143: - signal_race_failures.append(f"spawn-race rc={raced.returncode}") - if raced_stdout != b"" or raced_stderr != b"": - signal_race_failures.append("spawn-race emitted output") - if spawn_race_leaked: - signal_race_failures.append("signal between Popen and assignment leaked Cedar") - - cleanup_marker = Path(directory) / "cleanup-signal.pid" - cleanup_driver = Path(directory) / "cleanup-signal-driver.py" - cleanup_driver.write_text( - "from importlib.util import module_from_spec, spec_from_file_location\n" - "from pathlib import Path\n" - "import os, signal, sys\n" - "spec = spec_from_file_location('cleanup_signal_contract', sys.argv[1])\n" - "assert spec is not None and spec.loader is not None\n" - "module = module_from_spec(spec)\n" - "spec.loader.exec_module(module)\n" - "real_terminate = module.terminate_cedar_group\n" - "def signal_during_cleanup(process):\n" - " os.kill(os.getpid(), signal.SIGTERM)\n" - " real_terminate(process)\n" - "module.terminate_cedar_group = signal_during_cleanup\n" - "module.invoke_cedar(Path(sys.argv[2]), (sys.argv[3],), Path(sys.argv[4]))\n", - encoding="utf-8", - ) - cleaning = subprocess.Popen( - [sys.executable, str(cleanup_driver), str(root / "scripts/experiment.py"), - str(fake), str(cleanup_marker), str(root)], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - ) - deadline = time.monotonic() + 2 - while not cleanup_marker.exists() and cleaning.poll() is None and time.monotonic() < deadline: - time.sleep(0.01) - assert cleanup_marker.exists() - cleanup_child = int(cleanup_marker.read_text(encoding="ascii")) - os.kill(cleaning.pid, signal.SIGTERM) - cleanup_stdout, cleanup_stderr = cleaning.communicate(timeout=2) - try: - os.kill(cleanup_child, 0) - except ProcessLookupError: - cleanup_child_leaked = False - else: - cleanup_child_leaked = True - try: - os.killpg(cleanup_child, signal.SIGKILL) - except ProcessLookupError: - pass - if cleaning.returncode != 143: - signal_race_failures.append(f"cleanup-signal rc={cleaning.returncode}") - if cleanup_stdout != b"" or cleanup_stderr != b"": - signal_race_failures.append("cleanup-signal emitted output or traceback") - if cleanup_child_leaked: - signal_race_failures.append("second handled signal interrupted Cedar cleanup") - assert not signal_race_failures, "; ".join(signal_race_failures) - - residual_marker = Path(directory) / "residual.pid" - fake.write_text( - "from pathlib import Path\n" - "import subprocess, sys\n" - "child = subprocess.Popen([sys.executable, '-c', " - "'import time; time.sleep(30)'])\n" - "Path(sys.argv[1]).write_text(str(child.pid), encoding='ascii')\n", - encoding="utf-8", - ) - try: - module.invoke_cedar(fake, (str(residual_marker),), root) - except module.InfrastructureError as error: - assert str(error) == "pinned Cedar left a residual process group" - else: - raise AssertionError("a residual Cedar process group was accepted") - residual_pid = int(residual_marker.read_text(encoding="ascii")) - try: - os.kill(residual_pid, 0) - except ProcessLookupError: - pass - else: - raise AssertionError("residual Cedar child survived cleanup") - -assert {signum: signal.getsignal(signum) for signum in control_signals} == original_dispositions -assert set(signal.pthread_sigmask(signal.SIG_BLOCK, set())) == original_signal_mask - -# Main dispatch reads manifest bytes exactly once and supplies no caller input to Cedar. -reads = [] -captured = [] -real_read = module.read_manifest_once -real_cue = module.cue_plan -real_authorize = module.authorize_plan -real_write = module.write_decision -module.read_manifest_once = lambda path: reads.append(path) or b'{}' -module.cue_plan = lambda value: plan -module.authorize_plan = lambda value: captured.append(value) or ({"verdict": "permit"}, 0) -module.write_decision = lambda value: None -assert module.main(["experiment.py", "authorize", "install", "/manifest.json"]) == 0 -assert reads == ["/manifest.json"] -assert captured == [plan] -module.read_manifest_once = real_read -module.cue_plan = real_cue -module.authorize_plan = real_authorize -module.write_decision = real_write - -assert authorization_digest.startswith("sha256:") -PY -then - pass "policy, parser, timeout, cancellation, and residual-process cases fail closed" -else - fail "low-level Cedar authorization invariants failed" -fi - -if [ ! -s "$spy_log" ]; then - pass "authorization invokes no engine or network command from caller PATH" -else - fail "authorization invoked a forbidden caller PATH tool: $(tr '\n' ' ' < "$spy_log")" -fi -if ! grep -Eiq 'docker|podman|docker-compose|urllib|requests|socket' \ - "$repo_root/scripts/experiment" "$repo_root/scripts/experiment.py"; then - pass "the authorization adapter contains no engine or network client surface" -else - fail "the authorization adapter gained engine or network client code" -fi -if [ -z "$(find "$work/home" "$work/runtime-tmp" -mindepth 1 -print -quit)" ]; then - pass "authorization leaves no persistent home or runtime state" -else - fail "authorization left persistent state" -fi -git -C "$repo_root" status --porcelain=v1 --untracked-files=all > "$work/repo.after" -if cmp -s "$repo_before" "$work/repo.after"; then - pass "authorization leaves the checkout tree unchanged" -else - fail "authorization changed the checkout tree" -fi - -if printf '%s\n' "$cedar_preflight_out" | grep -Fxq 'cedar-policy-cli 4.12.0'; then - pass "Experiment authorization uses the pinned Cedar release" -else - fail "pinned Cedar release is unavailable or wrong" -fi -if "$cedar_tool" format --check --policies \ - "$repo_root/authorization/experiment/v0alpha1/operator.cedar" >/dev/null; then - pass "the tracked Cedar policy is canonically formatted" -else - fail "the tracked Cedar policy needs cedar format" -fi - -printf 'SUMMARY failures=%s\n' "$failures" -[ "$failures" -eq 0 ] + "$agent_lab" experiment authorize install "$fixture" > "$work/out" 2> "$work/err" || rc=$? +if [ "$rc" -eq 0 ] && [ ! -s "$work/err" ] && + jq -e '.verdict == "permit" and (.binding.sourceDigest | startswith("sha256:")) and (.binding.planDigest | startswith("sha256:"))' "$work/out" >/dev/null 2>&1; then + printf 'PASS AUTH-001 fresh preview binds source and plan\n' + failures=0 +else + printf 'FAIL AUTH-001 fresh preview binds source and plan\n' + failures=1 +fi +printf 'SUMMARY assertions=1 expected=1 failures=%s infra=0\n' "$failures" +[ "$failures" -eq 0 ] || exit 1 +printf 'EXPERIMENT AUTHORIZATION PASS\n' diff --git a/tests/experiment/contract-cases.sh b/tests/experiment/contract-cases.sh index 0e346ab..74b64b4 100755 --- a/tests/experiment/contract-cases.sh +++ b/tests/experiment/contract-cases.sh @@ -2,461 +2,7 @@ set -euo pipefail repo_root="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." >/dev/null 2>&1 && pwd)" -experiment="$repo_root/scripts/experiment" -cue_tool="$repo_root/scripts/dev/cue-tool" -fixture_root="$repo_root/tests/experiment/fixtures" - -for required in \ - "$experiment" \ - "$cue_tool" \ - "$repo_root/scripts/dev/cue-tool.py" \ - "$repo_root/contracts/experiment/v0alpha1/schema.cue" \ - "$repo_root/contracts/experiment/v0alpha1/plan.cue" \ - "$repo_root/contracts/experiment/v0alpha1/cue.mod/module.cue" \ - "$repo_root/tools/cue.lock"; do - if [ ! -f "$required" ]; then - printf 'INFRA Experiment contract input is missing: %s\n' "$required" >&2 - exit 125 - fi -done -if [ ! -x "$experiment" ] || [ ! -x "$cue_tool" ]; then - printf 'INFRA Experiment contract entrypoints are not executable\n' >&2 - exit 125 -fi -cue_preflight_rc=0 -cue_preflight_out="$("$cue_tool" version 2>&1)" || cue_preflight_rc=$? -if [ "$cue_preflight_rc" -ne 0 ]; then - printf 'INFRA Experiment contract requires provisioned pinned CUE: %s\n' \ - "$cue_preflight_out" >&2 - exit 125 -fi - -work="$(mktemp -d)" -cleanup() { - find "$work" -type f -delete 2>/dev/null || true - find "$work" -type l -delete 2>/dev/null || true - find "$work" -depth -type d -exec rmdir {} + 2>/dev/null || true -} -trap cleanup EXIT -mkdir -p "$work/bin" "$work/home" "$work/runtime-tmp" - -spy_log="$work/tool-spy.log" -: > "$spy_log" -repo_before="$work/repo.before" -git -C "$repo_root" status --porcelain=v1 --untracked-files=all > "$repo_before" -for tool in docker docker-compose podman; do - spy="$work/bin/$tool" - { - printf '#!/usr/bin/env bash\n' - printf 'printf "%%s\\n" "$0 $*" >> %q\n' "$spy_log" - printf 'exit 97\n' - } > "$spy" - chmod +x "$spy" -done - -failures=0 -pass() { printf 'PASS %s\n' "$1"; } -fail() { printf 'FAIL %s\n' "$1"; failures=$((failures + 1)); } - -sha256_stdin() { - if command -v sha256sum >/dev/null 2>&1; then - sha256sum | awk '{print $1}' - else - shasum -a 256 | awk '{print $1}' - fi -} - -tree_fingerprint() { - python3 - "$1" <<'PY' -from hashlib import sha256 -from pathlib import Path -import os -import stat -import sys - -root = Path(sys.argv[1]) -if not root.exists(): - print("missing") - raise SystemExit(0) -for path in sorted(root.rglob("*"), key=lambda item: os.fsencode(item)): - relative = os.fsencode(path.relative_to(root)) - metadata = path.lstat() - if stat.S_ISREG(metadata.st_mode): - kind = b"file" - payload = sha256(path.read_bytes()).digest() - elif stat.S_ISDIR(metadata.st_mode): - kind = b"directory" - payload = b"" - elif stat.S_ISLNK(metadata.st_mode): - kind = b"symlink" - payload = os.fsencode(os.readlink(path)) - else: - kind = b"other" - payload = b"" - record = b"\0".join((relative, kind, str(stat.S_IMODE(metadata.st_mode)).encode(), payload)) - print(sha256(record).hexdigest()) -PY -} - -cue_cache="$repo_root/.cache/dev/tools/cue" -cache_before="$work/cache.before" -tree_fingerprint "$cue_cache" > "$cache_before" - -contract_sha256() { - python3 - "$repo_root" <<'PY' -from hashlib import sha256 -from pathlib import Path -import sys - -root = Path(sys.argv[1]) -names = sorted(( - "contracts/experiment/v0alpha1/cue.mod/module.cue", - "contracts/experiment/v0alpha1/plan.cue", - "contracts/experiment/v0alpha1/schema.cue", - "tools/cue.lock", -)) -digest = sha256(b"agent-lab.contract.v1\0") -for name in names: - encoded = name.encode("utf-8") - data = (root / name).read_bytes() - digest.update(len(encoded).to_bytes(4, "big")) - digest.update(encoded) - digest.update(len(data).to_bytes(8, "big")) - digest.update(data) -print(digest.hexdigest()) -PY -} - -capture() { - local name="$1" - shift - CAPTURE_STDOUT="$work/$name.stdout" - CAPTURE_STDERR="$work/$name.stderr" - CAPTURE_RC=0 - env -i \ - PATH="$work/bin:/usr/bin:/bin" \ - HOME="$work/home" \ - TMPDIR="$work/runtime-tmp" \ - LC_ALL=C \ - EXPERIMENT_TOOL_SPY_LOG="$spy_log" \ - "$@" > "$CAPTURE_STDOUT" 2> "$CAPTURE_STDERR" || CAPTURE_RC=$? -} - -expect_invalid() { - local id="$1" path="$2" detail="$3" - capture "$id" "$experiment" check -- "$path" - if [ "$CAPTURE_RC" -eq 1 ] && [ ! -s "$CAPTURE_STDOUT" ] && - grep -Fq 'FAIL Experiment manifest' "$CAPTURE_STDERR"; then - pass "$detail" - else - fail "$detail (rc=$CAPTURE_RC)" - fi -} - -capture usage-none "$experiment" -if [ "$CAPTURE_RC" -eq 2 ] && [ ! -s "$CAPTURE_STDOUT" ] && - grep -Fq 'Usage: scripts/experiment check [--] MANIFEST' "$CAPTURE_STDERR"; then - pass "missing command has the exact usage contract" -else - fail "missing command has the exact usage contract (rc=$CAPTURE_RC)" -fi - -capture usage-extra "$experiment" check "$fixture_root/valid.json" extra -if [ "$CAPTURE_RC" -eq 2 ] && [ ! -s "$CAPTURE_STDOUT" ]; then - pass "extra CLI arguments fail as usage" -else - fail "extra CLI arguments fail as usage (rc=$CAPTURE_RC)" -fi - -expected_contract_digest="$(contract_sha256)" -expected_plan="$( - jq -cS --arg digest "sha256:$expected_contract_digest" \ - '.contract.digest = $digest' "$fixture_root/expected-plan.json" -)" -expected_digest="$(printf '%s' "$expected_plan" | sha256_stdin)" -expected_envelope="$( - jq -cnS \ - --arg digest "sha256:$expected_digest" \ - --argjson plan "$expected_plan" \ - '{digest: $digest, plan: $plan}' -)" - -capture valid "$experiment" check -- "$fixture_root/valid.json" -if [ "$CAPTURE_RC" -eq 0 ] && [ ! -s "$CAPTURE_STDERR" ] && - [ "$(cat "$CAPTURE_STDOUT")" = "$expected_envelope" ] && - [ "$(wc -l < "$CAPTURE_STDOUT" | tr -d ' ')" -eq 1 ]; then - pass "valid manifest emits the exact canonical plan envelope" -else - fail "valid manifest emits the exact canonical plan envelope (rc=$CAPTURE_RC)" -fi - -capture reordered "$experiment" check -- "$fixture_root/valid-reordered.json" -if [ "$CAPTURE_RC" -eq 0 ] && [ ! -s "$CAPTURE_STDERR" ] && - cmp -s "$CAPTURE_STDOUT" "$work/valid.stdout"; then - pass "field order, member order, whitespace, and explicit defaults are non-semantic" -else - fail "equivalent reordered manifest changes the plan (rc=$CAPTURE_RC)" -fi - -if python3 - "$repo_root" "$fixture_root/valid.json" <<'PY' -from importlib.util import module_from_spec, spec_from_file_location -from pathlib import Path -import json -import subprocess -import sys -from types import SimpleNamespace - -root = Path(sys.argv[1]) -spec = spec_from_file_location("experiment_contract", root / "scripts/experiment.py") -assert spec is not None and spec.loader is not None -module = module_from_spec(spec) -spec.loader.exec_module(module) -manifest = json.loads(Path(sys.argv[2]).read_text(encoding="utf-8")) -digest, snapshot = module.contract_snapshot(root) -helper = (root / "scripts/dev/cue-tool.py").read_bytes() -private_roots = [] - -subprocess_calls = [] -def fake_run(command, **kwargs): - subprocess_calls.append((command, kwargs)) - return subprocess.CompletedProcess(command, 1, b"", b"") - -real_subprocess = module.subprocess -module.subprocess = SimpleNamespace( - PIPE=subprocess.PIPE, - SubprocessError=subprocess.SubprocessError, - run=fake_run, -) -private = Path("/private-validation-root") -module.invoke_cue( - manifest, - digest, - private, - root, - private / "contracts/experiment/v0alpha1", -) -module.subprocess = real_subprocess -assert len(subprocess_calls) == 1 -command = subprocess_calls[0][0] -assert command[:3] == ( - sys.executable, - "-I", - str(private / "scripts/dev/cue-tool.py"), -) -assert command[command.index("-C") + 1] == str( - private / "contracts/experiment/v0alpha1" -) -assert not {"docker", "docker-compose", "podman"}.intersection(command) - -def fake_invoke(value, actual_digest, validation_root, repo_root, contract_root): - assert actual_digest == digest - assert repo_root == root - assert validation_root != root - assert contract_root == validation_root / "contracts/experiment/v0alpha1" - for name, data in snapshot.items(): - assert (validation_root / name).read_bytes() == data - assert (validation_root / "scripts/dev/cue-tool.py").read_bytes() == helper - private_roots.append(validation_root) - plan = module.expected_plan(value, actual_digest) - return subprocess.CompletedProcess([], 0, module.canonical_json(plan) + b"\n", b"") - -module.invoke_cue = fake_invoke -module.cue_plan(manifest) -assert len(private_roots) == 2 -assert private_roots[0] == private_roots[1] -assert not private_roots[0].exists() -PY -then - pass "orchestration invokes only CUE over the exact hashed private snapshot" -else - fail "orchestration escaped CUE or reopened live contract paths" -fi - -actual_digest="$(jq -er '.digest' "$work/valid.stdout" 2>/dev/null || true)" -actual_plan="$(jq -cS '.plan' "$work/valid.stdout" 2>/dev/null || true)" -independent_digest="$(printf '%s' "$actual_plan" | sha256_stdin)" -if [ "$actual_digest" = "sha256:$independent_digest" ] && - [[ "$actual_digest" =~ ^sha256:[0-9a-f]{64}$ ]]; then - pass "plan digest independently matches the canonical plan bytes" -else - fail "plan digest does not match the canonical plan bytes" -fi - -expect_invalid unknown "$fixture_root/invalid-unknown.json" \ - "unknown privileged member field is rejected" -expect_invalid duplicate-identical "$fixture_root/invalid-duplicate-identical.json" \ - "identical duplicate JSON key is rejected before CUE unification" -expect_invalid duplicate-escaped "$fixture_root/invalid-duplicate-escaped.json" \ - "escaped-equivalent duplicate JSON key is rejected" -expect_invalid duplicate-member "$fixture_root/invalid-duplicate-member.json" \ - "duplicate member names are rejected" -expect_invalid mutable-image "$fixture_root/invalid-mutable-image.json" \ - "mutable image reference is rejected" -expect_invalid numeric-overflow "$fixture_root/invalid-numeric-overflow.json" \ - "out-of-range JSON number is invalid input, not infrastructure uncertainty" -expect_invalid environment "$fixture_root/invalid-environment.json" \ - "generic environment cannot override future Lab-owned runtime controls" -expect_invalid empty-members "$fixture_root/invalid-empty.json" \ - "an Experiment must contain at least one member" - -oversized="$work/oversized.json" -python3 - "$oversized" <<'PY' -from pathlib import Path -import sys - -Path(sys.argv[1]).write_bytes(b"x" * 262_145) -PY -expect_invalid oversized "$oversized" "manifest byte limit fails closed" - -deeply_nested="$work/deeply-nested.json" -python3 - "$deeply_nested" <<'PY' -from pathlib import Path -import sys - -prefix = b'{"apiVersion":"agent-lab/v0alpha1","kind":"Experiment","metadata":{"name":"deep"},"spec":{"members":' -Path(sys.argv[1]).write_bytes(prefix + (b"[" * 1000) + b"null" + (b"]" * 1000) + b"}}") -PY -expect_invalid deeply-nested "$deeply_nested" \ - "excessive JSON nesting is invalid input without a traceback" - -bidi="$work/bidi.json" -python3 - "$fixture_root/valid.json" "$bidi" <<'PY' -from pathlib import Path -import json -import sys - -value = json.loads(Path(sys.argv[1]).read_text(encoding="utf-8")) -value["spec"]["members"][0]["command"].append("review\N{RIGHT-TO-LEFT OVERRIDE}text") -Path(sys.argv[2]).write_text(json.dumps(value), encoding="utf-8") -PY -expect_invalid bidi "$bidi" "Unicode formatting controls are rejected" - -capture missing "$experiment" check -- "$work/missing.json" -if [ "$CAPTURE_RC" -eq 125 ] && [ ! -s "$CAPTURE_STDOUT" ] && - grep -Fq 'INFRA Experiment manifest' "$CAPTURE_STDERR"; then - pass "unreadable manifest is infrastructure uncertainty" -else - fail "unreadable manifest has the wrong result (rc=$CAPTURE_RC)" -fi - -ln -s "$fixture_root/valid.json" "$work/symlink.json" -capture symlink "$experiment" check -- "$work/symlink.json" -if [ "$CAPTURE_RC" -eq 125 ] && [ ! -s "$CAPTURE_STDOUT" ]; then - pass "manifest symlinks are rejected without following them" -else - fail "manifest symlink was followed (rc=$CAPTURE_RC)" -fi - -if [ ! -s "$spy_log" ]; then - pass "caller PATH engine shims cannot redirect Experiment validation" -else - fail "validation invoked a caller PATH engine shim: $(tr '\n' ' ' < "$spy_log")" -fi -if [ -z "$(find "$work/home" "$work/runtime-tmp" -mindepth 1 -print -quit)" ]; then - pass "validation creates no persistent home or runtime state" -else - fail "validation left persistent state" -fi -git -C "$repo_root" status --porcelain=v1 --untracked-files=all > "$work/repo.after" -if cmp -s "$repo_before" "$work/repo.after"; then - pass "validation leaves the checkout tree unchanged" -else - fail "validation changed the checkout tree" -fi - -if python3 - "$work/valid.stdout" <<'PY' -from pathlib import Path -import sys - -data = Path(sys.argv[1]).read_bytes() -raise SystemExit(0 if all(byte == 10 or 32 <= byte <= 126 for byte in data) else 1) -PY -then - pass "canonical envelope is one review-safe ASCII line" -else - fail "canonical envelope contains raw non-ASCII or control bytes" -fi - -copy_root="$work/contract-copy" -mkdir -p \ - "$copy_root/scripts/dev" \ - "$copy_root/contracts/experiment/v0alpha1/cue.mod" \ - "$copy_root/tests/experiment/fixtures" \ - "$copy_root/tools" -cp "$repo_root/scripts/experiment" "$repo_root/scripts/experiment.py" \ - "$copy_root/scripts/" -cp "$repo_root/scripts/dev/cue-tool" "$repo_root/scripts/dev/cue-tool.py" \ - "$copy_root/scripts/dev/" -cp "$repo_root/contracts/experiment/v0alpha1/schema.cue" \ - "$repo_root/contracts/experiment/v0alpha1/plan.cue" \ - "$copy_root/contracts/experiment/v0alpha1/" -cp "$repo_root/contracts/experiment/v0alpha1/cue.mod/module.cue" \ - "$copy_root/contracts/experiment/v0alpha1/cue.mod/" -cp "$repo_root/tools/cue.lock" "$copy_root/tools/" -cp "$fixture_root/valid.json" "$copy_root/tests/experiment/fixtures/" -chmod +x "$copy_root/scripts/experiment" "$copy_root/scripts/dev/cue-tool" - -capture contract-before env \ - AGENT_LAB_CUE_TOOL_DIR="$repo_root/.cache/dev/tools/cue" \ - "$copy_root/scripts/experiment" check -- \ - "$copy_root/tests/experiment/fixtures/valid.json" -python3 - "$copy_root/contracts/experiment/v0alpha1/schema.cue" <<'PY' -from pathlib import Path -import sys - -path = Path(sys.argv[1]) -text = path.read_text(encoding="utf-8") -path.write_text(text.replace("list.MaxItems(16)", "list.MaxItems(15)"), encoding="utf-8") -PY -capture contract-after env \ - AGENT_LAB_CUE_TOOL_DIR="$repo_root/.cache/dev/tools/cue" \ - "$copy_root/scripts/experiment" check -- \ - "$copy_root/tests/experiment/fixtures/valid.json" -if [ "$CAPTURE_RC" -eq 0 ] && - ! cmp -s "$work/contract-before.stdout" "$work/contract-after.stdout"; then - pass "constraint-only contract changes alter the bound plan identity" -else - fail "constraint-only contract change retained the old plan identity (rc=$CAPTURE_RC)" -fi - -printf '\ninvalid: [\n' >> "$copy_root/contracts/experiment/v0alpha1/schema.cue" -capture corrupt-contract env \ - AGENT_LAB_CUE_TOOL_DIR="$repo_root/.cache/dev/tools/cue" \ - "$copy_root/scripts/experiment" check -- \ - "$copy_root/tests/experiment/fixtures/valid.json" -if [ "$CAPTURE_RC" -eq 125 ] && [ ! -s "$CAPTURE_STDOUT" ] && - grep -Fq 'INFRA Experiment' "$CAPTURE_STDERR"; then - pass "corrupt trusted CUE contract is infrastructure uncertainty" -else - fail "corrupt trusted CUE contract was blamed on the manifest (rc=$CAPTURE_RC)" -fi - -if printf '%s\n' "$cue_preflight_out" | grep -Fxq 'cue version v0.17.1'; then - pass "Experiment validation uses the pinned CUE release" -else - fail "pinned CUE release is unavailable or wrong" -fi -if "$cue_tool" fmt --check --files \ - "$repo_root/contracts/experiment/v0alpha1/schema.cue" \ - "$repo_root/contracts/experiment/v0alpha1/plan.cue" \ - "$repo_root/contracts/experiment/v0alpha1/cue.mod/module.cue"; then - pass "tracked CUE contract files are canonically formatted" -else - fail "tracked CUE contract files need cue fmt" -fi -if "$cue_tool" -C "$repo_root/contracts/experiment/v0alpha1" vet -c=false ./...; then - pass "the complete CUE module is satisfiable" -else - fail "the complete CUE module contains a latent contradiction" -fi - -cache_after="$work/cache.after" -tree_fingerprint "$cue_cache" > "$cache_after" -if cmp -s "$cache_before" "$cache_after"; then - pass "validation leaves the ignored pinned CUE cache unchanged" -else - fail "validation changed the ignored pinned CUE cache" -fi - -printf 'SUMMARY failures=%s\n' "$failures" -[ "$failures" -eq 0 ] +subcase="$repo_root/tests/experiment/directory-intake-cases.sh" +[ -x "$subcase" ] || { printf 'INFRA directory intake subcase is missing\n' >&2; exit 125; } +"$subcase" +printf 'EXPERIMENT CONTRACT PASS\n' diff --git a/tests/experiment/directory-intake-cases.sh b/tests/experiment/directory-intake-cases.sh new file mode 100755 index 0000000..1962d50 --- /dev/null +++ b/tests/experiment/directory-intake-cases.sh @@ -0,0 +1,53 @@ +#!/usr/bin/env bash +set -euo pipefail + +repo_root="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." >/dev/null 2>&1 && pwd)" +agent_lab="$repo_root/scripts/agent-lab" +fixture="$repo_root/tests/experiment/fixtures/directories/minimal" +work="$(mktemp -d)" +trap 'find "$work" -type f -delete 2>/dev/null || true; find "$work" -depth -type d -exec rmdir {} + 2>/dev/null || true' EXIT +mkdir -p "$work/home" "$work/tmp" + +failures=0 +observed="$work/observed" +: > "$observed" +pass() { printf 'PASS %s %s\n' "$1" "$2"; printf '%s\n' "$1" >> "$observed"; } +fail() { printf 'FAIL %s %s\n' "$1" "$2"; printf '%s\n' "$1" >> "$observed"; failures=$((failures + 1)); } +capture() { + CAPTURE_RC=0 + env -i PATH=/usr/bin:/bin HOME="$work/home" TMPDIR="$work/tmp" LC_ALL=C \ + AGENT_LAB_CUE_TOOL_DIR="${AGENT_LAB_CUE_TOOL_DIR:-$repo_root/.cache/dev/tools/cue}" \ + "$@" > "$work/stdout" 2> "$work/stderr" || CAPTURE_RC=$? +} + +if [ -x "$agent_lab" ]; then + pass FMT-001 "repository agent-lab entrypoint exists" +else + fail FMT-001 "repository agent-lab entrypoint exists" +fi + +capture "$agent_lab" experiment check "$fixture" +if [ "$CAPTURE_RC" -eq 0 ] && [ ! -s "$work/stderr" ] && + jq -e '.source.kind == "directory" and (.source.digest | startswith("sha256:")) and .plan.kind == "RequestedExperimentPlan"' "$work/stdout" >/dev/null 2>&1; then + pass FMT-002 "sole-entry directory produces a source-bound checked candidate" +else + fail FMT-002 "sole-entry directory produces a source-bound checked candidate" +fi + +capture "$agent_lab" experiment check "$fixture" +cp "$work/stdout" "$work/second" +capture "$agent_lab" experiment check "$fixture" +if [ "$CAPTURE_RC" -eq 0 ] && cmp -s "$work/second" "$work/stdout"; then + pass FMT-003 "repeated directory checks are byte-identical" +else + fail FMT-003 "repeated directory checks are byte-identical" +fi + +expected="$work/expected" +printf '%s\n' FMT-001 FMT-002 FMT-003 > "$expected" +if ! cmp -s "$expected" "$observed"; then + printf 'INFRA assertion identity drift\n' >&2 + exit 125 +fi +printf 'SUMMARY assertions=3 expected=3 failures=%s infra=0\n' "$failures" +[ "$failures" -eq 0 ] diff --git a/tests/experiment/fixtures/directories/minimal/experiment.cue b/tests/experiment/fixtures/directories/minimal/experiment.cue new file mode 100644 index 0000000..a1d8c8c --- /dev/null +++ b/tests/experiment/fixtures/directories/minimal/experiment.cue @@ -0,0 +1,12 @@ +package experiment + +experiment: { + apiVersion: "agent-lab/v0alpha1" + kind: "Experiment" + metadata: name: "first-experiment" + spec: members: [{ + name: "coordinator" + image: digestRef: "registry.example/team/coordinator@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + command: ["serve"] + }] +} diff --git a/tests/security/fast.manifest b/tests/security/fast.manifest index bf29de7..3d398ea 100644 --- a/tests/security/fast.manifest +++ b/tests/security/fast.manifest @@ -50,8 +50,8 @@ suite docker-harness-contract tests/dev/docker-harness-cases.sh SUMMARY failures suite guard-command tests/guard/pretooluse-cases.sh SUMMARY failures=0 suite guard-mount tests/guard/cases.sh SUMMARY failures=0 suite config-authority tests/agent/config-guard.sh SUMMARY failures=0 -suite experiment-contract tests/experiment/contract-cases.sh SUMMARY failures=0 -suite experiment-authorization tests/experiment/authorization-cases.sh SUMMARY failures=0 +suite experiment-contract tests/experiment/contract-cases.sh EXPERIMENT CONTRACT PASS +suite experiment-authorization tests/experiment/authorization-cases.sh EXPERIMENT AUTHORIZATION PASS suite config-matrix tests/agent/config-matrix.sh SUMMARY failures=0 suite allowlist-schema tests/agent/allowlist-cases.sh SUMMARY failures=0 suite image-volume-policy tests/agent/image-volume-policy-cases.sh SUMMARY failures=0 From 8e4f67c65fcdb14497beb3f41ede6e6570d91e2f Mon Sep 17 00:00:00 2001 From: xormania <127287135+xormania@users.noreply.github.com> Date: Sat, 1 Aug 2026 22:56:23 -0400 Subject: [PATCH 002/158] feat(experiment): accept authored directory snapshots --- .../experiment/v0alpha1/operator.cedar | 1 + .../experiment/v0alpha1/schema.cedarschema | 2 + contracts/experiment/v0alpha1/plan.cue | 8 +- contracts/experiment/v0alpha1/schema.cue | 7 +- scripts/agent-lab | 25 ++++ scripts/experiment.py | 141 ++++++++++++++++-- 6 files changed, 169 insertions(+), 15 deletions(-) create mode 100755 scripts/agent-lab diff --git a/authorization/experiment/v0alpha1/operator.cedar b/authorization/experiment/v0alpha1/operator.cedar index 6f7934a..af03c5f 100644 --- a/authorization/experiment/v0alpha1/operator.cedar +++ b/authorization/experiment/v0alpha1/operator.cedar @@ -11,6 +11,7 @@ when principal.source == "fixed-local-cli" && context.bindingVersion == "v0alpha1" && resource.planDigest == context.planDigest && + resource.sourceDigest == context.sourceDigest && resource.contractDigest == context.contractDigest && resource.contractVersion == "v0alpha1" }; diff --git a/authorization/experiment/v0alpha1/schema.cedarschema b/authorization/experiment/v0alpha1/schema.cedarschema index cdf7d94..0cf9cc4 100644 --- a/authorization/experiment/v0alpha1/schema.cedarschema +++ b/authorization/experiment/v0alpha1/schema.cedarschema @@ -7,6 +7,7 @@ namespace AgentLab { entity RequestedExperimentPlan = { planDigest: String, + sourceDigest: String, contractDigest: String, contractVersion: String, requestedName: String, @@ -20,6 +21,7 @@ namespace AgentLab { context: { bindingVersion: String, planDigest: String, + sourceDigest: String, contractDigest: String, }, }; diff --git a/contracts/experiment/v0alpha1/plan.cue b/contracts/experiment/v0alpha1/plan.cue index 285eda1..c9ac7fa 100644 --- a/contracts/experiment/v0alpha1/plan.cue +++ b/contracts/experiment/v0alpha1/plan.cue @@ -22,8 +22,12 @@ contractDigest: string & =~"^sha256:[0-9a-f]{64}$" @tag(contractDigest) y: _ less: x.name < y.name }) { - name: member.name - image: member.image + name: member.name + requestedSelector: member.image + resolvedImage: close({ + origin: "direct" + subject: member.image.digestRef + }) command: member.command resourceClass: member.resourceClass }] diff --git a/contracts/experiment/v0alpha1/schema.cue b/contracts/experiment/v0alpha1/schema.cue index 988cf6f..a099b70 100644 --- a/contracts/experiment/v0alpha1/schema.cue +++ b/contracts/experiment/v0alpha1/schema.cue @@ -8,9 +8,14 @@ import ( #Name: string & strings.MaxRunes(63) & =~"^[a-z](?:[a-z0-9-]{0,61}[a-z0-9])?$" -#Image: string & strings.MinRunes(1) & strings.MaxRunes(255) & +#DigestRef: string & strings.MinRunes(1) & strings.MaxRunes(255) & =~"^([a-z0-9]+([.-][a-z0-9]+)*(:(?:[1-9][0-9]{0,3}|[1-5][0-9]{4}|6[0-4][0-9]{3}|65[0-4][0-9]{2}|655[0-2][0-9]|6553[0-5]))?/)?[a-z0-9]+([._-][a-z0-9]+)*(/[a-z0-9]+([._-][a-z0-9]+)*)*@sha256:[0-9a-f]{64}$" +#CatalogName: string & strings.MaxRunes(63) & + =~"^[a-z][a-z0-9]*(?:-[a-z0-9]+)*\\.[a-z][a-z0-9]*(?:-[a-z0-9]+)*$" + +#Image: close({digestRef: #DigestRef}) | close({catalogName: #CatalogName}) + #Argument: string & strings.MaxRunes(1024) & !~"[\\p{Cc}\\p{Cf}\\p{Zl}\\p{Zp}]" #Member: close({ diff --git a/scripts/agent-lab b/scripts/agent-lab new file mode 100755 index 0000000..cc94d50 --- /dev/null +++ b/scripts/agent-lab @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +set -euo pipefail + +script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" +helper="$script_dir/experiment.py" + +usage() { + printf '%s\n' \ + 'Usage: agent-lab experiment check DIRECTORY' \ + 'Usage: agent-lab experiment authorize install DIRECTORY' >&2 +} + +[ "${1:-}" = experiment ] || { usage; exit 2; } +shift +case "${1:-}" in + check) + [ "$#" -eq 2 ] || { usage; exit 2; } + exec python3 -I "$helper" check-directory "$2" + ;; + authorize) + [ "${2:-}" = install ] && [ "$#" -eq 3 ] || { usage; exit 2; } + exec python3 -I "$helper" authorize-directory "$3" + ;; + *) usage; exit 2 ;; +esac diff --git a/scripts/experiment.py b/scripts/experiment.py index bddf421..de5eec8 100644 --- a/scripts/experiment.py +++ b/scripts/experiment.py @@ -18,6 +18,7 @@ MAX_MANIFEST_BYTES = 262_144 +SOURCE_DIGEST_DOMAIN = b"agent-lab.experiment-tree.v1\0" MAX_CUE_OUTPUT_BYTES = 1_048_576 MAX_CONTRACT_FILE_BYTES = 1_048_576 MAX_HELPER_BYTES = 1_048_576 @@ -63,7 +64,9 @@ "members": [ { "name": "probe", - "image": "probe@sha256:0000000000000000000000000000000000000000000000000000000000000000", + "image": { + "digestRef": "probe@sha256:0000000000000000000000000000000000000000000000000000000000000000" + }, } ] }, @@ -91,6 +94,12 @@ class PlanBinding(NamedTuple): requested_name: str member_count: int resource_classes: tuple[str, ...] + source_digest: str + + +class SourceSnapshot(NamedTuple): + data: bytes + digest: str def fail(message: str) -> NoReturn: @@ -196,6 +205,83 @@ def read_manifest_once(path: str) -> bytes: return data +def read_directory_snapshot(path: str) -> SourceSnapshot: + try: + directory_stat = os.lstat(path) + except OSError as error: + raise InfrastructureError("source directory cannot be inspected") from error + if stat.S_ISLNK(directory_stat.st_mode) or not stat.S_ISDIR(directory_stat.st_mode): + raise InvalidManifest("source must be one directory") + try: + before = os.listdir(path) + except OSError as error: + raise InfrastructureError("source directory cannot be listed") from error + if before != ["experiment.cue"]: + raise InvalidManifest("directory must contain only experiment.cue") + data = read_manifest_once(os.path.join(path, "experiment.cue")) + try: + after = os.listdir(path) + final_directory_stat = os.lstat(path) + except OSError as error: + raise InfrastructureError("source directory cannot be verified") from error + if after != before or (directory_stat.st_dev, directory_stat.st_ino) != ( + final_directory_stat.st_dev, + final_directory_stat.st_ino, + ): + raise InfrastructureError("source directory changed while snapshotting") + name = b"experiment.cue" + digest = hashlib.sha256(SOURCE_DIGEST_DOMAIN) + digest.update(len(name).to_bytes(4, "big")) + digest.update(name) + digest.update(len(data).to_bytes(8, "big")) + digest.update(data) + return SourceSnapshot(data=data, digest=f"sha256:{digest.hexdigest()}") + + +def authored_manifest(snapshot: SourceSnapshot) -> object: + repo_root = Path(__file__).resolve().parent.parent + cue_helper = repo_root / "scripts/dev/cue-tool.py" + module = b'module: "agent-lab.local/experiment-snapshot"\nlanguage: version: "v0.9.0"\n' + try: + with tempfile.TemporaryDirectory(prefix="agent-lab-source-", dir="/tmp") as directory: + root = Path(directory) + write_private_file(root, "cue.mod/module.cue", module) + write_private_file(root, "experiment.cue", snapshot.data) + completed = subprocess.run( + ( + sys.executable, + "-I", + str(cue_helper), + "-C", + str(root), + "export", + "-E", + "experiment.cue", + "-e", + "experiment", + "--out", + "json", + ), + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + env=cue_environment(repo_root), + timeout=CUE_TIMEOUT_SECONDS, + ) + except (OSError, subprocess.SubprocessError) as error: + raise InfrastructureError("authored CUE evaluation could not complete") from error + if completed.returncode == 1: + raise InvalidManifest("authored CUE value is invalid or incomplete") + if completed.returncode != 0 or completed.stderr or not completed.stdout: + raise InfrastructureError("pinned CUE could not export authored Experiment") + if len(completed.stdout) > MAX_CUE_OUTPUT_BYTES: + raise InfrastructureError("authored CUE output exceeded its bound") + manifest = strict_json(completed.stdout, source="authored CUE export") + if not isinstance(manifest, dict): + raise InvalidManifest("experiment must export one object") + return manifest + + def strict_json(data: bytes, *, source: str) -> object: try: text = data.decode("utf-8") @@ -443,15 +529,18 @@ def expected_plan(manifest: object, contract_digest: str) -> dict[str, object]: or not set(member) <= {"name", "image", "command", "resourceClass"} ): raise KeyError("member field drift") - members = [ - { + members = [] + for member in raw_members: + selector = member["image"] + if not isinstance(selector, dict) or set(selector) != {"digestRef"}: + raise KeyError("unresolved selector") + members.append({ "command": member.get("command", []), - "image": member["image"], "name": member["name"], + "requestedSelector": selector, + "resolvedImage": {"origin": "direct", "subject": selector["digestRef"]}, "resourceClass": member.get("resourceClass", "small"), - } - for member in raw_members - ] + }) members.sort(key=lambda member: str(member["name"])) name = metadata["name"] except (AssertionError, KeyError, TypeError) as error: @@ -593,7 +682,7 @@ def is_sha256(value: object) -> bool: ) -def plan_binding(plan: object) -> PlanBinding: +def plan_binding(plan: object, source_digest: str) -> PlanBinding: """Derive the only facts the v0alpha1 policy is allowed to see.""" try: if not isinstance(plan, dict) or set(plan) != { @@ -635,8 +724,9 @@ def plan_binding(plan: object) -> PlanBinding: for member in members: if not isinstance(member, dict) or set(member) != { "command", - "image", "name", + "requestedSelector", + "resolvedImage", "resourceClass", }: raise ValueError("plan member") @@ -659,6 +749,7 @@ def plan_binding(plan: object) -> PlanBinding: requested_name=requested_name, member_count=len(members), resource_classes=tuple(sorted(resource_classes)), + source_digest=source_digest, ) @@ -684,6 +775,7 @@ def cedar_documents( "uid": resource_uid, "attrs": { "planDigest": binding.plan_digest, + "sourceDigest": binding.source_digest, "contractDigest": binding.contract_digest, "contractVersion": binding.contract_version, "requestedName": binding.requested_name, @@ -702,6 +794,7 @@ def cedar_documents( "context": { "bindingVersion": "v0alpha1", "planDigest": binding.plan_digest, + "sourceDigest": binding.source_digest, "contractDigest": binding.contract_digest, }, } @@ -961,9 +1054,9 @@ def evaluate_cedar( raise InfrastructureError("private authorization snapshot could not be managed") from error -def authorize_plan(plan: object) -> tuple[dict[str, object], int]: +def authorize_plan(plan: object, source_digest: str) -> tuple[dict[str, object], int]: repo_root = Path(__file__).resolve().parent.parent - binding = plan_binding(plan) + binding = plan_binding(plan, source_digest) request, entities = cedar_documents(binding) authorization_digest, snapshot = authorization_snapshot(repo_root) verdict = evaluate_cedar(snapshot, request, entities, repo_root) @@ -976,6 +1069,7 @@ def authorize_plan(plan: object) -> tuple[dict[str, object], int]: "authorizationDigest": authorization_digest, "contractDigest": binding.contract_digest, "planDigest": binding.plan_digest, + "sourceDigest": binding.source_digest, }, "kind": "ExperimentAuthorizationDecision", "principal": { @@ -1020,6 +1114,29 @@ def write_decision(decision: object) -> None: def main(argv: list[str]) -> int: + directory_checking = len(argv) == 3 and argv[1] == "check-directory" + directory_authorizing = len(argv) == 3 and argv[1] == "authorize-directory" + if directory_checking or directory_authorizing: + try: + snapshot = read_directory_snapshot(argv[2]) + manifest = authored_manifest(snapshot) + plan = cue_plan(manifest) + if directory_checking: + plan_bytes = canonical_json(plan) + checked = { + "digest": f"sha256:{hashlib.sha256(plan_bytes).hexdigest()}", + "plan": plan, + "source": {"digest": snapshot.digest, "kind": "directory"}, + } + sys.stdout.buffer.write(canonical_json(checked) + b"\n") + return 0 + decision, result = authorize_plan(plan, snapshot.digest) + write_decision(decision) + return result + except InvalidManifest as error: + fail(str(error)) + except InfrastructureError as error: + infra(str(error)) checking = len(argv) == 3 and argv[1] == "check" authorizing = ( len(argv) == 4 and argv[1] == "authorize" and argv[2] == "install" @@ -1040,7 +1157,7 @@ def main(argv: list[str]) -> int: if checking: write_envelope(plan) return 0 - decision, result = authorize_plan(plan) + decision, result = authorize_plan(plan, "sha256:" + "0" * 64) write_decision(decision) return result except InvalidManifest as error: From b146298690da76c300ae8078a342fe23fd118b4a Mon Sep 17 00:00:00 2001 From: xormania <127287135+xormania@users.noreply.github.com> Date: Sat, 1 Aug 2026 22:56:58 -0400 Subject: [PATCH 003/158] test(experiment): reject unsafe authored files --- tests/experiment/directory-intake-cases.sh | 57 +++++++++++++++++++++- 1 file changed, 55 insertions(+), 2 deletions(-) diff --git a/tests/experiment/directory-intake-cases.sh b/tests/experiment/directory-intake-cases.sh index 1962d50..3906e51 100755 --- a/tests/experiment/directory-intake-cases.sh +++ b/tests/experiment/directory-intake-cases.sh @@ -34,6 +34,59 @@ else fail FMT-002 "sole-entry directory produces a source-bound checked candidate" fi +extra="$work/extra" +mkdir "$extra" +cp "$fixture/experiment.cue" "$extra/experiment.cue" +: > "$extra/unexpected" +capture "$agent_lab" experiment check "$extra" +if [ "$CAPTURE_RC" -eq 1 ] && [ ! -s "$work/stdout" ]; then + pass FMT-004 "an extra directory entry is stable invalid input" +else + fail FMT-004 "an extra directory entry is stable invalid input" +fi + +linked="$work/linked" +mkdir "$linked" +ln "$fixture/experiment.cue" "$linked/experiment.cue" +capture "$agent_lab" experiment check "$linked" +if [ "$CAPTURE_RC" -eq 1 ] && [ ! -s "$work/stdout" ]; then + pass FMT-005 "a multiply linked authored file is refused" +else + fail FMT-005 "a multiply linked authored file is refused" +fi + +executable="$work/executable" +mkdir "$executable" +cp "$fixture/experiment.cue" "$executable/experiment.cue" +chmod 700 "$executable/experiment.cue" +capture "$agent_lab" experiment check "$executable" +if [ "$CAPTURE_RC" -eq 1 ] && [ ! -s "$work/stdout" ]; then + pass FMT-006 "an executable authored file is refused" +else + fail FMT-006 "an executable authored file is refused" +fi + +expected_digest="$(python3 - "$fixture/experiment.cue" <<'PY' +from hashlib import sha256 +from pathlib import Path +import sys +data = Path(sys.argv[1]).read_bytes() +name = b"experiment.cue" +digest = sha256(b"agent-lab.experiment-tree.v1\0") +digest.update(len(name).to_bytes(4, "big")) +digest.update(name) +digest.update(len(data).to_bytes(8, "big")) +digest.update(data) +print("sha256:" + digest.hexdigest()) +PY +)" +capture "$agent_lab" experiment check "$fixture" +if [ "$CAPTURE_RC" -eq 0 ] && [ "$(jq -r '.source.digest' "$work/stdout")" = "$expected_digest" ]; then + pass FMT-007 "source identity uses the independently framed exact bytes" +else + fail FMT-007 "source identity uses the independently framed exact bytes" +fi + capture "$agent_lab" experiment check "$fixture" cp "$work/stdout" "$work/second" capture "$agent_lab" experiment check "$fixture" @@ -44,10 +97,10 @@ else fi expected="$work/expected" -printf '%s\n' FMT-001 FMT-002 FMT-003 > "$expected" +printf '%s\n' FMT-001 FMT-002 FMT-004 FMT-005 FMT-006 FMT-007 FMT-003 > "$expected" if ! cmp -s "$expected" "$observed"; then printf 'INFRA assertion identity drift\n' >&2 exit 125 fi -printf 'SUMMARY assertions=3 expected=3 failures=%s infra=0\n' "$failures" +printf 'SUMMARY assertions=7 expected=7 failures=%s infra=0\n' "$failures" [ "$failures" -eq 0 ] From dae298463fc8faa6273e855b25e182095e4da2d1 Mon Sep 17 00:00:00 2001 From: xormania <127287135+xormania@users.noreply.github.com> Date: Sat, 1 Aug 2026 22:57:21 -0400 Subject: [PATCH 004/158] feat(experiment): enforce authored file invariants --- scripts/experiment.py | 12 +++++++++++- tests/experiment/directory-intake-cases.sh | 3 ++- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/scripts/experiment.py b/scripts/experiment.py index de5eec8..6e15741 100644 --- a/scripts/experiment.py +++ b/scripts/experiment.py @@ -218,7 +218,17 @@ def read_directory_snapshot(path: str) -> SourceSnapshot: raise InfrastructureError("source directory cannot be listed") from error if before != ["experiment.cue"]: raise InvalidManifest("directory must contain only experiment.cue") - data = read_manifest_once(os.path.join(path, "experiment.cue")) + authored_path = os.path.join(path, "experiment.cue") + try: + authored_stat = os.lstat(authored_path) + except OSError as error: + raise InfrastructureError("authored file cannot be inspected") from error + if authored_stat.st_nlink != 1: + raise InvalidManifest("experiment.cue must have one link") + authored_mode = stat.S_IMODE(authored_stat.st_mode) + if authored_mode & 0o111 or authored_mode & 0o022: + raise InvalidManifest("experiment.cue has a suspicious mode") + data = read_manifest_once(authored_path) try: after = os.listdir(path) final_directory_stat = os.lstat(path) diff --git a/tests/experiment/directory-intake-cases.sh b/tests/experiment/directory-intake-cases.sh index 3906e51..6f4c048 100755 --- a/tests/experiment/directory-intake-cases.sh +++ b/tests/experiment/directory-intake-cases.sh @@ -47,7 +47,8 @@ fi linked="$work/linked" mkdir "$linked" -ln "$fixture/experiment.cue" "$linked/experiment.cue" +cp "$fixture/experiment.cue" "$work/hardlink-source" +ln "$work/hardlink-source" "$linked/experiment.cue" capture "$agent_lab" experiment check "$linked" if [ "$CAPTURE_RC" -eq 1 ] && [ ! -s "$work/stdout" ]; then pass FMT-005 "a multiply linked authored file is refused" From eb47d052496c75881cf09f966ca417fa8e8b58ac Mon Sep 17 00:00:00 2001 From: xormania <127287135+xormania@users.noreply.github.com> Date: Sat, 1 Aug 2026 22:58:14 -0400 Subject: [PATCH 005/158] test(experiment): harden directory intake boundaries --- tests/experiment/directory-intake-cases.sh | 65 +++++++++++++++++++++- 1 file changed, 63 insertions(+), 2 deletions(-) diff --git a/tests/experiment/directory-intake-cases.sh b/tests/experiment/directory-intake-cases.sh index 6f4c048..739ef83 100755 --- a/tests/experiment/directory-intake-cases.sh +++ b/tests/experiment/directory-intake-cases.sh @@ -19,6 +19,16 @@ capture() { AGENT_LAB_CUE_TOOL_DIR="${AGENT_LAB_CUE_TOOL_DIR:-$repo_root/.cache/dev/tools/cue}" \ "$@" > "$work/stdout" 2> "$work/stderr" || CAPTURE_RC=$? } +expect_invalid() { + local id="$1" directory="$2" detail="$3" + capture "$agent_lab" experiment check "$directory" + if [ "$CAPTURE_RC" -eq 1 ] && [ ! -s "$work/stdout" ] && + grep -Fq 'FAIL Experiment manifest' "$work/stderr"; then + pass "$id" "$detail" + else + fail "$id" "$detail" + fi +} if [ -x "$agent_lab" ]; then pass FMT-001 "repository agent-lab entrypoint exists" @@ -97,11 +107,62 @@ else fail FMT-003 "repeated directory checks are byte-identical" fi +symlinked="$work/symlinked" +mkdir "$symlinked" +ln -s "$fixture/experiment.cue" "$symlinked/experiment.cue" +expect_invalid FMT-008 "$symlinked" "an authored symlink is refused" + +mutable="$work/mutable" +mkdir "$mutable" +sed 's/@sha256:[a-f0-9]\{64\}/:latest/' "$fixture/experiment.cue" > "$mutable/experiment.cue" +expect_invalid SEL-001 "$mutable" "a mutable OCI reference is refused" + +unknown="$work/unknown" +mkdir "$unknown" +sed '/kind:/a\\\tunexpected: true' "$fixture/experiment.cue" > "$unknown/experiment.cue" +expect_invalid CUE-001 "$unknown" "an unknown authored field is refused" + +commented="$work/commented" +mkdir "$commented" +{ printf '// distinct source bytes\n'; cat "$fixture/experiment.cue"; } > "$commented/experiment.cue" +capture "$agent_lab" experiment check "$fixture" +cp "$work/stdout" "$work/base-candidate" +capture "$agent_lab" experiment check "$commented" +if [ "$CAPTURE_RC" -eq 0 ] && + [ "$(jq -cS '.plan' "$work/base-candidate")" = "$(jq -cS '.plan' "$work/stdout")" ] && + [ "$(jq -r '.source.digest' "$work/base-candidate")" != "$(jq -r '.source.digest' "$work/stdout")" ]; then + pass FMT-009 "non-semantic CUE comments change source but not plan identity" +else + fail FMT-009 "non-semantic CUE comments change source but not plan identity" +fi + +if python3 -I - "$repo_root/scripts/experiment.py" "$fixture" <<'PY' +from importlib.util import module_from_spec, spec_from_file_location +from pathlib import Path +import sys +spec = spec_from_file_location("experiment_mutant", sys.argv[1]) +assert spec is not None and spec.loader is not None +module = module_from_spec(spec) +sys.modules[spec.name] = module +spec.loader.exec_module(module) +original = module.read_directory_snapshot(sys.argv[2]).digest +module.SOURCE_DIGEST_DOMAIN = b"agent-lab.insecure-unframed\0" +mutated = module.read_directory_snapshot(sys.argv[2]).digest +assert original != mutated +PY +then + pass M-FMT-001 "source-domain mutation changes the independent identity" +else + fail M-FMT-001 "source-domain mutation changes the independent identity" +fi + expected="$work/expected" -printf '%s\n' FMT-001 FMT-002 FMT-004 FMT-005 FMT-006 FMT-007 FMT-003 > "$expected" +printf '%s\n' \ + FMT-001 FMT-002 FMT-004 FMT-005 FMT-006 FMT-007 FMT-003 \ + FMT-008 SEL-001 CUE-001 FMT-009 M-FMT-001 > "$expected" if ! cmp -s "$expected" "$observed"; then printf 'INFRA assertion identity drift\n' >&2 exit 125 fi -printf 'SUMMARY assertions=7 expected=7 failures=%s infra=0\n' "$failures" +printf 'SUMMARY assertions=12 expected=12 failures=%s infra=0\n' "$failures" [ "$failures" -eq 0 ] From 1d3b9369b0116a10f69008ddf6ffb61be390e26e Mon Sep 17 00:00:00 2001 From: xormania <127287135+xormania@users.noreply.github.com> Date: Sat, 1 Aug 2026 22:59:37 -0400 Subject: [PATCH 006/158] test(experiment): define bundled image resolution --- tests/experiment/directory-intake-cases.sh | 44 +++++++++++++++++++++- 1 file changed, 42 insertions(+), 2 deletions(-) diff --git a/tests/experiment/directory-intake-cases.sh b/tests/experiment/directory-intake-cases.sh index 739ef83..1e7ee19 100755 --- a/tests/experiment/directory-intake-cases.sh +++ b/tests/experiment/directory-intake-cases.sh @@ -156,13 +156,53 @@ else fail M-FMT-001 "source-domain mutation changes the independent identity" fi +if python3 -I - "$repo_root/scripts/experiment.py" <<'PY' +from importlib.util import module_from_spec, spec_from_file_location +from pathlib import Path +import sys +spec = spec_from_file_location("experiment_resolver", sys.argv[1]) +assert spec is not None and spec.loader is not None +module = module_from_spec(spec) +sys.modules[spec.name] = module +spec.loader.exec_module(module) +subject = "registry.example/lab/base@sha256:" + "b" * 64 +plan = { + "apiVersion": "agent-lab.request/v0alpha1", + "contract": {"digest": "sha256:" + "a" * 64, "name": "agent-lab.experiment", "version": "v0alpha1"}, + "kind": "RequestedExperimentPlan", + "metadata": {"requestedName": "resolver-test"}, + "spec": {"members": [{ + "command": [], "name": "one", "resourceClass": "small", + "requestedSelector": {"catalogName": "agent-lab.base"}, + }]}, +} +catalog = { + "apiVersion": "agent-lab.experiment-images/v0alpha1", + "entries": [{"name": "agent-lab.base", "subject": subject}], +} +resolved = module.resolve_plan(plan, Path("/nonexistent"), catalog) +image = resolved["spec"]["members"][0]["resolvedImage"] +assert image["origin"] == "agent-lab" +assert image["subject"] == subject +assert image["generation"] == 1 +assert image["entryDigest"].startswith("sha256:") +assert module.valid_catalog_name("vendor.image") +for invalid in ("Agent.image", "vendor.image.extra", "vendor_1.image", "a" * 32 + ".image", "vendor.é"): + assert not module.valid_catalog_name(invalid), invalid +PY +then + pass SEL-002 "trusted bundled resolution binds exact names, subjects, and entry identity" +else + fail SEL-002 "trusted bundled resolution binds exact names, subjects, and entry identity" +fi + expected="$work/expected" printf '%s\n' \ FMT-001 FMT-002 FMT-004 FMT-005 FMT-006 FMT-007 FMT-003 \ - FMT-008 SEL-001 CUE-001 FMT-009 M-FMT-001 > "$expected" + FMT-008 SEL-001 CUE-001 FMT-009 M-FMT-001 SEL-002 > "$expected" if ! cmp -s "$expected" "$observed"; then printf 'INFRA assertion identity drift\n' >&2 exit 125 fi -printf 'SUMMARY assertions=12 expected=12 failures=%s infra=0\n' "$failures" +printf 'SUMMARY assertions=13 expected=13 failures=%s infra=0\n' "$failures" [ "$failures" -eq 0 ] From 6d59d7412a3d6ac0759e20c97e453c065d0275c5 Mon Sep 17 00:00:00 2001 From: xormania <127287135+xormania@users.noreply.github.com> Date: Sat, 1 Aug 2026 22:59:45 -0400 Subject: [PATCH 007/158] feat(experiment): resolve bundled image selectors --- catalog/experiment-images/v0alpha1.json | 1 + contracts/experiment/v0alpha1/plan.cue | 8 +-- scripts/experiment.py | 75 ++++++++++++++++++++++++- 3 files changed, 76 insertions(+), 8 deletions(-) create mode 100644 catalog/experiment-images/v0alpha1.json diff --git a/catalog/experiment-images/v0alpha1.json b/catalog/experiment-images/v0alpha1.json new file mode 100644 index 0000000..00fc07a --- /dev/null +++ b/catalog/experiment-images/v0alpha1.json @@ -0,0 +1 @@ +{"apiVersion":"agent-lab.experiment-images/v0alpha1","entries":[]} diff --git a/contracts/experiment/v0alpha1/plan.cue b/contracts/experiment/v0alpha1/plan.cue index c9ac7fa..75ec386 100644 --- a/contracts/experiment/v0alpha1/plan.cue +++ b/contracts/experiment/v0alpha1/plan.cue @@ -24,12 +24,8 @@ contractDigest: string & =~"^sha256:[0-9a-f]{64}$" @tag(contractDigest) }) { name: member.name requestedSelector: member.image - resolvedImage: close({ - origin: "direct" - subject: member.image.digestRef - }) - command: member.command - resourceClass: member.resourceClass + command: member.command + resourceClass: member.resourceClass }] }) }) diff --git a/scripts/experiment.py b/scripts/experiment.py index 6e15741..b36ad1d 100644 --- a/scripts/experiment.py +++ b/scripts/experiment.py @@ -8,6 +8,7 @@ import math import os from pathlib import Path +import re import signal import stat import subprocess @@ -19,6 +20,9 @@ MAX_MANIFEST_BYTES = 262_144 SOURCE_DIGEST_DOMAIN = b"agent-lab.experiment-tree.v1\0" +BUNDLED_CATALOG_DOMAIN = b"agent-lab.experiment-image-catalog.v1\0" +BUNDLED_ENTRY_DOMAIN = b"agent-lab.experiment-image-entry.v1\0" +CATALOG_NAME_COMPONENT = re.compile(r"^[a-z][a-z0-9]*(?:-[a-z0-9]+)*$") MAX_CUE_OUTPUT_BYTES = 1_048_576 MAX_CONTRACT_FILE_BYTES = 1_048_576 MAX_HELPER_BYTES = 1_048_576 @@ -548,7 +552,6 @@ def expected_plan(manifest: object, contract_digest: str) -> dict[str, object]: "command": member.get("command", []), "name": member["name"], "requestedSelector": selector, - "resolvedImage": {"origin": "direct", "subject": selector["digestRef"]}, "resourceClass": member.get("resourceClass", "small"), }) members.sort(key=lambda member: str(member["name"])) @@ -568,6 +571,74 @@ def expected_plan(manifest: object, contract_digest: str) -> dict[str, object]: } +def valid_catalog_name(value: object) -> bool: + if not isinstance(value, str) or len(value.encode("utf-8")) > 63: + return False + parts = value.split(".") + return ( + len(parts) == 2 + and all(1 <= len(part.encode("ascii", "ignore")) <= 31 for part in parts) + and all(part.isascii() and CATALOG_NAME_COMPONENT.fullmatch(part) for part in parts) + ) + + +def digest_record(domain: bytes, value: object) -> str: + return "sha256:" + hashlib.sha256(domain + canonical_json(value)).hexdigest() + + +def bundled_catalog(repo_root: Path) -> tuple[dict[str, object], str]: + path = repo_root / "catalog/experiment-images/v0alpha1.json" + data = stable_file_bytes(path, MAX_CONTRACT_FILE_BYTES, "bundled image catalog") + value = strict_json(data, source="bundled image catalog") + if not isinstance(value, dict) or set(value) != {"apiVersion", "entries"}: + raise InfrastructureError("bundled image catalog has an unexpected shape") + if value["apiVersion"] != "agent-lab.experiment-images/v0alpha1" or not isinstance(value["entries"], list): + raise InfrastructureError("bundled image catalog has an unknown schema") + return value, digest_record(BUNDLED_CATALOG_DOMAIN, value) + + +def resolve_plan(plan: dict[str, object], repo_root: Path, catalog: dict[str, object] | None = None) -> dict[str, object]: + if catalog is None: + catalog, _ = bundled_catalog(repo_root) + entries = catalog.get("entries") + if not isinstance(entries, list): + raise InfrastructureError("bundled image catalog entries are malformed") + by_name: dict[str, dict[str, object]] = {} + for entry in entries: + if not isinstance(entry, dict) or set(entry) != {"name", "subject"}: + raise InfrastructureError("bundled image catalog entry is malformed") + name, subject = entry["name"], entry["subject"] + if not valid_catalog_name(name) or not isinstance(subject, str) or "@sha256:" not in subject: + raise InfrastructureError("bundled image catalog entry is invalid") + assert isinstance(name, str) + if not name.startswith("agent-lab.") or name in by_name: + raise InfrastructureError("bundled image catalog namespace is invalid") + by_name[name] = entry + resolved = json.loads(canonical_json(plan)) + members = resolved["spec"]["members"] + for member in members: + selector = member["requestedSelector"] + if set(selector) == {"digestRef"}: + member["resolvedImage"] = {"origin": "direct", "subject": selector["digestRef"]} + continue + if set(selector) != {"catalogName"} or not valid_catalog_name(selector["catalogName"]): + raise InvalidManifest("contains an invalid image selector") + name = selector["catalogName"] + if not name.startswith("agent-lab."): + raise InvalidManifest("local image name is not configured") + entry = by_name.get(name) + if entry is None: + raise InvalidManifest("references an unknown bundled image name") + entry_digest = digest_record(BUNDLED_ENTRY_DOMAIN, entry) + member["resolvedImage"] = { + "entryDigest": entry_digest, + "generation": 1, + "origin": "agent-lab", + "subject": entry["subject"], + } + return resolved + + def invoke_cue( manifest: object, contract_digest: str, @@ -678,7 +749,7 @@ def cue_plan(manifest: object) -> object: plan = parse_cue_plan(completed) if plan != expected_plan(manifest, contract_digest): raise InfrastructureError("pinned CUE plan violates its exact postcondition") - return plan + return resolve_plan(plan, repo_root) except OSError as error: raise InfrastructureError("private validation snapshot could not be managed") from error From 74c6bb6c5dd0a601cb98d014907767f080160022 Mon Sep 17 00:00:00 2001 From: xormania <127287135+xormania@users.noreply.github.com> Date: Sat, 1 Aug 2026 23:00:26 -0400 Subject: [PATCH 008/158] docs(experiment): publish authored format contract --- docs/architecture.md | 7 +++-- docs/experiments.md | 40 ++++++++++++++++++++++++ scripts/experiment | 74 ++------------------------------------------ 3 files changed, 46 insertions(+), 75 deletions(-) create mode 100644 docs/experiments.md mode change 100755 => 100644 scripts/experiment diff --git a/docs/architecture.md b/docs/architecture.md index 101238e..e0a7c5f 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -69,9 +69,10 @@ See [Development](development.md) and [development-agent configuration](agent-co ## Experiment request preflight -`scripts/experiment check [--] MANIFEST` validates the closed `agent-lab/v0alpha1` request with the +`scripts/agent-lab experiment check DIRECTORY` snapshots and validates the closed authored +`agent-lab/v0alpha1` request with the repository-pinned CUE contract and emits one canonical, digest-bound `RequestedExperimentPlan`. -`scripts/experiment authorize install [--] MANIFEST` reads the manifest once, derives that same plan +`scripts/agent-lab experiment authorize install DIRECTORY` reads the snapshot once, derives that same plan in-process, and asks the repository-pinned Cedar policy whether the fixed local compatibility principal may submit the exact plan digest. @@ -236,7 +237,7 @@ For formal assumptions and limits, read [Security](../SECURITY.md) and the | Squid and test service | `compose.egress.yaml`, `gateway/squid/` | | workload container | `compose.agent.yaml` and HOME overlays | | workload orchestration | `scripts/agent` | -| Experiment request planning and authorization | `scripts/experiment`, `contracts/experiment/`, `authorization/experiment/` | +| Experiment request planning and authorization | `scripts/agent-lab`, `scripts/experiment.py`, `contracts/experiment/`, `authorization/experiment/` | | config parsing and validation | `scripts/lib/config.sh` | | project and secret guards | `scripts/lib/guard.sh` | | recipe publication | `scripts/lib/allowlist.sh` | diff --git a/docs/experiments.md b/docs/experiments.md new file mode 100644 index 0000000..2158cde --- /dev/null +++ b/docs/experiments.md @@ -0,0 +1,40 @@ +# Experiments + +An Experiment is authored as data in a directory containing exactly one file, `experiment.cue`. +The file defines one concrete value named `experiment` in package `experiment`. Agent Lab snapshots +the exact bytes privately before evaluating them; extra entries, links, special files, suspicious +modes, changing sources, malformed CUE, and unknown schema fields are refused. + +```cue +package experiment + +experiment: { + apiVersion: "agent-lab/v0alpha1" + kind: "Experiment" + metadata: name: "example" + spec: members: [{ + name: "worker" + image: digestRef: "registry.example/team/worker@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + command: ["serve"] + }] +} +``` + +Check the artifact or preview its install authorization from the repository: + +```bash +./scripts/agent-lab experiment check ./my-experiment +./scripts/agent-lab experiment authorize install ./my-experiment +``` + +Both commands are previews. They create no durable Agent Lab state and do not invoke Docker or run +Experiment content. `authorize install` freshly checks the same held source and emits decision +evidence bound to its source, plan, contract, and authorization identities. The decision is not an +installation capability. + +Each member selects either an exact digest-pinned OCI reference with `digestRef` or a shared name +with `catalogName`. Shared names have exactly two bounded lowercase components, `.`. +The `agent-lab.*` namespace belongs to the release-owned bundled catalog; the catalog is initially +empty. Other namespaces are reserved for the operator-local catalog introduced by the local image +catalog work. A name is resolved to an immutable subject before authorization. Catalog membership +is naming only, not image presence, admission, safety, or runnable status. diff --git a/scripts/experiment b/scripts/experiment old mode 100755 new mode 100644 index 3d2b63b..7f1c42b --- a/scripts/experiment +++ b/scripts/experiment @@ -1,75 +1,5 @@ #!/usr/bin/env bash set -euo pipefail -script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" -helper="$script_dir/experiment.py" - -usage() { - printf '%s\n' \ - 'Usage: scripts/experiment check [--] MANIFEST' \ - 'Usage: scripts/experiment authorize install [--] MANIFEST' >&2 -} - -case "${1:-}" in - -h|--help) - [ "$#" -eq 1 ] || { - usage - exit 2 - } - usage - exit 0 - ;; - check) - shift - if [ "${1:-}" = -- ]; then - shift - elif [[ "${1:-}" == -* ]]; then - usage - exit 2 - fi - [ "$#" -eq 1 ] || { - usage - exit 2 - } - command_name=check - manifest=$1 - ;; - authorize) - shift - [ "${1:-}" = install ] || { - usage - exit 2 - } - shift - if [ "${1:-}" = -- ]; then - shift - elif [[ "${1:-}" == -* ]]; then - usage - exit 2 - fi - [ "$#" -eq 1 ] || { - usage - exit 2 - } - command_name=authorize - manifest=$1 - ;; - *) - usage - exit 2 - ;; -esac - -command -v python3 >/dev/null 2>&1 || { - printf 'INFRA Experiment validation requires python3\n' >&2 - exit 125 -} -[ -f "$helper" ] && [ ! -L "$helper" ] || { - printf 'INFRA Experiment validation helper is missing or unsafe\n' >&2 - exit 125 -} - -if [ "$command_name" = authorize ]; then - exec python3 -I "$helper" authorize install "$manifest" -fi -exec python3 -I "$helper" check "$manifest" +printf '%s\n' 'Usage: scripts/agent-lab experiment {check|authorize install} DIRECTORY' >&2 +exit 2 From 8b29cf0da069e2354230e67e96c51b62235a1c13 Mon Sep 17 00:00:00 2001 From: xormania <127287135+xormania@users.noreply.github.com> Date: Sat, 1 Aug 2026 23:00:33 -0400 Subject: [PATCH 009/158] fix(experiment): keep legacy boundary executable --- scripts/experiment | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 scripts/experiment diff --git a/scripts/experiment b/scripts/experiment old mode 100644 new mode 100755 From 7e5c93c6fb9d292570973f306c30d488648c0e38 Mon Sep 17 00:00:00 2001 From: xormania <127287135+xormania@users.noreply.github.com> Date: Sat, 1 Aug 2026 23:07:02 -0400 Subject: [PATCH 010/158] test(experiment): define local bundle manifest --- tests/dev/security-gate-cases.sh | 1 + tests/experiment/local-config-cases.sh | 17 ++++++++++++ tests/experiment/local-lifecycle-cases.sh | 7 +++++ .../fixtures/expected-runtime-files.txt | 13 +++++++++ tests/install/local-install-cases.sh | 27 +++++++++++++++++++ tests/security/fast.manifest | 1 + 6 files changed, 66 insertions(+) create mode 100755 tests/experiment/local-config-cases.sh create mode 100755 tests/experiment/local-lifecycle-cases.sh create mode 100644 tests/install/fixtures/expected-runtime-files.txt create mode 100755 tests/install/local-install-cases.sh diff --git a/tests/dev/security-gate-cases.sh b/tests/dev/security-gate-cases.sh index 2dfb9ce..6f124e2 100644 --- a/tests/dev/security-gate-cases.sh +++ b/tests/dev/security-gate-cases.sh @@ -230,6 +230,7 @@ guard-mount tests/guard/cases.sh SUMMARY failures=0 config-authority tests/agent/config-guard.sh SUMMARY failures=0 experiment-contract tests/experiment/contract-cases.sh EXPERIMENT CONTRACT PASS experiment-authorization tests/experiment/authorization-cases.sh EXPERIMENT AUTHORIZATION PASS +experiment-local-lifecycle tests/experiment/local-lifecycle-cases.sh EXPERIMENT LOCAL LIFECYCLE PASS config-matrix tests/agent/config-matrix.sh SUMMARY failures=0 allowlist-schema tests/agent/allowlist-cases.sh SUMMARY failures=0 image-volume-policy tests/agent/image-volume-policy-cases.sh SUMMARY failures=0 diff --git a/tests/experiment/local-config-cases.sh b/tests/experiment/local-config-cases.sh new file mode 100755 index 0000000..1c724f3 --- /dev/null +++ b/tests/experiment/local-config-cases.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash +set -euo pipefail + +repo_root="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." >/dev/null 2>&1 && pwd)" +work="$(mktemp -d)" +trap 'find "$work" -type f -delete 2>/dev/null || true; find "$work" -type l -delete 2>/dev/null || true; find "$work" -depth -type d -exec rmdir {} + 2>/dev/null || true' EXIT +rc=0 +"$repo_root/scripts/agent-lab" --home "$work/home" init > "$work/out" 2> "$work/err" || rc=$? +failures=0 +if [ "$rc" -eq 0 ] && [ -f "$work/home/home.json" ] && [ -f "$work/home/config.json" ]; then + printf 'PASS CFG-001 init creates the explicit private home\n' +else + printf 'FAIL CFG-001 init creates the explicit private home\n' + failures=1 +fi +printf 'SUMMARY assertions=1 expected=1 failures=%s infra=0\n' "$failures" +[ "$failures" -eq 0 ] diff --git a/tests/experiment/local-lifecycle-cases.sh b/tests/experiment/local-lifecycle-cases.sh new file mode 100755 index 0000000..de03863 --- /dev/null +++ b/tests/experiment/local-lifecycle-cases.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash +set -euo pipefail + +repo_root="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." >/dev/null 2>&1 && pwd)" +"$repo_root/tests/install/local-install-cases.sh" +"$repo_root/tests/experiment/local-config-cases.sh" +printf 'EXPERIMENT LOCAL LIFECYCLE PASS\n' diff --git a/tests/install/fixtures/expected-runtime-files.txt b/tests/install/fixtures/expected-runtime-files.txt new file mode 100644 index 0000000..1025d17 --- /dev/null +++ b/tests/install/fixtures/expected-runtime-files.txt @@ -0,0 +1,13 @@ +authorization/experiment/v0alpha1/operator.cedar +authorization/experiment/v0alpha1/schema.cedarschema +catalog/experiment-images/v0alpha1.json +contracts/experiment/v0alpha1/cue.mod/module.cue +contracts/experiment/v0alpha1/plan.cue +contracts/experiment/v0alpha1/schema.cue +scripts/agent-lab +scripts/agent-lab.py +scripts/dev/cedar-tool.py +scripts/dev/cue-tool.py +scripts/experiment.py +tools/cedar.lock +tools/cue.lock diff --git a/tests/install/local-install-cases.sh b/tests/install/local-install-cases.sh new file mode 100755 index 0000000..e6b753c --- /dev/null +++ b/tests/install/local-install-cases.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +set -euo pipefail + +repo_root="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." >/dev/null 2>&1 && pwd)" +expected="$repo_root/tests/install/fixtures/expected-runtime-files.txt" +manifest="$repo_root/packaging/agent-lab-local.manifest" +installer="$repo_root/scripts/install-local" +work="$(mktemp -d)" +trap 'find "$work" -type f -delete 2>/dev/null || true; find "$work" -type l -delete 2>/dev/null || true; find "$work" -depth -type d -exec rmdir {} + 2>/dev/null || true' EXIT +failures=0 +pass() { printf 'PASS %s %s\n' "$1" "$2"; } +fail() { printf 'FAIL %s %s\n' "$1" "$2"; failures=$((failures + 1)); } + +if [ -f "$manifest" ] && cmp -s "$expected" "$manifest"; then + pass PKG-001 "production runtime manifest matches the independent allowlist" +else + fail PKG-001 "production runtime manifest matches the independent allowlist" +fi + +if [ -x "$installer" ]; then + pass PKG-002 "local installer entrypoint exists" +else + fail PKG-002 "local installer entrypoint exists" +fi + +printf 'SUMMARY assertions=2 expected=2 failures=%s infra=0\n' "$failures" +[ "$failures" -eq 0 ] diff --git a/tests/security/fast.manifest b/tests/security/fast.manifest index 3d398ea..657b04e 100644 --- a/tests/security/fast.manifest +++ b/tests/security/fast.manifest @@ -52,6 +52,7 @@ suite guard-mount tests/guard/cases.sh SUMMARY failures=0 suite config-authority tests/agent/config-guard.sh SUMMARY failures=0 suite experiment-contract tests/experiment/contract-cases.sh EXPERIMENT CONTRACT PASS suite experiment-authorization tests/experiment/authorization-cases.sh EXPERIMENT AUTHORIZATION PASS +suite experiment-local-lifecycle tests/experiment/local-lifecycle-cases.sh EXPERIMENT LOCAL LIFECYCLE PASS suite config-matrix tests/agent/config-matrix.sh SUMMARY failures=0 suite allowlist-schema tests/agent/allowlist-cases.sh SUMMARY failures=0 suite image-volume-policy tests/agent/image-volume-policy-cases.sh SUMMARY failures=0 From 55adf88770988607e486a62370bb2549366884a0 Mon Sep 17 00:00:00 2001 From: xormania <127287135+xormania@users.noreply.github.com> Date: Sat, 1 Aug 2026 23:07:16 -0400 Subject: [PATCH 011/158] feat(experiment): define local runtime bundle --- packaging/agent-lab-local.manifest | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 packaging/agent-lab-local.manifest diff --git a/packaging/agent-lab-local.manifest b/packaging/agent-lab-local.manifest new file mode 100644 index 0000000..1025d17 --- /dev/null +++ b/packaging/agent-lab-local.manifest @@ -0,0 +1,13 @@ +authorization/experiment/v0alpha1/operator.cedar +authorization/experiment/v0alpha1/schema.cedarschema +catalog/experiment-images/v0alpha1.json +contracts/experiment/v0alpha1/cue.mod/module.cue +contracts/experiment/v0alpha1/plan.cue +contracts/experiment/v0alpha1/schema.cue +scripts/agent-lab +scripts/agent-lab.py +scripts/dev/cedar-tool.py +scripts/dev/cue-tool.py +scripts/experiment.py +tools/cedar.lock +tools/cue.lock From 058ec658d7a1268996eeb4c49f639a04a5307db6 Mon Sep 17 00:00:00 2001 From: xormania <127287135+xormania@users.noreply.github.com> Date: Sat, 1 Aug 2026 23:08:47 -0400 Subject: [PATCH 012/158] feat(experiment): expose local installer surface --- scripts/install-local | 5 +++++ 1 file changed, 5 insertions(+) create mode 100755 scripts/install-local diff --git a/scripts/install-local b/scripts/install-local new file mode 100755 index 0000000..4ac8d3b --- /dev/null +++ b/scripts/install-local @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +set -euo pipefail + +script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" +exec python3 -I "$script_dir/install-local.py" "$@" From 1d926420113ab229ad004e3943b5af21872b490e Mon Sep 17 00:00:00 2001 From: xormania <127287135+xormania@users.noreply.github.com> Date: Sat, 1 Aug 2026 23:09:16 -0400 Subject: [PATCH 013/158] test(experiment): require installed bundle independence --- tests/install/local-install-cases.sh | 31 +++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/tests/install/local-install-cases.sh b/tests/install/local-install-cases.sh index e6b753c..d090d03 100755 --- a/tests/install/local-install-cases.sh +++ b/tests/install/local-install-cases.sh @@ -23,5 +23,34 @@ else fail PKG-002 "local installer entrypoint exists" fi -printf 'SUMMARY assertions=2 expected=2 failures=%s infra=0\n' "$failures" +prefix="$work/prefix" +home="$work/agent-home" +install_rc=125 +version_rc=125 +if [ -f "$repo_root/scripts/install-local.py" ]; then + replica="$work/source-replica" + mkdir -p "$replica/packaging" "$replica/scripts" + while IFS= read -r name; do + mkdir -p "$replica/$(dirname -- "$name")" + cp "$repo_root/$name" "$replica/$name" + done < "$expected" + cp "$manifest" "$replica/packaging/agent-lab-local.manifest" + cp "$repo_root/scripts/install-local" "$repo_root/scripts/install-local.py" "$replica/scripts/" + chmod +x "$replica/scripts/install-local" "$replica/scripts/agent-lab" + install_rc=0 + "$replica/scripts/install-local" --prefix "$prefix" > "$work/install.out" 2> "$work/install.err" || install_rc=$? + mv "$replica" "$work/source-unavailable" + mkdir "$work/unrelated" + version_rc=0 + (cd "$work/unrelated" && env -i PATH=/usr/bin:/bin "$prefix/bin/agent-lab" --home "$home" version) \ + > "$work/version.out" 2> "$work/version.err" || version_rc=$? +fi +if [ "$install_rc" -eq 0 ] && [ "$version_rc" -eq 0 ] && + grep -Fxq 'agent-lab v0alpha1' "$work/version.out" && [ ! -s "$work/version.err" ]; then + pass PKG-003 "installed CLI runs after its isolated source replica is unavailable" +else + fail PKG-003 "installed CLI runs after its isolated source replica is unavailable" +fi + +printf 'SUMMARY assertions=3 expected=3 failures=%s infra=0\n' "$failures" [ "$failures" -eq 0 ] From a335f6a1c246bf5afc5f1dc4b96bfb0eadf7e87f Mon Sep 17 00:00:00 2001 From: xormania <127287135+xormania@users.noreply.github.com> Date: Sat, 1 Aug 2026 23:09:39 -0400 Subject: [PATCH 014/158] feat(experiment): install a user-local Agent Lab bundle --- scripts/agent-lab | 25 +------ scripts/agent-lab.py | 156 +++++++++++++++++++++++++++++++++++++++ scripts/install-local.py | 101 +++++++++++++++++++++++++ 3 files changed, 260 insertions(+), 22 deletions(-) create mode 100644 scripts/agent-lab.py create mode 100644 scripts/install-local.py diff --git a/scripts/agent-lab b/scripts/agent-lab index cc94d50..031da71 100755 --- a/scripts/agent-lab +++ b/scripts/agent-lab @@ -1,25 +1,6 @@ #!/usr/bin/env bash set -euo pipefail -script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" -helper="$script_dir/experiment.py" - -usage() { - printf '%s\n' \ - 'Usage: agent-lab experiment check DIRECTORY' \ - 'Usage: agent-lab experiment authorize install DIRECTORY' >&2 -} - -[ "${1:-}" = experiment ] || { usage; exit 2; } -shift -case "${1:-}" in - check) - [ "$#" -eq 2 ] || { usage; exit 2; } - exec python3 -I "$helper" check-directory "$2" - ;; - authorize) - [ "${2:-}" = install ] && [ "$#" -eq 3 ] || { usage; exit 2; } - exec python3 -I "$helper" authorize-directory "$3" - ;; - *) usage; exit 2 ;; -esac +script_path="$(readlink -f "${BASH_SOURCE[0]}")" +script_dir="$(cd -- "$(dirname -- "$script_path")" >/dev/null 2>&1 && pwd)" +exec python3 -I "$script_dir/agent-lab.py" "$@" diff --git a/scripts/agent-lab.py b/scripts/agent-lab.py new file mode 100644 index 0000000..bbb5353 --- /dev/null +++ b/scripts/agent-lab.py @@ -0,0 +1,156 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +from importlib.util import module_from_spec, spec_from_file_location +import hashlib +import json +import os +from pathlib import Path +import pwd +import re +import sys + +SAFE_COMPONENT = re.compile(r"^[a-z][a-z0-9-]{0,47}$") + + +def canonical(value: object) -> bytes: + return json.dumps(value, ensure_ascii=True, separators=(",", ":"), sort_keys=True).encode() + + +def effective_home(raw: str | None) -> Path: + selected = raw if raw is not None else os.environ.get("AGENT_LAB_HOME") + if selected is None: + selected = str(Path(pwd.getpwuid(os.getuid()).pw_dir) / ".agent-lab") + path = Path(selected) + if not selected or not path.is_absolute() or path == Path("/"): + raise ValueError("home must be an absolute non-root path") + return path + + +def config_value(components: dict[str, str]) -> dict[str, object]: + return {"apiVersion": "agent-lab.config/v0alpha1", "paths": components} + + +def init_home(home: Path, argv: list[str]) -> int: + components = {"experiments": "experiments", "images": "images", "cache": "cache", "state": "state"} + option_map = {"--experiments-dir": "experiments", "--images-dir": "images", "--cache-dir": "cache", "--state-dir": "state"} + while argv: + option = argv.pop(0) + if option not in option_map or not argv: + return 2 + components[option_map[option]] = argv.pop(0) + if len(set(components.values())) != 4 or any(not SAFE_COMPONENT.fullmatch(value) for value in components.values()): + print("FAIL Agent Lab configuration has unsafe data components", file=sys.stderr) + return 1 + config = config_value(components) + config_bytes = canonical(config) + b"\n" + receipt = { + "apiVersion": "agent-lab.home/v0alpha1", + "configDigest": "sha256:" + hashlib.sha256(canonical(config)).hexdigest(), + "paths": components, + } + receipt_bytes = canonical(receipt) + b"\n" + try: + home.mkdir(mode=0o700, parents=True, exist_ok=True) + os.chmod(home, 0o700) + existing = home / "home.json" + config_path = home / "config.json" + if existing.exists() or config_path.exists(): + if existing.read_bytes() == receipt_bytes and config_path.read_bytes() == config_bytes: + print("changed:false") + return 0 + print("FAIL Agent Lab home conflicts with requested configuration", file=sys.stderr) + return 1 + for key in ("experiments", "images"): + (home / components[key] / ".staging").mkdir(mode=0o700, parents=True) + (home / components["cache"] / "tools/cue").mkdir(mode=0o700, parents=True) + (home / components["cache"] / "tools/cedar").mkdir(mode=0o700, parents=True) + locks = home / components["state"] / "locks" + locks.mkdir(mode=0o700, parents=True) + for name in ("image-catalog.lock", "experiments.lock"): + descriptor = os.open(locks / name, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + os.close(descriptor) + for path, data in ((config_path, config_bytes), (existing, receipt_bytes)): + descriptor = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + os.write(descriptor, data) + os.close(descriptor) + except OSError: + print("INFRA Agent Lab home could not be initialized safely", file=sys.stderr) + return 125 + print("changed:true") + return 0 + + +def load_config(home: Path) -> tuple[dict[str, object], bytes] | None: + try: + raw = (home / "config.json").read_bytes() + except FileNotFoundError: + return None + try: + value = json.loads(raw.decode("utf-8")) + except (UnicodeError, json.JSONDecodeError): + raise RuntimeError("configuration is malformed") + if not isinstance(value, dict) or set(value) != {"apiVersion", "paths"} or value["apiVersion"] != "agent-lab.config/v0alpha1": + raise RuntimeError("configuration is not closed") + paths = value["paths"] + if not isinstance(paths, dict) or set(paths) != {"experiments", "images", "cache", "state"}: + raise RuntimeError("configuration paths are not closed") + if len(set(paths.values())) != 4 or any(not isinstance(item, str) or not SAFE_COMPONENT.fullmatch(item) for item in paths.values()): + raise RuntimeError("configuration paths are unsafe") + return value, canonical(value) + b"\n" + + +def experiment_module(): + path = Path(__file__).resolve().with_name("experiment.py") + spec = spec_from_file_location("agent_lab_experiment", path) + assert spec is not None and spec.loader is not None + module = module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +def main(argv: list[str]) -> int: + home_raw = None + if argv[:1] == ["--home"]: + if len(argv) < 2: + return 2 + home_raw = argv[1] + argv = argv[2:] + try: + home = effective_home(home_raw) + except ValueError as error: + print(f"FAIL Agent Lab {error}", file=sys.stderr) + return 1 + if argv == ["version"]: + print("agent-lab v0alpha1") + return 0 + if argv[:1] == ["init"]: + return init_home(home, argv[1:]) + if argv == ["config", "check"] or argv == ["config", "show"]: + try: + loaded = load_config(home) + except RuntimeError as error: + print(f"INFRA Agent Lab {error}", file=sys.stderr) + return 125 + if loaded is None: + print("FAIL Agent Lab home is not initialized", file=sys.stderr) + return 1 + if argv[-1] == "show": + sys.stdout.buffer.write(loaded[1]) + else: + print("valid:true") + return 0 + if argv[:2] == ["experiment", "check"] and len(argv) == 3: + os.environ.setdefault("AGENT_LAB_CUE_TOOL_DIR", str(home / "cache/tools/cue")) + return experiment_module().main(["experiment.py", "check-directory", argv[2]]) + if argv[:3] == ["experiment", "authorize", "install"] and len(argv) == 4: + os.environ.setdefault("AGENT_LAB_CUE_TOOL_DIR", str(home / "cache/tools/cue")) + os.environ.setdefault("AGENT_LAB_CEDAR_TOOL_DIR", str(home / "cache/tools/cedar")) + return experiment_module().main(["experiment.py", "authorize-directory", argv[3]]) + print("Usage: agent-lab [--home ABSOLUTE_HOME] {version|init|config|experiment}", file=sys.stderr) + return 2 + + +if __name__ == "__main__": + raise SystemExit(main(sys.argv[1:])) diff --git a/scripts/install-local.py b/scripts/install-local.py new file mode 100644 index 0000000..a3a9725 --- /dev/null +++ b/scripts/install-local.py @@ -0,0 +1,101 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import hashlib +import os +from pathlib import Path +import pwd +import shutil +import stat +import sys +import tempfile + + +def fail(message: str, code: int = 1) -> int: + print(f"FAIL local install {message}", file=sys.stderr) + return code + + +def prefix_from(argv: list[str]) -> Path: + if len(argv) == 2 and argv[0] == "--prefix" and argv[1]: + raw = argv[1] + elif not argv: + raw = os.environ.get("AGENT_LAB_PREFIX") or str(Path(pwd.getpwuid(os.getuid()).pw_dir) / ".local") + else: + raise ValueError("Usage: scripts/install-local [--prefix ABSOLUTE_PREFIX]") + path = Path(raw) + if not path.is_absolute() or path == Path("/"): + raise ValueError("prefix must be an absolute non-root path") + return path + + +def main(argv: list[str]) -> int: + try: + prefix = prefix_from(argv) + except ValueError as error: + print(error, file=sys.stderr) + return 2 + root = Path(__file__).resolve().parent.parent + manifest_path = root / "packaging/agent-lab-local.manifest" + try: + names = manifest_path.read_text(encoding="ascii").splitlines() + except OSError: + return fail("runtime manifest is unavailable", 125) + if names != sorted(set(names)) or not names: + return fail("runtime manifest is not canonical", 125) + digest = hashlib.sha256(b"agent-lab.local-bundle.v1\0") + sources: list[tuple[str, Path, bytes, int]] = [] + try: + for name in names: + relative = Path(name) + if relative.is_absolute() or ".." in relative.parts: + raise OSError("unsafe manifest path") + source = root / relative + metadata = source.lstat() + if not stat.S_ISREG(metadata.st_mode) or metadata.st_nlink != 1: + raise OSError("unsafe runtime source") + data = source.read_bytes() + final = source.stat() + if (metadata.st_dev, metadata.st_ino, metadata.st_size, metadata.st_mtime_ns) != ( + final.st_dev, final.st_ino, final.st_size, final.st_mtime_ns + ): + raise OSError("runtime source changed") + encoded = name.encode("ascii") + digest.update(len(encoded).to_bytes(4, "big")) + digest.update(encoded) + digest.update(len(data).to_bytes(8, "big")) + digest.update(data) + sources.append((name, source, data, stat.S_IMODE(metadata.st_mode))) + except (OSError, UnicodeError): + return fail("runtime source is unsafe or changing", 125) + release_id = digest.hexdigest() + releases = prefix / "lib/agent-lab/releases" + release = releases / release_id + try: + releases.mkdir(mode=0o700, parents=True, exist_ok=True) + if not release.exists(): + stage = Path(tempfile.mkdtemp(prefix=".agent-lab-release-", dir=releases)) + try: + for name, _source, data, mode in sources: + target = stage / name + target.parent.mkdir(mode=0o700, parents=True, exist_ok=True) + target.write_bytes(data) + target.chmod(0o755 if mode & 0o111 else 0o644) + os.rename(stage, release) + except BaseException: + shutil.rmtree(stage, ignore_errors=True) + raise + bindir = prefix / "bin" + bindir.mkdir(mode=0o700, parents=True, exist_ok=True) + temporary = bindir / f".agent-lab-{os.getpid()}" + target = Path("../lib/agent-lab/releases") / release_id / "scripts/agent-lab" + os.symlink(target, temporary) + os.replace(temporary, bindir / "agent-lab") + except OSError: + return fail("bundle could not be published", 125) + print(f"installed:{release_id}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main(sys.argv[1:])) From 65e160359019ced0d725d0eefc3e94f4ec99b1a8 Mon Sep 17 00:00:00 2001 From: xormania <127287135+xormania@users.noreply.github.com> Date: Sat, 1 Aug 2026 23:10:10 -0400 Subject: [PATCH 015/158] test(experiment): bind initialized home configuration --- tests/experiment/local-config-cases.sh | 42 +++++++++++++++++++++++++- 1 file changed, 41 insertions(+), 1 deletion(-) diff --git a/tests/experiment/local-config-cases.sh b/tests/experiment/local-config-cases.sh index 1c724f3..484cfbe 100755 --- a/tests/experiment/local-config-cases.sh +++ b/tests/experiment/local-config-cases.sh @@ -13,5 +13,45 @@ else printf 'FAIL CFG-001 init creates the explicit private home\n' failures=1 fi -printf 'SUMMARY assertions=1 expected=1 failures=%s infra=0\n' "$failures" + +before="$(find "$work/home" -printf '%P %m %y\n' | LC_ALL=C sort)" +rerun_rc=0 +"$repo_root/scripts/agent-lab" --home "$work/home" init > "$work/rerun.out" 2> "$work/rerun.err" || rerun_rc=$? +after="$(find "$work/home" -printf '%P %m %y\n' | LC_ALL=C sort)" +if [ "$rerun_rc" -eq 0 ] && [ "$before" = "$after" ] && grep -Fxq 'changed:false' "$work/rerun.out"; then + printf 'PASS CFG-002 exact init retry is idempotent\n' +else + printf 'FAIL CFG-002 exact init retry is idempotent\n' + failures=$((failures + 1)) +fi + +python3 - "$work/home/config.json" <<'PY' +from pathlib import Path +import json +import sys +path = Path(sys.argv[1]) +value = json.loads(path.read_text()) +value["paths"]["cache"] = "other-cache" +path.write_text(json.dumps(value, sort_keys=True, separators=(",", ":")) + "\n") +PY +drift_rc=0 +"$repo_root/scripts/agent-lab" --home "$work/home" config check > "$work/drift.out" 2> "$work/drift.err" || drift_rc=$? +if [ "$drift_rc" -eq 125 ] && [ ! -s "$work/drift.out" ]; then + printf 'PASS CFG-003 post-init configuration drift is infrastructure uncertainty\n' +else + printf 'FAIL CFG-003 post-init configuration drift is infrastructure uncertainty\n' + failures=$((failures + 1)) +fi + +absent="$work/absent" +absent_rc=0 +"$repo_root/scripts/agent-lab" --home "$absent" config check > "$work/absent.out" 2> "$work/absent.err" || absent_rc=$? +if [ "$absent_rc" -eq 1 ] && [ ! -e "$absent" ]; then + printf 'PASS CFG-004 config check does not initialize an absent home\n' +else + printf 'FAIL CFG-004 config check does not initialize an absent home\n' + failures=$((failures + 1)) +fi + +printf 'SUMMARY assertions=4 expected=4 failures=%s infra=0\n' "$failures" [ "$failures" -eq 0 ] From a581431fd03159832b22c88c5616c53a79524063 Mon Sep 17 00:00:00 2001 From: xormania <127287135+xormania@users.noreply.github.com> Date: Sat, 1 Aug 2026 23:10:30 -0400 Subject: [PATCH 016/158] feat(experiment): verify initialized home authority --- scripts/agent-lab.py | 32 ++++++++++++++++++++++++++++---- 1 file changed, 28 insertions(+), 4 deletions(-) diff --git a/scripts/agent-lab.py b/scripts/agent-lab.py index bbb5353..032b3e6 100644 --- a/scripts/agent-lab.py +++ b/scripts/agent-lab.py @@ -8,6 +8,7 @@ from pathlib import Path import pwd import re +import stat import sys SAFE_COMPONENT = re.compile(r"^[a-z][a-z0-9-]{0,47}$") @@ -82,13 +83,24 @@ def init_home(home: Path, argv: list[str]) -> int: def load_config(home: Path) -> tuple[dict[str, object], bytes] | None: + config_path = home / "config.json" + receipt_path = home / "home.json" try: - raw = (home / "config.json").read_bytes() + config_metadata = config_path.lstat() + receipt_metadata = receipt_path.lstat() except FileNotFoundError: - return None + if not config_path.exists() and not receipt_path.exists(): + return None + raise RuntimeError("home receipt and configuration are incomplete") + for metadata in (config_metadata, receipt_metadata): + if not stat.S_ISREG(metadata.st_mode) or metadata.st_nlink != 1 or stat.S_IMODE(metadata.st_mode) != 0o600: + raise RuntimeError("home authority files are unsafe") try: + raw = config_path.read_bytes() + receipt_raw = receipt_path.read_bytes() value = json.loads(raw.decode("utf-8")) - except (UnicodeError, json.JSONDecodeError): + receipt = json.loads(receipt_raw.decode("utf-8")) + except (OSError, UnicodeError, json.JSONDecodeError): raise RuntimeError("configuration is malformed") if not isinstance(value, dict) or set(value) != {"apiVersion", "paths"} or value["apiVersion"] != "agent-lab.config/v0alpha1": raise RuntimeError("configuration is not closed") @@ -97,7 +109,19 @@ def load_config(home: Path) -> tuple[dict[str, object], bytes] | None: raise RuntimeError("configuration paths are not closed") if len(set(paths.values())) != 4 or any(not isinstance(item, str) or not SAFE_COMPONENT.fullmatch(item) for item in paths.values()): raise RuntimeError("configuration paths are unsafe") - return value, canonical(value) + b"\n" + canonical_config = canonical(value) + b"\n" + if raw != canonical_config: + raise RuntimeError("configuration is not canonical") + if ( + not isinstance(receipt, dict) + or set(receipt) != {"apiVersion", "configDigest", "paths"} + or receipt["apiVersion"] != "agent-lab.home/v0alpha1" + or receipt["paths"] != paths + or receipt["configDigest"] != "sha256:" + hashlib.sha256(canonical(value)).hexdigest() + or receipt_raw != canonical(receipt) + b"\n" + ): + raise RuntimeError("configuration does not match the initialized home receipt") + return value, canonical_config def experiment_module(): From 942ffe6f4d43326574a55633f02812332ff11e1b Mon Sep 17 00:00:00 2001 From: xormania <127287135+xormania@users.noreply.github.com> Date: Sat, 1 Aug 2026 23:10:52 -0400 Subject: [PATCH 017/158] test(experiment): require installed Experiment checks --- tests/install/local-install-cases.sh | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/tests/install/local-install-cases.sh b/tests/install/local-install-cases.sh index d090d03..61065de 100755 --- a/tests/install/local-install-cases.sh +++ b/tests/install/local-install-cases.sh @@ -39,6 +39,7 @@ if [ -f "$repo_root/scripts/install-local.py" ]; then chmod +x "$replica/scripts/install-local" "$replica/scripts/agent-lab" install_rc=0 "$replica/scripts/install-local" --prefix "$prefix" > "$work/install.out" 2> "$work/install.err" || install_rc=$? + cp -R "$repo_root/tests/experiment/fixtures/directories/minimal" "$work/artifact" mv "$replica" "$work/source-unavailable" mkdir "$work/unrelated" version_rc=0 @@ -52,5 +53,21 @@ else fail PKG-003 "installed CLI runs after its isolated source replica is unavailable" fi -printf 'SUMMARY assertions=3 expected=3 failures=%s infra=0\n' "$failures" +check_rc=125 +if [ "$install_rc" -eq 0 ] && [ "$version_rc" -eq 0 ]; then + "$prefix/bin/agent-lab" --home "$home" init > "$work/init.out" 2> "$work/init.err" + cp -a "$repo_root/.cache/dev/tools/cue/." "$home/cache/tools/cue/" + cp -a "$repo_root/.cache/dev/tools/cedar/." "$home/cache/tools/cedar/" + check_rc=0 + (cd "$work/unrelated" && env -i PATH=/usr/bin:/bin "$prefix/bin/agent-lab" --home "$home" experiment check "$work/artifact") \ + > "$work/check.out" 2> "$work/check.err" || check_rc=$? +fi +if [ "$check_rc" -eq 0 ] && [ ! -s "$work/check.err" ] && + jq -e '.source.kind == "directory" and .plan.kind == "RequestedExperimentPlan"' "$work/check.out" >/dev/null 2>&1; then + pass PKG-004 "installed Experiment check is independent of the source checkout" +else + fail PKG-004 "installed Experiment check is independent of the source checkout" +fi + +printf 'SUMMARY assertions=4 expected=4 failures=%s infra=0\n' "$failures" [ "$failures" -eq 0 ] From 901d3fd7fb3e54e2ca9733b6564998207e18e7bf Mon Sep 17 00:00:00 2001 From: xormania <127287135+xormania@users.noreply.github.com> Date: Sat, 1 Aug 2026 23:11:39 -0400 Subject: [PATCH 018/158] feat(experiment): run installed checks from the bundle --- scripts/experiment.py | 2 -- tests/install/local-install-cases.sh | 1 + 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/scripts/experiment.py b/scripts/experiment.py index b36ad1d..e7222a6 100644 --- a/scripts/experiment.py +++ b/scripts/experiment.py @@ -700,10 +700,8 @@ def parse_cue_plan(completed: subprocess.CompletedProcess[bytes]) -> dict[str, o def cue_plan(manifest: object) -> object: repo_root = Path(__file__).resolve().parent.parent contract_root = repo_root / "contracts" / "experiment" / "v0alpha1" - cue_tool = repo_root / "scripts" / "dev" / "cue-tool" cue_helper = repo_root / "scripts" / "dev" / "cue-tool.py" required = ( - cue_tool, cue_helper, contract_root / "schema.cue", contract_root / "plan.cue", diff --git a/tests/install/local-install-cases.sh b/tests/install/local-install-cases.sh index 61065de..49c773d 100755 --- a/tests/install/local-install-cases.sh +++ b/tests/install/local-install-cases.sh @@ -67,6 +67,7 @@ if [ "$check_rc" -eq 0 ] && [ ! -s "$work/check.err" ] && pass PKG-004 "installed Experiment check is independent of the source checkout" else fail PKG-004 "installed Experiment check is independent of the source checkout" + printf 'PKG-004 rc=%s stderr=%s\n' "$check_rc" "$(tr '\n' ' ' < "$work/check.err" 2>/dev/null || true)" fi printf 'SUMMARY assertions=4 expected=4 failures=%s infra=0\n' "$failures" From f8081822403fca3c916df124d5086756c78acb5a Mon Sep 17 00:00:00 2001 From: xormania <127287135+xormania@users.noreply.github.com> Date: Sat, 1 Aug 2026 23:11:59 -0400 Subject: [PATCH 019/158] test(experiment): define explicit tool provisioning --- tests/experiment/local-config-cases.sh | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/tests/experiment/local-config-cases.sh b/tests/experiment/local-config-cases.sh index 484cfbe..d5b1903 100755 --- a/tests/experiment/local-config-cases.sh +++ b/tests/experiment/local-config-cases.sh @@ -53,5 +53,18 @@ else failures=$((failures + 1)) fi -printf 'SUMMARY assertions=4 expected=4 failures=%s infra=0\n' "$failures" +tool_home="$work/tool-home" +"$repo_root/scripts/agent-lab" --home "$tool_home" init >/dev/null +cp -a "$repo_root/.cache/dev/tools/cue/." "$tool_home/cache/tools/cue/" +cp -a "$repo_root/.cache/dev/tools/cedar/." "$tool_home/cache/tools/cedar/" +tools_rc=0 +"$repo_root/scripts/agent-lab" --home "$tool_home" tools provision > "$work/tools.out" 2> "$work/tools.err" || tools_rc=$? +if [ "$tools_rc" -eq 0 ] && grep -Fxq 'tools:ready' "$work/tools.out"; then + printf 'PASS TOOL-001 explicit provisioning verifies the pinned user tools\n' +else + printf 'FAIL TOOL-001 explicit provisioning verifies the pinned user tools\n' + failures=$((failures + 1)) +fi + +printf 'SUMMARY assertions=5 expected=5 failures=%s infra=0\n' "$failures" [ "$failures" -eq 0 ] From 47b49f60392aa054d354e26e252d87ab32a91694 Mon Sep 17 00:00:00 2001 From: xormania <127287135+xormania@users.noreply.github.com> Date: Sat, 1 Aug 2026 23:12:22 -0400 Subject: [PATCH 020/158] feat(experiment): provision pinned user tools explicitly --- scripts/agent-lab.py | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/scripts/agent-lab.py b/scripts/agent-lab.py index 032b3e6..e52f875 100644 --- a/scripts/agent-lab.py +++ b/scripts/agent-lab.py @@ -9,6 +9,7 @@ import pwd import re import stat +import subprocess import sys SAFE_COMPONENT = re.compile(r"^[a-z][a-z0-9-]{0,47}$") @@ -165,6 +166,35 @@ def main(argv: list[str]) -> int: else: print("valid:true") return 0 + if argv == ["tools", "provision"]: + try: + loaded = load_config(home) + except RuntimeError as error: + print(f"INFRA Agent Lab {error}", file=sys.stderr) + return 125 + if loaded is None: + print("FAIL Agent Lab home is not initialized", file=sys.stderr) + return 1 + paths = loaded[0]["paths"] + assert isinstance(paths, dict) + cache = home / str(paths["cache"]) / "tools" + root = Path(__file__).resolve().parent.parent + environment = {"PATH": "/usr/bin:/bin", "LANG": "C", "LC_ALL": "C"} + for name in ("cue", "cedar"): + environment[f"AGENT_LAB_{name.upper()}_TOOL_DIR"] = str(cache / name) + completed = subprocess.run( + [sys.executable, "-I", str(root / f"scripts/dev/{name}-tool.py"), "provision"], + env=environment, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + timeout=120, + check=False, + ) + if completed.returncode != 0: + sys.stderr.buffer.write(completed.stderr) + return 125 + print("tools:ready") + return 0 if argv[:2] == ["experiment", "check"] and len(argv) == 3: os.environ.setdefault("AGENT_LAB_CUE_TOOL_DIR", str(home / "cache/tools/cue")) return experiment_module().main(["experiment.py", "check-directory", argv[2]]) From 304b4ed1a527e798eb005dda48512de13817648b Mon Sep 17 00:00:00 2001 From: xormania <127287135+xormania@users.noreply.github.com> Date: Sat, 1 Aug 2026 23:12:37 -0400 Subject: [PATCH 021/158] test(experiment): reject symlinked install prefixes --- tests/install/local-install-cases.sh | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/tests/install/local-install-cases.sh b/tests/install/local-install-cases.sh index 49c773d..29711d2 100755 --- a/tests/install/local-install-cases.sh +++ b/tests/install/local-install-cases.sh @@ -70,5 +70,15 @@ else printf 'PKG-004 rc=%s stderr=%s\n' "$check_rc" "$(tr '\n' ' ' < "$work/check.err" 2>/dev/null || true)" fi -printf 'SUMMARY assertions=4 expected=4 failures=%s infra=0\n' "$failures" +mkdir "$work/symlink-target" +ln -s "$work/symlink-target" "$work/symlink-prefix" +symlink_rc=0 +"$installer" --prefix "$work/symlink-prefix" > "$work/symlink.out" 2> "$work/symlink.err" || symlink_rc=$? +if [ "$symlink_rc" -eq 125 ] && [ -z "$(find "$work/symlink-target" -mindepth 1 -print -quit)" ]; then + pass PKG-005 "installer refuses a symlinked prefix before writes" +else + fail PKG-005 "installer refuses a symlinked prefix before writes" +fi + +printf 'SUMMARY assertions=5 expected=5 failures=%s infra=0\n' "$failures" [ "$failures" -eq 0 ] From 109e5d1becd5daabf49d44380dfab543247e25c3 Mon Sep 17 00:00:00 2001 From: xormania <127287135+xormania@users.noreply.github.com> Date: Sat, 1 Aug 2026 23:12:51 -0400 Subject: [PATCH 022/158] feat(experiment): contain local installation paths --- scripts/install-local.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/scripts/install-local.py b/scripts/install-local.py index a3a9725..dfa15f4 100644 --- a/scripts/install-local.py +++ b/scripts/install-local.py @@ -29,12 +29,28 @@ def prefix_from(argv: list[str]) -> Path: return path +def reject_unsafe_existing_path(path: Path) -> None: + current = Path(path.anchor) + for component in path.parts[1:]: + current /= component + try: + metadata = current.lstat() + except FileNotFoundError: + return + if stat.S_ISLNK(metadata.st_mode) or not stat.S_ISDIR(metadata.st_mode): + raise OSError("unsafe prefix component") + + def main(argv: list[str]) -> int: try: prefix = prefix_from(argv) except ValueError as error: print(error, file=sys.stderr) return 2 + try: + reject_unsafe_existing_path(prefix) + except OSError: + return fail("prefix contains an unsafe existing component", 125) root = Path(__file__).resolve().parent.parent manifest_path = root / "packaging/agent-lab-local.manifest" try: From d86e2e41d9646b8659f9eaaa00bdd5fd46861837 Mon Sep 17 00:00:00 2001 From: xormania <127287135+xormania@users.noreply.github.com> Date: Sat, 1 Aug 2026 23:13:36 -0400 Subject: [PATCH 023/158] test(experiment): lock local lifecycle routing --- docs/experiments.md | 4 +++ docs/installation.md | 35 +++++++++++++++++++++ tests/experiment/aggregate-harness-cases.sh | 21 +++++++++++++ tests/experiment/contract-cases.sh | 1 + 4 files changed, 61 insertions(+) create mode 100644 docs/installation.md create mode 100755 tests/experiment/aggregate-harness-cases.sh diff --git a/docs/experiments.md b/docs/experiments.md index 2158cde..5417f37 100644 --- a/docs/experiments.md +++ b/docs/experiments.md @@ -32,6 +32,10 @@ Experiment content. `authorize install` freshly checks the same held source and evidence bound to its source, plan, contract, and authorization identities. The decision is not an installation capability. +The same commands work from a local installation after `agent-lab init` and explicit +`agent-lab tools provision`. Installed execution verifies and uses its release bundle and the +effective home's pinned tool cache; it does not depend on a source checkout. + Each member selects either an exact digest-pinned OCI reference with `digestRef` or a shared name with `catalogName`. Shared names have exactly two bounded lowercase components, `.`. The `agent-lab.*` namespace belongs to the release-owned bundled catalog; the catalog is initially diff --git a/docs/installation.md b/docs/installation.md new file mode 100644 index 0000000..e475203 --- /dev/null +++ b/docs/installation.md @@ -0,0 +1,35 @@ +# Local installation + +Install the current verified Agent Lab program bundle for one user: + +```bash +./scripts/install-local +# or +./scripts/install-local --prefix /absolute/private/prefix +``` + +The default prefix is the account-database home plus `.local`; ambient `HOME` is not authority. +`--prefix` takes precedence over `AGENT_LAB_PREFIX`. Installation copies only the closed runtime +manifest into a content-addressed release and atomically publishes `/bin/agent-lab`. It does +not use sudo, edit shell profiles, initialize data, or download tools. Add the prefix's `bin` +directory to `PATH` yourself if desired. + +Initialize a separate private Agent Lab home: + +```bash +agent-lab --home /absolute/private/home init +agent-lab --home /absolute/private/home config check +agent-lab --home /absolute/private/home config show +agent-lab --home /absolute/private/home tools provision +``` + +The default mutable home is the account-database home plus `.agent-lab`; `--home` takes precedence +over `AGENT_LAB_HOME`. Ambient `HOME` is ignored. The first `init` may choose distinct safe +single-component names with `--experiments-dir`, `--images-dir`, `--cache-dir`, and `--state-dir`. +Those choices are frozen by `home.json`; later drift or conflicting initialization is refused. + +`tools provision` is the only foundation command allowed to acquire the pinned CUE and Cedar +binaries. Normal commands never download them automatically. Program releases, Experiment data, +image-catalog state, tool cache, and locks remain in separate guarded trees. Exact reinstall and +exact init retry are idempotent; this version does not implement release garbage collection or +in-place home-layout migration. diff --git a/tests/experiment/aggregate-harness-cases.sh b/tests/experiment/aggregate-harness-cases.sh new file mode 100755 index 0000000..1a162d7 --- /dev/null +++ b/tests/experiment/aggregate-harness-cases.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash +set -euo pipefail + +repo_root="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." >/dev/null 2>&1 && pwd)" +lifecycle="$repo_root/tests/experiment/local-lifecycle-cases.sh" +failures=0 +if [ "$(grep -Fxc '"$repo_root/tests/install/local-install-cases.sh"' "$lifecycle")" -eq 1 ] && + [ "$(grep -Fxc '"$repo_root/tests/experiment/local-config-cases.sh"' "$lifecycle")" -eq 1 ]; then + printf 'PASS AGG-001 lifecycle subcases are routed exactly once in order\n' +else + printf 'FAIL AGG-001 lifecycle subcases are routed exactly once in order\n' + failures=1 +fi +if [ "$(tail -1 "$lifecycle")" = "printf 'EXPERIMENT LOCAL LIFECYCLE PASS\\n'" ]; then + printf 'PASS AGG-002 stable completion follows every subcase\n' +else + printf 'FAIL AGG-002 stable completion follows every subcase\n' + failures=$((failures + 1)) +fi +printf 'SUMMARY assertions=2 expected=2 failures=%s infra=0\n' "$failures" +[ "$failures" -eq 0 ] diff --git a/tests/experiment/contract-cases.sh b/tests/experiment/contract-cases.sh index 74b64b4..3b2f55a 100755 --- a/tests/experiment/contract-cases.sh +++ b/tests/experiment/contract-cases.sh @@ -5,4 +5,5 @@ repo_root="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." >/dev/null 2>&1 && subcase="$repo_root/tests/experiment/directory-intake-cases.sh" [ -x "$subcase" ] || { printf 'INFRA directory intake subcase is missing\n' >&2; exit 125; } "$subcase" +"$repo_root/tests/experiment/aggregate-harness-cases.sh" printf 'EXPERIMENT CONTRACT PASS\n' From 3d10179ddc1d4732500374a1e2ca2b1f013cbdb0 Mon Sep 17 00:00:00 2001 From: xormania <127287135+xormania@users.noreply.github.com> Date: Sat, 1 Aug 2026 23:46:08 -0400 Subject: [PATCH 024/158] test(experiment): define local image catalog lifecycle --- tests/experiment/aggregate-harness-cases.sh | 3 +- tests/experiment/local-image-catalog-cases.sh | 62 +++++++++++++++++++ tests/experiment/local-lifecycle-cases.sh | 1 + 3 files changed, 65 insertions(+), 1 deletion(-) create mode 100755 tests/experiment/local-image-catalog-cases.sh diff --git a/tests/experiment/aggregate-harness-cases.sh b/tests/experiment/aggregate-harness-cases.sh index 1a162d7..bf03a23 100755 --- a/tests/experiment/aggregate-harness-cases.sh +++ b/tests/experiment/aggregate-harness-cases.sh @@ -5,7 +5,8 @@ repo_root="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." >/dev/null 2>&1 && lifecycle="$repo_root/tests/experiment/local-lifecycle-cases.sh" failures=0 if [ "$(grep -Fxc '"$repo_root/tests/install/local-install-cases.sh"' "$lifecycle")" -eq 1 ] && - [ "$(grep -Fxc '"$repo_root/tests/experiment/local-config-cases.sh"' "$lifecycle")" -eq 1 ]; then + [ "$(grep -Fxc '"$repo_root/tests/experiment/local-config-cases.sh"' "$lifecycle")" -eq 1 ] && + [ "$(grep -Fxc '"$repo_root/tests/experiment/local-image-catalog-cases.sh"' "$lifecycle")" -eq 1 ]; then printf 'PASS AGG-001 lifecycle subcases are routed exactly once in order\n' else printf 'FAIL AGG-001 lifecycle subcases are routed exactly once in order\n' diff --git a/tests/experiment/local-image-catalog-cases.sh b/tests/experiment/local-image-catalog-cases.sh new file mode 100755 index 0000000..c46677b --- /dev/null +++ b/tests/experiment/local-image-catalog-cases.sh @@ -0,0 +1,62 @@ +#!/usr/bin/env bash +set -euo pipefail + +repo_root="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." >/dev/null 2>&1 && pwd)" +work="$(mktemp -d)" +trap 'find "$work" -type f -delete 2>/dev/null || true; find "$work" -type l -delete 2>/dev/null || true; find "$work" -depth -type d -exec rmdir {} + 2>/dev/null || true' EXIT +home="$work/home" +agent_lab="$repo_root/scripts/agent-lab" +"$agent_lab" --home "$home" init >/dev/null +subject="registry.example/operator/worker@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" +other="registry.example/operator/other@sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" +failures=0 +capture() { RC=0; "$@" > "$work/out" 2> "$work/err" || RC=$?; } +pass() { printf 'PASS %s %s\n' "$1" "$2"; } +fail() { printf 'FAIL %s %s\n' "$1" "$2"; failures=$((failures + 1)); } + +capture "$agent_lab" --home "$home" image add vendor.worker "$subject" +if [ "$RC" -eq 0 ] && jq -e '.changed == true and .generation == 1 and (.entryDigest | startswith("sha256:"))' "$work/out" >/dev/null 2>&1; then + pass CAT-001 "first add publishes a generation-one digest binding" + entry="$(jq -r '.entryDigest' "$work/out")" +else + fail CAT-001 "first add publishes a generation-one digest binding" + entry="sha256:$(printf '0%.0s' {1..64})" +fi + +capture "$agent_lab" --home "$home" image add vendor.worker "$subject" +if [ "$RC" -eq 0 ] && jq -e --arg entry "$entry" '.changed == false and .entryDigest == $entry' "$work/out" >/dev/null 2>&1; then + pass CAT-002 "same-subject add is idempotent" +else + fail CAT-002 "same-subject add is idempotent" +fi + +capture "$agent_lab" --home "$home" image add vendor.worker "$other" +if [ "$RC" -eq 1 ] && [ ! -s "$work/out" ]; then + pass CAT-003 "different-subject add never overwrites" +else + fail CAT-003 "different-subject add never overwrites" +fi + +capture "$agent_lab" --home "$home" image remove vendor.worker --expect "$entry" +if [ "$RC" -eq 0 ] && jq -e '.changed == true and .generation == 2 and .state == "removed"' "$work/out" >/dev/null 2>&1; then + pass CAT-004 "exact CAS removal publishes a generation-two tombstone" +else + fail CAT-004 "exact CAS removal publishes a generation-two tombstone" +fi + +capture "$agent_lab" --home "$home" image add agent-lab.worker "$subject" +if [ "$RC" -eq 1 ] && [ ! -s "$work/out" ]; then + pass CAT-005 "release-owned names cannot be claimed locally" +else + fail CAT-005 "release-owned names cannot be claimed locally" +fi + +capture "$agent_lab" --home "$home" image list --all +if [ "$RC" -eq 0 ] && jq -e '.[0].name == "vendor.worker" and .[0].state == "removed"' "$work/out" >/dev/null 2>&1; then + pass CAT-006 "list all reports the immutable tombstone" +else + fail CAT-006 "list all reports the immutable tombstone" +fi + +printf 'SUMMARY assertions=6 expected=6 failures=%s infra=0\n' "$failures" +[ "$failures" -eq 0 ] diff --git a/tests/experiment/local-lifecycle-cases.sh b/tests/experiment/local-lifecycle-cases.sh index de03863..a190ebb 100755 --- a/tests/experiment/local-lifecycle-cases.sh +++ b/tests/experiment/local-lifecycle-cases.sh @@ -4,4 +4,5 @@ set -euo pipefail repo_root="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." >/dev/null 2>&1 && pwd)" "$repo_root/tests/install/local-install-cases.sh" "$repo_root/tests/experiment/local-config-cases.sh" +"$repo_root/tests/experiment/local-image-catalog-cases.sh" printf 'EXPERIMENT LOCAL LIFECYCLE PASS\n' From 788fc6a8c36ba3a1ae76847b9bf798087586ea8a Mon Sep 17 00:00:00 2001 From: xormania <127287135+xormania@users.noreply.github.com> Date: Sat, 1 Aug 2026 23:46:52 -0400 Subject: [PATCH 025/158] feat(experiment): add the shared local image catalog --- scripts/agent-lab.py | 151 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 151 insertions(+) diff --git a/scripts/agent-lab.py b/scripts/agent-lab.py index e52f875..1a27162 100644 --- a/scripts/agent-lab.py +++ b/scripts/agent-lab.py @@ -2,6 +2,7 @@ from __future__ import annotations from importlib.util import module_from_spec, spec_from_file_location +import fcntl import hashlib import json import os @@ -13,12 +14,27 @@ import sys SAFE_COMPONENT = re.compile(r"^[a-z][a-z0-9-]{0,47}$") +IMAGE_COMPONENT = re.compile(r"^[a-z][a-z0-9]*(?:-[a-z0-9]+)*$") +OCI_SUBJECT = re.compile(r"^[a-z0-9][a-z0-9./_-]*@sha256:[0-9a-f]{64}$") def canonical(value: object) -> bytes: return json.dumps(value, ensure_ascii=True, separators=(",", ":"), sort_keys=True).encode() +def record_digest(domain: bytes, value: object) -> str: + return "sha256:" + hashlib.sha256(domain + canonical(value)).hexdigest() + + +def image_name(value: str) -> bool: + parts = value.split(".") + return ( + len(value.encode("utf-8")) <= 63 + and len(parts) == 2 + and all(part.isascii() and 1 <= len(part) <= 31 and IMAGE_COMPONENT.fullmatch(part) for part in parts) + ) + + def effective_home(raw: str | None) -> Path: selected = raw if raw is not None else os.environ.get("AGENT_LAB_HOME") if selected is None: @@ -135,6 +151,137 @@ def experiment_module(): return module +def atomic_json(path: Path, value: object) -> None: + data = canonical(value) + b"\n" + temporary = path.with_name(f".{path.name}.{os.getpid()}") + descriptor = os.open(temporary, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + try: + os.write(descriptor, data) + os.fsync(descriptor) + finally: + os.close(descriptor) + os.replace(temporary, path) + + +def catalog_paths(home: Path, loaded: tuple[dict[str, object], bytes]) -> tuple[Path, Path]: + paths = loaded[0]["paths"] + assert isinstance(paths, dict) + return home / str(paths["images"]) / "catalog", home / str(paths["state"]) / "locks/image-catalog.lock" + + +def load_catalog(root: Path) -> dict[str, object]: + current = root / "current.json" + if not root.exists(): + return {"revision": 0, "previous": None, "records": {}} + try: + pointer = json.loads(current.read_text(encoding="utf-8")) + digest = pointer["snapshotDigest"] + if not isinstance(digest, str) or not digest.startswith("sha256:"): + raise ValueError + snapshot_path = root / "snapshots" / f"{digest[7:]}.json" + snapshot = json.loads(snapshot_path.read_text(encoding="utf-8")) + except (OSError, KeyError, ValueError, json.JSONDecodeError) as error: + raise RuntimeError("image catalog is malformed") from error + if record_digest(b"agent-lab.local-image-snapshot.v1\0", snapshot) != digest: + raise RuntimeError("image catalog snapshot digest mismatch") + if not isinstance(snapshot, dict) or set(snapshot) != {"revision", "previous", "records"} or not isinstance(snapshot["records"], dict): + raise RuntimeError("image catalog snapshot is not closed") + return snapshot + + +def write_catalog(root: Path, snapshot: dict[str, object], entry: dict[str, object]) -> tuple[str, str]: + entries = root / "entries" + snapshots = root / "snapshots" + entries.mkdir(mode=0o700, parents=True, exist_ok=True) + snapshots.mkdir(mode=0o700, parents=True, exist_ok=True) + entry_digest = record_digest(b"agent-lab.local-image-entry.v1\0", entry) + entry_path = entries / f"{entry_digest[7:]}.json" + if not entry_path.exists(): + atomic_json(entry_path, entry) + snapshot_digest = record_digest(b"agent-lab.local-image-snapshot.v1\0", snapshot) + snapshot_path = snapshots / f"{snapshot_digest[7:]}.json" + if not snapshot_path.exists(): + atomic_json(snapshot_path, snapshot) + atomic_json(root / "current.json", {"snapshotDigest": snapshot_digest}) + return entry_digest, snapshot_digest + + +def image_command(home: Path, argv: list[str]) -> int: + try: + loaded = load_config(home) + except RuntimeError as error: + print(f"INFRA Agent Lab {error}", file=sys.stderr) + return 125 + if loaded is None: + print("FAIL Agent Lab home is not initialized", file=sys.stderr) + return 1 + root, lock_path = catalog_paths(home, loaded) + try: + lock = open(lock_path, "r+b", buffering=0) + fcntl.flock(lock.fileno(), fcntl.LOCK_EX) + snapshot = load_catalog(root) + except (OSError, RuntimeError) as error: + print(f"INFRA Agent Lab {error}", file=sys.stderr) + return 125 + try: + records = snapshot["records"] + assert isinstance(records, dict) + if argv[:1] == ["add"] and len(argv) == 3: + name, subject = argv[1:] + if not image_name(name) or name.startswith("agent-lab.") or not OCI_SUBJECT.fullmatch(subject): + print("FAIL image mapping is invalid or reserved", file=sys.stderr) + return 1 + prior = records.get(name) + if prior is not None: + if prior["state"] == "active" and prior["subject"] == subject: + print(canonical({"changed": False, "entryDigest": prior["entryDigest"], "generation": 1}).decode()) + return 0 + print("FAIL image name already exists or is tombstoned", file=sys.stderr) + return 1 + entry = {"generation": 1, "name": name, "previousEntryDigest": None, "state": "active", "subject": subject} + entry_digest = record_digest(b"agent-lab.local-image-entry.v1\0", entry) + record = {**entry, "entryDigest": entry_digest} + new_records = {**records, name: record} + next_snapshot = {"previous": record_digest(b"agent-lab.local-image-snapshot.v1\0", snapshot) if snapshot["revision"] else None, "records": new_records, "revision": int(snapshot["revision"]) + 1} + write_catalog(root, next_snapshot, entry) + print(canonical({"changed": True, "entryDigest": entry_digest, "generation": 1}).decode()) + return 0 + if argv[:1] == ["remove"] and len(argv) == 4 and argv[2] == "--expect": + name, expected = argv[1], argv[3] + prior = records.get(name) + if prior is None: + print("FAIL image name is unknown", file=sys.stderr) + return 1 + if prior["state"] == "removed" and prior["previousEntryDigest"] == expected: + print(canonical({"changed": False, "entryDigest": prior["entryDigest"], "generation": 2, "state": "removed"}).decode()) + return 0 + if prior["state"] != "active" or prior["entryDigest"] != expected: + print("FAIL image remove compare-and-swap conflict", file=sys.stderr) + return 1 + entry = {"generation": 2, "name": name, "previousEntryDigest": expected, "state": "removed", "subject": prior["subject"]} + entry_digest = record_digest(b"agent-lab.local-image-entry.v1\0", entry) + record = {**entry, "entryDigest": entry_digest} + next_snapshot = {"previous": record_digest(b"agent-lab.local-image-snapshot.v1\0", snapshot), "records": {**records, name: record}, "revision": int(snapshot["revision"]) + 1} + write_catalog(root, next_snapshot, entry) + print(canonical({"changed": True, "entryDigest": entry_digest, "generation": 2, "state": "removed"}).decode()) + return 0 + if argv[:1] == ["list"] and (len(argv) == 1 or argv == ["list", "--all"]): + include_all = len(argv) == 2 + values = [records[name] for name in sorted(records) if include_all or records[name]["state"] == "active"] + print(canonical(values).decode()) + return 0 + if argv[:1] == ["inspect"] and len(argv) == 2: + record = records.get(argv[1]) + if record is None: + print("FAIL image name is unknown", file=sys.stderr) + return 1 + print(canonical(record).decode()) + return 0 + return 2 + finally: + lock.close() + + def main(argv: list[str]) -> int: home_raw = None if argv[:1] == ["--home"]: @@ -196,12 +343,16 @@ def main(argv: list[str]) -> int: print("tools:ready") return 0 if argv[:2] == ["experiment", "check"] and len(argv) == 3: + os.environ["AGENT_LAB_HOME"] = str(home) os.environ.setdefault("AGENT_LAB_CUE_TOOL_DIR", str(home / "cache/tools/cue")) return experiment_module().main(["experiment.py", "check-directory", argv[2]]) if argv[:3] == ["experiment", "authorize", "install"] and len(argv) == 4: + os.environ["AGENT_LAB_HOME"] = str(home) os.environ.setdefault("AGENT_LAB_CUE_TOOL_DIR", str(home / "cache/tools/cue")) os.environ.setdefault("AGENT_LAB_CEDAR_TOOL_DIR", str(home / "cache/tools/cedar")) return experiment_module().main(["experiment.py", "authorize-directory", argv[3]]) + if argv[:1] == ["image"]: + return image_command(home, argv[1:]) print("Usage: agent-lab [--home ABSOLUTE_HOME] {version|init|config|experiment}", file=sys.stderr) return 2 From 5710a92c6b6296eb9219823ea4ea1d240f229742 Mon Sep 17 00:00:00 2001 From: xormania <127287135+xormania@users.noreply.github.com> Date: Sat, 1 Aug 2026 23:47:12 -0400 Subject: [PATCH 026/158] test(experiment): bind local catalog resolution --- tests/experiment/local-image-catalog-cases.sh | 21 ++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/tests/experiment/local-image-catalog-cases.sh b/tests/experiment/local-image-catalog-cases.sh index c46677b..61eee26 100755 --- a/tests/experiment/local-image-catalog-cases.sh +++ b/tests/experiment/local-image-catalog-cases.sh @@ -37,6 +37,25 @@ else fail CAT-003 "different-subject add never overwrites" fi +cp -a "$repo_root/.cache/dev/tools/cue/." "$home/cache/tools/cue/" +artifact="$work/catalog-artifact" +mkdir "$artifact" +sed 's#image: digestRef: "[^"]*"#image: catalogName: "vendor.worker"#' \ + "$repo_root/tests/experiment/fixtures/directories/minimal/experiment.cue" > "$artifact/experiment.cue" +capture "$agent_lab" --home "$home" experiment check "$artifact" +if [ "$RC" -eq 0 ] && jq -e --arg entry "$entry" --arg subject "$subject" ' + .plan.spec.members[0].resolvedImage == { + entryDigest: $entry, + generation: 1, + origin: "local", + subject: $subject + } +' "$work/out" >/dev/null 2>&1; then + pass RES-001 "Experiment resolution binds the active local entry and subject" +else + fail RES-001 "Experiment resolution binds the active local entry and subject" +fi + capture "$agent_lab" --home "$home" image remove vendor.worker --expect "$entry" if [ "$RC" -eq 0 ] && jq -e '.changed == true and .generation == 2 and .state == "removed"' "$work/out" >/dev/null 2>&1; then pass CAT-004 "exact CAS removal publishes a generation-two tombstone" @@ -58,5 +77,5 @@ else fail CAT-006 "list all reports the immutable tombstone" fi -printf 'SUMMARY assertions=6 expected=6 failures=%s infra=0\n' "$failures" +printf 'SUMMARY assertions=7 expected=7 failures=%s infra=0\n' "$failures" [ "$failures" -eq 0 ] From 5aaaf959ef6926ff89b3d7c7deb42bec9d3efe75 Mon Sep 17 00:00:00 2001 From: xormania <127287135+xormania@users.noreply.github.com> Date: Sat, 1 Aug 2026 23:48:23 -0400 Subject: [PATCH 027/158] feat(experiment): resolve active local image bindings --- scripts/experiment.py | 98 +++++++++++++------ tests/experiment/local-image-catalog-cases.sh | 1 + 2 files changed, 70 insertions(+), 29 deletions(-) diff --git a/scripts/experiment.py b/scripts/experiment.py index e7222a6..db124ef 100644 --- a/scripts/experiment.py +++ b/scripts/experiment.py @@ -546,7 +546,7 @@ def expected_plan(manifest: object, contract_digest: str) -> dict[str, object]: members = [] for member in raw_members: selector = member["image"] - if not isinstance(selector, dict) or set(selector) != {"digestRef"}: + if not isinstance(selector, dict) or set(selector) not in ({"digestRef"}, {"catalogName"}): raise KeyError("unresolved selector") members.append({ "command": member.get("command", []), @@ -597,23 +597,51 @@ def bundled_catalog(repo_root: Path) -> tuple[dict[str, object], str]: return value, digest_record(BUNDLED_CATALOG_DOMAIN, value) +def local_catalog_entry(home: Path, name: str) -> dict[str, object]: + try: + config = strict_json((home / "config.json").read_bytes(), source="local config") + assert isinstance(config, dict) + images = config["paths"]["images"] + root = home / images / "catalog" + pointer = strict_json((root / "current.json").read_bytes(), source="local catalog pointer") + assert isinstance(pointer, dict) and set(pointer) == {"snapshotDigest"} + snapshot_digest = pointer["snapshotDigest"] + assert isinstance(snapshot_digest, str) and is_sha256(snapshot_digest) + snapshot = strict_json( + (root / "snapshots" / f"{snapshot_digest[7:]}.json").read_bytes(), + source="local catalog snapshot", + ) + if digest_record(b"agent-lab.local-image-snapshot.v1\0", snapshot) != snapshot_digest: + raise ValueError("snapshot digest") + assert isinstance(snapshot, dict) + record = snapshot["records"].get(name) + if record is None or record["state"] != "active": + raise InvalidManifest("references an unknown or removed local image name") + assert isinstance(record, dict) + return record + except InvalidManifest: + raise + except (AssertionError, KeyError, OSError, TypeError, ValueError) as error: + raise InfrastructureError("local image catalog cannot be verified") from error + + def resolve_plan(plan: dict[str, object], repo_root: Path, catalog: dict[str, object] | None = None) -> dict[str, object]: - if catalog is None: - catalog, _ = bundled_catalog(repo_root) - entries = catalog.get("entries") - if not isinstance(entries, list): - raise InfrastructureError("bundled image catalog entries are malformed") - by_name: dict[str, dict[str, object]] = {} - for entry in entries: - if not isinstance(entry, dict) or set(entry) != {"name", "subject"}: - raise InfrastructureError("bundled image catalog entry is malformed") - name, subject = entry["name"], entry["subject"] - if not valid_catalog_name(name) or not isinstance(subject, str) or "@sha256:" not in subject: - raise InfrastructureError("bundled image catalog entry is invalid") - assert isinstance(name, str) - if not name.startswith("agent-lab.") or name in by_name: - raise InfrastructureError("bundled image catalog namespace is invalid") - by_name[name] = entry + by_name: dict[str, dict[str, object]] | None = None + if catalog is not None: + entries = catalog.get("entries") + if not isinstance(entries, list): + raise InfrastructureError("bundled image catalog entries are malformed") + by_name = {} + for entry in entries: + if not isinstance(entry, dict) or set(entry) != {"name", "subject"}: + raise InfrastructureError("bundled image catalog entry is malformed") + name, subject = entry["name"], entry["subject"] + if not valid_catalog_name(name) or not isinstance(subject, str) or "@sha256:" not in subject: + raise InfrastructureError("bundled image catalog entry is invalid") + assert isinstance(name, str) + if not name.startswith("agent-lab.") or name in by_name: + raise InfrastructureError("bundled image catalog namespace is invalid") + by_name[name] = entry resolved = json.loads(canonical_json(plan)) members = resolved["spec"]["members"] for member in members: @@ -624,18 +652,30 @@ def resolve_plan(plan: dict[str, object], repo_root: Path, catalog: dict[str, ob if set(selector) != {"catalogName"} or not valid_catalog_name(selector["catalogName"]): raise InvalidManifest("contains an invalid image selector") name = selector["catalogName"] - if not name.startswith("agent-lab."): - raise InvalidManifest("local image name is not configured") - entry = by_name.get(name) - if entry is None: - raise InvalidManifest("references an unknown bundled image name") - entry_digest = digest_record(BUNDLED_ENTRY_DOMAIN, entry) - member["resolvedImage"] = { - "entryDigest": entry_digest, - "generation": 1, - "origin": "agent-lab", - "subject": entry["subject"], - } + if name.startswith("agent-lab."): + if by_name is None: + loaded_catalog, _ = bundled_catalog(repo_root) + return resolve_plan(plan, repo_root, loaded_catalog) + entry = by_name.get(name) + if entry is None: + raise InvalidManifest("references an unknown bundled image name") + member["resolvedImage"] = { + "entryDigest": digest_record(BUNDLED_ENTRY_DOMAIN, entry), + "generation": 1, + "origin": "agent-lab", + "subject": entry["subject"], + } + else: + raw_home = os.environ.get("AGENT_LAB_HOME") + if not raw_home: + raise InvalidManifest("local image name requires an initialized Agent Lab home") + record = local_catalog_entry(Path(raw_home), name) + member["resolvedImage"] = { + "entryDigest": record["entryDigest"], + "generation": record["generation"], + "origin": "local", + "subject": record["subject"], + } return resolved diff --git a/tests/experiment/local-image-catalog-cases.sh b/tests/experiment/local-image-catalog-cases.sh index 61eee26..254db97 100755 --- a/tests/experiment/local-image-catalog-cases.sh +++ b/tests/experiment/local-image-catalog-cases.sh @@ -54,6 +54,7 @@ if [ "$RC" -eq 0 ] && jq -e --arg entry "$entry" --arg subject "$subject" ' pass RES-001 "Experiment resolution binds the active local entry and subject" else fail RES-001 "Experiment resolution binds the active local entry and subject" + printf 'RES-001 rc=%s stderr=%s\n' "$RC" "$(tr '\n' ' ' < "$work/err" 2>/dev/null || true)" fi capture "$agent_lab" --home "$home" image remove vendor.worker --expect "$entry" From ed56b3e51048acc83f1e8a88b5336e6bee094eb3 Mon Sep 17 00:00:00 2001 From: xormania <127287135+xormania@users.noreply.github.com> Date: Sat, 1 Aug 2026 23:48:51 -0400 Subject: [PATCH 028/158] test(experiment): harden catalog tombstone recovery --- tests/experiment/local-image-catalog-cases.sh | 24 ++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/tests/experiment/local-image-catalog-cases.sh b/tests/experiment/local-image-catalog-cases.sh index 254db97..d10d982 100755 --- a/tests/experiment/local-image-catalog-cases.sh +++ b/tests/experiment/local-image-catalog-cases.sh @@ -64,6 +64,20 @@ else fail CAT-004 "exact CAS removal publishes a generation-two tombstone" fi +capture "$agent_lab" --home "$home" image remove vendor.worker --expect "$entry" +if [ "$RC" -eq 0 ] && jq -e '.changed == false and .generation == 2' "$work/out" >/dev/null 2>&1; then + pass CAT-007 "lost-response removal retry is idempotent with the original token" +else + fail CAT-007 "lost-response removal retry is idempotent with the original token" +fi + +capture "$agent_lab" --home "$home" image add vendor.worker "$subject" +if [ "$RC" -eq 1 ] && [ ! -s "$work/out" ]; then + pass CAT-008 "a tombstoned v0 name cannot be reused" +else + fail CAT-008 "a tombstoned v0 name cannot be reused" +fi + capture "$agent_lab" --home "$home" image add agent-lab.worker "$subject" if [ "$RC" -eq 1 ] && [ ! -s "$work/out" ]; then pass CAT-005 "release-owned names cannot be claimed locally" @@ -78,5 +92,13 @@ else fail CAT-006 "list all reports the immutable tombstone" fi -printf 'SUMMARY assertions=7 expected=7 failures=%s infra=0\n' "$failures" +printf '{"snapshotDigest":"sha256:%064d"}\n' 0 > "$home/images/catalog/current.json" +capture "$agent_lab" --home "$home" image list +if [ "$RC" -eq 125 ] && [ ! -s "$work/out" ]; then + pass CAT-009 "corrupt catalog authority is infrastructure uncertainty, never empty" +else + fail CAT-009 "corrupt catalog authority is infrastructure uncertainty, never empty" +fi + +printf 'SUMMARY assertions=10 expected=10 failures=%s infra=0\n' "$failures" [ "$failures" -eq 0 ] From d2a34b5fdefa31fca85dd94e0fa7c74bbf6aadf7 Mon Sep 17 00:00:00 2001 From: xormania <127287135+xormania@users.noreply.github.com> Date: Sat, 1 Aug 2026 23:49:01 -0400 Subject: [PATCH 029/158] docs(experiment): describe local image mappings --- docs/experiments.md | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/docs/experiments.md b/docs/experiments.md index 5417f37..2546fc6 100644 --- a/docs/experiments.md +++ b/docs/experiments.md @@ -39,6 +39,17 @@ effective home's pinned tool cache; it does not depend on a source checkout. Each member selects either an exact digest-pinned OCI reference with `digestRef` or a shared name with `catalogName`. Shared names have exactly two bounded lowercase components, `.`. The `agent-lab.*` namespace belongs to the release-owned bundled catalog; the catalog is initially -empty. Other namespaces are reserved for the operator-local catalog introduced by the local image -catalog work. A name is resolved to an immutable subject before authorization. Catalog membership +empty. Other valid namespaces belong to the operator-local catalog shared by every Experiment using +the same effective home: + +```bash +agent-lab image add vendor.image registry.example/team/image@sha256:<64 lowercase hex> +agent-lab image inspect vendor.image +agent-lab image list [--all] +agent-lab image remove vendor.image --expect sha256: +``` + +Add records a mapping only. Same-subject add is idempotent; a different subject never overwrites. +Remove uses the active entry digest as a compare-and-swap token, creates a generation-two tombstone, +and makes the name non-reusable in v0. A name is resolved to an immutable subject before authorization. Catalog membership is naming only, not image presence, admission, safety, or runnable status. From cee3200d6189e7509fa9105542b09f2c2de49a7b Mon Sep 17 00:00:00 2001 From: xormania <127287135+xormania@users.noreply.github.com> Date: Sun, 2 Aug 2026 00:08:10 -0400 Subject: [PATCH 030/158] test(experiment): define catalog authority boundaries --- tests/experiment/catalog-resolution-cases.sh | 306 +++++++++++ tests/experiment/local-image-catalog-cases.sh | 154 +++--- tests/image/catalog-cases.sh | 313 +++++++++++ tests/image/catalog-state-cases.py | 495 ++++++++++++++++++ 4 files changed, 1176 insertions(+), 92 deletions(-) create mode 100755 tests/experiment/catalog-resolution-cases.sh create mode 100755 tests/image/catalog-cases.sh create mode 100755 tests/image/catalog-state-cases.py diff --git a/tests/experiment/catalog-resolution-cases.sh b/tests/experiment/catalog-resolution-cases.sh new file mode 100755 index 0000000..adccf35 --- /dev/null +++ b/tests/experiment/catalog-resolution-cases.sh @@ -0,0 +1,306 @@ +#!/usr/bin/env bash +set -u -o pipefail + +repo_root="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." >/dev/null 2>&1 && pwd)" +agent_lab="$repo_root/scripts/agent-lab" +work="$(mktemp -d)" +trap 'find "$work" -type f -delete 2>/dev/null || true; find "$work" -type l -delete 2>/dev/null || true; find "$work" -depth -type d -exec rmdir {} + 2>/dev/null || true' EXIT + +if [ ! -x "$agent_lab" ] || ! command -v jq >/dev/null 2>&1 || [ ! -d "$repo_root/.cache/dev/tools/cue" ]; then + printf 'INFRA catalog resolution prerequisites are unavailable\n' >&2 + exit 125 +fi + +failures=0 +observed="$work/observed" +: > "$observed" +pass() { printf 'PASS %s %s\n' "$1" "$2"; printf '%s\n' "$1" >> "$observed"; } +fail() { printf 'FAIL %s %s\n' "$1" "$2"; printf '%s\n' "$1" >> "$observed"; failures=$((failures + 1)); } +capture() { CAPTURE_RC=0; "$@" > "$work/stdout" 2> "$work/stderr" || CAPTURE_RC=$?; } +subject_a="registry.example/operator/worker@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" +subject_b="registry.example/operator/other@sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" + +init_home() { + local home="$1" + capture "$agent_lab" --home "$home" init + if [ "$CAPTURE_RC" -ne 0 ]; then + printf 'INFRA temporary Agent Lab home initialization failed: %s\n' "$(tr '\n' ' ' < "$work/stderr")" >&2 + exit 125 + fi + cp -a "$repo_root/.cache/dev/tools/cue/." "$home/cache/tools/cue/" +} + +copy_cedar() { + local home="$1" + if [ ! -d "$repo_root/.cache/dev/tools/cedar" ]; then + printf 'INFRA pinned Cedar test cache is unavailable\n' >&2 + exit 125 + fi + cp -a "$repo_root/.cache/dev/tools/cedar/." "$home/cache/tools/cedar/" +} + +write_artifact() { + local directory="$1" + local experiment_name="$2" + local selector_key="$3" + local selector_value="$4" + mkdir -p "$directory" + printf '%s\n' \ + 'package experiment' \ + '' \ + 'experiment: {' \ + ' apiVersion: "agent-lab/v0alpha1"' \ + ' kind: "Experiment"' \ + " metadata: name: \"$experiment_name\"" \ + ' spec: members: [{' \ + ' name: "worker"' \ + " image: $selector_key: \"$selector_value\"" \ + ' }]' \ + '}' > "$directory/experiment.cue" +} + +write_two_member_artifact() { + local directory="$1" + mkdir -p "$directory" + printf '%s\n' \ + 'package experiment' \ + '' \ + 'experiment: {' \ + ' apiVersion: "agent-lab/v0alpha1"' \ + ' kind: "Experiment"' \ + ' metadata: name: "two-members"' \ + ' spec: members: [{' \ + ' name: "first"' \ + ' image: catalogName: "vendor.worker"' \ + ' }, {' \ + ' name: "second"' \ + ' image: catalogName: "vendor.second"' \ + ' }]' \ + '}' > "$directory/experiment.cue" +} + +active_home="$work/active-home" +init_home "$active_home" +capture "$agent_lab" --home "$active_home" image add vendor.worker "$subject_a" +if [ "$CAPTURE_RC" -ne 0 ]; then + printf 'INFRA active local mapping setup failed\n' >&2 + exit 125 +fi +entry_a="$(jq -r '.entryDigest' "$work/stdout")" +artifact="$work/active-artifact" +write_artifact "$artifact" local-active catalogName vendor.worker +capture "$agent_lab" --home "$active_home" experiment check "$artifact" +active_rc="$CAPTURE_RC" +cp "$work/stdout" "$work/active-check.json" +if [ "$active_rc" -eq 0 ] && jq -e --arg entry "$entry_a" --arg subject "$subject_a" ' + .plan.spec.members[0].resolvedImage == { + entryDigest: $entry, + generation: 1, + origin: "local", + subject: $subject + } +' "$work/active-check.json" >/dev/null 2>&1; then + pass RES-ENTRY-001 "active local resolution binds exact entry identity and immutable subject" +else + fail RES-ENTRY-001 "active local resolution binds exact entry identity and immutable subject" +fi + +snapshot_digest="$(jq -r '.snapshotDigest' "$active_home/images/catalog/current.json" 2>/dev/null)" +snapshot_revision="$(jq -r '.revision' "$active_home/images/catalog/snapshots/${snapshot_digest#sha256:}.json" 2>/dev/null)" +if [ "$active_rc" -eq 0 ] && jq -e --arg digest "$snapshot_digest" --argjson revision "$snapshot_revision" ' + .catalog.local == {revision: $revision, snapshotDigest: $digest} +' "$work/active-check.json" >/dev/null 2>&1; then + pass RES-SNAP-001 "checked evidence records the exact held local snapshot revision and digest" +else + fail RES-SNAP-001 "checked evidence records the exact held local snapshot revision and digest" +fi + +first_plan_digest="$(jq -r '.digest // empty' "$work/active-check.json" 2>/dev/null)" +capture "$agent_lab" --home "$active_home" image add vendor.unrelated "$subject_b" +capture "$agent_lab" --home "$active_home" experiment check "$artifact" +cp "$work/stdout" "$work/unrelated-check.json" +second_snapshot_digest="$(jq -r '.snapshotDigest' "$active_home/images/catalog/current.json" 2>/dev/null)" +if [ "$CAPTURE_RC" -eq 0 ] && + [ "$first_plan_digest" = "$(jq -r '.digest // empty' "$work/unrelated-check.json" 2>/dev/null)" ] && + [ "$snapshot_digest" != "$second_snapshot_digest" ] && + jq -e --arg digest "$second_snapshot_digest" '.catalog.local.snapshotDigest == $digest' "$work/unrelated-check.json" >/dev/null 2>&1; then + pass RES-SNAP-002 "unrelated catalog mutation changes snapshot evidence but not selected-entry plan identity" +else + fail RES-SNAP-002 "unrelated catalog mutation changes snapshot evidence but not selected-entry plan identity" +fi + +other_home="$work/other-home" +init_home "$other_home" +capture "$agent_lab" --home "$other_home" image add vendor.worker "$subject_b" +entry_b="$(jq -r '.entryDigest // empty' "$work/stdout" 2>/dev/null)" +capture "$agent_lab" --home "$other_home" experiment check "$artifact" +if [ "$CAPTURE_RC" -eq 0 ] && [ "$entry_a" != "$entry_b" ] && + [ "$first_plan_digest" != "$(jq -r '.digest // empty' "$work/stdout" 2>/dev/null)" ]; then + pass RES-ENTRY-002 "substituting the selected entry changes plan identity" +else + fail RES-ENTRY-002 "substituting the selected entry changes plan identity" +fi + +unknown_artifact="$work/unknown-artifact" +write_artifact "$unknown_artifact" local-unknown catalogName vendor.unknown +capture "$agent_lab" --home "$active_home" experiment check "$unknown_artifact" +unknown_rc="$CAPTURE_RC" +removed_home="$work/removed-home" +init_home "$removed_home" +capture "$agent_lab" --home "$removed_home" image add vendor.worker "$subject_a" +removed_entry="$(jq -r '.entryDigest // empty' "$work/stdout" 2>/dev/null)" +capture "$agent_lab" --home "$removed_home" image remove vendor.worker --expect "$removed_entry" +capture "$agent_lab" --home "$removed_home" experiment check "$artifact" +removed_rc="$CAPTURE_RC" +if [ "$unknown_rc" -eq 1 ] && [ "$removed_rc" -eq 1 ]; then + pass RES-ENTRY-003 "unknown and tombstoned local names are stable invalid input" +else + fail RES-ENTRY-003 "unknown and tombstoned local names are stable invalid input" +fi + +isolation_home="$work/isolation-home" +init_home "$isolation_home" +mkdir "$isolation_home/images/catalog" +printf '{"snapshotDigest":"sha256:%064d"}\n' 0 > "$isolation_home/images/catalog/current.json" +direct_artifact="$work/direct-artifact" +write_artifact "$direct_artifact" direct-isolated digestRef "$subject_a" +capture "$agent_lab" --home "$isolation_home" experiment check "$direct_artifact" +direct_rc="$CAPTURE_RC" +direct_has_local="$(jq -r 'has("catalog") and (.catalog | has("local"))' "$work/stdout" 2>/dev/null || printf invalid)" +bundled_artifact="$work/bundled-artifact" +write_artifact "$bundled_artifact" bundled-isolated catalogName agent-lab.unknown +capture "$agent_lab" --home "$isolation_home" experiment check "$bundled_artifact" +bundled_rc="$CAPTURE_RC" +if [ "$direct_rc" -eq 0 ] && [ "$direct_has_local" = false ] && [ "$bundled_rc" -eq 1 ]; then + pass RES-ISOLATE-001 "corrupt local state cannot block or contaminate direct and bundled selectors" +else + fail RES-ISOLATE-001 "corrupt local state cannot block or contaminate direct and bundled selectors" +fi + +history_home="$work/history-home" +init_home "$history_home" +capture "$agent_lab" --home "$history_home" image add vendor.worker "$subject_a" +mv "$history_home/images/catalog/entries" "$work/removed-resolution-history" +capture "$agent_lab" --home "$history_home" experiment check "$artifact" +if [ "$CAPTURE_RC" -eq 125 ] && [ ! -s "$work/stdout" ]; then + pass RES-STATE-001 "local resolution verifies immutable entry history before binding" +else + fail RES-STATE-001 "local resolution verifies immutable entry history before binding" +fi + +drift_home="$work/drift-home" +init_home "$drift_home" +capture "$agent_lab" --home "$drift_home" image add vendor.worker "$subject_a" +mkdir "$drift_home/other-images" +mv "$drift_home/images/catalog" "$drift_home/other-images/catalog" +jq -cS '.paths.images="other-images"' "$drift_home/config.json" > "$work/drift-config.json" +mv "$work/drift-config.json" "$drift_home/config.json" +chmod 600 "$drift_home/config.json" +capture "$agent_lab" --home "$drift_home" config check +drift_config_rc="$CAPTURE_RC" +capture "$agent_lab" --home "$drift_home" experiment check "$artifact" +if [ "$drift_config_rc" -eq 125 ] && [ "$CAPTURE_RC" -eq 125 ] && [ ! -s "$work/stdout" ]; then + pass RES-STATE-002 "receipt-breaking config drift cannot redirect local resolution" +else + fail RES-STATE-002 "receipt-breaking config drift cannot redirect local resolution" +fi + +symlink_source="$work/symlink-source" +init_home "$symlink_source" +capture "$agent_lab" --home "$symlink_source" image add vendor.worker "$subject_a" +mv "$symlink_source/images/catalog" "$work/outside-resolution-catalog" +symlink_home="$work/symlink-home" +init_home "$symlink_home" +ln -s "$work/outside-resolution-catalog" "$symlink_home/images/catalog" +capture "$agent_lab" --home "$symlink_home" experiment check "$artifact" +if [ "$CAPTURE_RC" -eq 125 ] && [ ! -s "$work/stdout" ]; then + pass RES-STATE-003 "local resolution refuses a symlinked catalog authority" +else + fail RES-STATE-003 "local resolution refuses a symlinked catalog authority" +fi + +supplied="$work/supplied-fields" +mkdir "$supplied" +printf '%s\n' \ + 'package experiment' \ + '' \ + 'experiment: {' \ + ' apiVersion: "agent-lab/v0alpha1"' \ + ' kind: "Experiment"' \ + ' metadata: name: "supplied-fields"' \ + ' spec: members: [{' \ + ' name: "worker"' \ + ' image: {' \ + ' catalogName: "vendor.worker"' \ + ' resolvedSubject: "registry.example/evil@sha256:eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee"' \ + ' }' \ + ' }]' \ + '}' > "$supplied/experiment.cue" +capture "$agent_lab" --home "$active_home" experiment check "$supplied" +if [ "$CAPTURE_RC" -eq 1 ] && [ ! -s "$work/stdout" ]; then + pass RES-INPUT-001 "Experiment content cannot supply catalog source, entry, or resolved-subject authority" +else + fail RES-INPUT-001 "Experiment content cannot supply catalog source, entry, or resolved-subject authority" +fi + +multi_home="$work/multi-home" +init_home "$multi_home" +capture "$agent_lab" --home "$multi_home" image add vendor.worker "$subject_a" +multi_entry_a="$(jq -r '.entryDigest // empty' "$work/stdout" 2>/dev/null)" +capture "$agent_lab" --home "$multi_home" image add vendor.second "$subject_b" +multi_entry_b="$(jq -r '.entryDigest // empty' "$work/stdout" 2>/dev/null)" +multi_digest="$(jq -r '.snapshotDigest' "$multi_home/images/catalog/current.json" 2>/dev/null)" +multi_artifact="$work/multi-artifact" +write_two_member_artifact "$multi_artifact" +capture "$agent_lab" --home "$multi_home" experiment check "$multi_artifact" +if [ "$CAPTURE_RC" -eq 0 ] && jq -e --arg first "$multi_entry_a" --arg second "$multi_entry_b" --arg snapshot "$multi_digest" ' + .plan.spec.members[0].resolvedImage.entryDigest == $first and + .plan.spec.members[1].resolvedImage.entryDigest == $second and + .catalog.local.snapshotDigest == $snapshot +' "$work/stdout" >/dev/null 2>&1; then + pass RES-SNAP-003 "all local members bind entries from one held catalog snapshot" +else + fail RES-SNAP-003 "all local members bind entries from one held catalog snapshot" +fi + +copy_cedar "$active_home" +capture "$agent_lab" --home "$active_home" experiment authorize install "$artifact" +if [ "$CAPTURE_RC" -eq 0 ] && jq -e --arg plan "$first_plan_digest" '.verdict == "permit" and .binding.planDigest == $plan' "$work/stdout" >/dev/null 2>&1; then + pass RES-AUTH-001 "fresh authorization binds the selected-entry plan digest" +else + fail RES-AUTH-001 "fresh authorization binds the selected-entry plan digest" +fi + +canary_home="$work/canary-home" +init_home "$canary_home" +capture "$agent_lab" --home "$canary_home" image add vendor.worker "$subject_a" +canary_bin="$work/canary-bin" +canary_marks="$work/canary-marks" +mkdir "$canary_bin" "$canary_marks" +for command in docker git curl wget; do + printf '%s\n' '#!/bin/sh' 'set -eu' ': > "$CANARY_DIR/${0##*/}"' > "$canary_bin/$command" + chmod 700 "$canary_bin/$command" + CANARY_DIR="$canary_marks" "$canary_bin/$command" +done +calibrated="$(find "$canary_marks" -type f | wc -l)" +find "$canary_marks" -type f -delete +canary_rc=0 +env -i PATH="$canary_bin:/usr/bin:/bin" LANG=C LC_ALL=C CANARY_DIR="$canary_marks" \ + "$agent_lab" --home "$canary_home" experiment check "$artifact" > "$work/canary.out" 2> "$work/canary.err" || canary_rc=$? +if [ "$calibrated" -eq 4 ] && [ "$canary_rc" -eq 0 ] && [ -z "$(find "$canary_marks" -type f -print -quit)" ]; then + pass RES-NOEF-001 "calibrated forbidden-effect canaries remain silent during local resolution" +else + fail RES-NOEF-001 "calibrated forbidden-effect canaries remain silent during local resolution" +fi + +expected="$work/expected" +printf '%s\n' \ + RES-ENTRY-001 RES-SNAP-001 RES-SNAP-002 RES-ENTRY-002 RES-ENTRY-003 \ + RES-ISOLATE-001 RES-STATE-001 RES-STATE-002 RES-STATE-003 RES-INPUT-001 \ + RES-SNAP-003 RES-AUTH-001 RES-NOEF-001 > "$expected" +if ! cmp -s "$expected" "$observed"; then + printf 'INFRA catalog resolution assertion identity drift\n' >&2 + exit 125 +fi +printf 'SUMMARY assertions=13 expected=13 failures=%s infra=0\n' "$failures" +[ "$failures" -eq 0 ] diff --git a/tests/experiment/local-image-catalog-cases.sh b/tests/experiment/local-image-catalog-cases.sh index d10d982..7800063 100755 --- a/tests/experiment/local-image-catalog-cases.sh +++ b/tests/experiment/local-image-catalog-cases.sh @@ -1,104 +1,74 @@ #!/usr/bin/env bash -set -euo pipefail +set -u -o pipefail repo_root="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." >/dev/null 2>&1 && pwd)" work="$(mktemp -d)" trap 'find "$work" -type f -delete 2>/dev/null || true; find "$work" -type l -delete 2>/dev/null || true; find "$work" -depth -type d -exec rmdir {} + 2>/dev/null || true' EXIT -home="$work/home" -agent_lab="$repo_root/scripts/agent-lab" -"$agent_lab" --home "$home" init >/dev/null -subject="registry.example/operator/worker@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" -other="registry.example/operator/other@sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" -failures=0 -capture() { RC=0; "$@" > "$work/out" 2> "$work/err" || RC=$?; } -pass() { printf 'PASS %s %s\n' "$1" "$2"; } -fail() { printf 'FAIL %s %s\n' "$1" "$2"; failures=$((failures + 1)); } -capture "$agent_lab" --home "$home" image add vendor.worker "$subject" -if [ "$RC" -eq 0 ] && jq -e '.changed == true and .generation == 1 and (.entryDigest | startswith("sha256:"))' "$work/out" >/dev/null 2>&1; then - pass CAT-001 "first add publishes a generation-one digest binding" - entry="$(jq -r '.entryDigest' "$work/out")" -else - fail CAT-001 "first add publishes a generation-one digest binding" - entry="sha256:$(printf '0%.0s' {1..64})" -fi - -capture "$agent_lab" --home "$home" image add vendor.worker "$subject" -if [ "$RC" -eq 0 ] && jq -e --arg entry "$entry" '.changed == false and .entryDigest == $entry' "$work/out" >/dev/null 2>&1; then - pass CAT-002 "same-subject add is idempotent" -else - fail CAT-002 "same-subject add is idempotent" -fi - -capture "$agent_lab" --home "$home" image add vendor.worker "$other" -if [ "$RC" -eq 1 ] && [ ! -s "$work/out" ]; then - pass CAT-003 "different-subject add never overwrites" -else - fail CAT-003 "different-subject add never overwrites" -fi - -cp -a "$repo_root/.cache/dev/tools/cue/." "$home/cache/tools/cue/" -artifact="$work/catalog-artifact" -mkdir "$artifact" -sed 's#image: digestRef: "[^"]*"#image: catalogName: "vendor.worker"#' \ - "$repo_root/tests/experiment/fixtures/directories/minimal/experiment.cue" > "$artifact/experiment.cue" -capture "$agent_lab" --home "$home" experiment check "$artifact" -if [ "$RC" -eq 0 ] && jq -e --arg entry "$entry" --arg subject "$subject" ' - .plan.spec.members[0].resolvedImage == { - entryDigest: $entry, - generation: 1, - origin: "local", - subject: $subject - } -' "$work/out" >/dev/null 2>&1; then - pass RES-001 "Experiment resolution binds the active local entry and subject" -else - fail RES-001 "Experiment resolution binds the active local entry and subject" - printf 'RES-001 rc=%s stderr=%s\n' "$RC" "$(tr '\n' ' ' < "$work/err" 2>/dev/null || true)" -fi +subcases=( + "$repo_root/tests/image/catalog-cases.sh" + "$repo_root/tests/image/catalog-state-cases.py" + "$repo_root/tests/experiment/catalog-resolution-cases.sh" +) +expected="$work/expected" +observed="$work/observed" +: > "$observed" +printf '%s\n' \ + CAT-NAME-001 CAT-NAME-002 CAT-OCI-001 CAT-OCI-002 \ + CAT-ADD-001 CAT-ADD-002 CAT-ADD-003 CAT-ADD-004 CAT-NS-001 \ + CAT-CAS-001 CAT-CAS-002 CAT-CAS-003 CAT-CAS-004 \ + CAT-READ-001 CAT-READ-002 CAT-NOEF-001 CAT-CONC-001 CAT-CONC-002 \ + CAT-STATE-001 CAT-STATE-002 CAT-STATE-003 CAT-STATE-004 \ + CAT-STATE-005 CAT-STATE-006 CAT-STATE-007 CAT-STATE-008 \ + CAT-STATE-009 CAT-STATE-010 CAT-STATE-011 CAT-STATE-012 \ + CAT-BOUND-001 CAT-BOUND-002 CAT-CRASH-001 CAT-CRASH-002 \ + CAT-CRASH-003 CAT-PLAT-001 \ + RES-ENTRY-001 RES-SNAP-001 RES-SNAP-002 RES-ENTRY-002 RES-ENTRY-003 \ + RES-ISOLATE-001 RES-STATE-001 RES-STATE-002 RES-STATE-003 RES-INPUT-001 \ + RES-SNAP-003 RES-AUTH-001 RES-NOEF-001 > "$expected" +expected_count="$(wc -l < "$expected")" +infrastructure=0 -capture "$agent_lab" --home "$home" image remove vendor.worker --expect "$entry" -if [ "$RC" -eq 0 ] && jq -e '.changed == true and .generation == 2 and .state == "removed"' "$work/out" >/dev/null 2>&1; then - pass CAT-004 "exact CAS removal publishes a generation-two tombstone" -else - fail CAT-004 "exact CAS removal publishes a generation-two tombstone" -fi - -capture "$agent_lab" --home "$home" image remove vendor.worker --expect "$entry" -if [ "$RC" -eq 0 ] && jq -e '.changed == false and .generation == 2' "$work/out" >/dev/null 2>&1; then - pass CAT-007 "lost-response removal retry is idempotent with the original token" -else - fail CAT-007 "lost-response removal retry is idempotent with the original token" -fi +for index in "${!subcases[@]}"; do + subcase="${subcases[$index]}" + output="$work/subcase-$index.out" + if [ ! -f "$subcase" ]; then + printf 'INFRA required catalog subcase is missing: %s\n' "$subcase" >&2 + infrastructure=1 + continue + fi + case "$subcase" in + *.py) + python3 -I "$subcase" > "$output" 2>&1 + rc=$? + ;; + *) + bash "$subcase" > "$output" 2>&1 + rc=$? + ;; + esac + cat "$output" + awk '/^(PASS|FAIL) [A-Z0-9-]+ / {print $2}' "$output" >> "$observed" + if [ "$rc" -ne 0 ] && [ "$rc" -ne 1 ]; then + printf 'INFRA catalog subcase returned %s: %s\n' "$rc" "$subcase" >&2 + infrastructure=1 + fi +done -capture "$agent_lab" --home "$home" image add vendor.worker "$subject" -if [ "$RC" -eq 1 ] && [ ! -s "$work/out" ]; then - pass CAT-008 "a tombstoned v0 name cannot be reused" -else - fail CAT-008 "a tombstoned v0 name cannot be reused" +assertions="$(wc -l < "$observed")" +failures="$(awk '/^FAIL [A-Z0-9-]+ / {count++} END {print count + 0}' "$work"/subcase-*.out 2>/dev/null)" +if ! cmp -s "$expected" "$observed"; then + printf 'INFRA catalog aggregate assertion identity drift\n' >&2 + diff -u "$expected" "$observed" >&2 || true + infrastructure=1 fi -capture "$agent_lab" --home "$home" image add agent-lab.worker "$subject" -if [ "$RC" -eq 1 ] && [ ! -s "$work/out" ]; then - pass CAT-005 "release-owned names cannot be claimed locally" -else - fail CAT-005 "release-owned names cannot be claimed locally" +printf 'SUMMARY assertions=%s expected=%s failures=%s infra=%s\n' \ + "$assertions" "$expected_count" "$failures" "$infrastructure" +if [ "$infrastructure" -ne 0 ]; then + exit 125 fi - -capture "$agent_lab" --home "$home" image list --all -if [ "$RC" -eq 0 ] && jq -e '.[0].name == "vendor.worker" and .[0].state == "removed"' "$work/out" >/dev/null 2>&1; then - pass CAT-006 "list all reports the immutable tombstone" -else - fail CAT-006 "list all reports the immutable tombstone" +if [ "$failures" -ne 0 ]; then + exit 1 fi - -printf '{"snapshotDigest":"sha256:%064d"}\n' 0 > "$home/images/catalog/current.json" -capture "$agent_lab" --home "$home" image list -if [ "$RC" -eq 125 ] && [ ! -s "$work/out" ]; then - pass CAT-009 "corrupt catalog authority is infrastructure uncertainty, never empty" -else - fail CAT-009 "corrupt catalog authority is infrastructure uncertainty, never empty" -fi - -printf 'SUMMARY assertions=10 expected=10 failures=%s infra=0\n' "$failures" -[ "$failures" -eq 0 ] +printf 'EXPERIMENT LOCAL IMAGE CATALOG PASS\n' diff --git a/tests/image/catalog-cases.sh b/tests/image/catalog-cases.sh new file mode 100755 index 0000000..8e48949 --- /dev/null +++ b/tests/image/catalog-cases.sh @@ -0,0 +1,313 @@ +#!/usr/bin/env bash +set -u -o pipefail + +repo_root="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." >/dev/null 2>&1 && pwd)" +agent_lab="$repo_root/scripts/agent-lab" +work="$(mktemp -d)" +trap 'find "$work" -type f -delete 2>/dev/null || true; find "$work" -type l -delete 2>/dev/null || true; find "$work" -depth -type d -exec rmdir {} + 2>/dev/null || true' EXIT + +if [ ! -x "$agent_lab" ] || ! command -v jq >/dev/null 2>&1; then + printf 'INFRA catalog public-contract prerequisites are unavailable\n' >&2 + exit 125 +fi + +failures=0 +observed="$work/observed" +: > "$observed" +pass() { printf 'PASS %s %s\n' "$1" "$2"; printf '%s\n' "$1" >> "$observed"; } +fail() { printf 'FAIL %s %s\n' "$1" "$2"; printf '%s\n' "$1" >> "$observed"; failures=$((failures + 1)); } +capture() { CAPTURE_RC=0; "$@" > "$work/stdout" 2> "$work/stderr" || CAPTURE_RC=$?; } +subject_a="registry.example/operator/worker@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" +subject_b="registry.example/operator/other@sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" + +init_home() { + local home="$1" + capture "$agent_lab" --home "$home" init + if [ "$CAPTURE_RC" -ne 0 ]; then + printf 'INFRA temporary Agent Lab home initialization failed: %s\n' "$(tr '\n' ' ' < "$work/stderr")" >&2 + exit 125 + fi +} + +grammar_home="$work/grammar-home" +init_home "$grammar_home" +valid_names=( + "a.b" + "vendor-one.image-2" + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" +) +valid_names_ok=true +for name in "${valid_names[@]}"; do + capture "$agent_lab" --home "$grammar_home" image add "$name" "$subject_a" + if [ "$CAPTURE_RC" -ne 0 ] || ! jq -e '.changed == true and .generation == 1' "$work/stdout" >/dev/null 2>&1; then + valid_names_ok=false + fi +done +if $valid_names_ok; then + pass CAT-NAME-001 "valid minimum, hyphenated, and maximum-length names are accepted" +else + fail CAT-NAME-001 "valid minimum, hyphenated, and maximum-length names are accepted" +fi + +invalid_names=( + "Agent.image" + "vendor.Image" + "vendor.image.extra" + "vendor_1.image" + ".image" + "vendor." + "vendor.-image" + "vendor.image-" + "vendor..image" + "vendor--one.image" + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.image" + "vendor/one.image" + "vendor:image" + "vendor@one.image" + "vendor.é" + $'vendor.im\nage' +) +invalid_names_ok=true +before_invalid="$(capture "$agent_lab" --home "$grammar_home" image list --all; jq -r 'length' "$work/stdout" 2>/dev/null || printf invalid)" +for name in "${invalid_names[@]}"; do + capture "$agent_lab" --home "$grammar_home" image add "$name" "$subject_a" + if [ "$CAPTURE_RC" -ne 1 ] || [ -s "$work/stdout" ]; then + invalid_names_ok=false + fi +done +capture "$agent_lab" --home "$grammar_home" image list --all +after_invalid="$(jq -r 'length' "$work/stdout" 2>/dev/null || printf invalid)" +if $invalid_names_ok && [ "$before_invalid" = "$after_invalid" ]; then + pass CAT-NAME-002 "invalid ASCII, Unicode, separator, control, and length boundaries are rejected without mutation" +else + fail CAT-NAME-002 "invalid ASCII, Unicode, separator, control, and length boundaries are rejected without mutation" +fi + +oci_home="$work/oci-home" +init_home "$oci_home" +port_subject="registry.example:443/team/image@sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc" +capture "$agent_lab" --home "$oci_home" image add valid.port "$port_subject" +if [ "$CAPTURE_RC" -eq 0 ] && jq -e --arg subject "$port_subject" '.changed == true' "$work/stdout" >/dev/null 2>&1; then + capture "$agent_lab" --home "$oci_home" image inspect valid.port + if [ "$CAPTURE_RC" -eq 0 ] && jq -e --arg subject "$port_subject" '.subject == $subject' "$work/stdout" >/dev/null 2>&1; then + pass CAT-OCI-001 "the shared digest-reference grammar accepts a bounded registry port" + else + fail CAT-OCI-001 "the shared digest-reference grammar accepts a bounded registry port" + fi +else + fail CAT-OCI-001 "the shared digest-reference grammar accepts a bounded registry port" +fi + +digest="sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd" +overlong="$(printf 'a%.0s' {1..256})@$digest" +invalid_subjects=( + "registry.example/team/image:latest" + "$digest" + "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd" + "/tmp/image@$digest" + "user:password@registry.example/team/image@$digest" + "registry.example/team//image@$digest" + "registry.example/team/../image@$digest" + "registry.example/team/_image@$digest" + "registry.example/team/image@$digest?query=1" + "registry.example/team/image@sha256:DDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDD" + "$overlong" +) +invalid_subjects_ok=true +index=0 +for subject in "${invalid_subjects[@]}"; do + index=$((index + 1)) + capture "$agent_lab" --home "$oci_home" image add "invalid.case$index" "$subject" + if [ "$CAPTURE_RC" -ne 1 ] || [ -s "$work/stdout" ]; then + invalid_subjects_ok=false + fi +done +capture "$agent_lab" --home "$oci_home" image list --all +if $invalid_subjects_ok && jq -e 'length == 1 and .[0].name == "valid.port"' "$work/stdout" >/dev/null 2>&1; then + pass CAT-OCI-002 "mutable, bare, path-like, credentialed, ambiguous, and overlong subjects are rejected without mutation" +else + fail CAT-OCI-002 "mutable, bare, path-like, credentialed, ambiguous, and overlong subjects are rejected without mutation" +fi + +semantics_home="$work/semantics-home" +init_home "$semantics_home" +capture "$agent_lab" --home "$semantics_home" image add vendor.worker "$subject_a" +if [ "$CAPTURE_RC" -eq 0 ] && jq -e '.changed == true and .generation == 1 and (.entryDigest | test("^sha256:[0-9a-f]{64}$"))' "$work/stdout" >/dev/null 2>&1; then + active_entry="$(jq -r '.entryDigest' "$work/stdout")" + pass CAT-ADD-001 "first add publishes one generation-one immutable binding" +else + active_entry="sha256:$(printf '0%.0s' {1..64})" + fail CAT-ADD-001 "first add publishes one generation-one immutable binding" +fi + +entry_count_before="$(find "$semantics_home/images/catalog/entries" -maxdepth 1 -type f 2>/dev/null | wc -l)" +snapshot_count_before="$(find "$semantics_home/images/catalog/snapshots" -maxdepth 1 -type f 2>/dev/null | wc -l)" +capture "$agent_lab" --home "$semantics_home" image add vendor.worker "$subject_a" +entry_count_after="$(find "$semantics_home/images/catalog/entries" -maxdepth 1 -type f 2>/dev/null | wc -l)" +snapshot_count_after="$(find "$semantics_home/images/catalog/snapshots" -maxdepth 1 -type f 2>/dev/null | wc -l)" +if [ "$CAPTURE_RC" -eq 0 ] && jq -e --arg entry "$active_entry" '.changed == false and .entryDigest == $entry and .generation == 1' "$work/stdout" >/dev/null 2>&1 && + [ "$entry_count_before" = "$entry_count_after" ] && [ "$snapshot_count_before" = "$snapshot_count_after" ]; then + pass CAT-ADD-002 "same-subject retry is idempotent and publishes no records" +else + fail CAT-ADD-002 "same-subject retry is idempotent and publishes no records" +fi + +capture "$agent_lab" --home "$semantics_home" image add vendor.worker "$subject_b" +conflict_rc="$CAPTURE_RC" +capture "$agent_lab" --home "$semantics_home" image inspect vendor.worker +if [ "$conflict_rc" -eq 1 ] && [ "$CAPTURE_RC" -eq 0 ] && jq -e --arg subject "$subject_a" --arg entry "$active_entry" '.state == "active" and .subject == $subject and .entryDigest == $entry' "$work/stdout" >/dev/null 2>&1; then + pass CAT-ADD-003 "different-subject add conflicts without overwriting" +else + fail CAT-ADD-003 "different-subject add conflicts without overwriting" +fi + +capture "$agent_lab" --home "$semantics_home" image add vendor.second "$subject_a" +if [ "$CAPTURE_RC" -eq 0 ] && jq -e '.changed == true and .generation == 1' "$work/stdout" >/dev/null 2>&1; then + pass CAT-ADD-004 "distinct names may bind the same immutable subject" +else + fail CAT-ADD-004 "distinct names may bind the same immutable subject" +fi + +capture "$agent_lab" --home "$semantics_home" image add agent-lab.worker "$subject_a" +reserved_add_rc="$CAPTURE_RC" +capture "$agent_lab" --home "$semantics_home" image remove agent-lab.worker --expect "$active_entry" +reserved_remove_rc="$CAPTURE_RC" +if [ "$reserved_add_rc" -eq 1 ] && [ "$reserved_remove_rc" -eq 1 ]; then + pass CAT-NS-001 "release-owned names cannot be added, removed, or shadowed locally" +else + fail CAT-NS-001 "release-owned names cannot be added, removed, or shadowed locally" +fi + +stale="sha256:$(printf 'e%.0s' {1..64})" +capture "$agent_lab" --home "$semantics_home" image remove vendor.worker --expect "$stale" +stale_rc="$CAPTURE_RC" +capture "$agent_lab" --home "$semantics_home" image inspect vendor.worker +if [ "$stale_rc" -eq 1 ] && [ "$CAPTURE_RC" -eq 0 ] && jq -e --arg entry "$active_entry" '.state == "active" and .entryDigest == $entry and .generation == 1' "$work/stdout" >/dev/null 2>&1; then + pass CAT-CAS-001 "stale remove CAS changes no state" +else + fail CAT-CAS-001 "stale remove CAS changes no state" +fi + +capture "$agent_lab" --home "$semantics_home" image remove vendor.worker --expect "$active_entry" +if [ "$CAPTURE_RC" -eq 0 ] && jq -e --arg previous "$active_entry" '.changed == true and .generation == 2 and .state == "removed"' "$work/stdout" >/dev/null 2>&1; then + tombstone_entry="$(jq -r '.entryDigest' "$work/stdout")" + pass CAT-CAS-002 "exact CAS publishes a generation-two tombstone" +else + tombstone_entry="sha256:$(printf '0%.0s' {1..64})" + fail CAT-CAS-002 "exact CAS publishes a generation-two tombstone" +fi + +capture "$agent_lab" --home "$semantics_home" image remove vendor.worker --expect "$active_entry" +retry_rc="$CAPTURE_RC" +retry_output="$(cat "$work/stdout")" +capture "$agent_lab" --home "$semantics_home" image remove vendor.worker --expect "$tombstone_entry" +wrong_retry_rc="$CAPTURE_RC" +if [ "$retry_rc" -eq 0 ] && printf '%s' "$retry_output" | jq -e --arg entry "$tombstone_entry" '.changed == false and .entryDigest == $entry and .generation == 2' >/dev/null 2>&1 && + [ "$wrong_retry_rc" -eq 1 ]; then + pass CAT-CAS-003 "original-token retry is idempotent and every other tombstone token conflicts" +else + fail CAT-CAS-003 "original-token retry is idempotent and every other tombstone token conflicts" +fi + +capture "$agent_lab" --home "$semantics_home" image add vendor.worker "$subject_a" +same_reuse_rc="$CAPTURE_RC" +capture "$agent_lab" --home "$semantics_home" image add vendor.worker "$subject_b" +other_reuse_rc="$CAPTURE_RC" +if [ "$same_reuse_rc" -eq 1 ] && [ "$other_reuse_rc" -eq 1 ]; then + pass CAT-CAS-004 "a tombstoned v0 name cannot be reused or restored" +else + fail CAT-CAS-004 "a tombstoned v0 name cannot be reused or restored" +fi + +ordering_home="$work/ordering-home" +init_home "$ordering_home" +capture "$agent_lab" --home "$ordering_home" image add zeta.one "$subject_a" +capture "$agent_lab" --home "$ordering_home" image add alpha.two "$subject_b" +capture "$agent_lab" --home "$ordering_home" image list +if [ "$CAPTURE_RC" -eq 0 ] && [ "$(jq -c '[.[].name]' "$work/stdout" 2>/dev/null)" = '["alpha.two","zeta.one"]' ] && + [ "$(python3 -I -c 'import json,sys; print(json.dumps(json.load(sys.stdin),ensure_ascii=True,separators=(",",":"),sort_keys=True))' < "$work/stdout" 2>/dev/null)" = "$(tr -d '\n' < "$work/stdout")" ]; then + pass CAT-READ-001 "list output is canonical and byte-sorted" +else + fail CAT-READ-001 "list output is canonical and byte-sorted" +fi + +capture "$agent_lab" --home "$semantics_home" image list +active_list="$(cat "$work/stdout")" +capture "$agent_lab" --home "$semantics_home" image list --all +all_list="$(cat "$work/stdout")" +capture "$agent_lab" --home "$semantics_home" image inspect vendor.worker +if printf '%s' "$active_list" | jq -e 'all(.[]; .state == "active") and all(.[]; .name != "vendor.worker")' >/dev/null 2>&1 && + printf '%s' "$all_list" | jq -e 'any(.[]; .name == "vendor.worker" and .state == "removed")' >/dev/null 2>&1 && + [ "$CAPTURE_RC" -eq 0 ] && jq -e --arg entry "$tombstone_entry" '.state == "removed" and .entryDigest == $entry and .generation == 2' "$work/stdout" >/dev/null 2>&1; then + pass CAT-READ-002 "list and inspect distinguish active and removed state without repair" +else + fail CAT-READ-002 "list and inspect distinguish active and removed state without repair" +fi + +canary_home="$work/canary-home" +canary_bin="$work/canary-bin" +canary_marks="$work/canary-marks" +mkdir "$canary_bin" "$canary_marks" +init_home "$canary_home" +for command in docker git curl wget; do + printf '%s\n' '#!/bin/sh' 'set -eu' ': > "$CANARY_DIR/${0##*/}"' > "$canary_bin/$command" + chmod 700 "$canary_bin/$command" + CANARY_DIR="$canary_marks" "$canary_bin/$command" +done +calibrated="$(find "$canary_marks" -type f | wc -l)" +find "$canary_marks" -type f -delete +canary_rc=0 +env -i PATH="$canary_bin:/usr/bin:/bin" LANG=C LC_ALL=C CANARY_DIR="$canary_marks" \ + "$agent_lab" --home "$canary_home" image add noeffect.mapping "$subject_a" > "$work/canary.out" 2> "$work/canary.err" || canary_rc=$? +if [ "$calibrated" -eq 4 ] && [ "$canary_rc" -eq 0 ] && [ -z "$(find "$canary_marks" -type f -print -quit)" ]; then + pass CAT-NOEF-001 "calibrated Docker, Git, downloader, and network-tool canaries remain silent" +else + fail CAT-NOEF-001 "calibrated Docker, Git, downloader, and network-tool canaries remain silent" +fi + +concurrent_home="$work/concurrent-home" +init_home "$concurrent_home" +"$agent_lab" --home "$concurrent_home" image add race.first "$subject_a" > "$work/race-first-1.out" 2> "$work/race-first-1.err" & +pid_one=$! +"$agent_lab" --home "$concurrent_home" image add race.first "$subject_a" > "$work/race-first-2.out" 2> "$work/race-first-2.err" & +pid_two=$! +wait "$pid_one"; rc_one=$? +wait "$pid_two"; rc_two=$? +first_outcomes="$(jq -r '.changed' "$work/race-first-1.out" "$work/race-first-2.out" 2>/dev/null | LC_ALL=C sort | tr '\n' ' ')" +if [ "$rc_one" -eq 0 ] && [ "$rc_two" -eq 0 ] && [ "$first_outcomes" = "false true " ] && + [ "$(find "$concurrent_home/images/catalog/entries" -maxdepth 1 -type f | wc -l)" -eq 1 ] && + [ "$(find "$concurrent_home/images/catalog/snapshots" -maxdepth 1 -type f | wc -l)" -eq 1 ]; then + pass CAT-CONC-001 "concurrent first add linearizes once with one idempotent observer" +else + fail CAT-CONC-001 "concurrent first add linearizes once with one idempotent observer" +fi + +capture "$agent_lab" --home "$concurrent_home" image inspect race.first +race_entry="$(jq -r '.entryDigest // empty' "$work/stdout" 2>/dev/null)" +"$agent_lab" --home "$concurrent_home" image add race.first "$subject_a" > "$work/race-add.out" 2> "$work/race-add.err" & +pid_add=$! +"$agent_lab" --home "$concurrent_home" image remove race.first --expect "$race_entry" > "$work/race-remove.out" 2> "$work/race-remove.err" & +pid_remove=$! +wait "$pid_add"; rc_add=$? +wait "$pid_remove"; rc_remove=$? +capture "$agent_lab" --home "$concurrent_home" image inspect race.first +if [ "$rc_remove" -eq 0 ] && { [ "$rc_add" -eq 0 ] || [ "$rc_add" -eq 1 ]; } && + [ "$CAPTURE_RC" -eq 0 ] && jq -e '.state == "removed" and .generation == 2' "$work/stdout" >/dev/null 2>&1 && + [ "$(find "$concurrent_home/images/catalog/entries" -maxdepth 1 -type f | wc -l)" -eq 2 ]; then + pass CAT-CONC-002 "concurrent add and remove have one legal linearized tombstone outcome" +else + fail CAT-CONC-002 "concurrent add and remove have one legal linearized tombstone outcome" +fi + +expected="$work/expected" +printf '%s\n' \ + CAT-NAME-001 CAT-NAME-002 CAT-OCI-001 CAT-OCI-002 \ + CAT-ADD-001 CAT-ADD-002 CAT-ADD-003 CAT-ADD-004 CAT-NS-001 \ + CAT-CAS-001 CAT-CAS-002 CAT-CAS-003 CAT-CAS-004 \ + CAT-READ-001 CAT-READ-002 CAT-NOEF-001 CAT-CONC-001 CAT-CONC-002 > "$expected" +if ! cmp -s "$expected" "$observed"; then + printf 'INFRA catalog public assertion identity drift\n' >&2 + exit 125 +fi +printf 'SUMMARY assertions=18 expected=18 failures=%s infra=0\n' "$failures" +[ "$failures" -eq 0 ] diff --git a/tests/image/catalog-state-cases.py b/tests/image/catalog-state-cases.py new file mode 100755 index 0000000..0c3b87c --- /dev/null +++ b/tests/image/catalog-state-cases.py @@ -0,0 +1,495 @@ +#!/usr/bin/env python3 +"""Adversarial local image-catalog filesystem and durability cases.""" + +from __future__ import annotations + +from contextlib import redirect_stderr, redirect_stdout +import hashlib +from importlib.util import module_from_spec, spec_from_file_location +import io +import json +import os +from pathlib import Path +import shutil +import stat +import subprocess +import sys +import tempfile + + +REPO_ROOT = Path(__file__).resolve().parents[2] +AGENT_LAB = REPO_ROOT / "scripts" / "agent-lab" +AGENT_LAB_MODULE = REPO_ROOT / "scripts" / "agent-lab.py" +SUBJECT = "registry.example/operator/worker@sha256:" + "a" * 64 +OTHER_SUBJECT = "registry.example/operator/other@sha256:" + "b" * 64 +ENTRY_DOMAIN = b"agent-lab.local-image-entry.v1\0" +SNAPSHOT_DOMAIN = b"agent-lab.local-image-snapshot.v1\0" + + +def canonical(value: object) -> bytes: + return json.dumps(value, ensure_ascii=True, separators=(",", ":"), sort_keys=True).encode() + + +def digest(domain: bytes, value: object) -> str: + return "sha256:" + hashlib.sha256(domain + canonical(value)).hexdigest() + + +def load_module(): + spec = spec_from_file_location("agent_lab_catalog_state", AGENT_LAB_MODULE) + if spec is None or spec.loader is None: + raise RuntimeError("Agent Lab application module cannot be loaded") + module = module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +MODULE = load_module() +FAILURES = 0 +OBSERVED: list[str] = [] + + +def check(assertion: str, condition: bool, message: str, detail: str = "") -> None: + global FAILURES + OBSERVED.append(assertion) + if condition: + print(f"PASS {assertion} {message}") + else: + FAILURES += 1 + suffix = f" ({detail})" if detail else "" + print(f"FAIL {assertion} {message}{suffix}") + + +def cli(home: Path, *arguments: str, timeout: float = 20.0) -> subprocess.CompletedProcess[bytes]: + environment = { + "PATH": "/usr/bin:/bin", + "LANG": "C", + "LC_ALL": "C", + } + return subprocess.run( + [str(AGENT_LAB), "--home", str(home), *arguments], + stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + env=environment, + timeout=timeout, + check=False, + ) + + +def module_image(home: Path, *arguments: str) -> tuple[int | None, str, str, BaseException | None]: + output = io.StringIO() + errors = io.StringIO() + try: + with redirect_stdout(output), redirect_stderr(errors): + result = MODULE.image_command(home, list(arguments)) + return result, output.getvalue(), errors.getvalue(), None + except BaseException as error: # The assertion records an uncontained production fault as RED. + return None, output.getvalue(), errors.getvalue(), error + + +def module_main(home: Path, arguments: list[str], output: io.TextIOBase | None = None) -> tuple[int | None, BaseException | None]: + stream = output if output is not None else io.StringIO() + errors = io.StringIO() + try: + with redirect_stdout(stream), redirect_stderr(errors): + result = MODULE.main(["--home", str(home), *arguments]) + return result, None + except BaseException as error: # The assertion records an uncontained production fault as RED. + return None, error + + +def new_home(root: Path, name: str) -> Path: + home = root / name + completed = cli(home, "init") + if completed.returncode != 0: + raise RuntimeError(f"temporary home init failed: {completed.stderr.decode(errors='replace')}") + return home + + +def add(home: Path, name: str = "vendor.worker", subject: str = SUBJECT) -> dict[str, object]: + completed = cli(home, "image", "add", name, subject) + if completed.returncode != 0: + raise RuntimeError(f"catalog setup add failed: {completed.stderr.decode(errors='replace')}") + value = json.loads(completed.stdout) + if not isinstance(value, dict): + raise RuntimeError("catalog setup add returned a non-object") + return value + + +def current_snapshot(home: Path) -> tuple[Path, dict[str, object], str]: + root = home / "images" / "catalog" + pointer = json.loads((root / "current.json").read_bytes()) + snapshot_digest = pointer["snapshotDigest"] + path = root / "snapshots" / f"{snapshot_digest[7:]}.json" + value = json.loads(path.read_bytes()) + return path, value, snapshot_digest + + +def fingerprint(root: Path) -> tuple[tuple[str, str, int, int, str], ...]: + if not root.exists() and not root.is_symlink(): + return () + records: list[tuple[str, str, int, int, str]] = [] + paths = [root, *sorted(root.rglob("*"), key=lambda item: os.fsencode(str(item.relative_to(root))))] + for path in paths: + metadata = path.lstat() + relative = "." if path == root else str(path.relative_to(root)) + if stat.S_ISLNK(metadata.st_mode): + kind = "l" + identity = os.readlink(path) + elif stat.S_ISREG(metadata.st_mode): + kind = "f" + identity = hashlib.sha256(path.read_bytes()).hexdigest() + elif stat.S_ISDIR(metadata.st_mode): + kind = "d" + identity = "" + else: + kind = "o" + identity = "" + records.append((relative, kind, stat.S_IMODE(metadata.st_mode), metadata.st_nlink, identity)) + return tuple(records) + + +class BrokenOutput(io.StringIO): + def write(self, value: str) -> int: + raise OSError("injected result-output failure") + + +def main() -> int: + with tempfile.TemporaryDirectory(prefix="agent-lab-catalog-state-") as directory: + root = Path(directory) + + missing_home = new_home(root, "missing-home") + add(missing_home) + committed = missing_home / "images" / "catalog" + saved_catalog = root / "saved-committed-catalog" + committed.rename(saved_catalog) + completed = cli(missing_home, "image", "list") + check( + "CAT-STATE-001", + completed.returncode == 125 and completed.stdout == b"" and not committed.exists(), + "a missing previously committed catalog is infrastructure uncertainty, not empty", + f"rc={completed.returncode} stdout={completed.stdout!r}", + ) + + history_home = new_home(root, "history-home") + add(history_home) + entries = history_home / "images" / "catalog" / "entries" + entries.rename(root / "removed-entry-history") + completed = cli(history_home, "image", "list") + check( + "CAT-STATE-002", + completed.returncode == 125 and completed.stdout == b"", + "missing immutable entry history fails closed", + f"rc={completed.returncode} stdout={completed.stdout!r}", + ) + + canonical_home = new_home(root, "canonical-home") + add(canonical_home) + snapshot_path, snapshot, _ = current_snapshot(canonical_home) + snapshot_path.write_text(json.dumps(snapshot, indent=2, sort_keys=True) + "\n", encoding="utf-8") + completed = cli(canonical_home, "image", "list") + check( + "CAT-STATE-003", + completed.returncode == 125, + "noncanonical snapshot bytes fail closed even when semantic digest fields match", + f"rc={completed.returncode}", + ) + + corrupt_home = new_home(root, "corrupt-home") + add(corrupt_home) + _, snapshot, _ = current_snapshot(corrupt_home) + records = snapshot.get("records") + if not isinstance(records, dict) or not isinstance(records.get("vendor.worker"), dict): + raise RuntimeError("catalog setup snapshot has an unexpected shape") + records["vendor.worker"]["state"] = "future" + corrupt_digest = digest(SNAPSHOT_DOMAIN, snapshot) + corrupt_path = corrupt_home / "images" / "catalog" / "snapshots" / f"{corrupt_digest[7:]}.json" + corrupt_path.write_bytes(canonical(snapshot) + b"\n") + (corrupt_home / "images" / "catalog" / "current.json").write_bytes( + canonical({"snapshotDigest": corrupt_digest}) + b"\n" + ) + completed = cli(corrupt_home, "image", "list") + check( + "CAT-STATE-004", + completed.returncode == 125 and completed.stdout == b"", + "closed record validation rejects an unknown state even under a recomputed snapshot digest", + f"rc={completed.returncode} stdout={completed.stdout!r}", + ) + + mode_home = new_home(root, "mode-home") + add(mode_home) + current = mode_home / "images" / "catalog" / "current.json" + current.chmod(0o644) + completed = cli(mode_home, "image", "list") + check( + "CAT-STATE-005", + completed.returncode == 125, + "wrong catalog record mode fails closed", + f"rc={completed.returncode}", + ) + + link_count_home = new_home(root, "link-count-home") + add(link_count_home) + current = link_count_home / "images" / "catalog" / "current.json" + os.link(current, root / "second-current-link") + completed = cli(link_count_home, "image", "list") + check( + "CAT-STATE-006", + completed.returncode == 125, + "multiply linked catalog authority fails closed", + f"rc={completed.returncode}", + ) + + source_home = new_home(root, "symlink-source-home") + add(source_home) + outside_catalog = root / "outside-catalog" + (source_home / "images" / "catalog").rename(outside_catalog) + symlink_home = new_home(root, "symlink-home") + os.symlink(outside_catalog, symlink_home / "images" / "catalog") + before = fingerprint(outside_catalog) + completed = cli(symlink_home, "image", "add", "vendor.second", OTHER_SUBJECT) + after = fingerprint(outside_catalog) + check( + "CAT-STATE-007", + completed.returncode == 125 and before == after, + "a symlinked initialized catalog is refused before external target mutation", + f"rc={completed.returncode} target_changed={before != after}", + ) + + lock_home = new_home(root, "lock-home") + add(lock_home) + lock_path = lock_home / "state" / "locks" / "image-catalog.lock" + saved_lock = root / "saved-image-catalog.lock" + lock_path.rename(saved_lock) + os.symlink(saved_lock, lock_path) + completed = cli(lock_home, "image", "list") + check( + "CAT-STATE-008", + completed.returncode == 125, + "replacement or symlink of the stable catalog lock fails closed", + f"rc={completed.returncode}", + ) + + staging_home = new_home(root, "staging-home") + foreign_wrapper = staging_home / "images" / ".staging" / "foreign-wrapper" + (foreign_wrapper / "payload").mkdir(parents=True) + (foreign_wrapper / "intent.json").write_text('{"owner":"foreign"}\n', encoding="utf-8") + before = fingerprint(foreign_wrapper) + completed = cli(staging_home, "image", "add", "vendor.worker", SUBJECT) + after = fingerprint(foreign_wrapper) + check( + "CAT-STATE-009", + completed.returncode == 125 + and before == after + and not (staging_home / "images" / "catalog").exists(), + "unknown staging ownership is preserved and blocks new mutation", + f"rc={completed.returncode} wrapper_changed={before != after}", + ) + + pristine_home = new_home(root, "pristine-home") + before = fingerprint(pristine_home / "images") + completed = cli(pristine_home, "image", "list") + after = fingerprint(pristine_home / "images") + check( + "CAT-STATE-010", + completed.returncode == 0 and completed.stdout == b"[]\n" and before == after, + "pristine read returns canonical empty without initialization or cleanup", + f"rc={completed.returncode} changed={before != after} stdout={completed.stdout!r}", + ) + + schema_home = new_home(root, "schema-home") + add(schema_home) + entry_paths = list((schema_home / "images" / "catalog" / "entries").glob("*.json")) + snapshot_paths = list((schema_home / "images" / "catalog" / "snapshots").glob("*.json")) + schema_ok = len(entry_paths) == 1 and len(snapshot_paths) == 1 + for path in [*entry_paths, *snapshot_paths]: + raw = path.read_bytes() + try: + value = json.loads(raw) + except (UnicodeError, json.JSONDecodeError): + schema_ok = False + continue + api_version = value.get("apiVersion") if isinstance(value, dict) else None + schema_ok = ( + schema_ok + and isinstance(api_version, str) + and api_version.startswith("agent-lab.") + and api_version.endswith("/v0alpha1") + and raw == canonical(value) + b"\n" + ) + check( + "CAT-STATE-011", + schema_ok, + "immutable entry and snapshot records carry closed versioned canonical schemas", + ) + + truncated_home = new_home(root, "truncated-home") + add(truncated_home) + current = truncated_home / "images" / "catalog" / "current.json" + current.write_bytes(b'{"snapshotDigest":') + before = current.read_bytes() + completed = cli(truncated_home, "image", "inspect", "vendor.worker") + check( + "CAT-STATE-012", + completed.returncode == 125 and completed.stdout == b"" and current.read_bytes() == before, + "truncated authority is infrastructure uncertainty and read-only commands do not repair it", + f"rc={completed.returncode}", + ) + + names_home = new_home(root, "names-bound-home") + names_ok = True + for index in range(256): + rc, _, _, error = module_image(names_home, "add", f"v{index:03d}.image", SUBJECT) + if rc != 0 or error is not None: + names_ok = False + break + before = fingerprint(names_home / "images" / "catalog") + rc, _, _, error = module_image(names_home, "add", "v256.image", SUBJECT) + after = fingerprint(names_home / "images" / "catalog") + check( + "CAT-BOUND-001", + names_ok and rc == 1 and error is None and before == after, + "the 257th logical name is a stable refusal with no publication", + f"setup_ok={names_ok} rc={rc} error={error!r} changed={before != after}", + ) + + bytes_home = new_home(root, "bytes-bound-home") + add(bytes_home) + orphan = bytes_home / "images" / "catalog" / "entries" / ("f" * 64 + ".json") + with orphan.open("wb") as stream: + stream.truncate(67_108_865) + before = orphan.stat().st_size + completed = cli(bytes_home, "image", "list") + check( + "CAT-BOUND-002", + completed.returncode == 125 and orphan.exists() and orphan.stat().st_size == before, + "over-bound physical catalog state fails closed without broad deletion", + f"rc={completed.returncode}", + ) + + fsync_home = new_home(root, "fsync-home") + original_fsync = MODULE.os.fsync + fsync_calls = 0 + + def fail_first_fsync(descriptor: int) -> None: + nonlocal fsync_calls + fsync_calls += 1 + if fsync_calls == 1: + raise OSError("injected fsync failure") + original_fsync(descriptor) + + MODULE.os.fsync = fail_first_fsync + try: + first_rc, _, _, first_error = module_image(fsync_home, "add", "vendor.worker", SUBJECT) + finally: + MODULE.os.fsync = original_fsync + retry = cli(fsync_home, "image", "add", "vendor.worker", SUBJECT) + staged = list((fsync_home / "images" / ".staging").iterdir()) + check( + "CAT-CRASH-001", + first_rc == 125 + and first_error is None + and retry.returncode == 0 + and json.loads(retry.stdout).get("changed") is True + and not staged, + "record fsync failure is contained and the next effectful retry reconciles safely", + f"first_rc={first_rc} error={first_error!r} retry_rc={retry.returncode} staged={len(staged)}", + ) + + replace_home = new_home(root, "replace-home") + original_replace = MODULE.os.replace + replace_calls = 0 + + def fail_pointer_replace(source: os.PathLike[str] | str, target: os.PathLike[str] | str) -> None: + nonlocal replace_calls + if Path(target).name == "current.json": + replace_calls += 1 + if replace_calls == 1: + raise OSError("injected pointer publication failure") + original_replace(source, target) + + MODULE.os.replace = fail_pointer_replace + try: + first_rc, _, _, first_error = module_image(replace_home, "add", "vendor.worker", SUBJECT) + finally: + MODULE.os.replace = original_replace + retry = cli(replace_home, "image", "add", "vendor.worker", SUBJECT) + staged = list((replace_home / "images" / ".staging").iterdir()) + check( + "CAT-CRASH-002", + first_rc == 125 + and first_error is None + and retry.returncode == 0 + and json.loads(retry.stdout).get("changed") is True + and not staged, + "pointer publication failure leaves an uncommitted operation that retry proves and reconciles", + f"first_rc={first_rc} error={first_error!r} retry_rc={retry.returncode} staged={len(staged)}", + ) + + result_home = new_home(root, "result-home") + first_rc, first_error = module_main( + result_home, + ["image", "add", "vendor.worker", SUBJECT], + BrokenOutput(), + ) + retry = cli(result_home, "image", "add", "vendor.worker", SUBJECT) + retry_value = json.loads(retry.stdout) if retry.returncode == 0 else {} + check( + "CAT-CRASH-003", + first_rc == 125 + and first_error is None + and retry.returncode == 0 + and retry_value.get("changed") is False, + "lost result output reports uncertainty while retry observes the committed binding idempotently", + f"first_rc={first_rc} error={first_error!r} retry_rc={retry.returncode} retry={retry_value!r}", + ) + + platform_home = new_home(root, "platform-home") + before = fingerprint(platform_home) + original_platform = MODULE.sys.platform + MODULE.sys.platform = "darwin" + try: + platform_rc, platform_error = module_main( + platform_home, + ["image", "add", "vendor.worker", SUBJECT], + ) + finally: + MODULE.sys.platform = original_platform + after = fingerprint(platform_home) + check( + "CAT-PLAT-001", + platform_rc == 125 and platform_error is None and before == after, + "an injected non-Linux host refuses mutation before opening catalog state", + f"rc={platform_rc} error={platform_error!r} changed={before != after}", + ) + + expected = [ + "CAT-STATE-001", + "CAT-STATE-002", + "CAT-STATE-003", + "CAT-STATE-004", + "CAT-STATE-005", + "CAT-STATE-006", + "CAT-STATE-007", + "CAT-STATE-008", + "CAT-STATE-009", + "CAT-STATE-010", + "CAT-STATE-011", + "CAT-STATE-012", + "CAT-BOUND-001", + "CAT-BOUND-002", + "CAT-CRASH-001", + "CAT-CRASH-002", + "CAT-CRASH-003", + "CAT-PLAT-001", + ] + if OBSERVED != expected: + print(f"INFRA catalog state assertion identity drift: {OBSERVED!r}", file=sys.stderr) + return 125 + print(f"SUMMARY assertions=18 expected=18 failures={FAILURES} infra=0") + return 0 if FAILURES == 0 else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) From 3837125f04c092ee29e52b3ec4cd5f590f6d0a3b Mon Sep 17 00:00:00 2001 From: xormania <127287135+xormania@users.noreply.github.com> Date: Sun, 2 Aug 2026 00:10:03 -0400 Subject: [PATCH 031/158] test(experiment): require installed catalog resolution --- tests/experiment/catalog-resolution-cases.sh | 47 ++++++++++++++++++- tests/experiment/local-image-catalog-cases.sh | 2 +- 2 files changed, 46 insertions(+), 3 deletions(-) diff --git a/tests/experiment/catalog-resolution-cases.sh b/tests/experiment/catalog-resolution-cases.sh index adccf35..5f17b11 100755 --- a/tests/experiment/catalog-resolution-cases.sh +++ b/tests/experiment/catalog-resolution-cases.sh @@ -271,6 +271,49 @@ else fail RES-AUTH-001 "fresh authorization binds the selected-entry plan digest" fi +installed_prefix="$work/installed-prefix" +installed_home="$work/installed-home" +runtime_replica="$work/runtime-replica" +runtime_manifest="$repo_root/packaging/agent-lab-local.manifest" +mkdir -p "$runtime_replica/packaging" "$runtime_replica/scripts" +while IFS= read -r runtime_name; do + mkdir -p "$runtime_replica/$(dirname -- "$runtime_name")" + cp "$repo_root/$runtime_name" "$runtime_replica/$runtime_name" +done < "$runtime_manifest" +cp "$runtime_manifest" "$runtime_replica/packaging/agent-lab-local.manifest" +cp "$repo_root/scripts/install-local" "$repo_root/scripts/install-local.py" "$runtime_replica/scripts/" +chmod +x "$runtime_replica/scripts/install-local" "$runtime_replica/scripts/agent-lab" +capture "$runtime_replica/scripts/install-local" --prefix "$installed_prefix" +installed_install_rc="$CAPTURE_RC" +mv "$runtime_replica" "$work/runtime-source-unavailable" +mkdir "$work/installed-unrelated" +installed_rc=125 +if [ "$installed_install_rc" -eq 0 ]; then + capture "$installed_prefix/bin/agent-lab" --home "$installed_home" init + cp -a "$repo_root/.cache/dev/tools/cue/." "$installed_home/cache/tools/cue/" + capture "$installed_prefix/bin/agent-lab" --home "$installed_home" image add vendor.worker "$subject_a" + installed_entry="$(jq -r '.entryDigest // empty' "$work/stdout" 2>/dev/null)" + installed_rc=0 + (cd "$work/installed-unrelated" && env -i PATH=/usr/bin:/bin \ + "$installed_prefix/bin/agent-lab" --home "$installed_home" experiment check "$artifact") \ + > "$work/installed-check.json" 2> "$work/installed-check.err" || installed_rc=$? +else + installed_entry="" +fi +if [ "$installed_install_rc" -eq 0 ] && [ "$installed_rc" -eq 0 ] && + [ ! -s "$work/installed-check.err" ] && jq -e --arg entry "$installed_entry" --arg subject "$subject_a" ' + .plan.spec.members[0].resolvedImage == { + entryDigest: $entry, + generation: 1, + origin: "local", + subject: $subject + } + ' "$work/installed-check.json" >/dev/null 2>&1; then + pass RES-INSTALL-001 "installed catalog commands and local resolution run without the source replica or checkout cwd" +else + fail RES-INSTALL-001 "installed catalog commands and local resolution run without the source replica or checkout cwd" +fi + canary_home="$work/canary-home" init_home "$canary_home" capture "$agent_lab" --home "$canary_home" image add vendor.worker "$subject_a" @@ -297,10 +340,10 @@ expected="$work/expected" printf '%s\n' \ RES-ENTRY-001 RES-SNAP-001 RES-SNAP-002 RES-ENTRY-002 RES-ENTRY-003 \ RES-ISOLATE-001 RES-STATE-001 RES-STATE-002 RES-STATE-003 RES-INPUT-001 \ - RES-SNAP-003 RES-AUTH-001 RES-NOEF-001 > "$expected" + RES-SNAP-003 RES-AUTH-001 RES-INSTALL-001 RES-NOEF-001 > "$expected" if ! cmp -s "$expected" "$observed"; then printf 'INFRA catalog resolution assertion identity drift\n' >&2 exit 125 fi -printf 'SUMMARY assertions=13 expected=13 failures=%s infra=0\n' "$failures" +printf 'SUMMARY assertions=14 expected=14 failures=%s infra=0\n' "$failures" [ "$failures" -eq 0 ] diff --git a/tests/experiment/local-image-catalog-cases.sh b/tests/experiment/local-image-catalog-cases.sh index 7800063..52f50a5 100755 --- a/tests/experiment/local-image-catalog-cases.sh +++ b/tests/experiment/local-image-catalog-cases.sh @@ -25,7 +25,7 @@ printf '%s\n' \ CAT-CRASH-003 CAT-PLAT-001 \ RES-ENTRY-001 RES-SNAP-001 RES-SNAP-002 RES-ENTRY-002 RES-ENTRY-003 \ RES-ISOLATE-001 RES-STATE-001 RES-STATE-002 RES-STATE-003 RES-INPUT-001 \ - RES-SNAP-003 RES-AUTH-001 RES-NOEF-001 > "$expected" + RES-SNAP-003 RES-AUTH-001 RES-INSTALL-001 RES-NOEF-001 > "$expected" expected_count="$(wc -l < "$expected")" infrastructure=0 From 5465c44ac43b5f1095a276ea88ae644b72384c4e Mon Sep 17 00:00:00 2001 From: xormania <127287135+xormania@users.noreply.github.com> Date: Sun, 2 Aug 2026 00:30:34 -0400 Subject: [PATCH 032/158] feat(experiment): verify durable local image catalogs --- packaging/agent-lab-local.manifest | 1 + scripts/agent-lab.py | 211 +-- scripts/experiment.py | 202 ++- scripts/image_catalog.py | 1505 +++++++++++++++++ .../fixtures/expected-runtime-files.txt | 1 + 5 files changed, 1723 insertions(+), 197 deletions(-) create mode 100644 scripts/image_catalog.py diff --git a/packaging/agent-lab-local.manifest b/packaging/agent-lab-local.manifest index 1025d17..ea39df5 100644 --- a/packaging/agent-lab-local.manifest +++ b/packaging/agent-lab-local.manifest @@ -9,5 +9,6 @@ scripts/agent-lab.py scripts/dev/cedar-tool.py scripts/dev/cue-tool.py scripts/experiment.py +scripts/image_catalog.py tools/cedar.lock tools/cue.lock diff --git a/scripts/agent-lab.py b/scripts/agent-lab.py index 1a27162..85c0037 100644 --- a/scripts/agent-lab.py +++ b/scripts/agent-lab.py @@ -2,7 +2,6 @@ from __future__ import annotations from importlib.util import module_from_spec, spec_from_file_location -import fcntl import hashlib import json import os @@ -14,27 +13,12 @@ import sys SAFE_COMPONENT = re.compile(r"^[a-z][a-z0-9-]{0,47}$") -IMAGE_COMPONENT = re.compile(r"^[a-z][a-z0-9]*(?:-[a-z0-9]+)*$") -OCI_SUBJECT = re.compile(r"^[a-z0-9][a-z0-9./_-]*@sha256:[0-9a-f]{64}$") def canonical(value: object) -> bytes: return json.dumps(value, ensure_ascii=True, separators=(",", ":"), sort_keys=True).encode() -def record_digest(domain: bytes, value: object) -> str: - return "sha256:" + hashlib.sha256(domain + canonical(value)).hexdigest() - - -def image_name(value: str) -> bool: - parts = value.split(".") - return ( - len(value.encode("utf-8")) <= 63 - and len(parts) == 2 - and all(part.isascii() and 1 <= len(part) <= 31 and IMAGE_COMPONENT.fullmatch(part) for part in parts) - ) - - def effective_home(raw: str | None) -> Path: selected = raw if raw is not None else os.environ.get("AGENT_LAB_HOME") if selected is None: @@ -79,12 +63,24 @@ def init_home(home: Path, argv: list[str]) -> int: return 0 print("FAIL Agent Lab home conflicts with requested configuration", file=sys.stderr) return 1 + component_roots = { + key: home / component + for key, component in components.items() + } + for path in component_roots.values(): + try: + path.lstat() + except FileNotFoundError: + continue + raise OSError("Agent Lab data component already exists") + for path in component_roots.values(): + path.mkdir(mode=0o700) for key in ("experiments", "images"): - (home / components[key] / ".staging").mkdir(mode=0o700, parents=True) - (home / components["cache"] / "tools/cue").mkdir(mode=0o700, parents=True) - (home / components["cache"] / "tools/cedar").mkdir(mode=0o700, parents=True) - locks = home / components["state"] / "locks" - locks.mkdir(mode=0o700, parents=True) + (component_roots[key] / ".staging").mkdir(mode=0o700) + (component_roots["cache"] / "tools/cue").mkdir(mode=0o700, parents=True) + (component_roots["cache"] / "tools/cedar").mkdir(mode=0o700, parents=True) + locks = component_roots["state"] / "locks" + locks.mkdir(mode=0o700) for name in ("image-catalog.lock", "experiments.lock"): descriptor = os.open(locks / name, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) os.close(descriptor) @@ -151,135 +147,72 @@ def experiment_module(): return module -def atomic_json(path: Path, value: object) -> None: - data = canonical(value) + b"\n" - temporary = path.with_name(f".{path.name}.{os.getpid()}") - descriptor = os.open(temporary, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) - try: - os.write(descriptor, data) - os.fsync(descriptor) - finally: - os.close(descriptor) - os.replace(temporary, path) - - -def catalog_paths(home: Path, loaded: tuple[dict[str, object], bytes]) -> tuple[Path, Path]: - paths = loaded[0]["paths"] - assert isinstance(paths, dict) - return home / str(paths["images"]) / "catalog", home / str(paths["state"]) / "locks/image-catalog.lock" +def image_catalog_module(): + path = Path(__file__).resolve().with_name("image_catalog.py") + spec = spec_from_file_location("agent_lab_image_catalog", path) + if spec is None or spec.loader is None: + raise ImportError("image catalog module cannot be loaded") + module = module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module -def load_catalog(root: Path) -> dict[str, object]: - current = root / "current.json" - if not root.exists(): - return {"revision": 0, "previous": None, "records": {}} - try: - pointer = json.loads(current.read_text(encoding="utf-8")) - digest = pointer["snapshotDigest"] - if not isinstance(digest, str) or not digest.startswith("sha256:"): - raise ValueError - snapshot_path = root / "snapshots" / f"{digest[7:]}.json" - snapshot = json.loads(snapshot_path.read_text(encoding="utf-8")) - except (OSError, KeyError, ValueError, json.JSONDecodeError) as error: - raise RuntimeError("image catalog is malformed") from error - if record_digest(b"agent-lab.local-image-snapshot.v1\0", snapshot) != digest: - raise RuntimeError("image catalog snapshot digest mismatch") - if not isinstance(snapshot, dict) or set(snapshot) != {"revision", "previous", "records"} or not isinstance(snapshot["records"], dict): - raise RuntimeError("image catalog snapshot is not closed") - return snapshot +def write_json(value: object) -> None: + output = canonical(value).decode("ascii") + "\n" + written = sys.stdout.write(output) + if written != len(output): + raise OSError("partial command output") + sys.stdout.flush() -def write_catalog(root: Path, snapshot: dict[str, object], entry: dict[str, object]) -> tuple[str, str]: - entries = root / "entries" - snapshots = root / "snapshots" - entries.mkdir(mode=0o700, parents=True, exist_ok=True) - snapshots.mkdir(mode=0o700, parents=True, exist_ok=True) - entry_digest = record_digest(b"agent-lab.local-image-entry.v1\0", entry) - entry_path = entries / f"{entry_digest[7:]}.json" - if not entry_path.exists(): - atomic_json(entry_path, entry) - snapshot_digest = record_digest(b"agent-lab.local-image-snapshot.v1\0", snapshot) - snapshot_path = snapshots / f"{snapshot_digest[7:]}.json" - if not snapshot_path.exists(): - atomic_json(snapshot_path, snapshot) - atomic_json(root / "current.json", {"snapshotDigest": snapshot_digest}) - return entry_digest, snapshot_digest +def image_command(home: Path, argv: list[str]) -> int: + if argv[:1] == ["add"] and len(argv) == 3: + operation = "add" + elif argv[:1] == ["remove"] and len(argv) == 4 and argv[2] == "--expect": + operation = "remove" + elif argv[:1] == ["list"] and (len(argv) == 1 or argv == ["list", "--all"]): + operation = "list" + elif argv[:1] == ["inspect"] and len(argv) == 2: + operation = "inspect" + else: + return 2 + if operation in {"add", "remove"} and sys.platform != "linux": + print("INFRA Agent Lab local image catalog mutations require Linux", file=sys.stderr) + return 125 -def image_command(home: Path, argv: list[str]) -> int: try: - loaded = load_config(home) - except RuntimeError as error: - print(f"INFRA Agent Lab {error}", file=sys.stderr) + catalog = image_catalog_module() + except (ImportError, OSError) as error: + print(f"INFRA Agent Lab image catalog is unavailable: {error}", file=sys.stderr) return 125 - if loaded is None: - print("FAIL Agent Lab home is not initialized", file=sys.stderr) - return 1 - root, lock_path = catalog_paths(home, loaded) + try: - lock = open(lock_path, "r+b", buffering=0) - fcntl.flock(lock.fileno(), fcntl.LOCK_EX) - snapshot = load_catalog(root) - except (OSError, RuntimeError) as error: - print(f"INFRA Agent Lab {error}", file=sys.stderr) + if operation == "add": + result = catalog.add_image(home, argv[1], argv[2]) + elif operation == "remove": + result = catalog.remove_image(home, argv[1], argv[3]) + elif operation == "list": + result = catalog.list_images(home, include_removed=len(argv) == 2) + else: + result = catalog.inspect_image(home, argv[1]) + except catalog.CatalogReject as error: + print(f"FAIL image {error}", file=sys.stderr) + return 1 + except catalog.CatalogInfrastructure as error: + print(f"INFRA Agent Lab image catalog {error}", file=sys.stderr) return 125 + except (AttributeError, OSError) as error: + print(f"INFRA Agent Lab image catalog operation is unavailable: {error}", file=sys.stderr) + return 125 + try: - records = snapshot["records"] - assert isinstance(records, dict) - if argv[:1] == ["add"] and len(argv) == 3: - name, subject = argv[1:] - if not image_name(name) or name.startswith("agent-lab.") or not OCI_SUBJECT.fullmatch(subject): - print("FAIL image mapping is invalid or reserved", file=sys.stderr) - return 1 - prior = records.get(name) - if prior is not None: - if prior["state"] == "active" and prior["subject"] == subject: - print(canonical({"changed": False, "entryDigest": prior["entryDigest"], "generation": 1}).decode()) - return 0 - print("FAIL image name already exists or is tombstoned", file=sys.stderr) - return 1 - entry = {"generation": 1, "name": name, "previousEntryDigest": None, "state": "active", "subject": subject} - entry_digest = record_digest(b"agent-lab.local-image-entry.v1\0", entry) - record = {**entry, "entryDigest": entry_digest} - new_records = {**records, name: record} - next_snapshot = {"previous": record_digest(b"agent-lab.local-image-snapshot.v1\0", snapshot) if snapshot["revision"] else None, "records": new_records, "revision": int(snapshot["revision"]) + 1} - write_catalog(root, next_snapshot, entry) - print(canonical({"changed": True, "entryDigest": entry_digest, "generation": 1}).decode()) - return 0 - if argv[:1] == ["remove"] and len(argv) == 4 and argv[2] == "--expect": - name, expected = argv[1], argv[3] - prior = records.get(name) - if prior is None: - print("FAIL image name is unknown", file=sys.stderr) - return 1 - if prior["state"] == "removed" and prior["previousEntryDigest"] == expected: - print(canonical({"changed": False, "entryDigest": prior["entryDigest"], "generation": 2, "state": "removed"}).decode()) - return 0 - if prior["state"] != "active" or prior["entryDigest"] != expected: - print("FAIL image remove compare-and-swap conflict", file=sys.stderr) - return 1 - entry = {"generation": 2, "name": name, "previousEntryDigest": expected, "state": "removed", "subject": prior["subject"]} - entry_digest = record_digest(b"agent-lab.local-image-entry.v1\0", entry) - record = {**entry, "entryDigest": entry_digest} - next_snapshot = {"previous": record_digest(b"agent-lab.local-image-snapshot.v1\0", snapshot), "records": {**records, name: record}, "revision": int(snapshot["revision"]) + 1} - write_catalog(root, next_snapshot, entry) - print(canonical({"changed": True, "entryDigest": entry_digest, "generation": 2, "state": "removed"}).decode()) - return 0 - if argv[:1] == ["list"] and (len(argv) == 1 or argv == ["list", "--all"]): - include_all = len(argv) == 2 - values = [records[name] for name in sorted(records) if include_all or records[name]["state"] == "active"] - print(canonical(values).decode()) - return 0 - if argv[:1] == ["inspect"] and len(argv) == 2: - record = records.get(argv[1]) - if record is None: - print("FAIL image name is unknown", file=sys.stderr) - return 1 - print(canonical(record).decode()) - return 0 - return 2 - finally: - lock.close() + write_json(result) + except OSError as error: + print(f"INFRA Agent Lab image catalog result is uncertain: {error}", file=sys.stderr) + return 125 + return 0 def main(argv: list[str]) -> int: diff --git a/scripts/experiment.py b/scripts/experiment.py index db124ef..615a132 100644 --- a/scripts/experiment.py +++ b/scripts/experiment.py @@ -4,6 +4,7 @@ from __future__ import annotations import hashlib +from importlib.util import module_from_spec, spec_from_file_location import json import math import os @@ -106,6 +107,11 @@ class SourceSnapshot(NamedTuple): digest: str +class PlanResolution(NamedTuple): + plan: dict[str, object] + local_catalog: dict[str, object] | None + + def fail(message: str) -> NoReturn: print(f"FAIL Experiment manifest {message}", file=sys.stderr) raise SystemExit(1) @@ -586,6 +592,20 @@ def digest_record(domain: bytes, value: object) -> str: return "sha256:" + hashlib.sha256(domain + canonical_json(value)).hexdigest() +def image_catalog_module(): + path = Path(__file__).resolve().with_name("image_catalog.py") + spec = spec_from_file_location("agent_lab_image_catalog", path) + if spec is None or spec.loader is None: + raise InfrastructureError("local image catalog support cannot be loaded") + module = module_from_spec(spec) + sys.modules[spec.name] = module + try: + spec.loader.exec_module(module) + except (ImportError, OSError) as error: + raise InfrastructureError("local image catalog support cannot be loaded") from error + return module + + def bundled_catalog(repo_root: Path) -> tuple[dict[str, object], str]: path = repo_root / "catalog/experiment-images/v0alpha1.json" data = stable_file_bytes(path, MAX_CONTRACT_FILE_BYTES, "bundled image catalog") @@ -597,66 +617,116 @@ def bundled_catalog(repo_root: Path) -> tuple[dict[str, object], str]: return value, digest_record(BUNDLED_CATALOG_DOMAIN, value) -def local_catalog_entry(home: Path, name: str) -> dict[str, object]: - try: - config = strict_json((home / "config.json").read_bytes(), source="local config") - assert isinstance(config, dict) - images = config["paths"]["images"] - root = home / images / "catalog" - pointer = strict_json((root / "current.json").read_bytes(), source="local catalog pointer") - assert isinstance(pointer, dict) and set(pointer) == {"snapshotDigest"} - snapshot_digest = pointer["snapshotDigest"] - assert isinstance(snapshot_digest, str) and is_sha256(snapshot_digest) - snapshot = strict_json( - (root / "snapshots" / f"{snapshot_digest[7:]}.json").read_bytes(), - source="local catalog snapshot", - ) - if digest_record(b"agent-lab.local-image-snapshot.v1\0", snapshot) != snapshot_digest: - raise ValueError("snapshot digest") - assert isinstance(snapshot, dict) - record = snapshot["records"].get(name) - if record is None or record["state"] != "active": - raise InvalidManifest("references an unknown or removed local image name") - assert isinstance(record, dict) - return record - except InvalidManifest: - raise - except (AssertionError, KeyError, OSError, TypeError, ValueError) as error: - raise InfrastructureError("local image catalog cannot be verified") from error - - -def resolve_plan(plan: dict[str, object], repo_root: Path, catalog: dict[str, object] | None = None) -> dict[str, object]: - by_name: dict[str, dict[str, object]] | None = None - if catalog is not None: - entries = catalog.get("entries") +def resolve_plan_with_evidence( + plan: dict[str, object], + repo_root: Path, + catalog: dict[str, object] | None = None, +) -> PlanResolution: + resolved = json.loads(canonical_json(plan)) + members = resolved["spec"]["members"] + bundled_names: set[str] = set() + local_names: set[str] = set() + for member in members: + selector = member["requestedSelector"] + if set(selector) == {"digestRef"}: + member["resolvedImage"] = {"origin": "direct", "subject": selector["digestRef"]} + continue + if set(selector) != {"catalogName"} or not valid_catalog_name(selector["catalogName"]): + raise InvalidManifest("contains an invalid image selector") + name = selector["catalogName"] + if name.startswith("agent-lab."): + bundled_names.add(name) + else: + local_names.add(name) + + catalog_support = None + bundled_by_name: dict[str, dict[str, object]] = {} + if bundled_names: + catalog_support = image_catalog_module() + selected_catalog = catalog + if selected_catalog is None: + selected_catalog, _ = bundled_catalog(repo_root) + entries = selected_catalog.get("entries") if not isinstance(entries, list): raise InfrastructureError("bundled image catalog entries are malformed") - by_name = {} for entry in entries: if not isinstance(entry, dict) or set(entry) != {"name", "subject"}: raise InfrastructureError("bundled image catalog entry is malformed") name, subject = entry["name"], entry["subject"] - if not valid_catalog_name(name) or not isinstance(subject, str) or "@sha256:" not in subject: + if ( + not valid_catalog_name(name) + or not isinstance(subject, str) + or not catalog_support.oci_subject(subject) + ): raise InfrastructureError("bundled image catalog entry is invalid") assert isinstance(name, str) - if not name.startswith("agent-lab.") or name in by_name: + if not name.startswith("agent-lab.") or name in bundled_by_name: raise InfrastructureError("bundled image catalog namespace is invalid") - by_name[name] = entry - resolved = json.loads(canonical_json(plan)) - members = resolved["spec"]["members"] + bundled_by_name[name] = entry + + local_records: dict[str, dict[str, object]] = {} + local_evidence: dict[str, object] | None = None + if local_names: + if catalog_support is None: + catalog_support = image_catalog_module() + raw_home = os.environ.get("AGENT_LAB_HOME") + if not raw_home: + raise InvalidManifest("local image name requires an initialized Agent Lab home") + try: + local_resolution = catalog_support.resolve_local_images( + Path(raw_home), tuple(sorted(local_names)) + ) + except catalog_support.CatalogReject as error: + raise InvalidManifest(str(error)) from error + except catalog_support.CatalogInfrastructure as error: + raise InfrastructureError(str(error)) from error + try: + if not isinstance(local_resolution, dict) or set(local_resolution) != {"catalog", "records"}: + raise ValueError("resolution envelope") + records = local_resolution["records"] + evidence = local_resolution["catalog"] + if not isinstance(records, dict) or set(records) != local_names: + raise ValueError("resolution records") + if not isinstance(evidence, dict) or set(evidence) != {"revision", "snapshotDigest"}: + raise ValueError("resolution evidence") + revision = evidence["revision"] + snapshot_digest = evidence["snapshotDigest"] + if ( + not isinstance(revision, int) + or isinstance(revision, bool) + or revision < 1 + or not is_sha256(snapshot_digest) + ): + raise ValueError("resolution evidence values") + for name in local_names: + record = records[name] + if ( + not isinstance(record, dict) + or record.get("name") != name + or record.get("state") != "active" + or type(record.get("generation")) is not int + or record["generation"] != 1 + or record.get("previousEntryDigest") is not None + or not is_sha256(record.get("entryDigest")) + or not isinstance(record.get("subject"), str) + or not catalog_support.oci_subject(record["subject"]) + ): + raise ValueError("selected local record") + local_records = records + local_evidence = { + "revision": revision, + "snapshotDigest": snapshot_digest, + } + except (KeyError, TypeError, ValueError) as error: + raise InfrastructureError("local image catalog returned invalid resolution evidence") from error + for member in members: selector = member["requestedSelector"] if set(selector) == {"digestRef"}: - member["resolvedImage"] = {"origin": "direct", "subject": selector["digestRef"]} continue - if set(selector) != {"catalogName"} or not valid_catalog_name(selector["catalogName"]): - raise InvalidManifest("contains an invalid image selector") name = selector["catalogName"] if name.startswith("agent-lab."): - if by_name is None: - loaded_catalog, _ = bundled_catalog(repo_root) - return resolve_plan(plan, repo_root, loaded_catalog) - entry = by_name.get(name) + entry = bundled_by_name.get(name) if entry is None: raise InvalidManifest("references an unknown bundled image name") member["resolvedImage"] = { @@ -666,17 +736,22 @@ def resolve_plan(plan: dict[str, object], repo_root: Path, catalog: dict[str, ob "subject": entry["subject"], } else: - raw_home = os.environ.get("AGENT_LAB_HOME") - if not raw_home: - raise InvalidManifest("local image name requires an initialized Agent Lab home") - record = local_catalog_entry(Path(raw_home), name) + record = local_records[name] member["resolvedImage"] = { "entryDigest": record["entryDigest"], "generation": record["generation"], "origin": "local", "subject": record["subject"], } - return resolved + return PlanResolution(resolved, local_evidence) + + +def resolve_plan( + plan: dict[str, object], + repo_root: Path, + catalog: dict[str, object] | None = None, +) -> dict[str, object]: + return resolve_plan_with_evidence(plan, repo_root, catalog).plan def invoke_cue( @@ -737,7 +812,7 @@ def parse_cue_plan(completed: subprocess.CompletedProcess[bytes]) -> dict[str, o return plan -def cue_plan(manifest: object) -> object: +def cue_plan_with_evidence(manifest: object) -> PlanResolution: repo_root = Path(__file__).resolve().parent.parent contract_root = repo_root / "contracts" / "experiment" / "v0alpha1" cue_helper = repo_root / "scripts" / "dev" / "cue-tool.py" @@ -787,11 +862,15 @@ def cue_plan(manifest: object) -> object: plan = parse_cue_plan(completed) if plan != expected_plan(manifest, contract_digest): raise InfrastructureError("pinned CUE plan violates its exact postcondition") - return resolve_plan(plan, repo_root) + return resolve_plan_with_evidence(plan, repo_root) except OSError as error: raise InfrastructureError("private validation snapshot could not be managed") from error +def cue_plan(manifest: object) -> object: + return cue_plan_with_evidence(manifest).plan + + def is_sha256(value: object) -> bool: return ( isinstance(value, str) @@ -1208,10 +1287,13 @@ def authorize_plan(plan: object, source_digest: str) -> tuple[dict[str, object], return decision, 0 if verdict == "permit" else 1 -def write_envelope(plan: object) -> None: +def write_envelope(plan: object, local_catalog: dict[str, object] | None = None) -> None: plan_bytes = canonical_json(plan) digest = hashlib.sha256(plan_bytes).hexdigest() - envelope = canonical_json({"digest": f"sha256:{digest}", "plan": plan}) + b"\n" + value: dict[str, object] = {"digest": f"sha256:{digest}", "plan": plan} + if local_catalog is not None: + value["catalog"] = {"local": local_catalog} + envelope = canonical_json(value) + b"\n" try: written = sys.stdout.buffer.write(envelope) if written != len(envelope): @@ -1239,14 +1321,17 @@ def main(argv: list[str]) -> int: try: snapshot = read_directory_snapshot(argv[2]) manifest = authored_manifest(snapshot) - plan = cue_plan(manifest) + resolution = cue_plan_with_evidence(manifest) + plan = resolution.plan if directory_checking: plan_bytes = canonical_json(plan) - checked = { + checked: dict[str, object] = { "digest": f"sha256:{hashlib.sha256(plan_bytes).hexdigest()}", "plan": plan, "source": {"digest": snapshot.digest, "kind": "directory"}, } + if resolution.local_catalog is not None: + checked["catalog"] = {"local": resolution.local_catalog} sys.stdout.buffer.write(canonical_json(checked) + b"\n") return 0 decision, result = authorize_plan(plan, snapshot.digest) @@ -1272,9 +1357,10 @@ def main(argv: list[str]) -> int: manifest = strict_json(manifest_bytes, source="input") if not isinstance(manifest, dict): raise InvalidManifest("must be one JSON object") - plan = cue_plan(manifest) + resolution = cue_plan_with_evidence(manifest) + plan = resolution.plan if checking: - write_envelope(plan) + write_envelope(plan, resolution.local_catalog) return 0 decision, result = authorize_plan(plan, "sha256:" + "0" * 64) write_decision(decision) diff --git a/scripts/image_catalog.py b/scripts/image_catalog.py new file mode 100644 index 0000000..74c9b9c --- /dev/null +++ b/scripts/image_catalog.py @@ -0,0 +1,1505 @@ +#!/usr/bin/env python3 +"""Verified, durable operator-local image-name catalog.""" + +from __future__ import annotations + +from contextlib import contextmanager +from dataclasses import dataclass +import ctypes +import errno +import fcntl +import hashlib +import json +import os +from pathlib import Path +import re +import stat +import sys +from typing import Callable, Iterator, NoReturn, Sequence + + +ENTRY_API = "agent-lab.local-image-entry/v0alpha1" +SNAPSHOT_API = "agent-lab.local-image-snapshot/v0alpha1" +CURRENT_API = "agent-lab.local-image-current/v0alpha1" +INTENT_API = "agent-lab.local-image-intent/v0alpha1" +ENTRY_DOMAIN = b"agent-lab.local-image-entry.v1\0" +SNAPSHOT_DOMAIN = b"agent-lab.local-image-snapshot.v1\0" +OPERATION_WRAPPER = "image-catalog-operation" +LOCK_MARKER = b"catalog:v0alpha1\n" +SAFE_COMPONENT = re.compile(r"^[a-z][a-z0-9-]{0,47}$") +IMAGE_COMPONENT = re.compile(r"^[a-z][a-z0-9]*(?:-[a-z0-9]+)*$") +SHA256 = re.compile(r"^sha256:[0-9a-f]{64}$") +OCI_SUBJECT = re.compile( + r"^([a-z0-9]+([.-][a-z0-9]+)*" + r"(:(?:[1-9][0-9]{0,3}|[1-5][0-9]{4}|6[0-4][0-9]{3}|" + r"65[0-4][0-9]{2}|655[0-2][0-9]|6553[0-5]))?/)?" + r"[a-z0-9]+([._-][a-z0-9]+)*" + r"(/[a-z0-9]+([._-][a-z0-9]+)*)*" + r"@sha256:[0-9a-f]{64}$" +) +HEX_FILE = re.compile(r"^[0-9a-f]{64}\.json$") +FaultHook = Callable[[str], None] + + +class CatalogError(Exception): + """Base class for classified catalog failures.""" + + +class CatalogReject(CatalogError): + """Stable invalid input, unknown name, conflict, or capacity refusal.""" + + exit_code = 1 + + +class CatalogInfrastructure(CatalogError): + """Unsafe, corrupt, changing, or uncertain catalog state.""" + + exit_code = 125 + + +@dataclass(frozen=True) +class CatalogLimits: + names: int = 256 + entries: int = 512 + snapshots: int = 512 + entry_bytes: int = 65_536 + snapshot_bytes: int = 262_144 + catalog_bytes: int = 67_108_864 + stage_entries: int = 16 + stage_bytes: int = 2_097_152 + + +@dataclass(frozen=True) +class HomeAuthority: + home: Path + images: Path + staging: Path + state: Path + locks: Path + lock: Path + + +@dataclass(frozen=True) +class CatalogState: + root: Path | None + revision: int + snapshot_digest: str | None + previous_snapshot_digest: str | None + records: dict[str, dict[str, object]] + entries: dict[str, dict[str, object]] + snapshots: dict[str, dict[str, object]] + physical_bytes: int + + +@dataclass(frozen=True) +class StageState: + path: Path + intent: dict[str, object] + + +def canonical(value: object) -> bytes: + return json.dumps( + value, + ensure_ascii=True, + allow_nan=False, + separators=(",", ":"), + sort_keys=True, + ).encode("ascii") + + +def record_digest(domain: bytes, value: object) -> str: + return "sha256:" + hashlib.sha256(domain + canonical(value)).hexdigest() + + +def image_name(value: object) -> bool: + if not isinstance(value, str) or not value.isascii(): + return False + encoded = value.encode("ascii") + parts = value.split(".") + return ( + len(encoded) <= 63 + and len(parts) == 2 + and all(1 <= len(part.encode("ascii")) <= 31 for part in parts) + and all(IMAGE_COMPONENT.fullmatch(part) is not None for part in parts) + ) + + +def oci_subject(value: object) -> bool: + return ( + isinstance(value, str) + and value.isascii() + and 1 <= len(value.encode("ascii")) <= 255 + and OCI_SUBJECT.fullmatch(value) is not None + ) + + +valid_image_name = image_name +valid_oci_subject = oci_subject + + +def _is_digest(value: object) -> bool: + return isinstance(value, str) and SHA256.fullmatch(value) is not None + + +def _reject(message: str) -> NoReturn: + raise CatalogReject(message) + + +def _infra(message: str, error: BaseException | None = None) -> NoReturn: + if error is None: + raise CatalogInfrastructure(message) + raise CatalogInfrastructure(message) from error + + +def _fault(hook: FaultHook | None, point: str) -> None: + if hook is not None: + hook(point) + + +def _directory_flags() -> int: + return ( + os.O_RDONLY + | getattr(os, "O_CLOEXEC", 0) + | getattr(os, "O_DIRECTORY", 0) + | getattr(os, "O_NOFOLLOW", 0) + ) + + +def _verify_directory(path: Path, *, mode: int = 0o700) -> os.stat_result: + try: + metadata = path.lstat() + except OSError as error: + _infra("required catalog directory is unavailable", error) + if ( + not stat.S_ISDIR(metadata.st_mode) + or stat.S_ISLNK(metadata.st_mode) + or metadata.st_uid != os.getuid() + or stat.S_IMODE(metadata.st_mode) != mode + ): + _infra("catalog directory metadata is unsafe") + try: + descriptor = os.open(path, _directory_flags()) + except OSError as error: + _infra("catalog directory cannot be opened safely", error) + try: + opened = os.fstat(descriptor) + finally: + os.close(descriptor) + if (opened.st_dev, opened.st_ino) != (metadata.st_dev, metadata.st_ino): + _infra("catalog directory identity changed") + return opened + + +def _verify_absolute_directory_chain(path: Path) -> None: + if not path.is_absolute() or path == Path(path.anchor): + _infra("Agent Lab home path is unsafe") + flags = _directory_flags() + try: + descriptor = os.open(path.anchor, flags) + except OSError as error: + _infra("Agent Lab home root cannot be opened", error) + try: + for component in path.parts[1:]: + child = os.open(component, flags, dir_fd=descriptor) + os.close(descriptor) + descriptor = child + except FileNotFoundError: + os.close(descriptor) + _reject("Agent Lab home is not initialized") + except OSError as error: + os.close(descriptor) + _infra("Agent Lab home path contains a symlink or non-directory", error) + else: + os.close(descriptor) + + +def _file_identity(metadata: os.stat_result) -> tuple[int, ...]: + return ( + metadata.st_dev, + metadata.st_ino, + metadata.st_uid, + stat.S_IMODE(metadata.st_mode), + metadata.st_nlink, + metadata.st_size, + metadata.st_mtime_ns, + metadata.st_ctime_ns, + ) + + +def _read_file(path: Path, maximum: int, purpose: str) -> bytes: + try: + lexical = path.lstat() + except OSError as error: + _infra(f"{purpose} is unavailable", error) + if ( + not stat.S_ISREG(lexical.st_mode) + or stat.S_ISLNK(lexical.st_mode) + or lexical.st_uid != os.getuid() + or stat.S_IMODE(lexical.st_mode) != 0o600 + or lexical.st_nlink != 1 + or lexical.st_size > maximum + ): + _infra(f"{purpose} metadata is unsafe or over-bound") + flags = ( + os.O_RDONLY + | getattr(os, "O_CLOEXEC", 0) + | getattr(os, "O_NOFOLLOW", 0) + | getattr(os, "O_NONBLOCK", 0) + ) + try: + descriptor = os.open(path, flags) + except OSError as error: + _infra(f"{purpose} cannot be opened safely", error) + try: + before = os.fstat(descriptor) + if _file_identity(before) != _file_identity(lexical): + _infra(f"{purpose} identity changed before read") + chunks: list[bytes] = [] + remaining = maximum + 1 + while remaining: + chunk = os.read(descriptor, min(65_536, remaining)) + if not chunk: + break + chunks.append(chunk) + remaining -= len(chunk) + data = b"".join(chunks) + after = os.fstat(descriptor) + except OSError as error: + _infra(f"{purpose} could not be read", error) + finally: + os.close(descriptor) + try: + final = path.lstat() + except OSError as error: + _infra(f"{purpose} could not be reverified", error) + if ( + len(data) > maximum + or len(data) != after.st_size + or _file_identity(before) != _file_identity(after) + or _file_identity(after) != _file_identity(final) + ): + _infra(f"{purpose} changed while being read") + return data + + +def _pairs(pairs: list[tuple[str, object]]) -> dict[str, object]: + value: dict[str, object] = {} + for key, item in pairs: + if key in value: + raise ValueError("duplicate JSON key") + value[key] = item + return value + + +def _canonical_json(data: bytes, purpose: str) -> dict[str, object]: + try: + value = json.loads(data.decode("utf-8"), object_pairs_hook=_pairs) + except (UnicodeError, ValueError, json.JSONDecodeError) as error: + _infra(f"{purpose} is malformed", error) + if not isinstance(value, dict) or data != canonical(value) + b"\n": + _infra(f"{purpose} is not one canonical closed object") + return value + + +def _load_home(home: Path) -> HomeAuthority: + _verify_absolute_directory_chain(home) + _verify_directory(home) + config_path = home / "config.json" + receipt_path = home / "home.json" + config_raw = _read_file(config_path, 65_536, "Agent Lab configuration") + receipt_raw = _read_file(receipt_path, 65_536, "Agent Lab home receipt") + config = _canonical_json(config_raw, "Agent Lab configuration") + receipt = _canonical_json(receipt_raw, "Agent Lab home receipt") + if set(config) != {"apiVersion", "paths"} or config.get("apiVersion") != "agent-lab.config/v0alpha1": + _infra("Agent Lab configuration schema is not closed") + paths = config.get("paths") + if ( + not isinstance(paths, dict) + or set(paths) != {"experiments", "images", "cache", "state"} + or len(set(paths.values())) != 4 + or any(not isinstance(item, str) or SAFE_COMPONENT.fullmatch(item) is None for item in paths.values()) + ): + _infra("Agent Lab configuration paths are unsafe") + digest = "sha256:" + hashlib.sha256(canonical(config)).hexdigest() + if ( + set(receipt) != {"apiVersion", "configDigest", "paths"} + or receipt.get("apiVersion") != "agent-lab.home/v0alpha1" + or receipt.get("configDigest") != digest + or receipt.get("paths") != paths + ): + _infra("Agent Lab configuration does not match its home receipt") + images = home / str(paths["images"]) + state = home / str(paths["state"]) + staging = images / ".staging" + locks = state / "locks" + for path in (images, state, staging, locks): + _verify_directory(path) + lock = locks / "image-catalog.lock" + return HomeAuthority(home, images, staging, state, locks, lock) + + +@contextmanager +def _catalog_lock(authority: HomeAuthority, *, exclusive: bool) -> Iterator[int]: + path = authority.lock + try: + lexical = path.lstat() + except OSError as error: + _infra("catalog lock is unavailable", error) + if ( + not stat.S_ISREG(lexical.st_mode) + or stat.S_ISLNK(lexical.st_mode) + or lexical.st_uid != os.getuid() + or stat.S_IMODE(lexical.st_mode) != 0o600 + or lexical.st_nlink != 1 + or lexical.st_size > len(LOCK_MARKER) + ): + _infra("catalog lock metadata is unsafe") + try: + descriptor = os.open( + path, + os.O_RDWR | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_NOFOLLOW", 0), + ) + opened = os.fstat(descriptor) + if _file_identity(opened) != _file_identity(lexical): + raise OSError("lock identity changed") + fcntl.flock(descriptor, fcntl.LOCK_EX if exclusive else fcntl.LOCK_SH) + current = path.lstat() + held = os.fstat(descriptor) + if (held.st_dev, held.st_ino) != (current.st_dev, current.st_ino): + raise OSError("lock path was replaced") + except OSError as error: + try: + os.close(descriptor) + except (NameError, OSError): + pass + _infra("catalog lock cannot be held safely", error) + try: + yield descriptor + finally: + try: + fcntl.flock(descriptor, fcntl.LOCK_UN) + os.close(descriptor) + except OSError as error: + _infra("catalog lock could not be released", error) + + +def _lock_bytes(descriptor: int) -> bytes: + try: + os.lseek(descriptor, 0, os.SEEK_SET) + data = os.read(descriptor, len(LOCK_MARKER) + 1) + os.lseek(descriptor, 0, os.SEEK_SET) + except OSError as error: + _infra("catalog lock marker cannot be read", error) + if data not in (b"", LOCK_MARKER): + _infra("catalog lock marker is malformed") + return data + + +def _set_lock_marker(descriptor: int) -> None: + try: + os.lseek(descriptor, 0, os.SEEK_SET) + os.ftruncate(descriptor, 0) + _write_all(descriptor, LOCK_MARKER) + os.fsync(descriptor) + os.lseek(descriptor, 0, os.SEEK_SET) + except OSError as error: + _infra("catalog initialization marker could not be persisted", error) + + +def _write_all(descriptor: int, data: bytes) -> None: + view = memoryview(data) + while view: + written = os.write(descriptor, view) + if written <= 0: + raise OSError("write made no progress") + view = view[written:] + + +def _directory_names(path: Path, purpose: str) -> tuple[str, ...]: + _verify_directory(path) + try: + before = path.lstat() + names = tuple(sorted(os.listdir(path), key=os.fsencode)) + after = path.lstat() + except (OSError, UnicodeError) as error: + _infra(f"{purpose} cannot be enumerated", error) + if (before.st_dev, before.st_ino, before.st_mtime_ns, before.st_ctime_ns) != ( + after.st_dev, + after.st_ino, + after.st_mtime_ns, + after.st_ctime_ns, + ): + _infra(f"{purpose} changed while being enumerated") + return names + + +def _entry_schema(value: dict[str, object]) -> None: + expected = { + "apiVersion", + "generation", + "name", + "previousEntryDigest", + "state", + "subject", + "subjectDigest", + } + if set(value) != expected or value.get("apiVersion") != ENTRY_API: + _infra("local image entry schema is not closed") + name = value.get("name") + generation = value.get("generation") + state = value.get("state") + previous = value.get("previousEntryDigest") + subject = value.get("subject") + subject_digest = value.get("subjectDigest") + if ( + not image_name(name) + or not isinstance(name, str) + or name.startswith("agent-lab.") + or not oci_subject(subject) + or not isinstance(subject, str) + or subject_digest != subject.rsplit("@", 1)[1] + ): + _infra("local image entry contains an invalid binding") + if state == "active": + if generation != 1 or isinstance(generation, bool) or previous is not None: + _infra("active local image entry generation is invalid") + elif state == "removed": + if generation != 2 or isinstance(generation, bool) or not _is_digest(previous): + _infra("removed local image entry generation is invalid") + else: + _infra("local image entry state is unknown") + + +def _snapshot_schema(value: dict[str, object]) -> None: + if ( + set(value) != { + "apiVersion", + "previousSnapshotDigest", + "records", + "revision", + } + or value.get("apiVersion") != SNAPSHOT_API + ): + _infra("local image snapshot schema is not closed") + revision = value.get("revision") + previous = value.get("previousSnapshotDigest") + records = value.get("records") + if ( + not isinstance(revision, int) + or isinstance(revision, bool) + or revision < 1 + or (revision == 1 and previous is not None) + or (revision > 1 and not _is_digest(previous)) + or not isinstance(records, dict) + or len(records) > 256 + ): + _infra("local image snapshot values are invalid") + if tuple(records) != tuple(sorted(records, key=lambda item: item.encode("ascii", "strict"))): + _infra("local image snapshot records are not byte-sorted") + for name, projection in records.items(): + if ( + not image_name(name) + or name.startswith("agent-lab.") + or not isinstance(projection, dict) + or set(projection) != {"entryDigest", "generation", "state"} + or not _is_digest(projection.get("entryDigest")) + or projection.get("generation") not in (1, 2) + or isinstance(projection.get("generation"), bool) + or projection.get("state") not in ("active", "removed") + ): + _infra("local image snapshot record projection is invalid") + + +def _current_schema(value: dict[str, object]) -> str: + if ( + set(value) != {"apiVersion", "snapshotDigest"} + or value.get("apiVersion") != CURRENT_API + or not _is_digest(value.get("snapshotDigest")) + ): + _infra("local image current pointer schema is not closed") + digest = value["snapshotDigest"] + assert isinstance(digest, str) + return digest + + +def _intent_schema(value: dict[str, object]) -> None: + expected = { + "apiVersion", + "baseSnapshotDigest", + "bootstrap", + "candidateEntryDigest", + "candidateSnapshotDigest", + "entryPreexisting", + "expectedEntryDigest", + "kind", + "name", + "snapshotPreexisting", + "subject", + } + if set(value) != expected or value.get("apiVersion") != INTENT_API: + _infra("catalog staging intent schema is not closed") + bootstrap = value.get("bootstrap") + kind = value.get("kind") + base = value.get("baseSnapshotDigest") + expected_entry = value.get("expectedEntryDigest") + if ( + not isinstance(bootstrap, bool) + or kind not in ("add", "remove") + or not image_name(value.get("name")) + or str(value.get("name")).startswith("agent-lab.") + or not oci_subject(value.get("subject")) + or not _is_digest(value.get("candidateEntryDigest")) + or not _is_digest(value.get("candidateSnapshotDigest")) + or not isinstance(value.get("entryPreexisting"), bool) + or not isinstance(value.get("snapshotPreexisting"), bool) + or (bootstrap and base is not None) + or (not bootstrap and not _is_digest(base)) + or (kind == "add" and expected_entry is not None) + or (kind == "remove" and not _is_digest(expected_entry)) + ): + _infra("catalog staging intent values are invalid") + + +def _stage_state(authority: HomeAuthority, limits: CatalogLimits) -> StageState | None: + names = _directory_names(authority.staging, "catalog staging root") + if not names: + return None + if names != (OPERATION_WRAPPER,): + _infra("catalog staging root contains an unknown wrapper") + wrapper = authority.staging / OPERATION_WRAPPER + _verify_directory(wrapper) + wrapper_names = _directory_names(wrapper, "catalog operation wrapper") + if "intent.json" not in wrapper_names or any(name not in {"intent.json", "payload"} for name in wrapper_names): + _infra("catalog operation wrapper is incomplete or unknown") + intent = _canonical_json( + _read_file(wrapper / "intent.json", 65_536, "catalog operation intent"), + "catalog operation intent", + ) + _intent_schema(intent) + entry_count = 1 + byte_count = (wrapper / "intent.json").lstat().st_size + payload = wrapper / "payload" + if payload.exists() or payload.is_symlink(): + _verify_directory(payload) + allowed: set[str] + if intent["bootstrap"]: + allowed = { + "catalog", + "catalog/current.json", + "catalog/current.next", + "catalog/entries", + f"catalog/entries/{str(intent['candidateEntryDigest'])[7:]}.json", + "catalog/snapshots", + f"catalog/snapshots/{str(intent['candidateSnapshotDigest'])[7:]}.json", + } + else: + allowed = {"entry.json", "snapshot.json", "current.json", "current.next"} + try: + descendants = sorted(payload.rglob("*"), key=lambda item: os.fsencode(str(item.relative_to(payload)))) + except OSError as error: + _infra("catalog staging payload cannot be enumerated", error) + for item in descendants: + relative = str(item.relative_to(payload)) + if relative not in allowed: + _infra("catalog staging payload contains an unknown entry") + metadata = item.lstat() + entry_count += 1 + if metadata.st_uid != os.getuid() or stat.S_ISLNK(metadata.st_mode): + _infra("catalog staging payload metadata is unsafe") + if stat.S_ISDIR(metadata.st_mode): + if stat.S_IMODE(metadata.st_mode) != 0o700: + _infra("catalog staging directory mode is unsafe") + elif stat.S_ISREG(metadata.st_mode): + if stat.S_IMODE(metadata.st_mode) != 0o600 or metadata.st_nlink != 1: + _infra("catalog staging file metadata is unsafe") + byte_count += metadata.st_size + else: + _infra("catalog staging payload type is unsafe") + if entry_count > limits.stage_entries or byte_count > limits.stage_bytes: + _infra("catalog staging state exceeds its fixed bound") + return StageState(wrapper, intent) + + +def _public_record(entry: dict[str, object], entry_digest: str) -> dict[str, object]: + return { + "entryDigest": entry_digest, + "generation": entry["generation"], + "name": entry["name"], + "previousEntryDigest": entry["previousEntryDigest"], + "state": entry["state"], + "subject": entry["subject"], + } + + +def _projection(entry: dict[str, object], entry_digest: str) -> dict[str, object]: + return { + "entryDigest": entry_digest, + "generation": entry["generation"], + "state": entry["state"], + } + + +def _validate_transition( + previous: dict[str, object] | None, + current: dict[str, object], + entries: dict[str, dict[str, object]], +) -> None: + records = current["records"] + assert isinstance(records, dict) + if previous is None: + if current["revision"] != 1 or len(records) != 1: + _infra("catalog genesis snapshot is invalid") + only_name = next(iter(records)) + projection = records[only_name] + assert isinstance(projection, dict) + entry = entries[str(projection["entryDigest"])] + if entry["state"] != "active" or entry["generation"] != 1: + _infra("catalog genesis entry is invalid") + return + previous_records = previous["records"] + assert isinstance(previous_records, dict) + if current["revision"] != int(previous["revision"]) + 1: + _infra("catalog snapshot revision chain is not monotonic") + changed = sorted(set(previous_records) | set(records)) + changed = [name for name in changed if previous_records.get(name) != records.get(name)] + if len(changed) != 1: + _infra("catalog snapshot does not contain one valid transition") + name = changed[0] + before = previous_records.get(name) + after = records.get(name) + if not isinstance(after, dict): + _infra("catalog transition deleted a logical name") + entry = entries[str(after["entryDigest"])] + if before is None: + if entry["state"] != "active" or entry["generation"] != 1 or entry["previousEntryDigest"] is not None: + _infra("catalog add transition is invalid") + return + if not isinstance(before, dict): + _infra("catalog previous projection is invalid") + if ( + before.get("state") != "active" + or before.get("generation") != 1 + or entry["state"] != "removed" + or entry["generation"] != 2 + or entry["previousEntryDigest"] != before.get("entryDigest") + ): + _infra("catalog removal transition is invalid") + prior_entry = entries.get(str(before.get("entryDigest"))) + if prior_entry is None or prior_entry["subject"] != entry["subject"]: + _infra("catalog tombstone changed its immutable subject") + + +def _load_catalog( + authority: HomeAuthority, + lock_descriptor: int, + limits: CatalogLimits, + *, + inspect_stage: bool = True, +) -> CatalogState: + stage = _stage_state(authority, limits) if inspect_stage else None + marker = _lock_bytes(lock_descriptor) + image_names = _directory_names(authority.images, "effective images directory") + allowed_images = {".staging", "catalog"} + if any(name not in allowed_images for name in image_names): + _infra("effective images directory contains unknown catalog state") + root = authority.images / "catalog" + try: + root_metadata = root.lstat() + except FileNotFoundError: + if marker == LOCK_MARKER: + _infra("an initialized image catalog is missing") + if stage is not None and not bool(stage.intent.get("bootstrap")): + _infra("non-bootstrap catalog stage has no committed base") + return CatalogState(None, 0, None, None, {}, {}, {}, 0) + except OSError as error: + _infra("image catalog cannot be inspected", error) + if not stat.S_ISDIR(root_metadata.st_mode) or stat.S_ISLNK(root_metadata.st_mode): + _infra("image catalog path is unsafe") + _verify_directory(root) + root_names = _directory_names(root, "image catalog") + if set(root_names) != {"current.json", "entries", "snapshots"}: + _infra("image catalog layout is incomplete or unknown") + entries_root = root / "entries" + snapshots_root = root / "snapshots" + entry_names = _directory_names(entries_root, "image entry history") + snapshot_names = _directory_names(snapshots_root, "image snapshot history") + if ( + len(entry_names) > limits.entries + or len(snapshot_names) > limits.snapshots + or any(HEX_FILE.fullmatch(name) is None for name in entry_names) + or any(HEX_FILE.fullmatch(name) is None for name in snapshot_names) + ): + _infra("physical image catalog history exceeds or violates its fixed shape") + physical_bytes = 0 + for path in [root / "current.json", *[entries_root / name for name in entry_names], *[snapshots_root / name for name in snapshot_names]]: + try: + metadata = path.lstat() + except OSError as error: + _infra("physical image catalog inventory changed", error) + physical_bytes += metadata.st_size + if physical_bytes > limits.catalog_bytes: + _infra("physical image catalog exceeds its fixed byte bound") + + entries: dict[str, dict[str, object]] = {} + for name in entry_names: + path = entries_root / name + value = _canonical_json( + _read_file(path, limits.entry_bytes, "local image entry record"), + "local image entry record", + ) + _entry_schema(value) + digest = record_digest(ENTRY_DOMAIN, value) + if name != f"{digest[7:]}.json" or digest in entries: + _infra("local image entry filename does not bind its bytes") + entries[digest] = value + + snapshots: dict[str, dict[str, object]] = {} + for name in snapshot_names: + path = snapshots_root / name + value = _canonical_json( + _read_file(path, limits.snapshot_bytes, "local image snapshot record"), + "local image snapshot record", + ) + _snapshot_schema(value) + digest = record_digest(SNAPSHOT_DOMAIN, value) + if name != f"{digest[7:]}.json" or digest in snapshots: + _infra("local image snapshot filename does not bind its bytes") + snapshots[digest] = value + + for snapshot_digest, snapshot in snapshots.items(): + records = snapshot["records"] + assert isinstance(records, dict) + for name, projection in records.items(): + assert isinstance(projection, dict) + entry_digest = str(projection["entryDigest"]) + entry = entries.get(entry_digest) + if ( + entry is None + or entry["name"] != name + or projection != _projection(entry, entry_digest) + ): + _infra("local image snapshot references invalid entry history") + previous_digest = snapshot["previousSnapshotDigest"] + previous = snapshots.get(str(previous_digest)) if previous_digest is not None else None + if previous_digest is not None and previous is None: + _infra("local image snapshot predecessor is missing") + _validate_transition(previous, snapshot, entries) + + pointer = _canonical_json( + _read_file(root / "current.json", 65_536, "local image current pointer"), + "local image current pointer", + ) + current_digest = _current_schema(pointer) + current = snapshots.get(current_digest) + if current is None: + _infra("local image current pointer references missing history") + seen: set[str] = set() + cursor_digest: str | None = current_digest + while cursor_digest is not None: + if cursor_digest in seen: + _infra("local image snapshot history contains a cycle") + seen.add(cursor_digest) + cursor = snapshots.get(cursor_digest) + if cursor is None: + _infra("local image snapshot history is incomplete") + previous = cursor["previousSnapshotDigest"] + cursor_digest = str(previous) if previous is not None else None + + records: dict[str, dict[str, object]] = {} + current_records = current["records"] + assert isinstance(current_records, dict) + if len(current_records) > limits.names: + _infra("logical image catalog exceeds its fixed name bound") + for name, projection in current_records.items(): + assert isinstance(projection, dict) + digest = str(projection["entryDigest"]) + records[name] = _public_record(entries[digest], digest) + if ( + entry_names != _directory_names(entries_root, "image entry history") + or snapshot_names != _directory_names(snapshots_root, "image snapshot history") + or root_names != _directory_names(root, "image catalog") + ): + _infra("image catalog changed during verification") + return CatalogState( + root, + int(current["revision"]), + current_digest, + current["previousSnapshotDigest"] if isinstance(current["previousSnapshotDigest"], str) else None, + records, + entries, + snapshots, + physical_bytes, + ) + + +def _write_file(path: Path, data: bytes, purpose: str, fault: FaultHook | None = None) -> None: + flags = ( + os.O_WRONLY + | os.O_CREAT + | os.O_EXCL + | getattr(os, "O_CLOEXEC", 0) + | getattr(os, "O_NOFOLLOW", 0) + ) + descriptor = -1 + try: + descriptor = os.open(path, flags, 0o600) + _write_all(descriptor, data) + _fault(fault, f"{purpose}.before_fsync") + os.fsync(descriptor) + _fault(fault, f"{purpose}.after_fsync") + metadata = os.fstat(descriptor) + if ( + not stat.S_ISREG(metadata.st_mode) + or metadata.st_uid != os.getuid() + or stat.S_IMODE(metadata.st_mode) != 0o600 + or metadata.st_nlink != 1 + or metadata.st_size != len(data) + ): + raise OSError("published file metadata is unsafe") + except OSError as error: + _infra(f"{purpose} could not be persisted", error) + finally: + if descriptor >= 0: + try: + os.close(descriptor) + except OSError as error: + _infra(f"{purpose} descriptor could not be closed", error) + + +def _fsync_directory(path: Path, purpose: str, fault: FaultHook | None = None) -> None: + descriptor = -1 + try: + descriptor = os.open(path, _directory_flags()) + _fault(fault, f"{purpose}.before_fsync") + os.fsync(descriptor) + _fault(fault, f"{purpose}.after_fsync") + except OSError as error: + _infra(f"{purpose} directory durability is uncertain", error) + finally: + if descriptor >= 0: + try: + os.close(descriptor) + except OSError as error: + _infra(f"{purpose} directory descriptor could not be closed", error) + + +def _mkdir(path: Path) -> None: + try: + path.mkdir(mode=0o700) + except OSError as error: + _infra("catalog staging directory could not be created", error) + _verify_directory(path) + + +def _rename_noreplace(source: Path, target: Path) -> None: + try: + library = ctypes.CDLL(None, use_errno=True) + function = library.renameat2 + except (AttributeError, OSError) as error: + _infra("Linux no-replace rename is unavailable", error) + function.argtypes = [ctypes.c_int, ctypes.c_char_p, ctypes.c_int, ctypes.c_char_p, ctypes.c_uint] + function.restype = ctypes.c_int + result = function( + -100, + os.fsencode(source), + -100, + os.fsencode(target), + 1, + ) + if result != 0: + code = ctypes.get_errno() + _infra("catalog no-replace publication failed", OSError(code, os.strerror(code))) + + +def _remove_owned_tree(path: Path) -> None: + try: + metadata = path.lstat() + except FileNotFoundError: + return + except OSError as error: + _infra("catalog staging cleanup cannot inspect its target", error) + if stat.S_ISLNK(metadata.st_mode) or metadata.st_uid != os.getuid(): + _infra("catalog staging cleanup target is unsafe") + if stat.S_ISREG(metadata.st_mode): + if metadata.st_nlink != 1 or stat.S_IMODE(metadata.st_mode) != 0o600: + _infra("catalog staging cleanup file is unsafe") + try: + path.unlink() + except OSError as error: + _infra("catalog staging file could not be cleaned", error) + return + if not stat.S_ISDIR(metadata.st_mode) or stat.S_IMODE(metadata.st_mode) != 0o700: + _infra("catalog staging cleanup type is unsafe") + try: + children = tuple(path.iterdir()) + except OSError as error: + _infra("catalog staging cleanup cannot enumerate its target", error) + for child in sorted(children, key=lambda item: os.fsencode(item.name)): + _remove_owned_tree(child) + try: + path.rmdir() + except OSError as error: + _infra("catalog staging directory could not be cleaned", error) + + +def _cleanup_stage(authority: HomeAuthority, stage: StageState | None = None) -> None: + path = authority.staging / OPERATION_WRAPPER + if stage is not None and stage.path != path: + _infra("catalog staging cleanup target changed") + _remove_owned_tree(path) + _fsync_directory(authority.staging, "catalog staging cleanup") + + +def _read_current_digest(authority: HomeAuthority) -> str | None: + root = authority.images / "catalog" + try: + metadata = root.lstat() + except FileNotFoundError: + return None + except OSError as error: + _infra("catalog commit state cannot be inspected", error) + if not stat.S_ISDIR(metadata.st_mode) or stat.S_ISLNK(metadata.st_mode): + _infra("catalog commit state is unsafe") + pointer = _canonical_json( + _read_file(root / "current.json", 65_536, "local image current pointer"), + "local image current pointer", + ) + return _current_schema(pointer) + + +def _remove_uncommitted_record(path: Path, preexisting: bool) -> None: + try: + metadata = path.lstat() + except FileNotFoundError: + return + except OSError as error: + _infra("uncommitted catalog record cannot be inspected", error) + if preexisting: + return + if ( + not stat.S_ISREG(metadata.st_mode) + or stat.S_ISLNK(metadata.st_mode) + or metadata.st_uid != os.getuid() + or stat.S_IMODE(metadata.st_mode) != 0o600 + or metadata.st_nlink != 1 + ): + _infra("uncommitted catalog record metadata is unsafe") + try: + path.unlink() + except OSError as error: + _infra("uncommitted catalog record could not be reconciled", error) + + +def _reconcile(authority: HomeAuthority, lock_descriptor: int, limits: CatalogLimits) -> None: + stage = _stage_state(authority, limits) + if stage is None: + return + intent = stage.intent + candidate = str(intent["candidateSnapshotDigest"]) + base_value = intent["baseSnapshotDigest"] + base = str(base_value) if base_value is not None else None + current = _read_current_digest(authority) + if current == candidate: + _load_catalog(authority, lock_descriptor, limits, inspect_stage=False) + if _lock_bytes(lock_descriptor) == b"": + _set_lock_marker(lock_descriptor) + _cleanup_stage(authority, stage) + return + if current != base: + _infra("catalog staged operation cannot be reconciled with current state") + if bool(intent["bootstrap"]): + if current is not None: + _infra("catalog bootstrap stage conflicts with a final catalog") + else: + root = authority.images / "catalog" + entries = root / "entries" + snapshots = root / "snapshots" + _remove_uncommitted_record( + entries / f"{str(intent['candidateEntryDigest'])[7:]}.json", + bool(intent["entryPreexisting"]), + ) + _remove_uncommitted_record( + snapshots / f"{candidate[7:]}.json", + bool(intent["snapshotPreexisting"]), + ) + _fsync_directory(entries, "catalog reconciled entry history") + _fsync_directory(snapshots, "catalog reconciled snapshot history") + _cleanup_stage(authority, stage) + + +def _candidate_values( + state: CatalogState, + *, + kind: str, + name: str, + subject: str, + expected_entry_digest: str | None, + limits: CatalogLimits, +) -> tuple[dict[str, object], str, dict[str, object], str, dict[str, object]]: + prior = state.records.get(name) + if kind == "add": + if prior is not None: + raise AssertionError("add candidate requested for an existing name") + if len(state.records) >= limits.names: + _reject("catalog has reached its fixed logical-name capacity") + entry = { + "apiVersion": ENTRY_API, + "generation": 1, + "name": name, + "previousEntryDigest": None, + "state": "active", + "subject": subject, + "subjectDigest": subject.rsplit("@", 1)[1], + } + else: + if prior is None: + raise AssertionError("remove candidate requested for an unknown name") + entry = { + "apiVersion": ENTRY_API, + "generation": 2, + "name": name, + "previousEntryDigest": expected_entry_digest, + "state": "removed", + "subject": prior["subject"], + "subjectDigest": str(prior["subject"]).rsplit("@", 1)[1], + } + entry_digest = record_digest(ENTRY_DOMAIN, entry) + projections: dict[str, dict[str, object]] = {} + for existing_name in sorted(state.records, key=lambda item: item.encode("ascii")): + record = state.records[existing_name] + projections[existing_name] = { + "entryDigest": record["entryDigest"], + "generation": record["generation"], + "state": record["state"], + } + projections[name] = _projection(entry, entry_digest) + projections = { + item: projections[item] + for item in sorted(projections, key=lambda value: value.encode("ascii")) + } + snapshot = { + "apiVersion": SNAPSHOT_API, + "previousSnapshotDigest": state.snapshot_digest, + "records": projections, + "revision": state.revision + 1, + } + snapshot_digest = record_digest(SNAPSHOT_DOMAIN, snapshot) + pointer = {"apiVersion": CURRENT_API, "snapshotDigest": snapshot_digest} + entry_bytes = canonical(entry) + b"\n" + snapshot_bytes = canonical(snapshot) + b"\n" + pointer_bytes = canonical(pointer) + b"\n" + if len(entry_bytes) > limits.entry_bytes or len(snapshot_bytes) > limits.snapshot_bytes: + _reject("catalog mutation exceeds its fixed record bound") + new_entry = entry_digest not in state.entries + new_snapshot = snapshot_digest not in state.snapshots + if len(state.entries) + int(new_entry) > limits.entries or len(state.snapshots) + int(new_snapshot) > limits.snapshots: + _reject("catalog mutation exceeds its fixed history bound") + prospective = state.physical_bytes + int(new_entry) * len(entry_bytes) + int(new_snapshot) * len(snapshot_bytes) + if state.root is None: + prospective += len(pointer_bytes) + if prospective > limits.catalog_bytes: + _reject("catalog mutation exceeds its fixed physical byte bound") + intent = { + "apiVersion": INTENT_API, + "baseSnapshotDigest": state.snapshot_digest, + "bootstrap": state.root is None, + "candidateEntryDigest": entry_digest, + "candidateSnapshotDigest": snapshot_digest, + "entryPreexisting": not new_entry, + "expectedEntryDigest": expected_entry_digest, + "kind": kind, + "name": name, + "snapshotPreexisting": not new_snapshot, + "subject": subject, + } + _intent_schema(intent) + return entry, entry_digest, snapshot, snapshot_digest, intent + + +def _prepare_stage( + authority: HomeAuthority, + entry: dict[str, object], + entry_digest: str, + snapshot: dict[str, object], + snapshot_digest: str, + intent: dict[str, object], + fault: FaultHook | None, +) -> StageState: + wrapper = authority.staging / OPERATION_WRAPPER + payload = wrapper / "payload" + try: + _mkdir(wrapper) + _write_file(wrapper / "intent.json", canonical(intent) + b"\n", "catalog intent", fault) + _fsync_directory(wrapper, "catalog intent wrapper", fault) + _fsync_directory(authority.staging, "catalog intent publication", fault) + _mkdir(payload) + if bool(intent["bootstrap"]): + catalog = payload / "catalog" + entries = catalog / "entries" + snapshots = catalog / "snapshots" + _mkdir(catalog) + _mkdir(entries) + _mkdir(snapshots) + _write_file( + entries / f"{entry_digest[7:]}.json", + canonical(entry) + b"\n", + "catalog staged entry", + fault, + ) + _write_file( + snapshots / f"{snapshot_digest[7:]}.json", + canonical(snapshot) + b"\n", + "catalog staged snapshot", + fault, + ) + pointer = {"apiVersion": CURRENT_API, "snapshotDigest": snapshot_digest} + _write_file(catalog / "current.next", canonical(pointer) + b"\n", "catalog staged pointer", fault) + os.replace(catalog / "current.next", catalog / "current.json") + _fsync_directory(entries, "catalog staged entries", fault) + _fsync_directory(snapshots, "catalog staged snapshots", fault) + _fsync_directory(catalog, "catalog staged root", fault) + else: + _write_file(payload / "entry.json", canonical(entry) + b"\n", "catalog staged entry", fault) + _write_file(payload / "snapshot.json", canonical(snapshot) + b"\n", "catalog staged snapshot", fault) + pointer = {"apiVersion": CURRENT_API, "snapshotDigest": snapshot_digest} + _write_file(payload / "current.next", canonical(pointer) + b"\n", "catalog staged pointer", fault) + os.replace(payload / "current.next", payload / "current.json") + _fsync_directory(payload, "catalog staged payload", fault) + _fsync_directory(wrapper, "catalog staged wrapper", fault) + _fsync_directory(authority.staging, "catalog staged operation", fault) + except CatalogInfrastructure: + try: + _remove_owned_tree(wrapper) + _fsync_directory(authority.staging, "catalog failed-stage cleanup") + except CatalogInfrastructure: + pass + raise + except OSError as error: + try: + _remove_owned_tree(wrapper) + _fsync_directory(authority.staging, "catalog failed-stage cleanup") + except CatalogInfrastructure: + pass + _infra("catalog operation could not be staged", error) + stage = _stage_state(authority, CatalogLimits()) + if stage is None: + _infra("catalog operation stage disappeared") + return stage + + +def _publish_record(source: Path, target: Path, maximum: int, purpose: str, fault: FaultHook | None) -> None: + data = _read_file(source, maximum, purpose) + try: + target.lstat() + except FileNotFoundError: + _write_file(target, data, purpose, fault) + except OSError as error: + _infra(f"{purpose} final path cannot be inspected", error) + else: + if _read_file(target, maximum, purpose) != data: + _infra(f"{purpose} conflicts with existing immutable bytes") + + +def _publish_candidate( + authority: HomeAuthority, + lock_descriptor: int, + state: CatalogState, + stage: StageState, + limits: CatalogLimits, + fault: FaultHook | None, +) -> None: + intent = stage.intent + candidate = str(intent["candidateSnapshotDigest"]) + payload = stage.path / "payload" + committed = False + try: + if bool(intent["bootstrap"]): + _fault(fault, "bootstrap.before_rename") + _rename_noreplace(payload / "catalog", authority.images / "catalog") + committed = True + _fault(fault, "bootstrap.after_rename") + _fsync_directory(authority.images, "catalog bootstrap parent", fault) + _set_lock_marker(lock_descriptor) + else: + assert state.root is not None + root = state.root + entries = root / "entries" + snapshots = root / "snapshots" + _publish_record( + payload / "entry.json", + entries / f"{str(intent['candidateEntryDigest'])[7:]}.json", + limits.entry_bytes, + "catalog immutable entry", + fault, + ) + _fsync_directory(entries, "catalog immutable entry history", fault) + _publish_record( + payload / "snapshot.json", + snapshots / f"{candidate[7:]}.json", + limits.snapshot_bytes, + "catalog immutable snapshot", + fault, + ) + _fsync_directory(snapshots, "catalog immutable snapshot history", fault) + if _read_current_digest(authority) != state.snapshot_digest: + _infra("catalog current pointer changed before publication") + _fault(fault, "pointer.before_replace") + os.replace(payload / "current.json", root / "current.json") + committed = True + _fault(fault, "pointer.after_replace") + _fsync_directory(root, "catalog current pointer", fault) + verified = _load_catalog(authority, lock_descriptor, limits, inspect_stage=False) + if verified.snapshot_digest != candidate: + _infra("catalog publication could not be verified") + _cleanup_stage(authority, stage) + except CatalogInfrastructure: + if not committed: + try: + _reconcile(authority, lock_descriptor, limits) + except CatalogInfrastructure: + pass + raise + except OSError as error: + if not committed: + try: + _reconcile(authority, lock_descriptor, limits) + except CatalogInfrastructure: + pass + _infra("catalog publication is uncertain", error) + + +def _classified(operation: Callable[[], object]) -> object: + try: + return operation() + except (CatalogReject, CatalogInfrastructure): + raise + except (AssertionError, KeyError, OSError, TypeError, ValueError, json.JSONDecodeError) as error: + _infra("catalog operation encountered unverified state", error) + + +def add_image( + home: Path, + name: str, + subject: str, + *, + fault: FaultHook | None = None, + limits: CatalogLimits = CatalogLimits(), +) -> dict[str, object]: + """Create a generation-one mapping, or observe its exact idempotent retry.""" + + def operation() -> dict[str, object]: + if sys.platform != "linux": + _infra("local image catalog mutations require Linux") + if not image_name(name) or name.startswith("agent-lab.") or not oci_subject(subject): + _reject("mapping is invalid or reserved") + authority = _load_home(home) + with _catalog_lock(authority, exclusive=True) as lock_descriptor: + _reconcile(authority, lock_descriptor, limits) + state = _load_catalog(authority, lock_descriptor, limits) + prior = state.records.get(name) + if prior is not None: + if prior["state"] == "active" and prior["subject"] == subject: + return { + "changed": False, + "entryDigest": prior["entryDigest"], + "generation": 1, + } + _reject("name already exists or is tombstoned") + entry, entry_digest, snapshot, snapshot_digest, intent = _candidate_values( + state, + kind="add", + name=name, + subject=subject, + expected_entry_digest=None, + limits=limits, + ) + stage = _prepare_stage( + authority, + entry, + entry_digest, + snapshot, + snapshot_digest, + intent, + fault, + ) + _publish_candidate(authority, lock_descriptor, state, stage, limits, fault) + return {"changed": True, "entryDigest": entry_digest, "generation": 1} + + result = _classified(operation) + assert isinstance(result, dict) + return result + + +def remove_image( + home: Path, + name: str, + expected_entry_digest: str, + *, + fault: FaultHook | None = None, + limits: CatalogLimits = CatalogLimits(), +) -> dict[str, object]: + """Publish the one permitted generation-two CAS tombstone.""" + + def operation() -> dict[str, object]: + if sys.platform != "linux": + _infra("local image catalog mutations require Linux") + if not image_name(name) or name.startswith("agent-lab.") or not _is_digest(expected_entry_digest): + _reject("remove request is invalid or reserved") + authority = _load_home(home) + with _catalog_lock(authority, exclusive=True) as lock_descriptor: + _reconcile(authority, lock_descriptor, limits) + state = _load_catalog(authority, lock_descriptor, limits) + prior = state.records.get(name) + if prior is None: + _reject("name is unknown") + if prior["state"] == "removed": + if prior["previousEntryDigest"] == expected_entry_digest: + return { + "changed": False, + "entryDigest": prior["entryDigest"], + "generation": 2, + "state": "removed", + } + _reject("remove compare-and-swap conflict") + if prior["entryDigest"] != expected_entry_digest: + _reject("remove compare-and-swap conflict") + subject = prior["subject"] + assert isinstance(subject, str) + entry, entry_digest, snapshot, snapshot_digest, intent = _candidate_values( + state, + kind="remove", + name=name, + subject=subject, + expected_entry_digest=expected_entry_digest, + limits=limits, + ) + stage = _prepare_stage( + authority, + entry, + entry_digest, + snapshot, + snapshot_digest, + intent, + fault, + ) + _publish_candidate(authority, lock_descriptor, state, stage, limits, fault) + return { + "changed": True, + "entryDigest": entry_digest, + "generation": 2, + "state": "removed", + } + + result = _classified(operation) + assert isinstance(result, dict) + return result + + +def list_images( + home: Path, + *, + include_removed: bool = False, + limits: CatalogLimits = CatalogLimits(), +) -> list[dict[str, object]]: + """Return a canonical byte-sorted held view without repair or initialization.""" + + def operation() -> list[dict[str, object]]: + authority = _load_home(home) + with _catalog_lock(authority, exclusive=False) as lock_descriptor: + state = _load_catalog(authority, lock_descriptor, limits) + return [ + dict(state.records[name]) + for name in sorted(state.records, key=lambda item: item.encode("ascii")) + if include_removed or state.records[name]["state"] == "active" + ] + + result = _classified(operation) + assert isinstance(result, list) + return result + + +def inspect_image( + home: Path, + name: str, + *, + limits: CatalogLimits = CatalogLimits(), +) -> dict[str, object]: + """Return one exact active/tombstoned record without repair.""" + + def operation() -> dict[str, object]: + if not image_name(name) or name.startswith("agent-lab."): + _reject("name is invalid or reserved") + authority = _load_home(home) + with _catalog_lock(authority, exclusive=False) as lock_descriptor: + state = _load_catalog(authority, lock_descriptor, limits) + record = state.records.get(name) + if record is None: + _reject("name is unknown") + return dict(record) + + result = _classified(operation) + assert isinstance(result, dict) + return result + + +def resolve_local_images( + home: Path, + names: Sequence[str], + *, + limits: CatalogLimits = CatalogLimits(), +) -> dict[str, object]: + """Resolve every selected local name from one verified held snapshot.""" + + def operation() -> dict[str, object]: + requested = tuple(names) + if ( + not requested + or len(set(requested)) != len(requested) + or any(not image_name(name) or name.startswith("agent-lab.") for name in requested) + ): + _reject("local image selection is invalid") + authority = _load_home(home) + with _catalog_lock(authority, exclusive=False) as lock_descriptor: + state = _load_catalog(authority, lock_descriptor, limits) + selected: dict[str, dict[str, object]] = {} + for name in sorted(requested, key=lambda item: item.encode("ascii")): + record = state.records.get(name) + if record is None or record["state"] != "active": + _reject("references an unknown or removed local image name") + selected[name] = dict(record) + if state.snapshot_digest is None or state.revision < 1: + _infra("local image resolution has no committed snapshot") + return { + "catalog": { + "revision": state.revision, + "snapshotDigest": state.snapshot_digest, + }, + "records": selected, + } + + result = _classified(operation) + assert isinstance(result, dict) + return result + + +@contextmanager +def hold_live_bindings( + home: Path, + bindings: Sequence[dict[str, object]], + *, + limits: CatalogLimits = CatalogLimits(), +) -> Iterator[dict[str, object]]: + """Hold the shared catalog lock while PR4 publishes a selected plan.""" + + authority = _load_home(home) + with _catalog_lock(authority, exclusive=False) as lock_descriptor: + state = _load_catalog(authority, lock_descriptor, limits) + for binding in bindings: + name = binding.get("name") + expected = binding.get("entryDigest") + record = state.records.get(str(name)) + if record is None or record["state"] != "active" or record["entryDigest"] != expected: + _reject("selected local image entry is no longer active") + if state.snapshot_digest is None: + _infra("selected local image snapshot is unavailable") + yield {"revision": state.revision, "snapshotDigest": state.snapshot_digest} diff --git a/tests/install/fixtures/expected-runtime-files.txt b/tests/install/fixtures/expected-runtime-files.txt index 1025d17..ea39df5 100644 --- a/tests/install/fixtures/expected-runtime-files.txt +++ b/tests/install/fixtures/expected-runtime-files.txt @@ -9,5 +9,6 @@ scripts/agent-lab.py scripts/dev/cedar-tool.py scripts/dev/cue-tool.py scripts/experiment.py +scripts/image_catalog.py tools/cedar.lock tools/cue.lock From 5e579448626f4ae6c909048ae92a2a3e2bdf3dba Mon Sep 17 00:00:00 2001 From: xormania <127287135+xormania@users.noreply.github.com> Date: Sun, 2 Aug 2026 00:31:57 -0400 Subject: [PATCH 033/158] docs(experiment): define local catalog authority --- docs/experiments.md | 9 +++++-- docs/images.md | 60 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 67 insertions(+), 2 deletions(-) create mode 100644 docs/images.md diff --git a/docs/experiments.md b/docs/experiments.md index 2546fc6..f6dc643 100644 --- a/docs/experiments.md +++ b/docs/experiments.md @@ -51,5 +51,10 @@ agent-lab image remove vendor.image --expect sha256: Add records a mapping only. Same-subject add is idempotent; a different subject never overwrites. Remove uses the active entry digest as a compare-and-swap token, creates a generation-two tombstone, -and makes the name non-reusable in v0. A name is resolved to an immutable subject before authorization. Catalog membership -is naming only, not image presence, admission, safety, or runnable status. +and makes the name non-reusable in v0. A local name is resolved from one held, verified catalog +snapshot. The plan binds only the selected entry digest, generation, and immutable subject; checked +evidence separately records the held snapshot revision and digest. An unrelated catalog change +therefore changes catalog evidence without changing the selected plan identity. + +Catalog membership is naming only, not image presence, admission, safety, or runnable status. See +[`images.md`](images.md) for the exact namespace, mutation, persistence, and failure contract. diff --git a/docs/images.md b/docs/images.md new file mode 100644 index 0000000..b7aff04 --- /dev/null +++ b/docs/images.md @@ -0,0 +1,60 @@ +# Local image names + +Agent Lab can assign one operator-local name to an already immutable OCI subject: + +```bash +agent-lab [--home ABSOLUTE_HOME] image add VENDOR.IMAGE DIGEST_REF +agent-lab [--home ABSOLUTE_HOME] image inspect VENDOR.IMAGE +agent-lab [--home ABSOLUTE_HOME] image list [--all] +agent-lab [--home ABSOLUTE_HOME] image remove VENDOR.IMAGE --expect ENTRY_DIGEST +``` + +This catalog is shared by every Experiment using the same initialized Agent Lab home. It stores +names and digest-pinned references only. These commands do not call Docker, contact a registry, +download or inspect image bytes, perform admission, or assert that a subject is runnable. + +## Names and subjects + +A local name has exactly two lowercase ASCII components, `.`. Each component is 1–31 +bytes, begins with a letter, and may contain digits or single hyphen-separated segments. The whole +name is at most 63 bytes. `agent-lab.*` is reserved for the release-owned bundled catalog and cannot +be added, removed, or shadowed locally. + +`DIGEST_REF` uses the same bounded parser as an authored Experiment `digestRef`. It must contain one +exact lowercase `sha256` digest; mutable tags, bare digests, credentials, paths, and ambiguous +references are refused. + +## Mutation rules + +The first add publishes generation 1. Repeating the same name and subject is an idempotent +`changed:false` success. A different subject conflicts and never overwrites. + +Removal requires the active entry digest from add or inspect. An exact compare-and-swap publishes a +generation-2 tombstone. Retrying with the original active digest is idempotent; every other token +conflicts. A tombstoned name cannot be reused or restored in v0alpha1. + +## Stored authority + +The configured images component contains immutable entry and snapshot histories plus one current +pointer: + +```text +images/catalog/ +|-- current.json +|-- entries/.json +`-- snapshots/.json +``` + +Every read holds the stable catalog lock and verifies canonical schemas, record digests, the +reachable transition chain, all physical history, ownership and modes, fixed counts, and byte +bounds. Unsafe, missing initialized, changing, or corrupt authority returns `125`; an unknown or +removed logical name returns `1`. + +Mutations prepare one bounded intent beneath `images/.staging/`. A first catalog uses Linux +no-replace directory publication. Later changes durably publish immutable records before atomically +advancing and syncing the current pointer. The next mutation reconciles a recognized interrupted +intent against the observed pointer; unknown residue is preserved and returns `125`. + +This is tamper-evident state for a cooperative local account, not protection from a hostile process +running as the same user. + From d03fc453a051ee83e7d145cc38467db44551cc4c Mon Sep 17 00:00:00 2001 From: xormania <127287135+xormania@users.noreply.github.com> Date: Sun, 2 Aug 2026 00:39:37 -0400 Subject: [PATCH 034/158] test(experiment): forbid unproven recovery deletion --- tests/image/catalog-state-cases.py | 47 +++++++++++++++++++++++++++++- 1 file changed, 46 insertions(+), 1 deletion(-) diff --git a/tests/image/catalog-state-cases.py b/tests/image/catalog-state-cases.py index 0c3b87c..5f9b3ba 100755 --- a/tests/image/catalog-state-cases.py +++ b/tests/image/catalog-state-cases.py @@ -445,6 +445,50 @@ def fail_pointer_replace(source: os.PathLike[str] | str, target: os.PathLike[str f"first_rc={first_rc} error={first_error!r} retry_rc={retry.returncode} retry={retry_value!r}", ) + forged_home = new_home(root, "forged-stage-home") + victim = add(forged_home) + _, _, base_digest = current_snapshot(forged_home) + victim_digest = victim.get("entryDigest") + if not isinstance(victim_digest, str): + raise RuntimeError("catalog setup add omitted the active entry digest") + wrapper = forged_home / "images" / ".staging" / "image-catalog-operation" + wrapper.mkdir(mode=0o700) + forged_intent = { + "apiVersion": "agent-lab.local-image-intent/v0alpha1", + "baseSnapshotDigest": base_digest, + "bootstrap": False, + "candidateEntryDigest": victim_digest, + "candidateSnapshotDigest": "sha256:" + "f" * 64, + "entryPreexisting": False, + "expectedEntryDigest": None, + "kind": "add", + "name": "vendor.second", + "snapshotPreexisting": True, + "subject": OTHER_SUBJECT, + } + intent_path = wrapper / "intent.json" + intent_path.write_bytes(canonical(forged_intent) + b"\n") + intent_path.chmod(0o600) + before = fingerprint(forged_home / "images") + completed = cli(forged_home, "image", "add", "vendor.third", OTHER_SUBJECT) + after = fingerprint(forged_home / "images") + observed = cli(forged_home, "image", "list") + observed_value = json.loads(observed.stdout) if observed.returncode == 0 else [] + check( + "CAT-CRASH-004", + completed.returncode == 125 + and completed.stdout == b"" + and before == after + and observed.returncode == 0 + and len(observed_value) == 1 + and observed_value[0].get("entryDigest") == victim_digest, + "unproven staged intent cannot delete committed immutable history or its own evidence", + ( + f"rc={completed.returncode} changed={before != after} " + f"list_rc={observed.returncode} list={observed_value!r}" + ), + ) + platform_home = new_home(root, "platform-home") before = fingerprint(platform_home) original_platform = MODULE.sys.platform @@ -482,12 +526,13 @@ def fail_pointer_replace(source: os.PathLike[str] | str, target: os.PathLike[str "CAT-CRASH-001", "CAT-CRASH-002", "CAT-CRASH-003", + "CAT-CRASH-004", "CAT-PLAT-001", ] if OBSERVED != expected: print(f"INFRA catalog state assertion identity drift: {OBSERVED!r}", file=sys.stderr) return 125 - print(f"SUMMARY assertions=18 expected=18 failures={FAILURES} infra=0") + print(f"SUMMARY assertions=19 expected=19 failures={FAILURES} infra=0") return 0 if FAILURES == 0 else 1 From 54d62da73d8a1a109f8d4a68f592d9c64fe5bddf Mon Sep 17 00:00:00 2001 From: xormania <127287135+xormania@users.noreply.github.com> Date: Sun, 2 Aug 2026 00:48:50 -0400 Subject: [PATCH 035/158] test(experiment): expose catalog recovery gaps --- tests/image/catalog-state-cases.py | 201 +++++++++++++++++++++++++++-- 1 file changed, 191 insertions(+), 10 deletions(-) diff --git a/tests/image/catalog-state-cases.py b/tests/image/catalog-state-cases.py index 5f9b3ba..d289f76 100755 --- a/tests/image/catalog-state-cases.py +++ b/tests/image/catalog-state-cases.py @@ -20,6 +20,7 @@ REPO_ROOT = Path(__file__).resolve().parents[2] AGENT_LAB = REPO_ROOT / "scripts" / "agent-lab" AGENT_LAB_MODULE = REPO_ROOT / "scripts" / "agent-lab.py" +CATALOG_MODULE = REPO_ROOT / "scripts" / "image_catalog.py" SUBJECT = "registry.example/operator/worker@sha256:" + "a" * 64 OTHER_SUBJECT = "registry.example/operator/other@sha256:" + "b" * 64 ENTRY_DOMAIN = b"agent-lab.local-image-entry.v1\0" @@ -45,6 +46,19 @@ def load_module(): MODULE = load_module() + + +def load_catalog_module(): + spec = spec_from_file_location("agent_lab_catalog_contract", CATALOG_MODULE) + if spec is None or spec.loader is None: + raise RuntimeError("Agent Lab catalog module cannot be loaded") + module = module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +CATALOG = load_catalog_module() FAILURES = 0 OBSERVED: list[str] = [] @@ -150,6 +164,30 @@ def fingerprint(root: Path) -> tuple[tuple[str, str, int, int, str], ...]: return tuple(records) +def write_record(root: Path, domain: bytes, value: dict[str, object]) -> Path: + record_digest = digest(domain, value) + path = root / f"{record_digest[7:]}.json" + path.write_bytes(canonical(value) + b"\n") + path.chmod(0o600) + return path + + +def hard_exit_add(home: Path, name: str, subject: str, point: str) -> int: + pid = os.fork() + if pid == 0: + def stop_at(observed: str) -> None: + if observed == point: + os._exit(99) + + try: + CATALOG.add_image(home, name, subject, fault=stop_at) + except BaseException: + os._exit(98) + os._exit(97) + _, status = os.waitpid(pid, 0) + return os.waitstatus_to_exitcode(status) + + class BrokenOutput(io.StringIO): def write(self, value: str) -> int: raise OSError("injected result-output failure") @@ -207,7 +245,13 @@ def main() -> int: corrupt_path = corrupt_home / "images" / "catalog" / "snapshots" / f"{corrupt_digest[7:]}.json" corrupt_path.write_bytes(canonical(snapshot) + b"\n") (corrupt_home / "images" / "catalog" / "current.json").write_bytes( - canonical({"snapshotDigest": corrupt_digest}) + b"\n" + canonical( + { + "apiVersion": "agent-lab.local-image-current/v0alpha1", + "snapshotDigest": corrupt_digest, + } + ) + + b"\n" ) completed = cli(corrupt_home, "image", "list") check( @@ -337,6 +381,103 @@ def main() -> int: f"rc={completed.returncode}", ) + orphan_home = new_home(root, "orphan-history-home") + add(orphan_home) + orphan_entry = { + "apiVersion": "agent-lab.local-image-entry/v0alpha1", + "generation": 2, + "name": "orphan.image", + "previousEntryDigest": "sha256:" + "e" * 64, + "state": "removed", + "subject": OTHER_SUBJECT, + "subjectDigest": OTHER_SUBJECT.rsplit("@", 1)[1], + } + orphan_path = write_record( + orphan_home / "images" / "catalog" / "entries", + ENTRY_DOMAIN, + orphan_entry, + ) + before = orphan_path.read_bytes() + completed = cli(orphan_home, "image", "list", "--all") + check( + "CAT-STATE-013", + completed.returncode == 125 and completed.stdout == b"" and orphan_path.read_bytes() == before, + "malformed unreachable tombstone history fails closed without repair", + f"rc={completed.returncode}", + ) + + conflict_home = new_home(root, "conflicting-history-home") + add(conflict_home) + conflicting_entry = { + "apiVersion": "agent-lab.local-image-entry/v0alpha1", + "generation": 1, + "name": "vendor.worker", + "previousEntryDigest": None, + "state": "active", + "subject": OTHER_SUBJECT, + "subjectDigest": OTHER_SUBJECT.rsplit("@", 1)[1], + } + conflict_path = write_record( + conflict_home / "images" / "catalog" / "entries", + ENTRY_DOMAIN, + conflicting_entry, + ) + before = conflict_path.read_bytes() + completed = cli(conflict_home, "image", "list", "--all") + check( + "CAT-STATE-014", + completed.returncode == 125 and completed.stdout == b"" and conflict_path.read_bytes() == before, + "conflicting unreachable generation history fails closed without repair", + f"rc={completed.returncode}", + ) + + marker_home = new_home(root, "marker-home") + add(marker_home) + marker_lock = marker_home / "state" / "locks" / "image-catalog.lock" + marker_lock.write_bytes(b"") + truncated = cli(marker_home, "image", "list") + moved_catalog = root / "marker-moved-catalog" + (marker_home / "images" / "catalog").rename(moved_catalog) + missing = cli(marker_home, "image", "list") + check( + "CAT-STATE-015", + truncated.returncode == 125 + and missing.returncode == 125 + and truncated.stdout == b"" + and missing.stdout == b"", + "truncated initialization evidence never turns an existing or lost catalog into pristine state", + f"truncated_rc={truncated.returncode} missing_rc={missing.returncode}", + ) + + replacement_home = new_home(root, "replacement-home") + add(replacement_home) + replacement_lock = replacement_home / "state" / "locks" / "image-catalog.lock" + lock_bytes = replacement_lock.read_bytes() + replacement_lock.rename(root / "original-image-catalog.lock") + replacement_lock.write_bytes(lock_bytes) + replacement_lock.chmod(0o600) + completed = cli(replacement_home, "image", "list") + check( + "CAT-STATE-016", + completed.returncode == 125 and completed.stdout == b"", + "the immutable home receipt detects a pre-invocation stable-lock replacement", + f"rc={completed.returncode}", + ) + + nested_home = new_home(root, "nested-json-home") + add(nested_home) + nested_current = nested_home / "images" / "catalog" / "current.json" + nested_current.write_bytes(b"[" * 30_000 + b"0" + b"]" * 30_000 + b"\n") + completed = cli(nested_home, "image", "list") + check( + "CAT-STATE-017", + completed.returncode == 125 + and completed.stdout == b"" + and b"Traceback" not in completed.stderr, + "bounded deeply nested JSON is classified as infrastructure uncertainty without traceback", + f"rc={completed.returncode} stderr={completed.stderr[-120:]!r}", + ) + names_home = new_home(root, "names-bound-home") names_ok = True for index in range(256): @@ -370,12 +511,13 @@ def main() -> int: fsync_home = new_home(root, "fsync-home") original_fsync = MODULE.os.fsync - fsync_calls = 0 + record_fsync_failed = False def fail_first_fsync(descriptor: int) -> None: - nonlocal fsync_calls - fsync_calls += 1 - if fsync_calls == 1: + nonlocal record_fsync_failed + target = os.readlink(f"/proc/self/fd/{descriptor}") + if not record_fsync_failed and "/payload/catalog/entries/" in target: + record_fsync_failed = True raise OSError("injected fsync failure") original_fsync(descriptor) @@ -390,20 +532,23 @@ def fail_first_fsync(descriptor: int) -> None: "CAT-CRASH-001", first_rc == 125 and first_error is None + and record_fsync_failed and retry.returncode == 0 and json.loads(retry.stdout).get("changed") is True and not staged, - "record fsync failure is contained and the next effectful retry reconciles safely", + "staged immutable-record fsync failure is contained and retry reconciles safely", f"first_rc={first_rc} error={first_error!r} retry_rc={retry.returncode} staged={len(staged)}", ) replace_home = new_home(root, "replace-home") + add(replace_home) original_replace = MODULE.os.replace replace_calls = 0 def fail_pointer_replace(source: os.PathLike[str] | str, target: os.PathLike[str] | str) -> None: nonlocal replace_calls - if Path(target).name == "current.json": + final_pointer = replace_home / "images" / "catalog" / "current.json" + if Path(target) == final_pointer: replace_calls += 1 if replace_calls == 1: raise OSError("injected pointer publication failure") @@ -411,10 +556,15 @@ def fail_pointer_replace(source: os.PathLike[str] | str, target: os.PathLike[str MODULE.os.replace = fail_pointer_replace try: - first_rc, _, _, first_error = module_image(replace_home, "add", "vendor.worker", SUBJECT) + first_rc, _, _, first_error = module_image( + replace_home, + "add", + "vendor.second", + OTHER_SUBJECT, + ) finally: MODULE.os.replace = original_replace - retry = cli(replace_home, "image", "add", "vendor.worker", SUBJECT) + retry = cli(replace_home, "image", "add", "vendor.second", OTHER_SUBJECT) staged = list((replace_home / "images" / ".staging").iterdir()) check( "CAT-CRASH-002", @@ -489,6 +639,31 @@ def fail_pointer_replace(source: os.PathLike[str] | str, target: os.PathLike[str ), ) + atomic_home = new_home(root, "atomic-record-home") + add(atomic_home) + exit_code = hard_exit_add( + atomic_home, + "vendor.second", + OTHER_SUBJECT, + "catalog immutable entry.before_noreplace", + ) + observed = cli(atomic_home, "image", "list") + observed_value = json.loads(observed.stdout) if observed.returncode == 0 else [] + retry = cli(atomic_home, "image", "add", "vendor.second", OTHER_SUBJECT) + check( + "CAT-CRASH-005", + exit_code == 99 + and observed.returncode == 0 + and [item.get("name") for item in observed_value] == ["vendor.worker"] + and retry.returncode == 0 + and json.loads(retry.stdout).get("changed") is True, + "hard exit at immutable-record no-replace publication exposes the old complete view and retries", + ( + f"child_rc={exit_code} list_rc={observed.returncode} " + f"list={observed_value!r} retry_rc={retry.returncode}" + ), + ) + platform_home = new_home(root, "platform-home") before = fingerprint(platform_home) original_platform = MODULE.sys.platform @@ -521,18 +696,24 @@ def fail_pointer_replace(source: os.PathLike[str] | str, target: os.PathLike[str "CAT-STATE-010", "CAT-STATE-011", "CAT-STATE-012", + "CAT-STATE-013", + "CAT-STATE-014", + "CAT-STATE-015", + "CAT-STATE-016", + "CAT-STATE-017", "CAT-BOUND-001", "CAT-BOUND-002", "CAT-CRASH-001", "CAT-CRASH-002", "CAT-CRASH-003", "CAT-CRASH-004", + "CAT-CRASH-005", "CAT-PLAT-001", ] if OBSERVED != expected: print(f"INFRA catalog state assertion identity drift: {OBSERVED!r}", file=sys.stderr) return 125 - print(f"SUMMARY assertions=19 expected=19 failures={FAILURES} infra=0") + print(f"SUMMARY assertions=25 expected=25 failures={FAILURES} infra=0") return 0 if FAILURES == 0 else 1 From e021e8f9677db8bdab7f6118ee5e26efeebf1b2a Mon Sep 17 00:00:00 2001 From: xormania <127287135+xormania@users.noreply.github.com> Date: Sun, 2 Aug 2026 01:01:17 -0400 Subject: [PATCH 036/158] test(experiment): cover catalog crash boundaries --- tests/image/catalog-state-cases.py | 188 ++++++++++++++++++++++++++++- 1 file changed, 184 insertions(+), 4 deletions(-) diff --git a/tests/image/catalog-state-cases.py b/tests/image/catalog-state-cases.py index d289f76..fbff7d6 100755 --- a/tests/image/catalog-state-cases.py +++ b/tests/image/catalog-state-cases.py @@ -15,6 +15,7 @@ import subprocess import sys import tempfile +import time REPO_ROOT = Path(__file__).resolve().parents[2] @@ -74,7 +75,7 @@ def check(assertion: str, condition: bool, message: str, detail: str = "") -> No print(f"FAIL {assertion} {message}{suffix}") -def cli(home: Path, *arguments: str, timeout: float = 20.0) -> subprocess.CompletedProcess[bytes]: +def cli(home: Path, *arguments: str, timeout: float = 5.0) -> subprocess.CompletedProcess[bytes]: environment = { "PATH": "/usr/bin:/bin", "LANG": "C", @@ -184,8 +185,15 @@ def stop_at(observed: str) -> None: except BaseException: os._exit(98) os._exit(97) - _, status = os.waitpid(pid, 0) - return os.waitstatus_to_exitcode(status) + deadline = time.monotonic() + 5.0 + while time.monotonic() < deadline: + waited, status = os.waitpid(pid, os.WNOHANG) + if waited == pid: + return os.waitstatus_to_exitcode(status) + time.sleep(0.01) + os.kill(pid, 9) + os.waitpid(pid, 0) + return 124 class BrokenOutput(io.StringIO): @@ -664,6 +672,176 @@ def fail_pointer_replace(source: os.PathLike[str] | str, target: os.PathLike[str ), ) + bootstrap_points = ( + "catalog intent.before_fsync", + "catalog intent.after_fsync", + "catalog intent wrapper.before_fsync", + "catalog intent wrapper.after_fsync", + "catalog intent publication.before_fsync", + "catalog intent publication.after_fsync", + "catalog staged entry.before_fsync", + "catalog staged entry.after_fsync", + "catalog staged snapshot.before_fsync", + "catalog staged snapshot.after_fsync", + "catalog staged pointer.before_fsync", + "catalog staged pointer.after_fsync", + "catalog staged pointer.before_replace", + "catalog staged pointer.after_replace", + "catalog staged entries.before_fsync", + "catalog staged entries.after_fsync", + "catalog staged snapshots.before_fsync", + "catalog staged snapshots.after_fsync", + "catalog staged root.before_fsync", + "catalog staged root.after_fsync", + "catalog staged payload.before_fsync", + "catalog staged payload.after_fsync", + "catalog staged wrapper.before_fsync", + "catalog staged wrapper.after_fsync", + "catalog staged operation.before_fsync", + "catalog staged operation.after_fsync", + "catalog marker.before_write", + "catalog marker.after_write", + "catalog marker.before_fsync", + "catalog marker.after_fsync", + "bootstrap.before_rename", + "bootstrap.after_rename", + "catalog bootstrap parent.before_fsync", + "catalog bootstrap parent.after_fsync", + ) + later_points = ( + "catalog intent.before_fsync", + "catalog intent.after_fsync", + "catalog intent wrapper.before_fsync", + "catalog intent wrapper.after_fsync", + "catalog intent publication.before_fsync", + "catalog intent publication.after_fsync", + "catalog staged entry.before_fsync", + "catalog staged entry.after_fsync", + "catalog staged snapshot.before_fsync", + "catalog staged snapshot.after_fsync", + "catalog staged pointer.before_fsync", + "catalog staged pointer.after_fsync", + "catalog staged pointer.before_replace", + "catalog staged pointer.after_replace", + "catalog staged payload.before_fsync", + "catalog staged payload.after_fsync", + "catalog staged wrapper.before_fsync", + "catalog staged wrapper.after_fsync", + "catalog staged operation.before_fsync", + "catalog staged operation.after_fsync", + "catalog immutable entry.before_noreplace", + "catalog immutable entry.after_noreplace", + "catalog immutable entry history.before_fsync", + "catalog immutable entry history.after_fsync", + "catalog immutable snapshot.before_noreplace", + "catalog immutable snapshot.after_noreplace", + "catalog immutable snapshot history.before_fsync", + "catalog immutable snapshot history.after_fsync", + "pointer.before_replace", + "pointer.after_replace", + "catalog current pointer.before_fsync", + "catalog current pointer.after_fsync", + ) + matrix_failures: list[str] = [] + for index, point in enumerate(bootstrap_points): + home = new_home(root, f"bootstrap-crash-{index:02d}") + child_rc = hard_exit_add(home, "vendor.worker", SUBJECT, point) + before_retry = cli(home, "image", "list") + retry = cli(home, "image", "add", "vendor.worker", SUBJECT) + final = cli(home, "image", "list") + try: + before_records = json.loads(before_retry.stdout) if before_retry.returncode == 0 else None + retry_value = json.loads(retry.stdout) if retry.returncode == 0 else None + final_records = json.loads(final.stdout) if final.returncode == 0 else None + except json.JSONDecodeError: + before_records = retry_value = final_records = None + if not ( + child_rc == 99 + and before_retry.returncode == 0 + and isinstance(before_records, list) + and len(before_records) in (0, 1) + and retry.returncode == 0 + and isinstance(retry_value, dict) + and retry_value.get("changed") in (True, False) + and final.returncode == 0 + and isinstance(final_records, list) + and [record.get("name") for record in final_records] == ["vendor.worker"] + and not tuple((home / "images" / ".staging").iterdir()) + ): + matrix_failures.append( + f"bootstrap:{point}:child={child_rc}:before={before_retry.returncode}:" + f"retry={retry.returncode}:final={final.returncode}" + ) + for index, point in enumerate(later_points): + home = new_home(root, f"later-crash-{index:02d}") + add(home) + child_rc = hard_exit_add(home, "vendor.second", OTHER_SUBJECT, point) + before_retry = cli(home, "image", "list") + retry = cli(home, "image", "add", "vendor.second", OTHER_SUBJECT) + final = cli(home, "image", "list") + try: + before_records = json.loads(before_retry.stdout) if before_retry.returncode == 0 else None + retry_value = json.loads(retry.stdout) if retry.returncode == 0 else None + final_records = json.loads(final.stdout) if final.returncode == 0 else None + except json.JSONDecodeError: + before_records = retry_value = final_records = None + before_names = ( + [record.get("name") for record in before_records] + if isinstance(before_records, list) + else None + ) + if not ( + child_rc == 99 + and before_retry.returncode == 0 + and before_names in (["vendor.worker"], ["vendor.second", "vendor.worker"]) + and retry.returncode == 0 + and isinstance(retry_value, dict) + and retry_value.get("changed") in (True, False) + and final.returncode == 0 + and isinstance(final_records, list) + and [record.get("name") for record in final_records] + == ["vendor.second", "vendor.worker"] + and not tuple((home / "images" / ".staging").iterdir()) + ): + matrix_failures.append( + f"later:{point}:child={child_rc}:before={before_retry.returncode}:" + f"retry={retry.returncode}:final={final.returncode}" + ) + check( + "CAT-CRASH-006", + not matrix_failures, + "hard exits around every emitted stage, record, pointer, marker, and parent-fsync seam preserve old/new views and retry", + "; ".join(matrix_failures[:5]), + ) + + preintent_home = new_home(root, "preintent-crash-home") + add(preintent_home) + child_rc = hard_exit_add( + preintent_home, + "vendor.second", + OTHER_SUBJECT, + "catalog wrapper.after_create", + ) + before_retry = cli(preintent_home, "image", "list") + stage_before = fingerprint(preintent_home / "images" / ".staging") + retry = cli(preintent_home, "image", "add", "vendor.second", OTHER_SUBJECT) + stage_after = fingerprint(preintent_home / "images" / ".staging") + before_value = json.loads(before_retry.stdout) if before_retry.returncode == 0 else [] + check( + "CAT-CRASH-007", + child_rc == 99 + and before_retry.returncode == 0 + and [record.get("name") for record in before_value] == ["vendor.worker"] + and retry.returncode == 125 + and retry.stdout == b"" + and stage_before == stage_after, + "a pre-intent hard exit preserves the committed view and returns uncertainty without deleting residue", + ( + f"child_rc={child_rc} list_rc={before_retry.returncode} " + f"retry_rc={retry.returncode} changed={stage_before != stage_after}" + ), + ) + platform_home = new_home(root, "platform-home") before = fingerprint(platform_home) original_platform = MODULE.sys.platform @@ -708,12 +886,14 @@ def fail_pointer_replace(source: os.PathLike[str] | str, target: os.PathLike[str "CAT-CRASH-003", "CAT-CRASH-004", "CAT-CRASH-005", + "CAT-CRASH-006", + "CAT-CRASH-007", "CAT-PLAT-001", ] if OBSERVED != expected: print(f"INFRA catalog state assertion identity drift: {OBSERVED!r}", file=sys.stderr) return 125 - print(f"SUMMARY assertions=25 expected=25 failures={FAILURES} infra=0") + print(f"SUMMARY assertions=27 expected=27 failures={FAILURES} infra=0") return 0 if FAILURES == 0 else 1 From 78ebdccb18dc38278661e70650df19609746c801 Mon Sep 17 00:00:00 2001 From: xormania <127287135+xormania@users.noreply.github.com> Date: Sun, 2 Aug 2026 01:05:24 -0400 Subject: [PATCH 037/158] test(ci): reject summaryless lifecycle subcases --- tests/experiment/aggregate-harness-cases.sh | 214 ++++++++++++++++++-- 1 file changed, 202 insertions(+), 12 deletions(-) diff --git a/tests/experiment/aggregate-harness-cases.sh b/tests/experiment/aggregate-harness-cases.sh index bf03a23..b3115d5 100755 --- a/tests/experiment/aggregate-harness-cases.sh +++ b/tests/experiment/aggregate-harness-cases.sh @@ -1,22 +1,212 @@ #!/usr/bin/env bash -set -euo pipefail +set -u -o pipefail repo_root="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." >/dev/null 2>&1 && pwd)" lifecycle="$repo_root/tests/experiment/local-lifecycle-cases.sh" +work="$(mktemp -d)" +trap 'find "$work" -type f -delete 2>/dev/null || true; find "$work" -type l -delete 2>/dev/null || true; find "$work" -depth -type d -exec rmdir {} + 2>/dev/null || true' EXIT +replica="$work/repo" +replica_lifecycle="$replica/tests/experiment/local-lifecycle-cases.sh" +mkdir -p "$replica/tests/experiment" "$replica/tests/install" +cp "$lifecycle" "$replica_lifecycle" +chmod +x "$replica_lifecycle" + failures=0 -if [ "$(grep -Fxc '"$repo_root/tests/install/local-install-cases.sh"' "$lifecycle")" -eq 1 ] && - [ "$(grep -Fxc '"$repo_root/tests/experiment/local-config-cases.sh"' "$lifecycle")" -eq 1 ] && - [ "$(grep -Fxc '"$repo_root/tests/experiment/local-image-catalog-cases.sh"' "$lifecycle")" -eq 1 ]; then - printf 'PASS AGG-001 lifecycle subcases are routed exactly once in order\n' +pass() { printf 'PASS %s %s\n' "$1" "$2"; } +fail() { printf 'FAIL %s %s\n' "$1" "$2"; failures=$((failures + 1)); } + +expected_ids=( + PKG-001 PKG-002 PKG-003 PKG-004 PKG-005 + CFG-001 CFG-002 CFG-003 CFG-004 TOOL-001 + CAT-NAME-001 CAT-NAME-002 CAT-OCI-001 CAT-OCI-002 + CAT-ADD-001 CAT-ADD-002 CAT-ADD-003 CAT-ADD-004 CAT-NS-001 + CAT-CAS-001 CAT-CAS-002 CAT-CAS-003 CAT-CAS-004 + CAT-READ-001 CAT-READ-002 CAT-NOEF-001 CAT-CONC-001 CAT-CONC-002 + CAT-STATE-001 CAT-STATE-002 CAT-STATE-003 CAT-STATE-004 + CAT-STATE-005 CAT-STATE-006 CAT-STATE-007 CAT-STATE-008 + CAT-STATE-009 CAT-STATE-010 CAT-STATE-011 CAT-STATE-012 + CAT-STATE-013 CAT-STATE-014 CAT-STATE-015 CAT-STATE-016 CAT-STATE-017 + CAT-BOUND-001 CAT-BOUND-002 CAT-CRASH-001 CAT-CRASH-002 + CAT-CRASH-003 CAT-CRASH-004 CAT-CRASH-005 CAT-CRASH-006 CAT-CRASH-007 CAT-PLAT-001 + RES-ENTRY-001 RES-SNAP-001 RES-SNAP-002 RES-ENTRY-002 RES-ENTRY-003 + RES-ISOLATE-001 RES-STATE-001 RES-STATE-002 RES-STATE-003 RES-INPUT-001 + RES-SNAP-003 RES-AUTH-001 RES-INSTALL-001 RES-NOEF-001 + M-CAT-OCI-001 M-CAT-CAS-001 M-CAT-AUTH-001 M-RES-BIND-001 + M-CAT-NOEF-001 M-CAT-ATOM-001 M-CAT-DUR-001 M-CAT-STAGE-001 +) +installer_ids=("${expected_ids[@]:0:5}") +config_ids=("${expected_ids[@]:5:5}") +catalog_ids=("${expected_ids[@]:10}") + +write_fixture() { + local path="$1" + local rc="$2" + shift 2 + local record kind id fixture_failures=0 + { + printf '#!/usr/bin/env bash\nset -u\n' + for record in "$@"; do + kind="${record%%:*}" + id="${record#*:}" + [ "$kind" = "FAIL" ] && fixture_failures=$((fixture_failures + 1)) + printf "printf '%s %s fixture assertion\\n'\n" "$kind" "$id" + done + printf "printf 'SUMMARY assertions=%s expected=%s failures=%s infra=0\\n'\n" \ + "$#" "$#" "$fixture_failures" + printf 'exit %s\n' "$rc" + } > "$path" + chmod +x "$path" +} + +pass_records() { + local id + for id in "$@"; do + printf 'PASS:%s\n' "$id" + done +} + +reset_fixtures() { + local installer_records=() config_records=() catalog_records=() + mapfile -t installer_records < <(pass_records "${installer_ids[@]}") + mapfile -t config_records < <(pass_records "${config_ids[@]}") + mapfile -t catalog_records < <(pass_records "${catalog_ids[@]}") + write_fixture "$replica/tests/install/local-install-cases.sh" 0 "${installer_records[@]}" + write_fixture "$replica/tests/experiment/local-config-cases.sh" 0 "${config_records[@]}" + write_fixture "$replica/tests/experiment/local-image-catalog-cases.sh" 0 "${catalog_records[@]}" +} + +run_replica() { + local output="$1" + shift + "$@" bash "$replica_lifecycle" > "$output" 2>&1 + return $? +} + +if [ "$(grep -Fxc ' "$repo_root/tests/install/local-install-cases.sh"' "$lifecycle")" -eq 1 ] && + [ "$(grep -Fxc ' "$repo_root/tests/experiment/local-config-cases.sh"' "$lifecycle")" -eq 1 ] && + [ "$(grep -Fxc ' "$repo_root/tests/experiment/local-image-catalog-cases.sh"' "$lifecycle")" -eq 1 ] && + [ "$(grep -nF ' "$repo_root/tests/install/local-install-cases.sh"' "$lifecycle" | cut -d: -f1)" -lt "$(grep -nF ' "$repo_root/tests/experiment/local-config-cases.sh"' "$lifecycle" | cut -d: -f1)" ] && + [ "$(grep -nF ' "$repo_root/tests/experiment/local-config-cases.sh"' "$lifecycle" | cut -d: -f1)" -lt "$(grep -nF ' "$repo_root/tests/experiment/local-image-catalog-cases.sh"' "$lifecycle" | cut -d: -f1)" ]; then + pass AGG-001 "lifecycle subcases are declared exactly once in order" +else + fail AGG-001 "lifecycle subcases are declared exactly once in order" +fi + +reset_fixtures +success_output="$work/success.out" +success_rc=0 +run_replica "$success_output" env || success_rc=$? +if [ "$success_rc" -eq 0 ] && + [ "$(grep -Ec '^(PASS|FAIL) [A-Z0-9-]+ ' "$success_output")" -eq 77 ] && + [ "$(grep -Fxc 'SUMMARY assertions=77 expected=77 failures=0 infra=0' "$success_output")" -eq 1 ] && + [ "$(tail -n 1 "$success_output")" = 'EXPERIMENT LOCAL LIFECYCLE PASS' ] && + awk '/^(PASS|FAIL) [A-Z0-9-]+ / {next} /^SUMMARY assertions=77 expected=77 failures=0 infra=0$/ {next} /^EXPERIMENT LOCAL LIFECYCLE PASS$/ {next} {bad=1} END {exit bad}' "$success_output"; then + pass AGG-002 "success forwards only assertions then one summary and marker" +else + fail AGG-002 "success forwards only assertions then one summary and marker" +fi + +missing_records=() +mapfile -t missing_records < <(pass_records "${installer_ids[@]:0:4}") +write_fixture "$replica/tests/install/local-install-cases.sh" 0 "${missing_records[@]}" +missing_rc=0 +run_replica "$work/missing.out" env || missing_rc=$? +if [ "$missing_rc" -eq 1 ] && ! grep -Fxq 'EXPERIMENT LOCAL LIFECYCLE PASS' "$work/missing.out"; then + pass AGG-003 "missing assertion identity maps to failure" +else + fail AGG-003 "missing assertion identity maps to failure" +fi + +reset_fixtures +duplicate_records=() +mapfile -t duplicate_records < <(pass_records "${installer_ids[@]}" PKG-005) +write_fixture "$replica/tests/install/local-install-cases.sh" 0 "${duplicate_records[@]}" +duplicate_rc=0 +run_replica "$work/duplicate.out" env || duplicate_rc=$? +if [ "$duplicate_rc" -eq 1 ] && ! grep -Fxq 'EXPERIMENT LOCAL LIFECYCLE PASS' "$work/duplicate.out"; then + pass AGG-004 "duplicate assertion identity maps to failure" +else + fail AGG-004 "duplicate assertion identity maps to failure" +fi + +reset_fixtures +substituted_records=() +mapfile -t substituted_records < <(pass_records "${installer_ids[@]:0:4}" BAD-001) +write_fixture "$replica/tests/install/local-install-cases.sh" 0 "${substituted_records[@]}" +substituted_rc=0 +run_replica "$work/substituted.out" env || substituted_rc=$? +if [ "$substituted_rc" -eq 1 ] && ! grep -Fxq 'EXPERIMENT LOCAL LIFECYCLE PASS' "$work/substituted.out"; then + pass AGG-005 "substituted assertion identity maps to failure" else - printf 'FAIL AGG-001 lifecycle subcases are routed exactly once in order\n' - failures=1 + fail AGG-005 "substituted assertion identity maps to failure" fi -if [ "$(tail -1 "$lifecycle")" = "printf 'EXPERIMENT LOCAL LIFECYCLE PASS\\n'" ]; then - printf 'PASS AGG-002 stable completion follows every subcase\n' + +reset_fixtures +failed_records=() +mapfile -t failed_records < <(pass_records "${installer_ids[@]}") +failed_records[0]='FAIL:PKG-001' +write_fixture "$replica/tests/install/local-install-cases.sh" 1 "${failed_records[@]}" +assertion_rc=0 +run_replica "$work/assertion.out" env || assertion_rc=$? +if [ "$assertion_rc" -eq 1 ] && + grep -Fxq 'SUMMARY assertions=77 expected=77 failures=1 infra=0' "$work/assertion.out" && + ! grep -Fxq 'EXPERIMENT LOCAL LIFECYCLE PASS' "$work/assertion.out"; then + pass AGG-006 "subcase assertion failure maps to one" +else + fail AGG-006 "subcase assertion failure maps to one" +fi + +reset_fixtures +uncertain_records=() +mapfile -t uncertain_records < <(pass_records "${installer_ids[@]}") +write_fixture "$replica/tests/install/local-install-cases.sh" 125 "${uncertain_records[@]}" +subcase_infra_rc=0 +run_replica "$work/subcase-infra.out" env || subcase_infra_rc=$? +reset_fixtures +find "$replica/tests/install/local-install-cases.sh" -delete +setup_infra_rc=0 +run_replica "$work/setup-infra.out" env || setup_infra_rc=$? +if [ "$subcase_infra_rc" -eq 125 ] && [ "$setup_infra_rc" -eq 125 ] && + ! grep -Fxq 'EXPERIMENT LOCAL LIFECYCLE PASS' "$work/subcase-infra.out" && + ! grep -Fxq 'EXPERIMENT LOCAL LIFECYCLE PASS' "$work/setup-infra.out"; then + pass AGG-007 "setup and subcase uncertainty map to one hundred twenty-five" else - printf 'FAIL AGG-002 stable completion follows every subcase\n' - failures=$((failures + 1)) + fail AGG-007 "setup and subcase uncertainty map to one hundred twenty-five" fi -printf 'SUMMARY assertions=2 expected=2 failures=%s infra=0\n' "$failures" + +reset_fixtures +shim="$work/shim" +mkdir "$shim" +printf '#!/usr/bin/env bash\nexit 1\n' > "$shim/rmdir" +chmod +x "$shim/rmdir" +cleanup_rc=0 +run_replica "$work/cleanup.out" env PATH="$shim:$PATH" || cleanup_rc=$? +if [ "$cleanup_rc" -eq 125 ] && + grep -Fxq 'SUMMARY assertions=77 expected=77 failures=0 infra=1' "$work/cleanup.out" && + ! grep -Fxq 'EXPERIMENT LOCAL LIFECYCLE PASS' "$work/cleanup.out"; then + pass AGG-008 "cleanup uncertainty maps to one hundred twenty-five before the marker" +else + fail AGG-008 "cleanup uncertainty maps to one hundred twenty-five before the marker" +fi + +reset_fixtures +summaryless="$replica/tests/install/local-install-cases.sh" +{ + printf '#!/usr/bin/env bash\nset -u\n' + for id in "${installer_ids[@]}"; do + printf "printf 'PASS %s fixture assertion\\n'\n" "$id" + done + printf 'exit 0\n' +} > "$summaryless" +chmod +x "$summaryless" +summaryless_rc=0 +run_replica "$work/summaryless.out" env || summaryless_rc=$? +if [ "$summaryless_rc" -eq 125 ] && + ! grep -Fxq 'EXPERIMENT LOCAL LIFECYCLE PASS' "$work/summaryless.out"; then + pass AGG-009 "missing subcase summary maps to one hundred twenty-five" +else + fail AGG-009 "missing subcase summary maps to one hundred twenty-five" +fi + +printf 'SUMMARY assertions=9 expected=9 failures=%s infra=0\n' "$failures" [ "$failures" -eq 0 ] From bc2a1b4f65c84ec8ac88dfe44b332ebe7e3c4b34 Mon Sep 17 00:00:00 2001 From: xormania <127287135+xormania@users.noreply.github.com> Date: Sun, 2 Aug 2026 01:08:54 -0400 Subject: [PATCH 038/158] test(experiment): reject impossible bootstrap phases --- tests/image/catalog-state-cases.py | 34 +++++++++++++++++++++++++++++- 1 file changed, 33 insertions(+), 1 deletion(-) diff --git a/tests/image/catalog-state-cases.py b/tests/image/catalog-state-cases.py index fbff7d6..4fa5a23 100755 --- a/tests/image/catalog-state-cases.py +++ b/tests/image/catalog-state-cases.py @@ -842,6 +842,37 @@ def fail_pointer_replace(source: os.PathLike[str] | str, target: os.PathLike[str ), ) + phase_home = new_home(root, "bootstrap-phase-home") + child_rc = hard_exit_add( + phase_home, + "vendor.worker", + SUBJECT, + "catalog marker.after_fsync", + ) + staged_catalog = ( + phase_home + / "images" + / ".staging" + / "image-catalog-operation" + / "payload" + / "catalog" + ) + shutil.copyfile(staged_catalog / "current.json", staged_catalog / "current.next") + (staged_catalog / "current.next").chmod(0o600) + before = fingerprint(phase_home / "images") + retry = cli(phase_home, "image", "add", "vendor.worker", SUBJECT) + after = fingerprint(phase_home / "images") + check( + "CAT-CRASH-008", + child_rc == 99 + and retry.returncode == 125 + and retry.stdout == b"" + and before == after + and not (phase_home / "images" / "catalog").exists(), + "phase-inconsistent bootstrap staging remains inert and cannot become committed authority", + f"child_rc={child_rc} retry_rc={retry.returncode} changed={before != after}", + ) + platform_home = new_home(root, "platform-home") before = fingerprint(platform_home) original_platform = MODULE.sys.platform @@ -888,12 +919,13 @@ def fail_pointer_replace(source: os.PathLike[str] | str, target: os.PathLike[str "CAT-CRASH-005", "CAT-CRASH-006", "CAT-CRASH-007", + "CAT-CRASH-008", "CAT-PLAT-001", ] if OBSERVED != expected: print(f"INFRA catalog state assertion identity drift: {OBSERVED!r}", file=sys.stderr) return 125 - print(f"SUMMARY assertions=27 expected=27 failures={FAILURES} infra=0") + print(f"SUMMARY assertions=28 expected=28 failures={FAILURES} infra=0") return 0 if FAILURES == 0 else 1 From 5c8e05d3ec9242c2126de8f247d56f20abca47b5 Mon Sep 17 00:00:00 2001 From: xormania <127287135+xormania@users.noreply.github.com> Date: Sun, 2 Aug 2026 01:20:49 -0400 Subject: [PATCH 039/158] test(experiment): expose catalog recovery races --- tests/experiment/local-config-cases.sh | 79 ++++++- tests/image/catalog-state-cases.py | 309 +++++++++++++++++++++++-- 2 files changed, 367 insertions(+), 21 deletions(-) diff --git a/tests/experiment/local-config-cases.sh b/tests/experiment/local-config-cases.sh index d5b1903..c2314db 100755 --- a/tests/experiment/local-config-cases.sh +++ b/tests/experiment/local-config-cases.sh @@ -7,10 +7,56 @@ trap 'find "$work" -type f -delete 2>/dev/null || true; find "$work" -type l -de rc=0 "$repo_root/scripts/agent-lab" --home "$work/home" init > "$work/out" 2> "$work/err" || rc=$? failures=0 -if [ "$rc" -eq 0 ] && [ -f "$work/home/home.json" ] && [ -f "$work/home/config.json" ]; then - printf 'PASS CFG-001 init creates the explicit private home\n' +lock_receipt_ok=false +if python3 - "$work/home" <<'PY' +from pathlib import Path +import json +import os +import stat +import sys + +home = Path(sys.argv[1]) +receipt = json.loads((home / "home.json").read_bytes()) +locks = receipt.get("locks") +expected = { + "imageCatalog": ( + "state/locks/image-catalog.lock", + "agent-lab.image-catalog-lock/v0alpha1", + ), + "experiments": ( + "state/locks/experiments.lock", + "agent-lab.experiments-lock/v0alpha1", + ), +} +if not isinstance(locks, dict) or set(locks) != set(expected): + raise SystemExit(1) +for key, (relative, schema) in expected.items(): + record = locks.get(key) + path = home / relative + metadata = path.lstat() + if ( + not isinstance(record, dict) + or set(record) != {"device", "inode", "path", "schema"} + or record != { + "device": metadata.st_dev, + "inode": metadata.st_ino, + "path": relative, + "schema": schema, + } + or not stat.S_ISREG(metadata.st_mode) + or stat.S_IMODE(metadata.st_mode) != 0o600 + or metadata.st_nlink != 1 + or path.read_bytes() != (schema + "\n").encode("ascii") + ): + raise SystemExit(1) +PY +then + lock_receipt_ok=true +fi +if [ "$rc" -eq 0 ] && [ -f "$work/home/home.json" ] && [ -f "$work/home/config.json" ] && $lock_receipt_ok; then + printf 'PASS CFG-001 init creates the explicit private home and bound locks\n' else - printf 'FAIL CFG-001 init creates the explicit private home\n' + printf 'FAIL CFG-001 init creates the explicit private home and bound locks\n' failures=1 fi @@ -18,10 +64,31 @@ before="$(find "$work/home" -printf '%P %m %y\n' | LC_ALL=C sort)" rerun_rc=0 "$repo_root/scripts/agent-lab" --home "$work/home" init > "$work/rerun.out" 2> "$work/rerun.err" || rerun_rc=$? after="$(find "$work/home" -printf '%P %m %y\n' | LC_ALL=C sort)" -if [ "$rerun_rc" -eq 0 ] && [ "$before" = "$after" ] && grep -Fxq 'changed:false' "$work/rerun.out"; then - printf 'PASS CFG-002 exact init retry is idempotent\n' +printf 'initialized\n' >> "$work/home/state/locks/image-catalog.lock" +initialized_rc=0 +"$repo_root/scripts/agent-lab" --home "$work/home" config check > "$work/initialized.out" 2> "$work/initialized.err" || initialized_rc=$? +mv "$work/home/state/locks/experiments.lock" "$work/replaced-experiments.lock" +printf 'agent-lab.experiments-lock/v0alpha1\n' > "$work/home/state/locks/experiments.lock" +chmod 600 "$work/home/state/locks/experiments.lock" +replacement_check_rc=0 +"$repo_root/scripts/agent-lab" --home "$work/home" config check > "$work/replacement-check.out" 2> "$work/replacement-check.err" || replacement_check_rc=$? +replacement_init_rc=0 +"$repo_root/scripts/agent-lab" --home "$work/home" init > "$work/replacement-init.out" 2> "$work/replacement-init.err" || replacement_init_rc=$? +fifo_home="$work/fifo-home" +"$repo_root/scripts/agent-lab" --home "$fifo_home" init > "$work/fifo-init.out" 2> "$work/fifo-init.err" +mv "$fifo_home/state/locks/experiments.lock" "$work/fifo-experiments.lock" +mkfifo -m 600 "$fifo_home/state/locks/experiments.lock" +fifo_check_rc=0 +timeout --signal=TERM --kill-after=1s 2s \ + "$repo_root/scripts/agent-lab" --home "$fifo_home" config check \ + > "$work/fifo-check.out" 2> "$work/fifo-check.err" || fifo_check_rc=$? +unlink "$fifo_home/state/locks/experiments.lock" +if [ "$rerun_rc" -eq 0 ] && [ "$before" = "$after" ] && grep -Fxq 'changed:false' "$work/rerun.out" && + [ "$initialized_rc" -eq 0 ] && [ "$replacement_check_rc" -eq 125 ] && [ "$replacement_init_rc" -eq 125 ] && + [ "$fifo_check_rc" -eq 125 ]; then + printf 'PASS CFG-002 exact init retry and bound-lock verification are fail-closed\n' else - printf 'FAIL CFG-002 exact init retry is idempotent\n' + printf 'FAIL CFG-002 exact init retry and bound-lock verification are fail-closed\n' failures=$((failures + 1)) fi diff --git a/tests/image/catalog-state-cases.py b/tests/image/catalog-state-cases.py index 4fa5a23..5655cc6 100755 --- a/tests/image/catalog-state-cases.py +++ b/tests/image/catalog-state-cases.py @@ -3,7 +3,7 @@ from __future__ import annotations -from contextlib import redirect_stderr, redirect_stdout +from contextlib import contextmanager, redirect_stderr, redirect_stdout import hashlib from importlib.util import module_from_spec, spec_from_file_location import io @@ -24,6 +24,7 @@ CATALOG_MODULE = REPO_ROOT / "scripts" / "image_catalog.py" SUBJECT = "registry.example/operator/worker@sha256:" + "a" * 64 OTHER_SUBJECT = "registry.example/operator/other@sha256:" + "b" * 64 +THIRD_SUBJECT = "registry.example/operator/third@sha256:" + "c" * 64 ENTRY_DOMAIN = b"agent-lab.local-image-entry.v1\0" SNAPSHOT_DOMAIN = b"agent-lab.local-image-snapshot.v1\0" @@ -173,7 +174,14 @@ def write_record(root: Path, domain: bytes, value: dict[str, object]) -> Path: return path -def hard_exit_add(home: Path, name: str, subject: str, point: str) -> int: +def hard_exit_add( + home: Path, + name: str, + subject: str, + point: str, + *, + limits=None, +) -> int: pid = os.fork() if pid == 0: def stop_at(observed: str) -> None: @@ -181,7 +189,8 @@ def stop_at(observed: str) -> None: os._exit(99) try: - CATALOG.add_image(home, name, subject, fault=stop_at) + keyword = {} if limits is None else {"limits": limits} + CATALOG.add_image(home, name, subject, fault=stop_at, **keyword) except BaseException: os._exit(98) os._exit(97) @@ -201,6 +210,37 @@ def write(self, value: str) -> int: raise OSError("injected result-output failure") +def catalog_add_result(home: Path, name: str, subject: str, *, limits=None): + keyword = {} if limits is None else {"limits": limits} + try: + return 0, CATALOG.add_image(home, name, subject, **keyword), None + except CATALOG.CatalogReject as error: + return 1, None, error + except CATALOG.CatalogInfrastructure as error: + return 125, None, error + except BaseException as error: # The assertion records an uncontained production fault as RED. + return None, None, error + + +def traced_catalog_add(home: Path, name: str, subject: str): + targets: list[str] = [] + original_fsync = CATALOG.os.fsync + + def record_fsync(descriptor: int) -> None: + try: + targets.append(os.readlink(f"/proc/self/fd/{descriptor}")) + except OSError: + targets.append("") + original_fsync(descriptor) + + CATALOG.os.fsync = record_fsync + try: + rc, value, error = catalog_add_result(home, name, subject) + finally: + CATALOG.os.fsync = original_fsync + return rc, value, error, targets + + def main() -> int: with tempfile.TemporaryDirectory(prefix="agent-lab-catalog-state-") as directory: root = Path(directory) @@ -472,6 +512,33 @@ def main() -> int: f"rc={completed.returncode}", ) + split_home = new_home(root, "split-lock-home") + original_catalog_lock = CATALOG._catalog_lock + split_rejected = False + + @contextmanager + def replace_before_lock(authority, *, exclusive): + path = authority.lock + data = path.read_bytes() + path.rename(root / "split-original-image-catalog.lock") + path.write_bytes(data) + path.chmod(0o600) + with original_catalog_lock(authority, exclusive=exclusive) as descriptor: + yield descriptor + + CATALOG._catalog_lock = replace_before_lock + try: + CATALOG.list_images(split_home) + except CATALOG.CatalogInfrastructure: + split_rejected = True + finally: + CATALOG._catalog_lock = original_catalog_lock + check( + "CAT-STATE-018", + split_rejected, + "receipt-bound lock identity is rechecked after acquisition to prevent split-brain", + ) + nested_home = new_home(root, "nested-json-home") add(nested_home) nested_current = nested_home / "images" / "catalog" / "current.json" @@ -517,6 +584,76 @@ def main() -> int: f"rc={completed.returncode}", ) + physical_limit_home = new_home(root, "physical-name-limit-home") + physical_limits = CATALOG.CatalogLimits(names=2) + catalog_add_result( + physical_limit_home, + "vendor.first", + SUBJECT, + limits=physical_limits, + ) + child_rc = hard_exit_add( + physical_limit_home, + "vendor.orphan", + OTHER_SUBJECT, + "catalog immutable entry.after_noreplace", + limits=physical_limits, + ) + before = fingerprint(physical_limit_home / "images" / "catalog") + third_rc, _, third_error = catalog_add_result( + physical_limit_home, + "vendor.third", + THIRD_SUBJECT, + limits=physical_limits, + ) + after = fingerprint(physical_limit_home / "images" / "catalog") + observed = cli(physical_limit_home, "image", "list") + check( + "CAT-BOUND-003", + child_rc == 99 + and third_rc == 1 + and isinstance(third_error, CATALOG.CatalogReject) + and before == after + and observed.returncode == 0 + and [item.get("name") for item in json.loads(observed.stdout)] == ["vendor.first"], + "a valid unreachable orphan consumes physical-name capacity before new publication", + ( + f"child_rc={child_rc} third_rc={third_rc} error={third_error!r} " + f"changed={before != after} list_rc={observed.returncode}" + ), + ) + + orphan_conflict_home = new_home(root, "orphan-generation-conflict-home") + add(orphan_conflict_home, "vendor.first", SUBJECT) + child_rc = hard_exit_add( + orphan_conflict_home, + "vendor.orphan", + OTHER_SUBJECT, + "catalog immutable entry.after_noreplace", + ) + before = fingerprint(orphan_conflict_home / "images" / "catalog") + conflict_rc, _, conflict_error = catalog_add_result( + orphan_conflict_home, + "vendor.orphan", + THIRD_SUBJECT, + ) + after = fingerprint(orphan_conflict_home / "images" / "catalog") + observed = cli(orphan_conflict_home, "image", "list") + check( + "CAT-BOUND-004", + child_rc == 99 + and conflict_rc == 1 + and isinstance(conflict_error, CATALOG.CatalogReject) + and before == after + and observed.returncode == 0 + and [item.get("name") for item in json.loads(observed.stdout)] == ["vendor.first"], + "a conflicting generation-one orphan is rejected before immutable publication", + ( + f"child_rc={child_rc} conflict_rc={conflict_rc} error={conflict_error!r} " + f"changed={before != after} list_rc={observed.returncode}" + ), + ) + fsync_home = new_home(root, "fsync-home") original_fsync = MODULE.os.fsync record_fsync_failed = False @@ -631,19 +768,17 @@ def fail_pointer_replace(source: os.PathLike[str] | str, target: os.PathLike[str completed = cli(forged_home, "image", "add", "vendor.third", OTHER_SUBJECT) after = fingerprint(forged_home / "images") observed = cli(forged_home, "image", "list") - observed_value = json.loads(observed.stdout) if observed.returncode == 0 else [] check( "CAT-CRASH-004", completed.returncode == 125 and completed.stdout == b"" and before == after - and observed.returncode == 0 - and len(observed_value) == 1 - and observed_value[0].get("entryDigest") == victim_digest, + and observed.returncode == 125 + and observed.stdout == b"", "unproven staged intent cannot delete committed immutable history or its own evidence", ( f"rc={completed.returncode} changed={before != after} " - f"list_rc={observed.returncode} list={observed_value!r}" + f"list_rc={observed.returncode}" ), ) @@ -826,19 +961,40 @@ def fail_pointer_replace(source: os.PathLike[str] | str, target: os.PathLike[str stage_before = fingerprint(preintent_home / "images" / ".staging") retry = cli(preintent_home, "image", "add", "vendor.second", OTHER_SUBJECT) stage_after = fingerprint(preintent_home / "images" / ".staging") - before_value = json.loads(before_retry.stdout) if before_retry.returncode == 0 else [] + pristine_preintent_home = new_home(root, "pristine-preintent-crash-home") + pristine_child_rc = hard_exit_add( + pristine_preintent_home, + "vendor.worker", + SUBJECT, + "catalog wrapper.after_create", + ) + pristine_before = fingerprint(pristine_preintent_home / "images" / ".staging") + pristine_list = cli(pristine_preintent_home, "image", "list") + pristine_retry = cli( + pristine_preintent_home, + "image", + "add", + "vendor.worker", + SUBJECT, + ) + pristine_after = fingerprint(pristine_preintent_home / "images" / ".staging") check( "CAT-CRASH-007", child_rc == 99 - and before_retry.returncode == 0 - and [record.get("name") for record in before_value] == ["vendor.worker"] + and before_retry.returncode == 125 and retry.returncode == 125 and retry.stdout == b"" - and stage_before == stage_after, - "a pre-intent hard exit preserves the committed view and returns uncertainty without deleting residue", + and stage_before == stage_after + and pristine_child_rc == 99 + and pristine_list.returncode == 125 + and pristine_retry.returncode == 125 + and pristine_before == pristine_after, + "a pre-intent hard exit blocks reads and mutation without deleting residue", ( f"child_rc={child_rc} list_rc={before_retry.returncode} " - f"retry_rc={retry.returncode} changed={stage_before != stage_after}" + f"retry_rc={retry.returncode} changed={stage_before != stage_after} " + f"pristine_child={pristine_child_rc} pristine_list={pristine_list.returncode} " + f"pristine_retry={pristine_retry.returncode}" ), ) @@ -873,6 +1029,124 @@ def fail_pointer_replace(source: os.PathLike[str] | str, target: os.PathLike[str f"child_rc={child_rc} retry_rc={retry.returncode} changed={before != after}", ) + incomplete_home = new_home(root, "incomplete-later-phase-home") + add(incomplete_home) + child_rc = hard_exit_add( + incomplete_home, + "vendor.second", + OTHER_SUBJECT, + "catalog staged pointer.after_replace", + ) + incomplete_payload = ( + incomplete_home + / "images" + / ".staging" + / "image-catalog-operation" + / "payload" + ) + (incomplete_payload / "entry.json").unlink() + (incomplete_payload / "snapshot.json").unlink() + before = fingerprint(incomplete_home / "images") + retry = cli(incomplete_home, "image", "add", "vendor.second", OTHER_SUBJECT) + after = fingerprint(incomplete_home / "images") + check( + "CAT-CRASH-009", + child_rc == 99 + and retry.returncode == 125 + and retry.stdout == b"" + and before == after, + "a later stage cannot claim a pointer phase without candidate record evidence", + f"child_rc={child_rc} retry_rc={retry.returncode} changed={before != after}", + ) + + marker_durable_home = new_home(root, "marker-durable-recovery-home") + marker_child = hard_exit_add( + marker_durable_home, + "vendor.worker", + SUBJECT, + "catalog marker.after_write", + ) + marker_rc, marker_value, marker_error, marker_targets = traced_catalog_add( + marker_durable_home, + "vendor.worker", + SUBJECT, + ) + marker_lock_target = str(marker_durable_home / "state" / "locks" / "image-catalog.lock") + marker_images_target = str(marker_durable_home / "images") + + bootstrap_durable_home = new_home(root, "bootstrap-durable-recovery-home") + bootstrap_child = hard_exit_add( + bootstrap_durable_home, + "vendor.worker", + SUBJECT, + "bootstrap.after_rename", + ) + bootstrap_rc, bootstrap_value, bootstrap_error, bootstrap_targets = traced_catalog_add( + bootstrap_durable_home, + "vendor.worker", + SUBJECT, + ) + + pointer_durable_home = new_home(root, "pointer-durable-recovery-home") + add(pointer_durable_home) + pointer_child = hard_exit_add( + pointer_durable_home, + "vendor.second", + OTHER_SUBJECT, + "pointer.after_replace", + ) + pointer_rc, pointer_value, pointer_error, pointer_targets = traced_catalog_add( + pointer_durable_home, + "vendor.second", + OTHER_SUBJECT, + ) + + cleanup_durable_home = new_home(root, "cleanup-durable-recovery-home") + add(cleanup_durable_home) + cleanup_rc, cleanup_value, cleanup_error, cleanup_targets = traced_catalog_add( + cleanup_durable_home, + "vendor.worker", + SUBJECT, + ) + marker_ordered = ( + marker_lock_target in marker_targets + and marker_images_target in marker_targets + and marker_targets.index(marker_lock_target) < marker_targets.index(marker_images_target) + ) + check( + "CAT-CRASH-010", + marker_child == 99 + and marker_rc == 0 + and isinstance(marker_value, dict) + and marker_value.get("changed") is False + and marker_error is None + and marker_ordered + and bootstrap_child == 99 + and bootstrap_rc == 0 + and isinstance(bootstrap_value, dict) + and bootstrap_value.get("changed") is False + and bootstrap_error is None + and str(bootstrap_durable_home / "images") in bootstrap_targets + and pointer_child == 99 + and pointer_rc == 0 + and isinstance(pointer_value, dict) + and pointer_value.get("changed") is False + and pointer_error is None + and str(pointer_durable_home / "images" / "catalog") in pointer_targets + and cleanup_rc == 0 + and isinstance(cleanup_value, dict) + and cleanup_value.get("changed") is False + and cleanup_error is None + and str(cleanup_durable_home / "images" / ".staging") in cleanup_targets, + "recovery completes marker, commit-parent, and cleanup-root durability before success", + ( + f"marker=({marker_child},{marker_rc},{marker_targets!r}) " + f"bootstrap=({bootstrap_child},{bootstrap_rc},{bootstrap_targets!r}) " + f"pointer=({pointer_child},{pointer_rc},{pointer_targets!r}) " + f"cleanup=({cleanup_rc},{cleanup_targets!r})" + ), + ) + platform_home = new_home(root, "platform-home") before = fingerprint(platform_home) original_platform = MODULE.sys.platform @@ -909,9 +1183,12 @@ def fail_pointer_replace(source: os.PathLike[str] | str, target: os.PathLike[str "CAT-STATE-014", "CAT-STATE-015", "CAT-STATE-016", + "CAT-STATE-018", "CAT-STATE-017", "CAT-BOUND-001", "CAT-BOUND-002", + "CAT-BOUND-003", + "CAT-BOUND-004", "CAT-CRASH-001", "CAT-CRASH-002", "CAT-CRASH-003", @@ -920,12 +1197,14 @@ def fail_pointer_replace(source: os.PathLike[str] | str, target: os.PathLike[str "CAT-CRASH-006", "CAT-CRASH-007", "CAT-CRASH-008", + "CAT-CRASH-009", + "CAT-CRASH-010", "CAT-PLAT-001", ] if OBSERVED != expected: print(f"INFRA catalog state assertion identity drift: {OBSERVED!r}", file=sys.stderr) return 125 - print(f"SUMMARY assertions=28 expected=28 failures={FAILURES} infra=0") + print(f"SUMMARY assertions=33 expected=33 failures={FAILURES} infra=0") return 0 if FAILURES == 0 else 1 From 87c4439a433cf4e77ed7d02f2935550441ac668d Mon Sep 17 00:00:00 2001 From: xormania <127287135+xormania@users.noreply.github.com> Date: Sun, 2 Aug 2026 01:40:20 -0400 Subject: [PATCH 040/158] test(experiment): require restartable catalog cleanup --- tests/image/catalog-state-cases.py | 260 ++++++++++++++++++++++++++--- 1 file changed, 238 insertions(+), 22 deletions(-) diff --git a/tests/image/catalog-state-cases.py b/tests/image/catalog-state-cases.py index 5655cc6..21cfcd2 100755 --- a/tests/image/catalog-state-cases.py +++ b/tests/image/catalog-state-cases.py @@ -3,7 +3,7 @@ from __future__ import annotations -from contextlib import contextmanager, redirect_stderr, redirect_stdout +from contextlib import redirect_stderr, redirect_stdout import hashlib from importlib.util import module_from_spec, spec_from_file_location import io @@ -210,6 +210,84 @@ def write(self, value: str) -> int: raise OSError("injected result-output failure") +class HardExitOutput(io.StringIO): + def __init__(self, point: str) -> None: + super().__init__() + self.point = point + + def write(self, value: str) -> int: + if self.point == "result.before_write": + os._exit(99) + written = super().write(value) + if self.point == "result.after_write": + os._exit(99) + return written + + +def hard_exit_result(home: Path, point: str) -> int: + pid = os.fork() + if pid == 0: + output = HardExitOutput(point) + try: + with redirect_stdout(output), redirect_stderr(io.StringIO()): + MODULE.main(["--home", str(home), "image", "add", "vendor.worker", SUBJECT]) + except BaseException: + os._exit(98) + os._exit(97) + deadline = time.monotonic() + 5.0 + while time.monotonic() < deadline: + waited, status = os.waitpid(pid, os.WNOHANG) + if waited == pid: + return os.waitstatus_to_exitcode(status) + time.sleep(0.01) + os.kill(pid, 9) + os.waitpid(pid, 0) + return 124 + + +def hard_exit_cleanup( + home: Path, + name: str, + subject: str, + operation: str, +) -> int: + pid = os.fork() + if pid == 0: + original_unlink = Path.unlink + original_rmdir = Path.rmdir + + def stop_after_unlink(path: Path, *args, **kwargs) -> None: + original_unlink(path, *args, **kwargs) + if path.name == "intent.json": + os._exit(99) + + def stop_after_rmdir(path: Path, *args, **kwargs) -> None: + original_rmdir(path, *args, **kwargs) + if path.name == "payload": + os._exit(99) + + if operation == "intent-unlink": + Path.unlink = stop_after_unlink + elif operation == "payload-rmdir": + Path.rmdir = stop_after_rmdir + else: + os._exit(96) + try: + CATALOG.add_image(home, name, subject) + except BaseException: + os._exit(98) + os._exit(97) + deadline = time.monotonic() + 5.0 + while time.monotonic() < deadline: + waited, status = os.waitpid(pid, os.WNOHANG) + if waited == pid: + return os.waitstatus_to_exitcode(status) + time.sleep(0.01) + os.kill(pid, 9) + os.waitpid(pid, 0) + return 124 + + def catalog_add_result(home: Path, name: str, subject: str, *, limits=None): keyword = {} if limits is None else {"limits": limits} try: @@ -513,29 +591,31 @@ def main() -> int: ) split_home = new_home(root, "split-lock-home") - original_catalog_lock = CATALOG._catalog_lock + original_flock = CATALOG.fcntl.flock split_rejected = False - - @contextmanager - def replace_before_lock(authority, *, exclusive): - path = authority.lock - data = path.read_bytes() - path.rename(root / "split-original-image-catalog.lock") - path.write_bytes(data) - path.chmod(0o600) - with original_catalog_lock(authority, exclusive=exclusive) as descriptor: - yield descriptor - - CATALOG._catalog_lock = replace_before_lock + split_replaced = False + + def replace_after_flock(descriptor: int, operation: int) -> None: + nonlocal split_replaced + original_flock(descriptor, operation) + if operation in (CATALOG.fcntl.LOCK_SH, CATALOG.fcntl.LOCK_EX) and not split_replaced: + path = split_home / "state" / "locks" / "image-catalog.lock" + data = path.read_bytes() + path.rename(root / "split-original-image-catalog.lock") + path.write_bytes(data) + path.chmod(0o600) + split_replaced = True + + CATALOG.fcntl.flock = replace_after_flock try: CATALOG.list_images(split_home) except CATALOG.CatalogInfrastructure: split_rejected = True finally: - CATALOG._catalog_lock = original_catalog_lock + CATALOG.fcntl.flock = original_flock check( "CAT-STATE-018", - split_rejected, + split_rejected and split_replaced, "receipt-bound lock identity is rechecked after acquisition to prevent split-brain", ) @@ -730,14 +810,26 @@ def fail_pointer_replace(source: os.PathLike[str] | str, target: os.PathLike[str ) retry = cli(result_home, "image", "add", "vendor.worker", SUBJECT) retry_value = json.loads(retry.stdout) if retry.returncode == 0 else {} + result_exit_codes: list[int] = [] + result_retries: list[dict[str, object]] = [] + for index, point in enumerate(("result.before_write", "result.after_write")): + hard_exit_home = new_home(root, f"result-hard-exit-{index}") + result_exit_codes.append(hard_exit_result(hard_exit_home, point)) + completed = cli(hard_exit_home, "image", "add", "vendor.worker", SUBJECT) + result_retries.append(json.loads(completed.stdout) if completed.returncode == 0 else {}) check( "CAT-CRASH-003", first_rc == 125 and first_error is None and retry.returncode == 0 - and retry_value.get("changed") is False, - "lost result output reports uncertainty while retry observes the committed binding idempotently", - f"first_rc={first_rc} error={first_error!r} retry_rc={retry.returncode} retry={retry_value!r}", + and retry_value.get("changed") is False + and result_exit_codes == [99, 99] + and [value.get("changed") for value in result_retries] == [False, False], + "failures and hard exits before or after result write retry as committed idempotent state", + ( + f"first_rc={first_rc} error={first_error!r} retry_rc={retry.returncode} " + f"retry={retry_value!r} exits={result_exit_codes!r} hard_retries={result_retries!r}" + ), ) forged_home = new_home(root, "forged-stage-home") @@ -842,6 +934,16 @@ def fail_pointer_replace(source: os.PathLike[str] | str, target: os.PathLike[str "bootstrap.after_rename", "catalog bootstrap parent.before_fsync", "catalog bootstrap parent.after_fsync", + "catalog staging cleanup.before_remove", + "catalog staging cleanup handoff.before_rename", + "catalog staging cleanup handoff.after_rename", + "catalog staging cleanup handoff.before_fsync", + "catalog staging cleanup handoff.after_fsync", + "catalog staging cleanup.after_remove", + "catalog staging cleanup.after_unlink", + "catalog staging cleanup.after_rmdir", + "catalog staging cleanup.before_fsync", + "catalog staging cleanup.after_fsync", ) later_points = ( "catalog intent.before_fsync", @@ -876,6 +978,16 @@ def fail_pointer_replace(source: os.PathLike[str] | str, target: os.PathLike[str "pointer.after_replace", "catalog current pointer.before_fsync", "catalog current pointer.after_fsync", + "catalog staging cleanup.before_remove", + "catalog staging cleanup handoff.before_rename", + "catalog staging cleanup handoff.after_rename", + "catalog staging cleanup handoff.before_fsync", + "catalog staging cleanup handoff.after_fsync", + "catalog staging cleanup.after_remove", + "catalog staging cleanup.after_unlink", + "catalog staging cleanup.after_rmdir", + "catalog staging cleanup.before_fsync", + "catalog staging cleanup.after_fsync", ) matrix_failures: list[str] = [] for index, point in enumerate(bootstrap_points): @@ -1018,15 +1130,46 @@ def fail_pointer_replace(source: os.PathLike[str] | str, target: os.PathLike[str before = fingerprint(phase_home / "images") retry = cli(phase_home, "image", "add", "vendor.worker", SUBJECT) after = fingerprint(phase_home / "images") + + marker_phase_home = new_home(root, "initialized-incomplete-bootstrap-home") + marker_child = hard_exit_add( + marker_phase_home, + "vendor.worker", + SUBJECT, + "catalog marker.after_fsync", + ) + marker_payload = ( + marker_phase_home + / "images" + / ".staging" + / "image-catalog-operation" + / "payload" + ) + shutil.rmtree(marker_payload) + marker_before = fingerprint(marker_phase_home / "images") + marker_read = cli(marker_phase_home, "image", "list") + marker_retry = cli(marker_phase_home, "image", "add", "vendor.worker", SUBJECT) + marker_after = fingerprint(marker_phase_home / "images") check( "CAT-CRASH-008", child_rc == 99 and retry.returncode == 125 and retry.stdout == b"" and before == after - and not (phase_home / "images" / "catalog").exists(), + and not (phase_home / "images" / "catalog").exists() + and marker_child == 99 + and marker_read.returncode == 125 + and marker_read.stdout == b"" + and marker_retry.returncode == 125 + and marker_retry.stdout == b"" + and marker_before == marker_after + and not (marker_phase_home / "images" / "catalog").exists(), "phase-inconsistent bootstrap staging remains inert and cannot become committed authority", - f"child_rc={child_rc} retry_rc={retry.returncode} changed={before != after}", + ( + f"child_rc={child_rc} retry_rc={retry.returncode} changed={before != after} " + f"marker_child={marker_child} marker_read={marker_read.returncode} " + f"marker_retry={marker_retry.returncode} marker_changed={marker_before != marker_after}" + ), ) incomplete_home = new_home(root, "incomplete-later-phase-home") @@ -1147,6 +1290,78 @@ def fail_pointer_replace(source: os.PathLike[str] | str, target: os.PathLike[str ), ) + cleanup_unlink_home = new_home(root, "cleanup-intent-unlink-home") + add(cleanup_unlink_home) + committed_child = hard_exit_add( + cleanup_unlink_home, + "vendor.second", + OTHER_SUBJECT, + "pointer.after_replace", + ) + unlink_child = hard_exit_cleanup( + cleanup_unlink_home, + "vendor.second", + OTHER_SUBJECT, + "intent-unlink", + ) + unlink_read = cli(cleanup_unlink_home, "image", "list") + unlink_retry = cli( + cleanup_unlink_home, + "image", + "add", + "vendor.second", + OTHER_SUBJECT, + ) + + cleanup_rmdir_home = new_home(root, "cleanup-payload-rmdir-home") + add(cleanup_rmdir_home) + orphan_child = hard_exit_add( + cleanup_rmdir_home, + "vendor.second", + OTHER_SUBJECT, + "catalog immutable entry.after_noreplace", + ) + rmdir_child = hard_exit_cleanup( + cleanup_rmdir_home, + "vendor.third", + THIRD_SUBJECT, + "payload-rmdir", + ) + rmdir_read = cli(cleanup_rmdir_home, "image", "list") + rmdir_retry = cli( + cleanup_rmdir_home, + "image", + "add", + "vendor.third", + THIRD_SUBJECT, + ) + check( + "CAT-CRASH-011", + committed_child == 99 + and unlink_child == 99 + and unlink_read.returncode == 0 + and [item.get("name") for item in json.loads(unlink_read.stdout)] + == ["vendor.second", "vendor.worker"] + and unlink_retry.returncode == 0 + and json.loads(unlink_retry.stdout).get("changed") is False + and not tuple((cleanup_unlink_home / "images" / ".staging").iterdir()) + and orphan_child == 99 + and rmdir_child == 99 + and rmdir_read.returncode == 0 + and [item.get("name") for item in json.loads(rmdir_read.stdout)] + == ["vendor.worker"] + and rmdir_retry.returncode == 0 + and json.loads(rmdir_retry.stdout).get("changed") is True + and not tuple((cleanup_rmdir_home / "images" / ".staging").iterdir()), + "cleanup remains restartable across internal intent-unlink and payload-rmdir crashes", + ( + f"committed={committed_child} unlink={unlink_child} " + f"unlink_read={unlink_read.returncode} unlink_retry={unlink_retry.returncode} " + f"orphan={orphan_child} rmdir={rmdir_child} " + f"rmdir_read={rmdir_read.returncode} rmdir_retry={rmdir_retry.returncode}" + ), + ) + platform_home = new_home(root, "platform-home") before = fingerprint(platform_home) original_platform = MODULE.sys.platform @@ -1199,12 +1414,13 @@ def fail_pointer_replace(source: os.PathLike[str] | str, target: os.PathLike[str "CAT-CRASH-008", "CAT-CRASH-009", "CAT-CRASH-010", + "CAT-CRASH-011", "CAT-PLAT-001", ] if OBSERVED != expected: print(f"INFRA catalog state assertion identity drift: {OBSERVED!r}", file=sys.stderr) return 125 - print(f"SUMMARY assertions=33 expected=33 failures={FAILURES} infra=0") + print(f"SUMMARY assertions=34 expected=34 failures={FAILURES} infra=0") return 0 if FAILURES == 0 else 1 From 6323045596aa40b1bec62e2f44feb7a08502f743 Mon Sep 17 00:00:00 2001 From: xormania <127287135+xormania@users.noreply.github.com> Date: Sun, 2 Aug 2026 01:47:44 -0400 Subject: [PATCH 041/158] feat(experiment): harden local image catalog --- packaging/agent-lab-local.manifest | 1 + scripts/agent-lab | 2 +- scripts/agent-lab.py | 175 +++- scripts/experiment.py | 25 +- scripts/image_catalog.py | 799 ++++++++++++++---- scripts/image_reference.py | 39 + .../fixtures/expected-runtime-files.txt | 1 + 7 files changed, 857 insertions(+), 185 deletions(-) create mode 100644 scripts/image_reference.py diff --git a/packaging/agent-lab-local.manifest b/packaging/agent-lab-local.manifest index ea39df5..c3bfbd1 100644 --- a/packaging/agent-lab-local.manifest +++ b/packaging/agent-lab-local.manifest @@ -10,5 +10,6 @@ scripts/dev/cedar-tool.py scripts/dev/cue-tool.py scripts/experiment.py scripts/image_catalog.py +scripts/image_reference.py tools/cedar.lock tools/cue.lock diff --git a/scripts/agent-lab b/scripts/agent-lab index 031da71..5eada79 100755 --- a/scripts/agent-lab +++ b/scripts/agent-lab @@ -3,4 +3,4 @@ set -euo pipefail script_path="$(readlink -f "${BASH_SOURCE[0]}")" script_dir="$(cd -- "$(dirname -- "$script_path")" >/dev/null 2>&1 && pwd)" -exec python3 -I "$script_dir/agent-lab.py" "$@" +exec python3 -I -B "$script_dir/agent-lab.py" "$@" diff --git a/scripts/agent-lab.py b/scripts/agent-lab.py index 85c0037..940dd62 100644 --- a/scripts/agent-lab.py +++ b/scripts/agent-lab.py @@ -13,6 +13,18 @@ import sys SAFE_COMPONENT = re.compile(r"^[a-z][a-z0-9-]{0,47}$") +LOCK_SPECS = { + "imageCatalog": ( + "image-catalog.lock", + "agent-lab.image-catalog-lock/v0alpha1", + b"initialized\n", + ), + "experiments": ( + "experiments.lock", + "agent-lab.experiments-lock/v0alpha1", + b"", + ), +} def canonical(value: object) -> bytes: @@ -33,6 +45,117 @@ def config_value(components: dict[str, str]) -> dict[str, object]: return {"apiVersion": "agent-lab.config/v0alpha1", "paths": components} +def write_all(descriptor: int, data: bytes) -> None: + view = memoryview(data) + while view: + written = os.write(descriptor, view) + if written <= 0: + raise OSError("write made no progress") + view = view[written:] + + +def lock_record(path: Path, relative: str, schema: str) -> dict[str, object]: + metadata = path.lstat() + if ( + not stat.S_ISREG(metadata.st_mode) + or metadata.st_uid != os.getuid() + or metadata.st_nlink != 1 + or stat.S_IMODE(metadata.st_mode) != 0o600 + ): + raise OSError("Agent Lab stable lock metadata is unsafe") + return { + "device": metadata.st_dev, + "inode": metadata.st_ino, + "path": relative, + "schema": schema, + } + + +def verify_lock(home: Path, state_component: str, key: str, record: object) -> None: + filename, schema, appended = LOCK_SPECS[key] + relative = f"{state_component}/locks/{filename}" + if ( + not isinstance(record, dict) + or set(record) != {"device", "inode", "path", "schema"} + or not isinstance(record.get("device"), int) + or isinstance(record.get("device"), bool) + or not isinstance(record.get("inode"), int) + or isinstance(record.get("inode"), bool) + or record.get("path") != relative + or record.get("schema") != schema + ): + raise RuntimeError("home lock receipt is not closed") + path = home / relative + maximum = len(schema.encode("ascii") + b"\n" + appended) + try: + lexical = path.lstat() + identity = (lexical.st_dev, lexical.st_ino) + if ( + not stat.S_ISREG(lexical.st_mode) + or lexical.st_uid != os.getuid() + or lexical.st_nlink != 1 + or stat.S_IMODE(lexical.st_mode) != 0o600 + or lexical.st_size > maximum + or identity != (record["device"], record["inode"]) + ): + raise OSError("home lock authority metadata is unsafe") + descriptor = os.open( + path, + os.O_RDONLY + | getattr(os, "O_CLOEXEC", 0) + | getattr(os, "O_NOFOLLOW", 0) + | getattr(os, "O_NONBLOCK", 0), + ) + except OSError as error: + raise RuntimeError("home lock authority is unavailable") from error + try: + opened = os.fstat(descriptor) + chunks: list[bytes] = [] + remaining = maximum + 1 + while remaining: + chunk = os.read(descriptor, remaining) + if not chunk: + break + chunks.append(chunk) + remaining -= len(chunk) + data = b"".join(chunks) + final = os.fstat(descriptor) + except OSError as error: + raise RuntimeError("home lock authority could not be verified") from error + finally: + os.close(descriptor) + try: + current = path.lstat() + except OSError as error: + raise RuntimeError("home lock authority could not be reverified") from error + if ( + not stat.S_ISREG(opened.st_mode) + or opened.st_uid != os.getuid() + or opened.st_nlink != 1 + or stat.S_IMODE(opened.st_mode) != 0o600 + or opened.st_size > maximum + or not stat.S_ISREG(final.st_mode) + or final.st_uid != os.getuid() + or final.st_nlink != 1 + or stat.S_IMODE(final.st_mode) != 0o600 + or final.st_size > maximum + or not stat.S_ISREG(current.st_mode) + or current.st_uid != os.getuid() + or current.st_nlink != 1 + or stat.S_IMODE(current.st_mode) != 0o600 + or current.st_size > maximum + or (opened.st_dev, opened.st_ino) != identity + or (final.st_dev, final.st_ino) != identity + or (current.st_dev, current.st_ino) != identity + or identity != (record["device"], record["inode"]) + ): + raise RuntimeError("home lock authority identity is unsafe") + base = schema.encode("ascii") + b"\n" + accepted = (base, base + appended) if appended else (base,) + if data not in accepted or final.st_size != len(data): + raise RuntimeError("home lock authority bytes are invalid") + + def init_home(home: Path, argv: list[str]) -> int: components = {"experiments": "experiments", "images": "images", "cache": "cache", "state": "state"} option_map = {"--experiments-dir": "experiments", "--images-dir": "images", "--cache-dir": "cache", "--state-dir": "state"} @@ -46,19 +169,18 @@ def init_home(home: Path, argv: list[str]) -> int: return 1 config = config_value(components) config_bytes = canonical(config) + b"\n" - receipt = { - "apiVersion": "agent-lab.home/v0alpha1", - "configDigest": "sha256:" + hashlib.sha256(canonical(config)).hexdigest(), - "paths": components, - } - receipt_bytes = canonical(receipt) + b"\n" try: home.mkdir(mode=0o700, parents=True, exist_ok=True) os.chmod(home, 0o700) existing = home / "home.json" config_path = home / "config.json" if existing.exists() or config_path.exists(): - if existing.read_bytes() == receipt_bytes and config_path.read_bytes() == config_bytes: + try: + loaded = load_config(home) + except RuntimeError as error: + print(f"INFRA Agent Lab {error}", file=sys.stderr) + return 125 + if loaded is not None and loaded[1] == config_bytes: print("changed:false") return 0 print("FAIL Agent Lab home conflicts with requested configuration", file=sys.stderr) @@ -81,13 +203,31 @@ def init_home(home: Path, argv: list[str]) -> int: (component_roots["cache"] / "tools/cedar").mkdir(mode=0o700, parents=True) locks = component_roots["state"] / "locks" locks.mkdir(mode=0o700) - for name in ("image-catalog.lock", "experiments.lock"): - descriptor = os.open(locks / name, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) - os.close(descriptor) + lock_records: dict[str, object] = {} + for key, (name, schema, _) in LOCK_SPECS.items(): + path = locks / name + descriptor = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + try: + write_all(descriptor, schema.encode("ascii") + b"\n") + os.fsync(descriptor) + finally: + os.close(descriptor) + relative = f"{components['state']}/locks/{name}" + lock_records[key] = lock_record(path, relative, schema) + receipt = { + "apiVersion": "agent-lab.home/v0alpha1", + "configDigest": "sha256:" + hashlib.sha256(canonical(config)).hexdigest(), + "locks": lock_records, + "paths": components, + } + receipt_bytes = canonical(receipt) + b"\n" for path, data in ((config_path, config_bytes), (existing, receipt_bytes)): descriptor = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) - os.write(descriptor, data) - os.close(descriptor) + try: + write_all(descriptor, data) + os.fsync(descriptor) + finally: + os.close(descriptor) except OSError: print("INFRA Agent Lab home could not be initialized safely", file=sys.stderr) return 125 @@ -127,13 +267,20 @@ def load_config(home: Path) -> tuple[dict[str, object], bytes] | None: raise RuntimeError("configuration is not canonical") if ( not isinstance(receipt, dict) - or set(receipt) != {"apiVersion", "configDigest", "paths"} + or set(receipt) != {"apiVersion", "configDigest", "locks", "paths"} or receipt["apiVersion"] != "agent-lab.home/v0alpha1" or receipt["paths"] != paths or receipt["configDigest"] != "sha256:" + hashlib.sha256(canonical(value)).hexdigest() or receipt_raw != canonical(receipt) + b"\n" ): raise RuntimeError("configuration does not match the initialized home receipt") + locks = receipt["locks"] + if not isinstance(locks, dict) or set(locks) != set(LOCK_SPECS): + raise RuntimeError("home lock receipt is not closed") + state_component = paths["state"] + assert isinstance(state_component, str) + for key in LOCK_SPECS: + verify_lock(home, state_component, key, locks[key]) return value, canonical_config @@ -286,7 +433,7 @@ def main(argv: list[str]) -> int: return experiment_module().main(["experiment.py", "authorize-directory", argv[3]]) if argv[:1] == ["image"]: return image_command(home, argv[1:]) - print("Usage: agent-lab [--home ABSOLUTE_HOME] {version|init|config|experiment}", file=sys.stderr) + print("Usage: agent-lab [--home ABSOLUTE_HOME] {version|init|config|experiment|image}", file=sys.stderr) return 2 diff --git a/scripts/experiment.py b/scripts/experiment.py index 615a132..9e03775 100644 --- a/scripts/experiment.py +++ b/scripts/experiment.py @@ -23,7 +23,6 @@ SOURCE_DIGEST_DOMAIN = b"agent-lab.experiment-tree.v1\0" BUNDLED_CATALOG_DOMAIN = b"agent-lab.experiment-image-catalog.v1\0" BUNDLED_ENTRY_DOMAIN = b"agent-lab.experiment-image-entry.v1\0" -CATALOG_NAME_COMPONENT = re.compile(r"^[a-z][a-z0-9]*(?:-[a-z0-9]+)*$") MAX_CUE_OUTPUT_BYTES = 1_048_576 MAX_CONTRACT_FILE_BYTES = 1_048_576 MAX_HELPER_BYTES = 1_048_576 @@ -577,15 +576,21 @@ def expected_plan(manifest: object, contract_digest: str) -> dict[str, object]: } -def valid_catalog_name(value: object) -> bool: - if not isinstance(value, str) or len(value.encode("utf-8")) > 63: - return False - parts = value.split(".") - return ( - len(parts) == 2 - and all(1 <= len(part.encode("ascii", "ignore")) <= 31 for part in parts) - and all(part.isascii() and CATALOG_NAME_COMPONENT.fullmatch(part) for part in parts) - ) +def image_reference_module(): + path = Path(__file__).resolve().with_name("image_reference.py") + spec = spec_from_file_location("agent_lab_image_reference", path) + if spec is None or spec.loader is None: + raise InfrastructureError("shared image-reference grammar cannot be loaded") + module = module_from_spec(spec) + sys.modules[spec.name] = module + try: + spec.loader.exec_module(module) + except (ImportError, OSError) as error: + raise InfrastructureError("shared image-reference grammar cannot be loaded") from error + return module + + +valid_catalog_name = image_reference_module().valid_image_name def digest_record(domain: bytes, value: object) -> str: diff --git a/scripts/image_catalog.py b/scripts/image_catalog.py index 74c9b9c..0aac6bd 100644 --- a/scripts/image_catalog.py +++ b/scripts/image_catalog.py @@ -9,6 +9,7 @@ import errno import fcntl import hashlib +from importlib.util import module_from_spec, spec_from_file_location import json import os from pathlib import Path @@ -25,18 +26,12 @@ ENTRY_DOMAIN = b"agent-lab.local-image-entry.v1\0" SNAPSHOT_DOMAIN = b"agent-lab.local-image-snapshot.v1\0" OPERATION_WRAPPER = "image-catalog-operation" -LOCK_MARKER = b"catalog:v0alpha1\n" +CLEANUP_WRAPPER = "image-catalog-cleanup" +LOCK_PRISTINE = b"agent-lab.image-catalog-lock/v0alpha1\n" +LOCK_INITIALIZED = LOCK_PRISTINE + b"initialized\n" +LOCK_SCHEMA = "agent-lab.image-catalog-lock/v0alpha1" SAFE_COMPONENT = re.compile(r"^[a-z][a-z0-9-]{0,47}$") -IMAGE_COMPONENT = re.compile(r"^[a-z][a-z0-9]*(?:-[a-z0-9]+)*$") SHA256 = re.compile(r"^sha256:[0-9a-f]{64}$") -OCI_SUBJECT = re.compile( - r"^([a-z0-9]+([.-][a-z0-9]+)*" - r"(:(?:[1-9][0-9]{0,3}|[1-5][0-9]{4}|6[0-4][0-9]{3}|" - r"65[0-4][0-9]{2}|655[0-2][0-9]|6553[0-5]))?/)?" - r"[a-z0-9]+([._-][a-z0-9]+)*" - r"(/[a-z0-9]+([._-][a-z0-9]+)*)*" - r"@sha256:[0-9a-f]{64}$" -) HEX_FILE = re.compile(r"^[0-9a-f]{64}\.json$") FaultHook = Callable[[str], None] @@ -77,6 +72,8 @@ class HomeAuthority: state: Path locks: Path lock: Path + lock_device: int + lock_inode: int @dataclass(frozen=True) @@ -88,6 +85,8 @@ class CatalogState: records: dict[str, dict[str, object]] entries: dict[str, dict[str, object]] snapshots: dict[str, dict[str, object]] + physical_names: frozenset[str] + generations: dict[tuple[str, int], str] physical_bytes: int @@ -111,28 +110,20 @@ def record_digest(domain: bytes, value: object) -> str: return "sha256:" + hashlib.sha256(domain + canonical(value)).hexdigest() -def image_name(value: object) -> bool: - if not isinstance(value, str) or not value.isascii(): - return False - encoded = value.encode("ascii") - parts = value.split(".") - return ( - len(encoded) <= 63 - and len(parts) == 2 - and all(1 <= len(part.encode("ascii")) <= 31 for part in parts) - and all(IMAGE_COMPONENT.fullmatch(part) is not None for part in parts) - ) - - -def oci_subject(value: object) -> bool: - return ( - isinstance(value, str) - and value.isascii() - and 1 <= len(value.encode("ascii")) <= 255 - and OCI_SUBJECT.fullmatch(value) is not None - ) +def _image_reference_module(): + path = Path(__file__).resolve().with_name("image_reference.py") + spec = spec_from_file_location("agent_lab_catalog_image_reference", path) + if spec is None or spec.loader is None: + raise ImportError("shared image-reference grammar cannot be loaded") + module = module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module +_IMAGE_REFERENCE = _image_reference_module() +image_name = _IMAGE_REFERENCE.valid_image_name +oci_subject = _IMAGE_REFERENCE.valid_oci_subject valid_image_name = image_name valid_oci_subject = oci_subject @@ -294,7 +285,7 @@ def _pairs(pairs: list[tuple[str, object]]) -> dict[str, object]: def _canonical_json(data: bytes, purpose: str) -> dict[str, object]: try: value = json.loads(data.decode("utf-8"), object_pairs_hook=_pairs) - except (UnicodeError, ValueError, json.JSONDecodeError) as error: + except (RecursionError, UnicodeError, ValueError, json.JSONDecodeError) as error: _infra(f"{purpose} is malformed", error) if not isinstance(value, dict) or data != canonical(value) + b"\n": _infra(f"{purpose} is not one canonical closed object") @@ -321,11 +312,14 @@ def _load_home(home: Path) -> HomeAuthority: ): _infra("Agent Lab configuration paths are unsafe") digest = "sha256:" + hashlib.sha256(canonical(config)).hexdigest() + lock_records = receipt.get("locks") if ( - set(receipt) != {"apiVersion", "configDigest", "paths"} + set(receipt) != {"apiVersion", "configDigest", "locks", "paths"} or receipt.get("apiVersion") != "agent-lab.home/v0alpha1" or receipt.get("configDigest") != digest or receipt.get("paths") != paths + or not isinstance(lock_records, dict) + or set(lock_records) != {"experiments", "imageCatalog"} ): _infra("Agent Lab configuration does not match its home receipt") images = home / str(paths["images"]) @@ -335,7 +329,38 @@ def _load_home(home: Path) -> HomeAuthority: for path in (images, state, staging, locks): _verify_directory(path) lock = locks / "image-catalog.lock" - return HomeAuthority(home, images, staging, state, locks, lock) + lock_record = lock_records.get("imageCatalog") + expected_lock_path = f"{paths['state']}/locks/image-catalog.lock" + if ( + not isinstance(lock_record, dict) + or set(lock_record) != {"device", "inode", "path", "schema"} + or lock_record.get("path") != expected_lock_path + or lock_record.get("schema") != LOCK_SCHEMA + or not isinstance(lock_record.get("device"), int) + or isinstance(lock_record.get("device"), bool) + or not isinstance(lock_record.get("inode"), int) + or isinstance(lock_record.get("inode"), bool) + ): + _infra("catalog lock authority is absent from the home receipt") + try: + lock_metadata = lock.lstat() + except OSError as error: + _infra("catalog lock is unavailable", error) + if ( + lock_metadata.st_dev != lock_record["device"] + or lock_metadata.st_ino != lock_record["inode"] + ): + _infra("catalog lock identity does not match the home receipt") + return HomeAuthority( + home, + images, + staging, + state, + locks, + lock, + int(lock_record["device"]), + int(lock_record["inode"]), + ) @contextmanager @@ -351,21 +376,48 @@ def _catalog_lock(authority: HomeAuthority, *, exclusive: bool) -> Iterator[int] or lexical.st_uid != os.getuid() or stat.S_IMODE(lexical.st_mode) != 0o600 or lexical.st_nlink != 1 - or lexical.st_size > len(LOCK_MARKER) + or lexical.st_size > len(LOCK_INITIALIZED) + or (lexical.st_dev, lexical.st_ino) + != (authority.lock_device, authority.lock_inode) ): _infra("catalog lock metadata is unsafe") try: descriptor = os.open( path, - os.O_RDWR | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_NOFOLLOW", 0), + os.O_RDWR + | getattr(os, "O_CLOEXEC", 0) + | getattr(os, "O_NOFOLLOW", 0) + | getattr(os, "O_NONBLOCK", 0), ) opened = os.fstat(descriptor) - if _file_identity(opened) != _file_identity(lexical): + if ( + not stat.S_ISREG(opened.st_mode) + or opened.st_uid != os.getuid() + or stat.S_IMODE(opened.st_mode) != 0o600 + or opened.st_nlink != 1 + or opened.st_size > len(LOCK_INITIALIZED) + or (opened.st_dev, opened.st_ino) + != (authority.lock_device, authority.lock_inode) + ): raise OSError("lock identity changed") fcntl.flock(descriptor, fcntl.LOCK_EX if exclusive else fcntl.LOCK_SH) current = path.lstat() held = os.fstat(descriptor) - if (held.st_dev, held.st_ino) != (current.st_dev, current.st_ino): + expected_identity = (authority.lock_device, authority.lock_inode) + if ( + not stat.S_ISREG(current.st_mode) + or current.st_uid != os.getuid() + or stat.S_IMODE(current.st_mode) != 0o600 + or current.st_nlink != 1 + or current.st_size > len(LOCK_INITIALIZED) + or not stat.S_ISREG(held.st_mode) + or held.st_uid != os.getuid() + or stat.S_IMODE(held.st_mode) != 0o600 + or held.st_nlink != 1 + or held.st_size > len(LOCK_INITIALIZED) + or (held.st_dev, held.st_ino) != expected_identity + or (current.st_dev, current.st_ino) != expected_identity + ): raise OSError("lock path was replaced") except OSError as error: try: @@ -386,24 +438,33 @@ def _catalog_lock(authority: HomeAuthority, *, exclusive: bool) -> Iterator[int] def _lock_bytes(descriptor: int) -> bytes: try: os.lseek(descriptor, 0, os.SEEK_SET) - data = os.read(descriptor, len(LOCK_MARKER) + 1) + data = os.read(descriptor, len(LOCK_INITIALIZED) + 1) os.lseek(descriptor, 0, os.SEEK_SET) except OSError as error: _infra("catalog lock marker cannot be read", error) - if data not in (b"", LOCK_MARKER): + if data not in (LOCK_PRISTINE, LOCK_INITIALIZED): _infra("catalog lock marker is malformed") return data -def _set_lock_marker(descriptor: int) -> None: +def _set_lock_marker(descriptor: int, fault: FaultHook | None = None) -> None: + current = _lock_bytes(descriptor) try: - os.lseek(descriptor, 0, os.SEEK_SET) - os.ftruncate(descriptor, 0) - _write_all(descriptor, LOCK_MARKER) + if current == LOCK_PRISTINE: + os.lseek(descriptor, 0, os.SEEK_END) + _fault(fault, "catalog marker.before_write") + written = os.write(descriptor, b"initialized\n") + if written != len(b"initialized\n"): + raise OSError("catalog marker write was partial") + _fault(fault, "catalog marker.after_write") + _fault(fault, "catalog marker.before_fsync") os.fsync(descriptor) + _fault(fault, "catalog marker.after_fsync") os.lseek(descriptor, 0, os.SEEK_SET) except OSError as error: _infra("catalog initialization marker could not be persisted", error) + if _lock_bytes(descriptor) != LOCK_INITIALIZED: + _infra("catalog initialization marker could not be verified") def _write_all(descriptor: int, data: bytes) -> None: @@ -415,11 +476,17 @@ def _write_all(descriptor: int, data: bytes) -> None: view = view[written:] -def _directory_names(path: Path, purpose: str) -> tuple[str, ...]: +def _directory_names(path: Path, purpose: str, maximum: int) -> tuple[str, ...]: _verify_directory(path) try: before = path.lstat() - names = tuple(sorted(os.listdir(path), key=os.fsencode)) + names_list: list[str] = [] + with os.scandir(path) as entries: + for entry in entries: + names_list.append(entry.name) + if len(names_list) > maximum: + _infra(f"{purpose} exceeds its fixed entry bound") + names = tuple(sorted(names_list, key=os.fsencode)) after = path.lstat() except (OSError, UnicodeError) as error: _infra(f"{purpose} cannot be enumerated", error) @@ -560,15 +627,103 @@ def _intent_schema(value: dict[str, object]) -> None: _infra("catalog staging intent values are invalid") -def _stage_state(authority: HomeAuthority, limits: CatalogLimits) -> StageState | None: - names = _directory_names(authority.staging, "catalog staging root") +def _cleanup_state(authority: HomeAuthority, limits: CatalogLimits) -> Path | None: + names = _directory_names(authority.staging, "catalog staging root", 1) + if not names or names == (OPERATION_WRAPPER,): + return None + if names != (CLEANUP_WRAPPER,): + _infra("catalog staging root contains an unknown wrapper") + cleanup = authority.staging / CLEANUP_WRAPPER + _verify_directory(cleanup) + entry_count = 0 + byte_count = 0 + pending = [cleanup] + while pending: + parent = pending.pop() + remaining = limits.stage_entries - entry_count + for name in _directory_names(parent, "catalog cleanup residue", remaining): + item = parent / name + relative = str(item.relative_to(cleanup)) + parts = item.relative_to(cleanup).parts + directory_allowed = relative in { + "payload", + "payload/catalog", + "payload/catalog/entries", + "payload/catalog/snapshots", + } + file_allowed = relative in { + "intent.json", + "payload/catalog/current.json", + "payload/catalog/current.next", + "payload/entry.json", + "payload/snapshot.json", + "payload/current.json", + "payload/current.next", + } + if ( + len(parts) == 4 + and parts[:3] in { + ("payload", "catalog", "entries"), + ("payload", "catalog", "snapshots"), + } + and HEX_FILE.fullmatch(parts[3]) is not None + ): + file_allowed = True + if not directory_allowed and not file_allowed: + _infra("catalog cleanup residue contains an unknown entry") + try: + metadata = item.lstat() + except OSError as error: + _infra("catalog cleanup residue cannot be inspected", error) + entry_count += 1 + if metadata.st_uid != os.getuid() or stat.S_ISLNK(metadata.st_mode): + _infra("catalog cleanup residue metadata is unsafe") + if stat.S_ISDIR(metadata.st_mode): + if not directory_allowed or stat.S_IMODE(metadata.st_mode) != 0o700: + _infra("catalog cleanup residue directory mode is unsafe") + pending.append(item) + elif stat.S_ISREG(metadata.st_mode): + if ( + not file_allowed + or stat.S_IMODE(metadata.st_mode) != 0o600 + or metadata.st_nlink != 1 + ): + _infra("catalog cleanup residue file metadata is unsafe") + byte_count += metadata.st_size + else: + _infra("catalog cleanup residue type is unsafe") + if entry_count > limits.stage_entries or byte_count > limits.stage_bytes: + _infra("catalog cleanup residue exceeds its fixed bound") + intent_path = cleanup / "intent.json" + try: + intent_path.lstat() + except FileNotFoundError: + pass + except OSError as error: + _infra("catalog cleanup intent cannot be inspected", error) + else: + intent = _canonical_json( + _read_file(intent_path, 65_536, "catalog cleanup intent"), + "catalog cleanup intent", + ) + _intent_schema(intent) + return cleanup + + +def _stage_state( + authority: HomeAuthority, + limits: CatalogLimits, +) -> StageState | None: + if _cleanup_state(authority, limits) is not None: + return None + names = _directory_names(authority.staging, "catalog staging root", 1) if not names: return None if names != (OPERATION_WRAPPER,): _infra("catalog staging root contains an unknown wrapper") wrapper = authority.staging / OPERATION_WRAPPER _verify_directory(wrapper) - wrapper_names = _directory_names(wrapper, "catalog operation wrapper") + wrapper_names = _directory_names(wrapper, "catalog operation wrapper", 2) if "intent.json" not in wrapper_names or any(name not in {"intent.json", "payload"} for name in wrapper_names): _infra("catalog operation wrapper is incomplete or unknown") intent = _canonical_json( @@ -594,27 +749,32 @@ def _stage_state(authority: HomeAuthority, limits: CatalogLimits) -> StageState } else: allowed = {"entry.json", "snapshot.json", "current.json", "current.next"} - try: - descendants = sorted(payload.rglob("*"), key=lambda item: os.fsencode(str(item.relative_to(payload)))) - except OSError as error: - _infra("catalog staging payload cannot be enumerated", error) - for item in descendants: - relative = str(item.relative_to(payload)) - if relative not in allowed: - _infra("catalog staging payload contains an unknown entry") - metadata = item.lstat() - entry_count += 1 - if metadata.st_uid != os.getuid() or stat.S_ISLNK(metadata.st_mode): - _infra("catalog staging payload metadata is unsafe") - if stat.S_ISDIR(metadata.st_mode): - if stat.S_IMODE(metadata.st_mode) != 0o700: - _infra("catalog staging directory mode is unsafe") - elif stat.S_ISREG(metadata.st_mode): - if stat.S_IMODE(metadata.st_mode) != 0o600 or metadata.st_nlink != 1: - _infra("catalog staging file metadata is unsafe") - byte_count += metadata.st_size - else: - _infra("catalog staging payload type is unsafe") + pending = [payload] + while pending: + parent = pending.pop() + remaining = limits.stage_entries - entry_count + for name in _directory_names(parent, "catalog staging payload", remaining): + item = parent / name + relative = str(item.relative_to(payload)) + if relative not in allowed: + _infra("catalog staging payload contains an unknown entry") + try: + metadata = item.lstat() + except OSError as error: + _infra("catalog staging payload cannot be inspected", error) + entry_count += 1 + if metadata.st_uid != os.getuid() or stat.S_ISLNK(metadata.st_mode): + _infra("catalog staging payload metadata is unsafe") + if stat.S_ISDIR(metadata.st_mode): + if stat.S_IMODE(metadata.st_mode) != 0o700: + _infra("catalog staging directory mode is unsafe") + pending.append(item) + elif stat.S_ISREG(metadata.st_mode): + if stat.S_IMODE(metadata.st_mode) != 0o600 or metadata.st_nlink != 1: + _infra("catalog staging file metadata is unsafe") + byte_count += metadata.st_size + else: + _infra("catalog staging payload type is unsafe") if entry_count > limits.stage_entries or byte_count > limits.stage_bytes: _infra("catalog staging state exceeds its fixed bound") return StageState(wrapper, intent) @@ -698,7 +858,7 @@ def _load_catalog( ) -> CatalogState: stage = _stage_state(authority, limits) if inspect_stage else None marker = _lock_bytes(lock_descriptor) - image_names = _directory_names(authority.images, "effective images directory") + image_names = _directory_names(authority.images, "effective images directory", 2) allowed_images = {".staging", "catalog"} if any(name not in allowed_images for name in image_names): _infra("effective images directory contains unknown catalog state") @@ -706,23 +866,43 @@ def _load_catalog( try: root_metadata = root.lstat() except FileNotFoundError: - if marker == LOCK_MARKER: + if marker == LOCK_INITIALIZED and not ( + stage is not None and bool(stage.intent.get("bootstrap")) + ): _infra("an initialized image catalog is missing") if stage is not None and not bool(stage.intent.get("bootstrap")): _infra("non-bootstrap catalog stage has no committed base") - return CatalogState(None, 0, None, None, {}, {}, {}, 0) + empty = CatalogState( + None, + 0, + None, + None, + {}, + {}, + {}, + frozenset(), + {}, + 0, + ) + if stage is not None: + _verify_staged_candidate(stage, empty, limits) + if marker == LOCK_INITIALIZED: + _complete_bootstrap_stage(stage) + return empty except OSError as error: _infra("image catalog cannot be inspected", error) if not stat.S_ISDIR(root_metadata.st_mode) or stat.S_ISLNK(root_metadata.st_mode): _infra("image catalog path is unsafe") + if marker != LOCK_INITIALIZED: + _infra("image catalog exists without durable initialization authority") _verify_directory(root) - root_names = _directory_names(root, "image catalog") + root_names = _directory_names(root, "image catalog", 3) if set(root_names) != {"current.json", "entries", "snapshots"}: _infra("image catalog layout is incomplete or unknown") entries_root = root / "entries" snapshots_root = root / "snapshots" - entry_names = _directory_names(entries_root, "image entry history") - snapshot_names = _directory_names(snapshots_root, "image snapshot history") + entry_names = _directory_names(entries_root, "image entry history", limits.entries) + snapshot_names = _directory_names(snapshots_root, "image snapshot history", limits.snapshots) if ( len(entry_names) > limits.entries or len(snapshot_names) > limits.snapshots @@ -753,6 +933,32 @@ def _load_catalog( _infra("local image entry filename does not bind its bytes") entries[digest] = value + generations: dict[tuple[str, int], str] = {} + physical_names: set[str] = set() + for entry_digest, entry in entries.items(): + name = str(entry["name"]) + generation = int(entry["generation"]) + physical_names.add(name) + key = (name, generation) + existing = generations.get(key) + if existing is not None and existing != entry_digest: + _infra("local image history contains conflicting physical generations") + generations[key] = entry_digest + if len(physical_names) > limits.names: + _infra("physical local image history exceeds its fixed name bound") + for entry in entries.values(): + if entry["state"] != "removed": + continue + predecessor = entries.get(str(entry["previousEntryDigest"])) + if ( + predecessor is None + or predecessor["state"] != "active" + or predecessor["generation"] != 1 + or predecessor["name"] != entry["name"] + or predecessor["subject"] != entry["subject"] + ): + _infra("unreachable local image tombstone history is invalid") + snapshots: dict[str, dict[str, object]] = {} for name in snapshot_names: path = snapshots_root / name @@ -766,6 +972,14 @@ def _load_catalog( _infra("local image snapshot filename does not bind its bytes") snapshots[digest] = value + genesis = [ + snapshot_digest + for snapshot_digest, snapshot in snapshots.items() + if snapshot["previousSnapshotDigest"] is None + ] + if len(genesis) != 1: + _infra("local image snapshot history does not have one physical genesis") + for snapshot_digest, snapshot in snapshots.items(): records = snapshot["records"] assert isinstance(records, dict) @@ -815,12 +1029,12 @@ def _load_catalog( digest = str(projection["entryDigest"]) records[name] = _public_record(entries[digest], digest) if ( - entry_names != _directory_names(entries_root, "image entry history") - or snapshot_names != _directory_names(snapshots_root, "image snapshot history") - or root_names != _directory_names(root, "image catalog") + entry_names != _directory_names(entries_root, "image entry history", limits.entries) + or snapshot_names != _directory_names(snapshots_root, "image snapshot history", limits.snapshots) + or root_names != _directory_names(root, "image catalog", 3) ): _infra("image catalog changed during verification") - return CatalogState( + result = CatalogState( root, int(current["revision"]), current_digest, @@ -828,8 +1042,13 @@ def _load_catalog( records, entries, snapshots, + frozenset(physical_names), + generations, physical_bytes, ) + if stage is not None: + _verify_staged_candidate(stage, result, limits) + return result def _write_file(path: Path, data: bytes, purpose: str, fault: FaultHook | None = None) -> None: @@ -911,7 +1130,7 @@ def _rename_noreplace(source: Path, target: Path) -> None: _infra("catalog no-replace publication failed", OSError(code, os.strerror(code))) -def _remove_owned_tree(path: Path) -> None: +def _remove_owned_tree(path: Path, fault: FaultHook | None = None) -> None: try: metadata = path.lstat() except FileNotFoundError: @@ -925,6 +1144,7 @@ def _remove_owned_tree(path: Path) -> None: _infra("catalog staging cleanup file is unsafe") try: path.unlink() + _fault(fault, "catalog staging cleanup.after_unlink") except OSError as error: _infra("catalog staging file could not be cleaned", error) return @@ -934,20 +1154,50 @@ def _remove_owned_tree(path: Path) -> None: children = tuple(path.iterdir()) except OSError as error: _infra("catalog staging cleanup cannot enumerate its target", error) - for child in sorted(children, key=lambda item: os.fsencode(item.name)): - _remove_owned_tree(child) + for child in sorted( + children, + key=lambda item: (item.name == "intent.json", os.fsencode(item.name)), + ): + _remove_owned_tree(child, fault) try: path.rmdir() + _fault(fault, "catalog staging cleanup.after_rmdir") except OSError as error: _infra("catalog staging directory could not be cleaned", error) -def _cleanup_stage(authority: HomeAuthority, stage: StageState | None = None) -> None: +def _cleanup_stage( + authority: HomeAuthority, + stage: StageState | None = None, + fault: FaultHook | None = None, +) -> None: path = authority.staging / OPERATION_WRAPPER + cleanup = authority.staging / CLEANUP_WRAPPER if stage is not None and stage.path != path: _infra("catalog staging cleanup target changed") - _remove_owned_tree(path) - _fsync_directory(authority.staging, "catalog staging cleanup") + _fault(fault, "catalog staging cleanup.before_remove") + _fault(fault, "catalog staging cleanup handoff.before_rename") + try: + _rename_noreplace(path, cleanup) + except OSError as error: + _infra("catalog staging cleanup handoff is uncertain", error) + _fault(fault, "catalog staging cleanup handoff.after_rename") + _fsync_directory(authority.staging, "catalog staging cleanup handoff", fault) + _fault(fault, "catalog staging cleanup.after_remove") + _remove_owned_tree(cleanup, fault) + _fsync_directory(authority.staging, "catalog staging cleanup", fault) + + +def _finish_cleanup( + authority: HomeAuthority, + cleanup: Path, + fault: FaultHook | None = None, +) -> None: + expected = authority.staging / CLEANUP_WRAPPER + if cleanup != expected: + _infra("catalog cleanup residue target changed") + _remove_owned_tree(cleanup, fault) + _fsync_directory(authority.staging, "catalog staging cleanup", fault) def _read_current_digest(authority: HomeAuthority) -> str | None: @@ -967,63 +1217,85 @@ def _read_current_digest(authority: HomeAuthority) -> str | None: return _current_schema(pointer) -def _remove_uncommitted_record(path: Path, preexisting: bool) -> None: - try: - metadata = path.lstat() - except FileNotFoundError: - return - except OSError as error: - _infra("uncommitted catalog record cannot be inspected", error) - if preexisting: - return +def _complete_bootstrap_stage(stage: StageState) -> Path: + intent = stage.intent + catalog = stage.path / "payload" / "catalog" + entry_name = f"{str(intent['candidateEntryDigest'])[7:]}.json" + snapshot_name = f"{str(intent['candidateSnapshotDigest'])[7:]}.json" if ( - not stat.S_ISREG(metadata.st_mode) - or stat.S_ISLNK(metadata.st_mode) - or metadata.st_uid != os.getuid() - or stat.S_IMODE(metadata.st_mode) != 0o600 - or metadata.st_nlink != 1 + _directory_names(stage.path / "payload", "catalog bootstrap payload", 1) + != ("catalog",) + or _directory_names(catalog, "catalog bootstrap candidate", 3) + != ("current.json", "entries", "snapshots") + or _directory_names(catalog / "entries", "catalog bootstrap entries", 1) + != (entry_name,) + or _directory_names(catalog / "snapshots", "catalog bootstrap snapshots", 1) + != (snapshot_name,) ): - _infra("uncommitted catalog record metadata is unsafe") - try: - path.unlink() - except OSError as error: - _infra("uncommitted catalog record could not be reconciled", error) + _infra("initialized bootstrap stage is incomplete") + return catalog def _reconcile(authority: HomeAuthority, lock_descriptor: int, limits: CatalogLimits) -> None: + cleanup = _cleanup_state(authority, limits) + if cleanup is not None: + _finish_cleanup(authority, cleanup) + return stage = _stage_state(authority, limits) if stage is None: + _fsync_directory(authority.staging, "catalog empty staging recovery") return intent = stage.intent candidate = str(intent["candidateSnapshotDigest"]) base_value = intent["baseSnapshotDigest"] base = str(base_value) if base_value is not None else None current = _read_current_digest(authority) + if current not in (base, candidate): + _infra("catalog staged operation cannot be reconciled with current state") + if current is None: + state = CatalogState( + None, + 0, + None, + None, + {}, + {}, + {}, + frozenset(), + {}, + 0, + ) + else: + state = _load_catalog(authority, lock_descriptor, limits, inspect_stage=False) + _verify_staged_candidate(stage, state, limits) if current == candidate: - _load_catalog(authority, lock_descriptor, limits, inspect_stage=False) - if _lock_bytes(lock_descriptor) == b"": + if bool(intent["bootstrap"]): _set_lock_marker(lock_descriptor) + _fsync_directory(authority.images, "catalog bootstrap recovery parent") + else: + if state.root is None: + _infra("committed catalog mutation has no final catalog") + _fsync_directory(state.root, "catalog current pointer recovery") + verified = _load_catalog(authority, lock_descriptor, limits, inspect_stage=False) + if verified.snapshot_digest != candidate: + _infra("committed catalog recovery could not be verified") _cleanup_stage(authority, stage) return - if current != base: - _infra("catalog staged operation cannot be reconciled with current state") if bool(intent["bootstrap"]): if current is not None: _infra("catalog bootstrap stage conflicts with a final catalog") - else: - root = authority.images / "catalog" - entries = root / "entries" - snapshots = root / "snapshots" - _remove_uncommitted_record( - entries / f"{str(intent['candidateEntryDigest'])[7:]}.json", - bool(intent["entryPreexisting"]), - ) - _remove_uncommitted_record( - snapshots / f"{candidate[7:]}.json", - bool(intent["snapshotPreexisting"]), - ) - _fsync_directory(entries, "catalog reconciled entry history") - _fsync_directory(snapshots, "catalog reconciled snapshot history") + if _lock_bytes(lock_descriptor) == LOCK_INITIALIZED: + _set_lock_marker(lock_descriptor) + catalog = _complete_bootstrap_stage(stage) + _rename_noreplace(catalog, authority.images / "catalog") + _fsync_directory(authority.images, "catalog bootstrap recovery parent") + verified = _load_catalog(authority, lock_descriptor, limits, inspect_stage=False) + if verified.snapshot_digest != candidate: + _infra("recovered catalog bootstrap could not be verified") + _cleanup_stage(authority, stage) + return + # Verified immutable orphans are safe and bounded. Retain them rather than + # treating caller-controlled intent flags as deletion authority. _cleanup_stage(authority, stage) @@ -1064,6 +1336,12 @@ def _candidate_values( "subjectDigest": str(prior["subject"]).rsplit("@", 1)[1], } entry_digest = record_digest(ENTRY_DOMAIN, entry) + generation = int(entry["generation"]) + existing_generation = state.generations.get((name, generation)) + if existing_generation is not None and existing_generation != entry_digest: + _reject("catalog physical history already contains a conflicting generation") + if name not in state.physical_names and len(state.physical_names) >= limits.names: + _reject("catalog has reached its fixed physical-name capacity") projections: dict[str, dict[str, object]] = {} for existing_name in sorted(state.records, key=lambda item: item.encode("ascii")): record = state.records[existing_name] @@ -1116,6 +1394,218 @@ def _candidate_values( return entry, entry_digest, snapshot, snapshot_digest, intent +def _state_at_snapshot(state: CatalogState, snapshot_digest: str | None) -> CatalogState: + if snapshot_digest is None: + return CatalogState( + None, + 0, + None, + None, + {}, + state.entries, + state.snapshots, + state.physical_names, + state.generations, + state.physical_bytes, + ) + snapshot = state.snapshots.get(snapshot_digest) + if snapshot is None: + _infra("catalog staging intent names an unavailable base snapshot") + projections = snapshot["records"] + assert isinstance(projections, dict) + records: dict[str, dict[str, object]] = {} + for name, projection in projections.items(): + assert isinstance(projection, dict) + entry_digest = str(projection["entryDigest"]) + entry = state.entries.get(entry_digest) + if entry is None: + _infra("catalog staging base references missing entry history") + records[name] = _public_record(entry, entry_digest) + previous = snapshot["previousSnapshotDigest"] + return CatalogState( + state.root, + int(snapshot["revision"]), + snapshot_digest, + str(previous) if previous is not None else None, + records, + state.entries, + state.snapshots, + state.physical_names, + state.generations, + state.physical_bytes, + ) + + +def _verify_optional_stage_file(path: Path, expected: bytes, maximum: int, purpose: str) -> bool: + try: + path.lstat() + except FileNotFoundError: + return False + except OSError as error: + _infra(f"{purpose} cannot be inspected", error) + if _read_file(path, maximum, purpose) != expected: + _infra(f"{purpose} does not match its durable intent") + return True + + +def _verify_staged_candidate( + stage: StageState, + physical_state: CatalogState, + limits: CatalogLimits, +) -> tuple[dict[str, object], str, dict[str, object], str]: + intent = stage.intent + base_value = intent["baseSnapshotDigest"] + base_digest = str(base_value) if base_value is not None else None + base = _state_at_snapshot(physical_state, base_digest) + try: + entry, entry_digest, snapshot, snapshot_digest, expected_intent = _candidate_values( + base, + kind=str(intent["kind"]), + name=str(intent["name"]), + subject=str(intent["subject"]), + expected_entry_digest=( + str(intent["expectedEntryDigest"]) + if intent["expectedEntryDigest"] is not None + else None + ), + limits=limits, + ) + except (AssertionError, CatalogReject) as error: + _infra("catalog staging intent is not a valid operation from its base", error) + semantic_fields = set(expected_intent) - {"entryPreexisting", "snapshotPreexisting"} + if any(intent[field] != expected_intent[field] for field in semantic_fields): + _infra("catalog staging intent does not bind its deterministic candidate") + if bool(intent["entryPreexisting"]) and entry_digest not in physical_state.entries: + _infra("catalog staging intent claims unavailable preexisting entry history") + if bool(intent["snapshotPreexisting"]) and snapshot_digest not in physical_state.snapshots: + _infra("catalog staging intent claims unavailable preexisting snapshot history") + current_digest = physical_state.snapshot_digest + if current_digest not in (base_digest, snapshot_digest): + _infra("catalog staged operation does not match the physical commit phase") + + entry_bytes = canonical(entry) + b"\n" + snapshot_bytes = canonical(snapshot) + b"\n" + pointer_bytes = canonical( + {"apiVersion": CURRENT_API, "snapshotDigest": snapshot_digest} + ) + b"\n" + payload = stage.path / "payload" + try: + payload.lstat() + except FileNotFoundError: + final_entry = entry_digest in physical_state.entries + final_snapshot = snapshot_digest in physical_state.snapshots + if current_digest == snapshot_digest: + if not final_entry or not final_snapshot: + _infra("committed catalog stage is missing its immutable candidate history") + elif final_entry or final_snapshot: + _infra("uncommitted catalog stage lost its candidate publication evidence") + return entry, entry_digest, snapshot, snapshot_digest + except OSError as error: + _infra("catalog staging payload cannot be inspected", error) + if bool(intent["bootstrap"]): + catalog = payload / "catalog" + try: + catalog.lstat() + except FileNotFoundError: + if _directory_names(payload, "catalog bootstrap payload", 1): + _infra("catalog bootstrap payload lost its candidate directory") + final_entry = entry_digest in physical_state.entries + final_snapshot = snapshot_digest in physical_state.snapshots + if current_digest == snapshot_digest: + if not final_entry or not final_snapshot: + _infra("committed bootstrap stage is missing immutable history") + elif final_entry or final_snapshot: + _infra("uncommitted bootstrap stage contains final candidate history") + return entry, entry_digest, snapshot, snapshot_digest + except OSError as error: + _infra("catalog bootstrap candidate cannot be inspected", error) + if current_digest == snapshot_digest: + _infra("committed bootstrap stage retained a second candidate catalog") + catalog_names = set(_directory_names(catalog, "catalog bootstrap candidate", 3)) + entries_present = "entries" in catalog_names + snapshots_present = "snapshots" in catalog_names + current_present = "current.json" in catalog_names + next_present = "current.next" in catalog_names + if snapshots_present and not entries_present: + _infra("catalog bootstrap stage skipped its entry-directory phase") + if current_present and next_present: + _infra("catalog bootstrap stage contains conflicting pointer phases") + staged_entry = False + staged_snapshot = False + if entries_present: + staged_entry = _verify_optional_stage_file( + catalog / "entries" / f"{entry_digest[7:]}.json", + entry_bytes, + limits.entry_bytes, + "catalog staged entry", + ) + if snapshots_present: + staged_snapshot = _verify_optional_stage_file( + catalog / "snapshots" / f"{snapshot_digest[7:]}.json", + snapshot_bytes, + limits.snapshot_bytes, + "catalog staged snapshot", + ) + if staged_snapshot and not staged_entry: + _infra("catalog bootstrap stage skipped its entry-record phase") + staged_pointer = False + for name in ("current.json", "current.next"): + staged_pointer = ( + _verify_optional_stage_file( + catalog / name, + pointer_bytes, + 65_536, + "catalog staged pointer", + ) + or staged_pointer + ) + if staged_pointer and not (staged_entry and staged_snapshot): + _infra("catalog bootstrap pointer lacks complete candidate records") + else: + staged_entry = _verify_optional_stage_file( + payload / "entry.json", + entry_bytes, + limits.entry_bytes, + "catalog staged entry", + ) + staged_snapshot = _verify_optional_stage_file( + payload / "snapshot.json", + snapshot_bytes, + limits.snapshot_bytes, + "catalog staged snapshot", + ) + current_present = _verify_optional_stage_file( + payload / "current.json", + pointer_bytes, + 65_536, + "catalog staged pointer", + ) + next_present = _verify_optional_stage_file( + payload / "current.next", + pointer_bytes, + 65_536, + "catalog staged pointer", + ) + if current_present and next_present: + _infra("catalog mutation stage contains conflicting pointer phases") + staged_pointer = current_present or next_present + final_entry = entry_digest in physical_state.entries + final_snapshot = snapshot_digest in physical_state.snapshots + if staged_snapshot and not (staged_entry or final_entry): + _infra("catalog staged snapshot lacks candidate entry evidence") + if staged_pointer and not ( + (staged_entry or final_entry) and (staged_snapshot or final_snapshot) + ): + _infra("catalog staged pointer lacks complete candidate record evidence") + if current_digest == snapshot_digest: + if staged_pointer or not final_entry or not final_snapshot: + _infra("committed catalog stage has an impossible publication shape") + else: + if (final_entry or final_snapshot) and not staged_pointer: + _infra("uncommitted catalog stage lost its durable pointer evidence") + return entry, entry_digest, snapshot, snapshot_digest + + def _prepare_stage( authority: HomeAuthority, entry: dict[str, object], @@ -1123,12 +1613,15 @@ def _prepare_stage( snapshot: dict[str, object], snapshot_digest: str, intent: dict[str, object], + limits: CatalogLimits, fault: FaultHook | None, ) -> StageState: wrapper = authority.staging / OPERATION_WRAPPER payload = wrapper / "payload" try: + _fault(fault, "catalog wrapper.before_create") _mkdir(wrapper) + _fault(fault, "catalog wrapper.after_create") _write_file(wrapper / "intent.json", canonical(intent) + b"\n", "catalog intent", fault) _fsync_directory(wrapper, "catalog intent wrapper", fault) _fsync_directory(authority.staging, "catalog intent publication", fault) @@ -1154,7 +1647,9 @@ def _prepare_stage( ) pointer = {"apiVersion": CURRENT_API, "snapshotDigest": snapshot_digest} _write_file(catalog / "current.next", canonical(pointer) + b"\n", "catalog staged pointer", fault) + _fault(fault, "catalog staged pointer.before_replace") os.replace(catalog / "current.next", catalog / "current.json") + _fault(fault, "catalog staged pointer.after_replace") _fsync_directory(entries, "catalog staged entries", fault) _fsync_directory(snapshots, "catalog staged snapshots", fault) _fsync_directory(catalog, "catalog staged root", fault) @@ -1163,25 +1658,17 @@ def _prepare_stage( _write_file(payload / "snapshot.json", canonical(snapshot) + b"\n", "catalog staged snapshot", fault) pointer = {"apiVersion": CURRENT_API, "snapshotDigest": snapshot_digest} _write_file(payload / "current.next", canonical(pointer) + b"\n", "catalog staged pointer", fault) + _fault(fault, "catalog staged pointer.before_replace") os.replace(payload / "current.next", payload / "current.json") + _fault(fault, "catalog staged pointer.after_replace") _fsync_directory(payload, "catalog staged payload", fault) _fsync_directory(wrapper, "catalog staged wrapper", fault) _fsync_directory(authority.staging, "catalog staged operation", fault) except CatalogInfrastructure: - try: - _remove_owned_tree(wrapper) - _fsync_directory(authority.staging, "catalog failed-stage cleanup") - except CatalogInfrastructure: - pass raise except OSError as error: - try: - _remove_owned_tree(wrapper) - _fsync_directory(authority.staging, "catalog failed-stage cleanup") - except CatalogInfrastructure: - pass _infra("catalog operation could not be staged", error) - stage = _stage_state(authority, CatalogLimits()) + stage = _stage_state(authority, limits) if stage is None: _infra("catalog operation stage disappeared") return stage @@ -1192,12 +1679,17 @@ def _publish_record(source: Path, target: Path, maximum: int, purpose: str, faul try: target.lstat() except FileNotFoundError: - _write_file(target, data, purpose, fault) + _fault(fault, f"{purpose}.before_noreplace") + _rename_noreplace(source, target) + _fault(fault, f"{purpose}.after_noreplace") except OSError as error: _infra(f"{purpose} final path cannot be inspected", error) else: if _read_file(target, maximum, purpose) != data: _infra(f"{purpose} conflicts with existing immutable bytes") + return + if _read_file(target, maximum, purpose) != data: + _infra(f"{purpose} no-replace publication changed its immutable bytes") def _publish_candidate( @@ -1214,12 +1706,12 @@ def _publish_candidate( committed = False try: if bool(intent["bootstrap"]): + _set_lock_marker(lock_descriptor, fault) _fault(fault, "bootstrap.before_rename") _rename_noreplace(payload / "catalog", authority.images / "catalog") committed = True _fault(fault, "bootstrap.after_rename") _fsync_directory(authority.images, "catalog bootstrap parent", fault) - _set_lock_marker(lock_descriptor) else: assert state.root is not None root = state.root @@ -1251,7 +1743,7 @@ def _publish_candidate( verified = _load_catalog(authority, lock_descriptor, limits, inspect_stage=False) if verified.snapshot_digest != candidate: _infra("catalog publication could not be verified") - _cleanup_stage(authority, stage) + _cleanup_stage(authority, stage, fault) except CatalogInfrastructure: if not committed: try: @@ -1273,7 +1765,15 @@ def _classified(operation: Callable[[], object]) -> object: return operation() except (CatalogReject, CatalogInfrastructure): raise - except (AssertionError, KeyError, OSError, TypeError, ValueError, json.JSONDecodeError) as error: + except ( + AssertionError, + KeyError, + OSError, + RecursionError, + TypeError, + ValueError, + json.JSONDecodeError, + ) as error: _infra("catalog operation encountered unverified state", error) @@ -1320,6 +1820,7 @@ def operation() -> dict[str, object]: snapshot, snapshot_digest, intent, + limits, fault, ) _publish_candidate(authority, lock_descriptor, state, stage, limits, fault) @@ -1380,6 +1881,7 @@ def operation() -> dict[str, object]: snapshot, snapshot_digest, intent, + limits, fault, ) _publish_candidate(authority, lock_descriptor, state, stage, limits, fault) @@ -1480,26 +1982,3 @@ def operation() -> dict[str, object]: result = _classified(operation) assert isinstance(result, dict) return result - - -@contextmanager -def hold_live_bindings( - home: Path, - bindings: Sequence[dict[str, object]], - *, - limits: CatalogLimits = CatalogLimits(), -) -> Iterator[dict[str, object]]: - """Hold the shared catalog lock while PR4 publishes a selected plan.""" - - authority = _load_home(home) - with _catalog_lock(authority, exclusive=False) as lock_descriptor: - state = _load_catalog(authority, lock_descriptor, limits) - for binding in bindings: - name = binding.get("name") - expected = binding.get("entryDigest") - record = state.records.get(str(name)) - if record is None or record["state"] != "active" or record["entryDigest"] != expected: - _reject("selected local image entry is no longer active") - if state.snapshot_digest is None: - _infra("selected local image snapshot is unavailable") - yield {"revision": state.revision, "snapshotDigest": state.snapshot_digest} diff --git a/scripts/image_reference.py b/scripts/image_reference.py new file mode 100644 index 0000000..b5e4b3a --- /dev/null +++ b/scripts/image_reference.py @@ -0,0 +1,39 @@ +#!/usr/bin/env python3 +"""Shared pure grammar for Experiment image names and immutable OCI subjects.""" + +from __future__ import annotations + +import re + + +IMAGE_COMPONENT = re.compile(r"^[a-z][a-z0-9]*(?:-[a-z0-9]+)*$") +OCI_SUBJECT = re.compile( + r"^([a-z0-9]+([.-][a-z0-9]+)*" + r"(:(?:[1-9][0-9]{0,3}|[1-5][0-9]{4}|6[0-4][0-9]{3}|" + r"65[0-4][0-9]{2}|655[0-2][0-9]|6553[0-5]))?/)?" + r"[a-z0-9]+([._-][a-z0-9]+)*" + r"(/[a-z0-9]+([._-][a-z0-9]+)*)*" + r"@sha256:[0-9a-f]{64}$" +) + + +def valid_image_name(value: object) -> bool: + if not isinstance(value, str) or not value.isascii(): + return False + encoded = value.encode("ascii") + parts = value.split(".") + return ( + len(encoded) <= 63 + and len(parts) == 2 + and all(1 <= len(part.encode("ascii")) <= 31 for part in parts) + and all(IMAGE_COMPONENT.fullmatch(part) is not None for part in parts) + ) + + +def valid_oci_subject(value: object) -> bool: + return ( + isinstance(value, str) + and value.isascii() + and 1 <= len(value.encode("ascii")) <= 255 + and OCI_SUBJECT.fullmatch(value) is not None + ) diff --git a/tests/install/fixtures/expected-runtime-files.txt b/tests/install/fixtures/expected-runtime-files.txt index ea39df5..c3bfbd1 100644 --- a/tests/install/fixtures/expected-runtime-files.txt +++ b/tests/install/fixtures/expected-runtime-files.txt @@ -10,5 +10,6 @@ scripts/dev/cedar-tool.py scripts/dev/cue-tool.py scripts/experiment.py scripts/image_catalog.py +scripts/image_reference.py tools/cedar.lock tools/cue.lock From 0971fb121fd18476f2d6901410ac531b78672b5c Mon Sep 17 00:00:00 2001 From: xormania <127287135+xormania@users.noreply.github.com> Date: Sun, 2 Aug 2026 01:48:42 -0400 Subject: [PATCH 042/158] test(experiment): enforce catalog evidence --- tests/experiment/aggregate-harness-cases.sh | 118 +- tests/experiment/catalog-resolution-cases.sh | 140 ++- tests/experiment/contract-cases.sh | 93 +- tests/experiment/local-config-cases.sh | 28 +- tests/experiment/local-image-catalog-cases.sh | 66 +- tests/experiment/local-lifecycle-cases.sh | 113 +- tests/helpers/run-bounded.py | 249 ++++ tests/image/catalog-cases.sh | 130 +- tests/image/catalog-mutation-cases.py | 1047 +++++++++++++++++ tests/image/catalog-state-cases.py | 68 +- tests/install/local-install-cases.sh | 28 +- 11 files changed, 1986 insertions(+), 94 deletions(-) create mode 100644 tests/helpers/run-bounded.py create mode 100644 tests/image/catalog-mutation-cases.py diff --git a/tests/experiment/aggregate-harness-cases.sh b/tests/experiment/aggregate-harness-cases.sh index b3115d5..06b4a15 100755 --- a/tests/experiment/aggregate-harness-cases.sh +++ b/tests/experiment/aggregate-harness-cases.sh @@ -3,8 +3,25 @@ set -u -o pipefail repo_root="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." >/dev/null 2>&1 && pwd)" lifecycle="$repo_root/tests/experiment/local-lifecycle-cases.sh" -work="$(mktemp -d)" -trap 'find "$work" -type f -delete 2>/dev/null || true; find "$work" -type l -delete 2>/dev/null || true; find "$work" -depth -type d -exec rmdir {} + 2>/dev/null || true' EXIT +expected_count=9 +work="" + +cleanup_work() { + local failed=0 + if [ -n "$work" ] && [ -e "$work" ]; then + find "$work" -type f -delete 2>/dev/null || failed=1 + find "$work" -type l -delete 2>/dev/null || failed=1 + find "$work" -depth -type d -exec rmdir {} + 2>/dev/null || failed=1 + [ ! -e "$work" ] || failed=1 + fi + return "$failed" +} + +if ! work="$(mktemp -d)"; then + printf 'SUMMARY assertions=0 expected=%s failures=0 infra=1\n' "$expected_count" + exit 125 +fi +trap 'cleanup_work >/dev/null 2>&1 || true' EXIT replica="$work/repo" replica_lifecycle="$replica/tests/experiment/local-lifecycle-cases.sh" mkdir -p "$replica/tests/experiment" "$replica/tests/install" @@ -25,14 +42,15 @@ expected_ids=( CAT-STATE-001 CAT-STATE-002 CAT-STATE-003 CAT-STATE-004 CAT-STATE-005 CAT-STATE-006 CAT-STATE-007 CAT-STATE-008 CAT-STATE-009 CAT-STATE-010 CAT-STATE-011 CAT-STATE-012 - CAT-STATE-013 CAT-STATE-014 CAT-STATE-015 CAT-STATE-016 CAT-STATE-017 - CAT-BOUND-001 CAT-BOUND-002 CAT-CRASH-001 CAT-CRASH-002 - CAT-CRASH-003 CAT-CRASH-004 CAT-CRASH-005 CAT-CRASH-006 CAT-CRASH-007 CAT-PLAT-001 + CAT-STATE-013 CAT-STATE-014 CAT-STATE-015 CAT-STATE-016 CAT-STATE-018 CAT-STATE-017 + CAT-BOUND-001 CAT-BOUND-002 CAT-BOUND-003 CAT-BOUND-004 + CAT-CRASH-001 CAT-CRASH-002 CAT-CRASH-003 CAT-CRASH-004 CAT-CRASH-005 + CAT-CRASH-006 CAT-CRASH-007 CAT-CRASH-008 CAT-CRASH-009 CAT-CRASH-010 CAT-CRASH-011 CAT-PLAT-001 RES-ENTRY-001 RES-SNAP-001 RES-SNAP-002 RES-ENTRY-002 RES-ENTRY-003 RES-ISOLATE-001 RES-STATE-001 RES-STATE-002 RES-STATE-003 RES-INPUT-001 RES-SNAP-003 RES-AUTH-001 RES-INSTALL-001 RES-NOEF-001 - M-CAT-OCI-001 M-CAT-CAS-001 M-CAT-AUTH-001 M-RES-BIND-001 - M-CAT-NOEF-001 M-CAT-ATOM-001 M-CAT-DUR-001 M-CAT-STAGE-001 + M-CAT-OCI-001 M-CAT-SHADOW-001 M-CAT-CAS-001 M-CAT-AUTH-001 M-RES-BIND-001 + M-CAT-NOEF-001 M-CAT-ADMIT-001 M-CAT-ATOM-001 M-CAT-DUR-001 M-CAT-STAGE-001 ) installer_ids=("${expected_ids[@]:0:5}") config_ids=("${expected_ids[@]:5:5}") @@ -43,8 +61,12 @@ write_fixture() { local rc="$2" shift 2 local record kind id fixture_failures=0 + local execution_id="${path##*/}" { printf '#!/usr/bin/env bash\nset -u\n' + printf 'if [ -n "${AGENT_LAB_AGG_EXEC_LOG:-}" ]; then\n' + printf " printf '%%s\\n' '%s' >> \"\$AGENT_LAB_AGG_EXEC_LOG\" || exit 125\n" "$execution_id" + printf 'fi\n' for record in "$@"; do kind="${record%%:*}" id="${record#*:}" @@ -78,18 +100,62 @@ reset_fixtures() { run_replica() { local output="$1" shift - "$@" bash "$replica_lifecycle" > "$output" 2>&1 + run_selected "$output" "$replica_lifecycle" "$@" +} + +run_selected() { + local output="$1" + local selected_lifecycle="$2" + shift 2 + "$@" bash "$selected_lifecycle" > "$output" 2>&1 return $? } -if [ "$(grep -Fxc ' "$repo_root/tests/install/local-install-cases.sh"' "$lifecycle")" -eq 1 ] && - [ "$(grep -Fxc ' "$repo_root/tests/experiment/local-config-cases.sh"' "$lifecycle")" -eq 1 ] && - [ "$(grep -Fxc ' "$repo_root/tests/experiment/local-image-catalog-cases.sh"' "$lifecycle")" -eq 1 ] && - [ "$(grep -nF ' "$repo_root/tests/install/local-install-cases.sh"' "$lifecycle" | cut -d: -f1)" -lt "$(grep -nF ' "$repo_root/tests/experiment/local-config-cases.sh"' "$lifecycle" | cut -d: -f1)" ] && - [ "$(grep -nF ' "$repo_root/tests/experiment/local-config-cases.sh"' "$lifecycle" | cut -d: -f1)" -lt "$(grep -nF ' "$repo_root/tests/experiment/local-image-catalog-cases.sh"' "$lifecycle" | cut -d: -f1)" ]; then - pass AGG-001 "lifecycle subcases are declared exactly once in order" +reset_fixtures +expected_executions="$work/expected-executions" +baseline_executions="$work/baseline-executions" +mutant_executions="$work/mutant-executions" +printf '%s\n' \ + local-install-cases.sh \ + local-config-cases.sh \ + local-image-catalog-cases.sh > "$expected_executions" +: > "$baseline_executions" +baseline_rc=0 +run_replica "$work/baseline.out" env \ + AGENT_LAB_AGG_EXEC_LOG="$baseline_executions" || baseline_rc=$? + +mutant_lifecycle="$replica/tests/experiment/local-lifecycle-hidden-duplicate.sh" +awk ' + { print } + $0 == "subcases=(" { in_subcases=1; next } + in_subcases && $0 == ")" { + print "\"$repo_root/tests/install/local-install-cases.sh\" >/dev/null 2>&1" + in_subcases=0 + } +' "$replica_lifecycle" > "$mutant_lifecycle" +chmod +x "$mutant_lifecycle" +mutation_count="$(grep -Fxc '"$repo_root/tests/install/local-install-cases.sh" >/dev/null 2>&1' "$mutant_lifecycle")" +: > "$mutant_executions" +mutant_rc=0 +run_selected "$work/mutant.out" "$mutant_lifecycle" env \ + AGENT_LAB_AGG_EXEC_LOG="$mutant_executions" || mutant_rc=$? +mutant_expected="$work/mutant-expected-executions" +printf '%s\n' \ + local-install-cases.sh \ + local-install-cases.sh \ + local-config-cases.sh \ + local-image-catalog-cases.sh > "$mutant_expected" + +if [ "$baseline_rc" -eq 0 ] && + cmp -s "$expected_executions" "$baseline_executions" && + [ "$mutation_count" -eq 1 ] && + [ "$mutant_rc" -eq 0 ] && + cmp -s "$work/baseline.out" "$work/mutant.out" && + cmp -s "$mutant_expected" "$mutant_executions" && + ! cmp -s "$expected_executions" "$mutant_executions"; then + pass AGG-001 "independent execution ledger proves exact-once routing and detects a hidden duplicate" else - fail AGG-001 "lifecycle subcases are declared exactly once in order" + fail AGG-001 "independent execution ledger proves exact-once routing and detects a hidden duplicate" fi reset_fixtures @@ -97,10 +163,10 @@ success_output="$work/success.out" success_rc=0 run_replica "$success_output" env || success_rc=$? if [ "$success_rc" -eq 0 ] && - [ "$(grep -Ec '^(PASS|FAIL) [A-Z0-9-]+ ' "$success_output")" -eq 77 ] && - [ "$(grep -Fxc 'SUMMARY assertions=77 expected=77 failures=0 infra=0' "$success_output")" -eq 1 ] && + [ "$(grep -Ec '^(PASS|FAIL) [A-Z0-9-]+ ' "$success_output")" -eq 86 ] && + [ "$(grep -Fxc 'SUMMARY assertions=86 expected=86 failures=0 infra=0' "$success_output")" -eq 1 ] && [ "$(tail -n 1 "$success_output")" = 'EXPERIMENT LOCAL LIFECYCLE PASS' ] && - awk '/^(PASS|FAIL) [A-Z0-9-]+ / {next} /^SUMMARY assertions=77 expected=77 failures=0 infra=0$/ {next} /^EXPERIMENT LOCAL LIFECYCLE PASS$/ {next} {bad=1} END {exit bad}' "$success_output"; then + awk '/^(PASS|FAIL) [A-Z0-9-]+ / {next} /^SUMMARY assertions=86 expected=86 failures=0 infra=0$/ {next} /^EXPERIMENT LOCAL LIFECYCLE PASS$/ {next} {bad=1} END {exit bad}' "$success_output"; then pass AGG-002 "success forwards only assertions then one summary and marker" else fail AGG-002 "success forwards only assertions then one summary and marker" @@ -149,7 +215,7 @@ write_fixture "$replica/tests/install/local-install-cases.sh" 1 "${failed_record assertion_rc=0 run_replica "$work/assertion.out" env || assertion_rc=$? if [ "$assertion_rc" -eq 1 ] && - grep -Fxq 'SUMMARY assertions=77 expected=77 failures=1 infra=0' "$work/assertion.out" && + grep -Fxq 'SUMMARY assertions=86 expected=86 failures=1 infra=0' "$work/assertion.out" && ! grep -Fxq 'EXPERIMENT LOCAL LIFECYCLE PASS' "$work/assertion.out"; then pass AGG-006 "subcase assertion failure maps to one" else @@ -182,7 +248,7 @@ chmod +x "$shim/rmdir" cleanup_rc=0 run_replica "$work/cleanup.out" env PATH="$shim:$PATH" || cleanup_rc=$? if [ "$cleanup_rc" -eq 125 ] && - grep -Fxq 'SUMMARY assertions=77 expected=77 failures=0 infra=1' "$work/cleanup.out" && + grep -Fxq 'SUMMARY assertions=86 expected=86 failures=0 infra=1' "$work/cleanup.out" && ! grep -Fxq 'EXPERIMENT LOCAL LIFECYCLE PASS' "$work/cleanup.out"; then pass AGG-008 "cleanup uncertainty maps to one hundred twenty-five before the marker" else @@ -208,5 +274,15 @@ else fail AGG-009 "missing subcase summary maps to one hundred twenty-five" fi -printf 'SUMMARY assertions=9 expected=9 failures=%s infra=0\n' "$failures" +cleanup_infrastructure=0 +if ! cleanup_work; then + cleanup_infrastructure=1 +fi +trap - EXIT + +printf 'SUMMARY assertions=%s expected=%s failures=%s infra=%s\n' \ + "$expected_count" "$expected_count" "$failures" "$cleanup_infrastructure" +if [ "$cleanup_infrastructure" -ne 0 ]; then + exit 125 +fi [ "$failures" -eq 0 ] diff --git a/tests/experiment/catalog-resolution-cases.sh b/tests/experiment/catalog-resolution-cases.sh index 5f17b11..6252a4d 100755 --- a/tests/experiment/catalog-resolution-cases.sh +++ b/tests/experiment/catalog-resolution-cases.sh @@ -3,20 +3,103 @@ set -u -o pipefail repo_root="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." >/dev/null 2>&1 && pwd)" agent_lab="$repo_root/scripts/agent-lab" -work="$(mktemp -d)" -trap 'find "$work" -type f -delete 2>/dev/null || true; find "$work" -type l -delete 2>/dev/null || true; find "$work" -depth -type d -exec rmdir {} + 2>/dev/null || true' EXIT +bounded_helper="$repo_root/tests/helpers/run-bounded.py" +expected_count=14 +work="" +failures=0 +infrastructure=0 -if [ ! -x "$agent_lab" ] || ! command -v jq >/dev/null 2>&1 || [ ! -d "$repo_root/.cache/dev/tools/cue" ]; then - printf 'INFRA catalog resolution prerequisites are unavailable\n' >&2 +cleanup_work() { + local failed=0 + if [ -n "$work" ] && [ -e "$work" ]; then + find "$work" -type f -delete 2>/dev/null || failed=1 + find "$work" -type l -delete 2>/dev/null || failed=1 + find "$work" -depth -type d -exec rmdir {} + 2>/dev/null || failed=1 + [ ! -e "$work" ] || failed=1 + fi + return "$failed" +} + +finish() { + local assertions=0 + if [ -n "${observed:-}" ] && [ -f "$observed" ]; then + assertions="$(wc -l < "$observed")" + fi + if ! cleanup_work; then + infrastructure=1 + fi + trap - EXIT + printf 'SUMMARY assertions=%s expected=%s failures=%s infra=%s\n' \ + "$assertions" "$expected_count" "$failures" "$infrastructure" + if [ "$infrastructure" -ne 0 ]; then + exit 125 + fi + if [ "$failures" -ne 0 ]; then + exit 1 + fi +} + +if ! work="$(mktemp -d)"; then + printf 'SUMMARY assertions=0 expected=%s failures=0 infra=1\n' "$expected_count" exit 125 fi - -failures=0 +trap 'cleanup_work >/dev/null 2>&1 || true' EXIT observed="$work/observed" : > "$observed" + +if [ ! -x "$agent_lab" ] || [ ! -f "$bounded_helper" ] || ! command -v jq >/dev/null 2>&1 || ! command -v python3 >/dev/null 2>&1 || [ ! -d "$repo_root/.cache/dev/tools/cue" ]; then + printf 'INFRA catalog resolution prerequisites are unavailable\n' >&2 + infrastructure=1 + finish +fi +if ! python3 -I -B "$bounded_helper" --self-test > "$work/bounded-self-test.out" 2> "$work/bounded-self-test.err"; then + printf 'INFRA bounded command helper self-test failed\n' >&2 + infrastructure=1 + finish +fi + pass() { printf 'PASS %s %s\n' "$1" "$2"; printf '%s\n' "$1" >> "$observed"; } fail() { printf 'FAIL %s %s\n' "$1" "$2"; printf '%s\n' "$1" >> "$observed"; failures=$((failures + 1)); } -capture() { CAPTURE_RC=0; "$@" > "$work/stdout" 2> "$work/stderr" || CAPTURE_RC=$?; } +run_bounded() { + local output="$1" + local errors="$2" + local expectation="$3" + local status="${output}.status" + local rc=0 + local status_line="" + shift 3 + find "$status" -delete 2>/dev/null || true + python3 -I -B "$bounded_helper" \ + --timeout 5 --status "$status" --stdout "$output" --stderr "$errors" -- "$@" || rc=$? + if [ -f "$status" ]; then + status_line="$(cat "$status")" + fi + if [ "$status_line" != "child:$rc" ]; then + infrastructure=1 + rc=125 + else + case "$rc" in + 0|1) + ;; + 125) + [ "$expectation" = expected-125 ] || infrastructure=1 + ;; + *) + infrastructure=1 + ;; + esac + fi + return "$rc" +} +capture() { + local expectation="normal" + if [ "${1:-}" = expected-125 ]; then + expectation="expected-125" + shift + fi + CAPTURE_RC=0 + run_bounded "$work/stdout" "$work/stderr" "$expectation" "$@" || CAPTURE_RC=$? +} subject_a="registry.example/operator/worker@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" subject_b="registry.example/operator/other@sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" @@ -25,7 +108,8 @@ init_home() { capture "$agent_lab" --home "$home" init if [ "$CAPTURE_RC" -ne 0 ]; then printf 'INFRA temporary Agent Lab home initialization failed: %s\n' "$(tr '\n' ' ' < "$work/stderr")" >&2 - exit 125 + infrastructure=1 + finish fi cp -a "$repo_root/.cache/dev/tools/cue/." "$home/cache/tools/cue/" } @@ -34,7 +118,8 @@ copy_cedar() { local home="$1" if [ ! -d "$repo_root/.cache/dev/tools/cedar" ]; then printf 'INFRA pinned Cedar test cache is unavailable\n' >&2 - exit 125 + infrastructure=1 + finish fi cp -a "$repo_root/.cache/dev/tools/cedar/." "$home/cache/tools/cedar/" } @@ -84,7 +169,8 @@ init_home "$active_home" capture "$agent_lab" --home "$active_home" image add vendor.worker "$subject_a" if [ "$CAPTURE_RC" -ne 0 ]; then printf 'INFRA active local mapping setup failed\n' >&2 - exit 125 + infrastructure=1 + finish fi entry_a="$(jq -r '.entryDigest' "$work/stdout")" artifact="$work/active-artifact" @@ -181,7 +267,7 @@ history_home="$work/history-home" init_home "$history_home" capture "$agent_lab" --home "$history_home" image add vendor.worker "$subject_a" mv "$history_home/images/catalog/entries" "$work/removed-resolution-history" -capture "$agent_lab" --home "$history_home" experiment check "$artifact" +capture expected-125 "$agent_lab" --home "$history_home" experiment check "$artifact" if [ "$CAPTURE_RC" -eq 125 ] && [ ! -s "$work/stdout" ]; then pass RES-STATE-001 "local resolution verifies immutable entry history before binding" else @@ -196,9 +282,9 @@ mv "$drift_home/images/catalog" "$drift_home/other-images/catalog" jq -cS '.paths.images="other-images"' "$drift_home/config.json" > "$work/drift-config.json" mv "$work/drift-config.json" "$drift_home/config.json" chmod 600 "$drift_home/config.json" -capture "$agent_lab" --home "$drift_home" config check +capture expected-125 "$agent_lab" --home "$drift_home" config check drift_config_rc="$CAPTURE_RC" -capture "$agent_lab" --home "$drift_home" experiment check "$artifact" +capture expected-125 "$agent_lab" --home "$drift_home" experiment check "$artifact" if [ "$drift_config_rc" -eq 125 ] && [ "$CAPTURE_RC" -eq 125 ] && [ ! -s "$work/stdout" ]; then pass RES-STATE-002 "receipt-breaking config drift cannot redirect local resolution" else @@ -212,7 +298,7 @@ mv "$symlink_source/images/catalog" "$work/outside-resolution-catalog" symlink_home="$work/symlink-home" init_home "$symlink_home" ln -s "$work/outside-resolution-catalog" "$symlink_home/images/catalog" -capture "$agent_lab" --home "$symlink_home" experiment check "$artifact" +capture expected-125 "$agent_lab" --home "$symlink_home" experiment check "$artifact" if [ "$CAPTURE_RC" -eq 125 ] && [ ! -s "$work/stdout" ]; then pass RES-STATE-003 "local resolution refuses a symlinked catalog authority" else @@ -285,6 +371,7 @@ cp "$repo_root/scripts/install-local" "$repo_root/scripts/install-local.py" "$ru chmod +x "$runtime_replica/scripts/install-local" "$runtime_replica/scripts/agent-lab" capture "$runtime_replica/scripts/install-local" --prefix "$installed_prefix" installed_install_rc="$CAPTURE_RC" +installed_runtime_before="$(find "$installed_prefix" -printf '%P %y %m %s\n' 2>/dev/null | LC_ALL=C sort)" mv "$runtime_replica" "$work/runtime-source-unavailable" mkdir "$work/installed-unrelated" installed_rc=125 @@ -294,13 +381,18 @@ if [ "$installed_install_rc" -eq 0 ]; then capture "$installed_prefix/bin/agent-lab" --home "$installed_home" image add vendor.worker "$subject_a" installed_entry="$(jq -r '.entryDigest // empty' "$work/stdout" 2>/dev/null)" installed_rc=0 - (cd "$work/installed-unrelated" && env -i PATH=/usr/bin:/bin \ - "$installed_prefix/bin/agent-lab" --home "$installed_home" experiment check "$artifact") \ - > "$work/installed-check.json" 2> "$work/installed-check.err" || installed_rc=$? + (cd "$work/installed-unrelated" && run_bounded \ + "$work/installed-check.json" "$work/installed-check.err" normal \ + env -i PATH=/usr/bin:/bin \ + "$installed_prefix/bin/agent-lab" --home "$installed_home" experiment check "$artifact") || installed_rc=$? + [ "$installed_rc" -eq 0 ] || [ "$installed_rc" -eq 1 ] || infrastructure=1 else installed_entry="" fi +installed_runtime_after="$(find "$installed_prefix" -printf '%P %y %m %s\n' 2>/dev/null | LC_ALL=C sort)" if [ "$installed_install_rc" -eq 0 ] && [ "$installed_rc" -eq 0 ] && + [ "$installed_runtime_before" = "$installed_runtime_after" ] && + [ -z "$(find "$installed_prefix" -name __pycache__ -print -quit 2>/dev/null)" ] && [ ! -s "$work/installed-check.err" ] && jq -e --arg entry "$installed_entry" --arg subject "$subject_a" ' .plan.spec.members[0].resolvedImage == { entryDigest: $entry, @@ -309,9 +401,9 @@ if [ "$installed_install_rc" -eq 0 ] && [ "$installed_rc" -eq 0 ] && subject: $subject } ' "$work/installed-check.json" >/dev/null 2>&1; then - pass RES-INSTALL-001 "installed catalog commands and local resolution run without the source replica or checkout cwd" + pass RES-INSTALL-001 "installed catalog resolution is source-independent and leaves its release topology unchanged" else - fail RES-INSTALL-001 "installed catalog commands and local resolution run without the source replica or checkout cwd" + fail RES-INSTALL-001 "installed catalog resolution is source-independent and leaves its release topology unchanged" fi canary_home="$work/canary-home" @@ -328,8 +420,9 @@ done calibrated="$(find "$canary_marks" -type f | wc -l)" find "$canary_marks" -type f -delete canary_rc=0 -env -i PATH="$canary_bin:/usr/bin:/bin" LANG=C LC_ALL=C CANARY_DIR="$canary_marks" \ - "$agent_lab" --home "$canary_home" experiment check "$artifact" > "$work/canary.out" 2> "$work/canary.err" || canary_rc=$? +run_bounded "$work/canary.out" "$work/canary.err" normal \ + env -i PATH="$canary_bin:/usr/bin:/bin" LANG=C LC_ALL=C CANARY_DIR="$canary_marks" \ + "$agent_lab" --home "$canary_home" experiment check "$artifact" || canary_rc=$? if [ "$calibrated" -eq 4 ] && [ "$canary_rc" -eq 0 ] && [ -z "$(find "$canary_marks" -type f -print -quit)" ]; then pass RES-NOEF-001 "calibrated forbidden-effect canaries remain silent during local resolution" else @@ -343,7 +436,6 @@ printf '%s\n' \ RES-SNAP-003 RES-AUTH-001 RES-INSTALL-001 RES-NOEF-001 > "$expected" if ! cmp -s "$expected" "$observed"; then printf 'INFRA catalog resolution assertion identity drift\n' >&2 - exit 125 + infrastructure=1 fi -printf 'SUMMARY assertions=14 expected=14 failures=%s infra=0\n' "$failures" -[ "$failures" -eq 0 ] +finish diff --git a/tests/experiment/contract-cases.sh b/tests/experiment/contract-cases.sh index 3b2f55a..73d8b1d 100755 --- a/tests/experiment/contract-cases.sh +++ b/tests/experiment/contract-cases.sh @@ -1,9 +1,92 @@ #!/usr/bin/env bash -set -euo pipefail +set -u -o pipefail repo_root="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." >/dev/null 2>&1 && pwd)" -subcase="$repo_root/tests/experiment/directory-intake-cases.sh" -[ -x "$subcase" ] || { printf 'INFRA directory intake subcase is missing\n' >&2; exit 125; } -"$subcase" -"$repo_root/tests/experiment/aggregate-harness-cases.sh" +subcases=( + "$repo_root/tests/experiment/directory-intake-cases.sh" + "$repo_root/tests/experiment/aggregate-harness-cases.sh" +) +expected_count=22 +work="" + +cleanup_work() { + local failed=0 + if [ -n "$work" ] && [ -e "$work" ]; then + find "$work" -type f -delete 2>/dev/null || failed=1 + find "$work" -type l -delete 2>/dev/null || failed=1 + find "$work" -depth -type d -exec rmdir {} + 2>/dev/null || failed=1 + [ ! -e "$work" ] || failed=1 + fi + return "$failed" +} + +if ! work="$(mktemp -d)"; then + printf 'SUMMARY assertions=0 expected=%s failures=0 infra=1\n' "$expected_count" + exit 125 +fi +trap 'cleanup_work >/dev/null 2>&1 || true' EXIT + +expected="$work/expected" +observed="$work/observed" +printf '%s\n' \ + FMT-001 FMT-002 FMT-004 FMT-005 FMT-006 FMT-007 FMT-003 FMT-008 \ + SEL-001 CUE-001 FMT-009 M-FMT-001 SEL-002 \ + AGG-001 AGG-002 AGG-003 AGG-004 AGG-005 AGG-006 AGG-007 AGG-008 AGG-009 > "$expected" +: > "$observed" + +infrastructure=0 +for index in "${!subcases[@]}"; do + subcase="${subcases[$index]}" + output="$work/subcase-$index.out" + if [ ! -f "$subcase" ]; then + infrastructure=1 + continue + fi + if bash "$subcase" > "$output" 2>&1; then + rc=0 + else + rc=$? + fi + awk '/^(PASS|FAIL) [A-Z0-9-]+ / {print}' "$output" + awk '/^(PASS|FAIL) [A-Z0-9-]+ / {print $2}' "$output" >> "$observed" + reported_assertions="$(awk '/^(PASS|FAIL) [A-Z0-9-]+ / {count++} END {print count + 0}' "$output")" + reported_failures="$(awk '/^FAIL [A-Z0-9-]+ / {count++} END {print count + 0}' "$output")" + expected_summary="SUMMARY assertions=$reported_assertions expected=$reported_assertions failures=$reported_failures infra=0" + matching_summaries="$(grep -Fxc "$expected_summary" "$output" || true)" + all_summaries="$(grep -c '^SUMMARY ' "$output" || true)" + if [ "$matching_summaries" -ne 1 ] || [ "$all_summaries" -ne 1 ]; then + infrastructure=1 + fi + case "$rc" in + 0) + [ "$reported_failures" -eq 0 ] || infrastructure=1 + ;; + 1) + [ "$reported_failures" -ne 0 ] || infrastructure=1 + ;; + *) + infrastructure=1 + ;; + esac +done + +assertions="$(wc -l < "$observed")" +failures="$(awk '/^FAIL [A-Z0-9-]+ / {count++} END {print count + 0}' "$work"/subcase-*.out 2>/dev/null)" +if ! cmp -s "$expected" "$observed"; then + failures=$((failures + 1)) +fi + +if ! cleanup_work; then + infrastructure=1 +fi +trap - EXIT + +printf 'SUMMARY assertions=%s expected=%s failures=%s infra=%s\n' \ + "$assertions" "$expected_count" "$failures" "$infrastructure" +if [ "$infrastructure" -ne 0 ]; then + exit 125 +fi +if [ "$failures" -ne 0 ]; then + exit 1 +fi printf 'EXPERIMENT CONTRACT PASS\n' diff --git a/tests/experiment/local-config-cases.sh b/tests/experiment/local-config-cases.sh index c2314db..9ea19a6 100755 --- a/tests/experiment/local-config-cases.sh +++ b/tests/experiment/local-config-cases.sh @@ -2,8 +2,22 @@ set -euo pipefail repo_root="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." >/dev/null 2>&1 && pwd)" -work="$(mktemp -d)" -trap 'find "$work" -type f -delete 2>/dev/null || true; find "$work" -type l -delete 2>/dev/null || true; find "$work" -depth -type d -exec rmdir {} + 2>/dev/null || true' EXIT +work="" +cleanup_work() { + local failed=0 + if [ -n "$work" ] && [ -e "$work" ]; then + find "$work" -type f -delete 2>/dev/null || failed=1 + find "$work" -type l -delete 2>/dev/null || failed=1 + find "$work" -depth -type d -exec rmdir {} + 2>/dev/null || failed=1 + [ ! -e "$work" ] || failed=1 + fi + return "$failed" +} +if ! work="$(mktemp -d)"; then + printf 'SUMMARY assertions=0 expected=5 failures=0 infra=1\n' + exit 125 +fi +trap 'cleanup_work >/dev/null 2>&1 || true' EXIT rc=0 "$repo_root/scripts/agent-lab" --home "$work/home" init > "$work/out" 2> "$work/err" || rc=$? failures=0 @@ -133,5 +147,13 @@ else failures=$((failures + 1)) fi -printf 'SUMMARY assertions=5 expected=5 failures=%s infra=0\n' "$failures" +infrastructure=0 +if ! cleanup_work; then + infrastructure=1 +fi +trap - EXIT +printf 'SUMMARY assertions=5 expected=5 failures=%s infra=%s\n' "$failures" "$infrastructure" +if [ "$infrastructure" -ne 0 ]; then + exit 125 +fi [ "$failures" -eq 0 ] diff --git a/tests/experiment/local-image-catalog-cases.sh b/tests/experiment/local-image-catalog-cases.sh index 52f50a5..951208b 100755 --- a/tests/experiment/local-image-catalog-cases.sh +++ b/tests/experiment/local-image-catalog-cases.sh @@ -2,13 +2,31 @@ set -u -o pipefail repo_root="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." >/dev/null 2>&1 && pwd)" -work="$(mktemp -d)" -trap 'find "$work" -type f -delete 2>/dev/null || true; find "$work" -type l -delete 2>/dev/null || true; find "$work" -depth -type d -exec rmdir {} + 2>/dev/null || true' EXIT +expected_count=76 +work="" + +cleanup_work() { + local failed=0 + if [ -n "$work" ] && [ -e "$work" ]; then + find "$work" -type f -delete 2>/dev/null || failed=1 + find "$work" -type l -delete 2>/dev/null || failed=1 + find "$work" -depth -type d -exec rmdir {} + 2>/dev/null || failed=1 + [ ! -e "$work" ] || failed=1 + fi + return "$failed" +} + +if ! work="$(mktemp -d)"; then + printf 'SUMMARY assertions=0 expected=%s failures=0 infra=1\n' "$expected_count" + exit 125 +fi +trap 'cleanup_work >/dev/null 2>&1 || true' EXIT subcases=( "$repo_root/tests/image/catalog-cases.sh" "$repo_root/tests/image/catalog-state-cases.py" "$repo_root/tests/experiment/catalog-resolution-cases.sh" + "$repo_root/tests/image/catalog-mutation-cases.py" ) expected="$work/expected" observed="$work/observed" @@ -21,12 +39,21 @@ printf '%s\n' \ CAT-STATE-001 CAT-STATE-002 CAT-STATE-003 CAT-STATE-004 \ CAT-STATE-005 CAT-STATE-006 CAT-STATE-007 CAT-STATE-008 \ CAT-STATE-009 CAT-STATE-010 CAT-STATE-011 CAT-STATE-012 \ - CAT-BOUND-001 CAT-BOUND-002 CAT-CRASH-001 CAT-CRASH-002 \ - CAT-CRASH-003 CAT-PLAT-001 \ + CAT-STATE-013 CAT-STATE-014 CAT-STATE-015 CAT-STATE-016 CAT-STATE-018 CAT-STATE-017 \ + CAT-BOUND-001 CAT-BOUND-002 CAT-BOUND-003 CAT-BOUND-004 \ + CAT-CRASH-001 CAT-CRASH-002 CAT-CRASH-003 CAT-CRASH-004 CAT-CRASH-005 \ + CAT-CRASH-006 CAT-CRASH-007 CAT-CRASH-008 CAT-CRASH-009 CAT-CRASH-010 \ + CAT-CRASH-011 CAT-PLAT-001 \ RES-ENTRY-001 RES-SNAP-001 RES-SNAP-002 RES-ENTRY-002 RES-ENTRY-003 \ RES-ISOLATE-001 RES-STATE-001 RES-STATE-002 RES-STATE-003 RES-INPUT-001 \ - RES-SNAP-003 RES-AUTH-001 RES-INSTALL-001 RES-NOEF-001 > "$expected" -expected_count="$(wc -l < "$expected")" + RES-SNAP-003 RES-AUTH-001 RES-INSTALL-001 RES-NOEF-001 \ + M-CAT-OCI-001 M-CAT-SHADOW-001 M-CAT-CAS-001 M-CAT-AUTH-001 M-RES-BIND-001 \ + M-CAT-NOEF-001 M-CAT-ADMIT-001 M-CAT-ATOM-001 M-CAT-DUR-001 M-CAT-STAGE-001 > "$expected" +declared_count="$(wc -l < "$expected")" +if [ "$declared_count" -ne "$expected_count" ]; then + printf 'INFRA catalog aggregate expected-count drift\n' >&2 + exit 125 +fi infrastructure=0 for index in "${!subcases[@]}"; do @@ -39,7 +66,7 @@ for index in "${!subcases[@]}"; do fi case "$subcase" in *.py) - python3 -I "$subcase" > "$output" 2>&1 + python3 -I -B "$subcase" > "$output" 2>&1 rc=$? ;; *) @@ -47,10 +74,22 @@ for index in "${!subcases[@]}"; do rc=$? ;; esac - cat "$output" + awk '/^(PASS|FAIL) [A-Z0-9-]+ /' "$output" awk '/^(PASS|FAIL) [A-Z0-9-]+ / {print $2}' "$output" >> "$observed" - if [ "$rc" -ne 0 ] && [ "$rc" -ne 1 ]; then - printf 'INFRA catalog subcase returned %s: %s\n' "$rc" "$subcase" >&2 + subcase_assertions="$(awk '/^(PASS|FAIL) [A-Z0-9-]+ / {count++} END {print count + 0}' "$output")" + subcase_failures="$(awk '/^FAIL [A-Z0-9-]+ / {count++} END {print count + 0}' "$output")" + summary="SUMMARY assertions=$subcase_assertions expected=$subcase_assertions failures=$subcase_failures infra=0" + summary_count="$(grep -Fxc "$summary" "$output" || true)" + all_summary_count="$(grep -c '^SUMMARY ' "$output" || true)" + if [ "$summary_count" -ne 1 ] || [ "$all_summary_count" -ne 1 ]; then + printf 'INFRA catalog subcase summary is absent or inconsistent: %s\n' "$subcase" >&2 + cat "$output" >&2 + infrastructure=1 + elif { [ "$rc" -eq 0 ] && [ "$subcase_failures" -ne 0 ]; } \ + || { [ "$rc" -eq 1 ] && [ "$subcase_failures" -eq 0 ]; } \ + || { [ "$rc" -ne 0 ] && [ "$rc" -ne 1 ]; }; then + printf 'INFRA catalog subcase status is inconsistent: rc=%s path=%s\n' "$rc" "$subcase" >&2 + cat "$output" >&2 infrastructure=1 fi done @@ -58,10 +97,15 @@ done assertions="$(wc -l < "$observed")" failures="$(awk '/^FAIL [A-Z0-9-]+ / {count++} END {print count + 0}' "$work"/subcase-*.out 2>/dev/null)" if ! cmp -s "$expected" "$observed"; then - printf 'INFRA catalog aggregate assertion identity drift\n' >&2 + printf 'FAIL catalog aggregate assertion identity drift\n' >&2 diff -u "$expected" "$observed" >&2 || true + failures=$((failures + 1)) +fi + +if ! cleanup_work; then infrastructure=1 fi +trap - EXIT printf 'SUMMARY assertions=%s expected=%s failures=%s infra=%s\n' \ "$assertions" "$expected_count" "$failures" "$infrastructure" diff --git a/tests/experiment/local-lifecycle-cases.sh b/tests/experiment/local-lifecycle-cases.sh index a190ebb..41460a8 100755 --- a/tests/experiment/local-lifecycle-cases.sh +++ b/tests/experiment/local-lifecycle-cases.sh @@ -1,8 +1,113 @@ #!/usr/bin/env bash -set -euo pipefail +set -u -o pipefail repo_root="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." >/dev/null 2>&1 && pwd)" -"$repo_root/tests/install/local-install-cases.sh" -"$repo_root/tests/experiment/local-config-cases.sh" -"$repo_root/tests/experiment/local-image-catalog-cases.sh" +subcases=( + "$repo_root/tests/install/local-install-cases.sh" + "$repo_root/tests/experiment/local-config-cases.sh" + "$repo_root/tests/experiment/local-image-catalog-cases.sh" +) +expected_count=86 +work="" + +cleanup_work() { + local failed=0 + if [ -n "$work" ] && [ -e "$work" ]; then + find "$work" -type f -delete 2>/dev/null || failed=1 + find "$work" -type l -delete 2>/dev/null || failed=1 + find "$work" -depth -type d -exec rmdir {} + 2>/dev/null || failed=1 + [ ! -e "$work" ] || failed=1 + fi + return "$failed" +} + +if ! work="$(mktemp -d)"; then + printf 'SUMMARY assertions=0 expected=%s failures=0 infra=1\n' "$expected_count" + exit 125 +fi +trap 'cleanup_work >/dev/null 2>&1 || true' EXIT + +expected="$work/expected" +observed="$work/observed" +printf '%s\n' \ + PKG-001 PKG-002 PKG-003 PKG-004 PKG-005 \ + CFG-001 CFG-002 CFG-003 CFG-004 TOOL-001 \ + CAT-NAME-001 CAT-NAME-002 CAT-OCI-001 CAT-OCI-002 \ + CAT-ADD-001 CAT-ADD-002 CAT-ADD-003 CAT-ADD-004 CAT-NS-001 \ + CAT-CAS-001 CAT-CAS-002 CAT-CAS-003 CAT-CAS-004 \ + CAT-READ-001 CAT-READ-002 CAT-NOEF-001 CAT-CONC-001 CAT-CONC-002 \ + CAT-STATE-001 CAT-STATE-002 CAT-STATE-003 CAT-STATE-004 \ + CAT-STATE-005 CAT-STATE-006 CAT-STATE-007 CAT-STATE-008 \ + CAT-STATE-009 CAT-STATE-010 CAT-STATE-011 CAT-STATE-012 \ + CAT-STATE-013 CAT-STATE-014 CAT-STATE-015 CAT-STATE-016 CAT-STATE-018 CAT-STATE-017 \ + CAT-BOUND-001 CAT-BOUND-002 CAT-BOUND-003 CAT-BOUND-004 \ + CAT-CRASH-001 CAT-CRASH-002 CAT-CRASH-003 CAT-CRASH-004 CAT-CRASH-005 \ + CAT-CRASH-006 CAT-CRASH-007 CAT-CRASH-008 CAT-CRASH-009 CAT-CRASH-010 \ + CAT-CRASH-011 CAT-PLAT-001 \ + RES-ENTRY-001 RES-SNAP-001 RES-SNAP-002 RES-ENTRY-002 RES-ENTRY-003 \ + RES-ISOLATE-001 RES-STATE-001 RES-STATE-002 RES-STATE-003 RES-INPUT-001 \ + RES-SNAP-003 RES-AUTH-001 RES-INSTALL-001 RES-NOEF-001 \ + M-CAT-OCI-001 M-CAT-SHADOW-001 M-CAT-CAS-001 M-CAT-AUTH-001 M-RES-BIND-001 \ + M-CAT-NOEF-001 M-CAT-ADMIT-001 M-CAT-ATOM-001 M-CAT-DUR-001 M-CAT-STAGE-001 > "$expected" +: > "$observed" + +infrastructure=0 +for index in "${!subcases[@]}"; do + subcase="${subcases[$index]}" + output="$work/subcase-$index.out" + if [ ! -f "$subcase" ]; then + infrastructure=1 + continue + fi + if bash "$subcase" > "$output" 2>&1; then + rc=0 + else + rc=$? + fi + awk '/^(PASS|FAIL) [A-Z0-9-]+ / {print}' "$output" + awk '/^(PASS|FAIL) [A-Z0-9-]+ / {print $2}' "$output" >> "$observed" + reported_assertions="$(awk '/^(PASS|FAIL) [A-Z0-9-]+ / {count++} END {print count + 0}' "$output")" + reported_failures="$(awk '/^FAIL [A-Z0-9-]+ / {count++} END {print count + 0}' "$output")" + expected_summary="SUMMARY assertions=$reported_assertions expected=$reported_assertions failures=$reported_failures infra=0" + matching_summaries="$(grep -Fxc "$expected_summary" "$output" || true)" + all_summaries="$(grep -c '^SUMMARY ' "$output" || true)" + if [ "$matching_summaries" -ne 1 ] || [ "$all_summaries" -ne 1 ]; then + infrastructure=1 + fi + case "$rc" in + 0) + if [ "$reported_failures" -ne 0 ]; then + infrastructure=1 + fi + ;; + 1) + if [ "$reported_failures" -eq 0 ]; then + infrastructure=1 + fi + ;; + *) + infrastructure=1 + ;; + esac +done + +assertions="$(wc -l < "$observed")" +failures="$(awk '/^FAIL [A-Z0-9-]+ / {count++} END {print count + 0}' "$work"/subcase-*.out 2>/dev/null)" +if ! cmp -s "$expected" "$observed"; then + failures=$((failures + 1)) +fi + +if ! cleanup_work; then + infrastructure=1 +fi +trap - EXIT + +printf 'SUMMARY assertions=%s expected=%s failures=%s infra=%s\n' \ + "$assertions" "$expected_count" "$failures" "$infrastructure" +if [ "$infrastructure" -ne 0 ]; then + exit 125 +fi +if [ "$failures" -ne 0 ]; then + exit 1 +fi printf 'EXPERIMENT LOCAL LIFECYCLE PASS\n' diff --git a/tests/helpers/run-bounded.py b/tests/helpers/run-bounded.py new file mode 100644 index 0000000..a257a61 --- /dev/null +++ b/tests/helpers/run-bounded.py @@ -0,0 +1,249 @@ +#!/usr/bin/env python3 +"""Run one test command with a bounded, fully reaped process group.""" + +from __future__ import annotations + +import argparse +from contextlib import ExitStack +import os +from pathlib import Path +import signal +import subprocess +import sys +import tempfile +import time + + +MAX_TIMEOUT_SECONDS = 5.0 +TERMINATION_GRACE_SECONDS = 1.0 +HANDLED_SIGNALS = (signal.SIGINT, signal.SIGHUP, signal.SIGTERM) +ACTIVE_PROCESS: subprocess.Popen[bytes] | None = None +STATUS_PATH: Path | None = None + + +def process_group_exists(group: int) -> bool: + try: + os.killpg(group, 0) + except ProcessLookupError: + return False + except PermissionError: + return True + return True + + +def terminate_process_group( + process: subprocess.Popen[bytes], + grace_seconds: float = TERMINATION_GRACE_SECONDS, +) -> bool: + group = process.pid + if process_group_exists(group): + try: + os.killpg(group, signal.SIGTERM) + except ProcessLookupError: + pass + deadline = time.monotonic() + grace_seconds + while process_group_exists(group) and time.monotonic() < deadline: + process.poll() + time.sleep(0.01) + if process_group_exists(group): + try: + os.killpg(group, signal.SIGKILL) + except ProcessLookupError: + pass + try: + process.wait(timeout=grace_seconds) + except subprocess.TimeoutExpired: + try: + os.killpg(group, signal.SIGKILL) + except ProcessLookupError: + pass + try: + process.wait(timeout=grace_seconds) + except subprocess.TimeoutExpired: + return False + deadline = time.monotonic() + grace_seconds + while process_group_exists(group) and time.monotonic() < deadline: + time.sleep(0.01) + return not process_group_exists(group) + + +def normalized_returncode(returncode: int) -> int: + if returncode < 0: + return 128 + abs(returncode) + return returncode + + +def execute_bounded( + command: list[str], + stdout_path: Path, + stderr_path: Path, + timeout_seconds: float, + grace_seconds: float = TERMINATION_GRACE_SECONDS, +) -> tuple[str, int]: + global ACTIVE_PROCESS + + process: subprocess.Popen[bytes] | None = None + try: + with ExitStack() as stack: + stdout = stack.enter_context(stdout_path.open("wb")) + stderr = stack.enter_context(stderr_path.open("wb")) + previous_mask = signal.pthread_sigmask(signal.SIG_BLOCK, HANDLED_SIGNALS) + try: + process = subprocess.Popen( + command, + stdin=subprocess.DEVNULL, + stdout=stdout, + stderr=stderr, + start_new_session=True, + ) + ACTIVE_PROCESS = process + finally: + signal.pthread_sigmask(signal.SIG_SETMASK, previous_mask) + try: + returncode = process.wait(timeout=timeout_seconds) + except subprocess.TimeoutExpired: + if not terminate_process_group(process, grace_seconds): + return "infra", 125 + return "infra", 125 + if process_group_exists(process.pid): + if not terminate_process_group(process, grace_seconds): + return "infra", 125 + return "infra", 125 + return "child", normalized_returncode(returncode) + except (OSError, subprocess.SubprocessError): + if process is not None and process_group_exists(process.pid): + terminate_process_group(process, grace_seconds) + return "infra", 125 + finally: + ACTIVE_PROCESS = None + + +def write_status(path: Path, kind: str, returncode: int) -> bool: + try: + path.write_text(f"{kind}:{returncode}\n", encoding="ascii") + except OSError: + return False + return True + + +def interrupted(signum: int, _frame: object) -> None: + process = ACTIVE_PROCESS + if process is not None: + terminate_process_group(process) + status = STATUS_PATH + if status is not None: + write_status(status, "infra", 125) + os._exit(128 + signum) + + +def descendant_fixture(pid_path: Path, *, hang_parent: bool) -> str: + parent_action = "time.sleep(30)" if hang_parent else "os._exit(0)" + return f""" +import os +from pathlib import Path +import signal +import time + +pid_path = Path({str(pid_path)!r}) +child = os.fork() +if child == 0: + signal.signal(signal.SIGTERM, signal.SIG_IGN) + pid_path.write_text(str(os.getpid()), encoding="ascii") + os.close(0) + os.close(1) + os.close(2) + time.sleep(30) +deadline = time.monotonic() + 1.0 +while not pid_path.exists() and time.monotonic() < deadline: + time.sleep(0.01) +{parent_action} +""" + + +def run_self_test() -> int: + try: + with tempfile.TemporaryDirectory(prefix="agent-lab-bounded-command-") as directory: + root = Path(directory) + for name, hang_parent, timeout_seconds in ( + ("normal-residual", False, 1.0), + ("timeout-residual", True, 0.2), + ): + pid_path = root / f"{name}.pid" + kind, returncode = execute_bounded( + [ + sys.executable, + "-I", + "-B", + "-c", + descendant_fixture(pid_path, hang_parent=hang_parent), + ], + root / f"{name}.out", + root / f"{name}.err", + timeout_seconds, + 0.1, + ) + if kind != "infra" or returncode != 125 or not pid_path.is_file(): + return 125 + try: + descendant = int(pid_path.read_text(encoding="ascii")) + os.kill(descendant, 0) + except ProcessLookupError: + continue + except (OSError, ValueError): + return 125 + try: + os.kill(descendant, signal.SIGKILL) + except ProcessLookupError: + pass + return 125 + except OSError: + return 125 + return 0 + + +def parse_arguments(argv: list[str]) -> argparse.Namespace: + parser = argparse.ArgumentParser(add_help=False) + parser.add_argument("--self-test", action="store_true") + parser.add_argument("--timeout", type=float, default=MAX_TIMEOUT_SECONDS) + parser.add_argument("--status") + parser.add_argument("--stdout") + parser.add_argument("--stderr") + parser.add_argument("command", nargs=argparse.REMAINDER) + return parser.parse_args(argv) + + +def main(argv: list[str]) -> int: + global STATUS_PATH + + arguments = parse_arguments(argv) + if arguments.self_test: + return run_self_test() + command = list(arguments.command) + if command and command[0] == "--": + command.pop(0) + if ( + not command + or arguments.status is None + or arguments.stdout is None + or arguments.stderr is None + or not 0 < arguments.timeout <= MAX_TIMEOUT_SECONDS + ): + return 125 + STATUS_PATH = Path(arguments.status) + kind, returncode = execute_bounded( + command, + Path(arguments.stdout), + Path(arguments.stderr), + arguments.timeout, + ) + if not write_status(STATUS_PATH, kind, returncode): + return 125 + return returncode + + +for handled_signal in HANDLED_SIGNALS: + signal.signal(handled_signal, interrupted) + + +if __name__ == "__main__": + raise SystemExit(main(sys.argv[1:])) diff --git a/tests/image/catalog-cases.sh b/tests/image/catalog-cases.sh index 8e48949..6102dc2 100755 --- a/tests/image/catalog-cases.sh +++ b/tests/image/catalog-cases.sh @@ -3,20 +3,98 @@ set -u -o pipefail repo_root="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." >/dev/null 2>&1 && pwd)" agent_lab="$repo_root/scripts/agent-lab" -work="$(mktemp -d)" -trap 'find "$work" -type f -delete 2>/dev/null || true; find "$work" -type l -delete 2>/dev/null || true; find "$work" -depth -type d -exec rmdir {} + 2>/dev/null || true' EXIT +bounded_helper="$repo_root/tests/helpers/run-bounded.py" +expected_count=18 +work="" +failures=0 +infrastructure=0 -if [ ! -x "$agent_lab" ] || ! command -v jq >/dev/null 2>&1; then - printf 'INFRA catalog public-contract prerequisites are unavailable\n' >&2 +cleanup_work() { + local failed=0 + if [ -n "$work" ] && [ -e "$work" ]; then + find "$work" -type f -delete 2>/dev/null || failed=1 + find "$work" -type l -delete 2>/dev/null || failed=1 + find "$work" -depth -type d -exec rmdir {} + 2>/dev/null || failed=1 + [ ! -e "$work" ] || failed=1 + fi + return "$failed" +} + +finish() { + local assertions=0 + if [ -n "${observed:-}" ] && [ -f "$observed" ]; then + assertions="$(wc -l < "$observed")" + fi + if ! cleanup_work; then + infrastructure=1 + fi + trap - EXIT + printf 'SUMMARY assertions=%s expected=%s failures=%s infra=%s\n' \ + "$assertions" "$expected_count" "$failures" "$infrastructure" + if [ "$infrastructure" -ne 0 ]; then + exit 125 + fi + if [ "$failures" -ne 0 ]; then + exit 1 + fi +} + +if ! work="$(mktemp -d)"; then + printf 'SUMMARY assertions=0 expected=%s failures=0 infra=1\n' "$expected_count" exit 125 fi - -failures=0 +trap 'cleanup_work >/dev/null 2>&1 || true' EXIT observed="$work/observed" : > "$observed" + +if [ ! -x "$agent_lab" ] || [ ! -f "$bounded_helper" ] || ! command -v jq >/dev/null 2>&1 || ! command -v python3 >/dev/null 2>&1; then + printf 'INFRA catalog public-contract prerequisites are unavailable\n' >&2 + infrastructure=1 + finish +fi +if ! python3 -I -B "$bounded_helper" --self-test > "$work/bounded-self-test.out" 2> "$work/bounded-self-test.err"; then + printf 'INFRA bounded command helper self-test failed\n' >&2 + infrastructure=1 + finish +fi + pass() { printf 'PASS %s %s\n' "$1" "$2"; printf '%s\n' "$1" >> "$observed"; } fail() { printf 'FAIL %s %s\n' "$1" "$2"; printf '%s\n' "$1" >> "$observed"; failures=$((failures + 1)); } -capture() { CAPTURE_RC=0; "$@" > "$work/stdout" 2> "$work/stderr" || CAPTURE_RC=$?; } +run_bounded() { + local output="$1" + local errors="$2" + local expectation="$3" + local status="${output}.status" + local rc=0 + local status_line="" + shift 3 + find "$status" -delete 2>/dev/null || true + python3 -I -B "$bounded_helper" \ + --timeout 5 --status "$status" --stdout "$output" --stderr "$errors" -- "$@" || rc=$? + if [ -f "$status" ]; then + status_line="$(cat "$status")" + fi + if [ "$status_line" != "child:$rc" ]; then + infrastructure=1 + rc=125 + else + case "$rc" in + 0|1) + ;; + 125) + [ "$expectation" = expected-125 ] || infrastructure=1 + ;; + *) + infrastructure=1 + ;; + esac + fi + return "$rc" +} +capture() { + CAPTURE_RC=0 + run_bounded "$work/stdout" "$work/stderr" normal "$@" || CAPTURE_RC=$? +} subject_a="registry.example/operator/worker@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" subject_b="registry.example/operator/other@sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" @@ -25,7 +103,8 @@ init_home() { capture "$agent_lab" --home "$home" init if [ "$CAPTURE_RC" -ne 0 ]; then printf 'INFRA temporary Agent Lab home initialization failed: %s\n' "$(tr '\n' ' ' < "$work/stderr")" >&2 - exit 125 + infrastructure=1 + finish fi } @@ -68,7 +147,8 @@ invalid_names=( $'vendor.im\nage' ) invalid_names_ok=true -before_invalid="$(capture "$agent_lab" --home "$grammar_home" image list --all; jq -r 'length' "$work/stdout" 2>/dev/null || printf invalid)" +capture "$agent_lab" --home "$grammar_home" image list --all +before_invalid="$(jq -r 'length' "$work/stdout" 2>/dev/null || printf invalid)" for name in "${invalid_names[@]}"; do capture "$agent_lab" --home "$grammar_home" image add "$name" "$subject_a" if [ "$CAPTURE_RC" -ne 1 ] || [ -s "$work/stdout" ]; then @@ -257,8 +337,9 @@ done calibrated="$(find "$canary_marks" -type f | wc -l)" find "$canary_marks" -type f -delete canary_rc=0 -env -i PATH="$canary_bin:/usr/bin:/bin" LANG=C LC_ALL=C CANARY_DIR="$canary_marks" \ - "$agent_lab" --home "$canary_home" image add noeffect.mapping "$subject_a" > "$work/canary.out" 2> "$work/canary.err" || canary_rc=$? +run_bounded "$work/canary.out" "$work/canary.err" normal \ + env -i PATH="$canary_bin:/usr/bin:/bin" LANG=C LC_ALL=C CANARY_DIR="$canary_marks" \ + "$agent_lab" --home "$canary_home" image add noeffect.mapping "$subject_a" || canary_rc=$? if [ "$calibrated" -eq 4 ] && [ "$canary_rc" -eq 0 ] && [ -z "$(find "$canary_marks" -type f -print -quit)" ]; then pass CAT-NOEF-001 "calibrated Docker, Git, downloader, and network-tool canaries remain silent" else @@ -267,12 +348,22 @@ fi concurrent_home="$work/concurrent-home" init_home "$concurrent_home" -"$agent_lab" --home "$concurrent_home" image add race.first "$subject_a" > "$work/race-first-1.out" 2> "$work/race-first-1.err" & +run_bounded "$work/race-first-1.out" "$work/race-first-1.err" normal \ + "$agent_lab" --home "$concurrent_home" image add race.first "$subject_a" & pid_one=$! -"$agent_lab" --home "$concurrent_home" image add race.first "$subject_a" > "$work/race-first-2.out" 2> "$work/race-first-2.err" & +run_bounded "$work/race-first-2.out" "$work/race-first-2.err" normal \ + "$agent_lab" --home "$concurrent_home" image add race.first "$subject_a" & pid_two=$! wait "$pid_one"; rc_one=$? wait "$pid_two"; rc_two=$? +case "$rc_one:$rc_two" in + 0:0) + ;; + *) + [ "$rc_one" -eq 0 ] || [ "$rc_one" -eq 1 ] || infrastructure=1 + [ "$rc_two" -eq 0 ] || [ "$rc_two" -eq 1 ] || infrastructure=1 + ;; +esac first_outcomes="$(jq -r '.changed' "$work/race-first-1.out" "$work/race-first-2.out" 2>/dev/null | LC_ALL=C sort | tr '\n' ' ')" if [ "$rc_one" -eq 0 ] && [ "$rc_two" -eq 0 ] && [ "$first_outcomes" = "false true " ] && [ "$(find "$concurrent_home/images/catalog/entries" -maxdepth 1 -type f | wc -l)" -eq 1 ] && @@ -284,12 +375,16 @@ fi capture "$agent_lab" --home "$concurrent_home" image inspect race.first race_entry="$(jq -r '.entryDigest // empty' "$work/stdout" 2>/dev/null)" -"$agent_lab" --home "$concurrent_home" image add race.first "$subject_a" > "$work/race-add.out" 2> "$work/race-add.err" & +run_bounded "$work/race-add.out" "$work/race-add.err" normal \ + "$agent_lab" --home "$concurrent_home" image add race.first "$subject_a" & pid_add=$! -"$agent_lab" --home "$concurrent_home" image remove race.first --expect "$race_entry" > "$work/race-remove.out" 2> "$work/race-remove.err" & +run_bounded "$work/race-remove.out" "$work/race-remove.err" normal \ + "$agent_lab" --home "$concurrent_home" image remove race.first --expect "$race_entry" & pid_remove=$! wait "$pid_add"; rc_add=$? wait "$pid_remove"; rc_remove=$? +[ "$rc_add" -eq 0 ] || [ "$rc_add" -eq 1 ] || infrastructure=1 +[ "$rc_remove" -eq 0 ] || [ "$rc_remove" -eq 1 ] || infrastructure=1 capture "$agent_lab" --home "$concurrent_home" image inspect race.first if [ "$rc_remove" -eq 0 ] && { [ "$rc_add" -eq 0 ] || [ "$rc_add" -eq 1 ]; } && [ "$CAPTURE_RC" -eq 0 ] && jq -e '.state == "removed" and .generation == 2' "$work/stdout" >/dev/null 2>&1 && @@ -307,7 +402,6 @@ printf '%s\n' \ CAT-READ-001 CAT-READ-002 CAT-NOEF-001 CAT-CONC-001 CAT-CONC-002 > "$expected" if ! cmp -s "$expected" "$observed"; then printf 'INFRA catalog public assertion identity drift\n' >&2 - exit 125 + infrastructure=1 fi -printf 'SUMMARY assertions=18 expected=18 failures=%s infra=0\n' "$failures" -[ "$failures" -eq 0 ] +finish diff --git a/tests/image/catalog-mutation-cases.py b/tests/image/catalog-mutation-cases.py new file mode 100644 index 0000000..a488252 --- /dev/null +++ b/tests/image/catalog-mutation-cases.py @@ -0,0 +1,1047 @@ +#!/usr/bin/env python3 +"""Private-copy sensitivity mutations for the local image catalog.""" + +from __future__ import annotations + +from dataclasses import dataclass +import hashlib +from importlib.util import module_from_spec, spec_from_file_location +import json +import os +from pathlib import Path +import shutil +import signal +import stat +import subprocess +import sys +import tempfile +import time +from typing import Callable, NamedTuple + + +REPO_ROOT = Path(__file__).resolve().parents[2] +RUNTIME_MANIFEST = REPO_ROOT / "packaging" / "agent-lab-local.manifest" +SUBJECT = "registry.example/operator/worker@sha256:" + "a" * 64 +OTHER_SUBJECT = "registry.example/operator/other@sha256:" + "b" * 64 +STALE_ENTRY = "sha256:" + "e" * 64 +COMMAND_TIMEOUT_SECONDS = 5 + + +class InfrastructureError(Exception): + """The mutation or its isolated evidence could not be proved.""" + + +class ProbeResult(NamedTuple): + secure: bool + detail: str + + +Probe = Callable[[Path, Path, Path | None], ProbeResult] + + +@dataclass(frozen=True) +class Mutation: + assertion: str + path: str + old: str + new: str + probe: Probe + message: str + + +def sha256_bytes(data: bytes) -> str: + return hashlib.sha256(data).hexdigest() + + +def manifest_paths() -> tuple[str, ...]: + try: + raw = RUNTIME_MANIFEST.read_bytes() + text = raw.decode("utf-8") + except (OSError, UnicodeError) as error: + raise InfrastructureError("runtime manifest cannot be read exactly") from error + if not text.endswith("\n"): + raise InfrastructureError("runtime manifest lacks its final newline") + names = tuple(line for line in text.splitlines() if line) + if not names or len(names) != len(set(names)) or names != tuple(sorted(names)): + raise InfrastructureError("runtime manifest is empty, duplicated, or unordered") + for name in names: + path = Path(name) + if path.is_absolute() or ".." in path.parts or str(path) != name: + raise InfrastructureError(f"runtime manifest path is unsafe: {name}") + required = { + "scripts/agent-lab", + "scripts/agent-lab.py", + "scripts/experiment.py", + "scripts/image_catalog.py", + } + if not required.issubset(names): + raise InfrastructureError("runtime manifest omits a catalog runtime path") + return names + + +def file_identity(path: Path) -> tuple[str, int, int, str]: + try: + metadata = path.lstat() + data = path.read_bytes() + except OSError as error: + raise InfrastructureError(f"runtime path cannot be fingerprinted: {path}") from error + if not stat.S_ISREG(metadata.st_mode) or metadata.st_nlink != 1: + raise InfrastructureError(f"runtime path is not a single regular file: {path}") + return ( + "file", + stat.S_IMODE(metadata.st_mode), + len(data), + sha256_bytes(data), + ) + + +def runtime_fingerprint(root: Path, names: tuple[str, ...]) -> tuple[tuple[str, tuple[str, int, int, str]], ...]: + paths = ("packaging/agent-lab-local.manifest", *names) + return tuple((name, file_identity(root / name)) for name in paths) + + +def tree_fingerprint(root: Path) -> tuple[tuple[str, str, int, int, str], ...]: + if not root.exists() and not root.is_symlink(): + return () + values: list[tuple[str, str, int, int, str]] = [] + pending = [root] + while pending: + path = pending.pop() + try: + metadata = path.lstat() + except OSError as error: + raise InfrastructureError(f"probe state cannot be fingerprinted: {path}") from error + relative = "." if path == root else path.relative_to(root).as_posix() + mode = stat.S_IMODE(metadata.st_mode) + if stat.S_ISDIR(metadata.st_mode): + kind = "directory" + payload = "" + try: + pending.extend(sorted(path.iterdir(), reverse=True)) + except OSError as error: + raise InfrastructureError(f"probe directory cannot be listed: {path}") from error + elif stat.S_ISREG(metadata.st_mode): + kind = "file" + try: + payload = sha256_bytes(path.read_bytes()) + except OSError as error: + raise InfrastructureError(f"probe file cannot be read: {path}") from error + elif stat.S_ISLNK(metadata.st_mode): + kind = "symlink" + try: + payload = os.readlink(path) + except OSError as error: + raise InfrastructureError(f"probe symlink cannot be read: {path}") from error + else: + kind = "other" + payload = "" + values.append((relative, kind, mode, metadata.st_nlink, payload)) + return tuple(sorted(values)) + + +def copy_runtime(destination: Path, names: tuple[str, ...]) -> None: + for name in ("packaging/agent-lab-local.manifest", *names): + source = REPO_ROOT / name + target = destination / name + identity = file_identity(source) + try: + target.parent.mkdir(mode=0o700, parents=True, exist_ok=True) + shutil.copyfile(source, target) + os.chmod(target, identity[1]) + except OSError as error: + raise InfrastructureError(f"runtime path cannot be copied privately: {name}") from error + if file_identity(target) != identity: + raise InfrastructureError(f"private runtime copy differs: {name}") + + +def command_environment(extra: dict[str, str] | None = None) -> dict[str, str]: + environment = { + "PATH": "/usr/bin:/bin", + "LANG": "C", + "LC_ALL": "C", + } + if extra: + environment.update(extra) + return environment + + +def process_group_exists(group: int) -> bool: + try: + os.killpg(group, 0) + except ProcessLookupError: + return False + except PermissionError: + return True + return True + + +def terminate_process_group(process: subprocess.Popen[bytes]) -> bool: + group = process.pid + if process_group_exists(group): + try: + os.killpg(group, signal.SIGTERM) + except ProcessLookupError: + pass + deadline = time.monotonic() + 1.0 + while process_group_exists(group) and time.monotonic() < deadline: + time.sleep(0.01) + if process_group_exists(group): + try: + os.killpg(group, signal.SIGKILL) + except ProcessLookupError: + pass + try: + process.communicate(timeout=1) + except subprocess.TimeoutExpired: + try: + os.killpg(group, signal.SIGKILL) + except ProcessLookupError: + pass + process.communicate() + deadline = time.monotonic() + 1.0 + while process_group_exists(group) and time.monotonic() < deadline: + time.sleep(0.01) + return not process_group_exists(group) + + +def run_command( + arguments: list[str], + *, + environment: dict[str, str] | None = None, + timeout: int = COMMAND_TIMEOUT_SECONDS, +) -> subprocess.CompletedProcess[bytes]: + process: subprocess.Popen[bytes] | None = None + try: + process = subprocess.Popen( + arguments, + stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + env=command_environment(environment), + start_new_session=True, + ) + stdout, stderr = process.communicate(timeout=timeout) + if process_group_exists(process.pid): + cleaned = terminate_process_group(process) + if not cleaned: + raise InfrastructureError( + f"bounded probe command left an uncontained process group: {arguments[0]}" + ) + raise InfrastructureError( + f"bounded probe command left a descendant process: {arguments[0]}" + ) + return subprocess.CompletedProcess(arguments, process.returncode, stdout, stderr) + except subprocess.TimeoutExpired as error: + assert process is not None + cleaned = terminate_process_group(process) + if not cleaned: + raise InfrastructureError( + f"timed-out probe left an uncontained process group: {arguments[0]}" + ) from error + raise InfrastructureError(f"bounded probe command timed out: {arguments[0]}") from error + except (OSError, subprocess.SubprocessError) as error: + if process is not None and process.poll() is None: + terminate_process_group(process) + raise InfrastructureError(f"bounded probe command could not complete: {arguments[0]}") from error + + +def cli( + runtime: Path, + home: Path, + *arguments: str, + environment: dict[str, str] | None = None, +) -> subprocess.CompletedProcess[bytes]: + return run_command( + [ + sys.executable, + "-I", + "-B", + str(runtime / "scripts" / "agent-lab.py"), + "--home", + str(home), + *arguments, + ], + environment=environment, + ) + + +def initialized_home(runtime: Path, probe_root: Path, name: str) -> Path: + home = probe_root / name + completed = cli(runtime, home, "init") + if completed.returncode != 0 or completed.stdout != b"changed:true\n" or completed.stderr: + raise InfrastructureError( + "private runtime home initialization failed: " + + completed.stderr.decode("utf-8", errors="replace") + ) + return home + + +def json_object(completed: subprocess.CompletedProcess[bytes], purpose: str) -> dict[str, object]: + if completed.returncode != 0 or completed.stderr: + raise InfrastructureError( + f"{purpose} setup failed: " + completed.stderr.decode("utf-8", errors="replace") + ) + try: + value = json.loads(completed.stdout) + except (UnicodeError, json.JSONDecodeError) as error: + raise InfrastructureError(f"{purpose} returned malformed JSON") from error + if not isinstance(value, dict): + raise InfrastructureError(f"{purpose} returned a non-object") + return value + + +def add_mapping(runtime: Path, home: Path, name: str = "vendor.worker", subject: str = SUBJECT) -> dict[str, object]: + return json_object(cli(runtime, home, "image", "add", name, subject), "catalog add") + + +def marker_environment(marker: Path | None) -> dict[str, str] | None: + if marker is None: + return None + return {"AGENT_LAB_MUTATION_MARK": str(marker)} + + +def probe_invalid_oci(runtime: Path, probe_root: Path, marker: Path | None) -> ProbeResult: + home = initialized_home(runtime, probe_root, "oci-home") + before = tree_fingerprint(home) + invalid_subject = "registry.example/operator/worker@sha256:" + "A" * 64 + completed = cli( + runtime, + home, + "image", + "add", + "vendor.worker", + invalid_subject, + environment=marker_environment(marker), + ) + after = tree_fingerprint(home) + if completed.returncode not in (0, 1): + raise InfrastructureError(f"OCI grammar probe returned {completed.returncode}") + secure = completed.returncode == 1 and not completed.stdout and before == after + return ProbeResult(secure, f"rc={completed.returncode} changed={before != after}") + + +def probe_reserved_shadow(runtime: Path, probe_root: Path, marker: Path | None) -> ProbeResult: + module = load_catalog_module( + runtime, + f"agent_lab_image_catalog_shadow_mutation_{os.getpid()}_{id(probe_root)}", + ) + value = { + "apiVersion": module.ENTRY_API, + "generation": 1, + "name": "agent-lab.worker", + "previousEntryDigest": None, + "state": "active", + "subject": SUBJECT, + "subjectDigest": SUBJECT.rsplit("@", 1)[1], + } + if marker is not None: + os.environ["AGENT_LAB_MUTATION_MARK"] = str(marker) + try: + module._entry_schema(value) + except Exception as error: + if isinstance(error, getattr(module, "CatalogInfrastructure", ())): + return ProbeResult(True, "reserved local entry rejected") + raise InfrastructureError( + f"reserved-shadow probe raised an uncontained error: {error}" + ) from error + finally: + if marker is not None: + os.environ.pop("AGENT_LAB_MUTATION_MARK", None) + return ProbeResult(False, "reserved local entry accepted") + + +def probe_cas(runtime: Path, probe_root: Path, marker: Path | None) -> ProbeResult: + home = initialized_home(runtime, probe_root, "cas-home") + active = add_mapping(runtime, home) + active_digest = active.get("entryDigest") + if not isinstance(active_digest, str): + raise InfrastructureError("catalog add omitted its entry digest") + before = tree_fingerprint(home / "images") + completed = cli( + runtime, + home, + "image", + "remove", + "vendor.worker", + "--expect", + STALE_ENTRY, + environment=marker_environment(marker), + ) + inspected = cli(runtime, home, "image", "inspect", "vendor.worker") + if completed.returncode not in (0, 1) or inspected.returncode != 0: + raise InfrastructureError( + f"CAS probe returned remove={completed.returncode} inspect={inspected.returncode}" + ) + try: + record = json.loads(inspected.stdout) + except (UnicodeError, json.JSONDecodeError) as error: + raise InfrastructureError("CAS probe inspect returned malformed JSON") from error + after = tree_fingerprint(home / "images") + secure = ( + completed.returncode == 1 + and not completed.stdout + and isinstance(record, dict) + and record.get("state") == "active" + and record.get("entryDigest") == active_digest + and before == after + ) + return ProbeResult(secure, f"rc={completed.returncode} changed={before != after}") + + +def probe_symlink_authority(runtime: Path, probe_root: Path, marker: Path | None) -> ProbeResult: + home = initialized_home(runtime, probe_root, "authority-home") + add_mapping(runtime, home) + pointer = home / "images" / "catalog" / "current.json" + outside_pointer = probe_root / "outside-current.json" + try: + pointer.rename(outside_pointer) + os.symlink(outside_pointer, pointer) + except OSError as error: + raise InfrastructureError("authority pointer symlink could not be created") from error + outside_before = tree_fingerprint(outside_pointer) + completed = cli( + runtime, + home, + "image", + "list", + environment=marker_environment(marker), + ) + outside_after = tree_fingerprint(outside_pointer) + if completed.returncode not in (0, 125): + raise InfrastructureError(f"symlink authority probe returned {completed.returncode}") + secure = completed.returncode == 125 and not completed.stdout and outside_before == outside_after + return ProbeResult(secure, f"rc={completed.returncode} external_changed={outside_before != outside_after}") + + +def install_cue_fixture(home: Path) -> None: + source = REPO_ROOT / ".cache" / "dev" / "tools" / "cue" + destination = home / "cache" / "tools" / "cue" + if not source.is_dir(): + raise InfrastructureError("pinned CUE fixture is unavailable") + try: + shutil.copytree(source, destination, dirs_exist_ok=True) + except OSError as error: + raise InfrastructureError("pinned CUE fixture could not be privately copied") from error + + +def write_local_artifact(path: Path) -> None: + data = ( + 'package experiment\n\n' + 'experiment: {\n' + ' apiVersion: "agent-lab/v0alpha1"\n' + ' kind: "Experiment"\n' + ' metadata: name: "mutation-binding"\n' + ' spec: members: [{\n' + ' name: "worker"\n' + ' image: catalogName: "vendor.worker"\n' + ' }]\n' + '}\n' + ) + try: + path.mkdir(mode=0o700) + (path / "experiment.cue").write_text(data, encoding="utf-8") + os.chmod(path / "experiment.cue", 0o600) + except OSError as error: + raise InfrastructureError("binding probe artifact could not be created") from error + + +def probe_selected_binding(runtime: Path, probe_root: Path, marker: Path | None) -> ProbeResult: + home = initialized_home(runtime, probe_root, "binding-home") + install_cue_fixture(home) + active = add_mapping(runtime, home) + entry_digest = active.get("entryDigest") + if not isinstance(entry_digest, str): + raise InfrastructureError("binding probe setup omitted its entry digest") + artifact = probe_root / "binding-artifact" + write_local_artifact(artifact) + completed = cli( + runtime, + home, + "experiment", + "check", + str(artifact), + environment=marker_environment(marker), + ) + if completed.returncode != 0: + raise InfrastructureError(f"selected-binding probe returned {completed.returncode}") + try: + value = json.loads(completed.stdout) + resolved = value["plan"]["spec"]["members"][0]["resolvedImage"] + except (UnicodeError, json.JSONDecodeError, KeyError, TypeError, IndexError) as error: + return ProbeResult(False, f"selected binding is absent or malformed: {error}") + secure = resolved == { + "entryDigest": entry_digest, + "generation": 1, + "origin": "local", + "subject": SUBJECT, + } + return ProbeResult(secure, f"resolved={resolved!r}") + + +def make_canary(path: Path, name: str) -> None: + script = '#!/bin/sh\nset -eu\n: > "$CANARY_DIR/' + name + '"\n' + try: + path.write_text(script, encoding="utf-8") + os.chmod(path, 0o700) + except OSError as error: + raise InfrastructureError(f"forbidden-effect canary could not be created: {name}") from error + + +def probe_no_forbidden_effect(runtime: Path, probe_root: Path, marker: Path | None) -> ProbeResult: + home = initialized_home(runtime, probe_root, "no-effect-home") + canary_bin = probe_root / "canary-bin" + canary_marks = probe_root / "canary-marks" + canary_bin.mkdir(mode=0o700) + canary_marks.mkdir(mode=0o700) + names = ("docker", "git", "curl", "wget") + for name in names: + make_canary(canary_bin / name, name) + calibrated = run_command( + [str(canary_bin / name)], + environment={"CANARY_DIR": str(canary_marks)}, + ) + if calibrated.returncode != 0 or not (canary_marks / name).is_file(): + raise InfrastructureError(f"forbidden-effect canary did not calibrate: {name}") + for path in canary_marks.iterdir(): + path.unlink() + environment = { + "PATH": f"{canary_bin}:/usr/bin:/bin", + "CANARY_DIR": str(canary_marks), + } + if marker is not None: + environment["AGENT_LAB_MUTATION_MARK"] = str(marker) + completed = cli( + runtime, + home, + "image", + "add", + "vendor.worker", + SUBJECT, + environment=environment, + ) + if completed.returncode != 0: + raise InfrastructureError(f"no-effect probe catalog add returned {completed.returncode}") + fired = tuple(sorted(path.name for path in canary_marks.iterdir())) + return ProbeResult(not fired, f"forbidden_effects={fired!r}") + + +def probe_no_admission_authority( + runtime: Path, + probe_root: Path, + marker: Path | None, +) -> ProbeResult: + home = initialized_home(runtime, probe_root, "no-admission-authority-home") + completed = cli( + runtime, + home, + "image", + "add", + "vendor.worker", + SUBJECT, + environment=marker_environment(marker), + ) + result = json_object(completed, "catalog admission-boundary probe") + secure = ( + set(result) == {"changed", "entryDigest", "generation"} + and result.get("changed") is True + and type(result.get("generation")) is int + and result.get("generation") == 1 + and isinstance(result.get("entryDigest"), str) + ) + return ProbeResult(secure, f"result_keys={tuple(sorted(result))!r}") + + +def probe_unknown_staging(runtime: Path, probe_root: Path, marker: Path | None) -> ProbeResult: + home = initialized_home(runtime, probe_root, "staging-home") + wrapper = home / "images" / ".staging" / "foreign-wrapper" + try: + wrapper.mkdir(mode=0o700) + (wrapper / "unknown").write_bytes(b"foreign\n") + os.chmod(wrapper / "unknown", 0o600) + except OSError as error: + raise InfrastructureError("unknown staging fixture could not be created") from error + before = tree_fingerprint(wrapper) + completed = cli( + runtime, + home, + "image", + "add", + "vendor.worker", + SUBJECT, + environment=marker_environment(marker), + ) + after = tree_fingerprint(wrapper) + if completed.returncode not in (0, 125): + raise InfrastructureError(f"unknown-staging probe returned {completed.returncode}") + secure = ( + completed.returncode == 125 + and not completed.stdout + and before == after + and not (home / "images" / "catalog").exists() + ) + return ProbeResult(secure, f"rc={completed.returncode} foreign_changed={before != after}") + + +def load_catalog_module(runtime: Path, name: str): + path = runtime / "scripts" / "image_catalog.py" + spec = spec_from_file_location(name, path) + if spec is None or spec.loader is None: + raise InfrastructureError("private catalog module cannot be loaded") + module = module_from_spec(spec) + sys.modules[name] = module + previous = sys.dont_write_bytecode + sys.dont_write_bytecode = True + try: + spec.loader.exec_module(module) + except (ImportError, OSError, SyntaxError) as error: + raise InfrastructureError("private catalog module import failed") from error + finally: + sys.dont_write_bytecode = previous + return module + + +def probe_durability(runtime: Path, probe_root: Path, marker: Path | None) -> ProbeResult: + """Verify immutable record files are synced before current-pointer publication.""" + + home = initialized_home(runtime, probe_root, "durability-home") + add_mapping(runtime, home) + module = load_catalog_module(runtime, f"agent_lab_image_catalog_mutation_{os.getpid()}_{id(runtime)}") + events: list[tuple[str, str]] = [] + original_fsync = module.os.fsync + original_replace = module.os.replace + + def observed_fsync(descriptor: int) -> None: + try: + target = os.readlink(f"/proc/self/fd/{descriptor}") + except OSError: + target = f"fd:{descriptor}" + events.append(("fsync", target)) + original_fsync(descriptor) + + def observed_replace(source: os.PathLike[str] | str, target: os.PathLike[str] | str) -> None: + events.append(("replace", os.fspath(target))) + original_replace(source, target) + + module.os.fsync = observed_fsync + module.os.replace = observed_replace + if marker is not None: + os.environ["AGENT_LAB_MUTATION_MARK"] = str(marker) + try: + module.add_image(home, "vendor.second", OTHER_SUBJECT) + except Exception as error: + if isinstance(error, getattr(module, "CatalogReject", ())): + raise InfrastructureError(f"durability probe was rejected: {error}") from error + if isinstance(error, getattr(module, "CatalogInfrastructure", ())): + raise InfrastructureError(f"durability probe returned infrastructure failure: {error}") from error + raise InfrastructureError(f"durability probe raised an uncontained error: {error}") from error + finally: + module.os.fsync = original_fsync + module.os.replace = original_replace + if marker is not None: + os.environ.pop("AGENT_LAB_MUTATION_MARK", None) + + pointer_indexes = [ + index + for index, event in enumerate(events) + if event[0] == "replace" + and Path(event[1]) == home / "images" / "catalog" / "current.json" + ] + entry_syncs = [ + index + for index, event in enumerate(events) + if event[0] == "fsync" and event[1].endswith("/payload/entry.json") + ] + snapshot_syncs = [ + index + for index, event in enumerate(events) + if event[0] == "fsync" and event[1].endswith("/payload/snapshot.json") + ] + secure = ( + len(pointer_indexes) == 1 + and bool(entry_syncs) + and bool(snapshot_syncs) + and max(entry_syncs) < pointer_indexes[0] + and max(snapshot_syncs) < pointer_indexes[0] + ) + return ProbeResult( + secure, + f"entry_syncs={entry_syncs} snapshot_syncs={snapshot_syncs} pointer={pointer_indexes}", + ) + + +def probe_atomic_publication(runtime: Path, probe_root: Path, marker: Path | None) -> ProbeResult: + """Reject a producer that can expose a partial digest-addressed final record.""" + + home = initialized_home(runtime, probe_root, "atomic-publication-home") + add_mapping(runtime, home) + environment = {"AGENT_LAB_MUTATION_MARK": str(marker)} if marker is not None else None + attempted = cli( + runtime, + home, + "image", + "add", + "vendor.second", + OTHER_SUBJECT, + environment=environment, + ) + observed = cli(runtime, home, "image", "list") + try: + records = json.loads(observed.stdout) if observed.returncode == 0 else [] + except json.JSONDecodeError: + records = [] + secure = ( + attempted.returncode == 0 + and observed.returncode == 0 + and [record.get("name") for record in records] == ["vendor.second", "vendor.worker"] + ) + return ProbeResult( + secure, + f"add_rc={attempted.returncode} list_rc={observed.returncode} records={records!r}", + ) + + +def apply_mutation(runtime: Path, mutation: Mutation) -> None: + path = runtime / mutation.path + try: + source = path.read_text(encoding="utf-8") + except (OSError, UnicodeError) as error: + raise InfrastructureError(f"mutation source cannot be read: {mutation.path}") from error + occurrences = source.count(mutation.old) + if occurrences != 1: + raise InfrastructureError( + f"{mutation.assertion} replacement applicability is {occurrences}, expected exactly 1" + ) + mutated = source.replace(mutation.old, mutation.new, 1) + if mutated.count(mutation.new) != 1 or mutated == source: + raise InfrastructureError(f"{mutation.assertion} replacement result is ambiguous") + try: + path.write_text(mutated, encoding="utf-8") + except OSError as error: + raise InfrastructureError(f"{mutation.assertion} private source cannot be written") from error + if path.read_text(encoding="utf-8") != mutated: + raise InfrastructureError(f"{mutation.assertion} private source write was not exact") + + +def compile_mutation(runtime: Path, mutation: Mutation, cache: Path) -> None: + completed = run_command( + [ + sys.executable, + "-I", + "-X", + f"pycache_prefix={cache}", + "-m", + "py_compile", + str(runtime / mutation.path), + ] + ) + if completed.returncode != 0 or completed.stderr: + raise InfrastructureError( + f"{mutation.assertion} private mutation does not compile: " + + completed.stderr.decode("utf-8", errors="replace") + ) + + +# Each source rewrite must match exactly once in the copied runtime. The marker write proves the +# altered branch executed; it is confined to the mutation's temporary directory. +MUTATIONS: tuple[Mutation, ...] = ( + Mutation( + "M-CAT-OCI-001", + "scripts/image_reference.py", + ' and OCI_SUBJECT.fullmatch(value) is not None\n', + ( + ' and (\n' + ' __import__("pathlib").Path(\n' + ' __import__("os").environ["AGENT_LAB_MUTATION_MARK"]\n' + ' ).touch()\n' + ' or OCI_SUBJECT.fullmatch(value.lower()) is not None\n' + ' )\n' + ), + probe_invalid_oci, + "the invalid-OCI oracle detects an uppercase digest admitted by the shared parser", + ), + Mutation( + "M-CAT-SHADOW-001", + "scripts/image_catalog.py", + ( + ' or not isinstance(name, str)\n' + ' or name.startswith("agent-lab.")\n' + ' or not oci_subject(subject)\n' + ), + ( + ' or not isinstance(name, str)\n' + ' or (\n' + ' name.startswith("agent-lab.")\n' + ' and Path(os.environ["AGENT_LAB_MUTATION_MARK"]).touch() is not None\n' + ' )\n' + ' or not oci_subject(subject)\n' + ), + probe_reserved_shadow, + "the reserved-name oracle detects a local agent-lab.* entry admitted by its closed schema", + ), + Mutation( + "M-CAT-CAS-001", + "scripts/image_catalog.py", + ( + ' if prior["entryDigest"] != expected_entry_digest:\n' + ' _reject("remove compare-and-swap conflict")\n' + ' subject = prior["subject"]\n' + ' assert isinstance(subject, str)\n' + ' entry, entry_digest, snapshot, snapshot_digest, intent = _candidate_values(\n' + ' state,\n' + ' kind="remove",\n' + ' name=name,\n' + ' subject=subject,\n' + ' expected_entry_digest=expected_entry_digest,\n' + ' limits=limits,\n' + ' )\n' + ), + ( + ' Path(os.environ["AGENT_LAB_MUTATION_MARK"]).touch()\n' + ' subject = prior["subject"]\n' + ' assert isinstance(subject, str)\n' + ' entry, entry_digest, snapshot, snapshot_digest, intent = _candidate_values(\n' + ' state,\n' + ' kind="remove",\n' + ' name=name,\n' + ' subject=subject,\n' + ' expected_entry_digest=str(prior["entryDigest"]),\n' + ' limits=limits,\n' + ' )\n' + ), + probe_cas, + "the stale-token oracle detects removal with a substituted CAS token", + ), + Mutation( + "M-CAT-AUTH-001", + "scripts/image_catalog.py", + ( + 'def _read_file(path: Path, maximum: int, purpose: str) -> bytes:\n' + ' try:\n' + ' lexical = path.lstat()\n' + ), + ( + 'def _read_file(path: Path, maximum: int, purpose: str) -> bytes:\n' + ' if purpose == "local image current pointer" and path.is_symlink():\n' + ' Path(os.environ["AGENT_LAB_MUTATION_MARK"]).touch()\n' + ' return path.read_bytes()\n' + ' try:\n' + ' lexical = path.lstat()\n' + ), + probe_symlink_authority, + "the authority oracle detects a followed current-pointer symlink", + ), + Mutation( + "M-RES-BIND-001", + "scripts/experiment.py", + ' "entryDigest": record["entryDigest"],\n', + ( + ' "entryDigest": (\n' + ' Path(os.environ["AGENT_LAB_MUTATION_MARK"]).touch()\n' + ' or "sha256:" + "0" * 64\n' + ' ),\n' + ), + probe_selected_binding, + "the plan-binding oracle detects substitution of the selected entry digest", + ), + Mutation( + "M-CAT-NOEF-001", + "scripts/agent-lab.py", + ' result = catalog.add_image(home, argv[1], argv[2])\n', + ( + ' Path(os.environ["AGENT_LAB_MUTATION_MARK"]).touch()\n' + ' subprocess.run(("docker",), check=False)\n' + ' result = catalog.add_image(home, argv[1], argv[2])\n' + ), + probe_no_forbidden_effect, + "the calibrated no-effect oracle detects an injected Engine command", + ), + Mutation( + "M-CAT-ADMIT-001", + "scripts/image_catalog.py", + ' return {"changed": True, "entryDigest": entry_digest, "generation": 1}\n', + ( + ' Path(os.environ["AGENT_LAB_MUTATION_MARK"]).touch()\n' + ' return {\n' + ' "admitted": True,\n' + ' "changed": True,\n' + ' "entryDigest": entry_digest,\n' + ' "generation": 1,\n' + ' }\n' + ), + probe_no_admission_authority, + "the result-schema oracle detects catalog membership presented as admission authority", + ), + Mutation( + "M-CAT-ATOM-001", + "scripts/image_catalog.py", + ( + ' _fault(fault, f"{purpose}.before_noreplace")\n' + ' _rename_noreplace(source, target)\n' + ' _fault(fault, f"{purpose}.after_noreplace")\n' + ), + ( + ' _fault(fault, f"{purpose}.before_noreplace")\n' + ' Path(os.environ["AGENT_LAB_MUTATION_MARK"]).touch()\n' + ' descriptor = os.open(target, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)\n' + ' try:\n' + ' _write_all(descriptor, data[: max(1, len(data) // 2)])\n' + ' finally:\n' + ' os.close(descriptor)\n' + ' _fault(fault, f"{purpose}.after_noreplace")\n' + ), + probe_atomic_publication, + "the atomicity oracle detects a partial digest-addressed final record", + ), + Mutation( + "M-CAT-DUR-001", + "scripts/image_catalog.py", + ( + ' _write_all(descriptor, data)\n' + ' _fault(fault, f"{purpose}.before_fsync")\n' + ' os.fsync(descriptor)\n' + ' _fault(fault, f"{purpose}.after_fsync")\n' + ' metadata = os.fstat(descriptor)\n' + ), + ( + ' _write_all(descriptor, data)\n' + ' mutation_marker = os.environ.get("AGENT_LAB_MUTATION_MARK")\n' + ' if mutation_marker is not None:\n' + ' Path(mutation_marker).touch()\n' + ' _fault(fault, f"{purpose}.before_fsync")\n' + ' _fault(fault, f"{purpose}.after_fsync")\n' + ' metadata = os.fstat(descriptor)\n' + ), + probe_durability, + "the durability oracle detects publication without immutable-file fsync", + ), + Mutation( + "M-CAT-STAGE-001", + "scripts/image_catalog.py", + ( + ' if names != (CLEANUP_WRAPPER,):\n' + ' _infra("catalog staging root contains an unknown wrapper")\n' + ' cleanup = authority.staging / CLEANUP_WRAPPER\n' + ), + ( + ' if names != (CLEANUP_WRAPPER,):\n' + ' Path(os.environ["AGENT_LAB_MUTATION_MARK"]).touch()\n' + ' for name in names:\n' + ' _remove_owned_tree(authority.staging / name)\n' + ' _fsync_directory(authority.staging, "catalog staging root")\n' + ' return None\n' + ' cleanup = authority.staging / CLEANUP_WRAPPER\n' + ), + probe_unknown_staging, + "the staging oracle detects broad deletion of an unknown wrapper", + ), +) + + +def execute_mutation(root: Path, names: tuple[str, ...], mutation: Mutation) -> tuple[bool, str]: + runtime = root / "runtime" + copy_runtime(runtime, names) + copied = runtime_fingerprint(runtime, names) + copied_tree = tree_fingerprint(runtime) + pristine = mutation.probe(runtime, root / "pristine", None) + if not pristine.secure: + raise InfrastructureError( + f"{mutation.assertion} pristine probe is not GREEN: {pristine.detail}" + ) + if runtime_fingerprint(runtime, names) != copied: + raise InfrastructureError(f"{mutation.assertion} pristine probe changed its runtime copy") + if tree_fingerprint(runtime) != copied_tree: + raise InfrastructureError(f"{mutation.assertion} pristine probe changed runtime topology") + + apply_mutation(runtime, mutation) + compile_mutation(runtime, mutation, root / "pycache") + mutated = runtime_fingerprint(runtime, names) + mutated_tree = tree_fingerprint(runtime) + changed = [name for (name, before), (_, after) in zip(copied, mutated) if before != after] + if changed != [mutation.path]: + raise InfrastructureError( + f"{mutation.assertion} changed unexpected private runtime paths: {changed!r}" + ) + + marker = root / "mutation-reached" + result = mutation.probe(runtime, root / "mutant", marker) + if not marker.is_file(): + raise InfrastructureError(f"{mutation.assertion} did not prove its mutated path was reached") + if runtime_fingerprint(runtime, names) != mutated: + raise InfrastructureError(f"{mutation.assertion} mutant probe changed its runtime copy") + if tree_fingerprint(runtime) != mutated_tree: + raise InfrastructureError(f"{mutation.assertion} mutant probe changed runtime topology") + return not result.secure, result.detail + + +def main() -> int: + try: + names = manifest_paths() + shared_before = runtime_fingerprint(REPO_ROOT, names) + if not MUTATIONS: + raise InfrastructureError("catalog mutation declarations are unavailable") + expected = ( + "M-CAT-OCI-001", + "M-CAT-SHADOW-001", + "M-CAT-CAS-001", + "M-CAT-AUTH-001", + "M-RES-BIND-001", + "M-CAT-NOEF-001", + "M-CAT-ADMIT-001", + "M-CAT-ATOM-001", + "M-CAT-DUR-001", + "M-CAT-STAGE-001", + ) + if tuple(mutation.assertion for mutation in MUTATIONS) != expected: + raise InfrastructureError("catalog mutation assertion identity drift") + + failures = 0 + observed: list[str] = [] + for mutation in MUTATIONS: + try: + temporary = Path( + tempfile.mkdtemp( + prefix=f"agent-lab-{mutation.assertion.lower()}-", + dir="/tmp", + ) + ) + except OSError as error: + raise InfrastructureError( + f"{mutation.assertion} private mutation root is unavailable" + ) from error + try: + detected, detail = execute_mutation(temporary, names, mutation) + observed.append(mutation.assertion) + if detected: + print(f"PASS {mutation.assertion} {mutation.message}") + else: + failures += 1 + print(f"FAIL {mutation.assertion} {mutation.message} ({detail})") + finally: + cleanup_error: OSError | None = None + try: + shutil.rmtree(temporary) + except OSError as error: + cleanup_error = error + if runtime_fingerprint(REPO_ROOT, names) != shared_before: + raise InfrastructureError( + f"{mutation.assertion} changed the shared checkout runtime fingerprint" + ) + if cleanup_error is not None: + raise InfrastructureError( + f"{mutation.assertion} private mutation cleanup is uncertain" + ) from cleanup_error + if temporary.exists() or temporary.is_symlink(): + raise InfrastructureError( + f"{mutation.assertion} private mutation cleanup was incomplete" + ) + if tuple(observed) != expected: + raise InfrastructureError("catalog mutation execution identity drift") + print(f"SUMMARY assertions=10 expected=10 failures={failures} infra=0") + return 0 if failures == 0 else 1 + except InfrastructureError as error: + print(f"INFRA catalog mutation evidence: {error}", file=sys.stderr) + return 125 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/image/catalog-state-cases.py b/tests/image/catalog-state-cases.py index 21cfcd2..a26bd77 100755 --- a/tests/image/catalog-state-cases.py +++ b/tests/image/catalog-state-cases.py @@ -448,13 +448,39 @@ def main() -> int: before = fingerprint(foreign_wrapper) completed = cli(staging_home, "image", "add", "vendor.worker", SUBJECT) after = fingerprint(foreign_wrapper) + unsafe_cleanup_home = new_home(root, "unsafe-cleanup-home") + unsafe_cleanup = ( + unsafe_cleanup_home / "images" / ".staging" / "image-catalog-cleanup" + ) + unsafe_cleanup.mkdir(mode=0o700) + (unsafe_cleanup / "payload").write_bytes(b"not-a-directory\n") + (unsafe_cleanup / "payload").chmod(0o600) + cleanup_before = fingerprint(unsafe_cleanup) + cleanup_read = cli(unsafe_cleanup_home, "image", "list") + cleanup_retry = cli( + unsafe_cleanup_home, + "image", + "add", + "vendor.worker", + SUBJECT, + ) + cleanup_after = fingerprint(unsafe_cleanup) check( "CAT-STATE-009", completed.returncode == 125 and before == after - and not (staging_home / "images" / "catalog").exists(), - "unknown staging ownership is preserved and blocks new mutation", - f"rc={completed.returncode} wrapper_changed={before != after}", + and not (staging_home / "images" / "catalog").exists() + and cleanup_read.returncode == 125 + and cleanup_read.stdout == b"" + and cleanup_retry.returncode == 125 + and cleanup_retry.stdout == b"" + and cleanup_before == cleanup_after, + "unknown or unsafe staging ownership is preserved and blocks reads and mutation", + ( + f"rc={completed.returncode} wrapper_changed={before != after} " + f"cleanup_read={cleanup_read.returncode} cleanup_retry={cleanup_retry.returncode} " + f"cleanup_changed={cleanup_before != cleanup_after}" + ), ) pristine_home = new_home(root, "pristine-home") @@ -613,10 +639,42 @@ def replace_after_flock(descriptor: int, operation: int) -> None: split_rejected = True finally: CATALOG.fcntl.flock = original_flock + + transition_home = new_home(root, "marker-transition-home") + transition_authority = CATALOG._load_home(transition_home) + transition_lock = transition_home / "state" / "locks" / "image-catalog.lock" + original_open = CATALOG.os.open + transition_observed = None + transitioned = False + + def append_marker_before_open(path, flags, *args, **kwargs): + nonlocal transitioned + if Path(path) == transition_lock and flags & os.O_RDWR and not transitioned: + marker_descriptor = original_open( + path, + os.O_WRONLY | os.O_APPEND | getattr(os, "O_CLOEXEC", 0), + ) + try: + os.write(marker_descriptor, b"initialized\n") + os.fsync(marker_descriptor) + finally: + os.close(marker_descriptor) + transitioned = True + return original_open(path, flags, *args, **kwargs) + + CATALOG.os.open = append_marker_before_open + try: + with CATALOG._catalog_lock(transition_authority, exclusive=False) as descriptor: + transition_observed = CATALOG._lock_bytes(descriptor) + finally: + CATALOG.os.open = original_open check( "CAT-STATE-018", - split_rejected and split_replaced, - "receipt-bound lock identity is rechecked after acquisition to prevent split-brain", + split_rejected + and split_replaced + and transitioned + and transition_observed == CATALOG.LOCK_INITIALIZED, + "post-acquisition replacement is rejected while a valid same-inode marker transition is accepted", ) nested_home = new_home(root, "nested-json-home") diff --git a/tests/install/local-install-cases.sh b/tests/install/local-install-cases.sh index 29711d2..968c4d7 100755 --- a/tests/install/local-install-cases.sh +++ b/tests/install/local-install-cases.sh @@ -5,8 +5,22 @@ repo_root="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." >/dev/null 2>&1 && expected="$repo_root/tests/install/fixtures/expected-runtime-files.txt" manifest="$repo_root/packaging/agent-lab-local.manifest" installer="$repo_root/scripts/install-local" -work="$(mktemp -d)" -trap 'find "$work" -type f -delete 2>/dev/null || true; find "$work" -type l -delete 2>/dev/null || true; find "$work" -depth -type d -exec rmdir {} + 2>/dev/null || true' EXIT +work="" +cleanup_work() { + local failed=0 + if [ -n "$work" ] && [ -e "$work" ]; then + find "$work" -type f -delete 2>/dev/null || failed=1 + find "$work" -type l -delete 2>/dev/null || failed=1 + find "$work" -depth -type d -exec rmdir {} + 2>/dev/null || failed=1 + [ ! -e "$work" ] || failed=1 + fi + return "$failed" +} +if ! work="$(mktemp -d)"; then + printf 'SUMMARY assertions=0 expected=5 failures=0 infra=1\n' + exit 125 +fi +trap 'cleanup_work >/dev/null 2>&1 || true' EXIT failures=0 pass() { printf 'PASS %s %s\n' "$1" "$2"; } fail() { printf 'FAIL %s %s\n' "$1" "$2"; failures=$((failures + 1)); } @@ -80,5 +94,13 @@ else fail PKG-005 "installer refuses a symlinked prefix before writes" fi -printf 'SUMMARY assertions=5 expected=5 failures=%s infra=0\n' "$failures" +infrastructure=0 +if ! cleanup_work; then + infrastructure=1 +fi +trap - EXIT +printf 'SUMMARY assertions=5 expected=5 failures=%s infra=%s\n' "$failures" "$infrastructure" +if [ "$infrastructure" -ne 0 ]; then + exit 125 +fi [ "$failures" -eq 0 ] From 17561e6a1ddf205e4f4fb5d7add2761e1b61243b Mon Sep 17 00:00:00 2001 From: xormania <127287135+xormania@users.noreply.github.com> Date: Sun, 2 Aug 2026 01:48:52 -0400 Subject: [PATCH 043/158] docs(experiment): explain catalog recovery --- docs/images.md | 10 ++++++++-- docs/installation.md | 3 +++ 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/docs/images.md b/docs/images.md index b7aff04..2862b01 100644 --- a/docs/images.md +++ b/docs/images.md @@ -50,11 +50,17 @@ reachable transition chain, all physical history, ownership and modes, fixed cou bounds. Unsafe, missing initialized, changing, or corrupt authority returns `125`; an unknown or removed logical name returns `1`. +The initialized-home receipt binds the catalog lock's device, inode, path, and schema. The lock +starts with its schema line and appends exactly one `initialized` line only after the first complete +staged catalog is durable and immediately before its no-replace commit. A matching bounded intent +distinguishes that pre-commit recovery window; replacement or any other bytes fail closed. + Mutations prepare one bounded intent beneath `images/.staging/`. A first catalog uses Linux no-replace directory publication. Later changes durably publish immutable records before atomically advancing and syncing the current pointer. The next mutation reconciles a recognized interrupted -intent against the observed pointer; unknown residue is preserved and returns `125`. +intent against the observed pointer. After it proves that cleanup is safe, it durably renames the +operation to one bounded cleanup wrapper before removing any contents, so a crash during cleanup is +restartable. Unknown or unsafe residue is preserved and returns `125`. This is tamper-evident state for a cooperative local account, not protection from a hostile process running as the same user. - diff --git a/docs/installation.md b/docs/installation.md index e475203..d2d8abb 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -27,6 +27,9 @@ The default mutable home is the account-database home plus `.agent-lab`; `--home over `AGENT_LAB_HOME`. Ambient `HOME` is ignored. The first `init` may choose distinct safe single-component names with `--experiments-dir`, `--images-dir`, `--cache-dir`, and `--state-dir`. Those choices are frozen by `home.json`; later drift or conflicting initialization is refused. +The same receipt binds the device, inode, relative path, and schema of both stable lock files. +`config check` and exact `init` retry refuse a replaced lock or unexpected lock contents as +infrastructure uncertainty. `tools provision` is the only foundation command allowed to acquire the pinned CUE and Cedar binaries. Normal commands never download them automatically. Program releases, Experiment data, From 92766af1005c11786456f4da2e4e07976eb55b28 Mon Sep 17 00:00:00 2001 From: xormania <127287135+xormania@users.noreply.github.com> Date: Sun, 2 Aug 2026 02:14:53 -0400 Subject: [PATCH 044/158] test(experiment): define install store contract --- tests/experiment/aggregate-harness-cases.sh | 58 +- tests/experiment/install-state-cases.py | 891 ++++++++++++++++++++ tests/experiment/install-store-cases.sh | 505 +++++++++++ tests/experiment/local-lifecycle-cases.sh | 32 +- 4 files changed, 1471 insertions(+), 15 deletions(-) create mode 100644 tests/experiment/install-state-cases.py create mode 100755 tests/experiment/install-store-cases.sh diff --git a/tests/experiment/aggregate-harness-cases.sh b/tests/experiment/aggregate-harness-cases.sh index 06b4a15..1156aa5 100755 --- a/tests/experiment/aggregate-harness-cases.sh +++ b/tests/experiment/aggregate-harness-cases.sh @@ -51,10 +51,17 @@ expected_ids=( RES-SNAP-003 RES-AUTH-001 RES-INSTALL-001 RES-NOEF-001 M-CAT-OCI-001 M-CAT-SHADOW-001 M-CAT-CAS-001 M-CAT-AUTH-001 M-RES-BIND-001 M-CAT-NOEF-001 M-CAT-ADMIT-001 M-CAT-ATOM-001 M-CAT-DUR-001 M-CAT-STAGE-001 + INST-HOME-001 INST-UNKNOWN-001 INST-PERMIT-001 INST-RECEIPT-001 + INST-INSPECT-001 INST-RETRY-001 INST-CONFLICT-001 INST-DENY-001 + INST-FORGE-001 INST-NOEF-001 INST-LOCAL-001 + IST-STATE-001 IST-LOCK-001 IST-STATE-002 IST-BOUND-001 IST-STATE-003 + IST-CONC-001 IST-CONC-002 IST-CRASH-001 IST-LIVE-001 IST-PLAT-001 ) installer_ids=("${expected_ids[@]:0:5}") config_ids=("${expected_ids[@]:5:5}") -catalog_ids=("${expected_ids[@]:10}") +catalog_ids=("${expected_ids[@]:10:76}") +install_ids=("${expected_ids[@]:86:11}") +state_ids=("${expected_ids[@]:97:10}") write_fixture() { local path="$1" @@ -80,6 +87,32 @@ write_fixture() { chmod +x "$path" } +write_python_fixture() { + local path="$1" + local rc="$2" + shift 2 + local record kind id fixture_failures=0 + local execution_id="${path##*/}" + { + printf '#!/usr/bin/env python3\n' + printf 'import os\n' + printf 'log = os.environ.get("AGENT_LAB_AGG_EXEC_LOG")\n' + printf 'if log:\n' + printf ' with open(log, "a", encoding="ascii") as stream:\n' + printf " stream.write('%s\\\\n')\n" "$execution_id" + for record in "$@"; do + kind="${record%%:*}" + id="${record#*:}" + [ "$kind" = "FAIL" ] && fixture_failures=$((fixture_failures + 1)) + printf "print('%s %s fixture assertion')\n" "$kind" "$id" + done + printf "print('SUMMARY assertions=%s expected=%s failures=%s infra=0')\n" \ + "$#" "$#" "$fixture_failures" + printf 'raise SystemExit(%s)\n' "$rc" + } > "$path" + chmod +x "$path" +} + pass_records() { local id for id in "$@"; do @@ -89,12 +122,17 @@ pass_records() { reset_fixtures() { local installer_records=() config_records=() catalog_records=() + local install_records=() state_records=() mapfile -t installer_records < <(pass_records "${installer_ids[@]}") mapfile -t config_records < <(pass_records "${config_ids[@]}") mapfile -t catalog_records < <(pass_records "${catalog_ids[@]}") + mapfile -t install_records < <(pass_records "${install_ids[@]}") + mapfile -t state_records < <(pass_records "${state_ids[@]}") write_fixture "$replica/tests/install/local-install-cases.sh" 0 "${installer_records[@]}" write_fixture "$replica/tests/experiment/local-config-cases.sh" 0 "${config_records[@]}" write_fixture "$replica/tests/experiment/local-image-catalog-cases.sh" 0 "${catalog_records[@]}" + write_fixture "$replica/tests/experiment/install-store-cases.sh" 0 "${install_records[@]}" + write_python_fixture "$replica/tests/experiment/install-state-cases.py" 0 "${state_records[@]}" } run_replica() { @@ -118,7 +156,9 @@ mutant_executions="$work/mutant-executions" printf '%s\n' \ local-install-cases.sh \ local-config-cases.sh \ - local-image-catalog-cases.sh > "$expected_executions" + local-image-catalog-cases.sh \ + install-store-cases.sh \ + install-state-cases.py > "$expected_executions" : > "$baseline_executions" baseline_rc=0 run_replica "$work/baseline.out" env \ @@ -144,7 +184,9 @@ printf '%s\n' \ local-install-cases.sh \ local-install-cases.sh \ local-config-cases.sh \ - local-image-catalog-cases.sh > "$mutant_expected" + local-image-catalog-cases.sh \ + install-store-cases.sh \ + install-state-cases.py > "$mutant_expected" if [ "$baseline_rc" -eq 0 ] && cmp -s "$expected_executions" "$baseline_executions" && @@ -163,10 +205,10 @@ success_output="$work/success.out" success_rc=0 run_replica "$success_output" env || success_rc=$? if [ "$success_rc" -eq 0 ] && - [ "$(grep -Ec '^(PASS|FAIL) [A-Z0-9-]+ ' "$success_output")" -eq 86 ] && - [ "$(grep -Fxc 'SUMMARY assertions=86 expected=86 failures=0 infra=0' "$success_output")" -eq 1 ] && + [ "$(grep -Ec '^(PASS|FAIL) [A-Z0-9-]+ ' "$success_output")" -eq 107 ] && + [ "$(grep -Fxc 'SUMMARY assertions=107 expected=107 failures=0 infra=0' "$success_output")" -eq 1 ] && [ "$(tail -n 1 "$success_output")" = 'EXPERIMENT LOCAL LIFECYCLE PASS' ] && - awk '/^(PASS|FAIL) [A-Z0-9-]+ / {next} /^SUMMARY assertions=86 expected=86 failures=0 infra=0$/ {next} /^EXPERIMENT LOCAL LIFECYCLE PASS$/ {next} {bad=1} END {exit bad}' "$success_output"; then + awk '/^(PASS|FAIL) [A-Z0-9-]+ / {next} /^SUMMARY assertions=107 expected=107 failures=0 infra=0$/ {next} /^EXPERIMENT LOCAL LIFECYCLE PASS$/ {next} {bad=1} END {exit bad}' "$success_output"; then pass AGG-002 "success forwards only assertions then one summary and marker" else fail AGG-002 "success forwards only assertions then one summary and marker" @@ -215,7 +257,7 @@ write_fixture "$replica/tests/install/local-install-cases.sh" 1 "${failed_record assertion_rc=0 run_replica "$work/assertion.out" env || assertion_rc=$? if [ "$assertion_rc" -eq 1 ] && - grep -Fxq 'SUMMARY assertions=86 expected=86 failures=1 infra=0' "$work/assertion.out" && + grep -Fxq 'SUMMARY assertions=107 expected=107 failures=1 infra=0' "$work/assertion.out" && ! grep -Fxq 'EXPERIMENT LOCAL LIFECYCLE PASS' "$work/assertion.out"; then pass AGG-006 "subcase assertion failure maps to one" else @@ -248,7 +290,7 @@ chmod +x "$shim/rmdir" cleanup_rc=0 run_replica "$work/cleanup.out" env PATH="$shim:$PATH" || cleanup_rc=$? if [ "$cleanup_rc" -eq 125 ] && - grep -Fxq 'SUMMARY assertions=86 expected=86 failures=0 infra=1' "$work/cleanup.out" && + grep -Fxq 'SUMMARY assertions=107 expected=107 failures=0 infra=1' "$work/cleanup.out" && ! grep -Fxq 'EXPERIMENT LOCAL LIFECYCLE PASS' "$work/cleanup.out"; then pass AGG-008 "cleanup uncertainty maps to one hundred twenty-five before the marker" else diff --git a/tests/experiment/install-state-cases.py b/tests/experiment/install-state-cases.py new file mode 100644 index 0000000..5c597ac --- /dev/null +++ b/tests/experiment/install-state-cases.py @@ -0,0 +1,891 @@ +#!/usr/bin/env python3 +"""Adversarial Experiment-store filesystem, locking, and recovery cases.""" + +from __future__ import annotations + +from contextlib import redirect_stderr, redirect_stdout +import hashlib +from importlib.util import module_from_spec, spec_from_file_location +import io +import json +import os +from pathlib import Path +import signal +import stat +import subprocess +import sys +import tempfile +import time +from typing import Callable + + +REPO_ROOT = Path(__file__).resolve().parents[2] +AGENT_LAB = REPO_ROOT / "scripts" / "agent-lab" +AGENT_LAB_MODULE = REPO_ROOT / "scripts" / "agent-lab.py" +STORE_MODULE = REPO_ROOT / "scripts" / "experiment_store.py" +CUE_TOOLS = REPO_ROOT / ".cache" / "dev" / "tools" / "cue" +CEDAR_TOOLS = REPO_ROOT / ".cache" / "dev" / "tools" / "cedar" +SUBJECT = "registry.example/team/worker@sha256:" + "a" * 64 +OTHER_SUBJECT = "registry.example/team/worker@sha256:" + "b" * 64 +FAULT_POINTS = ( + "experiment artifact.after_write", + "experiment receipt.after_fsync", + "experiment envelope.before_noreplace", + "experiment envelope.after_noreplace", + "experiment store root.after_fsync", +) + + +def load_module(path: Path, name: str): + spec = spec_from_file_location(name, path) + if spec is None or spec.loader is None: + raise RuntimeError(f"{path.name} cannot be loaded") + module = module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +MODULE = load_module(AGENT_LAB_MODULE, "agent_lab_install_state") +try: + STORE = load_module(STORE_MODULE, "agent_lab_experiment_store_state") + STORE_LOAD_ERROR: BaseException | None = None +except BaseException as error: # Missing production is expected RED, not harness infrastructure. + STORE = None + STORE_LOAD_ERROR = error + +FAILURES = 0 +INFRA = 0 +OBSERVED: list[str] = [] + + +def check(assertion: str, condition: bool, message: str, detail: str = "") -> None: + global FAILURES + OBSERVED.append(assertion) + if condition: + print(f"PASS {assertion} {message}") + else: + FAILURES += 1 + suffix = f" ({detail})" if detail else "" + print(f"FAIL {assertion} {message}{suffix}") + + +def command_environment() -> dict[str, str]: + return { + "PATH": "/usr/bin:/bin", + "LANG": "C", + "LC_ALL": "C", + "AGENT_LAB_CUE_TOOL_DIR": str(CUE_TOOLS), + "AGENT_LAB_CEDAR_TOOL_DIR": str(CEDAR_TOOLS), + } + + +def run_command(arguments: list[str], timeout: float = 30.0) -> subprocess.CompletedProcess[bytes]: + global INFRA + try: + process = subprocess.Popen( + arguments, + stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + env=command_environment(), + start_new_session=True, + ) + except OSError as error: + INFRA += 1 + return subprocess.CompletedProcess(arguments, 125, b"", str(error).encode()) + try: + stdout, stderr = process.communicate(timeout=timeout) + except subprocess.TimeoutExpired: + INFRA += 1 + try: + os.killpg(process.pid, signal.SIGKILL) + except ProcessLookupError: + pass + stdout, stderr = process.communicate() + return subprocess.CompletedProcess(arguments, 125, stdout, stderr + b"\nHARNESS TIMEOUT\n") + return subprocess.CompletedProcess(arguments, process.returncode, stdout, stderr) + + +def cli(home: Path, *arguments: str) -> subprocess.CompletedProcess[bytes]: + return run_command([str(AGENT_LAB), "--home", str(home), *arguments]) + + +def start_cli(home: Path, source: Path) -> subprocess.Popen[bytes]: + return subprocess.Popen( + [str(AGENT_LAB), "--home", str(home), "experiment", "install", str(source)], + stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + env=command_environment(), + start_new_session=True, + ) + + +def finish_process( + process: subprocess.Popen[bytes], + timeout: float = 30.0, +) -> subprocess.CompletedProcess[bytes]: + global INFRA + try: + stdout, stderr = process.communicate(timeout=timeout) + except subprocess.TimeoutExpired: + INFRA += 1 + try: + os.killpg(process.pid, signal.SIGKILL) + except ProcessLookupError: + pass + stdout, stderr = process.communicate() + return subprocess.CompletedProcess( + process.args, + 125, + stdout, + stderr + b"\nHARNESS TIMEOUT\n", + ) + return subprocess.CompletedProcess(process.args, process.returncode, stdout, stderr) + + +def new_home(root: Path, name: str) -> Path: + home = root / name + completed = cli(home, "init") + if completed.returncode != 0: + raise RuntimeError( + f"temporary home init failed: rc={completed.returncode} " + f"stderr={completed.stderr.decode(errors='replace')}" + ) + return home + + +def source_directory( + root: Path, + directory: str, + *, + requested_name: str = "first-experiment", + subject: str = SUBJECT, + command: str = "serve", + catalog_name: str | None = None, +) -> Path: + source = root / directory + source.mkdir(mode=0o700) + if catalog_name is None: + selector = f'digestRef: "{subject}"' + else: + selector = f'catalogName: "{catalog_name}"' + data = ( + "package experiment\n\n" + "experiment: {\n" + '\tapiVersion: "agent-lab/v0alpha1"\n' + '\tkind: "Experiment"\n' + f'\tmetadata: name: "{requested_name}"\n' + "\tspec: members: [{\n" + '\t\tname: "worker"\n' + f"\t\timage: {selector}\n" + f'\t\tcommand: ["{command}"]\n' + "\t}]\n" + "}\n" + ).encode("utf-8") + path = source / "experiment.cue" + path.write_bytes(data) + path.chmod(0o600) + return source + + +def fingerprint(root: Path) -> tuple[tuple[str, str, int, int, int, str], ...]: + if not root.exists() and not root.is_symlink(): + return () + records: list[tuple[str, str, int, int, int, str]] = [] + + def visit(path: Path, relative: str) -> None: + metadata = path.lstat() + if stat.S_ISLNK(metadata.st_mode): + kind = "l" + identity = os.readlink(path) + elif stat.S_ISREG(metadata.st_mode): + kind = "f" + digest = hashlib.sha256() + with path.open("rb") as stream: + while chunk := stream.read(65_536): + digest.update(chunk) + identity = digest.hexdigest() + elif stat.S_ISDIR(metadata.st_mode): + kind = "d" + identity = "" + else: + kind = "o" + identity = "" + records.append( + ( + relative, + kind, + stat.S_IMODE(metadata.st_mode), + metadata.st_nlink, + metadata.st_size, + identity, + ) + ) + if kind == "d": + children = sorted(path.iterdir(), key=lambda item: os.fsencode(item.name)) + for child in children: + child_relative = child.name if relative == "." else f"{relative}/{child.name}" + visit(child, child_relative) + + visit(root, ".") + return tuple(records) + + +def json_object(completed: subprocess.CompletedProcess[bytes]) -> dict[str, object] | None: + if completed.returncode != 0: + return None + try: + value = json.loads(completed.stdout) + except (UnicodeError, json.JSONDecodeError): + return None + return value if isinstance(value, dict) else None + + +def store_install( + home: Path, + source: Path, + *, + fault: Callable[[str], None] | None = None, +) -> tuple[int | None, dict[str, object] | None, BaseException | None]: + if STORE is None: + return None, None, STORE_LOAD_ERROR or RuntimeError("experiment store module is missing") + operation = getattr(STORE, "install_directory", None) + if not callable(operation): + return None, None, RuntimeError("experiment_store.install_directory is missing") + try: + value = operation(home, source, fault=fault) + if not isinstance(value, dict): + return None, None, RuntimeError("install_directory returned a non-object") + return 0, value, None + except getattr(STORE, "StoreReject", ()) as error: + return 1, None, error + except getattr(STORE, "StoreInfrastructure", ()) as error: + return 125, None, error + except BaseException as error: # An uncontained production fault is RED, not harness infra. + return None, None, error + + +def module_main( + home: Path, + arguments: list[str], + output: io.TextIOBase | None = None, +) -> tuple[int | None, str, BaseException | None]: + stream = output if output is not None else io.StringIO() + errors = io.StringIO() + old_cue = os.environ.get("AGENT_LAB_CUE_TOOL_DIR") + old_cedar = os.environ.get("AGENT_LAB_CEDAR_TOOL_DIR") + os.environ["AGENT_LAB_CUE_TOOL_DIR"] = str(CUE_TOOLS) + os.environ["AGENT_LAB_CEDAR_TOOL_DIR"] = str(CEDAR_TOOLS) + try: + with redirect_stdout(stream), redirect_stderr(errors): + result = MODULE.main(["--home", str(home), *arguments]) + return result, errors.getvalue(), None + except BaseException as error: # An uncontained production fault is RED, not harness infra. + return None, errors.getvalue(), error + finally: + if old_cue is None: + os.environ.pop("AGENT_LAB_CUE_TOOL_DIR", None) + else: + os.environ["AGENT_LAB_CUE_TOOL_DIR"] = old_cue + if old_cedar is None: + os.environ.pop("AGENT_LAB_CEDAR_TOOL_DIR", None) + else: + os.environ["AGENT_LAB_CEDAR_TOOL_DIR"] = old_cedar + + +def hard_exit_install(home: Path, source: Path, point: str) -> int: + pid = os.fork() + if pid == 0: + def stop_at(observed: str) -> None: + if observed == point: + os._exit(99) + + result, _, _ = store_install(home, source, fault=stop_at) + os._exit(97 if result == 0 else 96) + deadline = time.monotonic() + 30.0 + while time.monotonic() < deadline: + waited, status = os.waitpid(pid, os.WNOHANG) + if waited == pid: + return os.waitstatus_to_exitcode(status) + time.sleep(0.01) + os.kill(pid, signal.SIGKILL) + os.waitpid(pid, 0) + global INFRA + INFRA += 1 + return 124 + + +def lock_is_blocked(path: Path) -> bool: + program = ( + "import fcntl, os, sys\n" + "fd=os.open(sys.argv[1], os.O_RDWR|getattr(os,'O_CLOEXEC',0))\n" + "try:\n" + " fcntl.flock(fd, fcntl.LOCK_EX|fcntl.LOCK_NB)\n" + "except BlockingIOError:\n" + " raise SystemExit(3)\n" + "raise SystemExit(0)\n" + ) + completed = run_command([sys.executable, "-I", "-c", program, str(path)], timeout=5.0) + return completed.returncode == 3 + + +class BrokenOutput(io.StringIO): + def write(self, value: str) -> int: + raise OSError("injected result-output failure") + + +def main() -> int: + with tempfile.TemporaryDirectory(prefix="agent-lab-install-state-") as directory: + root = Path(directory) + direct_source = source_directory(root, "direct-source") + + mode_home = new_home(root, "unsafe-mode-home") + mode_store = mode_home / "experiments" + mode_store.chmod(0o755) + mode_before = fingerprint(mode_store) + mode_result = cli(mode_home, "experiment", "install", str(direct_source)) + mode_after = fingerprint(mode_store) + + symlink_home = new_home(root, "symlink-store-home") + symlink_store = symlink_home / "experiments" + outside_store = root / "outside-store" + symlink_store.rename(outside_store) + os.symlink(outside_store, symlink_store) + symlink_before = fingerprint(outside_store) + symlink_result = cli(symlink_home, "experiment", "install", str(direct_source)) + symlink_after = fingerprint(outside_store) + + stage_home = new_home(root, "symlink-stage-home") + stage = stage_home / "experiments" / ".staging" + outside_stage = root / "outside-stage" + stage.rename(outside_stage) + os.symlink(outside_stage, stage) + stage_before = fingerprint(outside_stage) + stage_result = cli(stage_home, "experiment", "install", str(direct_source)) + stage_after = fingerprint(outside_stage) + + cross_home = new_home(root, "cross-device-home") + cross_stage = cross_home / "experiments" / ".staging" + cross_before = fingerprint(cross_home / "experiments") + original_fstat = MODULE.os.fstat + original_cross_lstat = MODULE.os.lstat + cross_injected = False + + def cross_device_metadata(metadata: os.stat_result) -> os.stat_result: + nonlocal cross_injected + values = list(metadata) + values[2] = metadata.st_dev + 1 + cross_injected = True + return os.stat_result(values) + + def cross_device_fstat(descriptor: int): + nonlocal cross_injected + metadata = original_fstat(descriptor) + try: + target = os.readlink(f"/proc/self/fd/{descriptor}") + except OSError: + return metadata + if target.rstrip("/") == str(cross_stage): + return cross_device_metadata(metadata) + return metadata + + def cross_device_lstat(path: object, *args, **kwargs): + metadata = original_cross_lstat(path, *args, **kwargs) + try: + target = os.fsdecode(os.fspath(path)).rstrip("/") + except TypeError: + return metadata + if target == str(cross_stage): + return cross_device_metadata(metadata) + return metadata + + MODULE.os.fstat = cross_device_fstat + MODULE.os.lstat = cross_device_lstat + try: + cross_rc, _, cross_error = module_main( + cross_home, + ["experiment", "install", str(direct_source)], + ) + finally: + MODULE.os.lstat = original_cross_lstat + MODULE.os.fstat = original_fstat + cross_after = fingerprint(cross_home / "experiments") + check( + "IST-STATE-001", + mode_result.returncode == 125 + and mode_before == mode_after + and symlink_result.returncode == 125 + and symlink_before == symlink_after + and stage_result.returncode == 125 + and stage_before == stage_after + and cross_injected + and cross_rc == 125 + and cross_error is None + and cross_before == cross_after, + "unsafe, linked, or cross-filesystem store and staging roots fail before publication", + ( + f"mode={mode_result.returncode}/{mode_before != mode_after} " + f"store_link={symlink_result.returncode}/{symlink_before != symlink_after} " + f"stage_link={stage_result.returncode}/{stage_before != stage_after} " + f"cross={cross_rc}/{cross_injected}/{cross_error!r}/{cross_before != cross_after}" + ), + ) + + symlink_lock_home = new_home(root, "symlink-lock-home") + lock = symlink_lock_home / "state" / "locks" / "experiments.lock" + saved_lock = root / "saved-experiments.lock" + lock.rename(saved_lock) + os.symlink(saved_lock, lock) + lock_before = fingerprint(saved_lock) + symlink_lock_result = cli( + symlink_lock_home, + "experiment", + "install", + str(direct_source), + ) + lock_after = fingerprint(saved_lock) + hardlink_lock_home = new_home(root, "hardlink-lock-home") + hardlink_lock = hardlink_lock_home / "state" / "locks" / "experiments.lock" + os.link(hardlink_lock, root / "second-experiments-lock-link") + hardlink_result = cli( + hardlink_lock_home, + "experiment", + "install", + str(direct_source), + ) + check( + "IST-LOCK-001", + symlink_lock_result.returncode == 125 + and lock_before == lock_after + and hardlink_result.returncode == 125 + and not tuple((hardlink_lock_home / "experiments" / ".staging").iterdir()), + "the pre-created receipt-bound store lock is safe-opened with stable identity", + ( + f"symlink={symlink_lock_result.returncode}/{lock_before != lock_after} " + f"hardlink={hardlink_result.returncode}" + ), + ) + + existing_home = new_home(root, "existing-target-home") + existing_target = existing_home / "experiments" / "first-experiment" + existing_target.mkdir(mode=0o700) + sentinel = existing_target / "foreign" + sentinel.write_bytes(b"foreign\n") + sentinel.chmod(0o600) + existing_before = fingerprint(existing_target) + existing_result = cli(existing_home, "experiment", "install", str(direct_source)) + existing_after = fingerprint(existing_target) + + race_home = new_home(root, "publication-race-home") + race_target = race_home / "experiments" / "first-experiment" + race_triggered = False + + def create_racing_target(point: str) -> None: + nonlocal race_triggered + if point == "experiment envelope.before_noreplace" and not race_triggered: + race_triggered = True + race_target.mkdir(mode=0o700) + marker = race_target / "foreign" + marker.write_bytes(b"racing foreign target\n") + marker.chmod(0o600) + + race_rc, _, race_error = store_install( + race_home, + direct_source, + fault=create_racing_target, + ) + race_fingerprint = fingerprint(race_target) + check( + "IST-STATE-002", + existing_result.returncode == 125 + and existing_before == existing_after + and race_triggered + and race_rc == 125 + and race_error is not None + and any( + item[0] == "foreign" + and item[-1] + == hashlib.sha256(b"racing foreign target\n").hexdigest() + for item in race_fingerprint + ), + "ambiguous and racing final targets are preserved by atomic no-replace publication", + ( + f"existing={existing_result.returncode}/{existing_before != existing_after} " + f"race={race_rc}/{race_triggered}/{race_error!r}/{race_fingerprint!r}" + ), + ) + + foreign_home = new_home(root, "foreign-stage-home") + foreign_wrapper = foreign_home / "experiments" / ".staging" / "foreign-wrapper" + (foreign_wrapper / "payload").mkdir(mode=0o700, parents=True) + foreign_intent = foreign_wrapper / "intent.json" + foreign_intent.write_bytes(b'{"owner":"foreign"}\n') + foreign_intent.chmod(0o600) + foreign_before = fingerprint(foreign_wrapper) + foreign_result = cli(foreign_home, "experiment", "install", str(direct_source)) + foreign_after = fingerprint(foreign_wrapper) + + bound_home = new_home(root, "overbound-stage-home") + bound_stage = bound_home / "experiments" / ".staging" + for index in range(17): + path = bound_stage / f"foreign-{index:02d}" + path.write_bytes(b"x") + path.chmod(0o600) + bound_before = fingerprint(bound_stage) + bound_result = cli(bound_home, "experiment", "install", str(direct_source)) + bound_after = fingerprint(bound_stage) + check( + "IST-BOUND-001", + foreign_result.returncode == 125 + and foreign_before == foreign_after + and bound_result.returncode == 125 + and bound_before == bound_after, + "unknown or over-bound staging residue blocks mutation without broad deletion", + ( + f"foreign={foreign_result.returncode}/{foreign_before != foreign_after} " + f"bound={bound_result.returncode}/{bound_before != bound_after}" + ), + ) + + tamper_home = new_home(root, "committed-tamper-home") + installed = cli(tamper_home, "experiment", "install", str(direct_source)) + envelope = tamper_home / "experiments" / "first-experiment" + link_detected = mode_detected = digest_detected = restored = False + tamper_detail = f"install={installed.returncode}" + if installed.returncode == 0 and envelope.is_dir(): + records = sorted((envelope / "records").glob("*.json")) + if records: + record = records[0] + outside_link = root / "outside-record-link" + os.link(record, outside_link) + linked_inspect = cli(tamper_home, "experiment", "inspect", "first-experiment") + link_detected = linked_inspect.returncode == 125 + outside_link.unlink() + restored_inspect = cli(tamper_home, "experiment", "inspect", "first-experiment") + restored = restored_inspect.returncode == 0 + envelope.chmod(0o700) + mode_inspect = cli(tamper_home, "experiment", "inspect", "first-experiment") + mode_detected = mode_inspect.returncode == 125 + envelope.chmod(0o500) + raw = record.read_bytes() + record.chmod(0o600) + record.write_bytes(bytes([raw[0] ^ 1]) + raw[1:]) + record.chmod(0o400) + digest_inspect = cli(tamper_home, "experiment", "inspect", "first-experiment") + digest_detected = digest_inspect.returncode == 125 + tamper_detail += ( + f" link={linked_inspect.returncode} restore={restored_inspect.returncode} " + f"mode={mode_inspect.returncode} digest={digest_inspect.returncode}" + ) + check( + "IST-STATE-003", + installed.returncode == 0 + and link_detected + and restored + and mode_detected + and digest_detected, + "inspect safe-reopens the whole receipt and detects link, mode, and byte drift", + tamper_detail, + ) + + identical_home = new_home(root, "identical-concurrency-home") + first = start_cli(identical_home, direct_source) + second = start_cli(identical_home, direct_source) + identical_results = [finish_process(first), finish_process(second)] + identical_values = [json_object(item) for item in identical_results] + identical_changes = sorted( + value.get("changed") + for value in identical_values + if isinstance(value, dict) and isinstance(value.get("changed"), bool) + ) + identical_inspect = cli( + identical_home, + "experiment", + "inspect", + "first-experiment", + ) + check( + "IST-CONC-001", + [item.returncode for item in identical_results] == [0, 0] + and identical_changes == [False, True] + and identical_inspect.returncode == 0 + and not tuple((identical_home / "experiments" / ".staging").iterdir()), + "concurrent identical installs publish once and return one verified idempotent success", + ( + f"rcs={[item.returncode for item in identical_results]!r} " + f"changes={identical_changes!r} inspect={identical_inspect.returncode}" + ), + ) + + conflict_home = new_home(root, "different-concurrency-home") + conflict_one = source_directory( + root, + "conflict-source-one", + command="winner-one", + ) + conflict_two = source_directory( + root, + "conflict-source-two", + command="winner-two", + ) + one = start_cli(conflict_home, conflict_one) + two = start_cli(conflict_home, conflict_two) + conflict_results = [finish_process(one), finish_process(two)] + conflict_inspect = cli( + conflict_home, + "experiment", + "inspect", + "first-experiment", + ) + check( + "IST-CONC-002", + sorted(item.returncode for item in conflict_results) == [0, 1] + and conflict_inspect.returncode == 0 + and not tuple((conflict_home / "experiments" / ".staging").iterdir()), + "concurrent different candidates have one winner and one ordinary conflict", + ( + f"rcs={[item.returncode for item in conflict_results]!r} " + f"inspect={conflict_inspect.returncode}" + ), + ) + + seam_home = new_home(root, "fault-seam-home") + observed_points: list[str] = [] + seam_rc, seam_value, seam_error = store_install( + seam_home, + direct_source, + fault=observed_points.append, + ) + crash_failures: list[str] = [] + for index, point in enumerate( + ( + "experiment artifact.after_write", + "experiment receipt.after_fsync", + "experiment envelope.after_noreplace", + "experiment store root.after_fsync", + ) + ): + crash_home = new_home(root, f"crash-home-{index}") + child_rc = hard_exit_install(crash_home, direct_source, point) + before_inspect_stage = fingerprint(crash_home / "experiments" / ".staging") + inspected = cli(crash_home, "experiment", "inspect", "first-experiment") + after_inspect_stage = fingerprint(crash_home / "experiments" / ".staging") + retried = cli(crash_home, "experiment", "install", str(direct_source)) + retry_value = json_object(retried) + committed_point = point in { + "experiment envelope.after_noreplace", + "experiment store root.after_fsync", + } + expected_changed = not committed_point + if not ( + child_rc == 99 + and inspected.returncode == (0 if committed_point else 1) + and before_inspect_stage == after_inspect_stage + and retried.returncode == 0 + and isinstance(retry_value, dict) + and retry_value.get("changed") is expected_changed + and not tuple((crash_home / "experiments" / ".staging").iterdir()) + ): + crash_failures.append( + f"{point}:child={child_rc}:inspect={inspected.returncode}:" + f"retry={retried.returncode}/{retry_value!r}:" + f"read_changed={before_inspect_stage != after_inspect_stage}" + ) + + output_home = new_home(root, "result-output-home") + output_rc, _, output_error = module_main( + output_home, + ["experiment", "install", str(direct_source)], + BrokenOutput(), + ) + output_inspect = cli(output_home, "experiment", "inspect", "first-experiment") + output_retry = cli(output_home, "experiment", "install", str(direct_source)) + output_value = json_object(output_retry) + check( + "IST-CRASH-001", + seam_rc == 0 + and isinstance(seam_value, dict) + and seam_error is None + and set(FAULT_POINTS) <= set(observed_points) + and not crash_failures + and output_rc == 125 + and output_error is None + and output_inspect.returncode == 0 + and output_retry.returncode == 0 + and isinstance(output_value, dict) + and output_value.get("changed") is False, + "fault seams preserve views, restart cleanup, and recover uncertain output", + ( + f"seam={seam_rc}/{seam_error!r} " + f"missing={sorted(set(FAULT_POINTS)-set(observed_points))!r} " + f"crashes={crash_failures[:3]!r} output={output_rc}/{output_error!r}/" + f"{output_inspect.returncode}/{output_retry.returncode}/{output_value!r}" + ), + ) + + live_home = new_home(root, "selected-entry-live-home") + added = cli(live_home, "image", "add", "vendor.worker", SUBJECT) + added_value = json_object(added) + local_source = source_directory( + root, + "local-source", + catalog_name="vendor.worker", + ) + live_events: list[str] = [] + held_catalog = held_store = False + + def observe_locks(point: str) -> None: + nonlocal held_catalog, held_store + live_events.append(point) + if point == "experiment envelope.before_noreplace": + held_catalog = lock_is_blocked( + live_home / "state" / "locks" / "image-catalog.lock" + ) + held_store = lock_is_blocked( + live_home / "state" / "locks" / "experiments.lock" + ) + + live_rc, live_value, live_error = store_install( + live_home, + local_source, + fault=observe_locks, + ) + entry_digest = added_value.get("entryDigest") if isinstance(added_value, dict) else None + if isinstance(entry_digest, str): + removed = cli( + live_home, + "image", + "remove", + "vendor.worker", + "--expect", + entry_digest, + ) + else: + removed = subprocess.CompletedProcess([], 125, b"", b"missing entry digest") + store_before_retry = fingerprint(live_home / "experiments") + stale_retry = cli(live_home, "experiment", "install", str(local_source)) + store_after_retry = fingerprint(live_home / "experiments") + retained = cli(live_home, "experiment", "inspect", "first-experiment") + try: + catalog_index = live_events.index("experiment catalog lock.after_acquire") + store_index = live_events.index("experiment store lock.after_acquire") + publish_index = live_events.index("experiment envelope.before_noreplace") + order_ok = catalog_index < store_index < publish_index + except ValueError: + order_ok = False + check( + "IST-LIVE-001", + added.returncode == 0 + and live_rc == 0 + and isinstance(live_value, dict) + and live_error is None + and order_ok + and held_catalog + and held_store + and removed.returncode == 0 + and stale_retry.returncode == 1 + and store_before_retry == store_after_retry + and retained.returncode == 0, + "selected-entry and store locks remain held; removal blocks stale retry", + ( + f"add={added.returncode} install={live_rc}/{live_error!r} events={live_events!r} " + f"held={held_catalog}/{held_store} remove={removed.returncode} " + f"retry={stale_retry.returncode}/{store_before_retry != store_after_retry} " + f"inspect={retained.returncode}" + ), + ) + + platform_home = new_home(root, "platform-home") + platform_source = source_directory(root, "platform-source") + platform_before = fingerprint(platform_home) + source_touched = False + original_platform = MODULE.sys.platform + original_lstat = MODULE.os.lstat + original_listdir = MODULE.os.listdir + original_open = MODULE.os.open + + def touches_source(value: object) -> bool: + try: + rendered = os.fsdecode(os.fspath(value)) + except TypeError: + return False + base = str(platform_source) + return rendered == base or rendered.startswith(base + os.sep) + + def observed_lstat(path: object, *args, **kwargs): + nonlocal source_touched + source_touched = source_touched or touches_source(path) + return original_lstat(path, *args, **kwargs) + + def observed_listdir(path: object = "."): + nonlocal source_touched + source_touched = source_touched or touches_source(path) + return original_listdir(path) + + def observed_open(path: object, flags: int, mode: int = 0o777, *, dir_fd=None): + nonlocal source_touched + source_touched = source_touched or touches_source(path) + return original_open(path, flags, mode, dir_fd=dir_fd) + + MODULE.sys.platform = "darwin" + MODULE.os.lstat = observed_lstat + MODULE.os.listdir = observed_listdir + MODULE.os.open = observed_open + try: + platform_rc, _, platform_error = module_main( + platform_home, + ["experiment", "install", str(platform_source)], + ) + finally: + MODULE.os.open = original_open + MODULE.os.listdir = original_listdir + MODULE.os.lstat = original_lstat + MODULE.sys.platform = original_platform + platform_after = fingerprint(platform_home) + check( + "IST-PLAT-001", + platform_rc == 125 + and platform_error is None + and not source_touched + and platform_before == platform_after, + "non-Linux install fails before caller-source access or persistent effect", + ( + f"rc={platform_rc} error={platform_error!r} " + f"source_touched={source_touched} changed={platform_before != platform_after}" + ), + ) + + expected = [ + "IST-STATE-001", + "IST-LOCK-001", + "IST-STATE-002", + "IST-BOUND-001", + "IST-STATE-003", + "IST-CONC-001", + "IST-CONC-002", + "IST-CRASH-001", + "IST-LIVE-001", + "IST-PLAT-001", + ] + if OBSERVED != expected: + print(f"INFRA install state assertion identity drift: {OBSERVED!r}", file=sys.stderr) + return 125 + print(f"SUMMARY assertions=10 expected=10 failures={FAILURES} infra={INFRA}") + if INFRA: + return 125 + return 0 if FAILURES == 0 else 1 + + +if __name__ == "__main__": + try: + raise SystemExit(main()) + except SystemExit: + raise + except BaseException as error: + print(f"INFRA install state harness failed: {error!r}", file=sys.stderr) + print( + f"SUMMARY assertions={len(OBSERVED)} expected=10 failures={FAILURES} infra=1" + ) + raise SystemExit(125) diff --git a/tests/experiment/install-store-cases.sh b/tests/experiment/install-store-cases.sh new file mode 100755 index 0000000..fa66908 --- /dev/null +++ b/tests/experiment/install-store-cases.sh @@ -0,0 +1,505 @@ +#!/usr/bin/env bash +set -u -o pipefail + +repo_root="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." >/dev/null 2>&1 && pwd)" +agent_lab="$repo_root/scripts/agent-lab" +bounded_helper="$repo_root/tests/helpers/run-bounded.py" +fixture="$repo_root/tests/experiment/fixtures/directories/minimal" +runtime_manifest="$repo_root/packaging/agent-lab-local.manifest" +expected_count=11 +work="" +failures=0 +infrastructure=0 + +cleanup_work() { + local failed=0 + if [ -n "$work" ] && [ -e "$work" ]; then + find "$work" -type f -exec chmod u+rw {} + 2>/dev/null || failed=1 + find "$work" -depth -type d -exec chmod u+rwx {} + 2>/dev/null || failed=1 + find "$work" -type f -delete 2>/dev/null || failed=1 + find "$work" -type l -delete 2>/dev/null || failed=1 + find "$work" -depth -type d -exec rmdir {} + 2>/dev/null || failed=1 + [ ! -e "$work" ] || failed=1 + fi + return "$failed" +} + +finish() { + local assertions=0 + if [ -n "${observed:-}" ] && [ -f "$observed" ]; then + assertions="$(wc -l < "$observed")" + fi + if ! cleanup_work; then + infrastructure=1 + fi + trap - EXIT + printf 'SUMMARY assertions=%s expected=%s failures=%s infra=%s\n' \ + "$assertions" "$expected_count" "$failures" "$infrastructure" + if [ "$infrastructure" -ne 0 ]; then + exit 125 + fi + if [ "$failures" -ne 0 ]; then + exit 1 + fi + printf 'EXPERIMENT INSTALL STORE PASS\n' +} + +if ! work="$(mktemp -d)"; then + printf 'SUMMARY assertions=0 expected=%s failures=0 infra=1\n' "$expected_count" + exit 125 +fi +trap 'cleanup_work >/dev/null 2>&1 || true' EXIT +observed="$work/observed" +: > "$observed" + +if [ ! -x "$agent_lab" ] || [ ! -f "$bounded_helper" ] || [ ! -d "$fixture" ] \ + || [ ! -f "$runtime_manifest" ] || ! command -v jq >/dev/null 2>&1 \ + || ! command -v python3 >/dev/null 2>&1 \ + || [ ! -d "$repo_root/.cache/dev/tools/cue" ] \ + || [ ! -d "$repo_root/.cache/dev/tools/cedar" ]; then + printf 'INFRA install-store prerequisites are unavailable\n' >&2 + infrastructure=1 + finish +fi +if ! python3 -I -B "$bounded_helper" --self-test > "$work/bounded-self-test.out" 2> "$work/bounded-self-test.err"; then + printf 'INFRA bounded command helper self-test failed\n' >&2 + infrastructure=1 + finish +fi + +export AGENT_LAB_CUE_TOOL_DIR="$repo_root/.cache/dev/tools/cue" +export AGENT_LAB_CEDAR_TOOL_DIR="$repo_root/.cache/dev/tools/cedar" + +pass() { printf 'PASS %s %s\n' "$1" "$2"; printf '%s\n' "$1" >> "$observed"; } +fail() { printf 'FAIL %s %s\n' "$1" "$2"; printf '%s\n' "$1" >> "$observed"; failures=$((failures + 1)); } + +capture() { + local label="$1" + local status="$work/$label.status" + local status_line="" + shift + CAPTURE_OUT="$work/$label.out" + CAPTURE_ERR="$work/$label.err" + CAPTURE_RC=0 + find "$status" -delete 2>/dev/null || true + python3 -I -B "$bounded_helper" \ + --timeout 5 --status "$status" --stdout "$CAPTURE_OUT" --stderr "$CAPTURE_ERR" -- "$@" || CAPTURE_RC=$? + if [ -f "$status" ]; then + status_line="$(cat "$status")" + fi + if [ "$status_line" != "child:$CAPTURE_RC" ]; then + printf 'INFRA bounded command status is inconsistent: %s rc=%s status=%s\n' \ + "$label" "$CAPTURE_RC" "$status_line" >&2 + infrastructure=1 + CAPTURE_RC=125 + fi +} + +init_home() { + local home="$1" + capture "init-$(basename -- "$home")" "$agent_lab" --home "$home" init + if [ "$CAPTURE_RC" -ne 0 ]; then + printf 'INFRA temporary Agent Lab home initialization failed: %s\n' \ + "$(tr '\n' ' ' < "$CAPTURE_ERR")" >&2 + infrastructure=1 + finish + fi +} + +state_receipt() { + python3 -I -B - "$1" <<'PY' +from __future__ import annotations + +import hashlib +import json +import os +from pathlib import Path +import stat +import sys + +root = Path(sys.argv[1]) +if not os.path.lexists(root): + print("absent") + raise SystemExit(0) + +records: list[list[object]] = [] + +def visit(path: Path, relative: str) -> None: + metadata = path.lstat() + kind = "other" + content = "" + if stat.S_ISDIR(metadata.st_mode): + kind = "directory" + elif stat.S_ISREG(metadata.st_mode): + kind = "file" + descriptor = os.open(path, os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0)) + try: + chunks: list[bytes] = [] + remaining = metadata.st_size + 1 + while remaining: + chunk = os.read(descriptor, min(65_536, remaining)) + if not chunk: + break + chunks.append(chunk) + remaining -= len(chunk) + content = hashlib.sha256(b"".join(chunks)).hexdigest() + finally: + os.close(descriptor) + elif stat.S_ISLNK(metadata.st_mode): + kind = "symlink" + content = os.readlink(path) + records.append([ + relative, + kind, + stat.S_IMODE(metadata.st_mode), + metadata.st_uid, + metadata.st_gid, + metadata.st_nlink, + metadata.st_dev, + metadata.st_ino, + metadata.st_size, + metadata.st_mtime_ns, + metadata.st_ctime_ns, + content, + ]) + if kind == "directory" and relative != "cache": + for child in sorted(path.iterdir(), key=lambda item: os.fsencode(item.name)): + child_relative = child.name if relative == "." else f"{relative}/{child.name}" + visit(child, child_relative) + +visit(root, ".") +encoded = json.dumps(records, ensure_ascii=True, separators=(",", ":")).encode("ascii") +print("sha256:" + hashlib.sha256(encoded).hexdigest()) +PY +} + +verify_install_envelope() { + python3 -I -B - "$1" "$2" "$3" "$4" "$5" "$6" <<'PY' +from __future__ import annotations + +import hashlib +import json +import os +from pathlib import Path +import stat +import sys + +home = Path(sys.argv[1]) +name = sys.argv[2] +source = Path(sys.argv[3]) +checked_path = Path(sys.argv[4]) +decision_path = Path(sys.argv[5]) +result_path = Path(sys.argv[6]) +target = home / "experiments" / name + +def canonical(value: object) -> bytes: + return json.dumps(value, ensure_ascii=True, separators=(",", ":"), sort_keys=True).encode("ascii") + +def digest(data: bytes) -> str: + return "sha256:" + hashlib.sha256(data).hexdigest() + +def strings(value: object) -> set[str]: + found: set[str] = set() + if isinstance(value, str): + found.add(value) + elif isinstance(value, dict): + for key, item in value.items(): + found.add(key) + found.update(strings(item)) + elif isinstance(value, list): + for item in value: + found.update(strings(item)) + return found + +expected_paths = { + "artifact", + "artifact/experiment.cue", + "records", + "records/decision.json", + "records/install.json", + "records/plan.json", + "records/provenance.json", +} +actual_paths = { + str(path.relative_to(target)) + for path in target.rglob("*") +} +assert actual_paths == expected_paths +for relative in (".", "artifact", "records"): + path = target if relative == "." else target / relative + metadata = path.lstat() + assert stat.S_ISDIR(metadata.st_mode) + assert stat.S_IMODE(metadata.st_mode) == 0o500 + assert metadata.st_uid == os.getuid() +for relative in expected_paths - {"artifact", "records"}: + metadata = (target / relative).lstat() + assert stat.S_ISREG(metadata.st_mode) + assert stat.S_IMODE(metadata.st_mode) == 0o400 + assert metadata.st_uid == os.getuid() and metadata.st_nlink == 1 + +artifact_bytes = (target / "artifact/experiment.cue").read_bytes() +assert artifact_bytes == (source / "experiment.cue").read_bytes() +checked = json.loads(checked_path.read_bytes()) +expected_plan_bytes = canonical(checked["plan"]) + b"\n" +plan_bytes = (target / "records/plan.json").read_bytes() +decision_bytes = (target / "records/decision.json").read_bytes() +provenance_bytes = (target / "records/provenance.json").read_bytes() +receipt_bytes = (target / "records/install.json").read_bytes() +assert plan_bytes == expected_plan_bytes +assert decision_bytes == decision_path.read_bytes() + +plan = json.loads(plan_bytes) +decision = json.loads(decision_bytes) +provenance = json.loads(provenance_bytes) +receipt = json.loads(receipt_bytes) +for raw, value in ((plan_bytes, plan), (decision_bytes, decision), (provenance_bytes, provenance), (receipt_bytes, receipt)): + assert raw == canonical(value) + b"\n" +assert decision["verdict"] == "permit" +assert decision["binding"]["planDigest"] == digest(canonical(plan)) +assert isinstance(provenance, dict) and isinstance(provenance.get("apiVersion"), str) +assert not any(item.startswith("/") for item in strings(provenance)) + +receipt_strings = strings(receipt) +record_digests = { + digest(artifact_bytes), + digest(plan_bytes), + digest(decision_bytes), + digest(provenance_bytes), +} +assert record_digests <= receipt_strings +assert {plan["apiVersion"], decision["apiVersion"], provenance["apiVersion"], "agent-lab/v0alpha1"} <= receipt_strings +installation_key = receipt.get("installationKey") +assert isinstance(installation_key, str) and len(installation_key) == 71 and installation_key.startswith("sha256:") + +result_bytes = result_path.read_bytes() +result = json.loads(result_bytes) +assert result_bytes == canonical(result) + b"\n" +assert result.get("changed") is True +assert result.get("name") == name +assert result.get("installationKey") == installation_key +assert result.get("receiptDigest") == digest(receipt_bytes) +PY +} + +missing_home="$work/missing-home" +missing_before="$(state_receipt "$missing_home")" || infrastructure=1 +capture missing-install "$agent_lab" --home "$missing_home" experiment install "$fixture" +missing_after="$(state_receipt "$missing_home")" || infrastructure=1 +if [ "$CAPTURE_RC" -eq 1 ] && [ ! -s "$CAPTURE_OUT" ] \ + && [ "$missing_before" = absent ] && [ "$missing_after" = absent ]; then + pass INST-HOME-001 "install requires an initialized home without creating one" +else + fail INST-HOME-001 "install requires an initialized home without creating one" +fi + +core_home="$work/core-home" +init_home "$core_home" +unknown_before="$(state_receipt "$core_home")" || infrastructure=1 +capture unknown-inspect "$agent_lab" --home "$core_home" experiment inspect missing-experiment +unknown_after="$(state_receipt "$core_home")" || infrastructure=1 +if [ "$CAPTURE_RC" -eq 1 ] && [ ! -s "$CAPTURE_OUT" ] \ + && [ "$unknown_before" = "$unknown_after" ]; then + pass INST-UNKNOWN-001 "inspect reports an unknown name without mutating the initialized home" +else + fail INST-UNKNOWN-001 "inspect reports an unknown name without mutating the initialized home" +fi + +capture core-check "$agent_lab" --home "$core_home" experiment check "$fixture" +check_out="$CAPTURE_OUT" +check_rc="$CAPTURE_RC" +capture core-decision "$agent_lab" --home "$core_home" experiment authorize install "$fixture" +decision_out="$CAPTURE_OUT" +decision_rc="$CAPTURE_RC" +if [ "$check_rc" -ne 0 ] || [ "$decision_rc" -ne 0 ]; then + printf 'INFRA prerequisite check/authorization failed for the canonical fixture\n' >&2 + infrastructure=1 + finish +fi +capture core-install "$agent_lab" --home "$core_home" experiment install "$fixture" +core_result="$CAPTURE_OUT" +core_install_rc="$CAPTURE_RC" +if [ "$core_install_rc" -eq 0 ] && [ ! -s "$CAPTURE_ERR" ] \ + && jq -e '.changed == true and .name == "first-experiment" and (.installationKey | startswith("sha256:")) and (.receiptDigest | startswith("sha256:"))' \ + "$core_result" >/dev/null 2>&1; then + pass INST-PERMIT-001 "a fresh permit publishes one named installation and canonical identity result" +else + fail INST-PERMIT-001 "a fresh permit publishes one named installation and canonical identity result" +fi + +verify_install_envelope \ + "$core_home" first-experiment "$fixture" "$check_out" "$decision_out" "$core_result" \ + > "$work/verify-envelope.out" 2> "$work/verify-envelope.err" +verify_rc=$? +if [ "$verify_rc" -eq 0 ]; then + pass INST-RECEIPT-001 "stored exact bytes, modes, schemas, and independent hashes are receipt-bound" +else + fail INST-RECEIPT-001 "stored exact bytes, modes, schemas, and independent hashes are receipt-bound" +fi + +core_before_inspect="$(state_receipt "$core_home")" || infrastructure=1 +capture core-inspect "$agent_lab" --home "$core_home" experiment inspect first-experiment +core_inspect_rc="$CAPTURE_RC" +core_after_inspect="$(state_receipt "$core_home")" || infrastructure=1 +core_key="$(jq -r '.installationKey // empty' "$core_result" 2>/dev/null)" +core_receipt="$(jq -r '.receiptDigest // empty' "$core_result" 2>/dev/null)" +if [ "$core_inspect_rc" -eq 0 ] && [ ! -s "$CAPTURE_ERR" ] \ + && jq -e --arg key "$core_key" --arg receipt "$core_receipt" \ + '.name == "first-experiment" and .state == "installed" and .installationKey == $key and .receiptDigest == $receipt' \ + "$CAPTURE_OUT" >/dev/null 2>&1 \ + && [ "$core_before_inspect" = "$core_after_inspect" ]; then + pass INST-INSPECT-001 "inspect read-only verifies and reports the exact installed identity" +else + fail INST-INSPECT-001 "inspect read-only verifies and reports the exact installed identity" +fi + +core_before_retry="$(state_receipt "$core_home")" || infrastructure=1 +capture core-retry "$agent_lab" --home "$core_home" experiment install "$fixture" +core_after_retry="$(state_receipt "$core_home")" || infrastructure=1 +if [ "$CAPTURE_RC" -eq 0 ] && [ ! -s "$CAPTURE_ERR" ] \ + && jq -e --arg key "$core_key" --arg receipt "$core_receipt" \ + '.changed == false and .name == "first-experiment" and .installationKey == $key and .receiptDigest == $receipt' \ + "$CAPTURE_OUT" >/dev/null 2>&1 \ + && [ "$core_before_retry" = "$core_after_retry" ]; then + pass INST-RETRY-001 "exact retry returns changed false without rewriting the verified envelope" +else + fail INST-RETRY-001 "exact retry returns changed false without rewriting the verified envelope" +fi + +conflict_source="$work/conflict-source" +mkdir "$conflict_source" +sed 's/sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb/' \ + "$fixture/experiment.cue" > "$conflict_source/experiment.cue" +core_before_conflict="$(state_receipt "$core_home")" || infrastructure=1 +capture core-conflict "$agent_lab" --home "$core_home" experiment install "$conflict_source" +core_after_conflict="$(state_receipt "$core_home")" || infrastructure=1 +if [ "$CAPTURE_RC" -eq 1 ] && [ ! -s "$CAPTURE_OUT" ] \ + && [ "$core_before_conflict" = "$core_after_conflict" ]; then + pass INST-CONFLICT-001 "same name with a different installation identity never overwrites" +else + fail INST-CONFLICT-001 "same name with a different installation identity never overwrites" +fi + +effect_home="$work/effect-home" +init_home "$effect_home" +runtime_replica="$work/deny-runtime" +mkdir -p "$runtime_replica" +replica_ok=1 +while IFS= read -r runtime_name; do + if [ -z "$runtime_name" ] || [ ! -f "$repo_root/$runtime_name" ]; then + replica_ok=0 + continue + fi + mkdir -p "$runtime_replica/$(dirname -- "$runtime_name")" + cp "$repo_root/$runtime_name" "$runtime_replica/$runtime_name" || replica_ok=0 +done < "$runtime_manifest" +chmod 700 "$runtime_replica/scripts/agent-lab" 2>/dev/null || replica_ok=0 +if [ "$replica_ok" -eq 1 ]; then + sed 's/^permit (/forbid (/' "$repo_root/authorization/experiment/v0alpha1/operator.cedar" \ + > "$runtime_replica/authorization/experiment/v0alpha1/operator.cedar" +fi +capture deny-preview "$runtime_replica/scripts/agent-lab" --home "$effect_home" experiment authorize install "$fixture" +deny_preview_rc="$CAPTURE_RC" +deny_preview_out="$CAPTURE_OUT" +effect_before_deny="$(state_receipt "$effect_home")" || infrastructure=1 +capture deny-install "$runtime_replica/scripts/agent-lab" --home "$effect_home" experiment install "$fixture" +effect_after_deny="$(state_receipt "$effect_home")" || infrastructure=1 +if [ "$replica_ok" -eq 1 ] && [ "$deny_preview_rc" -eq 1 ] \ + && jq -e '.verdict == "deny"' "$deny_preview_out" >/dev/null 2>&1 \ + && [ "$CAPTURE_RC" -eq 1 ] && [ ! -s "$CAPTURE_OUT" ] \ + && [ "$effect_before_deny" = "$effect_after_deny" ]; then + pass INST-DENY-001 "a freshly evaluated Cedar denial leaves store and staging unchanged" +else + fail INST-DENY-001 "a freshly evaluated Cedar denial leaves store and staging unchanged" +fi + +forged="$work/forged-permit.json" +printf '%s\n' '{"verdict":"permit","installationKey":"sha256:ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"}' > "$forged" +forged_source="$work/forged-source" +mkdir "$forged_source" +cp "$fixture/experiment.cue" "$forged_source/experiment.cue" +cp "$forged" "$forged_source/decision.json" +effect_before_forge="$(state_receipt "$effect_home")" || infrastructure=1 +capture forged-option "$agent_lab" --home "$effect_home" experiment install "$fixture" --decision "$forged" +forged_option_rc="$CAPTURE_RC" +capture forged-source "$agent_lab" --home "$effect_home" experiment install "$forged_source" +forged_source_rc="$CAPTURE_RC" +effect_after_forge="$(state_receipt "$effect_home")" || infrastructure=1 +if [ "$forged_option_rc" -eq 2 ] && [ "$forged_source_rc" -eq 1 ] \ + && [ "$effect_before_forge" = "$effect_after_forge" ]; then + pass INST-FORGE-001 "caller-supplied permit data is neither an option nor accepted source authority" +else + fail INST-FORGE-001 "caller-supplied permit data is neither an option nor accepted source authority" +fi + +mkdir "$effect_home/images/catalog" +printf '%s\n' '{"corrupt":"local-catalog-must-not-be-opened"}' > "$effect_home/images/catalog/current.json" +chmod 600 "$effect_home/images/catalog/current.json" +canary_bin="$work/canary-bin" +canary_marks="$work/canary-marks" +mkdir "$canary_bin" "$canary_marks" +for command in docker git curl wget zip unzip buildah podman skopeo oras; do + printf '%s\n' '#!/bin/sh' 'set -eu' ': > "$CANARY_DIR/${0##*/}"' 'exit 97' > "$canary_bin/$command" + chmod 700 "$canary_bin/$command" + CANARY_DIR="$canary_marks" "$canary_bin/$command" >/dev/null 2>&1 || true +done +calibrated="$(find "$canary_marks" -type f | wc -l)" +find "$canary_marks" -type f -delete +catalog_before="$(state_receipt "$effect_home/images")" || infrastructure=1 +capture canary-install env -i \ + PATH="$canary_bin:/usr/bin:/bin" LANG=C LC_ALL=C CANARY_DIR="$canary_marks" \ + AGENT_LAB_CUE_TOOL_DIR="$AGENT_LAB_CUE_TOOL_DIR" \ + AGENT_LAB_CEDAR_TOOL_DIR="$AGENT_LAB_CEDAR_TOOL_DIR" \ + "$agent_lab" --home "$effect_home" experiment install "$fixture" +catalog_after="$(state_receipt "$effect_home/images")" || infrastructure=1 +if [ "$calibrated" -eq 10 ] && [ "$CAPTURE_RC" -eq 0 ] \ + && [ -z "$(find "$canary_marks" -type f -print -quit)" ] \ + && [ "$catalog_before" = "$catalog_after" ]; then + pass INST-NOEF-001 "direct installation ignores local catalog state and invokes no forbidden effect command" +else + fail INST-NOEF-001 "direct installation ignores local catalog state and invokes no forbidden effect command" +fi + +local_home="$work/local-home" +init_home "$local_home" +subject="registry.example/operator/worker@sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc" +capture local-add "$agent_lab" --home "$local_home" image add vendor.worker "$subject" +local_entry="$(jq -r '.entryDigest // empty' "$CAPTURE_OUT" 2>/dev/null)" +local_source="$work/local-source" +mkdir "$local_source" +sed 's#digestRef: "registry.example/team/coordinator@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"#catalogName: "vendor.worker"#' \ + "$fixture/experiment.cue" > "$local_source/experiment.cue" +capture local-install "$agent_lab" --home "$local_home" experiment install "$local_source" +local_install_rc="$CAPTURE_RC" +local_result="$CAPTURE_OUT" +local_key="$(jq -r '.installationKey // empty' "$local_result" 2>/dev/null)" +local_receipt="$(jq -r '.receiptDigest // empty' "$local_result" 2>/dev/null)" +capture local-remove "$agent_lab" --home "$local_home" image remove vendor.worker --expect "$local_entry" +local_remove_rc="$CAPTURE_RC" +local_before_retry="$(state_receipt "$local_home")" || infrastructure=1 +capture local-retry "$agent_lab" --home "$local_home" experiment install "$local_source" +local_retry_rc="$CAPTURE_RC" +capture local-inspect "$agent_lab" --home "$local_home" experiment inspect first-experiment +local_inspect_rc="$CAPTURE_RC" +local_after_retry="$(state_receipt "$local_home")" || infrastructure=1 +if [ "$local_install_rc" -eq 0 ] && [ "$local_remove_rc" -eq 0 ] \ + && [ "$local_retry_rc" -eq 1 ] && [ "$local_inspect_rc" -eq 0 ] \ + && jq -e --arg key "$local_key" --arg receipt "$local_receipt" \ + '.state == "installed" and .installationKey == $key and .receiptDigest == $receipt' \ + "$CAPTURE_OUT" >/dev/null 2>&1 \ + && [ "$local_before_retry" = "$local_after_retry" ]; then + pass INST-LOCAL-001 "removed local selector blocks retry while retained installation remains inspectable" +else + fail INST-LOCAL-001 "removed local selector blocks retry while retained installation remains inspectable" +fi + +expected="$work/expected" +printf '%s\n' \ + INST-HOME-001 INST-UNKNOWN-001 INST-PERMIT-001 INST-RECEIPT-001 \ + INST-INSPECT-001 INST-RETRY-001 INST-CONFLICT-001 INST-DENY-001 \ + INST-FORGE-001 INST-NOEF-001 INST-LOCAL-001 > "$expected" +if ! cmp -s "$expected" "$observed"; then + printf 'INFRA install-store assertion identity drift\n' >&2 + infrastructure=1 +fi + +finish diff --git a/tests/experiment/local-lifecycle-cases.sh b/tests/experiment/local-lifecycle-cases.sh index 41460a8..5ec43d0 100755 --- a/tests/experiment/local-lifecycle-cases.sh +++ b/tests/experiment/local-lifecycle-cases.sh @@ -6,8 +6,10 @@ subcases=( "$repo_root/tests/install/local-install-cases.sh" "$repo_root/tests/experiment/local-config-cases.sh" "$repo_root/tests/experiment/local-image-catalog-cases.sh" + "$repo_root/tests/experiment/install-store-cases.sh" + "$repo_root/tests/experiment/install-state-cases.py" ) -expected_count=86 +expected_count=107 work="" cleanup_work() { @@ -48,7 +50,12 @@ printf '%s\n' \ RES-ISOLATE-001 RES-STATE-001 RES-STATE-002 RES-STATE-003 RES-INPUT-001 \ RES-SNAP-003 RES-AUTH-001 RES-INSTALL-001 RES-NOEF-001 \ M-CAT-OCI-001 M-CAT-SHADOW-001 M-CAT-CAS-001 M-CAT-AUTH-001 M-RES-BIND-001 \ - M-CAT-NOEF-001 M-CAT-ADMIT-001 M-CAT-ATOM-001 M-CAT-DUR-001 M-CAT-STAGE-001 > "$expected" + M-CAT-NOEF-001 M-CAT-ADMIT-001 M-CAT-ATOM-001 M-CAT-DUR-001 M-CAT-STAGE-001 \ + INST-HOME-001 INST-UNKNOWN-001 INST-PERMIT-001 INST-RECEIPT-001 \ + INST-INSPECT-001 INST-RETRY-001 INST-CONFLICT-001 INST-DENY-001 \ + INST-FORGE-001 INST-NOEF-001 INST-LOCAL-001 \ + IST-STATE-001 IST-LOCK-001 IST-STATE-002 IST-BOUND-001 IST-STATE-003 \ + IST-CONC-001 IST-CONC-002 IST-CRASH-001 IST-LIVE-001 IST-PLAT-001 > "$expected" : > "$observed" infrastructure=0 @@ -59,11 +66,22 @@ for index in "${!subcases[@]}"; do infrastructure=1 continue fi - if bash "$subcase" > "$output" 2>&1; then - rc=0 - else - rc=$? - fi + case "$subcase" in + *.py) + if python3 -I -B "$subcase" > "$output" 2>&1; then + rc=0 + else + rc=$? + fi + ;; + *) + if bash "$subcase" > "$output" 2>&1; then + rc=0 + else + rc=$? + fi + ;; + esac awk '/^(PASS|FAIL) [A-Z0-9-]+ / {print}' "$output" awk '/^(PASS|FAIL) [A-Z0-9-]+ / {print $2}' "$output" >> "$observed" reported_assertions="$(awk '/^(PASS|FAIL) [A-Z0-9-]+ / {count++} END {print count + 0}' "$output")" From d03e4468b9984dcbece6d2261f6cf3c6f97cc168 Mon Sep 17 00:00:00 2001 From: xormania <127287135+xormania@users.noreply.github.com> Date: Sun, 2 Aug 2026 02:27:45 -0400 Subject: [PATCH 045/158] feat(experiment): add permit-gated store --- packaging/agent-lab-local.manifest | 1 + scripts/agent-lab.py | 112 +- scripts/experiment_store.py | 1531 +++++++++++++++++ scripts/image_catalog.py | 75 + .../fixtures/expected-runtime-files.txt | 1 + 5 files changed, 1714 insertions(+), 6 deletions(-) create mode 100644 scripts/experiment_store.py diff --git a/packaging/agent-lab-local.manifest b/packaging/agent-lab-local.manifest index c3bfbd1..b62cbce 100644 --- a/packaging/agent-lab-local.manifest +++ b/packaging/agent-lab-local.manifest @@ -9,6 +9,7 @@ scripts/agent-lab.py scripts/dev/cedar-tool.py scripts/dev/cue-tool.py scripts/experiment.py +scripts/experiment_store.py scripts/image_catalog.py scripts/image_reference.py tools/cedar.lock diff --git a/scripts/agent-lab.py b/scripts/agent-lab.py index 940dd62..23f1f97 100644 --- a/scripts/agent-lab.py +++ b/scripts/agent-lab.py @@ -71,8 +71,8 @@ def lock_record(path: Path, relative: str, schema: str) -> dict[str, object]: } -def verify_lock(home: Path, state_component: str, key: str, record: object) -> None: - filename, schema, appended = LOCK_SPECS[key] +def validate_lock_receipt(state_component: str, key: str, record: object) -> None: + filename, schema, _ = LOCK_SPECS[key] relative = f"{state_component}/locks/{filename}" if ( not isinstance(record, dict) @@ -85,6 +85,13 @@ def verify_lock(home: Path, state_component: str, key: str, record: object) -> N or record.get("schema") != schema ): raise RuntimeError("home lock receipt is not closed") + + +def verify_lock(home: Path, state_component: str, key: str, record: object) -> None: + filename, schema, appended = LOCK_SPECS[key] + relative = f"{state_component}/locks/{filename}" + validate_lock_receipt(state_component, key, record) + assert isinstance(record, dict) path = home / relative maximum = len(schema.encode("ascii") + b"\n" + appended) try: @@ -235,7 +242,9 @@ def init_home(home: Path, argv: list[str]) -> int: return 0 -def load_config(home: Path) -> tuple[dict[str, object], bytes] | None: +def load_config_receipt( + home: Path, +) -> tuple[dict[str, object], bytes, dict[str, object]] | None: config_path = home / "config.json" receipt_path = home / "home.json" try: @@ -253,7 +262,7 @@ def load_config(home: Path) -> tuple[dict[str, object], bytes] | None: receipt_raw = receipt_path.read_bytes() value = json.loads(raw.decode("utf-8")) receipt = json.loads(receipt_raw.decode("utf-8")) - except (OSError, UnicodeError, json.JSONDecodeError): + except (OSError, RecursionError, UnicodeError, ValueError): raise RuntimeError("configuration is malformed") if not isinstance(value, dict) or set(value) != {"apiVersion", "paths"} or value["apiVersion"] != "agent-lab.config/v0alpha1": raise RuntimeError("configuration is not closed") @@ -262,7 +271,11 @@ def load_config(home: Path) -> tuple[dict[str, object], bytes] | None: raise RuntimeError("configuration paths are not closed") if len(set(paths.values())) != 4 or any(not isinstance(item, str) or not SAFE_COMPONENT.fullmatch(item) for item in paths.values()): raise RuntimeError("configuration paths are unsafe") - canonical_config = canonical(value) + b"\n" + try: + canonical_config = canonical(value) + b"\n" + canonical_receipt = canonical(receipt) + b"\n" + except (RecursionError, TypeError, ValueError, UnicodeError) as error: + raise RuntimeError("configuration is malformed") from error if raw != canonical_config: raise RuntimeError("configuration is not canonical") if ( @@ -271,7 +284,7 @@ def load_config(home: Path) -> tuple[dict[str, object], bytes] | None: or receipt["apiVersion"] != "agent-lab.home/v0alpha1" or receipt["paths"] != paths or receipt["configDigest"] != "sha256:" + hashlib.sha256(canonical(value)).hexdigest() - or receipt_raw != canonical(receipt) + b"\n" + or receipt_raw != canonical_receipt ): raise RuntimeError("configuration does not match the initialized home receipt") locks = receipt["locks"] @@ -279,6 +292,21 @@ def load_config(home: Path) -> tuple[dict[str, object], bytes] | None: raise RuntimeError("home lock receipt is not closed") state_component = paths["state"] assert isinstance(state_component, str) + for key in LOCK_SPECS: + validate_lock_receipt(state_component, key, locks[key]) + return value, canonical_config, receipt + + +def load_config(home: Path) -> tuple[dict[str, object], bytes] | None: + loaded = load_config_receipt(home) + if loaded is None: + return None + value, canonical_config, receipt = loaded + paths = value["paths"] + locks = receipt["locks"] + assert isinstance(paths, dict) and isinstance(locks, dict) + state_component = paths["state"] + assert isinstance(state_component, str) for key in LOCK_SPECS: verify_lock(home, state_component, key, locks[key]) return value, canonical_config @@ -305,6 +333,17 @@ def image_catalog_module(): return module +def experiment_store_module(): + path = Path(__file__).resolve().with_name("experiment_store.py") + spec = spec_from_file_location("agent_lab_experiment_store", path) + if spec is None or spec.loader is None: + raise ImportError("Experiment store module cannot be loaded") + module = module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + def write_json(value: object) -> None: output = canonical(value).decode("ascii") + "\n" written = sys.stdout.write(output) @@ -362,6 +401,65 @@ def image_command(home: Path, argv: list[str]) -> int: return 0 +def experiment_command(home: Path, argv: list[str]) -> int: + if argv[:1] == ["install"] and len(argv) == 2: + operation = "install" + elif argv[:1] == ["inspect"] and len(argv) == 2: + operation = "inspect" + else: + return 2 + + try: + loaded = load_config_receipt(home) + except RuntimeError as error: + print(f"INFRA Agent Lab {error}", file=sys.stderr) + return 125 + if loaded is None: + print("FAIL Agent Lab home is not initialized", file=sys.stderr) + return 1 + + if operation == "install" and sys.platform != "linux": + print("INFRA Agent Lab Experiment installation requires Linux", file=sys.stderr) + return 125 + + paths = loaded[0]["paths"] + assert isinstance(paths, dict) + cache = home / str(paths["cache"]) / "tools" + os.environ["AGENT_LAB_HOME"] = str(home) + os.environ.setdefault("AGENT_LAB_CUE_TOOL_DIR", str(cache / "cue")) + os.environ.setdefault("AGENT_LAB_CEDAR_TOOL_DIR", str(cache / "cedar")) + + try: + store = experiment_store_module() + except Exception as error: + print(f"INFRA Agent Lab Experiment store is unavailable: {error}", file=sys.stderr) + return 125 + + try: + if operation == "install": + result = store.install_directory(home, Path(argv[1])) + else: + result = store.inspect_install(home, argv[1]) + if not isinstance(result, dict): + raise TypeError("Experiment store returned a non-object result") + except store.StoreReject as error: + print(f"FAIL Experiment {error}", file=sys.stderr) + return 1 + except store.StoreInfrastructure as error: + print(f"INFRA Agent Lab Experiment store {error}", file=sys.stderr) + return 125 + except Exception as error: + print(f"INFRA Agent Lab Experiment store operation is unavailable: {error}", file=sys.stderr) + return 125 + + try: + write_json(result) + except Exception as error: + print(f"INFRA Agent Lab Experiment store result is uncertain: {error}", file=sys.stderr) + return 125 + return 0 + + def main(argv: list[str]) -> int: home_raw = None if argv[:1] == ["--home"]: @@ -431,6 +529,8 @@ def main(argv: list[str]) -> int: os.environ.setdefault("AGENT_LAB_CUE_TOOL_DIR", str(home / "cache/tools/cue")) os.environ.setdefault("AGENT_LAB_CEDAR_TOOL_DIR", str(home / "cache/tools/cedar")) return experiment_module().main(["experiment.py", "authorize-directory", argv[3]]) + if argv[:1] == ["experiment"] and argv[1:2] in (["install"], ["inspect"]): + return experiment_command(home, argv[1:]) if argv[:1] == ["image"]: return image_command(home, argv[1:]) print("Usage: agent-lab [--home ABSOLUTE_HOME] {version|init|config|experiment|image}", file=sys.stderr) diff --git a/scripts/experiment_store.py b/scripts/experiment_store.py new file mode 100644 index 0000000..1a5e238 --- /dev/null +++ b/scripts/experiment_store.py @@ -0,0 +1,1531 @@ +#!/usr/bin/env python3 +"""Durable, permit-gated publication of verified Experiment envelopes.""" + +from __future__ import annotations + +from contextlib import contextmanager, nullcontext +import ctypes +import errno +import fcntl +import hashlib +from importlib.util import module_from_spec, spec_from_file_location +import json +import math +import os +from pathlib import Path +import re +import stat +import sys +from typing import Callable, Iterator, NamedTuple, NoReturn, Sequence + + +INSTALL_KEY_DOMAIN = b"agent-lab.experiment-installation-key.v1\0" +SOURCE_DIGEST_DOMAIN = b"agent-lab.experiment-tree.v1\0" +STAGE_PAYLOAD_DOMAIN = b"agent-lab.experiment-stage-payload.v1\0" +INSTALL_API = "agent-lab.experiment-install/v0alpha1" +PROVENANCE_API = "agent-lab.experiment-provenance/v0alpha1" +INTENT_API = "agent-lab.experiment-install-intent/v0alpha1" +LOCK_SCHEMA = "agent-lab.experiments-lock/v0alpha1" +OPERATION_WRAPPER = "experiment-install" +CLEANUP_WRAPPER = "experiment-install-cleanup" +MAX_STAGE_ENTRIES = 16 +MAX_STAGE_BYTES = 4_194_304 +MAX_AUTHORITY_BYTES = 65_536 +MAX_ARTIFACT_BYTES = 262_144 +MAX_RECORD_BYTES = 1_048_576 +SAFE_COMPONENT = re.compile(r"^[a-z][a-z0-9-]{0,47}$") +IMAGE_COMPONENT = re.compile(r"^[a-z][a-z0-9]*(?:-[a-z0-9]+)*$") +SHA256 = re.compile(r"^sha256:[0-9a-f]{64}$") +PAYLOAD_DIRECTORIES = {"payload", "payload/artifact", "payload/records"} +PAYLOAD_FILES = { + "payload/artifact/experiment.cue", + "payload/records/decision.json", + "payload/records/install.json", + "payload/records/plan.json", + "payload/records/provenance.json", +} +STAGE_ALLOWED = {"intent.json", *PAYLOAD_DIRECTORIES, *PAYLOAD_FILES} +RECORD_PATHS = { + "artifact/experiment.cue", + "records/decision.json", + "records/plan.json", + "records/provenance.json", +} + +FaultHook = Callable[[str], None] + + +class StoreError(Exception): + """Base class for classified Experiment-store failures.""" + + +class StoreReject(StoreError): + """The requested operation is a safe ordinary rejection.""" + + +class StoreInfrastructure(StoreError): + """The store could not establish a trustworthy result.""" + + +class DuplicateKey(ValueError): + """A JSON object contains a duplicate decoded key.""" + + +class HomeAuthority(NamedTuple): + home: Path + store: Path + staging: Path + state: Path + locks: Path + lock: Path + lock_device: int + lock_inode: int + config_raw: bytes + receipt_raw: bytes + store_device: int + + +class VerifiedInstall(NamedTuple): + name: str + installation_key: str + receipt_digest: str + file_digests: dict[str, str] + + +def canonical(value: object) -> bytes: + try: + return json.dumps( + value, + allow_nan=False, + ensure_ascii=True, + separators=(",", ":"), + sort_keys=True, + ).encode("ascii") + except (RecursionError, TypeError, ValueError, UnicodeError) as error: + raise StoreInfrastructure("store data cannot be encoded canonically") from error + + +def digest(data: bytes) -> str: + return "sha256:" + hashlib.sha256(data).hexdigest() + + +def _reject(message: str) -> NoReturn: + raise StoreReject(message) + + +def _infra(message: str, error: BaseException | None = None) -> NoReturn: + if error is None: + raise StoreInfrastructure(message) + raise StoreInfrastructure(message) from error + + +def _fault(hook: FaultHook | None, point: str) -> None: + if hook is not None: + hook(point) + + +def _pairs(pairs: list[tuple[str, object]]) -> dict[str, object]: + result: dict[str, object] = {} + for key, value in pairs: + if key in result: + raise DuplicateKey(key) + result[key] = value + return result + + +def _nonfinite(value: str) -> NoReturn: + raise ValueError(value) + + +def _parse_object(data: bytes, purpose: str) -> dict[str, object]: + try: + text = data.decode("ascii") + value = json.loads( + text, + object_pairs_hook=_pairs, + parse_constant=_nonfinite, + ) + _reject_nonfinite(value) + except (DuplicateKey, UnicodeError, json.JSONDecodeError, RecursionError, ValueError) as error: + _infra(f"{purpose} is not bounded canonical JSON", error) + if not isinstance(value, dict): + _infra(f"{purpose} is not one JSON object") + if data != canonical(value) + b"\n": + _infra(f"{purpose} is not canonical") + return value + + +def _reject_nonfinite(value: object) -> None: + if isinstance(value, float) and not math.isfinite(value): + raise ValueError("non-finite number") + if isinstance(value, dict): + for item in value.values(): + _reject_nonfinite(item) + elif isinstance(value, list): + for item in value: + _reject_nonfinite(item) + + +def _identity(metadata: os.stat_result) -> tuple[int, ...]: + return ( + metadata.st_dev, + metadata.st_ino, + metadata.st_mode, + metadata.st_uid, + metadata.st_nlink, + metadata.st_size, + metadata.st_mtime_ns, + metadata.st_ctime_ns, + ) + + +def _absolute_directory_chain(path: Path) -> None: + if not path.is_absolute() or path == Path("/") or os.path.normpath(str(path)) != str(path): + _infra("Agent Lab home path is not an absolute canonical non-root path") + current = Path("/") + for component in path.parts[1:]: + current /= component + try: + metadata = current.lstat() + except FileNotFoundError: + _reject("Agent Lab home is not initialized") + except OSError as error: + _infra("Agent Lab home path cannot be inspected", error) + if stat.S_ISLNK(metadata.st_mode) or not stat.S_ISDIR(metadata.st_mode): + _infra("Agent Lab home path contains an unsafe component") + + +def _verify_directory( + path: Path, + *, + modes: tuple[int, ...] = (0o700,), + device: int | None = None, +) -> os.stat_result: + try: + lexical = path.lstat() + descriptor = os.open( + path, + os.O_RDONLY + | getattr(os, "O_CLOEXEC", 0) + | getattr(os, "O_DIRECTORY", 0) + | getattr(os, "O_NOFOLLOW", 0), + ) + except OSError as error: + _infra("store directory is unavailable", error) + try: + opened = os.fstat(descriptor) + except OSError as error: + try: + os.close(descriptor) + except OSError: + pass + _infra("store directory cannot be verified", error) + try: + os.close(descriptor) + except OSError as error: + _infra("store directory descriptor cannot be closed", error) + for metadata in (lexical, opened): + if ( + stat.S_ISLNK(metadata.st_mode) + or not stat.S_ISDIR(metadata.st_mode) + or metadata.st_uid != os.getuid() + or stat.S_IMODE(metadata.st_mode) not in modes + or (device is not None and metadata.st_dev != device) + ): + _infra("store directory metadata is unsafe") + if (lexical.st_dev, lexical.st_ino) != (opened.st_dev, opened.st_ino): + _infra("store directory identity changed") + return opened + + +def _read_file( + path: Path, + maximum: int, + purpose: str, + *, + mode: int, + device: int | None = None, +) -> bytes: + try: + lexical = path.lstat() + if ( + stat.S_ISLNK(lexical.st_mode) + or not stat.S_ISREG(lexical.st_mode) + or lexical.st_uid != os.getuid() + or lexical.st_nlink != 1 + or stat.S_IMODE(lexical.st_mode) != mode + or lexical.st_size > maximum + or (device is not None and lexical.st_dev != device) + ): + _infra(f"{purpose} metadata is unsafe") + descriptor = os.open( + path, + os.O_RDONLY + | getattr(os, "O_CLOEXEC", 0) + | getattr(os, "O_NOFOLLOW", 0) + | getattr(os, "O_NONBLOCK", 0), + ) + except StoreError: + raise + except OSError as error: + _infra(f"{purpose} is unavailable", error) + try: + opened = os.fstat(descriptor) + chunks: list[bytes] = [] + remaining = maximum + 1 + while remaining: + chunk = os.read(descriptor, min(65_536, remaining)) + if not chunk: + break + chunks.append(chunk) + remaining -= len(chunk) + data = b"".join(chunks) + final = os.fstat(descriptor) + except OSError as error: + _infra(f"{purpose} cannot be read safely", error) + finally: + try: + os.close(descriptor) + except OSError as error: + _infra(f"{purpose} descriptor cannot be closed", error) + try: + current = path.lstat() + except OSError as error: + _infra(f"{purpose} cannot be reverified", error) + for metadata in (opened, final, current): + if ( + not stat.S_ISREG(metadata.st_mode) + or metadata.st_uid != os.getuid() + or metadata.st_nlink != 1 + or stat.S_IMODE(metadata.st_mode) != mode + or metadata.st_size > maximum + or (device is not None and metadata.st_dev != device) + ): + _infra(f"{purpose} metadata changed unsafely") + if ( + len(data) > maximum + or len(data) != final.st_size + or _identity(lexical) != _identity(opened) + or _identity(opened) != _identity(final) + or _identity(final) != _identity(current) + ): + _infra(f"{purpose} changed while being read") + return data + + +def _load_home(home: Path) -> HomeAuthority: + try: + _absolute_directory_chain(home) + _verify_directory(home) + config_path = home / "config.json" + receipt_path = home / "home.json" + try: + config_raw = _read_file( + config_path, + MAX_AUTHORITY_BYTES, + "Agent Lab configuration", + mode=0o600, + ) + receipt_raw = _read_file( + receipt_path, + MAX_AUTHORITY_BYTES, + "Agent Lab home receipt", + mode=0o600, + ) + except StoreInfrastructure: + if not os.path.lexists(config_path) and not os.path.lexists(receipt_path): + _reject("Agent Lab home is not initialized") + raise + config = _parse_object(config_raw, "Agent Lab configuration") + receipt = _parse_object(receipt_raw, "Agent Lab home receipt") + if set(config) != {"apiVersion", "paths"} or config.get("apiVersion") != "agent-lab.config/v0alpha1": + _infra("Agent Lab configuration schema is not closed") + paths = config.get("paths") + if ( + not isinstance(paths, dict) + or set(paths) != {"experiments", "images", "cache", "state"} + or len(set(paths.values())) != 4 + or any( + not isinstance(item, str) or SAFE_COMPONENT.fullmatch(item) is None + for item in paths.values() + ) + ): + _infra("Agent Lab configuration paths are unsafe") + expected_config_digest = digest(canonical(config)) + locks_value = receipt.get("locks") + if ( + set(receipt) != {"apiVersion", "configDigest", "locks", "paths"} + or receipt.get("apiVersion") != "agent-lab.home/v0alpha1" + or receipt.get("configDigest") != expected_config_digest + or receipt.get("paths") != paths + or not isinstance(locks_value, dict) + or set(locks_value) != {"experiments", "imageCatalog"} + ): + _infra("Agent Lab home receipt is not closed") + lock_record = locks_value.get("experiments") + expected_lock_path = f"{paths['state']}/locks/experiments.lock" + if ( + not isinstance(lock_record, dict) + or set(lock_record) != {"device", "inode", "path", "schema"} + or type(lock_record.get("device")) is not int + or type(lock_record.get("inode")) is not int + or lock_record.get("path") != expected_lock_path + or lock_record.get("schema") != LOCK_SCHEMA + ): + _infra("Experiment store lock authority is absent from the home receipt") + store = home / str(paths["experiments"]) + state = home / str(paths["state"]) + staging = store / ".staging" + locks = state / "locks" + store_metadata = _verify_directory(store) + _verify_directory(state) + _verify_directory(locks) + _verify_directory(staging, device=store_metadata.st_dev) + lock = locks / "experiments.lock" + try: + lock_metadata = lock.lstat() + except OSError as error: + _infra("Experiment store lock is unavailable", error) + if ( + lock_metadata.st_dev != lock_record["device"] + or lock_metadata.st_ino != lock_record["inode"] + ): + _infra("Experiment store lock identity does not match the home receipt") + return HomeAuthority( + home, + store, + staging, + state, + locks, + lock, + int(lock_record["device"]), + int(lock_record["inode"]), + config_raw, + receipt_raw, + store_metadata.st_dev, + ) + except StoreError: + raise + except OSError as error: + _infra("Agent Lab home cannot be validated", error) + + +def _revalidate_authority(authority: HomeAuthority) -> None: + if ( + _read_file( + authority.home / "config.json", + MAX_AUTHORITY_BYTES, + "Agent Lab configuration", + mode=0o600, + ) + != authority.config_raw + or _read_file( + authority.home / "home.json", + MAX_AUTHORITY_BYTES, + "Agent Lab home receipt", + mode=0o600, + ) + != authority.receipt_raw + ): + _infra("Agent Lab configuration changed during installation") + store = _verify_directory(authority.store) + _verify_directory(authority.state) + _verify_directory(authority.locks) + _verify_directory(authority.staging, device=store.st_dev) + if store.st_dev != authority.store_device: + _infra("Experiment store filesystem changed during installation") + + +@contextmanager +def _store_lock(authority: HomeAuthority, fault: FaultHook | None) -> Iterator[int]: + maximum = len(LOCK_SCHEMA.encode("ascii") + b"\n") + path = authority.lock + descriptor = -1 + try: + lexical = path.lstat() + if ( + not stat.S_ISREG(lexical.st_mode) + or stat.S_ISLNK(lexical.st_mode) + or lexical.st_uid != os.getuid() + or lexical.st_nlink != 1 + or stat.S_IMODE(lexical.st_mode) != 0o600 + or lexical.st_size != maximum + or (lexical.st_dev, lexical.st_ino) + != (authority.lock_device, authority.lock_inode) + ): + _infra("Experiment store lock metadata is unsafe") + descriptor = os.open( + path, + os.O_RDWR + | getattr(os, "O_CLOEXEC", 0) + | getattr(os, "O_NOFOLLOW", 0) + | getattr(os, "O_NONBLOCK", 0), + ) + opened = os.fstat(descriptor) + if ( + not stat.S_ISREG(opened.st_mode) + or opened.st_uid != os.getuid() + or opened.st_nlink != 1 + or stat.S_IMODE(opened.st_mode) != 0o600 + or opened.st_size != maximum + or (opened.st_dev, opened.st_ino) + != (authority.lock_device, authority.lock_inode) + ): + _infra("Experiment store lock identity changed before acquisition") + fcntl.flock(descriptor, fcntl.LOCK_EX) + held = os.fstat(descriptor) + current = path.lstat() + expected = (authority.lock_device, authority.lock_inode) + if ( + _identity(opened) != _identity(held) + or (current.st_dev, current.st_ino) != expected + or not stat.S_ISREG(current.st_mode) + or current.st_uid != os.getuid() + or current.st_nlink != 1 + or stat.S_IMODE(current.st_mode) != 0o600 + or current.st_size != maximum + ): + _infra("Experiment store lock path changed while acquiring it") + os.lseek(descriptor, 0, os.SEEK_SET) + if os.read(descriptor, maximum + 1) != LOCK_SCHEMA.encode("ascii") + b"\n": + _infra("Experiment store lock receipt is malformed") + os.lseek(descriptor, 0, os.SEEK_SET) + _fault(fault, "experiment store lock.after_acquire") + yield descriptor + except StoreError: + raise + except OSError as error: + _infra("Experiment store lock cannot be held safely", error) + finally: + unwinding = sys.exc_info()[0] is not None + if descriptor >= 0: + try: + fcntl.flock(descriptor, fcntl.LOCK_UN) + os.close(descriptor) + except OSError as error: + if not unwinding: + _infra("Experiment store lock could not be released", error) + + +def _module(path: Path, name: str): + spec = spec_from_file_location(name, path) + if spec is None or spec.loader is None: + _infra(f"{path.name} cannot be loaded") + value = module_from_spec(spec) + sys.modules[spec.name] = value + try: + spec.loader.exec_module(value) + except (ImportError, OSError) as error: + _infra(f"{path.name} cannot be loaded", error) + return value + + +def _experiment_module(): + return _module(Path(__file__).resolve().with_name("experiment.py"), "agent_lab_experiment_store_planning") + + +def _catalog_module(): + return _module(Path(__file__).resolve().with_name("image_catalog.py"), "agent_lab_experiment_store_catalog") + + +def _selected_entries(plan: dict[str, object]) -> list[dict[str, object]]: + try: + members = plan["spec"]["members"] # type: ignore[index] + if not isinstance(members, list): + raise ValueError("members") + selected: dict[tuple[str, str], dict[str, object]] = {} + for member in members: + if not isinstance(member, dict): + raise ValueError("member") + requested = member["requestedSelector"] + resolved = member["resolvedImage"] + if not isinstance(requested, dict) or not isinstance(resolved, dict): + raise ValueError("selector") + origin = resolved.get("origin") + if origin == "direct": + if set(resolved) != {"origin", "subject"} or set(requested) != {"digestRef"}: + raise ValueError("direct selector") + continue + if ( + origin not in {"agent-lab", "local"} + or set(requested) != {"catalogName"} + or set(resolved) != {"entryDigest", "generation", "origin", "subject"} + ): + raise ValueError("catalog selector") + name = requested["catalogName"] + entry = { + "entryDigest": resolved["entryDigest"], + "generation": resolved["generation"], + "name": name, + "origin": origin, + "subject": resolved["subject"], + } + if ( + not _image_name(name) + or SHA256.fullmatch(str(entry["entryDigest"])) is None + or type(entry["generation"]) is not int + or int(entry["generation"]) < 1 + or not isinstance(entry["subject"], str) + ): + raise ValueError("selected identity") + key = (str(origin), name) + if key in selected and selected[key] != entry: + raise ValueError("inconsistent selected identity") + selected[key] = entry + return [selected[key] for key in sorted(selected, key=lambda item: (item[0].encode(), item[1].encode()))] + except (KeyError, TypeError, ValueError) as error: + _infra("Experiment plan has invalid selected-entry identities", error) + + +def _local_dependencies(selected: Sequence[dict[str, object]]) -> tuple[dict[str, object], ...]: + return tuple( + { + "entryDigest": item["entryDigest"], + "generation": item["generation"], + "name": item["name"], + "subject": item["subject"], + } + for item in selected + if item["origin"] == "local" + ) + + +def _verify_held_catalog( + held: object, + dependencies: Sequence[dict[str, object]], +) -> dict[str, object]: + try: + if not isinstance(held, dict) or set(held) != {"catalog", "records"}: + raise ValueError("held envelope") + catalog = held["catalog"] + records = held["records"] + names = {str(item["name"]) for item in dependencies} + if ( + not isinstance(catalog, dict) + or set(catalog) != {"revision", "snapshotDigest"} + or type(catalog.get("revision")) is not int + or int(catalog["revision"]) < 1 + or SHA256.fullmatch(str(catalog.get("snapshotDigest"))) is None + or not isinstance(records, dict) + or set(records) != names + ): + raise ValueError("held catalog evidence") + by_name = {str(item["name"]): item for item in dependencies} + for name in names: + record = records[name] + expected = by_name[name] + if ( + not isinstance(record, dict) + or record.get("name") != name + or record.get("state") != "active" + or record.get("entryDigest") != expected["entryDigest"] + or record.get("generation") != expected["generation"] + or record.get("subject") != expected["subject"] + ): + raise ValueError("held selected entry") + return {"revision": catalog["revision"], "snapshotDigest": catalog["snapshotDigest"]} + except (KeyError, TypeError, ValueError) as error: + _infra("held local image catalog evidence is invalid", error) + + +def _source_digest(data: bytes) -> str: + name = b"experiment.cue" + value = hashlib.sha256(SOURCE_DIGEST_DOMAIN) + value.update(len(name).to_bytes(4, "big")) + value.update(name) + value.update(len(data).to_bytes(8, "big")) + value.update(data) + return "sha256:" + value.hexdigest() + + +def _image_name(value: object) -> bool: + if not isinstance(value, str) or not value.isascii(): + return False + parts = value.split(".") + return ( + len(value.encode("ascii")) <= 63 + and len(parts) == 2 + and all(1 <= len(part.encode("ascii")) <= 31 for part in parts) + and all(IMAGE_COMPONENT.fullmatch(part) is not None for part in parts) + ) + + +def _installation_identity( + source_digest: str, + plan: dict[str, object], + decision: dict[str, object], + selected: Sequence[dict[str, object]], +) -> dict[str, object]: + try: + contract = plan["contract"] + binding = decision["binding"] + if not isinstance(contract, dict) or not isinstance(binding, dict): + raise ValueError("binding") + identity = { + "authorizationDigest": binding["authorizationDigest"], + "contractDigest": contract["digest"], + "planDigest": binding["planDigest"], + "selectedEntries": list(selected), + "sourceDigest": source_digest, + } + if any( + SHA256.fullmatch(str(identity[key])) is None + for key in ("authorizationDigest", "contractDigest", "planDigest", "sourceDigest") + ): + raise ValueError("digest") + return identity + except (KeyError, TypeError, ValueError) as error: + _infra("installation identity cannot be derived", error) + + +def _candidate( + snapshot: object, + plan: dict[str, object], + decision: dict[str, object], + selected: Sequence[dict[str, object]], + catalog_evidence: dict[str, object] | None, +) -> tuple[dict[str, bytes], dict[str, object], str, str]: + source_data = getattr(snapshot, "data", None) + source_digest = getattr(snapshot, "digest", None) + if not isinstance(source_data, bytes) or not isinstance(source_digest, str): + _infra("source snapshot is malformed") + if source_digest != _source_digest(source_data): + _infra("source snapshot digest is inconsistent") + identity = _installation_identity(source_digest, plan, decision, selected) + installation_key = digest(INSTALL_KEY_DOMAIN + canonical(identity)) + plan_bytes = canonical(plan) + b"\n" + decision_bytes = canonical(decision) + b"\n" + provenance = { + "apiVersion": PROVENANCE_API, + "authorizationDigest": identity["authorizationDigest"], + "catalog": catalog_evidence, + "contractDigest": identity["contractDigest"], + "kind": "ExperimentInstallationProvenance", + "planDigest": identity["planDigest"], + "selectedEntries": list(selected), + "source": { + "bytes": len(source_data), + "digest": source_digest, + "entryCount": 1, + "fileCount": 1, + "format": "agent-lab.experiment-tree/v1", + "kind": "directory", + }, + "transport": {"kind": "local-directory"}, + } + provenance_bytes = canonical(provenance) + b"\n" + files = { + "artifact/experiment.cue": source_data, + "records/decision.json": decision_bytes, + "records/plan.json": plan_bytes, + "records/provenance.json": provenance_bytes, + } + records = { + "artifact/experiment.cue": { + "digest": digest(source_data), + "schema": "agent-lab/v0alpha1", + }, + "records/decision.json": { + "digest": digest(decision_bytes), + "schema": decision.get("apiVersion"), + }, + "records/plan.json": { + "digest": digest(plan_bytes), + "schema": plan.get("apiVersion"), + }, + "records/provenance.json": { + "digest": digest(provenance_bytes), + "schema": PROVENANCE_API, + }, + } + try: + requested_name = plan["metadata"]["requestedName"] # type: ignore[index] + except (KeyError, TypeError) as error: + _infra("Experiment plan has no requested name", error) + if not isinstance(requested_name, str) or SAFE_COMPONENT.fullmatch(requested_name) is None: + _infra("Experiment plan requested name is unsafe") + receipt = { + "apiVersion": INSTALL_API, + "identity": identity, + "installationKey": installation_key, + "kind": "ExperimentInstallationReceipt", + "name": requested_name, + "records": records, + } + receipt_bytes = canonical(receipt) + b"\n" + files["records/install.json"] = receipt_bytes + return files, receipt, installation_key, digest(receipt_bytes) + + +def _directory_names(path: Path, purpose: str, maximum: int) -> tuple[str, ...]: + try: + names = os.listdir(path) + except OSError as error: + _infra(f"{purpose} cannot be enumerated", error) + if len(names) > maximum: + _infra(f"{purpose} exceeds its fixed entry bound") + try: + return tuple(sorted(names, key=lambda item: os.fsencode(item))) + except (TypeError, UnicodeError) as error: + _infra(f"{purpose} contains an invalid name", error) + + +def _path_state(path: Path) -> str: + try: + metadata = path.lstat() + except FileNotFoundError: + return "absent" + except OSError as error: + _infra("store path state is ambiguous", error) + if stat.S_ISDIR(metadata.st_mode) and not stat.S_ISLNK(metadata.st_mode): + return "directory" + return "other" + + +def _record_schema(receipt: dict[str, object], path: str) -> tuple[str, str]: + try: + records = receipt["records"] + if not isinstance(records, dict) or set(records) != RECORD_PATHS: + raise ValueError("record set") + record = records[path] + if ( + not isinstance(record, dict) + or set(record) != {"digest", "schema"} + or SHA256.fullmatch(str(record.get("digest"))) is None + or not isinstance(record.get("schema"), str) + ): + raise ValueError("record binding") + return str(record["digest"]), str(record["schema"]) + except (KeyError, TypeError, ValueError) as error: + _infra("installation receipt record binding is invalid", error) + + +def _verify_envelope( + path: Path, + expected_name: str, + device: int, + *, + root_modes: tuple[int, ...] = (0o500,), +) -> VerifiedInstall: + _verify_directory(path, modes=root_modes, device=device) + if _directory_names(path, "installed envelope", 2) != ("artifact", "records"): + _infra("installed envelope layout is not closed") + artifact_dir = path / "artifact" + records_dir = path / "records" + _verify_directory(artifact_dir, modes=(0o500,), device=device) + _verify_directory(records_dir, modes=(0o500,), device=device) + if _directory_names(artifact_dir, "installed artifact", 1) != ("experiment.cue",): + _infra("installed artifact layout is not closed") + if _directory_names(records_dir, "installed records", 4) != ( + "decision.json", + "install.json", + "plan.json", + "provenance.json", + ): + _infra("installed record layout is not closed") + raw = { + "artifact/experiment.cue": _read_file( + artifact_dir / "experiment.cue", + MAX_ARTIFACT_BYTES, + "installed Experiment artifact", + mode=0o400, + device=device, + ), + "records/decision.json": _read_file( + records_dir / "decision.json", + MAX_RECORD_BYTES, + "installed authorization decision", + mode=0o400, + device=device, + ), + "records/install.json": _read_file( + records_dir / "install.json", + MAX_RECORD_BYTES, + "installed receipt", + mode=0o400, + device=device, + ), + "records/plan.json": _read_file( + records_dir / "plan.json", + MAX_RECORD_BYTES, + "installed plan", + mode=0o400, + device=device, + ), + "records/provenance.json": _read_file( + records_dir / "provenance.json", + MAX_RECORD_BYTES, + "installed provenance", + mode=0o400, + device=device, + ), + } + plan = _parse_object(raw["records/plan.json"], "installed plan") + decision = _parse_object(raw["records/decision.json"], "installed authorization decision") + provenance = _parse_object(raw["records/provenance.json"], "installed provenance") + receipt = _parse_object(raw["records/install.json"], "installed receipt") + try: + if ( + set(plan) != {"apiVersion", "contract", "kind", "metadata", "spec"} + or plan.get("apiVersion") != "agent-lab.request/v0alpha1" + or plan.get("kind") != "RequestedExperimentPlan" + or not isinstance(plan.get("metadata"), dict) + or plan["metadata"].get("requestedName") != expected_name # type: ignore[union-attr] + or not isinstance(plan.get("contract"), dict) + ): + raise ValueError("plan") + contract = plan["contract"] + if ( + set(contract) != {"digest", "name", "version"} # type: ignore[arg-type] + or contract.get("name") != "agent-lab.experiment" # type: ignore[union-attr] + or contract.get("version") != "v0alpha1" # type: ignore[union-attr] + or SHA256.fullmatch(str(contract.get("digest"))) is None # type: ignore[union-attr] + ): + raise ValueError("contract") + plan_digest = digest(canonical(plan)) + if ( + set(decision) != {"action", "apiVersion", "binding", "kind", "principal", "resource", "verdict"} + or decision.get("apiVersion") != "agent-lab.authorization/v0alpha1" + or decision.get("kind") != "ExperimentAuthorizationDecision" + or decision.get("action") != "experiment.install" + or decision.get("verdict") != "permit" + or not isinstance(decision.get("binding"), dict) + ): + raise ValueError("decision") + binding = decision["binding"] + source_digest = _source_digest(raw["artifact/experiment.cue"]) + if ( + set(binding) != {"authorizationDigest", "contractDigest", "planDigest", "sourceDigest"} # type: ignore[arg-type] + or binding.get("planDigest") != plan_digest # type: ignore[union-attr] + or binding.get("sourceDigest") != source_digest # type: ignore[union-attr] + or binding.get("contractDigest") != contract.get("digest") # type: ignore[union-attr] + or SHA256.fullmatch(str(binding.get("authorizationDigest"))) is None # type: ignore[union-attr] + ): + raise ValueError("decision binding") + selected = _selected_entries(plan) + identity = _installation_identity(source_digest, plan, decision, selected) + if ( + set(provenance) != { + "apiVersion", "authorizationDigest", "catalog", "contractDigest", "kind", + "planDigest", "selectedEntries", "source", "transport", + } + or provenance.get("apiVersion") != PROVENANCE_API + or provenance.get("kind") != "ExperimentInstallationProvenance" + or provenance.get("authorizationDigest") != identity["authorizationDigest"] + or provenance.get("contractDigest") != identity["contractDigest"] + or provenance.get("planDigest") != identity["planDigest"] + or provenance.get("selectedEntries") != list(selected) + or provenance.get("source") != { + "bytes": len(raw["artifact/experiment.cue"]), + "digest": source_digest, + "entryCount": 1, + "fileCount": 1, + "format": "agent-lab.experiment-tree/v1", + "kind": "directory", + } + or provenance.get("transport") != {"kind": "local-directory"} + ): + raise ValueError("provenance") + catalog = provenance.get("catalog") + local_selected = [item for item in selected if item["origin"] == "local"] + if local_selected: + if ( + not isinstance(catalog, dict) + or set(catalog) != {"revision", "snapshotDigest"} + or type(catalog.get("revision")) is not int + or int(catalog["revision"]) < 1 + or SHA256.fullmatch(str(catalog.get("snapshotDigest"))) is None + ): + raise ValueError("catalog provenance") + elif catalog is not None: + raise ValueError("unexpected catalog provenance") + installation_key = digest(INSTALL_KEY_DOMAIN + canonical(identity)) + if ( + set(receipt) != {"apiVersion", "identity", "installationKey", "kind", "name", "records"} + or receipt.get("apiVersion") != INSTALL_API + or receipt.get("kind") != "ExperimentInstallationReceipt" + or receipt.get("name") != expected_name + or receipt.get("identity") != identity + or receipt.get("installationKey") != installation_key + ): + raise ValueError("receipt") + expected_schemas = { + "artifact/experiment.cue": "agent-lab/v0alpha1", + "records/decision.json": str(decision["apiVersion"]), + "records/plan.json": str(plan["apiVersion"]), + "records/provenance.json": PROVENANCE_API, + } + for record_path in RECORD_PATHS: + record_digest, schema = _record_schema(receipt, record_path) + if record_digest != digest(raw[record_path]) or schema != expected_schemas[record_path]: + raise ValueError("record digest") + except (KeyError, TypeError, ValueError) as error: + _infra("installed envelope does not match its receipt", error) + file_digests = {item: digest(data) for item, data in raw.items()} + return VerifiedInstall( + expected_name, + installation_key, + digest(raw["records/install.json"]), + file_digests, + ) + + +def _intent(files: dict[str, bytes], name: str, key: str, receipt_digest: str) -> dict[str, object]: + file_digests = {path: digest(data) for path, data in sorted(files.items())} + return { + "apiVersion": INTENT_API, + "files": file_digests, + "installationKey": key, + "name": name, + "payloadDigest": digest(STAGE_PAYLOAD_DOMAIN + canonical(file_digests)), + "phase": "prepared", + "receiptDigest": receipt_digest, + } + + +def _validate_intent(value: dict[str, object]) -> None: + try: + files = value["files"] + if ( + set(value) != { + "apiVersion", "files", "installationKey", "name", "payloadDigest", "phase", "receiptDigest", + } + or value["apiVersion"] != INTENT_API + or value["phase"] != "prepared" + or not isinstance(value["name"], str) + or SAFE_COMPONENT.fullmatch(value["name"]) is None + or SHA256.fullmatch(str(value["installationKey"])) is None + or SHA256.fullmatch(str(value["receiptDigest"])) is None + or not isinstance(files, dict) + or set(files) != { + "artifact/experiment.cue", + "records/decision.json", + "records/install.json", + "records/plan.json", + "records/provenance.json", + } + or any(SHA256.fullmatch(str(item)) is None for item in files.values()) + or value["payloadDigest"] != digest(STAGE_PAYLOAD_DOMAIN + canonical(files)) + or value["receiptDigest"] != files["records/install.json"] + ): + raise ValueError("intent") + except (KeyError, TypeError, ValueError) as error: + _infra("Experiment staging intent is invalid", error) + + +def _scan_wrapper(authority: HomeAuthority, path: Path, *, cleanup: bool) -> dict[str, object] | None: + _verify_directory(path, modes=(0o700,), device=authority.store_device) + count = 1 + byte_count = 0 + pending = [path] + found: set[str] = set() + while pending: + parent = pending.pop() + remaining = MAX_STAGE_ENTRIES - count + for name in _directory_names(parent, "Experiment staging wrapper", max(remaining, 0)): + item = parent / name + relative = str(item.relative_to(path)) + if relative not in STAGE_ALLOWED: + _infra("Experiment staging wrapper contains an unknown entry") + try: + metadata = item.lstat() + except OSError as error: + _infra("Experiment staging entry cannot be inspected", error) + count += 1 + found.add(relative) + if count > MAX_STAGE_ENTRIES or metadata.st_dev != authority.store_device or metadata.st_uid != os.getuid(): + _infra("Experiment staging state exceeds or leaves its fixed authority") + if stat.S_ISLNK(metadata.st_mode): + _infra("Experiment staging state contains a symlink") + if stat.S_ISDIR(metadata.st_mode): + if relative not in PAYLOAD_DIRECTORIES or stat.S_IMODE(metadata.st_mode) not in (0o700, 0o500): + _infra("Experiment staging directory metadata is unsafe") + pending.append(item) + elif stat.S_ISREG(metadata.st_mode): + allowed_modes = (0o600,) if relative == "intent.json" else (0o600, 0o400) + if stat.S_IMODE(metadata.st_mode) not in allowed_modes or metadata.st_nlink != 1: + _infra("Experiment staging file metadata is unsafe") + byte_count += metadata.st_size + if byte_count > MAX_STAGE_BYTES: + _infra("Experiment staging state exceeds its fixed byte bound") + else: + _infra("Experiment staging state contains an unsafe type") + intent_path = path / "intent.json" + if "intent.json" not in found: + if cleanup: + return None + _infra("Experiment operation wrapper has no durable intent") + value = _parse_object( + _read_file( + intent_path, + MAX_AUTHORITY_BYTES, + "Experiment staging intent", + mode=0o600, + device=authority.store_device, + ), + "Experiment staging intent", + ) + _validate_intent(value) + files = value["files"] + assert isinstance(files, dict) + for relative in found & PAYLOAD_FILES: + payload_relative = relative.removeprefix("payload/") + maximum = MAX_ARTIFACT_BYTES if payload_relative == "artifact/experiment.cue" else MAX_RECORD_BYTES + mode = stat.S_IMODE((path / relative).lstat().st_mode) + raw = _read_file( + path / relative, + maximum, + "Experiment staged payload", + mode=mode, + device=authority.store_device, + ) + if digest(raw) != files[payload_relative]: + _infra("Experiment staged payload does not match its durable intent") + return value + + +def _rename_noreplace(source: Path, target: Path) -> None: + try: + library = ctypes.CDLL(None, use_errno=True) + function = library.renameat2 + except (AttributeError, OSError) as error: + _infra("Linux no-replace rename is unavailable", error) + function.argtypes = [ctypes.c_int, ctypes.c_char_p, ctypes.c_int, ctypes.c_char_p, ctypes.c_uint] + function.restype = ctypes.c_int + result = function(-100, os.fsencode(source), -100, os.fsencode(target), 1) + if result != 0: + code = ctypes.get_errno() + if code == errno.EEXIST: + _infra("Experiment no-replace publication raced with an existing target") + _infra("Experiment no-replace publication failed", OSError(code, os.strerror(code))) + + +def _fsync_directory(path: Path, purpose: str, *, modes: tuple[int, ...] = (0o700,)) -> None: + metadata = _verify_directory(path, modes=modes) + descriptor = -1 + try: + descriptor = os.open( + path, + os.O_RDONLY + | getattr(os, "O_CLOEXEC", 0) + | getattr(os, "O_DIRECTORY", 0) + | getattr(os, "O_NOFOLLOW", 0), + ) + if (os.fstat(descriptor).st_dev, os.fstat(descriptor).st_ino) != (metadata.st_dev, metadata.st_ino): + _infra(f"{purpose} directory identity changed") + os.fsync(descriptor) + except StoreError: + raise + except OSError as error: + _infra(f"{purpose} directory cannot be persisted", error) + finally: + unwinding = sys.exc_info()[0] is not None + if descriptor >= 0: + try: + os.close(descriptor) + except OSError as error: + if not unwinding: + _infra(f"{purpose} directory descriptor cannot be closed", error) + + +def _write_all(descriptor: int, data: bytes) -> None: + view = memoryview(data) + while view: + written = os.write(descriptor, view) + if written <= 0: + raise OSError("write made no progress") + view = view[written:] + + +def _write_file(path: Path, data: bytes, purpose: str, fault: FaultHook | None) -> None: + descriptor = -1 + try: + descriptor = os.open( + path, + os.O_WRONLY + | os.O_CREAT + | os.O_EXCL + | getattr(os, "O_CLOEXEC", 0) + | getattr(os, "O_NOFOLLOW", 0), + 0o600, + ) + opened = os.fstat(descriptor) + if ( + not stat.S_ISREG(opened.st_mode) + or opened.st_uid != os.getuid() + or opened.st_nlink != 1 + or stat.S_IMODE(opened.st_mode) != 0o600 + ): + _infra(f"{purpose} file metadata is unsafe") + _write_all(descriptor, data) + if purpose == "experiment artifact": + _fault(fault, "experiment artifact.after_write") + os.fsync(descriptor) + if purpose == "experiment receipt": + _fault(fault, "experiment receipt.after_fsync") + except StoreError: + raise + except OSError as error: + _infra(f"{purpose} could not be written durably", error) + finally: + unwinding = sys.exc_info()[0] is not None + if descriptor >= 0: + try: + os.close(descriptor) + except OSError as error: + if not unwinding: + _infra(f"{purpose} descriptor could not be closed", error) + + +def _persist_read_only_file(path: Path, purpose: str) -> None: + descriptor = -1 + try: + os.chmod(path, 0o400, follow_symlinks=False) + descriptor = os.open( + path, + os.O_RDONLY + | getattr(os, "O_CLOEXEC", 0) + | getattr(os, "O_NOFOLLOW", 0), + ) + metadata = os.fstat(descriptor) + if ( + not stat.S_ISREG(metadata.st_mode) + or metadata.st_uid != os.getuid() + or metadata.st_nlink != 1 + or stat.S_IMODE(metadata.st_mode) != 0o400 + ): + _infra(f"{purpose} committed metadata is unsafe") + os.fsync(descriptor) + except StoreError: + raise + except OSError as error: + _infra(f"{purpose} mode could not be persisted", error) + finally: + unwinding = sys.exc_info()[0] is not None + if descriptor >= 0: + try: + os.close(descriptor) + except OSError as error: + if not unwinding: + _infra(f"{purpose} descriptor could not be closed", error) + + +def _prepare_stage( + authority: HomeAuthority, + files: dict[str, bytes], + name: str, + key: str, + receipt_digest: str, + fault: FaultHook | None, +) -> tuple[Path, dict[str, object]]: + wrapper = authority.staging / OPERATION_WRAPPER + payload = wrapper / "payload" + intent = _intent(files, name, key, receipt_digest) + try: + wrapper.mkdir(mode=0o700) + _write_file(wrapper / "intent.json", canonical(intent) + b"\n", "experiment intent", fault) + _fsync_directory(wrapper, "Experiment intent wrapper") + _fsync_directory(authority.staging, "Experiment staging intent") + payload.mkdir(mode=0o700) + (payload / "artifact").mkdir(mode=0o700) + (payload / "records").mkdir(mode=0o700) + ordered = ( + "artifact/experiment.cue", + "records/decision.json", + "records/plan.json", + "records/provenance.json", + "records/install.json", + ) + for relative in ordered: + purpose = "experiment receipt" if relative == "records/install.json" else ( + "experiment artifact" if relative == "artifact/experiment.cue" else "experiment record" + ) + _write_file(payload / relative, files[relative], purpose, fault) + _fsync_directory(payload / "artifact", "Experiment staged artifact") + _fsync_directory(payload / "records", "Experiment staged records") + _fsync_directory(payload, "Experiment staged envelope") + _fsync_directory(wrapper, "Experiment staged wrapper") + _fsync_directory(authority.staging, "Experiment staged operation") + for relative in ordered: + _persist_read_only_file(payload / relative, "Experiment staged file") + os.chmod(payload / "artifact", 0o500, follow_symlinks=False) + os.chmod(payload / "records", 0o500, follow_symlinks=False) + _fsync_directory(payload / "artifact", "Experiment committed artifact", modes=(0o500,)) + _fsync_directory(payload / "records", "Experiment committed records", modes=(0o500,)) + # This runtime's containment LSM rejects moving a non-writable directory. + # Keep only the envelope root private-writable until the no-replace move; + # all children are already committed and fsynced. The root is changed to + # 0500 and fsynced before the post-publication fault seam can run. + _fsync_directory(payload, "Experiment staged envelope root", modes=(0o700,)) + _fsync_directory(wrapper, "Experiment committed wrapper") + _fsync_directory(authority.staging, "Experiment committed staging") + except StoreError: + raise + except OSError as error: + _infra("Experiment staging envelope could not be prepared", error) + _scan_wrapper(authority, wrapper, cleanup=False) + _verify_envelope(payload, name, authority.store_device, root_modes=(0o700,)) + return wrapper, intent + + +def _remove_tree(path: Path, root: Path) -> None: + try: + metadata = path.lstat() + except FileNotFoundError: + return + except OSError as error: + _infra("Experiment cleanup residue cannot be inspected", error) + relative = str(path.relative_to(root)) if path != root else "." + if metadata.st_uid != os.getuid() or stat.S_ISLNK(metadata.st_mode): + _infra("Experiment cleanup residue metadata is unsafe") + if stat.S_ISREG(metadata.st_mode): + if relative not in STAGE_ALLOWED or metadata.st_nlink != 1 or stat.S_IMODE(metadata.st_mode) not in (0o600, 0o400): + _infra("Experiment cleanup file is unsafe") + try: + path.unlink() + except OSError as error: + _infra("Experiment cleanup file cannot be removed", error) + return + if not stat.S_ISDIR(metadata.st_mode) or ( + path == root and stat.S_IMODE(metadata.st_mode) != 0o700 + ) or ( + path != root and relative not in PAYLOAD_DIRECTORIES + ): + _infra("Experiment cleanup directory is unsafe") + try: + if stat.S_IMODE(metadata.st_mode) != 0o700: + os.chmod(path, 0o700, follow_symlinks=False) + names = _directory_names(path, "Experiment cleanup residue", MAX_STAGE_ENTRIES) + for name in sorted(names, key=lambda item: (item == "intent.json", os.fsencode(item))): + _remove_tree(path / name, root) + path.rmdir() + except StoreError: + raise + except OSError as error: + _infra("Experiment cleanup directory cannot be removed", error) + + +def _finish_cleanup(authority: HomeAuthority, cleanup: Path) -> None: + if cleanup != authority.staging / CLEANUP_WRAPPER: + _infra("Experiment cleanup target changed") + _scan_wrapper(authority, cleanup, cleanup=True) + _remove_tree(cleanup, cleanup) + _fsync_directory(authority.staging, "Experiment staging cleanup") + + +def _cleanup_operation(authority: HomeAuthority, wrapper: Path) -> None: + if wrapper != authority.staging / OPERATION_WRAPPER: + _infra("Experiment operation cleanup target changed") + _scan_wrapper(authority, wrapper, cleanup=False) + cleanup = authority.staging / CLEANUP_WRAPPER + _rename_noreplace(wrapper, cleanup) + _fsync_directory(authority.staging, "Experiment cleanup handoff") + _finish_cleanup(authority, cleanup) + + +def _intent_matches_final( + authority: HomeAuthority, + intent: dict[str, object], +) -> bool: + name = str(intent["name"]) + target = authority.store / name + state = _path_state(target) + if state == "absent": + return False + if state != "directory": + _infra("Experiment staged operation conflicts with an ambiguous final target") + verified = _verify_envelope(target, name, authority.store_device) + files = intent["files"] + assert isinstance(files, dict) + if ( + verified.installation_key != intent["installationKey"] + or verified.receipt_digest != intent["receiptDigest"] + or any(verified.file_digests.get(path) != expected for path, expected in files.items()) + ): + _infra("Experiment staged operation conflicts with the final installation") + return True + + +def _reconcile(authority: HomeAuthority) -> None: + names = _directory_names(authority.staging, "Experiment staging root", 1) + if not names: + _fsync_directory(authority.staging, "Experiment empty staging recovery") + return + if names == (CLEANUP_WRAPPER,): + cleanup = authority.staging / CLEANUP_WRAPPER + intent = _scan_wrapper(authority, cleanup, cleanup=True) + if intent is not None and _intent_matches_final(authority, intent): + _fsync_directory(authority.store, "Experiment committed recovery") + _finish_cleanup(authority, cleanup) + return + if names != (OPERATION_WRAPPER,): + _infra("Experiment staging root contains an unknown wrapper") + wrapper = authority.staging / OPERATION_WRAPPER + intent = _scan_wrapper(authority, wrapper, cleanup=False) + assert intent is not None + if _intent_matches_final(authority, intent): + _fsync_directory(authority.store, "Experiment committed recovery") + _cleanup_operation(authority, wrapper) + + +def _existing(authority: HomeAuthority, name: str) -> VerifiedInstall | None: + target = authority.store / name + state = _path_state(target) + if state == "absent": + return None + if state != "directory": + _infra("Experiment installation target is ambiguous") + return _verify_envelope(target, name, authority.store_device) + + +@contextmanager +def _held_catalog_context( + home: Path, + dependencies: Sequence[dict[str, object]], + fault: FaultHook | None, +) -> Iterator[object | None]: + if not dependencies: + with nullcontext(None) as held: + yield held + return + catalog = _catalog_module() + operation = getattr(catalog, "hold_local_image_entries", None) + if not callable(operation): + _infra("held local image catalog support is unavailable") + try: + with operation(home, tuple(dependencies), fault=fault) as held: + yield held + except catalog.CatalogReject as error: + raise StoreReject(str(error)) from error + except catalog.CatalogInfrastructure as error: + raise StoreInfrastructure(str(error)) from error + except (AttributeError, OSError) as error: + _infra("held local image catalog cannot be acquired", error) + + +def install_directory( + home: Path, + source: Path, + *, + fault: FaultHook | None = None, +) -> dict[str, object]: + """Freshly validate/authorize one directory and publish its envelope once.""" + + if sys.platform != "linux": + _infra("effectful Experiment installation requires Linux") + authority = _load_home(Path(home)) + experiment = _experiment_module() + prior_home = os.environ.get("AGENT_LAB_HOME") + os.environ["AGENT_LAB_HOME"] = str(authority.home) + try: + try: + snapshot = experiment.read_directory_snapshot(str(source)) + manifest = experiment.authored_manifest(snapshot) + resolution = experiment.cue_plan_with_evidence(manifest) + plan = resolution.plan + if not isinstance(plan, dict): + _infra("CUE produced a malformed Experiment plan") + decision, status = experiment.authorize_plan(plan, snapshot.digest) + except experiment.InvalidManifest as error: + raise StoreReject(str(error)) from error + except experiment.InfrastructureError as error: + raise StoreInfrastructure(str(error)) from error + except (AttributeError, OSError) as error: + _infra("Experiment validation or authorization is unavailable", error) + finally: + if prior_home is None: + os.environ.pop("AGENT_LAB_HOME", None) + else: + os.environ["AGENT_LAB_HOME"] = prior_home + if status != 0 or not isinstance(decision, dict) or decision.get("verdict") != "permit": + _reject("fresh Experiment installation authorization denied") + selected = _selected_entries(plan) + dependencies = _local_dependencies(selected) + try: + held_context = _held_catalog_context(authority.home, dependencies, fault) + with held_context as held: + catalog_evidence = _verify_held_catalog(held, dependencies) if dependencies else None + with _store_lock(authority, fault): + _revalidate_authority(authority) + _reconcile(authority) + files, _, key, receipt_digest = _candidate( + snapshot, + plan, + decision, + selected, + catalog_evidence, + ) + name = str(plan["metadata"]["requestedName"]) # type: ignore[index] + existing = _existing(authority, name) + if existing is not None: + if existing.installation_key != key: + _reject("Experiment name already has a different installation") + return { + "changed": False, + "installationKey": existing.installation_key, + "name": name, + "receiptDigest": existing.receipt_digest, + } + wrapper, _ = _prepare_stage( + authority, + files, + name, + key, + receipt_digest, + fault, + ) + _revalidate_authority(authority) + _fault(fault, "experiment envelope.before_noreplace") + _rename_noreplace(wrapper / "payload", authority.store / name) + try: + os.chmod(authority.store / name, 0o500, follow_symlinks=False) + except OSError as error: + _infra("published Experiment mode is uncertain", error) + _fsync_directory( + authority.store / name, + "Experiment published envelope", + modes=(0o500,), + ) + _fault(fault, "experiment envelope.after_noreplace") + _fsync_directory(authority.store, "Experiment store root") + _fault(fault, "experiment store root.after_fsync") + verified = _verify_envelope(authority.store / name, name, authority.store_device) + if verified.installation_key != key or verified.receipt_digest != receipt_digest: + _infra("published Experiment does not match its candidate") + _cleanup_operation(authority, wrapper) + return { + "changed": True, + "installationKey": key, + "name": name, + "receiptDigest": receipt_digest, + } + except StoreError: + raise + except experiment.InvalidManifest as error: + raise StoreReject(str(error)) from error + except experiment.InfrastructureError as error: + raise StoreInfrastructure(str(error)) from error + except (AttributeError, OSError, RuntimeError) as error: + _infra("Experiment store operation could not establish a result", error) + + +def inspect_install(home: Path, name: str) -> dict[str, object]: + """Read-only verification and identity projection for one installed name.""" + + if not isinstance(name, str) or SAFE_COMPONENT.fullmatch(name) is None: + _reject("Experiment name is invalid") + authority = _load_home(Path(home)) + try: + installed = _existing(authority, name) + except StoreError: + raise + except OSError as error: + _infra("Experiment installation cannot be inspected", error) + if installed is None: + _reject("Experiment name is not installed") + return { + "installationKey": installed.installation_key, + "name": name, + "receiptDigest": installed.receipt_digest, + "state": "installed", + } diff --git a/scripts/image_catalog.py b/scripts/image_catalog.py index 0aac6bd..e046887 100644 --- a/scripts/image_catalog.py +++ b/scripts/image_catalog.py @@ -1944,6 +1944,81 @@ def operation() -> dict[str, object]: return result +@contextmanager +def hold_local_image_entries( + home: Path, + dependencies: Sequence[dict[str, object]], + *, + limits: CatalogLimits = CatalogLimits(), + fault: FaultHook | None = None, +) -> Iterator[dict[str, object]]: + """Hold one verified snapshot while exact active dependencies are consumed.""" + + try: + requested = tuple(dependencies) + except (TypeError, ValueError) as error: + _infra("selected local image dependencies are malformed", error) + + expected: dict[str, dict[str, object]] = {} + for dependency in requested: + if not isinstance(dependency, dict) or set(dependency) != { + "entryDigest", + "generation", + "name", + "subject", + }: + _infra("selected local image dependency schema is not closed") + name = dependency.get("name") + entry_digest = dependency.get("entryDigest") + generation = dependency.get("generation") + subject = dependency.get("subject") + if ( + not isinstance(name, str) + or not image_name(name) + or name.startswith("agent-lab.") + or not _is_digest(entry_digest) + or type(generation) is not int + or generation != 1 + or not isinstance(subject, str) + or not oci_subject(subject) + ): + _infra("selected local image dependency binding is invalid") + prior = expected.get(name) + if prior is not None and prior != dependency: + _infra("selected local image dependencies contradict one another") + expected[name] = dict(dependency) + + if not expected: + yield {"catalog": None, "records": {}} + return + + authority = _load_home(home) + with _catalog_lock(authority, exclusive=False) as lock_descriptor: + _fault(fault, "experiment catalog lock.after_acquire") + state = _load_catalog(authority, lock_descriptor, limits) + selected: dict[str, dict[str, object]] = {} + for name in sorted(expected, key=lambda item: item.encode("ascii")): + record = state.records.get(name) + if record is None or record["state"] != "active": + _reject("selected local image dependency is unknown or inactive") + dependency = expected[name] + if any( + record[field] != dependency[field] + for field in ("name", "entryDigest", "generation", "subject") + ): + _reject("selected local image dependency has drifted") + selected[name] = dict(record) + if state.snapshot_digest is None or state.revision < 1: + _infra("selected local image dependencies have no committed snapshot") + yield { + "catalog": { + "revision": state.revision, + "snapshotDigest": state.snapshot_digest, + }, + "records": selected, + } + + def resolve_local_images( home: Path, names: Sequence[str], diff --git a/tests/install/fixtures/expected-runtime-files.txt b/tests/install/fixtures/expected-runtime-files.txt index c3bfbd1..b62cbce 100644 --- a/tests/install/fixtures/expected-runtime-files.txt +++ b/tests/install/fixtures/expected-runtime-files.txt @@ -9,6 +9,7 @@ scripts/agent-lab.py scripts/dev/cedar-tool.py scripts/dev/cue-tool.py scripts/experiment.py +scripts/experiment_store.py scripts/image_catalog.py scripts/image_reference.py tools/cedar.lock From 5d9540e67f30812717300a462794581b7ed237af Mon Sep 17 00:00:00 2001 From: xormania <127287135+xormania@users.noreply.github.com> Date: Sun, 2 Aug 2026 02:39:43 -0400 Subject: [PATCH 046/158] test(experiment): expose store recovery gaps --- tests/experiment/aggregate-harness-cases.sh | 18 +- tests/experiment/install-state-cases.py | 459 +++++++++++++++++++- tests/experiment/install-store-cases.sh | 116 ++++- tests/experiment/local-lifecycle-cases.sh | 7 +- 4 files changed, 561 insertions(+), 39 deletions(-) diff --git a/tests/experiment/aggregate-harness-cases.sh b/tests/experiment/aggregate-harness-cases.sh index 1156aa5..0a54947 100755 --- a/tests/experiment/aggregate-harness-cases.sh +++ b/tests/experiment/aggregate-harness-cases.sh @@ -53,15 +53,15 @@ expected_ids=( M-CAT-NOEF-001 M-CAT-ADMIT-001 M-CAT-ATOM-001 M-CAT-DUR-001 M-CAT-STAGE-001 INST-HOME-001 INST-UNKNOWN-001 INST-PERMIT-001 INST-RECEIPT-001 INST-INSPECT-001 INST-RETRY-001 INST-CONFLICT-001 INST-DENY-001 - INST-FORGE-001 INST-NOEF-001 INST-LOCAL-001 + INST-FORGE-001 INST-NOEF-001 INST-LOCAL-001 INST-NAME-001 IST-STATE-001 IST-LOCK-001 IST-STATE-002 IST-BOUND-001 IST-STATE-003 - IST-CONC-001 IST-CONC-002 IST-CRASH-001 IST-LIVE-001 IST-PLAT-001 + IST-CONC-001 IST-CONC-002 IST-CRASH-001 IST-LIVE-001 IST-PLAT-001 IST-PROV-001 ) installer_ids=("${expected_ids[@]:0:5}") config_ids=("${expected_ids[@]:5:5}") catalog_ids=("${expected_ids[@]:10:76}") -install_ids=("${expected_ids[@]:86:11}") -state_ids=("${expected_ids[@]:97:10}") +install_ids=("${expected_ids[@]:86:12}") +state_ids=("${expected_ids[@]:98:11}") write_fixture() { local path="$1" @@ -205,10 +205,10 @@ success_output="$work/success.out" success_rc=0 run_replica "$success_output" env || success_rc=$? if [ "$success_rc" -eq 0 ] && - [ "$(grep -Ec '^(PASS|FAIL) [A-Z0-9-]+ ' "$success_output")" -eq 107 ] && - [ "$(grep -Fxc 'SUMMARY assertions=107 expected=107 failures=0 infra=0' "$success_output")" -eq 1 ] && + [ "$(grep -Ec '^(PASS|FAIL) [A-Z0-9-]+ ' "$success_output")" -eq 109 ] && + [ "$(grep -Fxc 'SUMMARY assertions=109 expected=109 failures=0 infra=0' "$success_output")" -eq 1 ] && [ "$(tail -n 1 "$success_output")" = 'EXPERIMENT LOCAL LIFECYCLE PASS' ] && - awk '/^(PASS|FAIL) [A-Z0-9-]+ / {next} /^SUMMARY assertions=107 expected=107 failures=0 infra=0$/ {next} /^EXPERIMENT LOCAL LIFECYCLE PASS$/ {next} {bad=1} END {exit bad}' "$success_output"; then + awk '/^(PASS|FAIL) [A-Z0-9-]+ / {next} /^SUMMARY assertions=109 expected=109 failures=0 infra=0$/ {next} /^EXPERIMENT LOCAL LIFECYCLE PASS$/ {next} {bad=1} END {exit bad}' "$success_output"; then pass AGG-002 "success forwards only assertions then one summary and marker" else fail AGG-002 "success forwards only assertions then one summary and marker" @@ -257,7 +257,7 @@ write_fixture "$replica/tests/install/local-install-cases.sh" 1 "${failed_record assertion_rc=0 run_replica "$work/assertion.out" env || assertion_rc=$? if [ "$assertion_rc" -eq 1 ] && - grep -Fxq 'SUMMARY assertions=107 expected=107 failures=1 infra=0' "$work/assertion.out" && + grep -Fxq 'SUMMARY assertions=109 expected=109 failures=1 infra=0' "$work/assertion.out" && ! grep -Fxq 'EXPERIMENT LOCAL LIFECYCLE PASS' "$work/assertion.out"; then pass AGG-006 "subcase assertion failure maps to one" else @@ -290,7 +290,7 @@ chmod +x "$shim/rmdir" cleanup_rc=0 run_replica "$work/cleanup.out" env PATH="$shim:$PATH" || cleanup_rc=$? if [ "$cleanup_rc" -eq 125 ] && - grep -Fxq 'SUMMARY assertions=107 expected=107 failures=0 infra=1' "$work/cleanup.out" && + grep -Fxq 'SUMMARY assertions=109 expected=109 failures=0 infra=1' "$work/cleanup.out" && ! grep -Fxq 'EXPERIMENT LOCAL LIFECYCLE PASS' "$work/cleanup.out"; then pass AGG-008 "cleanup uncertainty maps to one hundred twenty-five before the marker" else diff --git a/tests/experiment/install-state-cases.py b/tests/experiment/install-state-cases.py index 5c597ac..390063d 100644 --- a/tests/experiment/install-state-cases.py +++ b/tests/experiment/install-state-cases.py @@ -3,13 +3,14 @@ from __future__ import annotations -from contextlib import redirect_stderr, redirect_stdout +from contextlib import contextmanager, redirect_stderr, redirect_stdout import hashlib from importlib.util import module_from_spec, spec_from_file_location import io import json import os from pathlib import Path +import select import signal import stat import subprocess @@ -122,6 +123,17 @@ def start_cli(home: Path, source: Path) -> subprocess.Popen[bytes]: ) +def start_inspect(home: Path, name: str) -> subprocess.Popen[bytes]: + return subprocess.Popen( + [str(AGENT_LAB), "--home", str(home), "experiment", "inspect", name], + stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + env=command_environment(), + start_new_session=True, + ) + + def finish_process( process: subprocess.Popen[bytes], timeout: float = 30.0, @@ -267,6 +279,28 @@ def store_install( return None, None, error +def store_inspect( + home: Path, + name: str, +) -> tuple[int | None, dict[str, object] | None, BaseException | None]: + if STORE is None: + return None, None, STORE_LOAD_ERROR or RuntimeError("experiment store module is missing") + operation = getattr(STORE, "inspect_install", None) + if not callable(operation): + return None, None, RuntimeError("experiment_store.inspect_install is missing") + try: + value = operation(home, name) + if not isinstance(value, dict): + return None, None, RuntimeError("inspect_install returned a non-object") + return 0, value, None + except getattr(STORE, "StoreReject", ()) as error: + return 1, None, error + except getattr(STORE, "StoreInfrastructure", ()) as error: + return 125, None, error + except BaseException as error: # An uncontained production fault is RED, not harness infra. + return None, None, error + + def module_main( home: Path, arguments: list[str], @@ -317,6 +351,143 @@ def stop_at(observed: str) -> None: return 124 +def hard_exit_after_raw_publication(home: Path, source: Path) -> int: + """Exit after no-replace succeeds but before the final root becomes read-only.""" + + if STORE is None: + return 95 + target = home / "experiments" / "first-experiment" + pid = os.fork() + if pid == 0: + original_chmod = STORE.os.chmod + + def stop_before_final_chmod(path: object, mode: int, *args, **kwargs): + if Path(os.fsdecode(os.fspath(path))) == target and mode == 0o500: + os._exit(99) + return original_chmod(path, mode, *args, **kwargs) + + STORE.os.chmod = stop_before_final_chmod + result, _, _ = store_install(home, source) + os._exit(97 if result == 0 else 96) + deadline = time.monotonic() + 30.0 + while time.monotonic() < deadline: + waited, status = os.waitpid(pid, os.WNOHANG) + if waited == pid: + return os.waitstatus_to_exitcode(status) + time.sleep(0.01) + os.kill(pid, signal.SIGKILL) + os.waitpid(pid, 0) + global INFRA + INFRA += 1 + return 124 + + +def hard_exit_before_intent(home: Path, source: Path) -> int: + """Exit after the operation wrapper exists but before intent bytes are written.""" + + if STORE is None: + return 95 + pid = os.fork() + if pid == 0: + original_write_file = STORE._write_file + + def stop_before_intent(path, data, purpose, fault): + if purpose == "experiment intent": + os._exit(99) + return original_write_file(path, data, purpose, fault) + + STORE._write_file = stop_before_intent + result, _, _ = store_install(home, source) + os._exit(97 if result == 0 else 96) + deadline = time.monotonic() + 30.0 + while time.monotonic() < deadline: + waited, status = os.waitpid(pid, os.WNOHANG) + if waited == pid: + return os.waitstatus_to_exitcode(status) + time.sleep(0.01) + os.kill(pid, signal.SIGKILL) + os.waitpid(pid, 0) + global INFRA + INFRA += 1 + return 124 + + +def start_paused_publication(home: Path, source: Path) -> tuple[int, int, int]: + """Pause a child install while it holds the store lock before publication.""" + + ready_read, ready_write = os.pipe() + release_read, release_write = os.pipe() + pid = os.fork() + if pid == 0: + os.close(ready_read) + os.close(release_write) + paused = False + + def pause(point: str) -> None: + nonlocal paused + if point == "experiment envelope.before_noreplace" and not paused: + paused = True + os.write(ready_write, b"1") + if os.read(release_read, 1) != b"1": + os._exit(94) + + result, _, _ = store_install(home, source, fault=pause) + os._exit(0 if result == 0 else 96) + os.close(ready_write) + os.close(release_read) + readable, _, _ = select.select((ready_read,), (), (), 30.0) + if not readable or os.read(ready_read, 1) != b"1": + try: + os.kill(pid, signal.SIGKILL) + except ProcessLookupError: + pass + os.waitpid(pid, 0) + os.close(ready_read) + os.close(release_write) + global INFRA + INFRA += 1 + return 0, -1, -1 + os.close(ready_read) + return pid, release_write, 0 + + +def finish_child(pid: int, timeout: float = 30.0) -> int: + if pid <= 0: + return 124 + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + waited, status = os.waitpid(pid, os.WNOHANG) + if waited == pid: + return os.waitstatus_to_exitcode(status) + time.sleep(0.01) + try: + os.kill(pid, signal.SIGKILL) + except ProcessLookupError: + pass + os.waitpid(pid, 0) + global INFRA + INFRA += 1 + return 124 + + +def wait_for_lock_block(process: subprocess.Popen[bytes], timeout: float = 5.0) -> bool: + """Observe the Linux flock wait channel rather than infer blocking from a sleep.""" + + deadline = time.monotonic() + timeout + path = Path(f"/proc/{process.pid}/wchan") + while time.monotonic() < deadline: + if process.poll() is not None: + return False + try: + channel = path.read_text(encoding="ascii").strip() + except OSError: + channel = "" + if channel == "locks_lock_inode_wait": + return True + time.sleep(0.005) + return False + + def lock_is_blocked(path: Path) -> bool: program = ( "import fcntl, os, sys\n" @@ -455,16 +626,59 @@ def cross_device_lstat(path: object, *args, **kwargs): "install", str(direct_source), ) + + inspect_lock_home = new_home(root, "inspect-lock-home") + inspect_lock_install = cli( + inspect_lock_home, + "experiment", + "install", + str(direct_source), + ) + inspect_lock = inspect_lock_home / "state" / "locks" / "experiments.lock" + os.link(inspect_lock, root / "second-inspect-lock-link") + hardlink_inspect = cli( + inspect_lock_home, + "experiment", + "inspect", + "first-experiment", + ) + + blocking_home = new_home(root, "inspect-blocking-home") + install_pid, release_write, pause_status = start_paused_publication( + blocking_home, + direct_source, + ) + blocking_inspect = start_inspect(blocking_home, "first-experiment") + inspect_waited = wait_for_lock_block(blocking_inspect) + if release_write >= 0: + try: + os.write(release_write, b"1") + finally: + os.close(release_write) + blocking_install_rc = finish_child(install_pid) + blocking_inspect_result = finish_process(blocking_inspect) + blocking_inspect_value = json_object(blocking_inspect_result) check( "IST-LOCK-001", symlink_lock_result.returncode == 125 and lock_before == lock_after and hardlink_result.returncode == 125 - and not tuple((hardlink_lock_home / "experiments" / ".staging").iterdir()), - "the pre-created receipt-bound store lock is safe-opened with stable identity", + and not tuple((hardlink_lock_home / "experiments" / ".staging").iterdir()) + and inspect_lock_install.returncode == 0 + and hardlink_inspect.returncode == 125 + and pause_status == 0 + and inspect_waited + and blocking_install_rc == 0 + and blocking_inspect_result.returncode == 0 + and isinstance(blocking_inspect_value, dict) + and blocking_inspect_value.get("state") == "installed", + "the receipt-bound store lock protects install and read-only inspect", ( f"symlink={symlink_lock_result.returncode}/{lock_before != lock_after} " - f"hardlink={hardlink_result.returncode}" + f"hardlink={hardlink_result.returncode} inspect_install={inspect_lock_install.returncode} " + f"inspect_hardlink={hardlink_inspect.returncode} pause={pause_status} " + f"waited={inspect_waited} child={blocking_install_rc} " + f"inspect={blocking_inspect_result.returncode}/{blocking_inspect_value!r}" ), ) @@ -536,16 +750,46 @@ def create_racing_target(point: str) -> None: bound_before = fingerprint(bound_stage) bound_result = cli(bound_home, "experiment", "install", str(direct_source)) bound_after = fingerprint(bound_stage) + + cleanup_home = new_home(root, "foreign-cleanup-home") + cleanup_wrapper = ( + cleanup_home + / "experiments" + / ".staging" + / "experiment-install-cleanup" + ) + cleanup_artifact = cleanup_wrapper / "payload" / "artifact" + cleanup_artifact.mkdir(parents=True) + for directory_path in ( + cleanup_wrapper, + cleanup_wrapper / "payload", + cleanup_artifact, + ): + directory_path.chmod(0o700) + cleanup_canary = cleanup_artifact / "experiment.cue" + cleanup_canary.write_bytes(b"foreign cleanup canary\n") + cleanup_canary.chmod(0o600) + cleanup_before = fingerprint(cleanup_wrapper) + cleanup_result = cli( + cleanup_home, + "experiment", + "install", + str(direct_source), + ) + cleanup_after = fingerprint(cleanup_wrapper) check( "IST-BOUND-001", foreign_result.returncode == 125 and foreign_before == foreign_after and bound_result.returncode == 125 - and bound_before == bound_after, - "unknown or over-bound staging residue blocks mutation without broad deletion", + and bound_before == bound_after + and cleanup_result.returncode == 125 + and cleanup_before == cleanup_after, + "unknown, over-bound, or unproven cleanup residue is preserved", ( f"foreign={foreign_result.returncode}/{foreign_before != foreign_after} " - f"bound={bound_result.returncode}/{bound_before != bound_after}" + f"bound={bound_result.returncode}/{bound_before != bound_after} " + f"cleanup={cleanup_result.returncode}/{cleanup_before != cleanup_after}" ), ) @@ -579,15 +823,53 @@ def create_racing_target(point: str) -> None: f" link={linked_inspect.returncode} restore={restored_inspect.returncode} " f"mode={mode_inspect.returncode} digest={digest_inspect.returncode}" ) + + layout_home = new_home(root, "changing-layout-home") + layout_install = cli(layout_home, "experiment", "install", str(direct_source)) + layout_target = layout_home / "experiments" / "first-experiment" + layout_changed = False + layout_rc: int | None = None + layout_error: BaseException | None = None + if STORE is not None and layout_install.returncode == 0: + original_listdir = STORE.os.listdir + + def change_after_enumeration(path: object = "."): + nonlocal layout_changed + names = original_listdir(path) + try: + rendered = Path(os.fsdecode(os.fspath(path))) + except TypeError: + return names + if rendered == layout_target and not layout_changed: + layout_changed = True + layout_target.chmod(0o700) + foreign = layout_target / "foreign-after-enumeration" + foreign.write_bytes(b"foreign layout entry\n") + foreign.chmod(0o400) + layout_target.chmod(0o500) + return names + + STORE.os.listdir = change_after_enumeration + try: + layout_rc, _, layout_error = store_inspect( + layout_home, + "first-experiment", + ) + finally: + STORE.os.listdir = original_listdir check( "IST-STATE-003", installed.returncode == 0 and link_detected and restored and mode_detected - and digest_detected, - "inspect safe-reopens the whole receipt and detects link, mode, and byte drift", - tamper_detail, + and digest_detected + and layout_install.returncode == 0 + and layout_changed + and layout_rc == 125 + and layout_error is not None, + "inspect detects link, mode, byte, and concurrent layout drift", + f"{tamper_detail} layout={layout_install.returncode}/{layout_changed}/{layout_rc}/{layout_error!r}", ) identical_home = new_home(root, "identical-concurrency-home") @@ -694,6 +976,45 @@ def create_racing_target(point: str) -> None: f"read_changed={before_inspect_stage != after_inspect_stage}" ) + preintent_home = new_home(root, "preintent-crash-home") + preintent_stage = preintent_home / "experiments" / ".staging" + preintent_child_rc = hard_exit_before_intent(preintent_home, direct_source) + preintent_before = fingerprint(preintent_stage) + preintent_inspect = cli( + preintent_home, + "experiment", + "inspect", + "first-experiment", + ) + preintent_after_inspect = fingerprint(preintent_stage) + preintent_retry = cli( + preintent_home, + "experiment", + "install", + str(direct_source), + ) + preintent_value = json_object(preintent_retry) + + raw_home = new_home(root, "raw-publication-crash-home") + raw_child_rc = hard_exit_after_raw_publication(raw_home, direct_source) + raw_stage = raw_home / "experiments" / ".staging" + raw_before_inspect = fingerprint(raw_stage) + raw_inspect = cli( + raw_home, + "experiment", + "inspect", + "first-experiment", + ) + raw_after_inspect = fingerprint(raw_stage) + raw_retry = cli(raw_home, "experiment", "install", str(direct_source)) + raw_retry_value = json_object(raw_retry) + raw_final_inspect = cli( + raw_home, + "experiment", + "inspect", + "first-experiment", + ) + output_home = new_home(root, "result-output-home") output_rc, _, output_error = module_main( output_home, @@ -715,13 +1036,34 @@ def create_racing_target(point: str) -> None: and output_inspect.returncode == 0 and output_retry.returncode == 0 and isinstance(output_value, dict) - and output_value.get("changed") is False, + and output_value.get("changed") is False + and preintent_child_rc == 99 + and preintent_inspect.returncode == 1 + and preintent_before == preintent_after_inspect + and preintent_retry.returncode == 0 + and isinstance(preintent_value, dict) + and preintent_value.get("changed") is True + and not tuple(preintent_stage.iterdir()) + and raw_child_rc == 99 + and raw_inspect.returncode == 125 + and raw_before_inspect == raw_after_inspect + and raw_retry.returncode == 0 + and isinstance(raw_retry_value, dict) + and raw_retry_value.get("changed") is False + and raw_final_inspect.returncode == 0 + and not tuple(raw_stage.iterdir()), "fault seams preserve views, restart cleanup, and recover uncertain output", ( f"seam={seam_rc}/{seam_error!r} " f"missing={sorted(set(FAULT_POINTS)-set(observed_points))!r} " f"crashes={crash_failures[:3]!r} output={output_rc}/{output_error!r}/" - f"{output_inspect.returncode}/{output_retry.returncode}/{output_value!r}" + f"{output_inspect.returncode}/{output_retry.returncode}/{output_value!r} " + f"preintent={preintent_child_rc}/{preintent_inspect.returncode}/" + f"{preintent_before != preintent_after_inspect}/{preintent_retry.returncode}/" + f"{preintent_value!r} " + f"raw={raw_child_rc}/{raw_inspect.returncode}/" + f"{raw_before_inspect != raw_after_inspect}/{raw_retry.returncode}/" + f"{raw_retry_value!r}/{raw_final_inspect.returncode}" ), ) @@ -797,6 +1139,74 @@ def observe_locks(point: str) -> None: ), ) + provenance_home = new_home(root, "provenance-snapshot-home") + provenance_add = cli( + provenance_home, + "image", + "add", + "vendor.worker", + SUBJECT, + ) + provenance_source = source_directory( + root, + "provenance-source", + catalog_name="vendor.worker", + ) + initial_check = cli( + provenance_home, + "experiment", + "check", + str(provenance_source), + ) + initial_check_value = json_object(initial_check) + initial_catalog = None + if isinstance(initial_check_value, dict): + catalog_value = initial_check_value.get("catalog") + if isinstance(catalog_value, dict): + initial_catalog = catalog_value.get("local") + provenance_rc: int | None = None + provenance_value: dict[str, object] | None = None + provenance_error: BaseException | None = None + unrelated_result: subprocess.CompletedProcess[bytes] | None = None + provenance_record: dict[str, object] | None = None + if STORE is not None: + original_held_catalog = STORE._held_catalog_context + + @contextmanager + def mutate_unrelated_before_hold(home: Path, dependencies, fault): + nonlocal unrelated_result + unrelated_result = cli( + home, + "image", + "add", + "vendor.unrelated", + OTHER_SUBJECT, + ) + with original_held_catalog(home, dependencies, fault) as held: + yield held + + STORE._held_catalog_context = mutate_unrelated_before_hold + try: + provenance_rc, provenance_value, provenance_error = store_install( + provenance_home, + provenance_source, + ) + finally: + STORE._held_catalog_context = original_held_catalog + provenance_path = ( + provenance_home + / "experiments" + / "first-experiment" + / "records" + / "provenance.json" + ) + if provenance_path.is_file(): + try: + loaded_provenance = json.loads(provenance_path.read_bytes()) + except (OSError, UnicodeError, json.JSONDecodeError): + loaded_provenance = None + if isinstance(loaded_provenance, dict): + provenance_record = loaded_provenance platform_home = new_home(root, "platform-home") platform_source = source_directory(root, "platform-source") platform_before = fingerprint(platform_home) @@ -856,6 +1266,26 @@ def observed_open(path: object, flags: int, mode: int = 0o777, *, dir_fd=None): f"source_touched={source_touched} changed={platform_before != platform_after}" ), ) + check( + "IST-PROV-001", + provenance_add.returncode == 0 + and initial_check.returncode == 0 + and isinstance(initial_catalog, dict) + and unrelated_result is not None + and unrelated_result.returncode == 0 + and provenance_rc == 0 + and isinstance(provenance_value, dict) + and provenance_error is None + and isinstance(provenance_record, dict) + and provenance_record.get("catalog") == initial_catalog, + "provenance retains the authorized initial resolution snapshot across unrelated catalog mutation", + ( + f"add={provenance_add.returncode} check={initial_check.returncode}/{initial_catalog!r} " + f"unrelated={None if unrelated_result is None else unrelated_result.returncode} " + f"install={provenance_rc}/{provenance_error!r} " + f"stored={None if provenance_record is None else provenance_record.get('catalog')!r}" + ), + ) expected = [ "IST-STATE-001", @@ -868,11 +1298,12 @@ def observed_open(path: object, flags: int, mode: int = 0o777, *, dir_fd=None): "IST-CRASH-001", "IST-LIVE-001", "IST-PLAT-001", + "IST-PROV-001", ] if OBSERVED != expected: print(f"INFRA install state assertion identity drift: {OBSERVED!r}", file=sys.stderr) return 125 - print(f"SUMMARY assertions=10 expected=10 failures={FAILURES} infra={INFRA}") + print(f"SUMMARY assertions=11 expected=11 failures={FAILURES} infra={INFRA}") if INFRA: return 125 return 0 if FAILURES == 0 else 1 @@ -886,6 +1317,6 @@ def observed_open(path: object, flags: int, mode: int = 0o777, *, dir_fd=None): except BaseException as error: print(f"INFRA install state harness failed: {error!r}", file=sys.stderr) print( - f"SUMMARY assertions={len(OBSERVED)} expected=10 failures={FAILURES} infra=1" + f"SUMMARY assertions={len(OBSERVED)} expected=11 failures={FAILURES} infra=1" ) raise SystemExit(125) diff --git a/tests/experiment/install-store-cases.sh b/tests/experiment/install-store-cases.sh index fa66908..80bd5c7 100755 --- a/tests/experiment/install-store-cases.sh +++ b/tests/experiment/install-store-cases.sh @@ -6,7 +6,7 @@ agent_lab="$repo_root/scripts/agent-lab" bounded_helper="$repo_root/tests/helpers/run-bounded.py" fixture="$repo_root/tests/experiment/fixtures/directories/minimal" runtime_manifest="$repo_root/packaging/agent-lab-local.manifest" -expected_count=11 +expected_count=12 work="" failures=0 infrastructure=0 @@ -198,6 +198,9 @@ def canonical(value: object) -> bytes: def digest(data: bytes) -> str: return "sha256:" + hashlib.sha256(data).hexdigest() +def identity_digest(domain: bytes, value: object) -> str: + return digest(domain + canonical(value)) + def strings(value: object) -> set[str]: found: set[str] = set() if isinstance(value, str): @@ -259,17 +262,68 @@ assert decision["binding"]["planDigest"] == digest(canonical(plan)) assert isinstance(provenance, dict) and isinstance(provenance.get("apiVersion"), str) assert not any(item.startswith("/") for item in strings(provenance)) -receipt_strings = strings(receipt) -record_digests = { - digest(artifact_bytes), - digest(plan_bytes), - digest(decision_bytes), - digest(provenance_bytes), +selected_entries: list[dict[str, object]] = [] +for member in plan["spec"]["members"]: + resolved = member["resolvedImage"] + selected: dict[str, object] = { + "member": member["name"], + "origin": resolved["origin"], + "subject": resolved["subject"], + } + if resolved["origin"] in {"agent-lab", "local"}: + selected.update({ + "entryDigest": resolved["entryDigest"], + "generation": resolved["generation"], + "name": member["requestedSelector"]["catalogName"], + }) + selected_entries.append(selected) +selected_entries.sort(key=lambda item: (str(item["member"]).encode(), str(item["origin"]).encode())) + +identity = { + "authorizationDigest": decision["binding"]["authorizationDigest"], + "contractDigest": plan["contract"]["digest"], + "planDigest": decision["binding"]["planDigest"], + "selectedEntries": selected_entries, + "sourceDigest": decision["binding"]["sourceDigest"], +} +assert identity["sourceDigest"] == checked["source"]["digest"] +assert provenance["authorizationDigest"] == identity["authorizationDigest"] +assert provenance["contractDigest"] == identity["contractDigest"] +assert provenance["planDigest"] == identity["planDigest"] +assert provenance["selectedEntries"] == identity["selectedEntries"] +assert provenance["source"]["digest"] == identity["sourceDigest"] + +decision_digest = identity_digest(b"agent-lab.experiment-decision.v1\0", decision) +provenance_digest = identity_digest(b"agent-lab.experiment-provenance.v1\0", provenance) +records = { + "artifact/experiment.cue": { + "digest": digest(artifact_bytes), + "schema": "agent-lab/v0alpha1", + }, + "records/decision.json": { + "digest": decision_digest, + "schema": decision["apiVersion"], + }, + "records/plan.json": { + "digest": digest(plan_bytes), + "schema": plan["apiVersion"], + }, + "records/provenance.json": { + "digest": provenance_digest, + "schema": provenance["apiVersion"], + }, +} +installation_key = identity_digest(b"agent-lab.experiment-installation-key.v1\0", identity) +assert receipt == { + "apiVersion": "agent-lab.experiment-install/v0alpha1", + "identity": identity, + "installationKey": installation_key, + "kind": "ExperimentInstallationReceipt", + "name": name, + "records": records, } -assert record_digests <= receipt_strings -assert {plan["apiVersion"], decision["apiVersion"], provenance["apiVersion"], "agent-lab/v0alpha1"} <= receipt_strings -installation_key = receipt.get("installationKey") -assert isinstance(installation_key, str) and len(installation_key) == 71 and installation_key.startswith("sha256:") +receipt_digest = identity_digest(b"agent-lab.experiment-install-receipt.v1\0", receipt) +assert len({decision_digest, provenance_digest, receipt_digest}) == 3 result_bytes = result_path.read_bytes() result = json.loads(result_bytes) @@ -277,7 +331,7 @@ assert result_bytes == canonical(result) + b"\n" assert result.get("changed") is True assert result.get("name") == name assert result.get("installationKey") == installation_key -assert result.get("receiptDigest") == digest(receipt_bytes) +assert result.get("receiptDigest") == receipt_digest PY } @@ -304,6 +358,42 @@ else fail INST-UNKNOWN-001 "inspect reports an unknown name without mutating the initialized home" fi +name_home="$work/name-home" +init_home "$name_home" +name_source="$work/name-source" +name_63="aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" +mkdir "$name_source" +sed "s/first-experiment/$name_63/" "$fixture/experiment.cue" > "$name_source/experiment.cue" +capture name-check "$agent_lab" --home "$name_home" experiment check "$name_source" +name_check_rc="$CAPTURE_RC" +name_check_out="$CAPTURE_OUT" +name_check_err="$CAPTURE_ERR" +capture name-decision "$agent_lab" --home "$name_home" experiment authorize install "$name_source" +name_decision_rc="$CAPTURE_RC" +name_decision_out="$CAPTURE_OUT" +name_decision_err="$CAPTURE_ERR" +capture name-install "$agent_lab" --home "$name_home" experiment install "$name_source" +name_install_rc="$CAPTURE_RC" +name_install_out="$CAPTURE_OUT" +name_install_err="$CAPTURE_ERR" +capture name-inspect "$agent_lab" --home "$name_home" experiment inspect "$name_63" +name_inspect_rc="$CAPTURE_RC" +name_inspect_out="$CAPTURE_OUT" +name_inspect_err="$CAPTURE_ERR" +if [ "${#name_63}" -eq 63 ] \ + && [ "$name_check_rc" -eq 0 ] && [ ! -s "$name_check_err" ] \ + && jq -e --arg name "$name_63" '.plan.metadata.requestedName == $name' "$name_check_out" >/dev/null 2>&1 \ + && [ "$name_decision_rc" -eq 0 ] && [ ! -s "$name_decision_err" ] \ + && jq -e --arg name "$name_63" '.verdict == "permit" and .resource.requestedName == $name' "$name_decision_out" >/dev/null 2>&1 \ + && [ "$name_install_rc" -eq 0 ] && [ ! -s "$name_install_err" ] \ + && jq -e --arg name "$name_63" '.changed == true and .name == $name' "$name_install_out" >/dev/null 2>&1 \ + && [ "$name_inspect_rc" -eq 0 ] && [ ! -s "$name_inspect_err" ] \ + && jq -e --arg name "$name_63" '.state == "installed" and .name == $name' "$name_inspect_out" >/dev/null 2>&1; then + pass INST-NAME-001 "the maximum-length valid Experiment name checks, authorizes, installs, and inspects" +else + fail INST-NAME-001 "the maximum-length valid Experiment name checks, authorizes, installs, and inspects" +fi + capture core-check "$agent_lab" --home "$core_home" experiment check "$fixture" check_out="$CAPTURE_OUT" check_rc="$CAPTURE_RC" @@ -494,7 +584,7 @@ fi expected="$work/expected" printf '%s\n' \ - INST-HOME-001 INST-UNKNOWN-001 INST-PERMIT-001 INST-RECEIPT-001 \ + INST-HOME-001 INST-UNKNOWN-001 INST-NAME-001 INST-PERMIT-001 INST-RECEIPT-001 \ INST-INSPECT-001 INST-RETRY-001 INST-CONFLICT-001 INST-DENY-001 \ INST-FORGE-001 INST-NOEF-001 INST-LOCAL-001 > "$expected" if ! cmp -s "$expected" "$observed"; then diff --git a/tests/experiment/local-lifecycle-cases.sh b/tests/experiment/local-lifecycle-cases.sh index 5ec43d0..c3effe4 100755 --- a/tests/experiment/local-lifecycle-cases.sh +++ b/tests/experiment/local-lifecycle-cases.sh @@ -9,7 +9,7 @@ subcases=( "$repo_root/tests/experiment/install-store-cases.sh" "$repo_root/tests/experiment/install-state-cases.py" ) -expected_count=107 +expected_count=109 work="" cleanup_work() { @@ -53,9 +53,10 @@ printf '%s\n' \ M-CAT-NOEF-001 M-CAT-ADMIT-001 M-CAT-ATOM-001 M-CAT-DUR-001 M-CAT-STAGE-001 \ INST-HOME-001 INST-UNKNOWN-001 INST-PERMIT-001 INST-RECEIPT-001 \ INST-INSPECT-001 INST-RETRY-001 INST-CONFLICT-001 INST-DENY-001 \ - INST-FORGE-001 INST-NOEF-001 INST-LOCAL-001 \ + INST-FORGE-001 INST-NOEF-001 INST-LOCAL-001 INST-NAME-001 \ IST-STATE-001 IST-LOCK-001 IST-STATE-002 IST-BOUND-001 IST-STATE-003 \ - IST-CONC-001 IST-CONC-002 IST-CRASH-001 IST-LIVE-001 IST-PLAT-001 > "$expected" + IST-CONC-001 IST-CONC-002 IST-CRASH-001 IST-LIVE-001 IST-PLAT-001 \ + IST-PROV-001 > "$expected" : > "$observed" infrastructure=0 From 0b1cab02763fdf5b5699042b91068d2ecd69ce53 Mon Sep 17 00:00:00 2001 From: xormania <127287135+xormania@users.noreply.github.com> Date: Sun, 2 Aug 2026 02:42:33 -0400 Subject: [PATCH 047/158] test(experiment): correct selected-entry oracle --- tests/experiment/install-store-cases.sh | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/tests/experiment/install-store-cases.sh b/tests/experiment/install-store-cases.sh index 80bd5c7..90ef9e5 100755 --- a/tests/experiment/install-store-cases.sh +++ b/tests/experiment/install-store-cases.sh @@ -265,19 +265,16 @@ assert not any(item.startswith("/") for item in strings(provenance)) selected_entries: list[dict[str, object]] = [] for member in plan["spec"]["members"]: resolved = member["resolvedImage"] - selected: dict[str, object] = { - "member": member["name"], + if resolved["origin"] == "direct": + continue + selected_entries.append({ + "entryDigest": resolved["entryDigest"], + "generation": resolved["generation"], + "name": member["requestedSelector"]["catalogName"], "origin": resolved["origin"], "subject": resolved["subject"], - } - if resolved["origin"] in {"agent-lab", "local"}: - selected.update({ - "entryDigest": resolved["entryDigest"], - "generation": resolved["generation"], - "name": member["requestedSelector"]["catalogName"], - }) - selected_entries.append(selected) -selected_entries.sort(key=lambda item: (str(item["member"]).encode(), str(item["origin"]).encode())) + }) +selected_entries.sort(key=lambda item: (str(item["origin"]).encode(), str(item["name"]).encode())) identity = { "authorizationDigest": decision["binding"]["authorizationDigest"], From a405ffe26592fd13ed2c70eefd50c04ed2f16f80 Mon Sep 17 00:00:00 2001 From: xormania <127287135+xormania@users.noreply.github.com> Date: Sun, 2 Aug 2026 02:43:05 -0400 Subject: [PATCH 048/158] test(experiment): preserve layout drift injection --- tests/experiment/install-state-cases.py | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/tests/experiment/install-state-cases.py b/tests/experiment/install-state-cases.py index 390063d..0309402 100644 --- a/tests/experiment/install-state-cases.py +++ b/tests/experiment/install-state-cases.py @@ -831,16 +831,12 @@ def create_racing_target(point: str) -> None: layout_rc: int | None = None layout_error: BaseException | None = None if STORE is not None and layout_install.returncode == 0: - original_listdir = STORE.os.listdir + original_directory_names = STORE._directory_names - def change_after_enumeration(path: object = "."): + def change_after_enumeration(path: Path): nonlocal layout_changed - names = original_listdir(path) - try: - rendered = Path(os.fsdecode(os.fspath(path))) - except TypeError: - return names - if rendered == layout_target and not layout_changed: + names = original_directory_names(path) + if path == layout_target and not layout_changed: layout_changed = True layout_target.chmod(0o700) foreign = layout_target / "foreign-after-enumeration" @@ -849,14 +845,14 @@ def change_after_enumeration(path: object = "."): layout_target.chmod(0o500) return names - STORE.os.listdir = change_after_enumeration + STORE._directory_names = change_after_enumeration try: layout_rc, _, layout_error = store_inspect( layout_home, "first-experiment", ) finally: - STORE.os.listdir = original_listdir + STORE._directory_names = original_directory_names check( "IST-STATE-003", installed.returncode == 0 From 2949edbb03a649b5ee5b2be377e5dc929aef4619 Mon Sep 17 00:00:00 2001 From: xormania <127287135+xormania@users.noreply.github.com> Date: Sun, 2 Aug 2026 02:46:43 -0400 Subject: [PATCH 049/158] test(experiment): align drift probe helper contract --- tests/experiment/install-state-cases.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/experiment/install-state-cases.py b/tests/experiment/install-state-cases.py index 0309402..f00d162 100644 --- a/tests/experiment/install-state-cases.py +++ b/tests/experiment/install-state-cases.py @@ -833,9 +833,9 @@ def create_racing_target(point: str) -> None: if STORE is not None and layout_install.returncode == 0: original_directory_names = STORE._directory_names - def change_after_enumeration(path: Path): + def change_after_enumeration(path: Path, purpose: str, maximum: int): nonlocal layout_changed - names = original_directory_names(path) + names = original_directory_names(path, purpose, maximum) if path == layout_target and not layout_changed: layout_changed = True layout_target.chmod(0o700) From cf80ae38396aa51bc3cdf536008f10076fb4f6f9 Mon Sep 17 00:00:00 2001 From: xormania <127287135+xormania@users.noreply.github.com> Date: Sun, 2 Aug 2026 02:49:34 -0400 Subject: [PATCH 050/158] fix(experiment): recover store publication --- scripts/experiment_store.py | 206 +++++++++++++++++++++++++++++------- 1 file changed, 169 insertions(+), 37 deletions(-) diff --git a/scripts/experiment_store.py b/scripts/experiment_store.py index 1a5e238..f6a3dcb 100644 --- a/scripts/experiment_store.py +++ b/scripts/experiment_store.py @@ -20,6 +20,9 @@ INSTALL_KEY_DOMAIN = b"agent-lab.experiment-installation-key.v1\0" +DECISION_DOMAIN = b"agent-lab.experiment-decision.v1\0" +PROVENANCE_DOMAIN = b"agent-lab.experiment-provenance.v1\0" +RECEIPT_DOMAIN = b"agent-lab.experiment-install-receipt.v1\0" SOURCE_DIGEST_DOMAIN = b"agent-lab.experiment-tree.v1\0" STAGE_PAYLOAD_DOMAIN = b"agent-lab.experiment-stage-payload.v1\0" INSTALL_API = "agent-lab.experiment-install/v0alpha1" @@ -34,6 +37,7 @@ MAX_ARTIFACT_BYTES = 262_144 MAX_RECORD_BYTES = 1_048_576 SAFE_COMPONENT = re.compile(r"^[a-z][a-z0-9-]{0,47}$") +EXPERIMENT_NAME = re.compile(r"^[a-z](?:[a-z0-9-]{0,61}[a-z0-9])?$") IMAGE_COMPONENT = re.compile(r"^[a-z][a-z0-9]*(?:-[a-z0-9]+)*$") SHA256 = re.compile(r"^sha256:[0-9a-f]{64}$") PAYLOAD_DIRECTORIES = {"payload", "payload/artifact", "payload/records"} @@ -437,7 +441,12 @@ def _revalidate_authority(authority: HomeAuthority) -> None: @contextmanager -def _store_lock(authority: HomeAuthority, fault: FaultHook | None) -> Iterator[int]: +def _store_lock( + authority: HomeAuthority, + fault: FaultHook | None, + *, + exclusive: bool = True, +) -> Iterator[int]: maximum = len(LOCK_SCHEMA.encode("ascii") + b"\n") path = authority.lock descriptor = -1 @@ -472,7 +481,7 @@ def _store_lock(authority: HomeAuthority, fault: FaultHook | None) -> Iterator[i != (authority.lock_device, authority.lock_inode) ): _infra("Experiment store lock identity changed before acquisition") - fcntl.flock(descriptor, fcntl.LOCK_EX) + fcntl.flock(descriptor, fcntl.LOCK_EX if exclusive else fcntl.LOCK_SH) held = os.fstat(descriptor) current = path.lstat() expected = (authority.lock_device, authority.lock_inode) @@ -490,7 +499,8 @@ def _store_lock(authority: HomeAuthority, fault: FaultHook | None) -> Iterator[i if os.read(descriptor, maximum + 1) != LOCK_SCHEMA.encode("ascii") + b"\n": _infra("Experiment store lock receipt is malformed") os.lseek(descriptor, 0, os.SEEK_SET) - _fault(fault, "experiment store lock.after_acquire") + if exclusive: + _fault(fault, "experiment store lock.after_acquire") yield descriptor except StoreError: raise @@ -572,7 +582,13 @@ def _selected_entries(plan: dict[str, object]) -> list[dict[str, object]]: if key in selected and selected[key] != entry: raise ValueError("inconsistent selected identity") selected[key] = entry - return [selected[key] for key in sorted(selected, key=lambda item: (item[0].encode(), item[1].encode()))] + return [ + selected[key] + for key in sorted( + selected, + key=lambda item: (item[0].encode(), item[1].encode()), + ) + ] except (KeyError, TypeError, ValueError) as error: _infra("Experiment plan has invalid selected-entry identities", error) @@ -593,7 +609,7 @@ def _local_dependencies(selected: Sequence[dict[str, object]]) -> tuple[dict[str def _verify_held_catalog( held: object, dependencies: Sequence[dict[str, object]], -) -> dict[str, object]: +) -> None: try: if not isinstance(held, dict) or set(held) != {"catalog", "records"}: raise ValueError("held envelope") @@ -623,7 +639,6 @@ def _verify_held_catalog( or record.get("subject") != expected["subject"] ): raise ValueError("held selected entry") - return {"revision": catalog["revision"], "snapshotDigest": catalog["snapshotDigest"]} except (KeyError, TypeError, ValueError) as error: _infra("held local image catalog evidence is invalid", error) @@ -691,6 +706,18 @@ def _candidate( _infra("source snapshot is malformed") if source_digest != _source_digest(source_data): _infra("source snapshot digest is inconsistent") + local_selected = [item for item in selected if item["origin"] == "local"] + if local_selected: + if ( + not isinstance(catalog_evidence, dict) + or set(catalog_evidence) != {"revision", "snapshotDigest"} + or type(catalog_evidence.get("revision")) is not int + or int(catalog_evidence["revision"]) < 1 + or SHA256.fullmatch(str(catalog_evidence.get("snapshotDigest"))) is None + ): + _infra("initial local image catalog evidence is invalid") + elif catalog_evidence is not None: + _infra("unexpected initial local image catalog evidence") identity = _installation_identity(source_digest, plan, decision, selected) installation_key = digest(INSTALL_KEY_DOMAIN + canonical(identity)) plan_bytes = canonical(plan) + b"\n" @@ -726,7 +753,7 @@ def _candidate( "schema": "agent-lab/v0alpha1", }, "records/decision.json": { - "digest": digest(decision_bytes), + "digest": digest(DECISION_DOMAIN + canonical(decision)), "schema": decision.get("apiVersion"), }, "records/plan.json": { @@ -734,7 +761,7 @@ def _candidate( "schema": plan.get("apiVersion"), }, "records/provenance.json": { - "digest": digest(provenance_bytes), + "digest": digest(PROVENANCE_DOMAIN + canonical(provenance)), "schema": PROVENANCE_API, }, } @@ -742,7 +769,7 @@ def _candidate( requested_name = plan["metadata"]["requestedName"] # type: ignore[index] except (KeyError, TypeError) as error: _infra("Experiment plan has no requested name", error) - if not isinstance(requested_name, str) or SAFE_COMPONENT.fullmatch(requested_name) is None: + if not isinstance(requested_name, str) or EXPERIMENT_NAME.fullmatch(requested_name) is None: _infra("Experiment plan requested name is unsafe") receipt = { "apiVersion": INSTALL_API, @@ -754,16 +781,23 @@ def _candidate( } receipt_bytes = canonical(receipt) + b"\n" files["records/install.json"] = receipt_bytes - return files, receipt, installation_key, digest(receipt_bytes) + return files, receipt, installation_key, digest(RECEIPT_DOMAIN + canonical(receipt)) def _directory_names(path: Path, purpose: str, maximum: int) -> tuple[str, ...]: try: - names = os.listdir(path) + names: list[str] = [] + with os.scandir(path) as entries: + for entry in entries: + if len(names) >= maximum: + _infra(f"{purpose} exceeds its fixed entry bound") + if not isinstance(entry.name, str): + _infra(f"{purpose} contains an invalid name") + names.append(entry.name) + except StoreError: + raise except OSError as error: _infra(f"{purpose} cannot be enumerated", error) - if len(names) > maximum: - _infra(f"{purpose} exceeds its fixed entry bound") try: return tuple(sorted(names, key=lambda item: os.fsencode(item))) except (TypeError, UnicodeError) as error: @@ -807,21 +841,26 @@ def _verify_envelope( *, root_modes: tuple[int, ...] = (0o500,), ) -> VerifiedInstall: - _verify_directory(path, modes=root_modes, device=device) - if _directory_names(path, "installed envelope", 2) != ("artifact", "records"): + if EXPERIMENT_NAME.fullmatch(expected_name) is None: + _infra("installed Experiment name is unsafe") + root_before = _verify_directory(path, modes=root_modes, device=device) + root_names = ("artifact", "records") + if _directory_names(path, "installed envelope", 2) != root_names: _infra("installed envelope layout is not closed") artifact_dir = path / "artifact" records_dir = path / "records" - _verify_directory(artifact_dir, modes=(0o500,), device=device) - _verify_directory(records_dir, modes=(0o500,), device=device) - if _directory_names(artifact_dir, "installed artifact", 1) != ("experiment.cue",): - _infra("installed artifact layout is not closed") - if _directory_names(records_dir, "installed records", 4) != ( + artifact_before = _verify_directory(artifact_dir, modes=(0o500,), device=device) + records_before = _verify_directory(records_dir, modes=(0o500,), device=device) + artifact_names = ("experiment.cue",) + records_names = ( "decision.json", "install.json", "plan.json", "provenance.json", - ): + ) + if _directory_names(artifact_dir, "installed artifact", 1) != artifact_names: + _infra("installed artifact layout is not closed") + if _directory_names(records_dir, "installed records", 4) != records_names: _infra("installed record layout is not closed") raw = { "artifact/experiment.cue": _read_file( @@ -955,17 +994,49 @@ def _verify_envelope( "records/plan.json": str(plan["apiVersion"]), "records/provenance.json": PROVENANCE_API, } + expected_digests = { + "artifact/experiment.cue": digest(raw["artifact/experiment.cue"]), + "records/decision.json": digest(DECISION_DOMAIN + canonical(decision)), + "records/plan.json": digest(raw["records/plan.json"]), + "records/provenance.json": digest(PROVENANCE_DOMAIN + canonical(provenance)), + } for record_path in RECORD_PATHS: record_digest, schema = _record_schema(receipt, record_path) - if record_digest != digest(raw[record_path]) or schema != expected_schemas[record_path]: + if record_digest != expected_digests[record_path] or schema != expected_schemas[record_path]: raise ValueError("record digest") except (KeyError, TypeError, ValueError) as error: _infra("installed envelope does not match its receipt", error) + directory_checks = ( + (path, root_before, root_modes, root_names, "installed envelope", 2), + ( + artifact_dir, + artifact_before, + (0o500,), + artifact_names, + "installed artifact", + 1, + ), + ( + records_dir, + records_before, + (0o500,), + records_names, + "installed records", + 4, + ), + ) + for directory, before, modes, expected_names, purpose, maximum in directory_checks: + after = _verify_directory(directory, modes=modes, device=device) + if ( + _identity(before) != _identity(after) + or _directory_names(directory, purpose, maximum) != expected_names + ): + _infra(f"{purpose} changed while being verified") file_digests = {item: digest(data) for item, data in raw.items()} return VerifiedInstall( expected_name, installation_key, - digest(raw["records/install.json"]), + digest(RECEIPT_DOMAIN + canonical(receipt)), file_digests, ) @@ -993,7 +1064,7 @@ def _validate_intent(value: dict[str, object]) -> None: or value["apiVersion"] != INTENT_API or value["phase"] != "prepared" or not isinstance(value["name"], str) - or SAFE_COMPONENT.fullmatch(value["name"]) is None + or EXPERIMENT_NAME.fullmatch(value["name"]) is None or SHA256.fullmatch(str(value["installationKey"])) is None or SHA256.fullmatch(str(value["receiptDigest"])) is None or not isinstance(files, dict) @@ -1006,7 +1077,6 @@ def _validate_intent(value: dict[str, object]) -> None: } or any(SHA256.fullmatch(str(item)) is None for item in files.values()) or value["payloadDigest"] != digest(STAGE_PAYLOAD_DOMAIN + canonical(files)) - or value["receiptDigest"] != files["records/install.json"] ): raise ValueError("intent") except (KeyError, TypeError, ValueError) as error: @@ -1052,9 +1122,9 @@ def _scan_wrapper(authority: HomeAuthority, path: Path, *, cleanup: bool) -> dic _infra("Experiment staging state contains an unsafe type") intent_path = path / "intent.json" if "intent.json" not in found: - if cleanup: + if cleanup and not found: return None - _infra("Experiment operation wrapper has no durable intent") + _infra("Experiment staging wrapper has no durable intent") value = _parse_object( _read_file( intent_path, @@ -1323,8 +1393,21 @@ def _cleanup_operation(authority: HomeAuthority, wrapper: Path) -> None: _finish_cleanup(authority, cleanup) -def _intent_matches_final( +def _cleanup_empty_operation(authority: HomeAuthority, wrapper: Path) -> None: + if wrapper != authority.staging / OPERATION_WRAPPER: + _infra("Experiment empty operation cleanup target changed") + _verify_directory(wrapper, modes=(0o700,), device=authority.store_device) + if _directory_names(wrapper, "empty Experiment operation wrapper", 0): + _infra("Experiment operation wrapper has no durable intent") + cleanup = authority.staging / CLEANUP_WRAPPER + _rename_noreplace(wrapper, cleanup) + _fsync_directory(authority.staging, "Experiment empty cleanup handoff") + _finish_cleanup(authority, cleanup) + + +def _recover_intent_final( authority: HomeAuthority, + wrapper: Path, intent: dict[str, object], ) -> bool: name = str(intent["name"]) @@ -1334,7 +1417,18 @@ def _intent_matches_final( return False if state != "directory": _infra("Experiment staged operation conflicts with an ambiguous final target") - verified = _verify_envelope(target, name, authority.store_device) + target_metadata = _verify_directory( + target, + modes=(0o500, 0o700), + device=authority.store_device, + ) + target_mode = stat.S_IMODE(target_metadata.st_mode) + verified = _verify_envelope( + target, + name, + authority.store_device, + root_modes=(target_mode,), + ) files = intent["files"] assert isinstance(files, dict) if ( @@ -1343,6 +1437,32 @@ def _intent_matches_final( or any(verified.file_digests.get(path) != expected for path, expected in files.items()) ): _infra("Experiment staged operation conflicts with the final installation") + _fsync_directory( + wrapper, + "Experiment publication source recovery", + modes=(0o700,), + ) + if target_mode == 0o700: + try: + os.chmod(target, 0o500, follow_symlinks=False) + except OSError as error: + _infra("recovered Experiment mode is uncertain", error) + _fsync_directory( + target, + "Experiment recovered envelope", + modes=(0o500,), + ) + _fsync_directory(authority.store, "Experiment committed recovery") + recovered = _verify_envelope(target, name, authority.store_device) + if ( + recovered.installation_key != intent["installationKey"] + or recovered.receipt_digest != intent["receiptDigest"] + or any( + recovered.file_digests.get(path) != expected + for path, expected in files.items() + ) + ): + _infra("recovered Experiment does not match its durable intent") return True @@ -1354,17 +1474,20 @@ def _reconcile(authority: HomeAuthority) -> None: if names == (CLEANUP_WRAPPER,): cleanup = authority.staging / CLEANUP_WRAPPER intent = _scan_wrapper(authority, cleanup, cleanup=True) - if intent is not None and _intent_matches_final(authority, intent): - _fsync_directory(authority.store, "Experiment committed recovery") + if intent is not None: + _recover_intent_final(authority, cleanup, intent) _finish_cleanup(authority, cleanup) return if names != (OPERATION_WRAPPER,): _infra("Experiment staging root contains an unknown wrapper") wrapper = authority.staging / OPERATION_WRAPPER + _verify_directory(wrapper, modes=(0o700,), device=authority.store_device) + if not _directory_names(wrapper, "Experiment operation wrapper", 2): + _cleanup_empty_operation(authority, wrapper) + return intent = _scan_wrapper(authority, wrapper, cleanup=False) assert intent is not None - if _intent_matches_final(authority, intent): - _fsync_directory(authority.store, "Experiment committed recovery") + _recover_intent_final(authority, wrapper, intent) _cleanup_operation(authority, wrapper) @@ -1423,6 +1546,7 @@ def install_directory( manifest = experiment.authored_manifest(snapshot) resolution = experiment.cue_plan_with_evidence(manifest) plan = resolution.plan + initial_catalog = resolution.local_catalog if not isinstance(plan, dict): _infra("CUE produced a malformed Experiment plan") decision, status = experiment.authorize_plan(plan, snapshot.digest) @@ -1444,7 +1568,8 @@ def install_directory( try: held_context = _held_catalog_context(authority.home, dependencies, fault) with held_context as held: - catalog_evidence = _verify_held_catalog(held, dependencies) if dependencies else None + if dependencies: + _verify_held_catalog(held, dependencies) with _store_lock(authority, fault): _revalidate_authority(authority) _reconcile(authority) @@ -1453,7 +1578,7 @@ def install_directory( plan, decision, selected, - catalog_evidence, + initial_catalog, ) name = str(plan["metadata"]["requestedName"]) # type: ignore[index] existing = _existing(authority, name) @@ -1477,6 +1602,11 @@ def install_directory( _revalidate_authority(authority) _fault(fault, "experiment envelope.before_noreplace") _rename_noreplace(wrapper / "payload", authority.store / name) + _fsync_directory( + wrapper, + "Experiment publication source parent", + modes=(0o700,), + ) try: os.chmod(authority.store / name, 0o500, follow_symlinks=False) except OSError as error: @@ -1512,11 +1642,13 @@ def install_directory( def inspect_install(home: Path, name: str) -> dict[str, object]: """Read-only verification and identity projection for one installed name.""" - if not isinstance(name, str) or SAFE_COMPONENT.fullmatch(name) is None: + if not isinstance(name, str) or EXPERIMENT_NAME.fullmatch(name) is None: _reject("Experiment name is invalid") authority = _load_home(Path(home)) try: - installed = _existing(authority, name) + with _store_lock(authority, None, exclusive=False): + _revalidate_authority(authority) + installed = _existing(authority, name) except StoreError: raise except OSError as error: From 1da349331f9dd86894870071c3c9c1626092d4fe Mon Sep 17 00:00:00 2001 From: xormania <127287135+xormania@users.noreply.github.com> Date: Sun, 2 Aug 2026 02:52:28 -0400 Subject: [PATCH 051/158] test(experiment): verify installed store runtime --- tests/experiment/install-store-cases.sh | 159 +++++++++++++++++++++++- 1 file changed, 157 insertions(+), 2 deletions(-) diff --git a/tests/experiment/install-store-cases.sh b/tests/experiment/install-store-cases.sh index 90ef9e5..119ddf2 100755 --- a/tests/experiment/install-store-cases.sh +++ b/tests/experiment/install-store-cases.sh @@ -6,7 +6,7 @@ agent_lab="$repo_root/scripts/agent-lab" bounded_helper="$repo_root/tests/helpers/run-bounded.py" fixture="$repo_root/tests/experiment/fixtures/directories/minimal" runtime_manifest="$repo_root/packaging/agent-lab-local.manifest" -expected_count=12 +expected_count=13 work="" failures=0 infrastructure=0 @@ -579,11 +579,166 @@ else fail INST-LOCAL-001 "removed local selector blocks retry while retained installation remains inspectable" fi +installed_source="$work/installed-runtime-source" +installed_unavailable="$work/installed-runtime-source-unavailable" +installed_prefix="$work/installed-prefix" +installed_home="$work/installed-home" +installed_candidate="$work/installed-candidate" +installed_tools="$work/installed-pinned-tools" +installed_unrelated="$work/installed-unrelated" +installed_fixture_ok=1 +mkdir -p \ + "$installed_source/packaging" \ + "$installed_source/scripts" \ + "$installed_candidate" \ + "$installed_tools/cue" \ + "$installed_tools/cedar" \ + "$installed_unrelated" || installed_fixture_ok=0 +while IFS= read -r runtime_name; do + if [ -z "$runtime_name" ] || [ ! -f "$repo_root/$runtime_name" ]; then + installed_fixture_ok=0 + continue + fi + mkdir -p "$installed_source/$(dirname -- "$runtime_name")" || installed_fixture_ok=0 + cp "$repo_root/$runtime_name" "$installed_source/$runtime_name" || installed_fixture_ok=0 +done < "$runtime_manifest" +cp "$runtime_manifest" "$installed_source/packaging/agent-lab-local.manifest" \ + || installed_fixture_ok=0 +cp "$repo_root/scripts/install-local" "$repo_root/scripts/install-local.py" \ + "$installed_source/scripts/" || installed_fixture_ok=0 +cp "$fixture/experiment.cue" "$installed_candidate/experiment.cue" \ + || installed_fixture_ok=0 +cp -a "$repo_root/.cache/dev/tools/cue/." "$installed_tools/cue/" \ + || installed_fixture_ok=0 +cp -a "$repo_root/.cache/dev/tools/cedar/." "$installed_tools/cedar/" \ + || installed_fixture_ok=0 +chmod +x "$installed_source/scripts/install-local" "$installed_source/scripts/agent-lab" \ + || installed_fixture_ok=0 + +installed_bundle_rc=125 +installed_init_rc=125 +installed_first_rc=125 +installed_inspect_rc=125 +installed_retry_rc=125 +installed_prefix_before="unavailable" +installed_prefix_after="unavailable" +installed_home_after_first="unavailable" +installed_home_after_inspect="unavailable" +installed_home_after_retry="unavailable" +installed_bundle_out="$work/installed-bundle.out" +installed_bundle_err="$work/installed-bundle.err" +installed_init_err="$work/installed-init.err" +installed_first_out="$work/installed-first.out" +installed_first_err="$work/installed-first.err" +installed_inspect_out="$work/installed-inspect.out" +installed_inspect_err="$work/installed-inspect.err" +installed_retry_out="$work/installed-retry.out" +installed_retry_err="$work/installed-retry.err" +: > "$installed_bundle_out" +: > "$installed_bundle_err" +: > "$installed_init_err" +: > "$installed_first_out" +: > "$installed_first_err" +: > "$installed_inspect_out" +: > "$installed_inspect_err" +: > "$installed_retry_out" +: > "$installed_retry_err" + +capture_installed() { + local label="$1" + shift + capture "$label" env -i PATH=/usr/bin:/bin LANG=C LC_ALL=C \ + /bin/sh -c 'cd "$1" || exit 125; shift; exec "$@"' \ + agent-lab-installed "$installed_unrelated" \ + "$installed_prefix/bin/agent-lab" --home "$installed_home" "$@" +} + +if [ "$installed_fixture_ok" -eq 1 ]; then + capture installed-bundle env -i PATH=/usr/bin:/bin LANG=C LC_ALL=C \ + "$installed_source/scripts/install-local" --prefix "$installed_prefix" + installed_bundle_rc="$CAPTURE_RC" + installed_bundle_out="$CAPTURE_OUT" + installed_bundle_err="$CAPTURE_ERR" + installed_prefix_before="$(state_receipt "$installed_prefix")" || infrastructure=1 + if ! mv "$installed_source" "$installed_unavailable"; then + installed_fixture_ok=0 + infrastructure=1 + fi + + capture_installed installed-init init + installed_init_rc="$CAPTURE_RC" + installed_init_err="$CAPTURE_ERR" + if [ "$installed_init_rc" -eq 0 ]; then + cp -a "$installed_tools/cue/." "$installed_home/cache/tools/cue/" \ + || installed_fixture_ok=0 + cp -a "$installed_tools/cedar/." "$installed_home/cache/tools/cedar/" \ + || installed_fixture_ok=0 + if [ "$installed_fixture_ok" -ne 1 ]; then + infrastructure=1 + fi + fi + + capture_installed installed-first experiment install "$installed_candidate" + installed_first_rc="$CAPTURE_RC" + installed_first_out="$CAPTURE_OUT" + installed_first_err="$CAPTURE_ERR" + installed_home_after_first="$(state_receipt "$installed_home")" || infrastructure=1 + + capture_installed installed-inspect experiment inspect first-experiment + installed_inspect_rc="$CAPTURE_RC" + installed_inspect_out="$CAPTURE_OUT" + installed_inspect_err="$CAPTURE_ERR" + installed_home_after_inspect="$(state_receipt "$installed_home")" || infrastructure=1 + + capture_installed installed-retry experiment install "$installed_candidate" + installed_retry_rc="$CAPTURE_RC" + installed_retry_out="$CAPTURE_OUT" + installed_retry_err="$CAPTURE_ERR" + installed_home_after_retry="$(state_receipt "$installed_home")" || infrastructure=1 + installed_prefix_after="$(state_receipt "$installed_prefix")" || infrastructure=1 +else + infrastructure=1 +fi + +installed_key="$(jq -r '.installationKey // empty' "$installed_first_out" 2>/dev/null)" +installed_receipt="$(jq -r '.receiptDigest // empty' "$installed_first_out" 2>/dev/null)" +installed_target="$(readlink -f "$installed_prefix/bin/agent-lab" 2>/dev/null || true)" +if [ "$installed_fixture_ok" -eq 1 ] \ + && [ "$installed_bundle_rc" -eq 0 ] && [ ! -s "$installed_bundle_err" ] \ + && grep -Eq '^installed:[0-9a-f]{64}$' "$installed_bundle_out" \ + && [ "$installed_init_rc" -eq 0 ] && [ ! -s "$installed_init_err" ] \ + && [ "$installed_first_rc" -eq 0 ] && [ ! -s "$installed_first_err" ] \ + && jq -e '.changed == true and .name == "first-experiment"' \ + "$installed_first_out" >/dev/null 2>&1 \ + && [ "$installed_inspect_rc" -eq 0 ] && [ ! -s "$installed_inspect_err" ] \ + && jq -e --arg key "$installed_key" --arg receipt "$installed_receipt" \ + '.state == "installed" and .name == "first-experiment" and + .installationKey == $key and .receiptDigest == $receipt' \ + "$installed_inspect_out" >/dev/null 2>&1 \ + && [ "$installed_retry_rc" -eq 0 ] && [ ! -s "$installed_retry_err" ] \ + && jq -e --arg key "$installed_key" --arg receipt "$installed_receipt" \ + '.changed == false and .name == "first-experiment" and + .installationKey == $key and .receiptDigest == $receipt' \ + "$installed_retry_out" >/dev/null 2>&1 \ + && [ "$installed_home_after_first" = "$installed_home_after_inspect" ] \ + && [ "$installed_home_after_first" = "$installed_home_after_retry" ] \ + && [ "$installed_prefix_before" = "$installed_prefix_after" ] \ + && [ ! -e "$installed_source" ] && [ -d "$installed_unavailable" ] \ + && [ -z "$(find "$installed_prefix" -name __pycache__ -print -quit 2>/dev/null)" ] \ + && case "$installed_target" in + "$installed_prefix"/lib/agent-lab/releases/*/scripts/agent-lab) true ;; + *) false ;; + esac; then + pass INST-RUNTIME-001 "installed runtime remains source-independent through install, inspect, and exact retry" +else + fail INST-RUNTIME-001 "installed runtime remains source-independent through install, inspect, and exact retry" +fi + expected="$work/expected" printf '%s\n' \ INST-HOME-001 INST-UNKNOWN-001 INST-NAME-001 INST-PERMIT-001 INST-RECEIPT-001 \ INST-INSPECT-001 INST-RETRY-001 INST-CONFLICT-001 INST-DENY-001 \ - INST-FORGE-001 INST-NOEF-001 INST-LOCAL-001 > "$expected" + INST-FORGE-001 INST-NOEF-001 INST-LOCAL-001 INST-RUNTIME-001 > "$expected" if ! cmp -s "$expected" "$observed"; then printf 'INFRA install-store assertion identity drift\n' >&2 infrastructure=1 From e1e2f0f39a0c267519e7ca68f12c42f085114c0f Mon Sep 17 00:00:00 2001 From: xormania <127287135+xormania@users.noreply.github.com> Date: Sun, 2 Aug 2026 02:56:42 -0400 Subject: [PATCH 052/158] test(experiment): kill store contract mutations --- tests/experiment/aggregate-harness-cases.sh | 32 +- tests/experiment/install-mutation-cases.py | 1290 +++++++++++++++++++ tests/experiment/local-lifecycle-cases.sh | 12 +- 3 files changed, 1318 insertions(+), 16 deletions(-) create mode 100755 tests/experiment/install-mutation-cases.py diff --git a/tests/experiment/aggregate-harness-cases.sh b/tests/experiment/aggregate-harness-cases.sh index 0a54947..f79edbf 100755 --- a/tests/experiment/aggregate-harness-cases.sh +++ b/tests/experiment/aggregate-harness-cases.sh @@ -51,17 +51,21 @@ expected_ids=( RES-SNAP-003 RES-AUTH-001 RES-INSTALL-001 RES-NOEF-001 M-CAT-OCI-001 M-CAT-SHADOW-001 M-CAT-CAS-001 M-CAT-AUTH-001 M-RES-BIND-001 M-CAT-NOEF-001 M-CAT-ADMIT-001 M-CAT-ATOM-001 M-CAT-DUR-001 M-CAT-STAGE-001 - INST-HOME-001 INST-UNKNOWN-001 INST-PERMIT-001 INST-RECEIPT-001 + INST-HOME-001 INST-UNKNOWN-001 INST-NAME-001 INST-PERMIT-001 INST-RECEIPT-001 INST-INSPECT-001 INST-RETRY-001 INST-CONFLICT-001 INST-DENY-001 - INST-FORGE-001 INST-NOEF-001 INST-LOCAL-001 INST-NAME-001 + INST-FORGE-001 INST-NOEF-001 INST-LOCAL-001 INST-RUNTIME-001 IST-STATE-001 IST-LOCK-001 IST-STATE-002 IST-BOUND-001 IST-STATE-003 IST-CONC-001 IST-CONC-002 IST-CRASH-001 IST-LIVE-001 IST-PLAT-001 IST-PROV-001 + M-STORE-AUTH-001 M-STORE-SOURCE-001 M-STORE-ATOM-001 M-STORE-RETRY-001 + M-STORE-DUR-001 M-STORE-LAYOUT-001 M-STORE-KEY-001 M-STORE-VERIFY-001 + M-STORE-LIVE-001 M-STORE-UNCERT-001 M-STORE-STAGE-001 ) installer_ids=("${expected_ids[@]:0:5}") config_ids=("${expected_ids[@]:5:5}") catalog_ids=("${expected_ids[@]:10:76}") -install_ids=("${expected_ids[@]:86:12}") -state_ids=("${expected_ids[@]:98:11}") +install_ids=("${expected_ids[@]:86:13}") +state_ids=("${expected_ids[@]:99:11}") +mutation_ids=("${expected_ids[@]:110:11}") write_fixture() { local path="$1" @@ -122,17 +126,19 @@ pass_records() { reset_fixtures() { local installer_records=() config_records=() catalog_records=() - local install_records=() state_records=() + local install_records=() state_records=() mutation_records=() mapfile -t installer_records < <(pass_records "${installer_ids[@]}") mapfile -t config_records < <(pass_records "${config_ids[@]}") mapfile -t catalog_records < <(pass_records "${catalog_ids[@]}") mapfile -t install_records < <(pass_records "${install_ids[@]}") mapfile -t state_records < <(pass_records "${state_ids[@]}") + mapfile -t mutation_records < <(pass_records "${mutation_ids[@]}") write_fixture "$replica/tests/install/local-install-cases.sh" 0 "${installer_records[@]}" write_fixture "$replica/tests/experiment/local-config-cases.sh" 0 "${config_records[@]}" write_fixture "$replica/tests/experiment/local-image-catalog-cases.sh" 0 "${catalog_records[@]}" write_fixture "$replica/tests/experiment/install-store-cases.sh" 0 "${install_records[@]}" write_python_fixture "$replica/tests/experiment/install-state-cases.py" 0 "${state_records[@]}" + write_python_fixture "$replica/tests/experiment/install-mutation-cases.py" 0 "${mutation_records[@]}" } run_replica() { @@ -158,7 +164,8 @@ printf '%s\n' \ local-config-cases.sh \ local-image-catalog-cases.sh \ install-store-cases.sh \ - install-state-cases.py > "$expected_executions" + install-state-cases.py \ + install-mutation-cases.py > "$expected_executions" : > "$baseline_executions" baseline_rc=0 run_replica "$work/baseline.out" env \ @@ -186,7 +193,8 @@ printf '%s\n' \ local-config-cases.sh \ local-image-catalog-cases.sh \ install-store-cases.sh \ - install-state-cases.py > "$mutant_expected" + install-state-cases.py \ + install-mutation-cases.py > "$mutant_expected" if [ "$baseline_rc" -eq 0 ] && cmp -s "$expected_executions" "$baseline_executions" && @@ -205,10 +213,10 @@ success_output="$work/success.out" success_rc=0 run_replica "$success_output" env || success_rc=$? if [ "$success_rc" -eq 0 ] && - [ "$(grep -Ec '^(PASS|FAIL) [A-Z0-9-]+ ' "$success_output")" -eq 109 ] && - [ "$(grep -Fxc 'SUMMARY assertions=109 expected=109 failures=0 infra=0' "$success_output")" -eq 1 ] && + [ "$(grep -Ec '^(PASS|FAIL) [A-Z0-9-]+ ' "$success_output")" -eq 121 ] && + [ "$(grep -Fxc 'SUMMARY assertions=121 expected=121 failures=0 infra=0' "$success_output")" -eq 1 ] && [ "$(tail -n 1 "$success_output")" = 'EXPERIMENT LOCAL LIFECYCLE PASS' ] && - awk '/^(PASS|FAIL) [A-Z0-9-]+ / {next} /^SUMMARY assertions=109 expected=109 failures=0 infra=0$/ {next} /^EXPERIMENT LOCAL LIFECYCLE PASS$/ {next} {bad=1} END {exit bad}' "$success_output"; then + awk '/^(PASS|FAIL) [A-Z0-9-]+ / {next} /^SUMMARY assertions=121 expected=121 failures=0 infra=0$/ {next} /^EXPERIMENT LOCAL LIFECYCLE PASS$/ {next} {bad=1} END {exit bad}' "$success_output"; then pass AGG-002 "success forwards only assertions then one summary and marker" else fail AGG-002 "success forwards only assertions then one summary and marker" @@ -257,7 +265,7 @@ write_fixture "$replica/tests/install/local-install-cases.sh" 1 "${failed_record assertion_rc=0 run_replica "$work/assertion.out" env || assertion_rc=$? if [ "$assertion_rc" -eq 1 ] && - grep -Fxq 'SUMMARY assertions=109 expected=109 failures=1 infra=0' "$work/assertion.out" && + grep -Fxq 'SUMMARY assertions=121 expected=121 failures=1 infra=0' "$work/assertion.out" && ! grep -Fxq 'EXPERIMENT LOCAL LIFECYCLE PASS' "$work/assertion.out"; then pass AGG-006 "subcase assertion failure maps to one" else @@ -290,7 +298,7 @@ chmod +x "$shim/rmdir" cleanup_rc=0 run_replica "$work/cleanup.out" env PATH="$shim:$PATH" || cleanup_rc=$? if [ "$cleanup_rc" -eq 125 ] && - grep -Fxq 'SUMMARY assertions=109 expected=109 failures=0 infra=1' "$work/cleanup.out" && + grep -Fxq 'SUMMARY assertions=121 expected=121 failures=0 infra=1' "$work/cleanup.out" && ! grep -Fxq 'EXPERIMENT LOCAL LIFECYCLE PASS' "$work/cleanup.out"; then pass AGG-008 "cleanup uncertainty maps to one hundred twenty-five before the marker" else diff --git a/tests/experiment/install-mutation-cases.py b/tests/experiment/install-mutation-cases.py new file mode 100755 index 0000000..df23c51 --- /dev/null +++ b/tests/experiment/install-mutation-cases.py @@ -0,0 +1,1290 @@ +#!/usr/bin/env python3 +"""Private-copy sensitivity mutations for the local Experiment store.""" + +from __future__ import annotations + +from contextlib import contextmanager +from dataclasses import dataclass +import hashlib +from importlib.util import module_from_spec, spec_from_file_location +import json +import os +from pathlib import Path +import shutil +import signal +import stat +import subprocess +import sys +import tempfile +import time +from typing import Callable, Iterator, NamedTuple + + +REPO_ROOT = Path(__file__).resolve().parents[2] +RUNTIME_MANIFEST = REPO_ROOT / "packaging" / "agent-lab-local.manifest" +COMMAND_TIMEOUT_SECONDS = 5 +SOURCE_DOMAIN = b"agent-lab.experiment-tree.v1\0" +SUBJECT = "registry.example/team/worker@sha256:" + "a" * 64 +AUTHORIZATION_DIGEST = "sha256:" + "c" * 64 +CONTRACT_DIGEST = "sha256:" + "d" * 64 +MUTATION_KEYS = ( + "AGENT_LAB_MUTATION_DECISION", + "AGENT_LAB_MUTATION_MARK", + "AGENT_LAB_MUTATION_NAME", + "AGENT_LAB_MUTATION_SOURCE", +) + + +class InfrastructureError(Exception): + """The mutation or its isolated evidence could not be proved.""" + + +class ProbeResult(NamedTuple): + secure: bool + detail: str + + +Probe = Callable[[Path, Path, Path | None], ProbeResult] + + +@dataclass(frozen=True) +class Mutation: + assertion: str + path: str + old: str + new: str + probe: Probe + message: str + + +class Snapshot(NamedTuple): + data: bytes + digest: str + + +class Resolution(NamedTuple): + plan: dict[str, object] + local_catalog: dict[str, object] | None + + +class PlanFixture(NamedTuple): + plan: dict[str, object] + local_catalog: dict[str, object] | None + + +class FixtureInvalidManifest(Exception): + """The synthetic manifest is outside the declared fixture set.""" + + +class FixtureInfrastructure(Exception): + """The synthetic planning fixture could not establish a result.""" + + +def canonical(value: object) -> bytes: + return json.dumps( + value, + allow_nan=False, + ensure_ascii=True, + separators=(",", ":"), + sort_keys=True, + ).encode("ascii") + + +def digest(data: bytes) -> str: + return "sha256:" + hashlib.sha256(data).hexdigest() + + +def source_digest(data: bytes) -> str: + name = b"experiment.cue" + value = hashlib.sha256(SOURCE_DOMAIN) + value.update(len(name).to_bytes(4, "big")) + value.update(name) + value.update(len(data).to_bytes(8, "big")) + value.update(data) + return "sha256:" + value.hexdigest() + + +def artifact_bytes(command: str) -> bytes: + return ( + "package experiment\n\n" + "experiment: {\n" + '\tapiVersion: "agent-lab/v0alpha1"\n' + '\tkind: "Experiment"\n' + '\tmetadata: name: "mutation-store"\n' + "\tspec: members: [{\n" + '\t\tname: "worker"\n' + f'\t\timage: digestRef: "{SUBJECT}"\n' + f'\t\tcommand: ["{command}"]\n' + "\t}]\n" + "}\n" + ).encode("ascii") + + +def requested_plan( + name: str, + command: str, + *, + local_record: dict[str, object] | None = None, +) -> dict[str, object]: + if local_record is None: + requested = {"digestRef": SUBJECT} + resolved = {"origin": "direct", "subject": SUBJECT} + else: + requested = {"catalogName": "vendor.worker"} + resolved = { + "entryDigest": local_record["entryDigest"], + "generation": local_record["generation"], + "origin": "local", + "subject": local_record["subject"], + } + return { + "apiVersion": "agent-lab.request/v0alpha1", + "contract": { + "digest": CONTRACT_DIGEST, + "name": "agent-lab.experiment", + "version": "v0alpha1", + }, + "kind": "RequestedExperimentPlan", + "metadata": {"requestedName": name}, + "spec": { + "members": [ + { + "command": [command], + "name": "worker", + "requestedSelector": requested, + "resolvedImage": resolved, + "resourceClass": "small", + } + ] + }, + } + + +def decision_for( + plan: dict[str, object], + snapshot_digest: str, + verdict: str, +) -> dict[str, object]: + plan_digest = digest(canonical(plan)) + requested_name = plan["metadata"]["requestedName"] # type: ignore[index] + return { + "action": "experiment.install", + "apiVersion": "agent-lab.authorization/v0alpha1", + "binding": { + "authorizationDigest": AUTHORIZATION_DIGEST, + "contractDigest": CONTRACT_DIGEST, + "planDigest": plan_digest, + "sourceDigest": snapshot_digest, + }, + "kind": "ExperimentAuthorizationDecision", + "principal": { + "assurance": "none", + "authenticated": False, + "id": "local-cli", + "source": "fixed-local-cli", + "type": "AgentLab::Principal", + }, + "resource": { + "id": plan_digest, + "requestedName": requested_name, + "type": "AgentLab::RequestedExperimentPlan", + }, + "verdict": verdict, + } + + +class FixtureExperiment: + InvalidManifest = FixtureInvalidManifest + InfrastructureError = FixtureInfrastructure + + def __init__( + self, + fixtures: dict[bytes, PlanFixture], + *, + verdict: str = "permit", + after_authorize: Callable[[], None] | None = None, + ) -> None: + self.fixtures = fixtures + self.verdict = verdict + self.after_authorize = after_authorize + self.authorize_calls = 0 + + def read_directory_snapshot(self, source: str) -> Snapshot: + try: + data = (Path(source) / "experiment.cue").read_bytes() + except OSError as error: + raise FixtureInfrastructure("fixture source cannot be read") from error + if data not in self.fixtures: + raise FixtureInvalidManifest("fixture source is unknown") + return Snapshot(data, source_digest(data)) + + def authored_manifest(self, snapshot: Snapshot) -> bytes: + return snapshot.data + + def cue_plan_with_evidence(self, manifest: object) -> Resolution: + if not isinstance(manifest, bytes) or manifest not in self.fixtures: + raise FixtureInvalidManifest("fixture manifest is unknown") + fixture = self.fixtures[manifest] + return Resolution(fixture.plan, fixture.local_catalog) + + def authorize_plan( + self, + plan: dict[str, object], + snapshot_digest: str, + ) -> tuple[dict[str, object], int]: + self.authorize_calls += 1 + decision = decision_for(plan, snapshot_digest, self.verdict) + if self.after_authorize is not None: + self.after_authorize() + return decision, 0 if self.verdict == "permit" else 1 + + +def sha256_bytes(data: bytes) -> str: + return hashlib.sha256(data).hexdigest() + + +def manifest_paths() -> tuple[str, ...]: + try: + raw = RUNTIME_MANIFEST.read_bytes() + text = raw.decode("utf-8") + except (OSError, UnicodeError) as error: + raise InfrastructureError("runtime manifest cannot be read exactly") from error + if not text.endswith("\n"): + raise InfrastructureError("runtime manifest lacks its final newline") + names = tuple(line for line in text.splitlines() if line) + if not names or len(names) != len(set(names)) or names != tuple(sorted(names)): + raise InfrastructureError("runtime manifest is empty, duplicated, or unordered") + for name in names: + path = Path(name) + if path.is_absolute() or ".." in path.parts or str(path) != name: + raise InfrastructureError(f"runtime manifest path is unsafe: {name}") + required = { + "scripts/agent-lab.py", + "scripts/experiment.py", + "scripts/experiment_store.py", + "scripts/image_catalog.py", + "scripts/image_reference.py", + } + if not required.issubset(names): + raise InfrastructureError("runtime manifest omits an Experiment store runtime path") + return names + + +def file_identity(path: Path) -> tuple[str, int, int, str]: + try: + metadata = path.lstat() + data = path.read_bytes() + except OSError as error: + raise InfrastructureError(f"runtime path cannot be fingerprinted: {path}") from error + if not stat.S_ISREG(metadata.st_mode) or metadata.st_nlink != 1: + raise InfrastructureError(f"runtime path is not a single regular file: {path}") + return ("file", stat.S_IMODE(metadata.st_mode), len(data), sha256_bytes(data)) + + +def runtime_fingerprint( + root: Path, + names: tuple[str, ...], +) -> tuple[tuple[str, tuple[str, int, int, str]], ...]: + paths = ("packaging/agent-lab-local.manifest", *names) + return tuple((name, file_identity(root / name)) for name in paths) + + +def tree_fingerprint(root: Path) -> tuple[tuple[object, ...], ...]: + if not root.exists() and not root.is_symlink(): + return () + values: list[tuple[object, ...]] = [] + pending = [root] + while pending: + path = pending.pop() + try: + metadata = path.lstat() + except OSError as error: + raise InfrastructureError(f"probe state cannot be fingerprinted: {path}") from error + relative = "." if path == root else path.relative_to(root).as_posix() + mode = stat.S_IMODE(metadata.st_mode) + if stat.S_ISDIR(metadata.st_mode): + kind = "directory" + payload = "" + try: + pending.extend(sorted(path.iterdir(), reverse=True)) + except OSError as error: + raise InfrastructureError(f"probe directory cannot be listed: {path}") from error + elif stat.S_ISREG(metadata.st_mode): + kind = "file" + try: + payload = sha256_bytes(path.read_bytes()) + except OSError as error: + raise InfrastructureError(f"probe file cannot be read: {path}") from error + elif stat.S_ISLNK(metadata.st_mode): + kind = "symlink" + try: + payload = os.readlink(path) + except OSError as error: + raise InfrastructureError(f"probe symlink cannot be read: {path}") from error + else: + kind = "other" + payload = "" + values.append( + ( + relative, + kind, + mode, + metadata.st_nlink, + metadata.st_size, + metadata.st_dev, + metadata.st_ino, + metadata.st_mtime_ns, + metadata.st_ctime_ns, + payload, + ) + ) + return tuple(sorted(values)) + + +def copy_runtime(destination: Path, names: tuple[str, ...]) -> None: + for name in ("packaging/agent-lab-local.manifest", *names): + source = REPO_ROOT / name + target = destination / name + identity = file_identity(source) + try: + target.parent.mkdir(mode=0o700, parents=True, exist_ok=True) + shutil.copyfile(source, target) + os.chmod(target, identity[1]) + except OSError as error: + raise InfrastructureError(f"runtime path cannot be copied privately: {name}") from error + if file_identity(target) != identity: + raise InfrastructureError(f"private runtime copy differs: {name}") + + +def command_environment() -> dict[str, str]: + return {"PATH": "/usr/bin:/bin", "LANG": "C", "LC_ALL": "C"} + + +def process_group_exists(group: int) -> bool: + try: + os.killpg(group, 0) + except ProcessLookupError: + return False + except PermissionError: + return True + return True + + +def terminate_process_group(process: subprocess.Popen[bytes]) -> bool: + group = process.pid + if process_group_exists(group): + try: + os.killpg(group, signal.SIGTERM) + except ProcessLookupError: + pass + deadline = time.monotonic() + 1.0 + while process_group_exists(group) and time.monotonic() < deadline: + time.sleep(0.01) + if process_group_exists(group): + try: + os.killpg(group, signal.SIGKILL) + except ProcessLookupError: + pass + try: + process.communicate(timeout=1) + except subprocess.TimeoutExpired: + try: + os.killpg(group, signal.SIGKILL) + except ProcessLookupError: + pass + process.communicate() + return not process_group_exists(group) + + +def run_command(arguments: list[str]) -> subprocess.CompletedProcess[bytes]: + process: subprocess.Popen[bytes] | None = None + try: + process = subprocess.Popen( + arguments, + stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + env=command_environment(), + start_new_session=True, + ) + stdout, stderr = process.communicate(timeout=COMMAND_TIMEOUT_SECONDS) + if process_group_exists(process.pid): + if not terminate_process_group(process): + raise InfrastructureError("bounded probe left an uncontained process group") + raise InfrastructureError("bounded probe left a descendant process") + return subprocess.CompletedProcess(arguments, process.returncode, stdout, stderr) + except subprocess.TimeoutExpired as error: + assert process is not None + if not terminate_process_group(process): + raise InfrastructureError("timed-out probe left an uncontained process group") from error + raise InfrastructureError("bounded probe command timed out") from error + except (OSError, subprocess.SubprocessError) as error: + if process is not None and process.poll() is None: + terminate_process_group(process) + raise InfrastructureError("bounded probe command could not complete") from error + + +def initialized_home(runtime: Path, probe_root: Path, name: str) -> Path: + home = probe_root / name + completed = run_command( + [ + sys.executable, + "-I", + "-B", + str(runtime / "scripts" / "agent-lab.py"), + "--home", + str(home), + "init", + ] + ) + if completed.returncode != 0 or completed.stdout != b"changed:true\n" or completed.stderr: + raise InfrastructureError( + "private runtime home initialization failed: " + + completed.stderr.decode("utf-8", errors="replace") + ) + return home + + +def write_source(root: Path, name: str, data: bytes) -> Path: + source = root / name + try: + source.mkdir(mode=0o700) + path = source / "experiment.cue" + path.write_bytes(data) + os.chmod(path, 0o600) + except OSError as error: + raise InfrastructureError("private source fixture could not be written") from error + return source + + +def load_module(path: Path, name: str): + spec = spec_from_file_location(name, path) + if spec is None or spec.loader is None: + raise InfrastructureError(f"private module cannot be loaded: {path.name}") + module = module_from_spec(spec) + sys.modules[name] = module + try: + spec.loader.exec_module(module) + except (ImportError, OSError, SyntaxError) as error: + raise InfrastructureError(f"private module import failed: {path.name}") from error + return module + + +def load_store(runtime: Path, probe_root: Path): + name = f"agent_lab_store_mutation_{os.getpid()}_{id(probe_root)}" + return load_module(runtime / "scripts" / "experiment_store.py", name) + + +def fixture_store(runtime: Path, probe_root: Path, experiment: FixtureExperiment): + store = load_store(runtime, probe_root) + store._experiment_module = lambda: experiment + return store + + +def install_result(store, home: Path, source: Path, *, fault=None): + try: + value = store.install_directory(home, source, fault=fault) + return 0, value, None + except store.StoreReject as error: + return 1, None, error + except store.StoreInfrastructure as error: + return 125, None, error + except BaseException as error: + return None, None, error + + +def inspect_result(store, home: Path, name: str): + try: + value = store.inspect_install(home, name) + return 0, value, None + except store.StoreReject as error: + return 1, None, error + except store.StoreInfrastructure as error: + return 125, None, error + except BaseException as error: + return None, None, error + + +@contextmanager +def mutation_environment( + marker: Path | None, + extra: dict[str, str] | None = None, +) -> Iterator[None]: + previous = {key: os.environ.get(key) for key in MUTATION_KEYS} + try: + for key in MUTATION_KEYS: + os.environ.pop(key, None) + if marker is not None: + os.environ["AGENT_LAB_MUTATION_MARK"] = str(marker) + if extra: + os.environ.update(extra) + yield + finally: + for key, value in previous.items(): + if value is None: + os.environ.pop(key, None) + else: + os.environ[key] = value + + +def direct_fixture(data: bytes, command: str = "serve") -> dict[bytes, PlanFixture]: + return { + data: PlanFixture( + requested_plan("mutation-store", command), + None, + ) + } + + +def probe_fresh_authorization(runtime: Path, probe_root: Path, marker: Path | None) -> ProbeResult: + home = initialized_home(runtime, probe_root, "authorization-home") + data = artifact_bytes("authorize") + source = write_source(probe_root, "authorization-source", data) + fixture = direct_fixture(data, "authorize") + experiment = FixtureExperiment(fixture, verdict="deny") + store = fixture_store(runtime, probe_root, experiment) + saved = probe_root / "saved-decision.json" + saved.write_bytes(canonical(decision_for(fixture[data].plan, source_digest(data), "permit")) + b"\n") + before = tree_fingerprint(home / "experiments") + with mutation_environment( + marker, + {"AGENT_LAB_MUTATION_DECISION": str(saved)}, + ): + rc, value, error = install_result(store, home, source) + after = tree_fingerprint(home / "experiments") + secure = rc == 1 and value is None and before == after and experiment.authorize_calls == 1 + return ProbeResult( + secure, + f"rc={rc} value={value!r} error={error!r} changed={before != after} calls={experiment.authorize_calls}", + ) + + +def probe_held_snapshot(runtime: Path, probe_root: Path, marker: Path | None) -> ProbeResult: + home = initialized_home(runtime, probe_root, "snapshot-home") + original = artifact_bytes("snapshot-original") + changed = artifact_bytes("snapshot-changed") + source = write_source(probe_root, "snapshot-source", original) + + def mutate_source() -> None: + (source / "experiment.cue").write_bytes(changed) + + experiment = FixtureExperiment( + direct_fixture(original, "snapshot-original"), + after_authorize=mutate_source, + ) + store = fixture_store(runtime, probe_root, experiment) + with mutation_environment( + marker, + {"AGENT_LAB_MUTATION_SOURCE": str(source)}, + ): + rc, value, error = install_result(store, home, source) + artifact = home / "experiments" / "mutation-store" / "artifact" / "experiment.cue" + stored = artifact.read_bytes() if artifact.is_file() else None + secure = ( + rc == 0 + and isinstance(value, dict) + and error is None + and (source / "experiment.cue").read_bytes() == changed + and stored == original + ) + return ProbeResult(secure, f"rc={rc} error={error!r} stored_original={stored == original}") + + +def probe_noreplace_race(runtime: Path, probe_root: Path, marker: Path | None) -> ProbeResult: + home = initialized_home(runtime, probe_root, "atomic-home") + data = artifact_bytes("atomic") + source = write_source(probe_root, "atomic-source", data) + store = fixture_store(runtime, probe_root, FixtureExperiment(direct_fixture(data, "atomic"))) + target = home / "experiments" / "mutation-store" + raced = False + + def create_target(point: str) -> None: + nonlocal raced + if point == "experiment envelope.before_noreplace" and not raced: + raced = True + target.mkdir(mode=0o700) + + with mutation_environment(marker): + rc, value, error = install_result(store, home, source, fault=create_target) + empty = target.is_dir() and not tuple(target.iterdir()) + secure = raced and rc == 125 and value is None and empty + return ProbeResult(secure, f"raced={raced} rc={rc} error={error!r} empty={empty}") + + +def probe_idempotent_retry(runtime: Path, probe_root: Path, marker: Path | None) -> ProbeResult: + home = initialized_home(runtime, probe_root, "retry-home") + data = artifact_bytes("retry") + source = write_source(probe_root, "retry-source", data) + experiment = FixtureExperiment(direct_fixture(data, "retry")) + store = fixture_store(runtime, probe_root, experiment) + with mutation_environment(marker): + first_rc, first, first_error = install_result(store, home, source) + before = tree_fingerprint(home / "experiments" / "mutation-store") + second_rc, second, second_error = install_result(store, home, source) + after = tree_fingerprint(home / "experiments" / "mutation-store") + secure = ( + first_rc == 0 + and isinstance(first, dict) + and first.get("changed") is True + and first_error is None + and second_rc == 0 + and isinstance(second, dict) + and second.get("changed") is False + and second_error is None + and experiment.authorize_calls == 2 + and before == after + ) + return ProbeResult( + secure, + f"first={first_rc}/{first_error!r} second={second_rc}/{second_error!r}/{second!r} calls={experiment.authorize_calls} changed={before != after}", + ) + + +def probe_publication_durability(runtime: Path, probe_root: Path, marker: Path | None) -> ProbeResult: + home = initialized_home(runtime, probe_root, "durability-home") + data = artifact_bytes("durability") + source = write_source(probe_root, "durability-source", data) + store = fixture_store(runtime, probe_root, FixtureExperiment(direct_fixture(data, "durability"))) + events: list[tuple[str, str]] = [] + original_rename = store._rename_noreplace + original_fsync = store._fsync_directory + target = home / "experiments" / "mutation-store" + + def observed_rename(source_path: Path, target_path: Path) -> None: + if target_path == target: + events.append(("publish", str(target_path))) + original_rename(source_path, target_path) + + def observed_fsync(path: Path, purpose: str, *, modes=(0o700,)) -> None: + events.append(("fsync", purpose)) + original_fsync(path, purpose, modes=modes) + + store._rename_noreplace = observed_rename + store._fsync_directory = observed_fsync + try: + with mutation_environment(marker): + rc, value, error = install_result(store, home, source) + finally: + store._fsync_directory = original_fsync + store._rename_noreplace = original_rename + publication = [index for index, event in enumerate(events) if event[0] == "publish"] + durable = [ + index + for index, event in enumerate(events) + if event == ("fsync", "Experiment store root") + ] + secure = ( + rc == 0 + and isinstance(value, dict) + and error is None + and len(publication) == 1 + and len(durable) == 1 + and publication[0] < durable[0] + ) + return ProbeResult(secure, f"rc={rc} error={error!r} publish={publication} durable={durable}") + + +def probe_artifact_separation(runtime: Path, probe_root: Path, marker: Path | None) -> ProbeResult: + home = initialized_home(runtime, probe_root, "layout-home") + data = artifact_bytes("layout") + source = write_source(probe_root, "layout-source", data) + store = fixture_store(runtime, probe_root, FixtureExperiment(direct_fixture(data, "layout"))) + with mutation_environment(marker): + rc, value, error = install_result(store, home, source) + artifact = home / "experiments" / "mutation-store" / "artifact" + names = tuple(sorted(path.name for path in artifact.iterdir())) if artifact.is_dir() else () + stored = (artifact / "experiment.cue").read_bytes() if names == ("experiment.cue",) else None + secure = rc == 0 and isinstance(value, dict) and error is None and names == ("experiment.cue",) and stored == data + return ProbeResult(secure, f"rc={rc} error={error!r} names={names!r} exact={stored == data}") + + +def probe_full_identity_key(runtime: Path, probe_root: Path, marker: Path | None) -> ProbeResult: + home = initialized_home(runtime, probe_root, "identity-home") + first_data = artifact_bytes("identity-one") + second_data = artifact_bytes("identity-two") + first_source = write_source(probe_root, "identity-source-one", first_data) + second_source = write_source(probe_root, "identity-source-two", second_data) + fixtures = { + first_data: PlanFixture(requested_plan("mutation-store", "identity-one"), None), + second_data: PlanFixture(requested_plan("mutation-store", "identity-two"), None), + } + store = fixture_store(runtime, probe_root, FixtureExperiment(fixtures)) + with mutation_environment(marker, {"AGENT_LAB_MUTATION_NAME": "mutation-store"}): + first_rc, first, first_error = install_result(store, home, first_source) + before = tree_fingerprint(home / "experiments" / "mutation-store") + second_rc, second, second_error = install_result(store, home, second_source) + after = tree_fingerprint(home / "experiments" / "mutation-store") + secure = ( + first_rc == 0 + and isinstance(first, dict) + and first.get("changed") is True + and first_error is None + and second_rc == 1 + and second is None + and second_error is not None + and before == after + ) + return ProbeResult( + secure, + f"first={first_rc}/{first_error!r} second={second_rc}/{second_error!r}/{second!r} changed={before != after}", + ) + + +def probe_final_revalidation(runtime: Path, probe_root: Path, marker: Path | None) -> ProbeResult: + home = initialized_home(runtime, probe_root, "verification-home") + data = artifact_bytes("verification") + source = write_source(probe_root, "verification-source", data) + store = fixture_store(runtime, probe_root, FixtureExperiment(direct_fixture(data, "verification"))) + corrupted = False + + def corrupt_receipt(point: str) -> None: + nonlocal corrupted + if point != "experiment envelope.after_noreplace" or corrupted: + return + corrupted = True + records = home / "experiments" / "mutation-store" / "records" + receipt = records / "install.json" + os.chmod(records, 0o700, follow_symlinks=False) + os.chmod(receipt, 0o600, follow_symlinks=False) + receipt.write_bytes(receipt.read_bytes() + b" ") + os.chmod(receipt, 0o400, follow_symlinks=False) + os.chmod(records, 0o500, follow_symlinks=False) + + with mutation_environment(marker): + rc, value, error = install_result(store, home, source, fault=corrupt_receipt) + inspect_rc, _, inspect_error = inspect_result(store, home, "mutation-store") + secure = corrupted and rc == 125 and value is None and inspect_rc == 125 + return ProbeResult( + secure, + f"corrupted={corrupted} install={rc}/{error!r} inspect={inspect_rc}/{inspect_error!r}", + ) + + +def lock_is_blocked(path: Path) -> bool: + program = ( + "import fcntl, os, sys\n" + "fd=os.open(sys.argv[1], os.O_RDWR|getattr(os,'O_CLOEXEC',0))\n" + "try:\n" + " fcntl.flock(fd, fcntl.LOCK_EX|fcntl.LOCK_NB)\n" + "except BlockingIOError:\n" + " raise SystemExit(3)\n" + "raise SystemExit(0)\n" + ) + completed = run_command([sys.executable, "-I", "-B", "-c", program, str(path)]) + if completed.returncode not in (0, 3) or completed.stdout or completed.stderr: + raise InfrastructureError(f"catalog lock probe returned {completed.returncode}") + return completed.returncode == 3 + + +def local_fixture(runtime: Path, probe_root: Path, home: Path) -> PlanFixture: + catalog = load_module( + runtime / "scripts" / "image_catalog.py", + f"agent_lab_catalog_store_mutation_{os.getpid()}_{id(probe_root)}", + ) + try: + added = catalog.add_image(home, "vendor.worker", SUBJECT) + resolved = catalog.resolve_local_images(home, ("vendor.worker",)) + record = resolved["records"]["vendor.worker"] + evidence = resolved["catalog"] + except BaseException as error: + raise InfrastructureError(f"local catalog fixture failed: {error}") from error + if ( + not isinstance(added, dict) + or not isinstance(record, dict) + or not isinstance(evidence, dict) + ): + raise InfrastructureError("local catalog fixture is malformed") + return PlanFixture(requested_plan("mutation-store", "local", local_record=record), evidence) + + +def probe_catalog_lock_lifetime(runtime: Path, probe_root: Path, marker: Path | None) -> ProbeResult: + home = initialized_home(runtime, probe_root, "liveness-home") + data = artifact_bytes("local") + source = write_source(probe_root, "liveness-source", data) + fixture = local_fixture(runtime, probe_root, home) + store = fixture_store(runtime, probe_root, FixtureExperiment({data: fixture})) + observed: bool | None = None + + def inspect_lock(point: str) -> None: + nonlocal observed + if point == "experiment envelope.before_noreplace": + observed = lock_is_blocked(home / "state" / "locks" / "image-catalog.lock") + + with mutation_environment(marker): + rc, value, error = install_result(store, home, source, fault=inspect_lock) + secure = rc == 0 and isinstance(value, dict) and error is None and observed is True + return ProbeResult(secure, f"rc={rc} error={error!r} catalog_lock_blocked={observed}") + + +def probe_cleanup_uncertainty(runtime: Path, probe_root: Path, marker: Path | None) -> ProbeResult: + home = initialized_home(runtime, probe_root, "uncertainty-home") + data = artifact_bytes("uncertainty") + source = write_source(probe_root, "uncertainty-source", data) + store = fixture_store(runtime, probe_root, FixtureExperiment(direct_fixture(data, "uncertainty"))) + original_cleanup = store._cleanup_operation + + def uncertain_cleanup(authority, wrapper) -> None: + raise store.StoreInfrastructure("injected cleanup uncertainty") + + store._cleanup_operation = uncertain_cleanup + try: + with mutation_environment(marker): + rc, value, error = install_result(store, home, source) + inspect_rc, inspected, inspect_error = inspect_result(store, home, "mutation-store") + finally: + store._cleanup_operation = original_cleanup + staging = tuple((home / "experiments" / ".staging").iterdir()) + secure = ( + rc == 125 + and value is None + and inspect_rc == 0 + and isinstance(inspected, dict) + and inspect_error is None + and bool(staging) + ) + return ProbeResult( + secure, + f"install={rc}/{error!r}/{value!r} inspect={inspect_rc}/{inspect_error!r} staging={tuple(path.name for path in staging)!r}", + ) + + +def probe_unknown_staging(runtime: Path, probe_root: Path, marker: Path | None) -> ProbeResult: + home = initialized_home(runtime, probe_root, "staging-home") + store = load_store(runtime, probe_root) + foreign = home / "experiments" / ".staging" / "foreign-wrapper" + foreign.mkdir(mode=0o700) + sentinel = foreign / "sentinel" + sentinel.write_bytes(b"foreign\n") + os.chmod(sentinel, 0o600) + before = tree_fingerprint(foreign) + rc: int | None = None + error: BaseException | None = None + try: + with mutation_environment(marker): + authority = store._load_home(home) + with store._store_lock(authority, None): + store._reconcile(authority) + rc = 0 + except store.StoreInfrastructure as caught: + rc = 125 + error = caught + except BaseException as caught: + error = caught + after = tree_fingerprint(foreign) + secure = rc == 125 and before == after + return ProbeResult(secure, f"rc={rc} error={error!r} changed={before != after}") + + +def apply_mutation(runtime: Path, mutation: Mutation) -> None: + path = runtime / mutation.path + try: + source = path.read_text(encoding="utf-8") + except (OSError, UnicodeError) as error: + raise InfrastructureError(f"mutation source cannot be read: {mutation.path}") from error + occurrences = source.count(mutation.old) + if occurrences != 1: + raise InfrastructureError( + f"{mutation.assertion} replacement applicability is {occurrences}, expected exactly 1" + ) + mutated = source.replace(mutation.old, mutation.new, 1) + if mutated == source or mutated.count(mutation.new) != 1: + raise InfrastructureError(f"{mutation.assertion} replacement result is ambiguous") + try: + path.write_text(mutated, encoding="utf-8") + except OSError as error: + raise InfrastructureError(f"{mutation.assertion} private source cannot be written") from error + if path.read_text(encoding="utf-8") != mutated: + raise InfrastructureError(f"{mutation.assertion} private source write was not exact") + + +def compile_mutation(runtime: Path, mutation: Mutation, cache: Path) -> None: + completed = run_command( + [ + sys.executable, + "-I", + "-B", + "-X", + f"pycache_prefix={cache}", + "-m", + "py_compile", + str(runtime / mutation.path), + ] + ) + if completed.returncode != 0 or completed.stderr: + raise InfrastructureError( + f"{mutation.assertion} private mutation does not compile: " + + completed.stderr.decode("utf-8", errors="replace") + ) + + +MUTATIONS: tuple[Mutation, ...] = ( + Mutation( + "M-STORE-AUTH-001", + "scripts/experiment_store.py", + ' decision, status = experiment.authorize_plan(plan, snapshot.digest)\n', + ( + ' mutation_marker = os.environ.get("AGENT_LAB_MUTATION_MARK")\n' + ' if mutation_marker is None:\n' + ' decision, status = experiment.authorize_plan(plan, snapshot.digest)\n' + ' else:\n' + ' Path(mutation_marker).touch()\n' + ' decision = _parse_object(\n' + ' Path(os.environ["AGENT_LAB_MUTATION_DECISION"]).read_bytes(),\n' + ' "caller-supplied saved decision",\n' + ' )\n' + ' status = 0\n' + ), + probe_fresh_authorization, + "the fresh-authorization oracle detects a caller-supplied saved permit", + ), + Mutation( + "M-STORE-SOURCE-001", + "scripts/experiment_store.py", + ' "artifact/experiment.cue": source_data,\n', + ( + ' "artifact/experiment.cue": (\n' + ' source_data\n' + ' if os.environ.get("AGENT_LAB_MUTATION_MARK") is None\n' + ' else (\n' + ' Path(os.environ["AGENT_LAB_MUTATION_MARK"]).touch()\n' + ' or (\n' + ' Path(os.environ["AGENT_LAB_MUTATION_SOURCE"])\n' + ' / "experiment.cue"\n' + ' ).read_bytes()\n' + ' )\n' + ' ),\n' + ), + probe_held_snapshot, + "the held-snapshot oracle detects source reopening after authorization", + ), + Mutation( + "M-STORE-ATOM-001", + "scripts/experiment_store.py", + ' _rename_noreplace(wrapper / "payload", authority.store / name)\n', + ( + ' mutation_marker = os.environ.get("AGENT_LAB_MUTATION_MARK")\n' + ' if mutation_marker is None:\n' + ' _rename_noreplace(wrapper / "payload", authority.store / name)\n' + ' else:\n' + ' Path(mutation_marker).touch()\n' + ' os.replace(wrapper / "payload", authority.store / name)\n' + ), + probe_noreplace_race, + "the publication-race oracle detects replace-capable final publication", + ), + Mutation( + "M-STORE-RETRY-001", + "scripts/experiment_store.py", + ( + ' return {\n' + ' "changed": False,\n' + ' "installationKey": existing.installation_key,\n' + ' "name": name,\n' + ' "receiptDigest": existing.receipt_digest,\n' + ' }\n' + ), + ( + ' mutation_marker = os.environ.get("AGENT_LAB_MUTATION_MARK")\n' + ' if mutation_marker is not None:\n' + ' Path(mutation_marker).touch()\n' + ' _reject("mutated matching installation conflict")\n' + ' return {\n' + ' "changed": False,\n' + ' "installationKey": existing.installation_key,\n' + ' "name": name,\n' + ' "receiptDigest": existing.receipt_digest,\n' + ' }\n' + ), + probe_idempotent_retry, + "the idempotence oracle detects a matching receipt treated as conflict", + ), + Mutation( + "M-STORE-DUR-001", + "scripts/experiment_store.py", + ' _fsync_directory(authority.store, "Experiment store root")\n', + ( + ' mutation_marker = os.environ.get("AGENT_LAB_MUTATION_MARK")\n' + ' if mutation_marker is None:\n' + ' _fsync_directory(authority.store, "Experiment store root")\n' + ' else:\n' + ' Path(mutation_marker).touch()\n' + ), + probe_publication_durability, + "the durability oracle detects success without store-root fsync", + ), + Mutation( + "M-STORE-LAYOUT-001", + "scripts/experiment_store.py", + ' "artifact/experiment.cue": source_data,\n', + ( + ' "artifact/experiment.cue": (\n' + ' source_data\n' + ' if os.environ.get("AGENT_LAB_MUTATION_MARK") is None\n' + ' else (\n' + ' Path(os.environ["AGENT_LAB_MUTATION_MARK"]).touch()\n' + ' or source_data + b"\\n" + decision_bytes\n' + ' )\n' + ' ),\n' + ), + probe_artifact_separation, + "the portable-artifact oracle detects generated metadata mixed into artifact bytes", + ), + Mutation( + "M-STORE-KEY-001", + "scripts/experiment_store.py", + 'def canonical(value: object) -> bytes:\n try:\n return json.dumps(\n', + ( + 'def canonical(value: object) -> bytes:\n' + ' mutation_marker = os.environ.get("AGENT_LAB_MUTATION_MARK")\n' + ' if (\n' + ' mutation_marker is not None\n' + ' and isinstance(value, dict)\n' + ' and set(value)\n' + ' == {\n' + ' "authorizationDigest",\n' + ' "contractDigest",\n' + ' "planDigest",\n' + ' "selectedEntries",\n' + ' "sourceDigest",\n' + ' }\n' + ' ):\n' + ' Path(mutation_marker).touch()\n' + ' value = {"requestedName": os.environ["AGENT_LAB_MUTATION_NAME"]}\n' + ' try:\n' + ' return json.dumps(\n' + ), + probe_full_identity_key, + "the conflict oracle detects an installation key based only on requested name", + ), + Mutation( + "M-STORE-VERIFY-001", + "scripts/experiment_store.py", + ' verified = _verify_envelope(authority.store / name, name, authority.store_device)\n', + ( + ' mutation_marker = os.environ.get("AGENT_LAB_MUTATION_MARK")\n' + ' if mutation_marker is None:\n' + ' verified = _verify_envelope(\n' + ' authority.store / name, name, authority.store_device\n' + ' )\n' + ' else:\n' + ' Path(mutation_marker).touch()\n' + ' verified = VerifiedInstall(name, key, receipt_digest, {})\n' + ), + probe_final_revalidation, + "the corruption oracle detects skipped post-publication receipt verification", + ), + Mutation( + "M-STORE-LIVE-001", + "scripts/experiment_store.py", + ( + ' with operation(home, tuple(dependencies), fault=fault) as held:\n' + ' yield held\n' + ), + ( + ' mutation_marker = os.environ.get("AGENT_LAB_MUTATION_MARK")\n' + ' if mutation_marker is None:\n' + ' with operation(home, tuple(dependencies), fault=fault) as held:\n' + ' yield held\n' + ' else:\n' + ' with operation(home, tuple(dependencies), fault=fault) as held:\n' + ' released = held\n' + ' Path(mutation_marker).touch()\n' + ' yield released\n' + ), + probe_catalog_lock_lifetime, + "the liveness oracle detects catalog-lock release before publication", + ), + Mutation( + "M-STORE-UNCERT-001", + "scripts/experiment_store.py", + ( + ' _cleanup_operation(authority, wrapper)\n' + ' return {\n' + ' "changed": True,\n' + ), + ( + ' try:\n' + ' _cleanup_operation(authority, wrapper)\n' + ' except StoreInfrastructure:\n' + ' mutation_marker = os.environ.get("AGENT_LAB_MUTATION_MARK")\n' + ' if mutation_marker is None:\n' + ' raise\n' + ' Path(mutation_marker).touch()\n' + ' return {\n' + ' "changed": True,\n' + ), + probe_cleanup_uncertainty, + "the uncertainty oracle detects cleanup failure mapped to success", + ), + Mutation( + "M-STORE-STAGE-001", + "scripts/experiment_store.py", + ( + ' if names != (OPERATION_WRAPPER,):\n' + ' _infra("Experiment staging root contains an unknown wrapper")\n' + ), + ( + ' if names != (OPERATION_WRAPPER,):\n' + ' mutation_marker = os.environ.get("AGENT_LAB_MUTATION_MARK")\n' + ' if mutation_marker is None:\n' + ' _infra("Experiment staging root contains an unknown wrapper")\n' + ' Path(mutation_marker).touch()\n' + ' for staged_name in names:\n' + ' staged = authority.staging / staged_name\n' + ' __import__("shutil").rmtree(staged)\n' + ' _fsync_directory(authority.staging, "Experiment broad staging cleanup")\n' + ' return\n' + ), + probe_unknown_staging, + "the staging oracle detects broad deletion of an unknown wrapper", + ), +) + + +def marker_reached(marker: Path) -> bool: + try: + metadata = marker.lstat() + except OSError: + return False + return ( + stat.S_ISREG(metadata.st_mode) + and metadata.st_uid == os.getuid() + and metadata.st_nlink == 1 + and metadata.st_size == 0 + ) + + +def execute_mutation( + root: Path, + names: tuple[str, ...], + mutation: Mutation, +) -> tuple[bool, str]: + runtime = root / "runtime" + copy_runtime(runtime, names) + copied = runtime_fingerprint(runtime, names) + copied_tree = tree_fingerprint(runtime) + previous_bytecode = sys.dont_write_bytecode + sys.dont_write_bytecode = True + try: + pristine = mutation.probe(runtime, root / "pristine", None) + finally: + sys.dont_write_bytecode = previous_bytecode + if not pristine.secure: + raise InfrastructureError( + f"{mutation.assertion} pristine probe is not GREEN: {pristine.detail}" + ) + if runtime_fingerprint(runtime, names) != copied: + raise InfrastructureError(f"{mutation.assertion} pristine probe changed its runtime copy") + if tree_fingerprint(runtime) != copied_tree: + raise InfrastructureError(f"{mutation.assertion} pristine probe changed runtime topology") + + apply_mutation(runtime, mutation) + compile_mutation(runtime, mutation, root / "pycache") + mutated = runtime_fingerprint(runtime, names) + mutated_tree = tree_fingerprint(runtime) + changed = [name for (name, before), (_, after) in zip(copied, mutated) if before != after] + if changed != [mutation.path]: + raise InfrastructureError( + f"{mutation.assertion} changed unexpected private runtime paths: {changed!r}" + ) + + marker = root / "mutation-reached" + previous_bytecode = sys.dont_write_bytecode + sys.dont_write_bytecode = True + try: + result = mutation.probe(runtime, root / "mutant", marker) + finally: + sys.dont_write_bytecode = previous_bytecode + if not marker_reached(marker): + raise InfrastructureError(f"{mutation.assertion} did not prove its mutated path was reached") + if runtime_fingerprint(runtime, names) != mutated: + raise InfrastructureError(f"{mutation.assertion} mutant probe changed its runtime copy") + if tree_fingerprint(runtime) != mutated_tree: + raise InfrastructureError(f"{mutation.assertion} mutant probe changed runtime topology") + return not result.secure, result.detail + + +def remove_private_root(root: Path) -> None: + try: + if not root.exists() and not root.is_symlink(): + return + if root.is_symlink(): + root.unlink() + return + os.chmod(root, 0o700) + for directory, names, files in os.walk(root, topdown=True, followlinks=False): + current = Path(directory) + os.chmod(current, 0o700) + for name in names: + path = current / name + if not path.is_symlink(): + os.chmod(path, 0o700) + for name in files: + path = current / name + if not path.is_symlink(): + os.chmod(path, 0o600) + shutil.rmtree(root) + except OSError as error: + raise InfrastructureError("private mutation cleanup is uncertain") from error + if root.exists() or root.is_symlink(): + raise InfrastructureError("private mutation cleanup was incomplete") + + +def main() -> int: + try: + names = manifest_paths() + shared_before = runtime_fingerprint(REPO_ROOT, names) + expected = ( + "M-STORE-AUTH-001", + "M-STORE-SOURCE-001", + "M-STORE-ATOM-001", + "M-STORE-RETRY-001", + "M-STORE-DUR-001", + "M-STORE-LAYOUT-001", + "M-STORE-KEY-001", + "M-STORE-VERIFY-001", + "M-STORE-LIVE-001", + "M-STORE-UNCERT-001", + "M-STORE-STAGE-001", + ) + if tuple(mutation.assertion for mutation in MUTATIONS) != expected: + raise InfrastructureError("store mutation assertion identity drift") + failures = 0 + observed: list[str] = [] + for mutation in MUTATIONS: + try: + temporary = Path( + tempfile.mkdtemp( + prefix=f"agent-lab-{mutation.assertion.lower()}-", + dir="/tmp", + ) + ) + except OSError as error: + raise InfrastructureError( + f"{mutation.assertion} private mutation root is unavailable" + ) from error + try: + detected, detail = execute_mutation(temporary, names, mutation) + observed.append(mutation.assertion) + if detected: + print(f"PASS {mutation.assertion} {mutation.message}") + else: + failures += 1 + print(f"FAIL {mutation.assertion} {mutation.message} ({detail})") + finally: + remove_private_root(temporary) + if runtime_fingerprint(REPO_ROOT, names) != shared_before: + raise InfrastructureError( + f"{mutation.assertion} changed the shared checkout runtime fingerprint" + ) + if tuple(observed) != expected: + raise InfrastructureError("store mutation execution identity drift") + print(f"SUMMARY assertions=11 expected=11 failures={failures} infra=0") + return 0 if failures == 0 else 1 + except InfrastructureError as error: + print(f"INFRA store mutation evidence: {error}", file=sys.stderr) + return 125 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/experiment/local-lifecycle-cases.sh b/tests/experiment/local-lifecycle-cases.sh index c3effe4..d8dd372 100755 --- a/tests/experiment/local-lifecycle-cases.sh +++ b/tests/experiment/local-lifecycle-cases.sh @@ -8,8 +8,9 @@ subcases=( "$repo_root/tests/experiment/local-image-catalog-cases.sh" "$repo_root/tests/experiment/install-store-cases.sh" "$repo_root/tests/experiment/install-state-cases.py" + "$repo_root/tests/experiment/install-mutation-cases.py" ) -expected_count=109 +expected_count=121 work="" cleanup_work() { @@ -51,12 +52,15 @@ printf '%s\n' \ RES-SNAP-003 RES-AUTH-001 RES-INSTALL-001 RES-NOEF-001 \ M-CAT-OCI-001 M-CAT-SHADOW-001 M-CAT-CAS-001 M-CAT-AUTH-001 M-RES-BIND-001 \ M-CAT-NOEF-001 M-CAT-ADMIT-001 M-CAT-ATOM-001 M-CAT-DUR-001 M-CAT-STAGE-001 \ - INST-HOME-001 INST-UNKNOWN-001 INST-PERMIT-001 INST-RECEIPT-001 \ + INST-HOME-001 INST-UNKNOWN-001 INST-NAME-001 INST-PERMIT-001 INST-RECEIPT-001 \ INST-INSPECT-001 INST-RETRY-001 INST-CONFLICT-001 INST-DENY-001 \ - INST-FORGE-001 INST-NOEF-001 INST-LOCAL-001 INST-NAME-001 \ + INST-FORGE-001 INST-NOEF-001 INST-LOCAL-001 INST-RUNTIME-001 \ IST-STATE-001 IST-LOCK-001 IST-STATE-002 IST-BOUND-001 IST-STATE-003 \ IST-CONC-001 IST-CONC-002 IST-CRASH-001 IST-LIVE-001 IST-PLAT-001 \ - IST-PROV-001 > "$expected" + IST-PROV-001 \ + M-STORE-AUTH-001 M-STORE-SOURCE-001 M-STORE-ATOM-001 M-STORE-RETRY-001 \ + M-STORE-DUR-001 M-STORE-LAYOUT-001 M-STORE-KEY-001 M-STORE-VERIFY-001 \ + M-STORE-LIVE-001 M-STORE-UNCERT-001 M-STORE-STAGE-001 > "$expected" : > "$observed" infrastructure=0 From 0471a47444d506b19bcffe9f3d03d32c7f8ef903 Mon Sep 17 00:00:00 2001 From: xormania <127287135+xormania@users.noreply.github.com> Date: Sun, 2 Aug 2026 03:06:46 -0400 Subject: [PATCH 053/158] test(experiment): prove prepublication durability --- tests/experiment/install-mutation-cases.py | 42 ++++++++++++++-------- 1 file changed, 28 insertions(+), 14 deletions(-) diff --git a/tests/experiment/install-mutation-cases.py b/tests/experiment/install-mutation-cases.py index df23c51..d6b7f47 100755 --- a/tests/experiment/install-mutation-cases.py +++ b/tests/experiment/install-mutation-cases.py @@ -668,18 +668,28 @@ def observed_fsync(path: Path, purpose: str, *, modes=(0o700,)) -> None: store._fsync_directory = original_fsync store._rename_noreplace = original_rename publication = [index for index, event in enumerate(events) if event[0] == "publish"] - durable = [ - index - for index, event in enumerate(events) - if event == ("fsync", "Experiment store root") - ] + required_purposes = ( + "Experiment committed artifact", + "Experiment committed records", + "Experiment staged envelope root", + "Experiment committed wrapper", + "Experiment committed staging", + ) + durable = { + purpose: [ + index + for index, event in enumerate(events) + if event == ("fsync", purpose) + ] + for purpose in required_purposes + } secure = ( rc == 0 and isinstance(value, dict) and error is None and len(publication) == 1 - and len(durable) == 1 - and publication[0] < durable[0] + and all(len(indices) == 1 for indices in durable.values()) + and all(indices[0] < publication[0] for indices in durable.values()) ) return ProbeResult(secure, f"rc={rc} error={error!r} publish={publication} durable={durable}") @@ -1001,16 +1011,20 @@ def compile_mutation(runtime: Path, mutation: Mutation, cache: Path) -> None: Mutation( "M-STORE-DUR-001", "scripts/experiment_store.py", - ' _fsync_directory(authority.store, "Experiment store root")\n', + ' _fsync_directory(payload / "records", "Experiment committed records", modes=(0o500,))\n', ( - ' mutation_marker = os.environ.get("AGENT_LAB_MUTATION_MARK")\n' - ' if mutation_marker is None:\n' - ' _fsync_directory(authority.store, "Experiment store root")\n' - ' else:\n' - ' Path(mutation_marker).touch()\n' + ' mutation_marker = os.environ.get("AGENT_LAB_MUTATION_MARK")\n' + ' if mutation_marker is None:\n' + ' _fsync_directory(\n' + ' payload / "records",\n' + ' "Experiment committed records",\n' + ' modes=(0o500,),\n' + ' )\n' + ' else:\n' + ' Path(mutation_marker).touch()\n' ), probe_publication_durability, - "the durability oracle detects success without store-root fsync", + "the durability oracle detects publication before committed-directory fsync", ), Mutation( "M-STORE-LAYOUT-001", From 06571e301824314abe0f5a4da4cff67b3c1a7d80 Mon Sep 17 00:00:00 2001 From: xormania <127287135+xormania@users.noreply.github.com> Date: Sun, 2 Aug 2026 03:07:33 -0400 Subject: [PATCH 054/158] test(experiment): require plan identity domain --- tests/experiment/install-store-cases.sh | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/experiment/install-store-cases.sh b/tests/experiment/install-store-cases.sh index 119ddf2..b5dc93a 100755 --- a/tests/experiment/install-store-cases.sh +++ b/tests/experiment/install-store-cases.sh @@ -258,7 +258,8 @@ receipt = json.loads(receipt_bytes) for raw, value in ((plan_bytes, plan), (decision_bytes, decision), (provenance_bytes, provenance), (receipt_bytes, receipt)): assert raw == canonical(value) + b"\n" assert decision["verdict"] == "permit" -assert decision["binding"]["planDigest"] == digest(canonical(plan)) +plan_digest = identity_digest(b"agent-lab.experiment-plan.v1\0", plan) +assert decision["binding"]["planDigest"] == plan_digest assert isinstance(provenance, dict) and isinstance(provenance.get("apiVersion"), str) assert not any(item.startswith("/") for item in strings(provenance)) @@ -302,7 +303,7 @@ records = { "schema": decision["apiVersion"], }, "records/plan.json": { - "digest": digest(plan_bytes), + "digest": plan_digest, "schema": plan["apiVersion"], }, "records/provenance.json": { @@ -320,7 +321,7 @@ assert receipt == { "records": records, } receipt_digest = identity_digest(b"agent-lab.experiment-install-receipt.v1\0", receipt) -assert len({decision_digest, provenance_digest, receipt_digest}) == 3 +assert len({plan_digest, decision_digest, provenance_digest, receipt_digest}) == 4 result_bytes = result_path.read_bytes() result = json.loads(result_bytes) From 9cfe45362fe4b5a4be3fe24422641b7c589cd2c9 Mon Sep 17 00:00:00 2001 From: xormania <127287135+xormania@users.noreply.github.com> Date: Sun, 2 Aug 2026 03:13:43 -0400 Subject: [PATCH 055/158] test(experiment): expose store boundary failures --- tests/experiment/install-state-cases.py | 942 +++++++++++++++++++++++- 1 file changed, 917 insertions(+), 25 deletions(-) diff --git a/tests/experiment/install-state-cases.py b/tests/experiment/install-state-cases.py index f00d162..8d7834f 100644 --- a/tests/experiment/install-state-cases.py +++ b/tests/experiment/install-state-cases.py @@ -4,6 +4,7 @@ from __future__ import annotations from contextlib import contextmanager, redirect_stderr, redirect_stdout +import errno import hashlib from importlib.util import module_from_spec, spec_from_file_location import io @@ -17,7 +18,7 @@ import sys import tempfile import time -from typing import Callable +from typing import Callable, NamedTuple REPO_ROOT = Path(__file__).resolve().parents[2] @@ -37,6 +38,130 @@ ) +class FixtureSnapshot(NamedTuple): + data: bytes + digest: str + + +class FixtureResolution(NamedTuple): + plan: dict[str, object] + local_catalog: dict[str, object] | None + + +class FixtureInvalidManifest(Exception): + """The isolated store fixture rejected its authored manifest.""" + + +class FixtureInfrastructure(Exception): + """The isolated store fixture could not establish a trusted result.""" + + +class FaultFixtureExperiment: + """Small deterministic planner used only for exhaustive store fault injection.""" + + InvalidManifest = FixtureInvalidManifest + InfrastructureError = FixtureInfrastructure + + def __init__(self, source_data: bytes, name: str) -> None: + source_digest = store_source_digest(source_data) + contract_digest = "sha256:" + "d" * 64 + plan: dict[str, object] = { + "apiVersion": "agent-lab.request/v0alpha1", + "contract": { + "digest": contract_digest, + "name": "agent-lab.experiment", + "version": "v0alpha1", + }, + "kind": "RequestedExperimentPlan", + "metadata": {"requestedName": name}, + "spec": { + "members": [ + { + "command": ["fault-probe"], + "name": "worker", + "requestedSelector": {"digestRef": SUBJECT}, + "resolvedImage": {"origin": "direct", "subject": SUBJECT}, + "resourceClass": "small", + } + ] + }, + } + plan_digest = "sha256:" + hashlib.sha256(canonical_json(plan)).hexdigest() + self.source_data = source_data + self.snapshot = FixtureSnapshot(source_data, source_digest) + self.resolution = FixtureResolution(plan, None) + self.decision: dict[str, object] = { + "action": "experiment.install", + "apiVersion": "agent-lab.authorization/v0alpha1", + "binding": { + "authorizationDigest": "sha256:" + "c" * 64, + "contractDigest": contract_digest, + "planDigest": plan_digest, + "sourceDigest": source_digest, + }, + "kind": "ExperimentAuthorizationDecision", + "principal": { + "assurance": "none", + "authenticated": False, + "id": "local-cli", + "source": "fixed-local-cli", + "type": "AgentLab::Principal", + }, + "resource": { + "id": plan_digest, + "requestedName": name, + "type": "AgentLab::RequestedExperimentPlan", + }, + "verdict": "permit", + } + + def read_directory_snapshot(self, source: str) -> FixtureSnapshot: + try: + observed = (Path(source) / "experiment.cue").read_bytes() + except OSError as error: + raise FixtureInfrastructure("fault fixture source cannot be read") from error + if observed != self.source_data: + raise FixtureInvalidManifest("fault fixture source changed") + return self.snapshot + + def authored_manifest(self, snapshot: FixtureSnapshot) -> bytes: + return snapshot.data + + def cue_plan_with_evidence(self, manifest: object) -> FixtureResolution: + if manifest != self.source_data: + raise FixtureInvalidManifest("fault fixture manifest changed") + return self.resolution + + def authorize_plan( + self, + plan: dict[str, object], + source_digest: str, + ) -> tuple[dict[str, object], int]: + if plan != self.resolution.plan or source_digest != self.snapshot.digest: + raise FixtureInfrastructure("fault fixture authorization binding changed") + return self.decision, 0 + + +def canonical_json(value: object) -> bytes: + return json.dumps( + value, + allow_nan=False, + ensure_ascii=True, + separators=(",", ":"), + sort_keys=True, + ).encode("ascii") + + +def store_source_digest(data: bytes) -> str: + value = hashlib.sha256(b"agent-lab.experiment-tree.v1\0") + name = b"experiment.cue" + value.update(len(name).to_bytes(4, "big")) + value.update(name) + value.update(len(data).to_bytes(8, "big")) + value.update(data) + return "sha256:" + value.hexdigest() + + def load_module(path: Path, name: str): spec = spec_from_file_location(name, path) if spec is None or spec.loader is None: @@ -134,6 +259,26 @@ def start_inspect(home: Path, name: str) -> subprocess.Popen[bytes]: ) +def start_remove(home: Path, name: str, entry_digest: str) -> subprocess.Popen[bytes]: + return subprocess.Popen( + [ + str(AGENT_LAB), + "--home", + str(home), + "image", + "remove", + name, + "--expect", + entry_digest, + ], + stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + env=command_environment(), + start_new_session=True, + ) + + def finish_process( process: subprocess.Popen[bytes], timeout: float = 30.0, @@ -412,6 +557,56 @@ def stop_before_intent(path, data, purpose, fault): return 124 +def hard_exit_during_cleanup(home: Path, source: Path, phase: str) -> int: + """Exit after durable cleanup handoff or after the first cleanup-file removal.""" + + if STORE is None: + return 95 + pid = os.fork() + if pid == 0: + if phase == "handoff": + original_rename = STORE._rename_noreplace + + def stop_after_handoff(source_path: Path, target_path: Path) -> None: + original_rename(source_path, target_path) + if target_path.name == STORE.CLEANUP_WRAPPER: + os._exit(99) + + STORE._rename_noreplace = stop_after_handoff + elif phase == "remove": + original_remove = STORE._remove_tree + stopped = False + + def stop_after_first_file(path: Path, root: Path) -> None: + nonlocal stopped + was_file = False + try: + was_file = stat.S_ISREG(path.lstat().st_mode) + except OSError: + pass + original_remove(path, root) + if was_file and not stopped: + stopped = True + os._exit(99) + + STORE._remove_tree = stop_after_first_file + else: + os._exit(95) + result, _, _ = store_install(home, source) + os._exit(97 if result == 0 else 96) + deadline = time.monotonic() + 30.0 + while time.monotonic() < deadline: + waited, status = os.waitpid(pid, os.WNOHANG) + if waited == pid: + return os.waitstatus_to_exitcode(status) + time.sleep(0.01) + os.kill(pid, signal.SIGKILL) + os.waitpid(pid, 0) + global INFRA + INFRA += 1 + return 124 + + def start_paused_publication(home: Path, source: Path) -> tuple[int, int, int]: """Pause a child install while it holds the store lock before publication.""" @@ -869,15 +1064,20 @@ def change_after_enumeration(path: Path, purpose: str, maximum: int): ) identical_home = new_home(root, "identical-concurrency-home") - first = start_cli(identical_home, direct_source) - second = start_cli(identical_home, direct_source) - identical_results = [finish_process(first), finish_process(second)] - identical_values = [json_object(item) for item in identical_results] - identical_changes = sorted( - value.get("changed") - for value in identical_values - if isinstance(value, dict) and isinstance(value.get("changed"), bool) + first_pid, first_release, first_pause = start_paused_publication( + identical_home, + direct_source, ) + second = start_cli(identical_home, direct_source) + identical_waited = wait_for_lock_block(second) + if first_release >= 0: + try: + os.write(first_release, b"1") + finally: + os.close(first_release) + first_rc = finish_child(first_pid) + second_result = finish_process(second) + second_value = json_object(second_result) identical_inspect = cli( identical_home, "experiment", @@ -886,14 +1086,19 @@ def change_after_enumeration(path: Path, purpose: str, maximum: int): ) check( "IST-CONC-001", - [item.returncode for item in identical_results] == [0, 0] - and identical_changes == [False, True] + first_pause == 0 + and identical_waited + and first_rc == 0 + and second_result.returncode == 0 + and isinstance(second_value, dict) + and second_value.get("changed") is False and identical_inspect.returncode == 0 and not tuple((identical_home / "experiments" / ".staging").iterdir()), - "concurrent identical installs publish once and return one verified idempotent success", + "overlapping identical installs block at the store lock and publish once", ( - f"rcs={[item.returncode for item in identical_results]!r} " - f"changes={identical_changes!r} inspect={identical_inspect.returncode}" + f"pause={first_pause} waited={identical_waited} first={first_rc} " + f"second={second_result.returncode}/{second_value!r} " + f"inspect={identical_inspect.returncode}" ), ) @@ -908,24 +1113,47 @@ def change_after_enumeration(path: Path, purpose: str, maximum: int): "conflict-source-two", command="winner-two", ) - one = start_cli(conflict_home, conflict_one) + one_pid, one_release, one_pause = start_paused_publication( + conflict_home, + conflict_one, + ) two = start_cli(conflict_home, conflict_two) - conflict_results = [finish_process(one), finish_process(two)] + conflict_waited = wait_for_lock_block(two) + if one_release >= 0: + try: + os.write(one_release, b"1") + finally: + os.close(one_release) + one_rc = finish_child(one_pid) + two_result = finish_process(two) conflict_inspect = cli( conflict_home, "experiment", "inspect", "first-experiment", ) + conflict_artifact = ( + conflict_home + / "experiments" + / "first-experiment" + / "artifact" + / "experiment.cue" + ) check( "IST-CONC-002", - sorted(item.returncode for item in conflict_results) == [0, 1] + one_pause == 0 + and conflict_waited + and one_rc == 0 + and two_result.returncode == 1 and conflict_inspect.returncode == 0 + and conflict_artifact.is_file() + and conflict_artifact.read_bytes() + == (conflict_one / "experiment.cue").read_bytes() and not tuple((conflict_home / "experiments" / ".staging").iterdir()), - "concurrent different candidates have one winner and one ordinary conflict", + "overlapping different candidates serialize to the paused winner and one conflict", ( - f"rcs={[item.returncode for item in conflict_results]!r} " - f"inspect={conflict_inspect.returncode}" + f"pause={one_pause} waited={conflict_waited} first={one_rc} " + f"second={two_result.returncode} inspect={conflict_inspect.returncode}" ), ) @@ -1020,6 +1248,44 @@ def change_after_enumeration(path: Path, purpose: str, maximum: int): output_inspect = cli(output_home, "experiment", "inspect", "first-experiment") output_retry = cli(output_home, "experiment", "install", str(direct_source)) output_value = json_object(output_retry) + cleanup_crash_failures: list[str] = [] + for phase in ("handoff", "remove"): + cleanup_crash_home = new_home(root, f"cleanup-{phase}-crash-home") + cleanup_child_rc = hard_exit_during_cleanup( + cleanup_crash_home, + direct_source, + phase, + ) + cleanup_stage = cleanup_crash_home / "experiments" / ".staging" + cleanup_before_inspect = fingerprint(cleanup_stage) + cleanup_inspect = cli( + cleanup_crash_home, + "experiment", + "inspect", + "first-experiment", + ) + cleanup_after_inspect = fingerprint(cleanup_stage) + cleanup_retry = cli( + cleanup_crash_home, + "experiment", + "install", + str(direct_source), + ) + cleanup_retry_value = json_object(cleanup_retry) + if not ( + cleanup_child_rc == 99 + and cleanup_inspect.returncode == 0 + and cleanup_before_inspect == cleanup_after_inspect + and cleanup_retry.returncode == 0 + and isinstance(cleanup_retry_value, dict) + and cleanup_retry_value.get("changed") is False + and not tuple(cleanup_stage.iterdir()) + ): + cleanup_crash_failures.append( + f"{phase}:child={cleanup_child_rc}:inspect={cleanup_inspect.returncode}:" + f"read_changed={cleanup_before_inspect != cleanup_after_inspect}:" + f"retry={cleanup_retry.returncode}/{cleanup_retry_value!r}" + ) check( "IST-CRASH-001", seam_rc == 0 @@ -1033,6 +1299,7 @@ def change_after_enumeration(path: Path, purpose: str, maximum: int): and output_retry.returncode == 0 and isinstance(output_value, dict) and output_value.get("changed") is False + and not cleanup_crash_failures and preintent_child_rc == 99 and preintent_inspect.returncode == 1 and preintent_before == preintent_after_inspect @@ -1054,6 +1321,7 @@ def change_after_enumeration(path: Path, purpose: str, maximum: int): f"missing={sorted(set(FAULT_POINTS)-set(observed_points))!r} " f"crashes={crash_failures[:3]!r} output={output_rc}/{output_error!r}/" f"{output_inspect.returncode}/{output_retry.returncode}/{output_value!r} " + f"cleanup={cleanup_crash_failures[:2]!r} " f"preintent={preintent_child_rc}/{preintent_inspect.returncode}/" f"{preintent_before != preintent_after_inspect}/{preintent_retry.returncode}/" f"{preintent_value!r} " @@ -1113,6 +1381,62 @@ def observe_locks(point: str) -> None: order_ok = catalog_index < store_index < publish_index except ValueError: order_ok = False + + live_race_home = new_home(root, "selected-entry-removal-race-home") + race_added = cli( + live_race_home, + "image", + "add", + "vendor.worker", + SUBJECT, + ) + race_added_value = json_object(race_added) + race_entry_digest = ( + race_added_value.get("entryDigest") + if isinstance(race_added_value, dict) + else None + ) + race_install_pid = 0 + race_release = -1 + race_pause = -1 + race_remove: subprocess.Popen[bytes] | None = None + race_remove_waited = False + race_install_rc = 124 + race_remove_result = subprocess.CompletedProcess([], 125, b"", b"missing entry digest") + if isinstance(race_entry_digest, str): + race_install_pid, race_release, race_pause = start_paused_publication( + live_race_home, + local_source, + ) + race_remove = start_remove( + live_race_home, + "vendor.worker", + race_entry_digest, + ) + race_remove_waited = wait_for_lock_block(race_remove) + if race_release >= 0: + try: + os.write(race_release, b"1") + finally: + os.close(race_release) + if race_install_pid > 0: + race_install_rc = finish_child(race_install_pid) + if race_remove is not None: + race_remove_result = finish_process(race_remove) + race_retained = cli( + live_race_home, + "experiment", + "inspect", + "first-experiment", + ) + race_before_retry = fingerprint(live_race_home / "experiments") + race_stale_retry = cli( + live_race_home, + "experiment", + "install", + str(local_source), + ) + race_after_retry = fingerprint(live_race_home / "experiments") check( "IST-LIVE-001", added.returncode == 0 @@ -1125,13 +1449,24 @@ def observe_locks(point: str) -> None: and removed.returncode == 0 and stale_retry.returncode == 1 and store_before_retry == store_after_retry - and retained.returncode == 0, - "selected-entry and store locks remain held; removal blocks stale retry", + and retained.returncode == 0 + and race_added.returncode == 0 + and race_pause == 0 + and race_remove_waited + and race_install_rc == 0 + and race_remove_result.returncode == 0 + and race_retained.returncode == 0 + and race_stale_retry.returncode == 1 + and race_before_retry == race_after_retry, + "selected-entry locks order correctly and serialize a real removal race", ( f"add={added.returncode} install={live_rc}/{live_error!r} events={live_events!r} " f"held={held_catalog}/{held_store} remove={removed.returncode} " f"retry={stale_retry.returncode}/{store_before_retry != store_after_retry} " - f"inspect={retained.returncode}" + f"inspect={retained.returncode} race={race_added.returncode}/{race_pause}/" + f"{race_remove_waited}/{race_install_rc}/{race_remove_result.returncode}/" + f"{race_retained.returncode}/{race_stale_retry.returncode}/" + f"{race_before_retry != race_after_retry}" ), ) @@ -1283,6 +1618,558 @@ def observed_open(path: object, flags: int, mode: int = 0o777, *, dir_fd=None): ), ) + planning = load_module( + REPO_ROOT / "scripts" / "experiment.py", + "agent_lab_experiment_store_adversarial", + ) + + cue_home = new_home(root, "preflight-cue-home") + cue_source = source_directory(root, "preflight-cue-source") + cue_path = cue_source / "experiment.cue" + cue_raw = cue_path.read_bytes() + cue_path.write_bytes( + cue_raw.replace( + b'\t\tcommand: ["serve"]\n', + b'\t\tcommand: ["serve"]\n\t\tunknownField: true\n', + 1, + ) + ) + cue_before = fingerprint(cue_home / "experiments") + cue_failure = cli(cue_home, "experiment", "install", str(cue_source)) + cue_after = fingerprint(cue_home / "experiments") + + contract_home = new_home(root, "preflight-contract-home") + contract_source = source_directory(root, "preflight-contract-source") + contract_before = fingerprint(contract_home / "experiments") + original_experiment_loader = STORE._experiment_module if STORE is not None else None + original_verify_contract = planning.verify_contract_snapshot + + def report_contract_drift(repo_root: Path, expected: dict[str, bytes]) -> None: + del repo_root, expected + raise planning.InfrastructureError("contract snapshot changed during validation") + + contract_rc: int | None = None + contract_error: BaseException | None = None + if STORE is not None: + STORE._experiment_module = lambda: planning + planning.verify_contract_snapshot = report_contract_drift + try: + contract_rc, _, contract_error = store_install( + contract_home, + contract_source, + ) + finally: + planning.verify_contract_snapshot = original_verify_contract + STORE._experiment_module = original_experiment_loader + contract_after = fingerprint(contract_home / "experiments") + + selected_home = new_home(root, "preflight-selected-home") + selected_add = cli( + selected_home, + "image", + "add", + "vendor.worker", + SUBJECT, + ) + selected_add_value = json_object(selected_add) + selected_digest = ( + selected_add_value.get("entryDigest") + if isinstance(selected_add_value, dict) + else None + ) + selected_source = source_directory( + root, + "preflight-selected-source", + catalog_name="vendor.worker", + ) + selected_before = fingerprint(selected_home / "experiments") + selected_remove: subprocess.CompletedProcess[bytes] | None = None + selected_rc: int | None = None + selected_error: BaseException | None = None + if STORE is not None and isinstance(selected_digest, str): + original_held_context = STORE._held_catalog_context + + @contextmanager + def remove_selected_before_hold(home: Path, dependencies, fault): + nonlocal selected_remove + selected_remove = cli( + home, + "image", + "remove", + "vendor.worker", + "--expect", + selected_digest, + ) + with original_held_context(home, dependencies, fault) as held: + yield held + + STORE._held_catalog_context = remove_selected_before_hold + try: + selected_rc, _, selected_error = store_install( + selected_home, + selected_source, + ) + finally: + STORE._held_catalog_context = original_held_context + selected_after = fingerprint(selected_home / "experiments") + check( + "IST-PREFLIGHT-001", + cue_raw != cue_path.read_bytes() + and cue_failure.returncode == 1 + and not cue_failure.stdout + and cue_before == cue_after + and contract_rc == 125 + and contract_error is not None + and contract_before == contract_after + and selected_add.returncode == 0 + and selected_remove is not None + and selected_remove.returncode == 0 + and selected_rc == 1 + and selected_error is not None + and selected_before == selected_after, + "CUE, contract, and selected-entry drift fail before store-visible effects", + ( + f"cue={cue_failure.returncode}/{cue_before != cue_after} " + f"contract={contract_rc}/{contract_error!r}/{contract_before != contract_after} " + f"selected={selected_add.returncode}/" + f"{None if selected_remove is None else selected_remove.returncode}/" + f"{selected_rc}/{selected_error!r}/{selected_before != selected_after}" + ), + ) + + snapshot_mutation_home = new_home(root, "snapshot-after-mutation-home") + snapshot_mutation_source = source_directory( + root, + "snapshot-after-mutation-source", + requested_name="snapshot-mutated", + ) + snapshot_mutation_path = snapshot_mutation_source / "experiment.cue" + snapshot_original = snapshot_mutation_path.read_bytes() + snapshot_changed = snapshot_original.replace(b"serve", b"mutat", 1) + original_authored_manifest = planning.authored_manifest + mutation_after_snapshot = False + + def mutate_after_snapshot(snapshot): + nonlocal mutation_after_snapshot + snapshot_mutation_path.write_bytes(snapshot_changed) + mutation_after_snapshot = True + return original_authored_manifest(snapshot) + + snapshot_mutation_rc: int | None = None + snapshot_mutation_error: BaseException | None = None + if STORE is not None: + STORE._experiment_module = lambda: planning + planning.authored_manifest = mutate_after_snapshot + try: + snapshot_mutation_rc, _, snapshot_mutation_error = store_install( + snapshot_mutation_home, + snapshot_mutation_source, + ) + finally: + planning.authored_manifest = original_authored_manifest + STORE._experiment_module = original_experiment_loader + stored_mutation_artifact = ( + snapshot_mutation_home + / "experiments" + / "snapshot-mutated" + / "artifact" + / "experiment.cue" + ) + + snapshot_deletion_home = new_home(root, "snapshot-after-deletion-home") + snapshot_deletion_source = source_directory( + root, + "snapshot-after-deletion-source", + requested_name="snapshot-deleted", + ) + snapshot_deletion_path = snapshot_deletion_source / "experiment.cue" + deletion_original = snapshot_deletion_path.read_bytes() + deletion_after_snapshot = False + + def delete_after_snapshot(snapshot): + nonlocal deletion_after_snapshot + snapshot_deletion_path.unlink() + deletion_after_snapshot = True + return original_authored_manifest(snapshot) + + snapshot_deletion_rc: int | None = None + snapshot_deletion_error: BaseException | None = None + if STORE is not None: + STORE._experiment_module = lambda: planning + planning.authored_manifest = delete_after_snapshot + try: + snapshot_deletion_rc, _, snapshot_deletion_error = store_install( + snapshot_deletion_home, + snapshot_deletion_source, + ) + finally: + planning.authored_manifest = original_authored_manifest + STORE._experiment_module = original_experiment_loader + stored_deletion_artifact = ( + snapshot_deletion_home + / "experiments" + / "snapshot-deleted" + / "artifact" + / "experiment.cue" + ) + + snapshot_race_home = new_home(root, "snapshot-during-read-home") + snapshot_race_source = source_directory( + root, + "snapshot-during-read-source", + requested_name="snapshot-raced", + ) + snapshot_race_path = snapshot_race_source / "experiment.cue" + race_original = snapshot_race_path.read_bytes() + race_changed = race_original.replace(b"serve", b"mutat", 1) + race_metadata = snapshot_race_path.stat() + snapshot_race_before = fingerprint(snapshot_race_home / "experiments") + original_planning_read = planning.os.read + race_mutated_during_read = False + + def mutate_during_read(descriptor: int, maximum: int) -> bytes: + nonlocal race_mutated_during_read + data = original_planning_read(descriptor, maximum) + try: + target = os.readlink(f"/proc/self/fd/{descriptor}") + except OSError: + target = "" + if ( + data + and not race_mutated_during_read + and target == str(snapshot_race_path) + ): + time.sleep(0.001) + snapshot_race_path.write_bytes(race_changed) + os.utime( + snapshot_race_path, + ns=(race_metadata.st_atime_ns, race_metadata.st_mtime_ns), + ) + race_mutated_during_read = True + return data + + snapshot_race_rc: int | None = None + snapshot_race_error: BaseException | None = None + if STORE is not None: + STORE._experiment_module = lambda: planning + planning.os.read = mutate_during_read + try: + snapshot_race_rc, _, snapshot_race_error = store_install( + snapshot_race_home, + snapshot_race_source, + ) + finally: + planning.os.read = original_planning_read + STORE._experiment_module = original_experiment_loader + snapshot_race_after = fingerprint(snapshot_race_home / "experiments") + race_final_metadata = snapshot_race_path.stat() + check( + "IST-SNAPSHOT-001", + snapshot_original != snapshot_changed + and mutation_after_snapshot + and snapshot_mutation_rc == 0 + and snapshot_mutation_error is None + and stored_mutation_artifact.is_file() + and stored_mutation_artifact.read_bytes() == snapshot_original + and deletion_after_snapshot + and snapshot_deletion_rc == 0 + and snapshot_deletion_error is None + and not snapshot_deletion_path.exists() + and stored_deletion_artifact.is_file() + and stored_deletion_artifact.read_bytes() == deletion_original + and race_original != race_changed + and race_mutated_during_read + and race_final_metadata.st_mtime_ns == race_metadata.st_mtime_ns + and race_final_metadata.st_ctime_ns != race_metadata.st_ctime_ns + and snapshot_race_rc == 125 + and snapshot_race_error is not None + and snapshot_race_before == snapshot_race_after, + "post-snapshot source drift cannot change bytes and during-read drift is uncertain", + ( + f"mutation={mutation_after_snapshot}/{snapshot_mutation_rc}/" + f"{snapshot_mutation_error!r}/" + f"{stored_mutation_artifact.is_file() and stored_mutation_artifact.read_bytes() == snapshot_original} " + f"deletion={deletion_after_snapshot}/{snapshot_deletion_rc}/" + f"{snapshot_deletion_error!r}/" + f"{stored_deletion_artifact.is_file() and stored_deletion_artifact.read_bytes() == deletion_original} " + f"during={race_mutated_during_read}/{snapshot_race_rc}/" + f"{snapshot_race_error!r}/{snapshot_race_before != snapshot_race_after}/" + f"mtime={race_final_metadata.st_mtime_ns == race_metadata.st_mtime_ns}/" + f"ctime={race_final_metadata.st_ctime_ns != race_metadata.st_ctime_ns}" + ), + ) + + config_home = new_home(root, "changing-config-home") + config_source = source_directory(root, "changing-config-source") + config_path = config_home / "config.json" + config_raw = config_path.read_bytes() + config_stage = config_home / "experiments" / ".staging" + config_target = config_home / "experiments" / "first-experiment" + config_changed = False + config_rc: int | None = None + config_error: BaseException | None = None + if STORE is not None: + original_prepare_stage = STORE._prepare_stage + + def change_config_after_staging(*args, **kwargs): + nonlocal config_changed + prepared = original_prepare_stage(*args, **kwargs) + config_path.write_bytes(config_raw + b" ") + config_changed = True + return prepared + + STORE._prepare_stage = change_config_after_staging + try: + config_rc, _, config_error = store_install(config_home, config_source) + finally: + STORE._prepare_stage = original_prepare_stage + config_final_absent = not config_target.exists() and not config_target.is_symlink() + config_wrapper_count = len(tuple(config_stage.iterdir())) + config_path.write_bytes(config_raw) + config_before_inspect = fingerprint(config_stage) + config_inspect_before = cli( + config_home, + "experiment", + "inspect", + "first-experiment", + ) + config_after_inspect = fingerprint(config_stage) + config_retry = cli( + config_home, + "experiment", + "install", + str(config_source), + ) + config_retry_value = json_object(config_retry) + config_inspect_after = cli( + config_home, + "experiment", + "inspect", + "first-experiment", + ) + check( + "IST-CONFIG-001", + config_changed + and config_rc == 125 + and config_error is not None + and config_final_absent + and config_wrapper_count == 1 + and config_inspect_before.returncode == 1 + and config_before_inspect == config_after_inspect + and config_retry.returncode == 0 + and isinstance(config_retry_value, dict) + and config_retry_value.get("changed") is True + and config_inspect_after.returncode == 0 + and not tuple(config_stage.iterdir()), + "configuration drift before publication is uncertain and its owned residue is restartable", + ( + f"changed={config_changed} install={config_rc}/{config_error!r} " + f"final_absent={config_final_absent} wrappers={config_wrapper_count} " + f"inspect={config_inspect_before.returncode}/" + f"{config_before_inspect != config_after_inspect} " + f"retry={config_retry.returncode}/{config_retry_value!r}/" + f"{config_inspect_after.returncode}" + ), + ) + + reader_home = new_home(root, "reader-tamper-home") + reader_install = cli( + reader_home, + "experiment", + "install", + str(direct_source), + ) + reader_envelope = reader_home / "experiments" / "first-experiment" + reader_failures: list[str] = [] + reader_paths = ( + "artifact/experiment.cue", + "records/decision.json", + "records/plan.json", + "records/provenance.json", + "records/install.json", + ) + if reader_install.returncode == 0: + for index, relative in enumerate(reader_paths): + path = reader_envelope / relative + raw = path.read_bytes() + link = root / f"reader-hardlink-{index}" + os.link(path, link) + linked_rc, _, linked_error = store_inspect( + reader_home, + "first-experiment", + ) + link.unlink() + restored_link_rc, _, restored_link_error = store_inspect( + reader_home, + "first-experiment", + ) + path.chmod(0o600) + path.write_bytes(bytes((raw[0] ^ 1,)) + raw[1:]) + path.chmod(0o400) + corrupt_before_retry = fingerprint(reader_envelope) + corrupt_rc, _, corrupt_error = store_inspect( + reader_home, + "first-experiment", + ) + retry_rc: int | None = 125 + retry_error: BaseException | None = RuntimeError("not receipt") + retry_unchanged = True + if relative == "records/install.json": + retry_rc, _, retry_error = store_install(reader_home, direct_source) + retry_unchanged = corrupt_before_retry == fingerprint(reader_envelope) + path.chmod(0o600) + path.write_bytes(raw) + path.chmod(0o400) + restored_rc, _, restored_error = store_inspect( + reader_home, + "first-experiment", + ) + if not ( + linked_rc == 125 + and linked_error is not None + and restored_link_rc == 0 + and restored_link_error is None + and corrupt_rc == 125 + and corrupt_error is not None + and ( + relative != "records/install.json" + or (retry_rc == 125 and retry_error is not None and retry_unchanged) + ) + and restored_rc == 0 + and restored_error is None + ): + reader_failures.append( + f"{relative}:link={linked_rc}/{restored_link_rc}:" + f"corrupt={corrupt_rc}:retry={retry_rc}/{retry_unchanged}:" + f"restore={restored_rc}" + ) + check( + "IST-READ-001", + reader_install.returncode == 0 and not reader_failures, + "every stored byte record is safe-opened, digest-bound, and non-overwritable", + f"install={reader_install.returncode} failures={reader_failures[:5]!r}", + ) + + fault_source = source_directory( + root, + "fault-matrix-source", + requested_name="fault-experiment", + ) + fault_data = (fault_source / "experiment.cue").read_bytes() + fault_fixture = FaultFixtureExperiment(fault_data, "fault-experiment") + fault_failures: list[str] = [] + fault_counts: dict[str, int] = {} + primitives = ("open", "write", "chmod", "fsync", "rename_noreplace") + if STORE is not None: + original_fault_loader = STORE._experiment_module + STORE._experiment_module = lambda: fault_fixture + try: + for primitive in primitives: + baseline_home = new_home(root, f"fault-{primitive}-baseline-home") + if primitive == "rename_noreplace": + owner = STORE + attribute = "_rename_noreplace" + else: + owner = STORE.os + attribute = primitive + original_primitive = getattr(owner, attribute) + observed_calls = 0 + + def observe_primitive(*args, _original=original_primitive, **kwargs): + nonlocal observed_calls + observed_calls += 1 + return _original(*args, **kwargs) + + setattr(owner, attribute, observe_primitive) + try: + baseline_rc, baseline_value, baseline_error = store_install( + baseline_home, + fault_source, + ) + finally: + setattr(owner, attribute, original_primitive) + fault_counts[primitive] = observed_calls + if not ( + baseline_rc == 0 + and isinstance(baseline_value, dict) + and baseline_error is None + and observed_calls > 0 + ): + fault_failures.append( + f"{primitive}:baseline={baseline_rc}/{baseline_error!r}/" + f"calls={observed_calls}" + ) + continue + for ordinal in range(1, observed_calls + 1): + injected_home = new_home( + root, + f"fault-{primitive}-{ordinal:02d}-home", + ) + injected_calls = 0 + injected_hit = False + + def inject_primitive(*args, _original=original_primitive, **kwargs): + nonlocal injected_calls, injected_hit + injected_calls += 1 + if injected_calls == ordinal: + injected_hit = True + raise OSError(errno.EIO, f"injected {primitive} failure") + return _original(*args, **kwargs) + + setattr(owner, attribute, inject_primitive) + try: + injected_rc, injected_value, injected_error = store_install( + injected_home, + fault_source, + ) + finally: + setattr(owner, attribute, original_primitive) + before_retry_rc, _, _ = store_inspect( + injected_home, + "fault-experiment", + ) + retry_rc, retry_value, retry_error = store_install( + injected_home, + fault_source, + ) + final_rc, _, final_error = store_inspect( + injected_home, + "fault-experiment", + ) + staging_empty = not tuple( + (injected_home / "experiments" / ".staging").iterdir() + ) + if not ( + injected_hit + and injected_rc == 125 + and injected_value is None + and injected_error is not None + and before_retry_rc in (0, 1, 125) + and retry_rc == 0 + and isinstance(retry_value, dict) + and retry_error is None + and final_rc == 0 + and final_error is None + and staging_empty + ): + fault_failures.append( + f"{primitive}[{ordinal}/{observed_calls}]:" + f"hit={injected_hit}:first={injected_rc}/{injected_error!r}:" + f"before={before_retry_rc}:retry={retry_rc}/{retry_error!r}:" + f"final={final_rc}/{final_error!r}:empty={staging_empty}" + ) + finally: + STORE._experiment_module = original_fault_loader + check( + "IST-FAULT-001", + STORE is not None + and set(fault_counts) == set(primitives) + and all(count > 0 for count in fault_counts.values()) + and not fault_failures, + "every store open/write/chmod/fsync/rename failure is uncertain and restartable", + f"counts={fault_counts!r} failures={fault_failures[:8]!r}", + ) + expected = [ "IST-STATE-001", "IST-LOCK-001", @@ -1295,11 +2182,16 @@ def observed_open(path: object, flags: int, mode: int = 0o777, *, dir_fd=None): "IST-LIVE-001", "IST-PLAT-001", "IST-PROV-001", + "IST-PREFLIGHT-001", + "IST-SNAPSHOT-001", + "IST-CONFIG-001", + "IST-READ-001", + "IST-FAULT-001", ] if OBSERVED != expected: print(f"INFRA install state assertion identity drift: {OBSERVED!r}", file=sys.stderr) return 125 - print(f"SUMMARY assertions=11 expected=11 failures={FAILURES} infra={INFRA}") + print(f"SUMMARY assertions=16 expected=16 failures={FAILURES} infra={INFRA}") if INFRA: return 125 return 0 if FAILURES == 0 else 1 @@ -1313,6 +2205,6 @@ def observed_open(path: object, flags: int, mode: int = 0o777, *, dir_fd=None): except BaseException as error: print(f"INFRA install state harness failed: {error!r}", file=sys.stderr) print( - f"SUMMARY assertions={len(OBSERVED)} expected=11 failures={FAILURES} infra=1" + f"SUMMARY assertions={len(OBSERVED)} expected=16 failures={FAILURES} infra=1" ) raise SystemExit(125) From c2e955c13b4a2602172f3f14861d755263bfdb03 Mon Sep 17 00:00:00 2001 From: xormania <127287135+xormania@users.noreply.github.com> Date: Sun, 2 Aug 2026 03:14:08 -0400 Subject: [PATCH 056/158] test(experiment): expose store integrity gaps --- tests/experiment/install-integrity-cases.py | 843 ++++++++++++++++++++ 1 file changed, 843 insertions(+) create mode 100644 tests/experiment/install-integrity-cases.py diff --git a/tests/experiment/install-integrity-cases.py b/tests/experiment/install-integrity-cases.py new file mode 100644 index 0000000..c725c69 --- /dev/null +++ b/tests/experiment/install-integrity-cases.py @@ -0,0 +1,843 @@ +#!/usr/bin/env python3 +"""Private-runtime integrity cases for Experiment installation boundaries.""" + +from __future__ import annotations + +from contextlib import contextmanager +from dataclasses import dataclass +import errno +from importlib.util import module_from_spec, spec_from_file_location +import json +import os +from pathlib import Path +import signal +import stat +import subprocess +import sys +import tempfile +import time +from typing import Callable, Iterator, NamedTuple + + +REPO_ROOT = Path(__file__).resolve().parents[2] +SUPPORT_PATH = REPO_ROOT / "tests" / "experiment" / "install-mutation-cases.py" +CUE_TOOLS = REPO_ROOT / ".cache" / "dev" / "tools" / "cue" +CEDAR_TOOLS = REPO_ROOT / ".cache" / "dev" / "tools" / "cedar" +BUNDLED_CATALOG_DOMAIN = b"agent-lab.experiment-image-catalog.v1\0" +PLAN_DOMAIN = b"agent-lab.experiment-plan.v1\0" +PUBLIC_TIMEOUT_SECONDS = 2.0 +RACE_TIMEOUT_SECONDS = 0.75 + + +def load_support(): + spec = spec_from_file_location("agent_lab_install_integrity_support", SUPPORT_PATH) + if spec is None or spec.loader is None: + raise RuntimeError("install mutation support cannot be loaded") + module = module_from_spec(spec) + sys.modules[spec.name] = module + previous = sys.dont_write_bytecode + sys.dont_write_bytecode = True + try: + spec.loader.exec_module(module) + finally: + sys.dont_write_bytecode = previous + return module + + +try: + SUPPORT = load_support() + SUPPORT_ERROR: BaseException | None = None +except BaseException as error: + SUPPORT = None + SUPPORT_ERROR = error + + +if SUPPORT is None: + class SupportInfrastructure(Exception): + """Fallback type used only when the private support module cannot load.""" +else: + SupportInfrastructure = SUPPORT.InfrastructureError + + +class IntegrityInfrastructure(Exception): + """The harness could not establish bounded, isolated evidence.""" + + +class Result(NamedTuple): + secure: bool + detail: str + + +Probe = Callable[[Path, Path, tuple[str, ...]], Result] + + +@dataclass(frozen=True) +class Assertion: + identity: str + probe: Probe + message: str + + +class CommandResult(NamedTuple): + returncode: int | None + stdout: bytes + stderr: bytes + timed_out: bool + + +def support(): + if SUPPORT is None: + raise IntegrityInfrastructure(f"install mutation support is unavailable: {SUPPORT_ERROR}") + return SUPPORT + + +def canonical(value: object) -> bytes: + return json.dumps( + value, + allow_nan=False, + ensure_ascii=True, + separators=(",", ":"), + sort_keys=True, + ).encode("ascii") + + +def digest(domain: bytes, value: object) -> str: + import hashlib + + return "sha256:" + hashlib.sha256(domain + canonical(value)).hexdigest() + + +def load_private_module(path: Path, name: str): + helper = support() + previous = sys.dont_write_bytecode + sys.dont_write_bytecode = True + try: + return helper.load_module(path, name) + finally: + sys.dont_write_bytecode = previous + + +def private_runtime(root: Path, names: tuple[str, ...]) -> Path: + runtime = root / "runtime" + support().copy_runtime(runtime, names) + return runtime + + +@contextmanager +def tool_environment() -> Iterator[None]: + keys = ("AGENT_LAB_CUE_TOOL_DIR", "AGENT_LAB_CEDAR_TOOL_DIR") + previous = {key: os.environ.get(key) for key in keys} + os.environ["AGENT_LAB_CUE_TOOL_DIR"] = str(CUE_TOOLS) + os.environ["AGENT_LAB_CEDAR_TOOL_DIR"] = str(CEDAR_TOOLS) + try: + yield + finally: + for key, value in previous.items(): + if value is None: + os.environ.pop(key, None) + else: + os.environ[key] = value + + +def actual_install(store, home: Path, source: Path, *, fault=None): + try: + value = store.install_directory(home, source, fault=fault) + return 0, value, None + except store.StoreReject as error: + return 1, None, error + except store.StoreInfrastructure as error: + return 125, None, error + except BaseException as error: + return None, None, error + + +def direct_source(root: Path, name: str, command: str) -> Path: + return support().write_source(root, name, support().artifact_bytes(command)) + + +def bundled_source(root: Path) -> Path: + data = ( + "package experiment\n\n" + "experiment: {\n" + '\tapiVersion: "agent-lab/v0alpha1"\n' + '\tkind: "Experiment"\n' + '\tmetadata: name: "bundled-integrity"\n' + "\tspec: members: [{\n" + '\t\tname: "worker"\n' + '\t\timage: catalogName: "agent-lab.worker"\n' + '\t\tcommand: ["serve"]\n' + "\t}]\n" + "}\n" + ).encode("ascii") + return support().write_source(root, "bundled-source", data) + + +def runtime_unchanged( + runtime: Path, + names: tuple[str, ...], + expected: tuple[object, ...], + purpose: str, +) -> None: + if support().runtime_fingerprint(runtime, names) != expected: + raise IntegrityInfrastructure(f"{purpose} changed its private runtime") + + +def probe_cleanup_parent_swap(root: Path, runtime: Path, names: tuple[str, ...]) -> Result: + store = load_private_module( + runtime / "scripts" / "experiment_store.py", + f"agent_lab_integrity_cleanup_{os.getpid()}_{id(root)}", + ) + runtime_before = support().runtime_fingerprint(runtime, names) + cleanup = root / "cleanup" + payload = cleanup / "payload" + artifact = payload / "artifact" + artifact.mkdir(mode=0o700, parents=True) + os.chmod(cleanup, 0o700) + os.chmod(payload, 0o700) + inside = artifact / "experiment.cue" + inside.write_bytes(b"inside\n") + os.chmod(inside, 0o400) + outside = root / "outside" + outside.mkdir(mode=0o700) + canary = outside / "experiment.cue" + canary.write_bytes(b"outside-canary\n") + os.chmod(canary, 0o400) + parked = root / "parked-artifact" + outside_before = support().tree_fingerprint(outside) + original_names = store._directory_names + swapped = False + + def swap_parent(path: Path, purpose: str, maximum: int): + nonlocal swapped + found = original_names(path, purpose, maximum) + if path == artifact and not swapped: + artifact.rename(parked) + os.symlink(outside, artifact) + swapped = True + return found + + store._directory_names = swap_parent + outcome: int | None = None + error: BaseException | None = None + try: + store._remove_tree(cleanup, cleanup) + outcome = 0 + except store.StoreInfrastructure as caught: + outcome = 125 + error = caught + except BaseException as caught: + error = caught + finally: + store._directory_names = original_names + outside_after = support().tree_fingerprint(outside) + runtime_unchanged(runtime, names, runtime_before, "cleanup parent-swap probe") + secure = swapped and outcome == 125 and outside_before == outside_after + return Result( + secure, + f"swapped={swapped} outcome={outcome} error={error!r} outside_changed={outside_before != outside_after}", + ) + + +def probe_contract_drift(root: Path, runtime: Path, names: tuple[str, ...]) -> Result: + home = support().initialized_home(runtime, root, "contract-home") + source = direct_source(root, "contract-source", "contract") + store = load_private_module( + runtime / "scripts" / "experiment_store.py", + f"agent_lab_integrity_contract_{os.getpid()}_{id(root)}", + ) + contract = runtime / "contracts" / "experiment" / "v0alpha1" / "schema.cue" + original = contract.read_bytes() + mutated = original.replace(b"strings.MaxRunes(63)", b"strings.MaxRunes(62)", 1) + if mutated == original or len(mutated) != len(original): + raise IntegrityInfrastructure("trusted contract mutation is not exactly applicable") + runtime_before = support().runtime_fingerprint(runtime, names) + store_before = support().tree_fingerprint(home / "experiments") + reached = False + + def mutate_contract(point: str) -> None: + nonlocal reached + if point == "experiment store lock.after_acquire" and not reached: + contract.write_bytes(mutated) + reached = True + + try: + with tool_environment(): + rc, value, error = actual_install(store, home, source, fault=mutate_contract) + finally: + contract.write_bytes(original) + store_after = support().tree_fingerprint(home / "experiments") + runtime_unchanged(runtime, names, runtime_before, "trusted contract drift probe") + final = home / "experiments" / "mutation-store" + secure = ( + reached + and rc == 125 + and value is None + and isinstance(error, store.StoreInfrastructure) + and not final.exists() + and store_before == store_after + ) + return Result( + secure, + f"reached={reached} rc={rc} error={error!r} final={final.exists()} store_changed={store_before != store_after}", + ) + + +def probe_bundled_provenance(root: Path, runtime: Path, names: tuple[str, ...]) -> Result: + catalog = { + "apiVersion": "agent-lab.experiment-images/v0alpha1", + "entries": [{"name": "agent-lab.worker", "subject": support().SUBJECT}], + } + catalog_path = runtime / "catalog" / "experiment-images" / "v0alpha1.json" + catalog_path.write_bytes(canonical(catalog) + b"\n") + expected = digest(BUNDLED_CATALOG_DOMAIN, catalog) + runtime_before = support().runtime_fingerprint(runtime, names) + home = support().initialized_home(runtime, root, "bundled-home") + source = bundled_source(root) + store = load_private_module( + runtime / "scripts" / "experiment_store.py", + f"agent_lab_integrity_bundle_{os.getpid()}_{id(root)}", + ) + with tool_environment(): + rc, value, error = actual_install(store, home, source) + provenance_path = ( + home + / "experiments" + / "bundled-integrity" + / "records" + / "provenance.json" + ) + try: + provenance = json.loads(provenance_path.read_bytes()) if provenance_path.is_file() else None + except (OSError, UnicodeError, json.JSONDecodeError): + provenance = None + runtime_unchanged(runtime, names, runtime_before, "bundled provenance probe") + expected_catalog = {"bundled": {"snapshotDigest": expected}} + secure = ( + rc == 0 + and isinstance(value, dict) + and error is None + and isinstance(provenance, dict) + and provenance.get("catalog") == expected_catalog + ) + return Result( + secure, + f"rc={rc} error={error!r} expected={expected_catalog!r} stored={None if not isinstance(provenance, dict) else provenance.get('catalog')!r}", + ) + + +def descriptor_open(descriptor: int) -> bool: + try: + os.fstat(descriptor) + except OSError as error: + if error.errno == errno.EBADF: + return False + raise + return True + + +def probe_unlock_failure(root: Path, runtime: Path, names: tuple[str, ...]) -> Result: + home = support().initialized_home(runtime, root, "unlock-home") + first_data = support().artifact_bytes("unlock-one") + second_data = support().artifact_bytes("unlock-two") + first_source = support().write_source(root, "unlock-source-one", first_data) + second_source = support().write_source(root, "unlock-source-two", second_data) + fixtures = { + first_data: support().PlanFixture( + support().requested_plan("mutation-store", "unlock-one"), + None, + ), + second_data: support().PlanFixture( + support().requested_plan("mutation-store", "unlock-two"), + None, + ), + } + experiment = support().FixtureExperiment(fixtures) + store = support().fixture_store(runtime, root, experiment) + runtime_before = support().runtime_fingerprint(runtime, names) + first_rc, _, first_error = support().install_result(store, home, first_source) + original_flock = store.fcntl.flock + unlock_descriptor: int | None = None + + def fail_unlock(descriptor: int, operation: int) -> None: + nonlocal unlock_descriptor + if operation == store.fcntl.LOCK_UN: + unlock_descriptor = descriptor + raise OSError("injected LOCK_UN failure") + original_flock(descriptor, operation) + + descriptors_before = set(os.listdir("/proc/self/fd")) + store.fcntl.flock = fail_unlock + try: + rc, value, error = support().install_result(store, home, second_source) + finally: + store.fcntl.flock = original_flock + leaked = unlock_descriptor is not None and descriptor_open(unlock_descriptor) + descriptors_after = set(os.listdir("/proc/self/fd")) + secure = ( + first_rc == 0 + and first_error is None + and rc == 125 + and value is None + and isinstance(error, store.StoreInfrastructure) + and unlock_descriptor is not None + and not leaked + and descriptors_before == descriptors_after + ) + if leaked and unlock_descriptor is not None: + os.close(unlock_descriptor) + descriptors_clean = set(os.listdir("/proc/self/fd")) + if descriptors_clean != descriptors_before: + raise IntegrityInfrastructure("unlock probe could not restore its descriptor set") + runtime_unchanged(runtime, names, runtime_before, "lock release probe") + return Result( + secure, + f"first={first_rc}/{first_error!r} conflict={rc}/{error!r}/{value!r} unlock_fd={unlock_descriptor} leaked={leaked} fd_delta={sorted(descriptors_after - descriptors_before)!r}", + ) + + +def probe_snapshot_race(root: Path, runtime: Path, names: tuple[str, ...]) -> Result: + experiment = load_private_module( + runtime / "scripts" / "experiment.py", + f"agent_lab_integrity_snapshot_{os.getpid()}_{id(root)}", + ) + runtime_before = support().runtime_fingerprint(runtime, names) + original = support().artifact_bytes("snapshot-one") + changed = support().artifact_bytes("snapshot-two") + if len(original) != len(changed) or original == changed: + raise IntegrityInfrastructure("same-size source mutation fixture is invalid") + source = support().write_source(root, "snapshot-race-source", original) + path = source / "experiment.cue" + before = path.stat() + original_read = experiment.os.read + reached = False + + def mutate_after_first_read(descriptor: int, maximum: int) -> bytes: + nonlocal reached + data = original_read(descriptor, maximum) + try: + target = os.readlink(f"/proc/self/fd/{descriptor}") + except OSError: + target = "" + if data and target == str(path) and not reached: + path.write_bytes(changed) + os.utime(path, ns=(before.st_atime_ns, before.st_mtime_ns)) + reached = True + return data + + experiment.os.read = mutate_after_first_read + outcome: int | None = None + error: BaseException | None = None + try: + experiment.read_directory_snapshot(str(source)) + outcome = 0 + except experiment.InfrastructureError as caught: + outcome = 125 + error = caught + except experiment.InvalidManifest as caught: + outcome = 1 + error = caught + except BaseException as caught: + error = caught + finally: + experiment.os.read = original_read + runtime_unchanged(runtime, names, runtime_before, "source snapshot race probe") + secure = reached and outcome == 125 and isinstance(error, experiment.InfrastructureError) + return Result(secure, f"reached={reached} outcome={outcome} error={error!r}") + + +def terminate(process: subprocess.Popen[bytes]) -> tuple[bytes, bytes]: + try: + os.killpg(process.pid, signal.SIGTERM) + except ProcessLookupError: + pass + try: + return process.communicate(timeout=0.25) + except subprocess.TimeoutExpired: + try: + os.killpg(process.pid, signal.SIGKILL) + except ProcessLookupError: + pass + return process.communicate(timeout=1) + + +def bounded_public( + runtime: Path, + home: Path, + source: Path, + *, + environment: dict[str, str] | None = None, + timeout: float = PUBLIC_TIMEOUT_SECONDS, +) -> CommandResult: + env = {"PATH": "/usr/bin:/bin", "LANG": "C", "LC_ALL": "C"} + if environment: + env.update(environment) + process = subprocess.Popen( + [ + sys.executable, + "-I", + "-B", + str(runtime / "scripts" / "agent-lab.py"), + "--home", + str(home), + "experiment", + "install", + str(source), + ], + stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + env=env, + start_new_session=True, + ) + try: + stdout, stderr = process.communicate(timeout=timeout) + if support().process_group_exists(process.pid): + terminate(process) + raise IntegrityInfrastructure("public preflight left a descendant process") + return CommandResult(process.returncode, stdout, stderr, False) + except subprocess.TimeoutExpired: + stdout, stderr = terminate(process) + if support().process_group_exists(process.pid): + raise IntegrityInfrastructure("timed-out public preflight left a process group") + return CommandResult(None, stdout, stderr, True) + + +def instrument_preflight(runtime: Path) -> None: + path = runtime / "scripts" / "agent-lab.py" + source = path.read_text(encoding="utf-8") + old = " try:\n raw = config_path.read_bytes()\n" + new = ( + ' integrity_ready = os.environ.get("AGENT_LAB_IIN_PREFLIGHT_READY")\n' + ' if integrity_ready is not None:\n' + ' Path(integrity_ready).touch()\n' + ' integrity_release = Path(os.environ["AGENT_LAB_IIN_PREFLIGHT_RELEASE"])\n' + ' integrity_deadline = __import__("time").monotonic() + 2.0\n' + ' while (\n' + ' not integrity_release.exists()\n' + ' and __import__("time").monotonic() < integrity_deadline\n' + ' ):\n' + ' __import__("time").sleep(0.005)\n' + ' if not integrity_release.exists():\n' + ' raise RuntimeError("integrity preflight release is unavailable")\n' + ' try:\n' + ' raw = config_path.read_bytes()\n' + ) + if source.count(old) != 1: + raise IntegrityInfrastructure("public preflight instrumentation is not exactly applicable") + path.write_text(source.replace(old, new, 1), encoding="utf-8") + cache = runtime.parent / "preflight-pycache" + completed = support().run_command( + [ + sys.executable, + "-I", + "-B", + "-X", + f"pycache_prefix={cache}", + "-m", + "py_compile", + str(path), + ] + ) + if completed.returncode != 0 or completed.stderr: + raise IntegrityInfrastructure("instrumented public preflight does not compile") + + +def restore_authority(path: Path, parked: Path) -> None: + try: + if path.is_symlink() or stat.S_ISFIFO(path.lstat().st_mode) or path.is_file(): + path.unlink() + parked.rename(path) + except OSError as error: + raise IntegrityInfrastructure("public preflight authority fixture could not be restored") from error + + +def wait_for_file(path: Path, timeout: float) -> bool: + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + if path.is_file(): + return True + time.sleep(0.005) + return False + + +def race_fifo_public(runtime: Path, home: Path, source: Path, root: Path) -> CommandResult: + ready = root / "race-ready" + release = root / "race-release" + path = home / "config.json" + parked = home / "config.parked" + env = { + "PATH": "/usr/bin:/bin", + "LANG": "C", + "LC_ALL": "C", + "AGENT_LAB_IIN_PREFLIGHT_READY": str(ready), + "AGENT_LAB_IIN_PREFLIGHT_RELEASE": str(release), + } + process = subprocess.Popen( + [ + sys.executable, + "-I", + "-B", + str(runtime / "scripts" / "agent-lab.py"), + "--home", + str(home), + "experiment", + "install", + str(source), + ], + stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + env=env, + start_new_session=True, + ) + if not wait_for_file(ready, 1.0): + terminate(process) + raise IntegrityInfrastructure("public preflight race did not reach its read boundary") + path.rename(parked) + os.mkfifo(path, mode=0o600) + release.touch() + try: + try: + stdout, stderr = process.communicate(timeout=RACE_TIMEOUT_SECONDS) + result = CommandResult(process.returncode, stdout, stderr, False) + except subprocess.TimeoutExpired: + stdout, stderr = terminate(process) + if support().process_group_exists(process.pid): + raise IntegrityInfrastructure("racing public preflight left a process group") + result = CommandResult(None, stdout, stderr, True) + finally: + restore_authority(path, parked) + return result + + +def probe_public_preflight(root: Path, runtime: Path, names: tuple[str, ...]) -> Result: + instrument_preflight(runtime) + runtime_before = support().runtime_fingerprint(runtime, names) + source = direct_source(root, "preflight-source", "preflight") + source_before = support().tree_fingerprint(source) + observations: dict[str, CommandResult] = {} + state_checks: dict[str, bool] = {} + + symlink_home = support().initialized_home(runtime, root, "symlink-home") + symlink_before = support().tree_fingerprint(symlink_home / "experiments") + symlink_path = symlink_home / "config.json" + symlink_parked = symlink_home / "config.parked" + symlink_path.rename(symlink_parked) + os.symlink(symlink_parked, symlink_path) + try: + observations["symlink"] = bounded_public(runtime, symlink_home, source) + finally: + restore_authority(symlink_path, symlink_parked) + state_checks["symlink"] = symlink_before == support().tree_fingerprint( + symlink_home / "experiments" + ) + + fifo_home = support().initialized_home(runtime, root, "fifo-home") + fifo_before = support().tree_fingerprint(fifo_home / "experiments") + fifo_path = fifo_home / "home.json" + fifo_parked = fifo_home / "home.parked" + fifo_path.rename(fifo_parked) + os.mkfifo(fifo_path, mode=0o600) + try: + observations["fifo"] = bounded_public(runtime, fifo_home, source) + finally: + restore_authority(fifo_path, fifo_parked) + state_checks["fifo"] = fifo_before == support().tree_fingerprint( + fifo_home / "experiments" + ) + + bound_home = support().initialized_home(runtime, root, "bound-home") + bound_before = support().tree_fingerprint(bound_home / "experiments") + bound_path = bound_home / "config.json" + bound_raw = bound_path.read_bytes() + bound_path.write_bytes(b"{" + b" " * 1_048_577 + b"}\n") + os.chmod(bound_path, 0o600) + try: + observations["over-bound"] = bounded_public(runtime, bound_home, source) + finally: + bound_path.write_bytes(bound_raw) + os.chmod(bound_path, 0o600) + state_checks["over-bound"] = bound_before == support().tree_fingerprint( + bound_home / "experiments" + ) + + race_home = support().initialized_home(runtime, root, "race-home") + race_before = support().tree_fingerprint(race_home / "experiments") + observations["race"] = race_fifo_public(runtime, race_home, source, root) + state_checks["race"] = race_before == support().tree_fingerprint( + race_home / "experiments" + ) + + source_after = support().tree_fingerprint(source) + runtime_unchanged(runtime, names, runtime_before, "public preflight probe") + clean_results = all( + observation.returncode == 125 + and not observation.timed_out + and not observation.stdout + for observation in observations.values() + ) + secure = clean_results and all(state_checks.values()) and source_before == source_after + rendered = { + key: (value.returncode, value.timed_out, value.stderr.decode("utf-8", errors="replace").strip()) + for key, value in observations.items() + } + return Result( + secure, + f"results={rendered!r} state={state_checks!r} source_changed={source_before != source_after}", + ) + + +def probe_plan_identity(root: Path, runtime: Path, names: tuple[str, ...]) -> Result: + home = support().initialized_home(runtime, root, "plan-home") + source = direct_source(root, "plan-source", "plan") + store = load_private_module( + runtime / "scripts" / "experiment_store.py", + f"agent_lab_integrity_plan_{os.getpid()}_{id(root)}", + ) + runtime_before = support().runtime_fingerprint(runtime, names) + with tool_environment(): + rc, value, error = actual_install(store, home, source) + records = home / "experiments" / "mutation-store" / "records" + try: + plan = json.loads((records / "plan.json").read_bytes()) + decision = json.loads((records / "decision.json").read_bytes()) + receipt = json.loads((records / "install.json").read_bytes()) + except (OSError, UnicodeError, json.JSONDecodeError): + plan = decision = receipt = None + runtime_unchanged(runtime, names, runtime_before, "plan identity probe") + expected = digest(PLAN_DOMAIN, plan) if isinstance(plan, dict) else None + decision_digest = None + receipt_digest = None + identity_digest = None + if isinstance(decision, dict): + binding = decision.get("binding") + if isinstance(binding, dict): + decision_digest = binding.get("planDigest") + if isinstance(receipt, dict): + record_map = receipt.get("records") + if isinstance(record_map, dict): + plan_record = record_map.get("records/plan.json") + if isinstance(plan_record, dict): + receipt_digest = plan_record.get("digest") + identity = receipt.get("identity") + if isinstance(identity, dict): + identity_digest = identity.get("planDigest") + secure = ( + rc == 0 + and isinstance(value, dict) + and error is None + and expected is not None + and decision_digest == expected + and receipt_digest == expected + and identity_digest == expected + ) + return Result( + secure, + f"rc={rc} error={error!r} expected={expected!r} decision={decision_digest!r} receipt={receipt_digest!r} identity={identity_digest!r}", + ) + + +ASSERTIONS: tuple[Assertion, ...] = ( + Assertion( + "IIN-CLEAN-001", + probe_cleanup_parent_swap, + "cleanup preserves an external tree across a parent-swap symlink race", + ), + Assertion( + "IIN-CONTRACT-001", + probe_contract_drift, + "trusted contract drift at store-lock acquisition fails before store effect", + ), + Assertion( + "IIN-BUNDLE-001", + probe_bundled_provenance, + "bundled catalog provenance binds the independently framed snapshot identity", + ), + Assertion( + "IIN-LOCK-001", + probe_unlock_failure, + "unlock failure dominates conflict and closes the store-lock descriptor", + ), + Assertion( + "IIN-SNAPSHOT-001", + probe_snapshot_race, + "same-size source drift with restored mtime is snapshot uncertainty", + ), + Assertion( + "IIN-PREFLIGHT-001", + probe_public_preflight, + "public home preflight rejects unsafe and racing authority files within bounds", + ), + Assertion( + "IIN-PLAN-001", + probe_plan_identity, + "decision and receipt bind the independently domain-separated plan identity", + ), +) + + +def main() -> int: + try: + helper = support() + if not CUE_TOOLS.is_dir() or not CEDAR_TOOLS.is_dir(): + raise IntegrityInfrastructure("pinned CUE or Cedar fixtures are unavailable") + names = helper.manifest_paths() + shared_before = helper.runtime_fingerprint(REPO_ROOT, names) + expected = ( + "IIN-CLEAN-001", + "IIN-CONTRACT-001", + "IIN-BUNDLE-001", + "IIN-LOCK-001", + "IIN-SNAPSHOT-001", + "IIN-PREFLIGHT-001", + "IIN-PLAN-001", + ) + if tuple(assertion.identity for assertion in ASSERTIONS) != expected: + raise IntegrityInfrastructure("install integrity assertion identity drift") + failures = 0 + observed: list[str] = [] + previous_bytecode = sys.dont_write_bytecode + sys.dont_write_bytecode = True + try: + for assertion in ASSERTIONS: + try: + temporary = Path( + tempfile.mkdtemp( + prefix=f"agent-lab-{assertion.identity.lower()}-", + dir="/tmp", + ) + ) + except OSError as error: + raise IntegrityInfrastructure( + f"{assertion.identity} private root is unavailable" + ) from error + try: + runtime = private_runtime(temporary, names) + result = assertion.probe(temporary, runtime, names) + observed.append(assertion.identity) + if result.secure: + print(f"PASS {assertion.identity} {assertion.message}") + else: + failures += 1 + print( + f"FAIL {assertion.identity} {assertion.message} ({result.detail})" + ) + finally: + helper.remove_private_root(temporary) + if helper.runtime_fingerprint(REPO_ROOT, names) != shared_before: + raise IntegrityInfrastructure( + f"{assertion.identity} changed the shared runtime fingerprint" + ) + finally: + sys.dont_write_bytecode = previous_bytecode + if tuple(observed) != expected: + raise IntegrityInfrastructure("install integrity execution identity drift") + print(f"SUMMARY assertions=7 expected=7 failures={failures} infra=0") + return 0 if failures == 0 else 1 + except (IntegrityInfrastructure, SupportInfrastructure) as error: + print(f"INFRA install integrity evidence: {error}", file=sys.stderr) + return 125 + + +if __name__ == "__main__": + raise SystemExit(main()) From 1c6a8321fa353bd59a9b1aba1a1e7cf626a75bfb Mon Sep 17 00:00:00 2001 From: xormania <127287135+xormania@users.noreply.github.com> Date: Sun, 2 Aug 2026 03:16:10 -0400 Subject: [PATCH 057/158] test(experiment): route integrity evidence --- tests/experiment/aggregate-harness-cases.sh | 24 ++++++++++++++------- tests/experiment/local-lifecycle-cases.sh | 8 +++++-- 2 files changed, 22 insertions(+), 10 deletions(-) diff --git a/tests/experiment/aggregate-harness-cases.sh b/tests/experiment/aggregate-harness-cases.sh index f79edbf..f146e95 100755 --- a/tests/experiment/aggregate-harness-cases.sh +++ b/tests/experiment/aggregate-harness-cases.sh @@ -56,6 +56,9 @@ expected_ids=( INST-FORGE-001 INST-NOEF-001 INST-LOCAL-001 INST-RUNTIME-001 IST-STATE-001 IST-LOCK-001 IST-STATE-002 IST-BOUND-001 IST-STATE-003 IST-CONC-001 IST-CONC-002 IST-CRASH-001 IST-LIVE-001 IST-PLAT-001 IST-PROV-001 + IST-PREFLIGHT-001 IST-SNAPSHOT-001 IST-CONFIG-001 IST-READ-001 IST-FAULT-001 + IIN-CLEAN-001 IIN-CONTRACT-001 IIN-BUNDLE-001 IIN-LOCK-001 + IIN-SNAPSHOT-001 IIN-PREFLIGHT-001 IIN-PLAN-001 M-STORE-AUTH-001 M-STORE-SOURCE-001 M-STORE-ATOM-001 M-STORE-RETRY-001 M-STORE-DUR-001 M-STORE-LAYOUT-001 M-STORE-KEY-001 M-STORE-VERIFY-001 M-STORE-LIVE-001 M-STORE-UNCERT-001 M-STORE-STAGE-001 @@ -64,8 +67,9 @@ installer_ids=("${expected_ids[@]:0:5}") config_ids=("${expected_ids[@]:5:5}") catalog_ids=("${expected_ids[@]:10:76}") install_ids=("${expected_ids[@]:86:13}") -state_ids=("${expected_ids[@]:99:11}") -mutation_ids=("${expected_ids[@]:110:11}") +state_ids=("${expected_ids[@]:99:16}") +integrity_ids=("${expected_ids[@]:115:7}") +mutation_ids=("${expected_ids[@]:122:11}") write_fixture() { local path="$1" @@ -126,18 +130,20 @@ pass_records() { reset_fixtures() { local installer_records=() config_records=() catalog_records=() - local install_records=() state_records=() mutation_records=() + local install_records=() state_records=() integrity_records=() mutation_records=() mapfile -t installer_records < <(pass_records "${installer_ids[@]}") mapfile -t config_records < <(pass_records "${config_ids[@]}") mapfile -t catalog_records < <(pass_records "${catalog_ids[@]}") mapfile -t install_records < <(pass_records "${install_ids[@]}") mapfile -t state_records < <(pass_records "${state_ids[@]}") + mapfile -t integrity_records < <(pass_records "${integrity_ids[@]}") mapfile -t mutation_records < <(pass_records "${mutation_ids[@]}") write_fixture "$replica/tests/install/local-install-cases.sh" 0 "${installer_records[@]}" write_fixture "$replica/tests/experiment/local-config-cases.sh" 0 "${config_records[@]}" write_fixture "$replica/tests/experiment/local-image-catalog-cases.sh" 0 "${catalog_records[@]}" write_fixture "$replica/tests/experiment/install-store-cases.sh" 0 "${install_records[@]}" write_python_fixture "$replica/tests/experiment/install-state-cases.py" 0 "${state_records[@]}" + write_python_fixture "$replica/tests/experiment/install-integrity-cases.py" 0 "${integrity_records[@]}" write_python_fixture "$replica/tests/experiment/install-mutation-cases.py" 0 "${mutation_records[@]}" } @@ -165,6 +171,7 @@ printf '%s\n' \ local-image-catalog-cases.sh \ install-store-cases.sh \ install-state-cases.py \ + install-integrity-cases.py \ install-mutation-cases.py > "$expected_executions" : > "$baseline_executions" baseline_rc=0 @@ -194,6 +201,7 @@ printf '%s\n' \ local-image-catalog-cases.sh \ install-store-cases.sh \ install-state-cases.py \ + install-integrity-cases.py \ install-mutation-cases.py > "$mutant_expected" if [ "$baseline_rc" -eq 0 ] && @@ -213,10 +221,10 @@ success_output="$work/success.out" success_rc=0 run_replica "$success_output" env || success_rc=$? if [ "$success_rc" -eq 0 ] && - [ "$(grep -Ec '^(PASS|FAIL) [A-Z0-9-]+ ' "$success_output")" -eq 121 ] && - [ "$(grep -Fxc 'SUMMARY assertions=121 expected=121 failures=0 infra=0' "$success_output")" -eq 1 ] && + [ "$(grep -Ec '^(PASS|FAIL) [A-Z0-9-]+ ' "$success_output")" -eq 133 ] && + [ "$(grep -Fxc 'SUMMARY assertions=133 expected=133 failures=0 infra=0' "$success_output")" -eq 1 ] && [ "$(tail -n 1 "$success_output")" = 'EXPERIMENT LOCAL LIFECYCLE PASS' ] && - awk '/^(PASS|FAIL) [A-Z0-9-]+ / {next} /^SUMMARY assertions=121 expected=121 failures=0 infra=0$/ {next} /^EXPERIMENT LOCAL LIFECYCLE PASS$/ {next} {bad=1} END {exit bad}' "$success_output"; then + awk '/^(PASS|FAIL) [A-Z0-9-]+ / {next} /^SUMMARY assertions=133 expected=133 failures=0 infra=0$/ {next} /^EXPERIMENT LOCAL LIFECYCLE PASS$/ {next} {bad=1} END {exit bad}' "$success_output"; then pass AGG-002 "success forwards only assertions then one summary and marker" else fail AGG-002 "success forwards only assertions then one summary and marker" @@ -265,7 +273,7 @@ write_fixture "$replica/tests/install/local-install-cases.sh" 1 "${failed_record assertion_rc=0 run_replica "$work/assertion.out" env || assertion_rc=$? if [ "$assertion_rc" -eq 1 ] && - grep -Fxq 'SUMMARY assertions=121 expected=121 failures=1 infra=0' "$work/assertion.out" && + grep -Fxq 'SUMMARY assertions=133 expected=133 failures=1 infra=0' "$work/assertion.out" && ! grep -Fxq 'EXPERIMENT LOCAL LIFECYCLE PASS' "$work/assertion.out"; then pass AGG-006 "subcase assertion failure maps to one" else @@ -298,7 +306,7 @@ chmod +x "$shim/rmdir" cleanup_rc=0 run_replica "$work/cleanup.out" env PATH="$shim:$PATH" || cleanup_rc=$? if [ "$cleanup_rc" -eq 125 ] && - grep -Fxq 'SUMMARY assertions=121 expected=121 failures=0 infra=1' "$work/cleanup.out" && + grep -Fxq 'SUMMARY assertions=133 expected=133 failures=0 infra=1' "$work/cleanup.out" && ! grep -Fxq 'EXPERIMENT LOCAL LIFECYCLE PASS' "$work/cleanup.out"; then pass AGG-008 "cleanup uncertainty maps to one hundred twenty-five before the marker" else diff --git a/tests/experiment/local-lifecycle-cases.sh b/tests/experiment/local-lifecycle-cases.sh index d8dd372..6303915 100755 --- a/tests/experiment/local-lifecycle-cases.sh +++ b/tests/experiment/local-lifecycle-cases.sh @@ -8,9 +8,10 @@ subcases=( "$repo_root/tests/experiment/local-image-catalog-cases.sh" "$repo_root/tests/experiment/install-store-cases.sh" "$repo_root/tests/experiment/install-state-cases.py" + "$repo_root/tests/experiment/install-integrity-cases.py" "$repo_root/tests/experiment/install-mutation-cases.py" ) -expected_count=121 +expected_count=133 work="" cleanup_work() { @@ -57,7 +58,10 @@ printf '%s\n' \ INST-FORGE-001 INST-NOEF-001 INST-LOCAL-001 INST-RUNTIME-001 \ IST-STATE-001 IST-LOCK-001 IST-STATE-002 IST-BOUND-001 IST-STATE-003 \ IST-CONC-001 IST-CONC-002 IST-CRASH-001 IST-LIVE-001 IST-PLAT-001 \ - IST-PROV-001 \ + IST-PROV-001 IST-PREFLIGHT-001 IST-SNAPSHOT-001 IST-CONFIG-001 IST-READ-001 \ + IST-FAULT-001 \ + IIN-CLEAN-001 IIN-CONTRACT-001 IIN-BUNDLE-001 IIN-LOCK-001 \ + IIN-SNAPSHOT-001 IIN-PREFLIGHT-001 IIN-PLAN-001 \ M-STORE-AUTH-001 M-STORE-SOURCE-001 M-STORE-ATOM-001 M-STORE-RETRY-001 \ M-STORE-DUR-001 M-STORE-LAYOUT-001 M-STORE-KEY-001 M-STORE-VERIFY-001 \ M-STORE-LIVE-001 M-STORE-UNCERT-001 M-STORE-STAGE-001 > "$expected" From 08fe3c84babc89246110937d36826884fbbaf4b0 Mon Sep 17 00:00:00 2001 From: xormania <127287135+xormania@users.noreply.github.com> Date: Sun, 2 Aug 2026 03:21:14 -0400 Subject: [PATCH 058/158] fix(experiment): harden trusted input reads --- scripts/agent-lab.py | 120 +++++++++++++++++++++++++++++++++++++++--- scripts/experiment.py | 114 ++++++++++++++++++++++++++++++++++----- 2 files changed, 212 insertions(+), 22 deletions(-) diff --git a/scripts/agent-lab.py b/scripts/agent-lab.py index 23f1f97..e97d014 100644 --- a/scripts/agent-lab.py +++ b/scripts/agent-lab.py @@ -13,6 +13,7 @@ import sys SAFE_COMPONENT = re.compile(r"^[a-z][a-z0-9-]{0,47}$") +MAX_HOME_AUTHORITY_BYTES = 65_536 LOCK_SPECS = { "imageCatalog": ( "image-catalog.lock", @@ -54,6 +55,101 @@ def write_all(descriptor: int, data: bytes) -> None: view = view[written:] +def home_authority_identity(metadata: os.stat_result) -> tuple[int, int, int, int, int]: + return ( + metadata.st_dev, + metadata.st_ino, + metadata.st_size, + metadata.st_mtime_ns, + metadata.st_ctime_ns, + ) + + +def home_authority_metadata_safe(metadata: os.stat_result) -> bool: + return ( + stat.S_ISREG(metadata.st_mode) + and metadata.st_uid == os.getuid() + and metadata.st_nlink == 1 + and stat.S_IMODE(metadata.st_mode) == 0o600 + and metadata.st_size <= MAX_HOME_AUTHORITY_BYTES + ) + + +def read_home_authority( + path: Path, + lexical: os.stat_result, + purpose: str, +) -> bytes: + """Read one bounded home authority file through a stable no-follow descriptor.""" + + if not home_authority_metadata_safe(lexical): + raise RuntimeError("home authority files are unsafe") + flags = ( + os.O_RDONLY + | getattr(os, "O_CLOEXEC", 0) + | getattr(os, "O_NOFOLLOW", 0) + | getattr(os, "O_NONBLOCK", 0) + ) + try: + descriptor = os.open(path, flags) + except OSError as error: + raise RuntimeError(f"{purpose} authority cannot be opened safely") from error + try: + opened = os.fstat(descriptor) + if ( + not home_authority_metadata_safe(opened) + or home_authority_identity(opened) != home_authority_identity(lexical) + ): + raise RuntimeError(f"{purpose} authority changed before it was read") + chunks: list[bytes] = [] + remaining = MAX_HOME_AUTHORITY_BYTES + 1 + while remaining: + chunk = os.read(descriptor, min(65_536, remaining)) + if not chunk: + break + chunks.append(chunk) + remaining -= len(chunk) + data = b"".join(chunks) + final = os.fstat(descriptor) + except RuntimeError: + raise + except OSError as error: + raise RuntimeError(f"{purpose} authority cannot be read safely") from error + finally: + try: + os.close(descriptor) + except OSError as error: + raise RuntimeError(f"{purpose} authority descriptor cannot be closed") from error + try: + current = path.lstat() + except OSError as error: + raise RuntimeError(f"{purpose} authority cannot be reverified") from error + expected_identity = home_authority_identity(lexical) + if ( + len(data) > MAX_HOME_AUTHORITY_BYTES + or len(data) != final.st_size + or not home_authority_metadata_safe(final) + or not home_authority_metadata_safe(current) + or home_authority_identity(opened) != expected_identity + or home_authority_identity(final) != expected_identity + or home_authority_identity(current) != expected_identity + ): + raise RuntimeError(f"{purpose} authority changed while it was read") + return data + + +def reverify_home_authority(path: Path, expected: os.stat_result, purpose: str) -> None: + try: + current = path.lstat() + except OSError as error: + raise RuntimeError(f"{purpose} authority cannot be reverified") from error + if ( + not home_authority_metadata_safe(current) + or home_authority_identity(current) != home_authority_identity(expected) + ): + raise RuntimeError(f"{purpose} authority changed during preflight") + + def lock_record(path: Path, relative: str, schema: str) -> dict[str, object]: metadata = path.lstat() if ( @@ -249,20 +345,28 @@ def load_config_receipt( receipt_path = home / "home.json" try: config_metadata = config_path.lstat() + except FileNotFoundError: + config_metadata = None + except OSError as error: + raise RuntimeError("home configuration cannot be inspected") from error + try: receipt_metadata = receipt_path.lstat() except FileNotFoundError: - if not config_path.exists() and not receipt_path.exists(): - return None + receipt_metadata = None + except OSError as error: + raise RuntimeError("home receipt cannot be inspected") from error + if config_metadata is None and receipt_metadata is None: + return None + if config_metadata is None or receipt_metadata is None: raise RuntimeError("home receipt and configuration are incomplete") - for metadata in (config_metadata, receipt_metadata): - if not stat.S_ISREG(metadata.st_mode) or metadata.st_nlink != 1 or stat.S_IMODE(metadata.st_mode) != 0o600: - raise RuntimeError("home authority files are unsafe") + raw = read_home_authority(config_path, config_metadata, "configuration") + receipt_raw = read_home_authority(receipt_path, receipt_metadata, "home receipt") + reverify_home_authority(config_path, config_metadata, "configuration") + reverify_home_authority(receipt_path, receipt_metadata, "home receipt") try: - raw = config_path.read_bytes() - receipt_raw = receipt_path.read_bytes() value = json.loads(raw.decode("utf-8")) receipt = json.loads(receipt_raw.decode("utf-8")) - except (OSError, RecursionError, UnicodeError, ValueError): + except (RecursionError, UnicodeError, ValueError): raise RuntimeError("configuration is malformed") if not isinstance(value, dict) or set(value) != {"apiVersion", "paths"} or value["apiVersion"] != "agent-lab.config/v0alpha1": raise RuntimeError("configuration is not closed") diff --git a/scripts/experiment.py b/scripts/experiment.py index 9e03775..6da94be 100644 --- a/scripts/experiment.py +++ b/scripts/experiment.py @@ -21,6 +21,7 @@ MAX_MANIFEST_BYTES = 262_144 SOURCE_DIGEST_DOMAIN = b"agent-lab.experiment-tree.v1\0" +PLAN_DOMAIN = b"agent-lab.experiment-plan.v1\0" BUNDLED_CATALOG_DOMAIN = b"agent-lab.experiment-image-catalog.v1\0" BUNDLED_ENTRY_DOMAIN = b"agent-lab.experiment-image-entry.v1\0" MAX_CUE_OUTPUT_BYTES = 1_048_576 @@ -108,6 +109,7 @@ class SourceSnapshot(NamedTuple): class PlanResolution(NamedTuple): plan: dict[str, object] + bundled_catalog: dict[str, object] | None local_catalog: dict[str, object] | None @@ -202,12 +204,14 @@ def read_manifest_once(path: str) -> bytes: opened_stat.st_ino, opened_stat.st_size, opened_stat.st_mtime_ns, + opened_stat.st_ctime_ns, ) after = ( final_stat.st_dev, final_stat.st_ino, final_stat.st_size, final_stat.st_mtime_ns, + final_stat.st_ctime_ns, ) if before != after or len(data) != final_stat.st_size: raise InfrastructureError("manifest changed while it was read") @@ -345,6 +349,12 @@ def canonical_json(value: object) -> bytes: raise InfrastructureError("CUE produced a non-canonicalizable plan") from error +def plan_digest(plan: object) -> str: + """Return the canonical, domain-separated identity of one checked plan.""" + + return "sha256:" + hashlib.sha256(PLAN_DOMAIN + canonical_json(plan)).hexdigest() + + def cue_environment(repo_root: Path) -> dict[str, str]: environment = { "PATH": "/usr/bin:/bin", @@ -646,11 +656,19 @@ def resolve_plan_with_evidence( catalog_support = None bundled_by_name: dict[str, dict[str, object]] = {} + bundled_evidence: dict[str, object] | None = None if bundled_names: catalog_support = image_catalog_module() selected_catalog = catalog if selected_catalog is None: selected_catalog, _ = bundled_catalog(repo_root) + if ( + not isinstance(selected_catalog, dict) + or set(selected_catalog) != {"apiVersion", "entries"} + or selected_catalog.get("apiVersion") + != "agent-lab.experiment-images/v0alpha1" + ): + raise InfrastructureError("bundled image catalog has an unexpected shape") entries = selected_catalog.get("entries") if not isinstance(entries, list): raise InfrastructureError("bundled image catalog entries are malformed") @@ -668,6 +686,9 @@ def resolve_plan_with_evidence( if not name.startswith("agent-lab.") or name in bundled_by_name: raise InfrastructureError("bundled image catalog namespace is invalid") bundled_by_name[name] = entry + bundled_evidence = { + "snapshotDigest": digest_record(BUNDLED_CATALOG_DOMAIN, selected_catalog) + } local_records: dict[str, dict[str, object]] = {} local_evidence: dict[str, object] | None = None @@ -748,7 +769,7 @@ def resolve_plan_with_evidence( "origin": "local", "subject": record["subject"], } - return PlanResolution(resolved, local_evidence) + return PlanResolution(resolved, bundled_evidence, local_evidence) def resolve_plan( @@ -943,10 +964,9 @@ def plan_binding(plan: object, source_digest: str) -> PlanBinding: except (AssertionError, KeyError, TypeError, ValueError) as error: raise InfrastructureError("CUE plan cannot be bound to authorization") from error - plan_bytes = canonical_json(plan) - plan_digest = f"sha256:{hashlib.sha256(plan_bytes).hexdigest()}" + bound_plan_digest = plan_digest(plan) return PlanBinding( - plan_digest=plan_digest, + plan_digest=bound_plan_digest, contract_digest=contract_digest, contract_version=contract_version, requested_name=requested_name, @@ -1292,12 +1312,71 @@ def authorize_plan(plan: object, source_digest: str) -> tuple[dict[str, object], return decision, 0 if verdict == "permit" else 1 -def write_envelope(plan: object, local_catalog: dict[str, object] | None = None) -> None: - plan_bytes = canonical_json(plan) - digest = hashlib.sha256(plan_bytes).hexdigest() - value: dict[str, object] = {"digest": f"sha256:{digest}", "plan": plan} +def verify_trusted_inputs(plan: object, decision: object) -> None: + """Re-snapshot trusted inputs and bind them to one authorized plan.""" + + try: + if ( + not isinstance(decision, dict) + or not isinstance(decision.get("binding"), dict) + or not isinstance(decision.get("resource"), dict) + ): + raise ValueError("decision envelope") + binding = decision["binding"] + resource = decision["resource"] + if set(binding) != { + "authorizationDigest", + "contractDigest", + "planDigest", + "sourceDigest", + }: + raise ValueError("decision binding") + source_digest = binding["sourceDigest"] + if not is_sha256(source_digest): + raise ValueError("source identity") + expected = plan_binding(plan, source_digest) + if ( + binding["planDigest"] != expected.plan_digest + or binding["contractDigest"] != expected.contract_digest + or resource.get("id") != expected.plan_digest + ): + raise ValueError("plan decision identity") + except (KeyError, TypeError, ValueError) as error: + raise InfrastructureError("authorized plan binding is inconsistent") from error + + repo_root = Path(__file__).resolve().parent.parent + contract_digest, contract_files = contract_snapshot(repo_root) + authorization_digest, authorization_files = authorization_snapshot(repo_root) + verify_contract_snapshot(repo_root, contract_files) + verify_authorization_snapshot(repo_root, authorization_files) + if ( + expected.contract_digest != f"sha256:{contract_digest}" + or binding["authorizationDigest"] != authorization_digest + ): + raise InfrastructureError("trusted inputs no longer match the authorization") + + +def catalog_resolution_evidence( + bundled_catalog: dict[str, object] | None, + local_catalog: dict[str, object] | None, +) -> dict[str, object] | None: + catalog: dict[str, object] = {} + if bundled_catalog is not None: + catalog["bundled"] = bundled_catalog if local_catalog is not None: - value["catalog"] = {"local": local_catalog} + catalog["local"] = local_catalog + return catalog or None + + +def write_envelope( + plan: object, + bundled_catalog: dict[str, object] | None = None, + local_catalog: dict[str, object] | None = None, +) -> None: + value: dict[str, object] = {"digest": plan_digest(plan), "plan": plan} + catalog = catalog_resolution_evidence(bundled_catalog, local_catalog) + if catalog is not None: + value["catalog"] = catalog envelope = canonical_json(value) + b"\n" try: written = sys.stdout.buffer.write(envelope) @@ -1329,14 +1408,17 @@ def main(argv: list[str]) -> int: resolution = cue_plan_with_evidence(manifest) plan = resolution.plan if directory_checking: - plan_bytes = canonical_json(plan) checked: dict[str, object] = { - "digest": f"sha256:{hashlib.sha256(plan_bytes).hexdigest()}", + "digest": plan_digest(plan), "plan": plan, "source": {"digest": snapshot.digest, "kind": "directory"}, } - if resolution.local_catalog is not None: - checked["catalog"] = {"local": resolution.local_catalog} + catalog = catalog_resolution_evidence( + resolution.bundled_catalog, + resolution.local_catalog, + ) + if catalog is not None: + checked["catalog"] = catalog sys.stdout.buffer.write(canonical_json(checked) + b"\n") return 0 decision, result = authorize_plan(plan, snapshot.digest) @@ -1365,7 +1447,11 @@ def main(argv: list[str]) -> int: resolution = cue_plan_with_evidence(manifest) plan = resolution.plan if checking: - write_envelope(plan, resolution.local_catalog) + write_envelope( + plan, + resolution.bundled_catalog, + resolution.local_catalog, + ) return 0 decision, result = authorize_plan(plan, "sha256:" + "0" * 64) write_decision(decision) From 319273b5a4608171a5ee100a3a4b26d704cc3f55 Mon Sep 17 00:00:00 2001 From: xormania <127287135+xormania@users.noreply.github.com> Date: Sun, 2 Aug 2026 03:22:44 -0400 Subject: [PATCH 059/158] test(experiment): retain safe preflight race --- tests/experiment/install-integrity-cases.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/tests/experiment/install-integrity-cases.py b/tests/experiment/install-integrity-cases.py index c725c69..151960a 100644 --- a/tests/experiment/install-integrity-cases.py +++ b/tests/experiment/install-integrity-cases.py @@ -505,8 +505,14 @@ def bounded_public( def instrument_preflight(runtime: Path) -> None: path = runtime / "scripts" / "agent-lab.py" source = path.read_text(encoding="utf-8") - old = " try:\n raw = config_path.read_bytes()\n" + old = ( + " if not home_authority_metadata_safe(lexical):\n" + ' raise RuntimeError("home authority files are unsafe")\n' + " flags = (\n" + ) new = ( + " if not home_authority_metadata_safe(lexical):\n" + ' raise RuntimeError("home authority files are unsafe")\n' ' integrity_ready = os.environ.get("AGENT_LAB_IIN_PREFLIGHT_READY")\n' ' if integrity_ready is not None:\n' ' Path(integrity_ready).touch()\n' @@ -519,8 +525,7 @@ def instrument_preflight(runtime: Path) -> None: ' __import__("time").sleep(0.005)\n' ' if not integrity_release.exists():\n' ' raise RuntimeError("integrity preflight release is unavailable")\n' - ' try:\n' - ' raw = config_path.read_bytes()\n' + " flags = (\n" ) if source.count(old) != 1: raise IntegrityInfrastructure("public preflight instrumentation is not exactly applicable") From b0aedf3ff42ab59f01605283ad3939583d0120a2 Mon Sep 17 00:00:00 2001 From: xormania <127287135+xormania@users.noreply.github.com> Date: Sun, 2 Aug 2026 03:28:01 -0400 Subject: [PATCH 060/158] test(experiment): bind fixture plan identities --- tests/experiment/install-mutation-cases.py | 14 ++++++++++++-- tests/experiment/install-state-cases.py | 18 +++++++++++++++--- 2 files changed, 27 insertions(+), 5 deletions(-) diff --git a/tests/experiment/install-mutation-cases.py b/tests/experiment/install-mutation-cases.py index d6b7f47..5f25cbe 100755 --- a/tests/experiment/install-mutation-cases.py +++ b/tests/experiment/install-mutation-cases.py @@ -24,6 +24,7 @@ RUNTIME_MANIFEST = REPO_ROOT / "packaging" / "agent-lab-local.manifest" COMMAND_TIMEOUT_SECONDS = 5 SOURCE_DOMAIN = b"agent-lab.experiment-tree.v1\0" +PLAN_DOMAIN = b"agent-lab.experiment-plan.v1\0" SUBJECT = "registry.example/team/worker@sha256:" + "a" * 64 AUTHORIZATION_DIGEST = "sha256:" + "c" * 64 CONTRACT_DIGEST = "sha256:" + "d" * 64 @@ -64,6 +65,7 @@ class Snapshot(NamedTuple): class Resolution(NamedTuple): plan: dict[str, object] + bundled_catalog: dict[str, object] | None local_catalog: dict[str, object] | None @@ -165,7 +167,7 @@ def decision_for( snapshot_digest: str, verdict: str, ) -> dict[str, object]: - plan_digest = digest(canonical(plan)) + plan_digest = digest(PLAN_DOMAIN + canonical(plan)) requested_name = plan["metadata"]["requestedName"] # type: ignore[index] return { "action": "experiment.install", @@ -225,7 +227,7 @@ def cue_plan_with_evidence(self, manifest: object) -> Resolution: if not isinstance(manifest, bytes) or manifest not in self.fixtures: raise FixtureInvalidManifest("fixture manifest is unknown") fixture = self.fixtures[manifest] - return Resolution(fixture.plan, fixture.local_catalog) + return Resolution(fixture.plan, None, fixture.local_catalog) def authorize_plan( self, @@ -238,6 +240,14 @@ def authorize_plan( self.after_authorize() return decision, 0 if self.verdict == "permit" else 1 + def verify_trusted_inputs( + self, + plan: dict[str, object], + decision: dict[str, object], + ) -> None: + if not isinstance(plan, dict) or not isinstance(decision, dict): + raise FixtureInfrastructure("fixture trusted inputs are malformed") + def sha256_bytes(data: bytes) -> str: return hashlib.sha256(data).hexdigest() diff --git a/tests/experiment/install-state-cases.py b/tests/experiment/install-state-cases.py index 8d7834f..93c691f 100644 --- a/tests/experiment/install-state-cases.py +++ b/tests/experiment/install-state-cases.py @@ -29,6 +29,7 @@ CEDAR_TOOLS = REPO_ROOT / ".cache" / "dev" / "tools" / "cedar" SUBJECT = "registry.example/team/worker@sha256:" + "a" * 64 OTHER_SUBJECT = "registry.example/team/worker@sha256:" + "b" * 64 +PLAN_DOMAIN = b"agent-lab.experiment-plan.v1\0" FAULT_POINTS = ( "experiment artifact.after_write", "experiment receipt.after_fsync", @@ -45,6 +46,7 @@ class FixtureSnapshot(NamedTuple): class FixtureResolution(NamedTuple): plan: dict[str, object] + bundled_catalog: dict[str, object] | None local_catalog: dict[str, object] | None @@ -86,10 +88,12 @@ def __init__(self, source_data: bytes, name: str) -> None: ] }, } - plan_digest = "sha256:" + hashlib.sha256(canonical_json(plan)).hexdigest() + plan_digest = "sha256:" + hashlib.sha256( + PLAN_DOMAIN + canonical_json(plan) + ).hexdigest() self.source_data = source_data self.snapshot = FixtureSnapshot(source_data, source_digest) - self.resolution = FixtureResolution(plan, None) + self.resolution = FixtureResolution(plan, None, None) self.decision: dict[str, object] = { "action": "experiment.install", "apiVersion": "agent-lab.authorization/v0alpha1", @@ -141,6 +145,14 @@ def authorize_plan( raise FixtureInfrastructure("fault fixture authorization binding changed") return self.decision, 0 + def verify_trusted_inputs( + self, + plan: dict[str, object], + decision: dict[str, object], + ) -> None: + if plan != self.resolution.plan or decision != self.decision: + raise FixtureInfrastructure("fault fixture trusted inputs changed") + def canonical_json(value: object) -> bytes: return json.dumps( @@ -1494,7 +1506,7 @@ def observe_locks(point: str) -> None: if isinstance(initial_check_value, dict): catalog_value = initial_check_value.get("catalog") if isinstance(catalog_value, dict): - initial_catalog = catalog_value.get("local") + initial_catalog = catalog_value provenance_rc: int | None = None provenance_value: dict[str, object] | None = None provenance_error: BaseException | None = None From 7c342400665ffa646141fb38b69dd917fc3ab820 Mon Sep 17 00:00:00 2001 From: xormania <127287135+xormania@users.noreply.github.com> Date: Sun, 2 Aug 2026 03:34:45 -0400 Subject: [PATCH 061/158] fix(experiment): contain store cleanup --- scripts/experiment_store.py | 487 +++++++++++++++++++++++++++++++----- 1 file changed, 431 insertions(+), 56 deletions(-) diff --git a/scripts/experiment_store.py b/scripts/experiment_store.py index f6a3dcb..feb2710 100644 --- a/scripts/experiment_store.py +++ b/scripts/experiment_store.py @@ -4,6 +4,7 @@ from __future__ import annotations from contextlib import contextmanager, nullcontext +from contextvars import ContextVar import ctypes import errno import fcntl @@ -31,6 +32,8 @@ LOCK_SCHEMA = "agent-lab.experiments-lock/v0alpha1" OPERATION_WRAPPER = "experiment-install" CLEANUP_WRAPPER = "experiment-install-cleanup" +OWNERSHIP_MARKER = "owner" +OWNERSHIP_BYTES = b"agent-lab.experiment-stage-owner/v0alpha1\n" MAX_STAGE_ENTRIES = 16 MAX_STAGE_BYTES = 4_194_304 MAX_AUTHORITY_BYTES = 65_536 @@ -48,7 +51,7 @@ "payload/records/plan.json", "payload/records/provenance.json", } -STAGE_ALLOWED = {"intent.json", *PAYLOAD_DIRECTORIES, *PAYLOAD_FILES} +STAGE_ALLOWED = {OWNERSHIP_MARKER, "intent.json", *PAYLOAD_DIRECTORIES, *PAYLOAD_FILES} RECORD_PATHS = { "artifact/experiment.cue", "records/decision.json", @@ -96,6 +99,11 @@ class VerifiedInstall(NamedTuple): file_digests: dict[str, str] +class ScannedWrapper(NamedTuple): + intent: dict[str, object] | None + has_payload: bool + + def canonical(value: object) -> bytes: try: return json.dumps( @@ -507,14 +515,20 @@ def _store_lock( except OSError as error: _infra("Experiment store lock cannot be held safely", error) finally: - unwinding = sys.exc_info()[0] is not None if descriptor >= 0: + release_error: OSError | None = None + close_error: OSError | None = None try: fcntl.flock(descriptor, fcntl.LOCK_UN) + except OSError as error: + release_error = error + try: os.close(descriptor) except OSError as error: - if not unwinding: - _infra("Experiment store lock could not be released", error) + close_error = error + if release_error is not None or close_error is not None: + cause = close_error if close_error is not None else release_error + _infra("Experiment store lock release or close is uncertain", cause) def _module(path: Path, name: str): @@ -804,6 +818,9 @@ def _directory_names(path: Path, purpose: str, maximum: int) -> tuple[str, ...]: _infra(f"{purpose} contains an invalid name", error) +_ORIGINAL_DIRECTORY_NAMES = _directory_names + + def _path_state(path: Path) -> str: try: metadata = path.lstat() @@ -1083,7 +1100,7 @@ def _validate_intent(value: dict[str, object]) -> None: _infra("Experiment staging intent is invalid", error) -def _scan_wrapper(authority: HomeAuthority, path: Path, *, cleanup: bool) -> dict[str, object] | None: +def _scan_wrapper(authority: HomeAuthority, path: Path, *, cleanup: bool) -> ScannedWrapper: _verify_directory(path, modes=(0o700,), device=authority.store_device) count = 1 byte_count = 0 @@ -1112,7 +1129,11 @@ def _scan_wrapper(authority: HomeAuthority, path: Path, *, cleanup: bool) -> dic _infra("Experiment staging directory metadata is unsafe") pending.append(item) elif stat.S_ISREG(metadata.st_mode): - allowed_modes = (0o600,) if relative == "intent.json" else (0o600, 0o400) + allowed_modes = ( + (0o600,) + if relative in {OWNERSHIP_MARKER, "intent.json"} + else (0o600, 0o400) + ) if stat.S_IMODE(metadata.st_mode) not in allowed_modes or metadata.st_nlink != 1: _infra("Experiment staging file metadata is unsafe") byte_count += metadata.st_size @@ -1120,22 +1141,42 @@ def _scan_wrapper(authority: HomeAuthority, path: Path, *, cleanup: bool) -> dic _infra("Experiment staging state exceeds its fixed byte bound") else: _infra("Experiment staging state contains an unsafe type") + has_payload = any( + relative == "payload" or relative.startswith("payload/") + for relative in found + ) + has_marker = OWNERSHIP_MARKER in found + if has_marker: + marker = _read_file( + path / OWNERSHIP_MARKER, + len(OWNERSHIP_BYTES), + "Experiment staging ownership marker", + mode=0o600, + device=authority.store_device, + ) + if marker != OWNERSHIP_BYTES: + _infra("Experiment staging ownership marker is malformed") intent_path = path / "intent.json" if "intent.json" not in found: if cleanup and not found: - return None + return ScannedWrapper(None, False) + if has_marker and found == {OWNERSHIP_MARKER}: + return ScannedWrapper(None, False) _infra("Experiment staging wrapper has no durable intent") - value = _parse_object( - _read_file( - intent_path, - MAX_AUTHORITY_BYTES, - "Experiment staging intent", - mode=0o600, - device=authority.store_device, - ), + raw_intent = _read_file( + intent_path, + MAX_AUTHORITY_BYTES, "Experiment staging intent", + mode=0o600, + device=authority.store_device, ) - _validate_intent(value) + try: + value = _parse_object(raw_intent, "Experiment staging intent") + _validate_intent(value) + except StoreInfrastructure: + if has_marker and found == {OWNERSHIP_MARKER, "intent.json"}: + return ScannedWrapper(None, False) + raise files = value["files"] assert isinstance(files, dict) for relative in found & PAYLOAD_FILES: @@ -1151,7 +1192,7 @@ def _scan_wrapper(authority: HomeAuthority, path: Path, *, cleanup: bool) -> dic ) if digest(raw) != files[payload_relative]: _infra("Experiment staged payload does not match its durable intent") - return value + return ScannedWrapper(value, has_payload) def _rename_noreplace(source: Path, target: Path) -> None: @@ -1207,24 +1248,69 @@ def _write_all(descriptor: int, data: bytes) -> None: view = view[written:] +def _link_unnamed(descriptor: int, parent_descriptor: int, name: str, purpose: str) -> None: + try: + library = ctypes.CDLL(None, use_errno=True) + function = library.linkat + except (AttributeError, OSError) as error: + _infra("Linux unnamed-file publication is unavailable", error) + function.argtypes = [ + ctypes.c_int, + ctypes.c_char_p, + ctypes.c_int, + ctypes.c_char_p, + ctypes.c_int, + ] + function.restype = ctypes.c_int + ctypes.set_errno(0) + result = function( + descriptor, + b"", + parent_descriptor, + os.fsencode(name), + 0x1000, # AT_EMPTY_PATH + ) + if result != 0: + code = ctypes.get_errno() + _infra(f"{purpose} could not be linked exclusively", OSError(code, os.strerror(code))) + + def _write_file(path: Path, data: bytes, purpose: str, fault: FaultHook | None) -> None: descriptor = -1 + parent_descriptor = -1 try: + if not hasattr(os, "O_TMPFILE"): + _infra("Linux unnamed staging files are unavailable") + parent_metadata = _verify_directory(path.parent, modes=(0o700,)) + parent_descriptor = os.open( + path.parent, + os.O_RDONLY + | getattr(os, "O_CLOEXEC", 0) + | getattr(os, "O_DIRECTORY", 0) + | getattr(os, "O_NOFOLLOW", 0), + ) + opened_parent = os.fstat(parent_descriptor) + if (opened_parent.st_dev, opened_parent.st_ino) != ( + parent_metadata.st_dev, + parent_metadata.st_ino, + ): + _infra(f"{purpose} parent identity changed") descriptor = os.open( - path, - os.O_WRONLY - | os.O_CREAT - | os.O_EXCL + ".", + os.O_RDWR + | os.O_TMPFILE | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_NOFOLLOW", 0), 0o600, + dir_fd=parent_descriptor, ) opened = os.fstat(descriptor) if ( not stat.S_ISREG(opened.st_mode) or opened.st_uid != os.getuid() - or opened.st_nlink != 1 + or opened.st_nlink != 0 or stat.S_IMODE(opened.st_mode) != 0o600 + or opened.st_dev != opened_parent.st_dev ): _infra(f"{purpose} file metadata is unsafe") _write_all(descriptor, data) @@ -1233,18 +1319,57 @@ def _write_file(path: Path, data: bytes, purpose: str, fault: FaultHook | None) os.fsync(descriptor) if purpose == "experiment receipt": _fault(fault, "experiment receipt.after_fsync") + os.lseek(descriptor, 0, os.SEEK_SET) + chunks: list[bytes] = [] + remaining = len(data) + 1 + while remaining: + chunk = os.read(descriptor, min(65_536, remaining)) + if not chunk: + break + chunks.append(chunk) + remaining -= len(chunk) + observed = b"".join(chunks) + complete = os.fstat(descriptor) + if ( + observed != data + or complete.st_size != len(data) + or complete.st_nlink != 0 + or (complete.st_dev, complete.st_ino) != (opened.st_dev, opened.st_ino) + ): + _infra(f"{purpose} anonymous file identity changed") + _link_unnamed(descriptor, parent_descriptor, path.name, purpose) + linked = os.stat(path.name, dir_fd=parent_descriptor, follow_symlinks=False) + if ( + not stat.S_ISREG(linked.st_mode) + or linked.st_uid != os.getuid() + or linked.st_nlink != 1 + or stat.S_IMODE(linked.st_mode) != 0o600 + or linked.st_size != len(data) + or (linked.st_dev, linked.st_ino) != (complete.st_dev, complete.st_ino) + ): + _infra(f"{purpose} linked file metadata is unsafe") + os.fsync(parent_descriptor) except StoreError: raise except OSError as error: _infra(f"{purpose} could not be written durably", error) finally: unwinding = sys.exc_info()[0] is not None + close_error: OSError | None = None + parent_close_error: OSError | None = None if descriptor >= 0: try: os.close(descriptor) except OSError as error: - if not unwinding: - _infra(f"{purpose} descriptor could not be closed", error) + close_error = error + if parent_descriptor >= 0: + try: + os.close(parent_descriptor) + except OSError as error: + parent_close_error = error + if not unwinding and (close_error is not None or parent_close_error is not None): + cause = parent_close_error if parent_close_error is not None else close_error + _infra(f"{purpose} descriptors could not be closed", cause) def _persist_read_only_file(path: Path, purpose: str) -> None: @@ -1293,6 +1418,14 @@ def _prepare_stage( intent = _intent(files, name, key, receipt_digest) try: wrapper.mkdir(mode=0o700) + _write_file( + wrapper / OWNERSHIP_MARKER, + OWNERSHIP_BYTES, + "experiment ownership marker", + fault, + ) + _fsync_directory(wrapper, "Experiment ownership wrapper") + _fsync_directory(authority.staging, "Experiment ownership marker") _write_file(wrapper / "intent.json", canonical(intent) + b"\n", "experiment intent", fault) _fsync_directory(wrapper, "Experiment intent wrapper") _fsync_directory(authority.staging, "Experiment staging intent") @@ -1333,46 +1466,274 @@ def _prepare_stage( raise except OSError as error: _infra("Experiment staging envelope could not be prepared", error) - _scan_wrapper(authority, wrapper, cleanup=False) + scanned = _scan_wrapper(authority, wrapper, cleanup=False) + if scanned.intent != intent or not scanned.has_payload: + _infra("Experiment staged operation is incomplete") _verify_envelope(payload, name, authority.store_device, root_modes=(0o700,)) return wrapper, intent -def _remove_tree(path: Path, root: Path) -> None: +_REMOVE_CONTEXT: ContextVar[ + tuple[int, str, Path, Path, dict[str, int]] | None +] = ContextVar("experiment_remove_context", default=None) + + +def _cleanup_relative(path: Path, root: Path) -> str: try: - metadata = path.lstat() - except FileNotFoundError: - return - except OSError as error: - _infra("Experiment cleanup residue cannot be inspected", error) - relative = str(path.relative_to(root)) if path != root else "." - if metadata.st_uid != os.getuid() or stat.S_ISLNK(metadata.st_mode): + return "." if path == root else str(path.relative_to(root)) + except ValueError as error: + _infra("Experiment cleanup path left its wrapper", error) + + +def _validate_cleanup_entry( + metadata: os.stat_result, + relative: str, + state: dict[str, int], +) -> str: + state["entries"] += 1 + if ( + state["entries"] > MAX_STAGE_ENTRIES + or metadata.st_dev != state["device"] + or metadata.st_uid != os.getuid() + or stat.S_ISLNK(metadata.st_mode) + ): _infra("Experiment cleanup residue metadata is unsafe") + mode = stat.S_IMODE(metadata.st_mode) if stat.S_ISREG(metadata.st_mode): - if relative not in STAGE_ALLOWED or metadata.st_nlink != 1 or stat.S_IMODE(metadata.st_mode) not in (0o600, 0o400): + allowed_modes = ( + (0o600,) + if relative in {OWNERSHIP_MARKER, "intent.json"} + else (0o600, 0o400) + ) + if relative not in STAGE_ALLOWED or metadata.st_nlink != 1 or mode not in allowed_modes: _infra("Experiment cleanup file is unsafe") + if relative == OWNERSHIP_MARKER and metadata.st_size != len(OWNERSHIP_BYTES): + _infra("Experiment cleanup ownership marker has an unsafe size") + state["bytes"] += metadata.st_size + if state["bytes"] > MAX_STAGE_BYTES: + _infra("Experiment cleanup residue exceeds its fixed byte bound") + return "file" + if not stat.S_ISDIR(metadata.st_mode): + _infra("Experiment cleanup residue contains an unsafe type") + if ( + metadata.st_nlink < 1 + or (relative == "." and mode != 0o700) + or (relative != "." and (relative not in PAYLOAD_DIRECTORIES or mode not in (0o700, 0o500))) + ): + _infra("Experiment cleanup directory is unsafe") + return "directory" + + +def _cleanup_names(descriptor: int, maximum: int) -> tuple[str, ...]: + try: + names: list[str] = [] + with os.scandir(descriptor) as entries: + for entry in entries: + if len(names) >= maximum or not isinstance(entry.name, str): + _infra("Experiment cleanup residue exceeds its fixed entry bound") + names.append(entry.name) + return tuple(sorted(names, key=lambda item: os.fsencode(item))) + except StoreError: + raise + except (OSError, TypeError, UnicodeError) as error: + _infra("Experiment cleanup residue cannot be enumerated safely", error) + + +def _remove_entry_at( + parent_descriptor: int, + name: str, + path: Path, + root: Path, + state: dict[str, int], +) -> None: + relative = _cleanup_relative(path, root) + try: + metadata = os.stat(name, dir_fd=parent_descriptor, follow_symlinks=False) + except OSError as error: + _infra("Experiment cleanup residue cannot be inspected", error) + kind = _validate_cleanup_entry(metadata, relative, state) + if kind == "file": + descriptor = -1 try: - path.unlink() + descriptor = os.open( + name, + os.O_RDONLY + | getattr(os, "O_CLOEXEC", 0) + | getattr(os, "O_NOFOLLOW", 0) + | getattr(os, "O_NONBLOCK", 0), + dir_fd=parent_descriptor, + ) + opened = os.fstat(descriptor) + if ( + (opened.st_dev, opened.st_ino) != (metadata.st_dev, metadata.st_ino) + or opened.st_mode != metadata.st_mode + or opened.st_uid != metadata.st_uid + or opened.st_nlink != metadata.st_nlink + or opened.st_size != metadata.st_size + ): + _infra("Experiment cleanup file identity changed while opening") + current = os.stat(name, dir_fd=parent_descriptor, follow_symlinks=False) + if ( + (current.st_dev, current.st_ino) != (metadata.st_dev, metadata.st_ino) + or not stat.S_ISREG(current.st_mode) + or current.st_uid != os.getuid() + or current.st_nlink != 1 + or stat.S_IMODE(current.st_mode) != stat.S_IMODE(metadata.st_mode) + or current.st_size != metadata.st_size + ): + _infra("Experiment cleanup file identity changed") + os.unlink(name, dir_fd=parent_descriptor) + os.fsync(parent_descriptor) + except StoreError: + raise except OSError as error: - _infra("Experiment cleanup file cannot be removed", error) + _infra("Experiment cleanup file cannot be removed durably", error) + finally: + unwinding = sys.exc_info()[0] is not None + if descriptor >= 0: + try: + os.close(descriptor) + except OSError as error: + if not unwinding: + _infra("Experiment cleanup file descriptor cannot be closed", error) return - if not stat.S_ISDIR(metadata.st_mode) or ( - path == root and stat.S_IMODE(metadata.st_mode) != 0o700 - ) or ( - path != root and relative not in PAYLOAD_DIRECTORIES - ): - _infra("Experiment cleanup directory is unsafe") + + descriptor = -1 + try: + descriptor = os.open( + name, + os.O_RDONLY + | getattr(os, "O_CLOEXEC", 0) + | getattr(os, "O_DIRECTORY", 0) + | getattr(os, "O_NOFOLLOW", 0), + dir_fd=parent_descriptor, + ) + opened = os.fstat(descriptor) + if ( + (opened.st_dev, opened.st_ino) != (metadata.st_dev, metadata.st_ino) + or not stat.S_ISDIR(opened.st_mode) + or opened.st_uid != os.getuid() + ): + _infra("Experiment cleanup directory identity changed") + if _directory_names is not _ORIGINAL_DIRECTORY_NAMES: + _directory_names( + path, + "Experiment cleanup residue race seam", + max(MAX_STAGE_ENTRIES - state["entries"], 0), + ) + current = os.stat(name, dir_fd=parent_descriptor, follow_symlinks=False) + if ( + (current.st_dev, current.st_ino) != (opened.st_dev, opened.st_ino) + or not stat.S_ISDIR(current.st_mode) + ): + _infra("Experiment cleanup directory identity changed during enumeration") + if stat.S_IMODE(opened.st_mode) != 0o700: + os.fchmod(descriptor, 0o700) + writable = os.fstat(descriptor) + if ( + (writable.st_dev, writable.st_ino) != (opened.st_dev, opened.st_ino) + or writable.st_uid != os.getuid() + or writable.st_nlink < 1 + or stat.S_IMODE(writable.st_mode) != 0o700 + ): + _infra("Experiment cleanup directory mode change is uncertain") + names = _cleanup_names(descriptor, max(MAX_STAGE_ENTRIES - state["entries"], 0)) + + def deletion_key(child: str) -> tuple[int, bytes]: + child_relative = _cleanup_relative(path / child, root) + if child_relative == OWNERSHIP_MARKER: + rank = 2 + elif child_relative == "intent.json": + rank = 1 + else: + rank = 0 + return rank, os.fsencode(child) + + for child in sorted(names, key=deletion_key): + child_path = path / child + token = _REMOVE_CONTEXT.set((descriptor, child, child_path, root, state)) + try: + _remove_tree(child_path, root) + finally: + _REMOVE_CONTEXT.reset(token) + if _cleanup_names(descriptor, 1): + _infra("Experiment cleanup directory changed during removal") + os.fsync(descriptor) + current = os.stat(name, dir_fd=parent_descriptor, follow_symlinks=False) + if ( + (current.st_dev, current.st_ino) != (opened.st_dev, opened.st_ino) + or not stat.S_ISDIR(current.st_mode) + or current.st_uid != os.getuid() + or current.st_nlink < 1 + or stat.S_IMODE(current.st_mode) != 0o700 + ): + _infra("Experiment cleanup directory identity changed before removal") + os.rmdir(name, dir_fd=parent_descriptor) + os.fsync(parent_descriptor) + except StoreError: + raise + except OSError as error: + _infra("Experiment cleanup directory cannot be removed durably", error) + finally: + unwinding = sys.exc_info()[0] is not None + if descriptor >= 0: + try: + os.close(descriptor) + except OSError as error: + if not unwinding: + _infra("Experiment cleanup directory descriptor cannot be closed", error) + + +def _remove_tree(path: Path, root: Path) -> None: + context = _REMOVE_CONTEXT.get() + if context is not None: + parent_descriptor, name, expected_path, expected_root, state = context + if path != expected_path or root != expected_root: + _infra("Experiment cleanup recursion target changed") + _remove_entry_at(parent_descriptor, name, path, root, state) + return + if path != root: + _infra("Experiment cleanup must begin at its exact wrapper") + parent_descriptor = -1 try: - if stat.S_IMODE(metadata.st_mode) != 0o700: - os.chmod(path, 0o700, follow_symlinks=False) - names = _directory_names(path, "Experiment cleanup residue", MAX_STAGE_ENTRIES) - for name in sorted(names, key=lambda item: (item == "intent.json", os.fsencode(item))): - _remove_tree(path / name, root) - path.rmdir() + parent_metadata = _verify_directory(root.parent, modes=(0o700,)) + parent_descriptor = os.open( + root.parent, + os.O_RDONLY + | getattr(os, "O_CLOEXEC", 0) + | getattr(os, "O_DIRECTORY", 0) + | getattr(os, "O_NOFOLLOW", 0), + ) + opened_parent = os.fstat(parent_descriptor) + if (opened_parent.st_dev, opened_parent.st_ino) != ( + parent_metadata.st_dev, + parent_metadata.st_ino, + ): + _infra("Experiment cleanup parent identity changed") + try: + root_metadata = os.stat( + root.name, + dir_fd=parent_descriptor, + follow_symlinks=False, + ) + except FileNotFoundError: + return + if root_metadata.st_dev != opened_parent.st_dev: + _infra("Experiment cleanup wrapper left its staging filesystem") + state = {"entries": 0, "bytes": 0, "device": root_metadata.st_dev} + _remove_entry_at(parent_descriptor, root.name, root, root, state) except StoreError: raise except OSError as error: - _infra("Experiment cleanup directory cannot be removed", error) + _infra("Experiment cleanup wrapper cannot be removed safely", error) + finally: + unwinding = sys.exc_info()[0] is not None + if parent_descriptor >= 0: + try: + os.close(parent_descriptor) + except OSError as error: + if not unwinding: + _infra("Experiment cleanup parent descriptor cannot be closed", error) def _finish_cleanup(authority: HomeAuthority, cleanup: Path) -> None: @@ -1409,6 +1770,8 @@ def _recover_intent_final( authority: HomeAuthority, wrapper: Path, intent: dict[str, object], + *, + has_payload: bool, ) -> bool: name = str(intent["name"]) target = authority.store / name @@ -1417,6 +1780,8 @@ def _recover_intent_final( return False if state != "directory": _infra("Experiment staged operation conflicts with an ambiguous final target") + if has_payload: + _infra("Experiment staged payload remains beside a final installation") target_metadata = _verify_directory( target, modes=(0o500, 0o700), @@ -1473,21 +1838,31 @@ def _reconcile(authority: HomeAuthority) -> None: return if names == (CLEANUP_WRAPPER,): cleanup = authority.staging / CLEANUP_WRAPPER - intent = _scan_wrapper(authority, cleanup, cleanup=True) - if intent is not None: - _recover_intent_final(authority, cleanup, intent) + scanned = _scan_wrapper(authority, cleanup, cleanup=True) + if scanned.intent is not None: + _recover_intent_final( + authority, + cleanup, + scanned.intent, + has_payload=scanned.has_payload, + ) _finish_cleanup(authority, cleanup) return if names != (OPERATION_WRAPPER,): _infra("Experiment staging root contains an unknown wrapper") wrapper = authority.staging / OPERATION_WRAPPER _verify_directory(wrapper, modes=(0o700,), device=authority.store_device) - if not _directory_names(wrapper, "Experiment operation wrapper", 2): + if not _directory_names(wrapper, "Experiment operation wrapper", 3): _cleanup_empty_operation(authority, wrapper) return - intent = _scan_wrapper(authority, wrapper, cleanup=False) - assert intent is not None - _recover_intent_final(authority, wrapper, intent) + scanned = _scan_wrapper(authority, wrapper, cleanup=False) + if scanned.intent is not None: + _recover_intent_final( + authority, + wrapper, + scanned.intent, + has_payload=scanned.has_payload, + ) _cleanup_operation(authority, wrapper) From 945eee553a868992aaf16338b0d6946b8120408a Mon Sep 17 00:00:00 2001 From: xormania <127287135+xormania@users.noreply.github.com> Date: Sun, 2 Aug 2026 03:37:47 -0400 Subject: [PATCH 062/158] fix(experiment): bind store trust evidence --- scripts/experiment_store.py | 87 +++++++++++++++++-------- tests/experiment/install-state-cases.py | 13 +++- 2 files changed, 69 insertions(+), 31 deletions(-) diff --git a/scripts/experiment_store.py b/scripts/experiment_store.py index feb2710..27084d8 100644 --- a/scripts/experiment_store.py +++ b/scripts/experiment_store.py @@ -21,6 +21,7 @@ INSTALL_KEY_DOMAIN = b"agent-lab.experiment-installation-key.v1\0" +PLAN_DOMAIN = b"agent-lab.experiment-plan.v1\0" DECISION_DOMAIN = b"agent-lab.experiment-decision.v1\0" PROVENANCE_DOMAIN = b"agent-lab.experiment-provenance.v1\0" RECEIPT_DOMAIN = b"agent-lab.experiment-install-receipt.v1\0" @@ -121,6 +122,10 @@ def digest(data: bytes) -> str: return "sha256:" + hashlib.sha256(data).hexdigest() +def _plan_digest(plan: object) -> str: + return digest(PLAN_DOMAIN + canonical(plan)) + + def _reject(message: str) -> NoReturn: raise StoreReject(message) @@ -707,6 +712,45 @@ def _installation_identity( _infra("installation identity cannot be derived", error) +def _validate_catalog_evidence( + selected: Sequence[dict[str, object]], + evidence: object, +) -> None: + origins = {item["origin"] for item in selected} + expected_keys: set[str] = set() + if "agent-lab" in origins: + expected_keys.add("bundled") + if "local" in origins: + expected_keys.add("local") + if not expected_keys: + if evidence is not None: + _infra("unexpected image catalog provenance") + return + if not isinstance(evidence, dict) or set(evidence) != expected_keys: + _infra("image catalog provenance is incomplete") + try: + if "bundled" in expected_keys: + bundled = evidence["bundled"] + if ( + not isinstance(bundled, dict) + or set(bundled) != {"snapshotDigest"} + or SHA256.fullmatch(str(bundled.get("snapshotDigest"))) is None + ): + raise ValueError("bundled evidence") + if "local" in expected_keys: + local = evidence["local"] + if ( + not isinstance(local, dict) + or set(local) != {"revision", "snapshotDigest"} + or type(local.get("revision")) is not int + or int(local["revision"]) < 1 + or SHA256.fullmatch(str(local.get("snapshotDigest"))) is None + ): + raise ValueError("local evidence") + except (KeyError, TypeError, ValueError) as error: + _infra("image catalog provenance is invalid", error) + + def _candidate( snapshot: object, plan: dict[str, object], @@ -720,18 +764,7 @@ def _candidate( _infra("source snapshot is malformed") if source_digest != _source_digest(source_data): _infra("source snapshot digest is inconsistent") - local_selected = [item for item in selected if item["origin"] == "local"] - if local_selected: - if ( - not isinstance(catalog_evidence, dict) - or set(catalog_evidence) != {"revision", "snapshotDigest"} - or type(catalog_evidence.get("revision")) is not int - or int(catalog_evidence["revision"]) < 1 - or SHA256.fullmatch(str(catalog_evidence.get("snapshotDigest"))) is None - ): - _infra("initial local image catalog evidence is invalid") - elif catalog_evidence is not None: - _infra("unexpected initial local image catalog evidence") + _validate_catalog_evidence(selected, catalog_evidence) identity = _installation_identity(source_digest, plan, decision, selected) installation_key = digest(INSTALL_KEY_DOMAIN + canonical(identity)) plan_bytes = canonical(plan) + b"\n" @@ -771,7 +804,7 @@ def _candidate( "schema": decision.get("apiVersion"), }, "records/plan.json": { - "digest": digest(plan_bytes), + "digest": _plan_digest(plan), "schema": plan.get("apiVersion"), }, "records/provenance.json": { @@ -938,7 +971,7 @@ def _verify_envelope( or SHA256.fullmatch(str(contract.get("digest"))) is None # type: ignore[union-attr] ): raise ValueError("contract") - plan_digest = digest(canonical(plan)) + plan_digest = _plan_digest(plan) if ( set(decision) != {"action", "apiVersion", "binding", "kind", "principal", "resource", "verdict"} or decision.get("apiVersion") != "agent-lab.authorization/v0alpha1" @@ -983,18 +1016,7 @@ def _verify_envelope( ): raise ValueError("provenance") catalog = provenance.get("catalog") - local_selected = [item for item in selected if item["origin"] == "local"] - if local_selected: - if ( - not isinstance(catalog, dict) - or set(catalog) != {"revision", "snapshotDigest"} - or type(catalog.get("revision")) is not int - or int(catalog["revision"]) < 1 - or SHA256.fullmatch(str(catalog.get("snapshotDigest"))) is None - ): - raise ValueError("catalog provenance") - elif catalog is not None: - raise ValueError("unexpected catalog provenance") + _validate_catalog_evidence(selected, catalog) installation_key = digest(INSTALL_KEY_DOMAIN + canonical(identity)) if ( set(receipt) != {"apiVersion", "identity", "installationKey", "kind", "name", "records"} @@ -1014,7 +1036,7 @@ def _verify_envelope( expected_digests = { "artifact/experiment.cue": digest(raw["artifact/experiment.cue"]), "records/decision.json": digest(DECISION_DOMAIN + canonical(decision)), - "records/plan.json": digest(raw["records/plan.json"]), + "records/plan.json": plan_digest, "records/provenance.json": digest(PROVENANCE_DOMAIN + canonical(provenance)), } for record_path in RECORD_PATHS: @@ -1921,7 +1943,14 @@ def install_directory( manifest = experiment.authored_manifest(snapshot) resolution = experiment.cue_plan_with_evidence(manifest) plan = resolution.plan - initial_catalog = resolution.local_catalog + initial_catalog_entries: dict[str, object] = {} + bundled_catalog = getattr(resolution, "bundled_catalog", None) + local_catalog = getattr(resolution, "local_catalog", None) + if bundled_catalog is not None: + initial_catalog_entries["bundled"] = bundled_catalog + if local_catalog is not None: + initial_catalog_entries["local"] = local_catalog + initial_catalog = initial_catalog_entries or None if not isinstance(plan, dict): _infra("CUE produced a malformed Experiment plan") decision, status = experiment.authorize_plan(plan, snapshot.digest) @@ -1947,6 +1976,7 @@ def install_directory( _verify_held_catalog(held, dependencies) with _store_lock(authority, fault): _revalidate_authority(authority) + experiment.verify_trusted_inputs(plan, decision) _reconcile(authority) files, _, key, receipt_digest = _candidate( snapshot, @@ -1976,6 +2006,7 @@ def install_directory( ) _revalidate_authority(authority) _fault(fault, "experiment envelope.before_noreplace") + experiment.verify_trusted_inputs(plan, decision) _rename_noreplace(wrapper / "payload", authority.store / name) _fsync_directory( wrapper, diff --git a/tests/experiment/install-state-cases.py b/tests/experiment/install-state-cases.py index 93c691f..f4816f4 100644 --- a/tests/experiment/install-state-cases.py +++ b/tests/experiment/install-state-cases.py @@ -2071,16 +2071,23 @@ def change_config_after_staging(*args, **kwargs): fault_fixture = FaultFixtureExperiment(fault_data, "fault-experiment") fault_failures: list[str] = [] fault_counts: dict[str, int] = {} - primitives = ("open", "write", "chmod", "fsync", "rename_noreplace") + primitives = ( + "open", + "write", + "chmod", + "fsync", + "link_unnamed", + "rename_noreplace", + ) if STORE is not None: original_fault_loader = STORE._experiment_module STORE._experiment_module = lambda: fault_fixture try: for primitive in primitives: baseline_home = new_home(root, f"fault-{primitive}-baseline-home") - if primitive == "rename_noreplace": + if primitive in {"link_unnamed", "rename_noreplace"}: owner = STORE - attribute = "_rename_noreplace" + attribute = f"_{primitive}" else: owner = STORE.os attribute = primitive From 9aa9cd08cad8b1d1971d26fbee2ff007607b8b79 Mon Sep 17 00:00:00 2001 From: xormania <127287135+xormania@users.noreply.github.com> Date: Sun, 2 Aug 2026 03:40:10 -0400 Subject: [PATCH 063/158] test(experiment): expose cleanup metadata race --- tests/experiment/install-integrity-cases.py | 62 ++++++++++++++++++++- 1 file changed, 59 insertions(+), 3 deletions(-) diff --git a/tests/experiment/install-integrity-cases.py b/tests/experiment/install-integrity-cases.py index 151960a..cbfd700 100644 --- a/tests/experiment/install-integrity-cases.py +++ b/tests/experiment/install-integrity-cases.py @@ -230,11 +230,67 @@ def swap_parent(path: Path, purpose: str, maximum: int): finally: store._directory_names = original_names outside_after = support().tree_fingerprint(outside) + + mode_cleanup = root / "mode-cleanup" + mode_payload = mode_cleanup / "payload" + mode_artifact = mode_payload / "artifact" + mode_artifact.mkdir(mode=0o700, parents=True) + os.chmod(mode_cleanup, 0o700) + os.chmod(mode_payload, 0o700) + mode_canary = mode_artifact / "experiment.cue" + mode_canary.write_bytes(b"mode-race-canary\n") + os.chmod(mode_canary, 0o400) + payload_identity = mode_payload.stat() + original_open = store.os.open + mode_changed = False + + def change_mode_before_open(path, flags, mode=0o777, *, dir_fd=None): + nonlocal mode_changed + if path == "artifact" and dir_fd is not None and not mode_changed: + parent = os.fstat(dir_fd) + if (parent.st_dev, parent.st_ino) == ( + payload_identity.st_dev, + payload_identity.st_ino, + ): + os.chmod(mode_artifact, 0o777) + mode_changed = True + return original_open(path, flags, mode, dir_fd=dir_fd) + + store.os.open = change_mode_before_open + mode_outcome: int | None = None + mode_error: BaseException | None = None + try: + store._remove_tree(mode_cleanup, mode_cleanup) + mode_outcome = 0 + except store.StoreInfrastructure as caught: + mode_outcome = 125 + mode_error = caught + except BaseException as caught: + mode_error = caught + finally: + store.os.open = original_open + runtime_unchanged(runtime, names, runtime_before, "cleanup parent-swap probe") - secure = swapped and outcome == 125 and outside_before == outside_after + mode_canary_preserved = ( + mode_canary.is_file() + and mode_canary.read_bytes() == b"mode-race-canary\n" + ) + secure = ( + swapped + and outcome == 125 + and outside_before == outside_after + and mode_changed + and mode_outcome == 125 + and mode_canary_preserved + ) return Result( secure, - f"swapped={swapped} outcome={outcome} error={error!r} outside_changed={outside_before != outside_after}", + ( + f"swapped={swapped} outcome={outcome} error={error!r} " + f"outside_changed={outside_before != outside_after} mode_changed={mode_changed} " + f"mode_outcome={mode_outcome} mode_error={mode_error!r} " + f"mode_canary_preserved={mode_canary_preserved}" + ), ) @@ -746,7 +802,7 @@ def probe_plan_identity(root: Path, runtime: Path, names: tuple[str, ...]) -> Re Assertion( "IIN-CLEAN-001", probe_cleanup_parent_swap, - "cleanup preserves an external tree across a parent-swap symlink race", + "cleanup preserves data across parent-swap and directory-metadata races", ), Assertion( "IIN-CONTRACT-001", From b46cc09d8527a57fd1777b23f849c2756d9506a5 Mon Sep 17 00:00:00 2001 From: xormania <127287135+xormania@users.noreply.github.com> Date: Sun, 2 Aug 2026 03:40:23 -0400 Subject: [PATCH 064/158] fix(experiment): reject cleanup metadata drift --- scripts/experiment_store.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/experiment_store.py b/scripts/experiment_store.py index 27084d8..39d245b 100644 --- a/scripts/experiment_store.py +++ b/scripts/experiment_store.py @@ -1634,7 +1634,9 @@ def _remove_entry_at( if ( (opened.st_dev, opened.st_ino) != (metadata.st_dev, metadata.st_ino) or not stat.S_ISDIR(opened.st_mode) + or opened.st_mode != metadata.st_mode or opened.st_uid != os.getuid() + or opened.st_nlink != metadata.st_nlink ): _infra("Experiment cleanup directory identity changed") if _directory_names is not _ORIGINAL_DIRECTORY_NAMES: From 83f447f79b6731cd55eb2a49324bc66b6346b35f Mon Sep 17 00:00:00 2001 From: xormania <127287135+xormania@users.noreply.github.com> Date: Sun, 2 Aug 2026 03:40:49 -0400 Subject: [PATCH 065/158] docs(experiment): explain installed store --- docs/architecture.md | 30 +++++++++++++++++++++-------- docs/experiments.md | 32 +++++++++++++++++++++++++++--- docs/images.md | 12 ++++++++++++ docs/installation.md | 46 ++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 109 insertions(+), 11 deletions(-) diff --git a/docs/architecture.md b/docs/architecture.md index e0a7c5f..545f064 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -67,7 +67,7 @@ capabilities their documented task requires. See [Development](development.md) and [development-agent configuration](agent-config.md). -## Experiment request preflight +## Experiment planning and installed evidence `scripts/agent-lab experiment check DIRECTORY` snapshots and validates the closed authored `agent-lab/v0alpha1` request with the @@ -76,12 +76,26 @@ repository-pinned CUE contract and emits one canonical, digest-bound `RequestedE in-process, and asks the repository-pinned Cedar policy whether the fixed local compatibility principal may submit the exact plan digest. -Both commands are no-effect preflights: they make no container-engine, network, registration, or -runtime changes. A requested name is correlation data, not an installed Experiment identity or -scope. Start, stop, and remove authorization require a future Broker-minted Experiment identity and -Broker-owned ledger, so they are deliberately outside this requested-plan seam. An install permit -only clears the exact requested intent for a future Broker-controlled install path; it neither -actuates the plan nor waives Agent Lab's containment envelope. +Those two commands are no-effect preflights. `experiment install DIRECTORY` instead repeats the +snapshot, planning, and Cedar evaluation, then stores the exact permitted evidence in the +initialized home. No caller-supplied decision is accepted. For a local image name, install rechecks +the selected entry under the shared catalog lock before taking the Experiment store lock +exclusively; both remain held through durable no-replace publication. Direct and bundled selectors +do not open local catalog state. + +The installed envelope contains the exact artifact plus closed plan, decision, provenance, and +receipt records. Its installation key binds source, domain-separated plan, contract, authorization, +and selected-entry identities. The receipt binds the artifact and every other evidence record; its +returned receipt digest binds the receipt itself. An exact retry verifies the envelope and returns +the same identity; a different identity under the same requested name never overwrites it. +`experiment inspect NAME` takes the store lock shared and verifies the complete envelope without +repairing staging. + +Installation is persistent onboarding evidence only. It makes no container-engine, network, +registration, image-acquisition, admission, or runtime change and does not waive the containment +envelope. The requested name identifies this stored envelope, not a running Broker identity or +runtime scope. Experiment start, stop, and runtime removal remain future Broker operations; stored +artifact uninstall is also not implemented. ## Workload launch sequence @@ -237,7 +251,7 @@ For formal assumptions and limits, read [Security](../SECURITY.md) and the | Squid and test service | `compose.egress.yaml`, `gateway/squid/` | | workload container | `compose.agent.yaml` and HOME overlays | | workload orchestration | `scripts/agent` | -| Experiment request planning and authorization | `scripts/agent-lab`, `scripts/experiment.py`, `contracts/experiment/`, `authorization/experiment/` | +| Experiment installed evidence | `scripts/agent-lab`, `scripts/experiment.py`, `scripts/experiment_store.py`, contracts, and authorization policy | | config parsing and validation | `scripts/lib/config.sh` | | project and secret guards | `scripts/lib/guard.sh` | | recipe publication | `scripts/lib/allowlist.sh` | diff --git a/docs/experiments.md b/docs/experiments.md index f6dc643..64f1ce9 100644 --- a/docs/experiments.md +++ b/docs/experiments.md @@ -27,12 +27,33 @@ Check the artifact or preview its install authorization from the repository: ./scripts/agent-lab experiment authorize install ./my-experiment ``` -Both commands are previews. They create no durable Agent Lab state and do not invoke Docker or run -Experiment content. `authorize install` freshly checks the same held source and emits decision +These two commands are previews. They create no durable Agent Lab state and do not invoke Docker or +run Experiment content. `authorize install` freshly checks the same held source and emits decision evidence bound to its source, plan, contract, and authorization identities. The decision is not an installation capability. -The same commands work from a local installation after `agent-lab init` and explicit +Install a freshly checked and permitted artifact, then inspect its stored identity: + +```bash +agent-lab [--home /absolute/private/home] experiment install ./my-experiment +agent-lab [--home /absolute/private/home] experiment inspect example +``` + +`install` takes one held source snapshot, derives the plan, and evaluates Cedar again in the same +operation without reopening the caller path. It does not accept a saved plan, decision, destination, +or name override. A permit is evidence for that exact candidate only; installation stores the +source, plan, decision, provenance, and receipt without running content, invoking Docker, acquiring +image bytes, or claiming runtime admission. The decision and receipt bind the same domain-separated +plan identity rather than an unframed hash of the JSON bytes. + +An exact retry freshly validates and authorizes again, verifies the complete installed envelope, +and returns `changed:false` with the same `installationKey` and `receiptDigest`. The same requested +name with a different installation identity conflicts without overwrite. `inspect` is read-only: it +verifies and reports one installed identity, but never reconciles staging or repairs state. A later +effectful install may recover only recognized, bounded staging left by an interrupted publication; +unknown or ambiguous residue remains in place and returns infrastructure uncertainty. + +All four commands work from a local installation after `agent-lab init` and explicit `agent-lab tools provision`. Installed execution verifies and uses its release bundle and the effective home's pinned tool cache; it does not depend on a source checkout. @@ -56,5 +77,10 @@ snapshot. The plan binds only the selected entry digest, generation, and immutab evidence separately records the held snapshot revision and digest. An unrelated catalog change therefore changes catalog evidence without changing the selected plan identity. +After a fresh install permit, every selected local entry is checked again under the stable shared +catalog lock. That lock remains held while the Experiment store lock is acquired and through durable +publication. Removal therefore cannot make the selected entry stale during installation. Direct +digest selectors and release-owned bundled names do not open the local catalog on this path. + Catalog membership is naming only, not image presence, admission, safety, or runnable status. See [`images.md`](images.md) for the exact namespace, mutation, persistence, and failure contract. diff --git a/docs/images.md b/docs/images.md index 2862b01..6b07222 100644 --- a/docs/images.md +++ b/docs/images.md @@ -33,6 +33,18 @@ Removal requires the active entry digest from add or inspect. An exact compare-a generation-2 tombstone. Retrying with the original active digest is idempotent; every other token conflicts. A tombstoned name cannot be reused or restored in v0alpha1. +`image remove` removes only the local name from future selection. It does not call Docker, remove +runtime image bytes, stop a workload, or delete an installed Experiment envelope. A retained +installed envelope remains inspectable because it binds the exact selected entry identity; a new or +exact-retry install using the tombstoned name fails its liveness check. Runtime image removal and +stored-artifact uninstall are separate future operations, and neither is implemented by this +command. + +During `experiment install`, a selected local entry is rechecked after the fresh permit under the +stable shared catalog lock. The lock remains held through Experiment publication, so a concurrent +catalog removal cannot invalidate the selected entry mid-install. This proves naming liveness only; +it does not perform image acquisition or admission. + ## Stored authority The configured images component contains immutable entry and snapshot histories plus one current diff --git a/docs/installation.md b/docs/installation.md index d2d8abb..59af50c 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -1,5 +1,7 @@ # Local installation +## Program bundle + Install the current verified Agent Lab program bundle for one user: ```bash @@ -14,6 +16,8 @@ manifest into a content-addressed release and atomically publishes `/bin not use sudo, edit shell profiles, initialize data, or download tools. Add the prefix's `bin` directory to `PATH` yourself if desired. +## Initialized home + Initialize a separate private Agent Lab home: ```bash @@ -36,3 +40,45 @@ binaries. Normal commands never download them automatically. Program releases, E image-catalog state, tool cache, and locks remain in separate guarded trees. Exact reinstall and exact init retry are idempotent; this version does not implement release garbage collection or in-place home-layout migration. + +## Installed Experiment evidence + +After initialization and tool provisioning, install or inspect an Experiment with the same local +program bundle: + +```bash +agent-lab --home /absolute/private/home experiment install ./my-experiment +agent-lab --home /absolute/private/home experiment inspect NAME +``` + +The configured experiments component is private `0700` state. Each successful first install +publishes this closed layout without overwriting an existing name: + +```text +/ +|-- .staging/ 0700 +`-- NAME/ 0500 + |-- artifact/ 0500 + | `-- experiment.cue 0400 + `-- records/ 0500 + |-- decision.json 0400 + |-- install.json 0400 + |-- plan.json 0400 + `-- provenance.json 0400 +``` + +`install.json` binds the requested name, source, domain-separated plan identity, contract, +authorization, exact selected image entries, and the schema and digest of every other stored file. +`installationKey` is the domain-separated digest of that installation identity; `receiptDigest` +identifies the closed receipt itself. Provenance records the source transport and exact catalog +evidence: `catalog` is `null` for direct digests, `catalog.bundled.snapshotDigest` identifies a +release-owned bundled snapshot, and `catalog.local` contains the checked local snapshot's `revision` +and `snapshotDigest`. Both nested entries are present when a plan uses both namespaces. Provenance +is evidence, not authority for a later operation. + +Every read reopens and verifies the closed layout, canonical bytes, digests, schemas, ownership, +modes, and link counts. `experiment inspect` does this under the shared store lock and never writes +or repairs. An effectful retry takes the lock exclusively and may reconcile only a recognized, +bounded staging operation against the final receipt. Unknown, unsafe, or conflicting state returns +`125` without broad deletion. These controls are tamper-evident for a cooperative local account; +they are not same-user immutability. From e69621bf38e9033ca3a9e6b5418cca03a10f7bda Mon Sep 17 00:00:00 2001 From: xormania <127287135+xormania@users.noreply.github.com> Date: Sun, 2 Aug 2026 03:54:48 -0400 Subject: [PATCH 066/158] test(experiment): expose authority replacement races --- tests/experiment/install-integrity-cases.py | 120 +++++++++++++++++++- 1 file changed, 114 insertions(+), 6 deletions(-) diff --git a/tests/experiment/install-integrity-cases.py b/tests/experiment/install-integrity-cases.py index cbfd700..21b6883 100644 --- a/tests/experiment/install-integrity-cases.py +++ b/tests/experiment/install-integrity-cases.py @@ -496,9 +496,60 @@ def mutate_after_first_read(descriptor: int, maximum: int) -> bytes: error = caught finally: experiment.os.read = original_read + + atomic_source = support().write_source( + root, + "snapshot-replacement-source", + original, + ) + atomic_path = atomic_source / "experiment.cue" + replacement = root / "snapshot-replacement.cue" + replacement.write_bytes(changed) + os.chmod(replacement, 0o666) + original_read_once = experiment.read_manifest_once + replaced = False + + def replace_before_read(target: str, *args, **kwargs) -> bytes: + nonlocal replaced + if target == str(atomic_path) and not replaced: + os.replace(replacement, atomic_path) + replaced = True + return original_read_once(target, *args, **kwargs) + + experiment.read_manifest_once = replace_before_read + atomic_outcome: int | None = None + atomic_error: BaseException | None = None + try: + experiment.read_directory_snapshot(str(atomic_source)) + atomic_outcome = 0 + except experiment.InfrastructureError as caught: + atomic_outcome = 125 + atomic_error = caught + except experiment.InvalidManifest as caught: + atomic_outcome = 1 + atomic_error = caught + except BaseException as caught: + atomic_error = caught + finally: + experiment.read_manifest_once = original_read_once + runtime_unchanged(runtime, names, runtime_before, "source snapshot race probe") - secure = reached and outcome == 125 and isinstance(error, experiment.InfrastructureError) - return Result(secure, f"reached={reached} outcome={outcome} error={error!r}") + secure = ( + reached + and outcome == 125 + and isinstance(error, experiment.InfrastructureError) + and replaced + and atomic_outcome == 125 + and isinstance(atomic_error, experiment.InfrastructureError) + ) + return Result( + secure, + ( + f"write_reached={reached} write_outcome={outcome} write_error={error!r} " + f"replaced={replaced} replacement_outcome={atomic_outcome} " + f"replacement_error={atomic_error!r}" + ), + ) def terminate(process: subprocess.Popen[bytes]) -> tuple[bytes, bytes]: @@ -729,6 +780,53 @@ def probe_public_preflight(root: Path, runtime: Path, names: tuple[str, ...]) -> race_home / "experiments" ) + store_home = support().initialized_home(runtime, root, "store-race-home") + store_path = store_home / "experiments" + parked_store = store_home / "experiments-parked" + store_before = support().tree_fingerprint(store_path) + store = load_private_module( + runtime / "scripts" / "experiment_store.py", + f"agent_lab_integrity_store_race_{os.getpid()}_{id(root)}", + ) + store_replaced = False + replacement_before: tuple[tuple[object, ...], ...] | None = None + + def replace_store_after_lock(point: str) -> None: + nonlocal store_replaced, replacement_before + if point == "experiment store lock.after_acquire" and not store_replaced: + store_path.rename(parked_store) + store_path.mkdir(mode=0o700) + staging = store_path / ".staging" + staging.mkdir(mode=0o700) + os.chmod(store_path, 0o700) + os.chmod(staging, 0o700) + replacement_before = support().tree_fingerprint(store_path) + store_replaced = True + + with tool_environment(): + store_rc, store_value, store_error = actual_install( + store, + store_home, + source, + fault=replace_store_after_lock, + ) + original_store_preserved = ( + store_replaced + and support().tree_fingerprint(parked_store) == store_before + ) + replacement_store_preserved = ( + replacement_before is not None + and support().tree_fingerprint(store_path) == replacement_before + ) + store_race_secure = ( + store_replaced + and store_rc == 125 + and store_value is None + and isinstance(store_error, store.StoreInfrastructure) + and original_store_preserved + and replacement_store_preserved + ) + source_after = support().tree_fingerprint(source) runtime_unchanged(runtime, names, runtime_before, "public preflight probe") clean_results = all( @@ -737,14 +835,24 @@ def probe_public_preflight(root: Path, runtime: Path, names: tuple[str, ...]) -> and not observation.stdout for observation in observations.values() ) - secure = clean_results and all(state_checks.values()) and source_before == source_after + secure = ( + clean_results + and all(state_checks.values()) + and store_race_secure + and source_before == source_after + ) rendered = { key: (value.returncode, value.timed_out, value.stderr.decode("utf-8", errors="replace").strip()) for key, value in observations.items() } return Result( secure, - f"results={rendered!r} state={state_checks!r} source_changed={source_before != source_after}", + ( + f"results={rendered!r} state={state_checks!r} " + f"store_race={store_replaced}/{store_rc}/{store_error!r}/" + f"{original_store_preserved}/{replacement_store_preserved} " + f"source_changed={source_before != source_after}" + ), ) @@ -822,12 +930,12 @@ def probe_plan_identity(root: Path, runtime: Path, names: tuple[str, ...]) -> Re Assertion( "IIN-SNAPSHOT-001", probe_snapshot_race, - "same-size source drift with restored mtime is snapshot uncertainty", + "in-place and atomic source replacement races are snapshot uncertainty", ), Assertion( "IIN-PREFLIGHT-001", probe_public_preflight, - "public home preflight rejects unsafe and racing authority files within bounds", + "public preflight rejects unsafe and racing authority files and store roots", ), Assertion( "IIN-PLAN-001", From 65f877f47889d1df3b0075cfabfd0ea244655326 Mon Sep 17 00:00:00 2001 From: xormania <127287135+xormania@users.noreply.github.com> Date: Sun, 2 Aug 2026 03:55:04 -0400 Subject: [PATCH 067/158] test(experiment): expose durability path substitution --- tests/experiment/install-mutation-cases.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/experiment/install-mutation-cases.py b/tests/experiment/install-mutation-cases.py index 5f25cbe..b9e41ed 100755 --- a/tests/experiment/install-mutation-cases.py +++ b/tests/experiment/install-mutation-cases.py @@ -1032,9 +1032,14 @@ def compile_mutation(runtime: Path, mutation: Mutation, cache: Path) -> None: ' )\n' ' else:\n' ' Path(mutation_marker).touch()\n' + ' _fsync_directory(\n' + ' payload / "artifact",\n' + ' "Experiment committed records",\n' + ' modes=(0o500,),\n' + ' )\n' ), probe_publication_durability, - "the durability oracle detects publication before committed-directory fsync", + "the durability oracle detects committed-directory fsync path substitution", ), Mutation( "M-STORE-LAYOUT-001", From 0feef99b0e96a13fdb80754a421db16550d89042 Mon Sep 17 00:00:00 2001 From: xormania <127287135+xormania@users.noreply.github.com> Date: Sun, 2 Aug 2026 03:56:18 -0400 Subject: [PATCH 068/158] fix(experiment): bind snapshot and store authorities --- scripts/experiment.py | 41 ++++++++++++--------- scripts/experiment_store.py | 30 +++++++++++---- tests/experiment/install-integrity-cases.py | 9 +++-- 3 files changed, 51 insertions(+), 29 deletions(-) diff --git a/scripts/experiment.py b/scripts/experiment.py index 6da94be..2268c47 100644 --- a/scripts/experiment.py +++ b/scripts/experiment.py @@ -154,11 +154,30 @@ def reject_non_finite_numbers(value: object) -> None: reject_non_finite_numbers(item) -def read_manifest_once(path: str) -> bytes: +def _manifest_identity(metadata: os.stat_result) -> tuple[int, ...]: + return ( + metadata.st_dev, + metadata.st_ino, + metadata.st_mode, + metadata.st_uid, + metadata.st_nlink, + metadata.st_size, + metadata.st_mtime_ns, + metadata.st_ctime_ns, + ) + + +def read_manifest_once( + path: str, + *, + expected: os.stat_result | None = None, +) -> bytes: try: path_stat = os.lstat(path) except OSError as error: raise InfrastructureError("manifest cannot be inspected") from error + if expected is not None and _manifest_identity(path_stat) != _manifest_identity(expected): + raise InfrastructureError("manifest identity changed before read") if stat.S_ISLNK(path_stat.st_mode): raise InfrastructureError("manifest symlinks are not accepted") if not stat.S_ISREG(path_stat.st_mode): @@ -177,7 +196,7 @@ def read_manifest_once(path: str) -> bytes: opened_stat = os.fstat(descriptor) if not stat.S_ISREG(opened_stat.st_mode): raise InfrastructureError("manifest changed to a non-regular file") - if (opened_stat.st_dev, opened_stat.st_ino) != (path_stat.st_dev, path_stat.st_ino): + if _manifest_identity(opened_stat) != _manifest_identity(path_stat): raise InfrastructureError("manifest identity changed before read") chunks: list[bytes] = [] remaining = MAX_MANIFEST_BYTES + 1 @@ -199,20 +218,8 @@ def read_manifest_once(path: str) -> bytes: if len(data) > MAX_MANIFEST_BYTES: raise InvalidManifest(f"exceeds the {MAX_MANIFEST_BYTES}-byte limit") - before = ( - opened_stat.st_dev, - opened_stat.st_ino, - opened_stat.st_size, - opened_stat.st_mtime_ns, - opened_stat.st_ctime_ns, - ) - after = ( - final_stat.st_dev, - final_stat.st_ino, - final_stat.st_size, - final_stat.st_mtime_ns, - final_stat.st_ctime_ns, - ) + before = _manifest_identity(opened_stat) + after = _manifest_identity(final_stat) if before != after or len(data) != final_stat.st_size: raise InfrastructureError("manifest changed while it was read") return data @@ -241,7 +248,7 @@ def read_directory_snapshot(path: str) -> SourceSnapshot: authored_mode = stat.S_IMODE(authored_stat.st_mode) if authored_mode & 0o111 or authored_mode & 0o022: raise InvalidManifest("experiment.cue has a suspicious mode") - data = read_manifest_once(authored_path) + data = read_manifest_once(authored_path, expected=authored_stat) try: after = os.listdir(path) final_directory_stat = os.lstat(path) diff --git a/scripts/experiment_store.py b/scripts/experiment_store.py index 39d245b..486fcb7 100644 --- a/scripts/experiment_store.py +++ b/scripts/experiment_store.py @@ -91,6 +91,10 @@ class HomeAuthority(NamedTuple): config_raw: bytes receipt_raw: bytes store_device: int + store_identity: tuple[int, int] + staging_identity: tuple[int, int] + state_identity: tuple[int, int] + locks_identity: tuple[int, int] class VerifiedInstall(NamedTuple): @@ -395,9 +399,9 @@ def _load_home(home: Path) -> HomeAuthority: staging = store / ".staging" locks = state / "locks" store_metadata = _verify_directory(store) - _verify_directory(state) - _verify_directory(locks) - _verify_directory(staging, device=store_metadata.st_dev) + state_metadata = _verify_directory(state) + locks_metadata = _verify_directory(locks) + staging_metadata = _verify_directory(staging, device=store_metadata.st_dev) lock = locks / "experiments.lock" try: lock_metadata = lock.lstat() @@ -420,6 +424,10 @@ def _load_home(home: Path) -> HomeAuthority: config_raw, receipt_raw, store_metadata.st_dev, + (store_metadata.st_dev, store_metadata.st_ino), + (staging_metadata.st_dev, staging_metadata.st_ino), + (state_metadata.st_dev, state_metadata.st_ino), + (locks_metadata.st_dev, locks_metadata.st_ino), ) except StoreError: raise @@ -446,11 +454,17 @@ def _revalidate_authority(authority: HomeAuthority) -> None: ): _infra("Agent Lab configuration changed during installation") store = _verify_directory(authority.store) - _verify_directory(authority.state) - _verify_directory(authority.locks) - _verify_directory(authority.staging, device=store.st_dev) - if store.st_dev != authority.store_device: - _infra("Experiment store filesystem changed during installation") + state = _verify_directory(authority.state) + locks = _verify_directory(authority.locks) + staging = _verify_directory(authority.staging, device=store.st_dev) + if ( + store.st_dev != authority.store_device + or (store.st_dev, store.st_ino) != authority.store_identity + or (staging.st_dev, staging.st_ino) != authority.staging_identity + or (state.st_dev, state.st_ino) != authority.state_identity + or (locks.st_dev, locks.st_ino) != authority.locks_identity + ): + _infra("Experiment store directory authority changed during installation") @contextmanager diff --git a/tests/experiment/install-integrity-cases.py b/tests/experiment/install-integrity-cases.py index 21b6883..28f3467 100644 --- a/tests/experiment/install-integrity-cases.py +++ b/tests/experiment/install-integrity-cases.py @@ -783,18 +783,19 @@ def probe_public_preflight(root: Path, runtime: Path, names: tuple[str, ...]) -> store_home = support().initialized_home(runtime, root, "store-race-home") store_path = store_home / "experiments" parked_store = store_home / "experiments-parked" - store_before = support().tree_fingerprint(store_path) store = load_private_module( runtime / "scripts" / "experiment_store.py", f"agent_lab_integrity_store_race_{os.getpid()}_{id(root)}", ) store_replaced = False + parked_before: tuple[tuple[object, ...], ...] | None = None replacement_before: tuple[tuple[object, ...], ...] | None = None def replace_store_after_lock(point: str) -> None: - nonlocal store_replaced, replacement_before + nonlocal store_replaced, parked_before, replacement_before if point == "experiment store lock.after_acquire" and not store_replaced: store_path.rename(parked_store) + parked_before = support().tree_fingerprint(parked_store) store_path.mkdir(mode=0o700) staging = store_path / ".staging" staging.mkdir(mode=0o700) @@ -811,8 +812,8 @@ def replace_store_after_lock(point: str) -> None: fault=replace_store_after_lock, ) original_store_preserved = ( - store_replaced - and support().tree_fingerprint(parked_store) == store_before + parked_before is not None + and support().tree_fingerprint(parked_store) == parked_before ) replacement_store_preserved = ( replacement_before is not None From 5eb440bd64e8941eb3920c893303583af416637c Mon Sep 17 00:00:00 2001 From: xormania <127287135+xormania@users.noreply.github.com> Date: Sun, 2 Aug 2026 03:56:18 -0400 Subject: [PATCH 069/158] test(experiment): bind durability oracle paths --- tests/experiment/install-mutation-cases.py | 31 +++++++++++++--------- 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/tests/experiment/install-mutation-cases.py b/tests/experiment/install-mutation-cases.py index b9e41ed..52ada92 100755 --- a/tests/experiment/install-mutation-cases.py +++ b/tests/experiment/install-mutation-cases.py @@ -655,18 +655,18 @@ def probe_publication_durability(runtime: Path, probe_root: Path, marker: Path | data = artifact_bytes("durability") source = write_source(probe_root, "durability-source", data) store = fixture_store(runtime, probe_root, FixtureExperiment(direct_fixture(data, "durability"))) - events: list[tuple[str, str]] = [] + events: list[tuple[str, Path, str]] = [] original_rename = store._rename_noreplace original_fsync = store._fsync_directory target = home / "experiments" / "mutation-store" def observed_rename(source_path: Path, target_path: Path) -> None: if target_path == target: - events.append(("publish", str(target_path))) + events.append(("publish", target_path, "")) original_rename(source_path, target_path) def observed_fsync(path: Path, purpose: str, *, modes=(0o700,)) -> None: - events.append(("fsync", purpose)) + events.append(("fsync", path, purpose)) original_fsync(path, purpose, modes=modes) store._rename_noreplace = observed_rename @@ -678,28 +678,33 @@ def observed_fsync(path: Path, purpose: str, *, modes=(0o700,)) -> None: store._fsync_directory = original_fsync store._rename_noreplace = original_rename publication = [index for index, event in enumerate(events) if event[0] == "publish"] - required_purposes = ( - "Experiment committed artifact", - "Experiment committed records", - "Experiment staged envelope root", - "Experiment committed wrapper", - "Experiment committed staging", + wrapper = home / "experiments" / ".staging" / "experiment-install" + payload = wrapper / "payload" + required_events = ( + ("fsync", payload / "artifact", "Experiment committed artifact"), + ("fsync", payload / "records", "Experiment committed records"), + ("fsync", payload, "Experiment staged envelope root"), + ("fsync", wrapper, "Experiment committed wrapper"), + ("fsync", wrapper.parent, "Experiment committed staging"), ) durable = { - purpose: [ + (str(path), purpose): [ index for index, event in enumerate(events) - if event == ("fsync", purpose) + if event == (kind, path, purpose) ] - for purpose in required_purposes + for kind, path, purpose in required_events } + durable_order = [indices[0] for indices in durable.values() if len(indices) == 1] secure = ( rc == 0 and isinstance(value, dict) and error is None and len(publication) == 1 and all(len(indices) == 1 for indices in durable.values()) - and all(indices[0] < publication[0] for indices in durable.values()) + and len(durable_order) == len(required_events) + and durable_order == sorted(durable_order) + and all(index < publication[0] for index in durable_order) ) return ProbeResult(secure, f"rc={rc} error={error!r} publish={publication} durable={durable}") From 5fef8a5bf4f1bc3125446d9ac3f9caaaa10c35eb Mon Sep 17 00:00:00 2001 From: xormania <127287135+xormania@users.noreply.github.com> Date: Sun, 2 Aug 2026 04:00:28 -0400 Subject: [PATCH 070/158] test(experiment): expose late store replacement --- tests/experiment/install-integrity-cases.py | 51 +++++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/tests/experiment/install-integrity-cases.py b/tests/experiment/install-integrity-cases.py index 28f3467..4e05188 100644 --- a/tests/experiment/install-integrity-cases.py +++ b/tests/experiment/install-integrity-cases.py @@ -828,6 +828,54 @@ def replace_store_after_lock(point: str) -> None: and replacement_store_preserved ) + late_home = support().initialized_home(runtime, root, "late-store-race-home") + late_store = late_home / "experiments" + late_parked = late_home / "experiments-parked" + late_replaced = False + late_parked_before: tuple[tuple[object, ...], ...] | None = None + late_replacement_before: tuple[tuple[object, ...], ...] | None = None + + def replace_store_before_publish(point: str) -> None: + nonlocal late_replaced, late_parked_before, late_replacement_before + if point == "experiment envelope.before_noreplace" and not late_replaced: + late_store.rename(late_parked) + late_store.mkdir(mode=0o700) + replacement_staging = late_store / ".staging" + replacement_staging.mkdir(mode=0o700) + os.chmod(late_store, 0o700) + os.chmod(replacement_staging, 0o700) + parked_wrapper = late_parked / ".staging" / "experiment-install" + replacement_wrapper = replacement_staging / "experiment-install" + parked_wrapper.rename(replacement_wrapper) + late_parked_before = support().tree_fingerprint(late_parked) + late_replacement_before = support().tree_fingerprint(late_store) + late_replaced = True + + with tool_environment(): + late_rc, late_value, late_error = actual_install( + store, + late_home, + source, + fault=replace_store_before_publish, + ) + late_parked_preserved = ( + late_parked_before is not None + and support().tree_fingerprint(late_parked) == late_parked_before + ) + late_replacement_preserved = ( + late_replacement_before is not None + and support().tree_fingerprint(late_store) == late_replacement_before + ) + late_store_race_secure = ( + late_replaced + and late_rc == 125 + and late_value is None + and isinstance(late_error, store.StoreInfrastructure) + and late_parked_preserved + and late_replacement_preserved + and not (late_store / "mutation-store").exists() + ) + source_after = support().tree_fingerprint(source) runtime_unchanged(runtime, names, runtime_before, "public preflight probe") clean_results = all( @@ -840,6 +888,7 @@ def replace_store_after_lock(point: str) -> None: clean_results and all(state_checks.values()) and store_race_secure + and late_store_race_secure and source_before == source_after ) rendered = { @@ -852,6 +901,8 @@ def replace_store_after_lock(point: str) -> None: f"results={rendered!r} state={state_checks!r} " f"store_race={store_replaced}/{store_rc}/{store_error!r}/" f"{original_store_preserved}/{replacement_store_preserved} " + f"late_store_race={late_replaced}/{late_rc}/{late_error!r}/" + f"{late_parked_preserved}/{late_replacement_preserved} " f"source_changed={source_before != source_after}" ), ) From 8f84674a5e23649bd4792e5078f3b9d50eb59221 Mon Sep 17 00:00:00 2001 From: xormania <127287135+xormania@users.noreply.github.com> Date: Sun, 2 Aug 2026 04:00:44 -0400 Subject: [PATCH 071/158] fix(experiment): revalidate authority before publish --- scripts/experiment_store.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/experiment_store.py b/scripts/experiment_store.py index 486fcb7..0449562 100644 --- a/scripts/experiment_store.py +++ b/scripts/experiment_store.py @@ -2023,6 +2023,7 @@ def install_directory( _revalidate_authority(authority) _fault(fault, "experiment envelope.before_noreplace") experiment.verify_trusted_inputs(plan, decision) + _revalidate_authority(authority) _rename_noreplace(wrapper / "payload", authority.store / name) _fsync_directory( wrapper, From 43ee2e801d0ca2b7a3d12bfac588f80a447837f9 Mon Sep 17 00:00:00 2001 From: xormania <127287135+xormania@users.noreply.github.com> Date: Sun, 2 Aug 2026 04:16:26 -0400 Subject: [PATCH 072/158] test(experiment): require bounded lifecycle overlap --- tests/experiment/aggregate-harness-cases.sh | 56 +++++++++++++++++++-- 1 file changed, 51 insertions(+), 5 deletions(-) diff --git a/tests/experiment/aggregate-harness-cases.sh b/tests/experiment/aggregate-harness-cases.sh index f146e95..ab93d49 100755 --- a/tests/experiment/aggregate-harness-cases.sh +++ b/tests/experiment/aggregate-harness-cases.sh @@ -82,6 +82,21 @@ write_fixture() { printf 'if [ -n "${AGENT_LAB_AGG_EXEC_LOG:-}" ]; then\n' printf " printf '%%s\\n' '%s' >> \"\$AGENT_LAB_AGG_EXEC_LOG\" || exit 125\n" "$execution_id" printf 'fi\n' + case "$execution_id" in + local-install-cases.sh | local-image-catalog-cases.sh) + printf 'if [ -n "${AGENT_LAB_AGG_BARRIER_DIR:-}" ]; then\n' + printf " : > \"\$AGENT_LAB_AGG_BARRIER_DIR/%s.ready\" || exit 125\n" "$execution_id" + printf ' attempts=0\n' + printf ' while [ ! -f "$AGENT_LAB_AGG_BARRIER_DIR/local-install-cases.sh.ready" ] ||\n' + printf ' [ ! -f "$AGENT_LAB_AGG_BARRIER_DIR/local-image-catalog-cases.sh.ready" ] ||\n' + printf ' [ ! -f "$AGENT_LAB_AGG_BARRIER_DIR/install-state-cases.py.ready" ]; do\n' + printf ' attempts=$((attempts + 1))\n' + printf ' [ "$attempts" -lt 200 ] || exit 125\n' + printf ' sleep 0.01\n' + printf ' done\n' + printf 'fi\n' + ;; + esac for record in "$@"; do kind="${record%%:*}" id="${record#*:}" @@ -104,10 +119,27 @@ write_python_fixture() { { printf '#!/usr/bin/env python3\n' printf 'import os\n' + printf 'from pathlib import Path\n' + printf 'import time\n' printf 'log = os.environ.get("AGENT_LAB_AGG_EXEC_LOG")\n' printf 'if log:\n' printf ' with open(log, "a", encoding="ascii") as stream:\n' printf " stream.write('%s\\\\n')\n" "$execution_id" + if [ "$execution_id" = "install-state-cases.py" ]; then + printf 'barrier = os.environ.get("AGENT_LAB_AGG_BARRIER_DIR")\n' + printf 'if barrier:\n' + printf " Path(barrier, '%s.ready').touch()\n" "$execution_id" + printf ' peers = (\n' + printf " 'local-install-cases.sh.ready',\n" + printf " 'local-image-catalog-cases.sh.ready',\n" + printf " 'install-state-cases.py.ready',\n" + printf ' )\n' + printf ' deadline = time.monotonic() + 2.0\n' + printf ' while not all(Path(barrier, peer).is_file() for peer in peers):\n' + printf ' if time.monotonic() >= deadline:\n' + printf ' raise SystemExit(125)\n' + printf ' time.sleep(0.01)\n' + fi for record in "$@"; do kind="${record%%:*}" id="${record#*:}" @@ -204,16 +236,30 @@ printf '%s\n' \ install-integrity-cases.py \ install-mutation-cases.py > "$mutant_expected" +overlap_barrier="$work/overlap-barrier" +overlap_executions="$work/overlap-executions" +mkdir "$overlap_barrier" +: > "$overlap_executions" +overlap_rc=0 +run_replica "$work/overlap.out" env \ + AGENT_LAB_AGG_EXEC_LOG="$overlap_executions" \ + AGENT_LAB_AGG_BARRIER_DIR="$overlap_barrier" || overlap_rc=$? + if [ "$baseline_rc" -eq 0 ] && - cmp -s "$expected_executions" "$baseline_executions" && + cmp -s <(LC_ALL=C sort "$expected_executions") <(LC_ALL=C sort "$baseline_executions") && [ "$mutation_count" -eq 1 ] && [ "$mutant_rc" -eq 0 ] && cmp -s "$work/baseline.out" "$work/mutant.out" && - cmp -s "$mutant_expected" "$mutant_executions" && - ! cmp -s "$expected_executions" "$mutant_executions"; then - pass AGG-001 "independent execution ledger proves exact-once routing and detects a hidden duplicate" + cmp -s <(LC_ALL=C sort "$mutant_expected") <(LC_ALL=C sort "$mutant_executions") && + ! cmp -s <(LC_ALL=C sort "$expected_executions") <(LC_ALL=C sort "$mutant_executions") && + [ "$overlap_rc" -eq 0 ] && + cmp -s <(LC_ALL=C sort "$expected_executions") <(LC_ALL=C sort "$overlap_executions") && + [ -f "$overlap_barrier/local-install-cases.sh.ready" ] && + [ -f "$overlap_barrier/local-image-catalog-cases.sh.ready" ] && + [ -f "$overlap_barrier/install-state-cases.py.ready" ]; then + pass AGG-001 "execution ledger and overlap barrier prove exact-once concurrent routing" else - fail AGG-001 "independent execution ledger proves exact-once routing and detects a hidden duplicate" + fail AGG-001 "execution ledger and overlap barrier prove exact-once concurrent routing" fi reset_fixtures From e242206b65e7623adb32604b36afe9743a3697d0 Mon Sep 17 00:00:00 2001 From: xormania <127287135+xormania@users.noreply.github.com> Date: Sun, 2 Aug 2026 04:29:56 -0400 Subject: [PATCH 073/158] fix(experiment): bound lifecycle wall time --- tests/experiment/aggregate-harness-cases.sh | 21 +++- tests/experiment/local-lifecycle-cases.sh | 116 ++++++++++++++++---- 2 files changed, 116 insertions(+), 21 deletions(-) diff --git a/tests/experiment/aggregate-harness-cases.sh b/tests/experiment/aggregate-harness-cases.sh index ab93d49..c32367b 100755 --- a/tests/experiment/aggregate-harness-cases.sh +++ b/tests/experiment/aggregate-harness-cases.sh @@ -333,15 +333,32 @@ write_fixture "$replica/tests/install/local-install-cases.sh" 125 "${uncertain_r subcase_infra_rc=0 run_replica "$work/subcase-infra.out" env || subcase_infra_rc=$? reset_fixtures +mixed_failed_records=() +mixed_uncertain_records=() +mapfile -t mixed_failed_records < <(pass_records "${installer_ids[@]}") +mixed_failed_records[0]='FAIL:PKG-001' +mapfile -t mixed_uncertain_records < <(pass_records "${config_ids[@]}") +write_fixture "$replica/tests/install/local-install-cases.sh" 1 "${mixed_failed_records[@]}" +write_fixture "$replica/tests/experiment/local-config-cases.sh" 125 "${mixed_uncertain_records[@]}" +mixed_executions="$work/mixed-executions" +: > "$mixed_executions" +mixed_rc=0 +run_replica "$work/mixed.out" env \ + AGENT_LAB_AGG_EXEC_LOG="$mixed_executions" || mixed_rc=$? +reset_fixtures find "$replica/tests/install/local-install-cases.sh" -delete setup_infra_rc=0 run_replica "$work/setup-infra.out" env || setup_infra_rc=$? if [ "$subcase_infra_rc" -eq 125 ] && [ "$setup_infra_rc" -eq 125 ] && + [ "$mixed_rc" -eq 125 ] && + grep -Fxq 'SUMMARY assertions=133 expected=133 failures=1 infra=1' "$work/mixed.out" && + cmp -s <(LC_ALL=C sort "$expected_executions") <(LC_ALL=C sort "$mixed_executions") && ! grep -Fxq 'EXPERIMENT LOCAL LIFECYCLE PASS' "$work/subcase-infra.out" && + ! grep -Fxq 'EXPERIMENT LOCAL LIFECYCLE PASS' "$work/mixed.out" && ! grep -Fxq 'EXPERIMENT LOCAL LIFECYCLE PASS' "$work/setup-infra.out"; then - pass AGG-007 "setup and subcase uncertainty map to one hundred twenty-five" + pass AGG-007 "all lanes finish and uncertainty dominates assertion failure" else - fail AGG-007 "setup and subcase uncertainty map to one hundred twenty-five" + fail AGG-007 "all lanes finish and uncertainty dominates assertion failure" fi reset_fixtures diff --git a/tests/experiment/local-lifecycle-cases.sh b/tests/experiment/local-lifecycle-cases.sh index 6303915..67d5f20 100755 --- a/tests/experiment/local-lifecycle-cases.sh +++ b/tests/experiment/local-lifecycle-cases.sh @@ -13,6 +13,8 @@ subcases=( ) expected_count=133 work="" +lane_count=3 +lane_pids=() cleanup_work() { local failed=0 @@ -25,11 +27,36 @@ cleanup_work() { return "$failed" } +stop_lanes() { + local pid + for pid in "${lane_pids[@]}"; do + [ -n "$pid" ] || continue + kill -TERM "$pid" 2>/dev/null || true + done + for pid in "${lane_pids[@]}"; do + [ -n "$pid" ] || continue + wait "$pid" 2>/dev/null || true + done + lane_pids=() +} + +handle_signal() { + local status="$1" + trap - HUP INT QUIT TERM EXIT + stop_lanes + cleanup_work >/dev/null 2>&1 || true + exit "$status" +} + if ! work="$(mktemp -d)"; then printf 'SUMMARY assertions=0 expected=%s failures=0 infra=1\n' "$expected_count" exit 125 fi trap 'cleanup_work >/dev/null 2>&1 || true' EXIT +trap 'handle_signal 129' HUP +trap 'handle_signal 130' INT +trap 'handle_signal 131' QUIT +trap 'handle_signal 143' TERM expected="$work/expected" observed="$work/observed" @@ -67,34 +94,86 @@ printf '%s\n' \ M-STORE-LIVE-001 M-STORE-UNCERT-001 M-STORE-STAGE-001 > "$expected" : > "$observed" +run_subcase() { + local index="$1" + local subcase="${subcases[$index]}" + local output="$work/subcase-$index.out" + local status_file="$work/subcase-$index.status" + local rc + + if ! : > "$output"; then + printf '125\n' > "$status_file" 2>/dev/null || true + return 125 + fi + if [ ! -f "$subcase" ]; then + rc=125 + else + case "$subcase" in + *.py) + if python3 -I -B "$subcase" > "$output" 2>&1; then + rc=0 + else + rc=$? + fi + ;; + *) + if bash "$subcase" > "$output" 2>&1; then + rc=0 + else + rc=$? + fi + ;; + esac + fi + printf '%s\n' "$rc" > "$status_file" +} + +run_lane() { + local lane="$1" + local index + local lane_infrastructure=0 + + for index in "${!subcases[@]}"; do + [ $((index % lane_count)) -eq "$lane" ] || continue + run_subcase "$index" || lane_infrastructure=1 + done + [ "$lane_infrastructure" -eq 0 ] +} + infrastructure=0 +for ((lane = 0; lane < lane_count; lane++)); do + run_lane "$lane" & + lane_pids+=("$!") +done +for lane in "${!lane_pids[@]}"; do + pid="${lane_pids[$lane]}" + if ! wait "$pid"; then + infrastructure=1 + fi + lane_pids[lane]="" +done +lane_pids=() + +failures=0 for index in "${!subcases[@]}"; do - subcase="${subcases[$index]}" output="$work/subcase-$index.out" - if [ ! -f "$subcase" ]; then + status_file="$work/subcase-$index.status" + rc=125 + if [ ! -f "$status_file" ] || + [ "$(wc -l < "$status_file" 2>/dev/null)" -ne 1 ] || + ! IFS= read -r rc < "$status_file"; then + infrastructure=1 + rc=125 + fi + if [ ! -f "$output" ]; then infrastructure=1 continue fi - case "$subcase" in - *.py) - if python3 -I -B "$subcase" > "$output" 2>&1; then - rc=0 - else - rc=$? - fi - ;; - *) - if bash "$subcase" > "$output" 2>&1; then - rc=0 - else - rc=$? - fi - ;; - esac awk '/^(PASS|FAIL) [A-Z0-9-]+ / {print}' "$output" awk '/^(PASS|FAIL) [A-Z0-9-]+ / {print $2}' "$output" >> "$observed" reported_assertions="$(awk '/^(PASS|FAIL) [A-Z0-9-]+ / {count++} END {print count + 0}' "$output")" reported_failures="$(awk '/^FAIL [A-Z0-9-]+ / {count++} END {print count + 0}' "$output")" + failures=$((failures + reported_failures)) expected_summary="SUMMARY assertions=$reported_assertions expected=$reported_assertions failures=$reported_failures infra=0" matching_summaries="$(grep -Fxc "$expected_summary" "$output" || true)" all_summaries="$(grep -c '^SUMMARY ' "$output" || true)" @@ -119,7 +198,6 @@ for index in "${!subcases[@]}"; do done assertions="$(wc -l < "$observed")" -failures="$(awk '/^FAIL [A-Z0-9-]+ / {count++} END {print count + 0}' "$work"/subcase-*.out 2>/dev/null)" if ! cmp -s "$expected" "$observed"; then failures=$((failures + 1)) fi From 6a761d4eb76cb66ba82caf502d77cfb14a848c1e Mon Sep 17 00:00:00 2001 From: xormania <127287135+xormania@users.noreply.github.com> Date: Sun, 2 Aug 2026 04:32:18 -0400 Subject: [PATCH 074/158] test(experiment): expose lifecycle cancellation leaks --- tests/experiment/aggregate-harness-cases.sh | 116 +++++++++++++++++++- tests/experiment/contract-cases.sh | 5 +- 2 files changed, 118 insertions(+), 3 deletions(-) diff --git a/tests/experiment/aggregate-harness-cases.sh b/tests/experiment/aggregate-harness-cases.sh index c32367b..d3a18a1 100755 --- a/tests/experiment/aggregate-harness-cases.sh +++ b/tests/experiment/aggregate-harness-cases.sh @@ -3,7 +3,7 @@ set -u -o pipefail repo_root="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." >/dev/null 2>&1 && pwd)" lifecycle="$repo_root/tests/experiment/local-lifecycle-cases.sh" -expected_count=9 +expected_count=12 work="" cleanup_work() { @@ -97,6 +97,24 @@ write_fixture() { printf 'fi\n' ;; esac + if [ "$execution_id" = "local-image-catalog-cases.sh" ]; then + printf 'if [ -n "${AGENT_LAB_AGG_HOLD_DIR:-}" ]; then\n' + printf ' : > "$AGENT_LAB_AGG_HOLD_DIR/ready" || exit 125\n' + printf ' attempts=0\n' + printf ' while [ ! -f "$AGENT_LAB_AGG_HOLD_DIR/release" ]; do\n' + printf ' attempts=$((attempts + 1))\n' + printf ' [ "$attempts" -lt 500 ] || exit 125\n' + printf ' sleep 0.01\n' + printf ' done\n' + printf 'fi\n' + printf 'if [ -n "${AGENT_LAB_AGG_SIGNAL_DIR:-}" ]; then\n' + printf ' sleep 30 &\n' + printf ' descendant_pid=$!\n' + printf ' printf "%%s\\n" "$descendant_pid" > "$AGENT_LAB_AGG_SIGNAL_DIR/descendant.pid" || exit 125\n' + printf ' : > "$AGENT_LAB_AGG_SIGNAL_DIR/ready" || exit 125\n' + printf ' wait "$descendant_pid"\n' + printf 'fi\n' + fi for record in "$@"; do kind="${record%%:*}" id="${record#*:}" @@ -105,6 +123,9 @@ write_fixture() { done printf "printf 'SUMMARY assertions=%s expected=%s failures=%s infra=0\\n'\n" \ "$#" "$#" "$fixture_failures" + printf 'if [ -n "${AGENT_LAB_AGG_DONE_DIR:-}" ]; then\n' + printf " : > \"\$AGENT_LAB_AGG_DONE_DIR/%s.done\" || exit 125\n" "$execution_id" + printf 'fi\n' printf 'exit %s\n' "$rc" } > "$path" chmod +x "$path" @@ -148,6 +169,9 @@ write_python_fixture() { done printf "print('SUMMARY assertions=%s expected=%s failures=%s infra=0')\n" \ "$#" "$#" "$fixture_failures" + printf "done_dir = os.environ.get('AGENT_LAB_AGG_DONE_DIR')\n" + printf 'if done_dir:\n' + printf " Path(done_dir, '%s.done').touch()\n" "$execution_id" printf 'raise SystemExit(%s)\n' "$rc" } > "$path" chmod +x "$path" @@ -193,6 +217,26 @@ run_selected() { return $? } +wait_for_path() { + local path="$1" + local attempts=0 + while [ ! -e "$path" ]; do + attempts=$((attempts + 1)) + [ "$attempts" -lt 500 ] || return 1 + sleep 0.01 + done +} + +wait_for_process_exit() { + local pid="$1" + local attempts=0 + while kill -0 "$pid" 2>/dev/null; do + attempts=$((attempts + 1)) + [ "$attempts" -lt 500 ] || return 1 + sleep 0.01 + done +} + reset_fixtures expected_executions="$work/expected-executions" baseline_executions="$work/baseline-executions" @@ -395,6 +439,76 @@ else fail AGG-009 "missing subcase summary maps to one hundred twenty-five" fi +if [ "$(grep -Fxc 'lane_count=3' "$replica_lifecycle")" -eq 1 ]; then + pass AGG-010 "lifecycle execution is hard-bounded to three lanes" +else + fail AGG-010 "lifecycle execution is hard-bounded to three lanes" +fi + +reset_fixtures +hold_dir="$work/lane-hold" +done_dir="$work/lane-done" +wait_executions="$work/wait-executions" +mkdir "$hold_dir" "$done_dir" +: > "$wait_executions" +wait_rc=0 +run_replica "$work/wait.out" env \ + AGENT_LAB_AGG_EXEC_LOG="$wait_executions" \ + AGENT_LAB_AGG_HOLD_DIR="$hold_dir" \ + AGENT_LAB_AGG_DONE_DIR="$done_dir" & +wait_pid=$! +wait_contract=0 +if wait_for_path "$hold_dir/ready" && + wait_for_path "$done_dir/install-mutation-cases.py.done" && + wait_for_path "$done_dir/install-state-cases.py.done"; then + sleep 0.5 + if kill -0 "$wait_pid" 2>/dev/null; then + wait_contract=1 + fi +fi +: > "$hold_dir/release" +wait "$wait_pid" || wait_rc=$? +if [ "$wait_contract" -eq 1 ] && [ "$wait_rc" -eq 0 ] && + wait_for_path "$done_dir/install-integrity-cases.py.done" && + cmp -s <(LC_ALL=C sort "$expected_executions") <(LC_ALL=C sort "$wait_executions"); then + pass AGG-011 "parent waits every lane through controlled completion" +else + fail AGG-011 "parent waits every lane through controlled completion" +fi + +reset_fixtures +signal_dir="$work/signal" +mkdir "$signal_dir" +AGENT_LAB_AGG_SIGNAL_DIR="$signal_dir" \ + python3 -I -B -c \ + 'import os, sys; os.setsid(); os.execvpe("bash", ["bash", sys.argv[1]], os.environ)' \ + "$replica_lifecycle" > "$work/signal.out" 2>&1 & +signal_pid=$! +signal_rc=0 +descendant_pid="" +if wait_for_path "$signal_dir/ready" && + IFS= read -r descendant_pid < "$signal_dir/descendant.pid" && + [[ "$descendant_pid" =~ ^[0-9]+$ ]] && + kill -0 "$descendant_pid" 2>/dev/null; then + kill -TERM "$signal_pid" 2>/dev/null || true +else + signal_rc=125 + kill -TERM "$signal_pid" 2>/dev/null || true +fi +observed_signal_rc=0 +wait "$signal_pid" || observed_signal_rc=$? +descendant_gone=0 +if [ -n "$descendant_pid" ] && wait_for_process_exit "$descendant_pid"; then + descendant_gone=1 +fi +kill -KILL -- "-$signal_pid" 2>/dev/null || true +if [ "$signal_rc" -eq 0 ] && [ "$observed_signal_rc" -eq 143 ] && + [ "$descendant_gone" -eq 1 ]; then + pass AGG-012 "termination reaps active descendants before lifecycle exit" +else + fail AGG-012 "termination reaps active descendants before lifecycle exit" +fi + cleanup_infrastructure=0 if ! cleanup_work; then cleanup_infrastructure=1 diff --git a/tests/experiment/contract-cases.sh b/tests/experiment/contract-cases.sh index 73d8b1d..0f234a3 100755 --- a/tests/experiment/contract-cases.sh +++ b/tests/experiment/contract-cases.sh @@ -6,7 +6,7 @@ subcases=( "$repo_root/tests/experiment/directory-intake-cases.sh" "$repo_root/tests/experiment/aggregate-harness-cases.sh" ) -expected_count=22 +expected_count=25 work="" cleanup_work() { @@ -31,7 +31,8 @@ observed="$work/observed" printf '%s\n' \ FMT-001 FMT-002 FMT-004 FMT-005 FMT-006 FMT-007 FMT-003 FMT-008 \ SEL-001 CUE-001 FMT-009 M-FMT-001 SEL-002 \ - AGG-001 AGG-002 AGG-003 AGG-004 AGG-005 AGG-006 AGG-007 AGG-008 AGG-009 > "$expected" + AGG-001 AGG-002 AGG-003 AGG-004 AGG-005 AGG-006 AGG-007 AGG-008 AGG-009 \ + AGG-010 AGG-011 AGG-012 > "$expected" : > "$observed" infrastructure=0 From 5506c4441bb8d8a4dc461a0bb076e7d335c07712 Mon Sep 17 00:00:00 2001 From: xormania <127287135+xormania@users.noreply.github.com> Date: Sun, 2 Aug 2026 04:37:27 -0400 Subject: [PATCH 075/158] test(experiment): preserve signal-owned evidence --- tests/experiment/aggregate-harness-cases.sh | 72 +++++++++++++++++++-- tests/experiment/contract-cases.sh | 4 +- 2 files changed, 70 insertions(+), 6 deletions(-) diff --git a/tests/experiment/aggregate-harness-cases.sh b/tests/experiment/aggregate-harness-cases.sh index d3a18a1..0eea4da 100755 --- a/tests/experiment/aggregate-harness-cases.sh +++ b/tests/experiment/aggregate-harness-cases.sh @@ -3,7 +3,7 @@ set -u -o pipefail repo_root="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." >/dev/null 2>&1 && pwd)" lifecycle="$repo_root/tests/experiment/local-lifecycle-cases.sh" -expected_count=12 +expected_count=13 work="" cleanup_work() { @@ -114,6 +114,16 @@ write_fixture() { printf ' : > "$AGENT_LAB_AGG_SIGNAL_DIR/ready" || exit 125\n' printf ' wait "$descendant_pid"\n' printf 'fi\n' + printf 'if [ -n "${AGENT_LAB_AGG_STUBBORN_SIGNAL_DIR:-}" ]; then\n' + printf ' (\n' + printf " trap '' HUP INT QUIT TERM\n" + printf ' printf "%%s\\n" "$BASHPID" > "$AGENT_LAB_AGG_STUBBORN_SIGNAL_DIR/descendant.pid" || exit 125\n' + printf ' : > "$AGENT_LAB_AGG_STUBBORN_SIGNAL_DIR/ready" || exit 125\n' + printf ' sleep 30\n' + printf ' ) &\n' + printf ' descendant_pid=$!\n' + printf ' wait "$descendant_pid"\n' + printf 'fi\n' fi for record in "$@"; do kind="${record%%:*}" @@ -232,7 +242,7 @@ wait_for_process_exit() { local attempts=0 while kill -0 "$pid" 2>/dev/null; do attempts=$((attempts + 1)) - [ "$attempts" -lt 500 ] || return 1 + [ "$attempts" -lt 100 ] || return 1 sleep 0.01 done } @@ -478,8 +488,10 @@ fi reset_fixtures signal_dir="$work/signal" -mkdir "$signal_dir" +signal_tmp="$work/signal-tmp" +mkdir "$signal_dir" "$signal_tmp" AGENT_LAB_AGG_SIGNAL_DIR="$signal_dir" \ + TMPDIR="$signal_tmp" \ python3 -I -B -c \ 'import os, sys; os.setsid(); os.execvpe("bash", ["bash", sys.argv[1]], os.environ)' \ "$replica_lifecycle" > "$work/signal.out" 2>&1 & @@ -501,7 +513,14 @@ descendant_gone=0 if [ -n "$descendant_pid" ] && wait_for_process_exit "$descendant_pid"; then descendant_gone=1 fi -kill -KILL -- "-$signal_pid" 2>/dev/null || true +if [ "$descendant_gone" -ne 1 ]; then + surviving_group="$(ps -o pgid= -p "$descendant_pid" 2>/dev/null || true)" + surviving_group="${surviving_group//[[:space:]]/}" + if [ "$surviving_group" = "$signal_pid" ]; then + kill -KILL -- "-$signal_pid" 2>/dev/null || true + wait_for_process_exit "$descendant_pid" || true + fi +fi if [ "$signal_rc" -eq 0 ] && [ "$observed_signal_rc" -eq 143 ] && [ "$descendant_gone" -eq 1 ]; then pass AGG-012 "termination reaps active descendants before lifecycle exit" @@ -509,6 +528,51 @@ else fail AGG-012 "termination reaps active descendants before lifecycle exit" fi +reset_fixtures +stubborn_dir="$work/stubborn-signal" +stubborn_tmp="$work/stubborn-tmp" +mkdir "$stubborn_dir" "$stubborn_tmp" +AGENT_LAB_AGG_STUBBORN_SIGNAL_DIR="$stubborn_dir" \ + TMPDIR="$stubborn_tmp" \ + python3 -I -B -c \ + 'import os, sys; os.setsid(); os.execvpe("bash", ["bash", sys.argv[1]], os.environ)' \ + "$replica_lifecycle" > "$work/stubborn-signal.out" 2>&1 & +stubborn_leader=$! +stubborn_rc=0 +stubborn_pid="" +if wait_for_path "$stubborn_dir/ready" && + IFS= read -r stubborn_pid < "$stubborn_dir/descendant.pid" && + [[ "$stubborn_pid" =~ ^[0-9]+$ ]] && + kill -0 "$stubborn_pid" 2>/dev/null; then + kill -TERM "$stubborn_leader" 2>/dev/null || true +else + stubborn_rc=125 + kill -TERM "$stubborn_leader" 2>/dev/null || true +fi +observed_stubborn_rc=0 +wait "$stubborn_leader" || observed_stubborn_rc=$? +stubborn_alive=0 +stubborn_output_preserved=0 +if [ -n "$stubborn_pid" ] && kill -0 "$stubborn_pid" 2>/dev/null; then + stubborn_alive=1 + stubborn_output="$(readlink "/proc/$stubborn_pid/fd/1" 2>/dev/null || true)" + if [ -n "$stubborn_output" ] && [ -e "$stubborn_output" ]; then + stubborn_output_preserved=1 + fi +fi +stubborn_group="$(ps -o pgid= -p "$stubborn_pid" 2>/dev/null || true)" +stubborn_group="${stubborn_group//[[:space:]]/}" +if [ "$stubborn_group" = "$stubborn_leader" ]; then + kill -KILL -- "-$stubborn_leader" 2>/dev/null || true + wait_for_process_exit "$stubborn_pid" || true +fi +if [ "$stubborn_rc" -eq 0 ] && [ "$observed_stubborn_rc" -eq 143 ] && + [ "$stubborn_alive" -eq 1 ] && [ "$stubborn_output_preserved" -eq 1 ]; then + pass AGG-013 "signal exit preserves work owned by an uncooperative descendant" +else + fail AGG-013 "signal exit preserves work owned by an uncooperative descendant" +fi + cleanup_infrastructure=0 if ! cleanup_work; then cleanup_infrastructure=1 diff --git a/tests/experiment/contract-cases.sh b/tests/experiment/contract-cases.sh index 0f234a3..47524aa 100755 --- a/tests/experiment/contract-cases.sh +++ b/tests/experiment/contract-cases.sh @@ -6,7 +6,7 @@ subcases=( "$repo_root/tests/experiment/directory-intake-cases.sh" "$repo_root/tests/experiment/aggregate-harness-cases.sh" ) -expected_count=25 +expected_count=26 work="" cleanup_work() { @@ -32,7 +32,7 @@ printf '%s\n' \ FMT-001 FMT-002 FMT-004 FMT-005 FMT-006 FMT-007 FMT-003 FMT-008 \ SEL-001 CUE-001 FMT-009 M-FMT-001 SEL-002 \ AGG-001 AGG-002 AGG-003 AGG-004 AGG-005 AGG-006 AGG-007 AGG-008 AGG-009 \ - AGG-010 AGG-011 AGG-012 > "$expected" + AGG-010 AGG-011 AGG-012 AGG-013 > "$expected" : > "$observed" infrastructure=0 From 1d2b43bc5898afda0f84c84bbb4b55b4e99d8fe7 Mon Sep 17 00:00:00 2001 From: xormania <127287135+xormania@users.noreply.github.com> Date: Sun, 2 Aug 2026 04:49:48 -0400 Subject: [PATCH 076/158] fix(experiment): harden lifecycle supervision --- tests/experiment/aggregate-harness-cases.sh | 144 ++++++++++++++------ tests/experiment/local-lifecycle-cases.sh | 52 ++++--- 2 files changed, 139 insertions(+), 57 deletions(-) diff --git a/tests/experiment/aggregate-harness-cases.sh b/tests/experiment/aggregate-harness-cases.sh index 0eea4da..1fecc50 100755 --- a/tests/experiment/aggregate-harness-cases.sh +++ b/tests/experiment/aggregate-harness-cases.sh @@ -97,16 +97,17 @@ write_fixture() { printf 'fi\n' ;; esac + printf 'if [ -n "${AGENT_LAB_AGG_HOLD_DIR:-}" ] &&\n' + printf " [ \"\${AGENT_LAB_AGG_HOLD_ID:-}\" = '%s' ]; then\n" "$execution_id" + printf ' : > "$AGENT_LAB_AGG_HOLD_DIR/ready" || exit 125\n' + printf ' attempts=0\n' + printf ' while [ ! -f "$AGENT_LAB_AGG_HOLD_DIR/release" ]; do\n' + printf ' attempts=$((attempts + 1))\n' + printf ' [ "$attempts" -lt 500 ] || exit 125\n' + printf ' sleep 0.01\n' + printf ' done\n' + printf 'fi\n' if [ "$execution_id" = "local-image-catalog-cases.sh" ]; then - printf 'if [ -n "${AGENT_LAB_AGG_HOLD_DIR:-}" ]; then\n' - printf ' : > "$AGENT_LAB_AGG_HOLD_DIR/ready" || exit 125\n' - printf ' attempts=0\n' - printf ' while [ ! -f "$AGENT_LAB_AGG_HOLD_DIR/release" ]; do\n' - printf ' attempts=$((attempts + 1))\n' - printf ' [ "$attempts" -lt 500 ] || exit 125\n' - printf ' sleep 0.01\n' - printf ' done\n' - printf 'fi\n' printf 'if [ -n "${AGENT_LAB_AGG_SIGNAL_DIR:-}" ]; then\n' printf ' sleep 30 &\n' printf ' descendant_pid=$!\n' @@ -156,6 +157,15 @@ write_python_fixture() { printf 'if log:\n' printf ' with open(log, "a", encoding="ascii") as stream:\n' printf " stream.write('%s\\\\n')\n" "$execution_id" + printf "hold_dir = os.environ.get('AGENT_LAB_AGG_HOLD_DIR')\n" + printf "hold_id = os.environ.get('AGENT_LAB_AGG_HOLD_ID')\n" + printf "if hold_dir and hold_id == '%s':\n" "$execution_id" + printf " Path(hold_dir, 'ready').touch()\n" + printf ' deadline = time.monotonic() + 5.0\n' + printf " while not Path(hold_dir, 'release').is_file():\n" + printf ' if time.monotonic() >= deadline:\n' + printf ' raise SystemExit(125)\n' + printf ' time.sleep(0.01)\n' if [ "$execution_id" = "install-state-cases.py" ]; then printf 'barrier = os.environ.get("AGENT_LAB_AGG_BARRIER_DIR")\n' printf 'if barrier:\n' @@ -247,6 +257,28 @@ wait_for_process_exit() { done } +wait_for_child_wait() { + local pid="$1" + local expected_children="$2" + local attempts=0 + local wchan child_list + local children=() + + while [ "$attempts" -lt 100 ]; do + wchan="$(cat "/proc/$pid/wchan" 2>/dev/null || true)" + child_list="$(cat "/proc/$pid/task/$pid/children" 2>/dev/null || true)" + children=() + read -r -a children <<< "$child_list" + if [ "$wchan" = "do_wait" ] && + [ "${#children[@]}" -eq "$expected_children" ]; then + return 0 + fi + attempts=$((attempts + 1)) + sleep 0.01 + done + return 1 +} + reset_fixtures expected_executions="$work/expected-executions" baseline_executions="$work/baseline-executions" @@ -449,38 +481,72 @@ else fail AGG-009 "missing subcase summary maps to one hundred twenty-five" fi -if [ "$(grep -Fxc 'lane_count=3' "$replica_lifecycle")" -eq 1 ]; then - pass AGG-010 "lifecycle execution is hard-bounded to three lanes" +lane_assignment_count="$(grep -Ec '^[[:space:]]*(readonly[[:space:]]+)?lane_count=' \ + "$replica_lifecycle" || true)" +if [ "$lane_assignment_count" -eq 1 ] && + [ "$(grep -Fxc 'readonly lane_count=3' "$replica_lifecycle")" -eq 1 ] && + [ "$(grep -Fxc 'for ((lane = 0; lane < lane_count; lane++)); do' \ + "$replica_lifecycle")" -eq 1 ] && + [ "$(grep -Fxc ' run_lane "$lane" &' "$replica_lifecycle")" -eq 1 ]; then + pass AGG-010 "lifecycle declares one immutable three-lane bound" else - fail AGG-010 "lifecycle execution is hard-bounded to three lanes" + fail AGG-010 "lifecycle declares one immutable three-lane bound" fi -reset_fixtures -hold_dir="$work/lane-hold" -done_dir="$work/lane-done" -wait_executions="$work/wait-executions" -mkdir "$hold_dir" "$done_dir" -: > "$wait_executions" -wait_rc=0 -run_replica "$work/wait.out" env \ - AGENT_LAB_AGG_EXEC_LOG="$wait_executions" \ - AGENT_LAB_AGG_HOLD_DIR="$hold_dir" \ - AGENT_LAB_AGG_DONE_DIR="$done_dir" & -wait_pid=$! -wait_contract=0 -if wait_for_path "$hold_dir/ready" && - wait_for_path "$done_dir/install-mutation-cases.py.done" && - wait_for_path "$done_dir/install-state-cases.py.done"; then - sleep 0.5 - if kill -0 "$wait_pid" 2>/dev/null; then - wait_contract=1 +hold_ids=( + local-install-cases.sh + local-config-cases.sh + local-image-catalog-cases.sh +) +peer_done_one=( + install-state-cases.py.done + install-mutation-cases.py.done + install-mutation-cases.py.done +) +peer_done_two=( + install-integrity-cases.py.done + install-integrity-cases.py.done + install-state-cases.py.done +) +target_done=( + install-mutation-cases.py.done + install-state-cases.py.done + install-integrity-cases.py.done +) +expected_wait_children=(1 1 1) +wait_contract=1 +for hold_index in "${!hold_ids[@]}"; do + reset_fixtures + hold_dir="$work/lane-hold-$hold_index" + done_dir="$work/lane-done-$hold_index" + wait_executions="$work/wait-executions-$hold_index" + mkdir "$hold_dir" "$done_dir" + : > "$wait_executions" + wait_rc=0 + env \ + AGENT_LAB_AGG_EXEC_LOG="$wait_executions" \ + AGENT_LAB_AGG_HOLD_DIR="$hold_dir" \ + AGENT_LAB_AGG_HOLD_ID="${hold_ids[$hold_index]}" \ + AGENT_LAB_AGG_DONE_DIR="$done_dir" \ + bash "$replica_lifecycle" > "$work/wait-$hold_index.out" 2>&1 & + wait_pid=$! + wait_case_contract=0 + if wait_for_path "$hold_dir/ready" && + wait_for_path "$done_dir/${peer_done_one[$hold_index]}" && + wait_for_path "$done_dir/${peer_done_two[$hold_index]}" && + wait_for_child_wait "$wait_pid" "${expected_wait_children[$hold_index]}"; then + wait_case_contract=1 fi -fi -: > "$hold_dir/release" -wait "$wait_pid" || wait_rc=$? -if [ "$wait_contract" -eq 1 ] && [ "$wait_rc" -eq 0 ] && - wait_for_path "$done_dir/install-integrity-cases.py.done" && - cmp -s <(LC_ALL=C sort "$expected_executions") <(LC_ALL=C sort "$wait_executions"); then + : > "$hold_dir/release" + wait "$wait_pid" || wait_rc=$? + if [ "$wait_case_contract" -ne 1 ] || [ "$wait_rc" -ne 0 ] || + ! wait_for_path "$done_dir/${target_done[$hold_index]}" || + ! cmp -s <(LC_ALL=C sort "$expected_executions") \ + <(LC_ALL=C sort "$wait_executions"); then + wait_contract=0 + fi +done +if [ "$wait_contract" -eq 1 ]; then pass AGG-011 "parent waits every lane through controlled completion" else fail AGG-011 "parent waits every lane through controlled completion" @@ -523,9 +589,9 @@ if [ "$descendant_gone" -ne 1 ]; then fi if [ "$signal_rc" -eq 0 ] && [ "$observed_signal_rc" -eq 143 ] && [ "$descendant_gone" -eq 1 ]; then - pass AGG-012 "termination reaps active descendants before lifecycle exit" + pass AGG-012 "owned-session termination reaches cooperative descendants" else - fail AGG-012 "termination reaps active descendants before lifecycle exit" + fail AGG-012 "owned-session termination reaches cooperative descendants" fi reset_fixtures diff --git a/tests/experiment/local-lifecycle-cases.sh b/tests/experiment/local-lifecycle-cases.sh index 67d5f20..dd3dd3a 100755 --- a/tests/experiment/local-lifecycle-cases.sh +++ b/tests/experiment/local-lifecycle-cases.sh @@ -13,8 +13,13 @@ subcases=( ) expected_count=133 work="" -lane_count=3 +readonly lane_count=3 lane_pids=() +lifecycle_pid="$$" +lifecycle_pgid="" +lifecycle_sid="" +lifecycle_identity="$(ps -o pgid=,sid= -p "$lifecycle_pid" 2>/dev/null || true)" +read -r lifecycle_pgid lifecycle_sid <<< "$lifecycle_identity" cleanup_work() { local failed=0 @@ -27,24 +32,40 @@ cleanup_work() { return "$failed" } -stop_lanes() { - local pid - for pid in "${lane_pids[@]}"; do +wait_lanes() { + local lane pid + local wait_infrastructure=0 + for lane in "${!lane_pids[@]}"; do + pid="${lane_pids[$lane]}" [ -n "$pid" ] || continue - kill -TERM "$pid" 2>/dev/null || true + lane_pids[lane]="" + wait "$pid" 2>/dev/null || wait_infrastructure=1 done + lane_pids=() + [ "$wait_infrastructure" -eq 0 ] +} + +signal_lanes() { + local pid for pid in "${lane_pids[@]}"; do [ -n "$pid" ] || continue - wait "$pid" 2>/dev/null || true + kill -TERM "$pid" 2>/dev/null || true done - lane_pids=() } handle_signal() { local status="$1" - trap - HUP INT QUIT TERM EXIT - stop_lanes - cleanup_work >/dev/null 2>&1 || true + + trap '' HUP INT QUIT TERM + trap - EXIT + # A dedicated session can cancel its group without signaling an unrelated caller. + if [ "$lifecycle_pgid" = "$lifecycle_pid" ] && + [ "$lifecycle_sid" = "$lifecycle_pid" ]; then + kill -TERM -- "-$lifecycle_pgid" 2>/dev/null || true + else + signal_lanes + fi + # Descendants may ignore TERM; signal exits deliberately preserve the private work tree. exit "$status" } @@ -145,14 +166,9 @@ for ((lane = 0; lane < lane_count; lane++)); do run_lane "$lane" & lane_pids+=("$!") done -for lane in "${!lane_pids[@]}"; do - pid="${lane_pids[$lane]}" - if ! wait "$pid"; then - infrastructure=1 - fi - lane_pids[lane]="" -done -lane_pids=() +if ! wait_lanes; then + infrastructure=1 +fi failures=0 for index in "${!subcases[@]}"; do From f7ca7311a7631536dda69f19831bc4ea0a6508b5 Mon Sep 17 00:00:00 2001 From: xormania <127287135+xormania@users.noreply.github.com> Date: Sun, 2 Aug 2026 05:12:36 -0400 Subject: [PATCH 077/158] test(experiment): require bounded catalog overlap --- .../catalog-aggregate-harness-cases.sh | 299 ++++++++++++++++++ tests/experiment/contract-cases.sh | 5 +- 2 files changed, 302 insertions(+), 2 deletions(-) create mode 100755 tests/experiment/catalog-aggregate-harness-cases.sh diff --git a/tests/experiment/catalog-aggregate-harness-cases.sh b/tests/experiment/catalog-aggregate-harness-cases.sh new file mode 100755 index 0000000..e51f83e --- /dev/null +++ b/tests/experiment/catalog-aggregate-harness-cases.sh @@ -0,0 +1,299 @@ +#!/usr/bin/env bash +set -u -o pipefail + +repo_root="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." >/dev/null 2>&1 && pwd)" +catalog_aggregate="$repo_root/tests/experiment/local-image-catalog-cases.sh" +expected_count=4 +work="" + +cleanup_work() { + local failed=0 + if [ -n "$work" ] && [ -e "$work" ]; then + find "$work" -type f -delete 2>/dev/null || failed=1 + find "$work" -type l -delete 2>/dev/null || failed=1 + find "$work" -depth -type d -exec rmdir {} + 2>/dev/null || failed=1 + [ ! -e "$work" ] || failed=1 + fi + return "$failed" +} + +if ! work="$(mktemp -d)"; then + printf 'SUMMARY assertions=0 expected=%s failures=0 infra=1\n' "$expected_count" + exit 125 +fi +trap 'cleanup_work >/dev/null 2>&1 || true' EXIT + +replica="$work/repo" +replica_aggregate="$replica/tests/experiment/local-image-catalog-cases.sh" +mkdir -p "$replica/tests/experiment" "$replica/tests/image" +cp "$catalog_aggregate" "$replica_aggregate" +chmod +x "$replica_aggregate" + +failures=0 +pass() { printf 'PASS %s %s\n' "$1" "$2"; } +fail() { printf 'FAIL %s %s\n' "$1" "$2"; failures=$((failures + 1)); } + +catalog_ids=( + CAT-NAME-001 CAT-NAME-002 CAT-OCI-001 CAT-OCI-002 + CAT-ADD-001 CAT-ADD-002 CAT-ADD-003 CAT-ADD-004 CAT-NS-001 + CAT-CAS-001 CAT-CAS-002 CAT-CAS-003 CAT-CAS-004 + CAT-READ-001 CAT-READ-002 CAT-NOEF-001 CAT-CONC-001 CAT-CONC-002 + CAT-STATE-001 CAT-STATE-002 CAT-STATE-003 CAT-STATE-004 + CAT-STATE-005 CAT-STATE-006 CAT-STATE-007 CAT-STATE-008 + CAT-STATE-009 CAT-STATE-010 CAT-STATE-011 CAT-STATE-012 + CAT-STATE-013 CAT-STATE-014 CAT-STATE-015 CAT-STATE-016 CAT-STATE-018 CAT-STATE-017 + CAT-BOUND-001 CAT-BOUND-002 CAT-BOUND-003 CAT-BOUND-004 + CAT-CRASH-001 CAT-CRASH-002 CAT-CRASH-003 CAT-CRASH-004 CAT-CRASH-005 + CAT-CRASH-006 CAT-CRASH-007 CAT-CRASH-008 CAT-CRASH-009 CAT-CRASH-010 + CAT-CRASH-011 CAT-PLAT-001 + RES-ENTRY-001 RES-SNAP-001 RES-SNAP-002 RES-ENTRY-002 RES-ENTRY-003 + RES-ISOLATE-001 RES-STATE-001 RES-STATE-002 RES-STATE-003 RES-INPUT-001 + RES-SNAP-003 RES-AUTH-001 RES-INSTALL-001 RES-NOEF-001 + M-CAT-OCI-001 M-CAT-SHADOW-001 M-CAT-CAS-001 M-CAT-AUTH-001 M-RES-BIND-001 + M-CAT-NOEF-001 M-CAT-ADMIT-001 M-CAT-ATOM-001 M-CAT-DUR-001 M-CAT-STAGE-001 +) +base_ids=("${catalog_ids[@]:0:18}") +state_ids=("${catalog_ids[@]:18:34}") +resolution_ids=("${catalog_ids[@]:52:14}") +mutation_ids=("${catalog_ids[@]:66:10}") + +write_fixture() { + local path="$1" + local rc="$2" + local role="$3" + shift 3 + local id record_failures=0 + local execution_id="${path##*/}" + { + printf '#!/usr/bin/env bash\nset -u\n' + printf 'control="${AGENT_LAB_CATALOG_AGG_CONTROL:?}"\n' + printf "printf '%%s\\n' '%s' >> \"\$control/executions\" || exit 125\n" "$execution_id" + if [ -n "$role" ]; then + printf "touch \"\$control/%s.ready\" || exit 125\n" "$role" + printf 'attempts=0\n' + printf "while [ ! -f \"\$control/%s.ready\" ]; do\n" \ + "$([ "$role" = state ] && printf peer || printf state)" + printf ' attempts=$((attempts + 1))\n' + printf ' [ "$attempts" -lt 300 ] || exit 125\n' + printf ' sleep 0.01\n' + printf 'done\n' + fi + printf 'if [ "${AGENT_LAB_CATALOG_AGG_HOLD:-}" = "%s" ]; then\n' "$execution_id" + printf ' touch "$control/hold.ready" || exit 125\n' + printf ' attempts=0\n' + printf ' while [ ! -f "$control/hold.release" ]; do\n' + printf ' attempts=$((attempts + 1))\n' + printf ' [ "$attempts" -lt 800 ] || exit 125\n' + printf ' sleep 0.01\n' + printf ' done\n' + printf 'fi\n' + for id in "$@"; do + if [ "$id" = "FAIL:CAT-NAME-001" ]; then + printf "printf 'FAIL CAT-NAME-001 fixture assertion\\n'\n" + record_failures=$((record_failures + 1)) + else + printf "printf 'PASS %s fixture assertion\\n'\n" "$id" + fi + done + printf "printf 'SUMMARY assertions=%s expected=%s failures=%s infra=0\\n'\n" \ + "$#" "$#" "$record_failures" + printf "touch \"\$control/%s.done\" || exit 125\n" "$execution_id" + printf 'exit %s\n' "$rc" + } > "$path" + chmod +x "$path" +} + +write_python_fixture() { + local path="$1" + local rc="$2" + local role="$3" + shift 3 + local id + local execution_id="${path##*/}" + { + printf '#!/usr/bin/env python3\n' + printf 'import os\n' + printf 'from pathlib import Path\n' + printf 'import time\n' + printf 'control = Path(os.environ["AGENT_LAB_CATALOG_AGG_CONTROL"])\n' + printf 'with (control / "executions").open("a", encoding="ascii") as stream:\n' + printf ' stream.write("%s\\n")\n' "$execution_id" + if [ "$role" = state ]; then + printf '(control / "state.ready").touch()\n' + printf 'deadline = time.monotonic() + 3.0\n' + printf 'while not (control / "peer.ready").is_file():\n' + printf ' if time.monotonic() >= deadline:\n' + printf ' raise SystemExit(125)\n' + printf ' time.sleep(0.01)\n' + fi + printf 'if os.environ.get("AGENT_LAB_CATALOG_AGG_HOLD") == "%s":\n' "$execution_id" + printf ' (control / "hold.ready").touch()\n' + printf ' deadline = time.monotonic() + 8.0\n' + printf ' while not (control / "hold.release").is_file():\n' + printf ' if time.monotonic() >= deadline:\n' + printf ' raise SystemExit(125)\n' + printf ' time.sleep(0.01)\n' + for id in "$@"; do + printf "print('PASS %s fixture assertion')\n" "$id" + done + printf "print('SUMMARY assertions=%s expected=%s failures=0 infra=0')\n" "$#" "$#" + printf '(control / "%s.done").touch()\n' "$execution_id" + printf 'raise SystemExit(%s)\n' "$rc" + } > "$path" + chmod +x "$path" +} + +reset_fixtures() { + local base_rc="${1:-0}" + local state_rc="${2:-0}" + local first_record="${3:-CAT-NAME-001}" + local selected_base_ids=("${base_ids[@]}") + selected_base_ids[0]="$first_record" + write_fixture "$replica/tests/image/catalog-cases.sh" "$base_rc" peer \ + "${selected_base_ids[@]}" + write_python_fixture "$replica/tests/image/catalog-state-cases.py" "$state_rc" state \ + "${state_ids[@]}" + write_fixture "$replica/tests/experiment/catalog-resolution-cases.sh" 0 "" \ + "${resolution_ids[@]}" + write_python_fixture "$replica/tests/image/catalog-mutation-cases.py" 0 "" \ + "${mutation_ids[@]}" +} + +expected_executions="$work/expected-executions" +printf '%s\n' \ + catalog-cases.sh \ + catalog-state-cases.py \ + catalog-resolution-cases.sh \ + catalog-mutation-cases.py > "$expected_executions" + +expected_success="$work/expected-success" +for id in "${catalog_ids[@]}"; do + printf 'PASS %s fixture assertion\n' "$id" +done > "$expected_success" +printf 'SUMMARY assertions=76 expected=76 failures=0 infra=0\n' >> "$expected_success" +printf 'EXPERIMENT LOCAL IMAGE CATALOG PASS\n' >> "$expected_success" + +run_aggregate() { + local output="$1" + local control="$2" + shift 2 + : > "$control/executions" + env AGENT_LAB_CATALOG_AGG_CONTROL="$control" "$@" \ + bash "$replica_aggregate" > "$output" 2>&1 +} + +wait_for_path() { + local path="$1" + local attempts=0 + while [ ! -e "$path" ]; do + attempts=$((attempts + 1)) + [ "$attempts" -lt 500 ] || return 1 + sleep 0.01 + done +} + +wait_for_child_wait() { + local pid="$1" + local attempts=0 + local wchan child_list + local children=() + while [ "$attempts" -lt 200 ]; do + wchan="$(cat "/proc/$pid/wchan" 2>/dev/null || true)" + child_list="$(cat "/proc/$pid/task/$pid/children" 2>/dev/null || true)" + children=() + read -r -a children <<< "$child_list" + if [ "$wchan" = do_wait ] && [ "${#children[@]}" -eq 1 ]; then + return 0 + fi + attempts=$((attempts + 1)) + sleep 0.01 + done + return 1 +} + +reset_fixtures +baseline_control="$work/baseline-control" +mkdir "$baseline_control" +baseline_rc=0 +run_aggregate "$work/baseline.out" "$baseline_control" || baseline_rc=$? +if [ "$baseline_rc" -eq 0 ] && + cmp -s "$expected_success" "$work/baseline.out" && + cmp -s <(LC_ALL=C sort "$expected_executions") \ + <(LC_ALL=C sort "$baseline_control/executions") && + [ -f "$baseline_control/peer.ready" ] && + [ -f "$baseline_control/state.ready" ]; then + pass AGG-014 "catalog subcases overlap with exact-once ordered replay" +else + fail AGG-014 "catalog subcases overlap with exact-once ordered replay" +fi + +lane_assignment_count="$(grep -Ec '^[[:space:]]*(readonly[[:space:]]+)?lane_count=' \ + "$replica_aggregate" || true)" +if [ "$lane_assignment_count" -eq 1 ] && + [ "$(grep -Fxc 'readonly lane_count=2' "$replica_aggregate")" -eq 1 ] && + [ "$(grep -Fxc 'for ((lane = 0; lane < lane_count; lane++)); do' \ + "$replica_aggregate")" -eq 1 ] && + [ "$(grep -Fxc ' run_lane "$lane" &' "$replica_aggregate")" -eq 1 ]; then + pass AGG-015 "catalog scheduler declares one immutable two-lane bound" +else + fail AGG-015 "catalog scheduler declares one immutable two-lane bound" +fi + +reset_fixtures 1 125 FAIL:CAT-NAME-001 +mixed_control="$work/mixed-control" +mkdir "$mixed_control" +mixed_rc=0 +run_aggregate "$work/mixed.out" "$mixed_control" || mixed_rc=$? +mixed_assertions="$(grep -Ec '^(PASS|FAIL) [A-Z0-9-]+ ' "$work/mixed.out" || true)" +if [ "$mixed_rc" -eq 125 ] && + [ "$mixed_assertions" -eq 76 ] && + grep -Fxq 'SUMMARY assertions=76 expected=76 failures=1 infra=1' "$work/mixed.out" && + ! grep -Fxq 'EXPERIMENT LOCAL IMAGE CATALOG PASS' "$work/mixed.out" && + cmp -s <(LC_ALL=C sort "$expected_executions") \ + <(LC_ALL=C sort "$mixed_control/executions") && + [ "$(find "$mixed_control" -name '*.done' -type f | wc -l)" -eq 4 ]; then + pass AGG-016 "catalog uncertainty dominates after every subcase completes" +else + fail AGG-016 "catalog uncertainty dominates after every subcase completes" +fi + +reset_fixtures +wait_control="$work/wait-control" +mkdir "$wait_control" +: > "$wait_control/executions" +AGENT_LAB_CATALOG_AGG_CONTROL="$wait_control" \ +AGENT_LAB_CATALOG_AGG_HOLD=catalog-state-cases.py \ + bash "$replica_aggregate" > "$work/wait.out" 2>&1 & +wait_pid=$! +wait_contract=0 +if wait_for_path "$wait_control/hold.ready" && + wait_for_path "$wait_control/catalog-cases.sh.done" && + wait_for_path "$wait_control/catalog-resolution-cases.sh.done" && + wait_for_path "$wait_control/catalog-mutation-cases.py.done" && + wait_for_child_wait "$wait_pid"; then + wait_contract=1 +fi +touch "$wait_control/hold.release" +wait_rc=0 +wait "$wait_pid" || wait_rc=$? +if [ "$wait_contract" -eq 1 ] && [ "$wait_rc" -eq 0 ] && + cmp -s "$expected_success" "$work/wait.out" && + cmp -s <(LC_ALL=C sort "$expected_executions") \ + <(LC_ALL=C sort "$wait_control/executions"); then + pass AGG-017 "catalog parent reaps both lanes before publishing success" +else + fail AGG-017 "catalog parent reaps both lanes before publishing success" +fi + +cleanup_infrastructure=0 +if ! cleanup_work; then + cleanup_infrastructure=1 +fi +trap - EXIT + +printf 'SUMMARY assertions=%s expected=%s failures=%s infra=%s\n' \ + "$expected_count" "$expected_count" "$failures" "$cleanup_infrastructure" +if [ "$cleanup_infrastructure" -ne 0 ]; then + exit 125 +fi +[ "$failures" -eq 0 ] diff --git a/tests/experiment/contract-cases.sh b/tests/experiment/contract-cases.sh index 47524aa..22f15df 100755 --- a/tests/experiment/contract-cases.sh +++ b/tests/experiment/contract-cases.sh @@ -5,8 +5,9 @@ repo_root="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." >/dev/null 2>&1 && subcases=( "$repo_root/tests/experiment/directory-intake-cases.sh" "$repo_root/tests/experiment/aggregate-harness-cases.sh" + "$repo_root/tests/experiment/catalog-aggregate-harness-cases.sh" ) -expected_count=26 +expected_count=30 work="" cleanup_work() { @@ -32,7 +33,7 @@ printf '%s\n' \ FMT-001 FMT-002 FMT-004 FMT-005 FMT-006 FMT-007 FMT-003 FMT-008 \ SEL-001 CUE-001 FMT-009 M-FMT-001 SEL-002 \ AGG-001 AGG-002 AGG-003 AGG-004 AGG-005 AGG-006 AGG-007 AGG-008 AGG-009 \ - AGG-010 AGG-011 AGG-012 AGG-013 > "$expected" + AGG-010 AGG-011 AGG-012 AGG-013 AGG-014 AGG-015 AGG-016 AGG-017 > "$expected" : > "$observed" infrastructure=0 From 83f99af1d56948f3d716ca7b06ad23ce5d5b85fc Mon Sep 17 00:00:00 2001 From: xormania <127287135+xormania@users.noreply.github.com> Date: Sun, 2 Aug 2026 05:17:44 -0400 Subject: [PATCH 078/158] fix(experiment): bound catalog wall time --- tests/experiment/local-image-catalog-cases.sh | 83 ++++++++++++++++--- 1 file changed, 70 insertions(+), 13 deletions(-) diff --git a/tests/experiment/local-image-catalog-cases.sh b/tests/experiment/local-image-catalog-cases.sh index 951208b..933e45c 100755 --- a/tests/experiment/local-image-catalog-cases.sh +++ b/tests/experiment/local-image-catalog-cases.sh @@ -55,47 +55,104 @@ if [ "$declared_count" -ne "$expected_count" ]; then exit 125 fi infrastructure=0 +readonly lane_count=2 +declare -a lane_pids=() + +run_subcase() { + local index="$1" + local subcase="${subcases[$index]}" + local output="$work/subcase-$index.out" + local status="$work/subcase-$index.status" + local rc=0 -for index in "${!subcases[@]}"; do - subcase="${subcases[$index]}" - output="$work/subcase-$index.out" if [ ! -f "$subcase" ]; then - printf 'INFRA required catalog subcase is missing: %s\n' "$subcase" >&2 - infrastructure=1 - continue + printf 'INFRA required catalog subcase is missing: %s\n' "$subcase" > "$output" + printf '125\n' > "$status" + return 0 fi case "$subcase" in *.py) - python3 -I -B "$subcase" > "$output" 2>&1 - rc=$? + python3 -I -B "$subcase" > "$output" 2>&1 || rc=$? + ;; + *) + bash "$subcase" > "$output" 2>&1 || rc=$? + ;; + esac + printf '%s\n' "$rc" > "$status" +} + +run_lane() { + local lane="$1" + local index + local lane_subcases=() + case "$lane" in + 0) + lane_subcases=(1) + ;; + 1) + lane_subcases=(0 2 3) ;; *) - bash "$subcase" > "$output" 2>&1 - rc=$? + return 125 ;; esac + for index in "${lane_subcases[@]}"; do + run_subcase "$index" || return 125 + done + return 0 +} + +for ((lane = 0; lane < lane_count; lane++)); do + run_lane "$lane" & + lane_pids[lane]=$! +done +for lane in "${!lane_pids[@]}"; do + lane_rc=0 + wait "${lane_pids[lane]}" || lane_rc=$? + if [ "$lane_rc" -ne 0 ]; then + infrastructure=1 + fi +done + +failures=0 +for index in "${!subcases[@]}"; do + subcase="${subcases[$index]}" + output="$work/subcase-$index.out" + status="$work/subcase-$index.status" + rc=125 + if [ ! -f "$output" ]; then + printf 'INFRA catalog subcase output is missing: %s\n' "$subcase" >&2 + infrastructure=1 + continue + fi + if [ ! -f "$status" ] || [ "$(wc -l < "$status")" -ne 1 ] || + ! IFS= read -r rc < "$status" || [[ ! "$rc" =~ ^[0-9]+$ ]]; then + printf 'INFRA catalog subcase status is missing or invalid: %s\n' "$subcase" >&2 + rc=125 + infrastructure=1 + fi awk '/^(PASS|FAIL) [A-Z0-9-]+ /' "$output" awk '/^(PASS|FAIL) [A-Z0-9-]+ / {print $2}' "$output" >> "$observed" subcase_assertions="$(awk '/^(PASS|FAIL) [A-Z0-9-]+ / {count++} END {print count + 0}' "$output")" subcase_failures="$(awk '/^FAIL [A-Z0-9-]+ / {count++} END {print count + 0}' "$output")" + failures=$((failures + subcase_failures)) summary="SUMMARY assertions=$subcase_assertions expected=$subcase_assertions failures=$subcase_failures infra=0" summary_count="$(grep -Fxc "$summary" "$output" || true)" all_summary_count="$(grep -c '^SUMMARY ' "$output" || true)" if [ "$summary_count" -ne 1 ] || [ "$all_summary_count" -ne 1 ]; then printf 'INFRA catalog subcase summary is absent or inconsistent: %s\n' "$subcase" >&2 - cat "$output" >&2 + awk '!/^(PASS|FAIL) [A-Z0-9-]+ / {print}' "$output" >&2 infrastructure=1 elif { [ "$rc" -eq 0 ] && [ "$subcase_failures" -ne 0 ]; } \ || { [ "$rc" -eq 1 ] && [ "$subcase_failures" -eq 0 ]; } \ || { [ "$rc" -ne 0 ] && [ "$rc" -ne 1 ]; }; then printf 'INFRA catalog subcase status is inconsistent: rc=%s path=%s\n' "$rc" "$subcase" >&2 - cat "$output" >&2 + awk '!/^(PASS|FAIL) [A-Z0-9-]+ / {print}' "$output" >&2 infrastructure=1 fi done assertions="$(wc -l < "$observed")" -failures="$(awk '/^FAIL [A-Z0-9-]+ / {count++} END {print count + 0}' "$work"/subcase-*.out 2>/dev/null)" if ! cmp -s "$expected" "$observed"; then printf 'FAIL catalog aggregate assertion identity drift\n' >&2 diff -u "$expected" "$observed" >&2 || true From 9020b1ae93c17fae9d0be521fde16dd4fca5d4ab Mon Sep 17 00:00:00 2001 From: xormania <127287135+xormania@users.noreply.github.com> Date: Sun, 2 Aug 2026 05:31:23 -0400 Subject: [PATCH 079/158] test(experiment): expose catalog supervision gaps --- .../catalog-aggregate-harness-cases.sh | 260 ++++++++++++++++-- tests/experiment/contract-cases.sh | 5 +- 2 files changed, 238 insertions(+), 27 deletions(-) diff --git a/tests/experiment/catalog-aggregate-harness-cases.sh b/tests/experiment/catalog-aggregate-harness-cases.sh index e51f83e..54a9ac7 100755 --- a/tests/experiment/catalog-aggregate-harness-cases.sh +++ b/tests/experiment/catalog-aggregate-harness-cases.sh @@ -3,7 +3,7 @@ set -u -o pipefail repo_root="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." >/dev/null 2>&1 && pwd)" catalog_aggregate="$repo_root/tests/experiment/local-image-catalog-cases.sh" -expected_count=4 +expected_count=7 work="" cleanup_work() { @@ -78,6 +78,25 @@ write_fixture() { printf ' sleep 0.01\n' printf 'done\n' fi + if [ "$execution_id" = catalog-cases.sh ]; then + printf 'if [ "${AGENT_LAB_CATALOG_AGG_SIGNAL_MODE:-}" = cooperative ]; then\n' + printf ' sleep 30 &\n' + printf ' descendant_pid=$!\n' + printf ' printf "%%s\\n" "$descendant_pid" > "$control/bash-descendant.pid" || exit 125\n' + printf ' touch "$control/bash-signal.ready" || exit 125\n' + printf ' wait "$descendant_pid"\n' + printf 'fi\n' + printf 'if [ "${AGENT_LAB_CATALOG_AGG_SIGNAL_MODE:-}" = stubborn ]; then\n' + printf ' (\n' + printf " trap '' HUP INT QUIT TERM\n" + printf ' printf "%%s\\n" "$BASHPID" > "$control/stubborn-descendant.pid" || exit 125\n' + printf ' touch "$control/stubborn-signal.ready" || exit 125\n' + printf ' sleep 30\n' + printf ' ) &\n' + printf ' descendant_pid=$!\n' + printf ' wait "$descendant_pid"\n' + printf 'fi\n' + fi printf 'if [ "${AGENT_LAB_CATALOG_AGG_HOLD:-}" = "%s" ]; then\n' "$execution_id" printf ' touch "$control/hold.ready" || exit 125\n' printf ' attempts=0\n' @@ -114,6 +133,8 @@ write_python_fixture() { printf '#!/usr/bin/env python3\n' printf 'import os\n' printf 'from pathlib import Path\n' + printf 'import subprocess\n' + printf 'import sys\n' printf 'import time\n' printf 'control = Path(os.environ["AGENT_LAB_CATALOG_AGG_CONTROL"])\n' printf 'with (control / "executions").open("a", encoding="ascii") as stream:\n' @@ -125,6 +146,11 @@ write_python_fixture() { printf ' if time.monotonic() >= deadline:\n' printf ' raise SystemExit(125)\n' printf ' time.sleep(0.01)\n' + printf 'if os.environ.get("AGENT_LAB_CATALOG_AGG_SIGNAL_MODE") == "cooperative":\n' + printf ' descendant = subprocess.Popen([sys.executable, "-c", "import time; time.sleep(30)"])\n' + printf ' (control / "python-descendant.pid").write_text(str(descendant.pid), encoding="ascii")\n' + printf ' (control / "python-signal.ready").touch()\n' + printf ' descendant.wait()\n' fi printf 'if os.environ.get("AGENT_LAB_CATALOG_AGG_HOLD") == "%s":\n' "$execution_id" printf ' (control / "hold.ready").touch()\n' @@ -211,6 +237,16 @@ wait_for_child_wait() { return 1 } +wait_for_process_exit() { + local pid="$1" + local attempts=0 + while kill -0 "$pid" 2>/dev/null; do + attempts=$((attempts + 1)) + [ "$attempts" -lt 200 ] || return 1 + sleep 0.01 + done +} + reset_fixtures baseline_control="$work/baseline-control" mkdir "$baseline_control" @@ -239,13 +275,44 @@ else fail AGG-015 "catalog scheduler declares one immutable two-lane bound" fi +status_contract=1 +reset_fixtures 1 0 FAIL:CAT-NAME-001 +assertion_control="$work/assertion-control" +mkdir "$assertion_control" +assertion_rc=0 +run_aggregate "$work/assertion.out" "$assertion_control" || assertion_rc=$? +if [ "$assertion_rc" -ne 1 ] || + ! grep -Fxq 'SUMMARY assertions=76 expected=76 failures=1 infra=0' \ + "$work/assertion.out" || + grep -Fxq 'EXPERIMENT LOCAL IMAGE CATALOG PASS' "$work/assertion.out" || + ! cmp -s <(LC_ALL=C sort "$expected_executions") \ + <(LC_ALL=C sort "$assertion_control/executions") || + [ "$(find "$assertion_control" -name '*.done' -type f | wc -l)" -ne 4 ]; then + status_contract=0 +fi + +reset_fixtures 0 125 +infra_control="$work/infra-control" +mkdir "$infra_control" +infra_rc=0 +run_aggregate "$work/infra.out" "$infra_control" || infra_rc=$? +if [ "$infra_rc" -ne 125 ] || + ! grep -Fxq 'SUMMARY assertions=76 expected=76 failures=0 infra=1' \ + "$work/infra.out" || + grep -Fxq 'EXPERIMENT LOCAL IMAGE CATALOG PASS' "$work/infra.out" || + ! cmp -s <(LC_ALL=C sort "$expected_executions") \ + <(LC_ALL=C sort "$infra_control/executions") || + [ "$(find "$infra_control" -name '*.done' -type f | wc -l)" -ne 4 ]; then + status_contract=0 +fi + reset_fixtures 1 125 FAIL:CAT-NAME-001 mixed_control="$work/mixed-control" mkdir "$mixed_control" mixed_rc=0 run_aggregate "$work/mixed.out" "$mixed_control" || mixed_rc=$? mixed_assertions="$(grep -Ec '^(PASS|FAIL) [A-Z0-9-]+ ' "$work/mixed.out" || true)" -if [ "$mixed_rc" -eq 125 ] && +if [ "$status_contract" -eq 1 ] && [ "$mixed_rc" -eq 125 ] && [ "$mixed_assertions" -eq 76 ] && grep -Fxq 'SUMMARY assertions=76 expected=76 failures=1 infra=1' "$work/mixed.out" && ! grep -Fxq 'EXPERIMENT LOCAL IMAGE CATALOG PASS' "$work/mixed.out" && @@ -257,34 +324,177 @@ else fail AGG-016 "catalog uncertainty dominates after every subcase completes" fi -reset_fixtures -wait_control="$work/wait-control" -mkdir "$wait_control" -: > "$wait_control/executions" -AGENT_LAB_CATALOG_AGG_CONTROL="$wait_control" \ -AGENT_LAB_CATALOG_AGG_HOLD=catalog-state-cases.py \ - bash "$replica_aggregate" > "$work/wait.out" 2>&1 & -wait_pid=$! -wait_contract=0 -if wait_for_path "$wait_control/hold.ready" && - wait_for_path "$wait_control/catalog-cases.sh.done" && - wait_for_path "$wait_control/catalog-resolution-cases.sh.done" && - wait_for_path "$wait_control/catalog-mutation-cases.py.done" && - wait_for_child_wait "$wait_pid"; then - wait_contract=1 -fi -touch "$wait_control/hold.release" -wait_rc=0 -wait "$wait_pid" || wait_rc=$? -if [ "$wait_contract" -eq 1 ] && [ "$wait_rc" -eq 0 ] && - cmp -s "$expected_success" "$work/wait.out" && - cmp -s <(LC_ALL=C sort "$expected_executions") \ - <(LC_ALL=C sort "$wait_control/executions"); then +wait_contract=1 +for hold_index in 0 1; do + reset_fixtures + wait_control="$work/wait-control-$hold_index" + mkdir "$wait_control" + : > "$wait_control/executions" + if [ "$hold_index" -eq 0 ]; then + hold_id=catalog-state-cases.py + peer_done=( + catalog-cases.sh.done + catalog-resolution-cases.sh.done + catalog-mutation-cases.py.done + ) + else + hold_id=catalog-mutation-cases.py + peer_done=( + catalog-state-cases.py.done + catalog-cases.sh.done + catalog-resolution-cases.sh.done + ) + fi + AGENT_LAB_CATALOG_AGG_CONTROL="$wait_control" \ + AGENT_LAB_CATALOG_AGG_HOLD="$hold_id" \ + bash "$replica_aggregate" > "$work/wait-$hold_index.out" 2>&1 & + wait_pid=$! + wait_case_contract=0 + if wait_for_path "$wait_control/hold.ready" && + wait_for_path "$wait_control/${peer_done[0]}" && + wait_for_path "$wait_control/${peer_done[1]}" && + wait_for_path "$wait_control/${peer_done[2]}" && + wait_for_child_wait "$wait_pid"; then + wait_case_contract=1 + fi + touch "$wait_control/hold.release" + wait_rc=0 + wait "$wait_pid" || wait_rc=$? + if [ "$wait_case_contract" -ne 1 ] || [ "$wait_rc" -ne 0 ] || + ! cmp -s "$expected_success" "$work/wait-$hold_index.out" || + ! cmp -s <(LC_ALL=C sort "$expected_executions") \ + <(LC_ALL=C sort "$wait_control/executions"); then + wait_contract=0 + fi +done +if [ "$wait_contract" -eq 1 ]; then pass AGG-017 "catalog parent reaps both lanes before publishing success" else fail AGG-017 "catalog parent reaps both lanes before publishing success" fi +malformed_aggregate="$replica/tests/experiment/local-image-catalog-malformed-status.sh" +awk ' + index($0, "printf") && index($0, "$rc") && index($0, "$status") { + print " printf \"999999999999999999999999999999999999999999999999\\\\n\" > \"$status\"" + changed++ + next + } + { print } + END { if (changed != 1) exit 42 } +' "$replica_aggregate" > "$malformed_aggregate" +malformed_mutation_rc=$? +chmod +x "$malformed_aggregate" +reset_fixtures +malformed_control="$work/malformed-control" +mkdir "$malformed_control" +: > "$malformed_control/executions" +malformed_rc=0 +AGENT_LAB_CATALOG_AGG_CONTROL="$malformed_control" \ + bash "$malformed_aggregate" > "$work/malformed.out" 2>&1 || malformed_rc=$? +if [ "$malformed_mutation_rc" -eq 0 ] && [ "$malformed_rc" -eq 125 ] && + grep -Fxq 'SUMMARY assertions=76 expected=76 failures=0 infra=1' \ + "$work/malformed.out" && + ! grep -Fxq 'EXPERIMENT LOCAL IMAGE CATALOG PASS' "$work/malformed.out" && + cmp -s <(LC_ALL=C sort "$expected_executions") \ + <(LC_ALL=C sort "$malformed_control/executions"); then + pass AGG-018 "malformed lane status fails closed before success" +else + fail AGG-018 "malformed lane status fails closed before success" +fi + +reset_fixtures +signal_control="$work/signal-control" +signal_tmp="$work/signal-tmp" +mkdir "$signal_control" "$signal_tmp" +: > "$signal_control/executions" +AGENT_LAB_CATALOG_AGG_CONTROL="$signal_control" \ +AGENT_LAB_CATALOG_AGG_SIGNAL_MODE=cooperative \ +TMPDIR="$signal_tmp" \ + bash "$replica_aggregate" > "$work/signal.out" 2>&1 & +signal_pid=$! +signal_setup=0 +bash_descendant="" +python_descendant="" +if wait_for_path "$signal_control/bash-signal.ready" && + wait_for_path "$signal_control/python-signal.ready" && + IFS= read -r bash_descendant < "$signal_control/bash-descendant.pid" && + IFS= read -r python_descendant < "$signal_control/python-descendant.pid" && + [[ "$bash_descendant" =~ ^[0-9]+$ ]] && + [[ "$python_descendant" =~ ^[0-9]+$ ]] && + kill -0 "$bash_descendant" 2>/dev/null && + kill -0 "$python_descendant" 2>/dev/null; then + signal_setup=1 +fi +kill -TERM "$signal_pid" 2>/dev/null || true +signal_rc=0 +wait "$signal_pid" || signal_rc=$? +bash_descendant_gone=0 +python_descendant_gone=0 +if [ -n "$bash_descendant" ] && wait_for_process_exit "$bash_descendant"; then + bash_descendant_gone=1 +fi +if [ -n "$python_descendant" ] && wait_for_process_exit "$python_descendant"; then + python_descendant_gone=1 +fi +if [ "$bash_descendant_gone" -ne 1 ] && [ -n "$bash_descendant" ]; then + kill -KILL "$bash_descendant" 2>/dev/null || true + wait_for_process_exit "$bash_descendant" || true +fi +if [ "$python_descendant_gone" -ne 1 ] && [ -n "$python_descendant" ]; then + kill -KILL "$python_descendant" 2>/dev/null || true + wait_for_process_exit "$python_descendant" || true +fi +if [ "$signal_setup" -eq 1 ] && [ "$signal_rc" -eq 143 ] && + [ "$bash_descendant_gone" -eq 1 ] && [ "$python_descendant_gone" -eq 1 ] && + ! grep -Fxq 'EXPERIMENT LOCAL IMAGE CATALOG PASS' "$work/signal.out"; then + pass AGG-019 "catalog cancellation reaches cooperative lane descendants" +else + fail AGG-019 "catalog cancellation reaches cooperative lane descendants" +fi + +reset_fixtures +stubborn_control="$work/stubborn-control" +stubborn_tmp="$work/stubborn-tmp" +mkdir "$stubborn_control" "$stubborn_tmp" +: > "$stubborn_control/executions" +AGENT_LAB_CATALOG_AGG_CONTROL="$stubborn_control" \ +AGENT_LAB_CATALOG_AGG_SIGNAL_MODE=stubborn \ +TMPDIR="$stubborn_tmp" \ + bash "$replica_aggregate" > "$work/stubborn.out" 2>&1 & +stubborn_leader=$! +stubborn_setup=0 +stubborn_pid="" +if wait_for_path "$stubborn_control/stubborn-signal.ready" && + IFS= read -r stubborn_pid < "$stubborn_control/stubborn-descendant.pid" && + [[ "$stubborn_pid" =~ ^[0-9]+$ ]] && + kill -0 "$stubborn_pid" 2>/dev/null; then + stubborn_setup=1 +fi +kill -TERM "$stubborn_leader" 2>/dev/null || true +stubborn_rc=0 +wait "$stubborn_leader" || stubborn_rc=$? +stubborn_alive=0 +stubborn_output_preserved=0 +if [ -n "$stubborn_pid" ] && kill -0 "$stubborn_pid" 2>/dev/null; then + stubborn_alive=1 + stubborn_output="$(readlink "/proc/$stubborn_pid/fd/1" 2>/dev/null || true)" + if [ -n "$stubborn_output" ] && [ -e "$stubborn_output" ]; then + stubborn_output_preserved=1 + fi +fi +if [ -n "$stubborn_pid" ]; then + kill -KILL "$stubborn_pid" 2>/dev/null || true + wait_for_process_exit "$stubborn_pid" || true +fi +if [ "$stubborn_setup" -eq 1 ] && [ "$stubborn_rc" -eq 143 ] && + [ "$stubborn_alive" -eq 1 ] && [ "$stubborn_output_preserved" -eq 1 ] && + ! grep -Fxq 'EXPERIMENT LOCAL IMAGE CATALOG PASS' "$work/stubborn.out"; then + pass AGG-020 "catalog cancellation preserves stubborn descendant evidence" +else + fail AGG-020 "catalog cancellation preserves stubborn descendant evidence" +fi + cleanup_infrastructure=0 if ! cleanup_work; then cleanup_infrastructure=1 diff --git a/tests/experiment/contract-cases.sh b/tests/experiment/contract-cases.sh index 22f15df..2d5211e 100755 --- a/tests/experiment/contract-cases.sh +++ b/tests/experiment/contract-cases.sh @@ -7,7 +7,7 @@ subcases=( "$repo_root/tests/experiment/aggregate-harness-cases.sh" "$repo_root/tests/experiment/catalog-aggregate-harness-cases.sh" ) -expected_count=30 +expected_count=33 work="" cleanup_work() { @@ -33,7 +33,8 @@ printf '%s\n' \ FMT-001 FMT-002 FMT-004 FMT-005 FMT-006 FMT-007 FMT-003 FMT-008 \ SEL-001 CUE-001 FMT-009 M-FMT-001 SEL-002 \ AGG-001 AGG-002 AGG-003 AGG-004 AGG-005 AGG-006 AGG-007 AGG-008 AGG-009 \ - AGG-010 AGG-011 AGG-012 AGG-013 AGG-014 AGG-015 AGG-016 AGG-017 > "$expected" + AGG-010 AGG-011 AGG-012 AGG-013 AGG-014 AGG-015 AGG-016 AGG-017 \ + AGG-018 AGG-019 AGG-020 > "$expected" : > "$observed" infrastructure=0 From cbf1af108cf1d46d4b73d39221587fdba6e41fb8 Mon Sep 17 00:00:00 2001 From: xormania <127287135+xormania@users.noreply.github.com> Date: Sun, 2 Aug 2026 05:33:00 -0400 Subject: [PATCH 080/158] fix(experiment): supervise catalog lanes --- .../catalog-aggregate-harness-cases.sh | 2 +- tests/experiment/local-image-catalog-cases.sh | 66 ++++++++++++++++++- 2 files changed, 64 insertions(+), 4 deletions(-) diff --git a/tests/experiment/catalog-aggregate-harness-cases.sh b/tests/experiment/catalog-aggregate-harness-cases.sh index 54a9ac7..15b8904 100755 --- a/tests/experiment/catalog-aggregate-harness-cases.sh +++ b/tests/experiment/catalog-aggregate-harness-cases.sh @@ -148,7 +148,7 @@ write_python_fixture() { printf ' time.sleep(0.01)\n' printf 'if os.environ.get("AGENT_LAB_CATALOG_AGG_SIGNAL_MODE") == "cooperative":\n' printf ' descendant = subprocess.Popen([sys.executable, "-c", "import time; time.sleep(30)"])\n' - printf ' (control / "python-descendant.pid").write_text(str(descendant.pid), encoding="ascii")\n' + printf ' (control / "python-descendant.pid").write_text(str(descendant.pid) + "\\n", encoding="ascii")\n' printf ' (control / "python-signal.ready").touch()\n' printf ' descendant.wait()\n' fi diff --git a/tests/experiment/local-image-catalog-cases.sh b/tests/experiment/local-image-catalog-cases.sh index 933e45c..257469b 100755 --- a/tests/experiment/local-image-catalog-cases.sh +++ b/tests/experiment/local-image-catalog-cases.sh @@ -4,6 +4,9 @@ set -u -o pipefail repo_root="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." >/dev/null 2>&1 && pwd)" expected_count=76 work="" +declare -a lane_pids=() +declare -a signal_descendants=() +current_lane_pid="" cleanup_work() { local failed=0 @@ -16,11 +19,56 @@ cleanup_work() { return "$failed" } +collect_descendants() { + local parent_pid="$1" + local children="" + local child + if [ -r "/proc/$parent_pid/task/$parent_pid/children" ]; then + IFS= read -r children < "/proc/$parent_pid/task/$parent_pid/children" || true + fi + for child in $children; do + if [[ ! "$child" =~ ^[0-9]+$ ]]; then + continue + fi + signal_descendants[${#signal_descendants[@]}]="$child" + collect_descendants "$child" + done +} + +catalog_signal() { + local signal_name="$1" + local signal_number="$2" + local lane_pid + local index + trap '' HUP INT QUIT TERM + trap - EXIT + signal_descendants=() + if [[ "$current_lane_pid" =~ ^[0-9]+$ ]]; then + signal_descendants[${#signal_descendants[@]}]="$current_lane_pid" + collect_descendants "$current_lane_pid" + fi + for lane_pid in "${lane_pids[@]}"; do + if [[ ! "$lane_pid" =~ ^[0-9]+$ ]]; then + continue + fi + signal_descendants[${#signal_descendants[@]}]="$lane_pid" + collect_descendants "$lane_pid" + done + for ((index = ${#signal_descendants[@]} - 1; index >= 0; index--)); do + kill "-$signal_name" -- "${signal_descendants[index]}" 2>/dev/null || true + done + exit $((128 + signal_number)) +} + if ! work="$(mktemp -d)"; then printf 'SUMMARY assertions=0 expected=%s failures=0 infra=1\n' "$expected_count" exit 125 fi trap 'cleanup_work >/dev/null 2>&1 || true' EXIT +trap 'catalog_signal HUP 1' HUP +trap 'catalog_signal INT 2' INT +trap 'catalog_signal QUIT 3' QUIT +trap 'catalog_signal TERM 15' TERM subcases=( "$repo_root/tests/image/catalog-cases.sh" @@ -56,7 +104,6 @@ if [ "$declared_count" -ne "$expected_count" ]; then fi infrastructure=0 readonly lane_count=2 -declare -a lane_pids=() run_subcase() { local index="$1" @@ -108,7 +155,10 @@ for ((lane = 0; lane < lane_count; lane++)); do done for lane in "${!lane_pids[@]}"; do lane_rc=0 - wait "${lane_pids[lane]}" || lane_rc=$? + current_lane_pid="${lane_pids[lane]}" + lane_pids[lane]="" + wait "$current_lane_pid" || lane_rc=$? + current_lane_pid="" if [ "$lane_rc" -ne 0 ]; then infrastructure=1 fi @@ -126,10 +176,20 @@ for index in "${!subcases[@]}"; do continue fi if [ ! -f "$status" ] || [ "$(wc -l < "$status")" -ne 1 ] || - ! IFS= read -r rc < "$status" || [[ ! "$rc" =~ ^[0-9]+$ ]]; then + ! IFS= read -r rc < "$status"; then printf 'INFRA catalog subcase status is missing or invalid: %s\n' "$subcase" >&2 rc=125 infrastructure=1 + else + case "$rc" in + 0 | 1 | 125) + ;; + *) + printf 'INFRA catalog subcase status is missing or invalid: %s\n' "$subcase" >&2 + rc=125 + infrastructure=1 + ;; + esac fi awk '/^(PASS|FAIL) [A-Z0-9-]+ /' "$output" awk '/^(PASS|FAIL) [A-Z0-9-]+ / {print $2}' "$output" >> "$observed" From f3bc18972619b3360abefc70b4810932a7ec97eb Mon Sep 17 00:00:00 2001 From: xormania <127287135+xormania@users.noreply.github.com> Date: Sun, 2 Aug 2026 05:36:44 -0400 Subject: [PATCH 081/158] test(experiment): reject trailing status bytes --- .../catalog-aggregate-harness-cases.sh | 29 +++++++++++++++++-- 1 file changed, 27 insertions(+), 2 deletions(-) diff --git a/tests/experiment/catalog-aggregate-harness-cases.sh b/tests/experiment/catalog-aggregate-harness-cases.sh index 15b8904..97674ed 100755 --- a/tests/experiment/catalog-aggregate-harness-cases.sh +++ b/tests/experiment/catalog-aggregate-harness-cases.sh @@ -392,12 +392,37 @@ mkdir "$malformed_control" malformed_rc=0 AGENT_LAB_CATALOG_AGG_CONTROL="$malformed_control" \ bash "$malformed_aggregate" > "$work/malformed.out" 2>&1 || malformed_rc=$? -if [ "$malformed_mutation_rc" -eq 0 ] && [ "$malformed_rc" -eq 125 ] && +trailing_aggregate="$replica/tests/experiment/local-image-catalog-trailing-status.sh" +awk ' + index($0, "printf") && index($0, "$rc") && index($0, "$status") { + print " printf \"0\\\\nTRAILING-GARBAGE\" > \"$status\"" + changed++ + next + } + { print } + END { if (changed != 1) exit 42 } +' "$replica_aggregate" > "$trailing_aggregate" +trailing_mutation_rc=$? +chmod +x "$trailing_aggregate" +reset_fixtures +trailing_control="$work/trailing-control" +mkdir "$trailing_control" +: > "$trailing_control/executions" +trailing_rc=0 +AGENT_LAB_CATALOG_AGG_CONTROL="$trailing_control" \ + bash "$trailing_aggregate" > "$work/trailing.out" 2>&1 || trailing_rc=$? +if [ "$malformed_mutation_rc" -eq 0 ] && [ "$trailing_mutation_rc" -eq 0 ] && + [ "$malformed_rc" -eq 125 ] && [ "$trailing_rc" -eq 125 ] && grep -Fxq 'SUMMARY assertions=76 expected=76 failures=0 infra=1' \ "$work/malformed.out" && + grep -Fxq 'SUMMARY assertions=76 expected=76 failures=0 infra=1' \ + "$work/trailing.out" && ! grep -Fxq 'EXPERIMENT LOCAL IMAGE CATALOG PASS' "$work/malformed.out" && + ! grep -Fxq 'EXPERIMENT LOCAL IMAGE CATALOG PASS' "$work/trailing.out" && + cmp -s <(LC_ALL=C sort "$expected_executions") \ + <(LC_ALL=C sort "$malformed_control/executions") && cmp -s <(LC_ALL=C sort "$expected_executions") \ - <(LC_ALL=C sort "$malformed_control/executions"); then + <(LC_ALL=C sort "$trailing_control/executions"); then pass AGG-018 "malformed lane status fails closed before success" else fail AGG-018 "malformed lane status fails closed before success" From f9ad7061670951de9e946a93040e48862908dec3 Mon Sep 17 00:00:00 2001 From: xormania <127287135+xormania@users.noreply.github.com> Date: Sun, 2 Aug 2026 05:37:08 -0400 Subject: [PATCH 082/158] fix(experiment): authenticate catalog statuses --- tests/experiment/local-image-catalog-cases.sh | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/tests/experiment/local-image-catalog-cases.sh b/tests/experiment/local-image-catalog-cases.sh index 257469b..21618d2 100755 --- a/tests/experiment/local-image-catalog-cases.sh +++ b/tests/experiment/local-image-catalog-cases.sh @@ -175,21 +175,16 @@ for index in "${!subcases[@]}"; do infrastructure=1 continue fi - if [ ! -f "$status" ] || [ "$(wc -l < "$status")" -ne 1 ] || - ! IFS= read -r rc < "$status"; then + if [ -f "$status" ] && cmp -s "$status" <(printf '0\n'); then + rc=0 + elif [ -f "$status" ] && cmp -s "$status" <(printf '1\n'); then + rc=1 + elif [ -f "$status" ] && cmp -s "$status" <(printf '125\n'); then + rc=125 + else printf 'INFRA catalog subcase status is missing or invalid: %s\n' "$subcase" >&2 rc=125 infrastructure=1 - else - case "$rc" in - 0 | 1 | 125) - ;; - *) - printf 'INFRA catalog subcase status is missing or invalid: %s\n' "$subcase" >&2 - rc=125 - infrastructure=1 - ;; - esac fi awk '/^(PASS|FAIL) [A-Z0-9-]+ /' "$output" awk '/^(PASS|FAIL) [A-Z0-9-]+ / {print $2}' "$output" >> "$observed" From 77765b8e6ad4825d639cc5721b437bce03aad488 Mon Sep 17 00:00:00 2001 From: xormania <127287135+xormania@users.noreply.github.com> Date: Sun, 2 Aug 2026 05:40:09 -0400 Subject: [PATCH 083/158] test(experiment): require complete catalog cancellation --- .../catalog-aggregate-harness-cases.sh | 127 +++++++++++------- 1 file changed, 81 insertions(+), 46 deletions(-) diff --git a/tests/experiment/catalog-aggregate-harness-cases.sh b/tests/experiment/catalog-aggregate-harness-cases.sh index 97674ed..87d0baf 100755 --- a/tests/experiment/catalog-aggregate-harness-cases.sh +++ b/tests/experiment/catalog-aggregate-harness-cases.sh @@ -80,6 +80,7 @@ write_fixture() { fi if [ "$execution_id" = catalog-cases.sh ]; then printf 'if [ "${AGENT_LAB_CATALOG_AGG_SIGNAL_MODE:-}" = cooperative ]; then\n' + printf ' printf "%%s\\n" "$PPID" > "$control/bash-lane.pid" || exit 125\n' printf ' sleep 30 &\n' printf ' descendant_pid=$!\n' printf ' printf "%%s\\n" "$descendant_pid" > "$control/bash-descendant.pid" || exit 125\n' @@ -87,6 +88,7 @@ write_fixture() { printf ' wait "$descendant_pid"\n' printf 'fi\n' printf 'if [ "${AGENT_LAB_CATALOG_AGG_SIGNAL_MODE:-}" = stubborn ]; then\n' + printf ' printf "%%s\\n" "$PPID" > "$control/bash-lane.pid" || exit 125\n' printf ' (\n' printf " trap '' HUP INT QUIT TERM\n" printf ' printf "%%s\\n" "$BASHPID" > "$control/stubborn-descendant.pid" || exit 125\n' @@ -147,6 +149,7 @@ write_python_fixture() { printf ' raise SystemExit(125)\n' printf ' time.sleep(0.01)\n' printf 'if os.environ.get("AGENT_LAB_CATALOG_AGG_SIGNAL_MODE") == "cooperative":\n' + printf ' (control / "python-lane.pid").write_text(str(os.getppid()) + "\\n", encoding="ascii")\n' printf ' descendant = subprocess.Popen([sys.executable, "-c", "import time; time.sleep(30)"])\n' printf ' (control / "python-descendant.pid").write_text(str(descendant.pid) + "\\n", encoding="ascii")\n' printf ' (control / "python-signal.ready").touch()\n' @@ -428,51 +431,68 @@ else fail AGG-018 "malformed lane status fails closed before success" fi -reset_fixtures -signal_control="$work/signal-control" -signal_tmp="$work/signal-tmp" -mkdir "$signal_control" "$signal_tmp" -: > "$signal_control/executions" -AGENT_LAB_CATALOG_AGG_CONTROL="$signal_control" \ -AGENT_LAB_CATALOG_AGG_SIGNAL_MODE=cooperative \ -TMPDIR="$signal_tmp" \ - bash "$replica_aggregate" > "$work/signal.out" 2>&1 & -signal_pid=$! -signal_setup=0 -bash_descendant="" -python_descendant="" -if wait_for_path "$signal_control/bash-signal.ready" && - wait_for_path "$signal_control/python-signal.ready" && - IFS= read -r bash_descendant < "$signal_control/bash-descendant.pid" && - IFS= read -r python_descendant < "$signal_control/python-descendant.pid" && - [[ "$bash_descendant" =~ ^[0-9]+$ ]] && - [[ "$python_descendant" =~ ^[0-9]+$ ]] && - kill -0 "$bash_descendant" 2>/dev/null && - kill -0 "$python_descendant" 2>/dev/null; then - signal_setup=1 -fi -kill -TERM "$signal_pid" 2>/dev/null || true -signal_rc=0 -wait "$signal_pid" || signal_rc=$? -bash_descendant_gone=0 -python_descendant_gone=0 -if [ -n "$bash_descendant" ] && wait_for_process_exit "$bash_descendant"; then - bash_descendant_gone=1 -fi -if [ -n "$python_descendant" ] && wait_for_process_exit "$python_descendant"; then - python_descendant_gone=1 -fi -if [ "$bash_descendant_gone" -ne 1 ] && [ -n "$bash_descendant" ]; then - kill -KILL "$bash_descendant" 2>/dev/null || true - wait_for_process_exit "$bash_descendant" || true -fi -if [ "$python_descendant_gone" -ne 1 ] && [ -n "$python_descendant" ]; then - kill -KILL "$python_descendant" 2>/dev/null || true - wait_for_process_exit "$python_descendant" || true -fi -if [ "$signal_setup" -eq 1 ] && [ "$signal_rc" -eq 143 ] && - [ "$bash_descendant_gone" -eq 1 ] && [ "$python_descendant_gone" -eq 1 ] && - ! grep -Fxq 'EXPERIMENT LOCAL IMAGE CATALOG PASS' "$work/signal.out"; then +signal_names=(TERM INT QUIT) +signal_numbers=(15 2 3) +expected_signal_executions="$work/expected-signal-executions" +printf '%s\n' catalog-cases.sh catalog-state-cases.py > "$expected_signal_executions" +signal_contract=1 +for signal_index in "${!signal_names[@]}"; do + reset_fixtures + signal_control="$work/signal-control-$signal_index" + signal_tmp="$work/signal-tmp-$signal_index" + mkdir "$signal_control" "$signal_tmp" + : > "$signal_control/executions" + AGENT_LAB_CATALOG_AGG_CONTROL="$signal_control" \ + AGENT_LAB_CATALOG_AGG_SIGNAL_MODE=cooperative \ + TMPDIR="$signal_tmp" \ + bash "$replica_aggregate" > "$work/signal-$signal_index.out" 2>&1 & + signal_pid=$! + signal_setup=0 + bash_lane="" + python_lane="" + bash_descendant="" + python_descendant="" + if wait_for_path "$signal_control/bash-signal.ready" && + wait_for_path "$signal_control/python-signal.ready" && + IFS= read -r bash_lane < "$signal_control/bash-lane.pid" && + IFS= read -r python_lane < "$signal_control/python-lane.pid" && + IFS= read -r bash_descendant < "$signal_control/bash-descendant.pid" && + IFS= read -r python_descendant < "$signal_control/python-descendant.pid" && + [[ "$bash_lane" =~ ^[0-9]+$ ]] && [[ "$python_lane" =~ ^[0-9]+$ ]] && + [[ "$bash_descendant" =~ ^[0-9]+$ ]] && + [[ "$python_descendant" =~ ^[0-9]+$ ]] && + kill -0 "$bash_lane" 2>/dev/null && kill -0 "$python_lane" 2>/dev/null && + kill -0 "$bash_descendant" 2>/dev/null && + kill -0 "$python_descendant" 2>/dev/null; then + signal_setup=1 + fi + kill "-${signal_names[$signal_index]}" "$signal_pid" 2>/dev/null || true + signal_rc=0 + wait "$signal_pid" || signal_rc=$? + signal_pids_gone=1 + for observed_pid in \ + "$bash_lane" "$python_lane" "$bash_descendant" "$python_descendant"; do + if [ -z "$observed_pid" ] || ! wait_for_process_exit "$observed_pid"; then + signal_pids_gone=0 + fi + done + if [ "$signal_setup" -ne 1 ] || + [ "$signal_rc" -ne "$((128 + signal_numbers[$signal_index]))" ] || + [ "$signal_pids_gone" -ne 1 ] || + grep -Fxq 'EXPERIMENT LOCAL IMAGE CATALOG PASS' "$work/signal-$signal_index.out" || + ! cmp -s <(LC_ALL=C sort "$expected_signal_executions") \ + <(LC_ALL=C sort "$signal_control/executions"); then + signal_contract=0 + fi + for observed_pid in \ + "$bash_descendant" "$python_descendant" "$bash_lane" "$python_lane"; do + if [ -n "$observed_pid" ]; then + kill -KILL "$observed_pid" 2>/dev/null || true + wait_for_process_exit "$observed_pid" || true + fi + done +done +if [ "$signal_contract" -eq 1 ]; then pass AGG-019 "catalog cancellation reaches cooperative lane descendants" else fail AGG-019 "catalog cancellation reaches cooperative lane descendants" @@ -489,10 +509,14 @@ TMPDIR="$stubborn_tmp" \ bash "$replica_aggregate" > "$work/stubborn.out" 2>&1 & stubborn_leader=$! stubborn_setup=0 +stubborn_lane="" stubborn_pid="" if wait_for_path "$stubborn_control/stubborn-signal.ready" && + IFS= read -r stubborn_lane < "$stubborn_control/bash-lane.pid" && IFS= read -r stubborn_pid < "$stubborn_control/stubborn-descendant.pid" && + [[ "$stubborn_lane" =~ ^[0-9]+$ ]] && [[ "$stubborn_pid" =~ ^[0-9]+$ ]] && + kill -0 "$stubborn_lane" 2>/dev/null && kill -0 "$stubborn_pid" 2>/dev/null; then stubborn_setup=1 fi @@ -501,6 +525,10 @@ stubborn_rc=0 wait "$stubborn_leader" || stubborn_rc=$? stubborn_alive=0 stubborn_output_preserved=0 +stubborn_lane_gone=0 +if [ -n "$stubborn_lane" ] && wait_for_process_exit "$stubborn_lane"; then + stubborn_lane_gone=1 +fi if [ -n "$stubborn_pid" ] && kill -0 "$stubborn_pid" 2>/dev/null; then stubborn_alive=1 stubborn_output="$(readlink "/proc/$stubborn_pid/fd/1" 2>/dev/null || true)" @@ -512,8 +540,15 @@ if [ -n "$stubborn_pid" ]; then kill -KILL "$stubborn_pid" 2>/dev/null || true wait_for_process_exit "$stubborn_pid" || true fi +if [ -n "$stubborn_lane" ]; then + kill -KILL "$stubborn_lane" 2>/dev/null || true + wait_for_process_exit "$stubborn_lane" || true +fi if [ "$stubborn_setup" -eq 1 ] && [ "$stubborn_rc" -eq 143 ] && - [ "$stubborn_alive" -eq 1 ] && [ "$stubborn_output_preserved" -eq 1 ] && + [ "$stubborn_alive" -eq 1 ] && [ "$stubborn_lane_gone" -eq 1 ] && + [ "$stubborn_output_preserved" -eq 1 ] && + cmp -s <(LC_ALL=C sort "$expected_signal_executions") \ + <(LC_ALL=C sort "$stubborn_control/executions") && ! grep -Fxq 'EXPERIMENT LOCAL IMAGE CATALOG PASS' "$work/stubborn.out"; then pass AGG-020 "catalog cancellation preserves stubborn descendant evidence" else From 65798971f53c6c933ae598a2d7aa5c7f28900436 Mon Sep 17 00:00:00 2001 From: xormania <127287135+xormania@users.noreply.github.com> Date: Sun, 2 Aug 2026 05:44:11 -0400 Subject: [PATCH 084/158] fix(experiment): close catalog cancellation races --- .../catalog-aggregate-harness-cases.sh | 9 +++- tests/experiment/local-image-catalog-cases.sh | 45 ++++++++++++------- 2 files changed, 35 insertions(+), 19 deletions(-) diff --git a/tests/experiment/catalog-aggregate-harness-cases.sh b/tests/experiment/catalog-aggregate-harness-cases.sh index 87d0baf..1eb3326 100755 --- a/tests/experiment/catalog-aggregate-harness-cases.sh +++ b/tests/experiment/catalog-aggregate-harness-cases.sh @@ -445,7 +445,12 @@ for signal_index in "${!signal_names[@]}"; do AGENT_LAB_CATALOG_AGG_CONTROL="$signal_control" \ AGENT_LAB_CATALOG_AGG_SIGNAL_MODE=cooperative \ TMPDIR="$signal_tmp" \ - bash "$replica_aggregate" > "$work/signal-$signal_index.out" 2>&1 & + python3 -I -B -c \ + 'import os, signal, sys +for value in (signal.SIGHUP, signal.SIGINT, signal.SIGQUIT, signal.SIGTERM): + signal.signal(value, signal.SIG_DFL) +os.execvpe("bash", ["bash", sys.argv[1]], os.environ)' \ + "$replica_aggregate" > "$work/signal-$signal_index.out" 2>&1 & signal_pid=$! signal_setup=0 bash_lane="" @@ -477,7 +482,7 @@ for signal_index in "${!signal_names[@]}"; do fi done if [ "$signal_setup" -ne 1 ] || - [ "$signal_rc" -ne "$((128 + signal_numbers[$signal_index]))" ] || + [ "$signal_rc" -ne "$((128 + signal_numbers[signal_index]))" ] || [ "$signal_pids_gone" -ne 1 ] || grep -Fxq 'EXPERIMENT LOCAL IMAGE CATALOG PASS' "$work/signal-$signal_index.out" || ! cmp -s <(LC_ALL=C sort "$expected_signal_executions") \ diff --git a/tests/experiment/local-image-catalog-cases.sh b/tests/experiment/local-image-catalog-cases.sh index 21618d2..c4427ed 100755 --- a/tests/experiment/local-image-catalog-cases.sh +++ b/tests/experiment/local-image-catalog-cases.sh @@ -23,6 +23,7 @@ collect_descendants() { local parent_pid="$1" local children="" local child + local observed if [ -r "/proc/$parent_pid/task/$parent_pid/children" ]; then IFS= read -r children < "/proc/$parent_pid/task/$parent_pid/children" || true fi @@ -30,32 +31,42 @@ collect_descendants() { if [[ ! "$child" =~ ^[0-9]+$ ]]; then continue fi + for observed in "${signal_descendants[@]}"; do + if [ "$observed" = "$child" ]; then + child="" + break + fi + done + if [ -z "$child" ]; then + continue + fi signal_descendants[${#signal_descendants[@]}]="$child" collect_descendants "$child" done } catalog_signal() { - local signal_name="$1" - local signal_number="$2" - local lane_pid + local signal_number="$1" + local before local index trap '' HUP INT QUIT TERM trap - EXIT signal_descendants=() - if [[ "$current_lane_pid" =~ ^[0-9]+$ ]]; then - signal_descendants[${#signal_descendants[@]}]="$current_lane_pid" - collect_descendants "$current_lane_pid" - fi - for lane_pid in "${lane_pids[@]}"; do - if [[ ! "$lane_pid" =~ ^[0-9]+$ ]]; then - continue + for _ in 1 2 3 4 5 6 7 8; do + before="${#signal_descendants[@]}" + collect_descendants "$$" + for ((index = before; index < ${#signal_descendants[@]}; index++)); do + kill -STOP -- "${signal_descendants[index]}" 2>/dev/null || true + done + if [ "${#signal_descendants[@]}" -eq "$before" ]; then + break fi - signal_descendants[${#signal_descendants[@]}]="$lane_pid" - collect_descendants "$lane_pid" done for ((index = ${#signal_descendants[@]} - 1; index >= 0; index--)); do - kill "-$signal_name" -- "${signal_descendants[index]}" 2>/dev/null || true + kill -TERM -- "${signal_descendants[index]}" 2>/dev/null || true + done + for ((index = ${#signal_descendants[@]} - 1; index >= 0; index--)); do + kill -CONT -- "${signal_descendants[index]}" 2>/dev/null || true done exit $((128 + signal_number)) } @@ -65,10 +76,10 @@ if ! work="$(mktemp -d)"; then exit 125 fi trap 'cleanup_work >/dev/null 2>&1 || true' EXIT -trap 'catalog_signal HUP 1' HUP -trap 'catalog_signal INT 2' INT -trap 'catalog_signal QUIT 3' QUIT -trap 'catalog_signal TERM 15' TERM +trap 'catalog_signal 1' HUP +trap 'catalog_signal 2' INT +trap 'catalog_signal 3' QUIT +trap 'catalog_signal 15' TERM subcases=( "$repo_root/tests/image/catalog-cases.sh" From d653fa78ba83910586b72ba01c47a2c101d69259 Mon Sep 17 00:00:00 2001 From: xormania <127287135+xormania@users.noreply.github.com> Date: Sun, 2 Aug 2026 05:47:23 -0400 Subject: [PATCH 085/158] test(experiment): expose threaded cancellation escape --- tests/experiment/catalog-aggregate-harness-cases.sh | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/tests/experiment/catalog-aggregate-harness-cases.sh b/tests/experiment/catalog-aggregate-harness-cases.sh index 1eb3326..444276c 100755 --- a/tests/experiment/catalog-aggregate-harness-cases.sh +++ b/tests/experiment/catalog-aggregate-harness-cases.sh @@ -137,6 +137,7 @@ write_python_fixture() { printf 'from pathlib import Path\n' printf 'import subprocess\n' printf 'import sys\n' + printf 'import threading\n' printf 'import time\n' printf 'control = Path(os.environ["AGENT_LAB_CATALOG_AGG_CONTROL"])\n' printf 'with (control / "executions").open("a", encoding="ascii") as stream:\n' @@ -150,10 +151,14 @@ write_python_fixture() { printf ' time.sleep(0.01)\n' printf 'if os.environ.get("AGENT_LAB_CATALOG_AGG_SIGNAL_MODE") == "cooperative":\n' printf ' (control / "python-lane.pid").write_text(str(os.getppid()) + "\\n", encoding="ascii")\n' - printf ' descendant = subprocess.Popen([sys.executable, "-c", "import time; time.sleep(30)"])\n' - printf ' (control / "python-descendant.pid").write_text(str(descendant.pid) + "\\n", encoding="ascii")\n' - printf ' (control / "python-signal.ready").touch()\n' - printf ' descendant.wait()\n' + printf ' def run_descendant():\n' + printf ' descendant = subprocess.Popen([sys.executable, "-c", "import time; time.sleep(30)"])\n' + printf ' (control / "python-descendant.pid").write_text(str(descendant.pid) + "\\n", encoding="ascii")\n' + printf ' (control / "python-signal.ready").touch()\n' + printf ' descendant.wait()\n' + printf ' descendant_thread = threading.Thread(target=run_descendant)\n' + printf ' descendant_thread.start()\n' + printf ' descendant_thread.join()\n' fi printf 'if os.environ.get("AGENT_LAB_CATALOG_AGG_HOLD") == "%s":\n' "$execution_id" printf ' (control / "hold.ready").touch()\n' From ed2183c16cd32ebeeb1d248b343fad8635d21b7a Mon Sep 17 00:00:00 2001 From: xormania <127287135+xormania@users.noreply.github.com> Date: Sun, 2 Aug 2026 05:47:48 -0400 Subject: [PATCH 086/158] fix(experiment): traverse threaded descendants --- tests/experiment/local-image-catalog-cases.sh | 33 +++++++++++-------- 1 file changed, 19 insertions(+), 14 deletions(-) diff --git a/tests/experiment/local-image-catalog-cases.sh b/tests/experiment/local-image-catalog-cases.sh index c4427ed..3a98456 100755 --- a/tests/experiment/local-image-catalog-cases.sh +++ b/tests/experiment/local-image-catalog-cases.sh @@ -22,26 +22,31 @@ cleanup_work() { collect_descendants() { local parent_pid="$1" local children="" + local children_file local child local observed - if [ -r "/proc/$parent_pid/task/$parent_pid/children" ]; then - IFS= read -r children < "/proc/$parent_pid/task/$parent_pid/children" || true - fi - for child in $children; do - if [[ ! "$child" =~ ^[0-9]+$ ]]; then + for children_file in /proc/"$parent_pid"/task/[0-9]*/children; do + if [ ! -r "$children_file" ]; then continue fi - for observed in "${signal_descendants[@]}"; do - if [ "$observed" = "$child" ]; then - child="" - break + children="" + IFS= read -r children < "$children_file" || true + for child in $children; do + if [[ ! "$child" =~ ^[0-9]+$ ]]; then + continue + fi + for observed in "${signal_descendants[@]}"; do + if [ "$observed" = "$child" ]; then + child="" + break + fi + done + if [ -z "$child" ]; then + continue fi + signal_descendants[${#signal_descendants[@]}]="$child" + collect_descendants "$child" done - if [ -z "$child" ]; then - continue - fi - signal_descendants[${#signal_descendants[@]}]="$child" - collect_descendants "$child" done } From 925f56ecaf4d89a135910e2d431d42e858219047 Mon Sep 17 00:00:00 2001 From: xormania <127287135+xormania@users.noreply.github.com> Date: Sun, 2 Aug 2026 05:49:06 -0400 Subject: [PATCH 087/158] fix(experiment): rescan catalog descendants --- tests/experiment/local-image-catalog-cases.sh | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tests/experiment/local-image-catalog-cases.sh b/tests/experiment/local-image-catalog-cases.sh index 3a98456..bdb75a3 100755 --- a/tests/experiment/local-image-catalog-cases.sh +++ b/tests/experiment/local-image-catalog-cases.sh @@ -24,6 +24,7 @@ collect_descendants() { local children="" local children_file local child + local known local observed for children_file in /proc/"$parent_pid"/task/[0-9]*/children; do if [ ! -r "$children_file" ]; then @@ -35,16 +36,16 @@ collect_descendants() { if [[ ! "$child" =~ ^[0-9]+$ ]]; then continue fi + known=0 for observed in "${signal_descendants[@]}"; do if [ "$observed" = "$child" ]; then - child="" + known=1 break fi done - if [ -z "$child" ]; then - continue + if [ "$known" -eq 0 ]; then + signal_descendants[${#signal_descendants[@]}]="$child" fi - signal_descendants[${#signal_descendants[@]}]="$child" collect_descendants "$child" done done From 38ce590bec7d4ef26811a94ef43cef8400556e84 Mon Sep 17 00:00:00 2001 From: xormania <127287135+xormania@users.noreply.github.com> Date: Sun, 2 Aug 2026 05:54:38 -0400 Subject: [PATCH 088/158] fix(experiment): freeze catalog trees top down --- tests/experiment/local-image-catalog-cases.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/experiment/local-image-catalog-cases.sh b/tests/experiment/local-image-catalog-cases.sh index bdb75a3..ca803d1 100755 --- a/tests/experiment/local-image-catalog-cases.sh +++ b/tests/experiment/local-image-catalog-cases.sh @@ -44,6 +44,9 @@ collect_descendants() { fi done if [ "$known" -eq 0 ]; then + if ! kill -STOP -- "$child" 2>/dev/null; then + continue + fi signal_descendants[${#signal_descendants[@]}]="$child" fi collect_descendants "$child" @@ -61,9 +64,6 @@ catalog_signal() { for _ in 1 2 3 4 5 6 7 8; do before="${#signal_descendants[@]}" collect_descendants "$$" - for ((index = before; index < ${#signal_descendants[@]}; index++)); do - kill -STOP -- "${signal_descendants[index]}" 2>/dev/null || true - done if [ "${#signal_descendants[@]}" -eq "$before" ]; then break fi From 6cb0b84cee44abacca0fa6c68920b4df149390ae Mon Sep 17 00:00:00 2001 From: xormania <127287135+xormania@users.noreply.github.com> Date: Sun, 2 Aug 2026 06:07:31 -0400 Subject: [PATCH 089/158] test(experiment): define zip intake surface --- tests/dev/security-gate-cases.sh | 1 + tests/experiment/contract-cases.sh | 5 +- tests/experiment/source-adapter-cases.sh | 92 ++++++++++++++++++++++++ tests/experiment/zip-intake-cases.sh | 75 +++++++++++++++++++ tests/security/fast.manifest | 1 + 5 files changed, 170 insertions(+), 4 deletions(-) create mode 100755 tests/experiment/source-adapter-cases.sh create mode 100755 tests/experiment/zip-intake-cases.sh diff --git a/tests/dev/security-gate-cases.sh b/tests/dev/security-gate-cases.sh index 6f124e2..05e61f8 100644 --- a/tests/dev/security-gate-cases.sh +++ b/tests/dev/security-gate-cases.sh @@ -231,6 +231,7 @@ config-authority tests/agent/config-guard.sh SUMMARY failures=0 experiment-contract tests/experiment/contract-cases.sh EXPERIMENT CONTRACT PASS experiment-authorization tests/experiment/authorization-cases.sh EXPERIMENT AUTHORIZATION PASS experiment-local-lifecycle tests/experiment/local-lifecycle-cases.sh EXPERIMENT LOCAL LIFECYCLE PASS +experiment-source-adapters tests/experiment/source-adapter-cases.sh EXPERIMENT SOURCE ADAPTERS PASS config-matrix tests/agent/config-matrix.sh SUMMARY failures=0 allowlist-schema tests/agent/allowlist-cases.sh SUMMARY failures=0 image-volume-policy tests/agent/image-volume-policy-cases.sh SUMMARY failures=0 diff --git a/tests/experiment/contract-cases.sh b/tests/experiment/contract-cases.sh index 2d5211e..5231a89 100755 --- a/tests/experiment/contract-cases.sh +++ b/tests/experiment/contract-cases.sh @@ -3,11 +3,10 @@ set -u -o pipefail repo_root="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." >/dev/null 2>&1 && pwd)" subcases=( - "$repo_root/tests/experiment/directory-intake-cases.sh" "$repo_root/tests/experiment/aggregate-harness-cases.sh" "$repo_root/tests/experiment/catalog-aggregate-harness-cases.sh" ) -expected_count=33 +expected_count=20 work="" cleanup_work() { @@ -30,8 +29,6 @@ trap 'cleanup_work >/dev/null 2>&1 || true' EXIT expected="$work/expected" observed="$work/observed" printf '%s\n' \ - FMT-001 FMT-002 FMT-004 FMT-005 FMT-006 FMT-007 FMT-003 FMT-008 \ - SEL-001 CUE-001 FMT-009 M-FMT-001 SEL-002 \ AGG-001 AGG-002 AGG-003 AGG-004 AGG-005 AGG-006 AGG-007 AGG-008 AGG-009 \ AGG-010 AGG-011 AGG-012 AGG-013 AGG-014 AGG-015 AGG-016 AGG-017 \ AGG-018 AGG-019 AGG-020 > "$expected" diff --git a/tests/experiment/source-adapter-cases.sh b/tests/experiment/source-adapter-cases.sh new file mode 100755 index 0000000..cb5e9c8 --- /dev/null +++ b/tests/experiment/source-adapter-cases.sh @@ -0,0 +1,92 @@ +#!/usr/bin/env bash +set -u -o pipefail + +repo_root="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." >/dev/null 2>&1 && pwd)" +subcases=( + "$repo_root/tests/experiment/directory-intake-cases.sh" + "$repo_root/tests/experiment/zip-intake-cases.sh" +) +expected_count=14 +work="" + +cleanup_work() { + local failed=0 + if [ -n "$work" ] && [ -e "$work" ]; then + find "$work" -type f -delete 2>/dev/null || failed=1 + find "$work" -type l -delete 2>/dev/null || failed=1 + find "$work" -depth -type d -exec rmdir {} + 2>/dev/null || failed=1 + [ ! -e "$work" ] || failed=1 + fi + return "$failed" +} + +if ! work="$(mktemp -d)"; then + printf 'SUMMARY assertions=0 expected=%s failures=0 infra=1\n' "$expected_count" + exit 125 +fi +trap 'cleanup_work >/dev/null 2>&1 || true' EXIT + +expected="$work/expected" +observed="$work/observed" +printf '%s\n' \ + FMT-001 FMT-002 FMT-004 FMT-005 FMT-006 FMT-007 FMT-003 \ + FMT-008 SEL-001 CUE-001 FMT-009 M-FMT-001 SEL-002 ZIP-001 > "$expected" +: > "$observed" + +infrastructure=0 +failures=0 +for index in "${!subcases[@]}"; do + subcase="${subcases[$index]}" + output="$work/subcase-$index.out" + if [ ! -f "$subcase" ]; then + infrastructure=1 + continue + fi + if bash "$subcase" > "$output" 2>&1; then + rc=0 + else + rc=$? + fi + awk '/^(PASS|FAIL) [A-Z0-9-]+ / {print}' "$output" + awk '/^(PASS|FAIL) [A-Z0-9-]+ / {print $2}' "$output" >> "$observed" + reported_assertions="$(awk '/^(PASS|FAIL) [A-Z0-9-]+ / {count++} END {print count + 0}' "$output")" + reported_failures="$(awk '/^FAIL [A-Z0-9-]+ / {count++} END {print count + 0}' "$output")" + failures=$((failures + reported_failures)) + expected_summary="SUMMARY assertions=$reported_assertions expected=$reported_assertions failures=$reported_failures infra=0" + matching_summaries="$(grep -Fxc "$expected_summary" "$output" || true)" + all_summaries="$(grep -c '^SUMMARY ' "$output" || true)" + if [ "$matching_summaries" -ne 1 ] || [ "$all_summaries" -ne 1 ]; then + infrastructure=1 + fi + case "$rc" in + 0) + [ "$reported_failures" -eq 0 ] || infrastructure=1 + ;; + 1) + [ "$reported_failures" -ne 0 ] || infrastructure=1 + ;; + *) + infrastructure=1 + ;; + esac +done + +assertions="$(wc -l < "$observed")" +if ! cmp -s "$expected" "$observed"; then + failures=$((failures + 1)) +fi + +if ! cleanup_work; then + infrastructure=1 +fi +trap - EXIT + +printf 'SUMMARY assertions=%s expected=%s failures=%s infra=%s\n' \ + "$assertions" "$expected_count" "$failures" "$infrastructure" +if [ "$infrastructure" -ne 0 ]; then + exit 125 +fi +if [ "$failures" -ne 0 ]; then + exit 1 +fi +printf 'EXPERIMENT SOURCE ADAPTERS PASS\n' diff --git a/tests/experiment/zip-intake-cases.sh b/tests/experiment/zip-intake-cases.sh new file mode 100755 index 0000000..371b448 --- /dev/null +++ b/tests/experiment/zip-intake-cases.sh @@ -0,0 +1,75 @@ +#!/usr/bin/env bash +set -euo pipefail + +repo_root="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." >/dev/null 2>&1 && pwd)" +agent_lab="$repo_root/scripts/agent-lab" +fixture="$repo_root/tests/experiment/fixtures/directories/minimal" +work="$(mktemp -d)" +trap 'find "$work" -type f -delete 2>/dev/null || true; find "$work" -depth -type d -exec rmdir {} + 2>/dev/null || true' EXIT +mkdir -p "$work/home" "$work/tmp" + +python3 -I - "$fixture/experiment.cue" "$work/stored.zip" "$work/deflated.zip" <<'PY' +from pathlib import Path +import stat +import sys +import zipfile + +source = Path(sys.argv[1]).read_bytes() +for target, method in ( + (Path(sys.argv[2]), zipfile.ZIP_STORED), + (Path(sys.argv[3]), zipfile.ZIP_DEFLATED), +): + info = zipfile.ZipInfo("experiment.cue", date_time=(1980, 1, 1, 0, 0, 0)) + info.compress_type = method + info.create_system = 3 + info.external_attr = (stat.S_IFREG | 0o600) << 16 + with zipfile.ZipFile(target, "w") as archive: + archive.writestr(info, source) +PY + +capture() { + local name="$1" + shift + CAPTURE_RC=0 + env -i PATH=/usr/bin:/bin HOME="$work/home" TMPDIR="$work/tmp" LC_ALL=C \ + AGENT_LAB_CUE_TOOL_DIR="${AGENT_LAB_CUE_TOOL_DIR:-$repo_root/.cache/dev/tools/cue}" \ + "$@" > "$work/$name.out" 2> "$work/$name.err" || CAPTURE_RC=$? +} + +failures=0 +observed="$work/observed" +: > "$observed" +pass() { printf 'PASS %s %s\n' "$1" "$2"; printf '%s\n' "$1" >> "$observed"; } +fail() { printf 'FAIL %s %s\n' "$1" "$2"; printf '%s\n' "$1" >> "$observed"; failures=$((failures + 1)); } + +capture directory "$agent_lab" experiment check "$fixture" +directory_rc="$CAPTURE_RC" +capture stored "$agent_lab" experiment check --zip "$work/stored.zip" +stored_rc="$CAPTURE_RC" +capture deflated "$agent_lab" experiment check --zip "$work/deflated.zip" +deflated_rc="$CAPTURE_RC" + +if [ "$directory_rc" -eq 0 ] && [ "$stored_rc" -eq 0 ] && [ "$deflated_rc" -eq 0 ] && + [ ! -s "$work/directory.err" ] && [ ! -s "$work/stored.err" ] && + [ ! -s "$work/deflated.err" ] && + [ "$(jq -cS '.plan' "$work/directory.out")" = "$(jq -cS '.plan' "$work/stored.out")" ] && + [ "$(jq -cS '.plan' "$work/directory.out")" = "$(jq -cS '.plan' "$work/deflated.out")" ] && + [ "$(jq -r '.source.digest' "$work/directory.out")" = "$(jq -r '.source.digest' "$work/stored.out")" ] && + [ "$(jq -r '.source.digest' "$work/directory.out")" = "$(jq -r '.source.digest' "$work/deflated.out")" ] && + jq -e '.source.kind == "zip" and (.source.archiveDigest | startswith("sha256:"))' \ + "$work/stored.out" >/dev/null 2>&1 && + jq -e '.source.kind == "zip" and (.source.archiveDigest | startswith("sha256:"))' \ + "$work/deflated.out" >/dev/null 2>&1; then + pass ZIP-001 "public zip check normalizes stored and deflated sources" +else + fail ZIP-001 "public zip check normalizes stored and deflated sources" +fi + +expected="$work/expected" +printf '%s\n' ZIP-001 > "$expected" +if ! cmp -s "$expected" "$observed"; then + printf 'INFRA assertion identity drift\n' >&2 + exit 125 +fi +printf 'SUMMARY assertions=1 expected=1 failures=%s infra=0\n' "$failures" +[ "$failures" -eq 0 ] diff --git a/tests/security/fast.manifest b/tests/security/fast.manifest index 657b04e..23cebcc 100644 --- a/tests/security/fast.manifest +++ b/tests/security/fast.manifest @@ -53,6 +53,7 @@ suite config-authority tests/agent/config-guard.sh SUMMARY failures=0 suite experiment-contract tests/experiment/contract-cases.sh EXPERIMENT CONTRACT PASS suite experiment-authorization tests/experiment/authorization-cases.sh EXPERIMENT AUTHORIZATION PASS suite experiment-local-lifecycle tests/experiment/local-lifecycle-cases.sh EXPERIMENT LOCAL LIFECYCLE PASS +suite experiment-source-adapters tests/experiment/source-adapter-cases.sh EXPERIMENT SOURCE ADAPTERS PASS suite config-matrix tests/agent/config-matrix.sh SUMMARY failures=0 suite allowlist-schema tests/agent/allowlist-cases.sh SUMMARY failures=0 suite image-volume-policy tests/agent/image-volume-policy-cases.sh SUMMARY failures=0 From 99116824fef8d44f862b12bb23b2f3de56ff89ea Mon Sep 17 00:00:00 2001 From: xormania <127287135+xormania@users.noreply.github.com> Date: Sun, 2 Aug 2026 06:09:02 -0400 Subject: [PATCH 090/158] test(experiment): preserve source suite ownership --- tests/experiment/contract-cases.sh | 5 ++++- tests/experiment/source-adapter-cases.sh | 6 ++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/tests/experiment/contract-cases.sh b/tests/experiment/contract-cases.sh index 5231a89..2d5211e 100755 --- a/tests/experiment/contract-cases.sh +++ b/tests/experiment/contract-cases.sh @@ -3,10 +3,11 @@ set -u -o pipefail repo_root="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." >/dev/null 2>&1 && pwd)" subcases=( + "$repo_root/tests/experiment/directory-intake-cases.sh" "$repo_root/tests/experiment/aggregate-harness-cases.sh" "$repo_root/tests/experiment/catalog-aggregate-harness-cases.sh" ) -expected_count=20 +expected_count=33 work="" cleanup_work() { @@ -29,6 +30,8 @@ trap 'cleanup_work >/dev/null 2>&1 || true' EXIT expected="$work/expected" observed="$work/observed" printf '%s\n' \ + FMT-001 FMT-002 FMT-004 FMT-005 FMT-006 FMT-007 FMT-003 FMT-008 \ + SEL-001 CUE-001 FMT-009 M-FMT-001 SEL-002 \ AGG-001 AGG-002 AGG-003 AGG-004 AGG-005 AGG-006 AGG-007 AGG-008 AGG-009 \ AGG-010 AGG-011 AGG-012 AGG-013 AGG-014 AGG-015 AGG-016 AGG-017 \ AGG-018 AGG-019 AGG-020 > "$expected" diff --git a/tests/experiment/source-adapter-cases.sh b/tests/experiment/source-adapter-cases.sh index cb5e9c8..d6f90ea 100755 --- a/tests/experiment/source-adapter-cases.sh +++ b/tests/experiment/source-adapter-cases.sh @@ -3,10 +3,9 @@ set -u -o pipefail repo_root="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." >/dev/null 2>&1 && pwd)" subcases=( - "$repo_root/tests/experiment/directory-intake-cases.sh" "$repo_root/tests/experiment/zip-intake-cases.sh" ) -expected_count=14 +expected_count=1 work="" cleanup_work() { @@ -29,8 +28,7 @@ trap 'cleanup_work >/dev/null 2>&1 || true' EXIT expected="$work/expected" observed="$work/observed" printf '%s\n' \ - FMT-001 FMT-002 FMT-004 FMT-005 FMT-006 FMT-007 FMT-003 \ - FMT-008 SEL-001 CUE-001 FMT-009 M-FMT-001 SEL-002 ZIP-001 > "$expected" + ZIP-001 > "$expected" : > "$observed" infrastructure=0 From ffb3e1dcddcbaeb2d4e5a569e619a2b4512622ae Mon Sep 17 00:00:00 2001 From: xormania <127287135+xormania@users.noreply.github.com> Date: Sun, 2 Aug 2026 06:09:20 -0400 Subject: [PATCH 091/158] feat(experiment): accept zip checks --- scripts/agent-lab.py | 4 +++ scripts/experiment.py | 79 +++++++++++++++++++++++++++++++++++++------ 2 files changed, 72 insertions(+), 11 deletions(-) diff --git a/scripts/agent-lab.py b/scripts/agent-lab.py index e97d014..512d644 100644 --- a/scripts/agent-lab.py +++ b/scripts/agent-lab.py @@ -624,6 +624,10 @@ def main(argv: list[str]) -> int: return 125 print("tools:ready") return 0 + if argv[:3] == ["experiment", "check", "--zip"] and len(argv) == 4: + os.environ["AGENT_LAB_HOME"] = str(home) + os.environ.setdefault("AGENT_LAB_CUE_TOOL_DIR", str(home / "cache/tools/cue")) + return experiment_module().main(["experiment.py", "check-zip", argv[3]]) if argv[:2] == ["experiment", "check"] and len(argv) == 3: os.environ["AGENT_LAB_HOME"] = str(home) os.environ.setdefault("AGENT_LAB_CUE_TOOL_DIR", str(home / "cache/tools/cue")) diff --git a/scripts/experiment.py b/scripts/experiment.py index 2268c47..05b385e 100644 --- a/scripts/experiment.py +++ b/scripts/experiment.py @@ -12,11 +12,13 @@ import re import signal import stat +import struct import subprocess import sys import tempfile import time from typing import NamedTuple, NoReturn +import zlib MAX_MANIFEST_BYTES = 262_144 @@ -105,6 +107,7 @@ class PlanBinding(NamedTuple): class SourceSnapshot(NamedTuple): data: bytes digest: str + transport: dict[str, object] class PlanResolution(NamedTuple): @@ -225,6 +228,16 @@ def read_manifest_once( return data +def source_digest(data: bytes) -> str: + name = b"experiment.cue" + digest = hashlib.sha256(SOURCE_DIGEST_DOMAIN) + digest.update(len(name).to_bytes(4, "big")) + digest.update(name) + digest.update(len(data).to_bytes(8, "big")) + digest.update(data) + return f"sha256:{digest.hexdigest()}" + + def read_directory_snapshot(path: str) -> SourceSnapshot: try: directory_stat = os.lstat(path) @@ -259,13 +272,53 @@ def read_directory_snapshot(path: str) -> SourceSnapshot: final_directory_stat.st_ino, ): raise InfrastructureError("source directory changed while snapshotting") - name = b"experiment.cue" - digest = hashlib.sha256(SOURCE_DIGEST_DOMAIN) - digest.update(len(name).to_bytes(4, "big")) - digest.update(name) - digest.update(len(data).to_bytes(8, "big")) - digest.update(data) - return SourceSnapshot(data=data, digest=f"sha256:{digest.hexdigest()}") + return SourceSnapshot( + data=data, + digest=source_digest(data), + transport={"kind": "directory"}, + ) + + +def read_zip_snapshot(path: str) -> SourceSnapshot: + """Read the first local ZIP member into the common source snapshot.""" + + try: + archive = Path(path).read_bytes() + ( + signature, + _version, + _flags, + method, + _modified_time, + _modified_date, + _crc, + compressed_size, + _expanded_size, + name_size, + extra_size, + ) = struct.unpack_from("<4s5H3I2H", archive) + if signature != b"PK\x03\x04": + raise ValueError("local header") + start = 30 + name_size + extra_size + payload = archive[start : start + compressed_size] + if method == 0: + data = payload + elif method == 8: + data = zlib.decompress(payload, -15) + else: + raise ValueError("compression method") + except OSError as error: + raise InfrastructureError("zip archive could not be read") from error + except (struct.error, ValueError, zlib.error) as error: + raise InvalidManifest("zip archive is malformed") from error + return SourceSnapshot( + data=data, + digest=source_digest(data), + transport={ + "archiveDigest": "sha256:" + hashlib.sha256(archive).hexdigest(), + "kind": "zip", + }, + ) def authored_manifest(snapshot: SourceSnapshot) -> object: @@ -1408,17 +1461,21 @@ def write_decision(decision: object) -> None: def main(argv: list[str]) -> int: directory_checking = len(argv) == 3 and argv[1] == "check-directory" directory_authorizing = len(argv) == 3 and argv[1] == "authorize-directory" - if directory_checking or directory_authorizing: + zip_checking = len(argv) == 3 and argv[1] == "check-zip" + if directory_checking or directory_authorizing or zip_checking: try: - snapshot = read_directory_snapshot(argv[2]) + if zip_checking: + snapshot = read_zip_snapshot(argv[2]) + else: + snapshot = read_directory_snapshot(argv[2]) manifest = authored_manifest(snapshot) resolution = cue_plan_with_evidence(manifest) plan = resolution.plan - if directory_checking: + if directory_checking or zip_checking: checked: dict[str, object] = { "digest": plan_digest(plan), "plan": plan, - "source": {"digest": snapshot.digest, "kind": "directory"}, + "source": {"digest": snapshot.digest, **snapshot.transport}, } catalog = catalog_resolution_evidence( resolution.bundled_catalog, From afc8be4b9a6fc12556e7baaab941de9fffd414ae Mon Sep 17 00:00:00 2001 From: xormania <127287135+xormania@users.noreply.github.com> Date: Sun, 2 Aug 2026 06:13:33 -0400 Subject: [PATCH 092/158] test(experiment): define hostile zip boundaries --- tests/experiment/source-adapter-cases.sh | 7 +- tests/experiment/zip-fixtures.py | 248 +++++++++++++++++++++++ tests/experiment/zip-intake-cases.sh | 107 ++++++++-- 3 files changed, 340 insertions(+), 22 deletions(-) create mode 100644 tests/experiment/zip-fixtures.py diff --git a/tests/experiment/source-adapter-cases.sh b/tests/experiment/source-adapter-cases.sh index d6f90ea..900041f 100755 --- a/tests/experiment/source-adapter-cases.sh +++ b/tests/experiment/source-adapter-cases.sh @@ -5,7 +5,7 @@ repo_root="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." >/dev/null 2>&1 && subcases=( "$repo_root/tests/experiment/zip-intake-cases.sh" ) -expected_count=1 +expected_count=16 work="" cleanup_work() { @@ -28,7 +28,10 @@ trap 'cleanup_work >/dev/null 2>&1 || true' EXIT expected="$work/expected" observed="$work/observed" printf '%s\n' \ - ZIP-001 > "$expected" + ZIP-001 ZIP-PATH-001 ZIP-COUNT-001 ZIP-TYPE-001 ZIP-META-001 \ + ZIP-FLAG-001 ZIP-METHOD-001 ZIP-ZIP64-001 ZIP-HEADER-001 ZIP-CRC-001 \ + ZIP-LENGTH-001 ZIP-SIZE-001 ZIP-BOMB-001 ZIP-TRUNC-001 ZIP-TRAIL-001 \ + ZIP-READ-001 > "$expected" : > "$observed" infrastructure=0 diff --git a/tests/experiment/zip-fixtures.py b/tests/experiment/zip-fixtures.py new file mode 100644 index 0000000..e550234 --- /dev/null +++ b/tests/experiment/zip-fixtures.py @@ -0,0 +1,248 @@ +#!/usr/bin/env python3 +"""Generate deterministic harmless ZIP intake fixtures for the shell contract.""" + +from __future__ import annotations + +from pathlib import Path +import stat +import struct +import sys +import warnings +import zipfile + + +LOCAL = b"PK\x03\x04" +CENTRAL = b"PK\x01\x02" +EOCD = b"PK\x05\x06" + + +def zip_bytes( + entries: list[tuple[str, bytes, int, int, bytes, bytes]], + *, + archive_comment: bytes = b"", +) -> bytes: + from io import BytesIO + + output = BytesIO() + with warnings.catch_warnings(): + warnings.simplefilter("ignore", UserWarning) + with zipfile.ZipFile(output, "w") as archive: + archive.comment = archive_comment + for name, data, method, mode, extra, comment in entries: + info = zipfile.ZipInfo(name, date_time=(1980, 1, 1, 0, 0, 0)) + info.compress_type = method + info.create_system = 3 + info.external_attr = mode << 16 + info.extra = extra + info.comment = comment + archive.writestr(info, data) + return output.getvalue() + + +def one( + name: str, + data: bytes, + method: int = zipfile.ZIP_STORED, + mode: int = stat.S_IFREG | 0o600, + extra: bytes = b"", + comment: bytes = b"", + archive_comment: bytes = b"", +) -> bytes: + return zip_bytes( + [(name, data, method, mode, extra, comment)], + archive_comment=archive_comment, + ) + + +def offsets(data: bytes) -> tuple[int, int, int]: + local = data.find(LOCAL) + central = data.rfind(CENTRAL) + eocd = data.rfind(EOCD) + if local != 0 or central < 0 or eocd < 0: + raise ValueError("fixture has no canonical ZIP records") + return local, central, eocd + + +def u16(data: bytearray, offset: int, value: int) -> None: + struct.pack_into(" None: + struct.pack_into(" bytes: + data = bytearray(base) + operation(data, *offsets(data)) + return bytes(data) + + +def replace_names(base: bytes, replacement: bytes, *, local: bool = True, central: bool = True) -> bytes: + if len(replacement) != len(b"experiment.cue"): + raise ValueError("replacement name must preserve record lengths") + data = bytearray(base) + local_offset, central_offset, _ = offsets(data) + if local: + data[local_offset + 30 : local_offset + 44] = replacement + if central: + data[central_offset + 46 : central_offset + 60] = replacement + return bytes(data) + + +def main() -> int: + if len(sys.argv) != 3: + return 2 + source = Path(sys.argv[1]).read_bytes() + root = Path(sys.argv[2]) + root.mkdir(parents=True, exist_ok=True) + + regular = stat.S_IFREG | 0o600 + stored = one("experiment.cue", source) + deflated = one("experiment.cue", source, zipfile.ZIP_DEFLATED) + fixtures: dict[str, bytes] = { + "stored.zip": stored, + "deflated.zip": deflated, + "wrong-case.zip": one("Experiment.cue", source), + "wrapper.zip": one("wrapper/experiment.cue", source), + "dotdot.zip": one("../experiment.cue", source), + "backslash.zip": one("folder\\experiment.cue", source), + "absolute.zip": one("/experiment.cue", source), + "drive.zip": one("C:/experiment.cue", source), + "unc.zip": one("//host/share/experiment.cue", source), + "nonascii.zip": one("experıment.cue", source), + "extra-entry.zip": zip_bytes( + [ + ("experiment.cue", source, zipfile.ZIP_STORED, regular, b"", b""), + ("metadata", b"harmless", zipfile.ZIP_STORED, regular, b"", b""), + ] + ), + "duplicate.zip": zip_bytes( + [ + ("experiment.cue", source, zipfile.ZIP_STORED, regular, b"", b""), + ("experiment.cue", source, zipfile.ZIP_STORED, regular, b"", b""), + ] + ), + "directory-type.zip": one("experiment.cue", source, mode=stat.S_IFDIR | 0o700), + "symlink-type.zip": one("experiment.cue", source, mode=stat.S_IFLNK | 0o777), + "fifo-type.zip": one("experiment.cue", source, mode=stat.S_IFIFO | 0o600), + "extra-field.zip": one("experiment.cue", source, extra=b"\xfe\xca\x00\x00"), + "file-comment.zip": one("experiment.cue", source, comment=b"metadata"), + "archive-comment.zip": one("experiment.cue", source, archive_comment=b"metadata"), + } + + fixtures["nul-name.zip"] = replace_names(stored, b"experiment.cu\x00") + fixtures["control-name.zip"] = replace_names(stored, b"experiment.cu\x01") + fixtures["encrypted.zip"] = mutate( + stored, + lambda data, local, central, _eocd: ( + u16(data, local + 6, struct.unpack_from("/dev/null || true; find "$work" -depth -type d -exec rmdir {} + 2>/dev/null || true' EXIT mkdir -p "$work/home" "$work/tmp" -python3 -I - "$fixture/experiment.cue" "$work/stored.zip" "$work/deflated.zip" <<'PY' -from pathlib import Path -import stat -import sys -import zipfile - -source = Path(sys.argv[1]).read_bytes() -for target, method in ( - (Path(sys.argv[2]), zipfile.ZIP_STORED), - (Path(sys.argv[3]), zipfile.ZIP_DEFLATED), -): - info = zipfile.ZipInfo("experiment.cue", date_time=(1980, 1, 1, 0, 0, 0)) - info.compress_type = method - info.create_system = 3 - info.external_attr = (stat.S_IFREG | 0o600) << 16 - with zipfile.ZipFile(target, "w") as archive: - archive.writestr(info, source) -PY +if ! python3 -I -B "$repo_root/tests/experiment/zip-fixtures.py" \ + "$fixture/experiment.cue" "$work"; then + printf 'SUMMARY assertions=0 expected=16 failures=0 infra=1\n' + exit 125 +fi capture() { local name="$1" @@ -41,6 +28,25 @@ observed="$work/observed" : > "$observed" pass() { printf 'PASS %s %s\n' "$1" "$2"; printf '%s\n' "$1" >> "$observed"; } fail() { printf 'FAIL %s %s\n' "$1" "$2"; printf '%s\n' "$1" >> "$observed"; failures=$((failures + 1)); } +expect_all_reject() { + local id="$1" code="$2" detail="$3" + shift 3 + local archive accepted=0 + for archive in "$@"; do + capture "reject-${archive%.zip}" "$agent_lab" experiment check --zip "$work/$archive" + if [ "$CAPTURE_RC" -ne 1 ] || [ -s "$work/reject-${archive%.zip}.out" ] || + [ "$(wc -l < "$work/reject-${archive%.zip}.err")" -ne 1 ] || + ! grep -Fq "FAIL Experiment manifest zip archive $code" \ + "$work/reject-${archive%.zip}.err"; then + accepted=1 + fi + done + if [ "$accepted" -eq 0 ]; then + pass "$id" "$detail" + else + fail "$id" "$detail" + fi +} capture directory "$agent_lab" experiment check "$fixture" directory_rc="$CAPTURE_RC" @@ -65,11 +71,72 @@ else fail ZIP-001 "public zip check normalizes stored and deflated sources" fi +expect_all_reject ZIP-PATH-001 ZIP-PATH "only the exact ASCII root member name is accepted" \ + wrong-case.zip wrapper.zip dotdot.zip backslash.zip absolute.zip drive.zip unc.zip \ + nul-name.zip control-name.zip nonascii.zip + +expect_all_reject ZIP-COUNT-001 ZIP-COUNT "the archive has exactly one member" \ + zero-count.zip extra-entry.zip duplicate.zip + +expect_all_reject ZIP-TYPE-001 ZIP-TYPE "the sole member is a regular file" \ + directory-type.zip symlink-type.zip fifo-type.zip + +expect_all_reject ZIP-META-001 ZIP-META "member and archive metadata are closed" \ + extra-field.zip file-comment.zip archive-comment.zip + +expect_all_reject ZIP-FLAG-001 ZIP-FLAG "encrypted and descriptor-based members are refused" \ + encrypted.zip strong-encrypted.zip data-descriptor.zip + +expect_all_reject ZIP-METHOD-001 ZIP-METHOD "only stored and raw deflate members are accepted" \ + unsupported-method.zip + +expect_all_reject ZIP-ZIP64-001 ZIP-ZIP64 "ZIP64 and multidisk records are refused" \ + zip64-version.zip zip64-sentinel.zip multidisk.zip + +expect_all_reject ZIP-HEADER-001 ZIP-HEADER "central and local headers agree exactly" \ + central-signature.zip central-name-mismatch.zip central-flags-mismatch.zip + +expect_all_reject ZIP-CRC-001 ZIP-CRC "member CRC is verified" bad-crc.zip + +expect_all_reject ZIP-LENGTH-001 ZIP-LENGTH "declared and decoded lengths agree" \ + bad-length.zip + +expect_all_reject ZIP-SIZE-001 ZIP-SIZE "archive and expanded source bounds apply before planning" \ + archive-over.zip expanded-over.zip deflate-bomb.zip + +expect_all_reject ZIP-BOMB-001 ZIP-BOMB "declared-small deflate expansion stops at the source bound" \ + declared-small-large.zip + +expect_all_reject ZIP-TRUNC-001 ZIP-TRUNC "truncated records and deflate streams are refused" \ + missing-central.zip truncated-deflate.zip + +expect_all_reject ZIP-TRAIL-001 ZIP-TRAIL "bytes after the canonical archive end are refused" trailing.zip + +ln -s "$work/stored.zip" "$work/archive-link.zip" +mkdir "$work/archive-directory" +capture missing "$agent_lab" experiment check --zip "$work/missing.zip" +missing_rc="$CAPTURE_RC" +capture linked "$agent_lab" experiment check --zip "$work/archive-link.zip" +linked_rc="$CAPTURE_RC" +capture archive-directory "$agent_lab" experiment check --zip "$work/archive-directory" +directory_path_rc="$CAPTURE_RC" +if [ "$missing_rc" -eq 125 ] && [ "$linked_rc" -eq 125 ] && + [ "$directory_path_rc" -eq 125 ] && [ ! -s "$work/missing.out" ] && + [ ! -s "$work/linked.out" ] && [ ! -s "$work/archive-directory.out" ]; then + pass ZIP-READ-001 "unavailable or unsafe archive paths are infrastructure failures" +else + fail ZIP-READ-001 "unavailable or unsafe archive paths are infrastructure failures" +fi + expected="$work/expected" -printf '%s\n' ZIP-001 > "$expected" +printf '%s\n' \ + ZIP-001 ZIP-PATH-001 ZIP-COUNT-001 ZIP-TYPE-001 ZIP-META-001 \ + ZIP-FLAG-001 ZIP-METHOD-001 ZIP-ZIP64-001 ZIP-HEADER-001 ZIP-CRC-001 \ + ZIP-LENGTH-001 ZIP-SIZE-001 ZIP-BOMB-001 ZIP-TRUNC-001 ZIP-TRAIL-001 \ + ZIP-READ-001 > "$expected" if ! cmp -s "$expected" "$observed"; then printf 'INFRA assertion identity drift\n' >&2 exit 125 fi -printf 'SUMMARY assertions=1 expected=1 failures=%s infra=0\n' "$failures" +printf 'SUMMARY assertions=16 expected=16 failures=%s infra=0\n' "$failures" [ "$failures" -eq 0 ] From f26bb97b049a51d273cadcff5a114c8ddf8668fa Mon Sep 17 00:00:00 2001 From: xormania <127287135+xormania@users.noreply.github.com> Date: Sun, 2 Aug 2026 06:16:04 -0400 Subject: [PATCH 093/158] fix(experiment): bound zip archive decoding --- scripts/experiment.py | 311 ++++++++++++++++++++++++++++++++++++++---- 1 file changed, 282 insertions(+), 29 deletions(-) diff --git a/scripts/experiment.py b/scripts/experiment.py index 05b385e..c41b3ec 100644 --- a/scripts/experiment.py +++ b/scripts/experiment.py @@ -22,6 +22,9 @@ MAX_MANIFEST_BYTES = 262_144 +MAX_ARCHIVE_BYTES = 1_048_576 +MAX_SOURCE_BYTES = MAX_MANIFEST_BYTES +ZIP_DECODE_TIMEOUT_SECONDS = 5 SOURCE_DIGEST_DOMAIN = b"agent-lab.experiment-tree.v1\0" PLAN_DOMAIN = b"agent-lab.experiment-plan.v1\0" BUNDLED_CATALOG_DOMAIN = b"agent-lab.experiment-image-catalog.v1\0" @@ -238,6 +241,138 @@ def source_digest(data: bytes) -> str: return f"sha256:{digest.hexdigest()}" +def _zip_reject(code: str, detail: str) -> NoReturn: + raise InvalidManifest(f"zip archive {code} {detail}") + + +def _read_zip_archive_once(path: str) -> bytes: + try: + path_stat = os.lstat(path) + except OSError as error: + raise InfrastructureError("zip archive ZIP-READ cannot be inspected") from error + if stat.S_ISLNK(path_stat.st_mode) or not stat.S_ISREG(path_stat.st_mode): + raise InfrastructureError("zip archive ZIP-READ is not a safe regular file") + if path_stat.st_size > MAX_ARCHIVE_BYTES: + _zip_reject("ZIP-SIZE", f"exceeds the {MAX_ARCHIVE_BYTES}-byte limit") + + flags = os.O_RDONLY | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_NONBLOCK", 0) + flags |= getattr(os, "O_NOFOLLOW", 0) + try: + descriptor = os.open(path, flags) + except OSError as error: + raise InfrastructureError("zip archive ZIP-READ cannot be opened") from error + + opened_stat: os.stat_result | None = None + final_stat: os.stat_result | None = None + try: + opened_stat = os.fstat(descriptor) + if ( + not stat.S_ISREG(opened_stat.st_mode) + or _manifest_identity(opened_stat) != _manifest_identity(path_stat) + ): + raise InfrastructureError("zip archive ZIP-READ identity changed before read") + chunks: list[bytes] = [] + remaining = MAX_ARCHIVE_BYTES + 1 + while remaining: + chunk = os.read(descriptor, min(65_536, remaining)) + if not chunk: + break + chunks.append(chunk) + remaining -= len(chunk) + archive = b"".join(chunks) + final_stat = os.fstat(descriptor) + except InfrastructureError: + raise + except OSError as error: + raise InfrastructureError("zip archive ZIP-READ could not be read completely") from error + finally: + try: + os.close(descriptor) + except OSError as error: + raise InfrastructureError("zip archive ZIP-READ descriptor could not be closed") from error + + if len(archive) > MAX_ARCHIVE_BYTES: + _zip_reject("ZIP-SIZE", f"exceeds the {MAX_ARCHIVE_BYTES}-byte limit") + assert opened_stat is not None and final_stat is not None + try: + current_stat = os.lstat(path) + except OSError as error: + raise InfrastructureError("zip archive ZIP-READ cannot be reverified") from error + expected = _manifest_identity(path_stat) + if ( + _manifest_identity(opened_stat) != expected + or _manifest_identity(final_stat) != expected + or _manifest_identity(current_stat) != expected + or len(archive) != final_stat.st_size + ): + raise InfrastructureError("zip archive ZIP-READ changed while being read") + return archive + + +def _zip_eocd(archive: bytes) -> tuple[int, tuple[object, ...]]: + eocd_offset = archive.rfind(b"PK\x05\x06") + if eocd_offset < 0 or len(archive) - eocd_offset < 22: + _zip_reject("ZIP-TRUNC", "has no complete end record") + try: + record = struct.unpack_from("<4s4H2IH", archive, eocd_offset) + except struct.error as error: + raise InvalidManifest("zip archive ZIP-TRUNC has an incomplete end record") from error + comment_size = int(record[-1]) + if eocd_offset + 22 + comment_size != len(archive): + _zip_reject("ZIP-TRAIL", "has bytes outside its canonical end") + if comment_size != 0: + _zip_reject("ZIP-META", "has an archive comment") + return eocd_offset, record + + +def _zip_decode(payload: bytes, expanded_size: int) -> bytes: + deadline = time.monotonic() + ZIP_DECODE_TIMEOUT_SECONDS + try: + decoder = zlib.decompressobj(-15) + output: list[bytes] = [] + produced = 0 + offset = 0 + while offset < len(payload): + if time.monotonic() >= deadline: + raise TimeoutError("zip decoder deadline") + chunk = payload[offset : offset + 65_536] + offset += len(chunk) + while chunk: + remaining = MAX_SOURCE_BYTES + 1 - produced + if remaining <= 0: + _zip_reject("ZIP-BOMB", "expands beyond the source limit") + decoded = decoder.decompress(chunk, remaining) + output.append(decoded) + produced += len(decoded) + chunk = decoder.unconsumed_tail + if produced > MAX_SOURCE_BYTES: + _zip_reject("ZIP-BOMB", "expands beyond the source limit") + if time.monotonic() >= deadline: + raise TimeoutError("zip decoder deadline") + remaining = MAX_SOURCE_BYTES + 1 - produced + decoded = decoder.flush(remaining) + output.append(decoded) + produced += len(decoded) + except InvalidManifest: + raise + except zlib.error as error: + raise InvalidManifest("zip archive ZIP-TRUNC has an invalid deflate stream") from error + except TimeoutError as error: + raise InfrastructureError("zip archive ZIP-TIMEOUT decoder deadline expired") from error + except Exception as error: + raise InfrastructureError("zip archive ZIP-DECODE decoder result is uncertain") from error + if produced > MAX_SOURCE_BYTES: + _zip_reject("ZIP-BOMB", "expands beyond the source limit") + if not decoder.eof: + _zip_reject("ZIP-TRUNC", "deflate stream ended early") + if decoder.unused_data or decoder.unconsumed_tail: + _zip_reject("ZIP-TRAIL", "deflate stream has unused input") + data = b"".join(output) + if len(data) != expanded_size: + _zip_reject("ZIP-LENGTH", "decoded length disagrees with its header") + return data + + def read_directory_snapshot(path: str) -> SourceSnapshot: try: directory_stat = os.lstat(path) @@ -280,41 +415,159 @@ def read_directory_snapshot(path: str) -> SourceSnapshot: def read_zip_snapshot(path: str) -> SourceSnapshot: - """Read the first local ZIP member into the common source snapshot.""" + """Inspect and bounded-decode one canonical in-memory ZIP source.""" + + archive = _read_zip_archive_once(path) + eocd_offset, eocd = _zip_eocd(archive) + ( + _signature, + disk_number, + central_disk, + disk_entries, + total_entries, + central_size, + central_offset, + _comment_size, + ) = eocd + if ( + disk_number != 0 + or central_disk != 0 + or disk_entries == 0xFFFF + or total_entries == 0xFFFF + or central_size == 0xFFFFFFFF + or central_offset == 0xFFFFFFFF + ): + _zip_reject("ZIP-ZIP64", "uses ZIP64 or multiple disks") + if disk_entries != 1 or total_entries != 1: + _zip_reject("ZIP-COUNT", "must contain exactly one entry") + if central_offset + central_size != eocd_offset: + _zip_reject("ZIP-HEADER", "central directory bounds disagree") + if central_size < 46 or central_offset < 30: + _zip_reject("ZIP-TRUNC", "central directory is incomplete") + try: + central = struct.unpack_from("<4s6H3I5H2I", archive, central_offset) + except struct.error as error: + raise InvalidManifest("zip archive ZIP-TRUNC central header is incomplete") from error + ( + central_signature, + version_made, + version_needed, + flags, + method, + modified_time, + modified_date, + crc, + compressed_size, + expanded_size, + name_size, + central_extra_size, + member_comment_size, + member_disk, + _internal_attributes, + external_attributes, + local_offset, + ) = central + if central_signature != b"PK\x01\x02": + _zip_reject("ZIP-HEADER", "central signature is invalid") + if ( + version_needed >= 45 + or (version_made & 0xFF) >= 45 + or compressed_size == 0xFFFFFFFF + or expanded_size == 0xFFFFFFFF + or local_offset == 0xFFFFFFFF + or member_disk != 0 + ): + _zip_reject("ZIP-ZIP64", "uses ZIP64 or multiple disks") + central_record_size = 46 + name_size + central_extra_size + member_comment_size + if central_record_size != central_size: + _zip_reject("ZIP-HEADER", "central record size disagrees") + if central_offset + central_record_size > eocd_offset: + _zip_reject("ZIP-TRUNC", "central record is incomplete") + if central_extra_size != 0 or member_comment_size != 0: + _zip_reject("ZIP-META", "contains member metadata") + if flags & ~0x800: + _zip_reject("ZIP-FLAG", "uses unsupported general-purpose flags") + if method not in (0, 8): + _zip_reject("ZIP-METHOD", "uses unsupported compression") + if expanded_size > MAX_SOURCE_BYTES: + _zip_reject("ZIP-SIZE", f"source exceeds the {MAX_SOURCE_BYTES}-byte limit") + central_name = archive[central_offset + 46 : central_offset + 46 + name_size] + create_system = version_made >> 8 + unix_mode = external_attributes >> 16 + unix_type = stat.S_IFMT(unix_mode) + dos_attributes = external_attributes & 0xFFFF + if ( + dos_attributes & 0x10 + or (create_system == 3 and unix_type not in (0, stat.S_IFREG)) + ): + _zip_reject("ZIP-TYPE", "member is not a regular file") + if local_offset != 0: + _zip_reject("ZIP-HEADER", "local header is not at the canonical offset") try: - archive = Path(path).read_bytes() - ( - signature, - _version, - _flags, - method, - _modified_time, - _modified_date, - _crc, - compressed_size, - _expanded_size, - name_size, - extra_size, - ) = struct.unpack_from("<4s5H3I2H", archive) - if signature != b"PK\x03\x04": - raise ValueError("local header") - start = 30 + name_size + extra_size - payload = archive[start : start + compressed_size] - if method == 0: - data = payload - elif method == 8: - data = zlib.decompress(payload, -15) - else: - raise ValueError("compression method") - except OSError as error: - raise InfrastructureError("zip archive could not be read") from error - except (struct.error, ValueError, zlib.error) as error: - raise InvalidManifest("zip archive is malformed") from error + local = struct.unpack_from("<4s5H3I2H", archive, local_offset) + except struct.error as error: + raise InvalidManifest("zip archive ZIP-TRUNC local header is incomplete") from error + ( + local_signature, + local_version, + local_flags, + local_method, + local_time, + local_date, + local_crc, + local_compressed_size, + local_expanded_size, + local_name_size, + local_extra_size, + ) = local + if local_signature != b"PK\x03\x04": + _zip_reject("ZIP-HEADER", "local signature is invalid") + local_record_size = 30 + local_name_size + local_extra_size + if local_record_size > central_offset: + _zip_reject("ZIP-TRUNC", "local header is incomplete") + local_name = archive[30 : 30 + local_name_size] + if local_extra_size != 0: + _zip_reject("ZIP-META", "contains local member metadata") + if ( + local_version != version_needed + or local_flags != flags + or local_method != method + or local_time != modified_time + or local_date != modified_date + or local_crc != crc + or local_compressed_size != compressed_size + or local_expanded_size != expanded_size + or local_name != central_name + ): + _zip_reject("ZIP-HEADER", "local and central records disagree") + try: + decoded_name = central_name.decode("ascii") + except UnicodeError as error: + raise InvalidManifest("zip archive ZIP-PATH member name is not ASCII") from error + if decoded_name != "experiment.cue": + _zip_reject("ZIP-PATH", "member name must be exactly experiment.cue") + payload_end = local_record_size + compressed_size + if payload_end != central_offset: + _zip_reject("ZIP-LENGTH", "compressed payload bounds disagree") + payload = archive[local_record_size:payload_end] + if len(payload) != compressed_size: + _zip_reject("ZIP-TRUNC", "compressed payload is incomplete") + if method == 0: + if compressed_size != expanded_size: + _zip_reject("ZIP-LENGTH", "stored member lengths disagree") + data = payload + else: + data = _zip_decode(payload, expanded_size) + if len(data) != expanded_size: + _zip_reject("ZIP-LENGTH", "decoded length disagrees with its header") + if (zlib.crc32(data) & 0xFFFFFFFF) != crc: + _zip_reject("ZIP-CRC", "member checksum disagrees") return SourceSnapshot( data=data, digest=source_digest(data), transport={ + "archiveBytes": len(archive), "archiveDigest": "sha256:" + hashlib.sha256(archive).hexdigest(), "kind": "zip", }, From 89574fa3317aaf96ca8fd29fa38a094d76a1b31c Mon Sep 17 00:00:00 2001 From: xormania <127287135+xormania@users.noreply.github.com> Date: Sun, 2 Aug 2026 06:17:08 -0400 Subject: [PATCH 094/158] test(experiment): require common zip authorization and install --- tests/experiment/source-adapter-cases.sh | 4 +- tests/experiment/zip-intake-cases.sh | 75 +++++++++++++++++++++++- 2 files changed, 74 insertions(+), 5 deletions(-) diff --git a/tests/experiment/source-adapter-cases.sh b/tests/experiment/source-adapter-cases.sh index 900041f..60d2189 100755 --- a/tests/experiment/source-adapter-cases.sh +++ b/tests/experiment/source-adapter-cases.sh @@ -5,7 +5,7 @@ repo_root="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." >/dev/null 2>&1 && subcases=( "$repo_root/tests/experiment/zip-intake-cases.sh" ) -expected_count=16 +expected_count=19 work="" cleanup_work() { @@ -31,7 +31,7 @@ printf '%s\n' \ ZIP-001 ZIP-PATH-001 ZIP-COUNT-001 ZIP-TYPE-001 ZIP-META-001 \ ZIP-FLAG-001 ZIP-METHOD-001 ZIP-ZIP64-001 ZIP-HEADER-001 ZIP-CRC-001 \ ZIP-LENGTH-001 ZIP-SIZE-001 ZIP-BOMB-001 ZIP-TRUNC-001 ZIP-TRAIL-001 \ - ZIP-READ-001 > "$expected" + ZIP-READ-001 ZIP-AUTH-001 ZIP-INSTALL-001 ZIP-RETRY-001 > "$expected" : > "$observed" infrastructure=0 diff --git a/tests/experiment/zip-intake-cases.sh b/tests/experiment/zip-intake-cases.sh index 01223a6..ce6e5c4 100755 --- a/tests/experiment/zip-intake-cases.sh +++ b/tests/experiment/zip-intake-cases.sh @@ -10,7 +10,7 @@ mkdir -p "$work/home" "$work/tmp" if ! python3 -I -B "$repo_root/tests/experiment/zip-fixtures.py" \ "$fixture/experiment.cue" "$work"; then - printf 'SUMMARY assertions=0 expected=16 failures=0 infra=1\n' + printf 'SUMMARY assertions=0 expected=19 failures=0 infra=1\n' exit 125 fi @@ -20,6 +20,7 @@ capture() { CAPTURE_RC=0 env -i PATH=/usr/bin:/bin HOME="$work/home" TMPDIR="$work/tmp" LC_ALL=C \ AGENT_LAB_CUE_TOOL_DIR="${AGENT_LAB_CUE_TOOL_DIR:-$repo_root/.cache/dev/tools/cue}" \ + AGENT_LAB_CEDAR_TOOL_DIR="${AGENT_LAB_CEDAR_TOOL_DIR:-$repo_root/.cache/dev/tools/cedar}" \ "$@" > "$work/$name.out" 2> "$work/$name.err" || CAPTURE_RC=$? } @@ -28,6 +29,16 @@ observed="$work/observed" : > "$observed" pass() { printf 'PASS %s %s\n' "$1" "$2"; printf '%s\n' "$1" >> "$observed"; } fail() { printf 'FAIL %s %s\n' "$1" "$2"; printf '%s\n' "$1" >> "$observed"; failures=$((failures + 1)); } +init_home() { + local label="$1" home="$2" + capture "$label" "$agent_lab" --home "$home" init + if [ "$CAPTURE_RC" -ne 0 ]; then + printf 'INFRA temporary Agent Lab home initialization failed\n' >&2 + printf 'SUMMARY assertions=%s expected=19 failures=%s infra=1\n' \ + "$(wc -l < "$observed")" "$failures" + exit 125 + fi +} expect_all_reject() { local id="$1" code="$2" detail="$3" shift 3 @@ -128,15 +139,73 @@ else fail ZIP-READ-001 "unavailable or unsafe archive paths are infrastructure failures" fi +capture directory-authorize "$agent_lab" experiment authorize install "$fixture" +directory_authorize_rc="$CAPTURE_RC" +capture zip-authorize "$agent_lab" experiment authorize install --zip "$work/stored.zip" +zip_authorize_rc="$CAPTURE_RC" +if [ "$directory_authorize_rc" -eq 0 ] && [ "$zip_authorize_rc" -eq 0 ] && + [ ! -s "$work/directory-authorize.err" ] && [ ! -s "$work/zip-authorize.err" ] && + cmp -s "$work/directory-authorize.out" "$work/zip-authorize.out" && + jq -e '.verdict == "permit" and (.binding.sourceDigest | startswith("sha256:"))' \ + "$work/zip-authorize.out" >/dev/null 2>&1; then + pass ZIP-AUTH-001 "zip authorization uses the directory decision path and identity" +else + fail ZIP-AUTH-001 "zip authorization uses the directory decision path and identity" +fi + +zip_home="$work/zip-home" +init_home zip-home-init "$zip_home" +capture zip-install "$agent_lab" --home "$zip_home" experiment install --zip "$work/stored.zip" +zip_install_rc="$CAPTURE_RC" +zip_target="$zip_home/experiments/first-experiment" +archive_digest="sha256:$(sha256sum "$work/stored.zip" | awk '{print $1}')" +archive_bytes="$(wc -c < "$work/stored.zip")" +source_identity="$(jq -r '.source.digest' "$work/directory.out")" +if [ "$zip_install_rc" -eq 0 ] && [ ! -s "$work/zip-install.err" ] && + jq -e '.changed == true and .name == "first-experiment"' \ + "$work/zip-install.out" >/dev/null 2>&1 && + cmp -s "$fixture/experiment.cue" "$zip_target/artifact/experiment.cue" && + jq -e --arg archive_digest "$archive_digest" --argjson archive_bytes "$archive_bytes" \ + --arg source_digest "$source_identity" \ + '.source.digest == $source_digest and + .transport == {archiveBytes: $archive_bytes, archiveDigest: $archive_digest, kind: "zip"}' \ + "$zip_target/records/provenance.json" >/dev/null 2>&1; then + pass ZIP-INSTALL-001 "zip install publishes the common artifact with closed archive provenance" +else + fail ZIP-INSTALL-001 "zip install publishes the common artifact with closed archive provenance" +fi + +retry_home="$work/retry-home" +init_home retry-home-init "$retry_home" +capture directory-first "$agent_lab" --home "$retry_home" experiment install "$fixture" +directory_first_rc="$CAPTURE_RC" +retry_receipt="$retry_home/experiments/first-experiment/records/install.json" +if [ "$directory_first_rc" -eq 0 ] && [ -f "$retry_receipt" ]; then + cp "$retry_receipt" "$work/retry-receipt-before" +else + : > "$work/retry-receipt-before" +fi +capture zip-retry "$agent_lab" --home "$retry_home" experiment install --zip "$work/deflated.zip" +zip_retry_rc="$CAPTURE_RC" +if [ "$directory_first_rc" -eq 0 ] && [ "$zip_retry_rc" -eq 0 ] && + [ ! -s "$work/directory-first.err" ] && [ ! -s "$work/zip-retry.err" ] && + jq -e '.changed == false and .name == "first-experiment"' \ + "$work/zip-retry.out" >/dev/null 2>&1 && + cmp -s "$work/retry-receipt-before" "$retry_receipt"; then + pass ZIP-RETRY-001 "equivalent zip retry preserves the directory installation receipt" +else + fail ZIP-RETRY-001 "equivalent zip retry preserves the directory installation receipt" +fi + expected="$work/expected" printf '%s\n' \ ZIP-001 ZIP-PATH-001 ZIP-COUNT-001 ZIP-TYPE-001 ZIP-META-001 \ ZIP-FLAG-001 ZIP-METHOD-001 ZIP-ZIP64-001 ZIP-HEADER-001 ZIP-CRC-001 \ ZIP-LENGTH-001 ZIP-SIZE-001 ZIP-BOMB-001 ZIP-TRUNC-001 ZIP-TRAIL-001 \ - ZIP-READ-001 > "$expected" + ZIP-READ-001 ZIP-AUTH-001 ZIP-INSTALL-001 ZIP-RETRY-001 > "$expected" if ! cmp -s "$expected" "$observed"; then printf 'INFRA assertion identity drift\n' >&2 exit 125 fi -printf 'SUMMARY assertions=16 expected=16 failures=%s infra=0\n' "$failures" +printf 'SUMMARY assertions=19 expected=19 failures=%s infra=0\n' "$failures" [ "$failures" -eq 0 ] From 60de36377285fbf66345e22ac05f9e8e5f043530 Mon Sep 17 00:00:00 2001 From: xormania <127287135+xormania@users.noreply.github.com> Date: Sun, 2 Aug 2026 06:19:28 -0400 Subject: [PATCH 095/158] feat(experiment): install zip snapshots through the common store --- scripts/agent-lab.py | 23 ++++++++--- scripts/experiment.py | 5 ++- scripts/experiment_store.py | 78 +++++++++++++++++++++++++++++++++---- 3 files changed, 92 insertions(+), 14 deletions(-) diff --git a/scripts/agent-lab.py b/scripts/agent-lab.py index 512d644..5f1f542 100644 --- a/scripts/agent-lab.py +++ b/scripts/agent-lab.py @@ -508,11 +508,20 @@ def image_command(home: Path, argv: list[str]) -> int: def experiment_command(home: Path, argv: list[str]) -> int: if argv[:1] == ["install"] and len(argv) == 2: operation = "install" + source_kind = "directory" + elif argv[:2] == ["install", "--zip"] and len(argv) == 3: + operation = "install" + source_kind = "zip" elif argv[:1] == ["inspect"] and len(argv) == 2: operation = "inspect" + source_kind = None else: return 2 + if operation == "install" and sys.platform != "linux": + print("INFRA Agent Lab Experiment installation requires Linux", file=sys.stderr) + return 125 + try: loaded = load_config_receipt(home) except RuntimeError as error: @@ -522,10 +531,6 @@ def experiment_command(home: Path, argv: list[str]) -> int: print("FAIL Agent Lab home is not initialized", file=sys.stderr) return 1 - if operation == "install" and sys.platform != "linux": - print("INFRA Agent Lab Experiment installation requires Linux", file=sys.stderr) - return 125 - paths = loaded[0]["paths"] assert isinstance(paths, dict) cache = home / str(paths["cache"]) / "tools" @@ -541,7 +546,10 @@ def experiment_command(home: Path, argv: list[str]) -> int: try: if operation == "install": - result = store.install_directory(home, Path(argv[1])) + if source_kind == "zip": + result = store.install_zip(home, Path(argv[2])) + else: + result = store.install_directory(home, Path(argv[1])) else: result = store.inspect_install(home, argv[1]) if not isinstance(result, dict): @@ -637,6 +645,11 @@ def main(argv: list[str]) -> int: os.environ.setdefault("AGENT_LAB_CUE_TOOL_DIR", str(home / "cache/tools/cue")) os.environ.setdefault("AGENT_LAB_CEDAR_TOOL_DIR", str(home / "cache/tools/cedar")) return experiment_module().main(["experiment.py", "authorize-directory", argv[3]]) + if argv[:4] == ["experiment", "authorize", "install", "--zip"] and len(argv) == 5: + os.environ["AGENT_LAB_HOME"] = str(home) + os.environ.setdefault("AGENT_LAB_CUE_TOOL_DIR", str(home / "cache/tools/cue")) + os.environ.setdefault("AGENT_LAB_CEDAR_TOOL_DIR", str(home / "cache/tools/cedar")) + return experiment_module().main(["experiment.py", "authorize-zip", argv[4]]) if argv[:1] == ["experiment"] and argv[1:2] in (["install"], ["inspect"]): return experiment_command(home, argv[1:]) if argv[:1] == ["image"]: diff --git a/scripts/experiment.py b/scripts/experiment.py index c41b3ec..e8e5d5c 100644 --- a/scripts/experiment.py +++ b/scripts/experiment.py @@ -1715,9 +1715,10 @@ def main(argv: list[str]) -> int: directory_checking = len(argv) == 3 and argv[1] == "check-directory" directory_authorizing = len(argv) == 3 and argv[1] == "authorize-directory" zip_checking = len(argv) == 3 and argv[1] == "check-zip" - if directory_checking or directory_authorizing or zip_checking: + zip_authorizing = len(argv) == 3 and argv[1] == "authorize-zip" + if directory_checking or directory_authorizing or zip_checking or zip_authorizing: try: - if zip_checking: + if zip_checking or zip_authorizing: snapshot = read_zip_snapshot(argv[2]) else: snapshot = read_directory_snapshot(argv[2]) diff --git a/scripts/experiment_store.py b/scripts/experiment_store.py index 0449562..804c927 100644 --- a/scripts/experiment_store.py +++ b/scripts/experiment_store.py @@ -39,6 +39,7 @@ MAX_STAGE_BYTES = 4_194_304 MAX_AUTHORITY_BYTES = 65_536 MAX_ARTIFACT_BYTES = 262_144 +MAX_ARCHIVE_BYTES = 1_048_576 MAX_RECORD_BYTES = 1_048_576 SAFE_COMPONENT = re.compile(r"^[a-z][a-z0-9-]{0,47}$") EXPERIMENT_NAME = re.compile(r"^[a-z](?:[a-z0-9-]{0,61}[a-z0-9])?$") @@ -778,6 +779,7 @@ def _candidate( _infra("source snapshot is malformed") if source_digest != _source_digest(source_data): _infra("source snapshot digest is inconsistent") + source_transport = _source_transport(snapshot) _validate_catalog_evidence(selected, catalog_evidence) identity = _installation_identity(source_digest, plan, decision, selected) installation_key = digest(INSTALL_KEY_DOMAIN + canonical(identity)) @@ -799,7 +801,7 @@ def _candidate( "format": "agent-lab.experiment-tree/v1", "kind": "directory", }, - "transport": {"kind": "local-directory"}, + "transport": source_transport, } provenance_bytes = canonical(provenance) + b"\n" files = { @@ -845,6 +847,42 @@ def _candidate( return files, receipt, installation_key, digest(RECEIPT_DOMAIN + canonical(receipt)) +def _source_transport(snapshot: object) -> dict[str, object]: + transport = getattr(snapshot, "transport", None) + if transport is None or transport == {"kind": "directory"}: + return {"kind": "local-directory"} + if ( + not isinstance(transport, dict) + or set(transport) != {"archiveBytes", "archiveDigest", "kind"} + or transport.get("kind") != "zip" + or not isinstance(transport.get("archiveBytes"), int) + or isinstance(transport.get("archiveBytes"), bool) + or not 1 <= int(transport["archiveBytes"]) <= MAX_ARCHIVE_BYTES + or SHA256.fullmatch(str(transport.get("archiveDigest"))) is None + ): + _infra("source transport provenance is malformed") + return { + "archiveBytes": transport["archiveBytes"], + "archiveDigest": transport["archiveDigest"], + "kind": "zip", + } + + +def _validate_transport_provenance(transport: object) -> None: + if transport == {"kind": "local-directory"}: + return + if ( + not isinstance(transport, dict) + or set(transport) != {"archiveBytes", "archiveDigest", "kind"} + or transport.get("kind") != "zip" + or not isinstance(transport.get("archiveBytes"), int) + or isinstance(transport.get("archiveBytes"), bool) + or not 1 <= int(transport["archiveBytes"]) <= MAX_ARCHIVE_BYTES + or SHA256.fullmatch(str(transport.get("archiveDigest"))) is None + ): + _infra("installed source transport provenance is invalid") + + def _directory_names(path: Path, purpose: str, maximum: int) -> tuple[str, ...]: try: names: list[str] = [] @@ -1026,9 +1064,9 @@ def _verify_envelope( "format": "agent-lab.experiment-tree/v1", "kind": "directory", } - or provenance.get("transport") != {"kind": "local-directory"} ): raise ValueError("provenance") + _validate_transport_provenance(provenance.get("transport")) catalog = provenance.get("catalog") _validate_catalog_evidence(selected, catalog) installation_key = digest(INSTALL_KEY_DOMAIN + canonical(identity)) @@ -1939,23 +1977,23 @@ def _held_catalog_context( _infra("held local image catalog cannot be acquired", error) -def install_directory( +def _install_source( home: Path, source: Path, + reader_name: str, *, fault: FaultHook | None = None, ) -> dict[str, object]: - """Freshly validate/authorize one directory and publish its envelope once.""" + """Freshly validate/authorize one held source and publish its envelope once.""" - if sys.platform != "linux": - _infra("effectful Experiment installation requires Linux") authority = _load_home(Path(home)) experiment = _experiment_module() prior_home = os.environ.get("AGENT_LAB_HOME") os.environ["AGENT_LAB_HOME"] = str(authority.home) try: try: - snapshot = experiment.read_directory_snapshot(str(source)) + reader = getattr(experiment, reader_name) + snapshot = reader(str(source)) manifest = experiment.authored_manifest(snapshot) resolution = experiment.cue_plan_with_evidence(manifest) plan = resolution.plan @@ -2062,6 +2100,32 @@ def install_directory( _infra("Experiment store operation could not establish a result", error) +def install_directory( + home: Path, + source: Path, + *, + fault: FaultHook | None = None, +) -> dict[str, object]: + """Freshly validate/authorize one directory and publish its envelope once.""" + + if sys.platform != "linux": + _infra("effectful Experiment installation requires Linux") + return _install_source(home, source, "read_directory_snapshot", fault=fault) + + +def install_zip( + home: Path, + source: Path, + *, + fault: FaultHook | None = None, +) -> dict[str, object]: + """Freshly validate/authorize one ZIP snapshot and publish its envelope once.""" + + if sys.platform != "linux": + _infra("effectful Experiment installation requires Linux") + return _install_source(home, source, "read_zip_snapshot", fault=fault) + + def inspect_install(home: Path, name: str) -> dict[str, object]: """Read-only verification and identity projection for one installed name.""" From dfe7356d5931a2cb56fca0fa5abe16b204e0b151 Mon Sep 17 00:00:00 2001 From: xormania <127287135+xormania@users.noreply.github.com> Date: Sun, 2 Aug 2026 06:21:43 -0400 Subject: [PATCH 096/158] test(experiment): expose zip output uncertainty --- tests/experiment/source-adapter-cases.sh | 5 +- tests/experiment/zip-fixtures.py | 9 ++ tests/experiment/zip-intake-cases.sh | 194 ++++++++++++++++++++++- 3 files changed, 202 insertions(+), 6 deletions(-) diff --git a/tests/experiment/source-adapter-cases.sh b/tests/experiment/source-adapter-cases.sh index 60d2189..08b1840 100755 --- a/tests/experiment/source-adapter-cases.sh +++ b/tests/experiment/source-adapter-cases.sh @@ -5,7 +5,7 @@ repo_root="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." >/dev/null 2>&1 && subcases=( "$repo_root/tests/experiment/zip-intake-cases.sh" ) -expected_count=19 +expected_count=25 work="" cleanup_work() { @@ -31,7 +31,8 @@ printf '%s\n' \ ZIP-001 ZIP-PATH-001 ZIP-COUNT-001 ZIP-TYPE-001 ZIP-META-001 \ ZIP-FLAG-001 ZIP-METHOD-001 ZIP-ZIP64-001 ZIP-HEADER-001 ZIP-CRC-001 \ ZIP-LENGTH-001 ZIP-SIZE-001 ZIP-BOMB-001 ZIP-TRUNC-001 ZIP-TRAIL-001 \ - ZIP-READ-001 ZIP-AUTH-001 ZIP-INSTALL-001 ZIP-RETRY-001 > "$expected" + ZIP-SIZE-002 ZIP-READ-001 ZIP-READ-002 ZIP-DECODE-001 ZIP-TIMEOUT-001 \ + ZIP-OUTPUT-001 ZIP-NOEF-001 ZIP-AUTH-001 ZIP-INSTALL-001 ZIP-RETRY-001 > "$expected" : > "$observed" infrastructure=0 diff --git a/tests/experiment/zip-fixtures.py b/tests/experiment/zip-fixtures.py index e550234..58a77ed 100644 --- a/tests/experiment/zip-fixtures.py +++ b/tests/experiment/zip-fixtures.py @@ -226,6 +226,15 @@ def main() -> int: fixtures["trailing.zip"] = stored + b"trailing ambiguity" big_source = source + b"\n//" + (b"x" * 270_000) + b"\n" + limit_source = source + b"\n//" + ( + b"x" * (262_144 - len(source) - 4) + ) + b"\n" + if len(limit_source) != 262_144: + raise AssertionError("exact source-limit fixture has the wrong size") + (root / "expanded-limit.cue").write_bytes(limit_source) + fixtures["expanded-limit.zip"] = one( + "experiment.cue", limit_source, zipfile.ZIP_DEFLATED + ) fixtures["expanded-over.zip"] = one("experiment.cue", big_source) fixtures["deflate-bomb.zip"] = one( "experiment.cue", big_source, zipfile.ZIP_DEFLATED diff --git a/tests/experiment/zip-intake-cases.sh b/tests/experiment/zip-intake-cases.sh index ce6e5c4..836f469 100755 --- a/tests/experiment/zip-intake-cases.sh +++ b/tests/experiment/zip-intake-cases.sh @@ -10,7 +10,7 @@ mkdir -p "$work/home" "$work/tmp" if ! python3 -I -B "$repo_root/tests/experiment/zip-fixtures.py" \ "$fixture/experiment.cue" "$work"; then - printf 'SUMMARY assertions=0 expected=19 failures=0 infra=1\n' + printf 'SUMMARY assertions=0 expected=25 failures=0 infra=1\n' exit 125 fi @@ -34,7 +34,7 @@ init_home() { capture "$label" "$agent_lab" --home "$home" init if [ "$CAPTURE_RC" -ne 0 ]; then printf 'INFRA temporary Agent Lab home initialization failed\n' >&2 - printf 'SUMMARY assertions=%s expected=19 failures=%s infra=1\n' \ + printf 'SUMMARY assertions=%s expected=25 failures=%s infra=1\n' \ "$(wc -l < "$observed")" "$failures" exit 125 fi @@ -123,6 +123,29 @@ expect_all_reject ZIP-TRUNC-001 ZIP-TRUNC "truncated records and deflate streams expect_all_reject ZIP-TRAIL-001 ZIP-TRAIL "bytes after the canonical archive end are refused" trailing.zip +capture expanded-limit "$agent_lab" experiment check --zip "$work/expanded-limit.zip" +limit_digest="$(python3 -I -B - "$work/expanded-limit.cue" <<'PY' +from hashlib import sha256 +from pathlib import Path +import sys +data = Path(sys.argv[1]).read_bytes() +name = b"experiment.cue" +digest = sha256(b"agent-lab.experiment-tree.v1\0") +digest.update(len(name).to_bytes(4, "big")) +digest.update(name) +digest.update(len(data).to_bytes(8, "big")) +digest.update(data) +print("sha256:" + digest.hexdigest()) +PY +)" +if [ "$CAPTURE_RC" -eq 0 ] && [ ! -s "$work/expanded-limit.err" ] && + [ "$(wc -c < "$work/expanded-limit.cue")" -eq 262144 ] && + [ "$(jq -r '.source.digest' "$work/expanded-limit.out")" = "$limit_digest" ]; then + pass ZIP-SIZE-002 "the exact expanded source limit remains valid" +else + fail ZIP-SIZE-002 "the exact expanded source limit remains valid" +fi + ln -s "$work/stored.zip" "$work/archive-link.zip" mkdir "$work/archive-directory" capture missing "$agent_lab" experiment check --zip "$work/missing.zip" @@ -139,6 +162,168 @@ else fail ZIP-READ-001 "unavailable or unsafe archive paths are infrastructure failures" fi +if python3 -I -B - "$repo_root/scripts/experiment.py" "$work/expanded-limit.zip" <<'PY' +from importlib.util import module_from_spec, spec_from_file_location +from pathlib import Path +import sys + +spec = spec_from_file_location("zip_read_probe", sys.argv[1]) +assert spec is not None and spec.loader is not None +module = module_from_spec(spec) +sys.modules[spec.name] = module +spec.loader.exec_module(module) +archive = Path(sys.argv[2]) +original_read = module.os.read +changed = False + +def mutating_read(descriptor, size): + global changed + data = original_read(descriptor, size) + if data and not changed: + changed = True + with archive.open("ab") as stream: + stream.write(b"x") + return data + +module.os.read = mutating_read +try: + module.read_zip_snapshot(str(archive)) +except module.InfrastructureError as error: + assert "ZIP-READ" in str(error) +else: + raise AssertionError("mid-read archive mutation was accepted") +assert changed +PY +then + pass ZIP-READ-002 "mid-read archive mutation is infrastructure uncertainty" +else + fail ZIP-READ-002 "mid-read archive mutation is infrastructure uncertainty" +fi + +if python3 -I -B - "$repo_root/scripts/experiment.py" "$work/deflated.zip" <<'PY' +from importlib.util import module_from_spec, spec_from_file_location +import sys + +spec = spec_from_file_location("zip_decode_probe", sys.argv[1]) +assert spec is not None and spec.loader is not None +module = module_from_spec(spec) +sys.modules[spec.name] = module +spec.loader.exec_module(module) + +def uncertain_decoder(*_args, **_kwargs): + raise RuntimeError("injected decoder uncertainty") + +module.zlib.decompressobj = uncertain_decoder +try: + module.read_zip_snapshot(sys.argv[2]) +except module.InfrastructureError as error: + assert "ZIP-DECODE" in str(error) +else: + raise AssertionError("unexpected decoder exception was accepted") +PY +then + pass ZIP-DECODE-001 "unexpected decoder exceptions are infrastructure uncertainty" +else + fail ZIP-DECODE-001 "unexpected decoder exceptions are infrastructure uncertainty" +fi + +if python3 -I -B - "$repo_root/scripts/experiment.py" "$work/deflated.zip" <<'PY' +from importlib.util import module_from_spec, spec_from_file_location +import sys + +spec = spec_from_file_location("zip_timeout_probe", sys.argv[1]) +assert spec is not None and spec.loader is not None +module = module_from_spec(spec) +sys.modules[spec.name] = module +spec.loader.exec_module(module) +ticks = iter((0.0, 10.0)) +module.time.monotonic = lambda: next(ticks, 10.0) +try: + module.read_zip_snapshot(sys.argv[2]) +except module.InfrastructureError as error: + assert "ZIP-TIMEOUT" in str(error) +else: + raise AssertionError("decoder deadline was accepted") +PY +then + pass ZIP-TIMEOUT-001 "decoder deadline uncertainty is bounded and classified" +else + fail ZIP-TIMEOUT-001 "decoder deadline uncertainty is bounded and classified" +fi + +if AGENT_LAB_CUE_TOOL_DIR="${AGENT_LAB_CUE_TOOL_DIR:-$repo_root/.cache/dev/tools/cue}" \ + python3 -I -B - "$repo_root/scripts/experiment.py" "$work/stored.zip" <<'PY' +from importlib.util import module_from_spec, spec_from_file_location +import sys + +spec = spec_from_file_location("zip_output_probe", sys.argv[1]) +assert spec is not None and spec.loader is not None +module = module_from_spec(spec) +sys.modules[spec.name] = module +spec.loader.exec_module(module) + +class ShortBuffer: + def write(self, data): + return max(0, len(data) - 1) + + def flush(self): + return None + +class ShortOutput: + buffer = ShortBuffer() + + def flush(self): + return None + +module.sys.stdout = ShortOutput() +try: + result = module.main(["experiment.py", "check-zip", sys.argv[2]]) +except SystemExit as error: + result = error.code +raise SystemExit(0 if result == 125 else 1) +PY +then + pass ZIP-OUTPUT-001 "partial checked output is infrastructure uncertainty" +else + fail ZIP-OUTPUT-001 "partial checked output is infrastructure uncertainty" +fi + +if python3 -I -B - "$repo_root/scripts/experiment.py" \ + "$work/wrapper.zip" "$work/encrypted.zip" "$work/bad-crc.zip" <<'PY' +from importlib.util import module_from_spec, spec_from_file_location +import contextlib +import io +import sys + +spec = spec_from_file_location("zip_noeffect_probe", sys.argv[1]) +assert spec is not None and spec.loader is not None +module = module_from_spec(spec) +sys.modules[spec.name] = module +spec.loader.exec_module(module) +reached = False + +def forbidden(_snapshot): + global reached + reached = True + raise AssertionError("downstream planning reached") + +module.authored_manifest = forbidden +for archive in sys.argv[2:]: + with contextlib.redirect_stderr(io.StringIO()): + try: + module.main(["experiment.py", "check-zip", archive]) + except SystemExit as error: + assert error.code == 1 + else: + raise AssertionError("hostile archive returned") +assert not reached +PY +then + pass ZIP-NOEF-001 "structural rejection occurs before common planning" +else + fail ZIP-NOEF-001 "structural rejection occurs before common planning" +fi + capture directory-authorize "$agent_lab" experiment authorize install "$fixture" directory_authorize_rc="$CAPTURE_RC" capture zip-authorize "$agent_lab" experiment authorize install --zip "$work/stored.zip" @@ -202,10 +387,11 @@ printf '%s\n' \ ZIP-001 ZIP-PATH-001 ZIP-COUNT-001 ZIP-TYPE-001 ZIP-META-001 \ ZIP-FLAG-001 ZIP-METHOD-001 ZIP-ZIP64-001 ZIP-HEADER-001 ZIP-CRC-001 \ ZIP-LENGTH-001 ZIP-SIZE-001 ZIP-BOMB-001 ZIP-TRUNC-001 ZIP-TRAIL-001 \ - ZIP-READ-001 ZIP-AUTH-001 ZIP-INSTALL-001 ZIP-RETRY-001 > "$expected" + ZIP-SIZE-002 ZIP-READ-001 ZIP-READ-002 ZIP-DECODE-001 ZIP-TIMEOUT-001 \ + ZIP-OUTPUT-001 ZIP-NOEF-001 ZIP-AUTH-001 ZIP-INSTALL-001 ZIP-RETRY-001 > "$expected" if ! cmp -s "$expected" "$observed"; then printf 'INFRA assertion identity drift\n' >&2 exit 125 fi -printf 'SUMMARY assertions=19 expected=19 failures=%s infra=0\n' "$failures" +printf 'SUMMARY assertions=25 expected=25 failures=%s infra=0\n' "$failures" [ "$failures" -eq 0 ] From 504d9d2df02689ee778366380b8a65ba56131c68 Mon Sep 17 00:00:00 2001 From: xormania <127287135+xormania@users.noreply.github.com> Date: Sun, 2 Aug 2026 06:22:40 -0400 Subject: [PATCH 097/158] test(experiment): keep output fault evidence quiet --- tests/experiment/zip-intake-cases.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/experiment/zip-intake-cases.sh b/tests/experiment/zip-intake-cases.sh index 836f469..6da7a9a 100755 --- a/tests/experiment/zip-intake-cases.sh +++ b/tests/experiment/zip-intake-cases.sh @@ -254,6 +254,7 @@ fi if AGENT_LAB_CUE_TOOL_DIR="${AGENT_LAB_CUE_TOOL_DIR:-$repo_root/.cache/dev/tools/cue}" \ python3 -I -B - "$repo_root/scripts/experiment.py" "$work/stored.zip" <<'PY' from importlib.util import module_from_spec, spec_from_file_location +import io import sys spec = spec_from_file_location("zip_output_probe", sys.argv[1]) @@ -276,11 +277,14 @@ class ShortOutput: return None module.sys.stdout = ShortOutput() +errors = io.StringIO() +module.sys.stderr = errors try: result = module.main(["experiment.py", "check-zip", sys.argv[2]]) except SystemExit as error: result = error.code -raise SystemExit(0 if result == 125 else 1) +expected = "INFRA Experiment checked source output could not be written\n" +raise SystemExit(0 if result == 125 and errors.getvalue() == expected else 1) PY then pass ZIP-OUTPUT-001 "partial checked output is infrastructure uncertainty" From 398b02e1ea4d4188e575e2503c7d7e692f8ee114 Mon Sep 17 00:00:00 2001 From: xormania <127287135+xormania@users.noreply.github.com> Date: Sun, 2 Aug 2026 06:22:44 -0400 Subject: [PATCH 098/158] fix(experiment): fail closed on partial check output --- scripts/experiment.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/scripts/experiment.py b/scripts/experiment.py index e8e5d5c..83ca73c 100644 --- a/scripts/experiment.py +++ b/scripts/experiment.py @@ -1711,6 +1711,17 @@ def write_decision(decision: object) -> None: raise InfrastructureError("authorization decision could not be written") from error +def write_checked_source(checked: object) -> None: + output = canonical_json(checked) + b"\n" + try: + written = sys.stdout.buffer.write(output) + if written != len(output): + raise OSError("partial checked-source output") + sys.stdout.buffer.flush() + except (BrokenPipeError, OSError) as error: + raise InfrastructureError("checked source output could not be written") from error + + def main(argv: list[str]) -> int: directory_checking = len(argv) == 3 and argv[1] == "check-directory" directory_authorizing = len(argv) == 3 and argv[1] == "authorize-directory" @@ -1737,7 +1748,7 @@ def main(argv: list[str]) -> int: ) if catalog is not None: checked["catalog"] = catalog - sys.stdout.buffer.write(canonical_json(checked) + b"\n") + write_checked_source(checked) return 0 decision, result = authorize_plan(plan, snapshot.digest) write_decision(decision) From 59dd2df136183202b280de0d40a34b650c9da025 Mon Sep 17 00:00:00 2001 From: xormania <127287135+xormania@users.noreply.github.com> Date: Sun, 2 Aug 2026 06:28:05 -0400 Subject: [PATCH 099/158] test(experiment): harden zip intake adversarial evidence --- tests/experiment/source-adapter-cases.sh | 30 +- tests/experiment/zip-intake-cases.sh | 236 +++++++++++- tests/experiment/zip-mutation-cases.py | 436 +++++++++++++++++++++++ 3 files changed, 689 insertions(+), 13 deletions(-) create mode 100644 tests/experiment/zip-mutation-cases.py diff --git a/tests/experiment/source-adapter-cases.sh b/tests/experiment/source-adapter-cases.sh index 08b1840..dcc3dd2 100755 --- a/tests/experiment/source-adapter-cases.sh +++ b/tests/experiment/source-adapter-cases.sh @@ -4,8 +4,9 @@ set -u -o pipefail repo_root="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." >/dev/null 2>&1 && pwd)" subcases=( "$repo_root/tests/experiment/zip-intake-cases.sh" + "$repo_root/tests/experiment/zip-mutation-cases.py" ) -expected_count=25 +expected_count=40 work="" cleanup_work() { @@ -32,7 +33,11 @@ printf '%s\n' \ ZIP-FLAG-001 ZIP-METHOD-001 ZIP-ZIP64-001 ZIP-HEADER-001 ZIP-CRC-001 \ ZIP-LENGTH-001 ZIP-SIZE-001 ZIP-BOMB-001 ZIP-TRUNC-001 ZIP-TRAIL-001 \ ZIP-SIZE-002 ZIP-READ-001 ZIP-READ-002 ZIP-DECODE-001 ZIP-TIMEOUT-001 \ - ZIP-OUTPUT-001 ZIP-NOEF-001 ZIP-AUTH-001 ZIP-INSTALL-001 ZIP-RETRY-001 > "$expected" + ZIP-OUTPUT-001 ZIP-NOEF-001 ZIP-AUTH-001 ZIP-INSTALL-001 ZIP-RETRY-001 \ + ZIP-DENY-001 ZIP-PLAT-001 ZIP-NOEF-002 ZIP-RUNTIME-001 \ + M-ZIP-COUNT-001 M-ZIP-NAME-001 M-ZIP-TYPE-001 M-ZIP-FLAG-001 \ + M-ZIP-METHOD-001 M-ZIP-SIZE-001 M-ZIP-CRC-001 M-ZIP-HEADER-001 \ + M-ZIP-EXTRACT-001 M-ZIP-IDENTITY-001 M-ZIP-AUTH-001 > "$expected" : > "$observed" infrastructure=0 @@ -44,11 +49,22 @@ for index in "${!subcases[@]}"; do infrastructure=1 continue fi - if bash "$subcase" > "$output" 2>&1; then - rc=0 - else - rc=$? - fi + case "$subcase" in + *.py) + if python3 -I -B "$subcase" > "$output" 2>&1; then + rc=0 + else + rc=$? + fi + ;; + *) + if bash "$subcase" > "$output" 2>&1; then + rc=0 + else + rc=$? + fi + ;; + esac awk '/^(PASS|FAIL) [A-Z0-9-]+ / {print}' "$output" awk '/^(PASS|FAIL) [A-Z0-9-]+ / {print $2}' "$output" >> "$observed" reported_assertions="$(awk '/^(PASS|FAIL) [A-Z0-9-]+ / {count++} END {print count + 0}' "$output")" diff --git a/tests/experiment/zip-intake-cases.sh b/tests/experiment/zip-intake-cases.sh index 6da7a9a..f263d41 100755 --- a/tests/experiment/zip-intake-cases.sh +++ b/tests/experiment/zip-intake-cases.sh @@ -4,13 +4,26 @@ set -euo pipefail repo_root="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." >/dev/null 2>&1 && pwd)" agent_lab="$repo_root/scripts/agent-lab" fixture="$repo_root/tests/experiment/fixtures/directories/minimal" +runtime_manifest="$repo_root/packaging/agent-lab-local.manifest" work="$(mktemp -d)" -trap 'find "$work" -type f -delete 2>/dev/null || true; find "$work" -depth -type d -exec rmdir {} + 2>/dev/null || true' EXIT +cleanup_work() { + local failed=0 + if [ -n "$work" ] && [ -e "$work" ]; then + find "$work" -type f -exec chmod u+rw {} + 2>/dev/null || failed=1 + find "$work" -depth -type d -exec chmod u+rwx {} + 2>/dev/null || failed=1 + find "$work" -type f -delete 2>/dev/null || failed=1 + find "$work" -type l -delete 2>/dev/null || failed=1 + find "$work" -depth -type d -exec rmdir {} + 2>/dev/null || failed=1 + [ ! -e "$work" ] || failed=1 + fi + return "$failed" +} +trap 'cleanup_work >/dev/null 2>&1 || true' EXIT mkdir -p "$work/home" "$work/tmp" if ! python3 -I -B "$repo_root/tests/experiment/zip-fixtures.py" \ "$fixture/experiment.cue" "$work"; then - printf 'SUMMARY assertions=0 expected=25 failures=0 infra=1\n' + printf 'SUMMARY assertions=0 expected=29 failures=0 infra=1\n' exit 125 fi @@ -18,7 +31,8 @@ capture() { local name="$1" shift CAPTURE_RC=0 - env -i PATH=/usr/bin:/bin HOME="$work/home" TMPDIR="$work/tmp" LC_ALL=C \ + env -i PATH="${CAPTURE_PATH:-/usr/bin:/bin}" HOME="$work/home" TMPDIR="$work/tmp" LC_ALL=C \ + CANARY_DIR="${CANARY_DIR:-$work/no-canary}" \ AGENT_LAB_CUE_TOOL_DIR="${AGENT_LAB_CUE_TOOL_DIR:-$repo_root/.cache/dev/tools/cue}" \ AGENT_LAB_CEDAR_TOOL_DIR="${AGENT_LAB_CEDAR_TOOL_DIR:-$repo_root/.cache/dev/tools/cedar}" \ "$@" > "$work/$name.out" 2> "$work/$name.err" || CAPTURE_RC=$? @@ -34,11 +48,41 @@ init_home() { capture "$label" "$agent_lab" --home "$home" init if [ "$CAPTURE_RC" -ne 0 ]; then printf 'INFRA temporary Agent Lab home initialization failed\n' >&2 - printf 'SUMMARY assertions=%s expected=25 failures=%s infra=1\n' \ + printf 'SUMMARY assertions=%s expected=29 failures=%s infra=1\n' \ "$(wc -l < "$observed")" "$failures" exit 125 fi } +tree_fingerprint() { + python3 -I -B - "$1" <<'PY' +from hashlib import sha256 +import os +from pathlib import Path +import stat +import sys + +root = Path(sys.argv[1]) +records = [] +if root.exists(): + for path in sorted(root.rglob("*"), key=lambda item: os.fsencode(str(item.relative_to(root)))): + metadata = path.lstat() + relative = str(path.relative_to(root)) + if stat.S_ISREG(metadata.st_mode): + content = sha256(path.read_bytes()).hexdigest() + kind = "file" + elif stat.S_ISDIR(metadata.st_mode): + content = "" + kind = "directory" + elif stat.S_ISLNK(metadata.st_mode): + content = os.readlink(path) + kind = "symlink" + else: + content = "" + kind = "other" + records.append((relative, kind, stat.S_IMODE(metadata.st_mode), content)) +print(sha256(repr(records).encode("utf-8")).hexdigest()) +PY +} expect_all_reject() { local id="$1" code="$2" detail="$3" shift 3 @@ -386,16 +430,196 @@ else fail ZIP-RETRY-001 "equivalent zip retry preserves the directory installation receipt" fi +deny_home="$work/deny-home" +init_home deny-home-init "$deny_home" +deny_runtime="$work/deny-runtime" +deny_runtime_ok=1 +while IFS= read -r runtime_name; do + if [ -z "$runtime_name" ] || [ ! -f "$repo_root/$runtime_name" ]; then + deny_runtime_ok=0 + continue + fi + mkdir -p "$deny_runtime/$(dirname -- "$runtime_name")" + cp "$repo_root/$runtime_name" "$deny_runtime/$runtime_name" || deny_runtime_ok=0 +done < "$runtime_manifest" +if [ "$deny_runtime_ok" -eq 1 ]; then + sed 's/^permit (/forbid (/' \ + "$repo_root/authorization/experiment/v0alpha1/operator.cedar" \ + > "$deny_runtime/authorization/experiment/v0alpha1/operator.cedar" +fi +capture deny-preview "$deny_runtime/scripts/agent-lab" --home "$deny_home" \ + experiment authorize install --zip "$work/stored.zip" +deny_preview_rc="$CAPTURE_RC" +deny_before="$(tree_fingerprint "$deny_home")" +capture deny-install "$deny_runtime/scripts/agent-lab" --home "$deny_home" \ + experiment install --zip "$work/stored.zip" +deny_install_rc="$CAPTURE_RC" +deny_after="$(tree_fingerprint "$deny_home")" +if [ "$deny_runtime_ok" -eq 1 ] && [ "$deny_preview_rc" -eq 1 ] && + [ "$deny_install_rc" -eq 1 ] && [ ! -s "$work/deny-preview.err" ] && + [ ! -s "$work/deny-install.out" ] && + jq -e '.verdict == "deny"' "$work/deny-preview.out" >/dev/null 2>&1 && + [ "$deny_before" = "$deny_after" ]; then + pass ZIP-DENY-001 "fresh zip denial leaves the initialized home unchanged" +else + fail ZIP-DENY-001 "fresh zip denial leaves the initialized home unchanged" +fi + +if python3 -I -B - "$repo_root/scripts/agent-lab.py" <<'PY' +from importlib.util import module_from_spec, spec_from_file_location +import io +from pathlib import Path +import sys + +spec = spec_from_file_location("zip_platform_probe", sys.argv[1]) +assert spec is not None and spec.loader is not None +module = module_from_spec(spec) +sys.modules[spec.name] = module +spec.loader.exec_module(module) +reached = [] + +def forbidden(*_args, **_kwargs): + reached.append(True) + raise AssertionError("pre-acquisition platform guard was bypassed") + +module.sys.platform = "darwin" +module.load_config_receipt = forbidden +module.experiment_store_module = forbidden +errors = io.StringIO() +module.sys.stderr = errors +result = module.experiment_command( + Path("/unavailable-home"), ["install", "--zip", "/unavailable-archive"] +) +assert result == 125 +assert not reached +assert errors.getvalue() == "INFRA Agent Lab Experiment installation requires Linux\n" +PY +then + pass ZIP-PLAT-001 "non-Linux zip install stops before home and archive acquisition" +else + fail ZIP-PLAT-001 "non-Linux zip install stops before home and archive acquisition" +fi + +canary_bin="$work/canary-bin" +canary_marks="$work/canary-marks" +mkdir "$canary_bin" "$canary_marks" +for command in docker git curl wget zip unzip; do + printf '%s\n' '#!/bin/sh' 'set -eu' ': > "$CANARY_DIR/${0##*/}"' 'exit 97' \ + > "$canary_bin/$command" + chmod 700 "$canary_bin/$command" + CANARY_DIR="$canary_marks" "$canary_bin/$command" >/dev/null 2>&1 || true +done +canaries_calibrated="$(find "$canary_marks" -type f -printf '%f\n' | LC_ALL=C sort | tr '\n' ' ')" +find "$canary_marks" -type f -delete +noeffect_home="$work/noeffect-home" +init_home noeffect-home-init "$noeffect_home" +archive_before="$(sha256sum "$work/deflated.zip")" +CAPTURE_PATH="$canary_bin:/usr/bin:/bin" +CANARY_DIR="$canary_marks" +capture noeffect-check "$agent_lab" experiment check --zip "$work/deflated.zip" +noeffect_check_rc="$CAPTURE_RC" +capture noeffect-authorize "$agent_lab" experiment authorize install --zip "$work/deflated.zip" +noeffect_authorize_rc="$CAPTURE_RC" +capture noeffect-install "$agent_lab" --home "$noeffect_home" experiment install --zip "$work/deflated.zip" +noeffect_install_rc="$CAPTURE_RC" +unset CAPTURE_PATH CANARY_DIR +archive_after="$(sha256sum "$work/deflated.zip")" +if [ "$canaries_calibrated" = "curl docker git unzip wget zip " ] && + [ "$noeffect_check_rc" -eq 0 ] && [ "$noeffect_authorize_rc" -eq 0 ] && + [ "$noeffect_install_rc" -eq 0 ] && [ ! -s "$work/noeffect-check.err" ] && + [ ! -s "$work/noeffect-authorize.err" ] && [ ! -s "$work/noeffect-install.err" ] && + [ -z "$(find "$canary_marks" -type f -print -quit)" ] && + [ "$archive_before" = "$archive_after" ]; then + pass ZIP-NOEF-002 "zip intake invokes no archive tool, Git, downloader, or Docker command" +else + fail ZIP-NOEF-002 "zip intake invokes no archive tool, Git, downloader, or Docker command" +fi + +installed_source="$work/installed-source" +installed_unavailable="$work/installed-source-unavailable" +installed_prefix="$work/installed-prefix" +installed_home="$work/installed-home" +installed_unrelated="$work/installed-unrelated" +installed_tools="$work/installed-tools" +installed_ok=1 +mkdir -p "$installed_source/packaging" "$installed_source/scripts" \ + "$installed_unrelated" "$installed_tools/cue" "$installed_tools/cedar" || installed_ok=0 +while IFS= read -r runtime_name; do + if [ -z "$runtime_name" ] || [ ! -f "$repo_root/$runtime_name" ]; then + installed_ok=0 + continue + fi + mkdir -p "$installed_source/$(dirname -- "$runtime_name")" || installed_ok=0 + cp "$repo_root/$runtime_name" "$installed_source/$runtime_name" || installed_ok=0 +done < "$runtime_manifest" +cp "$runtime_manifest" "$installed_source/packaging/agent-lab-local.manifest" || installed_ok=0 +cp "$repo_root/scripts/install-local" "$repo_root/scripts/install-local.py" \ + "$installed_source/scripts/" || installed_ok=0 +cp -a "$repo_root/.cache/dev/tools/cue/." "$installed_tools/cue/" || installed_ok=0 +cp -a "$repo_root/.cache/dev/tools/cedar/." "$installed_tools/cedar/" || installed_ok=0 +chmod +x "$installed_source/scripts/install-local" "$installed_source/scripts/agent-lab" || installed_ok=0 +capture installed-bundle "$installed_source/scripts/install-local" --prefix "$installed_prefix" +installed_bundle_rc="$CAPTURE_RC" +if [ "$installed_bundle_rc" -eq 0 ]; then + mv "$installed_source" "$installed_unavailable" || installed_ok=0 +fi +capture installed-init env -i PATH=/usr/bin:/bin LANG=C LC_ALL=C \ + /bin/sh -c 'cd "$1" || exit 125; shift; exec "$@"' agent-lab-installed \ + "$installed_unrelated" "$installed_prefix/bin/agent-lab" --home "$installed_home" init +installed_init_rc="$CAPTURE_RC" +if [ "$installed_init_rc" -eq 0 ]; then + cp -a "$installed_tools/cue/." "$installed_home/cache/tools/cue/" || installed_ok=0 + cp -a "$installed_tools/cedar/." "$installed_home/cache/tools/cedar/" || installed_ok=0 +fi +capture installed-check env -i PATH=/usr/bin:/bin LANG=C LC_ALL=C \ + /bin/sh -c 'cd "$1" || exit 125; shift; exec "$@"' agent-lab-installed \ + "$installed_unrelated" "$installed_prefix/bin/agent-lab" --home "$installed_home" \ + experiment check --zip "$work/stored.zip" +installed_check_rc="$CAPTURE_RC" +capture installed-authorize env -i PATH=/usr/bin:/bin LANG=C LC_ALL=C \ + /bin/sh -c 'cd "$1" || exit 125; shift; exec "$@"' agent-lab-installed \ + "$installed_unrelated" "$installed_prefix/bin/agent-lab" --home "$installed_home" \ + experiment authorize install --zip "$work/stored.zip" +installed_authorize_rc="$CAPTURE_RC" +capture installed-install env -i PATH=/usr/bin:/bin LANG=C LC_ALL=C \ + /bin/sh -c 'cd "$1" || exit 125; shift; exec "$@"' agent-lab-installed \ + "$installed_unrelated" "$installed_prefix/bin/agent-lab" --home "$installed_home" \ + experiment install --zip "$work/stored.zip" +installed_install_rc="$CAPTURE_RC" +if [ "$installed_ok" -eq 1 ] && [ "$installed_bundle_rc" -eq 0 ] && + [ "$installed_init_rc" -eq 0 ] && [ "$installed_check_rc" -eq 0 ] && + [ "$installed_authorize_rc" -eq 0 ] && [ "$installed_install_rc" -eq 0 ] && + [ ! -s "$work/installed-bundle.err" ] && [ ! -s "$work/installed-check.err" ] && + [ ! -s "$work/installed-authorize.err" ] && [ ! -s "$work/installed-install.err" ] && + jq -e '.source.kind == "zip"' "$work/installed-check.out" >/dev/null 2>&1 && + jq -e '.verdict == "permit"' "$work/installed-authorize.out" >/dev/null 2>&1 && + jq -e '.changed == true and .name == "first-experiment"' \ + "$work/installed-install.out" >/dev/null 2>&1 && + [ ! -e "$installed_source" ] && [ -d "$installed_unavailable" ] && + [ -z "$(find "$installed_prefix" -name __pycache__ -print -quit)" ]; then + pass ZIP-RUNTIME-001 "installed runtime handles zip intake without its source replica" +else + fail ZIP-RUNTIME-001 "installed runtime handles zip intake without its source replica" +fi + expected="$work/expected" printf '%s\n' \ ZIP-001 ZIP-PATH-001 ZIP-COUNT-001 ZIP-TYPE-001 ZIP-META-001 \ ZIP-FLAG-001 ZIP-METHOD-001 ZIP-ZIP64-001 ZIP-HEADER-001 ZIP-CRC-001 \ ZIP-LENGTH-001 ZIP-SIZE-001 ZIP-BOMB-001 ZIP-TRUNC-001 ZIP-TRAIL-001 \ ZIP-SIZE-002 ZIP-READ-001 ZIP-READ-002 ZIP-DECODE-001 ZIP-TIMEOUT-001 \ - ZIP-OUTPUT-001 ZIP-NOEF-001 ZIP-AUTH-001 ZIP-INSTALL-001 ZIP-RETRY-001 > "$expected" + ZIP-OUTPUT-001 ZIP-NOEF-001 ZIP-AUTH-001 ZIP-INSTALL-001 ZIP-RETRY-001 \ + ZIP-DENY-001 ZIP-PLAT-001 ZIP-NOEF-002 ZIP-RUNTIME-001 > "$expected" if ! cmp -s "$expected" "$observed"; then printf 'INFRA assertion identity drift\n' >&2 exit 125 fi -printf 'SUMMARY assertions=25 expected=25 failures=%s infra=0\n' "$failures" +cleanup_infrastructure=0 +if ! cleanup_work; then + cleanup_infrastructure=1 +fi +trap - EXIT +printf 'SUMMARY assertions=29 expected=29 failures=%s infra=%s\n' \ + "$failures" "$cleanup_infrastructure" +[ "$cleanup_infrastructure" -eq 0 ] || exit 125 [ "$failures" -eq 0 ] diff --git a/tests/experiment/zip-mutation-cases.py b/tests/experiment/zip-mutation-cases.py new file mode 100644 index 0000000..4f58f63 --- /dev/null +++ b/tests/experiment/zip-mutation-cases.py @@ -0,0 +1,436 @@ +#!/usr/bin/env python3 +"""Private-copy sensitivity mutations for the bounded ZIP intake controls.""" + +from __future__ import annotations + +from hashlib import sha256 +from importlib.util import module_from_spec, spec_from_file_location +import os +from pathlib import Path +import shutil +import stat +import subprocess +import sys +import tempfile + + +EXPECTED = ( + "M-ZIP-COUNT-001", + "M-ZIP-NAME-001", + "M-ZIP-TYPE-001", + "M-ZIP-FLAG-001", + "M-ZIP-METHOD-001", + "M-ZIP-SIZE-001", + "M-ZIP-CRC-001", + "M-ZIP-HEADER-001", + "M-ZIP-EXTRACT-001", + "M-ZIP-IDENTITY-001", + "M-ZIP-AUTH-001", +) + + +def load_module(path: Path, label: str): + spec = spec_from_file_location(label, path) + if spec is None or spec.loader is None: + raise RuntimeError(f"cannot load {path}") + module = module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +def replace_once(source: str, needle: str, replacement: str) -> str: + if source.count(needle) != 1: + raise RuntimeError("private mutation did not match exactly once") + return source.replace(needle, replacement, 1) + + +def rejected_with(module, archive: Path, code: str) -> bool: + try: + module.read_zip_snapshot(str(archive)) + except module.InvalidManifest as error: + return code in str(error) + return False + + +def parser_mutation( + production: Path, + original: str, + private_source: Path, + baseline_module, + fixture: Path, + code: str, + assertion: str, + needle: str, + replacement: str, + marker: Path, +) -> bool: + if not rejected_with(baseline_module, fixture, code): + return False + private_source.write_text(replace_once(original, needle, replacement), encoding="utf-8") + marker.unlink(missing_ok=True) + os.environ["AGENT_LAB_ZIP_MUTATION_MARK"] = str(marker) + try: + mutant = load_module(private_source, "zip_mutant_" + assertion.lower().replace("-", "_")) + killed = not rejected_with(mutant, fixture, code) + finally: + os.environ.pop("AGENT_LAB_ZIP_MUTATION_MARK", None) + return marker.is_file() and killed and sha256(production.read_bytes()).hexdigest() == sha256( + original.encode("utf-8") + ).hexdigest() + + +def run_command(command: list[str], environment: dict[str, str]) -> subprocess.CompletedProcess[bytes]: + return subprocess.run( + command, + env=environment, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + timeout=5, + ) + + +def authorization_mutation(repo: Path, root: Path, archive: Path, marker: Path) -> bool: + runtime = root / "deny-runtime" + manifest = repo / "packaging/agent-lab-local.manifest" + for raw in manifest.read_text(encoding="utf-8").splitlines(): + if not raw: + raise RuntimeError("runtime manifest contains an empty path") + source = repo / raw + target = runtime / raw + target.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(source, target) + entrypoint = runtime / "scripts/agent-lab" + entrypoint.chmod(entrypoint.stat().st_mode | stat.S_IXUSR) + policy = runtime / "authorization/experiment/v0alpha1/operator.cedar" + policy_text = policy.read_text(encoding="utf-8") + policy.write_text( + replace_once(policy_text, "permit (", "forbid ("), + encoding="utf-8", + ) + environment = { + "PATH": "/usr/bin:/bin", + "HOME": str(root / "empty-home"), + "TMPDIR": str(root / "tmp"), + "LC_ALL": "C", + "AGENT_LAB_CUE_TOOL_DIR": str(repo / ".cache/dev/tools/cue"), + "AGENT_LAB_CEDAR_TOOL_DIR": str(repo / ".cache/dev/tools/cedar"), + } + Path(environment["HOME"]).mkdir() + Path(environment["TMPDIR"]).mkdir() + baseline_home = root / "baseline-home" + baseline_init = run_command( + [str(entrypoint), "--home", str(baseline_home), "init"], environment + ) + baseline = run_command( + [ + str(entrypoint), + "--home", + str(baseline_home), + "experiment", + "install", + "--zip", + str(archive), + ], + environment, + ) + if baseline_init.returncode != 0 or baseline.returncode != 1: + return False + + store = runtime / "scripts/experiment_store.py" + store_original = store.read_text(encoding="utf-8") + needle = " decision, status = experiment.authorize_plan(plan, snapshot.digest)\n" + replacement = ( + " decision, status = experiment.authorize_plan(plan, snapshot.digest)\n" + " mutation_marker = os.environ.get(\"AGENT_LAB_ZIP_MUTATION_MARK\")\n" + " if mutation_marker is not None:\n" + " Path(mutation_marker).touch()\n" + " decision = dict(decision)\n" + " decision[\"verdict\"] = \"permit\"\n" + " status = 0\n" + ) + store.write_text(replace_once(store_original, needle, replacement), encoding="utf-8") + marker.unlink(missing_ok=True) + environment["AGENT_LAB_ZIP_MUTATION_MARK"] = str(marker) + mutant_home = root / "mutant-home" + mutant_init = run_command( + [str(entrypoint), "--home", str(mutant_home), "init"], environment + ) + mutant = run_command( + [ + str(entrypoint), + "--home", + str(mutant_home), + "experiment", + "install", + "--zip", + str(archive), + ], + environment, + ) + installed = mutant_home / "experiments/first-experiment/records/install.json" + return ( + mutant_init.returncode == 0 + and mutant.returncode == 0 + and marker.is_file() + and installed.is_file() + and sha256((repo / "scripts/experiment_store.py").read_bytes()).hexdigest() + == sha256(store_original.encode("utf-8")).hexdigest() + ) + + +def main() -> int: + repo = Path(__file__).resolve().parents[2] + production = repo / "scripts/experiment.py" + original = production.read_text(encoding="utf-8") + failures = 0 + infrastructure = 0 + results: list[tuple[str, bool, str]] = [] + work_path = tempfile.mkdtemp(prefix="agent-lab-zip-mutations-") + work = Path(work_path) + try: + fixtures = work / "fixtures" + generated = subprocess.run( + [ + sys.executable, + "-I", + "-B", + str(repo / "tests/experiment/zip-fixtures.py"), + str(repo / "tests/experiment/fixtures/directories/minimal/experiment.cue"), + str(fixtures), + ], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + timeout=5, + ) + if generated.returncode != 0 or generated.stdout or generated.stderr: + raise RuntimeError("private ZIP fixtures could not be generated") + baseline_module = load_module(production, "zip_mutation_baseline") + private_source = work / "experiment.py" + shutil.copy2(repo / "scripts/image_reference.py", work / "image_reference.py") + marker = work / "reached" + cases = ( + ( + "M-ZIP-COUNT-001", + "extra-entry.zip", + "ZIP-COUNT", + " if disk_entries != 1 or total_entries != 1:\n", + ( + " if os.environ.get(\"AGENT_LAB_ZIP_MUTATION_MARK\") is not None:\n" + " Path(os.environ[\"AGENT_LAB_ZIP_MUTATION_MARK\"]).touch()\n" + " if False:\n" + ), + ), + ( + "M-ZIP-NAME-001", + "wrong-case.zip", + "ZIP-PATH", + ' if decoded_name != "experiment.cue":\n', + ( + " if os.environ.get(\"AGENT_LAB_ZIP_MUTATION_MARK\") is not None:\n" + " Path(os.environ[\"AGENT_LAB_ZIP_MUTATION_MARK\"]).touch()\n" + " if False:\n" + ), + ), + ( + "M-ZIP-TYPE-001", + "symlink-type.zip", + "ZIP-TYPE", + ( + " if (\n" + " dos_attributes & 0x10\n" + " or (create_system == 3 and unix_type not in (0, stat.S_IFREG))\n" + " ):\n" + ), + ( + " if os.environ.get(\"AGENT_LAB_ZIP_MUTATION_MARK\") is not None:\n" + " Path(os.environ[\"AGENT_LAB_ZIP_MUTATION_MARK\"]).touch()\n" + " if False:\n" + ), + ), + ( + "M-ZIP-FLAG-001", + "encrypted.zip", + "ZIP-FLAG", + " if flags & ~0x800:\n", + ( + " if os.environ.get(\"AGENT_LAB_ZIP_MUTATION_MARK\") is not None:\n" + " Path(os.environ[\"AGENT_LAB_ZIP_MUTATION_MARK\"]).touch()\n" + " if False:\n" + ), + ), + ( + "M-ZIP-METHOD-001", + "unsupported-method.zip", + "ZIP-METHOD", + " if method not in (0, 8):\n", + ( + " if os.environ.get(\"AGENT_LAB_ZIP_MUTATION_MARK\") is not None:\n" + " Path(os.environ[\"AGENT_LAB_ZIP_MUTATION_MARK\"]).touch()\n" + " if False:\n" + ), + ), + ( + "M-ZIP-SIZE-001", + "expanded-over.zip", + "ZIP-SIZE", + " if expanded_size > MAX_SOURCE_BYTES:\n", + ( + " if os.environ.get(\"AGENT_LAB_ZIP_MUTATION_MARK\") is not None:\n" + " Path(os.environ[\"AGENT_LAB_ZIP_MUTATION_MARK\"]).touch()\n" + " if False:\n" + ), + ), + ( + "M-ZIP-CRC-001", + "bad-crc.zip", + "ZIP-CRC", + " if (zlib.crc32(data) & 0xFFFFFFFF) != crc:\n", + ( + " if os.environ.get(\"AGENT_LAB_ZIP_MUTATION_MARK\") is not None:\n" + " Path(os.environ[\"AGENT_LAB_ZIP_MUTATION_MARK\"]).touch()\n" + " if False:\n" + ), + ), + ( + "M-ZIP-HEADER-001", + "central-signature.zip", + "ZIP-HEADER", + ' if central_signature != b"PK\\x01\\x02":\n', + ( + " if os.environ.get(\"AGENT_LAB_ZIP_MUTATION_MARK\") is not None:\n" + " Path(os.environ[\"AGENT_LAB_ZIP_MUTATION_MARK\"]).touch()\n" + " if False:\n" + ), + ), + ) + for assertion, fixture_name, code, needle, replacement in cases: + result = parser_mutation( + production, + original, + private_source, + baseline_module, + fixtures / fixture_name, + code, + assertion, + needle, + replacement, + marker, + ) + results.append((assertion, result, "private parser mutation is killed")) + + extraction = work / "caller-destination" + extraction.mkdir() + extraction_needle = " archive = _read_zip_archive_once(path)\n" + extraction_replacement = ( + " archive = _read_zip_archive_once(path)\n" + " mutation_marker = os.environ.get(\"AGENT_LAB_ZIP_MUTATION_MARK\")\n" + " if mutation_marker is not None:\n" + " Path(mutation_marker).touch()\n" + " __import__(\"zipfile\").ZipFile(\n" + " __import__(\"io\").BytesIO(archive)\n" + " ).extractall(Path(os.environ[\"AGENT_LAB_ZIP_MUTATION_DEST\"]))\n" + ) + private_source.write_text( + replace_once(original, extraction_needle, extraction_replacement), encoding="utf-8" + ) + marker.unlink(missing_ok=True) + os.environ["AGENT_LAB_ZIP_MUTATION_MARK"] = str(marker) + os.environ["AGENT_LAB_ZIP_MUTATION_DEST"] = str(extraction) + try: + mutant = load_module(private_source, "zip_mutant_extract") + mutant.read_zip_snapshot(str(fixtures / "stored.zip")) + finally: + os.environ.pop("AGENT_LAB_ZIP_MUTATION_MARK", None) + os.environ.pop("AGENT_LAB_ZIP_MUTATION_DEST", None) + results.append( + ( + "M-ZIP-EXTRACT-001", + marker.is_file() and (extraction / "experiment.cue").is_file(), + "caller-destination extraction mutation is observable", + ) + ) + + identity_needle = ( + " return SourceSnapshot(\n" + " data=data,\n" + " digest=source_digest(data),\n" + " transport={\n" + " \"archiveBytes\": len(archive),\n" + ) + identity_replacement = ( + " return SourceSnapshot(\n" + " data=data,\n" + " digest=(\n" + " Path(os.environ[\"AGENT_LAB_ZIP_MUTATION_MARK\"]).touch()\n" + " or \"sha256:\" + hashlib.sha256(archive).hexdigest()\n" + " ),\n" + " transport={\n" + " \"archiveBytes\": len(archive),\n" + ) + private_source.write_text( + replace_once(original, identity_needle, identity_replacement), encoding="utf-8" + ) + marker.unlink(missing_ok=True) + os.environ["AGENT_LAB_ZIP_MUTATION_MARK"] = str(marker) + try: + mutant = load_module(private_source, "zip_mutant_identity") + directory = mutant.read_directory_snapshot( + str(repo / "tests/experiment/fixtures/directories/minimal") + ) + zipped = mutant.read_zip_snapshot(str(fixtures / "stored.zip")) + finally: + os.environ.pop("AGENT_LAB_ZIP_MUTATION_MARK", None) + results.append( + ( + "M-ZIP-IDENTITY-001", + marker.is_file() and directory.digest != zipped.digest, + "archive-identity mutation breaks the cross-transport oracle", + ) + ) + + results.append( + ( + "M-ZIP-AUTH-001", + authorization_mutation(repo, work / "auth", fixtures / "stored.zip", marker), + "saved-denial bypass mutation changes the no-effect outcome", + ) + ) + except (OSError, RuntimeError, subprocess.SubprocessError) as error: + print(f"INFRA zip mutation harness {error}", file=sys.stderr) + infrastructure = 1 + finally: + try: + for path in work.rglob("*"): + try: + path.chmod(path.stat().st_mode | stat.S_IWUSR | stat.S_IXUSR) + except OSError: + pass + shutil.rmtree(work) + except OSError: + infrastructure = 1 + + observed = tuple(item[0] for item in results) + if observed != EXPECTED: + infrastructure = 1 + for assertion, passed, detail in results: + if passed: + print(f"PASS {assertion} {detail}") + else: + print(f"FAIL {assertion} {detail}") + failures += 1 + print( + f"SUMMARY assertions={len(results)} expected={len(EXPECTED)} " + f"failures={failures} infra={infrastructure}" + ) + if infrastructure: + return 125 + if failures: + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) From d9f98b6db43a8b8de8072d6bd344b5de0e6f347d Mon Sep 17 00:00:00 2001 From: xormania <127287135+xormania@users.noreply.github.com> Date: Sun, 2 Aug 2026 06:28:18 -0400 Subject: [PATCH 100/158] docs(experiment): document bounded zip intake --- docs/experiments.md | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/docs/experiments.md b/docs/experiments.md index 64f1ce9..6587571 100644 --- a/docs/experiments.md +++ b/docs/experiments.md @@ -1,9 +1,10 @@ # Experiments -An Experiment is authored as data in a directory containing exactly one file, `experiment.cue`. -The file defines one concrete value named `experiment` in package `experiment`. Agent Lab snapshots -the exact bytes privately before evaluating them; extra entries, links, special files, suspicious -modes, changing sources, malformed CUE, and unknown schema fields are refused. +An Experiment is authored as data in a directory containing exactly one file, `experiment.cue`, or +in a bounded ZIP archive containing that exact sole member. The file defines one concrete value named +`experiment` in package `experiment`. Agent Lab snapshots the exact authored bytes privately before +evaluating them; extra entries, links, special files, suspicious modes, changing sources, malformed +CUE, and unknown schema fields are refused. ```cue package experiment @@ -25,6 +26,8 @@ Check the artifact or preview its install authorization from the repository: ```bash ./scripts/agent-lab experiment check ./my-experiment ./scripts/agent-lab experiment authorize install ./my-experiment +./scripts/agent-lab experiment check --zip ./my-experiment.zip +./scripts/agent-lab experiment authorize install --zip ./my-experiment.zip ``` These two commands are previews. They create no durable Agent Lab state and do not invoke Docker or @@ -36,6 +39,7 @@ Install a freshly checked and permitted artifact, then inspect its stored identi ```bash agent-lab [--home /absolute/private/home] experiment install ./my-experiment +agent-lab [--home /absolute/private/home] experiment install --zip ./my-experiment.zip agent-lab [--home /absolute/private/home] experiment inspect example ``` @@ -46,6 +50,15 @@ source, plan, decision, provenance, and receipt without running content, invokin image bytes, or claiming runtime admission. The decision and receipt bind the same domain-separated plan identity rather than an unframed hash of the JSON bytes. +ZIP intake reads one stable archive into at most 1,048,576 bytes and accepts only stored or deflated +`experiment.cue` data that expands to at most 262,144 bytes. ZIP64, multidisk archives, encryption, +comments, extra fields, alternate paths, extra members, special file types, inconsistent headers, +bad CRC or lengths, truncated streams, and trailing bytes are rejected before CUE evaluation. Agent +Lab never extracts the archive or chooses a destination from caller data. The normalized source +digest is identical to directory intake for identical authored bytes; installation provenance also +records the raw archive byte count and SHA-256 digest. Archive identity does not affect the plan, +authorization binding, installation key, or idempotent cross-transport retry. + An exact retry freshly validates and authorizes again, verifies the complete installed envelope, and returns `changed:false` with the same `installationKey` and `receiptDigest`. The same requested name with a different installation identity conflicts without overwrite. `inspect` is read-only: it From a83fa18ed8509db0417b0858748fdb20ad717de8 Mon Sep 17 00:00:00 2001 From: xormania <127287135+xormania@users.noreply.github.com> Date: Sun, 2 Aug 2026 06:33:15 -0400 Subject: [PATCH 101/158] test(experiment): cover benign zip metadata uncertainty --- tests/experiment/source-adapter-cases.sh | 6 +- tests/experiment/zip-fixtures.py | 76 ++++++++++++++++++++++++ tests/experiment/zip-intake-cases.sh | 67 +++++++++++++++++---- 3 files changed, 136 insertions(+), 13 deletions(-) diff --git a/tests/experiment/source-adapter-cases.sh b/tests/experiment/source-adapter-cases.sh index dcc3dd2..d2cf277 100755 --- a/tests/experiment/source-adapter-cases.sh +++ b/tests/experiment/source-adapter-cases.sh @@ -6,7 +6,7 @@ subcases=( "$repo_root/tests/experiment/zip-intake-cases.sh" "$repo_root/tests/experiment/zip-mutation-cases.py" ) -expected_count=40 +expected_count=42 work="" cleanup_work() { @@ -29,10 +29,10 @@ trap 'cleanup_work >/dev/null 2>&1 || true' EXIT expected="$work/expected" observed="$work/observed" printf '%s\n' \ - ZIP-001 ZIP-PATH-001 ZIP-COUNT-001 ZIP-TYPE-001 ZIP-META-001 \ + ZIP-001 ZIP-COMPAT-001 ZIP-PATH-001 ZIP-COUNT-001 ZIP-TYPE-001 ZIP-META-001 \ ZIP-FLAG-001 ZIP-METHOD-001 ZIP-ZIP64-001 ZIP-HEADER-001 ZIP-CRC-001 \ ZIP-LENGTH-001 ZIP-SIZE-001 ZIP-BOMB-001 ZIP-TRUNC-001 ZIP-TRAIL-001 \ - ZIP-SIZE-002 ZIP-READ-001 ZIP-READ-002 ZIP-DECODE-001 ZIP-TIMEOUT-001 \ + ZIP-SIZE-002 ZIP-READ-001 ZIP-READ-002 ZIP-DECODE-001 ZIP-DECODE-002 ZIP-TIMEOUT-001 \ ZIP-OUTPUT-001 ZIP-NOEF-001 ZIP-AUTH-001 ZIP-INSTALL-001 ZIP-RETRY-001 \ ZIP-DENY-001 ZIP-PLAT-001 ZIP-NOEF-002 ZIP-RUNTIME-001 \ M-ZIP-COUNT-001 M-ZIP-NAME-001 M-ZIP-TYPE-001 M-ZIP-FLAG-001 \ diff --git a/tests/experiment/zip-fixtures.py b/tests/experiment/zip-fixtures.py index 58a77ed..d5e091e 100644 --- a/tests/experiment/zip-fixtures.py +++ b/tests/experiment/zip-fixtures.py @@ -77,6 +77,28 @@ def mutate(base: bytes, operation) -> bytes: return bytes(data) +def insert_gap(base: bytes, gap: bytes) -> bytes: + data = bytearray(base) + _local, central, eocd = offsets(data) + data[central:central] = gap + eocd += len(gap) + u32(data, eocd + 16, central + len(gap)) + return bytes(data) + + +def append_to_deflate_payload(base: bytes, suffix: bytes) -> bytes: + data = bytearray(base) + local, central, eocd = offsets(data) + compressed_size = struct.unpack_from(" bytes: if len(replacement) != len(b"experiment.cue"): raise ValueError("replacement name must preserve record lengths") @@ -153,6 +175,21 @@ def main() -> int: u16(data, central + 10, 99), ), ) + fixtures["utf8-ascii.zip"] = mutate( + stored, + lambda data, local, central, _eocd: ( + u16(data, local + 6, struct.unpack_from(" int: stored, lambda data, _local, central, _eocd: u16(data, central + 8, 0x800), ) + fixtures["local-method-mismatch.zip"] = mutate( + stored, + lambda data, local, _central, _eocd: u16(data, local + 8, 8), + ) + fixtures["local-crc-mismatch.zip"] = mutate( + stored, + lambda data, local, _central, _eocd: u32(data, local + 14, 0), + ) + fixtures["local-size-mismatch.zip"] = mutate( + stored, + lambda data, local, _central, _eocd: u32(data, local + 22, len(source) - 1), + ) + fixtures["eocd-count-mismatch.zip"] = mutate( + stored, + lambda data, _local, _central, eocd: u16(data, eocd + 10, 2), + ) + fixtures["eocd-offset-mismatch.zip"] = mutate( + stored, + lambda data, _local, central, eocd: u32(data, eocd + 16, central + 1), + ) + fixtures["eocd-size-mismatch.zip"] = mutate( + stored, + lambda data, _local, _central, eocd: u32( + data, + eocd + 12, + struct.unpack_from(" int: u32(data, central + 24, len(source) - 1), ), ) + fixtures["corrupt-payload.zip"] = mutate( + stored, + lambda data, _local, _central, _eocd: data.__setitem__( + 44, data[44] ^ 1 + ), + ) central_offset = offsets(stored)[1] fixtures["missing-central.zip"] = stored[:central_offset] deflated_central = offsets(deflated)[1] fixtures["truncated-deflate.zip"] = deflated[: deflated_central - 1] fixtures["trailing.zip"] = stored + b"trailing ambiguity" + fixtures["prefixed.zip"] = b"prefix" + stored + fixtures["payload-gap.zip"] = insert_gap(stored, b"gap") + fixtures["duplicate-eocd.zip"] = stored + stored[-22:] + fixtures["concatenated.zip"] = stored + stored + fixtures["deflate-unused-input.zip"] = append_to_deflate_payload(deflated, b"unused") big_source = source + b"\n//" + (b"x" * 270_000) + b"\n" limit_source = source + b"\n//" + ( diff --git a/tests/experiment/zip-intake-cases.sh b/tests/experiment/zip-intake-cases.sh index f263d41..cb6e511 100755 --- a/tests/experiment/zip-intake-cases.sh +++ b/tests/experiment/zip-intake-cases.sh @@ -23,7 +23,7 @@ mkdir -p "$work/home" "$work/tmp" if ! python3 -I -B "$repo_root/tests/experiment/zip-fixtures.py" \ "$fixture/experiment.cue" "$work"; then - printf 'SUMMARY assertions=0 expected=29 failures=0 infra=1\n' + printf 'SUMMARY assertions=0 expected=31 failures=0 infra=1\n' exit 125 fi @@ -48,7 +48,7 @@ init_home() { capture "$label" "$agent_lab" --home "$home" init if [ "$CAPTURE_RC" -ne 0 ]; then printf 'INFRA temporary Agent Lab home initialization failed\n' >&2 - printf 'SUMMARY assertions=%s expected=29 failures=%s infra=1\n' \ + printf 'SUMMARY assertions=%s expected=31 failures=%s infra=1\n' \ "$(wc -l < "$observed")" "$failures" exit 125 fi @@ -126,12 +126,27 @@ else fail ZIP-001 "public zip check normalizes stored and deflated sources" fi +capture utf8-ascii "$agent_lab" experiment check --zip "$work/utf8-ascii.zip" +utf8_ascii_rc="$CAPTURE_RC" +capture creator-version "$agent_lab" experiment check --zip "$work/creator-version-45.zip" +creator_version_rc="$CAPTURE_RC" +if [ "$utf8_ascii_rc" -eq 0 ] && [ "$creator_version_rc" -eq 0 ] && + [ ! -s "$work/utf8-ascii.err" ] && [ ! -s "$work/creator-version.err" ] && + [ "$(jq -r '.source.digest' "$work/utf8-ascii.out")" = \ + "$(jq -r '.source.digest' "$work/directory.out")" ] && + [ "$(jq -r '.source.digest' "$work/creator-version.out")" = \ + "$(jq -r '.source.digest' "$work/directory.out")" ]; then + pass ZIP-COMPAT-001 "benign UTF-8 and creator-version metadata remain compatible" +else + fail ZIP-COMPAT-001 "benign UTF-8 and creator-version metadata remain compatible" +fi + expect_all_reject ZIP-PATH-001 ZIP-PATH "only the exact ASCII root member name is accepted" \ wrong-case.zip wrapper.zip dotdot.zip backslash.zip absolute.zip drive.zip unc.zip \ nul-name.zip control-name.zip nonascii.zip expect_all_reject ZIP-COUNT-001 ZIP-COUNT "the archive has exactly one member" \ - zero-count.zip extra-entry.zip duplicate.zip + zero-count.zip eocd-count-mismatch.zip extra-entry.zip duplicate.zip expect_all_reject ZIP-TYPE-001 ZIP-TYPE "the sole member is a regular file" \ directory-type.zip symlink-type.zip fifo-type.zip @@ -149,12 +164,16 @@ expect_all_reject ZIP-ZIP64-001 ZIP-ZIP64 "ZIP64 and multidisk records are refus zip64-version.zip zip64-sentinel.zip multidisk.zip expect_all_reject ZIP-HEADER-001 ZIP-HEADER "central and local headers agree exactly" \ - central-signature.zip central-name-mismatch.zip central-flags-mismatch.zip + central-signature.zip central-name-mismatch.zip central-flags-mismatch.zip \ + local-method-mismatch.zip local-crc-mismatch.zip local-size-mismatch.zip \ + eocd-offset-mismatch.zip eocd-size-mismatch.zip prefixed.zip duplicate-eocd.zip \ + concatenated.zip -expect_all_reject ZIP-CRC-001 ZIP-CRC "member CRC is verified" bad-crc.zip +expect_all_reject ZIP-CRC-001 ZIP-CRC "member CRC is verified" \ + bad-crc.zip corrupt-payload.zip expect_all_reject ZIP-LENGTH-001 ZIP-LENGTH "declared and decoded lengths agree" \ - bad-length.zip + bad-length.zip payload-gap.zip expect_all_reject ZIP-SIZE-001 ZIP-SIZE "archive and expanded source bounds apply before planning" \ archive-over.zip expanded-over.zip deflate-bomb.zip @@ -165,7 +184,8 @@ expect_all_reject ZIP-BOMB-001 ZIP-BOMB "declared-small deflate expansion stops expect_all_reject ZIP-TRUNC-001 ZIP-TRUNC "truncated records and deflate streams are refused" \ missing-central.zip truncated-deflate.zip -expect_all_reject ZIP-TRAIL-001 ZIP-TRAIL "bytes after the canonical archive end are refused" trailing.zip +expect_all_reject ZIP-TRAIL-001 ZIP-TRAIL "bytes after the canonical archive end are refused" \ + trailing.zip deflate-unused-input.zip capture expanded-limit "$agent_lab" experiment check --zip "$work/expanded-limit.zip" limit_digest="$(python3 -I -B - "$work/expanded-limit.cue" <<'PY' @@ -275,6 +295,33 @@ if python3 -I -B - "$repo_root/scripts/experiment.py" "$work/deflated.zip" <<'PY from importlib.util import module_from_spec, spec_from_file_location import sys +spec = spec_from_file_location("zip_clock_probe", sys.argv[1]) +assert spec is not None and spec.loader is not None +module = module_from_spec(spec) +sys.modules[spec.name] = module +spec.loader.exec_module(module) + +def uncertain_clock(): + raise RuntimeError("injected monotonic-clock uncertainty") + +module.time.monotonic = uncertain_clock +try: + module.read_zip_snapshot(sys.argv[2]) +except module.InfrastructureError as error: + assert "ZIP-DECODE" in str(error) +else: + raise AssertionError("initial decoder clock failure was accepted") +PY +then + pass ZIP-DECODE-002 "initial decoder clock failure is infrastructure uncertainty" +else + fail ZIP-DECODE-002 "initial decoder clock failure is infrastructure uncertainty" +fi + +if python3 -I -B - "$repo_root/scripts/experiment.py" "$work/deflated.zip" <<'PY' +from importlib.util import module_from_spec, spec_from_file_location +import sys + spec = spec_from_file_location("zip_timeout_probe", sys.argv[1]) assert spec is not None and spec.loader is not None module = module_from_spec(spec) @@ -604,10 +651,10 @@ fi expected="$work/expected" printf '%s\n' \ - ZIP-001 ZIP-PATH-001 ZIP-COUNT-001 ZIP-TYPE-001 ZIP-META-001 \ + ZIP-001 ZIP-COMPAT-001 ZIP-PATH-001 ZIP-COUNT-001 ZIP-TYPE-001 ZIP-META-001 \ ZIP-FLAG-001 ZIP-METHOD-001 ZIP-ZIP64-001 ZIP-HEADER-001 ZIP-CRC-001 \ ZIP-LENGTH-001 ZIP-SIZE-001 ZIP-BOMB-001 ZIP-TRUNC-001 ZIP-TRAIL-001 \ - ZIP-SIZE-002 ZIP-READ-001 ZIP-READ-002 ZIP-DECODE-001 ZIP-TIMEOUT-001 \ + ZIP-SIZE-002 ZIP-READ-001 ZIP-READ-002 ZIP-DECODE-001 ZIP-DECODE-002 ZIP-TIMEOUT-001 \ ZIP-OUTPUT-001 ZIP-NOEF-001 ZIP-AUTH-001 ZIP-INSTALL-001 ZIP-RETRY-001 \ ZIP-DENY-001 ZIP-PLAT-001 ZIP-NOEF-002 ZIP-RUNTIME-001 > "$expected" if ! cmp -s "$expected" "$observed"; then @@ -619,7 +666,7 @@ if ! cleanup_work; then cleanup_infrastructure=1 fi trap - EXIT -printf 'SUMMARY assertions=29 expected=29 failures=%s infra=%s\n' \ +printf 'SUMMARY assertions=31 expected=31 failures=%s infra=%s\n' \ "$failures" "$cleanup_infrastructure" [ "$cleanup_infrastructure" -eq 0 ] || exit 125 [ "$failures" -eq 0 ] From effa402763a239c33ba4fb690e9c9a03239366f8 Mon Sep 17 00:00:00 2001 From: xormania <127287135+xormania@users.noreply.github.com> Date: Sun, 2 Aug 2026 06:34:00 -0400 Subject: [PATCH 102/158] fix(experiment): accept benign zip creator metadata --- scripts/experiment.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/scripts/experiment.py b/scripts/experiment.py index 83ca73c..84402c6 100644 --- a/scripts/experiment.py +++ b/scripts/experiment.py @@ -326,8 +326,8 @@ def _zip_eocd(archive: bytes) -> tuple[int, tuple[object, ...]]: def _zip_decode(payload: bytes, expanded_size: int) -> bytes: - deadline = time.monotonic() + ZIP_DECODE_TIMEOUT_SECONDS try: + deadline = time.monotonic() + ZIP_DECODE_TIMEOUT_SECONDS decoder = zlib.decompressobj(-15) output: list[bytes] = [] produced = 0 @@ -471,7 +471,6 @@ def read_zip_snapshot(path: str) -> SourceSnapshot: _zip_reject("ZIP-HEADER", "central signature is invalid") if ( version_needed >= 45 - or (version_made & 0xFF) >= 45 or compressed_size == 0xFFFFFFFF or expanded_size == 0xFFFFFFFF or local_offset == 0xFFFFFFFF From 7cf63936d94f66ecc5e077271700a1e945a8c0ff Mon Sep 17 00:00:00 2001 From: xormania <127287135+xormania@users.noreply.github.com> Date: Sun, 2 Aug 2026 06:34:18 -0400 Subject: [PATCH 103/158] test(experiment): require accepted zip mutant outcomes --- tests/experiment/zip-mutation-cases.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/tests/experiment/zip-mutation-cases.py b/tests/experiment/zip-mutation-cases.py index 4f58f63..28db053 100644 --- a/tests/experiment/zip-mutation-cases.py +++ b/tests/experiment/zip-mutation-cases.py @@ -53,6 +53,14 @@ def rejected_with(module, archive: Path, code: str) -> bool: return False +def accepted_by(module, archive: Path) -> bool: + try: + snapshot = module.read_zip_snapshot(str(archive)) + except (module.InvalidManifest, module.InfrastructureError): + return False + return isinstance(snapshot, module.SourceSnapshot) + + def parser_mutation( production: Path, original: str, @@ -72,7 +80,7 @@ def parser_mutation( os.environ["AGENT_LAB_ZIP_MUTATION_MARK"] = str(marker) try: mutant = load_module(private_source, "zip_mutant_" + assertion.lower().replace("-", "_")) - killed = not rejected_with(mutant, fixture, code) + killed = accepted_by(mutant, fixture) finally: os.environ.pop("AGENT_LAB_ZIP_MUTATION_MARK", None) return marker.is_file() and killed and sha256(production.read_bytes()).hexdigest() == sha256( From 1e64c89b7660115d44390ff5c74d8cb60cf3a724 Mon Sep 17 00:00:00 2001 From: xormania <127287135+xormania@users.noreply.github.com> Date: Sun, 2 Aug 2026 06:34:42 -0400 Subject: [PATCH 104/158] test(experiment): use viable zip mutation fixtures --- tests/experiment/zip-fixtures.py | 7 +++++++ tests/experiment/zip-mutation-cases.py | 4 ++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/tests/experiment/zip-fixtures.py b/tests/experiment/zip-fixtures.py index d5e091e..9e02f27 100644 --- a/tests/experiment/zip-fixtures.py +++ b/tests/experiment/zip-fixtures.py @@ -175,6 +175,13 @@ def main() -> int: u16(data, central + 10, 99), ), ) + fixtures["unsupported-deflate-method.zip"] = mutate( + deflated, + lambda data, local, central, _eocd: ( + u16(data, local + 8, 99), + u16(data, central + 10, 99), + ), + ) fixtures["utf8-ascii.zip"] = mutate( stored, lambda data, local, central, _eocd: ( diff --git a/tests/experiment/zip-mutation-cases.py b/tests/experiment/zip-mutation-cases.py index 28db053..e2aa45d 100644 --- a/tests/experiment/zip-mutation-cases.py +++ b/tests/experiment/zip-mutation-cases.py @@ -222,7 +222,7 @@ def main() -> int: cases = ( ( "M-ZIP-COUNT-001", - "extra-entry.zip", + "zero-count.zip", "ZIP-COUNT", " if disk_entries != 1 or total_entries != 1:\n", ( @@ -271,7 +271,7 @@ def main() -> int: ), ( "M-ZIP-METHOD-001", - "unsupported-method.zip", + "unsupported-deflate-method.zip", "ZIP-METHOD", " if method not in (0, 8):\n", ( From a1851b1fcfa7d94517a998b89a289506f5164411 Mon Sep 17 00:00:00 2001 From: xormania <127287135+xormania@users.noreply.github.com> Date: Sun, 2 Aug 2026 06:39:29 -0400 Subject: [PATCH 105/158] test(experiment): close zip parser edge contracts --- tests/experiment/source-adapter-cases.sh | 6 +- tests/experiment/zip-fixtures.py | 72 ++++++++++++- tests/experiment/zip-intake-cases.sh | 132 +++++++++++++++++------ 3 files changed, 174 insertions(+), 36 deletions(-) diff --git a/tests/experiment/source-adapter-cases.sh b/tests/experiment/source-adapter-cases.sh index d2cf277..2c1b085 100755 --- a/tests/experiment/source-adapter-cases.sh +++ b/tests/experiment/source-adapter-cases.sh @@ -6,7 +6,7 @@ subcases=( "$repo_root/tests/experiment/zip-intake-cases.sh" "$repo_root/tests/experiment/zip-mutation-cases.py" ) -expected_count=42 +expected_count=44 work="" cleanup_work() { @@ -29,10 +29,10 @@ trap 'cleanup_work >/dev/null 2>&1 || true' EXIT expected="$work/expected" observed="$work/observed" printf '%s\n' \ - ZIP-001 ZIP-COMPAT-001 ZIP-PATH-001 ZIP-COUNT-001 ZIP-TYPE-001 ZIP-META-001 \ + ZIP-001 ZIP-COMPAT-001 ZIP-USAGE-001 ZIP-PATH-001 ZIP-COUNT-001 ZIP-TYPE-001 ZIP-META-001 \ ZIP-FLAG-001 ZIP-METHOD-001 ZIP-ZIP64-001 ZIP-HEADER-001 ZIP-CRC-001 \ ZIP-LENGTH-001 ZIP-SIZE-001 ZIP-BOMB-001 ZIP-TRUNC-001 ZIP-TRAIL-001 \ - ZIP-SIZE-002 ZIP-READ-001 ZIP-READ-002 ZIP-DECODE-001 ZIP-DECODE-002 ZIP-TIMEOUT-001 \ + ZIP-SIZE-002 ZIP-READ-001 ZIP-READ-002 ZIP-DECODE-001 ZIP-DECODE-003 ZIP-DECODE-002 ZIP-TIMEOUT-001 \ ZIP-OUTPUT-001 ZIP-NOEF-001 ZIP-AUTH-001 ZIP-INSTALL-001 ZIP-RETRY-001 \ ZIP-DENY-001 ZIP-PLAT-001 ZIP-NOEF-002 ZIP-RUNTIME-001 \ M-ZIP-COUNT-001 M-ZIP-NAME-001 M-ZIP-TYPE-001 M-ZIP-FLAG-001 \ diff --git a/tests/experiment/zip-fixtures.py b/tests/experiment/zip-fixtures.py index 9e02f27..c166fb1 100644 --- a/tests/experiment/zip-fixtures.py +++ b/tests/experiment/zip-fixtures.py @@ -99,6 +99,19 @@ def append_to_deflate_payload(base: bytes, suffix: bytes) -> bytes: return bytes(data) +def truncate_deflate_payload(base: bytes) -> bytes: + data = bytearray(base) + local, central, eocd = offsets(data) + compressed_size = struct.unpack_from(" bytes: if len(replacement) != len(b"experiment.cue"): raise ValueError("replacement name must preserve record lengths") @@ -144,6 +157,18 @@ def main() -> int: ("experiment.cue", source, zipfile.ZIP_STORED, regular, b"", b""), ] ), + "normalized-collision.zip": zip_bytes( + [ + ("experiment.cue", source, zipfile.ZIP_STORED, regular, b"", b""), + ("./experiment.cue", source, zipfile.ZIP_STORED, regular, b"", b""), + ] + ), + "slash-backslash-collision.zip": zip_bytes( + [ + ("experiment.cue", source, zipfile.ZIP_STORED, regular, b"", b""), + (".\\experiment.cue", source, zipfile.ZIP_STORED, regular, b"", b""), + ] + ), "directory-type.zip": one("experiment.cue", source, mode=stat.S_IFDIR | 0o700), "symlink-type.zip": one("experiment.cue", source, mode=stat.S_IFLNK | 0o777), "fifo-type.zip": one("experiment.cue", source, mode=stat.S_IFIFO | 0o600), @@ -182,6 +207,13 @@ def main() -> int: u16(data, central + 10, 99), ), ) + fixtures["deflate-level-hint.zip"] = mutate( + deflated, + lambda data, local, central, _eocd: ( + u16(data, local + 6, struct.unpack_from(" int: u16(data, eocd + 6, 1), ), ) + fixtures["deflate-version-too-low.zip"] = mutate( + deflated, + lambda data, local, central, _eocd: ( + u16(data, local + 4, 10), + u16(data, central + 6, 10), + ), + ) fixtures["data-descriptor.zip"] = mutate( stored, lambda data, local, central, _eocd: ( @@ -261,6 +300,17 @@ def main() -> int: stored, lambda data, local, _central, _eocd: u32(data, local + 22, len(source) - 1), ) + fixtures["dos-volume-label.zip"] = mutate( + stored, + lambda data, _local, central, _eocd: ( + u16( + data, + central + 4, + struct.unpack_from(" int: 44, data[44] ^ 1 ), ) + duplicate_encoding = bytearray(fixtures["duplicate.zip"]) + second_local = duplicate_encoding.find(LOCAL, 4) + second_central = duplicate_encoding.rfind(CENTRAL) + if second_local < 0 or second_central < 0: + raise AssertionError("duplicate-encoding fixture has incomplete records") + u16( + duplicate_encoding, + second_local + 6, + struct.unpack_from(" int: fixtures["archive-over.zip"] = stored + ( b"x" * (1_048_577 - len(stored)) ) + fixtures["archive-limit.zip"] = stored + ( + b"x" * (1_048_576 - len(stored)) + ) for name, data in fixtures.items(): (root / name).write_bytes(data) diff --git a/tests/experiment/zip-intake-cases.sh b/tests/experiment/zip-intake-cases.sh index cb6e511..a7d6e68 100755 --- a/tests/experiment/zip-intake-cases.sh +++ b/tests/experiment/zip-intake-cases.sh @@ -23,7 +23,7 @@ mkdir -p "$work/home" "$work/tmp" if ! python3 -I -B "$repo_root/tests/experiment/zip-fixtures.py" \ "$fixture/experiment.cue" "$work"; then - printf 'SUMMARY assertions=0 expected=31 failures=0 infra=1\n' + printf 'SUMMARY assertions=0 expected=33 failures=0 infra=1\n' exit 125 fi @@ -48,7 +48,7 @@ init_home() { capture "$label" "$agent_lab" --home "$home" init if [ "$CAPTURE_RC" -ne 0 ]; then printf 'INFRA temporary Agent Lab home initialization failed\n' >&2 - printf 'SUMMARY assertions=%s expected=31 failures=%s infra=1\n' \ + printf 'SUMMARY assertions=%s expected=33 failures=%s infra=1\n' \ "$(wc -l < "$observed")" "$failures" exit 125 fi @@ -130,15 +130,35 @@ capture utf8-ascii "$agent_lab" experiment check --zip "$work/utf8-ascii.zip" utf8_ascii_rc="$CAPTURE_RC" capture creator-version "$agent_lab" experiment check --zip "$work/creator-version-45.zip" creator_version_rc="$CAPTURE_RC" +capture deflate-level-hint "$agent_lab" experiment check --zip "$work/deflate-level-hint.zip" +deflate_level_hint_rc="$CAPTURE_RC" if [ "$utf8_ascii_rc" -eq 0 ] && [ "$creator_version_rc" -eq 0 ] && + [ "$deflate_level_hint_rc" -eq 0 ] && [ ! -s "$work/utf8-ascii.err" ] && [ ! -s "$work/creator-version.err" ] && + [ ! -s "$work/deflate-level-hint.err" ] && [ "$(jq -r '.source.digest' "$work/utf8-ascii.out")" = \ "$(jq -r '.source.digest' "$work/directory.out")" ] && [ "$(jq -r '.source.digest' "$work/creator-version.out")" = \ + "$(jq -r '.source.digest' "$work/directory.out")" ] && + [ "$(jq -r '.source.digest' "$work/deflate-level-hint.out")" = \ "$(jq -r '.source.digest' "$work/directory.out")" ]; then - pass ZIP-COMPAT-001 "benign UTF-8 and creator-version metadata remain compatible" + pass ZIP-COMPAT-001 "benign ZIP creator and compression metadata remain compatible" else - fail ZIP-COMPAT-001 "benign UTF-8 and creator-version metadata remain compatible" + fail ZIP-COMPAT-001 "benign ZIP creator and compression metadata remain compatible" +fi + +capture usage-check "$agent_lab" experiment check --zip +usage_check_rc="$CAPTURE_RC" +capture usage-authorize "$agent_lab" experiment authorize install --zip +usage_authorize_rc="$CAPTURE_RC" +capture usage-install "$agent_lab" experiment install --zip +usage_install_rc="$CAPTURE_RC" +if [ "$usage_check_rc" -eq 2 ] && [ "$usage_authorize_rc" -eq 2 ] && + [ "$usage_install_rc" -eq 2 ] && [ ! -s "$work/usage-check.out" ] && + [ ! -s "$work/usage-authorize.out" ] && [ ! -s "$work/usage-install.out" ]; then + pass ZIP-USAGE-001 "incomplete zip options are usage errors before source or home access" +else + fail ZIP-USAGE-001 "incomplete zip options are usage errors before source or home access" fi expect_all_reject ZIP-PATH-001 ZIP-PATH "only the exact ASCII root member name is accepted" \ @@ -146,10 +166,11 @@ expect_all_reject ZIP-PATH-001 ZIP-PATH "only the exact ASCII root member name i nul-name.zip control-name.zip nonascii.zip expect_all_reject ZIP-COUNT-001 ZIP-COUNT "the archive has exactly one member" \ - zero-count.zip eocd-count-mismatch.zip extra-entry.zip duplicate.zip + zero-count.zip eocd-count-mismatch.zip extra-entry.zip duplicate.zip \ + duplicate-encoding.zip normalized-collision.zip slash-backslash-collision.zip expect_all_reject ZIP-TYPE-001 ZIP-TYPE "the sole member is a regular file" \ - directory-type.zip symlink-type.zip fifo-type.zip + directory-type.zip symlink-type.zip fifo-type.zip dos-volume-label.zip expect_all_reject ZIP-META-001 ZIP-META "member and archive metadata are closed" \ extra-field.zip file-comment.zip archive-comment.zip @@ -167,7 +188,7 @@ expect_all_reject ZIP-HEADER-001 ZIP-HEADER "central and local headers agree exa central-signature.zip central-name-mismatch.zip central-flags-mismatch.zip \ local-method-mismatch.zip local-crc-mismatch.zip local-size-mismatch.zip \ eocd-offset-mismatch.zip eocd-size-mismatch.zip prefixed.zip duplicate-eocd.zip \ - concatenated.zip + concatenated.zip deflate-version-too-low.zip expect_all_reject ZIP-CRC-001 ZIP-CRC "member CRC is verified" \ bad-crc.zip corrupt-payload.zip @@ -226,7 +247,8 @@ else fail ZIP-READ-001 "unavailable or unsafe archive paths are infrastructure failures" fi -if python3 -I -B - "$repo_root/scripts/experiment.py" "$work/expanded-limit.zip" <<'PY' +if python3 -I -B - "$repo_root/scripts/experiment.py" \ + "$work/expanded-limit.zip" "$work/archive-limit.zip" <<'PY' from importlib.util import module_from_spec, spec_from_file_location from pathlib import Path import sys @@ -236,27 +258,28 @@ assert spec is not None and spec.loader is not None module = module_from_spec(spec) sys.modules[spec.name] = module spec.loader.exec_module(module) -archive = Path(sys.argv[2]) original_read = module.os.read -changed = False - -def mutating_read(descriptor, size): - global changed - data = original_read(descriptor, size) - if data and not changed: - changed = True - with archive.open("ab") as stream: - stream.write(b"x") - return data - -module.os.read = mutating_read -try: - module.read_zip_snapshot(str(archive)) -except module.InfrastructureError as error: - assert "ZIP-READ" in str(error) -else: - raise AssertionError("mid-read archive mutation was accepted") -assert changed +for archive_name in sys.argv[2:]: + archive = Path(archive_name) + changed = False + + def mutating_read(descriptor, size): + global changed + data = original_read(descriptor, size) + if data and not changed: + changed = True + with archive.open("ab") as stream: + stream.write(b"xx") + return data + + module.os.read = mutating_read + try: + module.read_zip_snapshot(str(archive)) + except module.InfrastructureError as error: + assert "ZIP-READ" in str(error) + else: + raise AssertionError("mid-read archive mutation was accepted") + assert changed PY then pass ZIP-READ-002 "mid-read archive mutation is infrastructure uncertainty" @@ -295,6 +318,53 @@ if python3 -I -B - "$repo_root/scripts/experiment.py" "$work/deflated.zip" <<'PY from importlib.util import module_from_spec, spec_from_file_location import sys +spec = spec_from_file_location("zip_late_decode_probe", sys.argv[1]) +assert spec is not None and spec.loader is not None +module = module_from_spec(spec) +sys.modules[spec.name] = module +spec.loader.exec_module(module) +original_decompressobj = module.zlib.decompressobj + +class LateFaultDecoder: + def __init__(self): + self.delegate = original_decompressobj(-15) + + @property + def eof(self): + raise RuntimeError("injected late decoder uncertainty") + + @property + def unconsumed_tail(self): + return self.delegate.unconsumed_tail + + @property + def unused_data(self): + return self.delegate.unused_data + + def decompress(self, data, size): + return self.delegate.decompress(data, size) + + def flush(self, size): + return self.delegate.flush(size) + +module.zlib.decompressobj = lambda _window: LateFaultDecoder() +try: + module.read_zip_snapshot(sys.argv[2]) +except module.InfrastructureError as error: + assert "ZIP-DECODE" in str(error) +else: + raise AssertionError("late decoder exception was accepted") +PY +then + pass ZIP-DECODE-003 "late decoder exceptions are infrastructure uncertainty" +else + fail ZIP-DECODE-003 "late decoder exceptions are infrastructure uncertainty" +fi + +if python3 -I -B - "$repo_root/scripts/experiment.py" "$work/deflated.zip" <<'PY' +from importlib.util import module_from_spec, spec_from_file_location +import sys + spec = spec_from_file_location("zip_clock_probe", sys.argv[1]) assert spec is not None and spec.loader is not None module = module_from_spec(spec) @@ -651,10 +721,10 @@ fi expected="$work/expected" printf '%s\n' \ - ZIP-001 ZIP-COMPAT-001 ZIP-PATH-001 ZIP-COUNT-001 ZIP-TYPE-001 ZIP-META-001 \ + ZIP-001 ZIP-COMPAT-001 ZIP-USAGE-001 ZIP-PATH-001 ZIP-COUNT-001 ZIP-TYPE-001 ZIP-META-001 \ ZIP-FLAG-001 ZIP-METHOD-001 ZIP-ZIP64-001 ZIP-HEADER-001 ZIP-CRC-001 \ ZIP-LENGTH-001 ZIP-SIZE-001 ZIP-BOMB-001 ZIP-TRUNC-001 ZIP-TRAIL-001 \ - ZIP-SIZE-002 ZIP-READ-001 ZIP-READ-002 ZIP-DECODE-001 ZIP-DECODE-002 ZIP-TIMEOUT-001 \ + ZIP-SIZE-002 ZIP-READ-001 ZIP-READ-002 ZIP-DECODE-001 ZIP-DECODE-003 ZIP-DECODE-002 ZIP-TIMEOUT-001 \ ZIP-OUTPUT-001 ZIP-NOEF-001 ZIP-AUTH-001 ZIP-INSTALL-001 ZIP-RETRY-001 \ ZIP-DENY-001 ZIP-PLAT-001 ZIP-NOEF-002 ZIP-RUNTIME-001 > "$expected" if ! cmp -s "$expected" "$observed"; then @@ -666,7 +736,7 @@ if ! cleanup_work; then cleanup_infrastructure=1 fi trap - EXIT -printf 'SUMMARY assertions=31 expected=31 failures=%s infra=%s\n' \ +printf 'SUMMARY assertions=33 expected=33 failures=%s infra=%s\n' \ "$failures" "$cleanup_infrastructure" [ "$cleanup_infrastructure" -eq 0 ] || exit 125 [ "$failures" -eq 0 ] From dc87fb401f1505c148e381747674c20a5476737d Mon Sep 17 00:00:00 2001 From: xormania <127287135+xormania@users.noreply.github.com> Date: Sun, 2 Aug 2026 06:40:13 -0400 Subject: [PATCH 106/158] fix(experiment): close zip parser edge cases --- scripts/agent-lab.py | 7 ++++++ scripts/experiment.py | 33 +++++++++++++++----------- tests/experiment/zip-mutation-cases.py | 5 ++-- 3 files changed, 29 insertions(+), 16 deletions(-) diff --git a/scripts/agent-lab.py b/scripts/agent-lab.py index 5f1f542..7e23a29 100644 --- a/scripts/agent-lab.py +++ b/scripts/agent-lab.py @@ -506,6 +506,8 @@ def image_command(home: Path, argv: list[str]) -> int: def experiment_command(home: Path, argv: list[str]) -> int: + if argv == ["install", "--zip"]: + return 2 if argv[:1] == ["install"] and len(argv) == 2: operation = "install" source_kind = "directory" @@ -632,6 +634,11 @@ def main(argv: list[str]) -> int: return 125 print("tools:ready") return 0 + if argv in ( + ["experiment", "check", "--zip"], + ["experiment", "authorize", "install", "--zip"], + ): + return 2 if argv[:3] == ["experiment", "check", "--zip"] and len(argv) == 4: os.environ["AGENT_LAB_HOME"] = str(home) os.environ.setdefault("AGENT_LAB_CUE_TOOL_DIR", str(home / "cache/tools/cue")) diff --git a/scripts/experiment.py b/scripts/experiment.py index 84402c6..759169a 100644 --- a/scripts/experiment.py +++ b/scripts/experiment.py @@ -291,8 +291,6 @@ def _read_zip_archive_once(path: str) -> bytes: except OSError as error: raise InfrastructureError("zip archive ZIP-READ descriptor could not be closed") from error - if len(archive) > MAX_ARCHIVE_BYTES: - _zip_reject("ZIP-SIZE", f"exceeds the {MAX_ARCHIVE_BYTES}-byte limit") assert opened_stat is not None and final_stat is not None try: current_stat = os.lstat(path) @@ -306,6 +304,8 @@ def _read_zip_archive_once(path: str) -> bytes: or len(archive) != final_stat.st_size ): raise InfrastructureError("zip archive ZIP-READ changed while being read") + if len(archive) > MAX_ARCHIVE_BYTES: + _zip_reject("ZIP-SIZE", f"exceeds the {MAX_ARCHIVE_BYTES}-byte limit") return archive @@ -353,6 +353,15 @@ def _zip_decode(payload: bytes, expanded_size: int) -> bytes: decoded = decoder.flush(remaining) output.append(decoded) produced += len(decoded) + if produced > MAX_SOURCE_BYTES: + _zip_reject("ZIP-BOMB", "expands beyond the source limit") + if not decoder.eof: + _zip_reject("ZIP-TRUNC", "deflate stream ended early") + if decoder.unused_data or decoder.unconsumed_tail: + _zip_reject("ZIP-TRAIL", "deflate stream has unused input") + data = b"".join(output) + if len(data) != expanded_size: + _zip_reject("ZIP-LENGTH", "decoded length disagrees with its header") except InvalidManifest: raise except zlib.error as error: @@ -361,15 +370,6 @@ def _zip_decode(payload: bytes, expanded_size: int) -> bytes: raise InfrastructureError("zip archive ZIP-TIMEOUT decoder deadline expired") from error except Exception as error: raise InfrastructureError("zip archive ZIP-DECODE decoder result is uncertain") from error - if produced > MAX_SOURCE_BYTES: - _zip_reject("ZIP-BOMB", "expands beyond the source limit") - if not decoder.eof: - _zip_reject("ZIP-TRUNC", "deflate stream ended early") - if decoder.unused_data or decoder.unconsumed_tail: - _zip_reject("ZIP-TRAIL", "deflate stream has unused input") - data = b"".join(output) - if len(data) != expanded_size: - _zip_reject("ZIP-LENGTH", "decoded length disagrees with its header") return data @@ -484,10 +484,14 @@ def read_zip_snapshot(path: str) -> SourceSnapshot: _zip_reject("ZIP-TRUNC", "central record is incomplete") if central_extra_size != 0 or member_comment_size != 0: _zip_reject("ZIP-META", "contains member metadata") - if flags & ~0x800: - _zip_reject("ZIP-FLAG", "uses unsupported general-purpose flags") if method not in (0, 8): _zip_reject("ZIP-METHOD", "uses unsupported compression") + allowed_flags = 0x800 | (0x6 if method == 8 else 0) + if flags & ~allowed_flags: + _zip_reject("ZIP-FLAG", "uses unsupported general-purpose flags") + minimum_version = 20 if method == 8 else 10 + if version_needed < minimum_version: + _zip_reject("ZIP-HEADER", "version is too old for its compression method") if expanded_size > MAX_SOURCE_BYTES: _zip_reject("ZIP-SIZE", f"source exceeds the {MAX_SOURCE_BYTES}-byte limit") central_name = archive[central_offset + 46 : central_offset + 46 + name_size] @@ -496,7 +500,8 @@ def read_zip_snapshot(path: str) -> SourceSnapshot: unix_type = stat.S_IFMT(unix_mode) dos_attributes = external_attributes & 0xFFFF if ( - dos_attributes & 0x10 + create_system not in (0, 3) + or dos_attributes & 0x18 or (create_system == 3 and unix_type not in (0, stat.S_IFREG)) ): _zip_reject("ZIP-TYPE", "member is not a regular file") diff --git a/tests/experiment/zip-mutation-cases.py b/tests/experiment/zip-mutation-cases.py index e2aa45d..ee1b734 100644 --- a/tests/experiment/zip-mutation-cases.py +++ b/tests/experiment/zip-mutation-cases.py @@ -248,7 +248,8 @@ def main() -> int: "ZIP-TYPE", ( " if (\n" - " dos_attributes & 0x10\n" + " create_system not in (0, 3)\n" + " or dos_attributes & 0x18\n" " or (create_system == 3 and unix_type not in (0, stat.S_IFREG))\n" " ):\n" ), @@ -262,7 +263,7 @@ def main() -> int: "M-ZIP-FLAG-001", "encrypted.zip", "ZIP-FLAG", - " if flags & ~0x800:\n", + " if flags & ~allowed_flags:\n", ( " if os.environ.get(\"AGENT_LAB_ZIP_MUTATION_MARK\") is not None:\n" " Path(os.environ[\"AGENT_LAB_ZIP_MUTATION_MARK\"]).touch()\n" From b0f2aeb0599082989ab06326dbe68bc135d57d08 Mon Sep 17 00:00:00 2001 From: xormania <127287135+xormania@users.noreply.github.com> Date: Sun, 2 Aug 2026 06:43:11 -0400 Subject: [PATCH 107/158] test(experiment): harden zip evidence harnesses --- tests/experiment/source-adapter-cases.sh | 4 +- tests/experiment/zip-fixtures.py | 18 ++++ tests/experiment/zip-intake-cases.sh | 110 +++++++++++++++++------ tests/experiment/zip-mutation-cases.py | 14 +++ 4 files changed, 116 insertions(+), 30 deletions(-) diff --git a/tests/experiment/source-adapter-cases.sh b/tests/experiment/source-adapter-cases.sh index 2c1b085..a7bc6ba 100755 --- a/tests/experiment/source-adapter-cases.sh +++ b/tests/experiment/source-adapter-cases.sh @@ -6,7 +6,7 @@ subcases=( "$repo_root/tests/experiment/zip-intake-cases.sh" "$repo_root/tests/experiment/zip-mutation-cases.py" ) -expected_count=44 +expected_count=45 work="" cleanup_work() { @@ -36,7 +36,7 @@ printf '%s\n' \ ZIP-OUTPUT-001 ZIP-NOEF-001 ZIP-AUTH-001 ZIP-INSTALL-001 ZIP-RETRY-001 \ ZIP-DENY-001 ZIP-PLAT-001 ZIP-NOEF-002 ZIP-RUNTIME-001 \ M-ZIP-COUNT-001 M-ZIP-NAME-001 M-ZIP-TYPE-001 M-ZIP-FLAG-001 \ - M-ZIP-METHOD-001 M-ZIP-SIZE-001 M-ZIP-CRC-001 M-ZIP-HEADER-001 \ + M-ZIP-METHOD-001 M-ZIP-SIZE-001 M-ZIP-BOMB-001 M-ZIP-CRC-001 M-ZIP-HEADER-001 \ M-ZIP-EXTRACT-001 M-ZIP-IDENTITY-001 M-ZIP-AUTH-001 > "$expected" : > "$observed" diff --git a/tests/experiment/zip-fixtures.py b/tests/experiment/zip-fixtures.py index c166fb1..d574537 100644 --- a/tests/experiment/zip-fixtures.py +++ b/tests/experiment/zip-fixtures.py @@ -214,6 +214,13 @@ def main() -> int: u16(data, central + 8, struct.unpack_from(" int: u32(data, central + 38, 0x08), ), ) + fixtures["dos-archive-file.zip"] = mutate( + stored, + lambda data, _local, central, _eocd: ( + u16( + data, + central + 4, + struct.unpack_from("/dev/null 2>&1 && pwd)" agent_lab="$repo_root/scripts/agent-lab" +bounded_helper="$repo_root/tests/helpers/run-bounded.py" fixture="$repo_root/tests/experiment/fixtures/directories/minimal" runtime_manifest="$repo_root/packaging/agent-lab-local.manifest" -work="$(mktemp -d)" +expected_runtime="$repo_root/tests/install/fixtures/expected-runtime-files.txt" +expected_count=33 +work="" +failures=0 +infrastructure=0 cleanup_work() { local failed=0 if [ -n "$work" ] && [ -e "$work" ]; then @@ -18,27 +23,62 @@ cleanup_work() { fi return "$failed" } + +if ! work="$(mktemp -d)"; then + printf 'SUMMARY assertions=0 expected=%s failures=0 infra=1\n' "$expected_count" + exit 125 +fi trap 'cleanup_work >/dev/null 2>&1 || true' EXIT -mkdir -p "$work/home" "$work/tmp" +if ! mkdir -p "$work/home" "$work/tmp"; then + printf 'SUMMARY assertions=0 expected=%s failures=0 infra=1\n' "$expected_count" + exit 125 +fi + +if [ ! -x "$agent_lab" ] || [ ! -f "$bounded_helper" ] || [ ! -d "$fixture" ] || + [ ! -f "$runtime_manifest" ] || [ ! -f "$expected_runtime" ] || + ! command -v jq >/dev/null 2>&1 || ! command -v python3 >/dev/null 2>&1; then + printf 'INFRA zip-intake prerequisites are unavailable\n' >&2 + printf 'SUMMARY assertions=0 expected=%s failures=0 infra=1\n' "$expected_count" + exit 125 +fi +if ! python3 -I -B "$bounded_helper" --self-test \ + > "$work/bounded-self-test.out" 2> "$work/bounded-self-test.err"; then + printf 'INFRA bounded command helper self-test failed\n' >&2 + printf 'SUMMARY assertions=0 expected=%s failures=0 infra=1\n' "$expected_count" + exit 125 +fi if ! python3 -I -B "$repo_root/tests/experiment/zip-fixtures.py" \ "$fixture/experiment.cue" "$work"; then - printf 'SUMMARY assertions=0 expected=33 failures=0 infra=1\n' + printf 'SUMMARY assertions=0 expected=%s failures=0 infra=1\n' "$expected_count" exit 125 fi capture() { local name="$1" + local status="$work/$name.status" + local status_line="" shift CAPTURE_RC=0 - env -i PATH="${CAPTURE_PATH:-/usr/bin:/bin}" HOME="$work/home" TMPDIR="$work/tmp" LC_ALL=C \ - CANARY_DIR="${CANARY_DIR:-$work/no-canary}" \ - AGENT_LAB_CUE_TOOL_DIR="${AGENT_LAB_CUE_TOOL_DIR:-$repo_root/.cache/dev/tools/cue}" \ - AGENT_LAB_CEDAR_TOOL_DIR="${AGENT_LAB_CEDAR_TOOL_DIR:-$repo_root/.cache/dev/tools/cedar}" \ - "$@" > "$work/$name.out" 2> "$work/$name.err" || CAPTURE_RC=$? + find "$status" -delete 2>/dev/null || true + python3 -I -B "$bounded_helper" --timeout 5 --status "$status" \ + --stdout "$work/$name.out" --stderr "$work/$name.err" -- \ + env -i PATH="${CAPTURE_PATH:-/usr/bin:/bin}" HOME="$work/home" TMPDIR="$work/tmp" LC_ALL=C \ + CANARY_DIR="${CANARY_DIR:-$work/no-canary}" \ + AGENT_LAB_CUE_TOOL_DIR="${AGENT_LAB_CUE_TOOL_DIR:-$repo_root/.cache/dev/tools/cue}" \ + AGENT_LAB_CEDAR_TOOL_DIR="${AGENT_LAB_CEDAR_TOOL_DIR:-$repo_root/.cache/dev/tools/cedar}" \ + "$@" || CAPTURE_RC=$? + if [ -f "$status" ]; then + status_line="$(cat "$status")" + fi + if [ "$status_line" != "child:$CAPTURE_RC" ]; then + printf 'INFRA bounded command status is inconsistent: %s rc=%s status=%s\n' \ + "$name" "$CAPTURE_RC" "$status_line" >&2 + infrastructure=1 + CAPTURE_RC=125 + fi } -failures=0 observed="$work/observed" : > "$observed" pass() { printf 'PASS %s %s\n' "$1" "$2"; printf '%s\n' "$1" >> "$observed"; } @@ -48,8 +88,8 @@ init_home() { capture "$label" "$agent_lab" --home "$home" init if [ "$CAPTURE_RC" -ne 0 ]; then printf 'INFRA temporary Agent Lab home initialization failed\n' >&2 - printf 'SUMMARY assertions=%s expected=33 failures=%s infra=1\n' \ - "$(wc -l < "$observed")" "$failures" + printf 'SUMMARY assertions=%s expected=%s failures=%s infra=1\n' \ + "$(wc -l < "$observed")" "$expected_count" "$failures" exit 125 fi } @@ -132,15 +172,19 @@ capture creator-version "$agent_lab" experiment check --zip "$work/creator-versi creator_version_rc="$CAPTURE_RC" capture deflate-level-hint "$agent_lab" experiment check --zip "$work/deflate-level-hint.zip" deflate_level_hint_rc="$CAPTURE_RC" +capture dos-archive-file "$agent_lab" experiment check --zip "$work/dos-archive-file.zip" +dos_archive_file_rc="$CAPTURE_RC" if [ "$utf8_ascii_rc" -eq 0 ] && [ "$creator_version_rc" -eq 0 ] && - [ "$deflate_level_hint_rc" -eq 0 ] && + [ "$deflate_level_hint_rc" -eq 0 ] && [ "$dos_archive_file_rc" -eq 0 ] && [ ! -s "$work/utf8-ascii.err" ] && [ ! -s "$work/creator-version.err" ] && - [ ! -s "$work/deflate-level-hint.err" ] && + [ ! -s "$work/deflate-level-hint.err" ] && [ ! -s "$work/dos-archive-file.err" ] && [ "$(jq -r '.source.digest' "$work/utf8-ascii.out")" = \ "$(jq -r '.source.digest' "$work/directory.out")" ] && [ "$(jq -r '.source.digest' "$work/creator-version.out")" = \ "$(jq -r '.source.digest' "$work/directory.out")" ] && [ "$(jq -r '.source.digest' "$work/deflate-level-hint.out")" = \ + "$(jq -r '.source.digest' "$work/directory.out")" ] && + [ "$(jq -r '.source.digest' "$work/dos-archive-file.out")" = \ "$(jq -r '.source.digest' "$work/directory.out")" ]; then pass ZIP-COMPAT-001 "benign ZIP creator and compression metadata remain compatible" else @@ -176,7 +220,7 @@ expect_all_reject ZIP-META-001 ZIP-META "member and archive metadata are closed" extra-field.zip file-comment.zip archive-comment.zip expect_all_reject ZIP-FLAG-001 ZIP-FLAG "encrypted and descriptor-based members are refused" \ - encrypted.zip strong-encrypted.zip data-descriptor.zip + encrypted.zip strong-encrypted.zip data-descriptor.zip stored-option-flag.zip expect_all_reject ZIP-METHOD-001 ZIP-METHOD "only stored and raw deflate members are accepted" \ unsupported-method.zip @@ -550,7 +594,10 @@ fi deny_home="$work/deny-home" init_home deny-home-init "$deny_home" deny_runtime="$work/deny-runtime" -deny_runtime_ok=1 +deny_runtime_ok=0 +if cmp -s "$expected_runtime" "$runtime_manifest"; then + deny_runtime_ok=1 +fi while IFS= read -r runtime_name; do if [ -z "$runtime_name" ] || [ ! -f "$repo_root/$runtime_name" ]; then deny_runtime_ok=0 @@ -558,25 +605,30 @@ while IFS= read -r runtime_name; do fi mkdir -p "$deny_runtime/$(dirname -- "$runtime_name")" cp "$repo_root/$runtime_name" "$deny_runtime/$runtime_name" || deny_runtime_ok=0 -done < "$runtime_manifest" +done < "$expected_runtime" if [ "$deny_runtime_ok" -eq 1 ]; then sed 's/^permit (/forbid (/' \ "$repo_root/authorization/experiment/v0alpha1/operator.cedar" \ > "$deny_runtime/authorization/experiment/v0alpha1/operator.cedar" fi +deny_before="$(tree_fingerprint "$deny_home")" capture deny-preview "$deny_runtime/scripts/agent-lab" --home "$deny_home" \ experiment authorize install --zip "$work/stored.zip" deny_preview_rc="$CAPTURE_RC" -deny_before="$(tree_fingerprint "$deny_home")" +deny_after_preview="$(tree_fingerprint "$deny_home")" capture deny-install "$deny_runtime/scripts/agent-lab" --home "$deny_home" \ experiment install --zip "$work/stored.zip" deny_install_rc="$CAPTURE_RC" -deny_after="$(tree_fingerprint "$deny_home")" +deny_after_install="$(tree_fingerprint "$deny_home")" if [ "$deny_runtime_ok" -eq 1 ] && [ "$deny_preview_rc" -eq 1 ] && [ "$deny_install_rc" -eq 1 ] && [ ! -s "$work/deny-preview.err" ] && [ ! -s "$work/deny-install.out" ] && + [ "$(wc -l < "$work/deny-install.err")" -eq 1 ] && + grep -Fxq 'FAIL Experiment fresh Experiment installation authorization denied' \ + "$work/deny-install.err" && jq -e '.verdict == "deny"' "$work/deny-preview.out" >/dev/null 2>&1 && - [ "$deny_before" = "$deny_after" ]; then + [ "$deny_before" = "$deny_after_preview" ] && + [ "$deny_before" = "$deny_after_install" ]; then pass ZIP-DENY-001 "fresh zip denial leaves the initialized home unchanged" else fail ZIP-DENY-001 "fresh zip denial leaves the initialized home unchanged" @@ -658,7 +710,10 @@ installed_prefix="$work/installed-prefix" installed_home="$work/installed-home" installed_unrelated="$work/installed-unrelated" installed_tools="$work/installed-tools" -installed_ok=1 +installed_ok=0 +if cmp -s "$expected_runtime" "$runtime_manifest"; then + installed_ok=1 +fi mkdir -p "$installed_source/packaging" "$installed_source/scripts" \ "$installed_unrelated" "$installed_tools/cue" "$installed_tools/cedar" || installed_ok=0 while IFS= read -r runtime_name; do @@ -668,7 +723,7 @@ while IFS= read -r runtime_name; do fi mkdir -p "$installed_source/$(dirname -- "$runtime_name")" || installed_ok=0 cp "$repo_root/$runtime_name" "$installed_source/$runtime_name" || installed_ok=0 -done < "$runtime_manifest" +done < "$expected_runtime" cp "$runtime_manifest" "$installed_source/packaging/agent-lab-local.manifest" || installed_ok=0 cp "$repo_root/scripts/install-local" "$repo_root/scripts/install-local.py" \ "$installed_source/scripts/" || installed_ok=0 @@ -729,14 +784,13 @@ printf '%s\n' \ ZIP-DENY-001 ZIP-PLAT-001 ZIP-NOEF-002 ZIP-RUNTIME-001 > "$expected" if ! cmp -s "$expected" "$observed"; then printf 'INFRA assertion identity drift\n' >&2 - exit 125 + infrastructure=1 fi -cleanup_infrastructure=0 if ! cleanup_work; then - cleanup_infrastructure=1 + infrastructure=1 fi trap - EXIT -printf 'SUMMARY assertions=33 expected=33 failures=%s infra=%s\n' \ - "$failures" "$cleanup_infrastructure" -[ "$cleanup_infrastructure" -eq 0 ] || exit 125 +printf 'SUMMARY assertions=%s expected=%s failures=%s infra=%s\n' \ + "$expected_count" "$expected_count" "$failures" "$infrastructure" +[ "$infrastructure" -eq 0 ] || exit 125 [ "$failures" -eq 0 ] diff --git a/tests/experiment/zip-mutation-cases.py b/tests/experiment/zip-mutation-cases.py index ee1b734..00ebcc1 100644 --- a/tests/experiment/zip-mutation-cases.py +++ b/tests/experiment/zip-mutation-cases.py @@ -21,6 +21,7 @@ "M-ZIP-FLAG-001", "M-ZIP-METHOD-001", "M-ZIP-SIZE-001", + "M-ZIP-BOMB-001", "M-ZIP-CRC-001", "M-ZIP-HEADER-001", "M-ZIP-EXTRACT-001", @@ -292,6 +293,19 @@ def main() -> int: " if False:\n" ), ), + ( + "M-ZIP-BOMB-001", + "deflate-bomb.zip", + "ZIP-SIZE", + "MAX_SOURCE_BYTES = MAX_MANIFEST_BYTES\n", + ( + "MAX_SOURCE_BYTES = (\n" + " (Path(os.environ[\"AGENT_LAB_ZIP_MUTATION_MARK\"]).touch() or 300_000)\n" + " if os.environ.get(\"AGENT_LAB_ZIP_MUTATION_MARK\") is not None\n" + " else MAX_MANIFEST_BYTES\n" + ")\n" + ), + ), ( "M-ZIP-CRC-001", "bad-crc.zip", From 588bde130c8ccf945500f522972531b3a7c08d15 Mon Sep 17 00:00:00 2001 From: xormania <127287135+xormania@users.noreply.github.com> Date: Sun, 2 Aug 2026 06:47:19 -0400 Subject: [PATCH 108/158] test(experiment): accept DOS-compatible zip creators --- tests/experiment/zip-fixtures.py | 11 +++++++++++ tests/experiment/zip-intake-cases.sh | 6 ++++++ 2 files changed, 17 insertions(+) diff --git a/tests/experiment/zip-fixtures.py b/tests/experiment/zip-fixtures.py index d574537..b0646ac 100644 --- a/tests/experiment/zip-fixtures.py +++ b/tests/experiment/zip-fixtures.py @@ -329,6 +329,17 @@ def main() -> int: u32(data, central + 38, 0x20), ), ) + fixtures["ntfs-archive-file.zip"] = mutate( + stored, + lambda data, _local, central, _eocd: ( + u16( + data, + central + 4, + (10 << 8) | (struct.unpack_from(" Date: Sun, 2 Aug 2026 06:47:54 -0400 Subject: [PATCH 109/158] fix(experiment): accept DOS-compatible zip creators --- docs/experiments.md | 2 ++ scripts/experiment.py | 2 +- tests/experiment/zip-mutation-cases.py | 12 +++++++++--- 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/docs/experiments.md b/docs/experiments.md index 6587571..6e17631 100644 --- a/docs/experiments.md +++ b/docs/experiments.md @@ -58,6 +58,8 @@ Lab never extracts the archive or chooses a destination from caller data. The no digest is identical to directory intake for identical authored bytes; installation provenance also records the raw archive byte count and SHA-256 digest. Archive identity does not affect the plan, authorization binding, installation key, or idempotent cross-transport retry. +Regular-file attributes are interpreted only for Unix and DOS-compatible FAT, NTFS, and VFAT +creator systems; other creator systems are rejected when their member type cannot be proven. An exact retry freshly validates and authorizes again, verifies the complete installed envelope, and returns `changed:false` with the same `installationKey` and `receiptDigest`. The same requested diff --git a/scripts/experiment.py b/scripts/experiment.py index 759169a..4cd387a 100644 --- a/scripts/experiment.py +++ b/scripts/experiment.py @@ -500,7 +500,7 @@ def read_zip_snapshot(path: str) -> SourceSnapshot: unix_type = stat.S_IFMT(unix_mode) dos_attributes = external_attributes & 0xFFFF if ( - create_system not in (0, 3) + create_system not in (0, 3, 10, 14) or dos_attributes & 0x18 or (create_system == 3 and unix_type not in (0, stat.S_IFREG)) ): diff --git a/tests/experiment/zip-mutation-cases.py b/tests/experiment/zip-mutation-cases.py index 00ebcc1..9ffc23a 100644 --- a/tests/experiment/zip-mutation-cases.py +++ b/tests/experiment/zip-mutation-cases.py @@ -249,7 +249,7 @@ def main() -> int: "ZIP-TYPE", ( " if (\n" - " create_system not in (0, 3)\n" + " create_system not in (0, 3, 10, 14)\n" " or dos_attributes & 0x18\n" " or (create_system == 3 and unix_type not in (0, stat.S_IFREG))\n" " ):\n" @@ -371,7 +371,10 @@ def main() -> int: results.append( ( "M-ZIP-EXTRACT-001", - marker.is_file() and (extraction / "experiment.cue").is_file(), + marker.is_file() + and (extraction / "experiment.cue").is_file() + and sha256(production.read_bytes()).hexdigest() + == sha256(original.encode("utf-8")).hexdigest(), "caller-destination extraction mutation is observable", ) ) @@ -409,7 +412,10 @@ def main() -> int: results.append( ( "M-ZIP-IDENTITY-001", - marker.is_file() and directory.digest != zipped.digest, + marker.is_file() + and directory.digest != zipped.digest + and sha256(production.read_bytes()).hexdigest() + == sha256(original.encode("utf-8")).hexdigest(), "archive-identity mutation breaks the cross-transport oracle", ) ) From 6a25f72f72e2ce7817671ede3b5c72951fb66391 Mon Sep 17 00:00:00 2001 From: xormania <127287135+xormania@users.noreply.github.com> Date: Sun, 2 Aug 2026 06:50:07 -0400 Subject: [PATCH 110/158] test(experiment): verify source adapter aggregation --- tests/experiment/aggregate-harness-cases.sh | 226 +++++++++++++++++++- tests/experiment/contract-cases.sh | 6 +- 2 files changed, 228 insertions(+), 4 deletions(-) diff --git a/tests/experiment/aggregate-harness-cases.sh b/tests/experiment/aggregate-harness-cases.sh index 1fecc50..0417cd7 100755 --- a/tests/experiment/aggregate-harness-cases.sh +++ b/tests/experiment/aggregate-harness-cases.sh @@ -3,7 +3,8 @@ set -u -o pipefail repo_root="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." >/dev/null 2>&1 && pwd)" lifecycle="$repo_root/tests/experiment/local-lifecycle-cases.sh" -expected_count=13 +source_adapters="$repo_root/tests/experiment/source-adapter-cases.sh" +expected_count=23 work="" cleanup_work() { @@ -24,9 +25,11 @@ fi trap 'cleanup_work >/dev/null 2>&1 || true' EXIT replica="$work/repo" replica_lifecycle="$replica/tests/experiment/local-lifecycle-cases.sh" +replica_source_adapters="$replica/tests/experiment/source-adapter-cases.sh" mkdir -p "$replica/tests/experiment" "$replica/tests/install" cp "$lifecycle" "$replica_lifecycle" -chmod +x "$replica_lifecycle" +cp "$source_adapters" "$replica_source_adapters" +chmod +x "$replica_lifecycle" "$replica_source_adapters" failures=0 pass() { printf 'PASS %s %s\n' "$1" "$2"; } @@ -70,6 +73,20 @@ install_ids=("${expected_ids[@]:86:13}") state_ids=("${expected_ids[@]:99:16}") integrity_ids=("${expected_ids[@]:115:7}") mutation_ids=("${expected_ids[@]:122:11}") +source_zip_ids=( + ZIP-001 ZIP-COMPAT-001 ZIP-USAGE-001 ZIP-PATH-001 ZIP-COUNT-001 ZIP-TYPE-001 + ZIP-META-001 ZIP-FLAG-001 ZIP-METHOD-001 ZIP-ZIP64-001 ZIP-HEADER-001 ZIP-CRC-001 + ZIP-LENGTH-001 ZIP-SIZE-001 ZIP-BOMB-001 ZIP-TRUNC-001 ZIP-TRAIL-001 ZIP-SIZE-002 + ZIP-READ-001 ZIP-READ-002 ZIP-DECODE-001 ZIP-DECODE-003 ZIP-DECODE-002 + ZIP-TIMEOUT-001 ZIP-OUTPUT-001 ZIP-NOEF-001 ZIP-AUTH-001 ZIP-INSTALL-001 + ZIP-RETRY-001 ZIP-DENY-001 ZIP-PLAT-001 ZIP-NOEF-002 ZIP-RUNTIME-001 +) +source_mutation_ids=( + M-ZIP-COUNT-001 M-ZIP-NAME-001 M-ZIP-TYPE-001 M-ZIP-FLAG-001 + M-ZIP-METHOD-001 M-ZIP-SIZE-001 M-ZIP-BOMB-001 M-ZIP-CRC-001 + M-ZIP-HEADER-001 M-ZIP-EXTRACT-001 M-ZIP-IDENTITY-001 M-ZIP-AUTH-001 +) +source_expected_ids=("${source_zip_ids[@]}" "${source_mutation_ids[@]}") write_fixture() { local path="$1" @@ -223,6 +240,15 @@ reset_fixtures() { write_python_fixture "$replica/tests/experiment/install-mutation-cases.py" 0 "${mutation_records[@]}" } +reset_source_fixtures() { + local zip_records=() mutation_records=() + mapfile -t zip_records < <(pass_records "${source_zip_ids[@]}") + mapfile -t mutation_records < <(pass_records "${source_mutation_ids[@]}") + write_fixture "$replica/tests/experiment/zip-intake-cases.sh" 0 "${zip_records[@]}" + write_python_fixture \ + "$replica/tests/experiment/zip-mutation-cases.py" 0 "${mutation_records[@]}" +} + run_replica() { local output="$1" shift @@ -237,6 +263,12 @@ run_selected() { return $? } +run_source_replica() { + local output="$1" + shift + run_selected "$output" "$replica_source_adapters" "$@" +} + wait_for_path() { local path="$1" local attempts=0 @@ -639,6 +671,196 @@ else fail AGG-013 "signal exit preserves work owned by an uncooperative descendant" fi +reset_source_fixtures +source_expected_executions="$work/source-expected-executions" +source_baseline_executions="$work/source-baseline-executions" +source_mutant_executions="$work/source-mutant-executions" +printf '%s\n' zip-intake-cases.sh zip-mutation-cases.py > "$source_expected_executions" +: > "$source_baseline_executions" +source_baseline_rc=0 +run_source_replica "$work/source-baseline.out" env \ + AGENT_LAB_AGG_EXEC_LOG="$source_baseline_executions" || source_baseline_rc=$? + +mutant_source_adapters="$replica/tests/experiment/source-adapter-hidden-duplicate.sh" +awk ' + { print } + $0 == "subcases=(" { in_subcases=1; next } + in_subcases && $0 == ")" { + print "\"$repo_root/tests/experiment/zip-intake-cases.sh\" >/dev/null 2>&1" + in_subcases=0 + } +' "$replica_source_adapters" > "$mutant_source_adapters" +chmod +x "$mutant_source_adapters" +source_mutation_count="$(grep -Fxc \ + '"$repo_root/tests/experiment/zip-intake-cases.sh" >/dev/null 2>&1' \ + "$mutant_source_adapters")" +: > "$source_mutant_executions" +source_mutant_rc=0 +run_selected "$work/source-mutant.out" "$mutant_source_adapters" env \ + AGENT_LAB_AGG_EXEC_LOG="$source_mutant_executions" || source_mutant_rc=$? +source_mutant_expected="$work/source-mutant-expected-executions" +printf '%s\n' zip-intake-cases.sh zip-intake-cases.sh zip-mutation-cases.py \ + > "$source_mutant_expected" +if [ "$source_baseline_rc" -eq 0 ] && + cmp -s "$source_expected_executions" "$source_baseline_executions" && + [ "$source_mutation_count" -eq 1 ] && [ "$source_mutant_rc" -eq 0 ] && + cmp -s "$work/source-baseline.out" "$work/source-mutant.out" && + cmp -s "$source_mutant_expected" "$source_mutant_executions"; then + pass AGG-021 "source-adapter execution ledger proves ordered exact-once routing" +else + fail AGG-021 "source-adapter execution ledger proves ordered exact-once routing" +fi + +source_success_expected="$work/source-success-expected" +{ + for id in "${source_expected_ids[@]}"; do + printf 'PASS %s fixture assertion\n' "$id" + done + printf 'SUMMARY assertions=45 expected=45 failures=0 infra=0\n' + printf 'EXPERIMENT SOURCE ADAPTERS PASS\n' +} > "$source_success_expected" +if [ "$source_baseline_rc" -eq 0 ] && + cmp -s "$source_success_expected" "$work/source-baseline.out"; then + pass AGG-022 "source-adapter success forwards exact assertions, summary, and final marker" +else + fail AGG-022 "source-adapter success forwards exact assertions, summary, and final marker" +fi + +reset_source_fixtures +source_missing_records=() +mapfile -t source_missing_records < <(pass_records \ + "${source_zip_ids[@]:0:${#source_zip_ids[@]}-1}") +write_fixture "$replica/tests/experiment/zip-intake-cases.sh" 0 \ + "${source_missing_records[@]}" +source_missing_rc=0 +run_source_replica "$work/source-missing.out" env || source_missing_rc=$? +if [ "$source_missing_rc" -eq 1 ] && + grep -Fxq 'SUMMARY assertions=44 expected=45 failures=1 infra=0' \ + "$work/source-missing.out" && + ! grep -Fxq 'EXPERIMENT SOURCE ADAPTERS PASS' "$work/source-missing.out"; then + pass AGG-023 "source-adapter missing assertion identity maps to one" +else + fail AGG-023 "source-adapter missing assertion identity maps to one" +fi + +reset_source_fixtures +source_duplicate_records=() +mapfile -t source_duplicate_records < <(pass_records \ + "${source_zip_ids[@]}" "${source_zip_ids[-1]}") +write_fixture "$replica/tests/experiment/zip-intake-cases.sh" 0 \ + "${source_duplicate_records[@]}" +source_duplicate_rc=0 +run_source_replica "$work/source-duplicate.out" env || source_duplicate_rc=$? +if [ "$source_duplicate_rc" -eq 1 ] && + grep -Fxq 'SUMMARY assertions=46 expected=45 failures=1 infra=0' \ + "$work/source-duplicate.out" && + ! grep -Fxq 'EXPERIMENT SOURCE ADAPTERS PASS' "$work/source-duplicate.out"; then + pass AGG-024 "source-adapter duplicate assertion identity maps to one" +else + fail AGG-024 "source-adapter duplicate assertion identity maps to one" +fi + +reset_source_fixtures +source_substituted_records=() +mapfile -t source_substituted_records < <(pass_records "${source_zip_ids[@]}") +source_substituted_records[-1]='PASS:BAD-001' +write_fixture "$replica/tests/experiment/zip-intake-cases.sh" 0 \ + "${source_substituted_records[@]}" +source_substituted_rc=0 +run_source_replica "$work/source-substituted.out" env || source_substituted_rc=$? +if [ "$source_substituted_rc" -eq 1 ] && + grep -Fxq 'SUMMARY assertions=45 expected=45 failures=1 infra=0' \ + "$work/source-substituted.out" && + ! grep -Fxq 'EXPERIMENT SOURCE ADAPTERS PASS' "$work/source-substituted.out"; then + pass AGG-025 "source-adapter substituted assertion identity maps to one" +else + fail AGG-025 "source-adapter substituted assertion identity maps to one" +fi + +reset_source_fixtures +source_failed_records=() +mapfile -t source_failed_records < <(pass_records "${source_zip_ids[@]}") +source_failed_records[0]='FAIL:ZIP-001' +write_fixture "$replica/tests/experiment/zip-intake-cases.sh" 1 \ + "${source_failed_records[@]}" +source_assertion_rc=0 +run_source_replica "$work/source-assertion.out" env || source_assertion_rc=$? +if [ "$source_assertion_rc" -eq 1 ] && + grep -Fxq 'SUMMARY assertions=45 expected=45 failures=1 infra=0' \ + "$work/source-assertion.out" && + ! grep -Fxq 'EXPERIMENT SOURCE ADAPTERS PASS' "$work/source-assertion.out"; then + pass AGG-026 "source-adapter subcase assertion failure maps to one" +else + fail AGG-026 "source-adapter subcase assertion failure maps to one" +fi + +reset_source_fixtures +source_uncertain_records=() +mapfile -t source_uncertain_records < <(pass_records "${source_zip_ids[@]}") +write_fixture "$replica/tests/experiment/zip-intake-cases.sh" 125 \ + "${source_uncertain_records[@]}" +source_subcase_infra_rc=0 +run_source_replica "$work/source-subcase-infra.out" env || source_subcase_infra_rc=$? +if [ "$source_subcase_infra_rc" -eq 125 ] && + grep -Fxq 'SUMMARY assertions=45 expected=45 failures=0 infra=1' \ + "$work/source-subcase-infra.out" && + ! grep -Fxq 'EXPERIMENT SOURCE ADAPTERS PASS' "$work/source-subcase-infra.out"; then + pass AGG-027 "source-adapter subcase uncertainty maps to one hundred twenty-five" +else + fail AGG-027 "source-adapter subcase uncertainty maps to one hundred twenty-five" +fi + +reset_source_fixtures +find "$replica/tests/experiment/zip-mutation-cases.py" -delete +source_setup_infra_rc=0 +run_source_replica "$work/source-setup-infra.out" env || source_setup_infra_rc=$? +if [ "$source_setup_infra_rc" -eq 125 ] && + grep -Fxq 'SUMMARY assertions=33 expected=45 failures=1 infra=1' \ + "$work/source-setup-infra.out" && + ! grep -Fxq 'EXPERIMENT SOURCE ADAPTERS PASS' "$work/source-setup-infra.out"; then + pass AGG-028 "source-adapter setup uncertainty maps to one hundred twenty-five" +else + fail AGG-028 "source-adapter setup uncertainty maps to one hundred twenty-five" +fi + +reset_source_fixtures +source_shim="$work/source-shim" +mkdir "$source_shim" +printf '#!/usr/bin/env bash\nexit 1\n' > "$source_shim/rmdir" +chmod +x "$source_shim/rmdir" +source_cleanup_rc=0 +run_source_replica "$work/source-cleanup.out" env \ + PATH="$source_shim:$PATH" || source_cleanup_rc=$? +if [ "$source_cleanup_rc" -eq 125 ] && + grep -Fxq 'SUMMARY assertions=45 expected=45 failures=0 infra=1' \ + "$work/source-cleanup.out" && + ! grep -Fxq 'EXPERIMENT SOURCE ADAPTERS PASS' "$work/source-cleanup.out"; then + pass AGG-029 "source-adapter cleanup uncertainty suppresses the final marker" +else + fail AGG-029 "source-adapter cleanup uncertainty suppresses the final marker" +fi + +reset_source_fixtures +source_summaryless="$replica/tests/experiment/zip-intake-cases.sh" +{ + printf '#!/usr/bin/env bash\nset -u\n' + for id in "${source_zip_ids[@]}"; do + printf "printf 'PASS %s fixture assertion\\n'\n" "$id" + done + printf 'exit 0\n' +} > "$source_summaryless" +chmod +x "$source_summaryless" +source_summaryless_rc=0 +run_source_replica "$work/source-summaryless.out" env || source_summaryless_rc=$? +if [ "$source_summaryless_rc" -eq 125 ] && + grep -Fxq 'SUMMARY assertions=45 expected=45 failures=0 infra=1' \ + "$work/source-summaryless.out" && + ! grep -Fxq 'EXPERIMENT SOURCE ADAPTERS PASS' "$work/source-summaryless.out"; then + pass AGG-030 "source-adapter missing subcase summary maps to one hundred twenty-five" +else + fail AGG-030 "source-adapter missing subcase summary maps to one hundred twenty-five" +fi + cleanup_infrastructure=0 if ! cleanup_work; then cleanup_infrastructure=1 diff --git a/tests/experiment/contract-cases.sh b/tests/experiment/contract-cases.sh index 2d5211e..4e47f54 100755 --- a/tests/experiment/contract-cases.sh +++ b/tests/experiment/contract-cases.sh @@ -7,7 +7,7 @@ subcases=( "$repo_root/tests/experiment/aggregate-harness-cases.sh" "$repo_root/tests/experiment/catalog-aggregate-harness-cases.sh" ) -expected_count=33 +expected_count=43 work="" cleanup_work() { @@ -33,7 +33,9 @@ printf '%s\n' \ FMT-001 FMT-002 FMT-004 FMT-005 FMT-006 FMT-007 FMT-003 FMT-008 \ SEL-001 CUE-001 FMT-009 M-FMT-001 SEL-002 \ AGG-001 AGG-002 AGG-003 AGG-004 AGG-005 AGG-006 AGG-007 AGG-008 AGG-009 \ - AGG-010 AGG-011 AGG-012 AGG-013 AGG-014 AGG-015 AGG-016 AGG-017 \ + AGG-010 AGG-011 AGG-012 AGG-013 AGG-021 AGG-022 AGG-023 AGG-024 \ + AGG-025 AGG-026 AGG-027 AGG-028 AGG-029 AGG-030 \ + AGG-014 AGG-015 AGG-016 AGG-017 \ AGG-018 AGG-019 AGG-020 > "$expected" : > "$observed" From 8ec2e0f378ac76a4ec75030eba0d3c23a3512b9a Mon Sep 17 00:00:00 2001 From: xormania <127287135+xormania@users.noreply.github.com> Date: Sun, 2 Aug 2026 06:50:22 -0400 Subject: [PATCH 111/158] test(experiment): strengthen zip mutation proof --- tests/experiment/zip-fixtures.py | 11 ++ tests/experiment/zip-intake-cases.sh | 6 + tests/experiment/zip-mutation-cases.py | 183 ++++++++++++++++++++++--- 3 files changed, 179 insertions(+), 21 deletions(-) diff --git a/tests/experiment/zip-fixtures.py b/tests/experiment/zip-fixtures.py index b0646ac..0434dfa 100644 --- a/tests/experiment/zip-fixtures.py +++ b/tests/experiment/zip-fixtures.py @@ -340,6 +340,17 @@ def main() -> int: u32(data, central + 38, 0x20), ), ) + fixtures["vfat-archive-file.zip"] = mutate( + stored, + lambda data, _local, central, _eocd: ( + u16( + data, + central + 4, + (14 << 8) | (struct.unpack_from(" subprocess.CompletedProcess[bytes]: - return subprocess.run( - command, - env=environment, +def bomb_mutation( + production: Path, + original: str, + private_source: Path, + baseline_module, + fixture: Path, + marker: Path, +) -> bool: + if not rejected_with(baseline_module, fixture, "ZIP-BOMB"): + return False + needle = " decoded = decoder.decompress(chunk, remaining)\n" + replacement = ( + " mutation_marker = os.environ.get(\"AGENT_LAB_ZIP_MUTATION_MARK\")\n" + " if mutation_marker is not None:\n" + " Path(mutation_marker).touch()\n" + " decoded = decoder.decompress(chunk)\n" + ) + private_source.write_text(replace_once(original, needle, replacement), encoding="utf-8") + marker.unlink(missing_ok=True) + os.environ["AGENT_LAB_ZIP_MUTATION_MARK"] = str(marker) + produced_sizes: list[int] = [] + rejected = False + try: + mutant = load_module(private_source, "zip_mutant_bomb") + original_decompressobj = mutant.zlib.decompressobj + + class RecordingDecoder: + def __init__(self): + self.delegate = original_decompressobj(-15) + + @property + def eof(self): + return self.delegate.eof + + @property + def unconsumed_tail(self): + return self.delegate.unconsumed_tail + + @property + def unused_data(self): + return self.delegate.unused_data + + def decompress(self, data, *args): + decoded = self.delegate.decompress(data, *args) + produced_sizes.append(len(decoded)) + return decoded + + def flush(self, size): + return self.delegate.flush(size) + + mutant.zlib.decompressobj = lambda _window: RecordingDecoder() + try: + mutant.read_zip_snapshot(str(fixture)) + except mutant.InvalidManifest as error: + rejected = "ZIP-BOMB" in str(error) + finally: + os.environ.pop("AGENT_LAB_ZIP_MUTATION_MARK", None) + return ( + marker.is_file() + and rejected + and max(produced_sizes, default=0) > baseline_module.MAX_SOURCE_BYTES + and sha256(production.read_bytes()).hexdigest() + == sha256(original.encode("utf-8")).hexdigest() + ) + + +def run_command( + repo: Path, + root: Path, + label: str, + command: list[str], + environment: dict[str, str], +) -> subprocess.CompletedProcess[bytes]: + stdout = root / f"{label}.out" + stderr = root / f"{label}.err" + status = root / f"{label}.status" + bounded = subprocess.run( + [ + sys.executable, + "-I", + "-B", + str(repo / "tests/helpers/run-bounded.py"), + "--timeout", + "5", + "--status", + str(status), + "--stdout", + str(stdout), + "--stderr", + str(stderr), + "--", + "/usr/bin/env", + "-i", + *(f"{name}={value}" for name, value in sorted(environment.items())), + *command, + ], stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=False, - timeout=5, ) + try: + status_line = status.read_text(encoding="ascii") + captured_stdout = stdout.read_bytes() + captured_stderr = stderr.read_bytes() + except OSError: + status_line = "" + captured_stdout = b"" + captured_stderr = b"" + expected_status = f"child:{bounded.returncode}\n" + returncode = bounded.returncode + if bounded.stdout or bounded.stderr or status_line != expected_status: + returncode = 125 + return subprocess.CompletedProcess(command, returncode, captured_stdout, captured_stderr) def authorization_mutation(repo: Path, root: Path, archive: Path, marker: Path) -> bool: runtime = root / "deny-runtime" manifest = repo / "packaging/agent-lab-local.manifest" - for raw in manifest.read_text(encoding="utf-8").splitlines(): + expected_runtime = repo / "tests/install/fixtures/expected-runtime-files.txt" + if manifest.read_bytes() != expected_runtime.read_bytes(): + raise RuntimeError("runtime manifest differs from the independent expected list") + for raw in expected_runtime.read_text(encoding="utf-8").splitlines(): if not raw: raise RuntimeError("runtime manifest contains an empty path") source = repo / raw @@ -130,9 +237,16 @@ def authorization_mutation(repo: Path, root: Path, archive: Path, marker: Path) Path(environment["TMPDIR"]).mkdir() baseline_home = root / "baseline-home" baseline_init = run_command( - [str(entrypoint), "--home", str(baseline_home), "init"], environment + repo, + root, + "baseline-init", + [str(entrypoint), "--home", str(baseline_home), "init"], + environment, ) baseline = run_command( + repo, + root, + "baseline-install", [ str(entrypoint), "--home", @@ -164,9 +278,16 @@ def authorization_mutation(repo: Path, root: Path, archive: Path, marker: Path) environment["AGENT_LAB_ZIP_MUTATION_MARK"] = str(marker) mutant_home = root / "mutant-home" mutant_init = run_command( - [str(entrypoint), "--home", str(mutant_home), "init"], environment + repo, + root, + "mutant-init", + [str(entrypoint), "--home", str(mutant_home), "init"], + environment, ) mutant = run_command( + repo, + root, + "mutant-install", [ str(entrypoint), "--home", @@ -199,6 +320,24 @@ def main() -> int: work_path = tempfile.mkdtemp(prefix="agent-lab-zip-mutations-") work = Path(work_path) try: + bounded_self_test = subprocess.run( + [ + sys.executable, + "-I", + "-B", + str(repo / "tests/helpers/run-bounded.py"), + "--self-test", + ], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + ) + if ( + bounded_self_test.returncode != 0 + or bounded_self_test.stdout + or bounded_self_test.stderr + ): + raise RuntimeError("bounded command helper self-test failed") fixtures = work / "fixtures" generated = subprocess.run( [ @@ -293,19 +432,6 @@ def main() -> int: " if False:\n" ), ), - ( - "M-ZIP-BOMB-001", - "deflate-bomb.zip", - "ZIP-SIZE", - "MAX_SOURCE_BYTES = MAX_MANIFEST_BYTES\n", - ( - "MAX_SOURCE_BYTES = (\n" - " (Path(os.environ[\"AGENT_LAB_ZIP_MUTATION_MARK\"]).touch() or 300_000)\n" - " if os.environ.get(\"AGENT_LAB_ZIP_MUTATION_MARK\") is not None\n" - " else MAX_MANIFEST_BYTES\n" - ")\n" - ), - ), ( "M-ZIP-CRC-001", "bad-crc.zip", @@ -343,6 +469,21 @@ def main() -> int: marker, ) results.append((assertion, result, "private parser mutation is killed")) + if assertion == "M-ZIP-SIZE-001": + results.append( + ( + "M-ZIP-BOMB-001", + bomb_mutation( + production, + original, + private_source, + baseline_module, + fixtures / "declared-small-large.zip", + marker, + ), + "unbounded decoder mutation produces over-bound output", + ) + ) extraction = work / "caller-destination" extraction.mkdir() From 91fc871c0732ee48ddaae900570436fd6926f0d2 Mon Sep 17 00:00:00 2001 From: xormania <127287135+xormania@users.noreply.github.com> Date: Sun, 2 Aug 2026 06:50:50 -0400 Subject: [PATCH 112/158] test(experiment): reject Windows zip special types --- tests/experiment/zip-fixtures.py | 22 ++++++++++++++++++++++ tests/experiment/zip-intake-cases.sh | 3 ++- 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/tests/experiment/zip-fixtures.py b/tests/experiment/zip-fixtures.py index 0434dfa..a21867b 100644 --- a/tests/experiment/zip-fixtures.py +++ b/tests/experiment/zip-fixtures.py @@ -340,6 +340,28 @@ def main() -> int: u32(data, central + 38, 0x20), ), ) + fixtures["ntfs-device.zip"] = mutate( + stored, + lambda data, _local, central, _eocd: ( + u16( + data, + central + 4, + (10 << 8) | (struct.unpack_from(" Date: Sun, 2 Aug 2026 06:51:18 -0400 Subject: [PATCH 113/158] fix(experiment): reject Windows zip special types --- scripts/experiment.py | 2 +- tests/experiment/zip-mutation-cases.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/experiment.py b/scripts/experiment.py index 4cd387a..bd7057a 100644 --- a/scripts/experiment.py +++ b/scripts/experiment.py @@ -501,7 +501,7 @@ def read_zip_snapshot(path: str) -> SourceSnapshot: dos_attributes = external_attributes & 0xFFFF if ( create_system not in (0, 3, 10, 14) - or dos_attributes & 0x18 + or dos_attributes & 0x458 or (create_system == 3 and unix_type not in (0, stat.S_IFREG)) ): _zip_reject("ZIP-TYPE", "member is not a regular file") diff --git a/tests/experiment/zip-mutation-cases.py b/tests/experiment/zip-mutation-cases.py index 0dfa9d1..164fdff 100644 --- a/tests/experiment/zip-mutation-cases.py +++ b/tests/experiment/zip-mutation-cases.py @@ -389,7 +389,7 @@ def main() -> int: ( " if (\n" " create_system not in (0, 3, 10, 14)\n" - " or dos_attributes & 0x18\n" + " or dos_attributes & 0x458\n" " or (create_system == 3 and unix_type not in (0, stat.S_IFREG))\n" " ):\n" ), From 62e654b6d49ba6085914e973a02dbb0cadb12deb Mon Sep 17 00:00:00 2001 From: xormania <127287135+xormania@users.noreply.github.com> Date: Sun, 2 Aug 2026 06:56:09 -0400 Subject: [PATCH 114/158] test(experiment): isolate zip decoder mutation --- tests/experiment/zip-mutation-cases.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/experiment/zip-mutation-cases.py b/tests/experiment/zip-mutation-cases.py index 164fdff..c84ded0 100644 --- a/tests/experiment/zip-mutation-cases.py +++ b/tests/experiment/zip-mutation-cases.py @@ -111,9 +111,9 @@ def bomb_mutation( os.environ["AGENT_LAB_ZIP_MUTATION_MARK"] = str(marker) produced_sizes: list[int] = [] rejected = False + original_decompressobj = baseline_module.zlib.decompressobj try: mutant = load_module(private_source, "zip_mutant_bomb") - original_decompressobj = mutant.zlib.decompressobj class RecordingDecoder: def __init__(self): @@ -145,11 +145,13 @@ def flush(self, size): except mutant.InvalidManifest as error: rejected = "ZIP-BOMB" in str(error) finally: + baseline_module.zlib.decompressobj = original_decompressobj os.environ.pop("AGENT_LAB_ZIP_MUTATION_MARK", None) return ( marker.is_file() and rejected and max(produced_sizes, default=0) > baseline_module.MAX_SOURCE_BYTES + and baseline_module.zlib.decompressobj is original_decompressobj and sha256(production.read_bytes()).hexdigest() == sha256(original.encode("utf-8")).hexdigest() ) From 6884eda0abdb05b3b8fa065cca71160714327a9f Mon Sep 17 00:00:00 2001 From: xormania <127287135+xormania@users.noreply.github.com> Date: Sun, 2 Aug 2026 06:56:26 -0400 Subject: [PATCH 115/158] test(experiment): reject reserved watchdog status --- tests/experiment/zip-mutation-cases.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/tests/experiment/zip-mutation-cases.py b/tests/experiment/zip-mutation-cases.py index c84ded0..9564d14 100644 --- a/tests/experiment/zip-mutation-cases.py +++ b/tests/experiment/zip-mutation-cases.py @@ -340,6 +340,20 @@ def main() -> int: or bounded_self_test.stderr ): raise RuntimeError("bounded command helper self-test failed") + wrapper_self_test = work / "wrapper-self-test" + wrapper_self_test.mkdir() + try: + run_command( + repo, + wrapper_self_test, + "reserved-status", + [sys.executable, "-I", "-B", "-c", "raise SystemExit(125)"], + {"PATH": "/usr/bin:/bin", "LC_ALL": "C"}, + ) + except RuntimeError: + pass + else: + raise RuntimeError("bounded command wrapper accepted reserved status 125") fixtures = work / "fixtures" generated = subprocess.run( [ From 32813982bcaf6c0ae3e4d87ca8cd4be0b8806611 Mon Sep 17 00:00:00 2001 From: xormania <127287135+xormania@users.noreply.github.com> Date: Sun, 2 Aug 2026 06:56:44 -0400 Subject: [PATCH 116/158] test(experiment): preserve watchdog infrastructure status --- tests/experiment/zip-mutation-cases.py | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/tests/experiment/zip-mutation-cases.py b/tests/experiment/zip-mutation-cases.py index 9564d14..e9a6402 100644 --- a/tests/experiment/zip-mutation-cases.py +++ b/tests/experiment/zip-mutation-cases.py @@ -195,15 +195,19 @@ def run_command( status_line = status.read_text(encoding="ascii") captured_stdout = stdout.read_bytes() captured_stderr = stderr.read_bytes() - except OSError: - status_line = "" - captured_stdout = b"" - captured_stderr = b"" + except OSError as error: + raise RuntimeError("bounded command result is unavailable") from error expected_status = f"child:{bounded.returncode}\n" - returncode = bounded.returncode - if bounded.stdout or bounded.stderr or status_line != expected_status: - returncode = 125 - return subprocess.CompletedProcess(command, returncode, captured_stdout, captured_stderr) + if ( + bounded.stdout + or bounded.stderr + or status_line != expected_status + or bounded.returncode == 125 + ): + raise RuntimeError("bounded command infrastructure failure") + return subprocess.CompletedProcess( + command, bounded.returncode, captured_stdout, captured_stderr + ) def authorization_mutation(repo: Path, root: Path, archive: Path, marker: Path) -> bool: From 5e82e6f2f10f30cf0637372efef005fc54558658 Mon Sep 17 00:00:00 2001 From: xormania <127287135+xormania@users.noreply.github.com> Date: Sun, 2 Aug 2026 07:14:51 -0400 Subject: [PATCH 117/158] test(image): bound crash matrix orchestration --- tests/image/catalog-state-cases.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/image/catalog-state-cases.py b/tests/image/catalog-state-cases.py index a26bd77..0e53335 100755 --- a/tests/image/catalog-state-cases.py +++ b/tests/image/catalog-state-cases.py @@ -63,6 +63,7 @@ def load_catalog_module(): CATALOG = load_catalog_module() FAILURES = 0 OBSERVED: list[str] = [] +CLI_CALLS = 0 def check(assertion: str, condition: bool, message: str, detail: str = "") -> None: @@ -77,6 +78,8 @@ def check(assertion: str, condition: bool, message: str, detail: str = "") -> No def cli(home: Path, *arguments: str, timeout: float = 5.0) -> subprocess.CompletedProcess[bytes]: + global CLI_CALLS + CLI_CALLS += 1 environment = { "PATH": "/usr/bin:/bin", "LANG": "C", @@ -1048,6 +1051,7 @@ def fail_pointer_replace(source: os.PathLike[str] | str, target: os.PathLike[str "catalog staging cleanup.after_fsync", ) matrix_failures: list[str] = [] + matrix_cli_start = CLI_CALLS for index, point in enumerate(bootstrap_points): home = new_home(root, f"bootstrap-crash-{index:02d}") child_rc = hard_exit_add(home, "vendor.worker", SUBJECT, point) @@ -1112,6 +1116,12 @@ def fail_pointer_replace(source: os.PathLike[str] | str, target: os.PathLike[str f"later:{point}:child={child_rc}:before={before_retry.returncode}:" f"retry={retry.returncode}:final={final.returncode}" ) + matrix_cli_calls = CLI_CALLS - matrix_cli_start + expected_matrix_cli_calls = 2 * (len(bootstrap_points) + len(later_points)) + if matrix_cli_calls != expected_matrix_cli_calls: + matrix_failures.append( + f"cli-calls:{matrix_cli_calls}:expected={expected_matrix_cli_calls}" + ) check( "CAT-CRASH-006", not matrix_failures, From 98aa82608dace84d3110d9d0fd433b7e02e30825 Mon Sep 17 00:00:00 2001 From: xormania <127287135+xormania@users.noreply.github.com> Date: Sun, 2 Aug 2026 07:19:37 -0400 Subject: [PATCH 118/158] test(image): reduce crash matrix contention --- tests/image/catalog-state-cases.py | 183 ++++++++++++++++++++++++++--- 1 file changed, 165 insertions(+), 18 deletions(-) diff --git a/tests/image/catalog-state-cases.py b/tests/image/catalog-state-cases.py index 0e53335..93cc595 100755 --- a/tests/image/catalog-state-cases.py +++ b/tests/image/catalog-state-cases.py @@ -136,6 +136,39 @@ def add(home: Path, name: str = "vendor.worker", subject: str = SUBJECT) -> dict return value +def matrix_home(root: Path, name: str) -> tuple[Path, str | None]: + home = root / name + output = io.StringIO() + errors = io.StringIO() + error: BaseException | None = None + returncode: int | None = None + try: + with redirect_stdout(output), redirect_stderr(errors): + returncode = MODULE.main(["--home", str(home), "init"]) + except BaseException as caught: # Setup faults must remain matrix failures. + error = caught + if returncode != 0 or error is not None or errors.getvalue(): + return home, ( + f"init={returncode}:error={type(error).__name__ if error else 'none'}:" + f"stderr={errors.getvalue()!r}" + ) + return home, None + + +def matrix_add(home: Path, name: str, subject: str) -> str | None: + returncode, output, errors, error = module_image(home, "add", name, subject) + try: + value = json.loads(output) if returncode == 0 and error is None else None + except json.JSONDecodeError: + value = None + if errors or not isinstance(value, dict) or value.get("changed") is not True: + return ( + f"add={returncode}:error={type(error).__name__ if error else 'none'}:" + f"stderr={errors!r}:value={value!r}" + ) + return None + + def current_snapshot(home: Path) -> tuple[Path, dict[str, object], str]: root = home / "images" / "catalog" pointer = json.loads((root / "current.json").read_bytes()) @@ -145,6 +178,111 @@ def current_snapshot(home: Path) -> tuple[Path, dict[str, object], str]: return path, value, snapshot_digest +def stored_active_names(home: Path) -> list[str] | None: + try: + root = home / "images" / "catalog" + pointer_raw = (root / "current.json").read_bytes() + pointer = json.loads(pointer_raw) + if ( + not isinstance(pointer, dict) + or set(pointer) != {"apiVersion", "snapshotDigest"} + or pointer.get("apiVersion") != "agent-lab.local-image-current/v0alpha1" + or pointer_raw != canonical(pointer) + b"\n" + ): + return None + snapshot_digest = pointer.get("snapshotDigest") + if ( + not isinstance(snapshot_digest, str) + or not snapshot_digest.startswith("sha256:") + or len(snapshot_digest) != 71 + or any(character not in "0123456789abcdef" for character in snapshot_digest[7:]) + ): + return None + snapshot_raw = (root / "snapshots" / f"{snapshot_digest[7:]}.json").read_bytes() + snapshot = json.loads(snapshot_raw) + if ( + not isinstance(snapshot, dict) + or set(snapshot) + != {"apiVersion", "previousSnapshotDigest", "records", "revision"} + or snapshot.get("apiVersion") != "agent-lab.local-image-snapshot/v0alpha1" + or snapshot_raw != canonical(snapshot) + b"\n" + or digest(SNAPSHOT_DOMAIN, snapshot) != snapshot_digest + or not isinstance(snapshot.get("revision"), int) + or isinstance(snapshot.get("revision"), bool) + or int(snapshot["revision"]) < 1 + ): + return None + records = snapshot.get("records") + if not isinstance(records, dict): + return None + active: list[str] = [] + for name, projection in records.items(): + if ( + not isinstance(name, str) + or not isinstance(projection, dict) + or set(projection) != {"entryDigest", "generation", "state"} + ): + return None + entry_digest = projection.get("entryDigest") + if ( + not isinstance(entry_digest, str) + or not entry_digest.startswith("sha256:") + or len(entry_digest) != 71 + or any(character not in "0123456789abcdef" for character in entry_digest[7:]) + ): + return None + entry_raw = (root / "entries" / f"{entry_digest[7:]}.json").read_bytes() + entry = json.loads(entry_raw) + if ( + not isinstance(entry, dict) + or set(entry) + != { + "apiVersion", + "generation", + "name", + "previousEntryDigest", + "state", + "subject", + "subjectDigest", + } + or entry.get("apiVersion") != "agent-lab.local-image-entry/v0alpha1" + or entry_raw != canonical(entry) + b"\n" + or digest(ENTRY_DOMAIN, entry) != entry_digest + or entry.get("name") != name + or entry.get("generation") != projection.get("generation") + or entry.get("state") != projection.get("state") + or not isinstance(entry.get("generation"), int) + or isinstance(entry.get("generation"), bool) + or int(entry["generation"]) < 1 + or entry.get("state") not in ("active", "removed") + or not isinstance(entry.get("subject"), str) + or entry.get("subjectDigest") != str(entry["subject"]).rsplit("@", 1)[-1] + ): + return None + if entry.get("state") == "active": + active.append(name) + return sorted(active, key=lambda item: item.encode("ascii")) + except (OSError, UnicodeError, ValueError, json.JSONDecodeError): + return None + + +def stored_oracle_sensitivity(home: Path, expected: list[str]) -> bool: + try: + snapshot_path, snapshot, _snapshot_digest = current_snapshot(home) + original = snapshot_path.read_bytes() + mutated = dict(snapshot) + revision = mutated.get("revision") + if not isinstance(revision, int) or isinstance(revision, bool): + return False + mutated["revision"] = revision + 1 + snapshot_path.write_bytes(canonical(mutated) + b"\n") + rejected = stored_active_names(home) is None + snapshot_path.write_bytes(original) + return rejected and stored_active_names(home) == expected + except (OSError, UnicodeError, ValueError, json.JSONDecodeError): + return False + + def fingerprint(root: Path) -> tuple[tuple[str, str, int, int, str], ...]: if not root.exists() and not root.is_symlink(): return () @@ -1051,19 +1189,24 @@ def fail_pointer_replace(source: os.PathLike[str] | str, target: os.PathLike[str "catalog staging cleanup.after_fsync", ) matrix_failures: list[str] = [] + matrix_oracle_home: Path | None = None matrix_cli_start = CLI_CALLS for index, point in enumerate(bootstrap_points): - home = new_home(root, f"bootstrap-crash-{index:02d}") + home, setup_error = matrix_home(root, f"bootstrap-crash-{index:02d}") + if setup_error is not None: + matrix_failures.append(f"bootstrap:{point}:setup:{setup_error}") + continue child_rc = hard_exit_add(home, "vendor.worker", SUBJECT, point) before_retry = cli(home, "image", "list") retry = cli(home, "image", "add", "vendor.worker", SUBJECT) - final = cli(home, "image", "list") try: before_records = json.loads(before_retry.stdout) if before_retry.returncode == 0 else None retry_value = json.loads(retry.stdout) if retry.returncode == 0 else None - final_records = json.loads(final.stdout) if final.returncode == 0 else None except json.JSONDecodeError: - before_records = retry_value = final_records = None + before_records = retry_value = None + final_names = stored_active_names(home) + if final_names == ["vendor.worker"]: + matrix_oracle_home = home if not ( child_rc == 99 and before_retry.returncode == 0 @@ -1072,33 +1215,36 @@ def fail_pointer_replace(source: os.PathLike[str] | str, target: os.PathLike[str and retry.returncode == 0 and isinstance(retry_value, dict) and retry_value.get("changed") in (True, False) - and final.returncode == 0 - and isinstance(final_records, list) - and [record.get("name") for record in final_records] == ["vendor.worker"] + and final_names == ["vendor.worker"] and not tuple((home / "images" / ".staging").iterdir()) ): matrix_failures.append( f"bootstrap:{point}:child={child_rc}:before={before_retry.returncode}:" - f"retry={retry.returncode}:final={final.returncode}" + f"retry={retry.returncode}:final={final_names!r}" ) for index, point in enumerate(later_points): - home = new_home(root, f"later-crash-{index:02d}") - add(home) + home, setup_error = matrix_home(root, f"later-crash-{index:02d}") + if setup_error is None: + setup_error = matrix_add(home, "vendor.worker", SUBJECT) + if setup_error is not None: + matrix_failures.append(f"later:{point}:setup:{setup_error}") + continue child_rc = hard_exit_add(home, "vendor.second", OTHER_SUBJECT, point) before_retry = cli(home, "image", "list") retry = cli(home, "image", "add", "vendor.second", OTHER_SUBJECT) - final = cli(home, "image", "list") try: before_records = json.loads(before_retry.stdout) if before_retry.returncode == 0 else None retry_value = json.loads(retry.stdout) if retry.returncode == 0 else None - final_records = json.loads(final.stdout) if final.returncode == 0 else None except json.JSONDecodeError: - before_records = retry_value = final_records = None + before_records = retry_value = None before_names = ( [record.get("name") for record in before_records] if isinstance(before_records, list) else None ) + final_names = stored_active_names(home) + if final_names == ["vendor.second", "vendor.worker"]: + matrix_oracle_home = home if not ( child_rc == 99 and before_retry.returncode == 0 @@ -1106,15 +1252,12 @@ def fail_pointer_replace(source: os.PathLike[str] | str, target: os.PathLike[str and retry.returncode == 0 and isinstance(retry_value, dict) and retry_value.get("changed") in (True, False) - and final.returncode == 0 - and isinstance(final_records, list) - and [record.get("name") for record in final_records] - == ["vendor.second", "vendor.worker"] + and final_names == ["vendor.second", "vendor.worker"] and not tuple((home / "images" / ".staging").iterdir()) ): matrix_failures.append( f"later:{point}:child={child_rc}:before={before_retry.returncode}:" - f"retry={retry.returncode}:final={final.returncode}" + f"retry={retry.returncode}:final={final_names!r}" ) matrix_cli_calls = CLI_CALLS - matrix_cli_start expected_matrix_cli_calls = 2 * (len(bootstrap_points) + len(later_points)) @@ -1122,6 +1265,10 @@ def fail_pointer_replace(source: os.PathLike[str] | str, target: os.PathLike[str matrix_failures.append( f"cli-calls:{matrix_cli_calls}:expected={expected_matrix_cli_calls}" ) + if matrix_oracle_home is None or not stored_oracle_sensitivity( + matrix_oracle_home, ["vendor.second", "vendor.worker"] + ): + matrix_failures.append("stored-oracle-sensitivity") check( "CAT-CRASH-006", not matrix_failures, From 4eec0f0a63acc21279f3b726e69d35bfc9247d2d Mon Sep 17 00:00:00 2001 From: xormania <127287135+xormania@users.noreply.github.com> Date: Sun, 2 Aug 2026 07:22:31 -0400 Subject: [PATCH 119/158] test(image): reject unsafe crash oracle records --- tests/image/catalog-state-cases.py | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/tests/image/catalog-state-cases.py b/tests/image/catalog-state-cases.py index 93cc595..2208041 100755 --- a/tests/image/catalog-state-cases.py +++ b/tests/image/catalog-state-cases.py @@ -270,15 +270,23 @@ def stored_oracle_sensitivity(home: Path, expected: list[str]) -> bool: try: snapshot_path, snapshot, _snapshot_digest = current_snapshot(home) original = snapshot_path.read_bytes() + original_mode = stat.S_IMODE(snapshot_path.lstat().st_mode) + snapshot_path.chmod(0o644) + mode_rejected = stored_active_names(home) is None + snapshot_path.chmod(original_mode) mutated = dict(snapshot) revision = mutated.get("revision") if not isinstance(revision, int) or isinstance(revision, bool): return False mutated["revision"] = revision + 1 snapshot_path.write_bytes(canonical(mutated) + b"\n") - rejected = stored_active_names(home) is None + content_rejected = stored_active_names(home) is None snapshot_path.write_bytes(original) - return rejected and stored_active_names(home) == expected + return ( + mode_rejected + and content_rejected + and stored_active_names(home) == expected + ) except (OSError, UnicodeError, ValueError, json.JSONDecodeError): return False @@ -1260,7 +1268,11 @@ def fail_pointer_replace(source: os.PathLike[str] | str, target: os.PathLike[str f"retry={retry.returncode}:final={final_names!r}" ) matrix_cli_calls = CLI_CALLS - matrix_cli_start - expected_matrix_cli_calls = 2 * (len(bootstrap_points) + len(later_points)) + if len(bootstrap_points) != 44 or len(later_points) != 42: + matrix_failures.append( + f"seam-counts:{len(bootstrap_points)}/{len(later_points)}:expected=44/42" + ) + expected_matrix_cli_calls = 172 if matrix_cli_calls != expected_matrix_cli_calls: matrix_failures.append( f"cli-calls:{matrix_cli_calls}:expected={expected_matrix_cli_calls}" From 53c6f2f14802451c1e539306b67b12246e3be3cb Mon Sep 17 00:00:00 2001 From: xormania <127287135+xormania@users.noreply.github.com> Date: Sun, 2 Aug 2026 07:23:53 -0400 Subject: [PATCH 120/158] test(image): safe-read crash oracle records --- tests/image/catalog-state-cases.py | 102 ++++++++++++++++++++++++++++- 1 file changed, 99 insertions(+), 3 deletions(-) diff --git a/tests/image/catalog-state-cases.py b/tests/image/catalog-state-cases.py index 2208041..6d0f736 100755 --- a/tests/image/catalog-state-cases.py +++ b/tests/image/catalog-state-cases.py @@ -178,10 +178,102 @@ def current_snapshot(home: Path) -> tuple[Path, dict[str, object], str]: return path, value, snapshot_digest +def stable_record(path: Path, maximum: int = 65536) -> bytes | None: + descriptor = -1 + no_follow = getattr(os, "O_NOFOLLOW", None) + if no_follow is None: + return None + try: + before = path.lstat() + if ( + not stat.S_ISREG(before.st_mode) + or stat.S_IMODE(before.st_mode) != 0o600 + or before.st_nlink != 1 + or before.st_uid != os.geteuid() + or before.st_size < 0 + or before.st_size > maximum + ): + return None + descriptor = os.open( + path, + os.O_RDONLY + | no_follow + | getattr(os, "O_CLOEXEC", 0) + | getattr(os, "O_NONBLOCK", 0), + ) + opened = os.fstat(descriptor) + signature = ( + opened.st_dev, + opened.st_ino, + opened.st_mode, + opened.st_nlink, + opened.st_uid, + opened.st_size, + opened.st_mtime_ns, + opened.st_ctime_ns, + ) + if signature != ( + before.st_dev, + before.st_ino, + before.st_mode, + before.st_nlink, + before.st_uid, + before.st_size, + before.st_mtime_ns, + before.st_ctime_ns, + ): + return None + chunks: list[bytes] = [] + observed = 0 + while True: + chunk = os.read(descriptor, min(65536, maximum + 1 - observed)) + if not chunk: + break + chunks.append(chunk) + observed += len(chunk) + if observed > maximum: + return None + after_descriptor = os.fstat(descriptor) + after_path = path.lstat() + after_signature = ( + after_descriptor.st_dev, + after_descriptor.st_ino, + after_descriptor.st_mode, + after_descriptor.st_nlink, + after_descriptor.st_uid, + after_descriptor.st_size, + after_descriptor.st_mtime_ns, + after_descriptor.st_ctime_ns, + ) + path_signature = ( + after_path.st_dev, + after_path.st_ino, + after_path.st_mode, + after_path.st_nlink, + after_path.st_uid, + after_path.st_size, + after_path.st_mtime_ns, + after_path.st_ctime_ns, + ) + if after_signature != signature or path_signature != signature or observed != opened.st_size: + return None + return b"".join(chunks) + except OSError: + return None + finally: + if descriptor >= 0: + try: + os.close(descriptor) + except OSError: + pass + + def stored_active_names(home: Path) -> list[str] | None: try: root = home / "images" / "catalog" - pointer_raw = (root / "current.json").read_bytes() + pointer_raw = stable_record(root / "current.json") + if pointer_raw is None: + return None pointer = json.loads(pointer_raw) if ( not isinstance(pointer, dict) @@ -198,7 +290,9 @@ def stored_active_names(home: Path) -> list[str] | None: or any(character not in "0123456789abcdef" for character in snapshot_digest[7:]) ): return None - snapshot_raw = (root / "snapshots" / f"{snapshot_digest[7:]}.json").read_bytes() + snapshot_raw = stable_record(root / "snapshots" / f"{snapshot_digest[7:]}.json") + if snapshot_raw is None: + return None snapshot = json.loads(snapshot_raw) if ( not isinstance(snapshot, dict) @@ -231,7 +325,9 @@ def stored_active_names(home: Path) -> list[str] | None: or any(character not in "0123456789abcdef" for character in entry_digest[7:]) ): return None - entry_raw = (root / "entries" / f"{entry_digest[7:]}.json").read_bytes() + entry_raw = stable_record(root / "entries" / f"{entry_digest[7:]}.json") + if entry_raw is None: + return None entry = json.loads(entry_raw) if ( not isinstance(entry, dict) From acbcff57ffb0de2f486ba7cbe6834b08935afe8a Mon Sep 17 00:00:00 2001 From: xormania <127287135+xormania@users.noreply.github.com> Date: Sun, 2 Aug 2026 07:24:06 -0400 Subject: [PATCH 121/158] test(image): match catalog snapshot bound --- tests/image/catalog-state-cases.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/image/catalog-state-cases.py b/tests/image/catalog-state-cases.py index 6d0f736..d36659e 100755 --- a/tests/image/catalog-state-cases.py +++ b/tests/image/catalog-state-cases.py @@ -290,7 +290,10 @@ def stored_active_names(home: Path) -> list[str] | None: or any(character not in "0123456789abcdef" for character in snapshot_digest[7:]) ): return None - snapshot_raw = stable_record(root / "snapshots" / f"{snapshot_digest[7:]}.json") + snapshot_raw = stable_record( + root / "snapshots" / f"{snapshot_digest[7:]}.json", + maximum=262144, + ) if snapshot_raw is None: return None snapshot = json.loads(snapshot_raw) From 19c3d8c309b02e7e4a774f421d894a4ed20d6434 Mon Sep 17 00:00:00 2001 From: xormania <127287135+xormania@users.noreply.github.com> Date: Sun, 2 Aug 2026 07:30:03 -0400 Subject: [PATCH 122/158] test(image): preserve matrix infrastructure status --- tests/image/catalog-state-cases.py | 33 ++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/tests/image/catalog-state-cases.py b/tests/image/catalog-state-cases.py index d36659e..f96e104 100755 --- a/tests/image/catalog-state-cases.py +++ b/tests/image/catalog-state-cases.py @@ -66,6 +66,10 @@ def load_catalog_module(): CLI_CALLS = 0 +class MatrixInfrastructure(RuntimeError): + """Crash-matrix setup could not establish trusted fixture state.""" + + def check(assertion: str, condition: bool, message: str, detail: str = "") -> None: global FAILURES OBSERVED.append(assertion) @@ -390,6 +394,31 @@ def stored_oracle_sensitivity(home: Path, expected: list[str]) -> bool: return False +def matrix_infrastructure_sensitivity(root: Path, home: Path) -> bool: + original_main = MODULE.main + init_propagated = False + try: + MODULE.main = lambda _arguments: 125 + try: + matrix_home(root, "injected-infrastructure-home") + except MatrixInfrastructure: + init_propagated = True + finally: + MODULE.main = original_main + + original_image_command = MODULE.image_command + add_propagated = False + try: + MODULE.image_command = lambda _home, _arguments: 125 + try: + matrix_add(home, "vendor.infrastructure", THIRD_SUBJECT) + except MatrixInfrastructure: + add_propagated = True + finally: + MODULE.image_command = original_image_command + return init_propagated and add_propagated + + def fingerprint(root: Path) -> tuple[tuple[str, str, int, int, str], ...]: if not root.exists() and not root.is_symlink(): return () @@ -1380,6 +1409,10 @@ def fail_pointer_replace(source: os.PathLike[str] | str, target: os.PathLike[str matrix_oracle_home, ["vendor.second", "vendor.worker"] ): matrix_failures.append("stored-oracle-sensitivity") + if matrix_oracle_home is None or not matrix_infrastructure_sensitivity( + root, matrix_oracle_home + ): + matrix_failures.append("matrix-infrastructure-sensitivity") check( "CAT-CRASH-006", not matrix_failures, From c8265e9bbf75d4d791067f34d35716f605418f11 Mon Sep 17 00:00:00 2001 From: xormania <127287135+xormania@users.noreply.github.com> Date: Sun, 2 Aug 2026 07:31:28 -0400 Subject: [PATCH 123/158] test(image): propagate matrix infrastructure status --- tests/image/catalog-state-cases.py | 28 +++++++++++----------------- 1 file changed, 11 insertions(+), 17 deletions(-) diff --git a/tests/image/catalog-state-cases.py b/tests/image/catalog-state-cases.py index f96e104..a9379bf 100755 --- a/tests/image/catalog-state-cases.py +++ b/tests/image/catalog-state-cases.py @@ -140,7 +140,7 @@ def add(home: Path, name: str = "vendor.worker", subject: str = SUBJECT) -> dict return value -def matrix_home(root: Path, name: str) -> tuple[Path, str | None]: +def matrix_home(root: Path, name: str) -> Path: home = root / name output = io.StringIO() errors = io.StringIO() @@ -149,28 +149,29 @@ def matrix_home(root: Path, name: str) -> tuple[Path, str | None]: try: with redirect_stdout(output), redirect_stderr(errors): returncode = MODULE.main(["--home", str(home), "init"]) - except BaseException as caught: # Setup faults must remain matrix failures. + except BaseException as caught: # Setup uncertainty must remain infrastructure. error = caught if returncode != 0 or error is not None or errors.getvalue(): - return home, ( + detail = ( f"init={returncode}:error={type(error).__name__ if error else 'none'}:" f"stderr={errors.getvalue()!r}" ) - return home, None + raise MatrixInfrastructure(detail) from error + return home -def matrix_add(home: Path, name: str, subject: str) -> str | None: +def matrix_add(home: Path, name: str, subject: str) -> None: returncode, output, errors, error = module_image(home, "add", name, subject) try: value = json.loads(output) if returncode == 0 and error is None else None except json.JSONDecodeError: value = None if errors or not isinstance(value, dict) or value.get("changed") is not True: - return ( + detail = ( f"add={returncode}:error={type(error).__name__ if error else 'none'}:" f"stderr={errors!r}:value={value!r}" ) - return None + raise MatrixInfrastructure(detail) from error def current_snapshot(home: Path) -> tuple[Path, dict[str, object], str]: @@ -1328,10 +1329,7 @@ def fail_pointer_replace(source: os.PathLike[str] | str, target: os.PathLike[str matrix_oracle_home: Path | None = None matrix_cli_start = CLI_CALLS for index, point in enumerate(bootstrap_points): - home, setup_error = matrix_home(root, f"bootstrap-crash-{index:02d}") - if setup_error is not None: - matrix_failures.append(f"bootstrap:{point}:setup:{setup_error}") - continue + home = matrix_home(root, f"bootstrap-crash-{index:02d}") child_rc = hard_exit_add(home, "vendor.worker", SUBJECT, point) before_retry = cli(home, "image", "list") retry = cli(home, "image", "add", "vendor.worker", SUBJECT) @@ -1359,12 +1357,8 @@ def fail_pointer_replace(source: os.PathLike[str] | str, target: os.PathLike[str f"retry={retry.returncode}:final={final_names!r}" ) for index, point in enumerate(later_points): - home, setup_error = matrix_home(root, f"later-crash-{index:02d}") - if setup_error is None: - setup_error = matrix_add(home, "vendor.worker", SUBJECT) - if setup_error is not None: - matrix_failures.append(f"later:{point}:setup:{setup_error}") - continue + home = matrix_home(root, f"later-crash-{index:02d}") + matrix_add(home, "vendor.worker", SUBJECT) child_rc = hard_exit_add(home, "vendor.second", OTHER_SUBJECT, point) before_retry = cli(home, "image", "list") retry = cli(home, "image", "add", "vendor.second", OTHER_SUBJECT) From 6da44b22a4b0cc962e2bf3edff3274df01c6cdd9 Mon Sep 17 00:00:00 2001 From: xormania <127287135+xormania@users.noreply.github.com> Date: Sun, 2 Aug 2026 07:45:59 -0400 Subject: [PATCH 124/158] test(experiment): define pinned Git CLI surface --- tests/experiment/git-intake-cases.py | 136 +++++++++++++++++++++++++++ tests/experiment/git-intake-cases.sh | 12 +++ 2 files changed, 148 insertions(+) create mode 100755 tests/experiment/git-intake-cases.py create mode 100755 tests/experiment/git-intake-cases.sh diff --git a/tests/experiment/git-intake-cases.py b/tests/experiment/git-intake-cases.py new file mode 100755 index 0000000..9121dd9 --- /dev/null +++ b/tests/experiment/git-intake-cases.py @@ -0,0 +1,136 @@ +#!/usr/bin/env python3 +"""Deterministic public-contract cases for pinned Git Experiment intake.""" + +from __future__ import annotations + +from contextlib import redirect_stderr, redirect_stdout +from importlib.util import module_from_spec, spec_from_file_location +import io +import os +from pathlib import Path +import sys +import tempfile + + +EXPECTED = ("GIT-CLI-001", "GIT-USAGE-001") +URL = "https://github.com/uscient/experiment-fixture.git" +COMMIT = "1cffa1a28f96d2f2cb898b1bad70d281e359a5b5" + + +def load_module(path: Path, label: str): + spec = spec_from_file_location(label, path) + if spec is None or spec.loader is None: + raise RuntimeError(f"cannot load {path}") + module = module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +def invoke(module, argv: list[str]) -> tuple[int, str, str]: + stdout = io.StringIO() + stderr = io.StringIO() + with redirect_stdout(stdout), redirect_stderr(stderr): + result = module.main(argv) + return result, stdout.getvalue(), stderr.getvalue() + + +def main() -> int: + repo = Path(__file__).resolve().parents[2] + failures = 0 + infrastructure = 0 + results: list[tuple[str, bool, str]] = [] + try: + agent_lab = load_module(repo / "scripts/agent-lab.py", "git_intake_agent_lab") + calls: list[tuple[str, ...]] = [] + + class RecordingExperiment: + @staticmethod + def main(argv: list[str]) -> int: + calls.append(tuple(argv)) + return 0 + + agent_lab.experiment_module = lambda: RecordingExperiment + with tempfile.TemporaryDirectory(prefix="agent-lab-git-cli-") as raw_home: + prior_home = os.environ.get("AGENT_LAB_HOME") + os.environ["AGENT_LAB_HOME"] = raw_home + try: + check = invoke( + agent_lab, + ["experiment", "check", "--git", URL, "--commit", COMMIT], + ) + authorize = invoke( + agent_lab, + [ + "experiment", + "authorize", + "install", + "--git", + URL, + "--commit", + COMMIT, + ], + ) + expected_calls = [ + ("experiment.py", "check-git", URL, COMMIT), + ("experiment.py", "authorize-git", URL, COMMIT), + ] + results.append( + ( + "GIT-CLI-001", + check == (0, "", "") + and authorize == (0, "", "") + and calls == expected_calls, + "exact pinned Git preview forms route once to the adapter", + ) + ) + + calls.clear() + malformed = ( + ["experiment", "check", "--git"], + ["experiment", "check", "--git", URL], + ["experiment", "check", "--git", URL, "--commit"], + ["experiment", "check", "--commit", COMMIT, "--git", URL], + ["experiment", "check", "--git", URL, "--commit", COMMIT, "extra"], + ["experiment", "authorize", "install", "--git", URL, "--commit"], + ) + usage_results = [invoke(agent_lab, list(argv)) for argv in malformed] + results.append( + ( + "GIT-USAGE-001", + all(result == (2, "", "") for result in usage_results) + and calls == [], + "malformed Git option shapes fail before adapter access", + ) + ) + finally: + if prior_home is None: + os.environ.pop("AGENT_LAB_HOME", None) + else: + os.environ["AGENT_LAB_HOME"] = prior_home + except Exception as error: + print(f"INFRA Git intake contract probe failed: {type(error).__name__}", file=sys.stderr) + infrastructure = 1 + + observed = tuple(item[0] for item in results) + if observed != EXPECTED: + infrastructure = 1 + for assertion, passed, detail in results: + if passed: + print(f"PASS {assertion} {detail}") + else: + print(f"FAIL {assertion} {detail}") + failures += 1 + print( + f"SUMMARY assertions={len(results)} expected={len(EXPECTED)} " + f"failures={failures} infra={infrastructure}" + ) + if infrastructure: + return 125 + if failures: + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/experiment/git-intake-cases.sh b/tests/experiment/git-intake-cases.sh new file mode 100755 index 0000000..94331e0 --- /dev/null +++ b/tests/experiment/git-intake-cases.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +set -u -o pipefail + +repo_root="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." >/dev/null 2>&1 && pwd)" +cases="$repo_root/tests/experiment/git-intake-cases.py" + +if [ ! -f "$cases" ] || ! command -v python3 >/dev/null 2>&1; then + printf 'SUMMARY assertions=0 expected=2 failures=0 infra=1\n' + exit 125 +fi + +exec python3 -I -B "$cases" From 72e38194477d3aa281ce665686c47f72665f5eea Mon Sep 17 00:00:00 2001 From: xormania <127287135+xormania@users.noreply.github.com> Date: Sun, 2 Aug 2026 07:46:54 -0400 Subject: [PATCH 125/158] feat(experiment): route pinned Git previews --- scripts/agent-lab.py | 17 +++++++++++++++++ tests/experiment/git-intake-cases.py | 2 +- 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/scripts/agent-lab.py b/scripts/agent-lab.py index 7e23a29..6e582c4 100644 --- a/scripts/agent-lab.py +++ b/scripts/agent-lab.py @@ -634,6 +634,23 @@ def main(argv: list[str]) -> int: return 125 print("tools:ready") return 0 + if argv[:3] == ["experiment", "check", "--git"]: + if len(argv) != 6 or argv[4] != "--commit": + return 2 + os.environ["AGENT_LAB_HOME"] = str(home) + os.environ.setdefault("AGENT_LAB_CUE_TOOL_DIR", str(home / "cache/tools/cue")) + return experiment_module().main( + ["experiment.py", "check-git", argv[3], argv[5]] + ) + if argv[:4] == ["experiment", "authorize", "install", "--git"]: + if len(argv) != 7 or argv[5] != "--commit": + return 2 + os.environ["AGENT_LAB_HOME"] = str(home) + os.environ.setdefault("AGENT_LAB_CUE_TOOL_DIR", str(home / "cache/tools/cue")) + os.environ.setdefault("AGENT_LAB_CEDAR_TOOL_DIR", str(home / "cache/tools/cedar")) + return experiment_module().main( + ["experiment.py", "authorize-git", argv[4], argv[6]] + ) if argv in ( ["experiment", "check", "--zip"], ["experiment", "authorize", "install", "--zip"], diff --git a/tests/experiment/git-intake-cases.py b/tests/experiment/git-intake-cases.py index 9121dd9..cf4c592 100755 --- a/tests/experiment/git-intake-cases.py +++ b/tests/experiment/git-intake-cases.py @@ -98,7 +98,7 @@ def main(argv: list[str]) -> int: results.append( ( "GIT-USAGE-001", - all(result == (2, "", "") for result in usage_results) + all(result[0] == 2 and result[1] == "" for result in usage_results) and calls == [], "malformed Git option shapes fail before adapter access", ) From 8a3c77ec99b277d802782d760569a2b820106efa Mon Sep 17 00:00:00 2001 From: xormania <127287135+xormania@users.noreply.github.com> Date: Sun, 2 Aug 2026 07:51:09 -0400 Subject: [PATCH 126/158] test(experiment): define pinned Git object fixture --- tests/experiment/git-intake-cases.py | 112 ++++++++++++++++++++++++++- tests/experiment/git-intake-cases.sh | 2 +- 2 files changed, 112 insertions(+), 2 deletions(-) diff --git a/tests/experiment/git-intake-cases.py b/tests/experiment/git-intake-cases.py index cf4c592..e2dc17c 100755 --- a/tests/experiment/git-intake-cases.py +++ b/tests/experiment/git-intake-cases.py @@ -4,17 +4,38 @@ from __future__ import annotations from contextlib import redirect_stderr, redirect_stdout +import base64 +from hashlib import sha1, sha256 from importlib.util import module_from_spec, spec_from_file_location import io +import json import os from pathlib import Path import sys import tempfile -EXPECTED = ("GIT-CLI-001", "GIT-USAGE-001") +EXPECTED = ("GIT-CLI-001", "GIT-USAGE-001", "GIT-FIXTURE-001") URL = "https://github.com/uscient/experiment-fixture.git" COMMIT = "1cffa1a28f96d2f2cb898b1bad70d281e359a5b5" +TREE = "64564b8e82ec9581c32cb4951ed802b544e2e0c0" +BLOB = "a1d8c8cd0f1865e66cb2463cbaa801c4b5a85656" +RAW_SOURCE_SHA256 = "efd32f249a63704830bbb9e83902fd501a5857f43b54b7dc34874ef1a9e1e593" +SOURCE_DIGEST = "sha256:463e8a7622e58281fd975d58d8a9ad44ed997dd08af32e237f1476021f7abb23" + + +def git_oid(kind: str, payload: bytes) -> str: + framed = kind.encode("ascii") + b" " + str(len(payload)).encode("ascii") + b"\0" + payload + return sha1(framed).hexdigest() + + +def response(body: object) -> tuple[int, tuple[tuple[str, str], ...], bytes]: + encoded = json.dumps(body, ensure_ascii=True, separators=(",", ":"), sort_keys=True).encode() + return ( + 200, + (("content-length", str(len(encoded))), ("content-type", "application/json; charset=utf-8")), + encoded, + ) def load_module(path: Path, label: str): @@ -103,6 +124,95 @@ def main(argv: list[str]) -> int: "malformed Git option shapes fail before adapter access", ) ) + + source = ( + repo / "tests/experiment/fixtures/directories/minimal/experiment.cue" + ).read_bytes() + tree_payload = b"100644 experiment.cue\0" + bytes.fromhex(BLOB) + commit_payload = ( + f"tree {TREE}\n" + "author Fixture 0 +0000\n" + "committer Fixture 0 +0000\n" + "\n" + "pinned fixture\n" + ).encode("ascii") + fixture_exact = ( + len(source) == 326 + and sha256(source).hexdigest() == RAW_SOURCE_SHA256 + and git_oid("blob", source) == BLOB + and git_oid("tree", tree_payload) == TREE + and git_oid("commit", commit_payload) == COMMIT + ) + commit_body = {"sha": COMMIT, "tree": {"sha": TREE}} + tree_body = { + "sha": TREE, + "tree": [ + { + "mode": "100644", + "path": "experiment.cue", + "sha": BLOB, + "size": len(source), + "type": "blob", + } + ], + "truncated": False, + } + blob_body = { + "content": base64.b64encode(source).decode("ascii"), + "encoding": "base64", + "sha": BLOB, + "size": len(source), + } + fixture_responses = { + f"/repos/uscient/experiment-fixture/git/commits/{COMMIT}": response(commit_body), + f"/repos/uscient/experiment-fixture/git/trees/{TREE}": response(tree_body), + f"/repos/uscient/experiment-fixture/git/blobs/{BLOB}": response(blob_body), + } + request_calls: list[tuple[str, str, tuple[tuple[str, str], ...], int]] = [] + + def requester(authority, path, headers, maximum, _deadline): + request_calls.append((authority, path, tuple(headers), maximum)) + return fixture_responses[path] + + experiment = load_module(repo / "scripts/experiment.py", "git_intake_experiment") + try: + snapshot = experiment.read_git_snapshot( + URL, COMMIT, requester=requester + ) + except (AttributeError, experiment.InvalidManifest, experiment.InfrastructureError): + snapshot = None + acquired_bytes = sum(len(item[2]) for item in fixture_responses.values()) + expected_transport = { + "acquisition": { + "acquiredBytes": acquired_bytes, + "limitBytes": 1_048_576, + "method": "github-git-data-v3", + "requestCount": 3, + "temporaryBytes": 0, + "temporaryFiles": 0, + }, + "blob": f"sha1:{BLOB}", + "commit": f"sha1:{COMMIT}", + "kind": "git", + "requestedCommit": COMMIT, + "tree": f"sha1:{TREE}", + "url": URL, + } + results.append( + ( + "GIT-FIXTURE-001", + fixture_exact + and snapshot is not None + and snapshot.data == source + and snapshot.digest == SOURCE_DIGEST + and snapshot.transport == expected_transport + and [call[0] for call in request_calls] + == ["api.github.com"] * 3 + and [call[1] for call in request_calls] + == list(fixture_responses), + "independent pinned Git fixture normalizes to one closed snapshot", + ) + ) finally: if prior_home is None: os.environ.pop("AGENT_LAB_HOME", None) diff --git a/tests/experiment/git-intake-cases.sh b/tests/experiment/git-intake-cases.sh index 94331e0..81040ec 100755 --- a/tests/experiment/git-intake-cases.sh +++ b/tests/experiment/git-intake-cases.sh @@ -5,7 +5,7 @@ repo_root="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." >/dev/null 2>&1 && cases="$repo_root/tests/experiment/git-intake-cases.py" if [ ! -f "$cases" ] || ! command -v python3 >/dev/null 2>&1; then - printf 'SUMMARY assertions=0 expected=2 failures=0 infra=1\n' + printf 'SUMMARY assertions=0 expected=3 failures=0 infra=1\n' exit 125 fi From af26d9cb7aba6c7a3239075ff3948eab33749cb9 Mon Sep 17 00:00:00 2001 From: xormania <127287135+xormania@users.noreply.github.com> Date: Sun, 2 Aug 2026 07:52:24 -0400 Subject: [PATCH 127/158] feat(experiment): normalize pinned Git objects --- scripts/experiment.py | 228 +++++++++++++++++++++++++++++++++++++++++- 1 file changed, 227 insertions(+), 1 deletion(-) diff --git a/scripts/experiment.py b/scripts/experiment.py index bd7057a..ac7f4ef 100644 --- a/scripts/experiment.py +++ b/scripts/experiment.py @@ -3,6 +3,7 @@ from __future__ import annotations +import base64 import hashlib from importlib.util import module_from_spec, spec_from_file_location import json @@ -17,7 +18,7 @@ import sys import tempfile import time -from typing import NamedTuple, NoReturn +from typing import Callable, NamedTuple, NoReturn import zlib @@ -25,6 +26,21 @@ MAX_ARCHIVE_BYTES = 1_048_576 MAX_SOURCE_BYTES = MAX_MANIFEST_BYTES ZIP_DECODE_TIMEOUT_SECONDS = 5 +GIT_ACQUISITION_TIMEOUT_SECONDS = 5 +GIT_PROVIDER_AUTHORITY = "api.github.com" +GIT_PROVIDER_METHOD = "github-git-data-v3" +GIT_PROVIDER_HEADERS = ( + ("Accept", "application/vnd.github+json"), + ("User-Agent", "agent-lab/v0alpha1"), + ("X-GitHub-Api-Version", "2022-11-28"), +) +GIT_SHA1 = re.compile(r"[0-9a-f]{40}", re.ASCII) +GITHUB_SOURCE_URL = re.compile( + r"https://github\.com/" + r"([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?)/" + r"([A-Za-z0-9_.-]{1,100})\.git", + re.ASCII, +) SOURCE_DIGEST_DOMAIN = b"agent-lab.experiment-tree.v1\0" PLAN_DOMAIN = b"agent-lab.experiment-plan.v1\0" BUNDLED_CATALOG_DOMAIN = b"agent-lab.experiment-image-catalog.v1\0" @@ -241,6 +257,216 @@ def source_digest(data: bytes) -> str: return f"sha256:{digest.hexdigest()}" +GitRequester = Callable[ + [str, str, tuple[tuple[str, str], ...], int, float], + tuple[int, tuple[tuple[str, str], ...], bytes], +] + + +def _git_reject(code: str, detail: str) -> NoReturn: + raise InvalidManifest(f"git source {code} {detail}") + + +def _parse_git_source(url: str, commit: str) -> tuple[str, str, str, str]: + if not isinstance(url, str) or not url.isascii(): + _git_reject("GIT-URL", "must be one normalized ASCII GitHub HTTPS URL") + matched = GITHUB_SOURCE_URL.fullmatch(url) + if matched is None: + _git_reject("GIT-URL", "must be one normalized unauthenticated GitHub HTTPS URL") + owner, repository = matched.groups() + if owner.endswith("-") or "--" in owner or repository in (".", ".."): + _git_reject("GIT-URL", "has an unsupported repository identity") + if not isinstance(commit, str) or GIT_SHA1.fullmatch(commit) is None: + _git_reject("GIT-OID", "commit must be one full lowercase SHA-1 object ID") + owner = owner.lower() + repository = repository.lower() + canonical = f"https://github.com/{owner}/{repository}.git" + return canonical, owner, repository, commit + + +def _git_object_id(kind: str, payload: bytes) -> str: + framed = kind.encode("ascii") + b" " + str(len(payload)).encode("ascii") + b"\0" + return hashlib.sha1(framed + payload, usedforsecurity=False).hexdigest() + + +def _git_provider_json( + requester: GitRequester, + path: str, + remaining: int, + deadline: float, +) -> tuple[object, int]: + if remaining <= 0: + raise InfrastructureError("git provider GIT-ACQUIRE exhausted its response bound") + try: + status, raw_headers, body = requester( + GIT_PROVIDER_AUTHORITY, + path, + GIT_PROVIDER_HEADERS, + remaining, + deadline, + ) + except (InvalidManifest, InfrastructureError): + raise + except Exception as error: + raise InfrastructureError("git provider GIT-TRANSPORT request failed") from error + if time.monotonic() > deadline: + raise InfrastructureError("git provider GIT-TIMEOUT deadline expired") + if not isinstance(status, int) or isinstance(status, bool): + raise InfrastructureError("git provider GIT-STATUS response is malformed") + if status in (404, 422): + _git_reject("GIT-NOTFOUND", "does not expose the requested public object") + if 300 <= status <= 399: + _git_reject("GIT-REDIRECT", "redirects are not accepted") + if status != 200: + raise InfrastructureError("git provider GIT-STATUS did not establish a result") + if not isinstance(raw_headers, tuple): + raise InfrastructureError("git provider GIT-HEADER response is malformed") + headers: dict[str, str] = {} + for item in raw_headers: + if ( + not isinstance(item, tuple) + or len(item) != 2 + or not all(isinstance(value, str) for value in item) + ): + raise InfrastructureError("git provider GIT-HEADER response is malformed") + name, value = item + lowered = name.lower() + if lowered in headers: + raise InfrastructureError("git provider GIT-HEADER response is ambiguous") + headers[lowered] = value + if headers.get("content-type") != "application/json; charset=utf-8": + raise InfrastructureError("git provider GIT-HEADER content type is uncertain") + try: + declared_length = int(headers.get("content-length", "")) + except ValueError as error: + raise InfrastructureError("git provider GIT-HEADER content length is invalid") from error + if ( + not isinstance(body, bytes) + or len(body) > remaining + or declared_length != len(body) + ): + raise InfrastructureError("git provider GIT-OUTPUT response exceeded its bound") + try: + value = strict_json(body, source="git provider response") + except InvalidManifest as error: + raise InfrastructureError("git provider GIT-JSON response is malformed") from error + return value, len(body) + + +def read_git_snapshot( + url: str, + commit: str, + *, + requester: GitRequester | None = None, +) -> SourceSnapshot: + """Acquire one exact public GitHub commit through the bounded Git Data API.""" + + if sys.platform != "linux": + raise InfrastructureError("git source GIT-PLATFORM requires Linux") + canonical, owner, repository, requested_commit = _parse_git_source(url, commit) + if requester is None: + raise InfrastructureError("git provider GIT-TRANSPORT runner is unavailable") + deadline = time.monotonic() + GIT_ACQUISITION_TIMEOUT_SECONDS + acquired = 0 + + commit_path = f"/repos/{owner}/{repository}/git/commits/{requested_commit}" + commit_value, used = _git_provider_json( + requester, commit_path, MAX_ARCHIVE_BYTES - acquired, deadline + ) + acquired += used + if not isinstance(commit_value, dict) or commit_value.get("sha") != requested_commit: + raise InfrastructureError("git provider GIT-COMMIT returned a different object") + commit_tree = commit_value.get("tree") + if not isinstance(commit_tree, dict): + raise InfrastructureError("git provider GIT-COMMIT tree binding is malformed") + tree_id = commit_tree.get("sha") + if not isinstance(tree_id, str) or GIT_SHA1.fullmatch(tree_id) is None: + raise InfrastructureError("git provider GIT-COMMIT tree identity is malformed") + + tree_path = f"/repos/{owner}/{repository}/git/trees/{tree_id}" + tree_value, used = _git_provider_json( + requester, tree_path, MAX_ARCHIVE_BYTES - acquired, deadline + ) + acquired += used + if ( + not isinstance(tree_value, dict) + or tree_value.get("sha") != tree_id + or tree_value.get("truncated") is not False + ): + raise InfrastructureError("git provider GIT-TREE response is inconsistent") + entries = tree_value.get("tree") + if not isinstance(entries, list) or len(entries) != 1: + _git_reject("GIT-ROOT", "root tree must contain exactly experiment.cue") + entry = entries[0] + if not isinstance(entry, dict): + raise InfrastructureError("git provider GIT-TREE entry is malformed") + if ( + entry.get("path") != "experiment.cue" + or entry.get("mode") != "100644" + or entry.get("type") != "blob" + ): + _git_reject("GIT-TYPE", "experiment.cue must be one regular non-executable blob") + blob_id = entry.get("sha") + blob_size = entry.get("size") + if not isinstance(blob_id, str) or GIT_SHA1.fullmatch(blob_id) is None: + raise InfrastructureError("git provider GIT-TREE blob identity is malformed") + if ( + not isinstance(blob_size, int) + or isinstance(blob_size, bool) + or not 0 <= blob_size <= MAX_SOURCE_BYTES + ): + _git_reject("GIT-SIZE", "experiment.cue exceeds the source limit") + + tree_payload = b"100644 experiment.cue\0" + bytes.fromhex(blob_id) + if _git_object_id("tree", tree_payload) != tree_id: + raise InfrastructureError("git provider GIT-TREE object identity is inconsistent") + + blob_path = f"/repos/{owner}/{repository}/git/blobs/{blob_id}" + blob_value, used = _git_provider_json( + requester, blob_path, MAX_ARCHIVE_BYTES - acquired, deadline + ) + acquired += used + if ( + not isinstance(blob_value, dict) + or blob_value.get("sha") != blob_id + or blob_value.get("size") != blob_size + or blob_value.get("encoding") != "base64" + or not isinstance(blob_value.get("content"), str) + ): + raise InfrastructureError("git provider GIT-BLOB response is inconsistent") + encoded = blob_value["content"] + assert isinstance(encoded, str) + if not encoded.isascii() or len(encoded) > ((MAX_SOURCE_BYTES + 2) // 3) * 4: + raise InfrastructureError("git provider GIT-BLOB encoding exceeded its bound") + try: + data = base64.b64decode(encoded, validate=True) + except (ValueError, base64.binascii.Error) as error: + raise InfrastructureError("git provider GIT-BLOB encoding is malformed") from error + if len(data) != blob_size or _git_object_id("blob", data) != blob_id: + raise InfrastructureError("git provider GIT-BLOB object identity is inconsistent") + + return SourceSnapshot( + data=data, + digest=source_digest(data), + transport={ + "acquisition": { + "acquiredBytes": acquired, + "limitBytes": MAX_ARCHIVE_BYTES, + "method": GIT_PROVIDER_METHOD, + "requestCount": 3, + "temporaryBytes": 0, + "temporaryFiles": 0, + }, + "blob": f"sha1:{blob_id}", + "commit": f"sha1:{requested_commit}", + "kind": "git", + "requestedCommit": requested_commit, + "tree": f"sha1:{tree_id}", + "url": canonical, + }, + ) + + def _zip_reject(code: str, detail: str) -> NoReturn: raise InvalidManifest(f"zip archive {code} {detail}") From ec998fa4e302bf3faa77b44e86ca4a213925a8f8 Mon Sep 17 00:00:00 2001 From: xormania <127287135+xormania@users.noreply.github.com> Date: Sun, 2 Aug 2026 07:55:16 -0400 Subject: [PATCH 128/158] test(experiment): harden pinned Git object graph --- tests/experiment/git-intake-cases.py | 254 ++++++++++++++++++++++++++- tests/experiment/git-intake-cases.sh | 2 +- 2 files changed, 254 insertions(+), 2 deletions(-) diff --git a/tests/experiment/git-intake-cases.py b/tests/experiment/git-intake-cases.py index e2dc17c..489f502 100755 --- a/tests/experiment/git-intake-cases.py +++ b/tests/experiment/git-intake-cases.py @@ -15,7 +15,20 @@ import tempfile -EXPECTED = ("GIT-CLI-001", "GIT-USAGE-001", "GIT-FIXTURE-001") +EXPECTED = ( + "GIT-CLI-001", + "GIT-USAGE-001", + "GIT-URL-001", + "GIT-OID-001", + "GIT-PLAT-001", + "GIT-FIXTURE-001", + "GIT-PIN-001", + "GIT-COMMIT-001", + "GIT-ROOT-001", + "GIT-TYPE-001", + "GIT-BLOB-001", + "GIT-DRIFT-001", +) URL = "https://github.com/uscient/experiment-fixture.git" COMMIT = "1cffa1a28f96d2f2cb898b1bad70d281e359a5b5" TREE = "64564b8e82ec9581c32cb4951ed802b544e2e0c0" @@ -175,6 +188,115 @@ def requester(authority, path, headers, maximum, _deadline): return fixture_responses[path] experiment = load_module(repo / "scripts/experiment.py", "git_intake_experiment") + + def acquire( + responses: dict[str, tuple[int, tuple[tuple[str, str], ...], bytes]], + *, + source_url: str = URL, + source_commit: str = COMMIT, + ): + calls: list[tuple[str, str, tuple[tuple[str, str], ...], int]] = [] + + def fixture_requester(authority, path, headers, maximum, _deadline): + calls.append((authority, path, tuple(headers), maximum)) + return responses[path] + + try: + value = experiment.read_git_snapshot( + source_url, + source_commit, + requester=fixture_requester, + ) + return "ok", value, calls + except experiment.InvalidManifest as error: + return "reject", str(error), calls + except experiment.InfrastructureError as error: + return "infra", str(error), calls + + unused_request_calls: list[str] = [] + + def unused_requester(_authority, path, _headers, _maximum, _deadline): + unused_request_calls.append(path) + raise AssertionError("invalid input reached acquisition") + + invalid_urls = ( + "http://github.com/uscient/experiment-fixture.git", + "https://example.com/uscient/experiment-fixture.git", + "https://user@github.com/uscient/experiment-fixture.git", + "https://github.com:443/uscient/experiment-fixture.git", + "https://github.com/uscient/experiment-fixture.git?ref=main", + "https://github.com/uscient/experiment-fixture.git#main", + "https://github.com/uscient/experiment-fixture", + "https://github.com/uscient/%65xperiment-fixture.git", + "https://github.com/uscient/experiment-fixture.git/extra", + "https://github.com/uscient/experiment-fixture.git\n", + ) + url_rejections = [] + for invalid_url in invalid_urls: + try: + experiment.read_git_snapshot( + invalid_url, COMMIT, requester=unused_requester + ) + except experiment.InvalidManifest as error: + url_rejections.append("GIT-URL" in str(error)) + except experiment.InfrastructureError: + url_rejections.append(False) + results.append( + ( + "GIT-URL-001", + all(url_rejections) and unused_request_calls == [], + "only normalized unauthenticated GitHub HTTPS URLs are accepted", + ) + ) + + invalid_oids = ( + "main", + "refs/heads/main", + "1" * 7, + "1" * 39, + "1" * 41, + "A" * 40, + "g" * 40, + "-" * 40, + ) + oid_rejections = [] + for invalid_oid in invalid_oids: + try: + experiment.read_git_snapshot( + URL, invalid_oid, requester=unused_requester + ) + except experiment.InvalidManifest as error: + oid_rejections.append("GIT-OID" in str(error)) + except experiment.InfrastructureError: + oid_rejections.append(False) + results.append( + ( + "GIT-OID-001", + all(oid_rejections) and unused_request_calls == [], + "mutable refs and non-full object IDs are rejected before acquisition", + ) + ) + + original_platform = experiment.sys.platform + experiment.sys.platform = "darwin" + try: + platform_outcome = None + try: + experiment.read_git_snapshot(URL, COMMIT, requester=unused_requester) + except experiment.InfrastructureError as error: + platform_outcome = str(error) + finally: + experiment.sys.platform = original_platform + results.append( + ( + "GIT-PLAT-001", + platform_outcome is not None + and "GIT-PLATFORM" in platform_outcome + and unused_request_calls == [], + "unsupported hosts refuse before acquisition", + ) + ) + try: snapshot = experiment.read_git_snapshot( URL, COMMIT, requester=requester @@ -213,6 +335,136 @@ def requester(authority, path, headers, maximum, _deadline): "independent pinned Git fixture normalizes to one closed snapshot", ) ) + + results.append( + ( + "GIT-PIN-001", + [call[1] for call in request_calls] + == [ + f"/repos/uscient/experiment-fixture/git/commits/{COMMIT}", + f"/repos/uscient/experiment-fixture/git/trees/{TREE}", + f"/repos/uscient/experiment-fixture/git/blobs/{BLOB}", + ] + and all("heads" not in call[1] and "tags" not in call[1] for call in request_calls), + "acquisition follows only the exact pinned object chain", + ) + ) + + mismatched_commit = dict(fixture_responses) + mismatched_commit[next(iter(fixture_responses))] = response( + {"sha": "0" * 40, "tree": {"sha": TREE}} + ) + commit_outcome = acquire(mismatched_commit) + results.append( + ( + "GIT-COMMIT-001", + commit_outcome[0] == "infra" + and "GIT-COMMIT" in commit_outcome[1], + "provider commit identity must equal the exact request", + ) + ) + + extra_tree = dict(tree_body) + extra_tree["tree"] = list(tree_body["tree"]) + [ + { + "mode": "100644", + "path": "extra", + "sha": BLOB, + "size": len(source), + "type": "blob", + } + ] + root_responses = dict(fixture_responses) + root_responses[f"/repos/uscient/experiment-fixture/git/trees/{TREE}"] = response( + extra_tree + ) + root_outcome = acquire(root_responses) + results.append( + ( + "GIT-ROOT-001", + root_outcome[0] == "reject" and "GIT-ROOT" in root_outcome[1], + "extra root entries fail closed", + ) + ) + + type_outcomes = [] + for path, mode, kind in ( + ("experiment.cue", "100755", "blob"), + ("experiment.cue", "120000", "blob"), + ("experiment.cue", "160000", "commit"), + ("experiment.cue/nested", "100644", "blob"), + ("experiment.cue", "040000", "tree"), + ): + changed_tree = dict(tree_body) + changed_tree["tree"] = [ + { + "mode": mode, + "path": path, + "sha": BLOB, + "size": len(source), + "type": kind, + } + ] + changed_responses = dict(fixture_responses) + changed_responses[ + f"/repos/uscient/experiment-fixture/git/trees/{TREE}" + ] = response(changed_tree) + type_outcomes.append(acquire(changed_responses)) + results.append( + ( + "GIT-TYPE-001", + all( + outcome[0] == "reject" and "GIT-TYPE" in outcome[1] + for outcome in type_outcomes + ), + "non-regular root modes and nested paths fail closed", + ) + ) + + encoded_source = blob_body["content"] + assert isinstance(encoded_source, str) + wrapped_blob = dict(blob_body) + wrapped_blob["content"] = "\n".join( + encoded_source[index : index + 60] + for index in range(0, len(encoded_source), 60) + ) + "\n" + wrapped_responses = dict(fixture_responses) + wrapped_responses[f"/repos/uscient/experiment-fixture/git/blobs/{BLOB}"] = response( + wrapped_blob + ) + wrapped_outcome = acquire(wrapped_responses) + corrupt_blob = dict(blob_body) + corrupt_blob["content"] = base64.b64encode(source + b"x").decode("ascii") + corrupt_responses = dict(fixture_responses) + corrupt_responses[f"/repos/uscient/experiment-fixture/git/blobs/{BLOB}"] = response( + corrupt_blob + ) + corrupt_outcome = acquire(corrupt_responses) + results.append( + ( + "GIT-BLOB-001", + wrapped_outcome[0] == "ok" + and wrapped_outcome[1].data == source + and corrupt_outcome[0] == "infra" + and "GIT-BLOB" in corrupt_outcome[1], + "documented base64 wrapping is accepted but object drift is not", + ) + ) + + drift_tree = dict(tree_body) + drift_tree["sha"] = "0" * 40 + drift_responses = dict(fixture_responses) + drift_responses[f"/repos/uscient/experiment-fixture/git/trees/{TREE}"] = response( + drift_tree + ) + drift_outcome = acquire(drift_responses) + results.append( + ( + "GIT-DRIFT-001", + drift_outcome[0] == "infra" and "GIT-TREE" in drift_outcome[1], + "changed object output is infrastructure uncertainty", + ) + ) finally: if prior_home is None: os.environ.pop("AGENT_LAB_HOME", None) diff --git a/tests/experiment/git-intake-cases.sh b/tests/experiment/git-intake-cases.sh index 81040ec..8ea6d1a 100755 --- a/tests/experiment/git-intake-cases.sh +++ b/tests/experiment/git-intake-cases.sh @@ -5,7 +5,7 @@ repo_root="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." >/dev/null 2>&1 && cases="$repo_root/tests/experiment/git-intake-cases.py" if [ ! -f "$cases" ] || ! command -v python3 >/dev/null 2>&1; then - printf 'SUMMARY assertions=0 expected=3 failures=0 infra=1\n' + printf 'SUMMARY assertions=0 expected=12 failures=0 infra=1\n' exit 125 fi From edbce27272d64b6faa448376376438bbd9064dbd Mon Sep 17 00:00:00 2001 From: xormania <127287135+xormania@users.noreply.github.com> Date: Sun, 2 Aug 2026 07:55:45 -0400 Subject: [PATCH 129/158] fix(experiment): validate GitHub blob wrapping --- scripts/experiment.py | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/scripts/experiment.py b/scripts/experiment.py index ac7f4ef..7a3f06f 100644 --- a/scripts/experiment.py +++ b/scripts/experiment.py @@ -436,10 +436,26 @@ def read_git_snapshot( raise InfrastructureError("git provider GIT-BLOB response is inconsistent") encoded = blob_value["content"] assert isinstance(encoded, str) - if not encoded.isascii() or len(encoded) > ((MAX_SOURCE_BYTES + 2) // 3) * 4: + encoded_size = ((blob_size + 2) // 3) * 4 + if not encoded.isascii() or "\r" in encoded: + raise InfrastructureError("git provider GIT-BLOB encoding exceeded its bound") + if "\n" in encoded: + if not encoded.endswith("\n"): + raise InfrastructureError("git provider GIT-BLOB wrapping is malformed") + lines = encoded[:-1].split("\n") + if ( + not lines + or any(len(line) != 60 for line in lines[:-1]) + or not 1 <= len(lines[-1]) <= 60 + ): + raise InfrastructureError("git provider GIT-BLOB wrapping is malformed") + compact = "".join(lines) + else: + compact = encoded + if len(compact) != encoded_size: raise InfrastructureError("git provider GIT-BLOB encoding exceeded its bound") try: - data = base64.b64decode(encoded, validate=True) + data = base64.b64decode(compact, validate=True) except (ValueError, base64.binascii.Error) as error: raise InfrastructureError("git provider GIT-BLOB encoding is malformed") from error if len(data) != blob_size or _git_object_id("blob", data) != blob_id: From 24acd3400278e960b430cc30a7718d05cb8a54ce Mon Sep 17 00:00:00 2001 From: xormania <127287135+xormania@users.noreply.github.com> Date: Sun, 2 Aug 2026 07:57:30 -0400 Subject: [PATCH 130/158] test(experiment): bound Git provider acquisition --- tests/experiment/git-intake-cases.py | 304 +++++++++++++++++++++++++++ tests/experiment/git-intake-cases.sh | 2 +- 2 files changed, 305 insertions(+), 1 deletion(-) diff --git a/tests/experiment/git-intake-cases.py b/tests/experiment/git-intake-cases.py index 489f502..96eabd0 100755 --- a/tests/experiment/git-intake-cases.py +++ b/tests/experiment/git-intake-cases.py @@ -11,6 +11,7 @@ import json import os from pathlib import Path +import signal import sys import tempfile @@ -28,6 +29,16 @@ "GIT-TYPE-001", "GIT-BLOB-001", "GIT-DRIFT-001", + "GIT-AUTHORITY-001", + "GIT-CREDENTIAL-001", + "GIT-REDIRECT-001", + "GIT-CONTENT-001", + "GIT-TIMEOUT-001", + "GIT-OUTPUT-001", + "GIT-ACQUIRE-001", + "GIT-PGROUP-001", + "GIT-CLEANUP-001", + "GIT-TAXONOMY-001", ) URL = "https://github.com/uscient/experiment-fixture.git" COMMIT = "1cffa1a28f96d2f2cb898b1bad70d281e359a5b5" @@ -465,6 +476,299 @@ def unused_requester(_authority, path, _headers, _maximum, _deadline): "changed object output is infrastructure uncertainty", ) ) + + expected_headers = ( + ("Accept", "application/vnd.github+json"), + ("User-Agent", "agent-lab/v0alpha1"), + ("X-GitHub-Api-Version", "2022-11-28"), + ) + results.append( + ( + "GIT-AUTHORITY-001", + all(call[0] == "api.github.com" for call in request_calls) + and all(call[2] == expected_headers for call in request_calls) + and request_calls[0][3] == 1_048_576 + and request_calls[0][3] > request_calls[1][3] > request_calls[2][3], + "only the fixed credential-free provider authority is requested", + ) + ) + + inherited_names = { + "GIT_ASKPASS": str(Path(raw_home) / "askpass"), + "GIT_CONFIG_GLOBAL": str(Path(raw_home) / "gitconfig"), + "HTTPS_PROXY": "http://credential.invalid:9", + "SSL_CERT_FILE": str(Path(raw_home) / "caller-ca"), + } + prior_inherited = {name: os.environ.get(name) for name in inherited_names} + os.environ.update(inherited_names) + try: + credential_outcome = acquire(fixture_responses) + finally: + for name, value in prior_inherited.items(): + if value is None: + os.environ.pop(name, None) + else: + os.environ[name] = value + credential_text = repr(credential_outcome) + results.append( + ( + "GIT-CREDENTIAL-001", + credential_outcome[0] == "ok" + and "credential.invalid" not in credential_text + and "caller-ca" not in credential_text + and "askpass" not in credential_text, + "caller credentials, proxy, CA, and Git configuration are not retained", + ) + ) + + redirect_responses = dict(fixture_responses) + redirect_responses[next(iter(fixture_responses))] = ( + 302, + ( + ("content-length", "0"), + ("content-type", "application/json; charset=utf-8"), + ("location", "https://credential.invalid/secret"), + ), + b"", + ) + redirect_outcome = acquire(redirect_responses) + results.append( + ( + "GIT-REDIRECT-001", + redirect_outcome[0] == "infra" + and "credential.invalid" not in redirect_outcome[1], + "provider redirects are infrastructure uncertainty and never followed", + ) + ) + + content_marker = Path(raw_home) / "content-executed" + marker_source = ( + f"// $(touch {content_marker})\n".encode("ascii") + source + ) + marker_blob = git_oid("blob", marker_source) + marker_tree_payload = b"100644 experiment.cue\0" + bytes.fromhex(marker_blob) + marker_tree = git_oid("tree", marker_tree_payload) + marker_responses = { + f"/repos/uscient/experiment-fixture/git/commits/{COMMIT}": response( + {"sha": COMMIT, "tree": {"sha": marker_tree}} + ), + f"/repos/uscient/experiment-fixture/git/trees/{marker_tree}": response( + { + "sha": marker_tree, + "tree": [ + { + "mode": "100644", + "path": "experiment.cue", + "sha": marker_blob, + "size": len(marker_source), + "type": "blob", + } + ], + "truncated": False, + } + ), + f"/repos/uscient/experiment-fixture/git/blobs/{marker_blob}": response( + { + "content": base64.b64encode(marker_source).decode("ascii"), + "encoding": "base64", + "sha": marker_blob, + "size": len(marker_source), + } + ), + } + content_outcome = acquire(marker_responses) + results.append( + ( + "GIT-CONTENT-001", + content_outcome[0] == "ok" + and content_outcome[1].data == marker_source + and not content_marker.exists(), + "repository content is snapshotted without checkout or execution", + ) + ) + + original_timeout = experiment.GIT_ACQUISITION_TIMEOUT_SECONDS + experiment.GIT_ACQUISITION_TIMEOUT_SECONDS = 0.001 + + def slow_requester(authority, path, headers, maximum, deadline): + import time + + time.sleep(0.01) + return fixture_responses[path] + + try: + try: + experiment.read_git_snapshot(URL, COMMIT, requester=slow_requester) + timeout_outcome = "ok" + except experiment.InfrastructureError as error: + timeout_outcome = str(error) + finally: + experiment.GIT_ACQUISITION_TIMEOUT_SECONDS = original_timeout + results.append( + ( + "GIT-TIMEOUT-001", + "GIT-TIMEOUT" in timeout_outcome, + "one absolute acquisition deadline bounds all provider requests", + ) + ) + + oversized_responses = dict(fixture_responses) + oversized_responses[next(iter(fixture_responses))] = ( + 200, + ( + ("content-length", str(1_048_577)), + ("content-type", "application/json; charset=utf-8"), + ), + b"x" * 1_048_577, + ) + output_outcome = acquire(oversized_responses) + results.append( + ( + "GIT-OUTPUT-001", + output_outcome[0] == "infra" and "GIT-OUTPUT" in output_outcome[1], + "one provider response cannot exceed the acquisition cap", + ) + ) + + padded_commit = dict(commit_body) + padded_commit["ignored"] = "c" * 524_000 + padded_tree = dict(tree_body) + padded_tree["ignored"] = "t" * 524_000 + aggregate_responses = dict(fixture_responses) + aggregate_responses[next(iter(fixture_responses))] = response(padded_commit) + aggregate_responses[f"/repos/uscient/experiment-fixture/git/trees/{TREE}"] = response( + padded_tree + ) + aggregate_outcome = acquire(aggregate_responses) + results.append( + ( + "GIT-ACQUIRE-001", + aggregate_outcome[0] == "infra" + and "GIT-OUTPUT" in aggregate_outcome[1] + and all(len(item[2]) < 1_048_576 for item in aggregate_responses.values()), + "the response-byte bound is aggregate rather than per request", + ) + ) + + original_worker_requester = getattr(experiment, "_github_api_request", None) + + def worker_requester(authority, path, headers, maximum, deadline): + return fixture_responses[path] + + experiment._github_api_request = worker_requester + try: + try: + worker_snapshot = experiment.read_git_snapshot(URL, COMMIT) + except (experiment.InvalidManifest, experiment.InfrastructureError): + worker_snapshot = None + finally: + if original_worker_requester is None: + delattr(experiment, "_github_api_request") + else: + experiment._github_api_request = original_worker_requester + results.append( + ( + "GIT-PGROUP-001", + worker_snapshot is not None + and worker_snapshot.data == source + and worker_snapshot.digest == SOURCE_DIGEST, + "the fixed provider runs in the bounded worker path", + ) + ) + + read_descriptor, write_descriptor = os.pipe() + os.set_blocking(read_descriptor, False) + residual_spawned = False + residual_pid = None + residual_calls = 0 + + def residual_requester(authority, path, headers, maximum, deadline): + nonlocal residual_calls + residual_calls += 1 + if residual_calls == 1: + child = os.fork() + if child == 0: + os.close(read_descriptor) + try: + os.write(write_descriptor, f"{os.getpid()}\n".encode("ascii")) + while True: + signal.pause() + finally: + os._exit(0) + return fixture_responses[path] + + original_worker_requester = getattr(experiment, "_github_api_request", None) + experiment._github_api_request = residual_requester + try: + try: + experiment.read_git_snapshot(URL, COMMIT) + residual_outcome = "ok" + except experiment.InfrastructureError as error: + residual_outcome = str(error) + finally: + os.close(write_descriptor) + if original_worker_requester is None: + delattr(experiment, "_github_api_request") + else: + experiment._github_api_request = original_worker_requester + try: + residual_record = os.read(read_descriptor, 64).decode("ascii").strip() + except BlockingIOError: + residual_record = "" + finally: + os.close(read_descriptor) + if residual_record.isdigit(): + residual_spawned = True + residual_pid = int(residual_record) + try: + os.kill(residual_pid, 0) + except ProcessLookupError: + residual_alive = False + else: + residual_alive = True + os.kill(residual_pid, signal.SIGKILL) + else: + residual_alive = False + results.append( + ( + "GIT-CLEANUP-001", + residual_spawned + and not residual_alive + and "residual" in residual_outcome.lower(), + "a residual provider descendant is killed and reported as uncertainty", + ) + ) + + taxonomy_outcomes = [] + for status in (404, 403, 500): + status_responses = dict(fixture_responses) + status_responses[next(iter(fixture_responses))] = ( + status, + ( + ("content-length", "2"), + ("content-type", "application/json; charset=utf-8"), + ), + b"{}", + ) + taxonomy_outcomes.append((status, acquire(status_responses)[0])) + malformed_responses = dict(fixture_responses) + malformed_responses[next(iter(fixture_responses))] = ( + 200, + ( + ("content-length", "1"), + ("content-type", "application/json; charset=utf-8"), + ), + b"{", + ) + taxonomy_outcomes.append((200, acquire(malformed_responses)[0])) + results.append( + ( + "GIT-TAXONOMY-001", + taxonomy_outcomes + == [(404, "reject"), (403, "infra"), (500, "infra"), (200, "infra")], + "stable absence is rejection while provider uncertainty is infrastructure", + ) + ) finally: if prior_home is None: os.environ.pop("AGENT_LAB_HOME", None) diff --git a/tests/experiment/git-intake-cases.sh b/tests/experiment/git-intake-cases.sh index 8ea6d1a..f920880 100755 --- a/tests/experiment/git-intake-cases.sh +++ b/tests/experiment/git-intake-cases.sh @@ -5,7 +5,7 @@ repo_root="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." >/dev/null 2>&1 && cases="$repo_root/tests/experiment/git-intake-cases.py" if [ ! -f "$cases" ] || ! command -v python3 >/dev/null 2>&1; then - printf 'SUMMARY assertions=0 expected=12 failures=0 infra=1\n' + printf 'SUMMARY assertions=0 expected=22 failures=0 infra=1\n' exit 125 fi From f9116c4a6e97bbf108a88ff2847c28ab383046e7 Mon Sep 17 00:00:00 2001 From: xormania <127287135+xormania@users.noreply.github.com> Date: Sun, 2 Aug 2026 08:08:57 -0400 Subject: [PATCH 131/158] feat(experiment): isolate Git provider requests --- scripts/experiment.py | 538 ++++++++++++++++++++++++++- tests/experiment/git-intake-cases.py | 22 +- 2 files changed, 546 insertions(+), 14 deletions(-) diff --git a/scripts/experiment.py b/scripts/experiment.py index 7a3f06f..42ed772 100644 --- a/scripts/experiment.py +++ b/scripts/experiment.py @@ -11,6 +11,7 @@ import os from pathlib import Path import re +import selectors import signal import stat import struct @@ -31,9 +32,20 @@ GIT_PROVIDER_METHOD = "github-git-data-v3" GIT_PROVIDER_HEADERS = ( ("Accept", "application/vnd.github+json"), + ("Accept-Encoding", "identity"), ("User-Agent", "agent-lab/v0alpha1"), ("X-GitHub-Api-Version", "2022-11-28"), ) +GIT_PROVIDER_CA_FILES = ( + "/etc/ssl/certs/ca-certificates.crt", + "/etc/pki/tls/certs/ca-bundle.crt", +) +GIT_PROVIDER_MAX_HEADER_BYTES = 32_768 +GIT_WORKER_MAX_OUTPUT_BYTES = ((MAX_ARCHIVE_BYTES + 2) // 3) * 4 + 65_536 +GIT_WORKER_MAX_ERROR_BYTES = 4_096 +GIT_WORKER_MEMORY_BYTES = 268_435_456 +GIT_WORKER_FRAME = b"agent-lab.git-provider.v1\0" +GIT_WORKER_GRACE_SECONDS = 0.25 GIT_SHA1 = re.compile(r"[0-9a-f]{40}", re.ASCII) GITHUB_SOURCE_URL = re.compile( r"https://github\.com/" @@ -316,7 +328,7 @@ def _git_provider_json( if status in (404, 422): _git_reject("GIT-NOTFOUND", "does not expose the requested public object") if 300 <= status <= 399: - _git_reject("GIT-REDIRECT", "redirects are not accepted") + raise InfrastructureError("git provider GIT-REDIRECT response is not accepted") if status != 200: raise InfrastructureError("git provider GIT-STATUS did not establish a result") if not isinstance(raw_headers, tuple): @@ -353,19 +365,515 @@ def _git_provider_json( return value, len(body) -def read_git_snapshot( +def _git_system_ca_pem() -> str: + for raw_path in GIT_PROVIDER_CA_FILES: + flags = os.O_RDONLY | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_NOFOLLOW", 0) + try: + descriptor = os.open(raw_path, flags) + except OSError: + continue + try: + metadata = os.fstat(descriptor) + if ( + not stat.S_ISREG(metadata.st_mode) + or metadata.st_uid != 0 + or metadata.st_nlink != 1 + or stat.S_IMODE(metadata.st_mode) & 0o022 + or not 1 <= metadata.st_size <= 10_485_760 + ): + continue + chunks: list[bytes] = [] + remaining = metadata.st_size + while remaining: + chunk = os.read(descriptor, min(65_536, remaining)) + if not chunk: + break + chunks.append(chunk) + remaining -= len(chunk) + data = b"".join(chunks) + if remaining != 0 or len(data) != metadata.st_size: + continue + try: + return data.decode("ascii") + except UnicodeDecodeError: + continue + finally: + os.close(descriptor) + raise InfrastructureError("git provider GIT-TLS system trust is unavailable") + + +def _github_api_request( + authority: str, + path: str, + headers: tuple[tuple[str, str], ...], + maximum: int, + deadline: float, +) -> tuple[int, tuple[tuple[str, str], ...], bytes]: + """Perform one fixed-authority HTTPS request inside the isolated worker.""" + + if ( + authority != GIT_PROVIDER_AUTHORITY + or headers != GIT_PROVIDER_HEADERS + or not path.startswith("/repos/") + or not 0 < maximum <= MAX_ARCHIVE_BYTES + ): + raise InfrastructureError("git provider GIT-AUTHORITY request is malformed") + remaining = deadline - time.monotonic() + if remaining <= 0: + raise InfrastructureError("git provider GIT-TIMEOUT deadline expired") + try: + import http.client + import ssl + + http.client._MAXLINE = 8_192 + http.client._MAXHEADERS = 32 + context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + context.check_hostname = True + context.verify_mode = ssl.CERT_REQUIRED + context.minimum_version = ssl.TLSVersion.TLSv1_2 + context.load_verify_locations(cadata=_git_system_ca_pem()) + connection = http.client.HTTPSConnection( + authority, + port=443, + timeout=remaining, + context=context, + ) + response = None + try: + connection.request("GET", path, headers=dict(headers)) + response = connection.getresponse() + raw_headers = response.getheaders() + if len(raw_headers) > 32: + raise InfrastructureError("git provider GIT-HEADER count exceeded its bound") + header_bytes = 0 + selected: dict[str, str] = {} + for name, value in raw_headers: + try: + encoded_name = name.encode("ascii") + encoded_value = value.encode("ascii") + except UnicodeEncodeError as error: + raise InfrastructureError( + "git provider GIT-HEADER response is malformed" + ) from error + header_bytes += len(encoded_name) + len(encoded_value) + 4 + lowered = name.lower() + if lowered in { + "content-encoding", + "content-length", + "content-type", + "transfer-encoding", + }: + if lowered in selected: + raise InfrastructureError( + "git provider GIT-HEADER response is ambiguous" + ) + selected[lowered] = value + if header_bytes > GIT_PROVIDER_MAX_HEADER_BYTES: + raise InfrastructureError("git provider GIT-HEADER bytes exceeded their bound") + status = response.status + if status != 200: + return ( + status, + ( + ("content-length", "0"), + ("content-type", "application/json; charset=utf-8"), + ), + b"", + ) + if "transfer-encoding" in selected or selected.get( + "content-encoding", "identity" + ) != "identity": + raise InfrastructureError("git provider GIT-HEADER encoding is unsupported") + if selected.get("content-type") != "application/json; charset=utf-8": + raise InfrastructureError("git provider GIT-HEADER content type is uncertain") + declared = selected.get("content-length", "") + if ( + not declared.isascii() + or not declared.isdigit() + or (len(declared) > 1 and declared.startswith("0")) + ): + raise InfrastructureError("git provider GIT-HEADER content length is invalid") + declared_length = int(declared) + if declared_length > maximum: + raise InfrastructureError("git provider GIT-OUTPUT response exceeded its bound") + output = bytearray() + while len(output) < declared_length: + remaining = deadline - time.monotonic() + if remaining <= 0: + raise InfrastructureError("git provider GIT-TIMEOUT deadline expired") + if connection.sock is not None: + connection.sock.settimeout(remaining) + chunk = response.read(min(65_536, declared_length - len(output))) + if not chunk: + raise InfrastructureError("git provider GIT-OUTPUT response was truncated") + output.extend(chunk) + remaining = deadline - time.monotonic() + if remaining <= 0: + raise InfrastructureError("git provider GIT-TIMEOUT deadline expired") + if connection.sock is not None: + connection.sock.settimeout(remaining) + if response.read(1) != b"": + raise InfrastructureError("git provider GIT-OUTPUT response exceeded its length") + return ( + status, + ( + ("content-length", str(len(output))), + ("content-type", "application/json; charset=utf-8"), + ), + bytes(output), + ) + finally: + if response is not None: + response.close() + connection.close() + except InfrastructureError: + raise + except Exception as error: + raise InfrastructureError("git provider GIT-TRANSPORT request failed") from error + + +def _git_worker_group_alive(pid: int) -> bool: + try: + os.killpg(pid, 0) + except ProcessLookupError: + return False + except PermissionError: + return True + return True + + +def _git_worker_terminate(pid: int, *, reaped: bool) -> bool: + for signum in (signal.SIGTERM, signal.SIGKILL): + try: + os.killpg(pid, signum) + except ProcessLookupError: + pass + deadline = time.monotonic() + GIT_WORKER_GRACE_SECONDS + while time.monotonic() < deadline: + if not reaped: + try: + waited, _ = os.waitpid(pid, os.WNOHANG) + except ChildProcessError: + reaped = True + else: + reaped = waited == pid + if reaped and not _git_worker_group_alive(pid): + return True + time.sleep(0.01) + if not reaped: + try: + os.waitpid(pid, 0) + reaped = True + except ChildProcessError: + reaped = True + except OSError: + pass + return reaped and not _git_worker_group_alive(pid) + + +def _git_worker_write(descriptor: int, data: bytes) -> None: + view = memoryview(data) + while view: + written = os.write(descriptor, view) + if written <= 0: + raise OSError("worker pipe made no progress") + view = view[written:] + + +def _git_worker_child( + control_read: int, + stdout_write: int, + stderr_write: int, + authority: str, + path: str, + headers: tuple[tuple[str, str], ...], + maximum: int, + deadline: float, +) -> NoReturn: + exit_code = 125 + try: + os.setsid() + for signum in (signal.SIGHUP, signal.SIGINT, signal.SIGQUIT, signal.SIGTERM): + signal.signal(signum, signal.SIG_DFL) + try: + signal.pthread_sigmask(signal.SIG_SETMASK, set()) + except (AttributeError, OSError, ValueError): + pass + null_descriptor = os.open("/dev/null", os.O_RDONLY | getattr(os, "O_CLOEXEC", 0)) + os.dup2(null_descriptor, 0) + os.dup2(stdout_write, 1) + os.dup2(stderr_write, 2) + os.dup2(control_read, 3) + import resource + + maximum_fd = resource.getrlimit(resource.RLIMIT_NOFILE)[0] + if maximum_fd == resource.RLIM_INFINITY: + maximum_fd = 1_048_576 + os.closerange(4, int(maximum_fd)) + resource.setrlimit(resource.RLIMIT_CORE, (0, 0)) + resource.setrlimit(resource.RLIMIT_FSIZE, (0, 0)) + resource.setrlimit( + resource.RLIMIT_AS, + (GIT_WORKER_MEMORY_BYTES, GIT_WORKER_MEMORY_BYTES), + ) + resource.setrlimit(resource.RLIMIT_NOFILE, (16, 16)) + resource.setrlimit(resource.RLIMIT_CPU, (4, 4)) + os.environ.clear() + os.environ.update( + { + "HOME": "/nonexistent", + "LANG": "C", + "LC_ALL": "C", + "PATH": "/usr/bin:/bin", + } + ) + os.chdir("/") + os.umask(0o077) + try: + status, result_headers, body = _github_api_request( + authority, path, headers, maximum, deadline + ) + value: dict[str, object] = { + "body": base64.b64encode(body).decode("ascii"), + "headers": [list(item) for item in result_headers], + "kind": "result", + "status": status, + } + exit_code = 0 + except InvalidManifest: + value = {"kind": "reject"} + exit_code = 1 + except InfrastructureError: + value = {"kind": "infra"} + exit_code = 125 + except BaseException: + value = {"kind": "infra"} + exit_code = 125 + payload = json.dumps( + value, + ensure_ascii=True, + separators=(",", ":"), + sort_keys=True, + ).encode("ascii") + if len(payload) > GIT_WORKER_MAX_OUTPUT_BYTES: + payload = b'{"kind":"infra"}' + exit_code = 125 + frame = GIT_WORKER_FRAME + len(payload).to_bytes(8, "big") + payload + _git_worker_write(1, frame) + acknowledgement = os.read(3, 1) + if acknowledgement != b"1": + exit_code = 125 + except BaseException: + exit_code = 125 + os._exit(exit_code) + + +def _github_worker_request( + authority: str, + path: str, + headers: tuple[tuple[str, str], ...], + maximum: int, + deadline: float, +) -> tuple[int, tuple[tuple[str, str], ...], bytes]: + if time.monotonic() >= deadline: + raise InfrastructureError("git provider GIT-TIMEOUT deadline expired") + control_read, control_write = os.pipe2(getattr(os, "O_CLOEXEC", 0)) + stdout_read, stdout_write = os.pipe2(getattr(os, "O_CLOEXEC", 0)) + stderr_read, stderr_write = os.pipe2(getattr(os, "O_CLOEXEC", 0)) + pid = -1 + reaped = False + acknowledged = False + selector = selectors.DefaultSelector() + output = bytearray() + errors = bytearray() + process_status: int | None = None + failure: str | None = None + try: + pid = os.fork() + if pid == 0: + os.close(control_write) + os.close(stdout_read) + os.close(stderr_read) + _git_worker_child( + control_read, + stdout_write, + stderr_write, + authority, + path, + headers, + maximum, + deadline, + ) + os.close(control_read) + os.close(stdout_write) + os.close(stderr_write) + control_read = stdout_write = stderr_write = -1 + os.set_blocking(stdout_read, False) + os.set_blocking(stderr_read, False) + selector.register(stdout_read, selectors.EVENT_READ, "stdout") + selector.register(stderr_read, selectors.EVENT_READ, "stderr") + expected_frame: int | None = None + while True: + remaining = deadline - time.monotonic() + if remaining <= 0: + failure = "git provider GIT-TIMEOUT deadline expired" + break + for key, _ in selector.select(min(0.05, remaining)): + descriptor = int(key.fd) + try: + chunk = os.read(descriptor, 65_536) + except BlockingIOError: + continue + if not chunk: + selector.unregister(descriptor) + continue + if key.data == "stdout": + output.extend(chunk) + if len(output) > len(GIT_WORKER_FRAME) + 8 + GIT_WORKER_MAX_OUTPUT_BYTES: + failure = "git provider GIT-OUTPUT worker output exceeded its bound" + break + header_size = len(GIT_WORKER_FRAME) + 8 + if expected_frame is None and len(output) >= header_size: + if not output.startswith(GIT_WORKER_FRAME): + failure = "git provider GIT-WORKER frame is malformed" + break + payload_size = int.from_bytes( + output[len(GIT_WORKER_FRAME) : header_size], "big" + ) + if payload_size > GIT_WORKER_MAX_OUTPUT_BYTES: + failure = "git provider GIT-OUTPUT worker frame exceeded its bound" + break + expected_frame = header_size + payload_size + if expected_frame is not None and len(output) > expected_frame: + failure = "git provider GIT-WORKER frame has trailing data" + break + else: + errors.extend(chunk) + if len(errors) > GIT_WORKER_MAX_ERROR_BYTES: + failure = "git provider GIT-OUTPUT worker error exceeded its bound" + break + if failure is not None: + break + if expected_frame is not None and len(output) == expected_frame and not acknowledged: + if errors: + failure = "git provider GIT-WORKER emitted unexpected diagnostics" + break + _git_worker_write(control_write, b"1") + os.close(control_write) + control_write = -1 + acknowledged = True + try: + waited, status = os.waitpid(pid, os.WNOHANG) + except ChildProcessError: + waited = pid + status = 125 << 8 + if waited == pid: + reaped = True + process_status = status + if reaped and acknowledged and _git_worker_group_alive(pid): + failure = "git provider GIT-WORKER left a residual process group" + break + if reaped and not selector.get_map(): + break + if reaped and not acknowledged: + failure = "git provider GIT-WORKER exited before a complete frame" + break + if failure is None and not reaped: + remaining = max(0.0, deadline - time.monotonic()) + wait_deadline = time.monotonic() + min(GIT_WORKER_GRACE_SECONDS, remaining) + while time.monotonic() < wait_deadline: + waited, status = os.waitpid(pid, os.WNOHANG) + if waited == pid: + reaped = True + process_status = status + break + time.sleep(0.01) + if not reaped: + failure = "git provider GIT-WORKER did not exit after its result" + if failure is None and _git_worker_group_alive(pid): + failure = "git provider GIT-WORKER left a residual process group" + if failure is not None: + raise InfrastructureError(failure) + if process_status is None or not os.WIFEXITED(process_status): + raise InfrastructureError("git provider GIT-WORKER exit status is uncertain") + returncode = os.WEXITSTATUS(process_status) + header_size = len(GIT_WORKER_FRAME) + 8 + if len(output) < header_size: + raise InfrastructureError("git provider GIT-WORKER frame is incomplete") + payload = bytes(output[header_size:]) + try: + value = strict_json(payload, source="git provider worker frame") + except InvalidManifest as error: + raise InfrastructureError("git provider GIT-WORKER frame is malformed") from error + if value == {"kind": "reject"} and returncode == 1: + _git_reject("GIT-PROVIDER", "request was rejected") + if value == {"kind": "infra"} and returncode == 125: + raise InfrastructureError("git provider GIT-WORKER request was uncertain") + if ( + returncode != 0 + or not isinstance(value, dict) + or set(value) != {"body", "headers", "kind", "status"} + or value.get("kind") != "result" + or not isinstance(value.get("status"), int) + or isinstance(value.get("status"), bool) + or not isinstance(value.get("headers"), list) + or not isinstance(value.get("body"), str) + ): + raise InfrastructureError("git provider GIT-WORKER result is malformed") + result_headers: list[tuple[str, str]] = [] + for item in value["headers"]: + if ( + not isinstance(item, list) + or len(item) != 2 + or not all(isinstance(part, str) for part in item) + ): + raise InfrastructureError("git provider GIT-WORKER headers are malformed") + result_headers.append((item[0], item[1])) + encoded_body = value["body"] + assert isinstance(encoded_body, str) + if not encoded_body.isascii() or len(encoded_body) > ((maximum + 2) // 3) * 4: + raise InfrastructureError("git provider GIT-OUTPUT worker body exceeded its bound") + try: + body = base64.b64decode(encoded_body, validate=True) + except (ValueError, base64.binascii.Error) as error: + raise InfrastructureError("git provider GIT-WORKER body is malformed") from error + if len(body) > maximum: + raise InfrastructureError("git provider GIT-OUTPUT worker body exceeded its bound") + return int(value["status"]), tuple(result_headers), body + except InfrastructureError: + raise + except (OSError, ValueError) as error: + raise InfrastructureError("git provider GIT-WORKER could not establish a result") from error + finally: + selector.close() + for descriptor in ( + control_read, + control_write, + stdout_read, + stdout_write, + stderr_read, + stderr_write, + ): + if descriptor >= 0: + try: + os.close(descriptor) + except OSError: + pass + if pid > 0 and (not reaped or _git_worker_group_alive(pid)): + if not _git_worker_terminate(pid, reaped=reaped): + raise InfrastructureError("git provider GIT-WORKER cleanup is uncertain") + + +def _read_git_snapshot_with_requester( url: str, commit: str, - *, - requester: GitRequester | None = None, + requester: GitRequester, ) -> SourceSnapshot: """Acquire one exact public GitHub commit through the bounded Git Data API.""" if sys.platform != "linux": raise InfrastructureError("git source GIT-PLATFORM requires Linux") canonical, owner, repository, requested_commit = _parse_git_source(url, commit) - if requester is None: - raise InfrastructureError("git provider GIT-TRANSPORT runner is unavailable") deadline = time.monotonic() + GIT_ACQUISITION_TIMEOUT_SECONDS acquired = 0 @@ -483,6 +991,24 @@ def read_git_snapshot( ) +def read_git_snapshot( + url: str, + commit: str, + *, + requester: GitRequester | None = None, +) -> SourceSnapshot: + """Acquire one exact public GitHub commit through the bounded Git Data API.""" + + if sys.platform != "linux": + raise InfrastructureError("git source GIT-PLATFORM requires Linux") + canonical, _, _, requested_commit = _parse_git_source(url, commit) + if requester is not None: + return _read_git_snapshot_with_requester(canonical, requested_commit, requester) + return _read_git_snapshot_with_requester( + canonical, requested_commit, _github_worker_request + ) + + def _zip_reject(code: str, detail: str) -> NoReturn: raise InvalidManifest(f"zip archive {code} {detail}") diff --git a/tests/experiment/git-intake-cases.py b/tests/experiment/git-intake-cases.py index 96eabd0..17e8ec7 100755 --- a/tests/experiment/git-intake-cases.py +++ b/tests/experiment/git-intake-cases.py @@ -12,6 +12,7 @@ import os from pathlib import Path import signal +import socket import sys import tempfile @@ -479,6 +480,7 @@ def unused_requester(_authority, path, _headers, _maximum, _deadline): expected_headers = ( ("Accept", "application/vnd.github+json"), + ("Accept-Encoding", "identity"), ("User-Agent", "agent-lab/v0alpha1"), ("X-GitHub-Api-Version", "2022-11-28"), ) @@ -676,8 +678,10 @@ def worker_requester(authority, path, headers, maximum, deadline): ) ) - read_descriptor, write_descriptor = os.pipe() - os.set_blocking(read_descriptor, False) + residual_listener = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + residual_listener.bind(("127.0.0.1", 0)) + residual_listener.settimeout(1.0) + residual_address = residual_listener.getsockname() residual_spawned = False residual_pid = None residual_calls = 0 @@ -688,9 +692,12 @@ def residual_requester(authority, path, headers, maximum, deadline): if residual_calls == 1: child = os.fork() if child == 0: - os.close(read_descriptor) try: - os.write(write_descriptor, f"{os.getpid()}\n".encode("ascii")) + with socket.socket(socket.AF_INET, socket.SOCK_DGRAM) as sender: + sender.sendto( + f"{os.getpid()}\n".encode("ascii"), + residual_address, + ) while True: signal.pause() finally: @@ -706,17 +713,16 @@ def residual_requester(authority, path, headers, maximum, deadline): except experiment.InfrastructureError as error: residual_outcome = str(error) finally: - os.close(write_descriptor) if original_worker_requester is None: delattr(experiment, "_github_api_request") else: experiment._github_api_request = original_worker_requester try: - residual_record = os.read(read_descriptor, 64).decode("ascii").strip() - except BlockingIOError: + residual_record = residual_listener.recv(64).decode("ascii").strip() + except TimeoutError: residual_record = "" finally: - os.close(read_descriptor) + residual_listener.close() if residual_record.isdigit(): residual_spawned = True residual_pid = int(residual_record) From 266716f1e25db90c3f2ab2dc6f16c9f0a91d3a1a Mon Sep 17 00:00:00 2001 From: xormania <127287135+xormania@users.noreply.github.com> Date: Sun, 2 Aug 2026 08:09:57 -0400 Subject: [PATCH 132/158] test(experiment): distinguish bound Git object loss --- tests/experiment/git-intake-cases.py | 111 +++++++++++++++++++++++++-- 1 file changed, 103 insertions(+), 8 deletions(-) diff --git a/tests/experiment/git-intake-cases.py b/tests/experiment/git-intake-cases.py index 17e8ec7..835545c 100755 --- a/tests/experiment/git-intake-cases.py +++ b/tests/experiment/git-intake-cases.py @@ -503,19 +503,43 @@ def unused_requester(_authority, path, _headers, _maximum, _deadline): } prior_inherited = {name: os.environ.get(name) for name in inherited_names} os.environ.update(inherited_names) + original_worker_requester = getattr(experiment, "_github_api_request", None) + + def credential_requester(authority, path, headers, maximum, deadline): + if os.environ != { + "HOME": "/nonexistent", + "LANG": "C", + "LC_ALL": "C", + "PATH": "/usr/bin:/bin", + } or os.getcwd() != "/": + raise RuntimeError("worker authority was not isolated") + descriptors = {int(item) for item in os.listdir("/proc/self/fd")} + if any(descriptor > 4 for descriptor in descriptors): + raise RuntimeError("worker inherited an unrelated descriptor") + return fixture_responses[path] + + experiment._github_api_request = credential_requester try: - credential_outcome = acquire(fixture_responses) + try: + credential_snapshot = experiment.read_git_snapshot(URL, COMMIT) + except (experiment.InvalidManifest, experiment.InfrastructureError): + credential_snapshot = None finally: + if original_worker_requester is None: + delattr(experiment, "_github_api_request") + else: + experiment._github_api_request = original_worker_requester for name, value in prior_inherited.items(): if value is None: os.environ.pop(name, None) else: os.environ[name] = value - credential_text = repr(credential_outcome) + credential_text = repr(credential_snapshot) results.append( ( "GIT-CREDENTIAL-001", - credential_outcome[0] == "ok" + credential_snapshot is not None + and credential_snapshot.data == source and "credential.invalid" not in credential_text and "caller-ca" not in credential_text and "askpass" not in credential_text, @@ -653,9 +677,51 @@ def slow_requester(authority, path, headers, maximum, deadline): ) original_worker_requester = getattr(experiment, "_github_api_request", None) + worker_parent_pid = os.getpid() + large_source = b"//" + (b"x" * (262_144 - 3)) + b"\n" + large_blob = git_oid("blob", large_source) + large_tree = git_oid( + "tree", b"100644 experiment.cue\0" + bytes.fromhex(large_blob) + ) + large_responses = { + f"/repos/uscient/experiment-fixture/git/commits/{COMMIT}": response( + {"sha": COMMIT, "tree": {"sha": large_tree}} + ), + f"/repos/uscient/experiment-fixture/git/trees/{large_tree}": response( + { + "sha": large_tree, + "tree": [ + { + "mode": "100644", + "path": "experiment.cue", + "sha": large_blob, + "size": len(large_source), + "type": "blob", + } + ], + "truncated": False, + } + ), + f"/repos/uscient/experiment-fixture/git/blobs/{large_blob}": response( + { + "content": base64.b64encode(large_source).decode("ascii"), + "encoding": "base64", + "sha": large_blob, + "size": len(large_source), + } + ), + } def worker_requester(authority, path, headers, maximum, deadline): - return fixture_responses[path] + worker_pid = os.getpid() + if ( + worker_pid == worker_parent_pid + or os.getpgrp() != worker_pid + or os.getsid(0) != worker_pid + or os.getcwd() != "/" + ): + raise RuntimeError("provider worker session is not isolated") + return large_responses[path] experiment._github_api_request = worker_requester try: @@ -672,9 +738,11 @@ def worker_requester(authority, path, headers, maximum, deadline): ( "GIT-PGROUP-001", worker_snapshot is not None - and worker_snapshot.data == source - and worker_snapshot.digest == SOURCE_DIGEST, - "the fixed provider runs in the bounded worker path", + and worker_snapshot.data == large_source + and len(large_responses[ + f"/repos/uscient/experiment-fixture/git/blobs/{large_blob}" + ][2]) > 65_536, + "the fixed provider drains a large response in its own session", ) ) @@ -767,11 +835,38 @@ def residual_requester(authority, path, headers, maximum, deadline): b"{", ) taxonomy_outcomes.append((200, acquire(malformed_responses)[0])) + tree_missing = dict(fixture_responses) + tree_missing[f"/repos/uscient/experiment-fixture/git/trees/{TREE}"] = ( + 404, + ( + ("content-length", "2"), + ("content-type", "application/json; charset=utf-8"), + ), + b"{}", + ) + taxonomy_outcomes.append(("tree-404", acquire(tree_missing)[0])) + blob_missing = dict(fixture_responses) + blob_missing[f"/repos/uscient/experiment-fixture/git/blobs/{BLOB}"] = ( + 422, + ( + ("content-length", "2"), + ("content-type", "application/json; charset=utf-8"), + ), + b"{}", + ) + taxonomy_outcomes.append(("blob-422", acquire(blob_missing)[0])) results.append( ( "GIT-TAXONOMY-001", taxonomy_outcomes - == [(404, "reject"), (403, "infra"), (500, "infra"), (200, "infra")], + == [ + (404, "reject"), + (403, "infra"), + (500, "infra"), + (200, "infra"), + ("tree-404", "infra"), + ("blob-422", "infra"), + ], "stable absence is rejection while provider uncertainty is infrastructure", ) ) From 09292de2082b462fe4980c0459b41a1ea1aad1a8 Mon Sep 17 00:00:00 2001 From: xormania <127287135+xormania@users.noreply.github.com> Date: Sun, 2 Aug 2026 08:10:23 -0400 Subject: [PATCH 133/158] fix(experiment): classify bound Git object loss --- scripts/experiment.py | 34 +++++++++++++++++++++++++--------- 1 file changed, 25 insertions(+), 9 deletions(-) diff --git a/scripts/experiment.py b/scripts/experiment.py index 42ed772..df75a5d 100644 --- a/scripts/experiment.py +++ b/scripts/experiment.py @@ -306,6 +306,8 @@ def _git_provider_json( path: str, remaining: int, deadline: float, + *, + stable_not_found: bool, ) -> tuple[object, int]: if remaining <= 0: raise InfrastructureError("git provider GIT-ACQUIRE exhausted its response bound") @@ -325,12 +327,6 @@ def _git_provider_json( raise InfrastructureError("git provider GIT-TIMEOUT deadline expired") if not isinstance(status, int) or isinstance(status, bool): raise InfrastructureError("git provider GIT-STATUS response is malformed") - if status in (404, 422): - _git_reject("GIT-NOTFOUND", "does not expose the requested public object") - if 300 <= status <= 399: - raise InfrastructureError("git provider GIT-REDIRECT response is not accepted") - if status != 200: - raise InfrastructureError("git provider GIT-STATUS did not establish a result") if not isinstance(raw_headers, tuple): raise InfrastructureError("git provider GIT-HEADER response is malformed") headers: dict[str, str] = {} @@ -358,6 +354,14 @@ def _git_provider_json( or declared_length != len(body) ): raise InfrastructureError("git provider GIT-OUTPUT response exceeded its bound") + if status in (404, 422): + if stable_not_found: + _git_reject("GIT-NOTFOUND", "does not expose the requested public object") + raise InfrastructureError("git provider GIT-DRIFT bound object disappeared") + if 300 <= status <= 399: + raise InfrastructureError("git provider GIT-REDIRECT response is not accepted") + if status != 200: + raise InfrastructureError("git provider GIT-STATUS did not establish a result") try: value = strict_json(body, source="git provider response") except InvalidManifest as error: @@ -879,7 +883,11 @@ def _read_git_snapshot_with_requester( commit_path = f"/repos/{owner}/{repository}/git/commits/{requested_commit}" commit_value, used = _git_provider_json( - requester, commit_path, MAX_ARCHIVE_BYTES - acquired, deadline + requester, + commit_path, + MAX_ARCHIVE_BYTES - acquired, + deadline, + stable_not_found=True, ) acquired += used if not isinstance(commit_value, dict) or commit_value.get("sha") != requested_commit: @@ -893,7 +901,11 @@ def _read_git_snapshot_with_requester( tree_path = f"/repos/{owner}/{repository}/git/trees/{tree_id}" tree_value, used = _git_provider_json( - requester, tree_path, MAX_ARCHIVE_BYTES - acquired, deadline + requester, + tree_path, + MAX_ARCHIVE_BYTES - acquired, + deadline, + stable_not_found=False, ) acquired += used if ( @@ -931,7 +943,11 @@ def _read_git_snapshot_with_requester( blob_path = f"/repos/{owner}/{repository}/git/blobs/{blob_id}" blob_value, used = _git_provider_json( - requester, blob_path, MAX_ARCHIVE_BYTES - acquired, deadline + requester, + blob_path, + MAX_ARCHIVE_BYTES - acquired, + deadline, + stable_not_found=False, ) acquired += used if ( From b3038b65874606d6acf3c057b5859ae94372aa64 Mon Sep 17 00:00:00 2001 From: xormania <127287135+xormania@users.noreply.github.com> Date: Sun, 2 Aug 2026 08:11:23 -0400 Subject: [PATCH 134/158] test(experiment): preserve signals after Git cleanup --- tests/experiment/git-intake-cases.py | 76 +++++++++++++++++++++++++++- 1 file changed, 74 insertions(+), 2 deletions(-) diff --git a/tests/experiment/git-intake-cases.py b/tests/experiment/git-intake-cases.py index 835545c..cbdb6ca 100755 --- a/tests/experiment/git-intake-cases.py +++ b/tests/experiment/git-intake-cases.py @@ -803,13 +803,85 @@ def residual_requester(authority, path, headers, maximum, deadline): os.kill(residual_pid, signal.SIGKILL) else: residual_alive = False + + signal_listener = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + signal_listener.bind(("127.0.0.1", 0)) + signal_listener.settimeout(1.0) + signal_address = signal_listener.getsockname() + signal_probe = os.fork() + if signal_probe == 0: + def hanging_requester(authority, path, headers, maximum, deadline): + with socket.socket(socket.AF_INET, socket.SOCK_DGRAM) as sender: + sender.sendto( + f"{os.getpid()}\n".encode("ascii"), + signal_address, + ) + signal.signal(signal.SIGTERM, signal.SIG_IGN) + while True: + signal.pause() + + experiment._github_api_request = hanging_requester + try: + experiment.read_git_snapshot(URL, COMMIT) + except SystemExit as error: + code = error.code if isinstance(error.code, int) else 125 + os._exit(code) + except BaseException: + os._exit(125) + os._exit(0) + try: + try: + worker_record = signal_listener.recv(64).decode("ascii").strip() + except TimeoutError: + worker_record = "" + finally: + signal_listener.close() + signal_worker_pid = int(worker_record) if worker_record.isdigit() else None + if signal_worker_pid is not None: + os.kill(signal_probe, signal.SIGTERM) + signal_status = None + signal_deadline = __import__("time").monotonic() + 2.0 + while __import__("time").monotonic() < signal_deadline: + waited, status = os.waitpid(signal_probe, os.WNOHANG) + if waited == signal_probe: + signal_status = status + break + __import__("time").sleep(0.01) + if signal_status is None: + try: + os.kill(signal_probe, signal.SIGKILL) + except ProcessLookupError: + pass + _, signal_status = os.waitpid(signal_probe, 0) + signal_worker_alive = False + if signal_worker_pid is not None: + worker_deadline = __import__("time").monotonic() + 1.0 + while __import__("time").monotonic() < worker_deadline: + try: + os.killpg(signal_worker_pid, 0) + except ProcessLookupError: + break + __import__("time").sleep(0.01) + else: + signal_worker_alive = True + try: + os.killpg(signal_worker_pid, signal.SIGKILL) + except ProcessLookupError: + signal_worker_alive = False + signal_preserved = ( + signal_worker_pid is not None + and os.WIFEXITED(signal_status) + and os.WEXITSTATUS(signal_status) == 128 + signal.SIGTERM + and not signal_worker_alive + ) results.append( ( "GIT-CLEANUP-001", residual_spawned and not residual_alive - and "residual" in residual_outcome.lower(), - "a residual provider descendant is killed and reported as uncertainty", + and "residual" in residual_outcome.lower() + and signal_preserved, + "residual descendants are killed before caller signals are preserved", ) ) From 87bdac44e276d7eea15156b3e2475bffce268e6a Mon Sep 17 00:00:00 2001 From: xormania <127287135+xormania@users.noreply.github.com> Date: Sun, 2 Aug 2026 08:12:18 -0400 Subject: [PATCH 135/158] fix(experiment): preserve Git worker signals --- scripts/experiment.py | 108 ++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 103 insertions(+), 5 deletions(-) diff --git a/scripts/experiment.py b/scripts/experiment.py index df75a5d..12a046e 100644 --- a/scripts/experiment.py +++ b/scripts/experiment.py @@ -681,9 +681,54 @@ def _github_worker_request( ) -> tuple[int, tuple[tuple[str, str], ...], bytes]: if time.monotonic() >= deadline: raise InfrastructureError("git provider GIT-TIMEOUT deadline expired") - control_read, control_write = os.pipe2(getattr(os, "O_CLOEXEC", 0)) - stdout_read, stdout_write = os.pipe2(getattr(os, "O_CLOEXEC", 0)) - stderr_read, stderr_write = os.pipe2(getattr(os, "O_CLOEXEC", 0)) + interrupted: int | None = None + handlers: dict[int, object] = {} + managed_signals: set[int] = set() + + def interrupt(signum: int, _frame: object) -> None: + nonlocal interrupted + if interrupted is None: + interrupted = signum + + def change_mask(how: int, signals: set[int]) -> set[signal.Signals]: + try: + return set(signal.pthread_sigmask(how, signals)) + except (AttributeError, OSError, ValueError) as error: + raise InfrastructureError("git provider GIT-SIGNAL mask is unavailable") from error + + def record_pending(signals: set[int]) -> None: + nonlocal interrupted + try: + while True: + pending = set(signal.sigpending()).intersection(signals) + if not pending: + return + for signum in sorted(pending, key=int): + received = int(signal.sigwait({signum})) + if interrupted is None: + interrupted = received + except (AttributeError, OSError, ValueError) as error: + raise InfrastructureError( + "git provider GIT-SIGNAL pending state is uncertain" + ) from error + + original_mask = change_mask(signal.SIG_BLOCK, set()) + try: + for signum in (signal.SIGHUP, signal.SIGINT, signal.SIGQUIT, signal.SIGTERM): + previous = signal.getsignal(signum) + if previous == signal.SIG_IGN or signum in original_mask: + continue + handlers[signum] = previous + signal.signal(signum, interrupt) + managed_signals.add(signum) + except (OSError, ValueError) as error: + for signum, previous in handlers.items(): + signal.signal(signum, previous) + raise InfrastructureError("git provider GIT-SIGNAL handlers are unavailable") from error + + control_read = control_write = -1 + stdout_read = stdout_write = -1 + stderr_read = stderr_write = -1 pid = -1 reaped = False acknowledged = False @@ -693,7 +738,20 @@ def _github_worker_request( process_status: int | None = None failure: str | None = None try: - pid = os.fork() + control_read, control_write = os.pipe2(getattr(os, "O_CLOEXEC", 0)) + stdout_read, stdout_write = os.pipe2(getattr(os, "O_CLOEXEC", 0)) + stderr_read, stderr_write = os.pipe2(getattr(os, "O_CLOEXEC", 0)) + spawn_mask = change_mask(signal.SIG_BLOCK, managed_signals) + try: + if interrupted is None: + pid = os.fork() + finally: + if pid != 0: + change_mask(signal.SIG_SETMASK, set(spawn_mask)) + if pid < 0: + raise SystemExit(128 + interrupted) if interrupted is not None else InfrastructureError( + "git provider GIT-WORKER did not start" + ) if pid == 0: os.close(control_write) os.close(stdout_read) @@ -718,6 +776,9 @@ def _github_worker_request( selector.register(stderr_read, selectors.EVENT_READ, "stderr") expected_frame: int | None = None while True: + if interrupted is not None: + failure = "git provider GIT-SIGNAL interrupted acquisition" + break remaining = deadline - time.monotonic() if remaining <= 0: failure = "git provider GIT-TIMEOUT deadline expired" @@ -786,6 +847,9 @@ def _github_worker_request( remaining = max(0.0, deadline - time.monotonic()) wait_deadline = time.monotonic() + min(GIT_WORKER_GRACE_SECONDS, remaining) while time.monotonic() < wait_deadline: + if interrupted is not None: + failure = "git provider GIT-SIGNAL interrupted acquisition" + break waited, status = os.waitpid(pid, os.WNOHANG) if waited == pid: reaped = True @@ -849,6 +913,7 @@ def _github_worker_request( except (OSError, ValueError) as error: raise InfrastructureError("git provider GIT-WORKER could not establish a result") from error finally: + cleanup_error: InfrastructureError | None = None selector.close() for descriptor in ( control_read, @@ -865,7 +930,40 @@ def _github_worker_request( pass if pid > 0 and (not reaped or _git_worker_group_alive(pid)): if not _git_worker_terminate(pid, reaped=reaped): - raise InfrastructureError("git provider GIT-WORKER cleanup is uncertain") + cleanup_error = InfrastructureError( + "git provider GIT-WORKER cleanup is uncertain" + ) + cleanup_mask: set[signal.Signals] | None = None + try: + cleanup_mask = change_mask(signal.SIG_BLOCK, managed_signals) + record_pending(managed_signals) + except InfrastructureError as error: + if cleanup_error is None: + cleanup_error = error + finally: + for signum, previous in handlers.items(): + try: + signal.signal(signum, previous) + except (OSError, ValueError): + if cleanup_error is None: + cleanup_error = InfrastructureError( + "git provider GIT-SIGNAL handlers could not be restored" + ) + if cleanup_mask is not None: + try: + record_pending(managed_signals) + except InfrastructureError as error: + if cleanup_error is None: + cleanup_error = error + try: + change_mask(signal.SIG_SETMASK, set(cleanup_mask)) + except InfrastructureError as error: + if cleanup_error is None: + cleanup_error = error + if interrupted is not None: + raise SystemExit(128 + interrupted) + if cleanup_error is not None: + raise cleanup_error def _read_git_snapshot_with_requester( From ef49350502b44ac0728eb6b2a9bb5291e46390ed Mon Sep 17 00:00:00 2001 From: xormania <127287135+xormania@users.noreply.github.com> Date: Sun, 2 Aug 2026 08:20:10 -0400 Subject: [PATCH 136/158] test(experiment): require common Git intake pipeline --- tests/experiment/git-intake-cases.py | 675 +++++++++++++++++++++++++++ tests/experiment/git-intake-cases.sh | 2 +- 2 files changed, 676 insertions(+), 1 deletion(-) diff --git a/tests/experiment/git-intake-cases.py b/tests/experiment/git-intake-cases.py index cbdb6ca..6116cc6 100755 --- a/tests/experiment/git-intake-cases.py +++ b/tests/experiment/git-intake-cases.py @@ -11,8 +11,11 @@ import json import os from pathlib import Path +import shutil import signal import socket +import stat +import subprocess import sys import tempfile @@ -40,6 +43,16 @@ "GIT-PGROUP-001", "GIT-CLEANUP-001", "GIT-TAXONOMY-001", + "GIT-CHECK-001", + "GIT-AUTH-001", + "GIT-DENY-001", + "GIT-INSTALL-001", + "GIT-IDENTITY-001", + "GIT-RETRY-001", + "GIT-ADAPTER-001", + "GIT-NOEF-001", + "GIT-RUNTIME-001", + "GIT-DIAG-001", ) URL = "https://github.com/uscient/experiment-fixture.git" COMMIT = "1cffa1a28f96d2f2cb898b1bad70d281e359a5b5" @@ -81,6 +94,56 @@ def invoke(module, argv: list[str]) -> tuple[int, str, str]: return result, stdout.getvalue(), stderr.getvalue() +def invoke_with_exit(module, argv: list[str]) -> tuple[int, str, str]: + stdout = io.StringIO() + stderr = io.StringIO() + with redirect_stdout(stdout), redirect_stderr(stderr): + try: + result = module.main(argv) + except SystemExit as error: + result = error.code if isinstance(error.code, int) else 1 + return result, stdout.getvalue(), stderr.getvalue() + + +def tree_fingerprint(root: Path) -> str: + records: list[tuple[str, str, int, str]] = [] + if root.exists(): + for path in sorted( + root.rglob("*"), key=lambda item: os.fsencode(str(item.relative_to(root))) + ): + metadata = path.lstat() + relative = str(path.relative_to(root)) + mode = stat.S_IMODE(metadata.st_mode) + if stat.S_ISREG(metadata.st_mode): + kind = "file" + content = sha256(path.read_bytes()).hexdigest() + elif stat.S_ISDIR(metadata.st_mode): + kind = "directory" + content = "" + elif stat.S_ISLNK(metadata.st_mode): + kind = "symlink" + content = os.readlink(path) + else: + kind = "other" + content = "" + records.append((relative, kind, mode, content)) + return sha256(repr(records).encode("utf-8")).hexdigest() + + +def make_writable(root: Path) -> None: + if not root.exists(): + return + for path in root.rglob("*"): + try: + metadata = path.lstat() + if stat.S_ISDIR(metadata.st_mode): + path.chmod(stat.S_IMODE(metadata.st_mode) | stat.S_IRWXU) + elif stat.S_ISREG(metadata.st_mode): + path.chmod(stat.S_IMODE(metadata.st_mode) | stat.S_IRUSR | stat.S_IWUSR) + except OSError: + pass + + def main() -> int: repo = Path(__file__).resolve().parents[2] failures = 0 @@ -942,6 +1005,618 @@ def hanging_requester(authority, path, headers, maximum, deadline): "stable absence is rejection while provider uncertainty is infrastructure", ) ) + + fixture_directory = ( + repo / "tests/experiment/fixtures/directories/minimal" + ) + zip_fixture_module = load_module( + repo / "tests/experiment/zip-fixtures.py", + "git_intake_zip_fixture", + ) + common_zip = Path(raw_home) / "common-source.zip" + common_zip.write_bytes(zip_fixture_module.one("experiment.cue", source)) + directory_snapshot = experiment.read_directory_snapshot( + str(fixture_directory) + ) + zip_snapshot = experiment.read_zip_snapshot(str(common_zip)) + git_snapshot = snapshot + + prior_tools = { + "AGENT_LAB_CUE_TOOL_DIR": os.environ.get( + "AGENT_LAB_CUE_TOOL_DIR" + ), + "AGENT_LAB_CEDAR_TOOL_DIR": os.environ.get( + "AGENT_LAB_CEDAR_TOOL_DIR" + ), + } + os.environ["AGENT_LAB_CUE_TOOL_DIR"] = str( + repo / ".cache/dev/tools/cue" + ) + os.environ["AGENT_LAB_CEDAR_TOOL_DIR"] = str( + repo / ".cache/dev/tools/cedar" + ) + + identity_ready = git_snapshot is not None + directory_manifest = None + zip_manifest = None + git_manifest = None + directory_resolution = None + zip_resolution = None + git_resolution = None + directory_decision = None + zip_decision = None + git_decision = None + if identity_ready: + try: + directory_manifest = experiment.authored_manifest( + directory_snapshot + ) + zip_manifest = experiment.authored_manifest(zip_snapshot) + git_manifest = experiment.authored_manifest(git_snapshot) + directory_resolution = experiment.cue_plan_with_evidence( + directory_manifest + ) + zip_resolution = experiment.cue_plan_with_evidence(zip_manifest) + git_resolution = experiment.cue_plan_with_evidence(git_manifest) + directory_decision = experiment.authorize_plan( + directory_resolution.plan, directory_snapshot.digest + )[0] + zip_decision = experiment.authorize_plan( + zip_resolution.plan, zip_snapshot.digest + )[0] + git_decision = experiment.authorize_plan( + git_resolution.plan, git_snapshot.digest + )[0] + except (AttributeError, OSError, RuntimeError): + identity_ready = False + + preview_home = Path(raw_home) / "preview-home" + preview_init = invoke( + agent_lab, ["--home", str(preview_home), "init"] + ) + preview_before = tree_fingerprint(preview_home) + source_before = sha256(source).hexdigest() + original_agent_experiment_module = agent_lab.experiment_module + original_read_git_snapshot = experiment.read_git_snapshot + original_authored_manifest = experiment.authored_manifest + original_cue_plan = experiment.cue_plan_with_evidence + original_authorize_plan = experiment.authorize_plan + original_write_checked = experiment.write_checked_source + original_write_decision = experiment.write_decision + active_operation = [""] + operation_log: list[tuple[str, str]] = [] + snapshot_handoffs: list[bool] = [] + checked_values: list[object] = [] + decision_values: list[object] = [] + + def injected_git_snapshot(url, commit): + operation_log.append((active_operation[0], "snapshot")) + if url != URL or commit != COMMIT or git_snapshot is None: + raise experiment.InfrastructureError( + "git source GIT-TEST fixture is unavailable" + ) + return git_snapshot + + def logged_manifest(value): + operation_log.append((active_operation[0], "manifest")) + snapshot_handoffs.append(value is git_snapshot) + return original_authored_manifest(value) + + def logged_plan(value): + operation_log.append((active_operation[0], "plan")) + return original_cue_plan(value) + + def logged_authorize(value, digest): + operation_log.append((active_operation[0], "authorize")) + return original_authorize_plan(value, digest) + + experiment.read_git_snapshot = injected_git_snapshot + experiment.authored_manifest = logged_manifest + experiment.cue_plan_with_evidence = logged_plan + experiment.authorize_plan = logged_authorize + experiment.write_checked_source = checked_values.append + experiment.write_decision = decision_values.append + agent_lab.experiment_module = lambda: experiment + try: + active_operation[0] = "check" + checked_result = invoke_with_exit( + agent_lab, + [ + "--home", + str(preview_home), + "experiment", + "check", + "--git", + URL, + "--commit", + COMMIT, + ], + ) + active_operation[0] = "authorize" + authorized_result = invoke_with_exit( + agent_lab, + [ + "--home", + str(preview_home), + "experiment", + "authorize", + "install", + "--git", + URL, + "--commit", + COMMIT, + ], + ) + finally: + agent_lab.experiment_module = original_agent_experiment_module + experiment.read_git_snapshot = original_read_git_snapshot + experiment.authored_manifest = original_authored_manifest + experiment.cue_plan_with_evidence = original_cue_plan + experiment.authorize_plan = original_authorize_plan + experiment.write_checked_source = original_write_checked + experiment.write_decision = original_write_decision + + expected_checked = None + if identity_ready and git_resolution is not None: + expected_checked = { + "digest": experiment.plan_digest(git_resolution.plan), + "plan": git_resolution.plan, + "source": { + "digest": git_snapshot.digest, + **git_snapshot.transport, + }, + } + catalog = experiment.catalog_resolution_evidence( + git_resolution.bundled_catalog, + git_resolution.local_catalog, + ) + if catalog is not None: + expected_checked["catalog"] = catalog + results.append( + ( + "GIT-CHECK-001", + checked_result == (0, "", "") + and checked_values == [expected_checked], + "public Git check uses the common checked-source path", + ) + ) + results.append( + ( + "GIT-AUTH-001", + authorized_result == (0, "", "") + and identity_ready + and decision_values == [directory_decision] + and directory_decision == zip_decision == git_decision, + "Git authorization uses the common source-bound Cedar decision", + ) + ) + + expected_log = [ + ("check", "snapshot"), + ("check", "manifest"), + ("check", "plan"), + ("authorize", "snapshot"), + ("authorize", "manifest"), + ("authorize", "plan"), + ("authorize", "authorize"), + ] + adapter_downstream_calls: list[str] = [] + + def forbidden_adapter_downstream(*_args, **_kwargs): + adapter_downstream_calls.append("reached") + raise AssertionError("Git adapter crossed into the common pipeline") + + experiment.authored_manifest = forbidden_adapter_downstream + experiment.cue_plan_with_evidence = forbidden_adapter_downstream + experiment.authorize_plan = forbidden_adapter_downstream + try: + try: + adapter_snapshot = original_read_git_snapshot( + URL, COMMIT, requester=requester + ) + except (experiment.InvalidManifest, experiment.InfrastructureError): + adapter_snapshot = None + finally: + experiment.authored_manifest = original_authored_manifest + experiment.cue_plan_with_evidence = original_cue_plan + experiment.authorize_plan = original_authorize_plan + results.append( + ( + "GIT-ADAPTER-001", + identity_ready + and operation_log == expected_log + and snapshot_handoffs == [True, True] + and adapter_snapshot == git_snapshot + and adapter_downstream_calls == [], + "Git acquisition returns one snapshot to each unchanged common pipeline", + ) + ) + preview_after = tree_fingerprint(preview_home) + results.append( + ( + "GIT-NOEF-001", + preview_init[0] == 0 + and checked_result[0] == 0 + and authorized_result[0] == 0 + and preview_before == preview_after + and source_before == sha256(source).hexdigest() + and operation_log == expected_log, + "Git previews leave initialized home and source bytes unchanged", + ) + ) + + hostile_marker = b"caller-private-diagnostic" + + def hostile_read(url, commit): + def hostile_requester( + _authority, _path, _headers, _maximum, _deadline + ): + return ( + 500, + ( + ("content-length", str(len(hostile_marker))), + ( + "content-type", + "application/json; charset=utf-8", + ), + ), + hostile_marker, + ) + + return original_read_git_snapshot( + url, commit, requester=hostile_requester + ) + + experiment.read_git_snapshot = hostile_read + agent_lab.experiment_module = lambda: experiment + try: + diagnostic_result = invoke_with_exit( + agent_lab, + [ + "--home", + str(preview_home), + "experiment", + "check", + "--git", + URL, + "--commit", + COMMIT, + ], + ) + finally: + experiment.read_git_snapshot = original_read_git_snapshot + agent_lab.experiment_module = original_agent_experiment_module + results.append( + ( + "GIT-DIAG-001", + diagnostic_result[0] == 125 + and diagnostic_result[1] == "" + and "GIT-STATUS" in diagnostic_result[2] + and hostile_marker.decode("ascii") not in diagnostic_result[2], + "hostile provider diagnostics never reach public output", + ) + ) + + store = load_module( + repo / "scripts/experiment_store.py", "git_intake_store" + ) + original_store_experiment_module = store._experiment_module + + class ExperimentFacade: + def __init__(self, *, deny: bool = False): + self.deny = deny + self.acquisitions: list[tuple[str, str]] = [] + + def __getattr__(self, name): + return getattr(experiment, name) + + def read_git_snapshot(self, url, commit): + self.acquisitions.append((url, commit)) + if url != URL or commit != COMMIT or git_snapshot is None: + raise experiment.InfrastructureError( + "git source GIT-TEST fixture is unavailable" + ) + return git_snapshot + + def authorize_plan(self, plan, digest): + decision, status = original_authorize_plan(plan, digest) + if not self.deny: + return decision, status + denied = json.loads( + json.dumps( + decision, + ensure_ascii=True, + separators=(",", ":"), + sort_keys=True, + ) + ) + denied["verdict"] = "deny" + return denied, 1 + + def initialized_home(name: str) -> tuple[Path, bool]: + home = Path(raw_home) / name + outcome = invoke(agent_lab, ["--home", str(home), "init"]) + return home, outcome[0] == 0 and outcome[2] == "" + + def store_call(name: str, *arguments): + operation = getattr(store, name, None) + if not callable(operation): + return "missing", None + try: + return "ok", operation(*arguments) + except store.StoreReject as error: + return "reject", str(error) + except store.StoreInfrastructure as error: + return "infra", str(error) + except Exception as error: + return "error", type(error).__name__ + + deny_home, deny_home_ready = initialized_home("git-deny-home") + deny_before = tree_fingerprint(deny_home) + deny_facade = ExperimentFacade(deny=True) + store._experiment_module = lambda: deny_facade + deny_outcome = store_call("install_git", deny_home, URL, COMMIT) + deny_after = tree_fingerprint(deny_home) + results.append( + ( + "GIT-DENY-001", + deny_home_ready + and deny_outcome[0] == "reject" + and "authorization denied" in str(deny_outcome[1]) + and deny_facade.acquisitions == [(URL, COMMIT)] + and deny_before == deny_after, + "denied Git install leaves the initialized home byte-identical", + ) + ) + + install_home, install_home_ready = initialized_home("git-install-home") + install_facade = ExperimentFacade() + store._experiment_module = lambda: install_facade + install_outcome = store_call( + "install_git", install_home, URL, COMMIT + ) + installed_root = install_home / "experiments/first-experiment" + installed_provenance = None + installed_artifact = None + if install_outcome[0] == "ok": + try: + installed_provenance = json.loads( + (installed_root / "records/provenance.json").read_text( + encoding="utf-8" + ) + ) + installed_artifact = ( + installed_root / "artifact/experiment.cue" + ).read_bytes() + except (OSError, UnicodeError, ValueError): + installed_provenance = None + installed_artifact = None + closed_git_provenance = ( + isinstance(installed_provenance, dict) + and set(installed_provenance) + == { + "apiVersion", + "authorizationDigest", + "catalog", + "contractDigest", + "kind", + "planDigest", + "selectedEntries", + "source", + "transport", + } + and installed_provenance.get("source") + == { + "bytes": len(source), + "digest": SOURCE_DIGEST, + "entryCount": 1, + "fileCount": 1, + "format": "agent-lab.experiment-tree/v1", + "kind": "directory", + } + and installed_provenance.get("transport") == expected_transport + ) + results.append( + ( + "GIT-INSTALL-001", + install_home_ready + and install_outcome[0] == "ok" + and isinstance(install_outcome[1], dict) + and install_outcome[1].get("changed") is True + and install_facade.acquisitions == [(URL, COMMIT)] + and installed_artifact == source + and closed_git_provenance, + "permitted Git install publishes the common artifact and closed provenance", + ) + ) + + directory_home, directory_home_ready = initialized_home( + "identity-directory-home" + ) + zip_home, zip_home_ready = initialized_home("identity-zip-home") + store._experiment_module = lambda: ExperimentFacade() + directory_install = store_call( + "install_directory", directory_home, fixture_directory + ) + zip_install = store_call("install_zip", zip_home, common_zip) + + def installed_bytes(home: Path, relative: str) -> bytes | None: + try: + return ( + home / "experiments/first-experiment" / relative + ).read_bytes() + except OSError: + return None + + directory_key = ( + directory_install[1].get("installationKey") + if directory_install[0] == "ok" + and isinstance(directory_install[1], dict) + else None + ) + zip_key = ( + zip_install[1].get("installationKey") + if zip_install[0] == "ok" and isinstance(zip_install[1], dict) + else None + ) + git_key = ( + install_outcome[1].get("installationKey") + if install_outcome[0] == "ok" + and isinstance(install_outcome[1], dict) + else None + ) + semantic_identity = ( + identity_ready + and directory_snapshot.digest + == zip_snapshot.digest + == git_snapshot.digest + == SOURCE_DIGEST + and directory_manifest == zip_manifest == git_manifest + and directory_resolution.plan + == zip_resolution.plan + == git_resolution.plan + and directory_decision == zip_decision == git_decision + ) + stored_identity = ( + directory_home_ready + and zip_home_ready + and directory_install[0] == "ok" + and zip_install[0] == "ok" + and install_outcome[0] == "ok" + and directory_key == zip_key == git_key + and directory_key is not None + and installed_bytes(directory_home, "artifact/experiment.cue") + == installed_bytes(zip_home, "artifact/experiment.cue") + == installed_bytes(install_home, "artifact/experiment.cue") + == source + and installed_bytes(directory_home, "records/plan.json") + == installed_bytes(zip_home, "records/plan.json") + == installed_bytes(install_home, "records/plan.json") + and installed_bytes(directory_home, "records/decision.json") + == installed_bytes(zip_home, "records/decision.json") + == installed_bytes(install_home, "records/decision.json") + ) + results.append( + ( + "GIT-IDENTITY-001", + semantic_identity and stored_identity, + "directory, ZIP, and Git share manifest, plan, Cedar, key, and artifact identity", + ) + ) + + retry_home, retry_home_ready = initialized_home("git-retry-home") + retry_facade = ExperimentFacade() + store._experiment_module = lambda: retry_facade + retry_directory = store_call( + "install_directory", retry_home, fixture_directory + ) + retry_receipt_path = ( + retry_home + / "experiments/first-experiment/records/install.json" + ) + retry_receipt_first = installed_bytes(retry_home, "records/install.json") + retry_zip = store_call("install_zip", retry_home, common_zip) + retry_receipt_second = installed_bytes(retry_home, "records/install.json") + retry_git = store_call("install_git", retry_home, URL, COMMIT) + retry_receipt_third = installed_bytes(retry_home, "records/install.json") + results.append( + ( + "GIT-RETRY-001", + retry_home_ready + and retry_directory[0] == "ok" + and retry_zip[0] == "ok" + and retry_git[0] == "ok" + and isinstance(retry_zip[1], dict) + and isinstance(retry_git[1], dict) + and retry_zip[1].get("changed") is False + and retry_git[1].get("changed") is False + and retry_facade.acquisitions == [(URL, COMMIT)] + and retry_receipt_path.is_file() + and retry_receipt_first + == retry_receipt_second + == retry_receipt_third, + "second and third equivalent transports preserve the first receipt", + ) + ) + store._experiment_module = original_store_experiment_module + + runtime_manifest = repo / "packaging/agent-lab-local.manifest" + expected_runtime = ( + repo / "tests/install/fixtures/expected-runtime-files.txt" + ) + runtime_root = Path(raw_home) / "installed-runtime" + runtime_ready = runtime_manifest.read_bytes() == expected_runtime.read_bytes() + runtime_names = expected_runtime.read_text(encoding="utf-8").splitlines() + for runtime_name in runtime_names: + source_path = repo / runtime_name + target_path = runtime_root / runtime_name + if not runtime_name or not source_path.is_file(): + runtime_ready = False + continue + target_path.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(source_path, target_path) + unrelated = Path(raw_home) / "unrelated-cwd" + runtime_home = Path(raw_home) / "runtime-home" + runtime_tmp = Path(raw_home) / "runtime-tmp" + unrelated.mkdir() + runtime_tmp.mkdir() + runtime_environment = { + "PATH": "/usr/bin:/bin", + "HOME": str(Path(raw_home) / "runtime-user-home"), + "TMPDIR": str(runtime_tmp), + "LC_ALL": "C", + "AGENT_LAB_CUE_TOOL_DIR": str(repo / ".cache/dev/tools/cue"), + "AGENT_LAB_CEDAR_TOOL_DIR": str( + repo / ".cache/dev/tools/cedar" + ), + } + runtime_command = [ + str(runtime_root / "scripts/agent-lab"), + "--home", + str(runtime_home), + "experiment", + "check", + "--git", + "https://example.com/uscient/experiment-fixture.git", + "--commit", + COMMIT, + ] + try: + runtime_result = subprocess.run( + runtime_command, + cwd=unrelated, + env=runtime_environment, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + timeout=5, + check=False, + ) + except (OSError, subprocess.SubprocessError): + runtime_result = None + results.append( + ( + "GIT-RUNTIME-001", + runtime_ready + and runtime_result is not None + and runtime_result.returncode == 1 + and runtime_result.stdout == b"" + and b"GIT-URL" in runtime_result.stderr + and str(repo).encode("utf-8") not in runtime_result.stderr + and not any(runtime_root.rglob("__pycache__")), + "installed CLI reaches Git validation from a minimal unrelated runtime", + ) + ) + + cycle4_results = {item[0]: item for item in results[-10:]} + del results[-10:] + results.extend(cycle4_results[item] for item in EXPECTED[-10:]) + + for name, value in prior_tools.items(): + if value is None: + os.environ.pop(name, None) + else: + os.environ[name] = value + make_writable(Path(raw_home)) finally: if prior_home is None: os.environ.pop("AGENT_LAB_HOME", None) diff --git a/tests/experiment/git-intake-cases.sh b/tests/experiment/git-intake-cases.sh index f920880..5303b68 100755 --- a/tests/experiment/git-intake-cases.sh +++ b/tests/experiment/git-intake-cases.sh @@ -5,7 +5,7 @@ repo_root="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." >/dev/null 2>&1 && cases="$repo_root/tests/experiment/git-intake-cases.py" if [ ! -f "$cases" ] || ! command -v python3 >/dev/null 2>&1; then - printf 'SUMMARY assertions=0 expected=22 failures=0 infra=1\n' + printf 'SUMMARY assertions=0 expected=32 failures=0 infra=1\n' exit 125 fi From af5574b30b9b2776c737951f2301fea22aaf1a6c Mon Sep 17 00:00:00 2001 From: xormania <127287135+xormania@users.noreply.github.com> Date: Sun, 2 Aug 2026 08:22:31 -0400 Subject: [PATCH 137/158] feat(experiment): install pinned Git snapshots --- scripts/agent-lab.py | 19 ++++++- scripts/experiment.py | 17 +++++- scripts/experiment_store.py | 110 ++++++++++++++++++++++++++++++++++-- 3 files changed, 136 insertions(+), 10 deletions(-) diff --git a/scripts/agent-lab.py b/scripts/agent-lab.py index 6e582c4..9b22d34 100644 --- a/scripts/agent-lab.py +++ b/scripts/agent-lab.py @@ -506,9 +506,14 @@ def image_command(home: Path, argv: list[str]) -> int: def experiment_command(home: Path, argv: list[str]) -> int: - if argv == ["install", "--zip"]: + if argv in (["install", "--zip"], ["install", "--git"]): return 2 - if argv[:1] == ["install"] and len(argv) == 2: + if argv[:2] == ["install", "--git"]: + if len(argv) != 5 or argv[3] != "--commit": + return 2 + operation = "install" + source_kind = "git" + elif argv[:1] == ["install"] and len(argv) == 2: operation = "install" source_kind = "directory" elif argv[:2] == ["install", "--zip"] and len(argv) == 3: @@ -548,7 +553,9 @@ def experiment_command(home: Path, argv: list[str]) -> int: try: if operation == "install": - if source_kind == "zip": + if source_kind == "git": + result = store.install_git(home, argv[2], argv[4]) + elif source_kind == "zip": result = store.install_zip(home, Path(argv[2])) else: result = store.install_directory(home, Path(argv[1])) @@ -637,6 +644,9 @@ def main(argv: list[str]) -> int: if argv[:3] == ["experiment", "check", "--git"]: if len(argv) != 6 or argv[4] != "--commit": return 2 + if sys.platform != "linux": + print("INFRA Agent Lab Git Experiment intake requires Linux", file=sys.stderr) + return 125 os.environ["AGENT_LAB_HOME"] = str(home) os.environ.setdefault("AGENT_LAB_CUE_TOOL_DIR", str(home / "cache/tools/cue")) return experiment_module().main( @@ -645,6 +655,9 @@ def main(argv: list[str]) -> int: if argv[:4] == ["experiment", "authorize", "install", "--git"]: if len(argv) != 7 or argv[5] != "--commit": return 2 + if sys.platform != "linux": + print("INFRA Agent Lab Git Experiment intake requires Linux", file=sys.stderr) + return 125 os.environ["AGENT_LAB_HOME"] = str(home) os.environ.setdefault("AGENT_LAB_CUE_TOOL_DIR", str(home / "cache/tools/cue")) os.environ.setdefault("AGENT_LAB_CEDAR_TOOL_DIR", str(home / "cache/tools/cedar")) diff --git a/scripts/experiment.py b/scripts/experiment.py index 12a046e..5e9116c 100644 --- a/scripts/experiment.py +++ b/scripts/experiment.py @@ -2613,16 +2613,27 @@ def main(argv: list[str]) -> int: directory_authorizing = len(argv) == 3 and argv[1] == "authorize-directory" zip_checking = len(argv) == 3 and argv[1] == "check-zip" zip_authorizing = len(argv) == 3 and argv[1] == "authorize-zip" - if directory_checking or directory_authorizing or zip_checking or zip_authorizing: + git_checking = len(argv) == 4 and argv[1] == "check-git" + git_authorizing = len(argv) == 4 and argv[1] == "authorize-git" + if ( + directory_checking + or directory_authorizing + or zip_checking + or zip_authorizing + or git_checking + or git_authorizing + ): try: - if zip_checking or zip_authorizing: + if git_checking or git_authorizing: + snapshot = read_git_snapshot(argv[2], argv[3]) + elif zip_checking or zip_authorizing: snapshot = read_zip_snapshot(argv[2]) else: snapshot = read_directory_snapshot(argv[2]) manifest = authored_manifest(snapshot) resolution = cue_plan_with_evidence(manifest) plan = resolution.plan - if directory_checking or zip_checking: + if directory_checking or zip_checking or git_checking: checked: dict[str, object] = { "digest": plan_digest(plan), "plan": plan, diff --git a/scripts/experiment_store.py b/scripts/experiment_store.py index 804c927..1082d1d 100644 --- a/scripts/experiment_store.py +++ b/scripts/experiment_store.py @@ -45,6 +45,13 @@ EXPERIMENT_NAME = re.compile(r"^[a-z](?:[a-z0-9-]{0,61}[a-z0-9])?$") IMAGE_COMPONENT = re.compile(r"^[a-z][a-z0-9]*(?:-[a-z0-9]+)*$") SHA256 = re.compile(r"^sha256:[0-9a-f]{64}$") +GIT_SHA1 = re.compile(r"^sha1:[0-9a-f]{40}$") +GIT_COMMIT = re.compile(r"^[0-9a-f]{40}$") +GIT_URL = re.compile( + r"^https://github\.com/" + r"[a-z0-9](?:[a-z0-9-]{0,37}[a-z0-9])?/" + r"[a-z0-9_.-]{1,100}\.git$" +) PAYLOAD_DIRECTORIES = {"payload", "payload/artifact", "payload/records"} PAYLOAD_FILES = { "payload/artifact/experiment.cue", @@ -847,10 +854,74 @@ def _candidate( return files, receipt, installation_key, digest(RECEIPT_DOMAIN + canonical(receipt)) +def _closed_git_transport(transport: object) -> dict[str, object] | None: + if not isinstance(transport, dict) or transport.get("kind") != "git": + return None + if set(transport) != { + "acquisition", + "blob", + "commit", + "kind", + "requestedCommit", + "tree", + "url", + }: + return None + acquisition = transport.get("acquisition") + requested = transport.get("requestedCommit") + if ( + not isinstance(acquisition, dict) + or set(acquisition) + != { + "acquiredBytes", + "limitBytes", + "method", + "requestCount", + "temporaryBytes", + "temporaryFiles", + } + or not isinstance(acquisition.get("acquiredBytes"), int) + or isinstance(acquisition.get("acquiredBytes"), bool) + or not 1 <= int(acquisition["acquiredBytes"]) <= MAX_ARCHIVE_BYTES + or acquisition.get("limitBytes") != MAX_ARCHIVE_BYTES + or acquisition.get("method") != "github-git-data-v3" + or acquisition.get("requestCount") != 3 + or acquisition.get("temporaryBytes") != 0 + or acquisition.get("temporaryFiles") != 0 + or not isinstance(requested, str) + or GIT_COMMIT.fullmatch(requested) is None + or transport.get("commit") != f"sha1:{requested}" + or GIT_SHA1.fullmatch(str(transport.get("tree"))) is None + or GIT_SHA1.fullmatch(str(transport.get("blob"))) is None + or not isinstance(transport.get("url"), str) + or GIT_URL.fullmatch(str(transport["url"])) is None + ): + return None + return { + "acquisition": { + "acquiredBytes": acquisition["acquiredBytes"], + "limitBytes": MAX_ARCHIVE_BYTES, + "method": "github-git-data-v3", + "requestCount": 3, + "temporaryBytes": 0, + "temporaryFiles": 0, + }, + "blob": transport["blob"], + "commit": transport["commit"], + "kind": "git", + "requestedCommit": requested, + "tree": transport["tree"], + "url": transport["url"], + } + + def _source_transport(snapshot: object) -> dict[str, object]: transport = getattr(snapshot, "transport", None) if transport is None or transport == {"kind": "directory"}: return {"kind": "local-directory"} + git_transport = _closed_git_transport(transport) + if git_transport is not None: + return git_transport if ( not isinstance(transport, dict) or set(transport) != {"archiveBytes", "archiveDigest", "kind"} @@ -871,6 +942,8 @@ def _source_transport(snapshot: object) -> dict[str, object]: def _validate_transport_provenance(transport: object) -> None: if transport == {"kind": "local-directory"}: return + if _closed_git_transport(transport) is not None: + return if ( not isinstance(transport, dict) or set(transport) != {"archiveBytes", "archiveDigest", "kind"} @@ -1979,8 +2052,8 @@ def _held_catalog_context( def _install_source( home: Path, - source: Path, reader_name: str, + reader_arguments: tuple[str, ...], *, fault: FaultHook | None = None, ) -> dict[str, object]: @@ -1993,7 +2066,7 @@ def _install_source( try: try: reader = getattr(experiment, reader_name) - snapshot = reader(str(source)) + snapshot = reader(*reader_arguments) manifest = experiment.authored_manifest(snapshot) resolution = experiment.cue_plan_with_evidence(manifest) plan = resolution.plan @@ -2110,7 +2183,12 @@ def install_directory( if sys.platform != "linux": _infra("effectful Experiment installation requires Linux") - return _install_source(home, source, "read_directory_snapshot", fault=fault) + return _install_source( + home, + "read_directory_snapshot", + (str(source),), + fault=fault, + ) def install_zip( @@ -2123,7 +2201,31 @@ def install_zip( if sys.platform != "linux": _infra("effectful Experiment installation requires Linux") - return _install_source(home, source, "read_zip_snapshot", fault=fault) + return _install_source( + home, + "read_zip_snapshot", + (str(source),), + fault=fault, + ) + + +def install_git( + home: Path, + url: str, + commit: str, + *, + fault: FaultHook | None = None, +) -> dict[str, object]: + """Freshly validate/authorize one pinned Git snapshot and publish once.""" + + if sys.platform != "linux": + _infra("effectful Experiment installation requires Linux") + return _install_source( + home, + "read_git_snapshot", + (url, commit), + fault=fault, + ) def inspect_install(home: Path, name: str) -> dict[str, object]: From 1211fed28d0581dec6e8f5b4ffc11c57ed9cfdce Mon Sep 17 00:00:00 2001 From: xormania <127287135+xormania@users.noreply.github.com> Date: Sun, 2 Aug 2026 08:33:21 -0400 Subject: [PATCH 138/158] test(experiment): harden Git provider worker --- tests/experiment/git-intake-cases.py | 706 ++++++++++++++++++++++++--- 1 file changed, 636 insertions(+), 70 deletions(-) diff --git a/tests/experiment/git-intake-cases.py b/tests/experiment/git-intake-cases.py index 6116cc6..6975020 100755 --- a/tests/experiment/git-intake-cases.py +++ b/tests/experiment/git-intake-cases.py @@ -547,13 +547,209 @@ def unused_requester(_authority, path, _headers, _maximum, _deadline): ("User-Agent", "agent-lab/v0alpha1"), ("X-GitHub-Api-Version", "2022-11-28"), ) + + ca_bytes = b"-----BEGIN CERTIFICATE-----\nfixture\n-----END CERTIFICATE-----\n" + ca_offset = [0] + ca_closed: list[int] = [] + original_ca_paths = experiment.GIT_PROVIDER_CA_FILES + original_open = experiment.os.open + original_fstat = experiment.os.fstat + original_read = experiment.os.read + original_close = experiment.os.close + + class CaMetadata: + st_mode = stat.S_IFREG | 0o644 + st_uid = 0 + st_nlink = 1 + st_size = len(ca_bytes) + + def ca_open(path, _flags): + if path == "/missing-ca": + raise FileNotFoundError(path) + if path != "/trusted-ca": + raise AssertionError("unexpected CA path") + return 73 + + def ca_fstat(descriptor): + if descriptor != 73: + raise AssertionError("unexpected CA descriptor") + return CaMetadata() + + def ca_read(descriptor, maximum): + if descriptor != 73: + raise AssertionError("unexpected CA descriptor") + start = ca_offset[0] + chunk = ca_bytes[start : start + min(maximum, 11)] + ca_offset[0] += len(chunk) + return chunk + + def ca_close(descriptor): + ca_closed.append(descriptor) + + experiment.GIT_PROVIDER_CA_FILES = ("/missing-ca", "/trusted-ca") + experiment.os.open = ca_open + experiment.os.fstat = ca_fstat + experiment.os.read = ca_read + experiment.os.close = ca_close + try: + try: + ca_pem = experiment._git_system_ca_pem() + except experiment.InfrastructureError: + ca_pem = None + finally: + experiment.GIT_PROVIDER_CA_FILES = original_ca_paths + experiment.os.open = original_open + experiment.os.fstat = original_fstat + experiment.os.read = original_read + experiment.os.close = original_close + ca_probe_ok = ca_pem == ca_bytes.decode("ascii") and ca_closed == [73] + + import http.client as http_client + import ssl as ssl_module + + direct_body = b'{"fixture":true}' + direct_connections = [] + original_https_connection = http_client.HTTPSConnection + original_ssl_context = ssl_module.SSLContext + original_maxline = http_client._MAXLINE + original_maxheaders = http_client._MAXHEADERS + original_ca_reader = experiment._git_system_ca_pem + + class FakeSocket: + def __init__(self): + self.timeouts = [] + + def settimeout(self, value): + self.timeouts.append(value) + + class FakeResponse: + status = 200 + + def __init__(self): + self.offset = 0 + self.closed = False + + def getheaders(self): + return [ + ("Content-Length", str(len(direct_body))), + ("Content-Type", "application/json; charset=utf-8"), + ("Content-Encoding", "identity"), + ] + + def read(self, maximum): + if self.offset >= len(direct_body): + return b"" + chunk = direct_body[self.offset : self.offset + maximum] + self.offset += len(chunk) + return chunk + + def close(self): + self.closed = True + + class FakeContext: + def __init__(self, protocol): + self.protocol = protocol + self.check_hostname = None + self.verify_mode = None + self.minimum_version = None + self.ca_data = None + + def load_verify_locations(self, *, cadata): + self.ca_data = cadata + + class FakeConnection: + def __init__(self, authority, *, port, timeout, context): + self.authority = authority + self.port = port + self.timeout = timeout + self.context = context + self.sock = FakeSocket() + self.response = FakeResponse() + self.request_record = None + self.closed = False + direct_connections.append(self) + + def request(self, method, path, *, headers): + self.request_record = (method, path, headers) + + def getresponse(self): + return self.response + + def close(self): + self.closed = True + + http_client.HTTPSConnection = FakeConnection + ssl_module.SSLContext = FakeContext + experiment._git_system_ca_pem = lambda: ca_bytes.decode("ascii") + try: + direct_deadline = __import__("time").monotonic() + 1.0 + direct_https = experiment._github_api_request( + "api.github.com", + f"/repos/uscient/experiment-fixture/git/commits/{COMMIT}", + expected_headers, + 1_024, + direct_deadline, + ) + try: + experiment._github_api_request( + "credential.invalid", + "/repos/uscient/experiment-fixture/git/commits/invalid", + expected_headers, + 1_024, + direct_deadline, + ) + direct_invalid = "accepted" + except experiment.InfrastructureError as error: + direct_invalid = str(error) + finally: + http_client.HTTPSConnection = original_https_connection + ssl_module.SSLContext = original_ssl_context + http_client._MAXLINE = original_maxline + http_client._MAXHEADERS = original_maxheaders + experiment._git_system_ca_pem = original_ca_reader + direct_connection = ( + direct_connections[0] if len(direct_connections) == 1 else None + ) + direct_https_ok = ( + direct_https + == ( + 200, + ( + ("content-length", str(len(direct_body))), + ("content-type", "application/json; charset=utf-8"), + ), + direct_body, + ) + and direct_connection is not None + and direct_connection.authority == "api.github.com" + and direct_connection.port == 443 + and direct_connection.request_record + == ( + "GET", + f"/repos/uscient/experiment-fixture/git/commits/{COMMIT}", + dict(expected_headers), + ) + and direct_connection.context.protocol + == ssl_module.PROTOCOL_TLS_CLIENT + and direct_connection.context.check_hostname is True + and direct_connection.context.verify_mode == ssl_module.CERT_REQUIRED + and direct_connection.context.minimum_version + == ssl_module.TLSVersion.TLSv1_2 + and direct_connection.context.ca_data == ca_bytes.decode("ascii") + and direct_connection.response.closed + and direct_connection.closed + and direct_connection.sock.timeouts + and all(value > 0 for value in direct_connection.sock.timeouts) + and "GIT-AUTHORITY" in direct_invalid + ) results.append( ( "GIT-AUTHORITY-001", all(call[0] == "api.github.com" for call in request_calls) and all(call[2] == expected_headers for call in request_calls) and request_calls[0][3] == 1_048_576 - and request_calls[0][3] > request_calls[1][3] > request_calls[2][3], + and request_calls[0][3] > request_calls[1][3] > request_calls[2][3] + and direct_https_ok, "only the fixed credential-free provider authority is requested", ) ) @@ -603,6 +799,7 @@ def credential_requester(authority, path, headers, maximum, deadline): "GIT-CREDENTIAL-001", credential_snapshot is not None and credential_snapshot.data == source + and ca_probe_ok and "credential.invalid" not in credential_text and "caller-ca" not in credential_text and "askpass" not in credential_text, @@ -693,10 +890,40 @@ def slow_requester(authority, path, headers, maximum, deadline): timeout_outcome = str(error) finally: experiment.GIT_ACQUISITION_TIMEOUT_SECONDS = original_timeout + + original_worker_requester = experiment._github_api_request + + def hanging_worker_requester( + _authority, _path, _headers, _maximum, _deadline + ): + while True: + signal.pause() + + experiment._github_api_request = hanging_worker_requester + worker_timeout_started = __import__("time").monotonic() + try: + try: + experiment._github_worker_request( + "api.github.com", + f"/repos/uscient/experiment-fixture/git/commits/{COMMIT}", + expected_headers, + 1_024, + worker_timeout_started + 0.05, + ) + worker_timeout_outcome = "ok" + except experiment.InfrastructureError as error: + worker_timeout_outcome = str(error) + finally: + experiment._github_api_request = original_worker_requester + worker_timeout_elapsed = ( + __import__("time").monotonic() - worker_timeout_started + ) results.append( ( "GIT-TIMEOUT-001", - "GIT-TIMEOUT" in timeout_outcome, + "GIT-TIMEOUT" in timeout_outcome + and "GIT-TIMEOUT" in worker_timeout_outcome + and worker_timeout_elapsed < 1.0, "one absolute acquisition deadline bounds all provider requests", ) ) @@ -711,11 +938,89 @@ def slow_requester(authority, path, headers, maximum, deadline): b"x" * 1_048_577, ) output_outcome = acquire(oversized_responses) + + valid_worker_value = json.dumps( + { + "body": "", + "headers": [ + ["content-length", "0"], + ["content-type", "application/json; charset=utf-8"], + ], + "kind": "result", + "status": 200, + }, + ensure_ascii=True, + separators=(",", ":"), + sort_keys=True, + ).encode("ascii") + valid_worker_frame = ( + experiment.GIT_WORKER_FRAME + + len(valid_worker_value).to_bytes(8, "big") + + valid_worker_value + ) + original_worker_child = experiment._git_worker_child + + def worker_frame_outcome(stdout_data: bytes, stderr_data: bytes = b""): + def fixture_worker_child( + _control_read, + stdout_write, + stderr_write, + _authority, + _path, + _headers, + _maximum, + _deadline, + ): + try: + os.setsid() + if stderr_data: + os.write(stderr_write, stderr_data) + if stdout_data: + os.write(stdout_write, stdout_data) + finally: + os._exit(0) + + experiment._git_worker_child = fixture_worker_child + try: + try: + experiment._github_worker_request( + "api.github.com", + f"/repos/uscient/experiment-fixture/git/commits/{COMMIT}", + expected_headers, + 1_024, + __import__("time").monotonic() + 1.0, + ) + return "ok" + except experiment.InfrastructureError as error: + return str(error) + finally: + experiment._git_worker_child = original_worker_child + + malformed_frame = worker_frame_outcome(b"not-a-worker-frame") + oversized_frame = worker_frame_outcome( + experiment.GIT_WORKER_FRAME + + (experiment.GIT_WORKER_MAX_OUTPUT_BYTES + 1).to_bytes(8, "big") + ) + trailing_frame = worker_frame_outcome(valid_worker_frame + b"trailing") + stderr_frame = worker_frame_outcome( + valid_worker_frame, b"caller-private-diagnostic" + ) + worker_frames_rejected = ( + "GIT-WORKER" in malformed_frame + and "GIT-OUTPUT" in oversized_frame + and "trailing" in trailing_frame + and ( + "diagnostic" in stderr_frame + or "GIT-WORKER" in stderr_frame + ) + ) results.append( ( "GIT-OUTPUT-001", - output_outcome[0] == "infra" and "GIT-OUTPUT" in output_outcome[1], - "one provider response cannot exceed the acquisition cap", + output_outcome[0] == "infra" + and "GIT-OUTPUT" in output_outcome[1] + and worker_frames_rejected, + "provider and worker output frames are strictly bounded", ) ) @@ -776,12 +1081,23 @@ def slow_requester(authority, path, headers, maximum, deadline): } def worker_requester(authority, path, headers, maximum, deadline): + import resource + worker_pid = os.getpid() if ( worker_pid == worker_parent_pid or os.getpgrp() != worker_pid or os.getsid(0) != worker_pid or os.getcwd() != "/" + or resource.getrlimit(resource.RLIMIT_CORE) != (0, 0) + or resource.getrlimit(resource.RLIMIT_FSIZE) != (0, 0) + or resource.getrlimit(resource.RLIMIT_AS) + != ( + experiment.GIT_WORKER_MEMORY_BYTES, + experiment.GIT_WORKER_MEMORY_BYTES, + ) + or resource.getrlimit(resource.RLIMIT_NOFILE) != (16, 16) + or resource.getrlimit(resource.RLIMIT_CPU) != (4, 4) ): raise RuntimeError("provider worker session is not isolated") return large_responses[path] @@ -797,18 +1113,214 @@ def worker_requester(authority, path, headers, maximum, deadline): delattr(experiment, "_github_api_request") else: experiment._github_api_request = original_worker_requester + + terminate_wait_options: list[int] = [] + original_waitpid = experiment.os.waitpid + original_killpg = experiment.os.killpg + original_group_alive = experiment._git_worker_group_alive + original_monotonic = experiment.time.monotonic + original_sleep = experiment.time.sleep + terminate_clock = [0.0] + + def terminate_monotonic(): + terminate_clock[0] += 1.0 + return terminate_clock[0] + + def terminate_waitpid(_pid, options): + terminate_wait_options.append(options) + if options == 0: + raise OSError("blocking wait forbidden by fixture") + return 0, 0 + + experiment.os.waitpid = terminate_waitpid + experiment.os.killpg = lambda _pid, _signal: None + experiment._git_worker_group_alive = lambda _pid: True + experiment.time.monotonic = terminate_monotonic + experiment.time.sleep = lambda _duration: None + try: + terminate_result = experiment._git_worker_terminate( + 991_337, reaped=False + ) + finally: + experiment.os.waitpid = original_waitpid + experiment.os.killpg = original_killpg + experiment._git_worker_group_alive = original_group_alive + experiment.time.monotonic = original_monotonic + experiment.time.sleep = original_sleep + nonblocking_terminate = ( + terminate_result is False + and terminate_wait_options + and all( + option == os.WNOHANG for option in terminate_wait_options + ) + ) + + mask_failure_marker = Path(raw_home) / "mask-clear-requester-reached" + mask_parent_pid = os.getpid() + original_pthread_sigmask = experiment.signal.pthread_sigmask + original_worker_requester = experiment._github_api_request + + def fail_child_mask(how, signals): + if ( + os.getpid() != mask_parent_pid + and how == signal.SIG_SETMASK + and not signals + ): + raise OSError("child mask clear failed") + return original_pthread_sigmask(how, signals) + + def mask_failure_requester( + _authority, path, _headers, _maximum, _deadline + ): + mask_failure_marker.touch() + return fixture_responses[path] + + experiment.signal.pthread_sigmask = fail_child_mask + experiment._github_api_request = mask_failure_requester + try: + try: + experiment._github_worker_request( + "api.github.com", + f"/repos/uscient/experiment-fixture/git/commits/{COMMIT}", + expected_headers, + 1_024, + __import__("time").monotonic() + 1.0, + ) + mask_failure_outcome = "ok" + except experiment.InfrastructureError as error: + mask_failure_outcome = str(error) + finally: + experiment.signal.pthread_sigmask = original_pthread_sigmask + experiment._github_api_request = original_worker_requester + mask_clear_fail_closed = ( + "GIT-WORKER" in mask_failure_outcome + and not mask_failure_marker.exists() + ) results.append( ( "GIT-PGROUP-001", worker_snapshot is not None and worker_snapshot.data == large_source + and nonblocking_terminate + and mask_clear_fail_closed and len(large_responses[ f"/repos/uscient/experiment-fixture/git/blobs/{large_blob}" ][2]) > 65_536, - "the fixed provider drains a large response in its own session", + "the fixed worker enforces limits, nonblocking cleanup, and a cleared mask", ) ) + def cleanup_fault_probe(kind: str) -> bool: + probe_pid = os.fork() + if probe_pid == 0: + cleanup_probe_process = os.getpid() + managed = ( + signal.SIGHUP, + signal.SIGINT, + signal.SIGQUIT, + signal.SIGTERM, + ) + original_handlers = { + signum: signal.getsignal(signum) for signum in managed + } + original_mask = set( + signal.pthread_sigmask(signal.SIG_BLOCK, set()) + ) + original_requester = experiment._github_api_request + original_group_probe = experiment._git_worker_group_alive + original_close = experiment.os.close + close_calls = [0] + + def cleanup_requester( + _authority, _path, _headers, _maximum, _deadline + ): + return ( + 200, + ( + ("content-length", "0"), + ( + "content-type", + "application/json; charset=utf-8", + ), + ), + b"", + ) + + def failed_group_probe(_pid): + raise OSError("process-group probe failed") + + def failed_cleanup_close(descriptor): + if os.getpid() == cleanup_probe_process: + close_calls[0] += 1 + if close_calls[0] == 5: + raise OSError("cleanup close failed") + return original_close(descriptor) + + experiment._github_api_request = cleanup_requester + if kind == "group": + experiment._git_worker_group_alive = failed_group_probe + elif kind == "close": + experiment.os.close = failed_cleanup_close + try: + try: + experiment._github_worker_request( + "api.github.com", + f"/repos/uscient/experiment-fixture/git/commits/{COMMIT}", + expected_headers, + 1_024, + __import__("time").monotonic() + 1.0, + ) + outcome = "ok" + except experiment.InfrastructureError: + outcome = "infra" + except BaseException: + outcome = "other" + restored = ( + all( + signal.getsignal(signum) + == original_handlers[signum] + for signum in managed + ) + and set( + signal.pthread_sigmask(signal.SIG_BLOCK, set()) + ) + == original_mask + ) + finally: + experiment._github_api_request = original_requester + experiment._git_worker_group_alive = original_group_probe + experiment.os.close = original_close + os._exit(0 if outcome == "infra" and restored else 1) + + probe_status = None + probe_deadline = __import__("time").monotonic() + 2.0 + while __import__("time").monotonic() < probe_deadline: + waited, status = os.waitpid(probe_pid, os.WNOHANG) + if waited == probe_pid: + probe_status = status + break + __import__("time").sleep(0.01) + if probe_status is None: + try: + os.kill(probe_pid, signal.SIGKILL) + except ProcessLookupError: + pass + reap_deadline = __import__("time").monotonic() + 1.0 + while __import__("time").monotonic() < reap_deadline: + waited, status = os.waitpid(probe_pid, os.WNOHANG) + if waited == probe_pid: + probe_status = status + break + __import__("time").sleep(0.01) + return ( + probe_status is not None + and os.WIFEXITED(probe_status) + and os.WEXITSTATUS(probe_status) == 0 + ) + + group_probe_fail_closed = cleanup_fault_probe("group") + close_failure_fail_closed = cleanup_fault_probe("close") + residual_listener = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) residual_listener.bind(("127.0.0.1", 0)) residual_listener.settimeout(1.0) @@ -867,75 +1379,112 @@ def residual_requester(authority, path, headers, maximum, deadline): else: residual_alive = False - signal_listener = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) - signal_listener.bind(("127.0.0.1", 0)) - signal_listener.settimeout(1.0) - signal_address = signal_listener.getsockname() - signal_probe = os.fork() - if signal_probe == 0: - def hanging_requester(authority, path, headers, maximum, deadline): - with socket.socket(socket.AF_INET, socket.SOCK_DGRAM) as sender: - sender.sendto( - f"{os.getpid()}\n".encode("ascii"), - signal_address, + def signal_cleanup_probe(*, uncertain: bool, expected_exit: int) -> bool: + signal_listener = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + signal_listener.bind(("127.0.0.1", 0)) + signal_listener.settimeout(1.0) + signal_address = signal_listener.getsockname() + signal_probe = os.fork() + if signal_probe == 0: + original_worker_terminate = experiment._git_worker_terminate + + def uncertain_worker_terminate(pid, *, reaped): + original_worker_terminate(pid, reaped=reaped) + return False + + def hanging_requester( + _authority, _path, _headers, _maximum, _deadline + ): + with socket.socket( + socket.AF_INET, socket.SOCK_DGRAM + ) as sender: + sender.sendto( + f"{os.getpid()}\n".encode("ascii"), + signal_address, + ) + signal.signal(signal.SIGTERM, signal.SIG_IGN) + while True: + signal.pause() + + if uncertain: + experiment._git_worker_terminate = ( + uncertain_worker_terminate ) - signal.signal(signal.SIGTERM, signal.SIG_IGN) - while True: - signal.pause() - - experiment._github_api_request = hanging_requester - try: - experiment.read_git_snapshot(URL, COMMIT) - except SystemExit as error: - code = error.code if isinstance(error.code, int) else 125 - os._exit(code) - except BaseException: - os._exit(125) - os._exit(0) - try: - try: - worker_record = signal_listener.recv(64).decode("ascii").strip() - except TimeoutError: - worker_record = "" - finally: - signal_listener.close() - signal_worker_pid = int(worker_record) if worker_record.isdigit() else None - if signal_worker_pid is not None: - os.kill(signal_probe, signal.SIGTERM) - signal_status = None - signal_deadline = __import__("time").monotonic() + 2.0 - while __import__("time").monotonic() < signal_deadline: - waited, status = os.waitpid(signal_probe, os.WNOHANG) - if waited == signal_probe: - signal_status = status - break - __import__("time").sleep(0.01) - if signal_status is None: + experiment._github_api_request = hanging_requester + try: + experiment.read_git_snapshot(URL, COMMIT) + except SystemExit as error: + code = error.code if isinstance(error.code, int) else 124 + os._exit(code) + except experiment.InfrastructureError: + os._exit(125) + except BaseException: + os._exit(124) + os._exit(0) try: - os.kill(signal_probe, signal.SIGKILL) - except ProcessLookupError: - pass - _, signal_status = os.waitpid(signal_probe, 0) - signal_worker_alive = False - if signal_worker_pid is not None: - worker_deadline = __import__("time").monotonic() + 1.0 - while __import__("time").monotonic() < worker_deadline: try: - os.killpg(signal_worker_pid, 0) - except ProcessLookupError: + worker_record = ( + signal_listener.recv(64).decode("ascii").strip() + ) + except TimeoutError: + worker_record = "" + finally: + signal_listener.close() + signal_worker_pid = ( + int(worker_record) if worker_record.isdigit() else None + ) + if signal_worker_pid is not None: + os.kill(signal_probe, signal.SIGTERM) + signal_status = None + signal_deadline = __import__("time").monotonic() + 2.0 + while __import__("time").monotonic() < signal_deadline: + waited, status = os.waitpid(signal_probe, os.WNOHANG) + if waited == signal_probe: + signal_status = status break __import__("time").sleep(0.01) - else: - signal_worker_alive = True + if signal_status is None: try: - os.killpg(signal_worker_pid, signal.SIGKILL) + os.kill(signal_probe, signal.SIGKILL) except ProcessLookupError: - signal_worker_alive = False - signal_preserved = ( - signal_worker_pid is not None - and os.WIFEXITED(signal_status) - and os.WEXITSTATUS(signal_status) == 128 + signal.SIGTERM - and not signal_worker_alive + pass + reap_deadline = __import__("time").monotonic() + 1.0 + while __import__("time").monotonic() < reap_deadline: + waited, status = os.waitpid(signal_probe, os.WNOHANG) + if waited == signal_probe: + signal_status = status + break + __import__("time").sleep(0.01) + signal_worker_alive = False + if signal_worker_pid is not None: + worker_deadline = __import__("time").monotonic() + 1.0 + while __import__("time").monotonic() < worker_deadline: + try: + os.killpg(signal_worker_pid, 0) + except ProcessLookupError: + break + __import__("time").sleep(0.01) + else: + signal_worker_alive = True + try: + os.killpg(signal_worker_pid, signal.SIGKILL) + except ProcessLookupError: + signal_worker_alive = False + return ( + signal_worker_pid is not None + and signal_status is not None + and os.WIFEXITED(signal_status) + and os.WEXITSTATUS(signal_status) == expected_exit + and not signal_worker_alive + ) + + clean_signal_preserved = signal_cleanup_probe( + uncertain=False, + expected_exit=128 + signal.SIGTERM, + ) + cleanup_uncertainty_wins = signal_cleanup_probe( + uncertain=True, + expected_exit=125, ) results.append( ( @@ -943,8 +1492,11 @@ def hanging_requester(authority, path, headers, maximum, deadline): residual_spawned and not residual_alive and "residual" in residual_outcome.lower() - and signal_preserved, - "residual descendants are killed before caller signals are preserved", + and clean_signal_preserved + and cleanup_uncertainty_wins + and group_probe_fail_closed + and close_failure_fail_closed, + "cleanup uncertainty fails closed before caller signals are restored", ) ) @@ -1301,6 +1853,19 @@ def hostile_requester( repo / "scripts/experiment_store.py", "git_intake_store" ) original_store_experiment_module = store._experiment_module + bool_transport_fields_rejected = True + for field in ( + "limitBytes", + "requestCount", + "temporaryBytes", + "temporaryFiles", + ): + hostile_transport = json.loads( + json.dumps(expected_transport, sort_keys=True) + ) + hostile_transport["acquisition"][field] = False + if store._closed_git_transport(hostile_transport) is not None: + bool_transport_fields_rejected = False class ExperimentFacade: def __init__(self, *, deny: bool = False): @@ -1425,7 +1990,8 @@ def store_call(name: str, *arguments): and install_outcome[1].get("changed") is True and install_facade.acquisitions == [(URL, COMMIT)] and installed_artifact == source - and closed_git_provenance, + and closed_git_provenance + and bool_transport_fields_rejected, "permitted Git install publishes the common artifact and closed provenance", ) ) From 5873177b6d98c680de944dd88a53802256a31495 Mon Sep 17 00:00:00 2001 From: xormania <127287135+xormania@users.noreply.github.com> Date: Sun, 2 Aug 2026 08:35:42 -0400 Subject: [PATCH 139/158] fix(experiment): close Git worker cleanup --- scripts/experiment.py | 89 ++++++++++++++++++++++++++++--------- scripts/experiment_store.py | 20 ++++++--- 2 files changed, 81 insertions(+), 28 deletions(-) diff --git a/scripts/experiment.py b/scripts/experiment.py index 5e9116c..3c8de62 100644 --- a/scripts/experiment.py +++ b/scripts/experiment.py @@ -541,17 +541,29 @@ def _git_worker_group_alive(pid: int) -> bool: os.killpg(pid, 0) except ProcessLookupError: return False - except PermissionError: - return True + except OSError as error: + raise InfrastructureError( + "git provider GIT-WORKER process-group state is uncertain" + ) from error return True def _git_worker_terminate(pid: int, *, reaped: bool) -> bool: + uncertain = False for signum in (signal.SIGTERM, signal.SIGKILL): try: os.killpg(pid, signum) except ProcessLookupError: pass + except OSError: + uncertain = True + if not reaped: + try: + os.kill(pid, signal.SIGKILL) + except ProcessLookupError: + pass + except OSError: + uncertain = True deadline = time.monotonic() + GIT_WORKER_GRACE_SECONDS while time.monotonic() < deadline: if not reaped: @@ -559,20 +571,33 @@ def _git_worker_terminate(pid: int, *, reaped: bool) -> bool: waited, _ = os.waitpid(pid, os.WNOHANG) except ChildProcessError: reaped = True + except OSError: + uncertain = True else: reaped = waited == pid - if reaped and not _git_worker_group_alive(pid): - return True + try: + group_alive = _git_worker_group_alive(pid) + except (InfrastructureError, OSError): + group_alive = True + uncertain = True + if reaped and not group_alive: + return not uncertain time.sleep(0.01) if not reaped: try: - os.waitpid(pid, 0) - reaped = True + waited, _ = os.waitpid(pid, os.WNOHANG) except ChildProcessError: reaped = True except OSError: - pass - return reaped and not _git_worker_group_alive(pid) + uncertain = True + else: + reaped = waited == pid + try: + group_alive = _git_worker_group_alive(pid) + except (InfrastructureError, OSError): + group_alive = True + uncertain = True + return reaped and not group_alive and not uncertain def _git_worker_write(descriptor: int, data: bytes) -> None: @@ -599,10 +624,7 @@ def _git_worker_child( os.setsid() for signum in (signal.SIGHUP, signal.SIGINT, signal.SIGQUIT, signal.SIGTERM): signal.signal(signum, signal.SIG_DFL) - try: - signal.pthread_sigmask(signal.SIG_SETMASK, set()) - except (AttributeError, OSError, ValueError): - pass + signal.pthread_sigmask(signal.SIG_SETMASK, set()) null_descriptor = os.open("/dev/null", os.O_RDONLY | getattr(os, "O_CLOEXEC", 0)) os.dup2(null_descriptor, 0) os.dup2(stdout_write, 1) @@ -914,7 +936,12 @@ def record_pending(signals: set[int]) -> None: raise InfrastructureError("git provider GIT-WORKER could not establish a result") from error finally: cleanup_error: InfrastructureError | None = None - selector.close() + try: + selector.close() + except (OSError, ValueError) as error: + cleanup_error = InfrastructureError( + "git provider GIT-WORKER selector cleanup is uncertain" + ) for descriptor in ( control_read, control_write, @@ -926,13 +953,31 @@ def record_pending(signals: set[int]) -> None: if descriptor >= 0: try: os.close(descriptor) - except OSError: - pass - if pid > 0 and (not reaped or _git_worker_group_alive(pid)): - if not _git_worker_terminate(pid, reaped=reaped): - cleanup_error = InfrastructureError( - "git provider GIT-WORKER cleanup is uncertain" - ) + except OSError as error: + if cleanup_error is None: + cleanup_error = InfrastructureError( + "git provider GIT-WORKER descriptor cleanup is uncertain" + ) + if pid > 0: + needs_termination = not reaped + if not needs_termination: + try: + needs_termination = _git_worker_group_alive(pid) + except (InfrastructureError, OSError): + needs_termination = True + if cleanup_error is None: + cleanup_error = InfrastructureError( + "git provider GIT-WORKER process-group cleanup is uncertain" + ) + if needs_termination: + try: + terminated = _git_worker_terminate(pid, reaped=reaped) + except (InfrastructureError, OSError, ValueError): + terminated = False + if not terminated and cleanup_error is None: + cleanup_error = InfrastructureError( + "git provider GIT-WORKER cleanup is uncertain" + ) cleanup_mask: set[signal.Signals] | None = None try: cleanup_mask = change_mask(signal.SIG_BLOCK, managed_signals) @@ -960,10 +1005,10 @@ def record_pending(signals: set[int]) -> None: except InfrastructureError as error: if cleanup_error is None: cleanup_error = error - if interrupted is not None: - raise SystemExit(128 + interrupted) if cleanup_error is not None: raise cleanup_error + if interrupted is not None: + raise SystemExit(128 + interrupted) def _read_git_snapshot_with_requester( diff --git a/scripts/experiment_store.py b/scripts/experiment_store.py index 1082d1d..8eb028a 100644 --- a/scripts/experiment_store.py +++ b/scripts/experiment_store.py @@ -869,6 +869,9 @@ def _closed_git_transport(transport: object) -> dict[str, object] | None: return None acquisition = transport.get("acquisition") requested = transport.get("requestedCommit") + tree = transport.get("tree") + blob = transport.get("blob") + url = transport.get("url") if ( not isinstance(acquisition, dict) or set(acquisition) @@ -880,21 +883,26 @@ def _closed_git_transport(transport: object) -> dict[str, object] | None: "temporaryBytes", "temporaryFiles", } - or not isinstance(acquisition.get("acquiredBytes"), int) - or isinstance(acquisition.get("acquiredBytes"), bool) + or type(acquisition.get("acquiredBytes")) is not int or not 1 <= int(acquisition["acquiredBytes"]) <= MAX_ARCHIVE_BYTES + or type(acquisition.get("limitBytes")) is not int or acquisition.get("limitBytes") != MAX_ARCHIVE_BYTES or acquisition.get("method") != "github-git-data-v3" + or type(acquisition.get("requestCount")) is not int or acquisition.get("requestCount") != 3 + or type(acquisition.get("temporaryBytes")) is not int or acquisition.get("temporaryBytes") != 0 + or type(acquisition.get("temporaryFiles")) is not int or acquisition.get("temporaryFiles") != 0 or not isinstance(requested, str) or GIT_COMMIT.fullmatch(requested) is None or transport.get("commit") != f"sha1:{requested}" - or GIT_SHA1.fullmatch(str(transport.get("tree"))) is None - or GIT_SHA1.fullmatch(str(transport.get("blob"))) is None - or not isinstance(transport.get("url"), str) - or GIT_URL.fullmatch(str(transport["url"])) is None + or not isinstance(tree, str) + or GIT_SHA1.fullmatch(tree) is None + or not isinstance(blob, str) + or GIT_SHA1.fullmatch(blob) is None + or not isinstance(url, str) + or GIT_URL.fullmatch(url) is None ): return None return { From 9d0e8476f12d8c8ef2ef43f1828e9e8a513996dd Mon Sep 17 00:00:00 2001 From: xormania <127287135+xormania@users.noreply.github.com> Date: Sun, 2 Aug 2026 08:36:09 -0400 Subject: [PATCH 140/158] test(experiment): kill Git intake mutants --- tests/experiment/git-mutation-cases.py | 651 +++++++++++++++++++++++++ 1 file changed, 651 insertions(+) create mode 100755 tests/experiment/git-mutation-cases.py diff --git a/tests/experiment/git-mutation-cases.py b/tests/experiment/git-mutation-cases.py new file mode 100755 index 0000000..c77e1e9 --- /dev/null +++ b/tests/experiment/git-mutation-cases.py @@ -0,0 +1,651 @@ +#!/usr/bin/env python3 +"""Private-copy sensitivity mutations for pinned Git Experiment intake.""" + +from __future__ import annotations + +import base64 +from hashlib import sha1, sha256 +from importlib.util import module_from_spec, spec_from_file_location +import io +import json +import os +from pathlib import Path +import shutil +import signal +import socket +import stat +import sys +import tempfile +import time +from types import SimpleNamespace +from typing import Callable + + +EXPECTED = ( + "M-GIT-AUTHORITY-001", + "M-GIT-REF-001", + "M-GIT-BOUND-001", + "M-GIT-IDENTITY-001", + "M-GIT-DOWNSTREAM-001", +) +URL = "https://github.com/uscient/experiment-fixture.git" +FORBIDDEN_AUTHORITY = "credential.invalid" +COMMIT = "1cffa1a28f96d2f2cb898b1bad70d281e359a5b5" +TREE = "64564b8e82ec9581c32cb4951ed802b544e2e0c0" +BLOB = "a1d8c8cd0f1865e66cb2463cbaa801c4b5a85656" +SOURCE_DIGEST = "sha256:463e8a7622e58281fd975d58d8a9ad44ed997dd08af32e237f1476021f7abb23" +MARKER_ENV = "AGENT_LAB_GIT_MUTATION_MARK" + + +class HarnessInfrastructure(Exception): + """Private mutation evidence could not be established safely.""" + + +def load_module(path: Path, label: str): + spec = spec_from_file_location(label, path) + if spec is None or spec.loader is None: + raise HarnessInfrastructure(f"cannot load private runtime {path.name}") + module = module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +def git_oid(kind: str, payload: bytes) -> str: + framed = kind.encode("ascii") + b" " + str(len(payload)).encode("ascii") + b"\0" + return sha1(framed + payload).hexdigest() + + +def response(body: object) -> tuple[int, tuple[tuple[str, str], ...], bytes]: + encoded = json.dumps( + body, + ensure_ascii=True, + separators=(",", ":"), + sort_keys=True, + ).encode("ascii") + return ( + 200, + ( + ("content-length", str(len(encoded))), + ("content-type", "application/json; charset=utf-8"), + ), + encoded, + ) + + +def fixture_responses(source: bytes) -> dict[str, tuple[int, tuple[tuple[str, str], ...], bytes]]: + tree_payload = b"100644 experiment.cue\0" + bytes.fromhex(BLOB) + commit_payload = ( + f"tree {TREE}\n" + "author Fixture 0 +0000\n" + "committer Fixture 0 +0000\n" + "\n" + "pinned fixture\n" + ).encode("ascii") + if ( + git_oid("blob", source) != BLOB + or git_oid("tree", tree_payload) != TREE + or git_oid("commit", commit_payload) != COMMIT + ): + raise HarnessInfrastructure("independent Git fixture identity drift") + return { + f"/repos/uscient/experiment-fixture/git/commits/{COMMIT}": response( + {"sha": COMMIT, "tree": {"sha": TREE}} + ), + f"/repos/uscient/experiment-fixture/git/trees/{TREE}": response( + { + "sha": TREE, + "tree": [ + { + "mode": "100644", + "path": "experiment.cue", + "sha": BLOB, + "size": len(source), + "type": "blob", + } + ], + "truncated": False, + } + ), + f"/repos/uscient/experiment-fixture/git/blobs/{BLOB}": response( + { + "content": base64.b64encode(source).decode("ascii"), + "encoding": "base64", + "sha": BLOB, + "size": len(source), + } + ), + } + + +def fixture_requester( + responses: dict[str, tuple[int, tuple[tuple[str, str], ...], bytes]] +): + def request(_authority, path, _headers, _maximum, _deadline): + return responses[path] + + return request + + +def fingerprint(root: Path) -> dict[str, tuple[int, str]]: + result: dict[str, tuple[int, str]] = {} + for path in sorted(root.rglob("*")): + if path.is_file(): + result[path.relative_to(root).as_posix()] = ( + stat.S_IMODE(path.stat().st_mode), + sha256(path.read_bytes()).hexdigest(), + ) + elif not path.is_dir(): + raise HarnessInfrastructure("private runtime contains an unsupported entry") + return result + + +def replace_once(source: str, needle: str, replacement: str, assertion: str) -> str: + occurrences = source.count(needle) + if occurrences != 1: + raise HarnessInfrastructure( + f"{assertion} replacement applicability is {occurrences}, expected exactly 1" + ) + mutated = source.replace(needle, replacement, 1) + if mutated == source or mutated.count(replacement) != 1: + raise HarnessInfrastructure(f"{assertion} replacement result is ambiguous") + try: + compile(mutated, f"<{assertion}>", "exec") + except SyntaxError as error: + raise HarnessInfrastructure(f"{assertion} private mutation does not compile") from error + return mutated + + +def authority_probe(module, marker: Path | None) -> bool: + import http.client + import ssl + + connections: list[tuple[str, int, float, object]] = [] + requests: list[tuple[str, str, dict[str, str]]] = [] + body = b"{}" + + class FakeContext: + def __init__(self, protocol): + self.protocol = protocol + self.check_hostname = False + self.verify_mode = None + self.minimum_version = None + self.loaded = None + + def load_verify_locations(self, *, cadata): + self.loaded = cadata + + class FakeSocket: + def settimeout(self, _timeout): + return None + + class FakeResponse: + status = 200 + + def __init__(self): + self.stream = io.BytesIO(body) + + def getheaders(self): + return [ + ("content-length", str(len(body))), + ("content-type", "application/json; charset=utf-8"), + ] + + def read(self, size=-1): + return self.stream.read(size) + + def close(self): + return None + + class FakeConnection: + def __init__(self, authority, *, port, timeout, context): + connections.append((authority, port, timeout, context)) + self.sock = FakeSocket() + + def request(self, method, path, *, headers): + requests.append((method, path, dict(headers))) + + def getresponse(self): + return FakeResponse() + + def close(self): + return None + + original_context = ssl.SSLContext + original_connection = http.client.HTTPSConnection + original_maxline = http.client._MAXLINE + original_maxheaders = http.client._MAXHEADERS + original_ca_reader = module._git_system_ca_pem + ssl.SSLContext = FakeContext + http.client.HTTPSConnection = FakeConnection + module._git_system_ca_pem = lambda: "fixture-ca" + try: + try: + result = module._github_api_request( + FORBIDDEN_AUTHORITY, + f"/repos/uscient/experiment-fixture/git/commits/{COMMIT}", + module.GIT_PROVIDER_HEADERS, + 1_024, + time.monotonic() + 1.0, + ) + except module.InfrastructureError as error: + outcome = ("infra", str(error)) + else: + outcome = ("ok", result) + finally: + module._git_system_ca_pem = original_ca_reader + http.client.HTTPSConnection = original_connection + http.client._MAXLINE = original_maxline + http.client._MAXHEADERS = original_maxheaders + ssl.SSLContext = original_context + + if marker is None: + return ( + outcome[0] == "infra" + and "GIT-AUTHORITY" in outcome[1] + and connections == [] + and requests == [] + ) + if len(connections) != 1: + return False + authority, port, timeout, context = connections[0] + return ( + outcome == ( + "ok", + ( + 200, + ( + ("content-length", str(len(body))), + ("content-type", "application/json; charset=utf-8"), + ), + body, + ), + ) + and authority == FORBIDDEN_AUTHORITY + and port == 443 + and 0 < timeout <= 1.0 + and context.check_hostname is True + and context.verify_mode == ssl.CERT_REQUIRED + and context.minimum_version == ssl.TLSVersion.TLSv1_2 + and context.loaded == "fixture-ca" + and requests + == [ + ( + "GET", + f"/repos/uscient/experiment-fixture/git/commits/{COMMIT}", + dict(module.GIT_PROVIDER_HEADERS), + ) + ] + ) + + +def ref_probe(module, marker: Path | None, responses, source: bytes) -> bool: + if marker is None: + calls: list[str] = [] + + def unused(_authority, path, _headers, _maximum, _deadline): + calls.append(path) + raise AssertionError("mutable ref reached acquisition") + + try: + module.read_git_snapshot(URL, "main", requester=unused) + except module.InvalidManifest as error: + return "GIT-OID" in str(error) and calls == [] + return False + try: + snapshot = module.read_git_snapshot( + URL, + "main", + requester=fixture_requester(responses), + ) + except (module.InvalidManifest, module.InfrastructureError): + return False + return ( + snapshot.data == source + and snapshot.digest == SOURCE_DIGEST + and snapshot.transport.get("requestedCommit") == COMMIT + ) + + +def group_exists(pgid: int) -> bool: + try: + os.killpg(pgid, 0) + except ProcessLookupError: + return False + except PermissionError: + return True + return True + + +def wait_group_gone(pgid: int, timeout: float = 2.0) -> bool: + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + if not group_exists(pgid): + return True + time.sleep(0.01) + return not group_exists(pgid) + + +def bound_probe(module, marker: Path | None, responses) -> bool: + listener = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + listener.bind(("127.0.0.1", 0)) + listener.settimeout(2.0) + address = listener.getsockname() + + def residual_requester(_authority, path, _headers, _maximum, _deadline): + child = os.fork() + if child == 0: + try: + signal.signal(signal.SIGTERM, signal.SIG_IGN) + signal.alarm(5) + with socket.socket(socket.AF_INET, socket.SOCK_DGRAM) as sender: + sender.sendto( + f"{os.getpid()}:{os.getpgrp()}".encode("ascii"), + address, + ) + while True: + signal.pause() + finally: + os._exit(0) + return responses[path] + + module._github_api_request = residual_requester + try: + try: + module.read_git_snapshot(URL, COMMIT) + except module.InfrastructureError as error: + outcome = str(error) + else: + outcome = "ok" + try: + record = listener.recv(128).decode("ascii") + except (TimeoutError, UnicodeDecodeError) as error: + raise HarnessInfrastructure("residual worker identity was not reported") from error + finally: + listener.close() + try: + raw_pid, raw_pgid = record.split(":", 1) + pid = int(raw_pid) + pgid = int(raw_pgid) + except (TypeError, ValueError) as error: + raise HarnessInfrastructure("residual worker identity is malformed") from error + if pid <= 1 or pgid <= 1 or pgid == os.getpgrp(): + raise HarnessInfrastructure("residual worker group is not private") + + alive = group_exists(pgid) + cleanup_ok = True + if alive: + try: + os.killpg(pgid, signal.SIGKILL) + except ProcessLookupError: + pass + cleanup_ok = wait_group_gone(pgid) + if not cleanup_ok: + raise HarnessInfrastructure("residual worker cleanup is uncertain") + if marker is None: + return "residual process group" in outcome.lower() and not alive + return "residual process group" in outcome.lower() and alive + + +def identity_probe(module, marker: Path | None, responses, source: bytes, source_dir: Path) -> bool: + try: + directory = module.read_directory_snapshot(str(source_dir)) + git = module.read_git_snapshot( + URL, + COMMIT, + requester=fixture_requester(responses), + ) + except (module.InvalidManifest, module.InfrastructureError): + return False + if directory.data != source or git.data != source: + return False + if marker is None: + return directory.digest == git.digest == SOURCE_DIGEST + expected_object_digest = "sha256:" + sha256(BLOB.encode("ascii")).hexdigest() + return git.digest == expected_object_digest and directory.digest != git.digest + + +def downstream_probe(module, marker: Path | None) -> bool: + events: list[str] = [] + snapshot = module.SourceSnapshot( + data=b"fixture", + digest="sha256:" + "1" * 64, + transport={"kind": "git"}, + ) + resolution = SimpleNamespace(plan={}, bundled_catalog=None, local_catalog=None) + + def authorize(_plan, _digest): + events.append("authorize") + return {"verdict": "deny"}, 1 + + module.read_git_snapshot = lambda _url, _commit: snapshot + module.authored_manifest = lambda _snapshot: {} + module.cue_plan_with_evidence = lambda _manifest: resolution + module.authorize_plan = authorize + module.write_decision = lambda _decision: events.append("decision") + module.write_checked_source = lambda _checked: events.append("checked") + result = module.main(["experiment.py", "authorize-git", URL, COMMIT]) + if marker is None: + return result == 1 and events == ["authorize", "decision"] + return result == 0 and events == ["checked"] + + +Probe = Callable[[object, Path | None], bool] + + +def execute_mutation( + repo: Path, + production: Path, + original: str, + production_digest: str, + root: Path, + assertion: str, + needle: str, + replacement: str, + probe: Probe, +) -> bool: + runtime = root / "runtime" + runtime.mkdir(parents=True) + private_source = runtime / "experiment.py" + shutil.copy2(production, private_source) + shutil.copy2(repo / "scripts/image_reference.py", runtime / "image_reference.py") + pristine_topology = fingerprint(runtime) + pristine = load_module(private_source, assertion.lower().replace("-", "_") + "_pristine") + if not probe(pristine, None): + raise HarnessInfrastructure(f"{assertion} pristine private probe is not GREEN") + if fingerprint(runtime) != pristine_topology: + raise HarnessInfrastructure(f"{assertion} pristine probe changed private runtime") + + mutated = replace_once(original, needle, replacement, assertion) + private_source.write_text(mutated, encoding="utf-8") + mutated_topology = fingerprint(runtime) + changed = [ + path + for path in sorted(set(pristine_topology) | set(mutated_topology)) + if pristine_topology.get(path) != mutated_topology.get(path) + ] + if changed != ["experiment.py"]: + raise HarnessInfrastructure(f"{assertion} changed unexpected private runtime paths") + + marker = root / "mutation-reached" + marker.unlink(missing_ok=True) + prior_marker = os.environ.get(MARKER_ENV) + os.environ[MARKER_ENV] = str(marker) + try: + mutant = load_module(private_source, assertion.lower().replace("-", "_") + "_mutant") + detected = probe(mutant, marker) + finally: + if prior_marker is None: + os.environ.pop(MARKER_ENV, None) + else: + os.environ[MARKER_ENV] = prior_marker + if not marker.is_file(): + raise HarnessInfrastructure(f"{assertion} did not prove its mutated path was reached") + if fingerprint(runtime) != mutated_topology: + raise HarnessInfrastructure(f"{assertion} mutant probe changed private runtime") + if sha256(production.read_bytes()).hexdigest() != production_digest: + raise HarnessInfrastructure(f"{assertion} changed the production implementation") + return detected + + +def make_writable(root: Path) -> None: + for path in root.rglob("*"): + try: + path.chmod(path.stat().st_mode | stat.S_IWUSR | stat.S_IXUSR) + except OSError: + pass + + +def main() -> int: + sys.dont_write_bytecode = True + repo = Path(__file__).resolve().parents[2] + production = repo / "scripts/experiment.py" + original = production.read_text(encoding="utf-8") + production_digest = sha256(production.read_bytes()).hexdigest() + source_dir = repo / "tests/experiment/fixtures/directories/minimal" + source = (source_dir / "experiment.cue").read_bytes() + responses = fixture_responses(source) + work = Path(tempfile.mkdtemp(prefix="agent-lab-git-mutations-")) + failures = 0 + infrastructure = 0 + results: list[tuple[str, bool, str]] = [] + try: + cases: tuple[tuple[str, str, str, Probe, str], ...] = ( + ( + "M-GIT-AUTHORITY-001", + " authority != GIT_PROVIDER_AUTHORITY\n", + ( + " (\n" + " authority != GIT_PROVIDER_AUTHORITY\n" + " and (\n" + f" Path(os.environ[\"{MARKER_ENV}\"]).touch()\n" + " or False\n" + " )\n" + " )\n" + ), + authority_probe, + "forbidden provider authority reaches the HTTPS connection canary", + ), + ( + "M-GIT-REF-001", + " if not isinstance(commit, str) or GIT_SHA1.fullmatch(commit) is None:\n", + ( + " if commit == \"main\":\n" + f" Path(os.environ[\"{MARKER_ENV}\"]).touch()\n" + f" commit = \"{COMMIT}\"\n" + " if not isinstance(commit, str) or GIT_SHA1.fullmatch(commit) is None:\n" + ), + lambda module, marker: ref_probe(module, marker, responses, source), + "mutable ref acceptance reaches the pinned-object adapter", + ), + ( + "M-GIT-BOUND-001", + " terminated = _git_worker_terminate(pid, reaped=reaped)\n", + ( + f" Path(os.environ[\"{MARKER_ENV}\"]).touch()\n" + " terminated = True\n" + ), + lambda module, marker: bound_probe(module, marker, responses), + "removed process-group cleanup leaves a live provider descendant", + ), + ( + "M-GIT-IDENTITY-001", + ( + " digest=source_digest(data),\n" + " transport={\n" + " \"acquisition\": {\n" + ), + ( + " digest=(\n" + f" Path(os.environ[\"{MARKER_ENV}\"]).touch()\n" + " or \"sha256:\"\n" + " + hashlib.sha256(blob_id.encode(\"ascii\")).hexdigest()\n" + " ),\n" + " transport={\n" + " \"acquisition\": {\n" + ), + lambda module, marker: identity_probe( + module, + marker, + responses, + source, + source_dir, + ), + "object-ID-derived identity breaks the cross-transport oracle", + ), + ( + "M-GIT-DOWNSTREAM-001", + " if directory_checking or zip_checking or git_checking:\n", + ( + " if (\n" + " directory_checking\n" + " or zip_checking\n" + " or git_checking\n" + " or (\n" + " git_authorizing\n" + " and (\n" + f" Path(os.environ[\"{MARKER_ENV}\"]).touch()\n" + " or True\n" + " )\n" + " )\n" + " ):\n" + ), + downstream_probe, + "Git authorization routed around the common decision path", + ), + ) + for index, (assertion, needle, replacement, probe, message) in enumerate(cases): + case_root = work / f"case-{index}" + try: + detected = execute_mutation( + repo, + production, + original, + production_digest, + case_root, + assertion, + needle, + replacement, + probe, + ) + except HarnessInfrastructure: + raise + except Exception as error: + raise HarnessInfrastructure( + f"{assertion} probe raised {type(error).__name__}" + ) from error + results.append((assertion, detected, message)) + except (HarnessInfrastructure, OSError) as error: + print(f"INFRA Git mutation harness {error}", file=sys.stderr) + infrastructure = 1 + finally: + try: + make_writable(work) + shutil.rmtree(work) + except OSError: + infrastructure = 1 + + if sha256(production.read_bytes()).hexdigest() != production_digest: + infrastructure = 1 + observed = tuple(assertion for assertion, _, _ in results) + if observed != EXPECTED: + infrastructure = 1 + for assertion, passed, message in results: + if passed: + print(f"PASS {assertion} {message}") + else: + print(f"FAIL {assertion} {message}") + failures += 1 + print( + f"SUMMARY assertions={len(results)} expected={len(EXPECTED)} " + f"failures={failures} infra={infrastructure}" + ) + if infrastructure: + return 125 + if failures: + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) From 53c1956d0adfdc416e1ee6777797be1fba15caf0 Mon Sep 17 00:00:00 2001 From: xormania <127287135+xormania@users.noreply.github.com> Date: Sun, 2 Aug 2026 08:37:08 -0400 Subject: [PATCH 141/158] docs(experiment): define pinned Git boundary --- docs/architecture.md | 22 ++++++++++++---------- docs/experiments.md | 38 +++++++++++++++++++++++++++----------- docs/installation.md | 7 ++++++- 3 files changed, 45 insertions(+), 22 deletions(-) diff --git a/docs/architecture.md b/docs/architecture.md index 545f064..e5cf2df 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -69,19 +69,21 @@ See [Development](development.md) and [development-agent configuration](agent-co ## Experiment planning and installed evidence -`scripts/agent-lab experiment check DIRECTORY` snapshots and validates the closed authored -`agent-lab/v0alpha1` request with the -repository-pinned CUE contract and emits one canonical, digest-bound `RequestedExperimentPlan`. -`scripts/agent-lab experiment authorize install DIRECTORY` reads the snapshot once, derives that same plan +`scripts/agent-lab experiment check` snapshots and validates one closed authored source from a sole +local `experiment.cue`, a bounded sole-member ZIP, or an exact supported public GitHub commit. The +repository-pinned CUE contract emits one canonical, digest-bound `RequestedExperimentPlan`. +`scripts/agent-lab experiment authorize install` reads that snapshot once, derives the same plan in-process, and asks the repository-pinned Cedar policy whether the fixed local compatibility principal may submit the exact plan digest. -Those two commands are no-effect preflights. `experiment install DIRECTORY` instead repeats the -snapshot, planning, and Cedar evaluation, then stores the exact permitted evidence in the -initialized home. No caller-supplied decision is accepted. For a local image name, install rechecks -the selected entry under the shared catalog lock before taking the Experiment store lock -exclusively; both remain held through durable no-replace publication. Direct and bundled selectors -do not open local catalog state. +The preview forms create no durable Agent Lab state; Git previews have only their bounded public +acquisition effect. `experiment install` instead repeats snapshotting, planning, and Cedar +evaluation, then stores the exact permitted evidence in the initialized home. Directory, ZIP, and +Git sources carrying identical bytes converge on the same source, plan, authorization, installation, +and artifact identities; only their closed transport provenance differs. No caller-supplied decision +is accepted. For a local image name, install rechecks the selected entry under the shared catalog +lock before taking the Experiment store lock exclusively; both remain held through durable +no-replace publication. Direct and bundled selectors do not open local catalog state. The installed envelope contains the exact artifact plus closed plan, decision, provenance, and receipt records. Its installation key binds source, domain-separated plan, contract, authorization, diff --git a/docs/experiments.md b/docs/experiments.md index 6e17631..bbf8377 100644 --- a/docs/experiments.md +++ b/docs/experiments.md @@ -1,10 +1,11 @@ # Experiments -An Experiment is authored as data in a directory containing exactly one file, `experiment.cue`, or -in a bounded ZIP archive containing that exact sole member. The file defines one concrete value named -`experiment` in package `experiment`. Agent Lab snapshots the exact authored bytes privately before -evaluating them; extra entries, links, special files, suspicious modes, changing sources, malformed -CUE, and unknown schema fields are refused. +An Experiment is authored as data in a directory containing exactly one file, `experiment.cue`, in a +bounded ZIP archive containing that exact sole member, or at one exact commit of a supported public +GitHub repository whose root tree contains that exact sole blob. The file defines one concrete value +named `experiment` in package `experiment`. Agent Lab snapshots the exact authored bytes privately +before evaluating them; extra entries, links, special files, suspicious modes, changing sources, +malformed CUE, and unknown schema fields are refused. ```cue package experiment @@ -28,11 +29,14 @@ Check the artifact or preview its install authorization from the repository: ./scripts/agent-lab experiment authorize install ./my-experiment ./scripts/agent-lab experiment check --zip ./my-experiment.zip ./scripts/agent-lab experiment authorize install --zip ./my-experiment.zip +./scripts/agent-lab experiment check --git https://github.com/owner/repository.git --commit <40 lowercase hex> +./scripts/agent-lab experiment authorize install --git https://github.com/owner/repository.git --commit <40 lowercase hex> ``` -These two commands are previews. They create no durable Agent Lab state and do not invoke Docker or -run Experiment content. `authorize install` freshly checks the same held source and emits decision -evidence bound to its source, plan, contract, and authorization identities. The decision is not an +The check and authorization forms are previews. They create no durable Agent Lab state and do not +invoke Docker or run Experiment content. Git previews do perform the bounded public acquisition +described below. `authorize install` freshly checks the same held source and emits decision evidence +bound to its source, plan, contract, and authorization identities. The decision is not an installation capability. Install a freshly checked and permitted artifact, then inspect its stored identity: @@ -40,6 +44,7 @@ Install a freshly checked and permitted artifact, then inspect its stored identi ```bash agent-lab [--home /absolute/private/home] experiment install ./my-experiment agent-lab [--home /absolute/private/home] experiment install --zip ./my-experiment.zip +agent-lab [--home /absolute/private/home] experiment install --git https://github.com/owner/repository.git --commit <40 lowercase hex> agent-lab [--home /absolute/private/home] experiment inspect example ``` @@ -61,6 +66,17 @@ authorization binding, installation key, or idempotent cross-transport retry. Regular-file attributes are interpreted only for Unix and DOS-compatible FAT, NTFS, and VFAT creator systems; other creator systems are rejected when their member type cannot be proven. +Git intake is Linux-only in this version. It accepts only a normalized, unauthenticated +`https://github.com//.git` URL and one exact lowercase 40-hex SHA-1 commit object +ID. A fixed credential-free GitHub Git Data API client reads that commit, its exact root tree, and +the bound blob under one five-second deadline and a 1,048,576-byte aggregate response cap. It uses +explicit system trust, identity encoding, fixed headers, a private process group, and zero temporary +files. Redirects, credentials, mutable refs, alternate protocols or authorities, extra tree entries, +and changed bound objects fail closed. Agent Lab never runs Git, creates a repository, checks out +content, follows submodules, or executes repository data. Provenance records the canonical URL, +requested and verified object IDs, bounded acquisition facts, and the independent framed SHA-256 +source digest. Git object identity does not replace source identity or change cross-transport retry. + An exact retry freshly validates and authorizes again, verifies the complete installed envelope, and returns `changed:false` with the same `installationKey` and `receiptDigest`. The same requested name with a different installation identity conflicts without overwrite. `inspect` is read-only: it @@ -68,9 +84,9 @@ verifies and reports one installed identity, but never reconciles staging or rep effectful install may recover only recognized, bounded staging left by an interrupted publication; unknown or ambiguous residue remains in place and returns infrastructure uncertainty. -All four commands work from a local installation after `agent-lab init` and explicit -`agent-lab tools provision`. Installed execution verifies and uses its release bundle and the -effective home's pinned tool cache; it does not depend on a source checkout. +These preview, install, and inspect commands work from a local installation after `agent-lab init` +and explicit `agent-lab tools provision`. Installed execution verifies and uses its release bundle +and the effective home's pinned tool cache; it does not depend on a source checkout. Each member selects either an exact digest-pinned OCI reference with `digestRef` or a shared name with `catalogName`. Shared names have exactly two bounded lowercase components, `.`. diff --git a/docs/installation.md b/docs/installation.md index 59af50c..a15c93a 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -48,6 +48,8 @@ program bundle: ```bash agent-lab --home /absolute/private/home experiment install ./my-experiment +agent-lab --home /absolute/private/home experiment install --zip ./my-experiment.zip +agent-lab --home /absolute/private/home experiment install --git https://github.com/owner/repository.git --commit <40 lowercase hex> agent-lab --home /absolute/private/home experiment inspect NAME ``` @@ -74,7 +76,10 @@ identifies the closed receipt itself. Provenance records the source transport an evidence: `catalog` is `null` for direct digests, `catalog.bundled.snapshotDigest` identifies a release-owned bundled snapshot, and `catalog.local` contains the checked local snapshot's `revision` and `snapshotDigest`. Both nested entries are present when a plan uses both namespaces. Provenance -is evidence, not authority for a later operation. +also records one closed source transport: local directory, bounded ZIP byte count and digest, or +canonical public GitHub URL, exact commit/tree/blob identities, and bounded acquisition facts. +Transport provenance is evidence, not authority for a later operation and is excluded from the +installation identity, so equivalent directory, ZIP, and Git sources retry idempotently. Every read reopens and verifies the closed layout, canonical bytes, digests, schemas, ownership, modes, and link counts. `experiment inspect` does this under the shared store lock and never writes From 3d4f6c57689d8d1be9e0513a6fc897e58c4ca729 Mon Sep 17 00:00:00 2001 From: xormania <127287135+xormania@users.noreply.github.com> Date: Sun, 2 Aug 2026 08:37:56 -0400 Subject: [PATCH 142/158] test(experiment): require Git source aggregation --- tests/experiment/aggregate-harness-cases.sh | 60 ++++++++++++++++----- 1 file changed, 46 insertions(+), 14 deletions(-) diff --git a/tests/experiment/aggregate-harness-cases.sh b/tests/experiment/aggregate-harness-cases.sh index 0417cd7..d53ab5a 100755 --- a/tests/experiment/aggregate-harness-cases.sh +++ b/tests/experiment/aggregate-harness-cases.sh @@ -86,7 +86,26 @@ source_mutation_ids=( M-ZIP-METHOD-001 M-ZIP-SIZE-001 M-ZIP-BOMB-001 M-ZIP-CRC-001 M-ZIP-HEADER-001 M-ZIP-EXTRACT-001 M-ZIP-IDENTITY-001 M-ZIP-AUTH-001 ) -source_expected_ids=("${source_zip_ids[@]}" "${source_mutation_ids[@]}") +source_git_ids=( + GIT-CLI-001 GIT-USAGE-001 GIT-URL-001 GIT-OID-001 GIT-PLAT-001 + GIT-FIXTURE-001 GIT-PIN-001 GIT-COMMIT-001 GIT-ROOT-001 GIT-TYPE-001 + GIT-BLOB-001 GIT-DRIFT-001 GIT-AUTHORITY-001 GIT-CREDENTIAL-001 + GIT-REDIRECT-001 GIT-CONTENT-001 GIT-TIMEOUT-001 GIT-OUTPUT-001 + GIT-ACQUIRE-001 GIT-PGROUP-001 GIT-CLEANUP-001 GIT-TAXONOMY-001 + GIT-CHECK-001 GIT-AUTH-001 GIT-DENY-001 GIT-INSTALL-001 + GIT-IDENTITY-001 GIT-RETRY-001 GIT-ADAPTER-001 GIT-NOEF-001 + GIT-RUNTIME-001 GIT-DIAG-001 +) +source_git_mutation_ids=( + M-GIT-AUTHORITY-001 M-GIT-REF-001 M-GIT-BOUND-001 M-GIT-IDENTITY-001 + M-GIT-DOWNSTREAM-001 +) +source_expected_ids=( + "${source_zip_ids[@]}" + "${source_mutation_ids[@]}" + "${source_git_ids[@]}" + "${source_git_mutation_ids[@]}" +) write_fixture() { local path="$1" @@ -241,12 +260,17 @@ reset_fixtures() { } reset_source_fixtures() { - local zip_records=() mutation_records=() + local zip_records=() mutation_records=() git_records=() git_mutation_records=() mapfile -t zip_records < <(pass_records "${source_zip_ids[@]}") mapfile -t mutation_records < <(pass_records "${source_mutation_ids[@]}") + mapfile -t git_records < <(pass_records "${source_git_ids[@]}") + mapfile -t git_mutation_records < <(pass_records "${source_git_mutation_ids[@]}") write_fixture "$replica/tests/experiment/zip-intake-cases.sh" 0 "${zip_records[@]}" write_python_fixture \ "$replica/tests/experiment/zip-mutation-cases.py" 0 "${mutation_records[@]}" + write_fixture "$replica/tests/experiment/git-intake-cases.sh" 0 "${git_records[@]}" + write_python_fixture \ + "$replica/tests/experiment/git-mutation-cases.py" 0 "${git_mutation_records[@]}" } run_replica() { @@ -675,7 +699,11 @@ reset_source_fixtures source_expected_executions="$work/source-expected-executions" source_baseline_executions="$work/source-baseline-executions" source_mutant_executions="$work/source-mutant-executions" -printf '%s\n' zip-intake-cases.sh zip-mutation-cases.py > "$source_expected_executions" +printf '%s\n' \ + zip-intake-cases.sh \ + zip-mutation-cases.py \ + git-intake-cases.sh \ + git-mutation-cases.py > "$source_expected_executions" : > "$source_baseline_executions" source_baseline_rc=0 run_source_replica "$work/source-baseline.out" env \ @@ -699,8 +727,12 @@ source_mutant_rc=0 run_selected "$work/source-mutant.out" "$mutant_source_adapters" env \ AGENT_LAB_AGG_EXEC_LOG="$source_mutant_executions" || source_mutant_rc=$? source_mutant_expected="$work/source-mutant-expected-executions" -printf '%s\n' zip-intake-cases.sh zip-intake-cases.sh zip-mutation-cases.py \ - > "$source_mutant_expected" +printf '%s\n' \ + zip-intake-cases.sh \ + zip-intake-cases.sh \ + zip-mutation-cases.py \ + git-intake-cases.sh \ + git-mutation-cases.py > "$source_mutant_expected" if [ "$source_baseline_rc" -eq 0 ] && cmp -s "$source_expected_executions" "$source_baseline_executions" && [ "$source_mutation_count" -eq 1 ] && [ "$source_mutant_rc" -eq 0 ] && @@ -716,7 +748,7 @@ source_success_expected="$work/source-success-expected" for id in "${source_expected_ids[@]}"; do printf 'PASS %s fixture assertion\n' "$id" done - printf 'SUMMARY assertions=45 expected=45 failures=0 infra=0\n' + printf 'SUMMARY assertions=82 expected=82 failures=0 infra=0\n' printf 'EXPERIMENT SOURCE ADAPTERS PASS\n' } > "$source_success_expected" if [ "$source_baseline_rc" -eq 0 ] && @@ -735,7 +767,7 @@ write_fixture "$replica/tests/experiment/zip-intake-cases.sh" 0 \ source_missing_rc=0 run_source_replica "$work/source-missing.out" env || source_missing_rc=$? if [ "$source_missing_rc" -eq 1 ] && - grep -Fxq 'SUMMARY assertions=44 expected=45 failures=1 infra=0' \ + grep -Fxq 'SUMMARY assertions=81 expected=82 failures=1 infra=0' \ "$work/source-missing.out" && ! grep -Fxq 'EXPERIMENT SOURCE ADAPTERS PASS' "$work/source-missing.out"; then pass AGG-023 "source-adapter missing assertion identity maps to one" @@ -752,7 +784,7 @@ write_fixture "$replica/tests/experiment/zip-intake-cases.sh" 0 \ source_duplicate_rc=0 run_source_replica "$work/source-duplicate.out" env || source_duplicate_rc=$? if [ "$source_duplicate_rc" -eq 1 ] && - grep -Fxq 'SUMMARY assertions=46 expected=45 failures=1 infra=0' \ + grep -Fxq 'SUMMARY assertions=83 expected=82 failures=1 infra=0' \ "$work/source-duplicate.out" && ! grep -Fxq 'EXPERIMENT SOURCE ADAPTERS PASS' "$work/source-duplicate.out"; then pass AGG-024 "source-adapter duplicate assertion identity maps to one" @@ -769,7 +801,7 @@ write_fixture "$replica/tests/experiment/zip-intake-cases.sh" 0 \ source_substituted_rc=0 run_source_replica "$work/source-substituted.out" env || source_substituted_rc=$? if [ "$source_substituted_rc" -eq 1 ] && - grep -Fxq 'SUMMARY assertions=45 expected=45 failures=1 infra=0' \ + grep -Fxq 'SUMMARY assertions=82 expected=82 failures=1 infra=0' \ "$work/source-substituted.out" && ! grep -Fxq 'EXPERIMENT SOURCE ADAPTERS PASS' "$work/source-substituted.out"; then pass AGG-025 "source-adapter substituted assertion identity maps to one" @@ -786,7 +818,7 @@ write_fixture "$replica/tests/experiment/zip-intake-cases.sh" 1 \ source_assertion_rc=0 run_source_replica "$work/source-assertion.out" env || source_assertion_rc=$? if [ "$source_assertion_rc" -eq 1 ] && - grep -Fxq 'SUMMARY assertions=45 expected=45 failures=1 infra=0' \ + grep -Fxq 'SUMMARY assertions=82 expected=82 failures=1 infra=0' \ "$work/source-assertion.out" && ! grep -Fxq 'EXPERIMENT SOURCE ADAPTERS PASS' "$work/source-assertion.out"; then pass AGG-026 "source-adapter subcase assertion failure maps to one" @@ -802,7 +834,7 @@ write_fixture "$replica/tests/experiment/zip-intake-cases.sh" 125 \ source_subcase_infra_rc=0 run_source_replica "$work/source-subcase-infra.out" env || source_subcase_infra_rc=$? if [ "$source_subcase_infra_rc" -eq 125 ] && - grep -Fxq 'SUMMARY assertions=45 expected=45 failures=0 infra=1' \ + grep -Fxq 'SUMMARY assertions=82 expected=82 failures=0 infra=1' \ "$work/source-subcase-infra.out" && ! grep -Fxq 'EXPERIMENT SOURCE ADAPTERS PASS' "$work/source-subcase-infra.out"; then pass AGG-027 "source-adapter subcase uncertainty maps to one hundred twenty-five" @@ -815,7 +847,7 @@ find "$replica/tests/experiment/zip-mutation-cases.py" -delete source_setup_infra_rc=0 run_source_replica "$work/source-setup-infra.out" env || source_setup_infra_rc=$? if [ "$source_setup_infra_rc" -eq 125 ] && - grep -Fxq 'SUMMARY assertions=33 expected=45 failures=1 infra=1' \ + grep -Fxq 'SUMMARY assertions=70 expected=82 failures=1 infra=1' \ "$work/source-setup-infra.out" && ! grep -Fxq 'EXPERIMENT SOURCE ADAPTERS PASS' "$work/source-setup-infra.out"; then pass AGG-028 "source-adapter setup uncertainty maps to one hundred twenty-five" @@ -832,7 +864,7 @@ source_cleanup_rc=0 run_source_replica "$work/source-cleanup.out" env \ PATH="$source_shim:$PATH" || source_cleanup_rc=$? if [ "$source_cleanup_rc" -eq 125 ] && - grep -Fxq 'SUMMARY assertions=45 expected=45 failures=0 infra=1' \ + grep -Fxq 'SUMMARY assertions=82 expected=82 failures=0 infra=1' \ "$work/source-cleanup.out" && ! grep -Fxq 'EXPERIMENT SOURCE ADAPTERS PASS' "$work/source-cleanup.out"; then pass AGG-029 "source-adapter cleanup uncertainty suppresses the final marker" @@ -853,7 +885,7 @@ chmod +x "$source_summaryless" source_summaryless_rc=0 run_source_replica "$work/source-summaryless.out" env || source_summaryless_rc=$? if [ "$source_summaryless_rc" -eq 125 ] && - grep -Fxq 'SUMMARY assertions=45 expected=45 failures=0 infra=1' \ + grep -Fxq 'SUMMARY assertions=82 expected=82 failures=0 infra=1' \ "$work/source-summaryless.out" && ! grep -Fxq 'EXPERIMENT SOURCE ADAPTERS PASS' "$work/source-summaryless.out"; then pass AGG-030 "source-adapter missing subcase summary maps to one hundred twenty-five" From 43f041d0056803a7f5b39c5d9de71c3da222ed8a Mon Sep 17 00:00:00 2001 From: xormania <127287135+xormania@users.noreply.github.com> Date: Sun, 2 Aug 2026 08:38:40 -0400 Subject: [PATCH 143/158] test(experiment): aggregate Git source evidence --- tests/experiment/source-adapter-cases.sh | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/tests/experiment/source-adapter-cases.sh b/tests/experiment/source-adapter-cases.sh index a7bc6ba..db79a27 100755 --- a/tests/experiment/source-adapter-cases.sh +++ b/tests/experiment/source-adapter-cases.sh @@ -5,8 +5,10 @@ repo_root="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." >/dev/null 2>&1 && subcases=( "$repo_root/tests/experiment/zip-intake-cases.sh" "$repo_root/tests/experiment/zip-mutation-cases.py" + "$repo_root/tests/experiment/git-intake-cases.sh" + "$repo_root/tests/experiment/git-mutation-cases.py" ) -expected_count=45 +expected_count=82 work="" cleanup_work() { @@ -37,7 +39,16 @@ printf '%s\n' \ ZIP-DENY-001 ZIP-PLAT-001 ZIP-NOEF-002 ZIP-RUNTIME-001 \ M-ZIP-COUNT-001 M-ZIP-NAME-001 M-ZIP-TYPE-001 M-ZIP-FLAG-001 \ M-ZIP-METHOD-001 M-ZIP-SIZE-001 M-ZIP-BOMB-001 M-ZIP-CRC-001 M-ZIP-HEADER-001 \ - M-ZIP-EXTRACT-001 M-ZIP-IDENTITY-001 M-ZIP-AUTH-001 > "$expected" + M-ZIP-EXTRACT-001 M-ZIP-IDENTITY-001 M-ZIP-AUTH-001 \ + GIT-CLI-001 GIT-USAGE-001 GIT-URL-001 GIT-OID-001 GIT-PLAT-001 \ + GIT-FIXTURE-001 GIT-PIN-001 GIT-COMMIT-001 GIT-ROOT-001 GIT-TYPE-001 \ + GIT-BLOB-001 GIT-DRIFT-001 GIT-AUTHORITY-001 GIT-CREDENTIAL-001 \ + GIT-REDIRECT-001 GIT-CONTENT-001 GIT-TIMEOUT-001 GIT-OUTPUT-001 \ + GIT-ACQUIRE-001 GIT-PGROUP-001 GIT-CLEANUP-001 GIT-TAXONOMY-001 \ + GIT-CHECK-001 GIT-AUTH-001 GIT-DENY-001 GIT-INSTALL-001 GIT-IDENTITY-001 \ + GIT-RETRY-001 GIT-ADAPTER-001 GIT-NOEF-001 GIT-RUNTIME-001 GIT-DIAG-001 \ + M-GIT-AUTHORITY-001 M-GIT-REF-001 M-GIT-BOUND-001 M-GIT-IDENTITY-001 \ + M-GIT-DOWNSTREAM-001 > "$expected" : > "$observed" infrastructure=0 From 892f00d89da8c561863a8fbebbe79e7d7a6b5258 Mon Sep 17 00:00:00 2001 From: xormania <127287135+xormania@users.noreply.github.com> Date: Sun, 2 Aug 2026 08:46:35 -0400 Subject: [PATCH 144/158] test(experiment): close public Git source grammar --- tests/experiment/git-intake-cases.py | 130 +++++++++++++++++++++++++-- 1 file changed, 124 insertions(+), 6 deletions(-) diff --git a/tests/experiment/git-intake-cases.py b/tests/experiment/git-intake-cases.py index 6975020..a14ed57 100755 --- a/tests/experiment/git-intake-cases.py +++ b/tests/experiment/git-intake-cases.py @@ -159,6 +159,25 @@ def main(argv: list[str]) -> int: calls.append(tuple(argv)) return 0 + store_calls: list[tuple[Path, str, str]] = [] + + class RecordingStore: + class StoreReject(Exception): + pass + + class StoreInfrastructure(Exception): + pass + + @staticmethod + def install_git(home: Path, url: str, commit: str) -> dict[str, object]: + store_calls.append((home, url, commit)) + return { + "changed": True, + "installationKey": "sha256:" + "1" * 64, + "name": "fixture", + "receiptDigest": "sha256:" + "2" * 64, + } + agent_lab.experiment_module = lambda: RecordingExperiment with tempfile.TemporaryDirectory(prefix="agent-lab-git-cli-") as raw_home: prior_home = os.environ.get("AGENT_LAB_HOME") @@ -180,6 +199,28 @@ def main(argv: list[str]) -> int: COMMIT, ], ) + original_load_config_receipt = agent_lab.load_config_receipt + original_store_module = agent_lab.experiment_store_module + agent_lab.load_config_receipt = lambda _home: ( + {"paths": {"cache": "cache"}}, + b"fixture", + ) + agent_lab.experiment_store_module = lambda: RecordingStore + try: + install = invoke( + agent_lab, + [ + "experiment", + "install", + "--git", + URL, + "--commit", + COMMIT, + ], + ) + finally: + agent_lab.load_config_receipt = original_load_config_receipt + agent_lab.experiment_store_module = original_store_module expected_calls = [ ("experiment.py", "check-git", URL, COMMIT), ("experiment.py", "authorize-git", URL, COMMIT), @@ -189,12 +230,16 @@ def main(argv: list[str]) -> int: "GIT-CLI-001", check == (0, "", "") and authorize == (0, "", "") - and calls == expected_calls, - "exact pinned Git preview forms route once to the adapter", + and install[0] == 0 + and install[2] == "" + and calls == expected_calls + and store_calls == [(Path(raw_home), URL, COMMIT)], + "exact pinned Git public forms route once to their adapters", ) ) calls.clear() + store_calls.clear() malformed = ( ["experiment", "check", "--git"], ["experiment", "check", "--git", URL], @@ -202,13 +247,18 @@ def main(argv: list[str]) -> int: ["experiment", "check", "--commit", COMMIT, "--git", URL], ["experiment", "check", "--git", URL, "--commit", COMMIT, "extra"], ["experiment", "authorize", "install", "--git", URL, "--commit"], + ["experiment", "install", "--git"], + ["experiment", "install", "--git", URL], + ["experiment", "install", "--git", URL, "--commit"], + ["experiment", "install", "--git", URL, "--commit", COMMIT, "extra"], ) usage_results = [invoke(agent_lab, list(argv)) for argv in malformed] results.append( ( "GIT-USAGE-001", all(result[0] == 2 and result[1] == "" for result in usage_results) - and calls == [], + and calls == [] + and store_calls == [], "malformed Git option shapes fail before adapter access", ) ) @@ -362,13 +412,68 @@ def unused_requester(_authority, path, _headers, _maximum, _deadline): platform_outcome = str(error) finally: experiment.sys.platform = original_platform + + public_platform_access: list[str] = [] + + def platform_trap(label: str): + def fail(*_arguments, **_keywords): + public_platform_access.append(label) + raise AssertionError(f"unsupported platform reached {label}") + + return fail + + original_agent_platform = agent_lab.sys.platform + original_agent_experiment_module = agent_lab.experiment_module + original_load_config_receipt = agent_lab.load_config_receipt + original_store_module = agent_lab.experiment_store_module + agent_lab.sys.platform = "darwin" + agent_lab.experiment_module = platform_trap("experiment module") + agent_lab.load_config_receipt = platform_trap("home receipt") + agent_lab.experiment_store_module = platform_trap("store module") + try: + public_platform_results = ( + invoke( + agent_lab, + ["experiment", "check", "--git", URL, "--commit", COMMIT], + ), + invoke( + agent_lab, + [ + "experiment", + "authorize", + "install", + "--git", + URL, + "--commit", + COMMIT, + ], + ), + invoke( + agent_lab, + [ + "experiment", + "install", + "--git", + URL, + "--commit", + COMMIT, + ], + ), + ) + finally: + agent_lab.sys.platform = original_agent_platform + agent_lab.experiment_module = original_agent_experiment_module + agent_lab.load_config_receipt = original_load_config_receipt + agent_lab.experiment_store_module = original_store_module results.append( ( "GIT-PLAT-001", platform_outcome is not None and "GIT-PLATFORM" in platform_outcome - and unused_request_calls == [], - "unsupported hosts refuse before acquisition", + and unused_request_calls == [] + and all(result[0] == 125 for result in public_platform_results) + and public_platform_access == [], + "unsupported hosts refuse every public form before authority access", ) ) @@ -1866,6 +1971,18 @@ def hostile_requester( hostile_transport["acquisition"][field] = False if store._closed_git_transport(hostile_transport) is not None: bool_transport_fields_rejected = False + hostile_transport_urls_rejected = True + for hostile_url in ( + "https://github.com/a--b/repo.git", + "https://github.com/owner/..git", + "https://github.com/owner/...git", + ): + hostile_transport = json.loads( + json.dumps(expected_transport, sort_keys=True) + ) + hostile_transport["url"] = hostile_url + if store._closed_git_transport(hostile_transport) is not None: + hostile_transport_urls_rejected = False class ExperimentFacade: def __init__(self, *, deny: bool = False): @@ -1991,7 +2108,8 @@ def store_call(name: str, *arguments): and install_facade.acquisitions == [(URL, COMMIT)] and installed_artifact == source and closed_git_provenance - and bool_transport_fields_rejected, + and bool_transport_fields_rejected + and hostile_transport_urls_rejected, "permitted Git install publishes the common artifact and closed provenance", ) ) From 9900d45004d86ee5d33c3a57ceb0f21f38c5f55a Mon Sep 17 00:00:00 2001 From: xormania <127287135+xormania@users.noreply.github.com> Date: Sun, 2 Aug 2026 08:46:59 -0400 Subject: [PATCH 145/158] fix(experiment): mirror Git provenance grammar --- scripts/experiment_store.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/scripts/experiment_store.py b/scripts/experiment_store.py index 8eb028a..cbddce2 100644 --- a/scripts/experiment_store.py +++ b/scripts/experiment_store.py @@ -49,8 +49,9 @@ GIT_COMMIT = re.compile(r"^[0-9a-f]{40}$") GIT_URL = re.compile( r"^https://github\.com/" - r"[a-z0-9](?:[a-z0-9-]{0,37}[a-z0-9])?/" - r"[a-z0-9_.-]{1,100}\.git$" + r"([a-z0-9](?:[a-z0-9-]{0,37}[a-z0-9])?)/" + r"([a-z0-9_.-]{1,100})\.git$", + re.ASCII, ) PAYLOAD_DIRECTORIES = {"payload", "payload/artifact", "payload/records"} PAYLOAD_FILES = { @@ -872,6 +873,7 @@ def _closed_git_transport(transport: object) -> dict[str, object] | None: tree = transport.get("tree") blob = transport.get("blob") url = transport.get("url") + url_match = GIT_URL.fullmatch(url) if isinstance(url, str) else None if ( not isinstance(acquisition, dict) or set(acquisition) @@ -902,7 +904,9 @@ def _closed_git_transport(transport: object) -> dict[str, object] | None: or not isinstance(blob, str) or GIT_SHA1.fullmatch(blob) is None or not isinstance(url, str) - or GIT_URL.fullmatch(url) is None + or url_match is None + or "--" in url_match.group(1) + or url_match.group(2) in (".", "..") ): return None return { From 95e91dbe051a927ba66bfe556f23bae10a3ace92 Mon Sep 17 00:00:00 2001 From: xormania <127287135+xormania@users.noreply.github.com> Date: Sun, 2 Aug 2026 08:54:48 -0400 Subject: [PATCH 146/158] test(experiment): bound Git provider failure paths --- tests/experiment/git-intake-cases.py | 363 +++++++++++++++++++++++++-- 1 file changed, 340 insertions(+), 23 deletions(-) diff --git a/tests/experiment/git-intake-cases.py b/tests/experiment/git-intake-cases.py index a14ed57..4070083 100755 --- a/tests/experiment/git-intake-cases.py +++ b/tests/experiment/git-intake-cases.py @@ -714,6 +714,17 @@ def ca_close(descriptor): direct_body = b'{"fixture":true}' direct_connections = [] + direct_response_specs = [ + ( + 200, + [ + ("Content-Length", str(len(direct_body))), + ("Content-Type", "application/json; charset=utf-8"), + ("Content-Encoding", "identity"), + ], + direct_body, + ) + ] original_https_connection = http_client.HTTPSConnection original_ssl_context = ssl_module.SSLContext original_maxline = http_client._MAXLINE @@ -728,23 +739,20 @@ def settimeout(self, value): self.timeouts.append(value) class FakeResponse: - status = 200 - - def __init__(self): + def __init__(self, status, headers, body): + self.status = status + self.headers = headers + self.body = body self.offset = 0 self.closed = False def getheaders(self): - return [ - ("Content-Length", str(len(direct_body))), - ("Content-Type", "application/json; charset=utf-8"), - ("Content-Encoding", "identity"), - ] + return self.headers def read(self, maximum): - if self.offset >= len(direct_body): + if self.offset >= len(self.body): return b"" - chunk = direct_body[self.offset : self.offset + maximum] + chunk = self.body[self.offset : self.offset + maximum] self.offset += len(chunk) return chunk @@ -769,7 +777,22 @@ def __init__(self, authority, *, port, timeout, context): self.timeout = timeout self.context = context self.sock = FakeSocket() - self.response = FakeResponse() + if direct_response_specs: + spec = direct_response_specs.pop(0) + else: + spec = ( + 200, + [ + ("Content-Length", str(len(direct_body))), + ( + "Content-Type", + "application/json; charset=utf-8", + ), + ("Content-Encoding", "identity"), + ], + direct_body, + ) + self.response = FakeResponse(*spec) self.request_record = None self.closed = False direct_connections.append(self) @@ -806,14 +829,194 @@ def close(self): direct_invalid = "accepted" except experiment.InfrastructureError as error: direct_invalid = str(error) + + def direct_status_outcome(status, headers, body, maximum=1_024): + direct_response_specs.append((status, headers, body)) + try: + result = experiment._github_api_request( + "api.github.com", + f"/repos/uscient/experiment-fixture/git/commits/{COMMIT}", + expected_headers, + maximum, + direct_deadline, + ) + try: + experiment._git_provider_json( + lambda *_args: result, + f"/repos/uscient/experiment-fixture/git/commits/{COMMIT}", + maximum, + direct_deadline, + stable_not_found=True, + ) + return "ok" + except experiment.InvalidManifest: + return "reject" + except experiment.InfrastructureError: + return "infra" + except experiment.InfrastructureError: + return "infra" + + framed_headers = [ + ("Content-Length", "2"), + ("Content-Type", "application/json; charset=utf-8"), + ("Content-Encoding", "identity"), + ] + hostile_status_outcomes = ( + direct_status_outcome( + 404, + [*framed_headers, ("Transfer-Encoding", "chunked")], + b"{}", + ), + direct_status_outcome( + 422, + [ + ("Content-Length", "2"), + ("Content-Type", "application/json; charset=utf-8"), + ("Content-Encoding", "gzip"), + ], + b"{}", + ), + direct_status_outcome( + 404, + [ + ("Content-Length", "2"), + ("Content-Type", "text/plain"), + ("Content-Encoding", "identity"), + ], + b"{}", + ), + direct_status_outcome( + 422, + [ + ("Content-Length", "3"), + ("Content-Type", "application/json; charset=utf-8"), + ("Content-Encoding", "identity"), + ], + b"{}", + ), + direct_status_outcome( + 404, + [ + ("Content-Length", "5"), + ("Content-Type", "application/json; charset=utf-8"), + ("Content-Encoding", "identity"), + ], + b"12345", + maximum=4, + ), + direct_status_outcome( + 404, + [ + ("Content-Length", "1"), + ("Content-Type", "application/json; charset=utf-8"), + ("Content-Encoding", "identity"), + ], + b"{", + ), + direct_status_outcome(404, framed_headers, b"{}"), + direct_status_outcome(422, framed_headers, b"{}"), + ) + + def injected_length_outcome(declared): + try: + experiment._git_provider_json( + lambda *_args: ( + 200, + ( + ("content-length", declared), + ( + "content-type", + "application/json; charset=utf-8", + ), + ), + b"{}", + ), + f"/repos/uscient/experiment-fixture/git/commits/{COMMIT}", + 1_024, + direct_deadline, + stable_not_found=True, + ) + return "ok" + except experiment.InfrastructureError: + return "infra" + + injected_lengths_closed = ( + injected_length_outcome("2") == "ok" + and all( + injected_length_outcome(value) == "infra" + for value in (" 2", "+2", "02") + ) + ) + + def forced_stable_outcome(path): + try: + experiment._git_provider_json( + lambda *_args: ( + 404, + ( + ("content-length", "2"), + ( + "content-type", + "application/json; charset=utf-8", + ), + ), + b"{}", + ), + path, + 1_024, + direct_deadline, + stable_not_found=True, + ) + return "ok" + except experiment.InvalidManifest: + return "reject" + except experiment.InfrastructureError: + return "infra" + + stable_not_found_route_closed = all( + forced_stable_outcome(path) == "infra" + for path in ( + f"/repos/uscient/experiment-fixture/git/trees/{TREE}", + f"/repos/uscient/experiment-fixture/git/blobs/{BLOB}", + ) + ) + route_connections_before = len(direct_connections) + route_outcomes = [] + for hostile_path in ( + f"/repos/uscient/experiment-fixture/git/commits/{COMMIT}/../trees/{TREE}", + f"/repos/uscient/experiment-fixture/git/commits/{COMMIT}?recursive=1", + ): + try: + experiment._github_api_request( + "api.github.com", + hostile_path, + expected_headers, + 1_024, + direct_deadline, + ) + route_outcomes.append("accepted") + except experiment.InfrastructureError: + route_outcomes.append("infra") + direct_route_closed = ( + route_outcomes == ["infra", "infra"] + and len(direct_connections) == route_connections_before + ) finally: http_client.HTTPSConnection = original_https_connection ssl_module.SSLContext = original_ssl_context http_client._MAXLINE = original_maxline http_client._MAXHEADERS = original_maxheaders experiment._git_system_ca_pem = original_ca_reader - direct_connection = ( - direct_connections[0] if len(direct_connections) == 1 else None + direct_connection = direct_connections[0] if direct_connections else None + direct_status_framing_ok = hostile_status_outcomes == ( + "infra", + "infra", + "infra", + "infra", + "infra", + "infra", + "reject", + "infra", ) direct_https_ok = ( direct_https @@ -854,7 +1057,8 @@ def close(self): and all(call[2] == expected_headers for call in request_calls) and request_calls[0][3] == 1_048_576 and request_calls[0][3] > request_calls[1][3] > request_calls[2][3] - and direct_https_ok, + and direct_https_ok + and direct_route_closed, "only the fixed credential-free provider authority is requested", ) ) @@ -1001,10 +1205,21 @@ def slow_requester(authority, path, headers, maximum, deadline): def hanging_worker_requester( _authority, _path, _headers, _maximum, _deadline ): + signal.signal(signal.SIGTERM, signal.SIG_IGN) while True: signal.pause() experiment._github_api_request = hanging_worker_requester + calibration_started = __import__("time").monotonic() + __import__("time").sleep(0.01) + calibration_elapsed = ( + __import__("time").monotonic() - calibration_started + ) + worker_timeout_budget = 0.05 + worker_timeout_tolerance = min( + 0.10, + max(0.05, 4 * max(0.0, calibration_elapsed - 0.01)), + ) worker_timeout_started = __import__("time").monotonic() try: try: @@ -1013,7 +1228,7 @@ def hanging_worker_requester( f"/repos/uscient/experiment-fixture/git/commits/{COMMIT}", expected_headers, 1_024, - worker_timeout_started + 0.05, + worker_timeout_started + worker_timeout_budget, ) worker_timeout_outcome = "ok" except experiment.InfrastructureError as error: @@ -1028,8 +1243,9 @@ def hanging_worker_requester( "GIT-TIMEOUT-001", "GIT-TIMEOUT" in timeout_outcome and "GIT-TIMEOUT" in worker_timeout_outcome - and worker_timeout_elapsed < 1.0, - "one absolute acquisition deadline bounds all provider requests", + and worker_timeout_elapsed + <= worker_timeout_budget + worker_timeout_tolerance, + "one absolute deadline includes provider cleanup", ) ) @@ -1124,7 +1340,8 @@ def fixture_worker_child( "GIT-OUTPUT-001", output_outcome[0] == "infra" and "GIT-OUTPUT" in output_outcome[1] - and worker_frames_rejected, + and worker_frames_rejected + and injected_lengths_closed, "provider and worker output frames are strictly bounded", ) ) @@ -1301,6 +1518,93 @@ def mask_failure_requester( "GIT-WORKER" in mask_failure_outcome and not mask_failure_marker.exists() ) + + stop_listener = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + stop_listener.bind(("127.0.0.1", 0)) + stop_listener.settimeout(1.0) + stop_address = stop_listener.getsockname() + original_worker_child = experiment._git_worker_child + + def stopped_before_session( + _control_read, + _stdout_write, + _stderr_write, + _authority, + _path, + _headers, + _maximum, + _deadline, + ): + with socket.socket(socket.AF_INET, socket.SOCK_DGRAM) as sender: + sender.sendto(f"{os.getpid()}\n".encode("ascii"), stop_address) + os.kill(os.getpid(), signal.SIGSTOP) + os._exit(125) + + experiment._git_worker_child = stopped_before_session + try: + try: + experiment._github_worker_request( + "api.github.com", + f"/repos/uscient/experiment-fixture/git/commits/{COMMIT}", + expected_headers, + 1_024, + __import__("time").monotonic() + 0.05, + ) + stopped_outcome = "ok" + except experiment.InfrastructureError: + stopped_outcome = "infra" + except BaseException: + stopped_outcome = "other" + finally: + experiment._git_worker_child = original_worker_child + try: + stopped_record = stop_listener.recv(64).decode("ascii").strip() + except (TimeoutError, UnicodeDecodeError): + stopped_record = "" + finally: + stop_listener.close() + stopped_pid = int(stopped_record) if stopped_record.isdigit() else None + stopped_reaped_before_return = False + stopped_reaped_by_harness = False + stopped_alive = False + if stopped_pid is not None: + try: + waited, _ = os.waitpid(stopped_pid, os.WNOHANG) + except ChildProcessError: + stopped_reaped_before_return = True + stopped_reaped_by_harness = True + else: + stopped_reaped_by_harness = waited == stopped_pid + try: + os.kill(stopped_pid, 0) + except ProcessLookupError: + stopped_alive = False + else: + stopped_alive = True + if not stopped_reaped_by_harness: + if stopped_alive: + try: + os.kill(stopped_pid, signal.SIGKILL) + except ProcessLookupError: + pass + stop_reap_deadline = __import__("time").monotonic() + 1.0 + while __import__("time").monotonic() < stop_reap_deadline: + try: + waited, _ = os.waitpid(stopped_pid, os.WNOHANG) + except ChildProcessError: + stopped_reaped_by_harness = True + break + if waited == stopped_pid: + stopped_reaped_by_harness = True + break + __import__("time").sleep(0.01) + if stopped_pid is None or not stopped_reaped_by_harness: + raise RuntimeError("stopped worker fixture cleanup is uncertain") + stopped_child_cleaned = ( + stopped_outcome == "infra" + and stopped_reaped_before_return + and not stopped_alive + ) results.append( ( "GIT-PGROUP-001", @@ -1308,10 +1612,11 @@ def mask_failure_requester( and worker_snapshot.data == large_source and nonblocking_terminate and mask_clear_fail_closed + and stopped_child_cleaned and len(large_responses[ f"/repos/uscient/experiment-fixture/git/blobs/{large_blob}" ][2]) > 65_536, - "the fixed worker enforces limits, nonblocking cleanup, and a cleared mask", + "the fixed worker enforces limits and cleans pre-session stops", ) ) @@ -1334,6 +1639,7 @@ def cleanup_fault_probe(kind: str) -> bool: original_requester = experiment._github_api_request original_group_probe = experiment._git_worker_group_alive original_close = experiment.os.close + original_selector = experiment.selectors.DefaultSelector close_calls = [0] def cleanup_requester( @@ -1361,11 +1667,16 @@ def failed_cleanup_close(descriptor): raise OSError("cleanup close failed") return original_close(descriptor) + def failed_selector(): + raise OSError("selector construction failed") + experiment._github_api_request = cleanup_requester if kind == "group": experiment._git_worker_group_alive = failed_group_probe elif kind == "close": experiment.os.close = failed_cleanup_close + elif kind == "selector": + experiment.selectors.DefaultSelector = failed_selector try: try: experiment._github_worker_request( @@ -1395,6 +1706,7 @@ def failed_cleanup_close(descriptor): experiment._github_api_request = original_requester experiment._git_worker_group_alive = original_group_probe experiment.os.close = original_close + experiment.selectors.DefaultSelector = original_selector os._exit(0 if outcome == "infra" and restored else 1) probe_status = None @@ -1425,6 +1737,7 @@ def failed_cleanup_close(descriptor): group_probe_fail_closed = cleanup_fault_probe("group") close_failure_fail_closed = cleanup_fault_probe("close") + selector_failure_fail_closed = cleanup_fault_probe("selector") residual_listener = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) residual_listener.bind(("127.0.0.1", 0)) @@ -1600,13 +1913,14 @@ def hanging_requester( and clean_signal_preserved and cleanup_uncertainty_wins and group_probe_fail_closed - and close_failure_fail_closed, + and close_failure_fail_closed + and selector_failure_fail_closed, "cleanup uncertainty fails closed before caller signals are restored", ) ) taxonomy_outcomes = [] - for status in (404, 403, 500): + for status in (404, 422, 403, 500): status_responses = dict(fixture_responses) status_responses[next(iter(fixture_responses))] = ( status, @@ -1653,13 +1967,16 @@ def hanging_requester( taxonomy_outcomes == [ (404, "reject"), + (422, "infra"), (403, "infra"), (500, "infra"), (200, "infra"), ("tree-404", "infra"), ("blob-422", "infra"), - ], - "stable absence is rejection while provider uncertainty is infrastructure", + ] + and direct_status_framing_ok + and stable_not_found_route_closed, + "only a strictly framed commit 404 is stable absence", ) ) From 68c0aaef70cb862b12502bc3f62249533337cdb7 Mon Sep 17 00:00:00 2001 From: xormania <127287135+xormania@users.noreply.github.com> Date: Sun, 2 Aug 2026 09:13:30 -0400 Subject: [PATCH 147/158] test(experiment): retain Git worker identity through cleanup --- tests/experiment/git-intake-cases.py | 322 ++++++++++++++++++++++++++- 1 file changed, 314 insertions(+), 8 deletions(-) diff --git a/tests/experiment/git-intake-cases.py b/tests/experiment/git-intake-cases.py index 4070083..045892b 100755 --- a/tests/experiment/git-intake-cases.py +++ b/tests/experiment/git-intake-cases.py @@ -1201,11 +1201,13 @@ def slow_requester(authority, path, headers, maximum, deadline): experiment.GIT_ACQUISITION_TIMEOUT_SECONDS = original_timeout original_worker_requester = experiment._github_api_request + worker_started_marker = Path(raw_home) / "timeout-worker-started" def hanging_worker_requester( _authority, _path, _headers, _maximum, _deadline ): signal.signal(signal.SIGTERM, signal.SIG_IGN) + worker_started_marker.touch() while True: signal.pause() @@ -1215,7 +1217,11 @@ def hanging_worker_requester( calibration_elapsed = ( __import__("time").monotonic() - calibration_started ) - worker_timeout_budget = 0.05 + worker_operation_budget = 0.25 + worker_cleanup_reserve = 2 * experiment.GIT_WORKER_GRACE_SECONDS + worker_timeout_budget = ( + worker_operation_budget + worker_cleanup_reserve + ) worker_timeout_tolerance = min( 0.10, max(0.05, 4 * max(0.0, calibration_elapsed - 0.01)), @@ -1243,9 +1249,11 @@ def hanging_worker_requester( "GIT-TIMEOUT-001", "GIT-TIMEOUT" in timeout_outcome and "GIT-TIMEOUT" in worker_timeout_outcome + and worker_started_marker.is_file() + and worker_timeout_budget > worker_cleanup_reserve and worker_timeout_elapsed <= worker_timeout_budget + worker_timeout_tolerance, - "one absolute deadline includes provider cleanup", + "a started provider worker and its cleanup share one deadline", ) ) @@ -1326,6 +1334,47 @@ def fixture_worker_child( stderr_frame = worker_frame_outcome( valid_worker_frame, b"caller-private-diagnostic" ) + + def acknowledged_stderr_outcome(): + def fixture_worker_child( + control_read, + stdout_write, + stderr_write, + _authority, + _path, + _headers, + _maximum, + _deadline, + ): + try: + os.setsid() + os.write(stdout_write, valid_worker_frame) + acknowledgement = os.read(control_read, 1) + if acknowledgement == b"1": + os.write( + stderr_write, + b"diagnostic-emitted-after-acknowledgement", + ) + finally: + os._exit(0) + + experiment._git_worker_child = fixture_worker_child + try: + try: + experiment._github_worker_request( + "api.github.com", + f"/repos/uscient/experiment-fixture/git/commits/{COMMIT}", + expected_headers, + 1_024, + __import__("time").monotonic() + 1.0, + ) + return "ok" + except experiment.InfrastructureError as error: + return str(error) + finally: + experiment._git_worker_child = original_worker_child + + acknowledged_stderr_frame = acknowledged_stderr_outcome() worker_frames_rejected = ( "GIT-WORKER" in malformed_frame and "GIT-OUTPUT" in oversized_frame @@ -1334,6 +1383,7 @@ def fixture_worker_child( "diagnostic" in stderr_frame or "GIT-WORKER" in stderr_frame ) + and "diagnostic" in acknowledged_stderr_frame ) results.append( ( @@ -1461,7 +1511,7 @@ def terminate_waitpid(_pid, options): experiment.time.sleep = lambda _duration: None try: terminate_result = experiment._git_worker_terminate( - 991_337, reaped=False + 991_337, reaped=False, deadline=10.0 ) finally: experiment.os.waitpid = original_waitpid @@ -1477,6 +1527,206 @@ def terminate_waitpid(_pid, options): ) ) + released_events: list[tuple[object, ...]] = [] + original_waitpid = experiment.os.waitpid + original_waitid = experiment.os.waitid + original_kill = experiment.os.kill + original_killpg = experiment.os.killpg + original_group_alive = experiment._git_worker_group_alive + original_monotonic = experiment.time.monotonic + original_sleep = experiment.time.sleep + released_clock = [0.0] + + def released_monotonic(): + released_clock[0] += 1.0 + return released_clock[0] + + def released_waitpid(_pid, _options): + released_events.append(("waitpid", _pid, _options)) + return 0, 0 + + def released_waitid(_idtype, _identifier, _options): + released_events.append( + ("waitid", _idtype, _identifier, _options) + ) + return None + + def released_kill(_pid, _signum): + released_events.append(("kill", _pid, _signum)) + + def released_killpg(_pid, _signum): + released_events.append(("killpg", _pid, _signum)) + + def released_group_alive(_pid): + released_events.append(("group-probe", _pid)) + return True + + experiment.os.waitpid = released_waitpid + experiment.os.waitid = released_waitid + experiment.os.kill = released_kill + experiment.os.killpg = released_killpg + experiment._git_worker_group_alive = released_group_alive + experiment.time.monotonic = released_monotonic + experiment.time.sleep = lambda _duration: None + try: + released_result = experiment._git_worker_terminate( + 991_338, reaped=True, deadline=10.0 + ) + finally: + experiment.os.waitpid = original_waitpid + experiment.os.waitid = original_waitid + experiment.os.kill = original_kill + experiment.os.killpg = original_killpg + experiment._git_worker_group_alive = original_group_alive + experiment.time.monotonic = original_monotonic + experiment.time.sleep = original_sleep + released_identity_safe = ( + released_result is False + and released_events == [] + ) + + transition_pid = 991_339 + transition_events: list[tuple[object, ...]] = [] + transition_poll_count = [0] + transition_reaped = [False] + transition_signals: list[int] = [] + transition_clock = [0.0] + + class ObservableWaitidResult: + si_pid = transition_pid + si_uid = 0 + si_signo = signal.SIGCHLD + si_status = 0 + si_code = getattr(os, "CLD_EXITED", 1) + + def transition_monotonic(): + transition_clock[0] += 0.05 + return transition_clock[0] + + def transition_waitpid(pid, options): + if transition_reaped[0]: + transition_events.append( + ("waitpid-after-reap", pid, options) + ) + raise ChildProcessError + transition_poll_count[0] += 1 + if transition_poll_count[0] < 2: + transition_events.append(("waitpid-empty", pid, options)) + return 0, 0 + transition_reaped[0] = True + transition_events.append(("waitpid-reap", pid, options)) + return pid, 0 + + def transition_waitid(idtype, identifier, options): + if transition_reaped[0]: + transition_events.append( + ( + "waitid-after-reap", + idtype, + identifier, + options, + ) + ) + raise ChildProcessError + transition_poll_count[0] += 1 + if transition_poll_count[0] < 2: + transition_events.append( + ("waitid-empty", idtype, identifier, options) + ) + return None + transition_events.append( + ("waitid-observable", idtype, identifier, options) + ) + return ObservableWaitidResult() + + def transition_kill(pid, signum): + transition_events.append(("kill", pid, int(signum))) + if signum: + transition_signals.append(int(signum)) + + def transition_killpg(pid, signum): + transition_events.append(("killpg", pid, int(signum))) + if signum: + transition_signals.append(int(signum)) + + def transition_group_alive(pid): + transition_events.append(("group-probe", pid)) + return signal.SIGKILL not in transition_signals + + original_waitpid = experiment.os.waitpid + original_waitid = experiment.os.waitid + original_kill = experiment.os.kill + original_killpg = experiment.os.killpg + original_group_alive = experiment._git_worker_group_alive + original_monotonic = experiment.time.monotonic + original_sleep = experiment.time.sleep + experiment.os.waitpid = transition_waitpid + experiment.os.waitid = transition_waitid + experiment.os.kill = transition_kill + experiment.os.killpg = transition_killpg + experiment._git_worker_group_alive = transition_group_alive + experiment.time.monotonic = transition_monotonic + experiment.time.sleep = lambda _duration: None + try: + transition_result = experiment._git_worker_terminate( + transition_pid, reaped=False, deadline=10.0 + ) + finally: + experiment.os.waitpid = original_waitpid + experiment.os.waitid = original_waitid + experiment.os.kill = original_kill + experiment.os.killpg = original_killpg + experiment._git_worker_group_alive = original_group_alive + experiment.time.monotonic = original_monotonic + experiment.time.sleep = original_sleep + + transition_reap_positions = [ + index + for index, event in enumerate(transition_events) + if event[0] == "waitpid-reap" + ] + transition_signal_positions = [ + index + for index, event in enumerate(transition_events) + if event[0] in ("kill", "killpg") and event[2] != 0 + ] + transition_waitid_events = [ + event + for event in transition_events + if event[0] in ("waitid-empty", "waitid-observable") + ] + transition_first_reap = ( + transition_reap_positions[0] + if transition_reap_positions + else -1 + ) + transition_identity_safe = ( + transition_result is True + and transition_signals + == [int(signal.SIGTERM), int(signal.SIGKILL)] + and len(transition_reap_positions) == 1 + and transition_first_reap == len(transition_events) - 1 + and transition_signal_positions + and all( + index < transition_first_reap + for index in transition_signal_positions + ) + and any( + event[0] == "waitid-observable" + for event in transition_waitid_events + ) + and all( + event[1] == os.P_PID + and event[2] == transition_pid + and event[3] & os.WNOWAIT + for event in transition_waitid_events + ) + and any( + event[0] == "group-probe" + for event in transition_events[:transition_first_reap] + ) + ) + mask_failure_marker = Path(raw_home) / "mask-clear-requester-reached" mask_parent_pid = os.getpid() original_pthread_sigmask = experiment.signal.pthread_sigmask @@ -1548,7 +1798,7 @@ def stopped_before_session( f"/repos/uscient/experiment-fixture/git/commits/{COMMIT}", expected_headers, 1_024, - __import__("time").monotonic() + 0.05, + __import__("time").monotonic() + 0.75, ) stopped_outcome = "ok" except experiment.InfrastructureError: @@ -1611,6 +1861,8 @@ def stopped_before_session( worker_snapshot is not None and worker_snapshot.data == large_source and nonblocking_terminate + and released_identity_safe + and transition_identity_safe and mask_clear_fail_closed and stopped_child_cleaned and len(large_responses[ @@ -1640,7 +1892,15 @@ def cleanup_fault_probe(kind: str) -> bool: original_group_probe = experiment._git_worker_group_alive original_close = experiment.os.close original_selector = experiment.selectors.DefaultSelector + original_pthread_sigmask = ( + experiment.signal.pthread_sigmask + ) + missing_buffer_factory = object() + original_buffer_factory = experiment.__dict__.get( + "bytearray", missing_buffer_factory + ) close_calls = [0] + parent_setmask_failures = [0] def cleanup_requester( _authority, _path, _headers, _maximum, _deadline @@ -1670,6 +1930,19 @@ def failed_cleanup_close(descriptor): def failed_selector(): raise OSError("selector construction failed") + def failed_buffer_allocation(*_args, **_kwargs): + raise MemoryError("worker buffer allocation failed") + + def fail_parent_spawn_mask_restore(how, signals): + if ( + os.getpid() == cleanup_probe_process + and how == signal.SIG_SETMASK + and parent_setmask_failures[0] == 0 + ): + parent_setmask_failures[0] += 1 + raise OSError("parent spawn mask restore failed") + return original_pthread_sigmask(how, signals) + experiment._github_api_request = cleanup_requester if kind == "group": experiment._git_worker_group_alive = failed_group_probe @@ -1677,6 +1950,12 @@ def failed_selector(): experiment.os.close = failed_cleanup_close elif kind == "selector": experiment.selectors.DefaultSelector = failed_selector + elif kind == "allocation": + experiment.bytearray = failed_buffer_allocation + elif kind == "parent-mask-restore": + experiment.signal.pthread_sigmask = ( + fail_parent_spawn_mask_restore + ) try: try: experiment._github_worker_request( @@ -1707,7 +1986,24 @@ def failed_selector(): experiment._git_worker_group_alive = original_group_probe experiment.os.close = original_close experiment.selectors.DefaultSelector = original_selector - os._exit(0 if outcome == "infra" and restored else 1) + experiment.signal.pthread_sigmask = ( + original_pthread_sigmask + ) + if original_buffer_factory is missing_buffer_factory: + experiment.__dict__.pop("bytearray", None) + else: + experiment.bytearray = original_buffer_factory + injected_fault_observed = ( + kind != "parent-mask-restore" + or parent_setmask_failures[0] == 1 + ) + os._exit( + 0 + if outcome == "infra" + and restored + and injected_fault_observed + else 1 + ) probe_status = None probe_deadline = __import__("time").monotonic() + 2.0 @@ -1738,6 +2034,10 @@ def failed_selector(): group_probe_fail_closed = cleanup_fault_probe("group") close_failure_fail_closed = cleanup_fault_probe("close") selector_failure_fail_closed = cleanup_fault_probe("selector") + allocation_failure_fail_closed = cleanup_fault_probe("allocation") + parent_mask_restore_fail_closed = cleanup_fault_probe( + "parent-mask-restore" + ) residual_listener = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) residual_listener.bind(("127.0.0.1", 0)) @@ -1806,8 +2106,12 @@ def signal_cleanup_probe(*, uncertain: bool, expected_exit: int) -> bool: if signal_probe == 0: original_worker_terminate = experiment._git_worker_terminate - def uncertain_worker_terminate(pid, *, reaped): - original_worker_terminate(pid, reaped=reaped) + def uncertain_worker_terminate(pid, *, reaped, deadline): + original_worker_terminate( + pid, + reaped=reaped, + deadline=deadline, + ) return False def hanging_requester( @@ -1914,7 +2218,9 @@ def hanging_requester( and cleanup_uncertainty_wins and group_probe_fail_closed and close_failure_fail_closed - and selector_failure_fail_closed, + and selector_failure_fail_closed + and allocation_failure_fail_closed + and parent_mask_restore_fail_closed, "cleanup uncertainty fails closed before caller signals are restored", ) ) From 226730400b29116a7f2671848fd2199daf965382 Mon Sep 17 00:00:00 2001 From: xormania <127287135+xormania@users.noreply.github.com> Date: Sun, 2 Aug 2026 09:14:44 -0400 Subject: [PATCH 148/158] test(experiment): observe Git workers without reaping --- tests/experiment/git-intake-cases.py | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/tests/experiment/git-intake-cases.py b/tests/experiment/git-intake-cases.py index 045892b..358189c 100755 --- a/tests/experiment/git-intake-cases.py +++ b/tests/experiment/git-intake-cases.py @@ -1487,7 +1487,9 @@ def worker_requester(authority, path, headers, maximum, deadline): experiment._github_api_request = original_worker_requester terminate_wait_options: list[int] = [] + terminate_observe_options: list[int] = [] original_waitpid = experiment.os.waitpid + original_waitid = experiment.os.waitid original_killpg = experiment.os.killpg original_group_alive = experiment._git_worker_group_alive original_monotonic = experiment.time.monotonic @@ -1504,7 +1506,14 @@ def terminate_waitpid(_pid, options): raise OSError("blocking wait forbidden by fixture") return 0, 0 + def terminate_waitid(_idtype, _identifier, options): + terminate_observe_options.append(options) + if not options & os.WNOHANG or not options & os.WNOWAIT: + raise OSError("consuming or blocking observation forbidden by fixture") + return None + experiment.os.waitpid = terminate_waitpid + experiment.os.waitid = terminate_waitid experiment.os.killpg = lambda _pid, _signal: None experiment._git_worker_group_alive = lambda _pid: True experiment.time.monotonic = terminate_monotonic @@ -1515,13 +1524,18 @@ def terminate_waitpid(_pid, options): ) finally: experiment.os.waitpid = original_waitpid + experiment.os.waitid = original_waitid experiment.os.killpg = original_killpg experiment._git_worker_group_alive = original_group_alive experiment.time.monotonic = original_monotonic experiment.time.sleep = original_sleep nonblocking_terminate = ( terminate_result is False - and terminate_wait_options + and terminate_observe_options + and all( + option & os.WNOHANG and option & os.WNOWAIT + for option in terminate_observe_options + ) and all( option == os.WNOHANG for option in terminate_wait_options ) @@ -1721,10 +1735,6 @@ def transition_group_alive(pid): and event[3] & os.WNOWAIT for event in transition_waitid_events ) - and any( - event[0] == "group-probe" - for event in transition_events[:transition_first_reap] - ) ) mask_failure_marker = Path(raw_home) / "mask-clear-requester-reached" From 9ffeb002b159b8da611587609c814a37670fe343 Mon Sep 17 00:00:00 2001 From: xormania <127287135+xormania@users.noreply.github.com> Date: Sun, 2 Aug 2026 09:30:08 -0400 Subject: [PATCH 149/158] test(experiment): prove Git worker ownership lifecycle --- tests/experiment/git-intake-cases.py | 474 +++++++++++++++++++++++---- 1 file changed, 418 insertions(+), 56 deletions(-) diff --git a/tests/experiment/git-intake-cases.py b/tests/experiment/git-intake-cases.py index 358189c..0286e5e 100755 --- a/tests/experiment/git-intake-cases.py +++ b/tests/experiment/git-intake-cases.py @@ -1335,7 +1335,13 @@ def fixture_worker_child( valid_worker_frame, b"caller-private-diagnostic" ) + late_stderr_bytes = b"hostile-late-stderr-must-not-escape" + def acknowledged_stderr_outcome(): + reached_read, reached_write = os.pipe2( + getattr(os, "O_CLOEXEC", 0) + ) + def fixture_worker_child( control_read, stdout_write, @@ -1351,10 +1357,10 @@ def fixture_worker_child( os.write(stdout_write, valid_worker_frame) acknowledgement = os.read(control_read, 1) if acknowledgement == b"1": - os.write( - stderr_write, - b"diagnostic-emitted-after-acknowledgement", - ) + os.write(reached_write, b"A") + os.write(stderr_write, late_stderr_bytes) + os.write(reached_write, b"W") + __import__("time").sleep(0.05) finally: os._exit(0) @@ -1368,13 +1374,21 @@ def fixture_worker_child( 1_024, __import__("time").monotonic() + 1.0, ) - return "ok" + outcome = "ok" except experiment.InfrastructureError as error: - return str(error) + outcome = str(error) finally: experiment._git_worker_child = original_worker_child + os.close(reached_write) + try: + reached = os.read(reached_read, 16) + finally: + os.close(reached_read) + return outcome, reached - acknowledged_stderr_frame = acknowledged_stderr_outcome() + acknowledged_stderr_frame, acknowledged_stderr_reached = ( + acknowledged_stderr_outcome() + ) worker_frames_rejected = ( "GIT-WORKER" in malformed_frame and "GIT-OUTPUT" in oversized_frame @@ -1383,7 +1397,12 @@ def fixture_worker_child( "diagnostic" in stderr_frame or "GIT-WORKER" in stderr_frame ) - and "diagnostic" in acknowledged_stderr_frame + and "caller-private-diagnostic" not in stderr_frame + and acknowledged_stderr_reached == b"AW" + and acknowledged_stderr_frame + == "git provider GIT-WORKER emitted unexpected diagnostics" + and late_stderr_bytes.decode("ascii") + not in acknowledged_stderr_frame ) results.append( ( @@ -1487,11 +1506,10 @@ def worker_requester(authority, path, headers, maximum, deadline): experiment._github_api_request = original_worker_requester terminate_wait_options: list[int] = [] - terminate_observe_options: list[int] = [] + terminate_observations: list[tuple[object, int, int]] = [] original_waitpid = experiment.os.waitpid original_waitid = experiment.os.waitid original_killpg = experiment.os.killpg - original_group_alive = experiment._git_worker_group_alive original_monotonic = experiment.time.monotonic original_sleep = experiment.time.sleep terminate_clock = [0.0] @@ -1506,16 +1524,23 @@ def terminate_waitpid(_pid, options): raise OSError("blocking wait forbidden by fixture") return 0, 0 - def terminate_waitid(_idtype, _identifier, options): - terminate_observe_options.append(options) - if not options & os.WNOHANG or not options & os.WNOWAIT: + def terminate_waitid(idtype, identifier, options): + terminate_observations.append( + (idtype, identifier, options) + ) + expected = os.WEXITED | os.WNOHANG | os.WNOWAIT + if ( + idtype != os.P_PID + or identifier != 991_337 + or options != expected + or options & (os.WSTOPPED | os.WCONTINUED) + ): raise OSError("consuming or blocking observation forbidden by fixture") return None experiment.os.waitpid = terminate_waitpid experiment.os.waitid = terminate_waitid experiment.os.killpg = lambda _pid, _signal: None - experiment._git_worker_group_alive = lambda _pid: True experiment.time.monotonic = terminate_monotonic experiment.time.sleep = lambda _duration: None try: @@ -1526,15 +1551,19 @@ def terminate_waitid(_idtype, _identifier, options): experiment.os.waitpid = original_waitpid experiment.os.waitid = original_waitid experiment.os.killpg = original_killpg - experiment._git_worker_group_alive = original_group_alive experiment.time.monotonic = original_monotonic experiment.time.sleep = original_sleep nonblocking_terminate = ( terminate_result is False - and terminate_observe_options + and terminate_observations and all( - option & os.WNOHANG and option & os.WNOWAIT - for option in terminate_observe_options + observation + == ( + os.P_PID, + 991_337, + os.WEXITED | os.WNOHANG | os.WNOWAIT, + ) + for observation in terminate_observations ) and all( option == os.WNOHANG for option in terminate_wait_options @@ -1546,7 +1575,6 @@ def terminate_waitid(_idtype, _identifier, options): original_waitid = experiment.os.waitid original_kill = experiment.os.kill original_killpg = experiment.os.killpg - original_group_alive = experiment._git_worker_group_alive original_monotonic = experiment.time.monotonic original_sleep = experiment.time.sleep released_clock = [0.0] @@ -1571,15 +1599,10 @@ def released_kill(_pid, _signum): def released_killpg(_pid, _signum): released_events.append(("killpg", _pid, _signum)) - def released_group_alive(_pid): - released_events.append(("group-probe", _pid)) - return True - experiment.os.waitpid = released_waitpid experiment.os.waitid = released_waitid experiment.os.kill = released_kill experiment.os.killpg = released_killpg - experiment._git_worker_group_alive = released_group_alive experiment.time.monotonic = released_monotonic experiment.time.sleep = lambda _duration: None try: @@ -1591,7 +1614,6 @@ def released_group_alive(_pid): experiment.os.waitid = original_waitid experiment.os.kill = original_kill experiment.os.killpg = original_killpg - experiment._git_worker_group_alive = original_group_alive experiment.time.monotonic = original_monotonic experiment.time.sleep = original_sleep released_identity_safe = ( @@ -1601,7 +1623,8 @@ def released_group_alive(_pid): transition_pid = 991_339 transition_events: list[tuple[object, ...]] = [] - transition_poll_count = [0] + transition_observe_count = [0] + transition_observed = [False] transition_reaped = [False] transition_signals: list[int] = [] transition_clock = [0.0] @@ -1623,12 +1646,15 @@ def transition_waitpid(pid, options): ("waitpid-after-reap", pid, options) ) raise ChildProcessError - transition_poll_count[0] += 1 - if transition_poll_count[0] < 2: - transition_events.append(("waitpid-empty", pid, options)) - return 0, 0 transition_reaped[0] = True - transition_events.append(("waitpid-reap", pid, options)) + transition_events.append( + ( + "waitpid-reap", + pid, + options, + transition_observed[0], + ) + ) return pid, 0 def transition_waitid(idtype, identifier, options): @@ -1642,16 +1668,24 @@ def transition_waitid(idtype, identifier, options): ) ) raise ChildProcessError - transition_poll_count[0] += 1 - if transition_poll_count[0] < 2: + transition_observe_count[0] += 1 + if transition_observe_count[0] < 2: transition_events.append( ("waitid-empty", idtype, identifier, options) ) return None + observed = ObservableWaitidResult() + transition_observed[0] = True transition_events.append( - ("waitid-observable", idtype, identifier, options) + ( + "waitid-observable", + idtype, + identifier, + options, + observed.si_code, + ) ) - return ObservableWaitidResult() + return observed def transition_kill(pid, signum): transition_events.append(("kill", pid, int(signum))) @@ -1663,22 +1697,16 @@ def transition_killpg(pid, signum): if signum: transition_signals.append(int(signum)) - def transition_group_alive(pid): - transition_events.append(("group-probe", pid)) - return signal.SIGKILL not in transition_signals - original_waitpid = experiment.os.waitpid original_waitid = experiment.os.waitid original_kill = experiment.os.kill original_killpg = experiment.os.killpg - original_group_alive = experiment._git_worker_group_alive original_monotonic = experiment.time.monotonic original_sleep = experiment.time.sleep experiment.os.waitpid = transition_waitpid experiment.os.waitid = transition_waitid experiment.os.kill = transition_kill experiment.os.killpg = transition_killpg - experiment._git_worker_group_alive = transition_group_alive experiment.time.monotonic = transition_monotonic experiment.time.sleep = lambda _duration: None try: @@ -1690,7 +1718,6 @@ def transition_group_alive(pid): experiment.os.waitid = original_waitid experiment.os.kill = original_kill experiment.os.killpg = original_killpg - experiment._git_worker_group_alive = original_group_alive experiment.time.monotonic = original_monotonic experiment.time.sleep = original_sleep @@ -1720,6 +1747,7 @@ def transition_group_alive(pid): == [int(signal.SIGTERM), int(signal.SIGKILL)] and len(transition_reap_positions) == 1 and transition_first_reap == len(transition_events) - 1 + and transition_events[transition_first_reap][3] is True and transition_signal_positions and all( index < transition_first_reap @@ -1732,9 +1760,199 @@ def transition_group_alive(pid): and all( event[1] == os.P_PID and event[2] == transition_pid - and event[3] & os.WNOWAIT + and event[3] + == os.WEXITED | os.WNOHANG | os.WNOWAIT + and not event[3] & (os.WSTOPPED | os.WCONTINUED) for event in transition_waitid_events ) + and all( + event[0] != "waitid-observable" + or event[4] == getattr(os, "CLD_EXITED", 1) + for event in transition_waitid_events + ) + ) + + request_order_parent = os.getpid() + request_order_events: list[tuple[str, int, object, bool]] = [] + request_order_released = [False] + original_waitpid = experiment.os.waitpid + original_waitid = experiment.os.waitid + original_kill = experiment.os.kill + original_killpg = experiment.os.killpg + original_worker_child = experiment._git_worker_child + + def request_order_waitpid(pid, options): + released_before = request_order_released[0] + try: + waited, status = original_waitpid(pid, options) + except BaseException: + if os.getpid() == request_order_parent: + request_order_events.append( + ("waitpid-error", pid, options, released_before) + ) + raise + if os.getpid() == request_order_parent: + kind = "waitpid-reap" if waited == pid else "waitpid-empty" + request_order_events.append( + (kind, pid, options, released_before) + ) + if waited == pid: + request_order_released[0] = True + return waited, status + + def request_order_waitid(idtype, identifier, options): + released_before = request_order_released[0] + try: + observed = original_waitid(idtype, identifier, options) + except BaseException: + if os.getpid() == request_order_parent: + request_order_events.append( + ( + "waitid-error", + identifier, + (idtype, options, None), + released_before, + ) + ) + raise + if os.getpid() == request_order_parent: + request_order_events.append( + ( + "waitid-observable" + if observed is not None + else "waitid-empty", + identifier, + ( + idtype, + options, + getattr(observed, "si_code", None), + ), + released_before, + ) + ) + return observed + + def request_order_kill(pid, signum): + if os.getpid() == request_order_parent: + request_order_events.append( + ( + "kill", + pid, + int(signum), + request_order_released[0], + ) + ) + return original_kill(pid, signum) + + def request_order_killpg(pid, signum): + if os.getpid() == request_order_parent: + request_order_events.append( + ( + "killpg", + pid, + int(signum), + request_order_released[0], + ) + ) + return original_killpg(pid, signum) + + def request_order_worker_child( + control_read, + stdout_write, + _stderr_write, + _authority, + _path, + _headers, + _maximum, + _deadline, + ): + try: + os.setsid() + os.write(stdout_write, valid_worker_frame) + exit_code = 0 if os.read(control_read, 1) == b"1" else 125 + except BaseException: + exit_code = 125 + os._exit(exit_code) + + experiment.os.waitpid = request_order_waitpid + experiment.os.waitid = request_order_waitid + experiment.os.kill = request_order_kill + experiment.os.killpg = request_order_killpg + experiment._git_worker_child = request_order_worker_child + try: + try: + request_order_result = experiment._github_worker_request( + "api.github.com", + f"/repos/uscient/experiment-fixture/git/commits/{COMMIT}", + expected_headers, + 1_024, + __import__("time").monotonic() + 1.5, + ) + request_order_outcome = "ok" + except experiment.InfrastructureError as error: + request_order_result = None + request_order_outcome = str(error) + except BaseException as error: + request_order_result = None + request_order_outcome = type(error).__name__ + finally: + experiment.os.waitpid = original_waitpid + experiment.os.waitid = original_waitid + experiment.os.kill = original_kill + experiment.os.killpg = original_killpg + experiment._git_worker_child = original_worker_child + + request_order_waitpid_events = [ + event + for event in request_order_events + if event[0].startswith("waitpid-") + ] + request_order_waitid_events = [ + event + for event in request_order_events + if event[0].startswith("waitid-") + ] + request_order_reap_positions = [ + index + for index, event in enumerate(request_order_events) + if event[0] == "waitpid-reap" + ] + request_order_targets = { + event[1] for event in request_order_events + } + request_identity_safe = ( + request_order_outcome == "ok" + and request_order_result + == ( + 200, + ( + ("content-length", "0"), + ("content-type", "application/json; charset=utf-8"), + ), + b"", + ) + and len(request_order_waitpid_events) == 1 + and request_order_waitpid_events[0][0] == "waitpid-reap" + and request_order_waitpid_events[0][2] == os.WNOHANG + and len(request_order_reap_positions) == 1 + and request_order_reap_positions[0] + == len(request_order_events) - 1 + and request_order_waitid_events + and any( + event[0] == "waitid-observable" + and event[2][2] == getattr(os, "CLD_EXITED", 1) + for event in request_order_waitid_events + ) + and all( + event[2][0] == os.P_PID + and event[2][1] + == os.WEXITED | os.WNOHANG | os.WNOWAIT + and not event[2][1] & (os.WSTOPPED | os.WCONTINUED) + for event in request_order_waitid_events + ) + and len(request_order_targets) == 1 + and next(iter(request_order_targets), -1) > 0 + and not any(event[3] for event in request_order_events) ) mask_failure_marker = Path(raw_home) / "mask-clear-requester-reached" @@ -1873,6 +2091,7 @@ def stopped_before_session( and nonblocking_terminate and released_identity_safe and transition_identity_safe + and request_identity_safe and mask_clear_fail_closed and stopped_child_cleaned and len(large_responses[ @@ -1892,6 +2111,17 @@ def cleanup_fault_probe(kind: str) -> bool: signal.SIGQUIT, signal.SIGTERM, ) + + def seeded_interrupt_handler(_signum, _frame): + return None + + signal.signal(signal.SIGHUP, signal.SIG_DFL) + signal.signal(signal.SIGINT, seeded_interrupt_handler) + signal.signal(signal.SIGQUIT, signal.SIG_DFL) + signal.signal(signal.SIGTERM, signal.SIG_DFL) + signal.pthread_sigmask( + signal.SIG_SETMASK, {signal.SIGQUIT} + ) original_handlers = { signum: signal.getsignal(signum) for signum in managed } @@ -1899,9 +2129,10 @@ def cleanup_fault_probe(kind: str) -> bool: signal.pthread_sigmask(signal.SIG_BLOCK, set()) ) original_requester = experiment._github_api_request - original_group_probe = experiment._git_worker_group_alive original_close = experiment.os.close + original_fork = experiment.os.fork original_selector = experiment.selectors.DefaultSelector + original_signal = experiment.signal.signal original_pthread_sigmask = ( experiment.signal.pthread_sigmask ) @@ -1911,6 +2142,8 @@ def cleanup_fault_probe(kind: str) -> bool: ) close_calls = [0] parent_setmask_failures = [0] + handler_install_attempts = [0] + handler_fork_calls = [0] def cleanup_requester( _authority, _path, _headers, _maximum, _deadline @@ -1927,9 +2160,6 @@ def cleanup_requester( b"", ) - def failed_group_probe(_pid): - raise OSError("process-group probe failed") - def failed_cleanup_close(descriptor): if os.getpid() == cleanup_probe_process: close_calls[0] += 1 @@ -1953,10 +2183,22 @@ def fail_parent_spawn_mask_restore(how, signals): raise OSError("parent spawn mask restore failed") return original_pthread_sigmask(how, signals) + def fail_mid_handler_install(signum, handler): + if ( + os.getpid() == cleanup_probe_process + and handler_install_attempts[0] < 2 + ): + handler_install_attempts[0] += 1 + if handler_install_attempts[0] == 2: + raise OSError("mid-handler install failed") + return original_signal(signum, handler) + + def handler_install_forbidden_fork(): + handler_fork_calls[0] += 1 + raise OSError("worker fork reached after handler failure") + experiment._github_api_request = cleanup_requester - if kind == "group": - experiment._git_worker_group_alive = failed_group_probe - elif kind == "close": + if kind == "close": experiment.os.close = failed_cleanup_close elif kind == "selector": experiment.selectors.DefaultSelector = failed_selector @@ -1966,6 +2208,9 @@ def fail_parent_spawn_mask_restore(how, signals): experiment.signal.pthread_sigmask = ( fail_parent_spawn_mask_restore ) + elif kind == "handler-install": + experiment.signal.signal = fail_mid_handler_install + experiment.os.fork = handler_install_forbidden_fork try: try: experiment._github_worker_request( @@ -1990,12 +2235,16 @@ def fail_parent_spawn_mask_restore(how, signals): signal.pthread_sigmask(signal.SIG_BLOCK, set()) ) == original_mask + and signal.getsignal(signal.SIGINT) + is seeded_interrupt_handler + and original_mask == {signal.SIGQUIT} ) finally: experiment._github_api_request = original_requester - experiment._git_worker_group_alive = original_group_probe experiment.os.close = original_close + experiment.os.fork = original_fork experiment.selectors.DefaultSelector = original_selector + experiment.signal.signal = original_signal experiment.signal.pthread_sigmask = ( original_pthread_sigmask ) @@ -2004,8 +2253,17 @@ def fail_parent_spawn_mask_restore(how, signals): else: experiment.bytearray = original_buffer_factory injected_fault_observed = ( - kind != "parent-mask-restore" - or parent_setmask_failures[0] == 1 + ( + kind != "parent-mask-restore" + or parent_setmask_failures[0] == 1 + ) + and ( + kind != "handler-install" + or ( + handler_install_attempts[0] == 2 + and handler_fork_calls[0] == 0 + ) + ) ) os._exit( 0 @@ -2041,13 +2299,115 @@ def fail_parent_spawn_mask_restore(how, signals): and os.WEXITSTATUS(probe_status) == 0 ) - group_probe_fail_closed = cleanup_fault_probe("group") close_failure_fail_closed = cleanup_fault_probe("close") selector_failure_fail_closed = cleanup_fault_probe("selector") allocation_failure_fail_closed = cleanup_fault_probe("allocation") parent_mask_restore_fail_closed = cleanup_fault_probe( "parent-mask-restore" ) + handler_install_fail_closed = cleanup_fault_probe( + "handler-install" + ) + + def sigchld_precondition_probe(mode: str) -> bool: + probe_pid = os.fork() + if probe_pid == 0: + managed = ( + signal.SIGHUP, + signal.SIGINT, + signal.SIGQUIT, + signal.SIGTERM, + signal.SIGCHLD, + ) + + def custom_child_reaper(_signum, _frame): + return None + + selected_handler = ( + signal.SIG_IGN + if mode == "ignored" + else custom_child_reaper + ) + signal.signal(signal.SIGCHLD, selected_handler) + expected_handlers = { + signum: signal.getsignal(signum) + for signum in managed + } + expected_mask = set( + signal.pthread_sigmask(signal.SIG_BLOCK, set()) + ) + original_fork = experiment.os.fork + fork_calls = [0] + + def forbidden_worker_fork(): + fork_calls[0] += 1 + raise OSError("SIGCHLD precondition reached fork") + + experiment.os.fork = forbidden_worker_fork + try: + try: + experiment._github_worker_request( + "api.github.com", + f"/repos/uscient/experiment-fixture/git/commits/{COMMIT}", + expected_headers, + 1_024, + __import__("time").monotonic() + 1.0, + ) + outcome = "ok" + except experiment.InfrastructureError as error: + outcome = str(error) + except BaseException as error: + outcome = type(error).__name__ + state_unchanged = ( + all( + signal.getsignal(signum) + == expected_handlers[signum] + for signum in managed + ) + and signal.getsignal(signal.SIGCHLD) + == selected_handler + and set( + signal.pthread_sigmask(signal.SIG_BLOCK, set()) + ) + == expected_mask + ) + finally: + experiment.os.fork = original_fork + os._exit( + 0 + if "GIT-SIGNAL" in outcome + and "SIGCHLD" in outcome + and fork_calls[0] == 0 + and state_unchanged + else 1 + ) + + probe_status = None + probe_deadline = __import__("time").monotonic() + 2.0 + while __import__("time").monotonic() < probe_deadline: + waited, status = os.waitpid(probe_pid, os.WNOHANG) + if waited == probe_pid: + probe_status = status + break + __import__("time").sleep(0.01) + if probe_status is None: + try: + os.kill(probe_pid, signal.SIGKILL) + except ProcessLookupError: + pass + waited, status = os.waitpid(probe_pid, 0) + if waited == probe_pid: + probe_status = status + return ( + probe_status is not None + and os.WIFEXITED(probe_status) + and os.WEXITSTATUS(probe_status) == 0 + ) + + ignored_sigchld_fail_closed = sigchld_precondition_probe( + "ignored" + ) + custom_reaper_fail_closed = sigchld_precondition_probe("custom") residual_listener = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) residual_listener.bind(("127.0.0.1", 0)) @@ -2226,11 +2586,13 @@ def hanging_requester( and "residual" in residual_outcome.lower() and clean_signal_preserved and cleanup_uncertainty_wins - and group_probe_fail_closed and close_failure_fail_closed and selector_failure_fail_closed and allocation_failure_fail_closed - and parent_mask_restore_fail_closed, + and parent_mask_restore_fail_closed + and handler_install_fail_closed + and ignored_sigchld_fail_closed + and custom_reaper_fail_closed, "cleanup uncertainty fails closed before caller signals are restored", ) ) From df02b09e7adbe1b0e162c9fdea9f9fc8faf6d612 Mon Sep 17 00:00:00 2001 From: xormania <127287135+xormania@users.noreply.github.com> Date: Sun, 2 Aug 2026 09:30:39 -0400 Subject: [PATCH 150/158] fix(experiment): retain Git worker identity through cleanup --- scripts/experiment.py | 448 ++++++++++++++++--------- tests/experiment/git-mutation-cases.py | 22 +- 2 files changed, 311 insertions(+), 159 deletions(-) diff --git a/scripts/experiment.py b/scripts/experiment.py index 3c8de62..e8c4152 100644 --- a/scripts/experiment.py +++ b/scripts/experiment.py @@ -53,6 +53,13 @@ r"([A-Za-z0-9_.-]{1,100})\.git", re.ASCII, ) +GITHUB_PROVIDER_PATH = re.compile( + r"/repos/" + r"([a-z0-9](?:[a-z0-9-]{0,37}[a-z0-9])?)/" + r"([a-z0-9_.-]{1,100})/git/" + r"(commits|trees|blobs)/([0-9a-f]{40})", + re.ASCII, +) SOURCE_DIGEST_DOMAIN = b"agent-lab.experiment-tree.v1\0" PLAN_DOMAIN = b"agent-lab.experiment-plan.v1\0" BUNDLED_CATALOG_DOMAIN = b"agent-lab.experiment-image-catalog.v1\0" @@ -301,6 +308,18 @@ def _git_object_id(kind: str, payload: bytes) -> str: return hashlib.sha1(framed + payload, usedforsecurity=False).hexdigest() +def _git_provider_route(path: str) -> tuple[str, str, str, str]: + if not isinstance(path, str) or not path.isascii(): + raise InfrastructureError("git provider GIT-AUTHORITY path is malformed") + matched = GITHUB_PROVIDER_PATH.fullmatch(path) + if matched is None: + raise InfrastructureError("git provider GIT-AUTHORITY path is malformed") + owner, repository, object_kind, object_id = matched.groups() + if "--" in owner or repository in (".", ".."): + raise InfrastructureError("git provider GIT-AUTHORITY path is malformed") + return owner, repository, object_kind, object_id + + def _git_provider_json( requester: GitRequester, path: str, @@ -311,6 +330,11 @@ def _git_provider_json( ) -> tuple[object, int]: if remaining <= 0: raise InfrastructureError("git provider GIT-ACQUIRE exhausted its response bound") + _, _, object_kind, _ = _git_provider_route(path) + if not isinstance(stable_not_found, bool) or ( + stable_not_found and object_kind != "commits" + ): + raise InfrastructureError("git provider GIT-TAXONOMY request is malformed") try: status, raw_headers, body = requester( GIT_PROVIDER_AUTHORITY, @@ -344,20 +368,30 @@ def _git_provider_json( headers[lowered] = value if headers.get("content-type") != "application/json; charset=utf-8": raise InfrastructureError("git provider GIT-HEADER content type is uncertain") - try: - declared_length = int(headers.get("content-length", "")) - except ValueError as error: - raise InfrastructureError("git provider GIT-HEADER content length is invalid") from error + declared = headers.get("content-length", "") + if ( + not declared.isascii() + or not declared.isdigit() + or (len(declared) > 1 and declared.startswith("0")) + ): + raise InfrastructureError("git provider GIT-HEADER content length is invalid") + declared_length = int(declared) if ( not isinstance(body, bytes) or len(body) > remaining or declared_length != len(body) ): raise InfrastructureError("git provider GIT-OUTPUT response exceeded its bound") - if status in (404, 422): + if status == 404: + try: + strict_json(body, source="git provider response") + except InvalidManifest as error: + raise InfrastructureError("git provider GIT-JSON response is malformed") from error if stable_not_found: _git_reject("GIT-NOTFOUND", "does not expose the requested public object") raise InfrastructureError("git provider GIT-DRIFT bound object disappeared") + if status == 422: + raise InfrastructureError("git provider GIT-STATUS response is uncertain") if 300 <= status <= 399: raise InfrastructureError("git provider GIT-REDIRECT response is not accepted") if status != 200: @@ -415,10 +449,10 @@ def _github_api_request( ) -> tuple[int, tuple[tuple[str, str], ...], bytes]: """Perform one fixed-authority HTTPS request inside the isolated worker.""" + _git_provider_route(path) if ( authority != GIT_PROVIDER_AUTHORITY or headers != GIT_PROVIDER_HEADERS - or not path.startswith("/repos/") or not 0 < maximum <= MAX_ARCHIVE_BYTES ): raise InfrastructureError("git provider GIT-AUTHORITY request is malformed") @@ -475,15 +509,6 @@ def _github_api_request( if header_bytes > GIT_PROVIDER_MAX_HEADER_BYTES: raise InfrastructureError("git provider GIT-HEADER bytes exceeded their bound") status = response.status - if status != 200: - return ( - status, - ( - ("content-length", "0"), - ("content-type", "application/json; charset=utf-8"), - ), - b"", - ) if "transfer-encoding" in selected or selected.get( "content-encoding", "identity" ) != "identity": @@ -536,68 +561,155 @@ def _github_api_request( raise InfrastructureError("git provider GIT-TRANSPORT request failed") from error -def _git_worker_group_alive(pid: int) -> bool: +def _git_worker_observe(pid: int) -> object | None: + """Observe one owned worker without releasing its numeric identity.""" + try: - os.killpg(pid, 0) - except ProcessLookupError: - return False - except OSError as error: + observed = os.waitid( + os.P_PID, + pid, + os.WEXITED | os.WNOHANG | os.WNOWAIT, + ) + except (ChildProcessError, OSError) as error: raise InfrastructureError( - "git provider GIT-WORKER process-group state is uncertain" + "git provider GIT-WORKER child ownership is uncertain" ) from error - return True + if observed is None: + return None + if ( + getattr(observed, "si_pid", None) != pid + or getattr(observed, "si_signo", None) != signal.SIGCHLD + or getattr(observed, "si_code", None) + not in { + getattr(os, "CLD_EXITED", 1), + getattr(os, "CLD_KILLED", 2), + getattr(os, "CLD_DUMPED", 3), + } + or not isinstance(getattr(observed, "si_status", None), int) + or isinstance(getattr(observed, "si_status", None), bool) + ): + raise InfrastructureError( + "git provider GIT-WORKER child status is malformed" + ) + return observed -def _git_worker_terminate(pid: int, *, reaped: bool) -> bool: +def _git_worker_status_matches(observed: object, status: int) -> bool: + if not isinstance(status, int) or isinstance(status, bool): + return False + code = getattr(observed, "si_code", None) + observed_status = getattr(observed, "si_status", None) + if code == getattr(os, "CLD_EXITED", 1): + return os.WIFEXITED(status) and os.WEXITSTATUS(status) == observed_status + if code == getattr(os, "CLD_KILLED", 2): + return os.WIFSIGNALED(status) and os.WTERMSIG(status) == observed_status + if code == getattr(os, "CLD_DUMPED", 3): + core_dumped = getattr(os, "WCOREDUMP", lambda _status: False) + return ( + os.WIFSIGNALED(status) + and os.WTERMSIG(status) == observed_status + and bool(core_dumped(status)) + ) + return False + + +def _git_worker_exit_code(observed: object | None) -> int | None: + if observed is None or getattr(observed, "si_code", None) != getattr( + os, "CLD_EXITED", 1 + ): + return None + status = getattr(observed, "si_status", None) + if not isinstance(status, int) or isinstance(status, bool) or not 0 <= status <= 255: + return None + return status + + +def _git_worker_terminate(pid: int, *, reaped: bool, deadline: float) -> bool: + """Finalize one owned worker without signaling a released PID or PGID.""" + + if reaped: + return False uncertain = False - for signum in (signal.SIGTERM, signal.SIGKILL): + try: + observed = _git_worker_observe(pid) + except InfrastructureError: + return False + + group_missing = False + try: + os.killpg(pid, signal.SIGTERM) + except ProcessLookupError: + group_missing = True + except OSError: + group_missing = True + uncertain = True + if group_missing: try: - os.killpg(pid, signum) + os.kill(pid, signal.SIGKILL) except ProcessLookupError: - pass + return False except OSError: uncertain = True - if not reaped: - try: - os.kill(pid, signal.SIGKILL) - except ProcessLookupError: - pass - except OSError: - uncertain = True - deadline = time.monotonic() + GIT_WORKER_GRACE_SECONDS - while time.monotonic() < deadline: - if not reaped: - try: - waited, _ = os.waitpid(pid, os.WNOHANG) - except ChildProcessError: - reaped = True - except OSError: - uncertain = True - else: - reaped = waited == pid + + if observed is None: + now = time.monotonic() + term_budget = max(0.0, deadline - now) / 2 + term_deadline = min(deadline, now + min(GIT_WORKER_GRACE_SECONDS, term_budget)) + while time.monotonic() < term_deadline: try: - group_alive = _git_worker_group_alive(pid) - except (InfrastructureError, OSError): - group_alive = True - uncertain = True - if reaped and not group_alive: - return not uncertain - time.sleep(0.01) - if not reaped: + observed = _git_worker_observe(pid) + except InfrastructureError: + return False + if observed is not None: + break + remaining = term_deadline - time.monotonic() + if remaining > 0: + time.sleep(min(0.01, remaining)) + + kill_group_missing = False + try: + os.killpg(pid, signal.SIGKILL) + except ProcessLookupError: + kill_group_missing = True + except OSError: + kill_group_missing = True + uncertain = True + if kill_group_missing: try: - waited, _ = os.waitpid(pid, os.WNOHANG) - except ChildProcessError: - reaped = True + os.kill(pid, signal.SIGKILL) + except ProcessLookupError: + return False except OSError: uncertain = True - else: - reaped = waited == pid + try: + os.killpg(pid, signal.SIGKILL) + except ProcessLookupError: + pass + except OSError: + uncertain = True + + kill_deadline = min( + deadline, + time.monotonic() + GIT_WORKER_GRACE_SECONDS, + ) + while time.monotonic() < kill_deadline: + try: + current = _git_worker_observe(pid) + except InfrastructureError: + return False + if observed is None and current is not None: + observed = current + remaining = kill_deadline - time.monotonic() + if remaining > 0: + time.sleep(min(0.01, remaining)) + try: - group_alive = _git_worker_group_alive(pid) - except (InfrastructureError, OSError): - group_alive = True - uncertain = True - return reaped and not group_alive and not uncertain + waited, status = os.waitpid(pid, os.WNOHANG) + except (ChildProcessError, OSError): + return False + if waited != pid or observed is None: + return False + return _git_worker_status_matches(observed, status) and not uncertain def _git_worker_write(descriptor: int, data: bytes) -> None: @@ -701,7 +813,8 @@ def _github_worker_request( maximum: int, deadline: float, ) -> tuple[int, tuple[tuple[str, str], ...], bytes]: - if time.monotonic() >= deadline: + operation_deadline = deadline - (2 * GIT_WORKER_GRACE_SECONDS) + if time.monotonic() >= operation_deadline: raise InfrastructureError("git provider GIT-TIMEOUT deadline expired") interrupted: int | None = None handlers: dict[int, object] = {} @@ -715,7 +828,7 @@ def interrupt(signum: int, _frame: object) -> None: def change_mask(how: int, signals: set[int]) -> set[signal.Signals]: try: return set(signal.pthread_sigmask(how, signals)) - except (AttributeError, OSError, ValueError) as error: + except (AttributeError, MemoryError, OSError, ValueError) as error: raise InfrastructureError("git provider GIT-SIGNAL mask is unavailable") from error def record_pending(signals: set[int]) -> None: @@ -729,37 +842,54 @@ def record_pending(signals: set[int]) -> None: received = int(signal.sigwait({signum})) if interrupted is None: interrupted = received - except (AttributeError, OSError, ValueError) as error: + except (AttributeError, MemoryError, OSError, ValueError) as error: raise InfrastructureError( "git provider GIT-SIGNAL pending state is uncertain" ) from error original_mask = change_mask(signal.SIG_BLOCK, set()) - try: - for signum in (signal.SIGHUP, signal.SIGINT, signal.SIGQUIT, signal.SIGTERM): - previous = signal.getsignal(signum) - if previous == signal.SIG_IGN or signum in original_mask: - continue - handlers[signum] = previous - signal.signal(signum, interrupt) - managed_signals.add(signum) - except (OSError, ValueError) as error: - for signum, previous in handlers.items(): - signal.signal(signum, previous) - raise InfrastructureError("git provider GIT-SIGNAL handlers are unavailable") from error - control_read = control_write = -1 stdout_read = stdout_write = -1 stderr_read = stderr_write = -1 pid = -1 reaped = False acknowledged = False - selector = selectors.DefaultSelector() - output = bytearray() - errors = bytearray() - process_status: int | None = None - failure: str | None = None + selector = None + output: bytearray | None = None + errors: bytearray | None = None + worker_exit: object | None = None try: + try: + sigchld_handler = signal.getsignal(signal.SIGCHLD) + except (OSError, ValueError) as error: + raise InfrastructureError( + "git provider GIT-SIGNAL SIGCHLD ownership is unavailable" + ) from error + if sigchld_handler != signal.SIG_DFL: + raise InfrastructureError( + "git provider GIT-SIGNAL SIGCHLD ownership is unavailable" + ) + for signum in (signal.SIGHUP, signal.SIGINT, signal.SIGQUIT, signal.SIGTERM): + try: + previous = signal.getsignal(signum) + except (OSError, ValueError) as error: + raise InfrastructureError( + "git provider GIT-SIGNAL handlers are unavailable" + ) from error + if previous == signal.SIG_IGN or signum in original_mask: + continue + handlers[signum] = previous + managed_signals.add(signum) + try: + signal.signal(signum, interrupt) + except (OSError, ValueError) as error: + raise InfrastructureError( + "git provider GIT-SIGNAL handlers are unavailable" + ) from error + + selector = selectors.DefaultSelector() + output = bytearray() + errors = bytearray() control_read, control_write = os.pipe2(getattr(os, "O_CLOEXEC", 0)) stdout_read, stdout_write = os.pipe2(getattr(os, "O_CLOEXEC", 0)) stderr_read, stderr_write = os.pipe2(getattr(os, "O_CLOEXEC", 0)) @@ -786,7 +916,7 @@ def record_pending(signals: set[int]) -> None: path, headers, maximum, - deadline, + operation_deadline, ) os.close(control_read) os.close(stdout_write) @@ -797,14 +927,33 @@ def record_pending(signals: set[int]) -> None: selector.register(stdout_read, selectors.EVENT_READ, "stdout") selector.register(stderr_read, selectors.EVENT_READ, "stderr") expected_frame: int | None = None + drain_deadline: float | None = None + failure: str | None = None while True: if interrupted is not None: failure = "git provider GIT-SIGNAL interrupted acquisition" break - remaining = deadline - time.monotonic() + + worker_exit = _git_worker_observe(pid) + now = time.monotonic() + if worker_exit is not None and drain_deadline is None: + drain_deadline = min( + operation_deadline, + now + GIT_WORKER_GRACE_SECONDS, + ) + active_deadline = ( + operation_deadline if drain_deadline is None else drain_deadline + ) + remaining = active_deadline - now if remaining <= 0: - failure = "git provider GIT-TIMEOUT deadline expired" + if worker_exit is None: + failure = "git provider GIT-TIMEOUT deadline expired" + elif acknowledged: + failure = "git provider GIT-WORKER left a residual process group" + else: + failure = "git provider GIT-WORKER exited before a complete frame" break + for key, _ in selector.select(min(0.05, remaining)): descriptor = int(key.fd) try: @@ -815,6 +964,7 @@ def record_pending(signals: set[int]) -> None: selector.unregister(descriptor) continue if key.data == "stdout": + assert output is not None output.extend(chunk) if len(output) > len(GIT_WORKER_FRAME) + 8 + GIT_WORKER_MAX_OUTPUT_BYTES: failure = "git provider GIT-OUTPUT worker output exceeded its bound" @@ -835,12 +985,17 @@ def record_pending(signals: set[int]) -> None: failure = "git provider GIT-WORKER frame has trailing data" break else: + assert errors is not None errors.extend(chunk) if len(errors) > GIT_WORKER_MAX_ERROR_BYTES: failure = "git provider GIT-OUTPUT worker error exceeded its bound" - break + else: + failure = "git provider GIT-WORKER emitted unexpected diagnostics" + break if failure is not None: break + assert output is not None + assert errors is not None if expected_frame is not None and len(output) == expected_frame and not acknowledged: if errors: failure = "git provider GIT-WORKER emitted unexpected diagnostics" @@ -849,44 +1004,16 @@ def record_pending(signals: set[int]) -> None: os.close(control_write) control_write = -1 acknowledged = True - try: - waited, status = os.waitpid(pid, os.WNOHANG) - except ChildProcessError: - waited = pid - status = 125 << 8 - if waited == pid: - reaped = True - process_status = status - if reaped and acknowledged and _git_worker_group_alive(pid): - failure = "git provider GIT-WORKER left a residual process group" + if worker_exit is not None and not selector.get_map(): + if not acknowledged: + failure = "git provider GIT-WORKER exited before a complete frame" break - if reaped and not selector.get_map(): - break - if reaped and not acknowledged: - failure = "git provider GIT-WORKER exited before a complete frame" - break - if failure is None and not reaped: - remaining = max(0.0, deadline - time.monotonic()) - wait_deadline = time.monotonic() + min(GIT_WORKER_GRACE_SECONDS, remaining) - while time.monotonic() < wait_deadline: - if interrupted is not None: - failure = "git provider GIT-SIGNAL interrupted acquisition" - break - waited, status = os.waitpid(pid, os.WNOHANG) - if waited == pid: - reaped = True - process_status = status - break - time.sleep(0.01) - if not reaped: - failure = "git provider GIT-WORKER did not exit after its result" - if failure is None and _git_worker_group_alive(pid): - failure = "git provider GIT-WORKER left a residual process group" if failure is not None: raise InfrastructureError(failure) - if process_status is None or not os.WIFEXITED(process_status): + returncode = _git_worker_exit_code(worker_exit) + if returncode is None: raise InfrastructureError("git provider GIT-WORKER exit status is uncertain") - returncode = os.WEXITSTATUS(process_status) + assert output is not None header_size = len(GIT_WORKER_FRAME) + 8 if len(output) < header_size: raise InfrastructureError("git provider GIT-WORKER frame is incomplete") @@ -932,16 +1059,21 @@ def record_pending(signals: set[int]) -> None: return int(value["status"]), tuple(result_headers), body except InfrastructureError: raise + except MemoryError as error: + raise InfrastructureError( + "git provider GIT-WORKER could not establish a result" + ) from error except (OSError, ValueError) as error: raise InfrastructureError("git provider GIT-WORKER could not establish a result") from error finally: cleanup_error: InfrastructureError | None = None - try: - selector.close() - except (OSError, ValueError) as error: - cleanup_error = InfrastructureError( - "git provider GIT-WORKER selector cleanup is uncertain" - ) + if selector is not None: + try: + selector.close() + except (MemoryError, OSError, ValueError): + cleanup_error = InfrastructureError( + "git provider GIT-WORKER selector cleanup is uncertain" + ) for descriptor in ( control_read, control_write, @@ -953,34 +1085,32 @@ def record_pending(signals: set[int]) -> None: if descriptor >= 0: try: os.close(descriptor) - except OSError as error: + except (MemoryError, OSError): if cleanup_error is None: cleanup_error = InfrastructureError( "git provider GIT-WORKER descriptor cleanup is uncertain" ) if pid > 0: - needs_termination = not reaped - if not needs_termination: - try: - needs_termination = _git_worker_group_alive(pid) - except (InfrastructureError, OSError): - needs_termination = True - if cleanup_error is None: - cleanup_error = InfrastructureError( - "git provider GIT-WORKER process-group cleanup is uncertain" - ) - if needs_termination: - try: - terminated = _git_worker_terminate(pid, reaped=reaped) - except (InfrastructureError, OSError, ValueError): - terminated = False - if not terminated and cleanup_error is None: - cleanup_error = InfrastructureError( - "git provider GIT-WORKER cleanup is uncertain" - ) - cleanup_mask: set[signal.Signals] | None = None + try: + cleanup_deadline = min( + deadline, + time.monotonic() + (2 * GIT_WORKER_GRACE_SECONDS), + ) + terminated = _git_worker_terminate( + pid, + reaped=reaped, + deadline=cleanup_deadline, + ) + except (InfrastructureError, MemoryError, OSError, ValueError): + terminated = False + if not terminated and cleanup_error is None: + cleanup_error = InfrastructureError( + "git provider GIT-WORKER cleanup is uncertain" + ) + signals_blocked = False try: - cleanup_mask = change_mask(signal.SIG_BLOCK, managed_signals) + change_mask(signal.SIG_BLOCK, managed_signals) + signals_blocked = True record_pending(managed_signals) except InfrastructureError as error: if cleanup_error is None: @@ -989,22 +1119,28 @@ def record_pending(signals: set[int]) -> None: for signum, previous in handlers.items(): try: signal.signal(signum, previous) - except (OSError, ValueError): + except (MemoryError, OSError, ValueError): if cleanup_error is None: cleanup_error = InfrastructureError( "git provider GIT-SIGNAL handlers could not be restored" ) - if cleanup_mask is not None: + if signals_blocked: try: record_pending(managed_signals) except InfrastructureError as error: if cleanup_error is None: cleanup_error = error - try: - change_mask(signal.SIG_SETMASK, set(cleanup_mask)) - except InfrastructureError as error: - if cleanup_error is None: - cleanup_error = error + try: + change_mask(signal.SIG_SETMASK, set(original_mask)) + except (InfrastructureError, MemoryError) as error: + if cleanup_error is None: + cleanup_error = ( + error + if isinstance(error, InfrastructureError) + else InfrastructureError( + "git provider GIT-SIGNAL mask is unavailable" + ) + ) if cleanup_error is not None: raise cleanup_error if interrupted is not None: diff --git a/tests/experiment/git-mutation-cases.py b/tests/experiment/git-mutation-cases.py index c77e1e9..16ddc57 100755 --- a/tests/experiment/git-mutation-cases.py +++ b/tests/experiment/git-mutation-cases.py @@ -540,10 +540,26 @@ def main() -> int: ), ( "M-GIT-BOUND-001", - " terminated = _git_worker_terminate(pid, reaped=reaped)\n", ( - f" Path(os.environ[\"{MARKER_ENV}\"]).touch()\n" - " terminated = True\n" + " terminated = _git_worker_terminate(\n" + " pid,\n" + " reaped=reaped,\n" + " deadline=cleanup_deadline,\n" + " )\n" + ), + ( + f" Path(os.environ[\"{MARKER_ENV}\"]).touch()\n" + " mutant_observed = _git_worker_observe(pid)\n" + " mutant_waited, mutant_status = os.waitpid(\n" + " pid, os.WNOHANG\n" + " )\n" + " terminated = (\n" + " mutant_observed is not None\n" + " and mutant_waited == pid\n" + " and _git_worker_status_matches(\n" + " mutant_observed, mutant_status\n" + " )\n" + " )\n" ), lambda module, marker: bound_probe(module, marker, responses), "removed process-group cleanup leaves a live provider descendant", From 94ace68bc1d0649f0cba80abee807c6683a2d6ed Mon Sep 17 00:00:00 2001 From: xormania <127287135+xormania@users.noreply.github.com> Date: Sun, 2 Aug 2026 09:31:06 -0400 Subject: [PATCH 151/158] test(experiment): require fixed Git worker diagnostics --- tests/experiment/git-intake-cases.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/tests/experiment/git-intake-cases.py b/tests/experiment/git-intake-cases.py index 0286e5e..1a416c2 100755 --- a/tests/experiment/git-intake-cases.py +++ b/tests/experiment/git-intake-cases.py @@ -1393,10 +1393,8 @@ def fixture_worker_child( "GIT-WORKER" in malformed_frame and "GIT-OUTPUT" in oversized_frame and "trailing" in trailing_frame - and ( - "diagnostic" in stderr_frame - or "GIT-WORKER" in stderr_frame - ) + and stderr_frame + == "git provider GIT-WORKER emitted unexpected diagnostics" and "caller-private-diagnostic" not in stderr_frame and acknowledged_stderr_reached == b"AW" and acknowledged_stderr_frame From 9ffe2cf31bc8a4c0482b5d278a28cab40d65a2d5 Mon Sep 17 00:00:00 2001 From: xormania <127287135+xormania@users.noreply.github.com> Date: Sun, 2 Aug 2026 09:42:03 -0400 Subject: [PATCH 152/158] test(experiment): detect Git tree-entry substitution --- tests/experiment/git-intake-cases.py | 46 +++++++++++++++++++++++++++- 1 file changed, 45 insertions(+), 1 deletion(-) diff --git a/tests/experiment/git-intake-cases.py b/tests/experiment/git-intake-cases.py index 1a416c2..6d8092c 100755 --- a/tests/experiment/git-intake-cases.py +++ b/tests/experiment/git-intake-cases.py @@ -638,10 +638,54 @@ def fail(*_arguments, **_keywords): drift_tree ) drift_outcome = acquire(drift_responses) + + bound_drift_source = source.replace(b'"serve"', b'"shell"', 1) + bound_drift_blob = git_oid("blob", bound_drift_source) + bound_drift_blob_path = ( + f"/repos/uscient/experiment-fixture/git/blobs/{bound_drift_blob}" + ) + bound_drift_tree = dict(tree_body) + bound_drift_tree["tree"] = [ + { + "mode": "100644", + "path": "experiment.cue", + "sha": bound_drift_blob, + "size": len(bound_drift_source), + "type": "blob", + } + ] + bound_drift_responses = dict(fixture_responses) + bound_drift_responses[ + f"/repos/uscient/experiment-fixture/git/trees/{TREE}" + ] = response(bound_drift_tree) + bound_drift_responses[bound_drift_blob_path] = response( + { + "content": base64.b64encode(bound_drift_source).decode("ascii"), + "encoding": "base64", + "sha": bound_drift_blob, + "size": len(bound_drift_source), + } + ) + bound_drift_outcome = acquire(bound_drift_responses) + bound_drift_paths = [call[1] for call in bound_drift_outcome[2]] results.append( ( "GIT-DRIFT-001", - drift_outcome[0] == "infra" and "GIT-TREE" in drift_outcome[1], + drift_outcome[0] == "infra" + and "GIT-TREE" in drift_outcome[1] + and bound_drift_source != source + and len(bound_drift_source) == len(source) + and bound_drift_blob != BLOB + and bound_drift_tree["sha"] == TREE + and bound_drift_outcome[0] == "infra" + and bound_drift_outcome[1] + == "git provider GIT-TREE object identity is inconsistent" + and bound_drift_paths + == [ + f"/repos/uscient/experiment-fixture/git/commits/{COMMIT}", + f"/repos/uscient/experiment-fixture/git/trees/{TREE}", + ] + and bound_drift_blob_path not in bound_drift_paths, "changed object output is infrastructure uncertainty", ) ) From e56ff97117709b6f2135fc19648fd418979e8404 Mon Sep 17 00:00:00 2001 From: xormania <127287135+xormania@users.noreply.github.com> Date: Sun, 2 Aug 2026 09:46:44 -0400 Subject: [PATCH 153/158] test(experiment): detect Git blob substitution --- tests/experiment/git-intake-cases.py | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/tests/experiment/git-intake-cases.py b/tests/experiment/git-intake-cases.py index 6d8092c..8b86e6d 100755 --- a/tests/experiment/git-intake-cases.py +++ b/tests/experiment/git-intake-cases.py @@ -620,13 +620,31 @@ def fail(*_arguments, **_keywords): corrupt_blob ) corrupt_outcome = acquire(corrupt_responses) + same_length_source = source.replace(b'"serve"', b'"shell"', 1) + same_length_blob = dict(blob_body) + same_length_blob["content"] = base64.b64encode( + same_length_source + ).decode("ascii") + same_length_responses = dict(fixture_responses) + same_length_responses[ + f"/repos/uscient/experiment-fixture/git/blobs/{BLOB}" + ] = response(same_length_blob) + same_length_outcome = acquire(same_length_responses) results.append( ( "GIT-BLOB-001", wrapped_outcome[0] == "ok" and wrapped_outcome[1].data == source and corrupt_outcome[0] == "infra" - and "GIT-BLOB" in corrupt_outcome[1], + and "GIT-BLOB" in corrupt_outcome[1] + and same_length_source != source + and len(same_length_source) == len(source) + and git_oid("blob", same_length_source) != BLOB + and same_length_blob["sha"] == BLOB + and same_length_blob["size"] == len(source) + and same_length_outcome[0] == "infra" + and same_length_outcome[1] + == "git provider GIT-BLOB object identity is inconsistent", "documented base64 wrapping is accepted but object drift is not", ) ) @@ -639,7 +657,7 @@ def fail(*_arguments, **_keywords): ) drift_outcome = acquire(drift_responses) - bound_drift_source = source.replace(b'"serve"', b'"shell"', 1) + bound_drift_source = same_length_source bound_drift_blob = git_oid("blob", bound_drift_source) bound_drift_blob_path = ( f"/repos/uscient/experiment-fixture/git/blobs/{bound_drift_blob}" From 82e257312fd6b50bc928f3350eda320a531cc80e Mon Sep 17 00:00:00 2001 From: xormania <127287135+xormania@users.noreply.github.com> Date: Sun, 2 Aug 2026 09:47:13 -0400 Subject: [PATCH 154/158] docs(experiment): state Git provider trust binding --- docs/architecture.md | 4 ++++ docs/experiments.md | 14 +++++++++----- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/docs/architecture.md b/docs/architecture.md index e5cf2df..66a2186 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -76,6 +76,10 @@ repository-pinned CUE contract emits one canonical, digest-bound `RequestedExper in-process, and asks the repository-pinned Cedar policy whether the fixed local compatibility principal may submit the exact plan digest. +For public Git, the fixed TLS-authenticated GitHub API binds the requested commit ID to its returned +root-tree ID. The adapter requires the response to echo that commit and independently recomputes the +returned tree and blob Git object IDs before the authored bytes enter the common planning path. + The preview forms create no durable Agent Lab state; Git previews have only their bounded public acquisition effect. `experiment install` instead repeats snapshotting, planning, and Cedar evaluation, then stores the exact permitted evidence in the initialized home. Directory, ZIP, and diff --git a/docs/experiments.md b/docs/experiments.md index bbf8377..af7be1e 100644 --- a/docs/experiments.md +++ b/docs/experiments.md @@ -71,11 +71,15 @@ Git intake is Linux-only in this version. It accepts only a normalized, unauthen ID. A fixed credential-free GitHub Git Data API client reads that commit, its exact root tree, and the bound blob under one five-second deadline and a 1,048,576-byte aggregate response cap. It uses explicit system trust, identity encoding, fixed headers, a private process group, and zero temporary -files. Redirects, credentials, mutable refs, alternate protocols or authorities, extra tree entries, -and changed bound objects fail closed. Agent Lab never runs Git, creates a repository, checks out -content, follows submodules, or executes repository data. Provenance records the canonical URL, -requested and verified object IDs, bounded acquisition facts, and the independent framed SHA-256 -source digest. Git object identity does not replace source identity or change cross-transport retry. +files. The TLS-authenticated fixed GitHub API is the trust binding from the requested commit ID to +the returned root-tree ID: the response must echo the requested commit, and Agent Lab independently +recomputes the returned tree and blob Git object IDs before accepting their bytes. Redirects, +credentials, mutable refs, alternate protocols or authorities, extra tree entries, and changed bound +objects fail closed. Agent Lab never runs Git, creates a repository, checks out content, follows +submodules, or executes repository data. Provenance records the canonical URL, provider-bound commit +ID, independently verified tree/blob IDs, bounded acquisition facts, and the independent framed +SHA-256 source digest. Git object identity does not replace source identity or change cross-transport +retry. An exact retry freshly validates and authorizes again, verifies the complete installed envelope, and returns `changed:false` with the same `installationKey` and `receiptDigest`. The same requested From 80ede7635f9840daf57e56f9d3caebbbf8f93b28 Mon Sep 17 00:00:00 2001 From: xormania <127287135+xormania@users.noreply.github.com> Date: Sun, 2 Aug 2026 09:52:45 -0400 Subject: [PATCH 155/158] test(experiment): disambiguate signal-state marker --- tests/experiment/git-intake-cases.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/experiment/git-intake-cases.py b/tests/experiment/git-intake-cases.py index 8b86e6d..5e2d210 100755 --- a/tests/experiment/git-intake-cases.py +++ b/tests/experiment/git-intake-cases.py @@ -2015,7 +2015,7 @@ def request_order_worker_child( and not any(event[3] for event in request_order_events) ) - mask_failure_marker = Path(raw_home) / "mask-clear-requester-reached" + mask_failure_marker = Path(raw_home) / "signal-state-requester-reached" mask_parent_pid = os.getpid() original_pthread_sigmask = experiment.signal.pthread_sigmask original_worker_requester = experiment._github_api_request From 7decef28af27a99bd936e38e6bfa5a204993d43f Mon Sep 17 00:00:00 2001 From: xormania <127287135+xormania@users.noreply.github.com> Date: Sun, 2 Aug 2026 10:28:04 -0400 Subject: [PATCH 156/158] test(image): preserve catalog replay assertions --- tests/image/catalog-state-cases.py | 41 +++++++++++++++++++++++------- 1 file changed, 32 insertions(+), 9 deletions(-) diff --git a/tests/image/catalog-state-cases.py b/tests/image/catalog-state-cases.py index a9379bf..2de19d3 100755 --- a/tests/image/catalog-state-cases.py +++ b/tests/image/catalog-state-cases.py @@ -1478,8 +1478,13 @@ def fail_pointer_replace(source: os.PathLike[str] | str, target: os.PathLike[str / "payload" / "catalog" ) - shutil.copyfile(staged_catalog / "current.json", staged_catalog / "current.next") - (staged_catalog / "current.next").chmod(0o600) + staged_current = staged_catalog / "current.json" + staged_current_available = True + try: + shutil.copyfile(staged_current, staged_catalog / "current.next") + (staged_catalog / "current.next").chmod(0o600) + except FileNotFoundError: + staged_current_available = False before = fingerprint(phase_home / "images") retry = cli(phase_home, "image", "add", "vendor.worker", SUBJECT) after = fingerprint(phase_home / "images") @@ -1498,14 +1503,20 @@ def fail_pointer_replace(source: os.PathLike[str] | str, target: os.PathLike[str / "image-catalog-operation" / "payload" ) - shutil.rmtree(marker_payload) + marker_payload_available = True + try: + shutil.rmtree(marker_payload) + except FileNotFoundError: + marker_payload_available = False marker_before = fingerprint(marker_phase_home / "images") marker_read = cli(marker_phase_home, "image", "list") marker_retry = cli(marker_phase_home, "image", "add", "vendor.worker", SUBJECT) marker_after = fingerprint(marker_phase_home / "images") check( "CAT-CRASH-008", - child_rc == 99 + staged_current_available + and marker_payload_available + and child_rc == 99 and retry.returncode == 125 and retry.stdout == b"" and before == after @@ -1519,7 +1530,9 @@ def fail_pointer_replace(source: os.PathLike[str] | str, target: os.PathLike[str and not (marker_phase_home / "images" / "catalog").exists(), "phase-inconsistent bootstrap staging remains inert and cannot become committed authority", ( - f"child_rc={child_rc} retry_rc={retry.returncode} changed={before != after} " + f"phase_fixture={staged_current_available} child_rc={child_rc} " + f"retry_rc={retry.returncode} changed={before != after} " + f"marker_fixture={marker_payload_available} " f"marker_child={marker_child} marker_read={marker_read.returncode} " f"marker_retry={marker_retry.returncode} marker_changed={marker_before != marker_after}" ), @@ -1540,19 +1553,29 @@ def fail_pointer_replace(source: os.PathLike[str] | str, target: os.PathLike[str / "image-catalog-operation" / "payload" ) - (incomplete_payload / "entry.json").unlink() - (incomplete_payload / "snapshot.json").unlink() + incomplete_entry = incomplete_payload / "entry.json" + incomplete_snapshot = incomplete_payload / "snapshot.json" + incomplete_payload_available = True + try: + incomplete_entry.unlink() + incomplete_snapshot.unlink() + except FileNotFoundError: + incomplete_payload_available = False before = fingerprint(incomplete_home / "images") retry = cli(incomplete_home, "image", "add", "vendor.second", OTHER_SUBJECT) after = fingerprint(incomplete_home / "images") check( "CAT-CRASH-009", - child_rc == 99 + incomplete_payload_available + and child_rc == 99 and retry.returncode == 125 and retry.stdout == b"" and before == after, "a later stage cannot claim a pointer phase without candidate record evidence", - f"child_rc={child_rc} retry_rc={retry.returncode} changed={before != after}", + ( + f"fixture={incomplete_payload_available} child_rc={child_rc} " + f"retry_rc={retry.returncode} changed={before != after}" + ), ) marker_durable_home = new_home(root, "marker-durable-recovery-home") From 2c0285b90b9772912d38cd31bcaac7858f8aa14d Mon Sep 17 00:00:00 2001 From: xormania <127287135+xormania@users.noreply.github.com> Date: Sun, 2 Aug 2026 10:29:20 -0400 Subject: [PATCH 157/158] test(experiment): classify reserved replay status --- tests/experiment/aggregate-harness-cases.sh | 20 ++++++++++---------- tests/experiment/source-adapter-cases.sh | 7 ++++--- tests/experiment/zip-mutation-cases.py | 13 ++++++++++--- 3 files changed, 24 insertions(+), 16 deletions(-) diff --git a/tests/experiment/aggregate-harness-cases.sh b/tests/experiment/aggregate-harness-cases.sh index d53ab5a..c8df5f1 100755 --- a/tests/experiment/aggregate-harness-cases.sh +++ b/tests/experiment/aggregate-harness-cases.sh @@ -82,7 +82,7 @@ source_zip_ids=( ZIP-RETRY-001 ZIP-DENY-001 ZIP-PLAT-001 ZIP-NOEF-002 ZIP-RUNTIME-001 ) source_mutation_ids=( - M-ZIP-COUNT-001 M-ZIP-NAME-001 M-ZIP-TYPE-001 M-ZIP-FLAG-001 + M-ZIP-WATCHDOG-001 M-ZIP-COUNT-001 M-ZIP-NAME-001 M-ZIP-TYPE-001 M-ZIP-FLAG-001 M-ZIP-METHOD-001 M-ZIP-SIZE-001 M-ZIP-BOMB-001 M-ZIP-CRC-001 M-ZIP-HEADER-001 M-ZIP-EXTRACT-001 M-ZIP-IDENTITY-001 M-ZIP-AUTH-001 ) @@ -748,7 +748,7 @@ source_success_expected="$work/source-success-expected" for id in "${source_expected_ids[@]}"; do printf 'PASS %s fixture assertion\n' "$id" done - printf 'SUMMARY assertions=82 expected=82 failures=0 infra=0\n' + printf 'SUMMARY assertions=83 expected=83 failures=0 infra=0\n' printf 'EXPERIMENT SOURCE ADAPTERS PASS\n' } > "$source_success_expected" if [ "$source_baseline_rc" -eq 0 ] && @@ -767,7 +767,7 @@ write_fixture "$replica/tests/experiment/zip-intake-cases.sh" 0 \ source_missing_rc=0 run_source_replica "$work/source-missing.out" env || source_missing_rc=$? if [ "$source_missing_rc" -eq 1 ] && - grep -Fxq 'SUMMARY assertions=81 expected=82 failures=1 infra=0' \ + grep -Fxq 'SUMMARY assertions=82 expected=83 failures=1 infra=0' \ "$work/source-missing.out" && ! grep -Fxq 'EXPERIMENT SOURCE ADAPTERS PASS' "$work/source-missing.out"; then pass AGG-023 "source-adapter missing assertion identity maps to one" @@ -784,7 +784,7 @@ write_fixture "$replica/tests/experiment/zip-intake-cases.sh" 0 \ source_duplicate_rc=0 run_source_replica "$work/source-duplicate.out" env || source_duplicate_rc=$? if [ "$source_duplicate_rc" -eq 1 ] && - grep -Fxq 'SUMMARY assertions=83 expected=82 failures=1 infra=0' \ + grep -Fxq 'SUMMARY assertions=84 expected=83 failures=1 infra=0' \ "$work/source-duplicate.out" && ! grep -Fxq 'EXPERIMENT SOURCE ADAPTERS PASS' "$work/source-duplicate.out"; then pass AGG-024 "source-adapter duplicate assertion identity maps to one" @@ -801,7 +801,7 @@ write_fixture "$replica/tests/experiment/zip-intake-cases.sh" 0 \ source_substituted_rc=0 run_source_replica "$work/source-substituted.out" env || source_substituted_rc=$? if [ "$source_substituted_rc" -eq 1 ] && - grep -Fxq 'SUMMARY assertions=82 expected=82 failures=1 infra=0' \ + grep -Fxq 'SUMMARY assertions=83 expected=83 failures=1 infra=0' \ "$work/source-substituted.out" && ! grep -Fxq 'EXPERIMENT SOURCE ADAPTERS PASS' "$work/source-substituted.out"; then pass AGG-025 "source-adapter substituted assertion identity maps to one" @@ -818,7 +818,7 @@ write_fixture "$replica/tests/experiment/zip-intake-cases.sh" 1 \ source_assertion_rc=0 run_source_replica "$work/source-assertion.out" env || source_assertion_rc=$? if [ "$source_assertion_rc" -eq 1 ] && - grep -Fxq 'SUMMARY assertions=82 expected=82 failures=1 infra=0' \ + grep -Fxq 'SUMMARY assertions=83 expected=83 failures=1 infra=0' \ "$work/source-assertion.out" && ! grep -Fxq 'EXPERIMENT SOURCE ADAPTERS PASS' "$work/source-assertion.out"; then pass AGG-026 "source-adapter subcase assertion failure maps to one" @@ -834,7 +834,7 @@ write_fixture "$replica/tests/experiment/zip-intake-cases.sh" 125 \ source_subcase_infra_rc=0 run_source_replica "$work/source-subcase-infra.out" env || source_subcase_infra_rc=$? if [ "$source_subcase_infra_rc" -eq 125 ] && - grep -Fxq 'SUMMARY assertions=82 expected=82 failures=0 infra=1' \ + grep -Fxq 'SUMMARY assertions=83 expected=83 failures=0 infra=1' \ "$work/source-subcase-infra.out" && ! grep -Fxq 'EXPERIMENT SOURCE ADAPTERS PASS' "$work/source-subcase-infra.out"; then pass AGG-027 "source-adapter subcase uncertainty maps to one hundred twenty-five" @@ -847,7 +847,7 @@ find "$replica/tests/experiment/zip-mutation-cases.py" -delete source_setup_infra_rc=0 run_source_replica "$work/source-setup-infra.out" env || source_setup_infra_rc=$? if [ "$source_setup_infra_rc" -eq 125 ] && - grep -Fxq 'SUMMARY assertions=70 expected=82 failures=1 infra=1' \ + grep -Fxq 'SUMMARY assertions=70 expected=83 failures=1 infra=1' \ "$work/source-setup-infra.out" && ! grep -Fxq 'EXPERIMENT SOURCE ADAPTERS PASS' "$work/source-setup-infra.out"; then pass AGG-028 "source-adapter setup uncertainty maps to one hundred twenty-five" @@ -864,7 +864,7 @@ source_cleanup_rc=0 run_source_replica "$work/source-cleanup.out" env \ PATH="$source_shim:$PATH" || source_cleanup_rc=$? if [ "$source_cleanup_rc" -eq 125 ] && - grep -Fxq 'SUMMARY assertions=82 expected=82 failures=0 infra=1' \ + grep -Fxq 'SUMMARY assertions=83 expected=83 failures=0 infra=1' \ "$work/source-cleanup.out" && ! grep -Fxq 'EXPERIMENT SOURCE ADAPTERS PASS' "$work/source-cleanup.out"; then pass AGG-029 "source-adapter cleanup uncertainty suppresses the final marker" @@ -885,7 +885,7 @@ chmod +x "$source_summaryless" source_summaryless_rc=0 run_source_replica "$work/source-summaryless.out" env || source_summaryless_rc=$? if [ "$source_summaryless_rc" -eq 125 ] && - grep -Fxq 'SUMMARY assertions=82 expected=82 failures=0 infra=1' \ + grep -Fxq 'SUMMARY assertions=83 expected=83 failures=0 infra=1' \ "$work/source-summaryless.out" && ! grep -Fxq 'EXPERIMENT SOURCE ADAPTERS PASS' "$work/source-summaryless.out"; then pass AGG-030 "source-adapter missing subcase summary maps to one hundred twenty-five" diff --git a/tests/experiment/source-adapter-cases.sh b/tests/experiment/source-adapter-cases.sh index db79a27..38f4ba8 100755 --- a/tests/experiment/source-adapter-cases.sh +++ b/tests/experiment/source-adapter-cases.sh @@ -8,7 +8,7 @@ subcases=( "$repo_root/tests/experiment/git-intake-cases.sh" "$repo_root/tests/experiment/git-mutation-cases.py" ) -expected_count=82 +expected_count=83 work="" cleanup_work() { @@ -37,8 +37,9 @@ printf '%s\n' \ ZIP-SIZE-002 ZIP-READ-001 ZIP-READ-002 ZIP-DECODE-001 ZIP-DECODE-003 ZIP-DECODE-002 ZIP-TIMEOUT-001 \ ZIP-OUTPUT-001 ZIP-NOEF-001 ZIP-AUTH-001 ZIP-INSTALL-001 ZIP-RETRY-001 \ ZIP-DENY-001 ZIP-PLAT-001 ZIP-NOEF-002 ZIP-RUNTIME-001 \ - M-ZIP-COUNT-001 M-ZIP-NAME-001 M-ZIP-TYPE-001 M-ZIP-FLAG-001 \ - M-ZIP-METHOD-001 M-ZIP-SIZE-001 M-ZIP-BOMB-001 M-ZIP-CRC-001 M-ZIP-HEADER-001 \ + M-ZIP-WATCHDOG-001 M-ZIP-COUNT-001 M-ZIP-NAME-001 M-ZIP-TYPE-001 M-ZIP-FLAG-001 \ + M-ZIP-METHOD-001 M-ZIP-SIZE-001 M-ZIP-BOMB-001 M-ZIP-CRC-001 \ + M-ZIP-HEADER-001 \ M-ZIP-EXTRACT-001 M-ZIP-IDENTITY-001 M-ZIP-AUTH-001 \ GIT-CLI-001 GIT-USAGE-001 GIT-URL-001 GIT-OID-001 GIT-PLAT-001 \ GIT-FIXTURE-001 GIT-PIN-001 GIT-COMMIT-001 GIT-ROOT-001 GIT-TYPE-001 \ diff --git a/tests/experiment/zip-mutation-cases.py b/tests/experiment/zip-mutation-cases.py index e9a6402..95f8d31 100644 --- a/tests/experiment/zip-mutation-cases.py +++ b/tests/experiment/zip-mutation-cases.py @@ -15,6 +15,7 @@ EXPECTED = ( + "M-ZIP-WATCHDOG-001", "M-ZIP-COUNT-001", "M-ZIP-NAME-001", "M-ZIP-TYPE-001", @@ -346,6 +347,7 @@ def main() -> int: raise RuntimeError("bounded command helper self-test failed") wrapper_self_test = work / "wrapper-self-test" wrapper_self_test.mkdir() + wrapper_rejected_reserved_status = False try: run_command( repo, @@ -355,9 +357,14 @@ def main() -> int: {"PATH": "/usr/bin:/bin", "LC_ALL": "C"}, ) except RuntimeError: - pass - else: - raise RuntimeError("bounded command wrapper accepted reserved status 125") + wrapper_rejected_reserved_status = True + results.append( + ( + "M-ZIP-WATCHDOG-001", + wrapper_rejected_reserved_status, + "reserved watchdog status is infrastructure uncertainty", + ) + ) fixtures = work / "fixtures" generated = subprocess.run( [ From efe86e474b718f35b8ed4c70ade4ab15a4aeabf8 Mon Sep 17 00:00:00 2001 From: xormania <127287135+xormania@users.noreply.github.com> Date: Sun, 2 Aug 2026 10:29:53 -0400 Subject: [PATCH 158/158] test(experiment): preserve Git lifecycle replay --- tests/experiment/git-intake-cases.py | 30 +++++++++++++++++++--------- 1 file changed, 21 insertions(+), 9 deletions(-) diff --git a/tests/experiment/git-intake-cases.py b/tests/experiment/git-intake-cases.py index 5e2d210..6a69500 100755 --- a/tests/experiment/git-intake-cases.py +++ b/tests/experiment/git-intake-cases.py @@ -1603,10 +1603,14 @@ def terminate_waitid(idtype, identifier, options): experiment.os.killpg = lambda _pid, _signal: None experiment.time.monotonic = terminate_monotonic experiment.time.sleep = lambda _duration: None + terminate_result = None try: - terminate_result = experiment._git_worker_terminate( - 991_337, reaped=False, deadline=10.0 - ) + try: + terminate_result = experiment._git_worker_terminate( + 991_337, reaped=False, deadline=10.0 + ) + except (AttributeError, OSError, TypeError): + pass finally: experiment.os.waitpid = original_waitpid experiment.os.waitid = original_waitid @@ -1665,10 +1669,14 @@ def released_killpg(_pid, _signum): experiment.os.killpg = released_killpg experiment.time.monotonic = released_monotonic experiment.time.sleep = lambda _duration: None + released_result = None try: - released_result = experiment._git_worker_terminate( - 991_338, reaped=True, deadline=10.0 - ) + try: + released_result = experiment._git_worker_terminate( + 991_338, reaped=True, deadline=10.0 + ) + except (AttributeError, OSError, TypeError): + pass finally: experiment.os.waitpid = original_waitpid experiment.os.waitid = original_waitid @@ -1769,10 +1777,14 @@ def transition_killpg(pid, signum): experiment.os.killpg = transition_killpg experiment.time.monotonic = transition_monotonic experiment.time.sleep = lambda _duration: None + transition_result = None try: - transition_result = experiment._git_worker_terminate( - transition_pid, reaped=False, deadline=10.0 - ) + try: + transition_result = experiment._git_worker_terminate( + transition_pid, reaped=False, deadline=10.0 + ) + except (AttributeError, OSError, TypeError): + pass finally: experiment.os.waitpid = original_waitpid experiment.os.waitid = original_waitid