diff --git a/authorization/experiment/v0alpha1/operator.cedar b/authorization/experiment/v0alpha1/operator.cedar index 6f7934a..af03c5f 100644 --- a/authorization/experiment/v0alpha1/operator.cedar +++ b/authorization/experiment/v0alpha1/operator.cedar @@ -11,6 +11,7 @@ when principal.source == "fixed-local-cli" && context.bindingVersion == "v0alpha1" && resource.planDigest == context.planDigest && + resource.sourceDigest == context.sourceDigest && resource.contractDigest == context.contractDigest && resource.contractVersion == "v0alpha1" }; diff --git a/authorization/experiment/v0alpha1/schema.cedarschema b/authorization/experiment/v0alpha1/schema.cedarschema index cdf7d94..0cf9cc4 100644 --- a/authorization/experiment/v0alpha1/schema.cedarschema +++ b/authorization/experiment/v0alpha1/schema.cedarschema @@ -7,6 +7,7 @@ namespace AgentLab { entity RequestedExperimentPlan = { planDigest: String, + sourceDigest: String, contractDigest: String, contractVersion: String, requestedName: String, @@ -20,6 +21,7 @@ namespace AgentLab { context: { bindingVersion: String, planDigest: String, + sourceDigest: String, contractDigest: String, }, }; diff --git a/catalog/experiment-images/v0alpha1.json b/catalog/experiment-images/v0alpha1.json new file mode 100644 index 0000000..00fc07a --- /dev/null +++ b/catalog/experiment-images/v0alpha1.json @@ -0,0 +1 @@ +{"apiVersion":"agent-lab.experiment-images/v0alpha1","entries":[]} diff --git a/contracts/experiment/v0alpha1/plan.cue b/contracts/experiment/v0alpha1/plan.cue index 285eda1..75ec386 100644 --- a/contracts/experiment/v0alpha1/plan.cue +++ b/contracts/experiment/v0alpha1/plan.cue @@ -22,10 +22,10 @@ contractDigest: string & =~"^sha256:[0-9a-f]{64}$" @tag(contractDigest) y: _ less: x.name < y.name }) { - name: member.name - image: member.image - command: member.command - resourceClass: member.resourceClass + name: member.name + requestedSelector: member.image + command: member.command + resourceClass: member.resourceClass }] }) }) diff --git a/contracts/experiment/v0alpha1/schema.cue b/contracts/experiment/v0alpha1/schema.cue index 988cf6f..a099b70 100644 --- a/contracts/experiment/v0alpha1/schema.cue +++ b/contracts/experiment/v0alpha1/schema.cue @@ -8,9 +8,14 @@ import ( #Name: string & strings.MaxRunes(63) & =~"^[a-z](?:[a-z0-9-]{0,61}[a-z0-9])?$" -#Image: string & strings.MinRunes(1) & strings.MaxRunes(255) & +#DigestRef: string & strings.MinRunes(1) & strings.MaxRunes(255) & =~"^([a-z0-9]+([.-][a-z0-9]+)*(:(?:[1-9][0-9]{0,3}|[1-5][0-9]{4}|6[0-4][0-9]{3}|65[0-4][0-9]{2}|655[0-2][0-9]|6553[0-5]))?/)?[a-z0-9]+([._-][a-z0-9]+)*(/[a-z0-9]+([._-][a-z0-9]+)*)*@sha256:[0-9a-f]{64}$" +#CatalogName: string & strings.MaxRunes(63) & + =~"^[a-z][a-z0-9]*(?:-[a-z0-9]+)*\\.[a-z][a-z0-9]*(?:-[a-z0-9]+)*$" + +#Image: close({digestRef: #DigestRef}) | close({catalogName: #CatalogName}) + #Argument: string & strings.MaxRunes(1024) & !~"[\\p{Cc}\\p{Cf}\\p{Zl}\\p{Zp}]" #Member: close({ diff --git a/docs/architecture.md b/docs/architecture.md index 101238e..66a2186 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -67,20 +67,41 @@ capabilities their documented task requires. See [Development](development.md) and [development-agent configuration](agent-config.md). -## Experiment request preflight +## Experiment planning and installed evidence -`scripts/experiment check [--] MANIFEST` validates the closed `agent-lab/v0alpha1` request with the -repository-pinned CUE contract and emits one canonical, digest-bound `RequestedExperimentPlan`. -`scripts/experiment authorize install [--] MANIFEST` reads the manifest once, derives that same plan +`scripts/agent-lab experiment check` snapshots and validates one closed authored source from a sole +local `experiment.cue`, a bounded sole-member ZIP, or an exact supported public GitHub commit. The +repository-pinned CUE contract emits one canonical, digest-bound `RequestedExperimentPlan`. +`scripts/agent-lab experiment authorize install` reads that snapshot once, derives the same plan in-process, and asks the repository-pinned Cedar policy whether the fixed local compatibility principal may submit the exact plan digest. -Both commands are no-effect preflights: they make no container-engine, network, registration, or -runtime changes. A requested name is correlation data, not an installed Experiment identity or -scope. Start, stop, and remove authorization require a future Broker-minted Experiment identity and -Broker-owned ledger, so they are deliberately outside this requested-plan seam. An install permit -only clears the exact requested intent for a future Broker-controlled install path; it neither -actuates the plan nor waives Agent Lab's containment envelope. +For public Git, the fixed TLS-authenticated GitHub API binds the requested commit ID to its returned +root-tree ID. The adapter requires the response to echo that commit and independently recomputes the +returned tree and blob Git object IDs before the authored bytes enter the common planning path. + +The preview forms create no durable Agent Lab state; Git previews have only their bounded public +acquisition effect. `experiment install` instead repeats snapshotting, planning, and Cedar +evaluation, then stores the exact permitted evidence in the initialized home. Directory, ZIP, and +Git sources carrying identical bytes converge on the same source, plan, authorization, installation, +and artifact identities; only their closed transport provenance differs. No caller-supplied decision +is accepted. For a local image name, install rechecks the selected entry under the shared catalog +lock before taking the Experiment store lock exclusively; both remain held through durable +no-replace publication. Direct and bundled selectors do not open local catalog state. + +The installed envelope contains the exact artifact plus closed plan, decision, provenance, and +receipt records. Its installation key binds source, domain-separated plan, contract, authorization, +and selected-entry identities. The receipt binds the artifact and every other evidence record; its +returned receipt digest binds the receipt itself. An exact retry verifies the envelope and returns +the same identity; a different identity under the same requested name never overwrites it. +`experiment inspect NAME` takes the store lock shared and verifies the complete envelope without +repairing staging. + +Installation is persistent onboarding evidence only. It makes no container-engine, network, +registration, image-acquisition, admission, or runtime change and does not waive the containment +envelope. The requested name identifies this stored envelope, not a running Broker identity or +runtime scope. Experiment start, stop, and runtime removal remain future Broker operations; stored +artifact uninstall is also not implemented. ## Workload launch sequence @@ -236,7 +257,7 @@ For formal assumptions and limits, read [Security](../SECURITY.md) and the | Squid and test service | `compose.egress.yaml`, `gateway/squid/` | | workload container | `compose.agent.yaml` and HOME overlays | | workload orchestration | `scripts/agent` | -| Experiment request planning and authorization | `scripts/experiment`, `contracts/experiment/`, `authorization/experiment/` | +| Experiment installed evidence | `scripts/agent-lab`, `scripts/experiment.py`, `scripts/experiment_store.py`, contracts, and authorization policy | | config parsing and validation | `scripts/lib/config.sh` | | project and secret guards | `scripts/lib/guard.sh` | | recipe publication | `scripts/lib/allowlist.sh` | diff --git a/docs/experiments.md b/docs/experiments.md new file mode 100644 index 0000000..af7be1e --- /dev/null +++ b/docs/experiments.md @@ -0,0 +1,121 @@ +# Experiments + +An Experiment is authored as data in a directory containing exactly one file, `experiment.cue`, in a +bounded ZIP archive containing that exact sole member, or at one exact commit of a supported public +GitHub repository whose root tree contains that exact sole blob. The file defines one concrete value +named `experiment` in package `experiment`. Agent Lab snapshots the exact authored bytes privately +before evaluating them; extra entries, links, special files, suspicious modes, changing sources, +malformed CUE, and unknown schema fields are refused. + +```cue +package experiment + +experiment: { + apiVersion: "agent-lab/v0alpha1" + kind: "Experiment" + metadata: name: "example" + spec: members: [{ + name: "worker" + image: digestRef: "registry.example/team/worker@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + command: ["serve"] + }] +} +``` + +Check the artifact or preview its install authorization from the repository: + +```bash +./scripts/agent-lab experiment check ./my-experiment +./scripts/agent-lab experiment authorize install ./my-experiment +./scripts/agent-lab experiment check --zip ./my-experiment.zip +./scripts/agent-lab experiment authorize install --zip ./my-experiment.zip +./scripts/agent-lab experiment check --git https://github.com/owner/repository.git --commit <40 lowercase hex> +./scripts/agent-lab experiment authorize install --git https://github.com/owner/repository.git --commit <40 lowercase hex> +``` + +The check and authorization forms are previews. They create no durable Agent Lab state and do not +invoke Docker or run Experiment content. Git previews do perform the bounded public acquisition +described below. `authorize install` freshly checks the same held source and emits decision evidence +bound to its source, plan, contract, and authorization identities. The decision is not an +installation capability. + +Install a freshly checked and permitted artifact, then inspect its stored identity: + +```bash +agent-lab [--home /absolute/private/home] experiment install ./my-experiment +agent-lab [--home /absolute/private/home] experiment install --zip ./my-experiment.zip +agent-lab [--home /absolute/private/home] experiment install --git https://github.com/owner/repository.git --commit <40 lowercase hex> +agent-lab [--home /absolute/private/home] experiment inspect example +``` + +`install` takes one held source snapshot, derives the plan, and evaluates Cedar again in the same +operation without reopening the caller path. It does not accept a saved plan, decision, destination, +or name override. A permit is evidence for that exact candidate only; installation stores the +source, plan, decision, provenance, and receipt without running content, invoking Docker, acquiring +image bytes, or claiming runtime admission. The decision and receipt bind the same domain-separated +plan identity rather than an unframed hash of the JSON bytes. + +ZIP intake reads one stable archive into at most 1,048,576 bytes and accepts only stored or deflated +`experiment.cue` data that expands to at most 262,144 bytes. ZIP64, multidisk archives, encryption, +comments, extra fields, alternate paths, extra members, special file types, inconsistent headers, +bad CRC or lengths, truncated streams, and trailing bytes are rejected before CUE evaluation. Agent +Lab never extracts the archive or chooses a destination from caller data. The normalized source +digest is identical to directory intake for identical authored bytes; installation provenance also +records the raw archive byte count and SHA-256 digest. Archive identity does not affect the plan, +authorization binding, installation key, or idempotent cross-transport retry. +Regular-file attributes are interpreted only for Unix and DOS-compatible FAT, NTFS, and VFAT +creator systems; other creator systems are rejected when their member type cannot be proven. + +Git intake is Linux-only in this version. It accepts only a normalized, unauthenticated +`https://github.com//.git` URL and one exact lowercase 40-hex SHA-1 commit object +ID. A fixed credential-free GitHub Git Data API client reads that commit, its exact root tree, and +the bound blob under one five-second deadline and a 1,048,576-byte aggregate response cap. It uses +explicit system trust, identity encoding, fixed headers, a private process group, and zero temporary +files. The TLS-authenticated fixed GitHub API is the trust binding from the requested commit ID to +the returned root-tree ID: the response must echo the requested commit, and Agent Lab independently +recomputes the returned tree and blob Git object IDs before accepting their bytes. Redirects, +credentials, mutable refs, alternate protocols or authorities, extra tree entries, and changed bound +objects fail closed. Agent Lab never runs Git, creates a repository, checks out content, follows +submodules, or executes repository data. Provenance records the canonical URL, provider-bound commit +ID, independently verified tree/blob IDs, bounded acquisition facts, and the independent framed +SHA-256 source digest. Git object identity does not replace source identity or change cross-transport +retry. + +An exact retry freshly validates and authorizes again, verifies the complete installed envelope, +and returns `changed:false` with the same `installationKey` and `receiptDigest`. The same requested +name with a different installation identity conflicts without overwrite. `inspect` is read-only: it +verifies and reports one installed identity, but never reconciles staging or repairs state. A later +effectful install may recover only recognized, bounded staging left by an interrupted publication; +unknown or ambiguous residue remains in place and returns infrastructure uncertainty. + +These preview, install, and inspect commands work from a local installation after `agent-lab init` +and explicit `agent-lab tools provision`. Installed execution verifies and uses its release bundle +and the effective home's pinned tool cache; it does not depend on a source checkout. + +Each member selects either an exact digest-pinned OCI reference with `digestRef` or a shared name +with `catalogName`. Shared names have exactly two bounded lowercase components, `.`. +The `agent-lab.*` namespace belongs to the release-owned bundled catalog; the catalog is initially +empty. Other valid namespaces belong to the operator-local catalog shared by every Experiment using +the same effective home: + +```bash +agent-lab image add vendor.image registry.example/team/image@sha256:<64 lowercase hex> +agent-lab image inspect vendor.image +agent-lab image list [--all] +agent-lab image remove vendor.image --expect sha256: +``` + +Add records a mapping only. Same-subject add is idempotent; a different subject never overwrites. +Remove uses the active entry digest as a compare-and-swap token, creates a generation-two tombstone, +and makes the name non-reusable in v0. A local name is resolved from one held, verified catalog +snapshot. The plan binds only the selected entry digest, generation, and immutable subject; checked +evidence separately records the held snapshot revision and digest. An unrelated catalog change +therefore changes catalog evidence without changing the selected plan identity. + +After a fresh install permit, every selected local entry is checked again under the stable shared +catalog lock. That lock remains held while the Experiment store lock is acquired and through durable +publication. Removal therefore cannot make the selected entry stale during installation. Direct +digest selectors and release-owned bundled names do not open the local catalog on this path. + +Catalog membership is naming only, not image presence, admission, safety, or runnable status. See +[`images.md`](images.md) for the exact namespace, mutation, persistence, and failure contract. diff --git a/docs/images.md b/docs/images.md new file mode 100644 index 0000000..6b07222 --- /dev/null +++ b/docs/images.md @@ -0,0 +1,78 @@ +# Local image names + +Agent Lab can assign one operator-local name to an already immutable OCI subject: + +```bash +agent-lab [--home ABSOLUTE_HOME] image add VENDOR.IMAGE DIGEST_REF +agent-lab [--home ABSOLUTE_HOME] image inspect VENDOR.IMAGE +agent-lab [--home ABSOLUTE_HOME] image list [--all] +agent-lab [--home ABSOLUTE_HOME] image remove VENDOR.IMAGE --expect ENTRY_DIGEST +``` + +This catalog is shared by every Experiment using the same initialized Agent Lab home. It stores +names and digest-pinned references only. These commands do not call Docker, contact a registry, +download or inspect image bytes, perform admission, or assert that a subject is runnable. + +## Names and subjects + +A local name has exactly two lowercase ASCII components, `.`. Each component is 1–31 +bytes, begins with a letter, and may contain digits or single hyphen-separated segments. The whole +name is at most 63 bytes. `agent-lab.*` is reserved for the release-owned bundled catalog and cannot +be added, removed, or shadowed locally. + +`DIGEST_REF` uses the same bounded parser as an authored Experiment `digestRef`. It must contain one +exact lowercase `sha256` digest; mutable tags, bare digests, credentials, paths, and ambiguous +references are refused. + +## Mutation rules + +The first add publishes generation 1. Repeating the same name and subject is an idempotent +`changed:false` success. A different subject conflicts and never overwrites. + +Removal requires the active entry digest from add or inspect. An exact compare-and-swap publishes a +generation-2 tombstone. Retrying with the original active digest is idempotent; every other token +conflicts. A tombstoned name cannot be reused or restored in v0alpha1. + +`image remove` removes only the local name from future selection. It does not call Docker, remove +runtime image bytes, stop a workload, or delete an installed Experiment envelope. A retained +installed envelope remains inspectable because it binds the exact selected entry identity; a new or +exact-retry install using the tombstoned name fails its liveness check. Runtime image removal and +stored-artifact uninstall are separate future operations, and neither is implemented by this +command. + +During `experiment install`, a selected local entry is rechecked after the fresh permit under the +stable shared catalog lock. The lock remains held through Experiment publication, so a concurrent +catalog removal cannot invalidate the selected entry mid-install. This proves naming liveness only; +it does not perform image acquisition or admission. + +## Stored authority + +The configured images component contains immutable entry and snapshot histories plus one current +pointer: + +```text +images/catalog/ +|-- current.json +|-- entries/.json +`-- snapshots/.json +``` + +Every read holds the stable catalog lock and verifies canonical schemas, record digests, the +reachable transition chain, all physical history, ownership and modes, fixed counts, and byte +bounds. Unsafe, missing initialized, changing, or corrupt authority returns `125`; an unknown or +removed logical name returns `1`. + +The initialized-home receipt binds the catalog lock's device, inode, path, and schema. The lock +starts with its schema line and appends exactly one `initialized` line only after the first complete +staged catalog is durable and immediately before its no-replace commit. A matching bounded intent +distinguishes that pre-commit recovery window; replacement or any other bytes fail closed. + +Mutations prepare one bounded intent beneath `images/.staging/`. A first catalog uses Linux +no-replace directory publication. Later changes durably publish immutable records before atomically +advancing and syncing the current pointer. The next mutation reconciles a recognized interrupted +intent against the observed pointer. After it proves that cleanup is safe, it durably renames the +operation to one bounded cleanup wrapper before removing any contents, so a crash during cleanup is +restartable. Unknown or unsafe residue is preserved and returns `125`. + +This is tamper-evident state for a cooperative local account, not protection from a hostile process +running as the same user. diff --git a/docs/installation.md b/docs/installation.md new file mode 100644 index 0000000..a15c93a --- /dev/null +++ b/docs/installation.md @@ -0,0 +1,89 @@ +# Local installation + +## Program bundle + +Install the current verified Agent Lab program bundle for one user: + +```bash +./scripts/install-local +# or +./scripts/install-local --prefix /absolute/private/prefix +``` + +The default prefix is the account-database home plus `.local`; ambient `HOME` is not authority. +`--prefix` takes precedence over `AGENT_LAB_PREFIX`. Installation copies only the closed runtime +manifest into a content-addressed release and atomically publishes `/bin/agent-lab`. It does +not use sudo, edit shell profiles, initialize data, or download tools. Add the prefix's `bin` +directory to `PATH` yourself if desired. + +## Initialized home + +Initialize a separate private Agent Lab home: + +```bash +agent-lab --home /absolute/private/home init +agent-lab --home /absolute/private/home config check +agent-lab --home /absolute/private/home config show +agent-lab --home /absolute/private/home tools provision +``` + +The default mutable home is the account-database home plus `.agent-lab`; `--home` takes precedence +over `AGENT_LAB_HOME`. Ambient `HOME` is ignored. The first `init` may choose distinct safe +single-component names with `--experiments-dir`, `--images-dir`, `--cache-dir`, and `--state-dir`. +Those choices are frozen by `home.json`; later drift or conflicting initialization is refused. +The same receipt binds the device, inode, relative path, and schema of both stable lock files. +`config check` and exact `init` retry refuse a replaced lock or unexpected lock contents as +infrastructure uncertainty. + +`tools provision` is the only foundation command allowed to acquire the pinned CUE and Cedar +binaries. Normal commands never download them automatically. Program releases, Experiment data, +image-catalog state, tool cache, and locks remain in separate guarded trees. Exact reinstall and +exact init retry are idempotent; this version does not implement release garbage collection or +in-place home-layout migration. + +## Installed Experiment evidence + +After initialization and tool provisioning, install or inspect an Experiment with the same local +program bundle: + +```bash +agent-lab --home /absolute/private/home experiment install ./my-experiment +agent-lab --home /absolute/private/home experiment install --zip ./my-experiment.zip +agent-lab --home /absolute/private/home experiment install --git https://github.com/owner/repository.git --commit <40 lowercase hex> +agent-lab --home /absolute/private/home experiment inspect NAME +``` + +The configured experiments component is private `0700` state. Each successful first install +publishes this closed layout without overwriting an existing name: + +```text +/ +|-- .staging/ 0700 +`-- NAME/ 0500 + |-- artifact/ 0500 + | `-- experiment.cue 0400 + `-- records/ 0500 + |-- decision.json 0400 + |-- install.json 0400 + |-- plan.json 0400 + `-- provenance.json 0400 +``` + +`install.json` binds the requested name, source, domain-separated plan identity, contract, +authorization, exact selected image entries, and the schema and digest of every other stored file. +`installationKey` is the domain-separated digest of that installation identity; `receiptDigest` +identifies the closed receipt itself. Provenance records the source transport and exact catalog +evidence: `catalog` is `null` for direct digests, `catalog.bundled.snapshotDigest` identifies a +release-owned bundled snapshot, and `catalog.local` contains the checked local snapshot's `revision` +and `snapshotDigest`. Both nested entries are present when a plan uses both namespaces. Provenance +also records one closed source transport: local directory, bounded ZIP byte count and digest, or +canonical public GitHub URL, exact commit/tree/blob identities, and bounded acquisition facts. +Transport provenance is evidence, not authority for a later operation and is excluded from the +installation identity, so equivalent directory, ZIP, and Git sources retry idempotently. + +Every read reopens and verifies the closed layout, canonical bytes, digests, schemas, ownership, +modes, and link counts. `experiment inspect` does this under the shared store lock and never writes +or repairs. An effectful retry takes the lock exclusively and may reconcile only a recognized, +bounded staging operation against the final receipt. Unknown, unsafe, or conflicting state returns +`125` without broad deletion. These controls are tamper-evident for a cooperative local account; +they are not same-user immutability. diff --git a/packaging/agent-lab-local.manifest b/packaging/agent-lab-local.manifest new file mode 100644 index 0000000..b62cbce --- /dev/null +++ b/packaging/agent-lab-local.manifest @@ -0,0 +1,16 @@ +authorization/experiment/v0alpha1/operator.cedar +authorization/experiment/v0alpha1/schema.cedarschema +catalog/experiment-images/v0alpha1.json +contracts/experiment/v0alpha1/cue.mod/module.cue +contracts/experiment/v0alpha1/plan.cue +contracts/experiment/v0alpha1/schema.cue +scripts/agent-lab +scripts/agent-lab.py +scripts/dev/cedar-tool.py +scripts/dev/cue-tool.py +scripts/experiment.py +scripts/experiment_store.py +scripts/image_catalog.py +scripts/image_reference.py +tools/cedar.lock +tools/cue.lock diff --git a/scripts/agent-lab b/scripts/agent-lab new file mode 100755 index 0000000..5eada79 --- /dev/null +++ b/scripts/agent-lab @@ -0,0 +1,6 @@ +#!/usr/bin/env bash +set -euo pipefail + +script_path="$(readlink -f "${BASH_SOURCE[0]}")" +script_dir="$(cd -- "$(dirname -- "$script_path")" >/dev/null 2>&1 && pwd)" +exec python3 -I -B "$script_dir/agent-lab.py" "$@" diff --git a/scripts/agent-lab.py b/scripts/agent-lab.py new file mode 100644 index 0000000..9b22d34 --- /dev/null +++ b/scripts/agent-lab.py @@ -0,0 +1,699 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +from importlib.util import module_from_spec, spec_from_file_location +import hashlib +import json +import os +from pathlib import Path +import pwd +import re +import stat +import subprocess +import sys + +SAFE_COMPONENT = re.compile(r"^[a-z][a-z0-9-]{0,47}$") +MAX_HOME_AUTHORITY_BYTES = 65_536 +LOCK_SPECS = { + "imageCatalog": ( + "image-catalog.lock", + "agent-lab.image-catalog-lock/v0alpha1", + b"initialized\n", + ), + "experiments": ( + "experiments.lock", + "agent-lab.experiments-lock/v0alpha1", + b"", + ), +} + + +def canonical(value: object) -> bytes: + return json.dumps(value, ensure_ascii=True, separators=(",", ":"), sort_keys=True).encode() + + +def effective_home(raw: str | None) -> Path: + selected = raw if raw is not None else os.environ.get("AGENT_LAB_HOME") + if selected is None: + selected = str(Path(pwd.getpwuid(os.getuid()).pw_dir) / ".agent-lab") + path = Path(selected) + if not selected or not path.is_absolute() or path == Path("/"): + raise ValueError("home must be an absolute non-root path") + return path + + +def config_value(components: dict[str, str]) -> dict[str, object]: + return {"apiVersion": "agent-lab.config/v0alpha1", "paths": components} + + +def write_all(descriptor: int, data: bytes) -> None: + view = memoryview(data) + while view: + written = os.write(descriptor, view) + if written <= 0: + raise OSError("write made no progress") + view = view[written:] + + +def home_authority_identity(metadata: os.stat_result) -> tuple[int, int, int, int, int]: + return ( + metadata.st_dev, + metadata.st_ino, + metadata.st_size, + metadata.st_mtime_ns, + metadata.st_ctime_ns, + ) + + +def home_authority_metadata_safe(metadata: os.stat_result) -> bool: + return ( + stat.S_ISREG(metadata.st_mode) + and metadata.st_uid == os.getuid() + and metadata.st_nlink == 1 + and stat.S_IMODE(metadata.st_mode) == 0o600 + and metadata.st_size <= MAX_HOME_AUTHORITY_BYTES + ) + + +def read_home_authority( + path: Path, + lexical: os.stat_result, + purpose: str, +) -> bytes: + """Read one bounded home authority file through a stable no-follow descriptor.""" + + if not home_authority_metadata_safe(lexical): + raise RuntimeError("home authority files are unsafe") + flags = ( + os.O_RDONLY + | getattr(os, "O_CLOEXEC", 0) + | getattr(os, "O_NOFOLLOW", 0) + | getattr(os, "O_NONBLOCK", 0) + ) + try: + descriptor = os.open(path, flags) + except OSError as error: + raise RuntimeError(f"{purpose} authority cannot be opened safely") from error + try: + opened = os.fstat(descriptor) + if ( + not home_authority_metadata_safe(opened) + or home_authority_identity(opened) != home_authority_identity(lexical) + ): + raise RuntimeError(f"{purpose} authority changed before it was read") + chunks: list[bytes] = [] + remaining = MAX_HOME_AUTHORITY_BYTES + 1 + while remaining: + chunk = os.read(descriptor, min(65_536, remaining)) + if not chunk: + break + chunks.append(chunk) + remaining -= len(chunk) + data = b"".join(chunks) + final = os.fstat(descriptor) + except RuntimeError: + raise + except OSError as error: + raise RuntimeError(f"{purpose} authority cannot be read safely") from error + finally: + try: + os.close(descriptor) + except OSError as error: + raise RuntimeError(f"{purpose} authority descriptor cannot be closed") from error + try: + current = path.lstat() + except OSError as error: + raise RuntimeError(f"{purpose} authority cannot be reverified") from error + expected_identity = home_authority_identity(lexical) + if ( + len(data) > MAX_HOME_AUTHORITY_BYTES + or len(data) != final.st_size + or not home_authority_metadata_safe(final) + or not home_authority_metadata_safe(current) + or home_authority_identity(opened) != expected_identity + or home_authority_identity(final) != expected_identity + or home_authority_identity(current) != expected_identity + ): + raise RuntimeError(f"{purpose} authority changed while it was read") + return data + + +def reverify_home_authority(path: Path, expected: os.stat_result, purpose: str) -> None: + try: + current = path.lstat() + except OSError as error: + raise RuntimeError(f"{purpose} authority cannot be reverified") from error + if ( + not home_authority_metadata_safe(current) + or home_authority_identity(current) != home_authority_identity(expected) + ): + raise RuntimeError(f"{purpose} authority changed during preflight") + + +def lock_record(path: Path, relative: str, schema: str) -> dict[str, object]: + metadata = path.lstat() + if ( + not stat.S_ISREG(metadata.st_mode) + or metadata.st_uid != os.getuid() + or metadata.st_nlink != 1 + or stat.S_IMODE(metadata.st_mode) != 0o600 + ): + raise OSError("Agent Lab stable lock metadata is unsafe") + return { + "device": metadata.st_dev, + "inode": metadata.st_ino, + "path": relative, + "schema": schema, + } + + +def validate_lock_receipt(state_component: str, key: str, record: object) -> None: + filename, schema, _ = LOCK_SPECS[key] + relative = f"{state_component}/locks/{filename}" + if ( + not isinstance(record, dict) + or set(record) != {"device", "inode", "path", "schema"} + or not isinstance(record.get("device"), int) + or isinstance(record.get("device"), bool) + or not isinstance(record.get("inode"), int) + or isinstance(record.get("inode"), bool) + or record.get("path") != relative + or record.get("schema") != schema + ): + raise RuntimeError("home lock receipt is not closed") + + +def verify_lock(home: Path, state_component: str, key: str, record: object) -> None: + filename, schema, appended = LOCK_SPECS[key] + relative = f"{state_component}/locks/{filename}" + validate_lock_receipt(state_component, key, record) + assert isinstance(record, dict) + path = home / relative + maximum = len(schema.encode("ascii") + b"\n" + appended) + try: + lexical = path.lstat() + identity = (lexical.st_dev, lexical.st_ino) + if ( + not stat.S_ISREG(lexical.st_mode) + or lexical.st_uid != os.getuid() + or lexical.st_nlink != 1 + or stat.S_IMODE(lexical.st_mode) != 0o600 + or lexical.st_size > maximum + or identity != (record["device"], record["inode"]) + ): + raise OSError("home lock authority metadata is unsafe") + descriptor = os.open( + path, + os.O_RDONLY + | getattr(os, "O_CLOEXEC", 0) + | getattr(os, "O_NOFOLLOW", 0) + | getattr(os, "O_NONBLOCK", 0), + ) + except OSError as error: + raise RuntimeError("home lock authority is unavailable") from error + try: + opened = os.fstat(descriptor) + chunks: list[bytes] = [] + remaining = maximum + 1 + while remaining: + chunk = os.read(descriptor, remaining) + if not chunk: + break + chunks.append(chunk) + remaining -= len(chunk) + data = b"".join(chunks) + final = os.fstat(descriptor) + except OSError as error: + raise RuntimeError("home lock authority could not be verified") from error + finally: + os.close(descriptor) + try: + current = path.lstat() + except OSError as error: + raise RuntimeError("home lock authority could not be reverified") from error + if ( + not stat.S_ISREG(opened.st_mode) + or opened.st_uid != os.getuid() + or opened.st_nlink != 1 + or stat.S_IMODE(opened.st_mode) != 0o600 + or opened.st_size > maximum + or not stat.S_ISREG(final.st_mode) + or final.st_uid != os.getuid() + or final.st_nlink != 1 + or stat.S_IMODE(final.st_mode) != 0o600 + or final.st_size > maximum + or not stat.S_ISREG(current.st_mode) + or current.st_uid != os.getuid() + or current.st_nlink != 1 + or stat.S_IMODE(current.st_mode) != 0o600 + or current.st_size > maximum + or (opened.st_dev, opened.st_ino) != identity + or (final.st_dev, final.st_ino) != identity + or (current.st_dev, current.st_ino) != identity + or identity != (record["device"], record["inode"]) + ): + raise RuntimeError("home lock authority identity is unsafe") + base = schema.encode("ascii") + b"\n" + accepted = (base, base + appended) if appended else (base,) + if data not in accepted or final.st_size != len(data): + raise RuntimeError("home lock authority bytes are invalid") + + +def init_home(home: Path, argv: list[str]) -> int: + components = {"experiments": "experiments", "images": "images", "cache": "cache", "state": "state"} + option_map = {"--experiments-dir": "experiments", "--images-dir": "images", "--cache-dir": "cache", "--state-dir": "state"} + while argv: + option = argv.pop(0) + if option not in option_map or not argv: + return 2 + components[option_map[option]] = argv.pop(0) + if len(set(components.values())) != 4 or any(not SAFE_COMPONENT.fullmatch(value) for value in components.values()): + print("FAIL Agent Lab configuration has unsafe data components", file=sys.stderr) + return 1 + config = config_value(components) + config_bytes = canonical(config) + b"\n" + try: + home.mkdir(mode=0o700, parents=True, exist_ok=True) + os.chmod(home, 0o700) + existing = home / "home.json" + config_path = home / "config.json" + if existing.exists() or config_path.exists(): + try: + loaded = load_config(home) + except RuntimeError as error: + print(f"INFRA Agent Lab {error}", file=sys.stderr) + return 125 + if loaded is not None and loaded[1] == config_bytes: + print("changed:false") + return 0 + print("FAIL Agent Lab home conflicts with requested configuration", file=sys.stderr) + return 1 + component_roots = { + key: home / component + for key, component in components.items() + } + for path in component_roots.values(): + try: + path.lstat() + except FileNotFoundError: + continue + raise OSError("Agent Lab data component already exists") + for path in component_roots.values(): + path.mkdir(mode=0o700) + for key in ("experiments", "images"): + (component_roots[key] / ".staging").mkdir(mode=0o700) + (component_roots["cache"] / "tools/cue").mkdir(mode=0o700, parents=True) + (component_roots["cache"] / "tools/cedar").mkdir(mode=0o700, parents=True) + locks = component_roots["state"] / "locks" + locks.mkdir(mode=0o700) + lock_records: dict[str, object] = {} + for key, (name, schema, _) in LOCK_SPECS.items(): + path = locks / name + descriptor = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + try: + write_all(descriptor, schema.encode("ascii") + b"\n") + os.fsync(descriptor) + finally: + os.close(descriptor) + relative = f"{components['state']}/locks/{name}" + lock_records[key] = lock_record(path, relative, schema) + receipt = { + "apiVersion": "agent-lab.home/v0alpha1", + "configDigest": "sha256:" + hashlib.sha256(canonical(config)).hexdigest(), + "locks": lock_records, + "paths": components, + } + receipt_bytes = canonical(receipt) + b"\n" + for path, data in ((config_path, config_bytes), (existing, receipt_bytes)): + descriptor = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + try: + write_all(descriptor, data) + os.fsync(descriptor) + finally: + os.close(descriptor) + except OSError: + print("INFRA Agent Lab home could not be initialized safely", file=sys.stderr) + return 125 + print("changed:true") + return 0 + + +def load_config_receipt( + home: Path, +) -> tuple[dict[str, object], bytes, dict[str, object]] | None: + config_path = home / "config.json" + receipt_path = home / "home.json" + try: + config_metadata = config_path.lstat() + except FileNotFoundError: + config_metadata = None + except OSError as error: + raise RuntimeError("home configuration cannot be inspected") from error + try: + receipt_metadata = receipt_path.lstat() + except FileNotFoundError: + receipt_metadata = None + except OSError as error: + raise RuntimeError("home receipt cannot be inspected") from error + if config_metadata is None and receipt_metadata is None: + return None + if config_metadata is None or receipt_metadata is None: + raise RuntimeError("home receipt and configuration are incomplete") + raw = read_home_authority(config_path, config_metadata, "configuration") + receipt_raw = read_home_authority(receipt_path, receipt_metadata, "home receipt") + reverify_home_authority(config_path, config_metadata, "configuration") + reverify_home_authority(receipt_path, receipt_metadata, "home receipt") + try: + value = json.loads(raw.decode("utf-8")) + receipt = json.loads(receipt_raw.decode("utf-8")) + except (RecursionError, UnicodeError, ValueError): + raise RuntimeError("configuration is malformed") + if not isinstance(value, dict) or set(value) != {"apiVersion", "paths"} or value["apiVersion"] != "agent-lab.config/v0alpha1": + raise RuntimeError("configuration is not closed") + paths = value["paths"] + if not isinstance(paths, dict) or set(paths) != {"experiments", "images", "cache", "state"}: + raise RuntimeError("configuration paths are not closed") + if len(set(paths.values())) != 4 or any(not isinstance(item, str) or not SAFE_COMPONENT.fullmatch(item) for item in paths.values()): + raise RuntimeError("configuration paths are unsafe") + try: + canonical_config = canonical(value) + b"\n" + canonical_receipt = canonical(receipt) + b"\n" + except (RecursionError, TypeError, ValueError, UnicodeError) as error: + raise RuntimeError("configuration is malformed") from error + if raw != canonical_config: + raise RuntimeError("configuration is not canonical") + if ( + not isinstance(receipt, dict) + or set(receipt) != {"apiVersion", "configDigest", "locks", "paths"} + or receipt["apiVersion"] != "agent-lab.home/v0alpha1" + or receipt["paths"] != paths + or receipt["configDigest"] != "sha256:" + hashlib.sha256(canonical(value)).hexdigest() + or receipt_raw != canonical_receipt + ): + raise RuntimeError("configuration does not match the initialized home receipt") + locks = receipt["locks"] + if not isinstance(locks, dict) or set(locks) != set(LOCK_SPECS): + raise RuntimeError("home lock receipt is not closed") + state_component = paths["state"] + assert isinstance(state_component, str) + for key in LOCK_SPECS: + validate_lock_receipt(state_component, key, locks[key]) + return value, canonical_config, receipt + + +def load_config(home: Path) -> tuple[dict[str, object], bytes] | None: + loaded = load_config_receipt(home) + if loaded is None: + return None + value, canonical_config, receipt = loaded + paths = value["paths"] + locks = receipt["locks"] + assert isinstance(paths, dict) and isinstance(locks, dict) + state_component = paths["state"] + assert isinstance(state_component, str) + for key in LOCK_SPECS: + verify_lock(home, state_component, key, locks[key]) + return value, canonical_config + + +def experiment_module(): + path = Path(__file__).resolve().with_name("experiment.py") + spec = spec_from_file_location("agent_lab_experiment", path) + assert spec is not None and spec.loader is not None + module = module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +def image_catalog_module(): + path = Path(__file__).resolve().with_name("image_catalog.py") + spec = spec_from_file_location("agent_lab_image_catalog", path) + if spec is None or spec.loader is None: + raise ImportError("image catalog module cannot be loaded") + module = module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +def experiment_store_module(): + path = Path(__file__).resolve().with_name("experiment_store.py") + spec = spec_from_file_location("agent_lab_experiment_store", path) + if spec is None or spec.loader is None: + raise ImportError("Experiment store module cannot be loaded") + module = module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +def write_json(value: object) -> None: + output = canonical(value).decode("ascii") + "\n" + written = sys.stdout.write(output) + if written != len(output): + raise OSError("partial command output") + sys.stdout.flush() + + +def image_command(home: Path, argv: list[str]) -> int: + if argv[:1] == ["add"] and len(argv) == 3: + operation = "add" + elif argv[:1] == ["remove"] and len(argv) == 4 and argv[2] == "--expect": + operation = "remove" + elif argv[:1] == ["list"] and (len(argv) == 1 or argv == ["list", "--all"]): + operation = "list" + elif argv[:1] == ["inspect"] and len(argv) == 2: + operation = "inspect" + else: + return 2 + + if operation in {"add", "remove"} and sys.platform != "linux": + print("INFRA Agent Lab local image catalog mutations require Linux", file=sys.stderr) + return 125 + + try: + catalog = image_catalog_module() + except (ImportError, OSError) as error: + print(f"INFRA Agent Lab image catalog is unavailable: {error}", file=sys.stderr) + return 125 + + try: + if operation == "add": + result = catalog.add_image(home, argv[1], argv[2]) + elif operation == "remove": + result = catalog.remove_image(home, argv[1], argv[3]) + elif operation == "list": + result = catalog.list_images(home, include_removed=len(argv) == 2) + else: + result = catalog.inspect_image(home, argv[1]) + except catalog.CatalogReject as error: + print(f"FAIL image {error}", file=sys.stderr) + return 1 + except catalog.CatalogInfrastructure as error: + print(f"INFRA Agent Lab image catalog {error}", file=sys.stderr) + return 125 + except (AttributeError, OSError) as error: + print(f"INFRA Agent Lab image catalog operation is unavailable: {error}", file=sys.stderr) + return 125 + + try: + write_json(result) + except OSError as error: + print(f"INFRA Agent Lab image catalog result is uncertain: {error}", file=sys.stderr) + return 125 + return 0 + + +def experiment_command(home: Path, argv: list[str]) -> int: + if argv in (["install", "--zip"], ["install", "--git"]): + return 2 + if argv[:2] == ["install", "--git"]: + if len(argv) != 5 or argv[3] != "--commit": + return 2 + operation = "install" + source_kind = "git" + elif argv[:1] == ["install"] and len(argv) == 2: + operation = "install" + source_kind = "directory" + elif argv[:2] == ["install", "--zip"] and len(argv) == 3: + operation = "install" + source_kind = "zip" + elif argv[:1] == ["inspect"] and len(argv) == 2: + operation = "inspect" + source_kind = None + else: + return 2 + + if operation == "install" and sys.platform != "linux": + print("INFRA Agent Lab Experiment installation requires Linux", file=sys.stderr) + return 125 + + try: + loaded = load_config_receipt(home) + except RuntimeError as error: + print(f"INFRA Agent Lab {error}", file=sys.stderr) + return 125 + if loaded is None: + print("FAIL Agent Lab home is not initialized", file=sys.stderr) + return 1 + + paths = loaded[0]["paths"] + assert isinstance(paths, dict) + cache = home / str(paths["cache"]) / "tools" + os.environ["AGENT_LAB_HOME"] = str(home) + os.environ.setdefault("AGENT_LAB_CUE_TOOL_DIR", str(cache / "cue")) + os.environ.setdefault("AGENT_LAB_CEDAR_TOOL_DIR", str(cache / "cedar")) + + try: + store = experiment_store_module() + except Exception as error: + print(f"INFRA Agent Lab Experiment store is unavailable: {error}", file=sys.stderr) + return 125 + + try: + if operation == "install": + if source_kind == "git": + result = store.install_git(home, argv[2], argv[4]) + elif source_kind == "zip": + result = store.install_zip(home, Path(argv[2])) + else: + result = store.install_directory(home, Path(argv[1])) + else: + result = store.inspect_install(home, argv[1]) + if not isinstance(result, dict): + raise TypeError("Experiment store returned a non-object result") + except store.StoreReject as error: + print(f"FAIL Experiment {error}", file=sys.stderr) + return 1 + except store.StoreInfrastructure as error: + print(f"INFRA Agent Lab Experiment store {error}", file=sys.stderr) + return 125 + except Exception as error: + print(f"INFRA Agent Lab Experiment store operation is unavailable: {error}", file=sys.stderr) + return 125 + + try: + write_json(result) + except Exception as error: + print(f"INFRA Agent Lab Experiment store result is uncertain: {error}", file=sys.stderr) + return 125 + return 0 + + +def main(argv: list[str]) -> int: + home_raw = None + if argv[:1] == ["--home"]: + if len(argv) < 2: + return 2 + home_raw = argv[1] + argv = argv[2:] + try: + home = effective_home(home_raw) + except ValueError as error: + print(f"FAIL Agent Lab {error}", file=sys.stderr) + return 1 + if argv == ["version"]: + print("agent-lab v0alpha1") + return 0 + if argv[:1] == ["init"]: + return init_home(home, argv[1:]) + if argv == ["config", "check"] or argv == ["config", "show"]: + try: + loaded = load_config(home) + except RuntimeError as error: + print(f"INFRA Agent Lab {error}", file=sys.stderr) + return 125 + if loaded is None: + print("FAIL Agent Lab home is not initialized", file=sys.stderr) + return 1 + if argv[-1] == "show": + sys.stdout.buffer.write(loaded[1]) + else: + print("valid:true") + return 0 + if argv == ["tools", "provision"]: + try: + loaded = load_config(home) + except RuntimeError as error: + print(f"INFRA Agent Lab {error}", file=sys.stderr) + return 125 + if loaded is None: + print("FAIL Agent Lab home is not initialized", file=sys.stderr) + return 1 + paths = loaded[0]["paths"] + assert isinstance(paths, dict) + cache = home / str(paths["cache"]) / "tools" + root = Path(__file__).resolve().parent.parent + environment = {"PATH": "/usr/bin:/bin", "LANG": "C", "LC_ALL": "C"} + for name in ("cue", "cedar"): + environment[f"AGENT_LAB_{name.upper()}_TOOL_DIR"] = str(cache / name) + completed = subprocess.run( + [sys.executable, "-I", str(root / f"scripts/dev/{name}-tool.py"), "provision"], + env=environment, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + timeout=120, + check=False, + ) + if completed.returncode != 0: + sys.stderr.buffer.write(completed.stderr) + return 125 + print("tools:ready") + return 0 + if argv[:3] == ["experiment", "check", "--git"]: + if len(argv) != 6 or argv[4] != "--commit": + return 2 + if sys.platform != "linux": + print("INFRA Agent Lab Git Experiment intake requires Linux", file=sys.stderr) + return 125 + os.environ["AGENT_LAB_HOME"] = str(home) + os.environ.setdefault("AGENT_LAB_CUE_TOOL_DIR", str(home / "cache/tools/cue")) + return experiment_module().main( + ["experiment.py", "check-git", argv[3], argv[5]] + ) + if argv[:4] == ["experiment", "authorize", "install", "--git"]: + if len(argv) != 7 or argv[5] != "--commit": + return 2 + if sys.platform != "linux": + print("INFRA Agent Lab Git Experiment intake requires Linux", file=sys.stderr) + return 125 + os.environ["AGENT_LAB_HOME"] = str(home) + os.environ.setdefault("AGENT_LAB_CUE_TOOL_DIR", str(home / "cache/tools/cue")) + os.environ.setdefault("AGENT_LAB_CEDAR_TOOL_DIR", str(home / "cache/tools/cedar")) + return experiment_module().main( + ["experiment.py", "authorize-git", argv[4], argv[6]] + ) + if argv in ( + ["experiment", "check", "--zip"], + ["experiment", "authorize", "install", "--zip"], + ): + return 2 + if argv[:3] == ["experiment", "check", "--zip"] and len(argv) == 4: + os.environ["AGENT_LAB_HOME"] = str(home) + os.environ.setdefault("AGENT_LAB_CUE_TOOL_DIR", str(home / "cache/tools/cue")) + return experiment_module().main(["experiment.py", "check-zip", argv[3]]) + if argv[:2] == ["experiment", "check"] and len(argv) == 3: + os.environ["AGENT_LAB_HOME"] = str(home) + os.environ.setdefault("AGENT_LAB_CUE_TOOL_DIR", str(home / "cache/tools/cue")) + return experiment_module().main(["experiment.py", "check-directory", argv[2]]) + if argv[:3] == ["experiment", "authorize", "install"] and len(argv) == 4: + os.environ["AGENT_LAB_HOME"] = str(home) + os.environ.setdefault("AGENT_LAB_CUE_TOOL_DIR", str(home / "cache/tools/cue")) + os.environ.setdefault("AGENT_LAB_CEDAR_TOOL_DIR", str(home / "cache/tools/cedar")) + return experiment_module().main(["experiment.py", "authorize-directory", argv[3]]) + if argv[:4] == ["experiment", "authorize", "install", "--zip"] and len(argv) == 5: + os.environ["AGENT_LAB_HOME"] = str(home) + os.environ.setdefault("AGENT_LAB_CUE_TOOL_DIR", str(home / "cache/tools/cue")) + os.environ.setdefault("AGENT_LAB_CEDAR_TOOL_DIR", str(home / "cache/tools/cedar")) + return experiment_module().main(["experiment.py", "authorize-zip", argv[4]]) + if argv[:1] == ["experiment"] and argv[1:2] in (["install"], ["inspect"]): + return experiment_command(home, argv[1:]) + if argv[:1] == ["image"]: + return image_command(home, argv[1:]) + print("Usage: agent-lab [--home ABSOLUTE_HOME] {version|init|config|experiment|image}", file=sys.stderr) + return 2 + + +if __name__ == "__main__": + raise SystemExit(main(sys.argv[1:])) diff --git a/scripts/experiment b/scripts/experiment index 3d2b63b..7f1c42b 100755 --- a/scripts/experiment +++ b/scripts/experiment @@ -1,75 +1,5 @@ #!/usr/bin/env bash set -euo pipefail -script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" -helper="$script_dir/experiment.py" - -usage() { - printf '%s\n' \ - 'Usage: scripts/experiment check [--] MANIFEST' \ - 'Usage: scripts/experiment authorize install [--] MANIFEST' >&2 -} - -case "${1:-}" in - -h|--help) - [ "$#" -eq 1 ] || { - usage - exit 2 - } - usage - exit 0 - ;; - check) - shift - if [ "${1:-}" = -- ]; then - shift - elif [[ "${1:-}" == -* ]]; then - usage - exit 2 - fi - [ "$#" -eq 1 ] || { - usage - exit 2 - } - command_name=check - manifest=$1 - ;; - authorize) - shift - [ "${1:-}" = install ] || { - usage - exit 2 - } - shift - if [ "${1:-}" = -- ]; then - shift - elif [[ "${1:-}" == -* ]]; then - usage - exit 2 - fi - [ "$#" -eq 1 ] || { - usage - exit 2 - } - command_name=authorize - manifest=$1 - ;; - *) - usage - exit 2 - ;; -esac - -command -v python3 >/dev/null 2>&1 || { - printf 'INFRA Experiment validation requires python3\n' >&2 - exit 125 -} -[ -f "$helper" ] && [ ! -L "$helper" ] || { - printf 'INFRA Experiment validation helper is missing or unsafe\n' >&2 - exit 125 -} - -if [ "$command_name" = authorize ]; then - exec python3 -I "$helper" authorize install "$manifest" -fi -exec python3 -I "$helper" check "$manifest" +printf '%s\n' 'Usage: scripts/agent-lab experiment {check|authorize install} DIRECTORY' >&2 +exit 2 diff --git a/scripts/experiment.py b/scripts/experiment.py index bddf421..e8c4152 100644 --- a/scripts/experiment.py +++ b/scripts/experiment.py @@ -3,21 +3,67 @@ from __future__ import annotations +import base64 import hashlib +from importlib.util import module_from_spec, spec_from_file_location import json import math import os from pathlib import Path +import re +import selectors import signal import stat +import struct import subprocess import sys import tempfile import time -from typing import NamedTuple, NoReturn +from typing import Callable, NamedTuple, NoReturn +import zlib MAX_MANIFEST_BYTES = 262_144 +MAX_ARCHIVE_BYTES = 1_048_576 +MAX_SOURCE_BYTES = MAX_MANIFEST_BYTES +ZIP_DECODE_TIMEOUT_SECONDS = 5 +GIT_ACQUISITION_TIMEOUT_SECONDS = 5 +GIT_PROVIDER_AUTHORITY = "api.github.com" +GIT_PROVIDER_METHOD = "github-git-data-v3" +GIT_PROVIDER_HEADERS = ( + ("Accept", "application/vnd.github+json"), + ("Accept-Encoding", "identity"), + ("User-Agent", "agent-lab/v0alpha1"), + ("X-GitHub-Api-Version", "2022-11-28"), +) +GIT_PROVIDER_CA_FILES = ( + "/etc/ssl/certs/ca-certificates.crt", + "/etc/pki/tls/certs/ca-bundle.crt", +) +GIT_PROVIDER_MAX_HEADER_BYTES = 32_768 +GIT_WORKER_MAX_OUTPUT_BYTES = ((MAX_ARCHIVE_BYTES + 2) // 3) * 4 + 65_536 +GIT_WORKER_MAX_ERROR_BYTES = 4_096 +GIT_WORKER_MEMORY_BYTES = 268_435_456 +GIT_WORKER_FRAME = b"agent-lab.git-provider.v1\0" +GIT_WORKER_GRACE_SECONDS = 0.25 +GIT_SHA1 = re.compile(r"[0-9a-f]{40}", re.ASCII) +GITHUB_SOURCE_URL = re.compile( + r"https://github\.com/" + r"([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?)/" + r"([A-Za-z0-9_.-]{1,100})\.git", + re.ASCII, +) +GITHUB_PROVIDER_PATH = re.compile( + r"/repos/" + r"([a-z0-9](?:[a-z0-9-]{0,37}[a-z0-9])?)/" + r"([a-z0-9_.-]{1,100})/git/" + r"(commits|trees|blobs)/([0-9a-f]{40})", + re.ASCII, +) +SOURCE_DIGEST_DOMAIN = b"agent-lab.experiment-tree.v1\0" +PLAN_DOMAIN = b"agent-lab.experiment-plan.v1\0" +BUNDLED_CATALOG_DOMAIN = b"agent-lab.experiment-image-catalog.v1\0" +BUNDLED_ENTRY_DOMAIN = b"agent-lab.experiment-image-entry.v1\0" MAX_CUE_OUTPUT_BYTES = 1_048_576 MAX_CONTRACT_FILE_BYTES = 1_048_576 MAX_HELPER_BYTES = 1_048_576 @@ -63,7 +109,9 @@ "members": [ { "name": "probe", - "image": "probe@sha256:0000000000000000000000000000000000000000000000000000000000000000", + "image": { + "digestRef": "probe@sha256:0000000000000000000000000000000000000000000000000000000000000000" + }, } ] }, @@ -91,6 +139,19 @@ class PlanBinding(NamedTuple): requested_name: str member_count: int resource_classes: tuple[str, ...] + source_digest: str + + +class SourceSnapshot(NamedTuple): + data: bytes + digest: str + transport: dict[str, object] + + +class PlanResolution(NamedTuple): + plan: dict[str, object] + bundled_catalog: dict[str, object] | None + local_catalog: dict[str, object] | None def fail(message: str) -> NoReturn: @@ -134,11 +195,30 @@ def reject_non_finite_numbers(value: object) -> None: reject_non_finite_numbers(item) -def read_manifest_once(path: str) -> bytes: +def _manifest_identity(metadata: os.stat_result) -> tuple[int, ...]: + return ( + metadata.st_dev, + metadata.st_ino, + metadata.st_mode, + metadata.st_uid, + metadata.st_nlink, + metadata.st_size, + metadata.st_mtime_ns, + metadata.st_ctime_ns, + ) + + +def read_manifest_once( + path: str, + *, + expected: os.stat_result | None = None, +) -> bytes: try: path_stat = os.lstat(path) except OSError as error: raise InfrastructureError("manifest cannot be inspected") from error + if expected is not None and _manifest_identity(path_stat) != _manifest_identity(expected): + raise InfrastructureError("manifest identity changed before read") if stat.S_ISLNK(path_stat.st_mode): raise InfrastructureError("manifest symlinks are not accepted") if not stat.S_ISREG(path_stat.st_mode): @@ -157,7 +237,7 @@ def read_manifest_once(path: str) -> bytes: opened_stat = os.fstat(descriptor) if not stat.S_ISREG(opened_stat.st_mode): raise InfrastructureError("manifest changed to a non-regular file") - if (opened_stat.st_dev, opened_stat.st_ino) != (path_stat.st_dev, path_stat.st_ino): + if _manifest_identity(opened_stat) != _manifest_identity(path_stat): raise InfrastructureError("manifest identity changed before read") chunks: list[bytes] = [] remaining = MAX_MANIFEST_BYTES + 1 @@ -179,23 +259,1432 @@ def read_manifest_once(path: str) -> bytes: if len(data) > MAX_MANIFEST_BYTES: raise InvalidManifest(f"exceeds the {MAX_MANIFEST_BYTES}-byte limit") - before = ( - opened_stat.st_dev, - opened_stat.st_ino, - opened_stat.st_size, - opened_stat.st_mtime_ns, - ) - after = ( - final_stat.st_dev, - final_stat.st_ino, - final_stat.st_size, - final_stat.st_mtime_ns, - ) + before = _manifest_identity(opened_stat) + after = _manifest_identity(final_stat) if before != after or len(data) != final_stat.st_size: raise InfrastructureError("manifest changed while it was read") return data +def source_digest(data: bytes) -> str: + name = b"experiment.cue" + digest = hashlib.sha256(SOURCE_DIGEST_DOMAIN) + digest.update(len(name).to_bytes(4, "big")) + digest.update(name) + digest.update(len(data).to_bytes(8, "big")) + digest.update(data) + return f"sha256:{digest.hexdigest()}" + + +GitRequester = Callable[ + [str, str, tuple[tuple[str, str], ...], int, float], + tuple[int, tuple[tuple[str, str], ...], bytes], +] + + +def _git_reject(code: str, detail: str) -> NoReturn: + raise InvalidManifest(f"git source {code} {detail}") + + +def _parse_git_source(url: str, commit: str) -> tuple[str, str, str, str]: + if not isinstance(url, str) or not url.isascii(): + _git_reject("GIT-URL", "must be one normalized ASCII GitHub HTTPS URL") + matched = GITHUB_SOURCE_URL.fullmatch(url) + if matched is None: + _git_reject("GIT-URL", "must be one normalized unauthenticated GitHub HTTPS URL") + owner, repository = matched.groups() + if owner.endswith("-") or "--" in owner or repository in (".", ".."): + _git_reject("GIT-URL", "has an unsupported repository identity") + if not isinstance(commit, str) or GIT_SHA1.fullmatch(commit) is None: + _git_reject("GIT-OID", "commit must be one full lowercase SHA-1 object ID") + owner = owner.lower() + repository = repository.lower() + canonical = f"https://github.com/{owner}/{repository}.git" + return canonical, owner, repository, commit + + +def _git_object_id(kind: str, payload: bytes) -> str: + framed = kind.encode("ascii") + b" " + str(len(payload)).encode("ascii") + b"\0" + return hashlib.sha1(framed + payload, usedforsecurity=False).hexdigest() + + +def _git_provider_route(path: str) -> tuple[str, str, str, str]: + if not isinstance(path, str) or not path.isascii(): + raise InfrastructureError("git provider GIT-AUTHORITY path is malformed") + matched = GITHUB_PROVIDER_PATH.fullmatch(path) + if matched is None: + raise InfrastructureError("git provider GIT-AUTHORITY path is malformed") + owner, repository, object_kind, object_id = matched.groups() + if "--" in owner or repository in (".", ".."): + raise InfrastructureError("git provider GIT-AUTHORITY path is malformed") + return owner, repository, object_kind, object_id + + +def _git_provider_json( + requester: GitRequester, + path: str, + remaining: int, + deadline: float, + *, + stable_not_found: bool, +) -> tuple[object, int]: + if remaining <= 0: + raise InfrastructureError("git provider GIT-ACQUIRE exhausted its response bound") + _, _, object_kind, _ = _git_provider_route(path) + if not isinstance(stable_not_found, bool) or ( + stable_not_found and object_kind != "commits" + ): + raise InfrastructureError("git provider GIT-TAXONOMY request is malformed") + try: + status, raw_headers, body = requester( + GIT_PROVIDER_AUTHORITY, + path, + GIT_PROVIDER_HEADERS, + remaining, + deadline, + ) + except (InvalidManifest, InfrastructureError): + raise + except Exception as error: + raise InfrastructureError("git provider GIT-TRANSPORT request failed") from error + if time.monotonic() > deadline: + raise InfrastructureError("git provider GIT-TIMEOUT deadline expired") + if not isinstance(status, int) or isinstance(status, bool): + raise InfrastructureError("git provider GIT-STATUS response is malformed") + if not isinstance(raw_headers, tuple): + raise InfrastructureError("git provider GIT-HEADER response is malformed") + headers: dict[str, str] = {} + for item in raw_headers: + if ( + not isinstance(item, tuple) + or len(item) != 2 + or not all(isinstance(value, str) for value in item) + ): + raise InfrastructureError("git provider GIT-HEADER response is malformed") + name, value = item + lowered = name.lower() + if lowered in headers: + raise InfrastructureError("git provider GIT-HEADER response is ambiguous") + headers[lowered] = value + if headers.get("content-type") != "application/json; charset=utf-8": + raise InfrastructureError("git provider GIT-HEADER content type is uncertain") + declared = headers.get("content-length", "") + if ( + not declared.isascii() + or not declared.isdigit() + or (len(declared) > 1 and declared.startswith("0")) + ): + raise InfrastructureError("git provider GIT-HEADER content length is invalid") + declared_length = int(declared) + if ( + not isinstance(body, bytes) + or len(body) > remaining + or declared_length != len(body) + ): + raise InfrastructureError("git provider GIT-OUTPUT response exceeded its bound") + if status == 404: + try: + strict_json(body, source="git provider response") + except InvalidManifest as error: + raise InfrastructureError("git provider GIT-JSON response is malformed") from error + if stable_not_found: + _git_reject("GIT-NOTFOUND", "does not expose the requested public object") + raise InfrastructureError("git provider GIT-DRIFT bound object disappeared") + if status == 422: + raise InfrastructureError("git provider GIT-STATUS response is uncertain") + if 300 <= status <= 399: + raise InfrastructureError("git provider GIT-REDIRECT response is not accepted") + if status != 200: + raise InfrastructureError("git provider GIT-STATUS did not establish a result") + try: + value = strict_json(body, source="git provider response") + except InvalidManifest as error: + raise InfrastructureError("git provider GIT-JSON response is malformed") from error + return value, len(body) + + +def _git_system_ca_pem() -> str: + for raw_path in GIT_PROVIDER_CA_FILES: + flags = os.O_RDONLY | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_NOFOLLOW", 0) + try: + descriptor = os.open(raw_path, flags) + except OSError: + continue + try: + metadata = os.fstat(descriptor) + if ( + not stat.S_ISREG(metadata.st_mode) + or metadata.st_uid != 0 + or metadata.st_nlink != 1 + or stat.S_IMODE(metadata.st_mode) & 0o022 + or not 1 <= metadata.st_size <= 10_485_760 + ): + continue + chunks: list[bytes] = [] + remaining = metadata.st_size + while remaining: + chunk = os.read(descriptor, min(65_536, remaining)) + if not chunk: + break + chunks.append(chunk) + remaining -= len(chunk) + data = b"".join(chunks) + if remaining != 0 or len(data) != metadata.st_size: + continue + try: + return data.decode("ascii") + except UnicodeDecodeError: + continue + finally: + os.close(descriptor) + raise InfrastructureError("git provider GIT-TLS system trust is unavailable") + + +def _github_api_request( + authority: str, + path: str, + headers: tuple[tuple[str, str], ...], + maximum: int, + deadline: float, +) -> tuple[int, tuple[tuple[str, str], ...], bytes]: + """Perform one fixed-authority HTTPS request inside the isolated worker.""" + + _git_provider_route(path) + if ( + authority != GIT_PROVIDER_AUTHORITY + or headers != GIT_PROVIDER_HEADERS + or not 0 < maximum <= MAX_ARCHIVE_BYTES + ): + raise InfrastructureError("git provider GIT-AUTHORITY request is malformed") + remaining = deadline - time.monotonic() + if remaining <= 0: + raise InfrastructureError("git provider GIT-TIMEOUT deadline expired") + try: + import http.client + import ssl + + http.client._MAXLINE = 8_192 + http.client._MAXHEADERS = 32 + context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + context.check_hostname = True + context.verify_mode = ssl.CERT_REQUIRED + context.minimum_version = ssl.TLSVersion.TLSv1_2 + context.load_verify_locations(cadata=_git_system_ca_pem()) + connection = http.client.HTTPSConnection( + authority, + port=443, + timeout=remaining, + context=context, + ) + response = None + try: + connection.request("GET", path, headers=dict(headers)) + response = connection.getresponse() + raw_headers = response.getheaders() + if len(raw_headers) > 32: + raise InfrastructureError("git provider GIT-HEADER count exceeded its bound") + header_bytes = 0 + selected: dict[str, str] = {} + for name, value in raw_headers: + try: + encoded_name = name.encode("ascii") + encoded_value = value.encode("ascii") + except UnicodeEncodeError as error: + raise InfrastructureError( + "git provider GIT-HEADER response is malformed" + ) from error + header_bytes += len(encoded_name) + len(encoded_value) + 4 + lowered = name.lower() + if lowered in { + "content-encoding", + "content-length", + "content-type", + "transfer-encoding", + }: + if lowered in selected: + raise InfrastructureError( + "git provider GIT-HEADER response is ambiguous" + ) + selected[lowered] = value + if header_bytes > GIT_PROVIDER_MAX_HEADER_BYTES: + raise InfrastructureError("git provider GIT-HEADER bytes exceeded their bound") + status = response.status + if "transfer-encoding" in selected or selected.get( + "content-encoding", "identity" + ) != "identity": + raise InfrastructureError("git provider GIT-HEADER encoding is unsupported") + if selected.get("content-type") != "application/json; charset=utf-8": + raise InfrastructureError("git provider GIT-HEADER content type is uncertain") + declared = selected.get("content-length", "") + if ( + not declared.isascii() + or not declared.isdigit() + or (len(declared) > 1 and declared.startswith("0")) + ): + raise InfrastructureError("git provider GIT-HEADER content length is invalid") + declared_length = int(declared) + if declared_length > maximum: + raise InfrastructureError("git provider GIT-OUTPUT response exceeded its bound") + output = bytearray() + while len(output) < declared_length: + remaining = deadline - time.monotonic() + if remaining <= 0: + raise InfrastructureError("git provider GIT-TIMEOUT deadline expired") + if connection.sock is not None: + connection.sock.settimeout(remaining) + chunk = response.read(min(65_536, declared_length - len(output))) + if not chunk: + raise InfrastructureError("git provider GIT-OUTPUT response was truncated") + output.extend(chunk) + remaining = deadline - time.monotonic() + if remaining <= 0: + raise InfrastructureError("git provider GIT-TIMEOUT deadline expired") + if connection.sock is not None: + connection.sock.settimeout(remaining) + if response.read(1) != b"": + raise InfrastructureError("git provider GIT-OUTPUT response exceeded its length") + return ( + status, + ( + ("content-length", str(len(output))), + ("content-type", "application/json; charset=utf-8"), + ), + bytes(output), + ) + finally: + if response is not None: + response.close() + connection.close() + except InfrastructureError: + raise + except Exception as error: + raise InfrastructureError("git provider GIT-TRANSPORT request failed") from error + + +def _git_worker_observe(pid: int) -> object | None: + """Observe one owned worker without releasing its numeric identity.""" + + try: + observed = os.waitid( + os.P_PID, + pid, + os.WEXITED | os.WNOHANG | os.WNOWAIT, + ) + except (ChildProcessError, OSError) as error: + raise InfrastructureError( + "git provider GIT-WORKER child ownership is uncertain" + ) from error + if observed is None: + return None + if ( + getattr(observed, "si_pid", None) != pid + or getattr(observed, "si_signo", None) != signal.SIGCHLD + or getattr(observed, "si_code", None) + not in { + getattr(os, "CLD_EXITED", 1), + getattr(os, "CLD_KILLED", 2), + getattr(os, "CLD_DUMPED", 3), + } + or not isinstance(getattr(observed, "si_status", None), int) + or isinstance(getattr(observed, "si_status", None), bool) + ): + raise InfrastructureError( + "git provider GIT-WORKER child status is malformed" + ) + return observed + + +def _git_worker_status_matches(observed: object, status: int) -> bool: + if not isinstance(status, int) or isinstance(status, bool): + return False + code = getattr(observed, "si_code", None) + observed_status = getattr(observed, "si_status", None) + if code == getattr(os, "CLD_EXITED", 1): + return os.WIFEXITED(status) and os.WEXITSTATUS(status) == observed_status + if code == getattr(os, "CLD_KILLED", 2): + return os.WIFSIGNALED(status) and os.WTERMSIG(status) == observed_status + if code == getattr(os, "CLD_DUMPED", 3): + core_dumped = getattr(os, "WCOREDUMP", lambda _status: False) + return ( + os.WIFSIGNALED(status) + and os.WTERMSIG(status) == observed_status + and bool(core_dumped(status)) + ) + return False + + +def _git_worker_exit_code(observed: object | None) -> int | None: + if observed is None or getattr(observed, "si_code", None) != getattr( + os, "CLD_EXITED", 1 + ): + return None + status = getattr(observed, "si_status", None) + if not isinstance(status, int) or isinstance(status, bool) or not 0 <= status <= 255: + return None + return status + + +def _git_worker_terminate(pid: int, *, reaped: bool, deadline: float) -> bool: + """Finalize one owned worker without signaling a released PID or PGID.""" + + if reaped: + return False + uncertain = False + try: + observed = _git_worker_observe(pid) + except InfrastructureError: + return False + + group_missing = False + try: + os.killpg(pid, signal.SIGTERM) + except ProcessLookupError: + group_missing = True + except OSError: + group_missing = True + uncertain = True + if group_missing: + try: + os.kill(pid, signal.SIGKILL) + except ProcessLookupError: + return False + except OSError: + uncertain = True + + if observed is None: + now = time.monotonic() + term_budget = max(0.0, deadline - now) / 2 + term_deadline = min(deadline, now + min(GIT_WORKER_GRACE_SECONDS, term_budget)) + while time.monotonic() < term_deadline: + try: + observed = _git_worker_observe(pid) + except InfrastructureError: + return False + if observed is not None: + break + remaining = term_deadline - time.monotonic() + if remaining > 0: + time.sleep(min(0.01, remaining)) + + kill_group_missing = False + try: + os.killpg(pid, signal.SIGKILL) + except ProcessLookupError: + kill_group_missing = True + except OSError: + kill_group_missing = True + uncertain = True + if kill_group_missing: + try: + os.kill(pid, signal.SIGKILL) + except ProcessLookupError: + return False + except OSError: + uncertain = True + try: + os.killpg(pid, signal.SIGKILL) + except ProcessLookupError: + pass + except OSError: + uncertain = True + + kill_deadline = min( + deadline, + time.monotonic() + GIT_WORKER_GRACE_SECONDS, + ) + while time.monotonic() < kill_deadline: + try: + current = _git_worker_observe(pid) + except InfrastructureError: + return False + if observed is None and current is not None: + observed = current + remaining = kill_deadline - time.monotonic() + if remaining > 0: + time.sleep(min(0.01, remaining)) + + try: + waited, status = os.waitpid(pid, os.WNOHANG) + except (ChildProcessError, OSError): + return False + if waited != pid or observed is None: + return False + return _git_worker_status_matches(observed, status) and not uncertain + + +def _git_worker_write(descriptor: int, data: bytes) -> None: + view = memoryview(data) + while view: + written = os.write(descriptor, view) + if written <= 0: + raise OSError("worker pipe made no progress") + view = view[written:] + + +def _git_worker_child( + control_read: int, + stdout_write: int, + stderr_write: int, + authority: str, + path: str, + headers: tuple[tuple[str, str], ...], + maximum: int, + deadline: float, +) -> NoReturn: + exit_code = 125 + try: + os.setsid() + for signum in (signal.SIGHUP, signal.SIGINT, signal.SIGQUIT, signal.SIGTERM): + signal.signal(signum, signal.SIG_DFL) + signal.pthread_sigmask(signal.SIG_SETMASK, set()) + null_descriptor = os.open("/dev/null", os.O_RDONLY | getattr(os, "O_CLOEXEC", 0)) + os.dup2(null_descriptor, 0) + os.dup2(stdout_write, 1) + os.dup2(stderr_write, 2) + os.dup2(control_read, 3) + import resource + + maximum_fd = resource.getrlimit(resource.RLIMIT_NOFILE)[0] + if maximum_fd == resource.RLIM_INFINITY: + maximum_fd = 1_048_576 + os.closerange(4, int(maximum_fd)) + resource.setrlimit(resource.RLIMIT_CORE, (0, 0)) + resource.setrlimit(resource.RLIMIT_FSIZE, (0, 0)) + resource.setrlimit( + resource.RLIMIT_AS, + (GIT_WORKER_MEMORY_BYTES, GIT_WORKER_MEMORY_BYTES), + ) + resource.setrlimit(resource.RLIMIT_NOFILE, (16, 16)) + resource.setrlimit(resource.RLIMIT_CPU, (4, 4)) + os.environ.clear() + os.environ.update( + { + "HOME": "/nonexistent", + "LANG": "C", + "LC_ALL": "C", + "PATH": "/usr/bin:/bin", + } + ) + os.chdir("/") + os.umask(0o077) + try: + status, result_headers, body = _github_api_request( + authority, path, headers, maximum, deadline + ) + value: dict[str, object] = { + "body": base64.b64encode(body).decode("ascii"), + "headers": [list(item) for item in result_headers], + "kind": "result", + "status": status, + } + exit_code = 0 + except InvalidManifest: + value = {"kind": "reject"} + exit_code = 1 + except InfrastructureError: + value = {"kind": "infra"} + exit_code = 125 + except BaseException: + value = {"kind": "infra"} + exit_code = 125 + payload = json.dumps( + value, + ensure_ascii=True, + separators=(",", ":"), + sort_keys=True, + ).encode("ascii") + if len(payload) > GIT_WORKER_MAX_OUTPUT_BYTES: + payload = b'{"kind":"infra"}' + exit_code = 125 + frame = GIT_WORKER_FRAME + len(payload).to_bytes(8, "big") + payload + _git_worker_write(1, frame) + acknowledgement = os.read(3, 1) + if acknowledgement != b"1": + exit_code = 125 + except BaseException: + exit_code = 125 + os._exit(exit_code) + + +def _github_worker_request( + authority: str, + path: str, + headers: tuple[tuple[str, str], ...], + maximum: int, + deadline: float, +) -> tuple[int, tuple[tuple[str, str], ...], bytes]: + operation_deadline = deadline - (2 * GIT_WORKER_GRACE_SECONDS) + if time.monotonic() >= operation_deadline: + raise InfrastructureError("git provider GIT-TIMEOUT deadline expired") + interrupted: int | None = None + handlers: dict[int, object] = {} + managed_signals: set[int] = set() + + def interrupt(signum: int, _frame: object) -> None: + nonlocal interrupted + if interrupted is None: + interrupted = signum + + def change_mask(how: int, signals: set[int]) -> set[signal.Signals]: + try: + return set(signal.pthread_sigmask(how, signals)) + except (AttributeError, MemoryError, OSError, ValueError) as error: + raise InfrastructureError("git provider GIT-SIGNAL mask is unavailable") from error + + def record_pending(signals: set[int]) -> None: + nonlocal interrupted + try: + while True: + pending = set(signal.sigpending()).intersection(signals) + if not pending: + return + for signum in sorted(pending, key=int): + received = int(signal.sigwait({signum})) + if interrupted is None: + interrupted = received + except (AttributeError, MemoryError, OSError, ValueError) as error: + raise InfrastructureError( + "git provider GIT-SIGNAL pending state is uncertain" + ) from error + + original_mask = change_mask(signal.SIG_BLOCK, set()) + control_read = control_write = -1 + stdout_read = stdout_write = -1 + stderr_read = stderr_write = -1 + pid = -1 + reaped = False + acknowledged = False + selector = None + output: bytearray | None = None + errors: bytearray | None = None + worker_exit: object | None = None + try: + try: + sigchld_handler = signal.getsignal(signal.SIGCHLD) + except (OSError, ValueError) as error: + raise InfrastructureError( + "git provider GIT-SIGNAL SIGCHLD ownership is unavailable" + ) from error + if sigchld_handler != signal.SIG_DFL: + raise InfrastructureError( + "git provider GIT-SIGNAL SIGCHLD ownership is unavailable" + ) + for signum in (signal.SIGHUP, signal.SIGINT, signal.SIGQUIT, signal.SIGTERM): + try: + previous = signal.getsignal(signum) + except (OSError, ValueError) as error: + raise InfrastructureError( + "git provider GIT-SIGNAL handlers are unavailable" + ) from error + if previous == signal.SIG_IGN or signum in original_mask: + continue + handlers[signum] = previous + managed_signals.add(signum) + try: + signal.signal(signum, interrupt) + except (OSError, ValueError) as error: + raise InfrastructureError( + "git provider GIT-SIGNAL handlers are unavailable" + ) from error + + selector = selectors.DefaultSelector() + output = bytearray() + errors = bytearray() + control_read, control_write = os.pipe2(getattr(os, "O_CLOEXEC", 0)) + stdout_read, stdout_write = os.pipe2(getattr(os, "O_CLOEXEC", 0)) + stderr_read, stderr_write = os.pipe2(getattr(os, "O_CLOEXEC", 0)) + spawn_mask = change_mask(signal.SIG_BLOCK, managed_signals) + try: + if interrupted is None: + pid = os.fork() + finally: + if pid != 0: + change_mask(signal.SIG_SETMASK, set(spawn_mask)) + if pid < 0: + raise SystemExit(128 + interrupted) if interrupted is not None else InfrastructureError( + "git provider GIT-WORKER did not start" + ) + if pid == 0: + os.close(control_write) + os.close(stdout_read) + os.close(stderr_read) + _git_worker_child( + control_read, + stdout_write, + stderr_write, + authority, + path, + headers, + maximum, + operation_deadline, + ) + os.close(control_read) + os.close(stdout_write) + os.close(stderr_write) + control_read = stdout_write = stderr_write = -1 + os.set_blocking(stdout_read, False) + os.set_blocking(stderr_read, False) + selector.register(stdout_read, selectors.EVENT_READ, "stdout") + selector.register(stderr_read, selectors.EVENT_READ, "stderr") + expected_frame: int | None = None + drain_deadline: float | None = None + failure: str | None = None + while True: + if interrupted is not None: + failure = "git provider GIT-SIGNAL interrupted acquisition" + break + + worker_exit = _git_worker_observe(pid) + now = time.monotonic() + if worker_exit is not None and drain_deadline is None: + drain_deadline = min( + operation_deadline, + now + GIT_WORKER_GRACE_SECONDS, + ) + active_deadline = ( + operation_deadline if drain_deadline is None else drain_deadline + ) + remaining = active_deadline - now + if remaining <= 0: + if worker_exit is None: + failure = "git provider GIT-TIMEOUT deadline expired" + elif acknowledged: + failure = "git provider GIT-WORKER left a residual process group" + else: + failure = "git provider GIT-WORKER exited before a complete frame" + break + + for key, _ in selector.select(min(0.05, remaining)): + descriptor = int(key.fd) + try: + chunk = os.read(descriptor, 65_536) + except BlockingIOError: + continue + if not chunk: + selector.unregister(descriptor) + continue + if key.data == "stdout": + assert output is not None + output.extend(chunk) + if len(output) > len(GIT_WORKER_FRAME) + 8 + GIT_WORKER_MAX_OUTPUT_BYTES: + failure = "git provider GIT-OUTPUT worker output exceeded its bound" + break + header_size = len(GIT_WORKER_FRAME) + 8 + if expected_frame is None and len(output) >= header_size: + if not output.startswith(GIT_WORKER_FRAME): + failure = "git provider GIT-WORKER frame is malformed" + break + payload_size = int.from_bytes( + output[len(GIT_WORKER_FRAME) : header_size], "big" + ) + if payload_size > GIT_WORKER_MAX_OUTPUT_BYTES: + failure = "git provider GIT-OUTPUT worker frame exceeded its bound" + break + expected_frame = header_size + payload_size + if expected_frame is not None and len(output) > expected_frame: + failure = "git provider GIT-WORKER frame has trailing data" + break + else: + assert errors is not None + errors.extend(chunk) + if len(errors) > GIT_WORKER_MAX_ERROR_BYTES: + failure = "git provider GIT-OUTPUT worker error exceeded its bound" + else: + failure = "git provider GIT-WORKER emitted unexpected diagnostics" + break + if failure is not None: + break + assert output is not None + assert errors is not None + if expected_frame is not None and len(output) == expected_frame and not acknowledged: + if errors: + failure = "git provider GIT-WORKER emitted unexpected diagnostics" + break + _git_worker_write(control_write, b"1") + os.close(control_write) + control_write = -1 + acknowledged = True + if worker_exit is not None and not selector.get_map(): + if not acknowledged: + failure = "git provider GIT-WORKER exited before a complete frame" + break + if failure is not None: + raise InfrastructureError(failure) + returncode = _git_worker_exit_code(worker_exit) + if returncode is None: + raise InfrastructureError("git provider GIT-WORKER exit status is uncertain") + assert output is not None + header_size = len(GIT_WORKER_FRAME) + 8 + if len(output) < header_size: + raise InfrastructureError("git provider GIT-WORKER frame is incomplete") + payload = bytes(output[header_size:]) + try: + value = strict_json(payload, source="git provider worker frame") + except InvalidManifest as error: + raise InfrastructureError("git provider GIT-WORKER frame is malformed") from error + if value == {"kind": "reject"} and returncode == 1: + _git_reject("GIT-PROVIDER", "request was rejected") + if value == {"kind": "infra"} and returncode == 125: + raise InfrastructureError("git provider GIT-WORKER request was uncertain") + if ( + returncode != 0 + or not isinstance(value, dict) + or set(value) != {"body", "headers", "kind", "status"} + or value.get("kind") != "result" + or not isinstance(value.get("status"), int) + or isinstance(value.get("status"), bool) + or not isinstance(value.get("headers"), list) + or not isinstance(value.get("body"), str) + ): + raise InfrastructureError("git provider GIT-WORKER result is malformed") + result_headers: list[tuple[str, str]] = [] + for item in value["headers"]: + if ( + not isinstance(item, list) + or len(item) != 2 + or not all(isinstance(part, str) for part in item) + ): + raise InfrastructureError("git provider GIT-WORKER headers are malformed") + result_headers.append((item[0], item[1])) + encoded_body = value["body"] + assert isinstance(encoded_body, str) + if not encoded_body.isascii() or len(encoded_body) > ((maximum + 2) // 3) * 4: + raise InfrastructureError("git provider GIT-OUTPUT worker body exceeded its bound") + try: + body = base64.b64decode(encoded_body, validate=True) + except (ValueError, base64.binascii.Error) as error: + raise InfrastructureError("git provider GIT-WORKER body is malformed") from error + if len(body) > maximum: + raise InfrastructureError("git provider GIT-OUTPUT worker body exceeded its bound") + return int(value["status"]), tuple(result_headers), body + except InfrastructureError: + raise + except MemoryError as error: + raise InfrastructureError( + "git provider GIT-WORKER could not establish a result" + ) from error + except (OSError, ValueError) as error: + raise InfrastructureError("git provider GIT-WORKER could not establish a result") from error + finally: + cleanup_error: InfrastructureError | None = None + if selector is not None: + try: + selector.close() + except (MemoryError, OSError, ValueError): + cleanup_error = InfrastructureError( + "git provider GIT-WORKER selector cleanup is uncertain" + ) + for descriptor in ( + control_read, + control_write, + stdout_read, + stdout_write, + stderr_read, + stderr_write, + ): + if descriptor >= 0: + try: + os.close(descriptor) + except (MemoryError, OSError): + if cleanup_error is None: + cleanup_error = InfrastructureError( + "git provider GIT-WORKER descriptor cleanup is uncertain" + ) + if pid > 0: + try: + cleanup_deadline = min( + deadline, + time.monotonic() + (2 * GIT_WORKER_GRACE_SECONDS), + ) + terminated = _git_worker_terminate( + pid, + reaped=reaped, + deadline=cleanup_deadline, + ) + except (InfrastructureError, MemoryError, OSError, ValueError): + terminated = False + if not terminated and cleanup_error is None: + cleanup_error = InfrastructureError( + "git provider GIT-WORKER cleanup is uncertain" + ) + signals_blocked = False + try: + change_mask(signal.SIG_BLOCK, managed_signals) + signals_blocked = True + record_pending(managed_signals) + except InfrastructureError as error: + if cleanup_error is None: + cleanup_error = error + finally: + for signum, previous in handlers.items(): + try: + signal.signal(signum, previous) + except (MemoryError, OSError, ValueError): + if cleanup_error is None: + cleanup_error = InfrastructureError( + "git provider GIT-SIGNAL handlers could not be restored" + ) + if signals_blocked: + try: + record_pending(managed_signals) + except InfrastructureError as error: + if cleanup_error is None: + cleanup_error = error + try: + change_mask(signal.SIG_SETMASK, set(original_mask)) + except (InfrastructureError, MemoryError) as error: + if cleanup_error is None: + cleanup_error = ( + error + if isinstance(error, InfrastructureError) + else InfrastructureError( + "git provider GIT-SIGNAL mask is unavailable" + ) + ) + if cleanup_error is not None: + raise cleanup_error + if interrupted is not None: + raise SystemExit(128 + interrupted) + + +def _read_git_snapshot_with_requester( + url: str, + commit: str, + requester: GitRequester, +) -> SourceSnapshot: + """Acquire one exact public GitHub commit through the bounded Git Data API.""" + + if sys.platform != "linux": + raise InfrastructureError("git source GIT-PLATFORM requires Linux") + canonical, owner, repository, requested_commit = _parse_git_source(url, commit) + deadline = time.monotonic() + GIT_ACQUISITION_TIMEOUT_SECONDS + acquired = 0 + + commit_path = f"/repos/{owner}/{repository}/git/commits/{requested_commit}" + commit_value, used = _git_provider_json( + requester, + commit_path, + MAX_ARCHIVE_BYTES - acquired, + deadline, + stable_not_found=True, + ) + acquired += used + if not isinstance(commit_value, dict) or commit_value.get("sha") != requested_commit: + raise InfrastructureError("git provider GIT-COMMIT returned a different object") + commit_tree = commit_value.get("tree") + if not isinstance(commit_tree, dict): + raise InfrastructureError("git provider GIT-COMMIT tree binding is malformed") + tree_id = commit_tree.get("sha") + if not isinstance(tree_id, str) or GIT_SHA1.fullmatch(tree_id) is None: + raise InfrastructureError("git provider GIT-COMMIT tree identity is malformed") + + tree_path = f"/repos/{owner}/{repository}/git/trees/{tree_id}" + tree_value, used = _git_provider_json( + requester, + tree_path, + MAX_ARCHIVE_BYTES - acquired, + deadline, + stable_not_found=False, + ) + acquired += used + if ( + not isinstance(tree_value, dict) + or tree_value.get("sha") != tree_id + or tree_value.get("truncated") is not False + ): + raise InfrastructureError("git provider GIT-TREE response is inconsistent") + entries = tree_value.get("tree") + if not isinstance(entries, list) or len(entries) != 1: + _git_reject("GIT-ROOT", "root tree must contain exactly experiment.cue") + entry = entries[0] + if not isinstance(entry, dict): + raise InfrastructureError("git provider GIT-TREE entry is malformed") + if ( + entry.get("path") != "experiment.cue" + or entry.get("mode") != "100644" + or entry.get("type") != "blob" + ): + _git_reject("GIT-TYPE", "experiment.cue must be one regular non-executable blob") + blob_id = entry.get("sha") + blob_size = entry.get("size") + if not isinstance(blob_id, str) or GIT_SHA1.fullmatch(blob_id) is None: + raise InfrastructureError("git provider GIT-TREE blob identity is malformed") + if ( + not isinstance(blob_size, int) + or isinstance(blob_size, bool) + or not 0 <= blob_size <= MAX_SOURCE_BYTES + ): + _git_reject("GIT-SIZE", "experiment.cue exceeds the source limit") + + tree_payload = b"100644 experiment.cue\0" + bytes.fromhex(blob_id) + if _git_object_id("tree", tree_payload) != tree_id: + raise InfrastructureError("git provider GIT-TREE object identity is inconsistent") + + blob_path = f"/repos/{owner}/{repository}/git/blobs/{blob_id}" + blob_value, used = _git_provider_json( + requester, + blob_path, + MAX_ARCHIVE_BYTES - acquired, + deadline, + stable_not_found=False, + ) + acquired += used + if ( + not isinstance(blob_value, dict) + or blob_value.get("sha") != blob_id + or blob_value.get("size") != blob_size + or blob_value.get("encoding") != "base64" + or not isinstance(blob_value.get("content"), str) + ): + raise InfrastructureError("git provider GIT-BLOB response is inconsistent") + encoded = blob_value["content"] + assert isinstance(encoded, str) + encoded_size = ((blob_size + 2) // 3) * 4 + if not encoded.isascii() or "\r" in encoded: + raise InfrastructureError("git provider GIT-BLOB encoding exceeded its bound") + if "\n" in encoded: + if not encoded.endswith("\n"): + raise InfrastructureError("git provider GIT-BLOB wrapping is malformed") + lines = encoded[:-1].split("\n") + if ( + not lines + or any(len(line) != 60 for line in lines[:-1]) + or not 1 <= len(lines[-1]) <= 60 + ): + raise InfrastructureError("git provider GIT-BLOB wrapping is malformed") + compact = "".join(lines) + else: + compact = encoded + if len(compact) != encoded_size: + raise InfrastructureError("git provider GIT-BLOB encoding exceeded its bound") + try: + data = base64.b64decode(compact, validate=True) + except (ValueError, base64.binascii.Error) as error: + raise InfrastructureError("git provider GIT-BLOB encoding is malformed") from error + if len(data) != blob_size or _git_object_id("blob", data) != blob_id: + raise InfrastructureError("git provider GIT-BLOB object identity is inconsistent") + + return SourceSnapshot( + data=data, + digest=source_digest(data), + transport={ + "acquisition": { + "acquiredBytes": acquired, + "limitBytes": MAX_ARCHIVE_BYTES, + "method": GIT_PROVIDER_METHOD, + "requestCount": 3, + "temporaryBytes": 0, + "temporaryFiles": 0, + }, + "blob": f"sha1:{blob_id}", + "commit": f"sha1:{requested_commit}", + "kind": "git", + "requestedCommit": requested_commit, + "tree": f"sha1:{tree_id}", + "url": canonical, + }, + ) + + +def read_git_snapshot( + url: str, + commit: str, + *, + requester: GitRequester | None = None, +) -> SourceSnapshot: + """Acquire one exact public GitHub commit through the bounded Git Data API.""" + + if sys.platform != "linux": + raise InfrastructureError("git source GIT-PLATFORM requires Linux") + canonical, _, _, requested_commit = _parse_git_source(url, commit) + if requester is not None: + return _read_git_snapshot_with_requester(canonical, requested_commit, requester) + return _read_git_snapshot_with_requester( + canonical, requested_commit, _github_worker_request + ) + + +def _zip_reject(code: str, detail: str) -> NoReturn: + raise InvalidManifest(f"zip archive {code} {detail}") + + +def _read_zip_archive_once(path: str) -> bytes: + try: + path_stat = os.lstat(path) + except OSError as error: + raise InfrastructureError("zip archive ZIP-READ cannot be inspected") from error + if stat.S_ISLNK(path_stat.st_mode) or not stat.S_ISREG(path_stat.st_mode): + raise InfrastructureError("zip archive ZIP-READ is not a safe regular file") + if path_stat.st_size > MAX_ARCHIVE_BYTES: + _zip_reject("ZIP-SIZE", f"exceeds the {MAX_ARCHIVE_BYTES}-byte limit") + + flags = os.O_RDONLY | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_NONBLOCK", 0) + flags |= getattr(os, "O_NOFOLLOW", 0) + try: + descriptor = os.open(path, flags) + except OSError as error: + raise InfrastructureError("zip archive ZIP-READ cannot be opened") from error + + opened_stat: os.stat_result | None = None + final_stat: os.stat_result | None = None + try: + opened_stat = os.fstat(descriptor) + if ( + not stat.S_ISREG(opened_stat.st_mode) + or _manifest_identity(opened_stat) != _manifest_identity(path_stat) + ): + raise InfrastructureError("zip archive ZIP-READ identity changed before read") + chunks: list[bytes] = [] + remaining = MAX_ARCHIVE_BYTES + 1 + while remaining: + chunk = os.read(descriptor, min(65_536, remaining)) + if not chunk: + break + chunks.append(chunk) + remaining -= len(chunk) + archive = b"".join(chunks) + final_stat = os.fstat(descriptor) + except InfrastructureError: + raise + except OSError as error: + raise InfrastructureError("zip archive ZIP-READ could not be read completely") from error + finally: + try: + os.close(descriptor) + except OSError as error: + raise InfrastructureError("zip archive ZIP-READ descriptor could not be closed") from error + + assert opened_stat is not None and final_stat is not None + try: + current_stat = os.lstat(path) + except OSError as error: + raise InfrastructureError("zip archive ZIP-READ cannot be reverified") from error + expected = _manifest_identity(path_stat) + if ( + _manifest_identity(opened_stat) != expected + or _manifest_identity(final_stat) != expected + or _manifest_identity(current_stat) != expected + or len(archive) != final_stat.st_size + ): + raise InfrastructureError("zip archive ZIP-READ changed while being read") + if len(archive) > MAX_ARCHIVE_BYTES: + _zip_reject("ZIP-SIZE", f"exceeds the {MAX_ARCHIVE_BYTES}-byte limit") + return archive + + +def _zip_eocd(archive: bytes) -> tuple[int, tuple[object, ...]]: + eocd_offset = archive.rfind(b"PK\x05\x06") + if eocd_offset < 0 or len(archive) - eocd_offset < 22: + _zip_reject("ZIP-TRUNC", "has no complete end record") + try: + record = struct.unpack_from("<4s4H2IH", archive, eocd_offset) + except struct.error as error: + raise InvalidManifest("zip archive ZIP-TRUNC has an incomplete end record") from error + comment_size = int(record[-1]) + if eocd_offset + 22 + comment_size != len(archive): + _zip_reject("ZIP-TRAIL", "has bytes outside its canonical end") + if comment_size != 0: + _zip_reject("ZIP-META", "has an archive comment") + return eocd_offset, record + + +def _zip_decode(payload: bytes, expanded_size: int) -> bytes: + try: + deadline = time.monotonic() + ZIP_DECODE_TIMEOUT_SECONDS + decoder = zlib.decompressobj(-15) + output: list[bytes] = [] + produced = 0 + offset = 0 + while offset < len(payload): + if time.monotonic() >= deadline: + raise TimeoutError("zip decoder deadline") + chunk = payload[offset : offset + 65_536] + offset += len(chunk) + while chunk: + remaining = MAX_SOURCE_BYTES + 1 - produced + if remaining <= 0: + _zip_reject("ZIP-BOMB", "expands beyond the source limit") + decoded = decoder.decompress(chunk, remaining) + output.append(decoded) + produced += len(decoded) + chunk = decoder.unconsumed_tail + if produced > MAX_SOURCE_BYTES: + _zip_reject("ZIP-BOMB", "expands beyond the source limit") + if time.monotonic() >= deadline: + raise TimeoutError("zip decoder deadline") + remaining = MAX_SOURCE_BYTES + 1 - produced + decoded = decoder.flush(remaining) + output.append(decoded) + produced += len(decoded) + if produced > MAX_SOURCE_BYTES: + _zip_reject("ZIP-BOMB", "expands beyond the source limit") + if not decoder.eof: + _zip_reject("ZIP-TRUNC", "deflate stream ended early") + if decoder.unused_data or decoder.unconsumed_tail: + _zip_reject("ZIP-TRAIL", "deflate stream has unused input") + data = b"".join(output) + if len(data) != expanded_size: + _zip_reject("ZIP-LENGTH", "decoded length disagrees with its header") + except InvalidManifest: + raise + except zlib.error as error: + raise InvalidManifest("zip archive ZIP-TRUNC has an invalid deflate stream") from error + except TimeoutError as error: + raise InfrastructureError("zip archive ZIP-TIMEOUT decoder deadline expired") from error + except Exception as error: + raise InfrastructureError("zip archive ZIP-DECODE decoder result is uncertain") from error + return data + + +def read_directory_snapshot(path: str) -> SourceSnapshot: + try: + directory_stat = os.lstat(path) + except OSError as error: + raise InfrastructureError("source directory cannot be inspected") from error + if stat.S_ISLNK(directory_stat.st_mode) or not stat.S_ISDIR(directory_stat.st_mode): + raise InvalidManifest("source must be one directory") + try: + before = os.listdir(path) + except OSError as error: + raise InfrastructureError("source directory cannot be listed") from error + if before != ["experiment.cue"]: + raise InvalidManifest("directory must contain only experiment.cue") + authored_path = os.path.join(path, "experiment.cue") + try: + authored_stat = os.lstat(authored_path) + except OSError as error: + raise InfrastructureError("authored file cannot be inspected") from error + if authored_stat.st_nlink != 1: + raise InvalidManifest("experiment.cue must have one link") + authored_mode = stat.S_IMODE(authored_stat.st_mode) + if authored_mode & 0o111 or authored_mode & 0o022: + raise InvalidManifest("experiment.cue has a suspicious mode") + data = read_manifest_once(authored_path, expected=authored_stat) + try: + after = os.listdir(path) + final_directory_stat = os.lstat(path) + except OSError as error: + raise InfrastructureError("source directory cannot be verified") from error + if after != before or (directory_stat.st_dev, directory_stat.st_ino) != ( + final_directory_stat.st_dev, + final_directory_stat.st_ino, + ): + raise InfrastructureError("source directory changed while snapshotting") + return SourceSnapshot( + data=data, + digest=source_digest(data), + transport={"kind": "directory"}, + ) + + +def read_zip_snapshot(path: str) -> SourceSnapshot: + """Inspect and bounded-decode one canonical in-memory ZIP source.""" + + archive = _read_zip_archive_once(path) + eocd_offset, eocd = _zip_eocd(archive) + ( + _signature, + disk_number, + central_disk, + disk_entries, + total_entries, + central_size, + central_offset, + _comment_size, + ) = eocd + if ( + disk_number != 0 + or central_disk != 0 + or disk_entries == 0xFFFF + or total_entries == 0xFFFF + or central_size == 0xFFFFFFFF + or central_offset == 0xFFFFFFFF + ): + _zip_reject("ZIP-ZIP64", "uses ZIP64 or multiple disks") + if disk_entries != 1 or total_entries != 1: + _zip_reject("ZIP-COUNT", "must contain exactly one entry") + if central_offset + central_size != eocd_offset: + _zip_reject("ZIP-HEADER", "central directory bounds disagree") + if central_size < 46 or central_offset < 30: + _zip_reject("ZIP-TRUNC", "central directory is incomplete") + try: + central = struct.unpack_from("<4s6H3I5H2I", archive, central_offset) + except struct.error as error: + raise InvalidManifest("zip archive ZIP-TRUNC central header is incomplete") from error + ( + central_signature, + version_made, + version_needed, + flags, + method, + modified_time, + modified_date, + crc, + compressed_size, + expanded_size, + name_size, + central_extra_size, + member_comment_size, + member_disk, + _internal_attributes, + external_attributes, + local_offset, + ) = central + if central_signature != b"PK\x01\x02": + _zip_reject("ZIP-HEADER", "central signature is invalid") + if ( + version_needed >= 45 + or compressed_size == 0xFFFFFFFF + or expanded_size == 0xFFFFFFFF + or local_offset == 0xFFFFFFFF + or member_disk != 0 + ): + _zip_reject("ZIP-ZIP64", "uses ZIP64 or multiple disks") + central_record_size = 46 + name_size + central_extra_size + member_comment_size + if central_record_size != central_size: + _zip_reject("ZIP-HEADER", "central record size disagrees") + if central_offset + central_record_size > eocd_offset: + _zip_reject("ZIP-TRUNC", "central record is incomplete") + if central_extra_size != 0 or member_comment_size != 0: + _zip_reject("ZIP-META", "contains member metadata") + if method not in (0, 8): + _zip_reject("ZIP-METHOD", "uses unsupported compression") + allowed_flags = 0x800 | (0x6 if method == 8 else 0) + if flags & ~allowed_flags: + _zip_reject("ZIP-FLAG", "uses unsupported general-purpose flags") + minimum_version = 20 if method == 8 else 10 + if version_needed < minimum_version: + _zip_reject("ZIP-HEADER", "version is too old for its compression method") + if expanded_size > MAX_SOURCE_BYTES: + _zip_reject("ZIP-SIZE", f"source exceeds the {MAX_SOURCE_BYTES}-byte limit") + central_name = archive[central_offset + 46 : central_offset + 46 + name_size] + create_system = version_made >> 8 + unix_mode = external_attributes >> 16 + unix_type = stat.S_IFMT(unix_mode) + dos_attributes = external_attributes & 0xFFFF + if ( + create_system not in (0, 3, 10, 14) + or dos_attributes & 0x458 + or (create_system == 3 and unix_type not in (0, stat.S_IFREG)) + ): + _zip_reject("ZIP-TYPE", "member is not a regular file") + if local_offset != 0: + _zip_reject("ZIP-HEADER", "local header is not at the canonical offset") + + try: + local = struct.unpack_from("<4s5H3I2H", archive, local_offset) + except struct.error as error: + raise InvalidManifest("zip archive ZIP-TRUNC local header is incomplete") from error + ( + local_signature, + local_version, + local_flags, + local_method, + local_time, + local_date, + local_crc, + local_compressed_size, + local_expanded_size, + local_name_size, + local_extra_size, + ) = local + if local_signature != b"PK\x03\x04": + _zip_reject("ZIP-HEADER", "local signature is invalid") + local_record_size = 30 + local_name_size + local_extra_size + if local_record_size > central_offset: + _zip_reject("ZIP-TRUNC", "local header is incomplete") + local_name = archive[30 : 30 + local_name_size] + if local_extra_size != 0: + _zip_reject("ZIP-META", "contains local member metadata") + if ( + local_version != version_needed + or local_flags != flags + or local_method != method + or local_time != modified_time + or local_date != modified_date + or local_crc != crc + or local_compressed_size != compressed_size + or local_expanded_size != expanded_size + or local_name != central_name + ): + _zip_reject("ZIP-HEADER", "local and central records disagree") + try: + decoded_name = central_name.decode("ascii") + except UnicodeError as error: + raise InvalidManifest("zip archive ZIP-PATH member name is not ASCII") from error + if decoded_name != "experiment.cue": + _zip_reject("ZIP-PATH", "member name must be exactly experiment.cue") + payload_end = local_record_size + compressed_size + if payload_end != central_offset: + _zip_reject("ZIP-LENGTH", "compressed payload bounds disagree") + payload = archive[local_record_size:payload_end] + if len(payload) != compressed_size: + _zip_reject("ZIP-TRUNC", "compressed payload is incomplete") + if method == 0: + if compressed_size != expanded_size: + _zip_reject("ZIP-LENGTH", "stored member lengths disagree") + data = payload + else: + data = _zip_decode(payload, expanded_size) + if len(data) != expanded_size: + _zip_reject("ZIP-LENGTH", "decoded length disagrees with its header") + if (zlib.crc32(data) & 0xFFFFFFFF) != crc: + _zip_reject("ZIP-CRC", "member checksum disagrees") + return SourceSnapshot( + data=data, + digest=source_digest(data), + transport={ + "archiveBytes": len(archive), + "archiveDigest": "sha256:" + hashlib.sha256(archive).hexdigest(), + "kind": "zip", + }, + ) + + +def authored_manifest(snapshot: SourceSnapshot) -> object: + repo_root = Path(__file__).resolve().parent.parent + cue_helper = repo_root / "scripts/dev/cue-tool.py" + module = b'module: "agent-lab.local/experiment-snapshot"\nlanguage: version: "v0.9.0"\n' + try: + with tempfile.TemporaryDirectory(prefix="agent-lab-source-", dir="/tmp") as directory: + root = Path(directory) + write_private_file(root, "cue.mod/module.cue", module) + write_private_file(root, "experiment.cue", snapshot.data) + completed = subprocess.run( + ( + sys.executable, + "-I", + str(cue_helper), + "-C", + str(root), + "export", + "-E", + "experiment.cue", + "-e", + "experiment", + "--out", + "json", + ), + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + env=cue_environment(repo_root), + timeout=CUE_TIMEOUT_SECONDS, + ) + except (OSError, subprocess.SubprocessError) as error: + raise InfrastructureError("authored CUE evaluation could not complete") from error + if completed.returncode == 1: + raise InvalidManifest("authored CUE value is invalid or incomplete") + if completed.returncode != 0 or completed.stderr or not completed.stdout: + raise InfrastructureError("pinned CUE could not export authored Experiment") + if len(completed.stdout) > MAX_CUE_OUTPUT_BYTES: + raise InfrastructureError("authored CUE output exceeded its bound") + manifest = strict_json(completed.stdout, source="authored CUE export") + if not isinstance(manifest, dict): + raise InvalidManifest("experiment must export one object") + return manifest + + def strict_json(data: bytes, *, source: str) -> object: try: text = data.decode("utf-8") @@ -240,6 +1729,12 @@ def canonical_json(value: object) -> bytes: raise InfrastructureError("CUE produced a non-canonicalizable plan") from error +def plan_digest(plan: object) -> str: + """Return the canonical, domain-separated identity of one checked plan.""" + + return "sha256:" + hashlib.sha256(PLAN_DOMAIN + canonical_json(plan)).hexdigest() + + def cue_environment(repo_root: Path) -> dict[str, str]: environment = { "PATH": "/usr/bin:/bin", @@ -443,15 +1938,17 @@ def expected_plan(manifest: object, contract_digest: str) -> dict[str, object]: or not set(member) <= {"name", "image", "command", "resourceClass"} ): raise KeyError("member field drift") - members = [ - { + members = [] + for member in raw_members: + selector = member["image"] + if not isinstance(selector, dict) or set(selector) not in ({"digestRef"}, {"catalogName"}): + raise KeyError("unresolved selector") + members.append({ "command": member.get("command", []), - "image": member["image"], "name": member["name"], + "requestedSelector": selector, "resourceClass": member.get("resourceClass", "small"), - } - for member in raw_members - ] + }) members.sort(key=lambda member: str(member["name"])) name = metadata["name"] except (AssertionError, KeyError, TypeError) as error: @@ -469,6 +1966,200 @@ def expected_plan(manifest: object, contract_digest: str) -> dict[str, object]: } +def image_reference_module(): + path = Path(__file__).resolve().with_name("image_reference.py") + spec = spec_from_file_location("agent_lab_image_reference", path) + if spec is None or spec.loader is None: + raise InfrastructureError("shared image-reference grammar cannot be loaded") + module = module_from_spec(spec) + sys.modules[spec.name] = module + try: + spec.loader.exec_module(module) + except (ImportError, OSError) as error: + raise InfrastructureError("shared image-reference grammar cannot be loaded") from error + return module + + +valid_catalog_name = image_reference_module().valid_image_name + + +def digest_record(domain: bytes, value: object) -> str: + return "sha256:" + hashlib.sha256(domain + canonical_json(value)).hexdigest() + + +def image_catalog_module(): + path = Path(__file__).resolve().with_name("image_catalog.py") + spec = spec_from_file_location("agent_lab_image_catalog", path) + if spec is None or spec.loader is None: + raise InfrastructureError("local image catalog support cannot be loaded") + module = module_from_spec(spec) + sys.modules[spec.name] = module + try: + spec.loader.exec_module(module) + except (ImportError, OSError) as error: + raise InfrastructureError("local image catalog support cannot be loaded") from error + return module + + +def bundled_catalog(repo_root: Path) -> tuple[dict[str, object], str]: + path = repo_root / "catalog/experiment-images/v0alpha1.json" + data = stable_file_bytes(path, MAX_CONTRACT_FILE_BYTES, "bundled image catalog") + value = strict_json(data, source="bundled image catalog") + if not isinstance(value, dict) or set(value) != {"apiVersion", "entries"}: + raise InfrastructureError("bundled image catalog has an unexpected shape") + if value["apiVersion"] != "agent-lab.experiment-images/v0alpha1" or not isinstance(value["entries"], list): + raise InfrastructureError("bundled image catalog has an unknown schema") + return value, digest_record(BUNDLED_CATALOG_DOMAIN, value) + + +def resolve_plan_with_evidence( + plan: dict[str, object], + repo_root: Path, + catalog: dict[str, object] | None = None, +) -> PlanResolution: + resolved = json.loads(canonical_json(plan)) + members = resolved["spec"]["members"] + bundled_names: set[str] = set() + local_names: set[str] = set() + for member in members: + selector = member["requestedSelector"] + if set(selector) == {"digestRef"}: + member["resolvedImage"] = {"origin": "direct", "subject": selector["digestRef"]} + continue + if set(selector) != {"catalogName"} or not valid_catalog_name(selector["catalogName"]): + raise InvalidManifest("contains an invalid image selector") + name = selector["catalogName"] + if name.startswith("agent-lab."): + bundled_names.add(name) + else: + local_names.add(name) + + catalog_support = None + bundled_by_name: dict[str, dict[str, object]] = {} + bundled_evidence: dict[str, object] | None = None + if bundled_names: + catalog_support = image_catalog_module() + selected_catalog = catalog + if selected_catalog is None: + selected_catalog, _ = bundled_catalog(repo_root) + if ( + not isinstance(selected_catalog, dict) + or set(selected_catalog) != {"apiVersion", "entries"} + or selected_catalog.get("apiVersion") + != "agent-lab.experiment-images/v0alpha1" + ): + raise InfrastructureError("bundled image catalog has an unexpected shape") + entries = selected_catalog.get("entries") + if not isinstance(entries, list): + raise InfrastructureError("bundled image catalog entries are malformed") + for entry in entries: + if not isinstance(entry, dict) or set(entry) != {"name", "subject"}: + raise InfrastructureError("bundled image catalog entry is malformed") + name, subject = entry["name"], entry["subject"] + if ( + not valid_catalog_name(name) + or not isinstance(subject, str) + or not catalog_support.oci_subject(subject) + ): + raise InfrastructureError("bundled image catalog entry is invalid") + assert isinstance(name, str) + if not name.startswith("agent-lab.") or name in bundled_by_name: + raise InfrastructureError("bundled image catalog namespace is invalid") + bundled_by_name[name] = entry + bundled_evidence = { + "snapshotDigest": digest_record(BUNDLED_CATALOG_DOMAIN, selected_catalog) + } + + local_records: dict[str, dict[str, object]] = {} + local_evidence: dict[str, object] | None = None + if local_names: + if catalog_support is None: + catalog_support = image_catalog_module() + raw_home = os.environ.get("AGENT_LAB_HOME") + if not raw_home: + raise InvalidManifest("local image name requires an initialized Agent Lab home") + try: + local_resolution = catalog_support.resolve_local_images( + Path(raw_home), tuple(sorted(local_names)) + ) + except catalog_support.CatalogReject as error: + raise InvalidManifest(str(error)) from error + except catalog_support.CatalogInfrastructure as error: + raise InfrastructureError(str(error)) from error + try: + if not isinstance(local_resolution, dict) or set(local_resolution) != {"catalog", "records"}: + raise ValueError("resolution envelope") + records = local_resolution["records"] + evidence = local_resolution["catalog"] + if not isinstance(records, dict) or set(records) != local_names: + raise ValueError("resolution records") + if not isinstance(evidence, dict) or set(evidence) != {"revision", "snapshotDigest"}: + raise ValueError("resolution evidence") + revision = evidence["revision"] + snapshot_digest = evidence["snapshotDigest"] + if ( + not isinstance(revision, int) + or isinstance(revision, bool) + or revision < 1 + or not is_sha256(snapshot_digest) + ): + raise ValueError("resolution evidence values") + for name in local_names: + record = records[name] + if ( + not isinstance(record, dict) + or record.get("name") != name + or record.get("state") != "active" + or type(record.get("generation")) is not int + or record["generation"] != 1 + or record.get("previousEntryDigest") is not None + or not is_sha256(record.get("entryDigest")) + or not isinstance(record.get("subject"), str) + or not catalog_support.oci_subject(record["subject"]) + ): + raise ValueError("selected local record") + local_records = records + local_evidence = { + "revision": revision, + "snapshotDigest": snapshot_digest, + } + except (KeyError, TypeError, ValueError) as error: + raise InfrastructureError("local image catalog returned invalid resolution evidence") from error + + for member in members: + selector = member["requestedSelector"] + if set(selector) == {"digestRef"}: + continue + name = selector["catalogName"] + if name.startswith("agent-lab."): + entry = bundled_by_name.get(name) + if entry is None: + raise InvalidManifest("references an unknown bundled image name") + member["resolvedImage"] = { + "entryDigest": digest_record(BUNDLED_ENTRY_DOMAIN, entry), + "generation": 1, + "origin": "agent-lab", + "subject": entry["subject"], + } + else: + record = local_records[name] + member["resolvedImage"] = { + "entryDigest": record["entryDigest"], + "generation": record["generation"], + "origin": "local", + "subject": record["subject"], + } + return PlanResolution(resolved, bundled_evidence, local_evidence) + + +def resolve_plan( + plan: dict[str, object], + repo_root: Path, + catalog: dict[str, object] | None = None, +) -> dict[str, object]: + return resolve_plan_with_evidence(plan, repo_root, catalog).plan + + def invoke_cue( manifest: object, contract_digest: str, @@ -527,13 +2218,11 @@ def parse_cue_plan(completed: subprocess.CompletedProcess[bytes]) -> dict[str, o return plan -def cue_plan(manifest: object) -> object: +def cue_plan_with_evidence(manifest: object) -> PlanResolution: repo_root = Path(__file__).resolve().parent.parent contract_root = repo_root / "contracts" / "experiment" / "v0alpha1" - cue_tool = repo_root / "scripts" / "dev" / "cue-tool" cue_helper = repo_root / "scripts" / "dev" / "cue-tool.py" required = ( - cue_tool, cue_helper, contract_root / "schema.cue", contract_root / "plan.cue", @@ -579,11 +2268,15 @@ def cue_plan(manifest: object) -> object: plan = parse_cue_plan(completed) if plan != expected_plan(manifest, contract_digest): raise InfrastructureError("pinned CUE plan violates its exact postcondition") - return plan + return resolve_plan_with_evidence(plan, repo_root) except OSError as error: raise InfrastructureError("private validation snapshot could not be managed") from error +def cue_plan(manifest: object) -> object: + return cue_plan_with_evidence(manifest).plan + + def is_sha256(value: object) -> bool: return ( isinstance(value, str) @@ -593,7 +2286,7 @@ def is_sha256(value: object) -> bool: ) -def plan_binding(plan: object) -> PlanBinding: +def plan_binding(plan: object, source_digest: str) -> PlanBinding: """Derive the only facts the v0alpha1 policy is allowed to see.""" try: if not isinstance(plan, dict) or set(plan) != { @@ -635,8 +2328,9 @@ def plan_binding(plan: object) -> PlanBinding: for member in members: if not isinstance(member, dict) or set(member) != { "command", - "image", "name", + "requestedSelector", + "resolvedImage", "resourceClass", }: raise ValueError("plan member") @@ -650,15 +2344,15 @@ def plan_binding(plan: object) -> PlanBinding: except (AssertionError, KeyError, TypeError, ValueError) as error: raise InfrastructureError("CUE plan cannot be bound to authorization") from error - plan_bytes = canonical_json(plan) - plan_digest = f"sha256:{hashlib.sha256(plan_bytes).hexdigest()}" + bound_plan_digest = plan_digest(plan) return PlanBinding( - plan_digest=plan_digest, + plan_digest=bound_plan_digest, contract_digest=contract_digest, contract_version=contract_version, requested_name=requested_name, member_count=len(members), resource_classes=tuple(sorted(resource_classes)), + source_digest=source_digest, ) @@ -684,6 +2378,7 @@ def cedar_documents( "uid": resource_uid, "attrs": { "planDigest": binding.plan_digest, + "sourceDigest": binding.source_digest, "contractDigest": binding.contract_digest, "contractVersion": binding.contract_version, "requestedName": binding.requested_name, @@ -702,6 +2397,7 @@ def cedar_documents( "context": { "bindingVersion": "v0alpha1", "planDigest": binding.plan_digest, + "sourceDigest": binding.source_digest, "contractDigest": binding.contract_digest, }, } @@ -961,9 +2657,9 @@ def evaluate_cedar( raise InfrastructureError("private authorization snapshot could not be managed") from error -def authorize_plan(plan: object) -> tuple[dict[str, object], int]: +def authorize_plan(plan: object, source_digest: str) -> tuple[dict[str, object], int]: repo_root = Path(__file__).resolve().parent.parent - binding = plan_binding(plan) + binding = plan_binding(plan, source_digest) request, entities = cedar_documents(binding) authorization_digest, snapshot = authorization_snapshot(repo_root) verdict = evaluate_cedar(snapshot, request, entities, repo_root) @@ -976,6 +2672,7 @@ def authorize_plan(plan: object) -> tuple[dict[str, object], int]: "authorizationDigest": authorization_digest, "contractDigest": binding.contract_digest, "planDigest": binding.plan_digest, + "sourceDigest": binding.source_digest, }, "kind": "ExperimentAuthorizationDecision", "principal": { @@ -995,10 +2692,72 @@ def authorize_plan(plan: object) -> tuple[dict[str, object], int]: return decision, 0 if verdict == "permit" else 1 -def write_envelope(plan: object) -> None: - plan_bytes = canonical_json(plan) - digest = hashlib.sha256(plan_bytes).hexdigest() - envelope = canonical_json({"digest": f"sha256:{digest}", "plan": plan}) + b"\n" +def verify_trusted_inputs(plan: object, decision: object) -> None: + """Re-snapshot trusted inputs and bind them to one authorized plan.""" + + try: + if ( + not isinstance(decision, dict) + or not isinstance(decision.get("binding"), dict) + or not isinstance(decision.get("resource"), dict) + ): + raise ValueError("decision envelope") + binding = decision["binding"] + resource = decision["resource"] + if set(binding) != { + "authorizationDigest", + "contractDigest", + "planDigest", + "sourceDigest", + }: + raise ValueError("decision binding") + source_digest = binding["sourceDigest"] + if not is_sha256(source_digest): + raise ValueError("source identity") + expected = plan_binding(plan, source_digest) + if ( + binding["planDigest"] != expected.plan_digest + or binding["contractDigest"] != expected.contract_digest + or resource.get("id") != expected.plan_digest + ): + raise ValueError("plan decision identity") + except (KeyError, TypeError, ValueError) as error: + raise InfrastructureError("authorized plan binding is inconsistent") from error + + repo_root = Path(__file__).resolve().parent.parent + contract_digest, contract_files = contract_snapshot(repo_root) + authorization_digest, authorization_files = authorization_snapshot(repo_root) + verify_contract_snapshot(repo_root, contract_files) + verify_authorization_snapshot(repo_root, authorization_files) + if ( + expected.contract_digest != f"sha256:{contract_digest}" + or binding["authorizationDigest"] != authorization_digest + ): + raise InfrastructureError("trusted inputs no longer match the authorization") + + +def catalog_resolution_evidence( + bundled_catalog: dict[str, object] | None, + local_catalog: dict[str, object] | None, +) -> dict[str, object] | None: + catalog: dict[str, object] = {} + if bundled_catalog is not None: + catalog["bundled"] = bundled_catalog + if local_catalog is not None: + catalog["local"] = local_catalog + return catalog or None + + +def write_envelope( + plan: object, + bundled_catalog: dict[str, object] | None = None, + local_catalog: dict[str, object] | None = None, +) -> None: + value: dict[str, object] = {"digest": plan_digest(plan), "plan": plan} + catalog = catalog_resolution_evidence(bundled_catalog, local_catalog) + if catalog is not None: + value["catalog"] = catalog + envelope = canonical_json(value) + b"\n" try: written = sys.stdout.buffer.write(envelope) if written != len(envelope): @@ -1019,7 +2778,63 @@ def write_decision(decision: object) -> None: raise InfrastructureError("authorization decision could not be written") from error +def write_checked_source(checked: object) -> None: + output = canonical_json(checked) + b"\n" + try: + written = sys.stdout.buffer.write(output) + if written != len(output): + raise OSError("partial checked-source output") + sys.stdout.buffer.flush() + except (BrokenPipeError, OSError) as error: + raise InfrastructureError("checked source output could not be written") from error + + def main(argv: list[str]) -> int: + directory_checking = len(argv) == 3 and argv[1] == "check-directory" + directory_authorizing = len(argv) == 3 and argv[1] == "authorize-directory" + zip_checking = len(argv) == 3 and argv[1] == "check-zip" + zip_authorizing = len(argv) == 3 and argv[1] == "authorize-zip" + git_checking = len(argv) == 4 and argv[1] == "check-git" + git_authorizing = len(argv) == 4 and argv[1] == "authorize-git" + if ( + directory_checking + or directory_authorizing + or zip_checking + or zip_authorizing + or git_checking + or git_authorizing + ): + try: + if git_checking or git_authorizing: + snapshot = read_git_snapshot(argv[2], argv[3]) + elif zip_checking or zip_authorizing: + snapshot = read_zip_snapshot(argv[2]) + else: + snapshot = read_directory_snapshot(argv[2]) + manifest = authored_manifest(snapshot) + resolution = cue_plan_with_evidence(manifest) + plan = resolution.plan + if directory_checking or zip_checking or git_checking: + checked: dict[str, object] = { + "digest": plan_digest(plan), + "plan": plan, + "source": {"digest": snapshot.digest, **snapshot.transport}, + } + catalog = catalog_resolution_evidence( + resolution.bundled_catalog, + resolution.local_catalog, + ) + if catalog is not None: + checked["catalog"] = catalog + write_checked_source(checked) + return 0 + decision, result = authorize_plan(plan, snapshot.digest) + write_decision(decision) + return result + except InvalidManifest as error: + fail(str(error)) + except InfrastructureError as error: + infra(str(error)) checking = len(argv) == 3 and argv[1] == "check" authorizing = ( len(argv) == 4 and argv[1] == "authorize" and argv[2] == "install" @@ -1036,11 +2851,16 @@ def main(argv: list[str]) -> int: manifest = strict_json(manifest_bytes, source="input") if not isinstance(manifest, dict): raise InvalidManifest("must be one JSON object") - plan = cue_plan(manifest) + resolution = cue_plan_with_evidence(manifest) + plan = resolution.plan if checking: - write_envelope(plan) + write_envelope( + plan, + resolution.bundled_catalog, + resolution.local_catalog, + ) return 0 - decision, result = authorize_plan(plan) + decision, result = authorize_plan(plan, "sha256:" + "0" * 64) write_decision(decision) return result except InvalidManifest as error: diff --git a/scripts/experiment_store.py b/scripts/experiment_store.py new file mode 100644 index 0000000..cbddce2 --- /dev/null +++ b/scripts/experiment_store.py @@ -0,0 +1,2264 @@ +#!/usr/bin/env python3 +"""Durable, permit-gated publication of verified Experiment envelopes.""" + +from __future__ import annotations + +from contextlib import contextmanager, nullcontext +from contextvars import ContextVar +import ctypes +import errno +import fcntl +import hashlib +from importlib.util import module_from_spec, spec_from_file_location +import json +import math +import os +from pathlib import Path +import re +import stat +import sys +from typing import Callable, Iterator, NamedTuple, NoReturn, Sequence + + +INSTALL_KEY_DOMAIN = b"agent-lab.experiment-installation-key.v1\0" +PLAN_DOMAIN = b"agent-lab.experiment-plan.v1\0" +DECISION_DOMAIN = b"agent-lab.experiment-decision.v1\0" +PROVENANCE_DOMAIN = b"agent-lab.experiment-provenance.v1\0" +RECEIPT_DOMAIN = b"agent-lab.experiment-install-receipt.v1\0" +SOURCE_DIGEST_DOMAIN = b"agent-lab.experiment-tree.v1\0" +STAGE_PAYLOAD_DOMAIN = b"agent-lab.experiment-stage-payload.v1\0" +INSTALL_API = "agent-lab.experiment-install/v0alpha1" +PROVENANCE_API = "agent-lab.experiment-provenance/v0alpha1" +INTENT_API = "agent-lab.experiment-install-intent/v0alpha1" +LOCK_SCHEMA = "agent-lab.experiments-lock/v0alpha1" +OPERATION_WRAPPER = "experiment-install" +CLEANUP_WRAPPER = "experiment-install-cleanup" +OWNERSHIP_MARKER = "owner" +OWNERSHIP_BYTES = b"agent-lab.experiment-stage-owner/v0alpha1\n" +MAX_STAGE_ENTRIES = 16 +MAX_STAGE_BYTES = 4_194_304 +MAX_AUTHORITY_BYTES = 65_536 +MAX_ARTIFACT_BYTES = 262_144 +MAX_ARCHIVE_BYTES = 1_048_576 +MAX_RECORD_BYTES = 1_048_576 +SAFE_COMPONENT = re.compile(r"^[a-z][a-z0-9-]{0,47}$") +EXPERIMENT_NAME = re.compile(r"^[a-z](?:[a-z0-9-]{0,61}[a-z0-9])?$") +IMAGE_COMPONENT = re.compile(r"^[a-z][a-z0-9]*(?:-[a-z0-9]+)*$") +SHA256 = re.compile(r"^sha256:[0-9a-f]{64}$") +GIT_SHA1 = re.compile(r"^sha1:[0-9a-f]{40}$") +GIT_COMMIT = re.compile(r"^[0-9a-f]{40}$") +GIT_URL = re.compile( + r"^https://github\.com/" + r"([a-z0-9](?:[a-z0-9-]{0,37}[a-z0-9])?)/" + r"([a-z0-9_.-]{1,100})\.git$", + re.ASCII, +) +PAYLOAD_DIRECTORIES = {"payload", "payload/artifact", "payload/records"} +PAYLOAD_FILES = { + "payload/artifact/experiment.cue", + "payload/records/decision.json", + "payload/records/install.json", + "payload/records/plan.json", + "payload/records/provenance.json", +} +STAGE_ALLOWED = {OWNERSHIP_MARKER, "intent.json", *PAYLOAD_DIRECTORIES, *PAYLOAD_FILES} +RECORD_PATHS = { + "artifact/experiment.cue", + "records/decision.json", + "records/plan.json", + "records/provenance.json", +} + +FaultHook = Callable[[str], None] + + +class StoreError(Exception): + """Base class for classified Experiment-store failures.""" + + +class StoreReject(StoreError): + """The requested operation is a safe ordinary rejection.""" + + +class StoreInfrastructure(StoreError): + """The store could not establish a trustworthy result.""" + + +class DuplicateKey(ValueError): + """A JSON object contains a duplicate decoded key.""" + + +class HomeAuthority(NamedTuple): + home: Path + store: Path + staging: Path + state: Path + locks: Path + lock: Path + lock_device: int + lock_inode: int + config_raw: bytes + receipt_raw: bytes + store_device: int + store_identity: tuple[int, int] + staging_identity: tuple[int, int] + state_identity: tuple[int, int] + locks_identity: tuple[int, int] + + +class VerifiedInstall(NamedTuple): + name: str + installation_key: str + receipt_digest: str + file_digests: dict[str, str] + + +class ScannedWrapper(NamedTuple): + intent: dict[str, object] | None + has_payload: bool + + +def canonical(value: object) -> bytes: + try: + return json.dumps( + value, + allow_nan=False, + ensure_ascii=True, + separators=(",", ":"), + sort_keys=True, + ).encode("ascii") + except (RecursionError, TypeError, ValueError, UnicodeError) as error: + raise StoreInfrastructure("store data cannot be encoded canonically") from error + + +def digest(data: bytes) -> str: + return "sha256:" + hashlib.sha256(data).hexdigest() + + +def _plan_digest(plan: object) -> str: + return digest(PLAN_DOMAIN + canonical(plan)) + + +def _reject(message: str) -> NoReturn: + raise StoreReject(message) + + +def _infra(message: str, error: BaseException | None = None) -> NoReturn: + if error is None: + raise StoreInfrastructure(message) + raise StoreInfrastructure(message) from error + + +def _fault(hook: FaultHook | None, point: str) -> None: + if hook is not None: + hook(point) + + +def _pairs(pairs: list[tuple[str, object]]) -> dict[str, object]: + result: dict[str, object] = {} + for key, value in pairs: + if key in result: + raise DuplicateKey(key) + result[key] = value + return result + + +def _nonfinite(value: str) -> NoReturn: + raise ValueError(value) + + +def _parse_object(data: bytes, purpose: str) -> dict[str, object]: + try: + text = data.decode("ascii") + value = json.loads( + text, + object_pairs_hook=_pairs, + parse_constant=_nonfinite, + ) + _reject_nonfinite(value) + except (DuplicateKey, UnicodeError, json.JSONDecodeError, RecursionError, ValueError) as error: + _infra(f"{purpose} is not bounded canonical JSON", error) + if not isinstance(value, dict): + _infra(f"{purpose} is not one JSON object") + if data != canonical(value) + b"\n": + _infra(f"{purpose} is not canonical") + return value + + +def _reject_nonfinite(value: object) -> None: + if isinstance(value, float) and not math.isfinite(value): + raise ValueError("non-finite number") + if isinstance(value, dict): + for item in value.values(): + _reject_nonfinite(item) + elif isinstance(value, list): + for item in value: + _reject_nonfinite(item) + + +def _identity(metadata: os.stat_result) -> tuple[int, ...]: + return ( + metadata.st_dev, + metadata.st_ino, + metadata.st_mode, + metadata.st_uid, + metadata.st_nlink, + metadata.st_size, + metadata.st_mtime_ns, + metadata.st_ctime_ns, + ) + + +def _absolute_directory_chain(path: Path) -> None: + if not path.is_absolute() or path == Path("/") or os.path.normpath(str(path)) != str(path): + _infra("Agent Lab home path is not an absolute canonical non-root path") + current = Path("/") + for component in path.parts[1:]: + current /= component + try: + metadata = current.lstat() + except FileNotFoundError: + _reject("Agent Lab home is not initialized") + except OSError as error: + _infra("Agent Lab home path cannot be inspected", error) + if stat.S_ISLNK(metadata.st_mode) or not stat.S_ISDIR(metadata.st_mode): + _infra("Agent Lab home path contains an unsafe component") + + +def _verify_directory( + path: Path, + *, + modes: tuple[int, ...] = (0o700,), + device: int | None = None, +) -> os.stat_result: + try: + lexical = path.lstat() + descriptor = os.open( + path, + os.O_RDONLY + | getattr(os, "O_CLOEXEC", 0) + | getattr(os, "O_DIRECTORY", 0) + | getattr(os, "O_NOFOLLOW", 0), + ) + except OSError as error: + _infra("store directory is unavailable", error) + try: + opened = os.fstat(descriptor) + except OSError as error: + try: + os.close(descriptor) + except OSError: + pass + _infra("store directory cannot be verified", error) + try: + os.close(descriptor) + except OSError as error: + _infra("store directory descriptor cannot be closed", error) + for metadata in (lexical, opened): + if ( + stat.S_ISLNK(metadata.st_mode) + or not stat.S_ISDIR(metadata.st_mode) + or metadata.st_uid != os.getuid() + or stat.S_IMODE(metadata.st_mode) not in modes + or (device is not None and metadata.st_dev != device) + ): + _infra("store directory metadata is unsafe") + if (lexical.st_dev, lexical.st_ino) != (opened.st_dev, opened.st_ino): + _infra("store directory identity changed") + return opened + + +def _read_file( + path: Path, + maximum: int, + purpose: str, + *, + mode: int, + device: int | None = None, +) -> bytes: + try: + lexical = path.lstat() + if ( + stat.S_ISLNK(lexical.st_mode) + or not stat.S_ISREG(lexical.st_mode) + or lexical.st_uid != os.getuid() + or lexical.st_nlink != 1 + or stat.S_IMODE(lexical.st_mode) != mode + or lexical.st_size > maximum + or (device is not None and lexical.st_dev != device) + ): + _infra(f"{purpose} metadata is unsafe") + descriptor = os.open( + path, + os.O_RDONLY + | getattr(os, "O_CLOEXEC", 0) + | getattr(os, "O_NOFOLLOW", 0) + | getattr(os, "O_NONBLOCK", 0), + ) + except StoreError: + raise + except OSError as error: + _infra(f"{purpose} is unavailable", error) + try: + opened = os.fstat(descriptor) + chunks: list[bytes] = [] + remaining = maximum + 1 + while remaining: + chunk = os.read(descriptor, min(65_536, remaining)) + if not chunk: + break + chunks.append(chunk) + remaining -= len(chunk) + data = b"".join(chunks) + final = os.fstat(descriptor) + except OSError as error: + _infra(f"{purpose} cannot be read safely", error) + finally: + try: + os.close(descriptor) + except OSError as error: + _infra(f"{purpose} descriptor cannot be closed", error) + try: + current = path.lstat() + except OSError as error: + _infra(f"{purpose} cannot be reverified", error) + for metadata in (opened, final, current): + if ( + not stat.S_ISREG(metadata.st_mode) + or metadata.st_uid != os.getuid() + or metadata.st_nlink != 1 + or stat.S_IMODE(metadata.st_mode) != mode + or metadata.st_size > maximum + or (device is not None and metadata.st_dev != device) + ): + _infra(f"{purpose} metadata changed unsafely") + if ( + len(data) > maximum + or len(data) != final.st_size + or _identity(lexical) != _identity(opened) + or _identity(opened) != _identity(final) + or _identity(final) != _identity(current) + ): + _infra(f"{purpose} changed while being read") + return data + + +def _load_home(home: Path) -> HomeAuthority: + try: + _absolute_directory_chain(home) + _verify_directory(home) + config_path = home / "config.json" + receipt_path = home / "home.json" + try: + config_raw = _read_file( + config_path, + MAX_AUTHORITY_BYTES, + "Agent Lab configuration", + mode=0o600, + ) + receipt_raw = _read_file( + receipt_path, + MAX_AUTHORITY_BYTES, + "Agent Lab home receipt", + mode=0o600, + ) + except StoreInfrastructure: + if not os.path.lexists(config_path) and not os.path.lexists(receipt_path): + _reject("Agent Lab home is not initialized") + raise + config = _parse_object(config_raw, "Agent Lab configuration") + receipt = _parse_object(receipt_raw, "Agent Lab home receipt") + if set(config) != {"apiVersion", "paths"} or config.get("apiVersion") != "agent-lab.config/v0alpha1": + _infra("Agent Lab configuration schema is not closed") + paths = config.get("paths") + if ( + not isinstance(paths, dict) + or set(paths) != {"experiments", "images", "cache", "state"} + or len(set(paths.values())) != 4 + or any( + not isinstance(item, str) or SAFE_COMPONENT.fullmatch(item) is None + for item in paths.values() + ) + ): + _infra("Agent Lab configuration paths are unsafe") + expected_config_digest = digest(canonical(config)) + locks_value = receipt.get("locks") + if ( + set(receipt) != {"apiVersion", "configDigest", "locks", "paths"} + or receipt.get("apiVersion") != "agent-lab.home/v0alpha1" + or receipt.get("configDigest") != expected_config_digest + or receipt.get("paths") != paths + or not isinstance(locks_value, dict) + or set(locks_value) != {"experiments", "imageCatalog"} + ): + _infra("Agent Lab home receipt is not closed") + lock_record = locks_value.get("experiments") + expected_lock_path = f"{paths['state']}/locks/experiments.lock" + if ( + not isinstance(lock_record, dict) + or set(lock_record) != {"device", "inode", "path", "schema"} + or type(lock_record.get("device")) is not int + or type(lock_record.get("inode")) is not int + or lock_record.get("path") != expected_lock_path + or lock_record.get("schema") != LOCK_SCHEMA + ): + _infra("Experiment store lock authority is absent from the home receipt") + store = home / str(paths["experiments"]) + state = home / str(paths["state"]) + staging = store / ".staging" + locks = state / "locks" + store_metadata = _verify_directory(store) + state_metadata = _verify_directory(state) + locks_metadata = _verify_directory(locks) + staging_metadata = _verify_directory(staging, device=store_metadata.st_dev) + lock = locks / "experiments.lock" + try: + lock_metadata = lock.lstat() + except OSError as error: + _infra("Experiment store lock is unavailable", error) + if ( + lock_metadata.st_dev != lock_record["device"] + or lock_metadata.st_ino != lock_record["inode"] + ): + _infra("Experiment store lock identity does not match the home receipt") + return HomeAuthority( + home, + store, + staging, + state, + locks, + lock, + int(lock_record["device"]), + int(lock_record["inode"]), + config_raw, + receipt_raw, + store_metadata.st_dev, + (store_metadata.st_dev, store_metadata.st_ino), + (staging_metadata.st_dev, staging_metadata.st_ino), + (state_metadata.st_dev, state_metadata.st_ino), + (locks_metadata.st_dev, locks_metadata.st_ino), + ) + except StoreError: + raise + except OSError as error: + _infra("Agent Lab home cannot be validated", error) + + +def _revalidate_authority(authority: HomeAuthority) -> None: + if ( + _read_file( + authority.home / "config.json", + MAX_AUTHORITY_BYTES, + "Agent Lab configuration", + mode=0o600, + ) + != authority.config_raw + or _read_file( + authority.home / "home.json", + MAX_AUTHORITY_BYTES, + "Agent Lab home receipt", + mode=0o600, + ) + != authority.receipt_raw + ): + _infra("Agent Lab configuration changed during installation") + store = _verify_directory(authority.store) + state = _verify_directory(authority.state) + locks = _verify_directory(authority.locks) + staging = _verify_directory(authority.staging, device=store.st_dev) + if ( + store.st_dev != authority.store_device + or (store.st_dev, store.st_ino) != authority.store_identity + or (staging.st_dev, staging.st_ino) != authority.staging_identity + or (state.st_dev, state.st_ino) != authority.state_identity + or (locks.st_dev, locks.st_ino) != authority.locks_identity + ): + _infra("Experiment store directory authority changed during installation") + + +@contextmanager +def _store_lock( + authority: HomeAuthority, + fault: FaultHook | None, + *, + exclusive: bool = True, +) -> Iterator[int]: + maximum = len(LOCK_SCHEMA.encode("ascii") + b"\n") + path = authority.lock + descriptor = -1 + try: + lexical = path.lstat() + if ( + not stat.S_ISREG(lexical.st_mode) + or stat.S_ISLNK(lexical.st_mode) + or lexical.st_uid != os.getuid() + or lexical.st_nlink != 1 + or stat.S_IMODE(lexical.st_mode) != 0o600 + or lexical.st_size != maximum + or (lexical.st_dev, lexical.st_ino) + != (authority.lock_device, authority.lock_inode) + ): + _infra("Experiment store lock metadata is unsafe") + descriptor = os.open( + path, + os.O_RDWR + | getattr(os, "O_CLOEXEC", 0) + | getattr(os, "O_NOFOLLOW", 0) + | getattr(os, "O_NONBLOCK", 0), + ) + opened = os.fstat(descriptor) + if ( + not stat.S_ISREG(opened.st_mode) + or opened.st_uid != os.getuid() + or opened.st_nlink != 1 + or stat.S_IMODE(opened.st_mode) != 0o600 + or opened.st_size != maximum + or (opened.st_dev, opened.st_ino) + != (authority.lock_device, authority.lock_inode) + ): + _infra("Experiment store lock identity changed before acquisition") + fcntl.flock(descriptor, fcntl.LOCK_EX if exclusive else fcntl.LOCK_SH) + held = os.fstat(descriptor) + current = path.lstat() + expected = (authority.lock_device, authority.lock_inode) + if ( + _identity(opened) != _identity(held) + or (current.st_dev, current.st_ino) != expected + or not stat.S_ISREG(current.st_mode) + or current.st_uid != os.getuid() + or current.st_nlink != 1 + or stat.S_IMODE(current.st_mode) != 0o600 + or current.st_size != maximum + ): + _infra("Experiment store lock path changed while acquiring it") + os.lseek(descriptor, 0, os.SEEK_SET) + if os.read(descriptor, maximum + 1) != LOCK_SCHEMA.encode("ascii") + b"\n": + _infra("Experiment store lock receipt is malformed") + os.lseek(descriptor, 0, os.SEEK_SET) + if exclusive: + _fault(fault, "experiment store lock.after_acquire") + yield descriptor + except StoreError: + raise + except OSError as error: + _infra("Experiment store lock cannot be held safely", error) + finally: + if descriptor >= 0: + release_error: OSError | None = None + close_error: OSError | None = None + try: + fcntl.flock(descriptor, fcntl.LOCK_UN) + except OSError as error: + release_error = error + try: + os.close(descriptor) + except OSError as error: + close_error = error + if release_error is not None or close_error is not None: + cause = close_error if close_error is not None else release_error + _infra("Experiment store lock release or close is uncertain", cause) + + +def _module(path: Path, name: str): + spec = spec_from_file_location(name, path) + if spec is None or spec.loader is None: + _infra(f"{path.name} cannot be loaded") + value = module_from_spec(spec) + sys.modules[spec.name] = value + try: + spec.loader.exec_module(value) + except (ImportError, OSError) as error: + _infra(f"{path.name} cannot be loaded", error) + return value + + +def _experiment_module(): + return _module(Path(__file__).resolve().with_name("experiment.py"), "agent_lab_experiment_store_planning") + + +def _catalog_module(): + return _module(Path(__file__).resolve().with_name("image_catalog.py"), "agent_lab_experiment_store_catalog") + + +def _selected_entries(plan: dict[str, object]) -> list[dict[str, object]]: + try: + members = plan["spec"]["members"] # type: ignore[index] + if not isinstance(members, list): + raise ValueError("members") + selected: dict[tuple[str, str], dict[str, object]] = {} + for member in members: + if not isinstance(member, dict): + raise ValueError("member") + requested = member["requestedSelector"] + resolved = member["resolvedImage"] + if not isinstance(requested, dict) or not isinstance(resolved, dict): + raise ValueError("selector") + origin = resolved.get("origin") + if origin == "direct": + if set(resolved) != {"origin", "subject"} or set(requested) != {"digestRef"}: + raise ValueError("direct selector") + continue + if ( + origin not in {"agent-lab", "local"} + or set(requested) != {"catalogName"} + or set(resolved) != {"entryDigest", "generation", "origin", "subject"} + ): + raise ValueError("catalog selector") + name = requested["catalogName"] + entry = { + "entryDigest": resolved["entryDigest"], + "generation": resolved["generation"], + "name": name, + "origin": origin, + "subject": resolved["subject"], + } + if ( + not _image_name(name) + or SHA256.fullmatch(str(entry["entryDigest"])) is None + or type(entry["generation"]) is not int + or int(entry["generation"]) < 1 + or not isinstance(entry["subject"], str) + ): + raise ValueError("selected identity") + key = (str(origin), name) + if key in selected and selected[key] != entry: + raise ValueError("inconsistent selected identity") + selected[key] = entry + return [ + selected[key] + for key in sorted( + selected, + key=lambda item: (item[0].encode(), item[1].encode()), + ) + ] + except (KeyError, TypeError, ValueError) as error: + _infra("Experiment plan has invalid selected-entry identities", error) + + +def _local_dependencies(selected: Sequence[dict[str, object]]) -> tuple[dict[str, object], ...]: + return tuple( + { + "entryDigest": item["entryDigest"], + "generation": item["generation"], + "name": item["name"], + "subject": item["subject"], + } + for item in selected + if item["origin"] == "local" + ) + + +def _verify_held_catalog( + held: object, + dependencies: Sequence[dict[str, object]], +) -> None: + try: + if not isinstance(held, dict) or set(held) != {"catalog", "records"}: + raise ValueError("held envelope") + catalog = held["catalog"] + records = held["records"] + names = {str(item["name"]) for item in dependencies} + if ( + not isinstance(catalog, dict) + or set(catalog) != {"revision", "snapshotDigest"} + or type(catalog.get("revision")) is not int + or int(catalog["revision"]) < 1 + or SHA256.fullmatch(str(catalog.get("snapshotDigest"))) is None + or not isinstance(records, dict) + or set(records) != names + ): + raise ValueError("held catalog evidence") + by_name = {str(item["name"]): item for item in dependencies} + for name in names: + record = records[name] + expected = by_name[name] + if ( + not isinstance(record, dict) + or record.get("name") != name + or record.get("state") != "active" + or record.get("entryDigest") != expected["entryDigest"] + or record.get("generation") != expected["generation"] + or record.get("subject") != expected["subject"] + ): + raise ValueError("held selected entry") + except (KeyError, TypeError, ValueError) as error: + _infra("held local image catalog evidence is invalid", error) + + +def _source_digest(data: bytes) -> str: + name = b"experiment.cue" + value = hashlib.sha256(SOURCE_DIGEST_DOMAIN) + value.update(len(name).to_bytes(4, "big")) + value.update(name) + value.update(len(data).to_bytes(8, "big")) + value.update(data) + return "sha256:" + value.hexdigest() + + +def _image_name(value: object) -> bool: + if not isinstance(value, str) or not value.isascii(): + return False + parts = value.split(".") + return ( + len(value.encode("ascii")) <= 63 + and len(parts) == 2 + and all(1 <= len(part.encode("ascii")) <= 31 for part in parts) + and all(IMAGE_COMPONENT.fullmatch(part) is not None for part in parts) + ) + + +def _installation_identity( + source_digest: str, + plan: dict[str, object], + decision: dict[str, object], + selected: Sequence[dict[str, object]], +) -> dict[str, object]: + try: + contract = plan["contract"] + binding = decision["binding"] + if not isinstance(contract, dict) or not isinstance(binding, dict): + raise ValueError("binding") + identity = { + "authorizationDigest": binding["authorizationDigest"], + "contractDigest": contract["digest"], + "planDigest": binding["planDigest"], + "selectedEntries": list(selected), + "sourceDigest": source_digest, + } + if any( + SHA256.fullmatch(str(identity[key])) is None + for key in ("authorizationDigest", "contractDigest", "planDigest", "sourceDigest") + ): + raise ValueError("digest") + return identity + except (KeyError, TypeError, ValueError) as error: + _infra("installation identity cannot be derived", error) + + +def _validate_catalog_evidence( + selected: Sequence[dict[str, object]], + evidence: object, +) -> None: + origins = {item["origin"] for item in selected} + expected_keys: set[str] = set() + if "agent-lab" in origins: + expected_keys.add("bundled") + if "local" in origins: + expected_keys.add("local") + if not expected_keys: + if evidence is not None: + _infra("unexpected image catalog provenance") + return + if not isinstance(evidence, dict) or set(evidence) != expected_keys: + _infra("image catalog provenance is incomplete") + try: + if "bundled" in expected_keys: + bundled = evidence["bundled"] + if ( + not isinstance(bundled, dict) + or set(bundled) != {"snapshotDigest"} + or SHA256.fullmatch(str(bundled.get("snapshotDigest"))) is None + ): + raise ValueError("bundled evidence") + if "local" in expected_keys: + local = evidence["local"] + if ( + not isinstance(local, dict) + or set(local) != {"revision", "snapshotDigest"} + or type(local.get("revision")) is not int + or int(local["revision"]) < 1 + or SHA256.fullmatch(str(local.get("snapshotDigest"))) is None + ): + raise ValueError("local evidence") + except (KeyError, TypeError, ValueError) as error: + _infra("image catalog provenance is invalid", error) + + +def _candidate( + snapshot: object, + plan: dict[str, object], + decision: dict[str, object], + selected: Sequence[dict[str, object]], + catalog_evidence: dict[str, object] | None, +) -> tuple[dict[str, bytes], dict[str, object], str, str]: + source_data = getattr(snapshot, "data", None) + source_digest = getattr(snapshot, "digest", None) + if not isinstance(source_data, bytes) or not isinstance(source_digest, str): + _infra("source snapshot is malformed") + if source_digest != _source_digest(source_data): + _infra("source snapshot digest is inconsistent") + source_transport = _source_transport(snapshot) + _validate_catalog_evidence(selected, catalog_evidence) + identity = _installation_identity(source_digest, plan, decision, selected) + installation_key = digest(INSTALL_KEY_DOMAIN + canonical(identity)) + plan_bytes = canonical(plan) + b"\n" + decision_bytes = canonical(decision) + b"\n" + provenance = { + "apiVersion": PROVENANCE_API, + "authorizationDigest": identity["authorizationDigest"], + "catalog": catalog_evidence, + "contractDigest": identity["contractDigest"], + "kind": "ExperimentInstallationProvenance", + "planDigest": identity["planDigest"], + "selectedEntries": list(selected), + "source": { + "bytes": len(source_data), + "digest": source_digest, + "entryCount": 1, + "fileCount": 1, + "format": "agent-lab.experiment-tree/v1", + "kind": "directory", + }, + "transport": source_transport, + } + provenance_bytes = canonical(provenance) + b"\n" + files = { + "artifact/experiment.cue": source_data, + "records/decision.json": decision_bytes, + "records/plan.json": plan_bytes, + "records/provenance.json": provenance_bytes, + } + records = { + "artifact/experiment.cue": { + "digest": digest(source_data), + "schema": "agent-lab/v0alpha1", + }, + "records/decision.json": { + "digest": digest(DECISION_DOMAIN + canonical(decision)), + "schema": decision.get("apiVersion"), + }, + "records/plan.json": { + "digest": _plan_digest(plan), + "schema": plan.get("apiVersion"), + }, + "records/provenance.json": { + "digest": digest(PROVENANCE_DOMAIN + canonical(provenance)), + "schema": PROVENANCE_API, + }, + } + try: + requested_name = plan["metadata"]["requestedName"] # type: ignore[index] + except (KeyError, TypeError) as error: + _infra("Experiment plan has no requested name", error) + if not isinstance(requested_name, str) or EXPERIMENT_NAME.fullmatch(requested_name) is None: + _infra("Experiment plan requested name is unsafe") + receipt = { + "apiVersion": INSTALL_API, + "identity": identity, + "installationKey": installation_key, + "kind": "ExperimentInstallationReceipt", + "name": requested_name, + "records": records, + } + receipt_bytes = canonical(receipt) + b"\n" + files["records/install.json"] = receipt_bytes + return files, receipt, installation_key, digest(RECEIPT_DOMAIN + canonical(receipt)) + + +def _closed_git_transport(transport: object) -> dict[str, object] | None: + if not isinstance(transport, dict) or transport.get("kind") != "git": + return None + if set(transport) != { + "acquisition", + "blob", + "commit", + "kind", + "requestedCommit", + "tree", + "url", + }: + return None + acquisition = transport.get("acquisition") + requested = transport.get("requestedCommit") + tree = transport.get("tree") + blob = transport.get("blob") + url = transport.get("url") + url_match = GIT_URL.fullmatch(url) if isinstance(url, str) else None + if ( + not isinstance(acquisition, dict) + or set(acquisition) + != { + "acquiredBytes", + "limitBytes", + "method", + "requestCount", + "temporaryBytes", + "temporaryFiles", + } + or type(acquisition.get("acquiredBytes")) is not int + or not 1 <= int(acquisition["acquiredBytes"]) <= MAX_ARCHIVE_BYTES + or type(acquisition.get("limitBytes")) is not int + or acquisition.get("limitBytes") != MAX_ARCHIVE_BYTES + or acquisition.get("method") != "github-git-data-v3" + or type(acquisition.get("requestCount")) is not int + or acquisition.get("requestCount") != 3 + or type(acquisition.get("temporaryBytes")) is not int + or acquisition.get("temporaryBytes") != 0 + or type(acquisition.get("temporaryFiles")) is not int + or acquisition.get("temporaryFiles") != 0 + or not isinstance(requested, str) + or GIT_COMMIT.fullmatch(requested) is None + or transport.get("commit") != f"sha1:{requested}" + or not isinstance(tree, str) + or GIT_SHA1.fullmatch(tree) is None + or not isinstance(blob, str) + or GIT_SHA1.fullmatch(blob) is None + or not isinstance(url, str) + or url_match is None + or "--" in url_match.group(1) + or url_match.group(2) in (".", "..") + ): + return None + return { + "acquisition": { + "acquiredBytes": acquisition["acquiredBytes"], + "limitBytes": MAX_ARCHIVE_BYTES, + "method": "github-git-data-v3", + "requestCount": 3, + "temporaryBytes": 0, + "temporaryFiles": 0, + }, + "blob": transport["blob"], + "commit": transport["commit"], + "kind": "git", + "requestedCommit": requested, + "tree": transport["tree"], + "url": transport["url"], + } + + +def _source_transport(snapshot: object) -> dict[str, object]: + transport = getattr(snapshot, "transport", None) + if transport is None or transport == {"kind": "directory"}: + return {"kind": "local-directory"} + git_transport = _closed_git_transport(transport) + if git_transport is not None: + return git_transport + if ( + not isinstance(transport, dict) + or set(transport) != {"archiveBytes", "archiveDigest", "kind"} + or transport.get("kind") != "zip" + or not isinstance(transport.get("archiveBytes"), int) + or isinstance(transport.get("archiveBytes"), bool) + or not 1 <= int(transport["archiveBytes"]) <= MAX_ARCHIVE_BYTES + or SHA256.fullmatch(str(transport.get("archiveDigest"))) is None + ): + _infra("source transport provenance is malformed") + return { + "archiveBytes": transport["archiveBytes"], + "archiveDigest": transport["archiveDigest"], + "kind": "zip", + } + + +def _validate_transport_provenance(transport: object) -> None: + if transport == {"kind": "local-directory"}: + return + if _closed_git_transport(transport) is not None: + return + if ( + not isinstance(transport, dict) + or set(transport) != {"archiveBytes", "archiveDigest", "kind"} + or transport.get("kind") != "zip" + or not isinstance(transport.get("archiveBytes"), int) + or isinstance(transport.get("archiveBytes"), bool) + or not 1 <= int(transport["archiveBytes"]) <= MAX_ARCHIVE_BYTES + or SHA256.fullmatch(str(transport.get("archiveDigest"))) is None + ): + _infra("installed source transport provenance is invalid") + + +def _directory_names(path: Path, purpose: str, maximum: int) -> tuple[str, ...]: + try: + names: list[str] = [] + with os.scandir(path) as entries: + for entry in entries: + if len(names) >= maximum: + _infra(f"{purpose} exceeds its fixed entry bound") + if not isinstance(entry.name, str): + _infra(f"{purpose} contains an invalid name") + names.append(entry.name) + except StoreError: + raise + except OSError as error: + _infra(f"{purpose} cannot be enumerated", error) + try: + return tuple(sorted(names, key=lambda item: os.fsencode(item))) + except (TypeError, UnicodeError) as error: + _infra(f"{purpose} contains an invalid name", error) + + +_ORIGINAL_DIRECTORY_NAMES = _directory_names + + +def _path_state(path: Path) -> str: + try: + metadata = path.lstat() + except FileNotFoundError: + return "absent" + except OSError as error: + _infra("store path state is ambiguous", error) + if stat.S_ISDIR(metadata.st_mode) and not stat.S_ISLNK(metadata.st_mode): + return "directory" + return "other" + + +def _record_schema(receipt: dict[str, object], path: str) -> tuple[str, str]: + try: + records = receipt["records"] + if not isinstance(records, dict) or set(records) != RECORD_PATHS: + raise ValueError("record set") + record = records[path] + if ( + not isinstance(record, dict) + or set(record) != {"digest", "schema"} + or SHA256.fullmatch(str(record.get("digest"))) is None + or not isinstance(record.get("schema"), str) + ): + raise ValueError("record binding") + return str(record["digest"]), str(record["schema"]) + except (KeyError, TypeError, ValueError) as error: + _infra("installation receipt record binding is invalid", error) + + +def _verify_envelope( + path: Path, + expected_name: str, + device: int, + *, + root_modes: tuple[int, ...] = (0o500,), +) -> VerifiedInstall: + if EXPERIMENT_NAME.fullmatch(expected_name) is None: + _infra("installed Experiment name is unsafe") + root_before = _verify_directory(path, modes=root_modes, device=device) + root_names = ("artifact", "records") + if _directory_names(path, "installed envelope", 2) != root_names: + _infra("installed envelope layout is not closed") + artifact_dir = path / "artifact" + records_dir = path / "records" + artifact_before = _verify_directory(artifact_dir, modes=(0o500,), device=device) + records_before = _verify_directory(records_dir, modes=(0o500,), device=device) + artifact_names = ("experiment.cue",) + records_names = ( + "decision.json", + "install.json", + "plan.json", + "provenance.json", + ) + if _directory_names(artifact_dir, "installed artifact", 1) != artifact_names: + _infra("installed artifact layout is not closed") + if _directory_names(records_dir, "installed records", 4) != records_names: + _infra("installed record layout is not closed") + raw = { + "artifact/experiment.cue": _read_file( + artifact_dir / "experiment.cue", + MAX_ARTIFACT_BYTES, + "installed Experiment artifact", + mode=0o400, + device=device, + ), + "records/decision.json": _read_file( + records_dir / "decision.json", + MAX_RECORD_BYTES, + "installed authorization decision", + mode=0o400, + device=device, + ), + "records/install.json": _read_file( + records_dir / "install.json", + MAX_RECORD_BYTES, + "installed receipt", + mode=0o400, + device=device, + ), + "records/plan.json": _read_file( + records_dir / "plan.json", + MAX_RECORD_BYTES, + "installed plan", + mode=0o400, + device=device, + ), + "records/provenance.json": _read_file( + records_dir / "provenance.json", + MAX_RECORD_BYTES, + "installed provenance", + mode=0o400, + device=device, + ), + } + plan = _parse_object(raw["records/plan.json"], "installed plan") + decision = _parse_object(raw["records/decision.json"], "installed authorization decision") + provenance = _parse_object(raw["records/provenance.json"], "installed provenance") + receipt = _parse_object(raw["records/install.json"], "installed receipt") + try: + if ( + set(plan) != {"apiVersion", "contract", "kind", "metadata", "spec"} + or plan.get("apiVersion") != "agent-lab.request/v0alpha1" + or plan.get("kind") != "RequestedExperimentPlan" + or not isinstance(plan.get("metadata"), dict) + or plan["metadata"].get("requestedName") != expected_name # type: ignore[union-attr] + or not isinstance(plan.get("contract"), dict) + ): + raise ValueError("plan") + contract = plan["contract"] + if ( + set(contract) != {"digest", "name", "version"} # type: ignore[arg-type] + or contract.get("name") != "agent-lab.experiment" # type: ignore[union-attr] + or contract.get("version") != "v0alpha1" # type: ignore[union-attr] + or SHA256.fullmatch(str(contract.get("digest"))) is None # type: ignore[union-attr] + ): + raise ValueError("contract") + plan_digest = _plan_digest(plan) + if ( + set(decision) != {"action", "apiVersion", "binding", "kind", "principal", "resource", "verdict"} + or decision.get("apiVersion") != "agent-lab.authorization/v0alpha1" + or decision.get("kind") != "ExperimentAuthorizationDecision" + or decision.get("action") != "experiment.install" + or decision.get("verdict") != "permit" + or not isinstance(decision.get("binding"), dict) + ): + raise ValueError("decision") + binding = decision["binding"] + source_digest = _source_digest(raw["artifact/experiment.cue"]) + if ( + set(binding) != {"authorizationDigest", "contractDigest", "planDigest", "sourceDigest"} # type: ignore[arg-type] + or binding.get("planDigest") != plan_digest # type: ignore[union-attr] + or binding.get("sourceDigest") != source_digest # type: ignore[union-attr] + or binding.get("contractDigest") != contract.get("digest") # type: ignore[union-attr] + or SHA256.fullmatch(str(binding.get("authorizationDigest"))) is None # type: ignore[union-attr] + ): + raise ValueError("decision binding") + selected = _selected_entries(plan) + identity = _installation_identity(source_digest, plan, decision, selected) + if ( + set(provenance) != { + "apiVersion", "authorizationDigest", "catalog", "contractDigest", "kind", + "planDigest", "selectedEntries", "source", "transport", + } + or provenance.get("apiVersion") != PROVENANCE_API + or provenance.get("kind") != "ExperimentInstallationProvenance" + or provenance.get("authorizationDigest") != identity["authorizationDigest"] + or provenance.get("contractDigest") != identity["contractDigest"] + or provenance.get("planDigest") != identity["planDigest"] + or provenance.get("selectedEntries") != list(selected) + or provenance.get("source") != { + "bytes": len(raw["artifact/experiment.cue"]), + "digest": source_digest, + "entryCount": 1, + "fileCount": 1, + "format": "agent-lab.experiment-tree/v1", + "kind": "directory", + } + ): + raise ValueError("provenance") + _validate_transport_provenance(provenance.get("transport")) + catalog = provenance.get("catalog") + _validate_catalog_evidence(selected, catalog) + installation_key = digest(INSTALL_KEY_DOMAIN + canonical(identity)) + if ( + set(receipt) != {"apiVersion", "identity", "installationKey", "kind", "name", "records"} + or receipt.get("apiVersion") != INSTALL_API + or receipt.get("kind") != "ExperimentInstallationReceipt" + or receipt.get("name") != expected_name + or receipt.get("identity") != identity + or receipt.get("installationKey") != installation_key + ): + raise ValueError("receipt") + expected_schemas = { + "artifact/experiment.cue": "agent-lab/v0alpha1", + "records/decision.json": str(decision["apiVersion"]), + "records/plan.json": str(plan["apiVersion"]), + "records/provenance.json": PROVENANCE_API, + } + expected_digests = { + "artifact/experiment.cue": digest(raw["artifact/experiment.cue"]), + "records/decision.json": digest(DECISION_DOMAIN + canonical(decision)), + "records/plan.json": plan_digest, + "records/provenance.json": digest(PROVENANCE_DOMAIN + canonical(provenance)), + } + for record_path in RECORD_PATHS: + record_digest, schema = _record_schema(receipt, record_path) + if record_digest != expected_digests[record_path] or schema != expected_schemas[record_path]: + raise ValueError("record digest") + except (KeyError, TypeError, ValueError) as error: + _infra("installed envelope does not match its receipt", error) + directory_checks = ( + (path, root_before, root_modes, root_names, "installed envelope", 2), + ( + artifact_dir, + artifact_before, + (0o500,), + artifact_names, + "installed artifact", + 1, + ), + ( + records_dir, + records_before, + (0o500,), + records_names, + "installed records", + 4, + ), + ) + for directory, before, modes, expected_names, purpose, maximum in directory_checks: + after = _verify_directory(directory, modes=modes, device=device) + if ( + _identity(before) != _identity(after) + or _directory_names(directory, purpose, maximum) != expected_names + ): + _infra(f"{purpose} changed while being verified") + file_digests = {item: digest(data) for item, data in raw.items()} + return VerifiedInstall( + expected_name, + installation_key, + digest(RECEIPT_DOMAIN + canonical(receipt)), + file_digests, + ) + + +def _intent(files: dict[str, bytes], name: str, key: str, receipt_digest: str) -> dict[str, object]: + file_digests = {path: digest(data) for path, data in sorted(files.items())} + return { + "apiVersion": INTENT_API, + "files": file_digests, + "installationKey": key, + "name": name, + "payloadDigest": digest(STAGE_PAYLOAD_DOMAIN + canonical(file_digests)), + "phase": "prepared", + "receiptDigest": receipt_digest, + } + + +def _validate_intent(value: dict[str, object]) -> None: + try: + files = value["files"] + if ( + set(value) != { + "apiVersion", "files", "installationKey", "name", "payloadDigest", "phase", "receiptDigest", + } + or value["apiVersion"] != INTENT_API + or value["phase"] != "prepared" + or not isinstance(value["name"], str) + or EXPERIMENT_NAME.fullmatch(value["name"]) is None + or SHA256.fullmatch(str(value["installationKey"])) is None + or SHA256.fullmatch(str(value["receiptDigest"])) is None + or not isinstance(files, dict) + or set(files) != { + "artifact/experiment.cue", + "records/decision.json", + "records/install.json", + "records/plan.json", + "records/provenance.json", + } + or any(SHA256.fullmatch(str(item)) is None for item in files.values()) + or value["payloadDigest"] != digest(STAGE_PAYLOAD_DOMAIN + canonical(files)) + ): + raise ValueError("intent") + except (KeyError, TypeError, ValueError) as error: + _infra("Experiment staging intent is invalid", error) + + +def _scan_wrapper(authority: HomeAuthority, path: Path, *, cleanup: bool) -> ScannedWrapper: + _verify_directory(path, modes=(0o700,), device=authority.store_device) + count = 1 + byte_count = 0 + pending = [path] + found: set[str] = set() + while pending: + parent = pending.pop() + remaining = MAX_STAGE_ENTRIES - count + for name in _directory_names(parent, "Experiment staging wrapper", max(remaining, 0)): + item = parent / name + relative = str(item.relative_to(path)) + if relative not in STAGE_ALLOWED: + _infra("Experiment staging wrapper contains an unknown entry") + try: + metadata = item.lstat() + except OSError as error: + _infra("Experiment staging entry cannot be inspected", error) + count += 1 + found.add(relative) + if count > MAX_STAGE_ENTRIES or metadata.st_dev != authority.store_device or metadata.st_uid != os.getuid(): + _infra("Experiment staging state exceeds or leaves its fixed authority") + if stat.S_ISLNK(metadata.st_mode): + _infra("Experiment staging state contains a symlink") + if stat.S_ISDIR(metadata.st_mode): + if relative not in PAYLOAD_DIRECTORIES or stat.S_IMODE(metadata.st_mode) not in (0o700, 0o500): + _infra("Experiment staging directory metadata is unsafe") + pending.append(item) + elif stat.S_ISREG(metadata.st_mode): + allowed_modes = ( + (0o600,) + if relative in {OWNERSHIP_MARKER, "intent.json"} + else (0o600, 0o400) + ) + if stat.S_IMODE(metadata.st_mode) not in allowed_modes or metadata.st_nlink != 1: + _infra("Experiment staging file metadata is unsafe") + byte_count += metadata.st_size + if byte_count > MAX_STAGE_BYTES: + _infra("Experiment staging state exceeds its fixed byte bound") + else: + _infra("Experiment staging state contains an unsafe type") + has_payload = any( + relative == "payload" or relative.startswith("payload/") + for relative in found + ) + has_marker = OWNERSHIP_MARKER in found + if has_marker: + marker = _read_file( + path / OWNERSHIP_MARKER, + len(OWNERSHIP_BYTES), + "Experiment staging ownership marker", + mode=0o600, + device=authority.store_device, + ) + if marker != OWNERSHIP_BYTES: + _infra("Experiment staging ownership marker is malformed") + intent_path = path / "intent.json" + if "intent.json" not in found: + if cleanup and not found: + return ScannedWrapper(None, False) + if has_marker and found == {OWNERSHIP_MARKER}: + return ScannedWrapper(None, False) + _infra("Experiment staging wrapper has no durable intent") + raw_intent = _read_file( + intent_path, + MAX_AUTHORITY_BYTES, + "Experiment staging intent", + mode=0o600, + device=authority.store_device, + ) + try: + value = _parse_object(raw_intent, "Experiment staging intent") + _validate_intent(value) + except StoreInfrastructure: + if has_marker and found == {OWNERSHIP_MARKER, "intent.json"}: + return ScannedWrapper(None, False) + raise + files = value["files"] + assert isinstance(files, dict) + for relative in found & PAYLOAD_FILES: + payload_relative = relative.removeprefix("payload/") + maximum = MAX_ARTIFACT_BYTES if payload_relative == "artifact/experiment.cue" else MAX_RECORD_BYTES + mode = stat.S_IMODE((path / relative).lstat().st_mode) + raw = _read_file( + path / relative, + maximum, + "Experiment staged payload", + mode=mode, + device=authority.store_device, + ) + if digest(raw) != files[payload_relative]: + _infra("Experiment staged payload does not match its durable intent") + return ScannedWrapper(value, has_payload) + + +def _rename_noreplace(source: Path, target: Path) -> None: + try: + library = ctypes.CDLL(None, use_errno=True) + function = library.renameat2 + except (AttributeError, OSError) as error: + _infra("Linux no-replace rename is unavailable", error) + function.argtypes = [ctypes.c_int, ctypes.c_char_p, ctypes.c_int, ctypes.c_char_p, ctypes.c_uint] + function.restype = ctypes.c_int + result = function(-100, os.fsencode(source), -100, os.fsencode(target), 1) + if result != 0: + code = ctypes.get_errno() + if code == errno.EEXIST: + _infra("Experiment no-replace publication raced with an existing target") + _infra("Experiment no-replace publication failed", OSError(code, os.strerror(code))) + + +def _fsync_directory(path: Path, purpose: str, *, modes: tuple[int, ...] = (0o700,)) -> None: + metadata = _verify_directory(path, modes=modes) + descriptor = -1 + try: + descriptor = os.open( + path, + os.O_RDONLY + | getattr(os, "O_CLOEXEC", 0) + | getattr(os, "O_DIRECTORY", 0) + | getattr(os, "O_NOFOLLOW", 0), + ) + if (os.fstat(descriptor).st_dev, os.fstat(descriptor).st_ino) != (metadata.st_dev, metadata.st_ino): + _infra(f"{purpose} directory identity changed") + os.fsync(descriptor) + except StoreError: + raise + except OSError as error: + _infra(f"{purpose} directory cannot be persisted", error) + finally: + unwinding = sys.exc_info()[0] is not None + if descriptor >= 0: + try: + os.close(descriptor) + except OSError as error: + if not unwinding: + _infra(f"{purpose} directory descriptor cannot be closed", error) + + +def _write_all(descriptor: int, data: bytes) -> None: + view = memoryview(data) + while view: + written = os.write(descriptor, view) + if written <= 0: + raise OSError("write made no progress") + view = view[written:] + + +def _link_unnamed(descriptor: int, parent_descriptor: int, name: str, purpose: str) -> None: + try: + library = ctypes.CDLL(None, use_errno=True) + function = library.linkat + except (AttributeError, OSError) as error: + _infra("Linux unnamed-file publication is unavailable", error) + function.argtypes = [ + ctypes.c_int, + ctypes.c_char_p, + ctypes.c_int, + ctypes.c_char_p, + ctypes.c_int, + ] + function.restype = ctypes.c_int + ctypes.set_errno(0) + result = function( + descriptor, + b"", + parent_descriptor, + os.fsencode(name), + 0x1000, # AT_EMPTY_PATH + ) + if result != 0: + code = ctypes.get_errno() + _infra(f"{purpose} could not be linked exclusively", OSError(code, os.strerror(code))) + + +def _write_file(path: Path, data: bytes, purpose: str, fault: FaultHook | None) -> None: + descriptor = -1 + parent_descriptor = -1 + try: + if not hasattr(os, "O_TMPFILE"): + _infra("Linux unnamed staging files are unavailable") + parent_metadata = _verify_directory(path.parent, modes=(0o700,)) + parent_descriptor = os.open( + path.parent, + os.O_RDONLY + | getattr(os, "O_CLOEXEC", 0) + | getattr(os, "O_DIRECTORY", 0) + | getattr(os, "O_NOFOLLOW", 0), + ) + opened_parent = os.fstat(parent_descriptor) + if (opened_parent.st_dev, opened_parent.st_ino) != ( + parent_metadata.st_dev, + parent_metadata.st_ino, + ): + _infra(f"{purpose} parent identity changed") + descriptor = os.open( + ".", + os.O_RDWR + | os.O_TMPFILE + | getattr(os, "O_CLOEXEC", 0) + | getattr(os, "O_NOFOLLOW", 0), + 0o600, + dir_fd=parent_descriptor, + ) + opened = os.fstat(descriptor) + if ( + not stat.S_ISREG(opened.st_mode) + or opened.st_uid != os.getuid() + or opened.st_nlink != 0 + or stat.S_IMODE(opened.st_mode) != 0o600 + or opened.st_dev != opened_parent.st_dev + ): + _infra(f"{purpose} file metadata is unsafe") + _write_all(descriptor, data) + if purpose == "experiment artifact": + _fault(fault, "experiment artifact.after_write") + os.fsync(descriptor) + if purpose == "experiment receipt": + _fault(fault, "experiment receipt.after_fsync") + os.lseek(descriptor, 0, os.SEEK_SET) + chunks: list[bytes] = [] + remaining = len(data) + 1 + while remaining: + chunk = os.read(descriptor, min(65_536, remaining)) + if not chunk: + break + chunks.append(chunk) + remaining -= len(chunk) + observed = b"".join(chunks) + complete = os.fstat(descriptor) + if ( + observed != data + or complete.st_size != len(data) + or complete.st_nlink != 0 + or (complete.st_dev, complete.st_ino) != (opened.st_dev, opened.st_ino) + ): + _infra(f"{purpose} anonymous file identity changed") + _link_unnamed(descriptor, parent_descriptor, path.name, purpose) + linked = os.stat(path.name, dir_fd=parent_descriptor, follow_symlinks=False) + if ( + not stat.S_ISREG(linked.st_mode) + or linked.st_uid != os.getuid() + or linked.st_nlink != 1 + or stat.S_IMODE(linked.st_mode) != 0o600 + or linked.st_size != len(data) + or (linked.st_dev, linked.st_ino) != (complete.st_dev, complete.st_ino) + ): + _infra(f"{purpose} linked file metadata is unsafe") + os.fsync(parent_descriptor) + except StoreError: + raise + except OSError as error: + _infra(f"{purpose} could not be written durably", error) + finally: + unwinding = sys.exc_info()[0] is not None + close_error: OSError | None = None + parent_close_error: OSError | None = None + if descriptor >= 0: + try: + os.close(descriptor) + except OSError as error: + close_error = error + if parent_descriptor >= 0: + try: + os.close(parent_descriptor) + except OSError as error: + parent_close_error = error + if not unwinding and (close_error is not None or parent_close_error is not None): + cause = parent_close_error if parent_close_error is not None else close_error + _infra(f"{purpose} descriptors could not be closed", cause) + + +def _persist_read_only_file(path: Path, purpose: str) -> None: + descriptor = -1 + try: + os.chmod(path, 0o400, follow_symlinks=False) + descriptor = os.open( + path, + os.O_RDONLY + | getattr(os, "O_CLOEXEC", 0) + | getattr(os, "O_NOFOLLOW", 0), + ) + metadata = os.fstat(descriptor) + if ( + not stat.S_ISREG(metadata.st_mode) + or metadata.st_uid != os.getuid() + or metadata.st_nlink != 1 + or stat.S_IMODE(metadata.st_mode) != 0o400 + ): + _infra(f"{purpose} committed metadata is unsafe") + os.fsync(descriptor) + except StoreError: + raise + except OSError as error: + _infra(f"{purpose} mode could not be persisted", error) + finally: + unwinding = sys.exc_info()[0] is not None + if descriptor >= 0: + try: + os.close(descriptor) + except OSError as error: + if not unwinding: + _infra(f"{purpose} descriptor could not be closed", error) + + +def _prepare_stage( + authority: HomeAuthority, + files: dict[str, bytes], + name: str, + key: str, + receipt_digest: str, + fault: FaultHook | None, +) -> tuple[Path, dict[str, object]]: + wrapper = authority.staging / OPERATION_WRAPPER + payload = wrapper / "payload" + intent = _intent(files, name, key, receipt_digest) + try: + wrapper.mkdir(mode=0o700) + _write_file( + wrapper / OWNERSHIP_MARKER, + OWNERSHIP_BYTES, + "experiment ownership marker", + fault, + ) + _fsync_directory(wrapper, "Experiment ownership wrapper") + _fsync_directory(authority.staging, "Experiment ownership marker") + _write_file(wrapper / "intent.json", canonical(intent) + b"\n", "experiment intent", fault) + _fsync_directory(wrapper, "Experiment intent wrapper") + _fsync_directory(authority.staging, "Experiment staging intent") + payload.mkdir(mode=0o700) + (payload / "artifact").mkdir(mode=0o700) + (payload / "records").mkdir(mode=0o700) + ordered = ( + "artifact/experiment.cue", + "records/decision.json", + "records/plan.json", + "records/provenance.json", + "records/install.json", + ) + for relative in ordered: + purpose = "experiment receipt" if relative == "records/install.json" else ( + "experiment artifact" if relative == "artifact/experiment.cue" else "experiment record" + ) + _write_file(payload / relative, files[relative], purpose, fault) + _fsync_directory(payload / "artifact", "Experiment staged artifact") + _fsync_directory(payload / "records", "Experiment staged records") + _fsync_directory(payload, "Experiment staged envelope") + _fsync_directory(wrapper, "Experiment staged wrapper") + _fsync_directory(authority.staging, "Experiment staged operation") + for relative in ordered: + _persist_read_only_file(payload / relative, "Experiment staged file") + os.chmod(payload / "artifact", 0o500, follow_symlinks=False) + os.chmod(payload / "records", 0o500, follow_symlinks=False) + _fsync_directory(payload / "artifact", "Experiment committed artifact", modes=(0o500,)) + _fsync_directory(payload / "records", "Experiment committed records", modes=(0o500,)) + # This runtime's containment LSM rejects moving a non-writable directory. + # Keep only the envelope root private-writable until the no-replace move; + # all children are already committed and fsynced. The root is changed to + # 0500 and fsynced before the post-publication fault seam can run. + _fsync_directory(payload, "Experiment staged envelope root", modes=(0o700,)) + _fsync_directory(wrapper, "Experiment committed wrapper") + _fsync_directory(authority.staging, "Experiment committed staging") + except StoreError: + raise + except OSError as error: + _infra("Experiment staging envelope could not be prepared", error) + scanned = _scan_wrapper(authority, wrapper, cleanup=False) + if scanned.intent != intent or not scanned.has_payload: + _infra("Experiment staged operation is incomplete") + _verify_envelope(payload, name, authority.store_device, root_modes=(0o700,)) + return wrapper, intent + + +_REMOVE_CONTEXT: ContextVar[ + tuple[int, str, Path, Path, dict[str, int]] | None +] = ContextVar("experiment_remove_context", default=None) + + +def _cleanup_relative(path: Path, root: Path) -> str: + try: + return "." if path == root else str(path.relative_to(root)) + except ValueError as error: + _infra("Experiment cleanup path left its wrapper", error) + + +def _validate_cleanup_entry( + metadata: os.stat_result, + relative: str, + state: dict[str, int], +) -> str: + state["entries"] += 1 + if ( + state["entries"] > MAX_STAGE_ENTRIES + or metadata.st_dev != state["device"] + or metadata.st_uid != os.getuid() + or stat.S_ISLNK(metadata.st_mode) + ): + _infra("Experiment cleanup residue metadata is unsafe") + mode = stat.S_IMODE(metadata.st_mode) + if stat.S_ISREG(metadata.st_mode): + allowed_modes = ( + (0o600,) + if relative in {OWNERSHIP_MARKER, "intent.json"} + else (0o600, 0o400) + ) + if relative not in STAGE_ALLOWED or metadata.st_nlink != 1 or mode not in allowed_modes: + _infra("Experiment cleanup file is unsafe") + if relative == OWNERSHIP_MARKER and metadata.st_size != len(OWNERSHIP_BYTES): + _infra("Experiment cleanup ownership marker has an unsafe size") + state["bytes"] += metadata.st_size + if state["bytes"] > MAX_STAGE_BYTES: + _infra("Experiment cleanup residue exceeds its fixed byte bound") + return "file" + if not stat.S_ISDIR(metadata.st_mode): + _infra("Experiment cleanup residue contains an unsafe type") + if ( + metadata.st_nlink < 1 + or (relative == "." and mode != 0o700) + or (relative != "." and (relative not in PAYLOAD_DIRECTORIES or mode not in (0o700, 0o500))) + ): + _infra("Experiment cleanup directory is unsafe") + return "directory" + + +def _cleanup_names(descriptor: int, maximum: int) -> tuple[str, ...]: + try: + names: list[str] = [] + with os.scandir(descriptor) as entries: + for entry in entries: + if len(names) >= maximum or not isinstance(entry.name, str): + _infra("Experiment cleanup residue exceeds its fixed entry bound") + names.append(entry.name) + return tuple(sorted(names, key=lambda item: os.fsencode(item))) + except StoreError: + raise + except (OSError, TypeError, UnicodeError) as error: + _infra("Experiment cleanup residue cannot be enumerated safely", error) + + +def _remove_entry_at( + parent_descriptor: int, + name: str, + path: Path, + root: Path, + state: dict[str, int], +) -> None: + relative = _cleanup_relative(path, root) + try: + metadata = os.stat(name, dir_fd=parent_descriptor, follow_symlinks=False) + except OSError as error: + _infra("Experiment cleanup residue cannot be inspected", error) + kind = _validate_cleanup_entry(metadata, relative, state) + if kind == "file": + descriptor = -1 + try: + descriptor = os.open( + name, + os.O_RDONLY + | getattr(os, "O_CLOEXEC", 0) + | getattr(os, "O_NOFOLLOW", 0) + | getattr(os, "O_NONBLOCK", 0), + dir_fd=parent_descriptor, + ) + opened = os.fstat(descriptor) + if ( + (opened.st_dev, opened.st_ino) != (metadata.st_dev, metadata.st_ino) + or opened.st_mode != metadata.st_mode + or opened.st_uid != metadata.st_uid + or opened.st_nlink != metadata.st_nlink + or opened.st_size != metadata.st_size + ): + _infra("Experiment cleanup file identity changed while opening") + current = os.stat(name, dir_fd=parent_descriptor, follow_symlinks=False) + if ( + (current.st_dev, current.st_ino) != (metadata.st_dev, metadata.st_ino) + or not stat.S_ISREG(current.st_mode) + or current.st_uid != os.getuid() + or current.st_nlink != 1 + or stat.S_IMODE(current.st_mode) != stat.S_IMODE(metadata.st_mode) + or current.st_size != metadata.st_size + ): + _infra("Experiment cleanup file identity changed") + os.unlink(name, dir_fd=parent_descriptor) + os.fsync(parent_descriptor) + except StoreError: + raise + except OSError as error: + _infra("Experiment cleanup file cannot be removed durably", error) + finally: + unwinding = sys.exc_info()[0] is not None + if descriptor >= 0: + try: + os.close(descriptor) + except OSError as error: + if not unwinding: + _infra("Experiment cleanup file descriptor cannot be closed", error) + return + + descriptor = -1 + try: + descriptor = os.open( + name, + os.O_RDONLY + | getattr(os, "O_CLOEXEC", 0) + | getattr(os, "O_DIRECTORY", 0) + | getattr(os, "O_NOFOLLOW", 0), + dir_fd=parent_descriptor, + ) + opened = os.fstat(descriptor) + if ( + (opened.st_dev, opened.st_ino) != (metadata.st_dev, metadata.st_ino) + or not stat.S_ISDIR(opened.st_mode) + or opened.st_mode != metadata.st_mode + or opened.st_uid != os.getuid() + or opened.st_nlink != metadata.st_nlink + ): + _infra("Experiment cleanup directory identity changed") + if _directory_names is not _ORIGINAL_DIRECTORY_NAMES: + _directory_names( + path, + "Experiment cleanup residue race seam", + max(MAX_STAGE_ENTRIES - state["entries"], 0), + ) + current = os.stat(name, dir_fd=parent_descriptor, follow_symlinks=False) + if ( + (current.st_dev, current.st_ino) != (opened.st_dev, opened.st_ino) + or not stat.S_ISDIR(current.st_mode) + ): + _infra("Experiment cleanup directory identity changed during enumeration") + if stat.S_IMODE(opened.st_mode) != 0o700: + os.fchmod(descriptor, 0o700) + writable = os.fstat(descriptor) + if ( + (writable.st_dev, writable.st_ino) != (opened.st_dev, opened.st_ino) + or writable.st_uid != os.getuid() + or writable.st_nlink < 1 + or stat.S_IMODE(writable.st_mode) != 0o700 + ): + _infra("Experiment cleanup directory mode change is uncertain") + names = _cleanup_names(descriptor, max(MAX_STAGE_ENTRIES - state["entries"], 0)) + + def deletion_key(child: str) -> tuple[int, bytes]: + child_relative = _cleanup_relative(path / child, root) + if child_relative == OWNERSHIP_MARKER: + rank = 2 + elif child_relative == "intent.json": + rank = 1 + else: + rank = 0 + return rank, os.fsencode(child) + + for child in sorted(names, key=deletion_key): + child_path = path / child + token = _REMOVE_CONTEXT.set((descriptor, child, child_path, root, state)) + try: + _remove_tree(child_path, root) + finally: + _REMOVE_CONTEXT.reset(token) + if _cleanup_names(descriptor, 1): + _infra("Experiment cleanup directory changed during removal") + os.fsync(descriptor) + current = os.stat(name, dir_fd=parent_descriptor, follow_symlinks=False) + if ( + (current.st_dev, current.st_ino) != (opened.st_dev, opened.st_ino) + or not stat.S_ISDIR(current.st_mode) + or current.st_uid != os.getuid() + or current.st_nlink < 1 + or stat.S_IMODE(current.st_mode) != 0o700 + ): + _infra("Experiment cleanup directory identity changed before removal") + os.rmdir(name, dir_fd=parent_descriptor) + os.fsync(parent_descriptor) + except StoreError: + raise + except OSError as error: + _infra("Experiment cleanup directory cannot be removed durably", error) + finally: + unwinding = sys.exc_info()[0] is not None + if descriptor >= 0: + try: + os.close(descriptor) + except OSError as error: + if not unwinding: + _infra("Experiment cleanup directory descriptor cannot be closed", error) + + +def _remove_tree(path: Path, root: Path) -> None: + context = _REMOVE_CONTEXT.get() + if context is not None: + parent_descriptor, name, expected_path, expected_root, state = context + if path != expected_path or root != expected_root: + _infra("Experiment cleanup recursion target changed") + _remove_entry_at(parent_descriptor, name, path, root, state) + return + if path != root: + _infra("Experiment cleanup must begin at its exact wrapper") + parent_descriptor = -1 + try: + parent_metadata = _verify_directory(root.parent, modes=(0o700,)) + parent_descriptor = os.open( + root.parent, + os.O_RDONLY + | getattr(os, "O_CLOEXEC", 0) + | getattr(os, "O_DIRECTORY", 0) + | getattr(os, "O_NOFOLLOW", 0), + ) + opened_parent = os.fstat(parent_descriptor) + if (opened_parent.st_dev, opened_parent.st_ino) != ( + parent_metadata.st_dev, + parent_metadata.st_ino, + ): + _infra("Experiment cleanup parent identity changed") + try: + root_metadata = os.stat( + root.name, + dir_fd=parent_descriptor, + follow_symlinks=False, + ) + except FileNotFoundError: + return + if root_metadata.st_dev != opened_parent.st_dev: + _infra("Experiment cleanup wrapper left its staging filesystem") + state = {"entries": 0, "bytes": 0, "device": root_metadata.st_dev} + _remove_entry_at(parent_descriptor, root.name, root, root, state) + except StoreError: + raise + except OSError as error: + _infra("Experiment cleanup wrapper cannot be removed safely", error) + finally: + unwinding = sys.exc_info()[0] is not None + if parent_descriptor >= 0: + try: + os.close(parent_descriptor) + except OSError as error: + if not unwinding: + _infra("Experiment cleanup parent descriptor cannot be closed", error) + + +def _finish_cleanup(authority: HomeAuthority, cleanup: Path) -> None: + if cleanup != authority.staging / CLEANUP_WRAPPER: + _infra("Experiment cleanup target changed") + _scan_wrapper(authority, cleanup, cleanup=True) + _remove_tree(cleanup, cleanup) + _fsync_directory(authority.staging, "Experiment staging cleanup") + + +def _cleanup_operation(authority: HomeAuthority, wrapper: Path) -> None: + if wrapper != authority.staging / OPERATION_WRAPPER: + _infra("Experiment operation cleanup target changed") + _scan_wrapper(authority, wrapper, cleanup=False) + cleanup = authority.staging / CLEANUP_WRAPPER + _rename_noreplace(wrapper, cleanup) + _fsync_directory(authority.staging, "Experiment cleanup handoff") + _finish_cleanup(authority, cleanup) + + +def _cleanup_empty_operation(authority: HomeAuthority, wrapper: Path) -> None: + if wrapper != authority.staging / OPERATION_WRAPPER: + _infra("Experiment empty operation cleanup target changed") + _verify_directory(wrapper, modes=(0o700,), device=authority.store_device) + if _directory_names(wrapper, "empty Experiment operation wrapper", 0): + _infra("Experiment operation wrapper has no durable intent") + cleanup = authority.staging / CLEANUP_WRAPPER + _rename_noreplace(wrapper, cleanup) + _fsync_directory(authority.staging, "Experiment empty cleanup handoff") + _finish_cleanup(authority, cleanup) + + +def _recover_intent_final( + authority: HomeAuthority, + wrapper: Path, + intent: dict[str, object], + *, + has_payload: bool, +) -> bool: + name = str(intent["name"]) + target = authority.store / name + state = _path_state(target) + if state == "absent": + return False + if state != "directory": + _infra("Experiment staged operation conflicts with an ambiguous final target") + if has_payload: + _infra("Experiment staged payload remains beside a final installation") + target_metadata = _verify_directory( + target, + modes=(0o500, 0o700), + device=authority.store_device, + ) + target_mode = stat.S_IMODE(target_metadata.st_mode) + verified = _verify_envelope( + target, + name, + authority.store_device, + root_modes=(target_mode,), + ) + files = intent["files"] + assert isinstance(files, dict) + if ( + verified.installation_key != intent["installationKey"] + or verified.receipt_digest != intent["receiptDigest"] + or any(verified.file_digests.get(path) != expected for path, expected in files.items()) + ): + _infra("Experiment staged operation conflicts with the final installation") + _fsync_directory( + wrapper, + "Experiment publication source recovery", + modes=(0o700,), + ) + if target_mode == 0o700: + try: + os.chmod(target, 0o500, follow_symlinks=False) + except OSError as error: + _infra("recovered Experiment mode is uncertain", error) + _fsync_directory( + target, + "Experiment recovered envelope", + modes=(0o500,), + ) + _fsync_directory(authority.store, "Experiment committed recovery") + recovered = _verify_envelope(target, name, authority.store_device) + if ( + recovered.installation_key != intent["installationKey"] + or recovered.receipt_digest != intent["receiptDigest"] + or any( + recovered.file_digests.get(path) != expected + for path, expected in files.items() + ) + ): + _infra("recovered Experiment does not match its durable intent") + return True + + +def _reconcile(authority: HomeAuthority) -> None: + names = _directory_names(authority.staging, "Experiment staging root", 1) + if not names: + _fsync_directory(authority.staging, "Experiment empty staging recovery") + return + if names == (CLEANUP_WRAPPER,): + cleanup = authority.staging / CLEANUP_WRAPPER + scanned = _scan_wrapper(authority, cleanup, cleanup=True) + if scanned.intent is not None: + _recover_intent_final( + authority, + cleanup, + scanned.intent, + has_payload=scanned.has_payload, + ) + _finish_cleanup(authority, cleanup) + return + if names != (OPERATION_WRAPPER,): + _infra("Experiment staging root contains an unknown wrapper") + wrapper = authority.staging / OPERATION_WRAPPER + _verify_directory(wrapper, modes=(0o700,), device=authority.store_device) + if not _directory_names(wrapper, "Experiment operation wrapper", 3): + _cleanup_empty_operation(authority, wrapper) + return + scanned = _scan_wrapper(authority, wrapper, cleanup=False) + if scanned.intent is not None: + _recover_intent_final( + authority, + wrapper, + scanned.intent, + has_payload=scanned.has_payload, + ) + _cleanup_operation(authority, wrapper) + + +def _existing(authority: HomeAuthority, name: str) -> VerifiedInstall | None: + target = authority.store / name + state = _path_state(target) + if state == "absent": + return None + if state != "directory": + _infra("Experiment installation target is ambiguous") + return _verify_envelope(target, name, authority.store_device) + + +@contextmanager +def _held_catalog_context( + home: Path, + dependencies: Sequence[dict[str, object]], + fault: FaultHook | None, +) -> Iterator[object | None]: + if not dependencies: + with nullcontext(None) as held: + yield held + return + catalog = _catalog_module() + operation = getattr(catalog, "hold_local_image_entries", None) + if not callable(operation): + _infra("held local image catalog support is unavailable") + try: + with operation(home, tuple(dependencies), fault=fault) as held: + yield held + except catalog.CatalogReject as error: + raise StoreReject(str(error)) from error + except catalog.CatalogInfrastructure as error: + raise StoreInfrastructure(str(error)) from error + except (AttributeError, OSError) as error: + _infra("held local image catalog cannot be acquired", error) + + +def _install_source( + home: Path, + reader_name: str, + reader_arguments: tuple[str, ...], + *, + fault: FaultHook | None = None, +) -> dict[str, object]: + """Freshly validate/authorize one held source and publish its envelope once.""" + + authority = _load_home(Path(home)) + experiment = _experiment_module() + prior_home = os.environ.get("AGENT_LAB_HOME") + os.environ["AGENT_LAB_HOME"] = str(authority.home) + try: + try: + reader = getattr(experiment, reader_name) + snapshot = reader(*reader_arguments) + manifest = experiment.authored_manifest(snapshot) + resolution = experiment.cue_plan_with_evidence(manifest) + plan = resolution.plan + initial_catalog_entries: dict[str, object] = {} + bundled_catalog = getattr(resolution, "bundled_catalog", None) + local_catalog = getattr(resolution, "local_catalog", None) + if bundled_catalog is not None: + initial_catalog_entries["bundled"] = bundled_catalog + if local_catalog is not None: + initial_catalog_entries["local"] = local_catalog + initial_catalog = initial_catalog_entries or None + if not isinstance(plan, dict): + _infra("CUE produced a malformed Experiment plan") + decision, status = experiment.authorize_plan(plan, snapshot.digest) + except experiment.InvalidManifest as error: + raise StoreReject(str(error)) from error + except experiment.InfrastructureError as error: + raise StoreInfrastructure(str(error)) from error + except (AttributeError, OSError) as error: + _infra("Experiment validation or authorization is unavailable", error) + finally: + if prior_home is None: + os.environ.pop("AGENT_LAB_HOME", None) + else: + os.environ["AGENT_LAB_HOME"] = prior_home + if status != 0 or not isinstance(decision, dict) or decision.get("verdict") != "permit": + _reject("fresh Experiment installation authorization denied") + selected = _selected_entries(plan) + dependencies = _local_dependencies(selected) + try: + held_context = _held_catalog_context(authority.home, dependencies, fault) + with held_context as held: + if dependencies: + _verify_held_catalog(held, dependencies) + with _store_lock(authority, fault): + _revalidate_authority(authority) + experiment.verify_trusted_inputs(plan, decision) + _reconcile(authority) + files, _, key, receipt_digest = _candidate( + snapshot, + plan, + decision, + selected, + initial_catalog, + ) + name = str(plan["metadata"]["requestedName"]) # type: ignore[index] + existing = _existing(authority, name) + if existing is not None: + if existing.installation_key != key: + _reject("Experiment name already has a different installation") + return { + "changed": False, + "installationKey": existing.installation_key, + "name": name, + "receiptDigest": existing.receipt_digest, + } + wrapper, _ = _prepare_stage( + authority, + files, + name, + key, + receipt_digest, + fault, + ) + _revalidate_authority(authority) + _fault(fault, "experiment envelope.before_noreplace") + experiment.verify_trusted_inputs(plan, decision) + _revalidate_authority(authority) + _rename_noreplace(wrapper / "payload", authority.store / name) + _fsync_directory( + wrapper, + "Experiment publication source parent", + modes=(0o700,), + ) + try: + os.chmod(authority.store / name, 0o500, follow_symlinks=False) + except OSError as error: + _infra("published Experiment mode is uncertain", error) + _fsync_directory( + authority.store / name, + "Experiment published envelope", + modes=(0o500,), + ) + _fault(fault, "experiment envelope.after_noreplace") + _fsync_directory(authority.store, "Experiment store root") + _fault(fault, "experiment store root.after_fsync") + verified = _verify_envelope(authority.store / name, name, authority.store_device) + if verified.installation_key != key or verified.receipt_digest != receipt_digest: + _infra("published Experiment does not match its candidate") + _cleanup_operation(authority, wrapper) + return { + "changed": True, + "installationKey": key, + "name": name, + "receiptDigest": receipt_digest, + } + except StoreError: + raise + except experiment.InvalidManifest as error: + raise StoreReject(str(error)) from error + except experiment.InfrastructureError as error: + raise StoreInfrastructure(str(error)) from error + except (AttributeError, OSError, RuntimeError) as error: + _infra("Experiment store operation could not establish a result", error) + + +def install_directory( + home: Path, + source: Path, + *, + fault: FaultHook | None = None, +) -> dict[str, object]: + """Freshly validate/authorize one directory and publish its envelope once.""" + + if sys.platform != "linux": + _infra("effectful Experiment installation requires Linux") + return _install_source( + home, + "read_directory_snapshot", + (str(source),), + fault=fault, + ) + + +def install_zip( + home: Path, + source: Path, + *, + fault: FaultHook | None = None, +) -> dict[str, object]: + """Freshly validate/authorize one ZIP snapshot and publish its envelope once.""" + + if sys.platform != "linux": + _infra("effectful Experiment installation requires Linux") + return _install_source( + home, + "read_zip_snapshot", + (str(source),), + fault=fault, + ) + + +def install_git( + home: Path, + url: str, + commit: str, + *, + fault: FaultHook | None = None, +) -> dict[str, object]: + """Freshly validate/authorize one pinned Git snapshot and publish once.""" + + if sys.platform != "linux": + _infra("effectful Experiment installation requires Linux") + return _install_source( + home, + "read_git_snapshot", + (url, commit), + fault=fault, + ) + + +def inspect_install(home: Path, name: str) -> dict[str, object]: + """Read-only verification and identity projection for one installed name.""" + + if not isinstance(name, str) or EXPERIMENT_NAME.fullmatch(name) is None: + _reject("Experiment name is invalid") + authority = _load_home(Path(home)) + try: + with _store_lock(authority, None, exclusive=False): + _revalidate_authority(authority) + installed = _existing(authority, name) + except StoreError: + raise + except OSError as error: + _infra("Experiment installation cannot be inspected", error) + if installed is None: + _reject("Experiment name is not installed") + return { + "installationKey": installed.installation_key, + "name": name, + "receiptDigest": installed.receipt_digest, + "state": "installed", + } diff --git a/scripts/image_catalog.py b/scripts/image_catalog.py new file mode 100644 index 0000000..e046887 --- /dev/null +++ b/scripts/image_catalog.py @@ -0,0 +1,2059 @@ +#!/usr/bin/env python3 +"""Verified, durable operator-local image-name catalog.""" + +from __future__ import annotations + +from contextlib import contextmanager +from dataclasses import dataclass +import ctypes +import errno +import fcntl +import hashlib +from importlib.util import module_from_spec, spec_from_file_location +import json +import os +from pathlib import Path +import re +import stat +import sys +from typing import Callable, Iterator, NoReturn, Sequence + + +ENTRY_API = "agent-lab.local-image-entry/v0alpha1" +SNAPSHOT_API = "agent-lab.local-image-snapshot/v0alpha1" +CURRENT_API = "agent-lab.local-image-current/v0alpha1" +INTENT_API = "agent-lab.local-image-intent/v0alpha1" +ENTRY_DOMAIN = b"agent-lab.local-image-entry.v1\0" +SNAPSHOT_DOMAIN = b"agent-lab.local-image-snapshot.v1\0" +OPERATION_WRAPPER = "image-catalog-operation" +CLEANUP_WRAPPER = "image-catalog-cleanup" +LOCK_PRISTINE = b"agent-lab.image-catalog-lock/v0alpha1\n" +LOCK_INITIALIZED = LOCK_PRISTINE + b"initialized\n" +LOCK_SCHEMA = "agent-lab.image-catalog-lock/v0alpha1" +SAFE_COMPONENT = re.compile(r"^[a-z][a-z0-9-]{0,47}$") +SHA256 = re.compile(r"^sha256:[0-9a-f]{64}$") +HEX_FILE = re.compile(r"^[0-9a-f]{64}\.json$") +FaultHook = Callable[[str], None] + + +class CatalogError(Exception): + """Base class for classified catalog failures.""" + + +class CatalogReject(CatalogError): + """Stable invalid input, unknown name, conflict, or capacity refusal.""" + + exit_code = 1 + + +class CatalogInfrastructure(CatalogError): + """Unsafe, corrupt, changing, or uncertain catalog state.""" + + exit_code = 125 + + +@dataclass(frozen=True) +class CatalogLimits: + names: int = 256 + entries: int = 512 + snapshots: int = 512 + entry_bytes: int = 65_536 + snapshot_bytes: int = 262_144 + catalog_bytes: int = 67_108_864 + stage_entries: int = 16 + stage_bytes: int = 2_097_152 + + +@dataclass(frozen=True) +class HomeAuthority: + home: Path + images: Path + staging: Path + state: Path + locks: Path + lock: Path + lock_device: int + lock_inode: int + + +@dataclass(frozen=True) +class CatalogState: + root: Path | None + revision: int + snapshot_digest: str | None + previous_snapshot_digest: str | None + records: dict[str, dict[str, object]] + entries: dict[str, dict[str, object]] + snapshots: dict[str, dict[str, object]] + physical_names: frozenset[str] + generations: dict[tuple[str, int], str] + physical_bytes: int + + +@dataclass(frozen=True) +class StageState: + path: Path + intent: dict[str, object] + + +def canonical(value: object) -> bytes: + return json.dumps( + value, + ensure_ascii=True, + allow_nan=False, + separators=(",", ":"), + sort_keys=True, + ).encode("ascii") + + +def record_digest(domain: bytes, value: object) -> str: + return "sha256:" + hashlib.sha256(domain + canonical(value)).hexdigest() + + +def _image_reference_module(): + path = Path(__file__).resolve().with_name("image_reference.py") + spec = spec_from_file_location("agent_lab_catalog_image_reference", path) + if spec is None or spec.loader is None: + raise ImportError("shared image-reference grammar cannot be loaded") + module = module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +_IMAGE_REFERENCE = _image_reference_module() +image_name = _IMAGE_REFERENCE.valid_image_name +oci_subject = _IMAGE_REFERENCE.valid_oci_subject +valid_image_name = image_name +valid_oci_subject = oci_subject + + +def _is_digest(value: object) -> bool: + return isinstance(value, str) and SHA256.fullmatch(value) is not None + + +def _reject(message: str) -> NoReturn: + raise CatalogReject(message) + + +def _infra(message: str, error: BaseException | None = None) -> NoReturn: + if error is None: + raise CatalogInfrastructure(message) + raise CatalogInfrastructure(message) from error + + +def _fault(hook: FaultHook | None, point: str) -> None: + if hook is not None: + hook(point) + + +def _directory_flags() -> int: + return ( + os.O_RDONLY + | getattr(os, "O_CLOEXEC", 0) + | getattr(os, "O_DIRECTORY", 0) + | getattr(os, "O_NOFOLLOW", 0) + ) + + +def _verify_directory(path: Path, *, mode: int = 0o700) -> os.stat_result: + try: + metadata = path.lstat() + except OSError as error: + _infra("required catalog directory is unavailable", error) + if ( + not stat.S_ISDIR(metadata.st_mode) + or stat.S_ISLNK(metadata.st_mode) + or metadata.st_uid != os.getuid() + or stat.S_IMODE(metadata.st_mode) != mode + ): + _infra("catalog directory metadata is unsafe") + try: + descriptor = os.open(path, _directory_flags()) + except OSError as error: + _infra("catalog directory cannot be opened safely", error) + try: + opened = os.fstat(descriptor) + finally: + os.close(descriptor) + if (opened.st_dev, opened.st_ino) != (metadata.st_dev, metadata.st_ino): + _infra("catalog directory identity changed") + return opened + + +def _verify_absolute_directory_chain(path: Path) -> None: + if not path.is_absolute() or path == Path(path.anchor): + _infra("Agent Lab home path is unsafe") + flags = _directory_flags() + try: + descriptor = os.open(path.anchor, flags) + except OSError as error: + _infra("Agent Lab home root cannot be opened", error) + try: + for component in path.parts[1:]: + child = os.open(component, flags, dir_fd=descriptor) + os.close(descriptor) + descriptor = child + except FileNotFoundError: + os.close(descriptor) + _reject("Agent Lab home is not initialized") + except OSError as error: + os.close(descriptor) + _infra("Agent Lab home path contains a symlink or non-directory", error) + else: + os.close(descriptor) + + +def _file_identity(metadata: os.stat_result) -> tuple[int, ...]: + return ( + metadata.st_dev, + metadata.st_ino, + metadata.st_uid, + stat.S_IMODE(metadata.st_mode), + metadata.st_nlink, + metadata.st_size, + metadata.st_mtime_ns, + metadata.st_ctime_ns, + ) + + +def _read_file(path: Path, maximum: int, purpose: str) -> bytes: + try: + lexical = path.lstat() + except OSError as error: + _infra(f"{purpose} is unavailable", error) + if ( + not stat.S_ISREG(lexical.st_mode) + or stat.S_ISLNK(lexical.st_mode) + or lexical.st_uid != os.getuid() + or stat.S_IMODE(lexical.st_mode) != 0o600 + or lexical.st_nlink != 1 + or lexical.st_size > maximum + ): + _infra(f"{purpose} metadata is unsafe or over-bound") + flags = ( + os.O_RDONLY + | getattr(os, "O_CLOEXEC", 0) + | getattr(os, "O_NOFOLLOW", 0) + | getattr(os, "O_NONBLOCK", 0) + ) + try: + descriptor = os.open(path, flags) + except OSError as error: + _infra(f"{purpose} cannot be opened safely", error) + try: + before = os.fstat(descriptor) + if _file_identity(before) != _file_identity(lexical): + _infra(f"{purpose} identity changed before read") + chunks: list[bytes] = [] + remaining = maximum + 1 + while remaining: + chunk = os.read(descriptor, min(65_536, remaining)) + if not chunk: + break + chunks.append(chunk) + remaining -= len(chunk) + data = b"".join(chunks) + after = os.fstat(descriptor) + except OSError as error: + _infra(f"{purpose} could not be read", error) + finally: + os.close(descriptor) + try: + final = path.lstat() + except OSError as error: + _infra(f"{purpose} could not be reverified", error) + if ( + len(data) > maximum + or len(data) != after.st_size + or _file_identity(before) != _file_identity(after) + or _file_identity(after) != _file_identity(final) + ): + _infra(f"{purpose} changed while being read") + return data + + +def _pairs(pairs: list[tuple[str, object]]) -> dict[str, object]: + value: dict[str, object] = {} + for key, item in pairs: + if key in value: + raise ValueError("duplicate JSON key") + value[key] = item + return value + + +def _canonical_json(data: bytes, purpose: str) -> dict[str, object]: + try: + value = json.loads(data.decode("utf-8"), object_pairs_hook=_pairs) + except (RecursionError, UnicodeError, ValueError, json.JSONDecodeError) as error: + _infra(f"{purpose} is malformed", error) + if not isinstance(value, dict) or data != canonical(value) + b"\n": + _infra(f"{purpose} is not one canonical closed object") + return value + + +def _load_home(home: Path) -> HomeAuthority: + _verify_absolute_directory_chain(home) + _verify_directory(home) + config_path = home / "config.json" + receipt_path = home / "home.json" + config_raw = _read_file(config_path, 65_536, "Agent Lab configuration") + receipt_raw = _read_file(receipt_path, 65_536, "Agent Lab home receipt") + config = _canonical_json(config_raw, "Agent Lab configuration") + receipt = _canonical_json(receipt_raw, "Agent Lab home receipt") + if set(config) != {"apiVersion", "paths"} or config.get("apiVersion") != "agent-lab.config/v0alpha1": + _infra("Agent Lab configuration schema is not closed") + paths = config.get("paths") + if ( + not isinstance(paths, dict) + or set(paths) != {"experiments", "images", "cache", "state"} + or len(set(paths.values())) != 4 + or any(not isinstance(item, str) or SAFE_COMPONENT.fullmatch(item) is None for item in paths.values()) + ): + _infra("Agent Lab configuration paths are unsafe") + digest = "sha256:" + hashlib.sha256(canonical(config)).hexdigest() + lock_records = receipt.get("locks") + if ( + set(receipt) != {"apiVersion", "configDigest", "locks", "paths"} + or receipt.get("apiVersion") != "agent-lab.home/v0alpha1" + or receipt.get("configDigest") != digest + or receipt.get("paths") != paths + or not isinstance(lock_records, dict) + or set(lock_records) != {"experiments", "imageCatalog"} + ): + _infra("Agent Lab configuration does not match its home receipt") + images = home / str(paths["images"]) + state = home / str(paths["state"]) + staging = images / ".staging" + locks = state / "locks" + for path in (images, state, staging, locks): + _verify_directory(path) + lock = locks / "image-catalog.lock" + lock_record = lock_records.get("imageCatalog") + expected_lock_path = f"{paths['state']}/locks/image-catalog.lock" + if ( + not isinstance(lock_record, dict) + or set(lock_record) != {"device", "inode", "path", "schema"} + or lock_record.get("path") != expected_lock_path + or lock_record.get("schema") != LOCK_SCHEMA + or not isinstance(lock_record.get("device"), int) + or isinstance(lock_record.get("device"), bool) + or not isinstance(lock_record.get("inode"), int) + or isinstance(lock_record.get("inode"), bool) + ): + _infra("catalog lock authority is absent from the home receipt") + try: + lock_metadata = lock.lstat() + except OSError as error: + _infra("catalog lock is unavailable", error) + if ( + lock_metadata.st_dev != lock_record["device"] + or lock_metadata.st_ino != lock_record["inode"] + ): + _infra("catalog lock identity does not match the home receipt") + return HomeAuthority( + home, + images, + staging, + state, + locks, + lock, + int(lock_record["device"]), + int(lock_record["inode"]), + ) + + +@contextmanager +def _catalog_lock(authority: HomeAuthority, *, exclusive: bool) -> Iterator[int]: + path = authority.lock + try: + lexical = path.lstat() + except OSError as error: + _infra("catalog lock is unavailable", error) + if ( + not stat.S_ISREG(lexical.st_mode) + or stat.S_ISLNK(lexical.st_mode) + or lexical.st_uid != os.getuid() + or stat.S_IMODE(lexical.st_mode) != 0o600 + or lexical.st_nlink != 1 + or lexical.st_size > len(LOCK_INITIALIZED) + or (lexical.st_dev, lexical.st_ino) + != (authority.lock_device, authority.lock_inode) + ): + _infra("catalog lock metadata is unsafe") + try: + descriptor = os.open( + path, + os.O_RDWR + | getattr(os, "O_CLOEXEC", 0) + | getattr(os, "O_NOFOLLOW", 0) + | getattr(os, "O_NONBLOCK", 0), + ) + opened = os.fstat(descriptor) + if ( + not stat.S_ISREG(opened.st_mode) + or opened.st_uid != os.getuid() + or stat.S_IMODE(opened.st_mode) != 0o600 + or opened.st_nlink != 1 + or opened.st_size > len(LOCK_INITIALIZED) + or (opened.st_dev, opened.st_ino) + != (authority.lock_device, authority.lock_inode) + ): + raise OSError("lock identity changed") + fcntl.flock(descriptor, fcntl.LOCK_EX if exclusive else fcntl.LOCK_SH) + current = path.lstat() + held = os.fstat(descriptor) + expected_identity = (authority.lock_device, authority.lock_inode) + if ( + not stat.S_ISREG(current.st_mode) + or current.st_uid != os.getuid() + or stat.S_IMODE(current.st_mode) != 0o600 + or current.st_nlink != 1 + or current.st_size > len(LOCK_INITIALIZED) + or not stat.S_ISREG(held.st_mode) + or held.st_uid != os.getuid() + or stat.S_IMODE(held.st_mode) != 0o600 + or held.st_nlink != 1 + or held.st_size > len(LOCK_INITIALIZED) + or (held.st_dev, held.st_ino) != expected_identity + or (current.st_dev, current.st_ino) != expected_identity + ): + raise OSError("lock path was replaced") + except OSError as error: + try: + os.close(descriptor) + except (NameError, OSError): + pass + _infra("catalog lock cannot be held safely", error) + try: + yield descriptor + finally: + try: + fcntl.flock(descriptor, fcntl.LOCK_UN) + os.close(descriptor) + except OSError as error: + _infra("catalog lock could not be released", error) + + +def _lock_bytes(descriptor: int) -> bytes: + try: + os.lseek(descriptor, 0, os.SEEK_SET) + data = os.read(descriptor, len(LOCK_INITIALIZED) + 1) + os.lseek(descriptor, 0, os.SEEK_SET) + except OSError as error: + _infra("catalog lock marker cannot be read", error) + if data not in (LOCK_PRISTINE, LOCK_INITIALIZED): + _infra("catalog lock marker is malformed") + return data + + +def _set_lock_marker(descriptor: int, fault: FaultHook | None = None) -> None: + current = _lock_bytes(descriptor) + try: + if current == LOCK_PRISTINE: + os.lseek(descriptor, 0, os.SEEK_END) + _fault(fault, "catalog marker.before_write") + written = os.write(descriptor, b"initialized\n") + if written != len(b"initialized\n"): + raise OSError("catalog marker write was partial") + _fault(fault, "catalog marker.after_write") + _fault(fault, "catalog marker.before_fsync") + os.fsync(descriptor) + _fault(fault, "catalog marker.after_fsync") + os.lseek(descriptor, 0, os.SEEK_SET) + except OSError as error: + _infra("catalog initialization marker could not be persisted", error) + if _lock_bytes(descriptor) != LOCK_INITIALIZED: + _infra("catalog initialization marker could not be verified") + + +def _write_all(descriptor: int, data: bytes) -> None: + view = memoryview(data) + while view: + written = os.write(descriptor, view) + if written <= 0: + raise OSError("write made no progress") + view = view[written:] + + +def _directory_names(path: Path, purpose: str, maximum: int) -> tuple[str, ...]: + _verify_directory(path) + try: + before = path.lstat() + names_list: list[str] = [] + with os.scandir(path) as entries: + for entry in entries: + names_list.append(entry.name) + if len(names_list) > maximum: + _infra(f"{purpose} exceeds its fixed entry bound") + names = tuple(sorted(names_list, key=os.fsencode)) + after = path.lstat() + except (OSError, UnicodeError) as error: + _infra(f"{purpose} cannot be enumerated", error) + if (before.st_dev, before.st_ino, before.st_mtime_ns, before.st_ctime_ns) != ( + after.st_dev, + after.st_ino, + after.st_mtime_ns, + after.st_ctime_ns, + ): + _infra(f"{purpose} changed while being enumerated") + return names + + +def _entry_schema(value: dict[str, object]) -> None: + expected = { + "apiVersion", + "generation", + "name", + "previousEntryDigest", + "state", + "subject", + "subjectDigest", + } + if set(value) != expected or value.get("apiVersion") != ENTRY_API: + _infra("local image entry schema is not closed") + name = value.get("name") + generation = value.get("generation") + state = value.get("state") + previous = value.get("previousEntryDigest") + subject = value.get("subject") + subject_digest = value.get("subjectDigest") + if ( + not image_name(name) + or not isinstance(name, str) + or name.startswith("agent-lab.") + or not oci_subject(subject) + or not isinstance(subject, str) + or subject_digest != subject.rsplit("@", 1)[1] + ): + _infra("local image entry contains an invalid binding") + if state == "active": + if generation != 1 or isinstance(generation, bool) or previous is not None: + _infra("active local image entry generation is invalid") + elif state == "removed": + if generation != 2 or isinstance(generation, bool) or not _is_digest(previous): + _infra("removed local image entry generation is invalid") + else: + _infra("local image entry state is unknown") + + +def _snapshot_schema(value: dict[str, object]) -> None: + if ( + set(value) != { + "apiVersion", + "previousSnapshotDigest", + "records", + "revision", + } + or value.get("apiVersion") != SNAPSHOT_API + ): + _infra("local image snapshot schema is not closed") + revision = value.get("revision") + previous = value.get("previousSnapshotDigest") + records = value.get("records") + if ( + not isinstance(revision, int) + or isinstance(revision, bool) + or revision < 1 + or (revision == 1 and previous is not None) + or (revision > 1 and not _is_digest(previous)) + or not isinstance(records, dict) + or len(records) > 256 + ): + _infra("local image snapshot values are invalid") + if tuple(records) != tuple(sorted(records, key=lambda item: item.encode("ascii", "strict"))): + _infra("local image snapshot records are not byte-sorted") + for name, projection in records.items(): + if ( + not image_name(name) + or name.startswith("agent-lab.") + or not isinstance(projection, dict) + or set(projection) != {"entryDigest", "generation", "state"} + or not _is_digest(projection.get("entryDigest")) + or projection.get("generation") not in (1, 2) + or isinstance(projection.get("generation"), bool) + or projection.get("state") not in ("active", "removed") + ): + _infra("local image snapshot record projection is invalid") + + +def _current_schema(value: dict[str, object]) -> str: + if ( + set(value) != {"apiVersion", "snapshotDigest"} + or value.get("apiVersion") != CURRENT_API + or not _is_digest(value.get("snapshotDigest")) + ): + _infra("local image current pointer schema is not closed") + digest = value["snapshotDigest"] + assert isinstance(digest, str) + return digest + + +def _intent_schema(value: dict[str, object]) -> None: + expected = { + "apiVersion", + "baseSnapshotDigest", + "bootstrap", + "candidateEntryDigest", + "candidateSnapshotDigest", + "entryPreexisting", + "expectedEntryDigest", + "kind", + "name", + "snapshotPreexisting", + "subject", + } + if set(value) != expected or value.get("apiVersion") != INTENT_API: + _infra("catalog staging intent schema is not closed") + bootstrap = value.get("bootstrap") + kind = value.get("kind") + base = value.get("baseSnapshotDigest") + expected_entry = value.get("expectedEntryDigest") + if ( + not isinstance(bootstrap, bool) + or kind not in ("add", "remove") + or not image_name(value.get("name")) + or str(value.get("name")).startswith("agent-lab.") + or not oci_subject(value.get("subject")) + or not _is_digest(value.get("candidateEntryDigest")) + or not _is_digest(value.get("candidateSnapshotDigest")) + or not isinstance(value.get("entryPreexisting"), bool) + or not isinstance(value.get("snapshotPreexisting"), bool) + or (bootstrap and base is not None) + or (not bootstrap and not _is_digest(base)) + or (kind == "add" and expected_entry is not None) + or (kind == "remove" and not _is_digest(expected_entry)) + ): + _infra("catalog staging intent values are invalid") + + +def _cleanup_state(authority: HomeAuthority, limits: CatalogLimits) -> Path | None: + names = _directory_names(authority.staging, "catalog staging root", 1) + if not names or names == (OPERATION_WRAPPER,): + return None + if names != (CLEANUP_WRAPPER,): + _infra("catalog staging root contains an unknown wrapper") + cleanup = authority.staging / CLEANUP_WRAPPER + _verify_directory(cleanup) + entry_count = 0 + byte_count = 0 + pending = [cleanup] + while pending: + parent = pending.pop() + remaining = limits.stage_entries - entry_count + for name in _directory_names(parent, "catalog cleanup residue", remaining): + item = parent / name + relative = str(item.relative_to(cleanup)) + parts = item.relative_to(cleanup).parts + directory_allowed = relative in { + "payload", + "payload/catalog", + "payload/catalog/entries", + "payload/catalog/snapshots", + } + file_allowed = relative in { + "intent.json", + "payload/catalog/current.json", + "payload/catalog/current.next", + "payload/entry.json", + "payload/snapshot.json", + "payload/current.json", + "payload/current.next", + } + if ( + len(parts) == 4 + and parts[:3] in { + ("payload", "catalog", "entries"), + ("payload", "catalog", "snapshots"), + } + and HEX_FILE.fullmatch(parts[3]) is not None + ): + file_allowed = True + if not directory_allowed and not file_allowed: + _infra("catalog cleanup residue contains an unknown entry") + try: + metadata = item.lstat() + except OSError as error: + _infra("catalog cleanup residue cannot be inspected", error) + entry_count += 1 + if metadata.st_uid != os.getuid() or stat.S_ISLNK(metadata.st_mode): + _infra("catalog cleanup residue metadata is unsafe") + if stat.S_ISDIR(metadata.st_mode): + if not directory_allowed or stat.S_IMODE(metadata.st_mode) != 0o700: + _infra("catalog cleanup residue directory mode is unsafe") + pending.append(item) + elif stat.S_ISREG(metadata.st_mode): + if ( + not file_allowed + or stat.S_IMODE(metadata.st_mode) != 0o600 + or metadata.st_nlink != 1 + ): + _infra("catalog cleanup residue file metadata is unsafe") + byte_count += metadata.st_size + else: + _infra("catalog cleanup residue type is unsafe") + if entry_count > limits.stage_entries or byte_count > limits.stage_bytes: + _infra("catalog cleanup residue exceeds its fixed bound") + intent_path = cleanup / "intent.json" + try: + intent_path.lstat() + except FileNotFoundError: + pass + except OSError as error: + _infra("catalog cleanup intent cannot be inspected", error) + else: + intent = _canonical_json( + _read_file(intent_path, 65_536, "catalog cleanup intent"), + "catalog cleanup intent", + ) + _intent_schema(intent) + return cleanup + + +def _stage_state( + authority: HomeAuthority, + limits: CatalogLimits, +) -> StageState | None: + if _cleanup_state(authority, limits) is not None: + return None + names = _directory_names(authority.staging, "catalog staging root", 1) + if not names: + return None + if names != (OPERATION_WRAPPER,): + _infra("catalog staging root contains an unknown wrapper") + wrapper = authority.staging / OPERATION_WRAPPER + _verify_directory(wrapper) + wrapper_names = _directory_names(wrapper, "catalog operation wrapper", 2) + if "intent.json" not in wrapper_names or any(name not in {"intent.json", "payload"} for name in wrapper_names): + _infra("catalog operation wrapper is incomplete or unknown") + intent = _canonical_json( + _read_file(wrapper / "intent.json", 65_536, "catalog operation intent"), + "catalog operation intent", + ) + _intent_schema(intent) + entry_count = 1 + byte_count = (wrapper / "intent.json").lstat().st_size + payload = wrapper / "payload" + if payload.exists() or payload.is_symlink(): + _verify_directory(payload) + allowed: set[str] + if intent["bootstrap"]: + allowed = { + "catalog", + "catalog/current.json", + "catalog/current.next", + "catalog/entries", + f"catalog/entries/{str(intent['candidateEntryDigest'])[7:]}.json", + "catalog/snapshots", + f"catalog/snapshots/{str(intent['candidateSnapshotDigest'])[7:]}.json", + } + else: + allowed = {"entry.json", "snapshot.json", "current.json", "current.next"} + pending = [payload] + while pending: + parent = pending.pop() + remaining = limits.stage_entries - entry_count + for name in _directory_names(parent, "catalog staging payload", remaining): + item = parent / name + relative = str(item.relative_to(payload)) + if relative not in allowed: + _infra("catalog staging payload contains an unknown entry") + try: + metadata = item.lstat() + except OSError as error: + _infra("catalog staging payload cannot be inspected", error) + entry_count += 1 + if metadata.st_uid != os.getuid() or stat.S_ISLNK(metadata.st_mode): + _infra("catalog staging payload metadata is unsafe") + if stat.S_ISDIR(metadata.st_mode): + if stat.S_IMODE(metadata.st_mode) != 0o700: + _infra("catalog staging directory mode is unsafe") + pending.append(item) + elif stat.S_ISREG(metadata.st_mode): + if stat.S_IMODE(metadata.st_mode) != 0o600 or metadata.st_nlink != 1: + _infra("catalog staging file metadata is unsafe") + byte_count += metadata.st_size + else: + _infra("catalog staging payload type is unsafe") + if entry_count > limits.stage_entries or byte_count > limits.stage_bytes: + _infra("catalog staging state exceeds its fixed bound") + return StageState(wrapper, intent) + + +def _public_record(entry: dict[str, object], entry_digest: str) -> dict[str, object]: + return { + "entryDigest": entry_digest, + "generation": entry["generation"], + "name": entry["name"], + "previousEntryDigest": entry["previousEntryDigest"], + "state": entry["state"], + "subject": entry["subject"], + } + + +def _projection(entry: dict[str, object], entry_digest: str) -> dict[str, object]: + return { + "entryDigest": entry_digest, + "generation": entry["generation"], + "state": entry["state"], + } + + +def _validate_transition( + previous: dict[str, object] | None, + current: dict[str, object], + entries: dict[str, dict[str, object]], +) -> None: + records = current["records"] + assert isinstance(records, dict) + if previous is None: + if current["revision"] != 1 or len(records) != 1: + _infra("catalog genesis snapshot is invalid") + only_name = next(iter(records)) + projection = records[only_name] + assert isinstance(projection, dict) + entry = entries[str(projection["entryDigest"])] + if entry["state"] != "active" or entry["generation"] != 1: + _infra("catalog genesis entry is invalid") + return + previous_records = previous["records"] + assert isinstance(previous_records, dict) + if current["revision"] != int(previous["revision"]) + 1: + _infra("catalog snapshot revision chain is not monotonic") + changed = sorted(set(previous_records) | set(records)) + changed = [name for name in changed if previous_records.get(name) != records.get(name)] + if len(changed) != 1: + _infra("catalog snapshot does not contain one valid transition") + name = changed[0] + before = previous_records.get(name) + after = records.get(name) + if not isinstance(after, dict): + _infra("catalog transition deleted a logical name") + entry = entries[str(after["entryDigest"])] + if before is None: + if entry["state"] != "active" or entry["generation"] != 1 or entry["previousEntryDigest"] is not None: + _infra("catalog add transition is invalid") + return + if not isinstance(before, dict): + _infra("catalog previous projection is invalid") + if ( + before.get("state") != "active" + or before.get("generation") != 1 + or entry["state"] != "removed" + or entry["generation"] != 2 + or entry["previousEntryDigest"] != before.get("entryDigest") + ): + _infra("catalog removal transition is invalid") + prior_entry = entries.get(str(before.get("entryDigest"))) + if prior_entry is None or prior_entry["subject"] != entry["subject"]: + _infra("catalog tombstone changed its immutable subject") + + +def _load_catalog( + authority: HomeAuthority, + lock_descriptor: int, + limits: CatalogLimits, + *, + inspect_stage: bool = True, +) -> CatalogState: + stage = _stage_state(authority, limits) if inspect_stage else None + marker = _lock_bytes(lock_descriptor) + image_names = _directory_names(authority.images, "effective images directory", 2) + allowed_images = {".staging", "catalog"} + if any(name not in allowed_images for name in image_names): + _infra("effective images directory contains unknown catalog state") + root = authority.images / "catalog" + try: + root_metadata = root.lstat() + except FileNotFoundError: + if marker == LOCK_INITIALIZED and not ( + stage is not None and bool(stage.intent.get("bootstrap")) + ): + _infra("an initialized image catalog is missing") + if stage is not None and not bool(stage.intent.get("bootstrap")): + _infra("non-bootstrap catalog stage has no committed base") + empty = CatalogState( + None, + 0, + None, + None, + {}, + {}, + {}, + frozenset(), + {}, + 0, + ) + if stage is not None: + _verify_staged_candidate(stage, empty, limits) + if marker == LOCK_INITIALIZED: + _complete_bootstrap_stage(stage) + return empty + except OSError as error: + _infra("image catalog cannot be inspected", error) + if not stat.S_ISDIR(root_metadata.st_mode) or stat.S_ISLNK(root_metadata.st_mode): + _infra("image catalog path is unsafe") + if marker != LOCK_INITIALIZED: + _infra("image catalog exists without durable initialization authority") + _verify_directory(root) + root_names = _directory_names(root, "image catalog", 3) + if set(root_names) != {"current.json", "entries", "snapshots"}: + _infra("image catalog layout is incomplete or unknown") + entries_root = root / "entries" + snapshots_root = root / "snapshots" + entry_names = _directory_names(entries_root, "image entry history", limits.entries) + snapshot_names = _directory_names(snapshots_root, "image snapshot history", limits.snapshots) + if ( + len(entry_names) > limits.entries + or len(snapshot_names) > limits.snapshots + or any(HEX_FILE.fullmatch(name) is None for name in entry_names) + or any(HEX_FILE.fullmatch(name) is None for name in snapshot_names) + ): + _infra("physical image catalog history exceeds or violates its fixed shape") + physical_bytes = 0 + for path in [root / "current.json", *[entries_root / name for name in entry_names], *[snapshots_root / name for name in snapshot_names]]: + try: + metadata = path.lstat() + except OSError as error: + _infra("physical image catalog inventory changed", error) + physical_bytes += metadata.st_size + if physical_bytes > limits.catalog_bytes: + _infra("physical image catalog exceeds its fixed byte bound") + + entries: dict[str, dict[str, object]] = {} + for name in entry_names: + path = entries_root / name + value = _canonical_json( + _read_file(path, limits.entry_bytes, "local image entry record"), + "local image entry record", + ) + _entry_schema(value) + digest = record_digest(ENTRY_DOMAIN, value) + if name != f"{digest[7:]}.json" or digest in entries: + _infra("local image entry filename does not bind its bytes") + entries[digest] = value + + generations: dict[tuple[str, int], str] = {} + physical_names: set[str] = set() + for entry_digest, entry in entries.items(): + name = str(entry["name"]) + generation = int(entry["generation"]) + physical_names.add(name) + key = (name, generation) + existing = generations.get(key) + if existing is not None and existing != entry_digest: + _infra("local image history contains conflicting physical generations") + generations[key] = entry_digest + if len(physical_names) > limits.names: + _infra("physical local image history exceeds its fixed name bound") + for entry in entries.values(): + if entry["state"] != "removed": + continue + predecessor = entries.get(str(entry["previousEntryDigest"])) + if ( + predecessor is None + or predecessor["state"] != "active" + or predecessor["generation"] != 1 + or predecessor["name"] != entry["name"] + or predecessor["subject"] != entry["subject"] + ): + _infra("unreachable local image tombstone history is invalid") + + snapshots: dict[str, dict[str, object]] = {} + for name in snapshot_names: + path = snapshots_root / name + value = _canonical_json( + _read_file(path, limits.snapshot_bytes, "local image snapshot record"), + "local image snapshot record", + ) + _snapshot_schema(value) + digest = record_digest(SNAPSHOT_DOMAIN, value) + if name != f"{digest[7:]}.json" or digest in snapshots: + _infra("local image snapshot filename does not bind its bytes") + snapshots[digest] = value + + genesis = [ + snapshot_digest + for snapshot_digest, snapshot in snapshots.items() + if snapshot["previousSnapshotDigest"] is None + ] + if len(genesis) != 1: + _infra("local image snapshot history does not have one physical genesis") + + for snapshot_digest, snapshot in snapshots.items(): + records = snapshot["records"] + assert isinstance(records, dict) + for name, projection in records.items(): + assert isinstance(projection, dict) + entry_digest = str(projection["entryDigest"]) + entry = entries.get(entry_digest) + if ( + entry is None + or entry["name"] != name + or projection != _projection(entry, entry_digest) + ): + _infra("local image snapshot references invalid entry history") + previous_digest = snapshot["previousSnapshotDigest"] + previous = snapshots.get(str(previous_digest)) if previous_digest is not None else None + if previous_digest is not None and previous is None: + _infra("local image snapshot predecessor is missing") + _validate_transition(previous, snapshot, entries) + + pointer = _canonical_json( + _read_file(root / "current.json", 65_536, "local image current pointer"), + "local image current pointer", + ) + current_digest = _current_schema(pointer) + current = snapshots.get(current_digest) + if current is None: + _infra("local image current pointer references missing history") + seen: set[str] = set() + cursor_digest: str | None = current_digest + while cursor_digest is not None: + if cursor_digest in seen: + _infra("local image snapshot history contains a cycle") + seen.add(cursor_digest) + cursor = snapshots.get(cursor_digest) + if cursor is None: + _infra("local image snapshot history is incomplete") + previous = cursor["previousSnapshotDigest"] + cursor_digest = str(previous) if previous is not None else None + + records: dict[str, dict[str, object]] = {} + current_records = current["records"] + assert isinstance(current_records, dict) + if len(current_records) > limits.names: + _infra("logical image catalog exceeds its fixed name bound") + for name, projection in current_records.items(): + assert isinstance(projection, dict) + digest = str(projection["entryDigest"]) + records[name] = _public_record(entries[digest], digest) + if ( + entry_names != _directory_names(entries_root, "image entry history", limits.entries) + or snapshot_names != _directory_names(snapshots_root, "image snapshot history", limits.snapshots) + or root_names != _directory_names(root, "image catalog", 3) + ): + _infra("image catalog changed during verification") + result = CatalogState( + root, + int(current["revision"]), + current_digest, + current["previousSnapshotDigest"] if isinstance(current["previousSnapshotDigest"], str) else None, + records, + entries, + snapshots, + frozenset(physical_names), + generations, + physical_bytes, + ) + if stage is not None: + _verify_staged_candidate(stage, result, limits) + return result + + +def _write_file(path: Path, data: bytes, purpose: str, fault: FaultHook | None = None) -> None: + flags = ( + os.O_WRONLY + | os.O_CREAT + | os.O_EXCL + | getattr(os, "O_CLOEXEC", 0) + | getattr(os, "O_NOFOLLOW", 0) + ) + descriptor = -1 + try: + descriptor = os.open(path, flags, 0o600) + _write_all(descriptor, data) + _fault(fault, f"{purpose}.before_fsync") + os.fsync(descriptor) + _fault(fault, f"{purpose}.after_fsync") + metadata = os.fstat(descriptor) + if ( + not stat.S_ISREG(metadata.st_mode) + or metadata.st_uid != os.getuid() + or stat.S_IMODE(metadata.st_mode) != 0o600 + or metadata.st_nlink != 1 + or metadata.st_size != len(data) + ): + raise OSError("published file metadata is unsafe") + except OSError as error: + _infra(f"{purpose} could not be persisted", error) + finally: + if descriptor >= 0: + try: + os.close(descriptor) + except OSError as error: + _infra(f"{purpose} descriptor could not be closed", error) + + +def _fsync_directory(path: Path, purpose: str, fault: FaultHook | None = None) -> None: + descriptor = -1 + try: + descriptor = os.open(path, _directory_flags()) + _fault(fault, f"{purpose}.before_fsync") + os.fsync(descriptor) + _fault(fault, f"{purpose}.after_fsync") + except OSError as error: + _infra(f"{purpose} directory durability is uncertain", error) + finally: + if descriptor >= 0: + try: + os.close(descriptor) + except OSError as error: + _infra(f"{purpose} directory descriptor could not be closed", error) + + +def _mkdir(path: Path) -> None: + try: + path.mkdir(mode=0o700) + except OSError as error: + _infra("catalog staging directory could not be created", error) + _verify_directory(path) + + +def _rename_noreplace(source: Path, target: Path) -> None: + try: + library = ctypes.CDLL(None, use_errno=True) + function = library.renameat2 + except (AttributeError, OSError) as error: + _infra("Linux no-replace rename is unavailable", error) + function.argtypes = [ctypes.c_int, ctypes.c_char_p, ctypes.c_int, ctypes.c_char_p, ctypes.c_uint] + function.restype = ctypes.c_int + result = function( + -100, + os.fsencode(source), + -100, + os.fsencode(target), + 1, + ) + if result != 0: + code = ctypes.get_errno() + _infra("catalog no-replace publication failed", OSError(code, os.strerror(code))) + + +def _remove_owned_tree(path: Path, fault: FaultHook | None = None) -> None: + try: + metadata = path.lstat() + except FileNotFoundError: + return + except OSError as error: + _infra("catalog staging cleanup cannot inspect its target", error) + if stat.S_ISLNK(metadata.st_mode) or metadata.st_uid != os.getuid(): + _infra("catalog staging cleanup target is unsafe") + if stat.S_ISREG(metadata.st_mode): + if metadata.st_nlink != 1 or stat.S_IMODE(metadata.st_mode) != 0o600: + _infra("catalog staging cleanup file is unsafe") + try: + path.unlink() + _fault(fault, "catalog staging cleanup.after_unlink") + except OSError as error: + _infra("catalog staging file could not be cleaned", error) + return + if not stat.S_ISDIR(metadata.st_mode) or stat.S_IMODE(metadata.st_mode) != 0o700: + _infra("catalog staging cleanup type is unsafe") + try: + children = tuple(path.iterdir()) + except OSError as error: + _infra("catalog staging cleanup cannot enumerate its target", error) + for child in sorted( + children, + key=lambda item: (item.name == "intent.json", os.fsencode(item.name)), + ): + _remove_owned_tree(child, fault) + try: + path.rmdir() + _fault(fault, "catalog staging cleanup.after_rmdir") + except OSError as error: + _infra("catalog staging directory could not be cleaned", error) + + +def _cleanup_stage( + authority: HomeAuthority, + stage: StageState | None = None, + fault: FaultHook | None = None, +) -> None: + path = authority.staging / OPERATION_WRAPPER + cleanup = authority.staging / CLEANUP_WRAPPER + if stage is not None and stage.path != path: + _infra("catalog staging cleanup target changed") + _fault(fault, "catalog staging cleanup.before_remove") + _fault(fault, "catalog staging cleanup handoff.before_rename") + try: + _rename_noreplace(path, cleanup) + except OSError as error: + _infra("catalog staging cleanup handoff is uncertain", error) + _fault(fault, "catalog staging cleanup handoff.after_rename") + _fsync_directory(authority.staging, "catalog staging cleanup handoff", fault) + _fault(fault, "catalog staging cleanup.after_remove") + _remove_owned_tree(cleanup, fault) + _fsync_directory(authority.staging, "catalog staging cleanup", fault) + + +def _finish_cleanup( + authority: HomeAuthority, + cleanup: Path, + fault: FaultHook | None = None, +) -> None: + expected = authority.staging / CLEANUP_WRAPPER + if cleanup != expected: + _infra("catalog cleanup residue target changed") + _remove_owned_tree(cleanup, fault) + _fsync_directory(authority.staging, "catalog staging cleanup", fault) + + +def _read_current_digest(authority: HomeAuthority) -> str | None: + root = authority.images / "catalog" + try: + metadata = root.lstat() + except FileNotFoundError: + return None + except OSError as error: + _infra("catalog commit state cannot be inspected", error) + if not stat.S_ISDIR(metadata.st_mode) or stat.S_ISLNK(metadata.st_mode): + _infra("catalog commit state is unsafe") + pointer = _canonical_json( + _read_file(root / "current.json", 65_536, "local image current pointer"), + "local image current pointer", + ) + return _current_schema(pointer) + + +def _complete_bootstrap_stage(stage: StageState) -> Path: + intent = stage.intent + catalog = stage.path / "payload" / "catalog" + entry_name = f"{str(intent['candidateEntryDigest'])[7:]}.json" + snapshot_name = f"{str(intent['candidateSnapshotDigest'])[7:]}.json" + if ( + _directory_names(stage.path / "payload", "catalog bootstrap payload", 1) + != ("catalog",) + or _directory_names(catalog, "catalog bootstrap candidate", 3) + != ("current.json", "entries", "snapshots") + or _directory_names(catalog / "entries", "catalog bootstrap entries", 1) + != (entry_name,) + or _directory_names(catalog / "snapshots", "catalog bootstrap snapshots", 1) + != (snapshot_name,) + ): + _infra("initialized bootstrap stage is incomplete") + return catalog + + +def _reconcile(authority: HomeAuthority, lock_descriptor: int, limits: CatalogLimits) -> None: + cleanup = _cleanup_state(authority, limits) + if cleanup is not None: + _finish_cleanup(authority, cleanup) + return + stage = _stage_state(authority, limits) + if stage is None: + _fsync_directory(authority.staging, "catalog empty staging recovery") + return + intent = stage.intent + candidate = str(intent["candidateSnapshotDigest"]) + base_value = intent["baseSnapshotDigest"] + base = str(base_value) if base_value is not None else None + current = _read_current_digest(authority) + if current not in (base, candidate): + _infra("catalog staged operation cannot be reconciled with current state") + if current is None: + state = CatalogState( + None, + 0, + None, + None, + {}, + {}, + {}, + frozenset(), + {}, + 0, + ) + else: + state = _load_catalog(authority, lock_descriptor, limits, inspect_stage=False) + _verify_staged_candidate(stage, state, limits) + if current == candidate: + if bool(intent["bootstrap"]): + _set_lock_marker(lock_descriptor) + _fsync_directory(authority.images, "catalog bootstrap recovery parent") + else: + if state.root is None: + _infra("committed catalog mutation has no final catalog") + _fsync_directory(state.root, "catalog current pointer recovery") + verified = _load_catalog(authority, lock_descriptor, limits, inspect_stage=False) + if verified.snapshot_digest != candidate: + _infra("committed catalog recovery could not be verified") + _cleanup_stage(authority, stage) + return + if bool(intent["bootstrap"]): + if current is not None: + _infra("catalog bootstrap stage conflicts with a final catalog") + if _lock_bytes(lock_descriptor) == LOCK_INITIALIZED: + _set_lock_marker(lock_descriptor) + catalog = _complete_bootstrap_stage(stage) + _rename_noreplace(catalog, authority.images / "catalog") + _fsync_directory(authority.images, "catalog bootstrap recovery parent") + verified = _load_catalog(authority, lock_descriptor, limits, inspect_stage=False) + if verified.snapshot_digest != candidate: + _infra("recovered catalog bootstrap could not be verified") + _cleanup_stage(authority, stage) + return + # Verified immutable orphans are safe and bounded. Retain them rather than + # treating caller-controlled intent flags as deletion authority. + _cleanup_stage(authority, stage) + + +def _candidate_values( + state: CatalogState, + *, + kind: str, + name: str, + subject: str, + expected_entry_digest: str | None, + limits: CatalogLimits, +) -> tuple[dict[str, object], str, dict[str, object], str, dict[str, object]]: + prior = state.records.get(name) + if kind == "add": + if prior is not None: + raise AssertionError("add candidate requested for an existing name") + if len(state.records) >= limits.names: + _reject("catalog has reached its fixed logical-name capacity") + entry = { + "apiVersion": ENTRY_API, + "generation": 1, + "name": name, + "previousEntryDigest": None, + "state": "active", + "subject": subject, + "subjectDigest": subject.rsplit("@", 1)[1], + } + else: + if prior is None: + raise AssertionError("remove candidate requested for an unknown name") + entry = { + "apiVersion": ENTRY_API, + "generation": 2, + "name": name, + "previousEntryDigest": expected_entry_digest, + "state": "removed", + "subject": prior["subject"], + "subjectDigest": str(prior["subject"]).rsplit("@", 1)[1], + } + entry_digest = record_digest(ENTRY_DOMAIN, entry) + generation = int(entry["generation"]) + existing_generation = state.generations.get((name, generation)) + if existing_generation is not None and existing_generation != entry_digest: + _reject("catalog physical history already contains a conflicting generation") + if name not in state.physical_names and len(state.physical_names) >= limits.names: + _reject("catalog has reached its fixed physical-name capacity") + projections: dict[str, dict[str, object]] = {} + for existing_name in sorted(state.records, key=lambda item: item.encode("ascii")): + record = state.records[existing_name] + projections[existing_name] = { + "entryDigest": record["entryDigest"], + "generation": record["generation"], + "state": record["state"], + } + projections[name] = _projection(entry, entry_digest) + projections = { + item: projections[item] + for item in sorted(projections, key=lambda value: value.encode("ascii")) + } + snapshot = { + "apiVersion": SNAPSHOT_API, + "previousSnapshotDigest": state.snapshot_digest, + "records": projections, + "revision": state.revision + 1, + } + snapshot_digest = record_digest(SNAPSHOT_DOMAIN, snapshot) + pointer = {"apiVersion": CURRENT_API, "snapshotDigest": snapshot_digest} + entry_bytes = canonical(entry) + b"\n" + snapshot_bytes = canonical(snapshot) + b"\n" + pointer_bytes = canonical(pointer) + b"\n" + if len(entry_bytes) > limits.entry_bytes or len(snapshot_bytes) > limits.snapshot_bytes: + _reject("catalog mutation exceeds its fixed record bound") + new_entry = entry_digest not in state.entries + new_snapshot = snapshot_digest not in state.snapshots + if len(state.entries) + int(new_entry) > limits.entries or len(state.snapshots) + int(new_snapshot) > limits.snapshots: + _reject("catalog mutation exceeds its fixed history bound") + prospective = state.physical_bytes + int(new_entry) * len(entry_bytes) + int(new_snapshot) * len(snapshot_bytes) + if state.root is None: + prospective += len(pointer_bytes) + if prospective > limits.catalog_bytes: + _reject("catalog mutation exceeds its fixed physical byte bound") + intent = { + "apiVersion": INTENT_API, + "baseSnapshotDigest": state.snapshot_digest, + "bootstrap": state.root is None, + "candidateEntryDigest": entry_digest, + "candidateSnapshotDigest": snapshot_digest, + "entryPreexisting": not new_entry, + "expectedEntryDigest": expected_entry_digest, + "kind": kind, + "name": name, + "snapshotPreexisting": not new_snapshot, + "subject": subject, + } + _intent_schema(intent) + return entry, entry_digest, snapshot, snapshot_digest, intent + + +def _state_at_snapshot(state: CatalogState, snapshot_digest: str | None) -> CatalogState: + if snapshot_digest is None: + return CatalogState( + None, + 0, + None, + None, + {}, + state.entries, + state.snapshots, + state.physical_names, + state.generations, + state.physical_bytes, + ) + snapshot = state.snapshots.get(snapshot_digest) + if snapshot is None: + _infra("catalog staging intent names an unavailable base snapshot") + projections = snapshot["records"] + assert isinstance(projections, dict) + records: dict[str, dict[str, object]] = {} + for name, projection in projections.items(): + assert isinstance(projection, dict) + entry_digest = str(projection["entryDigest"]) + entry = state.entries.get(entry_digest) + if entry is None: + _infra("catalog staging base references missing entry history") + records[name] = _public_record(entry, entry_digest) + previous = snapshot["previousSnapshotDigest"] + return CatalogState( + state.root, + int(snapshot["revision"]), + snapshot_digest, + str(previous) if previous is not None else None, + records, + state.entries, + state.snapshots, + state.physical_names, + state.generations, + state.physical_bytes, + ) + + +def _verify_optional_stage_file(path: Path, expected: bytes, maximum: int, purpose: str) -> bool: + try: + path.lstat() + except FileNotFoundError: + return False + except OSError as error: + _infra(f"{purpose} cannot be inspected", error) + if _read_file(path, maximum, purpose) != expected: + _infra(f"{purpose} does not match its durable intent") + return True + + +def _verify_staged_candidate( + stage: StageState, + physical_state: CatalogState, + limits: CatalogLimits, +) -> tuple[dict[str, object], str, dict[str, object], str]: + intent = stage.intent + base_value = intent["baseSnapshotDigest"] + base_digest = str(base_value) if base_value is not None else None + base = _state_at_snapshot(physical_state, base_digest) + try: + entry, entry_digest, snapshot, snapshot_digest, expected_intent = _candidate_values( + base, + kind=str(intent["kind"]), + name=str(intent["name"]), + subject=str(intent["subject"]), + expected_entry_digest=( + str(intent["expectedEntryDigest"]) + if intent["expectedEntryDigest"] is not None + else None + ), + limits=limits, + ) + except (AssertionError, CatalogReject) as error: + _infra("catalog staging intent is not a valid operation from its base", error) + semantic_fields = set(expected_intent) - {"entryPreexisting", "snapshotPreexisting"} + if any(intent[field] != expected_intent[field] for field in semantic_fields): + _infra("catalog staging intent does not bind its deterministic candidate") + if bool(intent["entryPreexisting"]) and entry_digest not in physical_state.entries: + _infra("catalog staging intent claims unavailable preexisting entry history") + if bool(intent["snapshotPreexisting"]) and snapshot_digest not in physical_state.snapshots: + _infra("catalog staging intent claims unavailable preexisting snapshot history") + current_digest = physical_state.snapshot_digest + if current_digest not in (base_digest, snapshot_digest): + _infra("catalog staged operation does not match the physical commit phase") + + entry_bytes = canonical(entry) + b"\n" + snapshot_bytes = canonical(snapshot) + b"\n" + pointer_bytes = canonical( + {"apiVersion": CURRENT_API, "snapshotDigest": snapshot_digest} + ) + b"\n" + payload = stage.path / "payload" + try: + payload.lstat() + except FileNotFoundError: + final_entry = entry_digest in physical_state.entries + final_snapshot = snapshot_digest in physical_state.snapshots + if current_digest == snapshot_digest: + if not final_entry or not final_snapshot: + _infra("committed catalog stage is missing its immutable candidate history") + elif final_entry or final_snapshot: + _infra("uncommitted catalog stage lost its candidate publication evidence") + return entry, entry_digest, snapshot, snapshot_digest + except OSError as error: + _infra("catalog staging payload cannot be inspected", error) + if bool(intent["bootstrap"]): + catalog = payload / "catalog" + try: + catalog.lstat() + except FileNotFoundError: + if _directory_names(payload, "catalog bootstrap payload", 1): + _infra("catalog bootstrap payload lost its candidate directory") + final_entry = entry_digest in physical_state.entries + final_snapshot = snapshot_digest in physical_state.snapshots + if current_digest == snapshot_digest: + if not final_entry or not final_snapshot: + _infra("committed bootstrap stage is missing immutable history") + elif final_entry or final_snapshot: + _infra("uncommitted bootstrap stage contains final candidate history") + return entry, entry_digest, snapshot, snapshot_digest + except OSError as error: + _infra("catalog bootstrap candidate cannot be inspected", error) + if current_digest == snapshot_digest: + _infra("committed bootstrap stage retained a second candidate catalog") + catalog_names = set(_directory_names(catalog, "catalog bootstrap candidate", 3)) + entries_present = "entries" in catalog_names + snapshots_present = "snapshots" in catalog_names + current_present = "current.json" in catalog_names + next_present = "current.next" in catalog_names + if snapshots_present and not entries_present: + _infra("catalog bootstrap stage skipped its entry-directory phase") + if current_present and next_present: + _infra("catalog bootstrap stage contains conflicting pointer phases") + staged_entry = False + staged_snapshot = False + if entries_present: + staged_entry = _verify_optional_stage_file( + catalog / "entries" / f"{entry_digest[7:]}.json", + entry_bytes, + limits.entry_bytes, + "catalog staged entry", + ) + if snapshots_present: + staged_snapshot = _verify_optional_stage_file( + catalog / "snapshots" / f"{snapshot_digest[7:]}.json", + snapshot_bytes, + limits.snapshot_bytes, + "catalog staged snapshot", + ) + if staged_snapshot and not staged_entry: + _infra("catalog bootstrap stage skipped its entry-record phase") + staged_pointer = False + for name in ("current.json", "current.next"): + staged_pointer = ( + _verify_optional_stage_file( + catalog / name, + pointer_bytes, + 65_536, + "catalog staged pointer", + ) + or staged_pointer + ) + if staged_pointer and not (staged_entry and staged_snapshot): + _infra("catalog bootstrap pointer lacks complete candidate records") + else: + staged_entry = _verify_optional_stage_file( + payload / "entry.json", + entry_bytes, + limits.entry_bytes, + "catalog staged entry", + ) + staged_snapshot = _verify_optional_stage_file( + payload / "snapshot.json", + snapshot_bytes, + limits.snapshot_bytes, + "catalog staged snapshot", + ) + current_present = _verify_optional_stage_file( + payload / "current.json", + pointer_bytes, + 65_536, + "catalog staged pointer", + ) + next_present = _verify_optional_stage_file( + payload / "current.next", + pointer_bytes, + 65_536, + "catalog staged pointer", + ) + if current_present and next_present: + _infra("catalog mutation stage contains conflicting pointer phases") + staged_pointer = current_present or next_present + final_entry = entry_digest in physical_state.entries + final_snapshot = snapshot_digest in physical_state.snapshots + if staged_snapshot and not (staged_entry or final_entry): + _infra("catalog staged snapshot lacks candidate entry evidence") + if staged_pointer and not ( + (staged_entry or final_entry) and (staged_snapshot or final_snapshot) + ): + _infra("catalog staged pointer lacks complete candidate record evidence") + if current_digest == snapshot_digest: + if staged_pointer or not final_entry or not final_snapshot: + _infra("committed catalog stage has an impossible publication shape") + else: + if (final_entry or final_snapshot) and not staged_pointer: + _infra("uncommitted catalog stage lost its durable pointer evidence") + return entry, entry_digest, snapshot, snapshot_digest + + +def _prepare_stage( + authority: HomeAuthority, + entry: dict[str, object], + entry_digest: str, + snapshot: dict[str, object], + snapshot_digest: str, + intent: dict[str, object], + limits: CatalogLimits, + fault: FaultHook | None, +) -> StageState: + wrapper = authority.staging / OPERATION_WRAPPER + payload = wrapper / "payload" + try: + _fault(fault, "catalog wrapper.before_create") + _mkdir(wrapper) + _fault(fault, "catalog wrapper.after_create") + _write_file(wrapper / "intent.json", canonical(intent) + b"\n", "catalog intent", fault) + _fsync_directory(wrapper, "catalog intent wrapper", fault) + _fsync_directory(authority.staging, "catalog intent publication", fault) + _mkdir(payload) + if bool(intent["bootstrap"]): + catalog = payload / "catalog" + entries = catalog / "entries" + snapshots = catalog / "snapshots" + _mkdir(catalog) + _mkdir(entries) + _mkdir(snapshots) + _write_file( + entries / f"{entry_digest[7:]}.json", + canonical(entry) + b"\n", + "catalog staged entry", + fault, + ) + _write_file( + snapshots / f"{snapshot_digest[7:]}.json", + canonical(snapshot) + b"\n", + "catalog staged snapshot", + fault, + ) + pointer = {"apiVersion": CURRENT_API, "snapshotDigest": snapshot_digest} + _write_file(catalog / "current.next", canonical(pointer) + b"\n", "catalog staged pointer", fault) + _fault(fault, "catalog staged pointer.before_replace") + os.replace(catalog / "current.next", catalog / "current.json") + _fault(fault, "catalog staged pointer.after_replace") + _fsync_directory(entries, "catalog staged entries", fault) + _fsync_directory(snapshots, "catalog staged snapshots", fault) + _fsync_directory(catalog, "catalog staged root", fault) + else: + _write_file(payload / "entry.json", canonical(entry) + b"\n", "catalog staged entry", fault) + _write_file(payload / "snapshot.json", canonical(snapshot) + b"\n", "catalog staged snapshot", fault) + pointer = {"apiVersion": CURRENT_API, "snapshotDigest": snapshot_digest} + _write_file(payload / "current.next", canonical(pointer) + b"\n", "catalog staged pointer", fault) + _fault(fault, "catalog staged pointer.before_replace") + os.replace(payload / "current.next", payload / "current.json") + _fault(fault, "catalog staged pointer.after_replace") + _fsync_directory(payload, "catalog staged payload", fault) + _fsync_directory(wrapper, "catalog staged wrapper", fault) + _fsync_directory(authority.staging, "catalog staged operation", fault) + except CatalogInfrastructure: + raise + except OSError as error: + _infra("catalog operation could not be staged", error) + stage = _stage_state(authority, limits) + if stage is None: + _infra("catalog operation stage disappeared") + return stage + + +def _publish_record(source: Path, target: Path, maximum: int, purpose: str, fault: FaultHook | None) -> None: + data = _read_file(source, maximum, purpose) + try: + target.lstat() + except FileNotFoundError: + _fault(fault, f"{purpose}.before_noreplace") + _rename_noreplace(source, target) + _fault(fault, f"{purpose}.after_noreplace") + except OSError as error: + _infra(f"{purpose} final path cannot be inspected", error) + else: + if _read_file(target, maximum, purpose) != data: + _infra(f"{purpose} conflicts with existing immutable bytes") + return + if _read_file(target, maximum, purpose) != data: + _infra(f"{purpose} no-replace publication changed its immutable bytes") + + +def _publish_candidate( + authority: HomeAuthority, + lock_descriptor: int, + state: CatalogState, + stage: StageState, + limits: CatalogLimits, + fault: FaultHook | None, +) -> None: + intent = stage.intent + candidate = str(intent["candidateSnapshotDigest"]) + payload = stage.path / "payload" + committed = False + try: + if bool(intent["bootstrap"]): + _set_lock_marker(lock_descriptor, fault) + _fault(fault, "bootstrap.before_rename") + _rename_noreplace(payload / "catalog", authority.images / "catalog") + committed = True + _fault(fault, "bootstrap.after_rename") + _fsync_directory(authority.images, "catalog bootstrap parent", fault) + else: + assert state.root is not None + root = state.root + entries = root / "entries" + snapshots = root / "snapshots" + _publish_record( + payload / "entry.json", + entries / f"{str(intent['candidateEntryDigest'])[7:]}.json", + limits.entry_bytes, + "catalog immutable entry", + fault, + ) + _fsync_directory(entries, "catalog immutable entry history", fault) + _publish_record( + payload / "snapshot.json", + snapshots / f"{candidate[7:]}.json", + limits.snapshot_bytes, + "catalog immutable snapshot", + fault, + ) + _fsync_directory(snapshots, "catalog immutable snapshot history", fault) + if _read_current_digest(authority) != state.snapshot_digest: + _infra("catalog current pointer changed before publication") + _fault(fault, "pointer.before_replace") + os.replace(payload / "current.json", root / "current.json") + committed = True + _fault(fault, "pointer.after_replace") + _fsync_directory(root, "catalog current pointer", fault) + verified = _load_catalog(authority, lock_descriptor, limits, inspect_stage=False) + if verified.snapshot_digest != candidate: + _infra("catalog publication could not be verified") + _cleanup_stage(authority, stage, fault) + except CatalogInfrastructure: + if not committed: + try: + _reconcile(authority, lock_descriptor, limits) + except CatalogInfrastructure: + pass + raise + except OSError as error: + if not committed: + try: + _reconcile(authority, lock_descriptor, limits) + except CatalogInfrastructure: + pass + _infra("catalog publication is uncertain", error) + + +def _classified(operation: Callable[[], object]) -> object: + try: + return operation() + except (CatalogReject, CatalogInfrastructure): + raise + except ( + AssertionError, + KeyError, + OSError, + RecursionError, + TypeError, + ValueError, + json.JSONDecodeError, + ) as error: + _infra("catalog operation encountered unverified state", error) + + +def add_image( + home: Path, + name: str, + subject: str, + *, + fault: FaultHook | None = None, + limits: CatalogLimits = CatalogLimits(), +) -> dict[str, object]: + """Create a generation-one mapping, or observe its exact idempotent retry.""" + + def operation() -> dict[str, object]: + if sys.platform != "linux": + _infra("local image catalog mutations require Linux") + if not image_name(name) or name.startswith("agent-lab.") or not oci_subject(subject): + _reject("mapping is invalid or reserved") + authority = _load_home(home) + with _catalog_lock(authority, exclusive=True) as lock_descriptor: + _reconcile(authority, lock_descriptor, limits) + state = _load_catalog(authority, lock_descriptor, limits) + prior = state.records.get(name) + if prior is not None: + if prior["state"] == "active" and prior["subject"] == subject: + return { + "changed": False, + "entryDigest": prior["entryDigest"], + "generation": 1, + } + _reject("name already exists or is tombstoned") + entry, entry_digest, snapshot, snapshot_digest, intent = _candidate_values( + state, + kind="add", + name=name, + subject=subject, + expected_entry_digest=None, + limits=limits, + ) + stage = _prepare_stage( + authority, + entry, + entry_digest, + snapshot, + snapshot_digest, + intent, + limits, + fault, + ) + _publish_candidate(authority, lock_descriptor, state, stage, limits, fault) + return {"changed": True, "entryDigest": entry_digest, "generation": 1} + + result = _classified(operation) + assert isinstance(result, dict) + return result + + +def remove_image( + home: Path, + name: str, + expected_entry_digest: str, + *, + fault: FaultHook | None = None, + limits: CatalogLimits = CatalogLimits(), +) -> dict[str, object]: + """Publish the one permitted generation-two CAS tombstone.""" + + def operation() -> dict[str, object]: + if sys.platform != "linux": + _infra("local image catalog mutations require Linux") + if not image_name(name) or name.startswith("agent-lab.") or not _is_digest(expected_entry_digest): + _reject("remove request is invalid or reserved") + authority = _load_home(home) + with _catalog_lock(authority, exclusive=True) as lock_descriptor: + _reconcile(authority, lock_descriptor, limits) + state = _load_catalog(authority, lock_descriptor, limits) + prior = state.records.get(name) + if prior is None: + _reject("name is unknown") + if prior["state"] == "removed": + if prior["previousEntryDigest"] == expected_entry_digest: + return { + "changed": False, + "entryDigest": prior["entryDigest"], + "generation": 2, + "state": "removed", + } + _reject("remove compare-and-swap conflict") + if prior["entryDigest"] != expected_entry_digest: + _reject("remove compare-and-swap conflict") + subject = prior["subject"] + assert isinstance(subject, str) + entry, entry_digest, snapshot, snapshot_digest, intent = _candidate_values( + state, + kind="remove", + name=name, + subject=subject, + expected_entry_digest=expected_entry_digest, + limits=limits, + ) + stage = _prepare_stage( + authority, + entry, + entry_digest, + snapshot, + snapshot_digest, + intent, + limits, + fault, + ) + _publish_candidate(authority, lock_descriptor, state, stage, limits, fault) + return { + "changed": True, + "entryDigest": entry_digest, + "generation": 2, + "state": "removed", + } + + result = _classified(operation) + assert isinstance(result, dict) + return result + + +def list_images( + home: Path, + *, + include_removed: bool = False, + limits: CatalogLimits = CatalogLimits(), +) -> list[dict[str, object]]: + """Return a canonical byte-sorted held view without repair or initialization.""" + + def operation() -> list[dict[str, object]]: + authority = _load_home(home) + with _catalog_lock(authority, exclusive=False) as lock_descriptor: + state = _load_catalog(authority, lock_descriptor, limits) + return [ + dict(state.records[name]) + for name in sorted(state.records, key=lambda item: item.encode("ascii")) + if include_removed or state.records[name]["state"] == "active" + ] + + result = _classified(operation) + assert isinstance(result, list) + return result + + +def inspect_image( + home: Path, + name: str, + *, + limits: CatalogLimits = CatalogLimits(), +) -> dict[str, object]: + """Return one exact active/tombstoned record without repair.""" + + def operation() -> dict[str, object]: + if not image_name(name) or name.startswith("agent-lab."): + _reject("name is invalid or reserved") + authority = _load_home(home) + with _catalog_lock(authority, exclusive=False) as lock_descriptor: + state = _load_catalog(authority, lock_descriptor, limits) + record = state.records.get(name) + if record is None: + _reject("name is unknown") + return dict(record) + + result = _classified(operation) + assert isinstance(result, dict) + return result + + +@contextmanager +def hold_local_image_entries( + home: Path, + dependencies: Sequence[dict[str, object]], + *, + limits: CatalogLimits = CatalogLimits(), + fault: FaultHook | None = None, +) -> Iterator[dict[str, object]]: + """Hold one verified snapshot while exact active dependencies are consumed.""" + + try: + requested = tuple(dependencies) + except (TypeError, ValueError) as error: + _infra("selected local image dependencies are malformed", error) + + expected: dict[str, dict[str, object]] = {} + for dependency in requested: + if not isinstance(dependency, dict) or set(dependency) != { + "entryDigest", + "generation", + "name", + "subject", + }: + _infra("selected local image dependency schema is not closed") + name = dependency.get("name") + entry_digest = dependency.get("entryDigest") + generation = dependency.get("generation") + subject = dependency.get("subject") + if ( + not isinstance(name, str) + or not image_name(name) + or name.startswith("agent-lab.") + or not _is_digest(entry_digest) + or type(generation) is not int + or generation != 1 + or not isinstance(subject, str) + or not oci_subject(subject) + ): + _infra("selected local image dependency binding is invalid") + prior = expected.get(name) + if prior is not None and prior != dependency: + _infra("selected local image dependencies contradict one another") + expected[name] = dict(dependency) + + if not expected: + yield {"catalog": None, "records": {}} + return + + authority = _load_home(home) + with _catalog_lock(authority, exclusive=False) as lock_descriptor: + _fault(fault, "experiment catalog lock.after_acquire") + state = _load_catalog(authority, lock_descriptor, limits) + selected: dict[str, dict[str, object]] = {} + for name in sorted(expected, key=lambda item: item.encode("ascii")): + record = state.records.get(name) + if record is None or record["state"] != "active": + _reject("selected local image dependency is unknown or inactive") + dependency = expected[name] + if any( + record[field] != dependency[field] + for field in ("name", "entryDigest", "generation", "subject") + ): + _reject("selected local image dependency has drifted") + selected[name] = dict(record) + if state.snapshot_digest is None or state.revision < 1: + _infra("selected local image dependencies have no committed snapshot") + yield { + "catalog": { + "revision": state.revision, + "snapshotDigest": state.snapshot_digest, + }, + "records": selected, + } + + +def resolve_local_images( + home: Path, + names: Sequence[str], + *, + limits: CatalogLimits = CatalogLimits(), +) -> dict[str, object]: + """Resolve every selected local name from one verified held snapshot.""" + + def operation() -> dict[str, object]: + requested = tuple(names) + if ( + not requested + or len(set(requested)) != len(requested) + or any(not image_name(name) or name.startswith("agent-lab.") for name in requested) + ): + _reject("local image selection is invalid") + authority = _load_home(home) + with _catalog_lock(authority, exclusive=False) as lock_descriptor: + state = _load_catalog(authority, lock_descriptor, limits) + selected: dict[str, dict[str, object]] = {} + for name in sorted(requested, key=lambda item: item.encode("ascii")): + record = state.records.get(name) + if record is None or record["state"] != "active": + _reject("references an unknown or removed local image name") + selected[name] = dict(record) + if state.snapshot_digest is None or state.revision < 1: + _infra("local image resolution has no committed snapshot") + return { + "catalog": { + "revision": state.revision, + "snapshotDigest": state.snapshot_digest, + }, + "records": selected, + } + + result = _classified(operation) + assert isinstance(result, dict) + return result diff --git a/scripts/image_reference.py b/scripts/image_reference.py new file mode 100644 index 0000000..b5e4b3a --- /dev/null +++ b/scripts/image_reference.py @@ -0,0 +1,39 @@ +#!/usr/bin/env python3 +"""Shared pure grammar for Experiment image names and immutable OCI subjects.""" + +from __future__ import annotations + +import re + + +IMAGE_COMPONENT = re.compile(r"^[a-z][a-z0-9]*(?:-[a-z0-9]+)*$") +OCI_SUBJECT = re.compile( + r"^([a-z0-9]+([.-][a-z0-9]+)*" + r"(:(?:[1-9][0-9]{0,3}|[1-5][0-9]{4}|6[0-4][0-9]{3}|" + r"65[0-4][0-9]{2}|655[0-2][0-9]|6553[0-5]))?/)?" + r"[a-z0-9]+([._-][a-z0-9]+)*" + r"(/[a-z0-9]+([._-][a-z0-9]+)*)*" + r"@sha256:[0-9a-f]{64}$" +) + + +def valid_image_name(value: object) -> bool: + if not isinstance(value, str) or not value.isascii(): + return False + encoded = value.encode("ascii") + parts = value.split(".") + return ( + len(encoded) <= 63 + and len(parts) == 2 + and all(1 <= len(part.encode("ascii")) <= 31 for part in parts) + and all(IMAGE_COMPONENT.fullmatch(part) is not None for part in parts) + ) + + +def valid_oci_subject(value: object) -> bool: + return ( + isinstance(value, str) + and value.isascii() + and 1 <= len(value.encode("ascii")) <= 255 + and OCI_SUBJECT.fullmatch(value) is not None + ) diff --git a/scripts/install-local b/scripts/install-local new file mode 100755 index 0000000..4ac8d3b --- /dev/null +++ b/scripts/install-local @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +set -euo pipefail + +script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" +exec python3 -I "$script_dir/install-local.py" "$@" diff --git a/scripts/install-local.py b/scripts/install-local.py new file mode 100644 index 0000000..dfa15f4 --- /dev/null +++ b/scripts/install-local.py @@ -0,0 +1,117 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import hashlib +import os +from pathlib import Path +import pwd +import shutil +import stat +import sys +import tempfile + + +def fail(message: str, code: int = 1) -> int: + print(f"FAIL local install {message}", file=sys.stderr) + return code + + +def prefix_from(argv: list[str]) -> Path: + if len(argv) == 2 and argv[0] == "--prefix" and argv[1]: + raw = argv[1] + elif not argv: + raw = os.environ.get("AGENT_LAB_PREFIX") or str(Path(pwd.getpwuid(os.getuid()).pw_dir) / ".local") + else: + raise ValueError("Usage: scripts/install-local [--prefix ABSOLUTE_PREFIX]") + path = Path(raw) + if not path.is_absolute() or path == Path("/"): + raise ValueError("prefix must be an absolute non-root path") + return path + + +def reject_unsafe_existing_path(path: Path) -> None: + current = Path(path.anchor) + for component in path.parts[1:]: + current /= component + try: + metadata = current.lstat() + except FileNotFoundError: + return + if stat.S_ISLNK(metadata.st_mode) or not stat.S_ISDIR(metadata.st_mode): + raise OSError("unsafe prefix component") + + +def main(argv: list[str]) -> int: + try: + prefix = prefix_from(argv) + except ValueError as error: + print(error, file=sys.stderr) + return 2 + try: + reject_unsafe_existing_path(prefix) + except OSError: + return fail("prefix contains an unsafe existing component", 125) + root = Path(__file__).resolve().parent.parent + manifest_path = root / "packaging/agent-lab-local.manifest" + try: + names = manifest_path.read_text(encoding="ascii").splitlines() + except OSError: + return fail("runtime manifest is unavailable", 125) + if names != sorted(set(names)) or not names: + return fail("runtime manifest is not canonical", 125) + digest = hashlib.sha256(b"agent-lab.local-bundle.v1\0") + sources: list[tuple[str, Path, bytes, int]] = [] + try: + for name in names: + relative = Path(name) + if relative.is_absolute() or ".." in relative.parts: + raise OSError("unsafe manifest path") + source = root / relative + metadata = source.lstat() + if not stat.S_ISREG(metadata.st_mode) or metadata.st_nlink != 1: + raise OSError("unsafe runtime source") + data = source.read_bytes() + final = source.stat() + if (metadata.st_dev, metadata.st_ino, metadata.st_size, metadata.st_mtime_ns) != ( + final.st_dev, final.st_ino, final.st_size, final.st_mtime_ns + ): + raise OSError("runtime source changed") + encoded = name.encode("ascii") + digest.update(len(encoded).to_bytes(4, "big")) + digest.update(encoded) + digest.update(len(data).to_bytes(8, "big")) + digest.update(data) + sources.append((name, source, data, stat.S_IMODE(metadata.st_mode))) + except (OSError, UnicodeError): + return fail("runtime source is unsafe or changing", 125) + release_id = digest.hexdigest() + releases = prefix / "lib/agent-lab/releases" + release = releases / release_id + try: + releases.mkdir(mode=0o700, parents=True, exist_ok=True) + if not release.exists(): + stage = Path(tempfile.mkdtemp(prefix=".agent-lab-release-", dir=releases)) + try: + for name, _source, data, mode in sources: + target = stage / name + target.parent.mkdir(mode=0o700, parents=True, exist_ok=True) + target.write_bytes(data) + target.chmod(0o755 if mode & 0o111 else 0o644) + os.rename(stage, release) + except BaseException: + shutil.rmtree(stage, ignore_errors=True) + raise + bindir = prefix / "bin" + bindir.mkdir(mode=0o700, parents=True, exist_ok=True) + temporary = bindir / f".agent-lab-{os.getpid()}" + target = Path("../lib/agent-lab/releases") / release_id / "scripts/agent-lab" + os.symlink(target, temporary) + os.replace(temporary, bindir / "agent-lab") + except OSError: + return fail("bundle could not be published", 125) + print(f"installed:{release_id}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main(sys.argv[1:])) diff --git a/tests/dev/security-gate-cases.sh b/tests/dev/security-gate-cases.sh index 7a382b2..05e61f8 100644 --- a/tests/dev/security-gate-cases.sh +++ b/tests/dev/security-gate-cases.sh @@ -228,8 +228,10 @@ docker-harness-contract tests/dev/docker-harness-cases.sh SUMMARY failures=0 guard-command tests/guard/pretooluse-cases.sh SUMMARY failures=0 guard-mount tests/guard/cases.sh SUMMARY failures=0 config-authority tests/agent/config-guard.sh SUMMARY failures=0 -experiment-contract tests/experiment/contract-cases.sh SUMMARY failures=0 -experiment-authorization tests/experiment/authorization-cases.sh SUMMARY failures=0 +experiment-contract tests/experiment/contract-cases.sh EXPERIMENT CONTRACT PASS +experiment-authorization tests/experiment/authorization-cases.sh EXPERIMENT AUTHORIZATION PASS +experiment-local-lifecycle tests/experiment/local-lifecycle-cases.sh EXPERIMENT LOCAL LIFECYCLE PASS +experiment-source-adapters tests/experiment/source-adapter-cases.sh EXPERIMENT SOURCE ADAPTERS PASS config-matrix tests/agent/config-matrix.sh SUMMARY failures=0 allowlist-schema tests/agent/allowlist-cases.sh SUMMARY failures=0 image-volume-policy tests/agent/image-volume-policy-cases.sh SUMMARY failures=0 diff --git a/tests/experiment/aggregate-harness-cases.sh b/tests/experiment/aggregate-harness-cases.sh new file mode 100755 index 0000000..c8df5f1 --- /dev/null +++ b/tests/experiment/aggregate-harness-cases.sh @@ -0,0 +1,907 @@ +#!/usr/bin/env bash +set -u -o pipefail + +repo_root="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." >/dev/null 2>&1 && pwd)" +lifecycle="$repo_root/tests/experiment/local-lifecycle-cases.sh" +source_adapters="$repo_root/tests/experiment/source-adapter-cases.sh" +expected_count=23 +work="" + +cleanup_work() { + local failed=0 + if [ -n "$work" ] && [ -e "$work" ]; then + find "$work" -type f -delete 2>/dev/null || failed=1 + find "$work" -type l -delete 2>/dev/null || failed=1 + find "$work" -depth -type d -exec rmdir {} + 2>/dev/null || failed=1 + [ ! -e "$work" ] || failed=1 + fi + return "$failed" +} + +if ! work="$(mktemp -d)"; then + printf 'SUMMARY assertions=0 expected=%s failures=0 infra=1\n' "$expected_count" + exit 125 +fi +trap 'cleanup_work >/dev/null 2>&1 || true' EXIT +replica="$work/repo" +replica_lifecycle="$replica/tests/experiment/local-lifecycle-cases.sh" +replica_source_adapters="$replica/tests/experiment/source-adapter-cases.sh" +mkdir -p "$replica/tests/experiment" "$replica/tests/install" +cp "$lifecycle" "$replica_lifecycle" +cp "$source_adapters" "$replica_source_adapters" +chmod +x "$replica_lifecycle" "$replica_source_adapters" + +failures=0 +pass() { printf 'PASS %s %s\n' "$1" "$2"; } +fail() { printf 'FAIL %s %s\n' "$1" "$2"; failures=$((failures + 1)); } + +expected_ids=( + PKG-001 PKG-002 PKG-003 PKG-004 PKG-005 + CFG-001 CFG-002 CFG-003 CFG-004 TOOL-001 + CAT-NAME-001 CAT-NAME-002 CAT-OCI-001 CAT-OCI-002 + CAT-ADD-001 CAT-ADD-002 CAT-ADD-003 CAT-ADD-004 CAT-NS-001 + CAT-CAS-001 CAT-CAS-002 CAT-CAS-003 CAT-CAS-004 + CAT-READ-001 CAT-READ-002 CAT-NOEF-001 CAT-CONC-001 CAT-CONC-002 + CAT-STATE-001 CAT-STATE-002 CAT-STATE-003 CAT-STATE-004 + CAT-STATE-005 CAT-STATE-006 CAT-STATE-007 CAT-STATE-008 + CAT-STATE-009 CAT-STATE-010 CAT-STATE-011 CAT-STATE-012 + CAT-STATE-013 CAT-STATE-014 CAT-STATE-015 CAT-STATE-016 CAT-STATE-018 CAT-STATE-017 + CAT-BOUND-001 CAT-BOUND-002 CAT-BOUND-003 CAT-BOUND-004 + CAT-CRASH-001 CAT-CRASH-002 CAT-CRASH-003 CAT-CRASH-004 CAT-CRASH-005 + CAT-CRASH-006 CAT-CRASH-007 CAT-CRASH-008 CAT-CRASH-009 CAT-CRASH-010 CAT-CRASH-011 CAT-PLAT-001 + RES-ENTRY-001 RES-SNAP-001 RES-SNAP-002 RES-ENTRY-002 RES-ENTRY-003 + RES-ISOLATE-001 RES-STATE-001 RES-STATE-002 RES-STATE-003 RES-INPUT-001 + RES-SNAP-003 RES-AUTH-001 RES-INSTALL-001 RES-NOEF-001 + M-CAT-OCI-001 M-CAT-SHADOW-001 M-CAT-CAS-001 M-CAT-AUTH-001 M-RES-BIND-001 + M-CAT-NOEF-001 M-CAT-ADMIT-001 M-CAT-ATOM-001 M-CAT-DUR-001 M-CAT-STAGE-001 + INST-HOME-001 INST-UNKNOWN-001 INST-NAME-001 INST-PERMIT-001 INST-RECEIPT-001 + INST-INSPECT-001 INST-RETRY-001 INST-CONFLICT-001 INST-DENY-001 + INST-FORGE-001 INST-NOEF-001 INST-LOCAL-001 INST-RUNTIME-001 + IST-STATE-001 IST-LOCK-001 IST-STATE-002 IST-BOUND-001 IST-STATE-003 + IST-CONC-001 IST-CONC-002 IST-CRASH-001 IST-LIVE-001 IST-PLAT-001 IST-PROV-001 + IST-PREFLIGHT-001 IST-SNAPSHOT-001 IST-CONFIG-001 IST-READ-001 IST-FAULT-001 + IIN-CLEAN-001 IIN-CONTRACT-001 IIN-BUNDLE-001 IIN-LOCK-001 + IIN-SNAPSHOT-001 IIN-PREFLIGHT-001 IIN-PLAN-001 + M-STORE-AUTH-001 M-STORE-SOURCE-001 M-STORE-ATOM-001 M-STORE-RETRY-001 + M-STORE-DUR-001 M-STORE-LAYOUT-001 M-STORE-KEY-001 M-STORE-VERIFY-001 + M-STORE-LIVE-001 M-STORE-UNCERT-001 M-STORE-STAGE-001 +) +installer_ids=("${expected_ids[@]:0:5}") +config_ids=("${expected_ids[@]:5:5}") +catalog_ids=("${expected_ids[@]:10:76}") +install_ids=("${expected_ids[@]:86:13}") +state_ids=("${expected_ids[@]:99:16}") +integrity_ids=("${expected_ids[@]:115:7}") +mutation_ids=("${expected_ids[@]:122:11}") +source_zip_ids=( + ZIP-001 ZIP-COMPAT-001 ZIP-USAGE-001 ZIP-PATH-001 ZIP-COUNT-001 ZIP-TYPE-001 + ZIP-META-001 ZIP-FLAG-001 ZIP-METHOD-001 ZIP-ZIP64-001 ZIP-HEADER-001 ZIP-CRC-001 + ZIP-LENGTH-001 ZIP-SIZE-001 ZIP-BOMB-001 ZIP-TRUNC-001 ZIP-TRAIL-001 ZIP-SIZE-002 + ZIP-READ-001 ZIP-READ-002 ZIP-DECODE-001 ZIP-DECODE-003 ZIP-DECODE-002 + ZIP-TIMEOUT-001 ZIP-OUTPUT-001 ZIP-NOEF-001 ZIP-AUTH-001 ZIP-INSTALL-001 + ZIP-RETRY-001 ZIP-DENY-001 ZIP-PLAT-001 ZIP-NOEF-002 ZIP-RUNTIME-001 +) +source_mutation_ids=( + M-ZIP-WATCHDOG-001 M-ZIP-COUNT-001 M-ZIP-NAME-001 M-ZIP-TYPE-001 M-ZIP-FLAG-001 + M-ZIP-METHOD-001 M-ZIP-SIZE-001 M-ZIP-BOMB-001 M-ZIP-CRC-001 + M-ZIP-HEADER-001 M-ZIP-EXTRACT-001 M-ZIP-IDENTITY-001 M-ZIP-AUTH-001 +) +source_git_ids=( + GIT-CLI-001 GIT-USAGE-001 GIT-URL-001 GIT-OID-001 GIT-PLAT-001 + GIT-FIXTURE-001 GIT-PIN-001 GIT-COMMIT-001 GIT-ROOT-001 GIT-TYPE-001 + GIT-BLOB-001 GIT-DRIFT-001 GIT-AUTHORITY-001 GIT-CREDENTIAL-001 + GIT-REDIRECT-001 GIT-CONTENT-001 GIT-TIMEOUT-001 GIT-OUTPUT-001 + GIT-ACQUIRE-001 GIT-PGROUP-001 GIT-CLEANUP-001 GIT-TAXONOMY-001 + GIT-CHECK-001 GIT-AUTH-001 GIT-DENY-001 GIT-INSTALL-001 + GIT-IDENTITY-001 GIT-RETRY-001 GIT-ADAPTER-001 GIT-NOEF-001 + GIT-RUNTIME-001 GIT-DIAG-001 +) +source_git_mutation_ids=( + M-GIT-AUTHORITY-001 M-GIT-REF-001 M-GIT-BOUND-001 M-GIT-IDENTITY-001 + M-GIT-DOWNSTREAM-001 +) +source_expected_ids=( + "${source_zip_ids[@]}" + "${source_mutation_ids[@]}" + "${source_git_ids[@]}" + "${source_git_mutation_ids[@]}" +) + +write_fixture() { + local path="$1" + local rc="$2" + shift 2 + local record kind id fixture_failures=0 + local execution_id="${path##*/}" + { + printf '#!/usr/bin/env bash\nset -u\n' + printf 'if [ -n "${AGENT_LAB_AGG_EXEC_LOG:-}" ]; then\n' + printf " printf '%%s\\n' '%s' >> \"\$AGENT_LAB_AGG_EXEC_LOG\" || exit 125\n" "$execution_id" + printf 'fi\n' + case "$execution_id" in + local-install-cases.sh | local-image-catalog-cases.sh) + printf 'if [ -n "${AGENT_LAB_AGG_BARRIER_DIR:-}" ]; then\n' + printf " : > \"\$AGENT_LAB_AGG_BARRIER_DIR/%s.ready\" || exit 125\n" "$execution_id" + printf ' attempts=0\n' + printf ' while [ ! -f "$AGENT_LAB_AGG_BARRIER_DIR/local-install-cases.sh.ready" ] ||\n' + printf ' [ ! -f "$AGENT_LAB_AGG_BARRIER_DIR/local-image-catalog-cases.sh.ready" ] ||\n' + printf ' [ ! -f "$AGENT_LAB_AGG_BARRIER_DIR/install-state-cases.py.ready" ]; do\n' + printf ' attempts=$((attempts + 1))\n' + printf ' [ "$attempts" -lt 200 ] || exit 125\n' + printf ' sleep 0.01\n' + printf ' done\n' + printf 'fi\n' + ;; + esac + printf 'if [ -n "${AGENT_LAB_AGG_HOLD_DIR:-}" ] &&\n' + printf " [ \"\${AGENT_LAB_AGG_HOLD_ID:-}\" = '%s' ]; then\n" "$execution_id" + printf ' : > "$AGENT_LAB_AGG_HOLD_DIR/ready" || exit 125\n' + printf ' attempts=0\n' + printf ' while [ ! -f "$AGENT_LAB_AGG_HOLD_DIR/release" ]; do\n' + printf ' attempts=$((attempts + 1))\n' + printf ' [ "$attempts" -lt 500 ] || exit 125\n' + printf ' sleep 0.01\n' + printf ' done\n' + printf 'fi\n' + if [ "$execution_id" = "local-image-catalog-cases.sh" ]; then + printf 'if [ -n "${AGENT_LAB_AGG_SIGNAL_DIR:-}" ]; then\n' + printf ' sleep 30 &\n' + printf ' descendant_pid=$!\n' + printf ' printf "%%s\\n" "$descendant_pid" > "$AGENT_LAB_AGG_SIGNAL_DIR/descendant.pid" || exit 125\n' + printf ' : > "$AGENT_LAB_AGG_SIGNAL_DIR/ready" || exit 125\n' + printf ' wait "$descendant_pid"\n' + printf 'fi\n' + printf 'if [ -n "${AGENT_LAB_AGG_STUBBORN_SIGNAL_DIR:-}" ]; then\n' + printf ' (\n' + printf " trap '' HUP INT QUIT TERM\n" + printf ' printf "%%s\\n" "$BASHPID" > "$AGENT_LAB_AGG_STUBBORN_SIGNAL_DIR/descendant.pid" || exit 125\n' + printf ' : > "$AGENT_LAB_AGG_STUBBORN_SIGNAL_DIR/ready" || exit 125\n' + printf ' sleep 30\n' + printf ' ) &\n' + printf ' descendant_pid=$!\n' + printf ' wait "$descendant_pid"\n' + printf 'fi\n' + fi + for record in "$@"; do + kind="${record%%:*}" + id="${record#*:}" + [ "$kind" = "FAIL" ] && fixture_failures=$((fixture_failures + 1)) + printf "printf '%s %s fixture assertion\\n'\n" "$kind" "$id" + done + printf "printf 'SUMMARY assertions=%s expected=%s failures=%s infra=0\\n'\n" \ + "$#" "$#" "$fixture_failures" + printf 'if [ -n "${AGENT_LAB_AGG_DONE_DIR:-}" ]; then\n' + printf " : > \"\$AGENT_LAB_AGG_DONE_DIR/%s.done\" || exit 125\n" "$execution_id" + printf 'fi\n' + printf 'exit %s\n' "$rc" + } > "$path" + chmod +x "$path" +} + +write_python_fixture() { + local path="$1" + local rc="$2" + shift 2 + local record kind id fixture_failures=0 + local execution_id="${path##*/}" + { + printf '#!/usr/bin/env python3\n' + printf 'import os\n' + printf 'from pathlib import Path\n' + printf 'import time\n' + printf 'log = os.environ.get("AGENT_LAB_AGG_EXEC_LOG")\n' + printf 'if log:\n' + printf ' with open(log, "a", encoding="ascii") as stream:\n' + printf " stream.write('%s\\\\n')\n" "$execution_id" + printf "hold_dir = os.environ.get('AGENT_LAB_AGG_HOLD_DIR')\n" + printf "hold_id = os.environ.get('AGENT_LAB_AGG_HOLD_ID')\n" + printf "if hold_dir and hold_id == '%s':\n" "$execution_id" + printf " Path(hold_dir, 'ready').touch()\n" + printf ' deadline = time.monotonic() + 5.0\n' + printf " while not Path(hold_dir, 'release').is_file():\n" + printf ' if time.monotonic() >= deadline:\n' + printf ' raise SystemExit(125)\n' + printf ' time.sleep(0.01)\n' + if [ "$execution_id" = "install-state-cases.py" ]; then + printf 'barrier = os.environ.get("AGENT_LAB_AGG_BARRIER_DIR")\n' + printf 'if barrier:\n' + printf " Path(barrier, '%s.ready').touch()\n" "$execution_id" + printf ' peers = (\n' + printf " 'local-install-cases.sh.ready',\n" + printf " 'local-image-catalog-cases.sh.ready',\n" + printf " 'install-state-cases.py.ready',\n" + printf ' )\n' + printf ' deadline = time.monotonic() + 2.0\n' + printf ' while not all(Path(barrier, peer).is_file() for peer in peers):\n' + printf ' if time.monotonic() >= deadline:\n' + printf ' raise SystemExit(125)\n' + printf ' time.sleep(0.01)\n' + fi + for record in "$@"; do + kind="${record%%:*}" + id="${record#*:}" + [ "$kind" = "FAIL" ] && fixture_failures=$((fixture_failures + 1)) + printf "print('%s %s fixture assertion')\n" "$kind" "$id" + done + printf "print('SUMMARY assertions=%s expected=%s failures=%s infra=0')\n" \ + "$#" "$#" "$fixture_failures" + printf "done_dir = os.environ.get('AGENT_LAB_AGG_DONE_DIR')\n" + printf 'if done_dir:\n' + printf " Path(done_dir, '%s.done').touch()\n" "$execution_id" + printf 'raise SystemExit(%s)\n' "$rc" + } > "$path" + chmod +x "$path" +} + +pass_records() { + local id + for id in "$@"; do + printf 'PASS:%s\n' "$id" + done +} + +reset_fixtures() { + local installer_records=() config_records=() catalog_records=() + local install_records=() state_records=() integrity_records=() mutation_records=() + mapfile -t installer_records < <(pass_records "${installer_ids[@]}") + mapfile -t config_records < <(pass_records "${config_ids[@]}") + mapfile -t catalog_records < <(pass_records "${catalog_ids[@]}") + mapfile -t install_records < <(pass_records "${install_ids[@]}") + mapfile -t state_records < <(pass_records "${state_ids[@]}") + mapfile -t integrity_records < <(pass_records "${integrity_ids[@]}") + mapfile -t mutation_records < <(pass_records "${mutation_ids[@]}") + write_fixture "$replica/tests/install/local-install-cases.sh" 0 "${installer_records[@]}" + write_fixture "$replica/tests/experiment/local-config-cases.sh" 0 "${config_records[@]}" + write_fixture "$replica/tests/experiment/local-image-catalog-cases.sh" 0 "${catalog_records[@]}" + write_fixture "$replica/tests/experiment/install-store-cases.sh" 0 "${install_records[@]}" + write_python_fixture "$replica/tests/experiment/install-state-cases.py" 0 "${state_records[@]}" + write_python_fixture "$replica/tests/experiment/install-integrity-cases.py" 0 "${integrity_records[@]}" + write_python_fixture "$replica/tests/experiment/install-mutation-cases.py" 0 "${mutation_records[@]}" +} + +reset_source_fixtures() { + local zip_records=() mutation_records=() git_records=() git_mutation_records=() + mapfile -t zip_records < <(pass_records "${source_zip_ids[@]}") + mapfile -t mutation_records < <(pass_records "${source_mutation_ids[@]}") + mapfile -t git_records < <(pass_records "${source_git_ids[@]}") + mapfile -t git_mutation_records < <(pass_records "${source_git_mutation_ids[@]}") + write_fixture "$replica/tests/experiment/zip-intake-cases.sh" 0 "${zip_records[@]}" + write_python_fixture \ + "$replica/tests/experiment/zip-mutation-cases.py" 0 "${mutation_records[@]}" + write_fixture "$replica/tests/experiment/git-intake-cases.sh" 0 "${git_records[@]}" + write_python_fixture \ + "$replica/tests/experiment/git-mutation-cases.py" 0 "${git_mutation_records[@]}" +} + +run_replica() { + local output="$1" + shift + run_selected "$output" "$replica_lifecycle" "$@" +} + +run_selected() { + local output="$1" + local selected_lifecycle="$2" + shift 2 + "$@" bash "$selected_lifecycle" > "$output" 2>&1 + return $? +} + +run_source_replica() { + local output="$1" + shift + run_selected "$output" "$replica_source_adapters" "$@" +} + +wait_for_path() { + local path="$1" + local attempts=0 + while [ ! -e "$path" ]; do + attempts=$((attempts + 1)) + [ "$attempts" -lt 500 ] || return 1 + sleep 0.01 + done +} + +wait_for_process_exit() { + local pid="$1" + local attempts=0 + while kill -0 "$pid" 2>/dev/null; do + attempts=$((attempts + 1)) + [ "$attempts" -lt 100 ] || return 1 + sleep 0.01 + done +} + +wait_for_child_wait() { + local pid="$1" + local expected_children="$2" + local attempts=0 + local wchan child_list + local children=() + + while [ "$attempts" -lt 100 ]; do + wchan="$(cat "/proc/$pid/wchan" 2>/dev/null || true)" + child_list="$(cat "/proc/$pid/task/$pid/children" 2>/dev/null || true)" + children=() + read -r -a children <<< "$child_list" + if [ "$wchan" = "do_wait" ] && + [ "${#children[@]}" -eq "$expected_children" ]; then + return 0 + fi + attempts=$((attempts + 1)) + sleep 0.01 + done + return 1 +} + +reset_fixtures +expected_executions="$work/expected-executions" +baseline_executions="$work/baseline-executions" +mutant_executions="$work/mutant-executions" +printf '%s\n' \ + local-install-cases.sh \ + local-config-cases.sh \ + local-image-catalog-cases.sh \ + install-store-cases.sh \ + install-state-cases.py \ + install-integrity-cases.py \ + install-mutation-cases.py > "$expected_executions" +: > "$baseline_executions" +baseline_rc=0 +run_replica "$work/baseline.out" env \ + AGENT_LAB_AGG_EXEC_LOG="$baseline_executions" || baseline_rc=$? + +mutant_lifecycle="$replica/tests/experiment/local-lifecycle-hidden-duplicate.sh" +awk ' + { print } + $0 == "subcases=(" { in_subcases=1; next } + in_subcases && $0 == ")" { + print "\"$repo_root/tests/install/local-install-cases.sh\" >/dev/null 2>&1" + in_subcases=0 + } +' "$replica_lifecycle" > "$mutant_lifecycle" +chmod +x "$mutant_lifecycle" +mutation_count="$(grep -Fxc '"$repo_root/tests/install/local-install-cases.sh" >/dev/null 2>&1' "$mutant_lifecycle")" +: > "$mutant_executions" +mutant_rc=0 +run_selected "$work/mutant.out" "$mutant_lifecycle" env \ + AGENT_LAB_AGG_EXEC_LOG="$mutant_executions" || mutant_rc=$? +mutant_expected="$work/mutant-expected-executions" +printf '%s\n' \ + local-install-cases.sh \ + local-install-cases.sh \ + local-config-cases.sh \ + local-image-catalog-cases.sh \ + install-store-cases.sh \ + install-state-cases.py \ + install-integrity-cases.py \ + install-mutation-cases.py > "$mutant_expected" + +overlap_barrier="$work/overlap-barrier" +overlap_executions="$work/overlap-executions" +mkdir "$overlap_barrier" +: > "$overlap_executions" +overlap_rc=0 +run_replica "$work/overlap.out" env \ + AGENT_LAB_AGG_EXEC_LOG="$overlap_executions" \ + AGENT_LAB_AGG_BARRIER_DIR="$overlap_barrier" || overlap_rc=$? + +if [ "$baseline_rc" -eq 0 ] && + cmp -s <(LC_ALL=C sort "$expected_executions") <(LC_ALL=C sort "$baseline_executions") && + [ "$mutation_count" -eq 1 ] && + [ "$mutant_rc" -eq 0 ] && + cmp -s "$work/baseline.out" "$work/mutant.out" && + cmp -s <(LC_ALL=C sort "$mutant_expected") <(LC_ALL=C sort "$mutant_executions") && + ! cmp -s <(LC_ALL=C sort "$expected_executions") <(LC_ALL=C sort "$mutant_executions") && + [ "$overlap_rc" -eq 0 ] && + cmp -s <(LC_ALL=C sort "$expected_executions") <(LC_ALL=C sort "$overlap_executions") && + [ -f "$overlap_barrier/local-install-cases.sh.ready" ] && + [ -f "$overlap_barrier/local-image-catalog-cases.sh.ready" ] && + [ -f "$overlap_barrier/install-state-cases.py.ready" ]; then + pass AGG-001 "execution ledger and overlap barrier prove exact-once concurrent routing" +else + fail AGG-001 "execution ledger and overlap barrier prove exact-once concurrent routing" +fi + +reset_fixtures +success_output="$work/success.out" +success_rc=0 +run_replica "$success_output" env || success_rc=$? +if [ "$success_rc" -eq 0 ] && + [ "$(grep -Ec '^(PASS|FAIL) [A-Z0-9-]+ ' "$success_output")" -eq 133 ] && + [ "$(grep -Fxc 'SUMMARY assertions=133 expected=133 failures=0 infra=0' "$success_output")" -eq 1 ] && + [ "$(tail -n 1 "$success_output")" = 'EXPERIMENT LOCAL LIFECYCLE PASS' ] && + awk '/^(PASS|FAIL) [A-Z0-9-]+ / {next} /^SUMMARY assertions=133 expected=133 failures=0 infra=0$/ {next} /^EXPERIMENT LOCAL LIFECYCLE PASS$/ {next} {bad=1} END {exit bad}' "$success_output"; then + pass AGG-002 "success forwards only assertions then one summary and marker" +else + fail AGG-002 "success forwards only assertions then one summary and marker" +fi + +missing_records=() +mapfile -t missing_records < <(pass_records "${installer_ids[@]:0:4}") +write_fixture "$replica/tests/install/local-install-cases.sh" 0 "${missing_records[@]}" +missing_rc=0 +run_replica "$work/missing.out" env || missing_rc=$? +if [ "$missing_rc" -eq 1 ] && ! grep -Fxq 'EXPERIMENT LOCAL LIFECYCLE PASS' "$work/missing.out"; then + pass AGG-003 "missing assertion identity maps to failure" +else + fail AGG-003 "missing assertion identity maps to failure" +fi + +reset_fixtures +duplicate_records=() +mapfile -t duplicate_records < <(pass_records "${installer_ids[@]}" PKG-005) +write_fixture "$replica/tests/install/local-install-cases.sh" 0 "${duplicate_records[@]}" +duplicate_rc=0 +run_replica "$work/duplicate.out" env || duplicate_rc=$? +if [ "$duplicate_rc" -eq 1 ] && ! grep -Fxq 'EXPERIMENT LOCAL LIFECYCLE PASS' "$work/duplicate.out"; then + pass AGG-004 "duplicate assertion identity maps to failure" +else + fail AGG-004 "duplicate assertion identity maps to failure" +fi + +reset_fixtures +substituted_records=() +mapfile -t substituted_records < <(pass_records "${installer_ids[@]:0:4}" BAD-001) +write_fixture "$replica/tests/install/local-install-cases.sh" 0 "${substituted_records[@]}" +substituted_rc=0 +run_replica "$work/substituted.out" env || substituted_rc=$? +if [ "$substituted_rc" -eq 1 ] && ! grep -Fxq 'EXPERIMENT LOCAL LIFECYCLE PASS' "$work/substituted.out"; then + pass AGG-005 "substituted assertion identity maps to failure" +else + fail AGG-005 "substituted assertion identity maps to failure" +fi + +reset_fixtures +failed_records=() +mapfile -t failed_records < <(pass_records "${installer_ids[@]}") +failed_records[0]='FAIL:PKG-001' +write_fixture "$replica/tests/install/local-install-cases.sh" 1 "${failed_records[@]}" +assertion_rc=0 +run_replica "$work/assertion.out" env || assertion_rc=$? +if [ "$assertion_rc" -eq 1 ] && + grep -Fxq 'SUMMARY assertions=133 expected=133 failures=1 infra=0' "$work/assertion.out" && + ! grep -Fxq 'EXPERIMENT LOCAL LIFECYCLE PASS' "$work/assertion.out"; then + pass AGG-006 "subcase assertion failure maps to one" +else + fail AGG-006 "subcase assertion failure maps to one" +fi + +reset_fixtures +uncertain_records=() +mapfile -t uncertain_records < <(pass_records "${installer_ids[@]}") +write_fixture "$replica/tests/install/local-install-cases.sh" 125 "${uncertain_records[@]}" +subcase_infra_rc=0 +run_replica "$work/subcase-infra.out" env || subcase_infra_rc=$? +reset_fixtures +mixed_failed_records=() +mixed_uncertain_records=() +mapfile -t mixed_failed_records < <(pass_records "${installer_ids[@]}") +mixed_failed_records[0]='FAIL:PKG-001' +mapfile -t mixed_uncertain_records < <(pass_records "${config_ids[@]}") +write_fixture "$replica/tests/install/local-install-cases.sh" 1 "${mixed_failed_records[@]}" +write_fixture "$replica/tests/experiment/local-config-cases.sh" 125 "${mixed_uncertain_records[@]}" +mixed_executions="$work/mixed-executions" +: > "$mixed_executions" +mixed_rc=0 +run_replica "$work/mixed.out" env \ + AGENT_LAB_AGG_EXEC_LOG="$mixed_executions" || mixed_rc=$? +reset_fixtures +find "$replica/tests/install/local-install-cases.sh" -delete +setup_infra_rc=0 +run_replica "$work/setup-infra.out" env || setup_infra_rc=$? +if [ "$subcase_infra_rc" -eq 125 ] && [ "$setup_infra_rc" -eq 125 ] && + [ "$mixed_rc" -eq 125 ] && + grep -Fxq 'SUMMARY assertions=133 expected=133 failures=1 infra=1' "$work/mixed.out" && + cmp -s <(LC_ALL=C sort "$expected_executions") <(LC_ALL=C sort "$mixed_executions") && + ! grep -Fxq 'EXPERIMENT LOCAL LIFECYCLE PASS' "$work/subcase-infra.out" && + ! grep -Fxq 'EXPERIMENT LOCAL LIFECYCLE PASS' "$work/mixed.out" && + ! grep -Fxq 'EXPERIMENT LOCAL LIFECYCLE PASS' "$work/setup-infra.out"; then + pass AGG-007 "all lanes finish and uncertainty dominates assertion failure" +else + fail AGG-007 "all lanes finish and uncertainty dominates assertion failure" +fi + +reset_fixtures +shim="$work/shim" +mkdir "$shim" +printf '#!/usr/bin/env bash\nexit 1\n' > "$shim/rmdir" +chmod +x "$shim/rmdir" +cleanup_rc=0 +run_replica "$work/cleanup.out" env PATH="$shim:$PATH" || cleanup_rc=$? +if [ "$cleanup_rc" -eq 125 ] && + grep -Fxq 'SUMMARY assertions=133 expected=133 failures=0 infra=1' "$work/cleanup.out" && + ! grep -Fxq 'EXPERIMENT LOCAL LIFECYCLE PASS' "$work/cleanup.out"; then + pass AGG-008 "cleanup uncertainty maps to one hundred twenty-five before the marker" +else + fail AGG-008 "cleanup uncertainty maps to one hundred twenty-five before the marker" +fi + +reset_fixtures +summaryless="$replica/tests/install/local-install-cases.sh" +{ + printf '#!/usr/bin/env bash\nset -u\n' + for id in "${installer_ids[@]}"; do + printf "printf 'PASS %s fixture assertion\\n'\n" "$id" + done + printf 'exit 0\n' +} > "$summaryless" +chmod +x "$summaryless" +summaryless_rc=0 +run_replica "$work/summaryless.out" env || summaryless_rc=$? +if [ "$summaryless_rc" -eq 125 ] && + ! grep -Fxq 'EXPERIMENT LOCAL LIFECYCLE PASS' "$work/summaryless.out"; then + pass AGG-009 "missing subcase summary maps to one hundred twenty-five" +else + fail AGG-009 "missing subcase summary maps to one hundred twenty-five" +fi + +lane_assignment_count="$(grep -Ec '^[[:space:]]*(readonly[[:space:]]+)?lane_count=' \ + "$replica_lifecycle" || true)" +if [ "$lane_assignment_count" -eq 1 ] && + [ "$(grep -Fxc 'readonly lane_count=3' "$replica_lifecycle")" -eq 1 ] && + [ "$(grep -Fxc 'for ((lane = 0; lane < lane_count; lane++)); do' \ + "$replica_lifecycle")" -eq 1 ] && + [ "$(grep -Fxc ' run_lane "$lane" &' "$replica_lifecycle")" -eq 1 ]; then + pass AGG-010 "lifecycle declares one immutable three-lane bound" +else + fail AGG-010 "lifecycle declares one immutable three-lane bound" +fi + +hold_ids=( + local-install-cases.sh + local-config-cases.sh + local-image-catalog-cases.sh +) +peer_done_one=( + install-state-cases.py.done + install-mutation-cases.py.done + install-mutation-cases.py.done +) +peer_done_two=( + install-integrity-cases.py.done + install-integrity-cases.py.done + install-state-cases.py.done +) +target_done=( + install-mutation-cases.py.done + install-state-cases.py.done + install-integrity-cases.py.done +) +expected_wait_children=(1 1 1) +wait_contract=1 +for hold_index in "${!hold_ids[@]}"; do + reset_fixtures + hold_dir="$work/lane-hold-$hold_index" + done_dir="$work/lane-done-$hold_index" + wait_executions="$work/wait-executions-$hold_index" + mkdir "$hold_dir" "$done_dir" + : > "$wait_executions" + wait_rc=0 + env \ + AGENT_LAB_AGG_EXEC_LOG="$wait_executions" \ + AGENT_LAB_AGG_HOLD_DIR="$hold_dir" \ + AGENT_LAB_AGG_HOLD_ID="${hold_ids[$hold_index]}" \ + AGENT_LAB_AGG_DONE_DIR="$done_dir" \ + bash "$replica_lifecycle" > "$work/wait-$hold_index.out" 2>&1 & + wait_pid=$! + wait_case_contract=0 + if wait_for_path "$hold_dir/ready" && + wait_for_path "$done_dir/${peer_done_one[$hold_index]}" && + wait_for_path "$done_dir/${peer_done_two[$hold_index]}" && + wait_for_child_wait "$wait_pid" "${expected_wait_children[$hold_index]}"; then + wait_case_contract=1 + fi + : > "$hold_dir/release" + wait "$wait_pid" || wait_rc=$? + if [ "$wait_case_contract" -ne 1 ] || [ "$wait_rc" -ne 0 ] || + ! wait_for_path "$done_dir/${target_done[$hold_index]}" || + ! cmp -s <(LC_ALL=C sort "$expected_executions") \ + <(LC_ALL=C sort "$wait_executions"); then + wait_contract=0 + fi +done +if [ "$wait_contract" -eq 1 ]; then + pass AGG-011 "parent waits every lane through controlled completion" +else + fail AGG-011 "parent waits every lane through controlled completion" +fi + +reset_fixtures +signal_dir="$work/signal" +signal_tmp="$work/signal-tmp" +mkdir "$signal_dir" "$signal_tmp" +AGENT_LAB_AGG_SIGNAL_DIR="$signal_dir" \ + TMPDIR="$signal_tmp" \ + python3 -I -B -c \ + 'import os, sys; os.setsid(); os.execvpe("bash", ["bash", sys.argv[1]], os.environ)' \ + "$replica_lifecycle" > "$work/signal.out" 2>&1 & +signal_pid=$! +signal_rc=0 +descendant_pid="" +if wait_for_path "$signal_dir/ready" && + IFS= read -r descendant_pid < "$signal_dir/descendant.pid" && + [[ "$descendant_pid" =~ ^[0-9]+$ ]] && + kill -0 "$descendant_pid" 2>/dev/null; then + kill -TERM "$signal_pid" 2>/dev/null || true +else + signal_rc=125 + kill -TERM "$signal_pid" 2>/dev/null || true +fi +observed_signal_rc=0 +wait "$signal_pid" || observed_signal_rc=$? +descendant_gone=0 +if [ -n "$descendant_pid" ] && wait_for_process_exit "$descendant_pid"; then + descendant_gone=1 +fi +if [ "$descendant_gone" -ne 1 ]; then + surviving_group="$(ps -o pgid= -p "$descendant_pid" 2>/dev/null || true)" + surviving_group="${surviving_group//[[:space:]]/}" + if [ "$surviving_group" = "$signal_pid" ]; then + kill -KILL -- "-$signal_pid" 2>/dev/null || true + wait_for_process_exit "$descendant_pid" || true + fi +fi +if [ "$signal_rc" -eq 0 ] && [ "$observed_signal_rc" -eq 143 ] && + [ "$descendant_gone" -eq 1 ]; then + pass AGG-012 "owned-session termination reaches cooperative descendants" +else + fail AGG-012 "owned-session termination reaches cooperative descendants" +fi + +reset_fixtures +stubborn_dir="$work/stubborn-signal" +stubborn_tmp="$work/stubborn-tmp" +mkdir "$stubborn_dir" "$stubborn_tmp" +AGENT_LAB_AGG_STUBBORN_SIGNAL_DIR="$stubborn_dir" \ + TMPDIR="$stubborn_tmp" \ + python3 -I -B -c \ + 'import os, sys; os.setsid(); os.execvpe("bash", ["bash", sys.argv[1]], os.environ)' \ + "$replica_lifecycle" > "$work/stubborn-signal.out" 2>&1 & +stubborn_leader=$! +stubborn_rc=0 +stubborn_pid="" +if wait_for_path "$stubborn_dir/ready" && + IFS= read -r stubborn_pid < "$stubborn_dir/descendant.pid" && + [[ "$stubborn_pid" =~ ^[0-9]+$ ]] && + kill -0 "$stubborn_pid" 2>/dev/null; then + kill -TERM "$stubborn_leader" 2>/dev/null || true +else + stubborn_rc=125 + kill -TERM "$stubborn_leader" 2>/dev/null || true +fi +observed_stubborn_rc=0 +wait "$stubborn_leader" || observed_stubborn_rc=$? +stubborn_alive=0 +stubborn_output_preserved=0 +if [ -n "$stubborn_pid" ] && kill -0 "$stubborn_pid" 2>/dev/null; then + stubborn_alive=1 + stubborn_output="$(readlink "/proc/$stubborn_pid/fd/1" 2>/dev/null || true)" + if [ -n "$stubborn_output" ] && [ -e "$stubborn_output" ]; then + stubborn_output_preserved=1 + fi +fi +stubborn_group="$(ps -o pgid= -p "$stubborn_pid" 2>/dev/null || true)" +stubborn_group="${stubborn_group//[[:space:]]/}" +if [ "$stubborn_group" = "$stubborn_leader" ]; then + kill -KILL -- "-$stubborn_leader" 2>/dev/null || true + wait_for_process_exit "$stubborn_pid" || true +fi +if [ "$stubborn_rc" -eq 0 ] && [ "$observed_stubborn_rc" -eq 143 ] && + [ "$stubborn_alive" -eq 1 ] && [ "$stubborn_output_preserved" -eq 1 ]; then + pass AGG-013 "signal exit preserves work owned by an uncooperative descendant" +else + fail AGG-013 "signal exit preserves work owned by an uncooperative descendant" +fi + +reset_source_fixtures +source_expected_executions="$work/source-expected-executions" +source_baseline_executions="$work/source-baseline-executions" +source_mutant_executions="$work/source-mutant-executions" +printf '%s\n' \ + zip-intake-cases.sh \ + zip-mutation-cases.py \ + git-intake-cases.sh \ + git-mutation-cases.py > "$source_expected_executions" +: > "$source_baseline_executions" +source_baseline_rc=0 +run_source_replica "$work/source-baseline.out" env \ + AGENT_LAB_AGG_EXEC_LOG="$source_baseline_executions" || source_baseline_rc=$? + +mutant_source_adapters="$replica/tests/experiment/source-adapter-hidden-duplicate.sh" +awk ' + { print } + $0 == "subcases=(" { in_subcases=1; next } + in_subcases && $0 == ")" { + print "\"$repo_root/tests/experiment/zip-intake-cases.sh\" >/dev/null 2>&1" + in_subcases=0 + } +' "$replica_source_adapters" > "$mutant_source_adapters" +chmod +x "$mutant_source_adapters" +source_mutation_count="$(grep -Fxc \ + '"$repo_root/tests/experiment/zip-intake-cases.sh" >/dev/null 2>&1' \ + "$mutant_source_adapters")" +: > "$source_mutant_executions" +source_mutant_rc=0 +run_selected "$work/source-mutant.out" "$mutant_source_adapters" env \ + AGENT_LAB_AGG_EXEC_LOG="$source_mutant_executions" || source_mutant_rc=$? +source_mutant_expected="$work/source-mutant-expected-executions" +printf '%s\n' \ + zip-intake-cases.sh \ + zip-intake-cases.sh \ + zip-mutation-cases.py \ + git-intake-cases.sh \ + git-mutation-cases.py > "$source_mutant_expected" +if [ "$source_baseline_rc" -eq 0 ] && + cmp -s "$source_expected_executions" "$source_baseline_executions" && + [ "$source_mutation_count" -eq 1 ] && [ "$source_mutant_rc" -eq 0 ] && + cmp -s "$work/source-baseline.out" "$work/source-mutant.out" && + cmp -s "$source_mutant_expected" "$source_mutant_executions"; then + pass AGG-021 "source-adapter execution ledger proves ordered exact-once routing" +else + fail AGG-021 "source-adapter execution ledger proves ordered exact-once routing" +fi + +source_success_expected="$work/source-success-expected" +{ + for id in "${source_expected_ids[@]}"; do + printf 'PASS %s fixture assertion\n' "$id" + done + printf 'SUMMARY assertions=83 expected=83 failures=0 infra=0\n' + printf 'EXPERIMENT SOURCE ADAPTERS PASS\n' +} > "$source_success_expected" +if [ "$source_baseline_rc" -eq 0 ] && + cmp -s "$source_success_expected" "$work/source-baseline.out"; then + pass AGG-022 "source-adapter success forwards exact assertions, summary, and final marker" +else + fail AGG-022 "source-adapter success forwards exact assertions, summary, and final marker" +fi + +reset_source_fixtures +source_missing_records=() +mapfile -t source_missing_records < <(pass_records \ + "${source_zip_ids[@]:0:${#source_zip_ids[@]}-1}") +write_fixture "$replica/tests/experiment/zip-intake-cases.sh" 0 \ + "${source_missing_records[@]}" +source_missing_rc=0 +run_source_replica "$work/source-missing.out" env || source_missing_rc=$? +if [ "$source_missing_rc" -eq 1 ] && + grep -Fxq 'SUMMARY assertions=82 expected=83 failures=1 infra=0' \ + "$work/source-missing.out" && + ! grep -Fxq 'EXPERIMENT SOURCE ADAPTERS PASS' "$work/source-missing.out"; then + pass AGG-023 "source-adapter missing assertion identity maps to one" +else + fail AGG-023 "source-adapter missing assertion identity maps to one" +fi + +reset_source_fixtures +source_duplicate_records=() +mapfile -t source_duplicate_records < <(pass_records \ + "${source_zip_ids[@]}" "${source_zip_ids[-1]}") +write_fixture "$replica/tests/experiment/zip-intake-cases.sh" 0 \ + "${source_duplicate_records[@]}" +source_duplicate_rc=0 +run_source_replica "$work/source-duplicate.out" env || source_duplicate_rc=$? +if [ "$source_duplicate_rc" -eq 1 ] && + grep -Fxq 'SUMMARY assertions=84 expected=83 failures=1 infra=0' \ + "$work/source-duplicate.out" && + ! grep -Fxq 'EXPERIMENT SOURCE ADAPTERS PASS' "$work/source-duplicate.out"; then + pass AGG-024 "source-adapter duplicate assertion identity maps to one" +else + fail AGG-024 "source-adapter duplicate assertion identity maps to one" +fi + +reset_source_fixtures +source_substituted_records=() +mapfile -t source_substituted_records < <(pass_records "${source_zip_ids[@]}") +source_substituted_records[-1]='PASS:BAD-001' +write_fixture "$replica/tests/experiment/zip-intake-cases.sh" 0 \ + "${source_substituted_records[@]}" +source_substituted_rc=0 +run_source_replica "$work/source-substituted.out" env || source_substituted_rc=$? +if [ "$source_substituted_rc" -eq 1 ] && + grep -Fxq 'SUMMARY assertions=83 expected=83 failures=1 infra=0' \ + "$work/source-substituted.out" && + ! grep -Fxq 'EXPERIMENT SOURCE ADAPTERS PASS' "$work/source-substituted.out"; then + pass AGG-025 "source-adapter substituted assertion identity maps to one" +else + fail AGG-025 "source-adapter substituted assertion identity maps to one" +fi + +reset_source_fixtures +source_failed_records=() +mapfile -t source_failed_records < <(pass_records "${source_zip_ids[@]}") +source_failed_records[0]='FAIL:ZIP-001' +write_fixture "$replica/tests/experiment/zip-intake-cases.sh" 1 \ + "${source_failed_records[@]}" +source_assertion_rc=0 +run_source_replica "$work/source-assertion.out" env || source_assertion_rc=$? +if [ "$source_assertion_rc" -eq 1 ] && + grep -Fxq 'SUMMARY assertions=83 expected=83 failures=1 infra=0' \ + "$work/source-assertion.out" && + ! grep -Fxq 'EXPERIMENT SOURCE ADAPTERS PASS' "$work/source-assertion.out"; then + pass AGG-026 "source-adapter subcase assertion failure maps to one" +else + fail AGG-026 "source-adapter subcase assertion failure maps to one" +fi + +reset_source_fixtures +source_uncertain_records=() +mapfile -t source_uncertain_records < <(pass_records "${source_zip_ids[@]}") +write_fixture "$replica/tests/experiment/zip-intake-cases.sh" 125 \ + "${source_uncertain_records[@]}" +source_subcase_infra_rc=0 +run_source_replica "$work/source-subcase-infra.out" env || source_subcase_infra_rc=$? +if [ "$source_subcase_infra_rc" -eq 125 ] && + grep -Fxq 'SUMMARY assertions=83 expected=83 failures=0 infra=1' \ + "$work/source-subcase-infra.out" && + ! grep -Fxq 'EXPERIMENT SOURCE ADAPTERS PASS' "$work/source-subcase-infra.out"; then + pass AGG-027 "source-adapter subcase uncertainty maps to one hundred twenty-five" +else + fail AGG-027 "source-adapter subcase uncertainty maps to one hundred twenty-five" +fi + +reset_source_fixtures +find "$replica/tests/experiment/zip-mutation-cases.py" -delete +source_setup_infra_rc=0 +run_source_replica "$work/source-setup-infra.out" env || source_setup_infra_rc=$? +if [ "$source_setup_infra_rc" -eq 125 ] && + grep -Fxq 'SUMMARY assertions=70 expected=83 failures=1 infra=1' \ + "$work/source-setup-infra.out" && + ! grep -Fxq 'EXPERIMENT SOURCE ADAPTERS PASS' "$work/source-setup-infra.out"; then + pass AGG-028 "source-adapter setup uncertainty maps to one hundred twenty-five" +else + fail AGG-028 "source-adapter setup uncertainty maps to one hundred twenty-five" +fi + +reset_source_fixtures +source_shim="$work/source-shim" +mkdir "$source_shim" +printf '#!/usr/bin/env bash\nexit 1\n' > "$source_shim/rmdir" +chmod +x "$source_shim/rmdir" +source_cleanup_rc=0 +run_source_replica "$work/source-cleanup.out" env \ + PATH="$source_shim:$PATH" || source_cleanup_rc=$? +if [ "$source_cleanup_rc" -eq 125 ] && + grep -Fxq 'SUMMARY assertions=83 expected=83 failures=0 infra=1' \ + "$work/source-cleanup.out" && + ! grep -Fxq 'EXPERIMENT SOURCE ADAPTERS PASS' "$work/source-cleanup.out"; then + pass AGG-029 "source-adapter cleanup uncertainty suppresses the final marker" +else + fail AGG-029 "source-adapter cleanup uncertainty suppresses the final marker" +fi + +reset_source_fixtures +source_summaryless="$replica/tests/experiment/zip-intake-cases.sh" +{ + printf '#!/usr/bin/env bash\nset -u\n' + for id in "${source_zip_ids[@]}"; do + printf "printf 'PASS %s fixture assertion\\n'\n" "$id" + done + printf 'exit 0\n' +} > "$source_summaryless" +chmod +x "$source_summaryless" +source_summaryless_rc=0 +run_source_replica "$work/source-summaryless.out" env || source_summaryless_rc=$? +if [ "$source_summaryless_rc" -eq 125 ] && + grep -Fxq 'SUMMARY assertions=83 expected=83 failures=0 infra=1' \ + "$work/source-summaryless.out" && + ! grep -Fxq 'EXPERIMENT SOURCE ADAPTERS PASS' "$work/source-summaryless.out"; then + pass AGG-030 "source-adapter missing subcase summary maps to one hundred twenty-five" +else + fail AGG-030 "source-adapter missing subcase summary maps to one hundred twenty-five" +fi + +cleanup_infrastructure=0 +if ! cleanup_work; then + cleanup_infrastructure=1 +fi +trap - EXIT + +printf 'SUMMARY assertions=%s expected=%s failures=%s infra=%s\n' \ + "$expected_count" "$expected_count" "$failures" "$cleanup_infrastructure" +if [ "$cleanup_infrastructure" -ne 0 ]; then + exit 125 +fi +[ "$failures" -eq 0 ] diff --git a/tests/experiment/authorization-cases.sh b/tests/experiment/authorization-cases.sh index 73439a7..e518e2d 100755 --- a/tests/experiment/authorization-cases.sh +++ b/tests/experiment/authorization-cases.sh @@ -2,622 +2,24 @@ set -euo pipefail repo_root="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." >/dev/null 2>&1 && pwd)" -experiment="$repo_root/scripts/experiment" -cedar_tool="$repo_root/scripts/dev/cedar-tool" -fixture_root="$repo_root/tests/experiment/fixtures" - +agent_lab="$repo_root/scripts/agent-lab" +fixture="$repo_root/tests/experiment/fixtures/directories/minimal" work="$(mktemp -d)" -cleanup() { - find "$work" -type f -delete 2>/dev/null || true - find "$work" -type l -delete 2>/dev/null || true - find "$work" -depth -type d -exec rmdir {} + 2>/dev/null || true -} -trap cleanup EXIT -mkdir -p "$work/bin" "$work/home" "$work/runtime-tmp" - -failures=0 -pass() { printf 'PASS %s\n' "$1"; } -fail() { printf 'FAIL %s\n' "$1"; failures=$((failures + 1)); } - -spy_log="$work/tool-spy.log" -: > "$spy_log" -for tool in docker docker-compose podman curl wget; do - spy="$work/bin/$tool" - { - printf '#!/usr/bin/env bash\n' - printf 'printf "%%s\\n" "$0 $*" >> %q\n' "$spy_log" - printf 'exit 97\n' - } > "$spy" - chmod +x "$spy" -done - -capture() { - local name="$1" - shift - CAPTURE_STDOUT="$work/$name.stdout" - CAPTURE_STDERR="$work/$name.stderr" - CAPTURE_RC=0 - env -i \ - PATH="$work/bin:/usr/bin:/bin" \ - HOME="$work/home" \ - TMPDIR="$work/runtime-tmp" \ - LC_ALL=C \ - AGENT_LAB_CUE_TOOL_DIR="${AGENT_LAB_CUE_TOOL_DIR:-$repo_root/.cache/dev/tools/cue}" \ - AGENT_LAB_CEDAR_TOOL_DIR="${AGENT_LAB_CEDAR_TOOL_DIR:-$repo_root/.cache/dev/tools/cedar}" \ - "$@" > "$CAPTURE_STDOUT" 2> "$CAPTURE_STDERR" || CAPTURE_RC=$? -} - -capture usage-authorize "$experiment" authorize -if [ "$CAPTURE_RC" -eq 2 ] && [ ! -s "$CAPTURE_STDOUT" ] && - grep -Fq 'Usage: scripts/experiment authorize install [--] MANIFEST' "$CAPTURE_STDERR"; then - pass "authorize requires one bounded install request" -else - fail "authorize usage is not exposed exactly (rc=$CAPTURE_RC)" -fi - -capture usage-action "$experiment" authorize start -- "$fixture_root/valid.json" -if [ "$CAPTURE_RC" -eq 2 ] && [ ! -s "$CAPTURE_STDOUT" ]; then - pass "runtime lifecycle actions are outside the requested-plan seam" -else - fail "authorize accepted a runtime lifecycle action (rc=$CAPTURE_RC)" -fi - -capture usage-override "$experiment" authorize install --principal attacker "$fixture_root/valid.json" -if [ "$CAPTURE_RC" -eq 2 ] && [ ! -s "$CAPTURE_STDOUT" ]; then - pass "the public command rejects caller-selected Cedar inputs" -else - fail "authorize accepted a caller-selected Cedar input (rc=$CAPTURE_RC)" -fi - -for required in \ - "$cedar_tool" \ - "$repo_root/scripts/dev/cedar-tool.py" \ - "$repo_root/tools/cedar.lock" \ - "$repo_root/authorization/experiment/v0alpha1/schema.cedarschema" \ - "$repo_root/authorization/experiment/v0alpha1/operator.cedar"; do - if [ ! -f "$required" ]; then - printf 'INFRA Experiment authorization input is missing: %s\n' "$required" >&2 - exit 125 - fi -done -if [ ! -x "$experiment" ] || [ ! -x "$cedar_tool" ]; then - printf 'INFRA Experiment authorization entrypoints are not executable\n' >&2 - exit 125 -fi - -cedar_preflight_rc=0 -cedar_preflight_out="$($cedar_tool --version 2>&1)" || cedar_preflight_rc=$? -if [ "$cedar_preflight_rc" -ne 0 ]; then - printf 'INFRA Experiment authorization requires provisioned pinned Cedar: %s\n' \ - "$cedar_preflight_out" >&2 - exit 125 -fi - -repo_before="$work/repo.before" -git -C "$repo_root" status --porcelain=v1 --untracked-files=all > "$repo_before" - -authorization_sha256() { - python3 - "$repo_root" <<'PY' -from hashlib import sha256 -from pathlib import Path -import sys - -root = Path(sys.argv[1]) -names = sorted(( - "authorization/experiment/v0alpha1/operator.cedar", - "authorization/experiment/v0alpha1/schema.cedarschema", - "tools/cedar.lock", -)) -digest = sha256(b"agent-lab.authorization-contract.v1\0") -for name in names: - encoded = name.encode("utf-8") - data = (root / name).read_bytes() - digest.update(len(encoded).to_bytes(4, "big")) - digest.update(encoded) - digest.update(len(data).to_bytes(8, "big")) - digest.update(data) -print(f"sha256:{digest.hexdigest()}") -PY -} - -capture checked "$experiment" check -- "$fixture_root/valid.json" -if [ "$CAPTURE_RC" -ne 0 ] || [ -s "$CAPTURE_STDERR" ]; then - printf 'INFRA baseline CUE contract failed before authorization tests\n' >&2 - exit 125 -fi -capture permitted "$experiment" authorize install -- "$fixture_root/valid.json" - -checked_plan_digest="$(jq -er '.digest' "$work/checked.stdout" 2>/dev/null || true)" -checked_contract_digest="$(jq -er '.plan.contract.digest' "$work/checked.stdout" 2>/dev/null || true)" -expected_authorization_digest="$(authorization_sha256)" -if [ "$CAPTURE_RC" -eq 0 ] && [ ! -s "$CAPTURE_STDERR" ] && - jq -e \ - --arg plan "$checked_plan_digest" \ - --arg contract "$checked_contract_digest" \ - --arg authorization "$expected_authorization_digest" ' - .apiVersion == "agent-lab.authorization/v0alpha1" and - .kind == "ExperimentAuthorizationDecision" and - .verdict == "permit" and - .action == "experiment.install" and - .principal == { - assurance: "none", - authenticated: false, - id: "legacy-local-operator", - source: "fixed-local-cli", - type: "AgentLab::Principal" - } and - .binding == { - authorizationDigest: $authorization, - contractDigest: $contract, - planDigest: $plan - } and - .resource == { - id: $plan, - requestedName: "first-experiment", - type: "AgentLab::RequestedExperimentPlan" - } - ' "$CAPTURE_STDOUT" >/dev/null 2>&1; then - pass "Cedar permits the fixed operator for the exact CUE plan" -else - fail "Cedar did not bind its permit to the exact CUE plan (rc=$CAPTURE_RC)" -fi - -if [ "$(wc -l < "$work/permitted.stdout" | tr -d ' ')" -eq 1 ] && - [ "$(jq -cS . "$work/permitted.stdout" 2>/dev/null || true)" = "$(cat "$work/permitted.stdout")" ] && - LC_ALL=C grep -Eq '^[ -~]+$' "$work/permitted.stdout"; then - pass "the authorization decision is one canonical review-safe ASCII line" -else - fail "the authorization decision is not canonical ASCII" -fi - -capture reordered "$experiment" authorize install -- "$fixture_root/valid-reordered.json" -if [ "$CAPTURE_RC" -eq 0 ] && [ ! -s "$CAPTURE_STDERR" ] && - cmp -s "$work/permitted.stdout" "$CAPTURE_STDOUT"; then - pass "equivalent manifests authorize the same requested plan" -else - fail "equivalent manifests changed authorization identity (rc=$CAPTURE_RC)" -fi - -different="$work/different.json" -jq '.spec.members[1].resourceClass = "standard"' "$fixture_root/valid.json" > "$different" -capture different "$experiment" authorize install -- "$different" -different_plan_digest="$(jq -er '.binding.planDigest' "$CAPTURE_STDOUT" 2>/dev/null || true)" -if [ "$CAPTURE_RC" -eq 0 ] && [ ! -s "$CAPTURE_STDERR" ] && - [ "$(jq -r '.resource.requestedName' "$CAPTURE_STDOUT" 2>/dev/null || true)" = first-experiment ] && - [ -n "$different_plan_digest" ] && [ "$different_plan_digest" != "$checked_plan_digest" ] && - [ "$(jq -r '.resource.id' "$CAPTURE_STDOUT" 2>/dev/null || true)" = "$different_plan_digest" ]; then - pass "same-name different intent receives a different digest resource" -else - fail "requested name became authorization identity (rc=$CAPTURE_RC)" -fi - -capture envelope "$experiment" authorize install -- "$work/checked.stdout" -if [ "$CAPTURE_RC" -eq 1 ] && [ ! -s "$CAPTURE_STDOUT" ] && - grep -Fq 'FAIL Experiment manifest' "$CAPTURE_STDERR"; then - pass "authorize revalidates manifest intent instead of accepting an envelope" -else - fail "authorize accepted or mishandled a caller-supplied plan envelope (rc=$CAPTURE_RC)" -fi - -copy_root="$work/authorization-copy" -mkdir -p \ - "$copy_root/scripts/dev" \ - "$copy_root/contracts/experiment/v0alpha1/cue.mod" \ - "$copy_root/authorization/experiment/v0alpha1" \ - "$copy_root/tests/experiment/fixtures" \ - "$copy_root/tools" -cp "$repo_root/scripts/experiment" "$repo_root/scripts/experiment.py" \ - "$copy_root/scripts/" -cp "$repo_root/scripts/dev/cue-tool" "$repo_root/scripts/dev/cue-tool.py" \ - "$repo_root/scripts/dev/cedar-tool.py" "$copy_root/scripts/dev/" -cp "$repo_root/contracts/experiment/v0alpha1/schema.cue" \ - "$repo_root/contracts/experiment/v0alpha1/plan.cue" \ - "$copy_root/contracts/experiment/v0alpha1/" -cp "$repo_root/contracts/experiment/v0alpha1/cue.mod/module.cue" \ - "$copy_root/contracts/experiment/v0alpha1/cue.mod/" -cp "$repo_root/authorization/experiment/v0alpha1/schema.cedarschema" \ - "$repo_root/authorization/experiment/v0alpha1/operator.cedar" \ - "$copy_root/authorization/experiment/v0alpha1/" -cp "$repo_root/tools/cue.lock" "$repo_root/tools/cedar.lock" "$copy_root/tools/" -cp "$fixture_root/valid.json" "$copy_root/tests/experiment/fixtures/" -chmod +x "$copy_root/scripts/experiment" "$copy_root/scripts/dev/cue-tool" - -printf '%s\n' \ - '' \ - '@id("emergency-stop-v0")' \ - 'forbid (principal, action, resource);' >> \ - "$copy_root/authorization/experiment/v0alpha1/operator.cedar" -capture public-deny env \ - AGENT_LAB_CUE_TOOL_DIR="${AGENT_LAB_CUE_TOOL_DIR:-$repo_root/.cache/dev/tools/cue}" \ - AGENT_LAB_CEDAR_TOOL_DIR="${AGENT_LAB_CEDAR_TOOL_DIR:-$repo_root/.cache/dev/tools/cedar}" \ - "$copy_root/scripts/experiment" authorize install -- \ - "$copy_root/tests/experiment/fixtures/valid.json" -if [ "$CAPTURE_RC" -eq 1 ] && [ ! -s "$CAPTURE_STDERR" ] && - [ "$(jq -r '.verdict' "$CAPTURE_STDOUT" 2>/dev/null || true)" = deny ] && - [ "$(jq -r '.binding.planDigest' "$CAPTURE_STDOUT" 2>/dev/null || true)" = "$checked_plan_digest" ] && - [ "$(jq -cS . "$CAPTURE_STDOUT" 2>/dev/null || true)" = "$(cat "$CAPTURE_STDOUT")" ]; then - pass "a strict repository forbid produces one canonical ordinary deny" -else - fail "ordinary Cedar deny was not translated canonically (rc=$CAPTURE_RC)" -fi - -printf '\nthis is not Cedar\n' >> \ - "$copy_root/authorization/experiment/v0alpha1/operator.cedar" -capture corrupt-policy env \ +trap 'find "$work" -type f -delete 2>/dev/null || true; find "$work" -depth -type d -exec rmdir {} + 2>/dev/null || true' EXIT +mkdir -p "$work/home" "$work/tmp" +rc=0 +env -i PATH=/usr/bin:/bin HOME="$work/home" TMPDIR="$work/tmp" LC_ALL=C \ AGENT_LAB_CUE_TOOL_DIR="${AGENT_LAB_CUE_TOOL_DIR:-$repo_root/.cache/dev/tools/cue}" \ AGENT_LAB_CEDAR_TOOL_DIR="${AGENT_LAB_CEDAR_TOOL_DIR:-$repo_root/.cache/dev/tools/cedar}" \ - "$copy_root/scripts/experiment" authorize install -- \ - "$copy_root/tests/experiment/fixtures/valid.json" -if [ "$CAPTURE_RC" -eq 125 ] && [ ! -s "$CAPTURE_STDOUT" ] && - grep -Fq 'INFRA Experiment strict Cedar policy validation was not exact' "$CAPTURE_STDERR"; then - pass "invalid trusted policy is infrastructure uncertainty, never a deny" -else - fail "invalid trusted policy was translated as a decision (rc=$CAPTURE_RC)" -fi - -if python3 - "$repo_root" "$fixture_root/valid.json" <<'PY' -from importlib.util import module_from_spec, spec_from_file_location -from pathlib import Path -import json -import os -import signal -import subprocess -import sys - -root = Path(sys.argv[1]) -spec = spec_from_file_location("experiment_authorization", root / "scripts/experiment.py") -assert spec is not None and spec.loader is not None -module = module_from_spec(spec) -spec.loader.exec_module(module) - -manifest = json.loads(Path(sys.argv[2]).read_text(encoding="utf-8")) -plan = module.cue_plan(manifest) -binding = module.plan_binding(plan) -request, entities = module.cedar_documents(binding) -authorization_digest, snapshot = module.authorization_snapshot(root) - -assert binding.plan_digest == request["context"]["planDigest"] -assert request["resource"] == ( - f'AgentLab::RequestedExperimentPlan::"{binding.plan_digest}"' -) -assert request["principal"] == ( - 'AgentLab::Principal::"legacy-local-operator"' -) - -observed_helpers = [] -observed_arguments = [] -real_invoke = module.invoke_cedar -def observing_invoke(helper, arguments, repository): - observed_helpers.append(helper) - observed_arguments.append(arguments) - assert str(helper).startswith("/tmp/agent-lab-authorization-") - assert helper.read_bytes() == snapshot[module.CEDAR_HELPER] - return real_invoke(helper, arguments, repository) -module.invoke_cedar = observing_invoke -assert module.evaluate_cedar(snapshot, request, entities, root) == "permit" -assert len(observed_helpers) == 2 -assert observed_arguments[0][2] == "validate" -assert "--deny-warnings" in observed_arguments[0] -assert observed_arguments[0][observed_arguments[0].index("--validation-mode") + 1] == "strict" -assert observed_arguments[1][2] == "authorize" -assert observed_arguments[1][observed_arguments[1].index("--request-validation") + 1] == "true" -module.invoke_cedar = real_invoke - -alternate_request = dict(request) -alternate_request["principal"] = 'AgentLab::Principal::"intruder"' -alternate_entities = json.loads(json.dumps(entities)) -alternate_entities.append({ - "uid": {"type": "AgentLab::Principal", "id": "intruder"}, - "attrs": {"authenticated": False, "assurance": "none", "source": "fixed-local-cli"}, - "parents": [], -}) -assert module.evaluate_cedar(snapshot, alternate_request, alternate_entities, root) == "deny" - -mismatch_request = json.loads(json.dumps(request)) -mismatch_request["context"]["planDigest"] = "sha256:" + ("f" * 64) -assert module.evaluate_cedar(snapshot, mismatch_request, entities, root) == "deny" - -contract_request = json.loads(json.dumps(request)) -contract_request["context"]["contractDigest"] = "sha256:" + ("e" * 64) -assert module.evaluate_cedar(snapshot, contract_request, entities, root) == "deny" - -version_request = json.loads(json.dumps(request)) -version_request["context"]["bindingVersion"] = "v0alpha2" -assert module.evaluate_cedar(snapshot, version_request, entities, root) == "deny" - -empty = dict(snapshot) -empty["authorization/experiment/v0alpha1/operator.cedar"] = b"" -assert module.evaluate_cedar(empty, request, entities, root) == "deny" - -forbidden = dict(snapshot) -forbidden["authorization/experiment/v0alpha1/operator.cedar"] += b'''\n@id("emergency-stop-v0") -forbid (principal, action, resource);\n''' -assert module.evaluate_cedar(forbidden, request, entities, root) == "deny" - -completed = subprocess.CompletedProcess([], 0, b"\nALLOW\n", b"") -assert module.parse_cedar_authorization(completed) == "permit" -completed = subprocess.CompletedProcess([], 2, b"\nDENY\n", b"") -assert module.parse_cedar_authorization(completed) == "deny" -for outcome in ( - subprocess.CompletedProcess([], 0, b"ALLOW\n", b""), - subprocess.CompletedProcess([], 0, b"\nALLOW\nextra", b""), - subprocess.CompletedProcess([], 0, b"\nALLOW\n", b"warning"), - subprocess.CompletedProcess([], 1, b"\nDENY\n", b""), - subprocess.CompletedProcess([], 2, b"\nALLOW\n", b""), - subprocess.CompletedProcess([], -9, b"", b""), -): - try: - module.parse_cedar_authorization(outcome) - except module.InfrastructureError: - pass - else: - raise AssertionError(f"ambiguous evaluator tuple accepted: {outcome}") - -# The production runner bounds bytes while the evaluator is live and kills timeouts. -from tempfile import TemporaryDirectory -import time -control_signals = (signal.SIGHUP, signal.SIGINT, signal.SIGQUIT, signal.SIGTERM) -original_dispositions = {signum: signal.getsignal(signum) for signum in control_signals} -original_signal_mask = set(signal.pthread_sigmask(signal.SIG_BLOCK, set())) -with TemporaryDirectory(prefix="cedar-runner-cases-", dir="/tmp") as directory: - fake = Path(directory) / "fake.py" - original_limit = module.MAX_CEDAR_OUTPUT_BYTES - original_timeout = module.CEDAR_TIMEOUT_SECONDS - module.MAX_CEDAR_OUTPUT_BYTES = 1024 - fake.write_text( - "import sys, time\n" - "sys.stdout.buffer.write(b'x' * 2048)\n" - "sys.stdout.buffer.flush()\n" - "time.sleep(5)\n", - encoding="utf-8", - ) - started = time.monotonic() - try: - module.invoke_cedar(fake, (), root) - except module.InfrastructureError as error: - assert str(error) == "pinned Cedar emitted overlong output" - else: - raise AssertionError("overlong live evaluator output was accepted") - assert time.monotonic() - started < 2 - - module.MAX_CEDAR_OUTPUT_BYTES = original_limit - module.CEDAR_TIMEOUT_SECONDS = 0.1 - fake.write_text("import time\ntime.sleep(5)\n", encoding="utf-8") - started = time.monotonic() - try: - module.invoke_cedar(fake, (), root) - except module.InfrastructureError as error: - assert str(error) == "pinned Cedar evaluation timed out" - else: - raise AssertionError("evaluator timeout was accepted") - assert time.monotonic() - started < 2 - module.MAX_CEDAR_OUTPUT_BYTES = original_limit - module.CEDAR_TIMEOUT_SECONDS = original_timeout - - marker = Path(directory) / "cancelled.pid" - fake.write_text( - "from pathlib import Path\n" - "import os, sys, time\n" - "Path(sys.argv[1]).write_text(str(os.getpid()), encoding='ascii')\n" - "time.sleep(30)\n", - encoding="utf-8", - ) - driver = Path(directory) / "driver.py" - driver.write_text( - "from importlib.util import module_from_spec, spec_from_file_location\n" - "from pathlib import Path\n" - "import sys\n" - "spec = spec_from_file_location('cancel_contract', sys.argv[1])\n" - "assert spec is not None and spec.loader is not None\n" - "module = module_from_spec(spec)\n" - "spec.loader.exec_module(module)\n" - "module.invoke_cedar(Path(sys.argv[2]), (sys.argv[3],), Path(sys.argv[4]))\n", - encoding="utf-8", - ) - parent = subprocess.Popen( - [sys.executable, str(driver), str(root / "scripts/experiment.py"), - str(fake), str(marker), str(root)], - ) - deadline = time.monotonic() + 2 - while not marker.exists() and parent.poll() is None and time.monotonic() < deadline: - time.sleep(0.01) - assert marker.exists() - evaluator_pid = int(marker.read_text(encoding="ascii")) - parent.terminate() - assert parent.wait(timeout=2) == 143 - deadline = time.monotonic() + 2 - while time.monotonic() < deadline: - try: - os.kill(evaluator_pid, 0) - except ProcessLookupError: - break - time.sleep(0.01) - else: - raise AssertionError("cancelled Cedar evaluator survived its parent") - - signal_race_failures = [] - spawn_race_pid = Path(directory) / "spawn-race.pid" - race_driver = Path(directory) / "spawn-race-driver.py" - race_driver.write_text( - "from importlib.util import module_from_spec, spec_from_file_location\n" - "from pathlib import Path\n" - "import os, signal, sys\n" - "spec = spec_from_file_location('spawn_race_contract', sys.argv[1])\n" - "assert spec is not None and spec.loader is not None\n" - "module = module_from_spec(spec)\n" - "spec.loader.exec_module(module)\n" - "real_popen = module.subprocess.Popen\n" - "def signal_after_spawn(*args, **kwargs):\n" - " child = real_popen(*args, **kwargs)\n" - " Path(sys.argv[3]).write_text(str(child.pid), encoding='ascii')\n" - " os.kill(os.getpid(), signal.SIGTERM)\n" - " return child\n" - "module.subprocess.Popen = signal_after_spawn\n" - "module.invoke_cedar(Path(sys.argv[2]), ('unused',), Path(sys.argv[4]))\n", - encoding="utf-8", - ) - raced = subprocess.Popen( - [sys.executable, str(race_driver), str(root / "scripts/experiment.py"), - str(fake), str(spawn_race_pid), str(root)], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - ) - raced_stdout, raced_stderr = raced.communicate(timeout=2) - assert spawn_race_pid.exists() - raced_child = int(spawn_race_pid.read_text(encoding="ascii")) - try: - os.kill(raced_child, 0) - except ProcessLookupError: - spawn_race_leaked = False - else: - spawn_race_leaked = True - try: - os.killpg(raced_child, signal.SIGKILL) - except ProcessLookupError: - pass - if raced.returncode != 143: - signal_race_failures.append(f"spawn-race rc={raced.returncode}") - if raced_stdout != b"" or raced_stderr != b"": - signal_race_failures.append("spawn-race emitted output") - if spawn_race_leaked: - signal_race_failures.append("signal between Popen and assignment leaked Cedar") - - cleanup_marker = Path(directory) / "cleanup-signal.pid" - cleanup_driver = Path(directory) / "cleanup-signal-driver.py" - cleanup_driver.write_text( - "from importlib.util import module_from_spec, spec_from_file_location\n" - "from pathlib import Path\n" - "import os, signal, sys\n" - "spec = spec_from_file_location('cleanup_signal_contract', sys.argv[1])\n" - "assert spec is not None and spec.loader is not None\n" - "module = module_from_spec(spec)\n" - "spec.loader.exec_module(module)\n" - "real_terminate = module.terminate_cedar_group\n" - "def signal_during_cleanup(process):\n" - " os.kill(os.getpid(), signal.SIGTERM)\n" - " real_terminate(process)\n" - "module.terminate_cedar_group = signal_during_cleanup\n" - "module.invoke_cedar(Path(sys.argv[2]), (sys.argv[3],), Path(sys.argv[4]))\n", - encoding="utf-8", - ) - cleaning = subprocess.Popen( - [sys.executable, str(cleanup_driver), str(root / "scripts/experiment.py"), - str(fake), str(cleanup_marker), str(root)], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - ) - deadline = time.monotonic() + 2 - while not cleanup_marker.exists() and cleaning.poll() is None and time.monotonic() < deadline: - time.sleep(0.01) - assert cleanup_marker.exists() - cleanup_child = int(cleanup_marker.read_text(encoding="ascii")) - os.kill(cleaning.pid, signal.SIGTERM) - cleanup_stdout, cleanup_stderr = cleaning.communicate(timeout=2) - try: - os.kill(cleanup_child, 0) - except ProcessLookupError: - cleanup_child_leaked = False - else: - cleanup_child_leaked = True - try: - os.killpg(cleanup_child, signal.SIGKILL) - except ProcessLookupError: - pass - if cleaning.returncode != 143: - signal_race_failures.append(f"cleanup-signal rc={cleaning.returncode}") - if cleanup_stdout != b"" or cleanup_stderr != b"": - signal_race_failures.append("cleanup-signal emitted output or traceback") - if cleanup_child_leaked: - signal_race_failures.append("second handled signal interrupted Cedar cleanup") - assert not signal_race_failures, "; ".join(signal_race_failures) - - residual_marker = Path(directory) / "residual.pid" - fake.write_text( - "from pathlib import Path\n" - "import subprocess, sys\n" - "child = subprocess.Popen([sys.executable, '-c', " - "'import time; time.sleep(30)'])\n" - "Path(sys.argv[1]).write_text(str(child.pid), encoding='ascii')\n", - encoding="utf-8", - ) - try: - module.invoke_cedar(fake, (str(residual_marker),), root) - except module.InfrastructureError as error: - assert str(error) == "pinned Cedar left a residual process group" - else: - raise AssertionError("a residual Cedar process group was accepted") - residual_pid = int(residual_marker.read_text(encoding="ascii")) - try: - os.kill(residual_pid, 0) - except ProcessLookupError: - pass - else: - raise AssertionError("residual Cedar child survived cleanup") - -assert {signum: signal.getsignal(signum) for signum in control_signals} == original_dispositions -assert set(signal.pthread_sigmask(signal.SIG_BLOCK, set())) == original_signal_mask - -# Main dispatch reads manifest bytes exactly once and supplies no caller input to Cedar. -reads = [] -captured = [] -real_read = module.read_manifest_once -real_cue = module.cue_plan -real_authorize = module.authorize_plan -real_write = module.write_decision -module.read_manifest_once = lambda path: reads.append(path) or b'{}' -module.cue_plan = lambda value: plan -module.authorize_plan = lambda value: captured.append(value) or ({"verdict": "permit"}, 0) -module.write_decision = lambda value: None -assert module.main(["experiment.py", "authorize", "install", "/manifest.json"]) == 0 -assert reads == ["/manifest.json"] -assert captured == [plan] -module.read_manifest_once = real_read -module.cue_plan = real_cue -module.authorize_plan = real_authorize -module.write_decision = real_write - -assert authorization_digest.startswith("sha256:") -PY -then - pass "policy, parser, timeout, cancellation, and residual-process cases fail closed" -else - fail "low-level Cedar authorization invariants failed" -fi - -if [ ! -s "$spy_log" ]; then - pass "authorization invokes no engine or network command from caller PATH" -else - fail "authorization invoked a forbidden caller PATH tool: $(tr '\n' ' ' < "$spy_log")" -fi -if ! grep -Eiq 'docker|podman|docker-compose|urllib|requests|socket' \ - "$repo_root/scripts/experiment" "$repo_root/scripts/experiment.py"; then - pass "the authorization adapter contains no engine or network client surface" -else - fail "the authorization adapter gained engine or network client code" -fi -if [ -z "$(find "$work/home" "$work/runtime-tmp" -mindepth 1 -print -quit)" ]; then - pass "authorization leaves no persistent home or runtime state" -else - fail "authorization left persistent state" -fi -git -C "$repo_root" status --porcelain=v1 --untracked-files=all > "$work/repo.after" -if cmp -s "$repo_before" "$work/repo.after"; then - pass "authorization leaves the checkout tree unchanged" -else - fail "authorization changed the checkout tree" -fi - -if printf '%s\n' "$cedar_preflight_out" | grep -Fxq 'cedar-policy-cli 4.12.0'; then - pass "Experiment authorization uses the pinned Cedar release" -else - fail "pinned Cedar release is unavailable or wrong" -fi -if "$cedar_tool" format --check --policies \ - "$repo_root/authorization/experiment/v0alpha1/operator.cedar" >/dev/null; then - pass "the tracked Cedar policy is canonically formatted" -else - fail "the tracked Cedar policy needs cedar format" -fi - -printf 'SUMMARY failures=%s\n' "$failures" -[ "$failures" -eq 0 ] + "$agent_lab" experiment authorize install "$fixture" > "$work/out" 2> "$work/err" || rc=$? +if [ "$rc" -eq 0 ] && [ ! -s "$work/err" ] && + jq -e '.verdict == "permit" and (.binding.sourceDigest | startswith("sha256:")) and (.binding.planDigest | startswith("sha256:"))' "$work/out" >/dev/null 2>&1; then + printf 'PASS AUTH-001 fresh preview binds source and plan\n' + failures=0 +else + printf 'FAIL AUTH-001 fresh preview binds source and plan\n' + failures=1 +fi +printf 'SUMMARY assertions=1 expected=1 failures=%s infra=0\n' "$failures" +[ "$failures" -eq 0 ] || exit 1 +printf 'EXPERIMENT AUTHORIZATION PASS\n' diff --git a/tests/experiment/catalog-aggregate-harness-cases.sh b/tests/experiment/catalog-aggregate-harness-cases.sh new file mode 100755 index 0000000..444276c --- /dev/null +++ b/tests/experiment/catalog-aggregate-harness-cases.sh @@ -0,0 +1,579 @@ +#!/usr/bin/env bash +set -u -o pipefail + +repo_root="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." >/dev/null 2>&1 && pwd)" +catalog_aggregate="$repo_root/tests/experiment/local-image-catalog-cases.sh" +expected_count=7 +work="" + +cleanup_work() { + local failed=0 + if [ -n "$work" ] && [ -e "$work" ]; then + find "$work" -type f -delete 2>/dev/null || failed=1 + find "$work" -type l -delete 2>/dev/null || failed=1 + find "$work" -depth -type d -exec rmdir {} + 2>/dev/null || failed=1 + [ ! -e "$work" ] || failed=1 + fi + return "$failed" +} + +if ! work="$(mktemp -d)"; then + printf 'SUMMARY assertions=0 expected=%s failures=0 infra=1\n' "$expected_count" + exit 125 +fi +trap 'cleanup_work >/dev/null 2>&1 || true' EXIT + +replica="$work/repo" +replica_aggregate="$replica/tests/experiment/local-image-catalog-cases.sh" +mkdir -p "$replica/tests/experiment" "$replica/tests/image" +cp "$catalog_aggregate" "$replica_aggregate" +chmod +x "$replica_aggregate" + +failures=0 +pass() { printf 'PASS %s %s\n' "$1" "$2"; } +fail() { printf 'FAIL %s %s\n' "$1" "$2"; failures=$((failures + 1)); } + +catalog_ids=( + CAT-NAME-001 CAT-NAME-002 CAT-OCI-001 CAT-OCI-002 + CAT-ADD-001 CAT-ADD-002 CAT-ADD-003 CAT-ADD-004 CAT-NS-001 + CAT-CAS-001 CAT-CAS-002 CAT-CAS-003 CAT-CAS-004 + CAT-READ-001 CAT-READ-002 CAT-NOEF-001 CAT-CONC-001 CAT-CONC-002 + CAT-STATE-001 CAT-STATE-002 CAT-STATE-003 CAT-STATE-004 + CAT-STATE-005 CAT-STATE-006 CAT-STATE-007 CAT-STATE-008 + CAT-STATE-009 CAT-STATE-010 CAT-STATE-011 CAT-STATE-012 + CAT-STATE-013 CAT-STATE-014 CAT-STATE-015 CAT-STATE-016 CAT-STATE-018 CAT-STATE-017 + CAT-BOUND-001 CAT-BOUND-002 CAT-BOUND-003 CAT-BOUND-004 + CAT-CRASH-001 CAT-CRASH-002 CAT-CRASH-003 CAT-CRASH-004 CAT-CRASH-005 + CAT-CRASH-006 CAT-CRASH-007 CAT-CRASH-008 CAT-CRASH-009 CAT-CRASH-010 + CAT-CRASH-011 CAT-PLAT-001 + RES-ENTRY-001 RES-SNAP-001 RES-SNAP-002 RES-ENTRY-002 RES-ENTRY-003 + RES-ISOLATE-001 RES-STATE-001 RES-STATE-002 RES-STATE-003 RES-INPUT-001 + RES-SNAP-003 RES-AUTH-001 RES-INSTALL-001 RES-NOEF-001 + M-CAT-OCI-001 M-CAT-SHADOW-001 M-CAT-CAS-001 M-CAT-AUTH-001 M-RES-BIND-001 + M-CAT-NOEF-001 M-CAT-ADMIT-001 M-CAT-ATOM-001 M-CAT-DUR-001 M-CAT-STAGE-001 +) +base_ids=("${catalog_ids[@]:0:18}") +state_ids=("${catalog_ids[@]:18:34}") +resolution_ids=("${catalog_ids[@]:52:14}") +mutation_ids=("${catalog_ids[@]:66:10}") + +write_fixture() { + local path="$1" + local rc="$2" + local role="$3" + shift 3 + local id record_failures=0 + local execution_id="${path##*/}" + { + printf '#!/usr/bin/env bash\nset -u\n' + printf 'control="${AGENT_LAB_CATALOG_AGG_CONTROL:?}"\n' + printf "printf '%%s\\n' '%s' >> \"\$control/executions\" || exit 125\n" "$execution_id" + if [ -n "$role" ]; then + printf "touch \"\$control/%s.ready\" || exit 125\n" "$role" + printf 'attempts=0\n' + printf "while [ ! -f \"\$control/%s.ready\" ]; do\n" \ + "$([ "$role" = state ] && printf peer || printf state)" + printf ' attempts=$((attempts + 1))\n' + printf ' [ "$attempts" -lt 300 ] || exit 125\n' + printf ' sleep 0.01\n' + printf 'done\n' + fi + if [ "$execution_id" = catalog-cases.sh ]; then + printf 'if [ "${AGENT_LAB_CATALOG_AGG_SIGNAL_MODE:-}" = cooperative ]; then\n' + printf ' printf "%%s\\n" "$PPID" > "$control/bash-lane.pid" || exit 125\n' + printf ' sleep 30 &\n' + printf ' descendant_pid=$!\n' + printf ' printf "%%s\\n" "$descendant_pid" > "$control/bash-descendant.pid" || exit 125\n' + printf ' touch "$control/bash-signal.ready" || exit 125\n' + printf ' wait "$descendant_pid"\n' + printf 'fi\n' + printf 'if [ "${AGENT_LAB_CATALOG_AGG_SIGNAL_MODE:-}" = stubborn ]; then\n' + printf ' printf "%%s\\n" "$PPID" > "$control/bash-lane.pid" || exit 125\n' + printf ' (\n' + printf " trap '' HUP INT QUIT TERM\n" + printf ' printf "%%s\\n" "$BASHPID" > "$control/stubborn-descendant.pid" || exit 125\n' + printf ' touch "$control/stubborn-signal.ready" || exit 125\n' + printf ' sleep 30\n' + printf ' ) &\n' + printf ' descendant_pid=$!\n' + printf ' wait "$descendant_pid"\n' + printf 'fi\n' + fi + printf 'if [ "${AGENT_LAB_CATALOG_AGG_HOLD:-}" = "%s" ]; then\n' "$execution_id" + printf ' touch "$control/hold.ready" || exit 125\n' + printf ' attempts=0\n' + printf ' while [ ! -f "$control/hold.release" ]; do\n' + printf ' attempts=$((attempts + 1))\n' + printf ' [ "$attempts" -lt 800 ] || exit 125\n' + printf ' sleep 0.01\n' + printf ' done\n' + printf 'fi\n' + for id in "$@"; do + if [ "$id" = "FAIL:CAT-NAME-001" ]; then + printf "printf 'FAIL CAT-NAME-001 fixture assertion\\n'\n" + record_failures=$((record_failures + 1)) + else + printf "printf 'PASS %s fixture assertion\\n'\n" "$id" + fi + done + printf "printf 'SUMMARY assertions=%s expected=%s failures=%s infra=0\\n'\n" \ + "$#" "$#" "$record_failures" + printf "touch \"\$control/%s.done\" || exit 125\n" "$execution_id" + printf 'exit %s\n' "$rc" + } > "$path" + chmod +x "$path" +} + +write_python_fixture() { + local path="$1" + local rc="$2" + local role="$3" + shift 3 + local id + local execution_id="${path##*/}" + { + printf '#!/usr/bin/env python3\n' + printf 'import os\n' + printf 'from pathlib import Path\n' + printf 'import subprocess\n' + printf 'import sys\n' + printf 'import threading\n' + printf 'import time\n' + printf 'control = Path(os.environ["AGENT_LAB_CATALOG_AGG_CONTROL"])\n' + printf 'with (control / "executions").open("a", encoding="ascii") as stream:\n' + printf ' stream.write("%s\\n")\n' "$execution_id" + if [ "$role" = state ]; then + printf '(control / "state.ready").touch()\n' + printf 'deadline = time.monotonic() + 3.0\n' + printf 'while not (control / "peer.ready").is_file():\n' + printf ' if time.monotonic() >= deadline:\n' + printf ' raise SystemExit(125)\n' + printf ' time.sleep(0.01)\n' + printf 'if os.environ.get("AGENT_LAB_CATALOG_AGG_SIGNAL_MODE") == "cooperative":\n' + printf ' (control / "python-lane.pid").write_text(str(os.getppid()) + "\\n", encoding="ascii")\n' + printf ' def run_descendant():\n' + printf ' descendant = subprocess.Popen([sys.executable, "-c", "import time; time.sleep(30)"])\n' + printf ' (control / "python-descendant.pid").write_text(str(descendant.pid) + "\\n", encoding="ascii")\n' + printf ' (control / "python-signal.ready").touch()\n' + printf ' descendant.wait()\n' + printf ' descendant_thread = threading.Thread(target=run_descendant)\n' + printf ' descendant_thread.start()\n' + printf ' descendant_thread.join()\n' + fi + printf 'if os.environ.get("AGENT_LAB_CATALOG_AGG_HOLD") == "%s":\n' "$execution_id" + printf ' (control / "hold.ready").touch()\n' + printf ' deadline = time.monotonic() + 8.0\n' + printf ' while not (control / "hold.release").is_file():\n' + printf ' if time.monotonic() >= deadline:\n' + printf ' raise SystemExit(125)\n' + printf ' time.sleep(0.01)\n' + for id in "$@"; do + printf "print('PASS %s fixture assertion')\n" "$id" + done + printf "print('SUMMARY assertions=%s expected=%s failures=0 infra=0')\n" "$#" "$#" + printf '(control / "%s.done").touch()\n' "$execution_id" + printf 'raise SystemExit(%s)\n' "$rc" + } > "$path" + chmod +x "$path" +} + +reset_fixtures() { + local base_rc="${1:-0}" + local state_rc="${2:-0}" + local first_record="${3:-CAT-NAME-001}" + local selected_base_ids=("${base_ids[@]}") + selected_base_ids[0]="$first_record" + write_fixture "$replica/tests/image/catalog-cases.sh" "$base_rc" peer \ + "${selected_base_ids[@]}" + write_python_fixture "$replica/tests/image/catalog-state-cases.py" "$state_rc" state \ + "${state_ids[@]}" + write_fixture "$replica/tests/experiment/catalog-resolution-cases.sh" 0 "" \ + "${resolution_ids[@]}" + write_python_fixture "$replica/tests/image/catalog-mutation-cases.py" 0 "" \ + "${mutation_ids[@]}" +} + +expected_executions="$work/expected-executions" +printf '%s\n' \ + catalog-cases.sh \ + catalog-state-cases.py \ + catalog-resolution-cases.sh \ + catalog-mutation-cases.py > "$expected_executions" + +expected_success="$work/expected-success" +for id in "${catalog_ids[@]}"; do + printf 'PASS %s fixture assertion\n' "$id" +done > "$expected_success" +printf 'SUMMARY assertions=76 expected=76 failures=0 infra=0\n' >> "$expected_success" +printf 'EXPERIMENT LOCAL IMAGE CATALOG PASS\n' >> "$expected_success" + +run_aggregate() { + local output="$1" + local control="$2" + shift 2 + : > "$control/executions" + env AGENT_LAB_CATALOG_AGG_CONTROL="$control" "$@" \ + bash "$replica_aggregate" > "$output" 2>&1 +} + +wait_for_path() { + local path="$1" + local attempts=0 + while [ ! -e "$path" ]; do + attempts=$((attempts + 1)) + [ "$attempts" -lt 500 ] || return 1 + sleep 0.01 + done +} + +wait_for_child_wait() { + local pid="$1" + local attempts=0 + local wchan child_list + local children=() + while [ "$attempts" -lt 200 ]; do + wchan="$(cat "/proc/$pid/wchan" 2>/dev/null || true)" + child_list="$(cat "/proc/$pid/task/$pid/children" 2>/dev/null || true)" + children=() + read -r -a children <<< "$child_list" + if [ "$wchan" = do_wait ] && [ "${#children[@]}" -eq 1 ]; then + return 0 + fi + attempts=$((attempts + 1)) + sleep 0.01 + done + return 1 +} + +wait_for_process_exit() { + local pid="$1" + local attempts=0 + while kill -0 "$pid" 2>/dev/null; do + attempts=$((attempts + 1)) + [ "$attempts" -lt 200 ] || return 1 + sleep 0.01 + done +} + +reset_fixtures +baseline_control="$work/baseline-control" +mkdir "$baseline_control" +baseline_rc=0 +run_aggregate "$work/baseline.out" "$baseline_control" || baseline_rc=$? +if [ "$baseline_rc" -eq 0 ] && + cmp -s "$expected_success" "$work/baseline.out" && + cmp -s <(LC_ALL=C sort "$expected_executions") \ + <(LC_ALL=C sort "$baseline_control/executions") && + [ -f "$baseline_control/peer.ready" ] && + [ -f "$baseline_control/state.ready" ]; then + pass AGG-014 "catalog subcases overlap with exact-once ordered replay" +else + fail AGG-014 "catalog subcases overlap with exact-once ordered replay" +fi + +lane_assignment_count="$(grep -Ec '^[[:space:]]*(readonly[[:space:]]+)?lane_count=' \ + "$replica_aggregate" || true)" +if [ "$lane_assignment_count" -eq 1 ] && + [ "$(grep -Fxc 'readonly lane_count=2' "$replica_aggregate")" -eq 1 ] && + [ "$(grep -Fxc 'for ((lane = 0; lane < lane_count; lane++)); do' \ + "$replica_aggregate")" -eq 1 ] && + [ "$(grep -Fxc ' run_lane "$lane" &' "$replica_aggregate")" -eq 1 ]; then + pass AGG-015 "catalog scheduler declares one immutable two-lane bound" +else + fail AGG-015 "catalog scheduler declares one immutable two-lane bound" +fi + +status_contract=1 +reset_fixtures 1 0 FAIL:CAT-NAME-001 +assertion_control="$work/assertion-control" +mkdir "$assertion_control" +assertion_rc=0 +run_aggregate "$work/assertion.out" "$assertion_control" || assertion_rc=$? +if [ "$assertion_rc" -ne 1 ] || + ! grep -Fxq 'SUMMARY assertions=76 expected=76 failures=1 infra=0' \ + "$work/assertion.out" || + grep -Fxq 'EXPERIMENT LOCAL IMAGE CATALOG PASS' "$work/assertion.out" || + ! cmp -s <(LC_ALL=C sort "$expected_executions") \ + <(LC_ALL=C sort "$assertion_control/executions") || + [ "$(find "$assertion_control" -name '*.done' -type f | wc -l)" -ne 4 ]; then + status_contract=0 +fi + +reset_fixtures 0 125 +infra_control="$work/infra-control" +mkdir "$infra_control" +infra_rc=0 +run_aggregate "$work/infra.out" "$infra_control" || infra_rc=$? +if [ "$infra_rc" -ne 125 ] || + ! grep -Fxq 'SUMMARY assertions=76 expected=76 failures=0 infra=1' \ + "$work/infra.out" || + grep -Fxq 'EXPERIMENT LOCAL IMAGE CATALOG PASS' "$work/infra.out" || + ! cmp -s <(LC_ALL=C sort "$expected_executions") \ + <(LC_ALL=C sort "$infra_control/executions") || + [ "$(find "$infra_control" -name '*.done' -type f | wc -l)" -ne 4 ]; then + status_contract=0 +fi + +reset_fixtures 1 125 FAIL:CAT-NAME-001 +mixed_control="$work/mixed-control" +mkdir "$mixed_control" +mixed_rc=0 +run_aggregate "$work/mixed.out" "$mixed_control" || mixed_rc=$? +mixed_assertions="$(grep -Ec '^(PASS|FAIL) [A-Z0-9-]+ ' "$work/mixed.out" || true)" +if [ "$status_contract" -eq 1 ] && [ "$mixed_rc" -eq 125 ] && + [ "$mixed_assertions" -eq 76 ] && + grep -Fxq 'SUMMARY assertions=76 expected=76 failures=1 infra=1' "$work/mixed.out" && + ! grep -Fxq 'EXPERIMENT LOCAL IMAGE CATALOG PASS' "$work/mixed.out" && + cmp -s <(LC_ALL=C sort "$expected_executions") \ + <(LC_ALL=C sort "$mixed_control/executions") && + [ "$(find "$mixed_control" -name '*.done' -type f | wc -l)" -eq 4 ]; then + pass AGG-016 "catalog uncertainty dominates after every subcase completes" +else + fail AGG-016 "catalog uncertainty dominates after every subcase completes" +fi + +wait_contract=1 +for hold_index in 0 1; do + reset_fixtures + wait_control="$work/wait-control-$hold_index" + mkdir "$wait_control" + : > "$wait_control/executions" + if [ "$hold_index" -eq 0 ]; then + hold_id=catalog-state-cases.py + peer_done=( + catalog-cases.sh.done + catalog-resolution-cases.sh.done + catalog-mutation-cases.py.done + ) + else + hold_id=catalog-mutation-cases.py + peer_done=( + catalog-state-cases.py.done + catalog-cases.sh.done + catalog-resolution-cases.sh.done + ) + fi + AGENT_LAB_CATALOG_AGG_CONTROL="$wait_control" \ + AGENT_LAB_CATALOG_AGG_HOLD="$hold_id" \ + bash "$replica_aggregate" > "$work/wait-$hold_index.out" 2>&1 & + wait_pid=$! + wait_case_contract=0 + if wait_for_path "$wait_control/hold.ready" && + wait_for_path "$wait_control/${peer_done[0]}" && + wait_for_path "$wait_control/${peer_done[1]}" && + wait_for_path "$wait_control/${peer_done[2]}" && + wait_for_child_wait "$wait_pid"; then + wait_case_contract=1 + fi + touch "$wait_control/hold.release" + wait_rc=0 + wait "$wait_pid" || wait_rc=$? + if [ "$wait_case_contract" -ne 1 ] || [ "$wait_rc" -ne 0 ] || + ! cmp -s "$expected_success" "$work/wait-$hold_index.out" || + ! cmp -s <(LC_ALL=C sort "$expected_executions") \ + <(LC_ALL=C sort "$wait_control/executions"); then + wait_contract=0 + fi +done +if [ "$wait_contract" -eq 1 ]; then + pass AGG-017 "catalog parent reaps both lanes before publishing success" +else + fail AGG-017 "catalog parent reaps both lanes before publishing success" +fi + +malformed_aggregate="$replica/tests/experiment/local-image-catalog-malformed-status.sh" +awk ' + index($0, "printf") && index($0, "$rc") && index($0, "$status") { + print " printf \"999999999999999999999999999999999999999999999999\\\\n\" > \"$status\"" + changed++ + next + } + { print } + END { if (changed != 1) exit 42 } +' "$replica_aggregate" > "$malformed_aggregate" +malformed_mutation_rc=$? +chmod +x "$malformed_aggregate" +reset_fixtures +malformed_control="$work/malformed-control" +mkdir "$malformed_control" +: > "$malformed_control/executions" +malformed_rc=0 +AGENT_LAB_CATALOG_AGG_CONTROL="$malformed_control" \ + bash "$malformed_aggregate" > "$work/malformed.out" 2>&1 || malformed_rc=$? +trailing_aggregate="$replica/tests/experiment/local-image-catalog-trailing-status.sh" +awk ' + index($0, "printf") && index($0, "$rc") && index($0, "$status") { + print " printf \"0\\\\nTRAILING-GARBAGE\" > \"$status\"" + changed++ + next + } + { print } + END { if (changed != 1) exit 42 } +' "$replica_aggregate" > "$trailing_aggregate" +trailing_mutation_rc=$? +chmod +x "$trailing_aggregate" +reset_fixtures +trailing_control="$work/trailing-control" +mkdir "$trailing_control" +: > "$trailing_control/executions" +trailing_rc=0 +AGENT_LAB_CATALOG_AGG_CONTROL="$trailing_control" \ + bash "$trailing_aggregate" > "$work/trailing.out" 2>&1 || trailing_rc=$? +if [ "$malformed_mutation_rc" -eq 0 ] && [ "$trailing_mutation_rc" -eq 0 ] && + [ "$malformed_rc" -eq 125 ] && [ "$trailing_rc" -eq 125 ] && + grep -Fxq 'SUMMARY assertions=76 expected=76 failures=0 infra=1' \ + "$work/malformed.out" && + grep -Fxq 'SUMMARY assertions=76 expected=76 failures=0 infra=1' \ + "$work/trailing.out" && + ! grep -Fxq 'EXPERIMENT LOCAL IMAGE CATALOG PASS' "$work/malformed.out" && + ! grep -Fxq 'EXPERIMENT LOCAL IMAGE CATALOG PASS' "$work/trailing.out" && + cmp -s <(LC_ALL=C sort "$expected_executions") \ + <(LC_ALL=C sort "$malformed_control/executions") && + cmp -s <(LC_ALL=C sort "$expected_executions") \ + <(LC_ALL=C sort "$trailing_control/executions"); then + pass AGG-018 "malformed lane status fails closed before success" +else + fail AGG-018 "malformed lane status fails closed before success" +fi + +signal_names=(TERM INT QUIT) +signal_numbers=(15 2 3) +expected_signal_executions="$work/expected-signal-executions" +printf '%s\n' catalog-cases.sh catalog-state-cases.py > "$expected_signal_executions" +signal_contract=1 +for signal_index in "${!signal_names[@]}"; do + reset_fixtures + signal_control="$work/signal-control-$signal_index" + signal_tmp="$work/signal-tmp-$signal_index" + mkdir "$signal_control" "$signal_tmp" + : > "$signal_control/executions" + AGENT_LAB_CATALOG_AGG_CONTROL="$signal_control" \ + AGENT_LAB_CATALOG_AGG_SIGNAL_MODE=cooperative \ + TMPDIR="$signal_tmp" \ + python3 -I -B -c \ + 'import os, signal, sys +for value in (signal.SIGHUP, signal.SIGINT, signal.SIGQUIT, signal.SIGTERM): + signal.signal(value, signal.SIG_DFL) +os.execvpe("bash", ["bash", sys.argv[1]], os.environ)' \ + "$replica_aggregate" > "$work/signal-$signal_index.out" 2>&1 & + signal_pid=$! + signal_setup=0 + bash_lane="" + python_lane="" + bash_descendant="" + python_descendant="" + if wait_for_path "$signal_control/bash-signal.ready" && + wait_for_path "$signal_control/python-signal.ready" && + IFS= read -r bash_lane < "$signal_control/bash-lane.pid" && + IFS= read -r python_lane < "$signal_control/python-lane.pid" && + IFS= read -r bash_descendant < "$signal_control/bash-descendant.pid" && + IFS= read -r python_descendant < "$signal_control/python-descendant.pid" && + [[ "$bash_lane" =~ ^[0-9]+$ ]] && [[ "$python_lane" =~ ^[0-9]+$ ]] && + [[ "$bash_descendant" =~ ^[0-9]+$ ]] && + [[ "$python_descendant" =~ ^[0-9]+$ ]] && + kill -0 "$bash_lane" 2>/dev/null && kill -0 "$python_lane" 2>/dev/null && + kill -0 "$bash_descendant" 2>/dev/null && + kill -0 "$python_descendant" 2>/dev/null; then + signal_setup=1 + fi + kill "-${signal_names[$signal_index]}" "$signal_pid" 2>/dev/null || true + signal_rc=0 + wait "$signal_pid" || signal_rc=$? + signal_pids_gone=1 + for observed_pid in \ + "$bash_lane" "$python_lane" "$bash_descendant" "$python_descendant"; do + if [ -z "$observed_pid" ] || ! wait_for_process_exit "$observed_pid"; then + signal_pids_gone=0 + fi + done + if [ "$signal_setup" -ne 1 ] || + [ "$signal_rc" -ne "$((128 + signal_numbers[signal_index]))" ] || + [ "$signal_pids_gone" -ne 1 ] || + grep -Fxq 'EXPERIMENT LOCAL IMAGE CATALOG PASS' "$work/signal-$signal_index.out" || + ! cmp -s <(LC_ALL=C sort "$expected_signal_executions") \ + <(LC_ALL=C sort "$signal_control/executions"); then + signal_contract=0 + fi + for observed_pid in \ + "$bash_descendant" "$python_descendant" "$bash_lane" "$python_lane"; do + if [ -n "$observed_pid" ]; then + kill -KILL "$observed_pid" 2>/dev/null || true + wait_for_process_exit "$observed_pid" || true + fi + done +done +if [ "$signal_contract" -eq 1 ]; then + pass AGG-019 "catalog cancellation reaches cooperative lane descendants" +else + fail AGG-019 "catalog cancellation reaches cooperative lane descendants" +fi + +reset_fixtures +stubborn_control="$work/stubborn-control" +stubborn_tmp="$work/stubborn-tmp" +mkdir "$stubborn_control" "$stubborn_tmp" +: > "$stubborn_control/executions" +AGENT_LAB_CATALOG_AGG_CONTROL="$stubborn_control" \ +AGENT_LAB_CATALOG_AGG_SIGNAL_MODE=stubborn \ +TMPDIR="$stubborn_tmp" \ + bash "$replica_aggregate" > "$work/stubborn.out" 2>&1 & +stubborn_leader=$! +stubborn_setup=0 +stubborn_lane="" +stubborn_pid="" +if wait_for_path "$stubborn_control/stubborn-signal.ready" && + IFS= read -r stubborn_lane < "$stubborn_control/bash-lane.pid" && + IFS= read -r stubborn_pid < "$stubborn_control/stubborn-descendant.pid" && + [[ "$stubborn_lane" =~ ^[0-9]+$ ]] && + [[ "$stubborn_pid" =~ ^[0-9]+$ ]] && + kill -0 "$stubborn_lane" 2>/dev/null && + kill -0 "$stubborn_pid" 2>/dev/null; then + stubborn_setup=1 +fi +kill -TERM "$stubborn_leader" 2>/dev/null || true +stubborn_rc=0 +wait "$stubborn_leader" || stubborn_rc=$? +stubborn_alive=0 +stubborn_output_preserved=0 +stubborn_lane_gone=0 +if [ -n "$stubborn_lane" ] && wait_for_process_exit "$stubborn_lane"; then + stubborn_lane_gone=1 +fi +if [ -n "$stubborn_pid" ] && kill -0 "$stubborn_pid" 2>/dev/null; then + stubborn_alive=1 + stubborn_output="$(readlink "/proc/$stubborn_pid/fd/1" 2>/dev/null || true)" + if [ -n "$stubborn_output" ] && [ -e "$stubborn_output" ]; then + stubborn_output_preserved=1 + fi +fi +if [ -n "$stubborn_pid" ]; then + kill -KILL "$stubborn_pid" 2>/dev/null || true + wait_for_process_exit "$stubborn_pid" || true +fi +if [ -n "$stubborn_lane" ]; then + kill -KILL "$stubborn_lane" 2>/dev/null || true + wait_for_process_exit "$stubborn_lane" || true +fi +if [ "$stubborn_setup" -eq 1 ] && [ "$stubborn_rc" -eq 143 ] && + [ "$stubborn_alive" -eq 1 ] && [ "$stubborn_lane_gone" -eq 1 ] && + [ "$stubborn_output_preserved" -eq 1 ] && + cmp -s <(LC_ALL=C sort "$expected_signal_executions") \ + <(LC_ALL=C sort "$stubborn_control/executions") && + ! grep -Fxq 'EXPERIMENT LOCAL IMAGE CATALOG PASS' "$work/stubborn.out"; then + pass AGG-020 "catalog cancellation preserves stubborn descendant evidence" +else + fail AGG-020 "catalog cancellation preserves stubborn descendant evidence" +fi + +cleanup_infrastructure=0 +if ! cleanup_work; then + cleanup_infrastructure=1 +fi +trap - EXIT + +printf 'SUMMARY assertions=%s expected=%s failures=%s infra=%s\n' \ + "$expected_count" "$expected_count" "$failures" "$cleanup_infrastructure" +if [ "$cleanup_infrastructure" -ne 0 ]; then + exit 125 +fi +[ "$failures" -eq 0 ] diff --git a/tests/experiment/catalog-resolution-cases.sh b/tests/experiment/catalog-resolution-cases.sh new file mode 100755 index 0000000..6252a4d --- /dev/null +++ b/tests/experiment/catalog-resolution-cases.sh @@ -0,0 +1,441 @@ +#!/usr/bin/env bash +set -u -o pipefail + +repo_root="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." >/dev/null 2>&1 && pwd)" +agent_lab="$repo_root/scripts/agent-lab" +bounded_helper="$repo_root/tests/helpers/run-bounded.py" +expected_count=14 +work="" +failures=0 +infrastructure=0 + +cleanup_work() { + local failed=0 + if [ -n "$work" ] && [ -e "$work" ]; then + find "$work" -type f -delete 2>/dev/null || failed=1 + find "$work" -type l -delete 2>/dev/null || failed=1 + find "$work" -depth -type d -exec rmdir {} + 2>/dev/null || failed=1 + [ ! -e "$work" ] || failed=1 + fi + return "$failed" +} + +finish() { + local assertions=0 + if [ -n "${observed:-}" ] && [ -f "$observed" ]; then + assertions="$(wc -l < "$observed")" + fi + if ! cleanup_work; then + infrastructure=1 + fi + trap - EXIT + printf 'SUMMARY assertions=%s expected=%s failures=%s infra=%s\n' \ + "$assertions" "$expected_count" "$failures" "$infrastructure" + if [ "$infrastructure" -ne 0 ]; then + exit 125 + fi + if [ "$failures" -ne 0 ]; then + exit 1 + fi +} + +if ! work="$(mktemp -d)"; then + printf 'SUMMARY assertions=0 expected=%s failures=0 infra=1\n' "$expected_count" + exit 125 +fi +trap 'cleanup_work >/dev/null 2>&1 || true' EXIT +observed="$work/observed" +: > "$observed" + +if [ ! -x "$agent_lab" ] || [ ! -f "$bounded_helper" ] || ! command -v jq >/dev/null 2>&1 || ! command -v python3 >/dev/null 2>&1 || [ ! -d "$repo_root/.cache/dev/tools/cue" ]; then + printf 'INFRA catalog resolution prerequisites are unavailable\n' >&2 + infrastructure=1 + finish +fi +if ! python3 -I -B "$bounded_helper" --self-test > "$work/bounded-self-test.out" 2> "$work/bounded-self-test.err"; then + printf 'INFRA bounded command helper self-test failed\n' >&2 + infrastructure=1 + finish +fi + +pass() { printf 'PASS %s %s\n' "$1" "$2"; printf '%s\n' "$1" >> "$observed"; } +fail() { printf 'FAIL %s %s\n' "$1" "$2"; printf '%s\n' "$1" >> "$observed"; failures=$((failures + 1)); } +run_bounded() { + local output="$1" + local errors="$2" + local expectation="$3" + local status="${output}.status" + local rc=0 + local status_line="" + shift 3 + find "$status" -delete 2>/dev/null || true + python3 -I -B "$bounded_helper" \ + --timeout 5 --status "$status" --stdout "$output" --stderr "$errors" -- "$@" || rc=$? + if [ -f "$status" ]; then + status_line="$(cat "$status")" + fi + if [ "$status_line" != "child:$rc" ]; then + infrastructure=1 + rc=125 + else + case "$rc" in + 0|1) + ;; + 125) + [ "$expectation" = expected-125 ] || infrastructure=1 + ;; + *) + infrastructure=1 + ;; + esac + fi + return "$rc" +} +capture() { + local expectation="normal" + if [ "${1:-}" = expected-125 ]; then + expectation="expected-125" + shift + fi + CAPTURE_RC=0 + run_bounded "$work/stdout" "$work/stderr" "$expectation" "$@" || CAPTURE_RC=$? +} +subject_a="registry.example/operator/worker@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" +subject_b="registry.example/operator/other@sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" + +init_home() { + local home="$1" + capture "$agent_lab" --home "$home" init + if [ "$CAPTURE_RC" -ne 0 ]; then + printf 'INFRA temporary Agent Lab home initialization failed: %s\n' "$(tr '\n' ' ' < "$work/stderr")" >&2 + infrastructure=1 + finish + fi + cp -a "$repo_root/.cache/dev/tools/cue/." "$home/cache/tools/cue/" +} + +copy_cedar() { + local home="$1" + if [ ! -d "$repo_root/.cache/dev/tools/cedar" ]; then + printf 'INFRA pinned Cedar test cache is unavailable\n' >&2 + infrastructure=1 + finish + fi + cp -a "$repo_root/.cache/dev/tools/cedar/." "$home/cache/tools/cedar/" +} + +write_artifact() { + local directory="$1" + local experiment_name="$2" + local selector_key="$3" + local selector_value="$4" + mkdir -p "$directory" + printf '%s\n' \ + 'package experiment' \ + '' \ + 'experiment: {' \ + ' apiVersion: "agent-lab/v0alpha1"' \ + ' kind: "Experiment"' \ + " metadata: name: \"$experiment_name\"" \ + ' spec: members: [{' \ + ' name: "worker"' \ + " image: $selector_key: \"$selector_value\"" \ + ' }]' \ + '}' > "$directory/experiment.cue" +} + +write_two_member_artifact() { + local directory="$1" + mkdir -p "$directory" + printf '%s\n' \ + 'package experiment' \ + '' \ + 'experiment: {' \ + ' apiVersion: "agent-lab/v0alpha1"' \ + ' kind: "Experiment"' \ + ' metadata: name: "two-members"' \ + ' spec: members: [{' \ + ' name: "first"' \ + ' image: catalogName: "vendor.worker"' \ + ' }, {' \ + ' name: "second"' \ + ' image: catalogName: "vendor.second"' \ + ' }]' \ + '}' > "$directory/experiment.cue" +} + +active_home="$work/active-home" +init_home "$active_home" +capture "$agent_lab" --home "$active_home" image add vendor.worker "$subject_a" +if [ "$CAPTURE_RC" -ne 0 ]; then + printf 'INFRA active local mapping setup failed\n' >&2 + infrastructure=1 + finish +fi +entry_a="$(jq -r '.entryDigest' "$work/stdout")" +artifact="$work/active-artifact" +write_artifact "$artifact" local-active catalogName vendor.worker +capture "$agent_lab" --home "$active_home" experiment check "$artifact" +active_rc="$CAPTURE_RC" +cp "$work/stdout" "$work/active-check.json" +if [ "$active_rc" -eq 0 ] && jq -e --arg entry "$entry_a" --arg subject "$subject_a" ' + .plan.spec.members[0].resolvedImage == { + entryDigest: $entry, + generation: 1, + origin: "local", + subject: $subject + } +' "$work/active-check.json" >/dev/null 2>&1; then + pass RES-ENTRY-001 "active local resolution binds exact entry identity and immutable subject" +else + fail RES-ENTRY-001 "active local resolution binds exact entry identity and immutable subject" +fi + +snapshot_digest="$(jq -r '.snapshotDigest' "$active_home/images/catalog/current.json" 2>/dev/null)" +snapshot_revision="$(jq -r '.revision' "$active_home/images/catalog/snapshots/${snapshot_digest#sha256:}.json" 2>/dev/null)" +if [ "$active_rc" -eq 0 ] && jq -e --arg digest "$snapshot_digest" --argjson revision "$snapshot_revision" ' + .catalog.local == {revision: $revision, snapshotDigest: $digest} +' "$work/active-check.json" >/dev/null 2>&1; then + pass RES-SNAP-001 "checked evidence records the exact held local snapshot revision and digest" +else + fail RES-SNAP-001 "checked evidence records the exact held local snapshot revision and digest" +fi + +first_plan_digest="$(jq -r '.digest // empty' "$work/active-check.json" 2>/dev/null)" +capture "$agent_lab" --home "$active_home" image add vendor.unrelated "$subject_b" +capture "$agent_lab" --home "$active_home" experiment check "$artifact" +cp "$work/stdout" "$work/unrelated-check.json" +second_snapshot_digest="$(jq -r '.snapshotDigest' "$active_home/images/catalog/current.json" 2>/dev/null)" +if [ "$CAPTURE_RC" -eq 0 ] && + [ "$first_plan_digest" = "$(jq -r '.digest // empty' "$work/unrelated-check.json" 2>/dev/null)" ] && + [ "$snapshot_digest" != "$second_snapshot_digest" ] && + jq -e --arg digest "$second_snapshot_digest" '.catalog.local.snapshotDigest == $digest' "$work/unrelated-check.json" >/dev/null 2>&1; then + pass RES-SNAP-002 "unrelated catalog mutation changes snapshot evidence but not selected-entry plan identity" +else + fail RES-SNAP-002 "unrelated catalog mutation changes snapshot evidence but not selected-entry plan identity" +fi + +other_home="$work/other-home" +init_home "$other_home" +capture "$agent_lab" --home "$other_home" image add vendor.worker "$subject_b" +entry_b="$(jq -r '.entryDigest // empty' "$work/stdout" 2>/dev/null)" +capture "$agent_lab" --home "$other_home" experiment check "$artifact" +if [ "$CAPTURE_RC" -eq 0 ] && [ "$entry_a" != "$entry_b" ] && + [ "$first_plan_digest" != "$(jq -r '.digest // empty' "$work/stdout" 2>/dev/null)" ]; then + pass RES-ENTRY-002 "substituting the selected entry changes plan identity" +else + fail RES-ENTRY-002 "substituting the selected entry changes plan identity" +fi + +unknown_artifact="$work/unknown-artifact" +write_artifact "$unknown_artifact" local-unknown catalogName vendor.unknown +capture "$agent_lab" --home "$active_home" experiment check "$unknown_artifact" +unknown_rc="$CAPTURE_RC" +removed_home="$work/removed-home" +init_home "$removed_home" +capture "$agent_lab" --home "$removed_home" image add vendor.worker "$subject_a" +removed_entry="$(jq -r '.entryDigest // empty' "$work/stdout" 2>/dev/null)" +capture "$agent_lab" --home "$removed_home" image remove vendor.worker --expect "$removed_entry" +capture "$agent_lab" --home "$removed_home" experiment check "$artifact" +removed_rc="$CAPTURE_RC" +if [ "$unknown_rc" -eq 1 ] && [ "$removed_rc" -eq 1 ]; then + pass RES-ENTRY-003 "unknown and tombstoned local names are stable invalid input" +else + fail RES-ENTRY-003 "unknown and tombstoned local names are stable invalid input" +fi + +isolation_home="$work/isolation-home" +init_home "$isolation_home" +mkdir "$isolation_home/images/catalog" +printf '{"snapshotDigest":"sha256:%064d"}\n' 0 > "$isolation_home/images/catalog/current.json" +direct_artifact="$work/direct-artifact" +write_artifact "$direct_artifact" direct-isolated digestRef "$subject_a" +capture "$agent_lab" --home "$isolation_home" experiment check "$direct_artifact" +direct_rc="$CAPTURE_RC" +direct_has_local="$(jq -r 'has("catalog") and (.catalog | has("local"))' "$work/stdout" 2>/dev/null || printf invalid)" +bundled_artifact="$work/bundled-artifact" +write_artifact "$bundled_artifact" bundled-isolated catalogName agent-lab.unknown +capture "$agent_lab" --home "$isolation_home" experiment check "$bundled_artifact" +bundled_rc="$CAPTURE_RC" +if [ "$direct_rc" -eq 0 ] && [ "$direct_has_local" = false ] && [ "$bundled_rc" -eq 1 ]; then + pass RES-ISOLATE-001 "corrupt local state cannot block or contaminate direct and bundled selectors" +else + fail RES-ISOLATE-001 "corrupt local state cannot block or contaminate direct and bundled selectors" +fi + +history_home="$work/history-home" +init_home "$history_home" +capture "$agent_lab" --home "$history_home" image add vendor.worker "$subject_a" +mv "$history_home/images/catalog/entries" "$work/removed-resolution-history" +capture expected-125 "$agent_lab" --home "$history_home" experiment check "$artifact" +if [ "$CAPTURE_RC" -eq 125 ] && [ ! -s "$work/stdout" ]; then + pass RES-STATE-001 "local resolution verifies immutable entry history before binding" +else + fail RES-STATE-001 "local resolution verifies immutable entry history before binding" +fi + +drift_home="$work/drift-home" +init_home "$drift_home" +capture "$agent_lab" --home "$drift_home" image add vendor.worker "$subject_a" +mkdir "$drift_home/other-images" +mv "$drift_home/images/catalog" "$drift_home/other-images/catalog" +jq -cS '.paths.images="other-images"' "$drift_home/config.json" > "$work/drift-config.json" +mv "$work/drift-config.json" "$drift_home/config.json" +chmod 600 "$drift_home/config.json" +capture expected-125 "$agent_lab" --home "$drift_home" config check +drift_config_rc="$CAPTURE_RC" +capture expected-125 "$agent_lab" --home "$drift_home" experiment check "$artifact" +if [ "$drift_config_rc" -eq 125 ] && [ "$CAPTURE_RC" -eq 125 ] && [ ! -s "$work/stdout" ]; then + pass RES-STATE-002 "receipt-breaking config drift cannot redirect local resolution" +else + fail RES-STATE-002 "receipt-breaking config drift cannot redirect local resolution" +fi + +symlink_source="$work/symlink-source" +init_home "$symlink_source" +capture "$agent_lab" --home "$symlink_source" image add vendor.worker "$subject_a" +mv "$symlink_source/images/catalog" "$work/outside-resolution-catalog" +symlink_home="$work/symlink-home" +init_home "$symlink_home" +ln -s "$work/outside-resolution-catalog" "$symlink_home/images/catalog" +capture expected-125 "$agent_lab" --home "$symlink_home" experiment check "$artifact" +if [ "$CAPTURE_RC" -eq 125 ] && [ ! -s "$work/stdout" ]; then + pass RES-STATE-003 "local resolution refuses a symlinked catalog authority" +else + fail RES-STATE-003 "local resolution refuses a symlinked catalog authority" +fi + +supplied="$work/supplied-fields" +mkdir "$supplied" +printf '%s\n' \ + 'package experiment' \ + '' \ + 'experiment: {' \ + ' apiVersion: "agent-lab/v0alpha1"' \ + ' kind: "Experiment"' \ + ' metadata: name: "supplied-fields"' \ + ' spec: members: [{' \ + ' name: "worker"' \ + ' image: {' \ + ' catalogName: "vendor.worker"' \ + ' resolvedSubject: "registry.example/evil@sha256:eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee"' \ + ' }' \ + ' }]' \ + '}' > "$supplied/experiment.cue" +capture "$agent_lab" --home "$active_home" experiment check "$supplied" +if [ "$CAPTURE_RC" -eq 1 ] && [ ! -s "$work/stdout" ]; then + pass RES-INPUT-001 "Experiment content cannot supply catalog source, entry, or resolved-subject authority" +else + fail RES-INPUT-001 "Experiment content cannot supply catalog source, entry, or resolved-subject authority" +fi + +multi_home="$work/multi-home" +init_home "$multi_home" +capture "$agent_lab" --home "$multi_home" image add vendor.worker "$subject_a" +multi_entry_a="$(jq -r '.entryDigest // empty' "$work/stdout" 2>/dev/null)" +capture "$agent_lab" --home "$multi_home" image add vendor.second "$subject_b" +multi_entry_b="$(jq -r '.entryDigest // empty' "$work/stdout" 2>/dev/null)" +multi_digest="$(jq -r '.snapshotDigest' "$multi_home/images/catalog/current.json" 2>/dev/null)" +multi_artifact="$work/multi-artifact" +write_two_member_artifact "$multi_artifact" +capture "$agent_lab" --home "$multi_home" experiment check "$multi_artifact" +if [ "$CAPTURE_RC" -eq 0 ] && jq -e --arg first "$multi_entry_a" --arg second "$multi_entry_b" --arg snapshot "$multi_digest" ' + .plan.spec.members[0].resolvedImage.entryDigest == $first and + .plan.spec.members[1].resolvedImage.entryDigest == $second and + .catalog.local.snapshotDigest == $snapshot +' "$work/stdout" >/dev/null 2>&1; then + pass RES-SNAP-003 "all local members bind entries from one held catalog snapshot" +else + fail RES-SNAP-003 "all local members bind entries from one held catalog snapshot" +fi + +copy_cedar "$active_home" +capture "$agent_lab" --home "$active_home" experiment authorize install "$artifact" +if [ "$CAPTURE_RC" -eq 0 ] && jq -e --arg plan "$first_plan_digest" '.verdict == "permit" and .binding.planDigest == $plan' "$work/stdout" >/dev/null 2>&1; then + pass RES-AUTH-001 "fresh authorization binds the selected-entry plan digest" +else + fail RES-AUTH-001 "fresh authorization binds the selected-entry plan digest" +fi + +installed_prefix="$work/installed-prefix" +installed_home="$work/installed-home" +runtime_replica="$work/runtime-replica" +runtime_manifest="$repo_root/packaging/agent-lab-local.manifest" +mkdir -p "$runtime_replica/packaging" "$runtime_replica/scripts" +while IFS= read -r runtime_name; do + mkdir -p "$runtime_replica/$(dirname -- "$runtime_name")" + cp "$repo_root/$runtime_name" "$runtime_replica/$runtime_name" +done < "$runtime_manifest" +cp "$runtime_manifest" "$runtime_replica/packaging/agent-lab-local.manifest" +cp "$repo_root/scripts/install-local" "$repo_root/scripts/install-local.py" "$runtime_replica/scripts/" +chmod +x "$runtime_replica/scripts/install-local" "$runtime_replica/scripts/agent-lab" +capture "$runtime_replica/scripts/install-local" --prefix "$installed_prefix" +installed_install_rc="$CAPTURE_RC" +installed_runtime_before="$(find "$installed_prefix" -printf '%P %y %m %s\n' 2>/dev/null | LC_ALL=C sort)" +mv "$runtime_replica" "$work/runtime-source-unavailable" +mkdir "$work/installed-unrelated" +installed_rc=125 +if [ "$installed_install_rc" -eq 0 ]; then + capture "$installed_prefix/bin/agent-lab" --home "$installed_home" init + cp -a "$repo_root/.cache/dev/tools/cue/." "$installed_home/cache/tools/cue/" + capture "$installed_prefix/bin/agent-lab" --home "$installed_home" image add vendor.worker "$subject_a" + installed_entry="$(jq -r '.entryDigest // empty' "$work/stdout" 2>/dev/null)" + installed_rc=0 + (cd "$work/installed-unrelated" && run_bounded \ + "$work/installed-check.json" "$work/installed-check.err" normal \ + env -i PATH=/usr/bin:/bin \ + "$installed_prefix/bin/agent-lab" --home "$installed_home" experiment check "$artifact") || installed_rc=$? + [ "$installed_rc" -eq 0 ] || [ "$installed_rc" -eq 1 ] || infrastructure=1 +else + installed_entry="" +fi +installed_runtime_after="$(find "$installed_prefix" -printf '%P %y %m %s\n' 2>/dev/null | LC_ALL=C sort)" +if [ "$installed_install_rc" -eq 0 ] && [ "$installed_rc" -eq 0 ] && + [ "$installed_runtime_before" = "$installed_runtime_after" ] && + [ -z "$(find "$installed_prefix" -name __pycache__ -print -quit 2>/dev/null)" ] && + [ ! -s "$work/installed-check.err" ] && jq -e --arg entry "$installed_entry" --arg subject "$subject_a" ' + .plan.spec.members[0].resolvedImage == { + entryDigest: $entry, + generation: 1, + origin: "local", + subject: $subject + } + ' "$work/installed-check.json" >/dev/null 2>&1; then + pass RES-INSTALL-001 "installed catalog resolution is source-independent and leaves its release topology unchanged" +else + fail RES-INSTALL-001 "installed catalog resolution is source-independent and leaves its release topology unchanged" +fi + +canary_home="$work/canary-home" +init_home "$canary_home" +capture "$agent_lab" --home "$canary_home" image add vendor.worker "$subject_a" +canary_bin="$work/canary-bin" +canary_marks="$work/canary-marks" +mkdir "$canary_bin" "$canary_marks" +for command in docker git curl wget; do + printf '%s\n' '#!/bin/sh' 'set -eu' ': > "$CANARY_DIR/${0##*/}"' > "$canary_bin/$command" + chmod 700 "$canary_bin/$command" + CANARY_DIR="$canary_marks" "$canary_bin/$command" +done +calibrated="$(find "$canary_marks" -type f | wc -l)" +find "$canary_marks" -type f -delete +canary_rc=0 +run_bounded "$work/canary.out" "$work/canary.err" normal \ + env -i PATH="$canary_bin:/usr/bin:/bin" LANG=C LC_ALL=C CANARY_DIR="$canary_marks" \ + "$agent_lab" --home "$canary_home" experiment check "$artifact" || canary_rc=$? +if [ "$calibrated" -eq 4 ] && [ "$canary_rc" -eq 0 ] && [ -z "$(find "$canary_marks" -type f -print -quit)" ]; then + pass RES-NOEF-001 "calibrated forbidden-effect canaries remain silent during local resolution" +else + fail RES-NOEF-001 "calibrated forbidden-effect canaries remain silent during local resolution" +fi + +expected="$work/expected" +printf '%s\n' \ + RES-ENTRY-001 RES-SNAP-001 RES-SNAP-002 RES-ENTRY-002 RES-ENTRY-003 \ + RES-ISOLATE-001 RES-STATE-001 RES-STATE-002 RES-STATE-003 RES-INPUT-001 \ + RES-SNAP-003 RES-AUTH-001 RES-INSTALL-001 RES-NOEF-001 > "$expected" +if ! cmp -s "$expected" "$observed"; then + printf 'INFRA catalog resolution assertion identity drift\n' >&2 + infrastructure=1 +fi +finish diff --git a/tests/experiment/contract-cases.sh b/tests/experiment/contract-cases.sh index 0e346ab..4e47f54 100755 --- a/tests/experiment/contract-cases.sh +++ b/tests/experiment/contract-cases.sh @@ -1,462 +1,97 @@ #!/usr/bin/env bash -set -euo pipefail +set -u -o pipefail repo_root="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." >/dev/null 2>&1 && pwd)" -experiment="$repo_root/scripts/experiment" -cue_tool="$repo_root/scripts/dev/cue-tool" -fixture_root="$repo_root/tests/experiment/fixtures" - -for required in \ - "$experiment" \ - "$cue_tool" \ - "$repo_root/scripts/dev/cue-tool.py" \ - "$repo_root/contracts/experiment/v0alpha1/schema.cue" \ - "$repo_root/contracts/experiment/v0alpha1/plan.cue" \ - "$repo_root/contracts/experiment/v0alpha1/cue.mod/module.cue" \ - "$repo_root/tools/cue.lock"; do - if [ ! -f "$required" ]; then - printf 'INFRA Experiment contract input is missing: %s\n' "$required" >&2 - exit 125 +subcases=( + "$repo_root/tests/experiment/directory-intake-cases.sh" + "$repo_root/tests/experiment/aggregate-harness-cases.sh" + "$repo_root/tests/experiment/catalog-aggregate-harness-cases.sh" +) +expected_count=43 +work="" + +cleanup_work() { + local failed=0 + if [ -n "$work" ] && [ -e "$work" ]; then + find "$work" -type f -delete 2>/dev/null || failed=1 + find "$work" -type l -delete 2>/dev/null || failed=1 + find "$work" -depth -type d -exec rmdir {} + 2>/dev/null || failed=1 + [ ! -e "$work" ] || failed=1 fi -done -if [ ! -x "$experiment" ] || [ ! -x "$cue_tool" ]; then - printf 'INFRA Experiment contract entrypoints are not executable\n' >&2 - exit 125 -fi -cue_preflight_rc=0 -cue_preflight_out="$("$cue_tool" version 2>&1)" || cue_preflight_rc=$? -if [ "$cue_preflight_rc" -ne 0 ]; then - printf 'INFRA Experiment contract requires provisioned pinned CUE: %s\n' \ - "$cue_preflight_out" >&2 - exit 125 -fi - -work="$(mktemp -d)" -cleanup() { - find "$work" -type f -delete 2>/dev/null || true - find "$work" -type l -delete 2>/dev/null || true - find "$work" -depth -type d -exec rmdir {} + 2>/dev/null || true + return "$failed" } -trap cleanup EXIT -mkdir -p "$work/bin" "$work/home" "$work/runtime-tmp" - -spy_log="$work/tool-spy.log" -: > "$spy_log" -repo_before="$work/repo.before" -git -C "$repo_root" status --porcelain=v1 --untracked-files=all > "$repo_before" -for tool in docker docker-compose podman; do - spy="$work/bin/$tool" - { - printf '#!/usr/bin/env bash\n' - printf 'printf "%%s\\n" "$0 $*" >> %q\n' "$spy_log" - printf 'exit 97\n' - } > "$spy" - chmod +x "$spy" -done -failures=0 -pass() { printf 'PASS %s\n' "$1"; } -fail() { printf 'FAIL %s\n' "$1"; failures=$((failures + 1)); } - -sha256_stdin() { - if command -v sha256sum >/dev/null 2>&1; then - sha256sum | awk '{print $1}' - else - shasum -a 256 | awk '{print $1}' +if ! work="$(mktemp -d)"; then + printf 'SUMMARY assertions=0 expected=%s failures=0 infra=1\n' "$expected_count" + exit 125 +fi +trap 'cleanup_work >/dev/null 2>&1 || true' EXIT + +expected="$work/expected" +observed="$work/observed" +printf '%s\n' \ + FMT-001 FMT-002 FMT-004 FMT-005 FMT-006 FMT-007 FMT-003 FMT-008 \ + SEL-001 CUE-001 FMT-009 M-FMT-001 SEL-002 \ + AGG-001 AGG-002 AGG-003 AGG-004 AGG-005 AGG-006 AGG-007 AGG-008 AGG-009 \ + AGG-010 AGG-011 AGG-012 AGG-013 AGG-021 AGG-022 AGG-023 AGG-024 \ + AGG-025 AGG-026 AGG-027 AGG-028 AGG-029 AGG-030 \ + AGG-014 AGG-015 AGG-016 AGG-017 \ + AGG-018 AGG-019 AGG-020 > "$expected" +: > "$observed" + +infrastructure=0 +for index in "${!subcases[@]}"; do + subcase="${subcases[$index]}" + output="$work/subcase-$index.out" + if [ ! -f "$subcase" ]; then + infrastructure=1 + continue fi -} - -tree_fingerprint() { - python3 - "$1" <<'PY' -from hashlib import sha256 -from pathlib import Path -import os -import stat -import sys - -root = Path(sys.argv[1]) -if not root.exists(): - print("missing") - raise SystemExit(0) -for path in sorted(root.rglob("*"), key=lambda item: os.fsencode(item)): - relative = os.fsencode(path.relative_to(root)) - metadata = path.lstat() - if stat.S_ISREG(metadata.st_mode): - kind = b"file" - payload = sha256(path.read_bytes()).digest() - elif stat.S_ISDIR(metadata.st_mode): - kind = b"directory" - payload = b"" - elif stat.S_ISLNK(metadata.st_mode): - kind = b"symlink" - payload = os.fsencode(os.readlink(path)) - else: - kind = b"other" - payload = b"" - record = b"\0".join((relative, kind, str(stat.S_IMODE(metadata.st_mode)).encode(), payload)) - print(sha256(record).hexdigest()) -PY -} - -cue_cache="$repo_root/.cache/dev/tools/cue" -cache_before="$work/cache.before" -tree_fingerprint "$cue_cache" > "$cache_before" - -contract_sha256() { - python3 - "$repo_root" <<'PY' -from hashlib import sha256 -from pathlib import Path -import sys - -root = Path(sys.argv[1]) -names = sorted(( - "contracts/experiment/v0alpha1/cue.mod/module.cue", - "contracts/experiment/v0alpha1/plan.cue", - "contracts/experiment/v0alpha1/schema.cue", - "tools/cue.lock", -)) -digest = sha256(b"agent-lab.contract.v1\0") -for name in names: - encoded = name.encode("utf-8") - data = (root / name).read_bytes() - digest.update(len(encoded).to_bytes(4, "big")) - digest.update(encoded) - digest.update(len(data).to_bytes(8, "big")) - digest.update(data) -print(digest.hexdigest()) -PY -} - -capture() { - local name="$1" - shift - CAPTURE_STDOUT="$work/$name.stdout" - CAPTURE_STDERR="$work/$name.stderr" - CAPTURE_RC=0 - env -i \ - PATH="$work/bin:/usr/bin:/bin" \ - HOME="$work/home" \ - TMPDIR="$work/runtime-tmp" \ - LC_ALL=C \ - EXPERIMENT_TOOL_SPY_LOG="$spy_log" \ - "$@" > "$CAPTURE_STDOUT" 2> "$CAPTURE_STDERR" || CAPTURE_RC=$? -} - -expect_invalid() { - local id="$1" path="$2" detail="$3" - capture "$id" "$experiment" check -- "$path" - if [ "$CAPTURE_RC" -eq 1 ] && [ ! -s "$CAPTURE_STDOUT" ] && - grep -Fq 'FAIL Experiment manifest' "$CAPTURE_STDERR"; then - pass "$detail" + if bash "$subcase" > "$output" 2>&1; then + rc=0 else - fail "$detail (rc=$CAPTURE_RC)" + rc=$? fi -} - -capture usage-none "$experiment" -if [ "$CAPTURE_RC" -eq 2 ] && [ ! -s "$CAPTURE_STDOUT" ] && - grep -Fq 'Usage: scripts/experiment check [--] MANIFEST' "$CAPTURE_STDERR"; then - pass "missing command has the exact usage contract" -else - fail "missing command has the exact usage contract (rc=$CAPTURE_RC)" -fi - -capture usage-extra "$experiment" check "$fixture_root/valid.json" extra -if [ "$CAPTURE_RC" -eq 2 ] && [ ! -s "$CAPTURE_STDOUT" ]; then - pass "extra CLI arguments fail as usage" -else - fail "extra CLI arguments fail as usage (rc=$CAPTURE_RC)" -fi - -expected_contract_digest="$(contract_sha256)" -expected_plan="$( - jq -cS --arg digest "sha256:$expected_contract_digest" \ - '.contract.digest = $digest' "$fixture_root/expected-plan.json" -)" -expected_digest="$(printf '%s' "$expected_plan" | sha256_stdin)" -expected_envelope="$( - jq -cnS \ - --arg digest "sha256:$expected_digest" \ - --argjson plan "$expected_plan" \ - '{digest: $digest, plan: $plan}' -)" - -capture valid "$experiment" check -- "$fixture_root/valid.json" -if [ "$CAPTURE_RC" -eq 0 ] && [ ! -s "$CAPTURE_STDERR" ] && - [ "$(cat "$CAPTURE_STDOUT")" = "$expected_envelope" ] && - [ "$(wc -l < "$CAPTURE_STDOUT" | tr -d ' ')" -eq 1 ]; then - pass "valid manifest emits the exact canonical plan envelope" -else - fail "valid manifest emits the exact canonical plan envelope (rc=$CAPTURE_RC)" -fi - -capture reordered "$experiment" check -- "$fixture_root/valid-reordered.json" -if [ "$CAPTURE_RC" -eq 0 ] && [ ! -s "$CAPTURE_STDERR" ] && - cmp -s "$CAPTURE_STDOUT" "$work/valid.stdout"; then - pass "field order, member order, whitespace, and explicit defaults are non-semantic" -else - fail "equivalent reordered manifest changes the plan (rc=$CAPTURE_RC)" -fi - -if python3 - "$repo_root" "$fixture_root/valid.json" <<'PY' -from importlib.util import module_from_spec, spec_from_file_location -from pathlib import Path -import json -import subprocess -import sys -from types import SimpleNamespace - -root = Path(sys.argv[1]) -spec = spec_from_file_location("experiment_contract", root / "scripts/experiment.py") -assert spec is not None and spec.loader is not None -module = module_from_spec(spec) -spec.loader.exec_module(module) -manifest = json.loads(Path(sys.argv[2]).read_text(encoding="utf-8")) -digest, snapshot = module.contract_snapshot(root) -helper = (root / "scripts/dev/cue-tool.py").read_bytes() -private_roots = [] - -subprocess_calls = [] -def fake_run(command, **kwargs): - subprocess_calls.append((command, kwargs)) - return subprocess.CompletedProcess(command, 1, b"", b"") - -real_subprocess = module.subprocess -module.subprocess = SimpleNamespace( - PIPE=subprocess.PIPE, - SubprocessError=subprocess.SubprocessError, - run=fake_run, -) -private = Path("/private-validation-root") -module.invoke_cue( - manifest, - digest, - private, - root, - private / "contracts/experiment/v0alpha1", -) -module.subprocess = real_subprocess -assert len(subprocess_calls) == 1 -command = subprocess_calls[0][0] -assert command[:3] == ( - sys.executable, - "-I", - str(private / "scripts/dev/cue-tool.py"), -) -assert command[command.index("-C") + 1] == str( - private / "contracts/experiment/v0alpha1" -) -assert not {"docker", "docker-compose", "podman"}.intersection(command) - -def fake_invoke(value, actual_digest, validation_root, repo_root, contract_root): - assert actual_digest == digest - assert repo_root == root - assert validation_root != root - assert contract_root == validation_root / "contracts/experiment/v0alpha1" - for name, data in snapshot.items(): - assert (validation_root / name).read_bytes() == data - assert (validation_root / "scripts/dev/cue-tool.py").read_bytes() == helper - private_roots.append(validation_root) - plan = module.expected_plan(value, actual_digest) - return subprocess.CompletedProcess([], 0, module.canonical_json(plan) + b"\n", b"") - -module.invoke_cue = fake_invoke -module.cue_plan(manifest) -assert len(private_roots) == 2 -assert private_roots[0] == private_roots[1] -assert not private_roots[0].exists() -PY -then - pass "orchestration invokes only CUE over the exact hashed private snapshot" -else - fail "orchestration escaped CUE or reopened live contract paths" -fi - -actual_digest="$(jq -er '.digest' "$work/valid.stdout" 2>/dev/null || true)" -actual_plan="$(jq -cS '.plan' "$work/valid.stdout" 2>/dev/null || true)" -independent_digest="$(printf '%s' "$actual_plan" | sha256_stdin)" -if [ "$actual_digest" = "sha256:$independent_digest" ] && - [[ "$actual_digest" =~ ^sha256:[0-9a-f]{64}$ ]]; then - pass "plan digest independently matches the canonical plan bytes" -else - fail "plan digest does not match the canonical plan bytes" -fi - -expect_invalid unknown "$fixture_root/invalid-unknown.json" \ - "unknown privileged member field is rejected" -expect_invalid duplicate-identical "$fixture_root/invalid-duplicate-identical.json" \ - "identical duplicate JSON key is rejected before CUE unification" -expect_invalid duplicate-escaped "$fixture_root/invalid-duplicate-escaped.json" \ - "escaped-equivalent duplicate JSON key is rejected" -expect_invalid duplicate-member "$fixture_root/invalid-duplicate-member.json" \ - "duplicate member names are rejected" -expect_invalid mutable-image "$fixture_root/invalid-mutable-image.json" \ - "mutable image reference is rejected" -expect_invalid numeric-overflow "$fixture_root/invalid-numeric-overflow.json" \ - "out-of-range JSON number is invalid input, not infrastructure uncertainty" -expect_invalid environment "$fixture_root/invalid-environment.json" \ - "generic environment cannot override future Lab-owned runtime controls" -expect_invalid empty-members "$fixture_root/invalid-empty.json" \ - "an Experiment must contain at least one member" - -oversized="$work/oversized.json" -python3 - "$oversized" <<'PY' -from pathlib import Path -import sys - -Path(sys.argv[1]).write_bytes(b"x" * 262_145) -PY -expect_invalid oversized "$oversized" "manifest byte limit fails closed" - -deeply_nested="$work/deeply-nested.json" -python3 - "$deeply_nested" <<'PY' -from pathlib import Path -import sys - -prefix = b'{"apiVersion":"agent-lab/v0alpha1","kind":"Experiment","metadata":{"name":"deep"},"spec":{"members":' -Path(sys.argv[1]).write_bytes(prefix + (b"[" * 1000) + b"null" + (b"]" * 1000) + b"}}") -PY -expect_invalid deeply-nested "$deeply_nested" \ - "excessive JSON nesting is invalid input without a traceback" - -bidi="$work/bidi.json" -python3 - "$fixture_root/valid.json" "$bidi" <<'PY' -from pathlib import Path -import json -import sys - -value = json.loads(Path(sys.argv[1]).read_text(encoding="utf-8")) -value["spec"]["members"][0]["command"].append("review\N{RIGHT-TO-LEFT OVERRIDE}text") -Path(sys.argv[2]).write_text(json.dumps(value), encoding="utf-8") -PY -expect_invalid bidi "$bidi" "Unicode formatting controls are rejected" - -capture missing "$experiment" check -- "$work/missing.json" -if [ "$CAPTURE_RC" -eq 125 ] && [ ! -s "$CAPTURE_STDOUT" ] && - grep -Fq 'INFRA Experiment manifest' "$CAPTURE_STDERR"; then - pass "unreadable manifest is infrastructure uncertainty" -else - fail "unreadable manifest has the wrong result (rc=$CAPTURE_RC)" -fi - -ln -s "$fixture_root/valid.json" "$work/symlink.json" -capture symlink "$experiment" check -- "$work/symlink.json" -if [ "$CAPTURE_RC" -eq 125 ] && [ ! -s "$CAPTURE_STDOUT" ]; then - pass "manifest symlinks are rejected without following them" -else - fail "manifest symlink was followed (rc=$CAPTURE_RC)" -fi - -if [ ! -s "$spy_log" ]; then - pass "caller PATH engine shims cannot redirect Experiment validation" -else - fail "validation invoked a caller PATH engine shim: $(tr '\n' ' ' < "$spy_log")" -fi -if [ -z "$(find "$work/home" "$work/runtime-tmp" -mindepth 1 -print -quit)" ]; then - pass "validation creates no persistent home or runtime state" -else - fail "validation left persistent state" -fi -git -C "$repo_root" status --porcelain=v1 --untracked-files=all > "$work/repo.after" -if cmp -s "$repo_before" "$work/repo.after"; then - pass "validation leaves the checkout tree unchanged" -else - fail "validation changed the checkout tree" -fi - -if python3 - "$work/valid.stdout" <<'PY' -from pathlib import Path -import sys - -data = Path(sys.argv[1]).read_bytes() -raise SystemExit(0 if all(byte == 10 or 32 <= byte <= 126 for byte in data) else 1) -PY -then - pass "canonical envelope is one review-safe ASCII line" -else - fail "canonical envelope contains raw non-ASCII or control bytes" -fi - -copy_root="$work/contract-copy" -mkdir -p \ - "$copy_root/scripts/dev" \ - "$copy_root/contracts/experiment/v0alpha1/cue.mod" \ - "$copy_root/tests/experiment/fixtures" \ - "$copy_root/tools" -cp "$repo_root/scripts/experiment" "$repo_root/scripts/experiment.py" \ - "$copy_root/scripts/" -cp "$repo_root/scripts/dev/cue-tool" "$repo_root/scripts/dev/cue-tool.py" \ - "$copy_root/scripts/dev/" -cp "$repo_root/contracts/experiment/v0alpha1/schema.cue" \ - "$repo_root/contracts/experiment/v0alpha1/plan.cue" \ - "$copy_root/contracts/experiment/v0alpha1/" -cp "$repo_root/contracts/experiment/v0alpha1/cue.mod/module.cue" \ - "$copy_root/contracts/experiment/v0alpha1/cue.mod/" -cp "$repo_root/tools/cue.lock" "$copy_root/tools/" -cp "$fixture_root/valid.json" "$copy_root/tests/experiment/fixtures/" -chmod +x "$copy_root/scripts/experiment" "$copy_root/scripts/dev/cue-tool" - -capture contract-before env \ - AGENT_LAB_CUE_TOOL_DIR="$repo_root/.cache/dev/tools/cue" \ - "$copy_root/scripts/experiment" check -- \ - "$copy_root/tests/experiment/fixtures/valid.json" -python3 - "$copy_root/contracts/experiment/v0alpha1/schema.cue" <<'PY' -from pathlib import Path -import sys + awk '/^(PASS|FAIL) [A-Z0-9-]+ / {print}' "$output" + awk '/^(PASS|FAIL) [A-Z0-9-]+ / {print $2}' "$output" >> "$observed" + reported_assertions="$(awk '/^(PASS|FAIL) [A-Z0-9-]+ / {count++} END {print count + 0}' "$output")" + reported_failures="$(awk '/^FAIL [A-Z0-9-]+ / {count++} END {print count + 0}' "$output")" + expected_summary="SUMMARY assertions=$reported_assertions expected=$reported_assertions failures=$reported_failures infra=0" + matching_summaries="$(grep -Fxc "$expected_summary" "$output" || true)" + all_summaries="$(grep -c '^SUMMARY ' "$output" || true)" + if [ "$matching_summaries" -ne 1 ] || [ "$all_summaries" -ne 1 ]; then + infrastructure=1 + fi + case "$rc" in + 0) + [ "$reported_failures" -eq 0 ] || infrastructure=1 + ;; + 1) + [ "$reported_failures" -ne 0 ] || infrastructure=1 + ;; + *) + infrastructure=1 + ;; + esac +done -path = Path(sys.argv[1]) -text = path.read_text(encoding="utf-8") -path.write_text(text.replace("list.MaxItems(16)", "list.MaxItems(15)"), encoding="utf-8") -PY -capture contract-after env \ - AGENT_LAB_CUE_TOOL_DIR="$repo_root/.cache/dev/tools/cue" \ - "$copy_root/scripts/experiment" check -- \ - "$copy_root/tests/experiment/fixtures/valid.json" -if [ "$CAPTURE_RC" -eq 0 ] && - ! cmp -s "$work/contract-before.stdout" "$work/contract-after.stdout"; then - pass "constraint-only contract changes alter the bound plan identity" -else - fail "constraint-only contract change retained the old plan identity (rc=$CAPTURE_RC)" +assertions="$(wc -l < "$observed")" +failures="$(awk '/^FAIL [A-Z0-9-]+ / {count++} END {print count + 0}' "$work"/subcase-*.out 2>/dev/null)" +if ! cmp -s "$expected" "$observed"; then + failures=$((failures + 1)) fi -printf '\ninvalid: [\n' >> "$copy_root/contracts/experiment/v0alpha1/schema.cue" -capture corrupt-contract env \ - AGENT_LAB_CUE_TOOL_DIR="$repo_root/.cache/dev/tools/cue" \ - "$copy_root/scripts/experiment" check -- \ - "$copy_root/tests/experiment/fixtures/valid.json" -if [ "$CAPTURE_RC" -eq 125 ] && [ ! -s "$CAPTURE_STDOUT" ] && - grep -Fq 'INFRA Experiment' "$CAPTURE_STDERR"; then - pass "corrupt trusted CUE contract is infrastructure uncertainty" -else - fail "corrupt trusted CUE contract was blamed on the manifest (rc=$CAPTURE_RC)" +if ! cleanup_work; then + infrastructure=1 fi +trap - EXIT -if printf '%s\n' "$cue_preflight_out" | grep -Fxq 'cue version v0.17.1'; then - pass "Experiment validation uses the pinned CUE release" -else - fail "pinned CUE release is unavailable or wrong" -fi -if "$cue_tool" fmt --check --files \ - "$repo_root/contracts/experiment/v0alpha1/schema.cue" \ - "$repo_root/contracts/experiment/v0alpha1/plan.cue" \ - "$repo_root/contracts/experiment/v0alpha1/cue.mod/module.cue"; then - pass "tracked CUE contract files are canonically formatted" -else - fail "tracked CUE contract files need cue fmt" -fi -if "$cue_tool" -C "$repo_root/contracts/experiment/v0alpha1" vet -c=false ./...; then - pass "the complete CUE module is satisfiable" -else - fail "the complete CUE module contains a latent contradiction" +printf 'SUMMARY assertions=%s expected=%s failures=%s infra=%s\n' \ + "$assertions" "$expected_count" "$failures" "$infrastructure" +if [ "$infrastructure" -ne 0 ]; then + exit 125 fi - -cache_after="$work/cache.after" -tree_fingerprint "$cue_cache" > "$cache_after" -if cmp -s "$cache_before" "$cache_after"; then - pass "validation leaves the ignored pinned CUE cache unchanged" -else - fail "validation changed the ignored pinned CUE cache" +if [ "$failures" -ne 0 ]; then + exit 1 fi - -printf 'SUMMARY failures=%s\n' "$failures" -[ "$failures" -eq 0 ] +printf 'EXPERIMENT CONTRACT PASS\n' diff --git a/tests/experiment/directory-intake-cases.sh b/tests/experiment/directory-intake-cases.sh new file mode 100755 index 0000000..1e7ee19 --- /dev/null +++ b/tests/experiment/directory-intake-cases.sh @@ -0,0 +1,208 @@ +#!/usr/bin/env bash +set -euo pipefail + +repo_root="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." >/dev/null 2>&1 && pwd)" +agent_lab="$repo_root/scripts/agent-lab" +fixture="$repo_root/tests/experiment/fixtures/directories/minimal" +work="$(mktemp -d)" +trap 'find "$work" -type f -delete 2>/dev/null || true; find "$work" -depth -type d -exec rmdir {} + 2>/dev/null || true' EXIT +mkdir -p "$work/home" "$work/tmp" + +failures=0 +observed="$work/observed" +: > "$observed" +pass() { printf 'PASS %s %s\n' "$1" "$2"; printf '%s\n' "$1" >> "$observed"; } +fail() { printf 'FAIL %s %s\n' "$1" "$2"; printf '%s\n' "$1" >> "$observed"; failures=$((failures + 1)); } +capture() { + CAPTURE_RC=0 + env -i PATH=/usr/bin:/bin HOME="$work/home" TMPDIR="$work/tmp" LC_ALL=C \ + AGENT_LAB_CUE_TOOL_DIR="${AGENT_LAB_CUE_TOOL_DIR:-$repo_root/.cache/dev/tools/cue}" \ + "$@" > "$work/stdout" 2> "$work/stderr" || CAPTURE_RC=$? +} +expect_invalid() { + local id="$1" directory="$2" detail="$3" + capture "$agent_lab" experiment check "$directory" + if [ "$CAPTURE_RC" -eq 1 ] && [ ! -s "$work/stdout" ] && + grep -Fq 'FAIL Experiment manifest' "$work/stderr"; then + pass "$id" "$detail" + else + fail "$id" "$detail" + fi +} + +if [ -x "$agent_lab" ]; then + pass FMT-001 "repository agent-lab entrypoint exists" +else + fail FMT-001 "repository agent-lab entrypoint exists" +fi + +capture "$agent_lab" experiment check "$fixture" +if [ "$CAPTURE_RC" -eq 0 ] && [ ! -s "$work/stderr" ] && + jq -e '.source.kind == "directory" and (.source.digest | startswith("sha256:")) and .plan.kind == "RequestedExperimentPlan"' "$work/stdout" >/dev/null 2>&1; then + pass FMT-002 "sole-entry directory produces a source-bound checked candidate" +else + fail FMT-002 "sole-entry directory produces a source-bound checked candidate" +fi + +extra="$work/extra" +mkdir "$extra" +cp "$fixture/experiment.cue" "$extra/experiment.cue" +: > "$extra/unexpected" +capture "$agent_lab" experiment check "$extra" +if [ "$CAPTURE_RC" -eq 1 ] && [ ! -s "$work/stdout" ]; then + pass FMT-004 "an extra directory entry is stable invalid input" +else + fail FMT-004 "an extra directory entry is stable invalid input" +fi + +linked="$work/linked" +mkdir "$linked" +cp "$fixture/experiment.cue" "$work/hardlink-source" +ln "$work/hardlink-source" "$linked/experiment.cue" +capture "$agent_lab" experiment check "$linked" +if [ "$CAPTURE_RC" -eq 1 ] && [ ! -s "$work/stdout" ]; then + pass FMT-005 "a multiply linked authored file is refused" +else + fail FMT-005 "a multiply linked authored file is refused" +fi + +executable="$work/executable" +mkdir "$executable" +cp "$fixture/experiment.cue" "$executable/experiment.cue" +chmod 700 "$executable/experiment.cue" +capture "$agent_lab" experiment check "$executable" +if [ "$CAPTURE_RC" -eq 1 ] && [ ! -s "$work/stdout" ]; then + pass FMT-006 "an executable authored file is refused" +else + fail FMT-006 "an executable authored file is refused" +fi + +expected_digest="$(python3 - "$fixture/experiment.cue" <<'PY' +from hashlib import sha256 +from pathlib import Path +import sys +data = Path(sys.argv[1]).read_bytes() +name = b"experiment.cue" +digest = sha256(b"agent-lab.experiment-tree.v1\0") +digest.update(len(name).to_bytes(4, "big")) +digest.update(name) +digest.update(len(data).to_bytes(8, "big")) +digest.update(data) +print("sha256:" + digest.hexdigest()) +PY +)" +capture "$agent_lab" experiment check "$fixture" +if [ "$CAPTURE_RC" -eq 0 ] && [ "$(jq -r '.source.digest' "$work/stdout")" = "$expected_digest" ]; then + pass FMT-007 "source identity uses the independently framed exact bytes" +else + fail FMT-007 "source identity uses the independently framed exact bytes" +fi + +capture "$agent_lab" experiment check "$fixture" +cp "$work/stdout" "$work/second" +capture "$agent_lab" experiment check "$fixture" +if [ "$CAPTURE_RC" -eq 0 ] && cmp -s "$work/second" "$work/stdout"; then + pass FMT-003 "repeated directory checks are byte-identical" +else + fail FMT-003 "repeated directory checks are byte-identical" +fi + +symlinked="$work/symlinked" +mkdir "$symlinked" +ln -s "$fixture/experiment.cue" "$symlinked/experiment.cue" +expect_invalid FMT-008 "$symlinked" "an authored symlink is refused" + +mutable="$work/mutable" +mkdir "$mutable" +sed 's/@sha256:[a-f0-9]\{64\}/:latest/' "$fixture/experiment.cue" > "$mutable/experiment.cue" +expect_invalid SEL-001 "$mutable" "a mutable OCI reference is refused" + +unknown="$work/unknown" +mkdir "$unknown" +sed '/kind:/a\\\tunexpected: true' "$fixture/experiment.cue" > "$unknown/experiment.cue" +expect_invalid CUE-001 "$unknown" "an unknown authored field is refused" + +commented="$work/commented" +mkdir "$commented" +{ printf '// distinct source bytes\n'; cat "$fixture/experiment.cue"; } > "$commented/experiment.cue" +capture "$agent_lab" experiment check "$fixture" +cp "$work/stdout" "$work/base-candidate" +capture "$agent_lab" experiment check "$commented" +if [ "$CAPTURE_RC" -eq 0 ] && + [ "$(jq -cS '.plan' "$work/base-candidate")" = "$(jq -cS '.plan' "$work/stdout")" ] && + [ "$(jq -r '.source.digest' "$work/base-candidate")" != "$(jq -r '.source.digest' "$work/stdout")" ]; then + pass FMT-009 "non-semantic CUE comments change source but not plan identity" +else + fail FMT-009 "non-semantic CUE comments change source but not plan identity" +fi + +if python3 -I - "$repo_root/scripts/experiment.py" "$fixture" <<'PY' +from importlib.util import module_from_spec, spec_from_file_location +from pathlib import Path +import sys +spec = spec_from_file_location("experiment_mutant", sys.argv[1]) +assert spec is not None and spec.loader is not None +module = module_from_spec(spec) +sys.modules[spec.name] = module +spec.loader.exec_module(module) +original = module.read_directory_snapshot(sys.argv[2]).digest +module.SOURCE_DIGEST_DOMAIN = b"agent-lab.insecure-unframed\0" +mutated = module.read_directory_snapshot(sys.argv[2]).digest +assert original != mutated +PY +then + pass M-FMT-001 "source-domain mutation changes the independent identity" +else + fail M-FMT-001 "source-domain mutation changes the independent identity" +fi + +if python3 -I - "$repo_root/scripts/experiment.py" <<'PY' +from importlib.util import module_from_spec, spec_from_file_location +from pathlib import Path +import sys +spec = spec_from_file_location("experiment_resolver", sys.argv[1]) +assert spec is not None and spec.loader is not None +module = module_from_spec(spec) +sys.modules[spec.name] = module +spec.loader.exec_module(module) +subject = "registry.example/lab/base@sha256:" + "b" * 64 +plan = { + "apiVersion": "agent-lab.request/v0alpha1", + "contract": {"digest": "sha256:" + "a" * 64, "name": "agent-lab.experiment", "version": "v0alpha1"}, + "kind": "RequestedExperimentPlan", + "metadata": {"requestedName": "resolver-test"}, + "spec": {"members": [{ + "command": [], "name": "one", "resourceClass": "small", + "requestedSelector": {"catalogName": "agent-lab.base"}, + }]}, +} +catalog = { + "apiVersion": "agent-lab.experiment-images/v0alpha1", + "entries": [{"name": "agent-lab.base", "subject": subject}], +} +resolved = module.resolve_plan(plan, Path("/nonexistent"), catalog) +image = resolved["spec"]["members"][0]["resolvedImage"] +assert image["origin"] == "agent-lab" +assert image["subject"] == subject +assert image["generation"] == 1 +assert image["entryDigest"].startswith("sha256:") +assert module.valid_catalog_name("vendor.image") +for invalid in ("Agent.image", "vendor.image.extra", "vendor_1.image", "a" * 32 + ".image", "vendor.é"): + assert not module.valid_catalog_name(invalid), invalid +PY +then + pass SEL-002 "trusted bundled resolution binds exact names, subjects, and entry identity" +else + fail SEL-002 "trusted bundled resolution binds exact names, subjects, and entry identity" +fi + +expected="$work/expected" +printf '%s\n' \ + FMT-001 FMT-002 FMT-004 FMT-005 FMT-006 FMT-007 FMT-003 \ + FMT-008 SEL-001 CUE-001 FMT-009 M-FMT-001 SEL-002 > "$expected" +if ! cmp -s "$expected" "$observed"; then + printf 'INFRA assertion identity drift\n' >&2 + exit 125 +fi +printf 'SUMMARY assertions=13 expected=13 failures=%s infra=0\n' "$failures" +[ "$failures" -eq 0 ] diff --git a/tests/experiment/fixtures/directories/minimal/experiment.cue b/tests/experiment/fixtures/directories/minimal/experiment.cue new file mode 100644 index 0000000..a1d8c8c --- /dev/null +++ b/tests/experiment/fixtures/directories/minimal/experiment.cue @@ -0,0 +1,12 @@ +package experiment + +experiment: { + apiVersion: "agent-lab/v0alpha1" + kind: "Experiment" + metadata: name: "first-experiment" + spec: members: [{ + name: "coordinator" + image: digestRef: "registry.example/team/coordinator@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + command: ["serve"] + }] +} diff --git a/tests/experiment/git-intake-cases.py b/tests/experiment/git-intake-cases.py new file mode 100755 index 0000000..6a69500 --- /dev/null +++ b/tests/experiment/git-intake-cases.py @@ -0,0 +1,3401 @@ +#!/usr/bin/env python3 +"""Deterministic public-contract cases for pinned Git Experiment intake.""" + +from __future__ import annotations + +from contextlib import redirect_stderr, redirect_stdout +import base64 +from hashlib import sha1, sha256 +from importlib.util import module_from_spec, spec_from_file_location +import io +import json +import os +from pathlib import Path +import shutil +import signal +import socket +import stat +import subprocess +import sys +import tempfile + + +EXPECTED = ( + "GIT-CLI-001", + "GIT-USAGE-001", + "GIT-URL-001", + "GIT-OID-001", + "GIT-PLAT-001", + "GIT-FIXTURE-001", + "GIT-PIN-001", + "GIT-COMMIT-001", + "GIT-ROOT-001", + "GIT-TYPE-001", + "GIT-BLOB-001", + "GIT-DRIFT-001", + "GIT-AUTHORITY-001", + "GIT-CREDENTIAL-001", + "GIT-REDIRECT-001", + "GIT-CONTENT-001", + "GIT-TIMEOUT-001", + "GIT-OUTPUT-001", + "GIT-ACQUIRE-001", + "GIT-PGROUP-001", + "GIT-CLEANUP-001", + "GIT-TAXONOMY-001", + "GIT-CHECK-001", + "GIT-AUTH-001", + "GIT-DENY-001", + "GIT-INSTALL-001", + "GIT-IDENTITY-001", + "GIT-RETRY-001", + "GIT-ADAPTER-001", + "GIT-NOEF-001", + "GIT-RUNTIME-001", + "GIT-DIAG-001", +) +URL = "https://github.com/uscient/experiment-fixture.git" +COMMIT = "1cffa1a28f96d2f2cb898b1bad70d281e359a5b5" +TREE = "64564b8e82ec9581c32cb4951ed802b544e2e0c0" +BLOB = "a1d8c8cd0f1865e66cb2463cbaa801c4b5a85656" +RAW_SOURCE_SHA256 = "efd32f249a63704830bbb9e83902fd501a5857f43b54b7dc34874ef1a9e1e593" +SOURCE_DIGEST = "sha256:463e8a7622e58281fd975d58d8a9ad44ed997dd08af32e237f1476021f7abb23" + + +def git_oid(kind: str, payload: bytes) -> str: + framed = kind.encode("ascii") + b" " + str(len(payload)).encode("ascii") + b"\0" + payload + return sha1(framed).hexdigest() + + +def response(body: object) -> tuple[int, tuple[tuple[str, str], ...], bytes]: + encoded = json.dumps(body, ensure_ascii=True, separators=(",", ":"), sort_keys=True).encode() + return ( + 200, + (("content-length", str(len(encoded))), ("content-type", "application/json; charset=utf-8")), + encoded, + ) + + +def load_module(path: Path, label: str): + spec = spec_from_file_location(label, path) + if spec is None or spec.loader is None: + raise RuntimeError(f"cannot load {path}") + module = module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +def invoke(module, argv: list[str]) -> tuple[int, str, str]: + stdout = io.StringIO() + stderr = io.StringIO() + with redirect_stdout(stdout), redirect_stderr(stderr): + result = module.main(argv) + return result, stdout.getvalue(), stderr.getvalue() + + +def invoke_with_exit(module, argv: list[str]) -> tuple[int, str, str]: + stdout = io.StringIO() + stderr = io.StringIO() + with redirect_stdout(stdout), redirect_stderr(stderr): + try: + result = module.main(argv) + except SystemExit as error: + result = error.code if isinstance(error.code, int) else 1 + return result, stdout.getvalue(), stderr.getvalue() + + +def tree_fingerprint(root: Path) -> str: + records: list[tuple[str, str, int, str]] = [] + if root.exists(): + for path in sorted( + root.rglob("*"), key=lambda item: os.fsencode(str(item.relative_to(root))) + ): + metadata = path.lstat() + relative = str(path.relative_to(root)) + mode = stat.S_IMODE(metadata.st_mode) + if stat.S_ISREG(metadata.st_mode): + kind = "file" + content = sha256(path.read_bytes()).hexdigest() + elif stat.S_ISDIR(metadata.st_mode): + kind = "directory" + content = "" + elif stat.S_ISLNK(metadata.st_mode): + kind = "symlink" + content = os.readlink(path) + else: + kind = "other" + content = "" + records.append((relative, kind, mode, content)) + return sha256(repr(records).encode("utf-8")).hexdigest() + + +def make_writable(root: Path) -> None: + if not root.exists(): + return + for path in root.rglob("*"): + try: + metadata = path.lstat() + if stat.S_ISDIR(metadata.st_mode): + path.chmod(stat.S_IMODE(metadata.st_mode) | stat.S_IRWXU) + elif stat.S_ISREG(metadata.st_mode): + path.chmod(stat.S_IMODE(metadata.st_mode) | stat.S_IRUSR | stat.S_IWUSR) + except OSError: + pass + + +def main() -> int: + repo = Path(__file__).resolve().parents[2] + failures = 0 + infrastructure = 0 + results: list[tuple[str, bool, str]] = [] + try: + agent_lab = load_module(repo / "scripts/agent-lab.py", "git_intake_agent_lab") + calls: list[tuple[str, ...]] = [] + + class RecordingExperiment: + @staticmethod + def main(argv: list[str]) -> int: + calls.append(tuple(argv)) + return 0 + + store_calls: list[tuple[Path, str, str]] = [] + + class RecordingStore: + class StoreReject(Exception): + pass + + class StoreInfrastructure(Exception): + pass + + @staticmethod + def install_git(home: Path, url: str, commit: str) -> dict[str, object]: + store_calls.append((home, url, commit)) + return { + "changed": True, + "installationKey": "sha256:" + "1" * 64, + "name": "fixture", + "receiptDigest": "sha256:" + "2" * 64, + } + + agent_lab.experiment_module = lambda: RecordingExperiment + with tempfile.TemporaryDirectory(prefix="agent-lab-git-cli-") as raw_home: + prior_home = os.environ.get("AGENT_LAB_HOME") + os.environ["AGENT_LAB_HOME"] = raw_home + try: + check = invoke( + agent_lab, + ["experiment", "check", "--git", URL, "--commit", COMMIT], + ) + authorize = invoke( + agent_lab, + [ + "experiment", + "authorize", + "install", + "--git", + URL, + "--commit", + COMMIT, + ], + ) + original_load_config_receipt = agent_lab.load_config_receipt + original_store_module = agent_lab.experiment_store_module + agent_lab.load_config_receipt = lambda _home: ( + {"paths": {"cache": "cache"}}, + b"fixture", + ) + agent_lab.experiment_store_module = lambda: RecordingStore + try: + install = invoke( + agent_lab, + [ + "experiment", + "install", + "--git", + URL, + "--commit", + COMMIT, + ], + ) + finally: + agent_lab.load_config_receipt = original_load_config_receipt + agent_lab.experiment_store_module = original_store_module + expected_calls = [ + ("experiment.py", "check-git", URL, COMMIT), + ("experiment.py", "authorize-git", URL, COMMIT), + ] + results.append( + ( + "GIT-CLI-001", + check == (0, "", "") + and authorize == (0, "", "") + and install[0] == 0 + and install[2] == "" + and calls == expected_calls + and store_calls == [(Path(raw_home), URL, COMMIT)], + "exact pinned Git public forms route once to their adapters", + ) + ) + + calls.clear() + store_calls.clear() + malformed = ( + ["experiment", "check", "--git"], + ["experiment", "check", "--git", URL], + ["experiment", "check", "--git", URL, "--commit"], + ["experiment", "check", "--commit", COMMIT, "--git", URL], + ["experiment", "check", "--git", URL, "--commit", COMMIT, "extra"], + ["experiment", "authorize", "install", "--git", URL, "--commit"], + ["experiment", "install", "--git"], + ["experiment", "install", "--git", URL], + ["experiment", "install", "--git", URL, "--commit"], + ["experiment", "install", "--git", URL, "--commit", COMMIT, "extra"], + ) + usage_results = [invoke(agent_lab, list(argv)) for argv in malformed] + results.append( + ( + "GIT-USAGE-001", + all(result[0] == 2 and result[1] == "" for result in usage_results) + and calls == [] + and store_calls == [], + "malformed Git option shapes fail before adapter access", + ) + ) + + source = ( + repo / "tests/experiment/fixtures/directories/minimal/experiment.cue" + ).read_bytes() + tree_payload = b"100644 experiment.cue\0" + bytes.fromhex(BLOB) + commit_payload = ( + f"tree {TREE}\n" + "author Fixture 0 +0000\n" + "committer Fixture 0 +0000\n" + "\n" + "pinned fixture\n" + ).encode("ascii") + fixture_exact = ( + len(source) == 326 + and sha256(source).hexdigest() == RAW_SOURCE_SHA256 + and git_oid("blob", source) == BLOB + and git_oid("tree", tree_payload) == TREE + and git_oid("commit", commit_payload) == COMMIT + ) + commit_body = {"sha": COMMIT, "tree": {"sha": TREE}} + tree_body = { + "sha": TREE, + "tree": [ + { + "mode": "100644", + "path": "experiment.cue", + "sha": BLOB, + "size": len(source), + "type": "blob", + } + ], + "truncated": False, + } + blob_body = { + "content": base64.b64encode(source).decode("ascii"), + "encoding": "base64", + "sha": BLOB, + "size": len(source), + } + fixture_responses = { + f"/repos/uscient/experiment-fixture/git/commits/{COMMIT}": response(commit_body), + f"/repos/uscient/experiment-fixture/git/trees/{TREE}": response(tree_body), + f"/repos/uscient/experiment-fixture/git/blobs/{BLOB}": response(blob_body), + } + request_calls: list[tuple[str, str, tuple[tuple[str, str], ...], int]] = [] + + def requester(authority, path, headers, maximum, _deadline): + request_calls.append((authority, path, tuple(headers), maximum)) + return fixture_responses[path] + + experiment = load_module(repo / "scripts/experiment.py", "git_intake_experiment") + + def acquire( + responses: dict[str, tuple[int, tuple[tuple[str, str], ...], bytes]], + *, + source_url: str = URL, + source_commit: str = COMMIT, + ): + calls: list[tuple[str, str, tuple[tuple[str, str], ...], int]] = [] + + def fixture_requester(authority, path, headers, maximum, _deadline): + calls.append((authority, path, tuple(headers), maximum)) + return responses[path] + + try: + value = experiment.read_git_snapshot( + source_url, + source_commit, + requester=fixture_requester, + ) + return "ok", value, calls + except experiment.InvalidManifest as error: + return "reject", str(error), calls + except experiment.InfrastructureError as error: + return "infra", str(error), calls + + unused_request_calls: list[str] = [] + + def unused_requester(_authority, path, _headers, _maximum, _deadline): + unused_request_calls.append(path) + raise AssertionError("invalid input reached acquisition") + + invalid_urls = ( + "http://github.com/uscient/experiment-fixture.git", + "https://example.com/uscient/experiment-fixture.git", + "https://user@github.com/uscient/experiment-fixture.git", + "https://github.com:443/uscient/experiment-fixture.git", + "https://github.com/uscient/experiment-fixture.git?ref=main", + "https://github.com/uscient/experiment-fixture.git#main", + "https://github.com/uscient/experiment-fixture", + "https://github.com/uscient/%65xperiment-fixture.git", + "https://github.com/uscient/experiment-fixture.git/extra", + "https://github.com/uscient/experiment-fixture.git\n", + ) + url_rejections = [] + for invalid_url in invalid_urls: + try: + experiment.read_git_snapshot( + invalid_url, COMMIT, requester=unused_requester + ) + except experiment.InvalidManifest as error: + url_rejections.append("GIT-URL" in str(error)) + except experiment.InfrastructureError: + url_rejections.append(False) + results.append( + ( + "GIT-URL-001", + all(url_rejections) and unused_request_calls == [], + "only normalized unauthenticated GitHub HTTPS URLs are accepted", + ) + ) + + invalid_oids = ( + "main", + "refs/heads/main", + "1" * 7, + "1" * 39, + "1" * 41, + "A" * 40, + "g" * 40, + "-" * 40, + ) + oid_rejections = [] + for invalid_oid in invalid_oids: + try: + experiment.read_git_snapshot( + URL, invalid_oid, requester=unused_requester + ) + except experiment.InvalidManifest as error: + oid_rejections.append("GIT-OID" in str(error)) + except experiment.InfrastructureError: + oid_rejections.append(False) + results.append( + ( + "GIT-OID-001", + all(oid_rejections) and unused_request_calls == [], + "mutable refs and non-full object IDs are rejected before acquisition", + ) + ) + + original_platform = experiment.sys.platform + experiment.sys.platform = "darwin" + try: + platform_outcome = None + try: + experiment.read_git_snapshot(URL, COMMIT, requester=unused_requester) + except experiment.InfrastructureError as error: + platform_outcome = str(error) + finally: + experiment.sys.platform = original_platform + + public_platform_access: list[str] = [] + + def platform_trap(label: str): + def fail(*_arguments, **_keywords): + public_platform_access.append(label) + raise AssertionError(f"unsupported platform reached {label}") + + return fail + + original_agent_platform = agent_lab.sys.platform + original_agent_experiment_module = agent_lab.experiment_module + original_load_config_receipt = agent_lab.load_config_receipt + original_store_module = agent_lab.experiment_store_module + agent_lab.sys.platform = "darwin" + agent_lab.experiment_module = platform_trap("experiment module") + agent_lab.load_config_receipt = platform_trap("home receipt") + agent_lab.experiment_store_module = platform_trap("store module") + try: + public_platform_results = ( + invoke( + agent_lab, + ["experiment", "check", "--git", URL, "--commit", COMMIT], + ), + invoke( + agent_lab, + [ + "experiment", + "authorize", + "install", + "--git", + URL, + "--commit", + COMMIT, + ], + ), + invoke( + agent_lab, + [ + "experiment", + "install", + "--git", + URL, + "--commit", + COMMIT, + ], + ), + ) + finally: + agent_lab.sys.platform = original_agent_platform + agent_lab.experiment_module = original_agent_experiment_module + agent_lab.load_config_receipt = original_load_config_receipt + agent_lab.experiment_store_module = original_store_module + results.append( + ( + "GIT-PLAT-001", + platform_outcome is not None + and "GIT-PLATFORM" in platform_outcome + and unused_request_calls == [] + and all(result[0] == 125 for result in public_platform_results) + and public_platform_access == [], + "unsupported hosts refuse every public form before authority access", + ) + ) + + try: + snapshot = experiment.read_git_snapshot( + URL, COMMIT, requester=requester + ) + except (AttributeError, experiment.InvalidManifest, experiment.InfrastructureError): + snapshot = None + acquired_bytes = sum(len(item[2]) for item in fixture_responses.values()) + expected_transport = { + "acquisition": { + "acquiredBytes": acquired_bytes, + "limitBytes": 1_048_576, + "method": "github-git-data-v3", + "requestCount": 3, + "temporaryBytes": 0, + "temporaryFiles": 0, + }, + "blob": f"sha1:{BLOB}", + "commit": f"sha1:{COMMIT}", + "kind": "git", + "requestedCommit": COMMIT, + "tree": f"sha1:{TREE}", + "url": URL, + } + results.append( + ( + "GIT-FIXTURE-001", + fixture_exact + and snapshot is not None + and snapshot.data == source + and snapshot.digest == SOURCE_DIGEST + and snapshot.transport == expected_transport + and [call[0] for call in request_calls] + == ["api.github.com"] * 3 + and [call[1] for call in request_calls] + == list(fixture_responses), + "independent pinned Git fixture normalizes to one closed snapshot", + ) + ) + + results.append( + ( + "GIT-PIN-001", + [call[1] for call in request_calls] + == [ + f"/repos/uscient/experiment-fixture/git/commits/{COMMIT}", + f"/repos/uscient/experiment-fixture/git/trees/{TREE}", + f"/repos/uscient/experiment-fixture/git/blobs/{BLOB}", + ] + and all("heads" not in call[1] and "tags" not in call[1] for call in request_calls), + "acquisition follows only the exact pinned object chain", + ) + ) + + mismatched_commit = dict(fixture_responses) + mismatched_commit[next(iter(fixture_responses))] = response( + {"sha": "0" * 40, "tree": {"sha": TREE}} + ) + commit_outcome = acquire(mismatched_commit) + results.append( + ( + "GIT-COMMIT-001", + commit_outcome[0] == "infra" + and "GIT-COMMIT" in commit_outcome[1], + "provider commit identity must equal the exact request", + ) + ) + + extra_tree = dict(tree_body) + extra_tree["tree"] = list(tree_body["tree"]) + [ + { + "mode": "100644", + "path": "extra", + "sha": BLOB, + "size": len(source), + "type": "blob", + } + ] + root_responses = dict(fixture_responses) + root_responses[f"/repos/uscient/experiment-fixture/git/trees/{TREE}"] = response( + extra_tree + ) + root_outcome = acquire(root_responses) + results.append( + ( + "GIT-ROOT-001", + root_outcome[0] == "reject" and "GIT-ROOT" in root_outcome[1], + "extra root entries fail closed", + ) + ) + + type_outcomes = [] + for path, mode, kind in ( + ("experiment.cue", "100755", "blob"), + ("experiment.cue", "120000", "blob"), + ("experiment.cue", "160000", "commit"), + ("experiment.cue/nested", "100644", "blob"), + ("experiment.cue", "040000", "tree"), + ): + changed_tree = dict(tree_body) + changed_tree["tree"] = [ + { + "mode": mode, + "path": path, + "sha": BLOB, + "size": len(source), + "type": kind, + } + ] + changed_responses = dict(fixture_responses) + changed_responses[ + f"/repos/uscient/experiment-fixture/git/trees/{TREE}" + ] = response(changed_tree) + type_outcomes.append(acquire(changed_responses)) + results.append( + ( + "GIT-TYPE-001", + all( + outcome[0] == "reject" and "GIT-TYPE" in outcome[1] + for outcome in type_outcomes + ), + "non-regular root modes and nested paths fail closed", + ) + ) + + encoded_source = blob_body["content"] + assert isinstance(encoded_source, str) + wrapped_blob = dict(blob_body) + wrapped_blob["content"] = "\n".join( + encoded_source[index : index + 60] + for index in range(0, len(encoded_source), 60) + ) + "\n" + wrapped_responses = dict(fixture_responses) + wrapped_responses[f"/repos/uscient/experiment-fixture/git/blobs/{BLOB}"] = response( + wrapped_blob + ) + wrapped_outcome = acquire(wrapped_responses) + corrupt_blob = dict(blob_body) + corrupt_blob["content"] = base64.b64encode(source + b"x").decode("ascii") + corrupt_responses = dict(fixture_responses) + corrupt_responses[f"/repos/uscient/experiment-fixture/git/blobs/{BLOB}"] = response( + corrupt_blob + ) + corrupt_outcome = acquire(corrupt_responses) + same_length_source = source.replace(b'"serve"', b'"shell"', 1) + same_length_blob = dict(blob_body) + same_length_blob["content"] = base64.b64encode( + same_length_source + ).decode("ascii") + same_length_responses = dict(fixture_responses) + same_length_responses[ + f"/repos/uscient/experiment-fixture/git/blobs/{BLOB}" + ] = response(same_length_blob) + same_length_outcome = acquire(same_length_responses) + results.append( + ( + "GIT-BLOB-001", + wrapped_outcome[0] == "ok" + and wrapped_outcome[1].data == source + and corrupt_outcome[0] == "infra" + and "GIT-BLOB" in corrupt_outcome[1] + and same_length_source != source + and len(same_length_source) == len(source) + and git_oid("blob", same_length_source) != BLOB + and same_length_blob["sha"] == BLOB + and same_length_blob["size"] == len(source) + and same_length_outcome[0] == "infra" + and same_length_outcome[1] + == "git provider GIT-BLOB object identity is inconsistent", + "documented base64 wrapping is accepted but object drift is not", + ) + ) + + drift_tree = dict(tree_body) + drift_tree["sha"] = "0" * 40 + drift_responses = dict(fixture_responses) + drift_responses[f"/repos/uscient/experiment-fixture/git/trees/{TREE}"] = response( + drift_tree + ) + drift_outcome = acquire(drift_responses) + + bound_drift_source = same_length_source + bound_drift_blob = git_oid("blob", bound_drift_source) + bound_drift_blob_path = ( + f"/repos/uscient/experiment-fixture/git/blobs/{bound_drift_blob}" + ) + bound_drift_tree = dict(tree_body) + bound_drift_tree["tree"] = [ + { + "mode": "100644", + "path": "experiment.cue", + "sha": bound_drift_blob, + "size": len(bound_drift_source), + "type": "blob", + } + ] + bound_drift_responses = dict(fixture_responses) + bound_drift_responses[ + f"/repos/uscient/experiment-fixture/git/trees/{TREE}" + ] = response(bound_drift_tree) + bound_drift_responses[bound_drift_blob_path] = response( + { + "content": base64.b64encode(bound_drift_source).decode("ascii"), + "encoding": "base64", + "sha": bound_drift_blob, + "size": len(bound_drift_source), + } + ) + bound_drift_outcome = acquire(bound_drift_responses) + bound_drift_paths = [call[1] for call in bound_drift_outcome[2]] + results.append( + ( + "GIT-DRIFT-001", + drift_outcome[0] == "infra" + and "GIT-TREE" in drift_outcome[1] + and bound_drift_source != source + and len(bound_drift_source) == len(source) + and bound_drift_blob != BLOB + and bound_drift_tree["sha"] == TREE + and bound_drift_outcome[0] == "infra" + and bound_drift_outcome[1] + == "git provider GIT-TREE object identity is inconsistent" + and bound_drift_paths + == [ + f"/repos/uscient/experiment-fixture/git/commits/{COMMIT}", + f"/repos/uscient/experiment-fixture/git/trees/{TREE}", + ] + and bound_drift_blob_path not in bound_drift_paths, + "changed object output is infrastructure uncertainty", + ) + ) + + expected_headers = ( + ("Accept", "application/vnd.github+json"), + ("Accept-Encoding", "identity"), + ("User-Agent", "agent-lab/v0alpha1"), + ("X-GitHub-Api-Version", "2022-11-28"), + ) + + ca_bytes = b"-----BEGIN CERTIFICATE-----\nfixture\n-----END CERTIFICATE-----\n" + ca_offset = [0] + ca_closed: list[int] = [] + original_ca_paths = experiment.GIT_PROVIDER_CA_FILES + original_open = experiment.os.open + original_fstat = experiment.os.fstat + original_read = experiment.os.read + original_close = experiment.os.close + + class CaMetadata: + st_mode = stat.S_IFREG | 0o644 + st_uid = 0 + st_nlink = 1 + st_size = len(ca_bytes) + + def ca_open(path, _flags): + if path == "/missing-ca": + raise FileNotFoundError(path) + if path != "/trusted-ca": + raise AssertionError("unexpected CA path") + return 73 + + def ca_fstat(descriptor): + if descriptor != 73: + raise AssertionError("unexpected CA descriptor") + return CaMetadata() + + def ca_read(descriptor, maximum): + if descriptor != 73: + raise AssertionError("unexpected CA descriptor") + start = ca_offset[0] + chunk = ca_bytes[start : start + min(maximum, 11)] + ca_offset[0] += len(chunk) + return chunk + + def ca_close(descriptor): + ca_closed.append(descriptor) + + experiment.GIT_PROVIDER_CA_FILES = ("/missing-ca", "/trusted-ca") + experiment.os.open = ca_open + experiment.os.fstat = ca_fstat + experiment.os.read = ca_read + experiment.os.close = ca_close + try: + try: + ca_pem = experiment._git_system_ca_pem() + except experiment.InfrastructureError: + ca_pem = None + finally: + experiment.GIT_PROVIDER_CA_FILES = original_ca_paths + experiment.os.open = original_open + experiment.os.fstat = original_fstat + experiment.os.read = original_read + experiment.os.close = original_close + ca_probe_ok = ca_pem == ca_bytes.decode("ascii") and ca_closed == [73] + + import http.client as http_client + import ssl as ssl_module + + direct_body = b'{"fixture":true}' + direct_connections = [] + direct_response_specs = [ + ( + 200, + [ + ("Content-Length", str(len(direct_body))), + ("Content-Type", "application/json; charset=utf-8"), + ("Content-Encoding", "identity"), + ], + direct_body, + ) + ] + original_https_connection = http_client.HTTPSConnection + original_ssl_context = ssl_module.SSLContext + original_maxline = http_client._MAXLINE + original_maxheaders = http_client._MAXHEADERS + original_ca_reader = experiment._git_system_ca_pem + + class FakeSocket: + def __init__(self): + self.timeouts = [] + + def settimeout(self, value): + self.timeouts.append(value) + + class FakeResponse: + def __init__(self, status, headers, body): + self.status = status + self.headers = headers + self.body = body + self.offset = 0 + self.closed = False + + def getheaders(self): + return self.headers + + def read(self, maximum): + if self.offset >= len(self.body): + return b"" + chunk = self.body[self.offset : self.offset + maximum] + self.offset += len(chunk) + return chunk + + def close(self): + self.closed = True + + class FakeContext: + def __init__(self, protocol): + self.protocol = protocol + self.check_hostname = None + self.verify_mode = None + self.minimum_version = None + self.ca_data = None + + def load_verify_locations(self, *, cadata): + self.ca_data = cadata + + class FakeConnection: + def __init__(self, authority, *, port, timeout, context): + self.authority = authority + self.port = port + self.timeout = timeout + self.context = context + self.sock = FakeSocket() + if direct_response_specs: + spec = direct_response_specs.pop(0) + else: + spec = ( + 200, + [ + ("Content-Length", str(len(direct_body))), + ( + "Content-Type", + "application/json; charset=utf-8", + ), + ("Content-Encoding", "identity"), + ], + direct_body, + ) + self.response = FakeResponse(*spec) + self.request_record = None + self.closed = False + direct_connections.append(self) + + def request(self, method, path, *, headers): + self.request_record = (method, path, headers) + + def getresponse(self): + return self.response + + def close(self): + self.closed = True + + http_client.HTTPSConnection = FakeConnection + ssl_module.SSLContext = FakeContext + experiment._git_system_ca_pem = lambda: ca_bytes.decode("ascii") + try: + direct_deadline = __import__("time").monotonic() + 1.0 + direct_https = experiment._github_api_request( + "api.github.com", + f"/repos/uscient/experiment-fixture/git/commits/{COMMIT}", + expected_headers, + 1_024, + direct_deadline, + ) + try: + experiment._github_api_request( + "credential.invalid", + "/repos/uscient/experiment-fixture/git/commits/invalid", + expected_headers, + 1_024, + direct_deadline, + ) + direct_invalid = "accepted" + except experiment.InfrastructureError as error: + direct_invalid = str(error) + + def direct_status_outcome(status, headers, body, maximum=1_024): + direct_response_specs.append((status, headers, body)) + try: + result = experiment._github_api_request( + "api.github.com", + f"/repos/uscient/experiment-fixture/git/commits/{COMMIT}", + expected_headers, + maximum, + direct_deadline, + ) + try: + experiment._git_provider_json( + lambda *_args: result, + f"/repos/uscient/experiment-fixture/git/commits/{COMMIT}", + maximum, + direct_deadline, + stable_not_found=True, + ) + return "ok" + except experiment.InvalidManifest: + return "reject" + except experiment.InfrastructureError: + return "infra" + except experiment.InfrastructureError: + return "infra" + + framed_headers = [ + ("Content-Length", "2"), + ("Content-Type", "application/json; charset=utf-8"), + ("Content-Encoding", "identity"), + ] + hostile_status_outcomes = ( + direct_status_outcome( + 404, + [*framed_headers, ("Transfer-Encoding", "chunked")], + b"{}", + ), + direct_status_outcome( + 422, + [ + ("Content-Length", "2"), + ("Content-Type", "application/json; charset=utf-8"), + ("Content-Encoding", "gzip"), + ], + b"{}", + ), + direct_status_outcome( + 404, + [ + ("Content-Length", "2"), + ("Content-Type", "text/plain"), + ("Content-Encoding", "identity"), + ], + b"{}", + ), + direct_status_outcome( + 422, + [ + ("Content-Length", "3"), + ("Content-Type", "application/json; charset=utf-8"), + ("Content-Encoding", "identity"), + ], + b"{}", + ), + direct_status_outcome( + 404, + [ + ("Content-Length", "5"), + ("Content-Type", "application/json; charset=utf-8"), + ("Content-Encoding", "identity"), + ], + b"12345", + maximum=4, + ), + direct_status_outcome( + 404, + [ + ("Content-Length", "1"), + ("Content-Type", "application/json; charset=utf-8"), + ("Content-Encoding", "identity"), + ], + b"{", + ), + direct_status_outcome(404, framed_headers, b"{}"), + direct_status_outcome(422, framed_headers, b"{}"), + ) + + def injected_length_outcome(declared): + try: + experiment._git_provider_json( + lambda *_args: ( + 200, + ( + ("content-length", declared), + ( + "content-type", + "application/json; charset=utf-8", + ), + ), + b"{}", + ), + f"/repos/uscient/experiment-fixture/git/commits/{COMMIT}", + 1_024, + direct_deadline, + stable_not_found=True, + ) + return "ok" + except experiment.InfrastructureError: + return "infra" + + injected_lengths_closed = ( + injected_length_outcome("2") == "ok" + and all( + injected_length_outcome(value) == "infra" + for value in (" 2", "+2", "02") + ) + ) + + def forced_stable_outcome(path): + try: + experiment._git_provider_json( + lambda *_args: ( + 404, + ( + ("content-length", "2"), + ( + "content-type", + "application/json; charset=utf-8", + ), + ), + b"{}", + ), + path, + 1_024, + direct_deadline, + stable_not_found=True, + ) + return "ok" + except experiment.InvalidManifest: + return "reject" + except experiment.InfrastructureError: + return "infra" + + stable_not_found_route_closed = all( + forced_stable_outcome(path) == "infra" + for path in ( + f"/repos/uscient/experiment-fixture/git/trees/{TREE}", + f"/repos/uscient/experiment-fixture/git/blobs/{BLOB}", + ) + ) + route_connections_before = len(direct_connections) + route_outcomes = [] + for hostile_path in ( + f"/repos/uscient/experiment-fixture/git/commits/{COMMIT}/../trees/{TREE}", + f"/repos/uscient/experiment-fixture/git/commits/{COMMIT}?recursive=1", + ): + try: + experiment._github_api_request( + "api.github.com", + hostile_path, + expected_headers, + 1_024, + direct_deadline, + ) + route_outcomes.append("accepted") + except experiment.InfrastructureError: + route_outcomes.append("infra") + direct_route_closed = ( + route_outcomes == ["infra", "infra"] + and len(direct_connections) == route_connections_before + ) + finally: + http_client.HTTPSConnection = original_https_connection + ssl_module.SSLContext = original_ssl_context + http_client._MAXLINE = original_maxline + http_client._MAXHEADERS = original_maxheaders + experiment._git_system_ca_pem = original_ca_reader + direct_connection = direct_connections[0] if direct_connections else None + direct_status_framing_ok = hostile_status_outcomes == ( + "infra", + "infra", + "infra", + "infra", + "infra", + "infra", + "reject", + "infra", + ) + direct_https_ok = ( + direct_https + == ( + 200, + ( + ("content-length", str(len(direct_body))), + ("content-type", "application/json; charset=utf-8"), + ), + direct_body, + ) + and direct_connection is not None + and direct_connection.authority == "api.github.com" + and direct_connection.port == 443 + and direct_connection.request_record + == ( + "GET", + f"/repos/uscient/experiment-fixture/git/commits/{COMMIT}", + dict(expected_headers), + ) + and direct_connection.context.protocol + == ssl_module.PROTOCOL_TLS_CLIENT + and direct_connection.context.check_hostname is True + and direct_connection.context.verify_mode == ssl_module.CERT_REQUIRED + and direct_connection.context.minimum_version + == ssl_module.TLSVersion.TLSv1_2 + and direct_connection.context.ca_data == ca_bytes.decode("ascii") + and direct_connection.response.closed + and direct_connection.closed + and direct_connection.sock.timeouts + and all(value > 0 for value in direct_connection.sock.timeouts) + and "GIT-AUTHORITY" in direct_invalid + ) + results.append( + ( + "GIT-AUTHORITY-001", + all(call[0] == "api.github.com" for call in request_calls) + and all(call[2] == expected_headers for call in request_calls) + and request_calls[0][3] == 1_048_576 + and request_calls[0][3] > request_calls[1][3] > request_calls[2][3] + and direct_https_ok + and direct_route_closed, + "only the fixed credential-free provider authority is requested", + ) + ) + + inherited_names = { + "GIT_ASKPASS": str(Path(raw_home) / "askpass"), + "GIT_CONFIG_GLOBAL": str(Path(raw_home) / "gitconfig"), + "HTTPS_PROXY": "http://credential.invalid:9", + "SSL_CERT_FILE": str(Path(raw_home) / "caller-ca"), + } + prior_inherited = {name: os.environ.get(name) for name in inherited_names} + os.environ.update(inherited_names) + original_worker_requester = getattr(experiment, "_github_api_request", None) + + def credential_requester(authority, path, headers, maximum, deadline): + if os.environ != { + "HOME": "/nonexistent", + "LANG": "C", + "LC_ALL": "C", + "PATH": "/usr/bin:/bin", + } or os.getcwd() != "/": + raise RuntimeError("worker authority was not isolated") + descriptors = {int(item) for item in os.listdir("/proc/self/fd")} + if any(descriptor > 4 for descriptor in descriptors): + raise RuntimeError("worker inherited an unrelated descriptor") + return fixture_responses[path] + + experiment._github_api_request = credential_requester + try: + try: + credential_snapshot = experiment.read_git_snapshot(URL, COMMIT) + except (experiment.InvalidManifest, experiment.InfrastructureError): + credential_snapshot = None + finally: + if original_worker_requester is None: + delattr(experiment, "_github_api_request") + else: + experiment._github_api_request = original_worker_requester + for name, value in prior_inherited.items(): + if value is None: + os.environ.pop(name, None) + else: + os.environ[name] = value + credential_text = repr(credential_snapshot) + results.append( + ( + "GIT-CREDENTIAL-001", + credential_snapshot is not None + and credential_snapshot.data == source + and ca_probe_ok + and "credential.invalid" not in credential_text + and "caller-ca" not in credential_text + and "askpass" not in credential_text, + "caller credentials, proxy, CA, and Git configuration are not retained", + ) + ) + + redirect_responses = dict(fixture_responses) + redirect_responses[next(iter(fixture_responses))] = ( + 302, + ( + ("content-length", "0"), + ("content-type", "application/json; charset=utf-8"), + ("location", "https://credential.invalid/secret"), + ), + b"", + ) + redirect_outcome = acquire(redirect_responses) + results.append( + ( + "GIT-REDIRECT-001", + redirect_outcome[0] == "infra" + and "credential.invalid" not in redirect_outcome[1], + "provider redirects are infrastructure uncertainty and never followed", + ) + ) + + content_marker = Path(raw_home) / "content-executed" + marker_source = ( + f"// $(touch {content_marker})\n".encode("ascii") + source + ) + marker_blob = git_oid("blob", marker_source) + marker_tree_payload = b"100644 experiment.cue\0" + bytes.fromhex(marker_blob) + marker_tree = git_oid("tree", marker_tree_payload) + marker_responses = { + f"/repos/uscient/experiment-fixture/git/commits/{COMMIT}": response( + {"sha": COMMIT, "tree": {"sha": marker_tree}} + ), + f"/repos/uscient/experiment-fixture/git/trees/{marker_tree}": response( + { + "sha": marker_tree, + "tree": [ + { + "mode": "100644", + "path": "experiment.cue", + "sha": marker_blob, + "size": len(marker_source), + "type": "blob", + } + ], + "truncated": False, + } + ), + f"/repos/uscient/experiment-fixture/git/blobs/{marker_blob}": response( + { + "content": base64.b64encode(marker_source).decode("ascii"), + "encoding": "base64", + "sha": marker_blob, + "size": len(marker_source), + } + ), + } + content_outcome = acquire(marker_responses) + results.append( + ( + "GIT-CONTENT-001", + content_outcome[0] == "ok" + and content_outcome[1].data == marker_source + and not content_marker.exists(), + "repository content is snapshotted without checkout or execution", + ) + ) + + original_timeout = experiment.GIT_ACQUISITION_TIMEOUT_SECONDS + experiment.GIT_ACQUISITION_TIMEOUT_SECONDS = 0.001 + + def slow_requester(authority, path, headers, maximum, deadline): + import time + + time.sleep(0.01) + return fixture_responses[path] + + try: + try: + experiment.read_git_snapshot(URL, COMMIT, requester=slow_requester) + timeout_outcome = "ok" + except experiment.InfrastructureError as error: + timeout_outcome = str(error) + finally: + experiment.GIT_ACQUISITION_TIMEOUT_SECONDS = original_timeout + + original_worker_requester = experiment._github_api_request + worker_started_marker = Path(raw_home) / "timeout-worker-started" + + def hanging_worker_requester( + _authority, _path, _headers, _maximum, _deadline + ): + signal.signal(signal.SIGTERM, signal.SIG_IGN) + worker_started_marker.touch() + while True: + signal.pause() + + experiment._github_api_request = hanging_worker_requester + calibration_started = __import__("time").monotonic() + __import__("time").sleep(0.01) + calibration_elapsed = ( + __import__("time").monotonic() - calibration_started + ) + worker_operation_budget = 0.25 + worker_cleanup_reserve = 2 * experiment.GIT_WORKER_GRACE_SECONDS + worker_timeout_budget = ( + worker_operation_budget + worker_cleanup_reserve + ) + worker_timeout_tolerance = min( + 0.10, + max(0.05, 4 * max(0.0, calibration_elapsed - 0.01)), + ) + worker_timeout_started = __import__("time").monotonic() + try: + try: + experiment._github_worker_request( + "api.github.com", + f"/repos/uscient/experiment-fixture/git/commits/{COMMIT}", + expected_headers, + 1_024, + worker_timeout_started + worker_timeout_budget, + ) + worker_timeout_outcome = "ok" + except experiment.InfrastructureError as error: + worker_timeout_outcome = str(error) + finally: + experiment._github_api_request = original_worker_requester + worker_timeout_elapsed = ( + __import__("time").monotonic() - worker_timeout_started + ) + results.append( + ( + "GIT-TIMEOUT-001", + "GIT-TIMEOUT" in timeout_outcome + and "GIT-TIMEOUT" in worker_timeout_outcome + and worker_started_marker.is_file() + and worker_timeout_budget > worker_cleanup_reserve + and worker_timeout_elapsed + <= worker_timeout_budget + worker_timeout_tolerance, + "a started provider worker and its cleanup share one deadline", + ) + ) + + oversized_responses = dict(fixture_responses) + oversized_responses[next(iter(fixture_responses))] = ( + 200, + ( + ("content-length", str(1_048_577)), + ("content-type", "application/json; charset=utf-8"), + ), + b"x" * 1_048_577, + ) + output_outcome = acquire(oversized_responses) + + valid_worker_value = json.dumps( + { + "body": "", + "headers": [ + ["content-length", "0"], + ["content-type", "application/json; charset=utf-8"], + ], + "kind": "result", + "status": 200, + }, + ensure_ascii=True, + separators=(",", ":"), + sort_keys=True, + ).encode("ascii") + valid_worker_frame = ( + experiment.GIT_WORKER_FRAME + + len(valid_worker_value).to_bytes(8, "big") + + valid_worker_value + ) + original_worker_child = experiment._git_worker_child + + def worker_frame_outcome(stdout_data: bytes, stderr_data: bytes = b""): + def fixture_worker_child( + _control_read, + stdout_write, + stderr_write, + _authority, + _path, + _headers, + _maximum, + _deadline, + ): + try: + os.setsid() + if stderr_data: + os.write(stderr_write, stderr_data) + if stdout_data: + os.write(stdout_write, stdout_data) + finally: + os._exit(0) + + experiment._git_worker_child = fixture_worker_child + try: + try: + experiment._github_worker_request( + "api.github.com", + f"/repos/uscient/experiment-fixture/git/commits/{COMMIT}", + expected_headers, + 1_024, + __import__("time").monotonic() + 1.0, + ) + return "ok" + except experiment.InfrastructureError as error: + return str(error) + finally: + experiment._git_worker_child = original_worker_child + + malformed_frame = worker_frame_outcome(b"not-a-worker-frame") + oversized_frame = worker_frame_outcome( + experiment.GIT_WORKER_FRAME + + (experiment.GIT_WORKER_MAX_OUTPUT_BYTES + 1).to_bytes(8, "big") + ) + trailing_frame = worker_frame_outcome(valid_worker_frame + b"trailing") + stderr_frame = worker_frame_outcome( + valid_worker_frame, b"caller-private-diagnostic" + ) + + late_stderr_bytes = b"hostile-late-stderr-must-not-escape" + + def acknowledged_stderr_outcome(): + reached_read, reached_write = os.pipe2( + getattr(os, "O_CLOEXEC", 0) + ) + + def fixture_worker_child( + control_read, + stdout_write, + stderr_write, + _authority, + _path, + _headers, + _maximum, + _deadline, + ): + try: + os.setsid() + os.write(stdout_write, valid_worker_frame) + acknowledgement = os.read(control_read, 1) + if acknowledgement == b"1": + os.write(reached_write, b"A") + os.write(stderr_write, late_stderr_bytes) + os.write(reached_write, b"W") + __import__("time").sleep(0.05) + finally: + os._exit(0) + + experiment._git_worker_child = fixture_worker_child + try: + try: + experiment._github_worker_request( + "api.github.com", + f"/repos/uscient/experiment-fixture/git/commits/{COMMIT}", + expected_headers, + 1_024, + __import__("time").monotonic() + 1.0, + ) + outcome = "ok" + except experiment.InfrastructureError as error: + outcome = str(error) + finally: + experiment._git_worker_child = original_worker_child + os.close(reached_write) + try: + reached = os.read(reached_read, 16) + finally: + os.close(reached_read) + return outcome, reached + + acknowledged_stderr_frame, acknowledged_stderr_reached = ( + acknowledged_stderr_outcome() + ) + worker_frames_rejected = ( + "GIT-WORKER" in malformed_frame + and "GIT-OUTPUT" in oversized_frame + and "trailing" in trailing_frame + and stderr_frame + == "git provider GIT-WORKER emitted unexpected diagnostics" + and "caller-private-diagnostic" not in stderr_frame + and acknowledged_stderr_reached == b"AW" + and acknowledged_stderr_frame + == "git provider GIT-WORKER emitted unexpected diagnostics" + and late_stderr_bytes.decode("ascii") + not in acknowledged_stderr_frame + ) + results.append( + ( + "GIT-OUTPUT-001", + output_outcome[0] == "infra" + and "GIT-OUTPUT" in output_outcome[1] + and worker_frames_rejected + and injected_lengths_closed, + "provider and worker output frames are strictly bounded", + ) + ) + + padded_commit = dict(commit_body) + padded_commit["ignored"] = "c" * 524_000 + padded_tree = dict(tree_body) + padded_tree["ignored"] = "t" * 524_000 + aggregate_responses = dict(fixture_responses) + aggregate_responses[next(iter(fixture_responses))] = response(padded_commit) + aggregate_responses[f"/repos/uscient/experiment-fixture/git/trees/{TREE}"] = response( + padded_tree + ) + aggregate_outcome = acquire(aggregate_responses) + results.append( + ( + "GIT-ACQUIRE-001", + aggregate_outcome[0] == "infra" + and "GIT-OUTPUT" in aggregate_outcome[1] + and all(len(item[2]) < 1_048_576 for item in aggregate_responses.values()), + "the response-byte bound is aggregate rather than per request", + ) + ) + + original_worker_requester = getattr(experiment, "_github_api_request", None) + worker_parent_pid = os.getpid() + large_source = b"//" + (b"x" * (262_144 - 3)) + b"\n" + large_blob = git_oid("blob", large_source) + large_tree = git_oid( + "tree", b"100644 experiment.cue\0" + bytes.fromhex(large_blob) + ) + large_responses = { + f"/repos/uscient/experiment-fixture/git/commits/{COMMIT}": response( + {"sha": COMMIT, "tree": {"sha": large_tree}} + ), + f"/repos/uscient/experiment-fixture/git/trees/{large_tree}": response( + { + "sha": large_tree, + "tree": [ + { + "mode": "100644", + "path": "experiment.cue", + "sha": large_blob, + "size": len(large_source), + "type": "blob", + } + ], + "truncated": False, + } + ), + f"/repos/uscient/experiment-fixture/git/blobs/{large_blob}": response( + { + "content": base64.b64encode(large_source).decode("ascii"), + "encoding": "base64", + "sha": large_blob, + "size": len(large_source), + } + ), + } + + def worker_requester(authority, path, headers, maximum, deadline): + import resource + + worker_pid = os.getpid() + if ( + worker_pid == worker_parent_pid + or os.getpgrp() != worker_pid + or os.getsid(0) != worker_pid + or os.getcwd() != "/" + or resource.getrlimit(resource.RLIMIT_CORE) != (0, 0) + or resource.getrlimit(resource.RLIMIT_FSIZE) != (0, 0) + or resource.getrlimit(resource.RLIMIT_AS) + != ( + experiment.GIT_WORKER_MEMORY_BYTES, + experiment.GIT_WORKER_MEMORY_BYTES, + ) + or resource.getrlimit(resource.RLIMIT_NOFILE) != (16, 16) + or resource.getrlimit(resource.RLIMIT_CPU) != (4, 4) + ): + raise RuntimeError("provider worker session is not isolated") + return large_responses[path] + + experiment._github_api_request = worker_requester + try: + try: + worker_snapshot = experiment.read_git_snapshot(URL, COMMIT) + except (experiment.InvalidManifest, experiment.InfrastructureError): + worker_snapshot = None + finally: + if original_worker_requester is None: + delattr(experiment, "_github_api_request") + else: + experiment._github_api_request = original_worker_requester + + terminate_wait_options: list[int] = [] + terminate_observations: list[tuple[object, int, int]] = [] + original_waitpid = experiment.os.waitpid + original_waitid = experiment.os.waitid + original_killpg = experiment.os.killpg + original_monotonic = experiment.time.monotonic + original_sleep = experiment.time.sleep + terminate_clock = [0.0] + + def terminate_monotonic(): + terminate_clock[0] += 1.0 + return terminate_clock[0] + + def terminate_waitpid(_pid, options): + terminate_wait_options.append(options) + if options == 0: + raise OSError("blocking wait forbidden by fixture") + return 0, 0 + + def terminate_waitid(idtype, identifier, options): + terminate_observations.append( + (idtype, identifier, options) + ) + expected = os.WEXITED | os.WNOHANG | os.WNOWAIT + if ( + idtype != os.P_PID + or identifier != 991_337 + or options != expected + or options & (os.WSTOPPED | os.WCONTINUED) + ): + raise OSError("consuming or blocking observation forbidden by fixture") + return None + + experiment.os.waitpid = terminate_waitpid + experiment.os.waitid = terminate_waitid + experiment.os.killpg = lambda _pid, _signal: None + experiment.time.monotonic = terminate_monotonic + experiment.time.sleep = lambda _duration: None + terminate_result = None + try: + try: + terminate_result = experiment._git_worker_terminate( + 991_337, reaped=False, deadline=10.0 + ) + except (AttributeError, OSError, TypeError): + pass + finally: + experiment.os.waitpid = original_waitpid + experiment.os.waitid = original_waitid + experiment.os.killpg = original_killpg + experiment.time.monotonic = original_monotonic + experiment.time.sleep = original_sleep + nonblocking_terminate = ( + terminate_result is False + and terminate_observations + and all( + observation + == ( + os.P_PID, + 991_337, + os.WEXITED | os.WNOHANG | os.WNOWAIT, + ) + for observation in terminate_observations + ) + and all( + option == os.WNOHANG for option in terminate_wait_options + ) + ) + + released_events: list[tuple[object, ...]] = [] + original_waitpid = experiment.os.waitpid + original_waitid = experiment.os.waitid + original_kill = experiment.os.kill + original_killpg = experiment.os.killpg + original_monotonic = experiment.time.monotonic + original_sleep = experiment.time.sleep + released_clock = [0.0] + + def released_monotonic(): + released_clock[0] += 1.0 + return released_clock[0] + + def released_waitpid(_pid, _options): + released_events.append(("waitpid", _pid, _options)) + return 0, 0 + + def released_waitid(_idtype, _identifier, _options): + released_events.append( + ("waitid", _idtype, _identifier, _options) + ) + return None + + def released_kill(_pid, _signum): + released_events.append(("kill", _pid, _signum)) + + def released_killpg(_pid, _signum): + released_events.append(("killpg", _pid, _signum)) + + experiment.os.waitpid = released_waitpid + experiment.os.waitid = released_waitid + experiment.os.kill = released_kill + experiment.os.killpg = released_killpg + experiment.time.monotonic = released_monotonic + experiment.time.sleep = lambda _duration: None + released_result = None + try: + try: + released_result = experiment._git_worker_terminate( + 991_338, reaped=True, deadline=10.0 + ) + except (AttributeError, OSError, TypeError): + pass + finally: + experiment.os.waitpid = original_waitpid + experiment.os.waitid = original_waitid + experiment.os.kill = original_kill + experiment.os.killpg = original_killpg + experiment.time.monotonic = original_monotonic + experiment.time.sleep = original_sleep + released_identity_safe = ( + released_result is False + and released_events == [] + ) + + transition_pid = 991_339 + transition_events: list[tuple[object, ...]] = [] + transition_observe_count = [0] + transition_observed = [False] + transition_reaped = [False] + transition_signals: list[int] = [] + transition_clock = [0.0] + + class ObservableWaitidResult: + si_pid = transition_pid + si_uid = 0 + si_signo = signal.SIGCHLD + si_status = 0 + si_code = getattr(os, "CLD_EXITED", 1) + + def transition_monotonic(): + transition_clock[0] += 0.05 + return transition_clock[0] + + def transition_waitpid(pid, options): + if transition_reaped[0]: + transition_events.append( + ("waitpid-after-reap", pid, options) + ) + raise ChildProcessError + transition_reaped[0] = True + transition_events.append( + ( + "waitpid-reap", + pid, + options, + transition_observed[0], + ) + ) + return pid, 0 + + def transition_waitid(idtype, identifier, options): + if transition_reaped[0]: + transition_events.append( + ( + "waitid-after-reap", + idtype, + identifier, + options, + ) + ) + raise ChildProcessError + transition_observe_count[0] += 1 + if transition_observe_count[0] < 2: + transition_events.append( + ("waitid-empty", idtype, identifier, options) + ) + return None + observed = ObservableWaitidResult() + transition_observed[0] = True + transition_events.append( + ( + "waitid-observable", + idtype, + identifier, + options, + observed.si_code, + ) + ) + return observed + + def transition_kill(pid, signum): + transition_events.append(("kill", pid, int(signum))) + if signum: + transition_signals.append(int(signum)) + + def transition_killpg(pid, signum): + transition_events.append(("killpg", pid, int(signum))) + if signum: + transition_signals.append(int(signum)) + + original_waitpid = experiment.os.waitpid + original_waitid = experiment.os.waitid + original_kill = experiment.os.kill + original_killpg = experiment.os.killpg + original_monotonic = experiment.time.monotonic + original_sleep = experiment.time.sleep + experiment.os.waitpid = transition_waitpid + experiment.os.waitid = transition_waitid + experiment.os.kill = transition_kill + experiment.os.killpg = transition_killpg + experiment.time.monotonic = transition_monotonic + experiment.time.sleep = lambda _duration: None + transition_result = None + try: + try: + transition_result = experiment._git_worker_terminate( + transition_pid, reaped=False, deadline=10.0 + ) + except (AttributeError, OSError, TypeError): + pass + finally: + experiment.os.waitpid = original_waitpid + experiment.os.waitid = original_waitid + experiment.os.kill = original_kill + experiment.os.killpg = original_killpg + experiment.time.monotonic = original_monotonic + experiment.time.sleep = original_sleep + + transition_reap_positions = [ + index + for index, event in enumerate(transition_events) + if event[0] == "waitpid-reap" + ] + transition_signal_positions = [ + index + for index, event in enumerate(transition_events) + if event[0] in ("kill", "killpg") and event[2] != 0 + ] + transition_waitid_events = [ + event + for event in transition_events + if event[0] in ("waitid-empty", "waitid-observable") + ] + transition_first_reap = ( + transition_reap_positions[0] + if transition_reap_positions + else -1 + ) + transition_identity_safe = ( + transition_result is True + and transition_signals + == [int(signal.SIGTERM), int(signal.SIGKILL)] + and len(transition_reap_positions) == 1 + and transition_first_reap == len(transition_events) - 1 + and transition_events[transition_first_reap][3] is True + and transition_signal_positions + and all( + index < transition_first_reap + for index in transition_signal_positions + ) + and any( + event[0] == "waitid-observable" + for event in transition_waitid_events + ) + and all( + event[1] == os.P_PID + and event[2] == transition_pid + and event[3] + == os.WEXITED | os.WNOHANG | os.WNOWAIT + and not event[3] & (os.WSTOPPED | os.WCONTINUED) + for event in transition_waitid_events + ) + and all( + event[0] != "waitid-observable" + or event[4] == getattr(os, "CLD_EXITED", 1) + for event in transition_waitid_events + ) + ) + + request_order_parent = os.getpid() + request_order_events: list[tuple[str, int, object, bool]] = [] + request_order_released = [False] + original_waitpid = experiment.os.waitpid + original_waitid = experiment.os.waitid + original_kill = experiment.os.kill + original_killpg = experiment.os.killpg + original_worker_child = experiment._git_worker_child + + def request_order_waitpid(pid, options): + released_before = request_order_released[0] + try: + waited, status = original_waitpid(pid, options) + except BaseException: + if os.getpid() == request_order_parent: + request_order_events.append( + ("waitpid-error", pid, options, released_before) + ) + raise + if os.getpid() == request_order_parent: + kind = "waitpid-reap" if waited == pid else "waitpid-empty" + request_order_events.append( + (kind, pid, options, released_before) + ) + if waited == pid: + request_order_released[0] = True + return waited, status + + def request_order_waitid(idtype, identifier, options): + released_before = request_order_released[0] + try: + observed = original_waitid(idtype, identifier, options) + except BaseException: + if os.getpid() == request_order_parent: + request_order_events.append( + ( + "waitid-error", + identifier, + (idtype, options, None), + released_before, + ) + ) + raise + if os.getpid() == request_order_parent: + request_order_events.append( + ( + "waitid-observable" + if observed is not None + else "waitid-empty", + identifier, + ( + idtype, + options, + getattr(observed, "si_code", None), + ), + released_before, + ) + ) + return observed + + def request_order_kill(pid, signum): + if os.getpid() == request_order_parent: + request_order_events.append( + ( + "kill", + pid, + int(signum), + request_order_released[0], + ) + ) + return original_kill(pid, signum) + + def request_order_killpg(pid, signum): + if os.getpid() == request_order_parent: + request_order_events.append( + ( + "killpg", + pid, + int(signum), + request_order_released[0], + ) + ) + return original_killpg(pid, signum) + + def request_order_worker_child( + control_read, + stdout_write, + _stderr_write, + _authority, + _path, + _headers, + _maximum, + _deadline, + ): + try: + os.setsid() + os.write(stdout_write, valid_worker_frame) + exit_code = 0 if os.read(control_read, 1) == b"1" else 125 + except BaseException: + exit_code = 125 + os._exit(exit_code) + + experiment.os.waitpid = request_order_waitpid + experiment.os.waitid = request_order_waitid + experiment.os.kill = request_order_kill + experiment.os.killpg = request_order_killpg + experiment._git_worker_child = request_order_worker_child + try: + try: + request_order_result = experiment._github_worker_request( + "api.github.com", + f"/repos/uscient/experiment-fixture/git/commits/{COMMIT}", + expected_headers, + 1_024, + __import__("time").monotonic() + 1.5, + ) + request_order_outcome = "ok" + except experiment.InfrastructureError as error: + request_order_result = None + request_order_outcome = str(error) + except BaseException as error: + request_order_result = None + request_order_outcome = type(error).__name__ + finally: + experiment.os.waitpid = original_waitpid + experiment.os.waitid = original_waitid + experiment.os.kill = original_kill + experiment.os.killpg = original_killpg + experiment._git_worker_child = original_worker_child + + request_order_waitpid_events = [ + event + for event in request_order_events + if event[0].startswith("waitpid-") + ] + request_order_waitid_events = [ + event + for event in request_order_events + if event[0].startswith("waitid-") + ] + request_order_reap_positions = [ + index + for index, event in enumerate(request_order_events) + if event[0] == "waitpid-reap" + ] + request_order_targets = { + event[1] for event in request_order_events + } + request_identity_safe = ( + request_order_outcome == "ok" + and request_order_result + == ( + 200, + ( + ("content-length", "0"), + ("content-type", "application/json; charset=utf-8"), + ), + b"", + ) + and len(request_order_waitpid_events) == 1 + and request_order_waitpid_events[0][0] == "waitpid-reap" + and request_order_waitpid_events[0][2] == os.WNOHANG + and len(request_order_reap_positions) == 1 + and request_order_reap_positions[0] + == len(request_order_events) - 1 + and request_order_waitid_events + and any( + event[0] == "waitid-observable" + and event[2][2] == getattr(os, "CLD_EXITED", 1) + for event in request_order_waitid_events + ) + and all( + event[2][0] == os.P_PID + and event[2][1] + == os.WEXITED | os.WNOHANG | os.WNOWAIT + and not event[2][1] & (os.WSTOPPED | os.WCONTINUED) + for event in request_order_waitid_events + ) + and len(request_order_targets) == 1 + and next(iter(request_order_targets), -1) > 0 + and not any(event[3] for event in request_order_events) + ) + + mask_failure_marker = Path(raw_home) / "signal-state-requester-reached" + mask_parent_pid = os.getpid() + original_pthread_sigmask = experiment.signal.pthread_sigmask + original_worker_requester = experiment._github_api_request + + def fail_child_mask(how, signals): + if ( + os.getpid() != mask_parent_pid + and how == signal.SIG_SETMASK + and not signals + ): + raise OSError("child mask clear failed") + return original_pthread_sigmask(how, signals) + + def mask_failure_requester( + _authority, path, _headers, _maximum, _deadline + ): + mask_failure_marker.touch() + return fixture_responses[path] + + experiment.signal.pthread_sigmask = fail_child_mask + experiment._github_api_request = mask_failure_requester + try: + try: + experiment._github_worker_request( + "api.github.com", + f"/repos/uscient/experiment-fixture/git/commits/{COMMIT}", + expected_headers, + 1_024, + __import__("time").monotonic() + 1.0, + ) + mask_failure_outcome = "ok" + except experiment.InfrastructureError as error: + mask_failure_outcome = str(error) + finally: + experiment.signal.pthread_sigmask = original_pthread_sigmask + experiment._github_api_request = original_worker_requester + mask_clear_fail_closed = ( + "GIT-WORKER" in mask_failure_outcome + and not mask_failure_marker.exists() + ) + + stop_listener = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + stop_listener.bind(("127.0.0.1", 0)) + stop_listener.settimeout(1.0) + stop_address = stop_listener.getsockname() + original_worker_child = experiment._git_worker_child + + def stopped_before_session( + _control_read, + _stdout_write, + _stderr_write, + _authority, + _path, + _headers, + _maximum, + _deadline, + ): + with socket.socket(socket.AF_INET, socket.SOCK_DGRAM) as sender: + sender.sendto(f"{os.getpid()}\n".encode("ascii"), stop_address) + os.kill(os.getpid(), signal.SIGSTOP) + os._exit(125) + + experiment._git_worker_child = stopped_before_session + try: + try: + experiment._github_worker_request( + "api.github.com", + f"/repos/uscient/experiment-fixture/git/commits/{COMMIT}", + expected_headers, + 1_024, + __import__("time").monotonic() + 0.75, + ) + stopped_outcome = "ok" + except experiment.InfrastructureError: + stopped_outcome = "infra" + except BaseException: + stopped_outcome = "other" + finally: + experiment._git_worker_child = original_worker_child + try: + stopped_record = stop_listener.recv(64).decode("ascii").strip() + except (TimeoutError, UnicodeDecodeError): + stopped_record = "" + finally: + stop_listener.close() + stopped_pid = int(stopped_record) if stopped_record.isdigit() else None + stopped_reaped_before_return = False + stopped_reaped_by_harness = False + stopped_alive = False + if stopped_pid is not None: + try: + waited, _ = os.waitpid(stopped_pid, os.WNOHANG) + except ChildProcessError: + stopped_reaped_before_return = True + stopped_reaped_by_harness = True + else: + stopped_reaped_by_harness = waited == stopped_pid + try: + os.kill(stopped_pid, 0) + except ProcessLookupError: + stopped_alive = False + else: + stopped_alive = True + if not stopped_reaped_by_harness: + if stopped_alive: + try: + os.kill(stopped_pid, signal.SIGKILL) + except ProcessLookupError: + pass + stop_reap_deadline = __import__("time").monotonic() + 1.0 + while __import__("time").monotonic() < stop_reap_deadline: + try: + waited, _ = os.waitpid(stopped_pid, os.WNOHANG) + except ChildProcessError: + stopped_reaped_by_harness = True + break + if waited == stopped_pid: + stopped_reaped_by_harness = True + break + __import__("time").sleep(0.01) + if stopped_pid is None or not stopped_reaped_by_harness: + raise RuntimeError("stopped worker fixture cleanup is uncertain") + stopped_child_cleaned = ( + stopped_outcome == "infra" + and stopped_reaped_before_return + and not stopped_alive + ) + results.append( + ( + "GIT-PGROUP-001", + worker_snapshot is not None + and worker_snapshot.data == large_source + and nonblocking_terminate + and released_identity_safe + and transition_identity_safe + and request_identity_safe + and mask_clear_fail_closed + and stopped_child_cleaned + and len(large_responses[ + f"/repos/uscient/experiment-fixture/git/blobs/{large_blob}" + ][2]) > 65_536, + "the fixed worker enforces limits and cleans pre-session stops", + ) + ) + + def cleanup_fault_probe(kind: str) -> bool: + probe_pid = os.fork() + if probe_pid == 0: + cleanup_probe_process = os.getpid() + managed = ( + signal.SIGHUP, + signal.SIGINT, + signal.SIGQUIT, + signal.SIGTERM, + ) + + def seeded_interrupt_handler(_signum, _frame): + return None + + signal.signal(signal.SIGHUP, signal.SIG_DFL) + signal.signal(signal.SIGINT, seeded_interrupt_handler) + signal.signal(signal.SIGQUIT, signal.SIG_DFL) + signal.signal(signal.SIGTERM, signal.SIG_DFL) + signal.pthread_sigmask( + signal.SIG_SETMASK, {signal.SIGQUIT} + ) + original_handlers = { + signum: signal.getsignal(signum) for signum in managed + } + original_mask = set( + signal.pthread_sigmask(signal.SIG_BLOCK, set()) + ) + original_requester = experiment._github_api_request + original_close = experiment.os.close + original_fork = experiment.os.fork + original_selector = experiment.selectors.DefaultSelector + original_signal = experiment.signal.signal + original_pthread_sigmask = ( + experiment.signal.pthread_sigmask + ) + missing_buffer_factory = object() + original_buffer_factory = experiment.__dict__.get( + "bytearray", missing_buffer_factory + ) + close_calls = [0] + parent_setmask_failures = [0] + handler_install_attempts = [0] + handler_fork_calls = [0] + + def cleanup_requester( + _authority, _path, _headers, _maximum, _deadline + ): + return ( + 200, + ( + ("content-length", "0"), + ( + "content-type", + "application/json; charset=utf-8", + ), + ), + b"", + ) + + def failed_cleanup_close(descriptor): + if os.getpid() == cleanup_probe_process: + close_calls[0] += 1 + if close_calls[0] == 5: + raise OSError("cleanup close failed") + return original_close(descriptor) + + def failed_selector(): + raise OSError("selector construction failed") + + def failed_buffer_allocation(*_args, **_kwargs): + raise MemoryError("worker buffer allocation failed") + + def fail_parent_spawn_mask_restore(how, signals): + if ( + os.getpid() == cleanup_probe_process + and how == signal.SIG_SETMASK + and parent_setmask_failures[0] == 0 + ): + parent_setmask_failures[0] += 1 + raise OSError("parent spawn mask restore failed") + return original_pthread_sigmask(how, signals) + + def fail_mid_handler_install(signum, handler): + if ( + os.getpid() == cleanup_probe_process + and handler_install_attempts[0] < 2 + ): + handler_install_attempts[0] += 1 + if handler_install_attempts[0] == 2: + raise OSError("mid-handler install failed") + return original_signal(signum, handler) + + def handler_install_forbidden_fork(): + handler_fork_calls[0] += 1 + raise OSError("worker fork reached after handler failure") + + experiment._github_api_request = cleanup_requester + if kind == "close": + experiment.os.close = failed_cleanup_close + elif kind == "selector": + experiment.selectors.DefaultSelector = failed_selector + elif kind == "allocation": + experiment.bytearray = failed_buffer_allocation + elif kind == "parent-mask-restore": + experiment.signal.pthread_sigmask = ( + fail_parent_spawn_mask_restore + ) + elif kind == "handler-install": + experiment.signal.signal = fail_mid_handler_install + experiment.os.fork = handler_install_forbidden_fork + try: + try: + experiment._github_worker_request( + "api.github.com", + f"/repos/uscient/experiment-fixture/git/commits/{COMMIT}", + expected_headers, + 1_024, + __import__("time").monotonic() + 1.0, + ) + outcome = "ok" + except experiment.InfrastructureError: + outcome = "infra" + except BaseException: + outcome = "other" + restored = ( + all( + signal.getsignal(signum) + == original_handlers[signum] + for signum in managed + ) + and set( + signal.pthread_sigmask(signal.SIG_BLOCK, set()) + ) + == original_mask + and signal.getsignal(signal.SIGINT) + is seeded_interrupt_handler + and original_mask == {signal.SIGQUIT} + ) + finally: + experiment._github_api_request = original_requester + experiment.os.close = original_close + experiment.os.fork = original_fork + experiment.selectors.DefaultSelector = original_selector + experiment.signal.signal = original_signal + experiment.signal.pthread_sigmask = ( + original_pthread_sigmask + ) + if original_buffer_factory is missing_buffer_factory: + experiment.__dict__.pop("bytearray", None) + else: + experiment.bytearray = original_buffer_factory + injected_fault_observed = ( + ( + kind != "parent-mask-restore" + or parent_setmask_failures[0] == 1 + ) + and ( + kind != "handler-install" + or ( + handler_install_attempts[0] == 2 + and handler_fork_calls[0] == 0 + ) + ) + ) + os._exit( + 0 + if outcome == "infra" + and restored + and injected_fault_observed + else 1 + ) + + probe_status = None + probe_deadline = __import__("time").monotonic() + 2.0 + while __import__("time").monotonic() < probe_deadline: + waited, status = os.waitpid(probe_pid, os.WNOHANG) + if waited == probe_pid: + probe_status = status + break + __import__("time").sleep(0.01) + if probe_status is None: + try: + os.kill(probe_pid, signal.SIGKILL) + except ProcessLookupError: + pass + reap_deadline = __import__("time").monotonic() + 1.0 + while __import__("time").monotonic() < reap_deadline: + waited, status = os.waitpid(probe_pid, os.WNOHANG) + if waited == probe_pid: + probe_status = status + break + __import__("time").sleep(0.01) + return ( + probe_status is not None + and os.WIFEXITED(probe_status) + and os.WEXITSTATUS(probe_status) == 0 + ) + + close_failure_fail_closed = cleanup_fault_probe("close") + selector_failure_fail_closed = cleanup_fault_probe("selector") + allocation_failure_fail_closed = cleanup_fault_probe("allocation") + parent_mask_restore_fail_closed = cleanup_fault_probe( + "parent-mask-restore" + ) + handler_install_fail_closed = cleanup_fault_probe( + "handler-install" + ) + + def sigchld_precondition_probe(mode: str) -> bool: + probe_pid = os.fork() + if probe_pid == 0: + managed = ( + signal.SIGHUP, + signal.SIGINT, + signal.SIGQUIT, + signal.SIGTERM, + signal.SIGCHLD, + ) + + def custom_child_reaper(_signum, _frame): + return None + + selected_handler = ( + signal.SIG_IGN + if mode == "ignored" + else custom_child_reaper + ) + signal.signal(signal.SIGCHLD, selected_handler) + expected_handlers = { + signum: signal.getsignal(signum) + for signum in managed + } + expected_mask = set( + signal.pthread_sigmask(signal.SIG_BLOCK, set()) + ) + original_fork = experiment.os.fork + fork_calls = [0] + + def forbidden_worker_fork(): + fork_calls[0] += 1 + raise OSError("SIGCHLD precondition reached fork") + + experiment.os.fork = forbidden_worker_fork + try: + try: + experiment._github_worker_request( + "api.github.com", + f"/repos/uscient/experiment-fixture/git/commits/{COMMIT}", + expected_headers, + 1_024, + __import__("time").monotonic() + 1.0, + ) + outcome = "ok" + except experiment.InfrastructureError as error: + outcome = str(error) + except BaseException as error: + outcome = type(error).__name__ + state_unchanged = ( + all( + signal.getsignal(signum) + == expected_handlers[signum] + for signum in managed + ) + and signal.getsignal(signal.SIGCHLD) + == selected_handler + and set( + signal.pthread_sigmask(signal.SIG_BLOCK, set()) + ) + == expected_mask + ) + finally: + experiment.os.fork = original_fork + os._exit( + 0 + if "GIT-SIGNAL" in outcome + and "SIGCHLD" in outcome + and fork_calls[0] == 0 + and state_unchanged + else 1 + ) + + probe_status = None + probe_deadline = __import__("time").monotonic() + 2.0 + while __import__("time").monotonic() < probe_deadline: + waited, status = os.waitpid(probe_pid, os.WNOHANG) + if waited == probe_pid: + probe_status = status + break + __import__("time").sleep(0.01) + if probe_status is None: + try: + os.kill(probe_pid, signal.SIGKILL) + except ProcessLookupError: + pass + waited, status = os.waitpid(probe_pid, 0) + if waited == probe_pid: + probe_status = status + return ( + probe_status is not None + and os.WIFEXITED(probe_status) + and os.WEXITSTATUS(probe_status) == 0 + ) + + ignored_sigchld_fail_closed = sigchld_precondition_probe( + "ignored" + ) + custom_reaper_fail_closed = sigchld_precondition_probe("custom") + + residual_listener = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + residual_listener.bind(("127.0.0.1", 0)) + residual_listener.settimeout(1.0) + residual_address = residual_listener.getsockname() + residual_spawned = False + residual_pid = None + residual_calls = 0 + + def residual_requester(authority, path, headers, maximum, deadline): + nonlocal residual_calls + residual_calls += 1 + if residual_calls == 1: + child = os.fork() + if child == 0: + try: + with socket.socket(socket.AF_INET, socket.SOCK_DGRAM) as sender: + sender.sendto( + f"{os.getpid()}\n".encode("ascii"), + residual_address, + ) + while True: + signal.pause() + finally: + os._exit(0) + return fixture_responses[path] + + original_worker_requester = getattr(experiment, "_github_api_request", None) + experiment._github_api_request = residual_requester + try: + try: + experiment.read_git_snapshot(URL, COMMIT) + residual_outcome = "ok" + except experiment.InfrastructureError as error: + residual_outcome = str(error) + finally: + if original_worker_requester is None: + delattr(experiment, "_github_api_request") + else: + experiment._github_api_request = original_worker_requester + try: + residual_record = residual_listener.recv(64).decode("ascii").strip() + except TimeoutError: + residual_record = "" + finally: + residual_listener.close() + if residual_record.isdigit(): + residual_spawned = True + residual_pid = int(residual_record) + try: + os.kill(residual_pid, 0) + except ProcessLookupError: + residual_alive = False + else: + residual_alive = True + os.kill(residual_pid, signal.SIGKILL) + else: + residual_alive = False + + def signal_cleanup_probe(*, uncertain: bool, expected_exit: int) -> bool: + signal_listener = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + signal_listener.bind(("127.0.0.1", 0)) + signal_listener.settimeout(1.0) + signal_address = signal_listener.getsockname() + signal_probe = os.fork() + if signal_probe == 0: + original_worker_terminate = experiment._git_worker_terminate + + def uncertain_worker_terminate(pid, *, reaped, deadline): + original_worker_terminate( + pid, + reaped=reaped, + deadline=deadline, + ) + return False + + def hanging_requester( + _authority, _path, _headers, _maximum, _deadline + ): + with socket.socket( + socket.AF_INET, socket.SOCK_DGRAM + ) as sender: + sender.sendto( + f"{os.getpid()}\n".encode("ascii"), + signal_address, + ) + signal.signal(signal.SIGTERM, signal.SIG_IGN) + while True: + signal.pause() + + if uncertain: + experiment._git_worker_terminate = ( + uncertain_worker_terminate + ) + experiment._github_api_request = hanging_requester + try: + experiment.read_git_snapshot(URL, COMMIT) + except SystemExit as error: + code = error.code if isinstance(error.code, int) else 124 + os._exit(code) + except experiment.InfrastructureError: + os._exit(125) + except BaseException: + os._exit(124) + os._exit(0) + try: + try: + worker_record = ( + signal_listener.recv(64).decode("ascii").strip() + ) + except TimeoutError: + worker_record = "" + finally: + signal_listener.close() + signal_worker_pid = ( + int(worker_record) if worker_record.isdigit() else None + ) + if signal_worker_pid is not None: + os.kill(signal_probe, signal.SIGTERM) + signal_status = None + signal_deadline = __import__("time").monotonic() + 2.0 + while __import__("time").monotonic() < signal_deadline: + waited, status = os.waitpid(signal_probe, os.WNOHANG) + if waited == signal_probe: + signal_status = status + break + __import__("time").sleep(0.01) + if signal_status is None: + try: + os.kill(signal_probe, signal.SIGKILL) + except ProcessLookupError: + pass + reap_deadline = __import__("time").monotonic() + 1.0 + while __import__("time").monotonic() < reap_deadline: + waited, status = os.waitpid(signal_probe, os.WNOHANG) + if waited == signal_probe: + signal_status = status + break + __import__("time").sleep(0.01) + signal_worker_alive = False + if signal_worker_pid is not None: + worker_deadline = __import__("time").monotonic() + 1.0 + while __import__("time").monotonic() < worker_deadline: + try: + os.killpg(signal_worker_pid, 0) + except ProcessLookupError: + break + __import__("time").sleep(0.01) + else: + signal_worker_alive = True + try: + os.killpg(signal_worker_pid, signal.SIGKILL) + except ProcessLookupError: + signal_worker_alive = False + return ( + signal_worker_pid is not None + and signal_status is not None + and os.WIFEXITED(signal_status) + and os.WEXITSTATUS(signal_status) == expected_exit + and not signal_worker_alive + ) + + clean_signal_preserved = signal_cleanup_probe( + uncertain=False, + expected_exit=128 + signal.SIGTERM, + ) + cleanup_uncertainty_wins = signal_cleanup_probe( + uncertain=True, + expected_exit=125, + ) + results.append( + ( + "GIT-CLEANUP-001", + residual_spawned + and not residual_alive + and "residual" in residual_outcome.lower() + and clean_signal_preserved + and cleanup_uncertainty_wins + and close_failure_fail_closed + and selector_failure_fail_closed + and allocation_failure_fail_closed + and parent_mask_restore_fail_closed + and handler_install_fail_closed + and ignored_sigchld_fail_closed + and custom_reaper_fail_closed, + "cleanup uncertainty fails closed before caller signals are restored", + ) + ) + + taxonomy_outcomes = [] + for status in (404, 422, 403, 500): + status_responses = dict(fixture_responses) + status_responses[next(iter(fixture_responses))] = ( + status, + ( + ("content-length", "2"), + ("content-type", "application/json; charset=utf-8"), + ), + b"{}", + ) + taxonomy_outcomes.append((status, acquire(status_responses)[0])) + malformed_responses = dict(fixture_responses) + malformed_responses[next(iter(fixture_responses))] = ( + 200, + ( + ("content-length", "1"), + ("content-type", "application/json; charset=utf-8"), + ), + b"{", + ) + taxonomy_outcomes.append((200, acquire(malformed_responses)[0])) + tree_missing = dict(fixture_responses) + tree_missing[f"/repos/uscient/experiment-fixture/git/trees/{TREE}"] = ( + 404, + ( + ("content-length", "2"), + ("content-type", "application/json; charset=utf-8"), + ), + b"{}", + ) + taxonomy_outcomes.append(("tree-404", acquire(tree_missing)[0])) + blob_missing = dict(fixture_responses) + blob_missing[f"/repos/uscient/experiment-fixture/git/blobs/{BLOB}"] = ( + 422, + ( + ("content-length", "2"), + ("content-type", "application/json; charset=utf-8"), + ), + b"{}", + ) + taxonomy_outcomes.append(("blob-422", acquire(blob_missing)[0])) + results.append( + ( + "GIT-TAXONOMY-001", + taxonomy_outcomes + == [ + (404, "reject"), + (422, "infra"), + (403, "infra"), + (500, "infra"), + (200, "infra"), + ("tree-404", "infra"), + ("blob-422", "infra"), + ] + and direct_status_framing_ok + and stable_not_found_route_closed, + "only a strictly framed commit 404 is stable absence", + ) + ) + + fixture_directory = ( + repo / "tests/experiment/fixtures/directories/minimal" + ) + zip_fixture_module = load_module( + repo / "tests/experiment/zip-fixtures.py", + "git_intake_zip_fixture", + ) + common_zip = Path(raw_home) / "common-source.zip" + common_zip.write_bytes(zip_fixture_module.one("experiment.cue", source)) + directory_snapshot = experiment.read_directory_snapshot( + str(fixture_directory) + ) + zip_snapshot = experiment.read_zip_snapshot(str(common_zip)) + git_snapshot = snapshot + + prior_tools = { + "AGENT_LAB_CUE_TOOL_DIR": os.environ.get( + "AGENT_LAB_CUE_TOOL_DIR" + ), + "AGENT_LAB_CEDAR_TOOL_DIR": os.environ.get( + "AGENT_LAB_CEDAR_TOOL_DIR" + ), + } + os.environ["AGENT_LAB_CUE_TOOL_DIR"] = str( + repo / ".cache/dev/tools/cue" + ) + os.environ["AGENT_LAB_CEDAR_TOOL_DIR"] = str( + repo / ".cache/dev/tools/cedar" + ) + + identity_ready = git_snapshot is not None + directory_manifest = None + zip_manifest = None + git_manifest = None + directory_resolution = None + zip_resolution = None + git_resolution = None + directory_decision = None + zip_decision = None + git_decision = None + if identity_ready: + try: + directory_manifest = experiment.authored_manifest( + directory_snapshot + ) + zip_manifest = experiment.authored_manifest(zip_snapshot) + git_manifest = experiment.authored_manifest(git_snapshot) + directory_resolution = experiment.cue_plan_with_evidence( + directory_manifest + ) + zip_resolution = experiment.cue_plan_with_evidence(zip_manifest) + git_resolution = experiment.cue_plan_with_evidence(git_manifest) + directory_decision = experiment.authorize_plan( + directory_resolution.plan, directory_snapshot.digest + )[0] + zip_decision = experiment.authorize_plan( + zip_resolution.plan, zip_snapshot.digest + )[0] + git_decision = experiment.authorize_plan( + git_resolution.plan, git_snapshot.digest + )[0] + except (AttributeError, OSError, RuntimeError): + identity_ready = False + + preview_home = Path(raw_home) / "preview-home" + preview_init = invoke( + agent_lab, ["--home", str(preview_home), "init"] + ) + preview_before = tree_fingerprint(preview_home) + source_before = sha256(source).hexdigest() + original_agent_experiment_module = agent_lab.experiment_module + original_read_git_snapshot = experiment.read_git_snapshot + original_authored_manifest = experiment.authored_manifest + original_cue_plan = experiment.cue_plan_with_evidence + original_authorize_plan = experiment.authorize_plan + original_write_checked = experiment.write_checked_source + original_write_decision = experiment.write_decision + active_operation = [""] + operation_log: list[tuple[str, str]] = [] + snapshot_handoffs: list[bool] = [] + checked_values: list[object] = [] + decision_values: list[object] = [] + + def injected_git_snapshot(url, commit): + operation_log.append((active_operation[0], "snapshot")) + if url != URL or commit != COMMIT or git_snapshot is None: + raise experiment.InfrastructureError( + "git source GIT-TEST fixture is unavailable" + ) + return git_snapshot + + def logged_manifest(value): + operation_log.append((active_operation[0], "manifest")) + snapshot_handoffs.append(value is git_snapshot) + return original_authored_manifest(value) + + def logged_plan(value): + operation_log.append((active_operation[0], "plan")) + return original_cue_plan(value) + + def logged_authorize(value, digest): + operation_log.append((active_operation[0], "authorize")) + return original_authorize_plan(value, digest) + + experiment.read_git_snapshot = injected_git_snapshot + experiment.authored_manifest = logged_manifest + experiment.cue_plan_with_evidence = logged_plan + experiment.authorize_plan = logged_authorize + experiment.write_checked_source = checked_values.append + experiment.write_decision = decision_values.append + agent_lab.experiment_module = lambda: experiment + try: + active_operation[0] = "check" + checked_result = invoke_with_exit( + agent_lab, + [ + "--home", + str(preview_home), + "experiment", + "check", + "--git", + URL, + "--commit", + COMMIT, + ], + ) + active_operation[0] = "authorize" + authorized_result = invoke_with_exit( + agent_lab, + [ + "--home", + str(preview_home), + "experiment", + "authorize", + "install", + "--git", + URL, + "--commit", + COMMIT, + ], + ) + finally: + agent_lab.experiment_module = original_agent_experiment_module + experiment.read_git_snapshot = original_read_git_snapshot + experiment.authored_manifest = original_authored_manifest + experiment.cue_plan_with_evidence = original_cue_plan + experiment.authorize_plan = original_authorize_plan + experiment.write_checked_source = original_write_checked + experiment.write_decision = original_write_decision + + expected_checked = None + if identity_ready and git_resolution is not None: + expected_checked = { + "digest": experiment.plan_digest(git_resolution.plan), + "plan": git_resolution.plan, + "source": { + "digest": git_snapshot.digest, + **git_snapshot.transport, + }, + } + catalog = experiment.catalog_resolution_evidence( + git_resolution.bundled_catalog, + git_resolution.local_catalog, + ) + if catalog is not None: + expected_checked["catalog"] = catalog + results.append( + ( + "GIT-CHECK-001", + checked_result == (0, "", "") + and checked_values == [expected_checked], + "public Git check uses the common checked-source path", + ) + ) + results.append( + ( + "GIT-AUTH-001", + authorized_result == (0, "", "") + and identity_ready + and decision_values == [directory_decision] + and directory_decision == zip_decision == git_decision, + "Git authorization uses the common source-bound Cedar decision", + ) + ) + + expected_log = [ + ("check", "snapshot"), + ("check", "manifest"), + ("check", "plan"), + ("authorize", "snapshot"), + ("authorize", "manifest"), + ("authorize", "plan"), + ("authorize", "authorize"), + ] + adapter_downstream_calls: list[str] = [] + + def forbidden_adapter_downstream(*_args, **_kwargs): + adapter_downstream_calls.append("reached") + raise AssertionError("Git adapter crossed into the common pipeline") + + experiment.authored_manifest = forbidden_adapter_downstream + experiment.cue_plan_with_evidence = forbidden_adapter_downstream + experiment.authorize_plan = forbidden_adapter_downstream + try: + try: + adapter_snapshot = original_read_git_snapshot( + URL, COMMIT, requester=requester + ) + except (experiment.InvalidManifest, experiment.InfrastructureError): + adapter_snapshot = None + finally: + experiment.authored_manifest = original_authored_manifest + experiment.cue_plan_with_evidence = original_cue_plan + experiment.authorize_plan = original_authorize_plan + results.append( + ( + "GIT-ADAPTER-001", + identity_ready + and operation_log == expected_log + and snapshot_handoffs == [True, True] + and adapter_snapshot == git_snapshot + and adapter_downstream_calls == [], + "Git acquisition returns one snapshot to each unchanged common pipeline", + ) + ) + preview_after = tree_fingerprint(preview_home) + results.append( + ( + "GIT-NOEF-001", + preview_init[0] == 0 + and checked_result[0] == 0 + and authorized_result[0] == 0 + and preview_before == preview_after + and source_before == sha256(source).hexdigest() + and operation_log == expected_log, + "Git previews leave initialized home and source bytes unchanged", + ) + ) + + hostile_marker = b"caller-private-diagnostic" + + def hostile_read(url, commit): + def hostile_requester( + _authority, _path, _headers, _maximum, _deadline + ): + return ( + 500, + ( + ("content-length", str(len(hostile_marker))), + ( + "content-type", + "application/json; charset=utf-8", + ), + ), + hostile_marker, + ) + + return original_read_git_snapshot( + url, commit, requester=hostile_requester + ) + + experiment.read_git_snapshot = hostile_read + agent_lab.experiment_module = lambda: experiment + try: + diagnostic_result = invoke_with_exit( + agent_lab, + [ + "--home", + str(preview_home), + "experiment", + "check", + "--git", + URL, + "--commit", + COMMIT, + ], + ) + finally: + experiment.read_git_snapshot = original_read_git_snapshot + agent_lab.experiment_module = original_agent_experiment_module + results.append( + ( + "GIT-DIAG-001", + diagnostic_result[0] == 125 + and diagnostic_result[1] == "" + and "GIT-STATUS" in diagnostic_result[2] + and hostile_marker.decode("ascii") not in diagnostic_result[2], + "hostile provider diagnostics never reach public output", + ) + ) + + store = load_module( + repo / "scripts/experiment_store.py", "git_intake_store" + ) + original_store_experiment_module = store._experiment_module + bool_transport_fields_rejected = True + for field in ( + "limitBytes", + "requestCount", + "temporaryBytes", + "temporaryFiles", + ): + hostile_transport = json.loads( + json.dumps(expected_transport, sort_keys=True) + ) + hostile_transport["acquisition"][field] = False + if store._closed_git_transport(hostile_transport) is not None: + bool_transport_fields_rejected = False + hostile_transport_urls_rejected = True + for hostile_url in ( + "https://github.com/a--b/repo.git", + "https://github.com/owner/..git", + "https://github.com/owner/...git", + ): + hostile_transport = json.loads( + json.dumps(expected_transport, sort_keys=True) + ) + hostile_transport["url"] = hostile_url + if store._closed_git_transport(hostile_transport) is not None: + hostile_transport_urls_rejected = False + + class ExperimentFacade: + def __init__(self, *, deny: bool = False): + self.deny = deny + self.acquisitions: list[tuple[str, str]] = [] + + def __getattr__(self, name): + return getattr(experiment, name) + + def read_git_snapshot(self, url, commit): + self.acquisitions.append((url, commit)) + if url != URL or commit != COMMIT or git_snapshot is None: + raise experiment.InfrastructureError( + "git source GIT-TEST fixture is unavailable" + ) + return git_snapshot + + def authorize_plan(self, plan, digest): + decision, status = original_authorize_plan(plan, digest) + if not self.deny: + return decision, status + denied = json.loads( + json.dumps( + decision, + ensure_ascii=True, + separators=(",", ":"), + sort_keys=True, + ) + ) + denied["verdict"] = "deny" + return denied, 1 + + def initialized_home(name: str) -> tuple[Path, bool]: + home = Path(raw_home) / name + outcome = invoke(agent_lab, ["--home", str(home), "init"]) + return home, outcome[0] == 0 and outcome[2] == "" + + def store_call(name: str, *arguments): + operation = getattr(store, name, None) + if not callable(operation): + return "missing", None + try: + return "ok", operation(*arguments) + except store.StoreReject as error: + return "reject", str(error) + except store.StoreInfrastructure as error: + return "infra", str(error) + except Exception as error: + return "error", type(error).__name__ + + deny_home, deny_home_ready = initialized_home("git-deny-home") + deny_before = tree_fingerprint(deny_home) + deny_facade = ExperimentFacade(deny=True) + store._experiment_module = lambda: deny_facade + deny_outcome = store_call("install_git", deny_home, URL, COMMIT) + deny_after = tree_fingerprint(deny_home) + results.append( + ( + "GIT-DENY-001", + deny_home_ready + and deny_outcome[0] == "reject" + and "authorization denied" in str(deny_outcome[1]) + and deny_facade.acquisitions == [(URL, COMMIT)] + and deny_before == deny_after, + "denied Git install leaves the initialized home byte-identical", + ) + ) + + install_home, install_home_ready = initialized_home("git-install-home") + install_facade = ExperimentFacade() + store._experiment_module = lambda: install_facade + install_outcome = store_call( + "install_git", install_home, URL, COMMIT + ) + installed_root = install_home / "experiments/first-experiment" + installed_provenance = None + installed_artifact = None + if install_outcome[0] == "ok": + try: + installed_provenance = json.loads( + (installed_root / "records/provenance.json").read_text( + encoding="utf-8" + ) + ) + installed_artifact = ( + installed_root / "artifact/experiment.cue" + ).read_bytes() + except (OSError, UnicodeError, ValueError): + installed_provenance = None + installed_artifact = None + closed_git_provenance = ( + isinstance(installed_provenance, dict) + and set(installed_provenance) + == { + "apiVersion", + "authorizationDigest", + "catalog", + "contractDigest", + "kind", + "planDigest", + "selectedEntries", + "source", + "transport", + } + and installed_provenance.get("source") + == { + "bytes": len(source), + "digest": SOURCE_DIGEST, + "entryCount": 1, + "fileCount": 1, + "format": "agent-lab.experiment-tree/v1", + "kind": "directory", + } + and installed_provenance.get("transport") == expected_transport + ) + results.append( + ( + "GIT-INSTALL-001", + install_home_ready + and install_outcome[0] == "ok" + and isinstance(install_outcome[1], dict) + and install_outcome[1].get("changed") is True + and install_facade.acquisitions == [(URL, COMMIT)] + and installed_artifact == source + and closed_git_provenance + and bool_transport_fields_rejected + and hostile_transport_urls_rejected, + "permitted Git install publishes the common artifact and closed provenance", + ) + ) + + directory_home, directory_home_ready = initialized_home( + "identity-directory-home" + ) + zip_home, zip_home_ready = initialized_home("identity-zip-home") + store._experiment_module = lambda: ExperimentFacade() + directory_install = store_call( + "install_directory", directory_home, fixture_directory + ) + zip_install = store_call("install_zip", zip_home, common_zip) + + def installed_bytes(home: Path, relative: str) -> bytes | None: + try: + return ( + home / "experiments/first-experiment" / relative + ).read_bytes() + except OSError: + return None + + directory_key = ( + directory_install[1].get("installationKey") + if directory_install[0] == "ok" + and isinstance(directory_install[1], dict) + else None + ) + zip_key = ( + zip_install[1].get("installationKey") + if zip_install[0] == "ok" and isinstance(zip_install[1], dict) + else None + ) + git_key = ( + install_outcome[1].get("installationKey") + if install_outcome[0] == "ok" + and isinstance(install_outcome[1], dict) + else None + ) + semantic_identity = ( + identity_ready + and directory_snapshot.digest + == zip_snapshot.digest + == git_snapshot.digest + == SOURCE_DIGEST + and directory_manifest == zip_manifest == git_manifest + and directory_resolution.plan + == zip_resolution.plan + == git_resolution.plan + and directory_decision == zip_decision == git_decision + ) + stored_identity = ( + directory_home_ready + and zip_home_ready + and directory_install[0] == "ok" + and zip_install[0] == "ok" + and install_outcome[0] == "ok" + and directory_key == zip_key == git_key + and directory_key is not None + and installed_bytes(directory_home, "artifact/experiment.cue") + == installed_bytes(zip_home, "artifact/experiment.cue") + == installed_bytes(install_home, "artifact/experiment.cue") + == source + and installed_bytes(directory_home, "records/plan.json") + == installed_bytes(zip_home, "records/plan.json") + == installed_bytes(install_home, "records/plan.json") + and installed_bytes(directory_home, "records/decision.json") + == installed_bytes(zip_home, "records/decision.json") + == installed_bytes(install_home, "records/decision.json") + ) + results.append( + ( + "GIT-IDENTITY-001", + semantic_identity and stored_identity, + "directory, ZIP, and Git share manifest, plan, Cedar, key, and artifact identity", + ) + ) + + retry_home, retry_home_ready = initialized_home("git-retry-home") + retry_facade = ExperimentFacade() + store._experiment_module = lambda: retry_facade + retry_directory = store_call( + "install_directory", retry_home, fixture_directory + ) + retry_receipt_path = ( + retry_home + / "experiments/first-experiment/records/install.json" + ) + retry_receipt_first = installed_bytes(retry_home, "records/install.json") + retry_zip = store_call("install_zip", retry_home, common_zip) + retry_receipt_second = installed_bytes(retry_home, "records/install.json") + retry_git = store_call("install_git", retry_home, URL, COMMIT) + retry_receipt_third = installed_bytes(retry_home, "records/install.json") + results.append( + ( + "GIT-RETRY-001", + retry_home_ready + and retry_directory[0] == "ok" + and retry_zip[0] == "ok" + and retry_git[0] == "ok" + and isinstance(retry_zip[1], dict) + and isinstance(retry_git[1], dict) + and retry_zip[1].get("changed") is False + and retry_git[1].get("changed") is False + and retry_facade.acquisitions == [(URL, COMMIT)] + and retry_receipt_path.is_file() + and retry_receipt_first + == retry_receipt_second + == retry_receipt_third, + "second and third equivalent transports preserve the first receipt", + ) + ) + store._experiment_module = original_store_experiment_module + + runtime_manifest = repo / "packaging/agent-lab-local.manifest" + expected_runtime = ( + repo / "tests/install/fixtures/expected-runtime-files.txt" + ) + runtime_root = Path(raw_home) / "installed-runtime" + runtime_ready = runtime_manifest.read_bytes() == expected_runtime.read_bytes() + runtime_names = expected_runtime.read_text(encoding="utf-8").splitlines() + for runtime_name in runtime_names: + source_path = repo / runtime_name + target_path = runtime_root / runtime_name + if not runtime_name or not source_path.is_file(): + runtime_ready = False + continue + target_path.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(source_path, target_path) + unrelated = Path(raw_home) / "unrelated-cwd" + runtime_home = Path(raw_home) / "runtime-home" + runtime_tmp = Path(raw_home) / "runtime-tmp" + unrelated.mkdir() + runtime_tmp.mkdir() + runtime_environment = { + "PATH": "/usr/bin:/bin", + "HOME": str(Path(raw_home) / "runtime-user-home"), + "TMPDIR": str(runtime_tmp), + "LC_ALL": "C", + "AGENT_LAB_CUE_TOOL_DIR": str(repo / ".cache/dev/tools/cue"), + "AGENT_LAB_CEDAR_TOOL_DIR": str( + repo / ".cache/dev/tools/cedar" + ), + } + runtime_command = [ + str(runtime_root / "scripts/agent-lab"), + "--home", + str(runtime_home), + "experiment", + "check", + "--git", + "https://example.com/uscient/experiment-fixture.git", + "--commit", + COMMIT, + ] + try: + runtime_result = subprocess.run( + runtime_command, + cwd=unrelated, + env=runtime_environment, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + timeout=5, + check=False, + ) + except (OSError, subprocess.SubprocessError): + runtime_result = None + results.append( + ( + "GIT-RUNTIME-001", + runtime_ready + and runtime_result is not None + and runtime_result.returncode == 1 + and runtime_result.stdout == b"" + and b"GIT-URL" in runtime_result.stderr + and str(repo).encode("utf-8") not in runtime_result.stderr + and not any(runtime_root.rglob("__pycache__")), + "installed CLI reaches Git validation from a minimal unrelated runtime", + ) + ) + + cycle4_results = {item[0]: item for item in results[-10:]} + del results[-10:] + results.extend(cycle4_results[item] for item in EXPECTED[-10:]) + + for name, value in prior_tools.items(): + if value is None: + os.environ.pop(name, None) + else: + os.environ[name] = value + make_writable(Path(raw_home)) + finally: + if prior_home is None: + os.environ.pop("AGENT_LAB_HOME", None) + else: + os.environ["AGENT_LAB_HOME"] = prior_home + except Exception as error: + print(f"INFRA Git intake contract probe failed: {type(error).__name__}", file=sys.stderr) + infrastructure = 1 + + observed = tuple(item[0] for item in results) + if observed != EXPECTED: + infrastructure = 1 + for assertion, passed, detail in results: + if passed: + print(f"PASS {assertion} {detail}") + else: + print(f"FAIL {assertion} {detail}") + failures += 1 + print( + f"SUMMARY assertions={len(results)} expected={len(EXPECTED)} " + f"failures={failures} infra={infrastructure}" + ) + if infrastructure: + return 125 + if failures: + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/experiment/git-intake-cases.sh b/tests/experiment/git-intake-cases.sh new file mode 100755 index 0000000..5303b68 --- /dev/null +++ b/tests/experiment/git-intake-cases.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +set -u -o pipefail + +repo_root="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." >/dev/null 2>&1 && pwd)" +cases="$repo_root/tests/experiment/git-intake-cases.py" + +if [ ! -f "$cases" ] || ! command -v python3 >/dev/null 2>&1; then + printf 'SUMMARY assertions=0 expected=32 failures=0 infra=1\n' + exit 125 +fi + +exec python3 -I -B "$cases" diff --git a/tests/experiment/git-mutation-cases.py b/tests/experiment/git-mutation-cases.py new file mode 100755 index 0000000..16ddc57 --- /dev/null +++ b/tests/experiment/git-mutation-cases.py @@ -0,0 +1,667 @@ +#!/usr/bin/env python3 +"""Private-copy sensitivity mutations for pinned Git Experiment intake.""" + +from __future__ import annotations + +import base64 +from hashlib import sha1, sha256 +from importlib.util import module_from_spec, spec_from_file_location +import io +import json +import os +from pathlib import Path +import shutil +import signal +import socket +import stat +import sys +import tempfile +import time +from types import SimpleNamespace +from typing import Callable + + +EXPECTED = ( + "M-GIT-AUTHORITY-001", + "M-GIT-REF-001", + "M-GIT-BOUND-001", + "M-GIT-IDENTITY-001", + "M-GIT-DOWNSTREAM-001", +) +URL = "https://github.com/uscient/experiment-fixture.git" +FORBIDDEN_AUTHORITY = "credential.invalid" +COMMIT = "1cffa1a28f96d2f2cb898b1bad70d281e359a5b5" +TREE = "64564b8e82ec9581c32cb4951ed802b544e2e0c0" +BLOB = "a1d8c8cd0f1865e66cb2463cbaa801c4b5a85656" +SOURCE_DIGEST = "sha256:463e8a7622e58281fd975d58d8a9ad44ed997dd08af32e237f1476021f7abb23" +MARKER_ENV = "AGENT_LAB_GIT_MUTATION_MARK" + + +class HarnessInfrastructure(Exception): + """Private mutation evidence could not be established safely.""" + + +def load_module(path: Path, label: str): + spec = spec_from_file_location(label, path) + if spec is None or spec.loader is None: + raise HarnessInfrastructure(f"cannot load private runtime {path.name}") + module = module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +def git_oid(kind: str, payload: bytes) -> str: + framed = kind.encode("ascii") + b" " + str(len(payload)).encode("ascii") + b"\0" + return sha1(framed + payload).hexdigest() + + +def response(body: object) -> tuple[int, tuple[tuple[str, str], ...], bytes]: + encoded = json.dumps( + body, + ensure_ascii=True, + separators=(",", ":"), + sort_keys=True, + ).encode("ascii") + return ( + 200, + ( + ("content-length", str(len(encoded))), + ("content-type", "application/json; charset=utf-8"), + ), + encoded, + ) + + +def fixture_responses(source: bytes) -> dict[str, tuple[int, tuple[tuple[str, str], ...], bytes]]: + tree_payload = b"100644 experiment.cue\0" + bytes.fromhex(BLOB) + commit_payload = ( + f"tree {TREE}\n" + "author Fixture 0 +0000\n" + "committer Fixture 0 +0000\n" + "\n" + "pinned fixture\n" + ).encode("ascii") + if ( + git_oid("blob", source) != BLOB + or git_oid("tree", tree_payload) != TREE + or git_oid("commit", commit_payload) != COMMIT + ): + raise HarnessInfrastructure("independent Git fixture identity drift") + return { + f"/repos/uscient/experiment-fixture/git/commits/{COMMIT}": response( + {"sha": COMMIT, "tree": {"sha": TREE}} + ), + f"/repos/uscient/experiment-fixture/git/trees/{TREE}": response( + { + "sha": TREE, + "tree": [ + { + "mode": "100644", + "path": "experiment.cue", + "sha": BLOB, + "size": len(source), + "type": "blob", + } + ], + "truncated": False, + } + ), + f"/repos/uscient/experiment-fixture/git/blobs/{BLOB}": response( + { + "content": base64.b64encode(source).decode("ascii"), + "encoding": "base64", + "sha": BLOB, + "size": len(source), + } + ), + } + + +def fixture_requester( + responses: dict[str, tuple[int, tuple[tuple[str, str], ...], bytes]] +): + def request(_authority, path, _headers, _maximum, _deadline): + return responses[path] + + return request + + +def fingerprint(root: Path) -> dict[str, tuple[int, str]]: + result: dict[str, tuple[int, str]] = {} + for path in sorted(root.rglob("*")): + if path.is_file(): + result[path.relative_to(root).as_posix()] = ( + stat.S_IMODE(path.stat().st_mode), + sha256(path.read_bytes()).hexdigest(), + ) + elif not path.is_dir(): + raise HarnessInfrastructure("private runtime contains an unsupported entry") + return result + + +def replace_once(source: str, needle: str, replacement: str, assertion: str) -> str: + occurrences = source.count(needle) + if occurrences != 1: + raise HarnessInfrastructure( + f"{assertion} replacement applicability is {occurrences}, expected exactly 1" + ) + mutated = source.replace(needle, replacement, 1) + if mutated == source or mutated.count(replacement) != 1: + raise HarnessInfrastructure(f"{assertion} replacement result is ambiguous") + try: + compile(mutated, f"<{assertion}>", "exec") + except SyntaxError as error: + raise HarnessInfrastructure(f"{assertion} private mutation does not compile") from error + return mutated + + +def authority_probe(module, marker: Path | None) -> bool: + import http.client + import ssl + + connections: list[tuple[str, int, float, object]] = [] + requests: list[tuple[str, str, dict[str, str]]] = [] + body = b"{}" + + class FakeContext: + def __init__(self, protocol): + self.protocol = protocol + self.check_hostname = False + self.verify_mode = None + self.minimum_version = None + self.loaded = None + + def load_verify_locations(self, *, cadata): + self.loaded = cadata + + class FakeSocket: + def settimeout(self, _timeout): + return None + + class FakeResponse: + status = 200 + + def __init__(self): + self.stream = io.BytesIO(body) + + def getheaders(self): + return [ + ("content-length", str(len(body))), + ("content-type", "application/json; charset=utf-8"), + ] + + def read(self, size=-1): + return self.stream.read(size) + + def close(self): + return None + + class FakeConnection: + def __init__(self, authority, *, port, timeout, context): + connections.append((authority, port, timeout, context)) + self.sock = FakeSocket() + + def request(self, method, path, *, headers): + requests.append((method, path, dict(headers))) + + def getresponse(self): + return FakeResponse() + + def close(self): + return None + + original_context = ssl.SSLContext + original_connection = http.client.HTTPSConnection + original_maxline = http.client._MAXLINE + original_maxheaders = http.client._MAXHEADERS + original_ca_reader = module._git_system_ca_pem + ssl.SSLContext = FakeContext + http.client.HTTPSConnection = FakeConnection + module._git_system_ca_pem = lambda: "fixture-ca" + try: + try: + result = module._github_api_request( + FORBIDDEN_AUTHORITY, + f"/repos/uscient/experiment-fixture/git/commits/{COMMIT}", + module.GIT_PROVIDER_HEADERS, + 1_024, + time.monotonic() + 1.0, + ) + except module.InfrastructureError as error: + outcome = ("infra", str(error)) + else: + outcome = ("ok", result) + finally: + module._git_system_ca_pem = original_ca_reader + http.client.HTTPSConnection = original_connection + http.client._MAXLINE = original_maxline + http.client._MAXHEADERS = original_maxheaders + ssl.SSLContext = original_context + + if marker is None: + return ( + outcome[0] == "infra" + and "GIT-AUTHORITY" in outcome[1] + and connections == [] + and requests == [] + ) + if len(connections) != 1: + return False + authority, port, timeout, context = connections[0] + return ( + outcome == ( + "ok", + ( + 200, + ( + ("content-length", str(len(body))), + ("content-type", "application/json; charset=utf-8"), + ), + body, + ), + ) + and authority == FORBIDDEN_AUTHORITY + and port == 443 + and 0 < timeout <= 1.0 + and context.check_hostname is True + and context.verify_mode == ssl.CERT_REQUIRED + and context.minimum_version == ssl.TLSVersion.TLSv1_2 + and context.loaded == "fixture-ca" + and requests + == [ + ( + "GET", + f"/repos/uscient/experiment-fixture/git/commits/{COMMIT}", + dict(module.GIT_PROVIDER_HEADERS), + ) + ] + ) + + +def ref_probe(module, marker: Path | None, responses, source: bytes) -> bool: + if marker is None: + calls: list[str] = [] + + def unused(_authority, path, _headers, _maximum, _deadline): + calls.append(path) + raise AssertionError("mutable ref reached acquisition") + + try: + module.read_git_snapshot(URL, "main", requester=unused) + except module.InvalidManifest as error: + return "GIT-OID" in str(error) and calls == [] + return False + try: + snapshot = module.read_git_snapshot( + URL, + "main", + requester=fixture_requester(responses), + ) + except (module.InvalidManifest, module.InfrastructureError): + return False + return ( + snapshot.data == source + and snapshot.digest == SOURCE_DIGEST + and snapshot.transport.get("requestedCommit") == COMMIT + ) + + +def group_exists(pgid: int) -> bool: + try: + os.killpg(pgid, 0) + except ProcessLookupError: + return False + except PermissionError: + return True + return True + + +def wait_group_gone(pgid: int, timeout: float = 2.0) -> bool: + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + if not group_exists(pgid): + return True + time.sleep(0.01) + return not group_exists(pgid) + + +def bound_probe(module, marker: Path | None, responses) -> bool: + listener = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + listener.bind(("127.0.0.1", 0)) + listener.settimeout(2.0) + address = listener.getsockname() + + def residual_requester(_authority, path, _headers, _maximum, _deadline): + child = os.fork() + if child == 0: + try: + signal.signal(signal.SIGTERM, signal.SIG_IGN) + signal.alarm(5) + with socket.socket(socket.AF_INET, socket.SOCK_DGRAM) as sender: + sender.sendto( + f"{os.getpid()}:{os.getpgrp()}".encode("ascii"), + address, + ) + while True: + signal.pause() + finally: + os._exit(0) + return responses[path] + + module._github_api_request = residual_requester + try: + try: + module.read_git_snapshot(URL, COMMIT) + except module.InfrastructureError as error: + outcome = str(error) + else: + outcome = "ok" + try: + record = listener.recv(128).decode("ascii") + except (TimeoutError, UnicodeDecodeError) as error: + raise HarnessInfrastructure("residual worker identity was not reported") from error + finally: + listener.close() + try: + raw_pid, raw_pgid = record.split(":", 1) + pid = int(raw_pid) + pgid = int(raw_pgid) + except (TypeError, ValueError) as error: + raise HarnessInfrastructure("residual worker identity is malformed") from error + if pid <= 1 or pgid <= 1 or pgid == os.getpgrp(): + raise HarnessInfrastructure("residual worker group is not private") + + alive = group_exists(pgid) + cleanup_ok = True + if alive: + try: + os.killpg(pgid, signal.SIGKILL) + except ProcessLookupError: + pass + cleanup_ok = wait_group_gone(pgid) + if not cleanup_ok: + raise HarnessInfrastructure("residual worker cleanup is uncertain") + if marker is None: + return "residual process group" in outcome.lower() and not alive + return "residual process group" in outcome.lower() and alive + + +def identity_probe(module, marker: Path | None, responses, source: bytes, source_dir: Path) -> bool: + try: + directory = module.read_directory_snapshot(str(source_dir)) + git = module.read_git_snapshot( + URL, + COMMIT, + requester=fixture_requester(responses), + ) + except (module.InvalidManifest, module.InfrastructureError): + return False + if directory.data != source or git.data != source: + return False + if marker is None: + return directory.digest == git.digest == SOURCE_DIGEST + expected_object_digest = "sha256:" + sha256(BLOB.encode("ascii")).hexdigest() + return git.digest == expected_object_digest and directory.digest != git.digest + + +def downstream_probe(module, marker: Path | None) -> bool: + events: list[str] = [] + snapshot = module.SourceSnapshot( + data=b"fixture", + digest="sha256:" + "1" * 64, + transport={"kind": "git"}, + ) + resolution = SimpleNamespace(plan={}, bundled_catalog=None, local_catalog=None) + + def authorize(_plan, _digest): + events.append("authorize") + return {"verdict": "deny"}, 1 + + module.read_git_snapshot = lambda _url, _commit: snapshot + module.authored_manifest = lambda _snapshot: {} + module.cue_plan_with_evidence = lambda _manifest: resolution + module.authorize_plan = authorize + module.write_decision = lambda _decision: events.append("decision") + module.write_checked_source = lambda _checked: events.append("checked") + result = module.main(["experiment.py", "authorize-git", URL, COMMIT]) + if marker is None: + return result == 1 and events == ["authorize", "decision"] + return result == 0 and events == ["checked"] + + +Probe = Callable[[object, Path | None], bool] + + +def execute_mutation( + repo: Path, + production: Path, + original: str, + production_digest: str, + root: Path, + assertion: str, + needle: str, + replacement: str, + probe: Probe, +) -> bool: + runtime = root / "runtime" + runtime.mkdir(parents=True) + private_source = runtime / "experiment.py" + shutil.copy2(production, private_source) + shutil.copy2(repo / "scripts/image_reference.py", runtime / "image_reference.py") + pristine_topology = fingerprint(runtime) + pristine = load_module(private_source, assertion.lower().replace("-", "_") + "_pristine") + if not probe(pristine, None): + raise HarnessInfrastructure(f"{assertion} pristine private probe is not GREEN") + if fingerprint(runtime) != pristine_topology: + raise HarnessInfrastructure(f"{assertion} pristine probe changed private runtime") + + mutated = replace_once(original, needle, replacement, assertion) + private_source.write_text(mutated, encoding="utf-8") + mutated_topology = fingerprint(runtime) + changed = [ + path + for path in sorted(set(pristine_topology) | set(mutated_topology)) + if pristine_topology.get(path) != mutated_topology.get(path) + ] + if changed != ["experiment.py"]: + raise HarnessInfrastructure(f"{assertion} changed unexpected private runtime paths") + + marker = root / "mutation-reached" + marker.unlink(missing_ok=True) + prior_marker = os.environ.get(MARKER_ENV) + os.environ[MARKER_ENV] = str(marker) + try: + mutant = load_module(private_source, assertion.lower().replace("-", "_") + "_mutant") + detected = probe(mutant, marker) + finally: + if prior_marker is None: + os.environ.pop(MARKER_ENV, None) + else: + os.environ[MARKER_ENV] = prior_marker + if not marker.is_file(): + raise HarnessInfrastructure(f"{assertion} did not prove its mutated path was reached") + if fingerprint(runtime) != mutated_topology: + raise HarnessInfrastructure(f"{assertion} mutant probe changed private runtime") + if sha256(production.read_bytes()).hexdigest() != production_digest: + raise HarnessInfrastructure(f"{assertion} changed the production implementation") + return detected + + +def make_writable(root: Path) -> None: + for path in root.rglob("*"): + try: + path.chmod(path.stat().st_mode | stat.S_IWUSR | stat.S_IXUSR) + except OSError: + pass + + +def main() -> int: + sys.dont_write_bytecode = True + repo = Path(__file__).resolve().parents[2] + production = repo / "scripts/experiment.py" + original = production.read_text(encoding="utf-8") + production_digest = sha256(production.read_bytes()).hexdigest() + source_dir = repo / "tests/experiment/fixtures/directories/minimal" + source = (source_dir / "experiment.cue").read_bytes() + responses = fixture_responses(source) + work = Path(tempfile.mkdtemp(prefix="agent-lab-git-mutations-")) + failures = 0 + infrastructure = 0 + results: list[tuple[str, bool, str]] = [] + try: + cases: tuple[tuple[str, str, str, Probe, str], ...] = ( + ( + "M-GIT-AUTHORITY-001", + " authority != GIT_PROVIDER_AUTHORITY\n", + ( + " (\n" + " authority != GIT_PROVIDER_AUTHORITY\n" + " and (\n" + f" Path(os.environ[\"{MARKER_ENV}\"]).touch()\n" + " or False\n" + " )\n" + " )\n" + ), + authority_probe, + "forbidden provider authority reaches the HTTPS connection canary", + ), + ( + "M-GIT-REF-001", + " if not isinstance(commit, str) or GIT_SHA1.fullmatch(commit) is None:\n", + ( + " if commit == \"main\":\n" + f" Path(os.environ[\"{MARKER_ENV}\"]).touch()\n" + f" commit = \"{COMMIT}\"\n" + " if not isinstance(commit, str) or GIT_SHA1.fullmatch(commit) is None:\n" + ), + lambda module, marker: ref_probe(module, marker, responses, source), + "mutable ref acceptance reaches the pinned-object adapter", + ), + ( + "M-GIT-BOUND-001", + ( + " terminated = _git_worker_terminate(\n" + " pid,\n" + " reaped=reaped,\n" + " deadline=cleanup_deadline,\n" + " )\n" + ), + ( + f" Path(os.environ[\"{MARKER_ENV}\"]).touch()\n" + " mutant_observed = _git_worker_observe(pid)\n" + " mutant_waited, mutant_status = os.waitpid(\n" + " pid, os.WNOHANG\n" + " )\n" + " terminated = (\n" + " mutant_observed is not None\n" + " and mutant_waited == pid\n" + " and _git_worker_status_matches(\n" + " mutant_observed, mutant_status\n" + " )\n" + " )\n" + ), + lambda module, marker: bound_probe(module, marker, responses), + "removed process-group cleanup leaves a live provider descendant", + ), + ( + "M-GIT-IDENTITY-001", + ( + " digest=source_digest(data),\n" + " transport={\n" + " \"acquisition\": {\n" + ), + ( + " digest=(\n" + f" Path(os.environ[\"{MARKER_ENV}\"]).touch()\n" + " or \"sha256:\"\n" + " + hashlib.sha256(blob_id.encode(\"ascii\")).hexdigest()\n" + " ),\n" + " transport={\n" + " \"acquisition\": {\n" + ), + lambda module, marker: identity_probe( + module, + marker, + responses, + source, + source_dir, + ), + "object-ID-derived identity breaks the cross-transport oracle", + ), + ( + "M-GIT-DOWNSTREAM-001", + " if directory_checking or zip_checking or git_checking:\n", + ( + " if (\n" + " directory_checking\n" + " or zip_checking\n" + " or git_checking\n" + " or (\n" + " git_authorizing\n" + " and (\n" + f" Path(os.environ[\"{MARKER_ENV}\"]).touch()\n" + " or True\n" + " )\n" + " )\n" + " ):\n" + ), + downstream_probe, + "Git authorization routed around the common decision path", + ), + ) + for index, (assertion, needle, replacement, probe, message) in enumerate(cases): + case_root = work / f"case-{index}" + try: + detected = execute_mutation( + repo, + production, + original, + production_digest, + case_root, + assertion, + needle, + replacement, + probe, + ) + except HarnessInfrastructure: + raise + except Exception as error: + raise HarnessInfrastructure( + f"{assertion} probe raised {type(error).__name__}" + ) from error + results.append((assertion, detected, message)) + except (HarnessInfrastructure, OSError) as error: + print(f"INFRA Git mutation harness {error}", file=sys.stderr) + infrastructure = 1 + finally: + try: + make_writable(work) + shutil.rmtree(work) + except OSError: + infrastructure = 1 + + if sha256(production.read_bytes()).hexdigest() != production_digest: + infrastructure = 1 + observed = tuple(assertion for assertion, _, _ in results) + if observed != EXPECTED: + infrastructure = 1 + for assertion, passed, message in results: + if passed: + print(f"PASS {assertion} {message}") + else: + print(f"FAIL {assertion} {message}") + failures += 1 + print( + f"SUMMARY assertions={len(results)} expected={len(EXPECTED)} " + f"failures={failures} infra={infrastructure}" + ) + if infrastructure: + return 125 + if failures: + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/experiment/install-integrity-cases.py b/tests/experiment/install-integrity-cases.py new file mode 100644 index 0000000..4e05188 --- /dev/null +++ b/tests/experiment/install-integrity-cases.py @@ -0,0 +1,1064 @@ +#!/usr/bin/env python3 +"""Private-runtime integrity cases for Experiment installation boundaries.""" + +from __future__ import annotations + +from contextlib import contextmanager +from dataclasses import dataclass +import errno +from importlib.util import module_from_spec, spec_from_file_location +import json +import os +from pathlib import Path +import signal +import stat +import subprocess +import sys +import tempfile +import time +from typing import Callable, Iterator, NamedTuple + + +REPO_ROOT = Path(__file__).resolve().parents[2] +SUPPORT_PATH = REPO_ROOT / "tests" / "experiment" / "install-mutation-cases.py" +CUE_TOOLS = REPO_ROOT / ".cache" / "dev" / "tools" / "cue" +CEDAR_TOOLS = REPO_ROOT / ".cache" / "dev" / "tools" / "cedar" +BUNDLED_CATALOG_DOMAIN = b"agent-lab.experiment-image-catalog.v1\0" +PLAN_DOMAIN = b"agent-lab.experiment-plan.v1\0" +PUBLIC_TIMEOUT_SECONDS = 2.0 +RACE_TIMEOUT_SECONDS = 0.75 + + +def load_support(): + spec = spec_from_file_location("agent_lab_install_integrity_support", SUPPORT_PATH) + if spec is None or spec.loader is None: + raise RuntimeError("install mutation support cannot be loaded") + module = module_from_spec(spec) + sys.modules[spec.name] = module + previous = sys.dont_write_bytecode + sys.dont_write_bytecode = True + try: + spec.loader.exec_module(module) + finally: + sys.dont_write_bytecode = previous + return module + + +try: + SUPPORT = load_support() + SUPPORT_ERROR: BaseException | None = None +except BaseException as error: + SUPPORT = None + SUPPORT_ERROR = error + + +if SUPPORT is None: + class SupportInfrastructure(Exception): + """Fallback type used only when the private support module cannot load.""" +else: + SupportInfrastructure = SUPPORT.InfrastructureError + + +class IntegrityInfrastructure(Exception): + """The harness could not establish bounded, isolated evidence.""" + + +class Result(NamedTuple): + secure: bool + detail: str + + +Probe = Callable[[Path, Path, tuple[str, ...]], Result] + + +@dataclass(frozen=True) +class Assertion: + identity: str + probe: Probe + message: str + + +class CommandResult(NamedTuple): + returncode: int | None + stdout: bytes + stderr: bytes + timed_out: bool + + +def support(): + if SUPPORT is None: + raise IntegrityInfrastructure(f"install mutation support is unavailable: {SUPPORT_ERROR}") + return SUPPORT + + +def canonical(value: object) -> bytes: + return json.dumps( + value, + allow_nan=False, + ensure_ascii=True, + separators=(",", ":"), + sort_keys=True, + ).encode("ascii") + + +def digest(domain: bytes, value: object) -> str: + import hashlib + + return "sha256:" + hashlib.sha256(domain + canonical(value)).hexdigest() + + +def load_private_module(path: Path, name: str): + helper = support() + previous = sys.dont_write_bytecode + sys.dont_write_bytecode = True + try: + return helper.load_module(path, name) + finally: + sys.dont_write_bytecode = previous + + +def private_runtime(root: Path, names: tuple[str, ...]) -> Path: + runtime = root / "runtime" + support().copy_runtime(runtime, names) + return runtime + + +@contextmanager +def tool_environment() -> Iterator[None]: + keys = ("AGENT_LAB_CUE_TOOL_DIR", "AGENT_LAB_CEDAR_TOOL_DIR") + previous = {key: os.environ.get(key) for key in keys} + os.environ["AGENT_LAB_CUE_TOOL_DIR"] = str(CUE_TOOLS) + os.environ["AGENT_LAB_CEDAR_TOOL_DIR"] = str(CEDAR_TOOLS) + try: + yield + finally: + for key, value in previous.items(): + if value is None: + os.environ.pop(key, None) + else: + os.environ[key] = value + + +def actual_install(store, home: Path, source: Path, *, fault=None): + try: + value = store.install_directory(home, source, fault=fault) + return 0, value, None + except store.StoreReject as error: + return 1, None, error + except store.StoreInfrastructure as error: + return 125, None, error + except BaseException as error: + return None, None, error + + +def direct_source(root: Path, name: str, command: str) -> Path: + return support().write_source(root, name, support().artifact_bytes(command)) + + +def bundled_source(root: Path) -> Path: + data = ( + "package experiment\n\n" + "experiment: {\n" + '\tapiVersion: "agent-lab/v0alpha1"\n' + '\tkind: "Experiment"\n' + '\tmetadata: name: "bundled-integrity"\n' + "\tspec: members: [{\n" + '\t\tname: "worker"\n' + '\t\timage: catalogName: "agent-lab.worker"\n' + '\t\tcommand: ["serve"]\n' + "\t}]\n" + "}\n" + ).encode("ascii") + return support().write_source(root, "bundled-source", data) + + +def runtime_unchanged( + runtime: Path, + names: tuple[str, ...], + expected: tuple[object, ...], + purpose: str, +) -> None: + if support().runtime_fingerprint(runtime, names) != expected: + raise IntegrityInfrastructure(f"{purpose} changed its private runtime") + + +def probe_cleanup_parent_swap(root: Path, runtime: Path, names: tuple[str, ...]) -> Result: + store = load_private_module( + runtime / "scripts" / "experiment_store.py", + f"agent_lab_integrity_cleanup_{os.getpid()}_{id(root)}", + ) + runtime_before = support().runtime_fingerprint(runtime, names) + cleanup = root / "cleanup" + payload = cleanup / "payload" + artifact = payload / "artifact" + artifact.mkdir(mode=0o700, parents=True) + os.chmod(cleanup, 0o700) + os.chmod(payload, 0o700) + inside = artifact / "experiment.cue" + inside.write_bytes(b"inside\n") + os.chmod(inside, 0o400) + outside = root / "outside" + outside.mkdir(mode=0o700) + canary = outside / "experiment.cue" + canary.write_bytes(b"outside-canary\n") + os.chmod(canary, 0o400) + parked = root / "parked-artifact" + outside_before = support().tree_fingerprint(outside) + original_names = store._directory_names + swapped = False + + def swap_parent(path: Path, purpose: str, maximum: int): + nonlocal swapped + found = original_names(path, purpose, maximum) + if path == artifact and not swapped: + artifact.rename(parked) + os.symlink(outside, artifact) + swapped = True + return found + + store._directory_names = swap_parent + outcome: int | None = None + error: BaseException | None = None + try: + store._remove_tree(cleanup, cleanup) + outcome = 0 + except store.StoreInfrastructure as caught: + outcome = 125 + error = caught + except BaseException as caught: + error = caught + finally: + store._directory_names = original_names + outside_after = support().tree_fingerprint(outside) + + mode_cleanup = root / "mode-cleanup" + mode_payload = mode_cleanup / "payload" + mode_artifact = mode_payload / "artifact" + mode_artifact.mkdir(mode=0o700, parents=True) + os.chmod(mode_cleanup, 0o700) + os.chmod(mode_payload, 0o700) + mode_canary = mode_artifact / "experiment.cue" + mode_canary.write_bytes(b"mode-race-canary\n") + os.chmod(mode_canary, 0o400) + payload_identity = mode_payload.stat() + original_open = store.os.open + mode_changed = False + + def change_mode_before_open(path, flags, mode=0o777, *, dir_fd=None): + nonlocal mode_changed + if path == "artifact" and dir_fd is not None and not mode_changed: + parent = os.fstat(dir_fd) + if (parent.st_dev, parent.st_ino) == ( + payload_identity.st_dev, + payload_identity.st_ino, + ): + os.chmod(mode_artifact, 0o777) + mode_changed = True + return original_open(path, flags, mode, dir_fd=dir_fd) + + store.os.open = change_mode_before_open + mode_outcome: int | None = None + mode_error: BaseException | None = None + try: + store._remove_tree(mode_cleanup, mode_cleanup) + mode_outcome = 0 + except store.StoreInfrastructure as caught: + mode_outcome = 125 + mode_error = caught + except BaseException as caught: + mode_error = caught + finally: + store.os.open = original_open + + runtime_unchanged(runtime, names, runtime_before, "cleanup parent-swap probe") + mode_canary_preserved = ( + mode_canary.is_file() + and mode_canary.read_bytes() == b"mode-race-canary\n" + ) + secure = ( + swapped + and outcome == 125 + and outside_before == outside_after + and mode_changed + and mode_outcome == 125 + and mode_canary_preserved + ) + return Result( + secure, + ( + f"swapped={swapped} outcome={outcome} error={error!r} " + f"outside_changed={outside_before != outside_after} mode_changed={mode_changed} " + f"mode_outcome={mode_outcome} mode_error={mode_error!r} " + f"mode_canary_preserved={mode_canary_preserved}" + ), + ) + + +def probe_contract_drift(root: Path, runtime: Path, names: tuple[str, ...]) -> Result: + home = support().initialized_home(runtime, root, "contract-home") + source = direct_source(root, "contract-source", "contract") + store = load_private_module( + runtime / "scripts" / "experiment_store.py", + f"agent_lab_integrity_contract_{os.getpid()}_{id(root)}", + ) + contract = runtime / "contracts" / "experiment" / "v0alpha1" / "schema.cue" + original = contract.read_bytes() + mutated = original.replace(b"strings.MaxRunes(63)", b"strings.MaxRunes(62)", 1) + if mutated == original or len(mutated) != len(original): + raise IntegrityInfrastructure("trusted contract mutation is not exactly applicable") + runtime_before = support().runtime_fingerprint(runtime, names) + store_before = support().tree_fingerprint(home / "experiments") + reached = False + + def mutate_contract(point: str) -> None: + nonlocal reached + if point == "experiment store lock.after_acquire" and not reached: + contract.write_bytes(mutated) + reached = True + + try: + with tool_environment(): + rc, value, error = actual_install(store, home, source, fault=mutate_contract) + finally: + contract.write_bytes(original) + store_after = support().tree_fingerprint(home / "experiments") + runtime_unchanged(runtime, names, runtime_before, "trusted contract drift probe") + final = home / "experiments" / "mutation-store" + secure = ( + reached + and rc == 125 + and value is None + and isinstance(error, store.StoreInfrastructure) + and not final.exists() + and store_before == store_after + ) + return Result( + secure, + f"reached={reached} rc={rc} error={error!r} final={final.exists()} store_changed={store_before != store_after}", + ) + + +def probe_bundled_provenance(root: Path, runtime: Path, names: tuple[str, ...]) -> Result: + catalog = { + "apiVersion": "agent-lab.experiment-images/v0alpha1", + "entries": [{"name": "agent-lab.worker", "subject": support().SUBJECT}], + } + catalog_path = runtime / "catalog" / "experiment-images" / "v0alpha1.json" + catalog_path.write_bytes(canonical(catalog) + b"\n") + expected = digest(BUNDLED_CATALOG_DOMAIN, catalog) + runtime_before = support().runtime_fingerprint(runtime, names) + home = support().initialized_home(runtime, root, "bundled-home") + source = bundled_source(root) + store = load_private_module( + runtime / "scripts" / "experiment_store.py", + f"agent_lab_integrity_bundle_{os.getpid()}_{id(root)}", + ) + with tool_environment(): + rc, value, error = actual_install(store, home, source) + provenance_path = ( + home + / "experiments" + / "bundled-integrity" + / "records" + / "provenance.json" + ) + try: + provenance = json.loads(provenance_path.read_bytes()) if provenance_path.is_file() else None + except (OSError, UnicodeError, json.JSONDecodeError): + provenance = None + runtime_unchanged(runtime, names, runtime_before, "bundled provenance probe") + expected_catalog = {"bundled": {"snapshotDigest": expected}} + secure = ( + rc == 0 + and isinstance(value, dict) + and error is None + and isinstance(provenance, dict) + and provenance.get("catalog") == expected_catalog + ) + return Result( + secure, + f"rc={rc} error={error!r} expected={expected_catalog!r} stored={None if not isinstance(provenance, dict) else provenance.get('catalog')!r}", + ) + + +def descriptor_open(descriptor: int) -> bool: + try: + os.fstat(descriptor) + except OSError as error: + if error.errno == errno.EBADF: + return False + raise + return True + + +def probe_unlock_failure(root: Path, runtime: Path, names: tuple[str, ...]) -> Result: + home = support().initialized_home(runtime, root, "unlock-home") + first_data = support().artifact_bytes("unlock-one") + second_data = support().artifact_bytes("unlock-two") + first_source = support().write_source(root, "unlock-source-one", first_data) + second_source = support().write_source(root, "unlock-source-two", second_data) + fixtures = { + first_data: support().PlanFixture( + support().requested_plan("mutation-store", "unlock-one"), + None, + ), + second_data: support().PlanFixture( + support().requested_plan("mutation-store", "unlock-two"), + None, + ), + } + experiment = support().FixtureExperiment(fixtures) + store = support().fixture_store(runtime, root, experiment) + runtime_before = support().runtime_fingerprint(runtime, names) + first_rc, _, first_error = support().install_result(store, home, first_source) + original_flock = store.fcntl.flock + unlock_descriptor: int | None = None + + def fail_unlock(descriptor: int, operation: int) -> None: + nonlocal unlock_descriptor + if operation == store.fcntl.LOCK_UN: + unlock_descriptor = descriptor + raise OSError("injected LOCK_UN failure") + original_flock(descriptor, operation) + + descriptors_before = set(os.listdir("/proc/self/fd")) + store.fcntl.flock = fail_unlock + try: + rc, value, error = support().install_result(store, home, second_source) + finally: + store.fcntl.flock = original_flock + leaked = unlock_descriptor is not None and descriptor_open(unlock_descriptor) + descriptors_after = set(os.listdir("/proc/self/fd")) + secure = ( + first_rc == 0 + and first_error is None + and rc == 125 + and value is None + and isinstance(error, store.StoreInfrastructure) + and unlock_descriptor is not None + and not leaked + and descriptors_before == descriptors_after + ) + if leaked and unlock_descriptor is not None: + os.close(unlock_descriptor) + descriptors_clean = set(os.listdir("/proc/self/fd")) + if descriptors_clean != descriptors_before: + raise IntegrityInfrastructure("unlock probe could not restore its descriptor set") + runtime_unchanged(runtime, names, runtime_before, "lock release probe") + return Result( + secure, + f"first={first_rc}/{first_error!r} conflict={rc}/{error!r}/{value!r} unlock_fd={unlock_descriptor} leaked={leaked} fd_delta={sorted(descriptors_after - descriptors_before)!r}", + ) + + +def probe_snapshot_race(root: Path, runtime: Path, names: tuple[str, ...]) -> Result: + experiment = load_private_module( + runtime / "scripts" / "experiment.py", + f"agent_lab_integrity_snapshot_{os.getpid()}_{id(root)}", + ) + runtime_before = support().runtime_fingerprint(runtime, names) + original = support().artifact_bytes("snapshot-one") + changed = support().artifact_bytes("snapshot-two") + if len(original) != len(changed) or original == changed: + raise IntegrityInfrastructure("same-size source mutation fixture is invalid") + source = support().write_source(root, "snapshot-race-source", original) + path = source / "experiment.cue" + before = path.stat() + original_read = experiment.os.read + reached = False + + def mutate_after_first_read(descriptor: int, maximum: int) -> bytes: + nonlocal reached + data = original_read(descriptor, maximum) + try: + target = os.readlink(f"/proc/self/fd/{descriptor}") + except OSError: + target = "" + if data and target == str(path) and not reached: + path.write_bytes(changed) + os.utime(path, ns=(before.st_atime_ns, before.st_mtime_ns)) + reached = True + return data + + experiment.os.read = mutate_after_first_read + outcome: int | None = None + error: BaseException | None = None + try: + experiment.read_directory_snapshot(str(source)) + outcome = 0 + except experiment.InfrastructureError as caught: + outcome = 125 + error = caught + except experiment.InvalidManifest as caught: + outcome = 1 + error = caught + except BaseException as caught: + error = caught + finally: + experiment.os.read = original_read + + atomic_source = support().write_source( + root, + "snapshot-replacement-source", + original, + ) + atomic_path = atomic_source / "experiment.cue" + replacement = root / "snapshot-replacement.cue" + replacement.write_bytes(changed) + os.chmod(replacement, 0o666) + original_read_once = experiment.read_manifest_once + replaced = False + + def replace_before_read(target: str, *args, **kwargs) -> bytes: + nonlocal replaced + if target == str(atomic_path) and not replaced: + os.replace(replacement, atomic_path) + replaced = True + return original_read_once(target, *args, **kwargs) + + experiment.read_manifest_once = replace_before_read + atomic_outcome: int | None = None + atomic_error: BaseException | None = None + try: + experiment.read_directory_snapshot(str(atomic_source)) + atomic_outcome = 0 + except experiment.InfrastructureError as caught: + atomic_outcome = 125 + atomic_error = caught + except experiment.InvalidManifest as caught: + atomic_outcome = 1 + atomic_error = caught + except BaseException as caught: + atomic_error = caught + finally: + experiment.read_manifest_once = original_read_once + + runtime_unchanged(runtime, names, runtime_before, "source snapshot race probe") + secure = ( + reached + and outcome == 125 + and isinstance(error, experiment.InfrastructureError) + and replaced + and atomic_outcome == 125 + and isinstance(atomic_error, experiment.InfrastructureError) + ) + return Result( + secure, + ( + f"write_reached={reached} write_outcome={outcome} write_error={error!r} " + f"replaced={replaced} replacement_outcome={atomic_outcome} " + f"replacement_error={atomic_error!r}" + ), + ) + + +def terminate(process: subprocess.Popen[bytes]) -> tuple[bytes, bytes]: + try: + os.killpg(process.pid, signal.SIGTERM) + except ProcessLookupError: + pass + try: + return process.communicate(timeout=0.25) + except subprocess.TimeoutExpired: + try: + os.killpg(process.pid, signal.SIGKILL) + except ProcessLookupError: + pass + return process.communicate(timeout=1) + + +def bounded_public( + runtime: Path, + home: Path, + source: Path, + *, + environment: dict[str, str] | None = None, + timeout: float = PUBLIC_TIMEOUT_SECONDS, +) -> CommandResult: + env = {"PATH": "/usr/bin:/bin", "LANG": "C", "LC_ALL": "C"} + if environment: + env.update(environment) + process = subprocess.Popen( + [ + sys.executable, + "-I", + "-B", + str(runtime / "scripts" / "agent-lab.py"), + "--home", + str(home), + "experiment", + "install", + str(source), + ], + stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + env=env, + start_new_session=True, + ) + try: + stdout, stderr = process.communicate(timeout=timeout) + if support().process_group_exists(process.pid): + terminate(process) + raise IntegrityInfrastructure("public preflight left a descendant process") + return CommandResult(process.returncode, stdout, stderr, False) + except subprocess.TimeoutExpired: + stdout, stderr = terminate(process) + if support().process_group_exists(process.pid): + raise IntegrityInfrastructure("timed-out public preflight left a process group") + return CommandResult(None, stdout, stderr, True) + + +def instrument_preflight(runtime: Path) -> None: + path = runtime / "scripts" / "agent-lab.py" + source = path.read_text(encoding="utf-8") + old = ( + " if not home_authority_metadata_safe(lexical):\n" + ' raise RuntimeError("home authority files are unsafe")\n' + " flags = (\n" + ) + new = ( + " if not home_authority_metadata_safe(lexical):\n" + ' raise RuntimeError("home authority files are unsafe")\n' + ' integrity_ready = os.environ.get("AGENT_LAB_IIN_PREFLIGHT_READY")\n' + ' if integrity_ready is not None:\n' + ' Path(integrity_ready).touch()\n' + ' integrity_release = Path(os.environ["AGENT_LAB_IIN_PREFLIGHT_RELEASE"])\n' + ' integrity_deadline = __import__("time").monotonic() + 2.0\n' + ' while (\n' + ' not integrity_release.exists()\n' + ' and __import__("time").monotonic() < integrity_deadline\n' + ' ):\n' + ' __import__("time").sleep(0.005)\n' + ' if not integrity_release.exists():\n' + ' raise RuntimeError("integrity preflight release is unavailable")\n' + " flags = (\n" + ) + if source.count(old) != 1: + raise IntegrityInfrastructure("public preflight instrumentation is not exactly applicable") + path.write_text(source.replace(old, new, 1), encoding="utf-8") + cache = runtime.parent / "preflight-pycache" + completed = support().run_command( + [ + sys.executable, + "-I", + "-B", + "-X", + f"pycache_prefix={cache}", + "-m", + "py_compile", + str(path), + ] + ) + if completed.returncode != 0 or completed.stderr: + raise IntegrityInfrastructure("instrumented public preflight does not compile") + + +def restore_authority(path: Path, parked: Path) -> None: + try: + if path.is_symlink() or stat.S_ISFIFO(path.lstat().st_mode) or path.is_file(): + path.unlink() + parked.rename(path) + except OSError as error: + raise IntegrityInfrastructure("public preflight authority fixture could not be restored") from error + + +def wait_for_file(path: Path, timeout: float) -> bool: + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + if path.is_file(): + return True + time.sleep(0.005) + return False + + +def race_fifo_public(runtime: Path, home: Path, source: Path, root: Path) -> CommandResult: + ready = root / "race-ready" + release = root / "race-release" + path = home / "config.json" + parked = home / "config.parked" + env = { + "PATH": "/usr/bin:/bin", + "LANG": "C", + "LC_ALL": "C", + "AGENT_LAB_IIN_PREFLIGHT_READY": str(ready), + "AGENT_LAB_IIN_PREFLIGHT_RELEASE": str(release), + } + process = subprocess.Popen( + [ + sys.executable, + "-I", + "-B", + str(runtime / "scripts" / "agent-lab.py"), + "--home", + str(home), + "experiment", + "install", + str(source), + ], + stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + env=env, + start_new_session=True, + ) + if not wait_for_file(ready, 1.0): + terminate(process) + raise IntegrityInfrastructure("public preflight race did not reach its read boundary") + path.rename(parked) + os.mkfifo(path, mode=0o600) + release.touch() + try: + try: + stdout, stderr = process.communicate(timeout=RACE_TIMEOUT_SECONDS) + result = CommandResult(process.returncode, stdout, stderr, False) + except subprocess.TimeoutExpired: + stdout, stderr = terminate(process) + if support().process_group_exists(process.pid): + raise IntegrityInfrastructure("racing public preflight left a process group") + result = CommandResult(None, stdout, stderr, True) + finally: + restore_authority(path, parked) + return result + + +def probe_public_preflight(root: Path, runtime: Path, names: tuple[str, ...]) -> Result: + instrument_preflight(runtime) + runtime_before = support().runtime_fingerprint(runtime, names) + source = direct_source(root, "preflight-source", "preflight") + source_before = support().tree_fingerprint(source) + observations: dict[str, CommandResult] = {} + state_checks: dict[str, bool] = {} + + symlink_home = support().initialized_home(runtime, root, "symlink-home") + symlink_before = support().tree_fingerprint(symlink_home / "experiments") + symlink_path = symlink_home / "config.json" + symlink_parked = symlink_home / "config.parked" + symlink_path.rename(symlink_parked) + os.symlink(symlink_parked, symlink_path) + try: + observations["symlink"] = bounded_public(runtime, symlink_home, source) + finally: + restore_authority(symlink_path, symlink_parked) + state_checks["symlink"] = symlink_before == support().tree_fingerprint( + symlink_home / "experiments" + ) + + fifo_home = support().initialized_home(runtime, root, "fifo-home") + fifo_before = support().tree_fingerprint(fifo_home / "experiments") + fifo_path = fifo_home / "home.json" + fifo_parked = fifo_home / "home.parked" + fifo_path.rename(fifo_parked) + os.mkfifo(fifo_path, mode=0o600) + try: + observations["fifo"] = bounded_public(runtime, fifo_home, source) + finally: + restore_authority(fifo_path, fifo_parked) + state_checks["fifo"] = fifo_before == support().tree_fingerprint( + fifo_home / "experiments" + ) + + bound_home = support().initialized_home(runtime, root, "bound-home") + bound_before = support().tree_fingerprint(bound_home / "experiments") + bound_path = bound_home / "config.json" + bound_raw = bound_path.read_bytes() + bound_path.write_bytes(b"{" + b" " * 1_048_577 + b"}\n") + os.chmod(bound_path, 0o600) + try: + observations["over-bound"] = bounded_public(runtime, bound_home, source) + finally: + bound_path.write_bytes(bound_raw) + os.chmod(bound_path, 0o600) + state_checks["over-bound"] = bound_before == support().tree_fingerprint( + bound_home / "experiments" + ) + + race_home = support().initialized_home(runtime, root, "race-home") + race_before = support().tree_fingerprint(race_home / "experiments") + observations["race"] = race_fifo_public(runtime, race_home, source, root) + state_checks["race"] = race_before == support().tree_fingerprint( + race_home / "experiments" + ) + + store_home = support().initialized_home(runtime, root, "store-race-home") + store_path = store_home / "experiments" + parked_store = store_home / "experiments-parked" + store = load_private_module( + runtime / "scripts" / "experiment_store.py", + f"agent_lab_integrity_store_race_{os.getpid()}_{id(root)}", + ) + store_replaced = False + parked_before: tuple[tuple[object, ...], ...] | None = None + replacement_before: tuple[tuple[object, ...], ...] | None = None + + def replace_store_after_lock(point: str) -> None: + nonlocal store_replaced, parked_before, replacement_before + if point == "experiment store lock.after_acquire" and not store_replaced: + store_path.rename(parked_store) + parked_before = support().tree_fingerprint(parked_store) + store_path.mkdir(mode=0o700) + staging = store_path / ".staging" + staging.mkdir(mode=0o700) + os.chmod(store_path, 0o700) + os.chmod(staging, 0o700) + replacement_before = support().tree_fingerprint(store_path) + store_replaced = True + + with tool_environment(): + store_rc, store_value, store_error = actual_install( + store, + store_home, + source, + fault=replace_store_after_lock, + ) + original_store_preserved = ( + parked_before is not None + and support().tree_fingerprint(parked_store) == parked_before + ) + replacement_store_preserved = ( + replacement_before is not None + and support().tree_fingerprint(store_path) == replacement_before + ) + store_race_secure = ( + store_replaced + and store_rc == 125 + and store_value is None + and isinstance(store_error, store.StoreInfrastructure) + and original_store_preserved + and replacement_store_preserved + ) + + late_home = support().initialized_home(runtime, root, "late-store-race-home") + late_store = late_home / "experiments" + late_parked = late_home / "experiments-parked" + late_replaced = False + late_parked_before: tuple[tuple[object, ...], ...] | None = None + late_replacement_before: tuple[tuple[object, ...], ...] | None = None + + def replace_store_before_publish(point: str) -> None: + nonlocal late_replaced, late_parked_before, late_replacement_before + if point == "experiment envelope.before_noreplace" and not late_replaced: + late_store.rename(late_parked) + late_store.mkdir(mode=0o700) + replacement_staging = late_store / ".staging" + replacement_staging.mkdir(mode=0o700) + os.chmod(late_store, 0o700) + os.chmod(replacement_staging, 0o700) + parked_wrapper = late_parked / ".staging" / "experiment-install" + replacement_wrapper = replacement_staging / "experiment-install" + parked_wrapper.rename(replacement_wrapper) + late_parked_before = support().tree_fingerprint(late_parked) + late_replacement_before = support().tree_fingerprint(late_store) + late_replaced = True + + with tool_environment(): + late_rc, late_value, late_error = actual_install( + store, + late_home, + source, + fault=replace_store_before_publish, + ) + late_parked_preserved = ( + late_parked_before is not None + and support().tree_fingerprint(late_parked) == late_parked_before + ) + late_replacement_preserved = ( + late_replacement_before is not None + and support().tree_fingerprint(late_store) == late_replacement_before + ) + late_store_race_secure = ( + late_replaced + and late_rc == 125 + and late_value is None + and isinstance(late_error, store.StoreInfrastructure) + and late_parked_preserved + and late_replacement_preserved + and not (late_store / "mutation-store").exists() + ) + + source_after = support().tree_fingerprint(source) + runtime_unchanged(runtime, names, runtime_before, "public preflight probe") + clean_results = all( + observation.returncode == 125 + and not observation.timed_out + and not observation.stdout + for observation in observations.values() + ) + secure = ( + clean_results + and all(state_checks.values()) + and store_race_secure + and late_store_race_secure + and source_before == source_after + ) + rendered = { + key: (value.returncode, value.timed_out, value.stderr.decode("utf-8", errors="replace").strip()) + for key, value in observations.items() + } + return Result( + secure, + ( + f"results={rendered!r} state={state_checks!r} " + f"store_race={store_replaced}/{store_rc}/{store_error!r}/" + f"{original_store_preserved}/{replacement_store_preserved} " + f"late_store_race={late_replaced}/{late_rc}/{late_error!r}/" + f"{late_parked_preserved}/{late_replacement_preserved} " + f"source_changed={source_before != source_after}" + ), + ) + + +def probe_plan_identity(root: Path, runtime: Path, names: tuple[str, ...]) -> Result: + home = support().initialized_home(runtime, root, "plan-home") + source = direct_source(root, "plan-source", "plan") + store = load_private_module( + runtime / "scripts" / "experiment_store.py", + f"agent_lab_integrity_plan_{os.getpid()}_{id(root)}", + ) + runtime_before = support().runtime_fingerprint(runtime, names) + with tool_environment(): + rc, value, error = actual_install(store, home, source) + records = home / "experiments" / "mutation-store" / "records" + try: + plan = json.loads((records / "plan.json").read_bytes()) + decision = json.loads((records / "decision.json").read_bytes()) + receipt = json.loads((records / "install.json").read_bytes()) + except (OSError, UnicodeError, json.JSONDecodeError): + plan = decision = receipt = None + runtime_unchanged(runtime, names, runtime_before, "plan identity probe") + expected = digest(PLAN_DOMAIN, plan) if isinstance(plan, dict) else None + decision_digest = None + receipt_digest = None + identity_digest = None + if isinstance(decision, dict): + binding = decision.get("binding") + if isinstance(binding, dict): + decision_digest = binding.get("planDigest") + if isinstance(receipt, dict): + record_map = receipt.get("records") + if isinstance(record_map, dict): + plan_record = record_map.get("records/plan.json") + if isinstance(plan_record, dict): + receipt_digest = plan_record.get("digest") + identity = receipt.get("identity") + if isinstance(identity, dict): + identity_digest = identity.get("planDigest") + secure = ( + rc == 0 + and isinstance(value, dict) + and error is None + and expected is not None + and decision_digest == expected + and receipt_digest == expected + and identity_digest == expected + ) + return Result( + secure, + f"rc={rc} error={error!r} expected={expected!r} decision={decision_digest!r} receipt={receipt_digest!r} identity={identity_digest!r}", + ) + + +ASSERTIONS: tuple[Assertion, ...] = ( + Assertion( + "IIN-CLEAN-001", + probe_cleanup_parent_swap, + "cleanup preserves data across parent-swap and directory-metadata races", + ), + Assertion( + "IIN-CONTRACT-001", + probe_contract_drift, + "trusted contract drift at store-lock acquisition fails before store effect", + ), + Assertion( + "IIN-BUNDLE-001", + probe_bundled_provenance, + "bundled catalog provenance binds the independently framed snapshot identity", + ), + Assertion( + "IIN-LOCK-001", + probe_unlock_failure, + "unlock failure dominates conflict and closes the store-lock descriptor", + ), + Assertion( + "IIN-SNAPSHOT-001", + probe_snapshot_race, + "in-place and atomic source replacement races are snapshot uncertainty", + ), + Assertion( + "IIN-PREFLIGHT-001", + probe_public_preflight, + "public preflight rejects unsafe and racing authority files and store roots", + ), + Assertion( + "IIN-PLAN-001", + probe_plan_identity, + "decision and receipt bind the independently domain-separated plan identity", + ), +) + + +def main() -> int: + try: + helper = support() + if not CUE_TOOLS.is_dir() or not CEDAR_TOOLS.is_dir(): + raise IntegrityInfrastructure("pinned CUE or Cedar fixtures are unavailable") + names = helper.manifest_paths() + shared_before = helper.runtime_fingerprint(REPO_ROOT, names) + expected = ( + "IIN-CLEAN-001", + "IIN-CONTRACT-001", + "IIN-BUNDLE-001", + "IIN-LOCK-001", + "IIN-SNAPSHOT-001", + "IIN-PREFLIGHT-001", + "IIN-PLAN-001", + ) + if tuple(assertion.identity for assertion in ASSERTIONS) != expected: + raise IntegrityInfrastructure("install integrity assertion identity drift") + failures = 0 + observed: list[str] = [] + previous_bytecode = sys.dont_write_bytecode + sys.dont_write_bytecode = True + try: + for assertion in ASSERTIONS: + try: + temporary = Path( + tempfile.mkdtemp( + prefix=f"agent-lab-{assertion.identity.lower()}-", + dir="/tmp", + ) + ) + except OSError as error: + raise IntegrityInfrastructure( + f"{assertion.identity} private root is unavailable" + ) from error + try: + runtime = private_runtime(temporary, names) + result = assertion.probe(temporary, runtime, names) + observed.append(assertion.identity) + if result.secure: + print(f"PASS {assertion.identity} {assertion.message}") + else: + failures += 1 + print( + f"FAIL {assertion.identity} {assertion.message} ({result.detail})" + ) + finally: + helper.remove_private_root(temporary) + if helper.runtime_fingerprint(REPO_ROOT, names) != shared_before: + raise IntegrityInfrastructure( + f"{assertion.identity} changed the shared runtime fingerprint" + ) + finally: + sys.dont_write_bytecode = previous_bytecode + if tuple(observed) != expected: + raise IntegrityInfrastructure("install integrity execution identity drift") + print(f"SUMMARY assertions=7 expected=7 failures={failures} infra=0") + return 0 if failures == 0 else 1 + except (IntegrityInfrastructure, SupportInfrastructure) as error: + print(f"INFRA install integrity evidence: {error}", file=sys.stderr) + return 125 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/experiment/install-mutation-cases.py b/tests/experiment/install-mutation-cases.py new file mode 100755 index 0000000..52ada92 --- /dev/null +++ b/tests/experiment/install-mutation-cases.py @@ -0,0 +1,1324 @@ +#!/usr/bin/env python3 +"""Private-copy sensitivity mutations for the local Experiment store.""" + +from __future__ import annotations + +from contextlib import contextmanager +from dataclasses import dataclass +import hashlib +from importlib.util import module_from_spec, spec_from_file_location +import json +import os +from pathlib import Path +import shutil +import signal +import stat +import subprocess +import sys +import tempfile +import time +from typing import Callable, Iterator, NamedTuple + + +REPO_ROOT = Path(__file__).resolve().parents[2] +RUNTIME_MANIFEST = REPO_ROOT / "packaging" / "agent-lab-local.manifest" +COMMAND_TIMEOUT_SECONDS = 5 +SOURCE_DOMAIN = b"agent-lab.experiment-tree.v1\0" +PLAN_DOMAIN = b"agent-lab.experiment-plan.v1\0" +SUBJECT = "registry.example/team/worker@sha256:" + "a" * 64 +AUTHORIZATION_DIGEST = "sha256:" + "c" * 64 +CONTRACT_DIGEST = "sha256:" + "d" * 64 +MUTATION_KEYS = ( + "AGENT_LAB_MUTATION_DECISION", + "AGENT_LAB_MUTATION_MARK", + "AGENT_LAB_MUTATION_NAME", + "AGENT_LAB_MUTATION_SOURCE", +) + + +class InfrastructureError(Exception): + """The mutation or its isolated evidence could not be proved.""" + + +class ProbeResult(NamedTuple): + secure: bool + detail: str + + +Probe = Callable[[Path, Path, Path | None], ProbeResult] + + +@dataclass(frozen=True) +class Mutation: + assertion: str + path: str + old: str + new: str + probe: Probe + message: str + + +class Snapshot(NamedTuple): + data: bytes + digest: str + + +class Resolution(NamedTuple): + plan: dict[str, object] + bundled_catalog: dict[str, object] | None + local_catalog: dict[str, object] | None + + +class PlanFixture(NamedTuple): + plan: dict[str, object] + local_catalog: dict[str, object] | None + + +class FixtureInvalidManifest(Exception): + """The synthetic manifest is outside the declared fixture set.""" + + +class FixtureInfrastructure(Exception): + """The synthetic planning fixture could not establish a result.""" + + +def canonical(value: object) -> bytes: + return json.dumps( + value, + allow_nan=False, + ensure_ascii=True, + separators=(",", ":"), + sort_keys=True, + ).encode("ascii") + + +def digest(data: bytes) -> str: + return "sha256:" + hashlib.sha256(data).hexdigest() + + +def source_digest(data: bytes) -> str: + name = b"experiment.cue" + value = hashlib.sha256(SOURCE_DOMAIN) + value.update(len(name).to_bytes(4, "big")) + value.update(name) + value.update(len(data).to_bytes(8, "big")) + value.update(data) + return "sha256:" + value.hexdigest() + + +def artifact_bytes(command: str) -> bytes: + return ( + "package experiment\n\n" + "experiment: {\n" + '\tapiVersion: "agent-lab/v0alpha1"\n' + '\tkind: "Experiment"\n' + '\tmetadata: name: "mutation-store"\n' + "\tspec: members: [{\n" + '\t\tname: "worker"\n' + f'\t\timage: digestRef: "{SUBJECT}"\n' + f'\t\tcommand: ["{command}"]\n' + "\t}]\n" + "}\n" + ).encode("ascii") + + +def requested_plan( + name: str, + command: str, + *, + local_record: dict[str, object] | None = None, +) -> dict[str, object]: + if local_record is None: + requested = {"digestRef": SUBJECT} + resolved = {"origin": "direct", "subject": SUBJECT} + else: + requested = {"catalogName": "vendor.worker"} + resolved = { + "entryDigest": local_record["entryDigest"], + "generation": local_record["generation"], + "origin": "local", + "subject": local_record["subject"], + } + return { + "apiVersion": "agent-lab.request/v0alpha1", + "contract": { + "digest": CONTRACT_DIGEST, + "name": "agent-lab.experiment", + "version": "v0alpha1", + }, + "kind": "RequestedExperimentPlan", + "metadata": {"requestedName": name}, + "spec": { + "members": [ + { + "command": [command], + "name": "worker", + "requestedSelector": requested, + "resolvedImage": resolved, + "resourceClass": "small", + } + ] + }, + } + + +def decision_for( + plan: dict[str, object], + snapshot_digest: str, + verdict: str, +) -> dict[str, object]: + plan_digest = digest(PLAN_DOMAIN + canonical(plan)) + requested_name = plan["metadata"]["requestedName"] # type: ignore[index] + return { + "action": "experiment.install", + "apiVersion": "agent-lab.authorization/v0alpha1", + "binding": { + "authorizationDigest": AUTHORIZATION_DIGEST, + "contractDigest": CONTRACT_DIGEST, + "planDigest": plan_digest, + "sourceDigest": snapshot_digest, + }, + "kind": "ExperimentAuthorizationDecision", + "principal": { + "assurance": "none", + "authenticated": False, + "id": "local-cli", + "source": "fixed-local-cli", + "type": "AgentLab::Principal", + }, + "resource": { + "id": plan_digest, + "requestedName": requested_name, + "type": "AgentLab::RequestedExperimentPlan", + }, + "verdict": verdict, + } + + +class FixtureExperiment: + InvalidManifest = FixtureInvalidManifest + InfrastructureError = FixtureInfrastructure + + def __init__( + self, + fixtures: dict[bytes, PlanFixture], + *, + verdict: str = "permit", + after_authorize: Callable[[], None] | None = None, + ) -> None: + self.fixtures = fixtures + self.verdict = verdict + self.after_authorize = after_authorize + self.authorize_calls = 0 + + def read_directory_snapshot(self, source: str) -> Snapshot: + try: + data = (Path(source) / "experiment.cue").read_bytes() + except OSError as error: + raise FixtureInfrastructure("fixture source cannot be read") from error + if data not in self.fixtures: + raise FixtureInvalidManifest("fixture source is unknown") + return Snapshot(data, source_digest(data)) + + def authored_manifest(self, snapshot: Snapshot) -> bytes: + return snapshot.data + + def cue_plan_with_evidence(self, manifest: object) -> Resolution: + if not isinstance(manifest, bytes) or manifest not in self.fixtures: + raise FixtureInvalidManifest("fixture manifest is unknown") + fixture = self.fixtures[manifest] + return Resolution(fixture.plan, None, fixture.local_catalog) + + def authorize_plan( + self, + plan: dict[str, object], + snapshot_digest: str, + ) -> tuple[dict[str, object], int]: + self.authorize_calls += 1 + decision = decision_for(plan, snapshot_digest, self.verdict) + if self.after_authorize is not None: + self.after_authorize() + return decision, 0 if self.verdict == "permit" else 1 + + def verify_trusted_inputs( + self, + plan: dict[str, object], + decision: dict[str, object], + ) -> None: + if not isinstance(plan, dict) or not isinstance(decision, dict): + raise FixtureInfrastructure("fixture trusted inputs are malformed") + + +def sha256_bytes(data: bytes) -> str: + return hashlib.sha256(data).hexdigest() + + +def manifest_paths() -> tuple[str, ...]: + try: + raw = RUNTIME_MANIFEST.read_bytes() + text = raw.decode("utf-8") + except (OSError, UnicodeError) as error: + raise InfrastructureError("runtime manifest cannot be read exactly") from error + if not text.endswith("\n"): + raise InfrastructureError("runtime manifest lacks its final newline") + names = tuple(line for line in text.splitlines() if line) + if not names or len(names) != len(set(names)) or names != tuple(sorted(names)): + raise InfrastructureError("runtime manifest is empty, duplicated, or unordered") + for name in names: + path = Path(name) + if path.is_absolute() or ".." in path.parts or str(path) != name: + raise InfrastructureError(f"runtime manifest path is unsafe: {name}") + required = { + "scripts/agent-lab.py", + "scripts/experiment.py", + "scripts/experiment_store.py", + "scripts/image_catalog.py", + "scripts/image_reference.py", + } + if not required.issubset(names): + raise InfrastructureError("runtime manifest omits an Experiment store runtime path") + return names + + +def file_identity(path: Path) -> tuple[str, int, int, str]: + try: + metadata = path.lstat() + data = path.read_bytes() + except OSError as error: + raise InfrastructureError(f"runtime path cannot be fingerprinted: {path}") from error + if not stat.S_ISREG(metadata.st_mode) or metadata.st_nlink != 1: + raise InfrastructureError(f"runtime path is not a single regular file: {path}") + return ("file", stat.S_IMODE(metadata.st_mode), len(data), sha256_bytes(data)) + + +def runtime_fingerprint( + root: Path, + names: tuple[str, ...], +) -> tuple[tuple[str, tuple[str, int, int, str]], ...]: + paths = ("packaging/agent-lab-local.manifest", *names) + return tuple((name, file_identity(root / name)) for name in paths) + + +def tree_fingerprint(root: Path) -> tuple[tuple[object, ...], ...]: + if not root.exists() and not root.is_symlink(): + return () + values: list[tuple[object, ...]] = [] + pending = [root] + while pending: + path = pending.pop() + try: + metadata = path.lstat() + except OSError as error: + raise InfrastructureError(f"probe state cannot be fingerprinted: {path}") from error + relative = "." if path == root else path.relative_to(root).as_posix() + mode = stat.S_IMODE(metadata.st_mode) + if stat.S_ISDIR(metadata.st_mode): + kind = "directory" + payload = "" + try: + pending.extend(sorted(path.iterdir(), reverse=True)) + except OSError as error: + raise InfrastructureError(f"probe directory cannot be listed: {path}") from error + elif stat.S_ISREG(metadata.st_mode): + kind = "file" + try: + payload = sha256_bytes(path.read_bytes()) + except OSError as error: + raise InfrastructureError(f"probe file cannot be read: {path}") from error + elif stat.S_ISLNK(metadata.st_mode): + kind = "symlink" + try: + payload = os.readlink(path) + except OSError as error: + raise InfrastructureError(f"probe symlink cannot be read: {path}") from error + else: + kind = "other" + payload = "" + values.append( + ( + relative, + kind, + mode, + metadata.st_nlink, + metadata.st_size, + metadata.st_dev, + metadata.st_ino, + metadata.st_mtime_ns, + metadata.st_ctime_ns, + payload, + ) + ) + return tuple(sorted(values)) + + +def copy_runtime(destination: Path, names: tuple[str, ...]) -> None: + for name in ("packaging/agent-lab-local.manifest", *names): + source = REPO_ROOT / name + target = destination / name + identity = file_identity(source) + try: + target.parent.mkdir(mode=0o700, parents=True, exist_ok=True) + shutil.copyfile(source, target) + os.chmod(target, identity[1]) + except OSError as error: + raise InfrastructureError(f"runtime path cannot be copied privately: {name}") from error + if file_identity(target) != identity: + raise InfrastructureError(f"private runtime copy differs: {name}") + + +def command_environment() -> dict[str, str]: + return {"PATH": "/usr/bin:/bin", "LANG": "C", "LC_ALL": "C"} + + +def process_group_exists(group: int) -> bool: + try: + os.killpg(group, 0) + except ProcessLookupError: + return False + except PermissionError: + return True + return True + + +def terminate_process_group(process: subprocess.Popen[bytes]) -> bool: + group = process.pid + if process_group_exists(group): + try: + os.killpg(group, signal.SIGTERM) + except ProcessLookupError: + pass + deadline = time.monotonic() + 1.0 + while process_group_exists(group) and time.monotonic() < deadline: + time.sleep(0.01) + if process_group_exists(group): + try: + os.killpg(group, signal.SIGKILL) + except ProcessLookupError: + pass + try: + process.communicate(timeout=1) + except subprocess.TimeoutExpired: + try: + os.killpg(group, signal.SIGKILL) + except ProcessLookupError: + pass + process.communicate() + return not process_group_exists(group) + + +def run_command(arguments: list[str]) -> subprocess.CompletedProcess[bytes]: + process: subprocess.Popen[bytes] | None = None + try: + process = subprocess.Popen( + arguments, + stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + env=command_environment(), + start_new_session=True, + ) + stdout, stderr = process.communicate(timeout=COMMAND_TIMEOUT_SECONDS) + if process_group_exists(process.pid): + if not terminate_process_group(process): + raise InfrastructureError("bounded probe left an uncontained process group") + raise InfrastructureError("bounded probe left a descendant process") + return subprocess.CompletedProcess(arguments, process.returncode, stdout, stderr) + except subprocess.TimeoutExpired as error: + assert process is not None + if not terminate_process_group(process): + raise InfrastructureError("timed-out probe left an uncontained process group") from error + raise InfrastructureError("bounded probe command timed out") from error + except (OSError, subprocess.SubprocessError) as error: + if process is not None and process.poll() is None: + terminate_process_group(process) + raise InfrastructureError("bounded probe command could not complete") from error + + +def initialized_home(runtime: Path, probe_root: Path, name: str) -> Path: + home = probe_root / name + completed = run_command( + [ + sys.executable, + "-I", + "-B", + str(runtime / "scripts" / "agent-lab.py"), + "--home", + str(home), + "init", + ] + ) + if completed.returncode != 0 or completed.stdout != b"changed:true\n" or completed.stderr: + raise InfrastructureError( + "private runtime home initialization failed: " + + completed.stderr.decode("utf-8", errors="replace") + ) + return home + + +def write_source(root: Path, name: str, data: bytes) -> Path: + source = root / name + try: + source.mkdir(mode=0o700) + path = source / "experiment.cue" + path.write_bytes(data) + os.chmod(path, 0o600) + except OSError as error: + raise InfrastructureError("private source fixture could not be written") from error + return source + + +def load_module(path: Path, name: str): + spec = spec_from_file_location(name, path) + if spec is None or spec.loader is None: + raise InfrastructureError(f"private module cannot be loaded: {path.name}") + module = module_from_spec(spec) + sys.modules[name] = module + try: + spec.loader.exec_module(module) + except (ImportError, OSError, SyntaxError) as error: + raise InfrastructureError(f"private module import failed: {path.name}") from error + return module + + +def load_store(runtime: Path, probe_root: Path): + name = f"agent_lab_store_mutation_{os.getpid()}_{id(probe_root)}" + return load_module(runtime / "scripts" / "experiment_store.py", name) + + +def fixture_store(runtime: Path, probe_root: Path, experiment: FixtureExperiment): + store = load_store(runtime, probe_root) + store._experiment_module = lambda: experiment + return store + + +def install_result(store, home: Path, source: Path, *, fault=None): + try: + value = store.install_directory(home, source, fault=fault) + return 0, value, None + except store.StoreReject as error: + return 1, None, error + except store.StoreInfrastructure as error: + return 125, None, error + except BaseException as error: + return None, None, error + + +def inspect_result(store, home: Path, name: str): + try: + value = store.inspect_install(home, name) + return 0, value, None + except store.StoreReject as error: + return 1, None, error + except store.StoreInfrastructure as error: + return 125, None, error + except BaseException as error: + return None, None, error + + +@contextmanager +def mutation_environment( + marker: Path | None, + extra: dict[str, str] | None = None, +) -> Iterator[None]: + previous = {key: os.environ.get(key) for key in MUTATION_KEYS} + try: + for key in MUTATION_KEYS: + os.environ.pop(key, None) + if marker is not None: + os.environ["AGENT_LAB_MUTATION_MARK"] = str(marker) + if extra: + os.environ.update(extra) + yield + finally: + for key, value in previous.items(): + if value is None: + os.environ.pop(key, None) + else: + os.environ[key] = value + + +def direct_fixture(data: bytes, command: str = "serve") -> dict[bytes, PlanFixture]: + return { + data: PlanFixture( + requested_plan("mutation-store", command), + None, + ) + } + + +def probe_fresh_authorization(runtime: Path, probe_root: Path, marker: Path | None) -> ProbeResult: + home = initialized_home(runtime, probe_root, "authorization-home") + data = artifact_bytes("authorize") + source = write_source(probe_root, "authorization-source", data) + fixture = direct_fixture(data, "authorize") + experiment = FixtureExperiment(fixture, verdict="deny") + store = fixture_store(runtime, probe_root, experiment) + saved = probe_root / "saved-decision.json" + saved.write_bytes(canonical(decision_for(fixture[data].plan, source_digest(data), "permit")) + b"\n") + before = tree_fingerprint(home / "experiments") + with mutation_environment( + marker, + {"AGENT_LAB_MUTATION_DECISION": str(saved)}, + ): + rc, value, error = install_result(store, home, source) + after = tree_fingerprint(home / "experiments") + secure = rc == 1 and value is None and before == after and experiment.authorize_calls == 1 + return ProbeResult( + secure, + f"rc={rc} value={value!r} error={error!r} changed={before != after} calls={experiment.authorize_calls}", + ) + + +def probe_held_snapshot(runtime: Path, probe_root: Path, marker: Path | None) -> ProbeResult: + home = initialized_home(runtime, probe_root, "snapshot-home") + original = artifact_bytes("snapshot-original") + changed = artifact_bytes("snapshot-changed") + source = write_source(probe_root, "snapshot-source", original) + + def mutate_source() -> None: + (source / "experiment.cue").write_bytes(changed) + + experiment = FixtureExperiment( + direct_fixture(original, "snapshot-original"), + after_authorize=mutate_source, + ) + store = fixture_store(runtime, probe_root, experiment) + with mutation_environment( + marker, + {"AGENT_LAB_MUTATION_SOURCE": str(source)}, + ): + rc, value, error = install_result(store, home, source) + artifact = home / "experiments" / "mutation-store" / "artifact" / "experiment.cue" + stored = artifact.read_bytes() if artifact.is_file() else None + secure = ( + rc == 0 + and isinstance(value, dict) + and error is None + and (source / "experiment.cue").read_bytes() == changed + and stored == original + ) + return ProbeResult(secure, f"rc={rc} error={error!r} stored_original={stored == original}") + + +def probe_noreplace_race(runtime: Path, probe_root: Path, marker: Path | None) -> ProbeResult: + home = initialized_home(runtime, probe_root, "atomic-home") + data = artifact_bytes("atomic") + source = write_source(probe_root, "atomic-source", data) + store = fixture_store(runtime, probe_root, FixtureExperiment(direct_fixture(data, "atomic"))) + target = home / "experiments" / "mutation-store" + raced = False + + def create_target(point: str) -> None: + nonlocal raced + if point == "experiment envelope.before_noreplace" and not raced: + raced = True + target.mkdir(mode=0o700) + + with mutation_environment(marker): + rc, value, error = install_result(store, home, source, fault=create_target) + empty = target.is_dir() and not tuple(target.iterdir()) + secure = raced and rc == 125 and value is None and empty + return ProbeResult(secure, f"raced={raced} rc={rc} error={error!r} empty={empty}") + + +def probe_idempotent_retry(runtime: Path, probe_root: Path, marker: Path | None) -> ProbeResult: + home = initialized_home(runtime, probe_root, "retry-home") + data = artifact_bytes("retry") + source = write_source(probe_root, "retry-source", data) + experiment = FixtureExperiment(direct_fixture(data, "retry")) + store = fixture_store(runtime, probe_root, experiment) + with mutation_environment(marker): + first_rc, first, first_error = install_result(store, home, source) + before = tree_fingerprint(home / "experiments" / "mutation-store") + second_rc, second, second_error = install_result(store, home, source) + after = tree_fingerprint(home / "experiments" / "mutation-store") + secure = ( + first_rc == 0 + and isinstance(first, dict) + and first.get("changed") is True + and first_error is None + and second_rc == 0 + and isinstance(second, dict) + and second.get("changed") is False + and second_error is None + and experiment.authorize_calls == 2 + and before == after + ) + return ProbeResult( + secure, + f"first={first_rc}/{first_error!r} second={second_rc}/{second_error!r}/{second!r} calls={experiment.authorize_calls} changed={before != after}", + ) + + +def probe_publication_durability(runtime: Path, probe_root: Path, marker: Path | None) -> ProbeResult: + home = initialized_home(runtime, probe_root, "durability-home") + data = artifact_bytes("durability") + source = write_source(probe_root, "durability-source", data) + store = fixture_store(runtime, probe_root, FixtureExperiment(direct_fixture(data, "durability"))) + events: list[tuple[str, Path, str]] = [] + original_rename = store._rename_noreplace + original_fsync = store._fsync_directory + target = home / "experiments" / "mutation-store" + + def observed_rename(source_path: Path, target_path: Path) -> None: + if target_path == target: + events.append(("publish", target_path, "")) + original_rename(source_path, target_path) + + def observed_fsync(path: Path, purpose: str, *, modes=(0o700,)) -> None: + events.append(("fsync", path, purpose)) + original_fsync(path, purpose, modes=modes) + + store._rename_noreplace = observed_rename + store._fsync_directory = observed_fsync + try: + with mutation_environment(marker): + rc, value, error = install_result(store, home, source) + finally: + store._fsync_directory = original_fsync + store._rename_noreplace = original_rename + publication = [index for index, event in enumerate(events) if event[0] == "publish"] + wrapper = home / "experiments" / ".staging" / "experiment-install" + payload = wrapper / "payload" + required_events = ( + ("fsync", payload / "artifact", "Experiment committed artifact"), + ("fsync", payload / "records", "Experiment committed records"), + ("fsync", payload, "Experiment staged envelope root"), + ("fsync", wrapper, "Experiment committed wrapper"), + ("fsync", wrapper.parent, "Experiment committed staging"), + ) + durable = { + (str(path), purpose): [ + index + for index, event in enumerate(events) + if event == (kind, path, purpose) + ] + for kind, path, purpose in required_events + } + durable_order = [indices[0] for indices in durable.values() if len(indices) == 1] + secure = ( + rc == 0 + and isinstance(value, dict) + and error is None + and len(publication) == 1 + and all(len(indices) == 1 for indices in durable.values()) + and len(durable_order) == len(required_events) + and durable_order == sorted(durable_order) + and all(index < publication[0] for index in durable_order) + ) + return ProbeResult(secure, f"rc={rc} error={error!r} publish={publication} durable={durable}") + + +def probe_artifact_separation(runtime: Path, probe_root: Path, marker: Path | None) -> ProbeResult: + home = initialized_home(runtime, probe_root, "layout-home") + data = artifact_bytes("layout") + source = write_source(probe_root, "layout-source", data) + store = fixture_store(runtime, probe_root, FixtureExperiment(direct_fixture(data, "layout"))) + with mutation_environment(marker): + rc, value, error = install_result(store, home, source) + artifact = home / "experiments" / "mutation-store" / "artifact" + names = tuple(sorted(path.name for path in artifact.iterdir())) if artifact.is_dir() else () + stored = (artifact / "experiment.cue").read_bytes() if names == ("experiment.cue",) else None + secure = rc == 0 and isinstance(value, dict) and error is None and names == ("experiment.cue",) and stored == data + return ProbeResult(secure, f"rc={rc} error={error!r} names={names!r} exact={stored == data}") + + +def probe_full_identity_key(runtime: Path, probe_root: Path, marker: Path | None) -> ProbeResult: + home = initialized_home(runtime, probe_root, "identity-home") + first_data = artifact_bytes("identity-one") + second_data = artifact_bytes("identity-two") + first_source = write_source(probe_root, "identity-source-one", first_data) + second_source = write_source(probe_root, "identity-source-two", second_data) + fixtures = { + first_data: PlanFixture(requested_plan("mutation-store", "identity-one"), None), + second_data: PlanFixture(requested_plan("mutation-store", "identity-two"), None), + } + store = fixture_store(runtime, probe_root, FixtureExperiment(fixtures)) + with mutation_environment(marker, {"AGENT_LAB_MUTATION_NAME": "mutation-store"}): + first_rc, first, first_error = install_result(store, home, first_source) + before = tree_fingerprint(home / "experiments" / "mutation-store") + second_rc, second, second_error = install_result(store, home, second_source) + after = tree_fingerprint(home / "experiments" / "mutation-store") + secure = ( + first_rc == 0 + and isinstance(first, dict) + and first.get("changed") is True + and first_error is None + and second_rc == 1 + and second is None + and second_error is not None + and before == after + ) + return ProbeResult( + secure, + f"first={first_rc}/{first_error!r} second={second_rc}/{second_error!r}/{second!r} changed={before != after}", + ) + + +def probe_final_revalidation(runtime: Path, probe_root: Path, marker: Path | None) -> ProbeResult: + home = initialized_home(runtime, probe_root, "verification-home") + data = artifact_bytes("verification") + source = write_source(probe_root, "verification-source", data) + store = fixture_store(runtime, probe_root, FixtureExperiment(direct_fixture(data, "verification"))) + corrupted = False + + def corrupt_receipt(point: str) -> None: + nonlocal corrupted + if point != "experiment envelope.after_noreplace" or corrupted: + return + corrupted = True + records = home / "experiments" / "mutation-store" / "records" + receipt = records / "install.json" + os.chmod(records, 0o700, follow_symlinks=False) + os.chmod(receipt, 0o600, follow_symlinks=False) + receipt.write_bytes(receipt.read_bytes() + b" ") + os.chmod(receipt, 0o400, follow_symlinks=False) + os.chmod(records, 0o500, follow_symlinks=False) + + with mutation_environment(marker): + rc, value, error = install_result(store, home, source, fault=corrupt_receipt) + inspect_rc, _, inspect_error = inspect_result(store, home, "mutation-store") + secure = corrupted and rc == 125 and value is None and inspect_rc == 125 + return ProbeResult( + secure, + f"corrupted={corrupted} install={rc}/{error!r} inspect={inspect_rc}/{inspect_error!r}", + ) + + +def lock_is_blocked(path: Path) -> bool: + program = ( + "import fcntl, os, sys\n" + "fd=os.open(sys.argv[1], os.O_RDWR|getattr(os,'O_CLOEXEC',0))\n" + "try:\n" + " fcntl.flock(fd, fcntl.LOCK_EX|fcntl.LOCK_NB)\n" + "except BlockingIOError:\n" + " raise SystemExit(3)\n" + "raise SystemExit(0)\n" + ) + completed = run_command([sys.executable, "-I", "-B", "-c", program, str(path)]) + if completed.returncode not in (0, 3) or completed.stdout or completed.stderr: + raise InfrastructureError(f"catalog lock probe returned {completed.returncode}") + return completed.returncode == 3 + + +def local_fixture(runtime: Path, probe_root: Path, home: Path) -> PlanFixture: + catalog = load_module( + runtime / "scripts" / "image_catalog.py", + f"agent_lab_catalog_store_mutation_{os.getpid()}_{id(probe_root)}", + ) + try: + added = catalog.add_image(home, "vendor.worker", SUBJECT) + resolved = catalog.resolve_local_images(home, ("vendor.worker",)) + record = resolved["records"]["vendor.worker"] + evidence = resolved["catalog"] + except BaseException as error: + raise InfrastructureError(f"local catalog fixture failed: {error}") from error + if ( + not isinstance(added, dict) + or not isinstance(record, dict) + or not isinstance(evidence, dict) + ): + raise InfrastructureError("local catalog fixture is malformed") + return PlanFixture(requested_plan("mutation-store", "local", local_record=record), evidence) + + +def probe_catalog_lock_lifetime(runtime: Path, probe_root: Path, marker: Path | None) -> ProbeResult: + home = initialized_home(runtime, probe_root, "liveness-home") + data = artifact_bytes("local") + source = write_source(probe_root, "liveness-source", data) + fixture = local_fixture(runtime, probe_root, home) + store = fixture_store(runtime, probe_root, FixtureExperiment({data: fixture})) + observed: bool | None = None + + def inspect_lock(point: str) -> None: + nonlocal observed + if point == "experiment envelope.before_noreplace": + observed = lock_is_blocked(home / "state" / "locks" / "image-catalog.lock") + + with mutation_environment(marker): + rc, value, error = install_result(store, home, source, fault=inspect_lock) + secure = rc == 0 and isinstance(value, dict) and error is None and observed is True + return ProbeResult(secure, f"rc={rc} error={error!r} catalog_lock_blocked={observed}") + + +def probe_cleanup_uncertainty(runtime: Path, probe_root: Path, marker: Path | None) -> ProbeResult: + home = initialized_home(runtime, probe_root, "uncertainty-home") + data = artifact_bytes("uncertainty") + source = write_source(probe_root, "uncertainty-source", data) + store = fixture_store(runtime, probe_root, FixtureExperiment(direct_fixture(data, "uncertainty"))) + original_cleanup = store._cleanup_operation + + def uncertain_cleanup(authority, wrapper) -> None: + raise store.StoreInfrastructure("injected cleanup uncertainty") + + store._cleanup_operation = uncertain_cleanup + try: + with mutation_environment(marker): + rc, value, error = install_result(store, home, source) + inspect_rc, inspected, inspect_error = inspect_result(store, home, "mutation-store") + finally: + store._cleanup_operation = original_cleanup + staging = tuple((home / "experiments" / ".staging").iterdir()) + secure = ( + rc == 125 + and value is None + and inspect_rc == 0 + and isinstance(inspected, dict) + and inspect_error is None + and bool(staging) + ) + return ProbeResult( + secure, + f"install={rc}/{error!r}/{value!r} inspect={inspect_rc}/{inspect_error!r} staging={tuple(path.name for path in staging)!r}", + ) + + +def probe_unknown_staging(runtime: Path, probe_root: Path, marker: Path | None) -> ProbeResult: + home = initialized_home(runtime, probe_root, "staging-home") + store = load_store(runtime, probe_root) + foreign = home / "experiments" / ".staging" / "foreign-wrapper" + foreign.mkdir(mode=0o700) + sentinel = foreign / "sentinel" + sentinel.write_bytes(b"foreign\n") + os.chmod(sentinel, 0o600) + before = tree_fingerprint(foreign) + rc: int | None = None + error: BaseException | None = None + try: + with mutation_environment(marker): + authority = store._load_home(home) + with store._store_lock(authority, None): + store._reconcile(authority) + rc = 0 + except store.StoreInfrastructure as caught: + rc = 125 + error = caught + except BaseException as caught: + error = caught + after = tree_fingerprint(foreign) + secure = rc == 125 and before == after + return ProbeResult(secure, f"rc={rc} error={error!r} changed={before != after}") + + +def apply_mutation(runtime: Path, mutation: Mutation) -> None: + path = runtime / mutation.path + try: + source = path.read_text(encoding="utf-8") + except (OSError, UnicodeError) as error: + raise InfrastructureError(f"mutation source cannot be read: {mutation.path}") from error + occurrences = source.count(mutation.old) + if occurrences != 1: + raise InfrastructureError( + f"{mutation.assertion} replacement applicability is {occurrences}, expected exactly 1" + ) + mutated = source.replace(mutation.old, mutation.new, 1) + if mutated == source or mutated.count(mutation.new) != 1: + raise InfrastructureError(f"{mutation.assertion} replacement result is ambiguous") + try: + path.write_text(mutated, encoding="utf-8") + except OSError as error: + raise InfrastructureError(f"{mutation.assertion} private source cannot be written") from error + if path.read_text(encoding="utf-8") != mutated: + raise InfrastructureError(f"{mutation.assertion} private source write was not exact") + + +def compile_mutation(runtime: Path, mutation: Mutation, cache: Path) -> None: + completed = run_command( + [ + sys.executable, + "-I", + "-B", + "-X", + f"pycache_prefix={cache}", + "-m", + "py_compile", + str(runtime / mutation.path), + ] + ) + if completed.returncode != 0 or completed.stderr: + raise InfrastructureError( + f"{mutation.assertion} private mutation does not compile: " + + completed.stderr.decode("utf-8", errors="replace") + ) + + +MUTATIONS: tuple[Mutation, ...] = ( + Mutation( + "M-STORE-AUTH-001", + "scripts/experiment_store.py", + ' decision, status = experiment.authorize_plan(plan, snapshot.digest)\n', + ( + ' mutation_marker = os.environ.get("AGENT_LAB_MUTATION_MARK")\n' + ' if mutation_marker is None:\n' + ' decision, status = experiment.authorize_plan(plan, snapshot.digest)\n' + ' else:\n' + ' Path(mutation_marker).touch()\n' + ' decision = _parse_object(\n' + ' Path(os.environ["AGENT_LAB_MUTATION_DECISION"]).read_bytes(),\n' + ' "caller-supplied saved decision",\n' + ' )\n' + ' status = 0\n' + ), + probe_fresh_authorization, + "the fresh-authorization oracle detects a caller-supplied saved permit", + ), + Mutation( + "M-STORE-SOURCE-001", + "scripts/experiment_store.py", + ' "artifact/experiment.cue": source_data,\n', + ( + ' "artifact/experiment.cue": (\n' + ' source_data\n' + ' if os.environ.get("AGENT_LAB_MUTATION_MARK") is None\n' + ' else (\n' + ' Path(os.environ["AGENT_LAB_MUTATION_MARK"]).touch()\n' + ' or (\n' + ' Path(os.environ["AGENT_LAB_MUTATION_SOURCE"])\n' + ' / "experiment.cue"\n' + ' ).read_bytes()\n' + ' )\n' + ' ),\n' + ), + probe_held_snapshot, + "the held-snapshot oracle detects source reopening after authorization", + ), + Mutation( + "M-STORE-ATOM-001", + "scripts/experiment_store.py", + ' _rename_noreplace(wrapper / "payload", authority.store / name)\n', + ( + ' mutation_marker = os.environ.get("AGENT_LAB_MUTATION_MARK")\n' + ' if mutation_marker is None:\n' + ' _rename_noreplace(wrapper / "payload", authority.store / name)\n' + ' else:\n' + ' Path(mutation_marker).touch()\n' + ' os.replace(wrapper / "payload", authority.store / name)\n' + ), + probe_noreplace_race, + "the publication-race oracle detects replace-capable final publication", + ), + Mutation( + "M-STORE-RETRY-001", + "scripts/experiment_store.py", + ( + ' return {\n' + ' "changed": False,\n' + ' "installationKey": existing.installation_key,\n' + ' "name": name,\n' + ' "receiptDigest": existing.receipt_digest,\n' + ' }\n' + ), + ( + ' mutation_marker = os.environ.get("AGENT_LAB_MUTATION_MARK")\n' + ' if mutation_marker is not None:\n' + ' Path(mutation_marker).touch()\n' + ' _reject("mutated matching installation conflict")\n' + ' return {\n' + ' "changed": False,\n' + ' "installationKey": existing.installation_key,\n' + ' "name": name,\n' + ' "receiptDigest": existing.receipt_digest,\n' + ' }\n' + ), + probe_idempotent_retry, + "the idempotence oracle detects a matching receipt treated as conflict", + ), + Mutation( + "M-STORE-DUR-001", + "scripts/experiment_store.py", + ' _fsync_directory(payload / "records", "Experiment committed records", modes=(0o500,))\n', + ( + ' mutation_marker = os.environ.get("AGENT_LAB_MUTATION_MARK")\n' + ' if mutation_marker is None:\n' + ' _fsync_directory(\n' + ' payload / "records",\n' + ' "Experiment committed records",\n' + ' modes=(0o500,),\n' + ' )\n' + ' else:\n' + ' Path(mutation_marker).touch()\n' + ' _fsync_directory(\n' + ' payload / "artifact",\n' + ' "Experiment committed records",\n' + ' modes=(0o500,),\n' + ' )\n' + ), + probe_publication_durability, + "the durability oracle detects committed-directory fsync path substitution", + ), + Mutation( + "M-STORE-LAYOUT-001", + "scripts/experiment_store.py", + ' "artifact/experiment.cue": source_data,\n', + ( + ' "artifact/experiment.cue": (\n' + ' source_data\n' + ' if os.environ.get("AGENT_LAB_MUTATION_MARK") is None\n' + ' else (\n' + ' Path(os.environ["AGENT_LAB_MUTATION_MARK"]).touch()\n' + ' or source_data + b"\\n" + decision_bytes\n' + ' )\n' + ' ),\n' + ), + probe_artifact_separation, + "the portable-artifact oracle detects generated metadata mixed into artifact bytes", + ), + Mutation( + "M-STORE-KEY-001", + "scripts/experiment_store.py", + 'def canonical(value: object) -> bytes:\n try:\n return json.dumps(\n', + ( + 'def canonical(value: object) -> bytes:\n' + ' mutation_marker = os.environ.get("AGENT_LAB_MUTATION_MARK")\n' + ' if (\n' + ' mutation_marker is not None\n' + ' and isinstance(value, dict)\n' + ' and set(value)\n' + ' == {\n' + ' "authorizationDigest",\n' + ' "contractDigest",\n' + ' "planDigest",\n' + ' "selectedEntries",\n' + ' "sourceDigest",\n' + ' }\n' + ' ):\n' + ' Path(mutation_marker).touch()\n' + ' value = {"requestedName": os.environ["AGENT_LAB_MUTATION_NAME"]}\n' + ' try:\n' + ' return json.dumps(\n' + ), + probe_full_identity_key, + "the conflict oracle detects an installation key based only on requested name", + ), + Mutation( + "M-STORE-VERIFY-001", + "scripts/experiment_store.py", + ' verified = _verify_envelope(authority.store / name, name, authority.store_device)\n', + ( + ' mutation_marker = os.environ.get("AGENT_LAB_MUTATION_MARK")\n' + ' if mutation_marker is None:\n' + ' verified = _verify_envelope(\n' + ' authority.store / name, name, authority.store_device\n' + ' )\n' + ' else:\n' + ' Path(mutation_marker).touch()\n' + ' verified = VerifiedInstall(name, key, receipt_digest, {})\n' + ), + probe_final_revalidation, + "the corruption oracle detects skipped post-publication receipt verification", + ), + Mutation( + "M-STORE-LIVE-001", + "scripts/experiment_store.py", + ( + ' with operation(home, tuple(dependencies), fault=fault) as held:\n' + ' yield held\n' + ), + ( + ' mutation_marker = os.environ.get("AGENT_LAB_MUTATION_MARK")\n' + ' if mutation_marker is None:\n' + ' with operation(home, tuple(dependencies), fault=fault) as held:\n' + ' yield held\n' + ' else:\n' + ' with operation(home, tuple(dependencies), fault=fault) as held:\n' + ' released = held\n' + ' Path(mutation_marker).touch()\n' + ' yield released\n' + ), + probe_catalog_lock_lifetime, + "the liveness oracle detects catalog-lock release before publication", + ), + Mutation( + "M-STORE-UNCERT-001", + "scripts/experiment_store.py", + ( + ' _cleanup_operation(authority, wrapper)\n' + ' return {\n' + ' "changed": True,\n' + ), + ( + ' try:\n' + ' _cleanup_operation(authority, wrapper)\n' + ' except StoreInfrastructure:\n' + ' mutation_marker = os.environ.get("AGENT_LAB_MUTATION_MARK")\n' + ' if mutation_marker is None:\n' + ' raise\n' + ' Path(mutation_marker).touch()\n' + ' return {\n' + ' "changed": True,\n' + ), + probe_cleanup_uncertainty, + "the uncertainty oracle detects cleanup failure mapped to success", + ), + Mutation( + "M-STORE-STAGE-001", + "scripts/experiment_store.py", + ( + ' if names != (OPERATION_WRAPPER,):\n' + ' _infra("Experiment staging root contains an unknown wrapper")\n' + ), + ( + ' if names != (OPERATION_WRAPPER,):\n' + ' mutation_marker = os.environ.get("AGENT_LAB_MUTATION_MARK")\n' + ' if mutation_marker is None:\n' + ' _infra("Experiment staging root contains an unknown wrapper")\n' + ' Path(mutation_marker).touch()\n' + ' for staged_name in names:\n' + ' staged = authority.staging / staged_name\n' + ' __import__("shutil").rmtree(staged)\n' + ' _fsync_directory(authority.staging, "Experiment broad staging cleanup")\n' + ' return\n' + ), + probe_unknown_staging, + "the staging oracle detects broad deletion of an unknown wrapper", + ), +) + + +def marker_reached(marker: Path) -> bool: + try: + metadata = marker.lstat() + except OSError: + return False + return ( + stat.S_ISREG(metadata.st_mode) + and metadata.st_uid == os.getuid() + and metadata.st_nlink == 1 + and metadata.st_size == 0 + ) + + +def execute_mutation( + root: Path, + names: tuple[str, ...], + mutation: Mutation, +) -> tuple[bool, str]: + runtime = root / "runtime" + copy_runtime(runtime, names) + copied = runtime_fingerprint(runtime, names) + copied_tree = tree_fingerprint(runtime) + previous_bytecode = sys.dont_write_bytecode + sys.dont_write_bytecode = True + try: + pristine = mutation.probe(runtime, root / "pristine", None) + finally: + sys.dont_write_bytecode = previous_bytecode + if not pristine.secure: + raise InfrastructureError( + f"{mutation.assertion} pristine probe is not GREEN: {pristine.detail}" + ) + if runtime_fingerprint(runtime, names) != copied: + raise InfrastructureError(f"{mutation.assertion} pristine probe changed its runtime copy") + if tree_fingerprint(runtime) != copied_tree: + raise InfrastructureError(f"{mutation.assertion} pristine probe changed runtime topology") + + apply_mutation(runtime, mutation) + compile_mutation(runtime, mutation, root / "pycache") + mutated = runtime_fingerprint(runtime, names) + mutated_tree = tree_fingerprint(runtime) + changed = [name for (name, before), (_, after) in zip(copied, mutated) if before != after] + if changed != [mutation.path]: + raise InfrastructureError( + f"{mutation.assertion} changed unexpected private runtime paths: {changed!r}" + ) + + marker = root / "mutation-reached" + previous_bytecode = sys.dont_write_bytecode + sys.dont_write_bytecode = True + try: + result = mutation.probe(runtime, root / "mutant", marker) + finally: + sys.dont_write_bytecode = previous_bytecode + if not marker_reached(marker): + raise InfrastructureError(f"{mutation.assertion} did not prove its mutated path was reached") + if runtime_fingerprint(runtime, names) != mutated: + raise InfrastructureError(f"{mutation.assertion} mutant probe changed its runtime copy") + if tree_fingerprint(runtime) != mutated_tree: + raise InfrastructureError(f"{mutation.assertion} mutant probe changed runtime topology") + return not result.secure, result.detail + + +def remove_private_root(root: Path) -> None: + try: + if not root.exists() and not root.is_symlink(): + return + if root.is_symlink(): + root.unlink() + return + os.chmod(root, 0o700) + for directory, names, files in os.walk(root, topdown=True, followlinks=False): + current = Path(directory) + os.chmod(current, 0o700) + for name in names: + path = current / name + if not path.is_symlink(): + os.chmod(path, 0o700) + for name in files: + path = current / name + if not path.is_symlink(): + os.chmod(path, 0o600) + shutil.rmtree(root) + except OSError as error: + raise InfrastructureError("private mutation cleanup is uncertain") from error + if root.exists() or root.is_symlink(): + raise InfrastructureError("private mutation cleanup was incomplete") + + +def main() -> int: + try: + names = manifest_paths() + shared_before = runtime_fingerprint(REPO_ROOT, names) + expected = ( + "M-STORE-AUTH-001", + "M-STORE-SOURCE-001", + "M-STORE-ATOM-001", + "M-STORE-RETRY-001", + "M-STORE-DUR-001", + "M-STORE-LAYOUT-001", + "M-STORE-KEY-001", + "M-STORE-VERIFY-001", + "M-STORE-LIVE-001", + "M-STORE-UNCERT-001", + "M-STORE-STAGE-001", + ) + if tuple(mutation.assertion for mutation in MUTATIONS) != expected: + raise InfrastructureError("store mutation assertion identity drift") + failures = 0 + observed: list[str] = [] + for mutation in MUTATIONS: + try: + temporary = Path( + tempfile.mkdtemp( + prefix=f"agent-lab-{mutation.assertion.lower()}-", + dir="/tmp", + ) + ) + except OSError as error: + raise InfrastructureError( + f"{mutation.assertion} private mutation root is unavailable" + ) from error + try: + detected, detail = execute_mutation(temporary, names, mutation) + observed.append(mutation.assertion) + if detected: + print(f"PASS {mutation.assertion} {mutation.message}") + else: + failures += 1 + print(f"FAIL {mutation.assertion} {mutation.message} ({detail})") + finally: + remove_private_root(temporary) + if runtime_fingerprint(REPO_ROOT, names) != shared_before: + raise InfrastructureError( + f"{mutation.assertion} changed the shared checkout runtime fingerprint" + ) + if tuple(observed) != expected: + raise InfrastructureError("store mutation execution identity drift") + print(f"SUMMARY assertions=11 expected=11 failures={failures} infra=0") + return 0 if failures == 0 else 1 + except InfrastructureError as error: + print(f"INFRA store mutation evidence: {error}", file=sys.stderr) + return 125 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/experiment/install-state-cases.py b/tests/experiment/install-state-cases.py new file mode 100644 index 0000000..f4816f4 --- /dev/null +++ b/tests/experiment/install-state-cases.py @@ -0,0 +1,2229 @@ +#!/usr/bin/env python3 +"""Adversarial Experiment-store filesystem, locking, and recovery cases.""" + +from __future__ import annotations + +from contextlib import contextmanager, redirect_stderr, redirect_stdout +import errno +import hashlib +from importlib.util import module_from_spec, spec_from_file_location +import io +import json +import os +from pathlib import Path +import select +import signal +import stat +import subprocess +import sys +import tempfile +import time +from typing import Callable, NamedTuple + + +REPO_ROOT = Path(__file__).resolve().parents[2] +AGENT_LAB = REPO_ROOT / "scripts" / "agent-lab" +AGENT_LAB_MODULE = REPO_ROOT / "scripts" / "agent-lab.py" +STORE_MODULE = REPO_ROOT / "scripts" / "experiment_store.py" +CUE_TOOLS = REPO_ROOT / ".cache" / "dev" / "tools" / "cue" +CEDAR_TOOLS = REPO_ROOT / ".cache" / "dev" / "tools" / "cedar" +SUBJECT = "registry.example/team/worker@sha256:" + "a" * 64 +OTHER_SUBJECT = "registry.example/team/worker@sha256:" + "b" * 64 +PLAN_DOMAIN = b"agent-lab.experiment-plan.v1\0" +FAULT_POINTS = ( + "experiment artifact.after_write", + "experiment receipt.after_fsync", + "experiment envelope.before_noreplace", + "experiment envelope.after_noreplace", + "experiment store root.after_fsync", +) + + +class FixtureSnapshot(NamedTuple): + data: bytes + digest: str + + +class FixtureResolution(NamedTuple): + plan: dict[str, object] + bundled_catalog: dict[str, object] | None + local_catalog: dict[str, object] | None + + +class FixtureInvalidManifest(Exception): + """The isolated store fixture rejected its authored manifest.""" + + +class FixtureInfrastructure(Exception): + """The isolated store fixture could not establish a trusted result.""" + + +class FaultFixtureExperiment: + """Small deterministic planner used only for exhaustive store fault injection.""" + + InvalidManifest = FixtureInvalidManifest + InfrastructureError = FixtureInfrastructure + + def __init__(self, source_data: bytes, name: str) -> None: + source_digest = store_source_digest(source_data) + contract_digest = "sha256:" + "d" * 64 + plan: dict[str, object] = { + "apiVersion": "agent-lab.request/v0alpha1", + "contract": { + "digest": contract_digest, + "name": "agent-lab.experiment", + "version": "v0alpha1", + }, + "kind": "RequestedExperimentPlan", + "metadata": {"requestedName": name}, + "spec": { + "members": [ + { + "command": ["fault-probe"], + "name": "worker", + "requestedSelector": {"digestRef": SUBJECT}, + "resolvedImage": {"origin": "direct", "subject": SUBJECT}, + "resourceClass": "small", + } + ] + }, + } + plan_digest = "sha256:" + hashlib.sha256( + PLAN_DOMAIN + canonical_json(plan) + ).hexdigest() + self.source_data = source_data + self.snapshot = FixtureSnapshot(source_data, source_digest) + self.resolution = FixtureResolution(plan, None, None) + self.decision: dict[str, object] = { + "action": "experiment.install", + "apiVersion": "agent-lab.authorization/v0alpha1", + "binding": { + "authorizationDigest": "sha256:" + "c" * 64, + "contractDigest": contract_digest, + "planDigest": plan_digest, + "sourceDigest": source_digest, + }, + "kind": "ExperimentAuthorizationDecision", + "principal": { + "assurance": "none", + "authenticated": False, + "id": "local-cli", + "source": "fixed-local-cli", + "type": "AgentLab::Principal", + }, + "resource": { + "id": plan_digest, + "requestedName": name, + "type": "AgentLab::RequestedExperimentPlan", + }, + "verdict": "permit", + } + + def read_directory_snapshot(self, source: str) -> FixtureSnapshot: + try: + observed = (Path(source) / "experiment.cue").read_bytes() + except OSError as error: + raise FixtureInfrastructure("fault fixture source cannot be read") from error + if observed != self.source_data: + raise FixtureInvalidManifest("fault fixture source changed") + return self.snapshot + + def authored_manifest(self, snapshot: FixtureSnapshot) -> bytes: + return snapshot.data + + def cue_plan_with_evidence(self, manifest: object) -> FixtureResolution: + if manifest != self.source_data: + raise FixtureInvalidManifest("fault fixture manifest changed") + return self.resolution + + def authorize_plan( + self, + plan: dict[str, object], + source_digest: str, + ) -> tuple[dict[str, object], int]: + if plan != self.resolution.plan or source_digest != self.snapshot.digest: + raise FixtureInfrastructure("fault fixture authorization binding changed") + return self.decision, 0 + + def verify_trusted_inputs( + self, + plan: dict[str, object], + decision: dict[str, object], + ) -> None: + if plan != self.resolution.plan or decision != self.decision: + raise FixtureInfrastructure("fault fixture trusted inputs changed") + + +def canonical_json(value: object) -> bytes: + return json.dumps( + value, + allow_nan=False, + ensure_ascii=True, + separators=(",", ":"), + sort_keys=True, + ).encode("ascii") + + +def store_source_digest(data: bytes) -> str: + value = hashlib.sha256(b"agent-lab.experiment-tree.v1\0") + name = b"experiment.cue" + value.update(len(name).to_bytes(4, "big")) + value.update(name) + value.update(len(data).to_bytes(8, "big")) + value.update(data) + return "sha256:" + value.hexdigest() + + +def load_module(path: Path, name: str): + spec = spec_from_file_location(name, path) + if spec is None or spec.loader is None: + raise RuntimeError(f"{path.name} cannot be loaded") + module = module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +MODULE = load_module(AGENT_LAB_MODULE, "agent_lab_install_state") +try: + STORE = load_module(STORE_MODULE, "agent_lab_experiment_store_state") + STORE_LOAD_ERROR: BaseException | None = None +except BaseException as error: # Missing production is expected RED, not harness infrastructure. + STORE = None + STORE_LOAD_ERROR = error + +FAILURES = 0 +INFRA = 0 +OBSERVED: list[str] = [] + + +def check(assertion: str, condition: bool, message: str, detail: str = "") -> None: + global FAILURES + OBSERVED.append(assertion) + if condition: + print(f"PASS {assertion} {message}") + else: + FAILURES += 1 + suffix = f" ({detail})" if detail else "" + print(f"FAIL {assertion} {message}{suffix}") + + +def command_environment() -> dict[str, str]: + return { + "PATH": "/usr/bin:/bin", + "LANG": "C", + "LC_ALL": "C", + "AGENT_LAB_CUE_TOOL_DIR": str(CUE_TOOLS), + "AGENT_LAB_CEDAR_TOOL_DIR": str(CEDAR_TOOLS), + } + + +def run_command(arguments: list[str], timeout: float = 30.0) -> subprocess.CompletedProcess[bytes]: + global INFRA + try: + process = subprocess.Popen( + arguments, + stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + env=command_environment(), + start_new_session=True, + ) + except OSError as error: + INFRA += 1 + return subprocess.CompletedProcess(arguments, 125, b"", str(error).encode()) + try: + stdout, stderr = process.communicate(timeout=timeout) + except subprocess.TimeoutExpired: + INFRA += 1 + try: + os.killpg(process.pid, signal.SIGKILL) + except ProcessLookupError: + pass + stdout, stderr = process.communicate() + return subprocess.CompletedProcess(arguments, 125, stdout, stderr + b"\nHARNESS TIMEOUT\n") + return subprocess.CompletedProcess(arguments, process.returncode, stdout, stderr) + + +def cli(home: Path, *arguments: str) -> subprocess.CompletedProcess[bytes]: + return run_command([str(AGENT_LAB), "--home", str(home), *arguments]) + + +def start_cli(home: Path, source: Path) -> subprocess.Popen[bytes]: + return subprocess.Popen( + [str(AGENT_LAB), "--home", str(home), "experiment", "install", str(source)], + stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + env=command_environment(), + start_new_session=True, + ) + + +def start_inspect(home: Path, name: str) -> subprocess.Popen[bytes]: + return subprocess.Popen( + [str(AGENT_LAB), "--home", str(home), "experiment", "inspect", name], + stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + env=command_environment(), + start_new_session=True, + ) + + +def start_remove(home: Path, name: str, entry_digest: str) -> subprocess.Popen[bytes]: + return subprocess.Popen( + [ + str(AGENT_LAB), + "--home", + str(home), + "image", + "remove", + name, + "--expect", + entry_digest, + ], + stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + env=command_environment(), + start_new_session=True, + ) + + +def finish_process( + process: subprocess.Popen[bytes], + timeout: float = 30.0, +) -> subprocess.CompletedProcess[bytes]: + global INFRA + try: + stdout, stderr = process.communicate(timeout=timeout) + except subprocess.TimeoutExpired: + INFRA += 1 + try: + os.killpg(process.pid, signal.SIGKILL) + except ProcessLookupError: + pass + stdout, stderr = process.communicate() + return subprocess.CompletedProcess( + process.args, + 125, + stdout, + stderr + b"\nHARNESS TIMEOUT\n", + ) + return subprocess.CompletedProcess(process.args, process.returncode, stdout, stderr) + + +def new_home(root: Path, name: str) -> Path: + home = root / name + completed = cli(home, "init") + if completed.returncode != 0: + raise RuntimeError( + f"temporary home init failed: rc={completed.returncode} " + f"stderr={completed.stderr.decode(errors='replace')}" + ) + return home + + +def source_directory( + root: Path, + directory: str, + *, + requested_name: str = "first-experiment", + subject: str = SUBJECT, + command: str = "serve", + catalog_name: str | None = None, +) -> Path: + source = root / directory + source.mkdir(mode=0o700) + if catalog_name is None: + selector = f'digestRef: "{subject}"' + else: + selector = f'catalogName: "{catalog_name}"' + data = ( + "package experiment\n\n" + "experiment: {\n" + '\tapiVersion: "agent-lab/v0alpha1"\n' + '\tkind: "Experiment"\n' + f'\tmetadata: name: "{requested_name}"\n' + "\tspec: members: [{\n" + '\t\tname: "worker"\n' + f"\t\timage: {selector}\n" + f'\t\tcommand: ["{command}"]\n' + "\t}]\n" + "}\n" + ).encode("utf-8") + path = source / "experiment.cue" + path.write_bytes(data) + path.chmod(0o600) + return source + + +def fingerprint(root: Path) -> tuple[tuple[str, str, int, int, int, str], ...]: + if not root.exists() and not root.is_symlink(): + return () + records: list[tuple[str, str, int, int, int, str]] = [] + + def visit(path: Path, relative: str) -> None: + metadata = path.lstat() + if stat.S_ISLNK(metadata.st_mode): + kind = "l" + identity = os.readlink(path) + elif stat.S_ISREG(metadata.st_mode): + kind = "f" + digest = hashlib.sha256() + with path.open("rb") as stream: + while chunk := stream.read(65_536): + digest.update(chunk) + identity = digest.hexdigest() + elif stat.S_ISDIR(metadata.st_mode): + kind = "d" + identity = "" + else: + kind = "o" + identity = "" + records.append( + ( + relative, + kind, + stat.S_IMODE(metadata.st_mode), + metadata.st_nlink, + metadata.st_size, + identity, + ) + ) + if kind == "d": + children = sorted(path.iterdir(), key=lambda item: os.fsencode(item.name)) + for child in children: + child_relative = child.name if relative == "." else f"{relative}/{child.name}" + visit(child, child_relative) + + visit(root, ".") + return tuple(records) + + +def json_object(completed: subprocess.CompletedProcess[bytes]) -> dict[str, object] | None: + if completed.returncode != 0: + return None + try: + value = json.loads(completed.stdout) + except (UnicodeError, json.JSONDecodeError): + return None + return value if isinstance(value, dict) else None + + +def store_install( + home: Path, + source: Path, + *, + fault: Callable[[str], None] | None = None, +) -> tuple[int | None, dict[str, object] | None, BaseException | None]: + if STORE is None: + return None, None, STORE_LOAD_ERROR or RuntimeError("experiment store module is missing") + operation = getattr(STORE, "install_directory", None) + if not callable(operation): + return None, None, RuntimeError("experiment_store.install_directory is missing") + try: + value = operation(home, source, fault=fault) + if not isinstance(value, dict): + return None, None, RuntimeError("install_directory returned a non-object") + return 0, value, None + except getattr(STORE, "StoreReject", ()) as error: + return 1, None, error + except getattr(STORE, "StoreInfrastructure", ()) as error: + return 125, None, error + except BaseException as error: # An uncontained production fault is RED, not harness infra. + return None, None, error + + +def store_inspect( + home: Path, + name: str, +) -> tuple[int | None, dict[str, object] | None, BaseException | None]: + if STORE is None: + return None, None, STORE_LOAD_ERROR or RuntimeError("experiment store module is missing") + operation = getattr(STORE, "inspect_install", None) + if not callable(operation): + return None, None, RuntimeError("experiment_store.inspect_install is missing") + try: + value = operation(home, name) + if not isinstance(value, dict): + return None, None, RuntimeError("inspect_install returned a non-object") + return 0, value, None + except getattr(STORE, "StoreReject", ()) as error: + return 1, None, error + except getattr(STORE, "StoreInfrastructure", ()) as error: + return 125, None, error + except BaseException as error: # An uncontained production fault is RED, not harness infra. + return None, None, error + + +def module_main( + home: Path, + arguments: list[str], + output: io.TextIOBase | None = None, +) -> tuple[int | None, str, BaseException | None]: + stream = output if output is not None else io.StringIO() + errors = io.StringIO() + old_cue = os.environ.get("AGENT_LAB_CUE_TOOL_DIR") + old_cedar = os.environ.get("AGENT_LAB_CEDAR_TOOL_DIR") + os.environ["AGENT_LAB_CUE_TOOL_DIR"] = str(CUE_TOOLS) + os.environ["AGENT_LAB_CEDAR_TOOL_DIR"] = str(CEDAR_TOOLS) + try: + with redirect_stdout(stream), redirect_stderr(errors): + result = MODULE.main(["--home", str(home), *arguments]) + return result, errors.getvalue(), None + except BaseException as error: # An uncontained production fault is RED, not harness infra. + return None, errors.getvalue(), error + finally: + if old_cue is None: + os.environ.pop("AGENT_LAB_CUE_TOOL_DIR", None) + else: + os.environ["AGENT_LAB_CUE_TOOL_DIR"] = old_cue + if old_cedar is None: + os.environ.pop("AGENT_LAB_CEDAR_TOOL_DIR", None) + else: + os.environ["AGENT_LAB_CEDAR_TOOL_DIR"] = old_cedar + + +def hard_exit_install(home: Path, source: Path, point: str) -> int: + pid = os.fork() + if pid == 0: + def stop_at(observed: str) -> None: + if observed == point: + os._exit(99) + + result, _, _ = store_install(home, source, fault=stop_at) + os._exit(97 if result == 0 else 96) + deadline = time.monotonic() + 30.0 + while time.monotonic() < deadline: + waited, status = os.waitpid(pid, os.WNOHANG) + if waited == pid: + return os.waitstatus_to_exitcode(status) + time.sleep(0.01) + os.kill(pid, signal.SIGKILL) + os.waitpid(pid, 0) + global INFRA + INFRA += 1 + return 124 + + +def hard_exit_after_raw_publication(home: Path, source: Path) -> int: + """Exit after no-replace succeeds but before the final root becomes read-only.""" + + if STORE is None: + return 95 + target = home / "experiments" / "first-experiment" + pid = os.fork() + if pid == 0: + original_chmod = STORE.os.chmod + + def stop_before_final_chmod(path: object, mode: int, *args, **kwargs): + if Path(os.fsdecode(os.fspath(path))) == target and mode == 0o500: + os._exit(99) + return original_chmod(path, mode, *args, **kwargs) + + STORE.os.chmod = stop_before_final_chmod + result, _, _ = store_install(home, source) + os._exit(97 if result == 0 else 96) + deadline = time.monotonic() + 30.0 + while time.monotonic() < deadline: + waited, status = os.waitpid(pid, os.WNOHANG) + if waited == pid: + return os.waitstatus_to_exitcode(status) + time.sleep(0.01) + os.kill(pid, signal.SIGKILL) + os.waitpid(pid, 0) + global INFRA + INFRA += 1 + return 124 + + +def hard_exit_before_intent(home: Path, source: Path) -> int: + """Exit after the operation wrapper exists but before intent bytes are written.""" + + if STORE is None: + return 95 + pid = os.fork() + if pid == 0: + original_write_file = STORE._write_file + + def stop_before_intent(path, data, purpose, fault): + if purpose == "experiment intent": + os._exit(99) + return original_write_file(path, data, purpose, fault) + + STORE._write_file = stop_before_intent + result, _, _ = store_install(home, source) + os._exit(97 if result == 0 else 96) + deadline = time.monotonic() + 30.0 + while time.monotonic() < deadline: + waited, status = os.waitpid(pid, os.WNOHANG) + if waited == pid: + return os.waitstatus_to_exitcode(status) + time.sleep(0.01) + os.kill(pid, signal.SIGKILL) + os.waitpid(pid, 0) + global INFRA + INFRA += 1 + return 124 + + +def hard_exit_during_cleanup(home: Path, source: Path, phase: str) -> int: + """Exit after durable cleanup handoff or after the first cleanup-file removal.""" + + if STORE is None: + return 95 + pid = os.fork() + if pid == 0: + if phase == "handoff": + original_rename = STORE._rename_noreplace + + def stop_after_handoff(source_path: Path, target_path: Path) -> None: + original_rename(source_path, target_path) + if target_path.name == STORE.CLEANUP_WRAPPER: + os._exit(99) + + STORE._rename_noreplace = stop_after_handoff + elif phase == "remove": + original_remove = STORE._remove_tree + stopped = False + + def stop_after_first_file(path: Path, root: Path) -> None: + nonlocal stopped + was_file = False + try: + was_file = stat.S_ISREG(path.lstat().st_mode) + except OSError: + pass + original_remove(path, root) + if was_file and not stopped: + stopped = True + os._exit(99) + + STORE._remove_tree = stop_after_first_file + else: + os._exit(95) + result, _, _ = store_install(home, source) + os._exit(97 if result == 0 else 96) + deadline = time.monotonic() + 30.0 + while time.monotonic() < deadline: + waited, status = os.waitpid(pid, os.WNOHANG) + if waited == pid: + return os.waitstatus_to_exitcode(status) + time.sleep(0.01) + os.kill(pid, signal.SIGKILL) + os.waitpid(pid, 0) + global INFRA + INFRA += 1 + return 124 + + +def start_paused_publication(home: Path, source: Path) -> tuple[int, int, int]: + """Pause a child install while it holds the store lock before publication.""" + + ready_read, ready_write = os.pipe() + release_read, release_write = os.pipe() + pid = os.fork() + if pid == 0: + os.close(ready_read) + os.close(release_write) + paused = False + + def pause(point: str) -> None: + nonlocal paused + if point == "experiment envelope.before_noreplace" and not paused: + paused = True + os.write(ready_write, b"1") + if os.read(release_read, 1) != b"1": + os._exit(94) + + result, _, _ = store_install(home, source, fault=pause) + os._exit(0 if result == 0 else 96) + os.close(ready_write) + os.close(release_read) + readable, _, _ = select.select((ready_read,), (), (), 30.0) + if not readable or os.read(ready_read, 1) != b"1": + try: + os.kill(pid, signal.SIGKILL) + except ProcessLookupError: + pass + os.waitpid(pid, 0) + os.close(ready_read) + os.close(release_write) + global INFRA + INFRA += 1 + return 0, -1, -1 + os.close(ready_read) + return pid, release_write, 0 + + +def finish_child(pid: int, timeout: float = 30.0) -> int: + if pid <= 0: + return 124 + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + waited, status = os.waitpid(pid, os.WNOHANG) + if waited == pid: + return os.waitstatus_to_exitcode(status) + time.sleep(0.01) + try: + os.kill(pid, signal.SIGKILL) + except ProcessLookupError: + pass + os.waitpid(pid, 0) + global INFRA + INFRA += 1 + return 124 + + +def wait_for_lock_block(process: subprocess.Popen[bytes], timeout: float = 5.0) -> bool: + """Observe the Linux flock wait channel rather than infer blocking from a sleep.""" + + deadline = time.monotonic() + timeout + path = Path(f"/proc/{process.pid}/wchan") + while time.monotonic() < deadline: + if process.poll() is not None: + return False + try: + channel = path.read_text(encoding="ascii").strip() + except OSError: + channel = "" + if channel == "locks_lock_inode_wait": + return True + time.sleep(0.005) + return False + + +def lock_is_blocked(path: Path) -> bool: + program = ( + "import fcntl, os, sys\n" + "fd=os.open(sys.argv[1], os.O_RDWR|getattr(os,'O_CLOEXEC',0))\n" + "try:\n" + " fcntl.flock(fd, fcntl.LOCK_EX|fcntl.LOCK_NB)\n" + "except BlockingIOError:\n" + " raise SystemExit(3)\n" + "raise SystemExit(0)\n" + ) + completed = run_command([sys.executable, "-I", "-c", program, str(path)], timeout=5.0) + return completed.returncode == 3 + + +class BrokenOutput(io.StringIO): + def write(self, value: str) -> int: + raise OSError("injected result-output failure") + + +def main() -> int: + with tempfile.TemporaryDirectory(prefix="agent-lab-install-state-") as directory: + root = Path(directory) + direct_source = source_directory(root, "direct-source") + + mode_home = new_home(root, "unsafe-mode-home") + mode_store = mode_home / "experiments" + mode_store.chmod(0o755) + mode_before = fingerprint(mode_store) + mode_result = cli(mode_home, "experiment", "install", str(direct_source)) + mode_after = fingerprint(mode_store) + + symlink_home = new_home(root, "symlink-store-home") + symlink_store = symlink_home / "experiments" + outside_store = root / "outside-store" + symlink_store.rename(outside_store) + os.symlink(outside_store, symlink_store) + symlink_before = fingerprint(outside_store) + symlink_result = cli(symlink_home, "experiment", "install", str(direct_source)) + symlink_after = fingerprint(outside_store) + + stage_home = new_home(root, "symlink-stage-home") + stage = stage_home / "experiments" / ".staging" + outside_stage = root / "outside-stage" + stage.rename(outside_stage) + os.symlink(outside_stage, stage) + stage_before = fingerprint(outside_stage) + stage_result = cli(stage_home, "experiment", "install", str(direct_source)) + stage_after = fingerprint(outside_stage) + + cross_home = new_home(root, "cross-device-home") + cross_stage = cross_home / "experiments" / ".staging" + cross_before = fingerprint(cross_home / "experiments") + original_fstat = MODULE.os.fstat + original_cross_lstat = MODULE.os.lstat + cross_injected = False + + def cross_device_metadata(metadata: os.stat_result) -> os.stat_result: + nonlocal cross_injected + values = list(metadata) + values[2] = metadata.st_dev + 1 + cross_injected = True + return os.stat_result(values) + + def cross_device_fstat(descriptor: int): + nonlocal cross_injected + metadata = original_fstat(descriptor) + try: + target = os.readlink(f"/proc/self/fd/{descriptor}") + except OSError: + return metadata + if target.rstrip("/") == str(cross_stage): + return cross_device_metadata(metadata) + return metadata + + def cross_device_lstat(path: object, *args, **kwargs): + metadata = original_cross_lstat(path, *args, **kwargs) + try: + target = os.fsdecode(os.fspath(path)).rstrip("/") + except TypeError: + return metadata + if target == str(cross_stage): + return cross_device_metadata(metadata) + return metadata + + MODULE.os.fstat = cross_device_fstat + MODULE.os.lstat = cross_device_lstat + try: + cross_rc, _, cross_error = module_main( + cross_home, + ["experiment", "install", str(direct_source)], + ) + finally: + MODULE.os.lstat = original_cross_lstat + MODULE.os.fstat = original_fstat + cross_after = fingerprint(cross_home / "experiments") + check( + "IST-STATE-001", + mode_result.returncode == 125 + and mode_before == mode_after + and symlink_result.returncode == 125 + and symlink_before == symlink_after + and stage_result.returncode == 125 + and stage_before == stage_after + and cross_injected + and cross_rc == 125 + and cross_error is None + and cross_before == cross_after, + "unsafe, linked, or cross-filesystem store and staging roots fail before publication", + ( + f"mode={mode_result.returncode}/{mode_before != mode_after} " + f"store_link={symlink_result.returncode}/{symlink_before != symlink_after} " + f"stage_link={stage_result.returncode}/{stage_before != stage_after} " + f"cross={cross_rc}/{cross_injected}/{cross_error!r}/{cross_before != cross_after}" + ), + ) + + symlink_lock_home = new_home(root, "symlink-lock-home") + lock = symlink_lock_home / "state" / "locks" / "experiments.lock" + saved_lock = root / "saved-experiments.lock" + lock.rename(saved_lock) + os.symlink(saved_lock, lock) + lock_before = fingerprint(saved_lock) + symlink_lock_result = cli( + symlink_lock_home, + "experiment", + "install", + str(direct_source), + ) + lock_after = fingerprint(saved_lock) + hardlink_lock_home = new_home(root, "hardlink-lock-home") + hardlink_lock = hardlink_lock_home / "state" / "locks" / "experiments.lock" + os.link(hardlink_lock, root / "second-experiments-lock-link") + hardlink_result = cli( + hardlink_lock_home, + "experiment", + "install", + str(direct_source), + ) + + inspect_lock_home = new_home(root, "inspect-lock-home") + inspect_lock_install = cli( + inspect_lock_home, + "experiment", + "install", + str(direct_source), + ) + inspect_lock = inspect_lock_home / "state" / "locks" / "experiments.lock" + os.link(inspect_lock, root / "second-inspect-lock-link") + hardlink_inspect = cli( + inspect_lock_home, + "experiment", + "inspect", + "first-experiment", + ) + + blocking_home = new_home(root, "inspect-blocking-home") + install_pid, release_write, pause_status = start_paused_publication( + blocking_home, + direct_source, + ) + blocking_inspect = start_inspect(blocking_home, "first-experiment") + inspect_waited = wait_for_lock_block(blocking_inspect) + if release_write >= 0: + try: + os.write(release_write, b"1") + finally: + os.close(release_write) + blocking_install_rc = finish_child(install_pid) + blocking_inspect_result = finish_process(blocking_inspect) + blocking_inspect_value = json_object(blocking_inspect_result) + check( + "IST-LOCK-001", + symlink_lock_result.returncode == 125 + and lock_before == lock_after + and hardlink_result.returncode == 125 + and not tuple((hardlink_lock_home / "experiments" / ".staging").iterdir()) + and inspect_lock_install.returncode == 0 + and hardlink_inspect.returncode == 125 + and pause_status == 0 + and inspect_waited + and blocking_install_rc == 0 + and blocking_inspect_result.returncode == 0 + and isinstance(blocking_inspect_value, dict) + and blocking_inspect_value.get("state") == "installed", + "the receipt-bound store lock protects install and read-only inspect", + ( + f"symlink={symlink_lock_result.returncode}/{lock_before != lock_after} " + f"hardlink={hardlink_result.returncode} inspect_install={inspect_lock_install.returncode} " + f"inspect_hardlink={hardlink_inspect.returncode} pause={pause_status} " + f"waited={inspect_waited} child={blocking_install_rc} " + f"inspect={blocking_inspect_result.returncode}/{blocking_inspect_value!r}" + ), + ) + + existing_home = new_home(root, "existing-target-home") + existing_target = existing_home / "experiments" / "first-experiment" + existing_target.mkdir(mode=0o700) + sentinel = existing_target / "foreign" + sentinel.write_bytes(b"foreign\n") + sentinel.chmod(0o600) + existing_before = fingerprint(existing_target) + existing_result = cli(existing_home, "experiment", "install", str(direct_source)) + existing_after = fingerprint(existing_target) + + race_home = new_home(root, "publication-race-home") + race_target = race_home / "experiments" / "first-experiment" + race_triggered = False + + def create_racing_target(point: str) -> None: + nonlocal race_triggered + if point == "experiment envelope.before_noreplace" and not race_triggered: + race_triggered = True + race_target.mkdir(mode=0o700) + marker = race_target / "foreign" + marker.write_bytes(b"racing foreign target\n") + marker.chmod(0o600) + + race_rc, _, race_error = store_install( + race_home, + direct_source, + fault=create_racing_target, + ) + race_fingerprint = fingerprint(race_target) + check( + "IST-STATE-002", + existing_result.returncode == 125 + and existing_before == existing_after + and race_triggered + and race_rc == 125 + and race_error is not None + and any( + item[0] == "foreign" + and item[-1] + == hashlib.sha256(b"racing foreign target\n").hexdigest() + for item in race_fingerprint + ), + "ambiguous and racing final targets are preserved by atomic no-replace publication", + ( + f"existing={existing_result.returncode}/{existing_before != existing_after} " + f"race={race_rc}/{race_triggered}/{race_error!r}/{race_fingerprint!r}" + ), + ) + + foreign_home = new_home(root, "foreign-stage-home") + foreign_wrapper = foreign_home / "experiments" / ".staging" / "foreign-wrapper" + (foreign_wrapper / "payload").mkdir(mode=0o700, parents=True) + foreign_intent = foreign_wrapper / "intent.json" + foreign_intent.write_bytes(b'{"owner":"foreign"}\n') + foreign_intent.chmod(0o600) + foreign_before = fingerprint(foreign_wrapper) + foreign_result = cli(foreign_home, "experiment", "install", str(direct_source)) + foreign_after = fingerprint(foreign_wrapper) + + bound_home = new_home(root, "overbound-stage-home") + bound_stage = bound_home / "experiments" / ".staging" + for index in range(17): + path = bound_stage / f"foreign-{index:02d}" + path.write_bytes(b"x") + path.chmod(0o600) + bound_before = fingerprint(bound_stage) + bound_result = cli(bound_home, "experiment", "install", str(direct_source)) + bound_after = fingerprint(bound_stage) + + cleanup_home = new_home(root, "foreign-cleanup-home") + cleanup_wrapper = ( + cleanup_home + / "experiments" + / ".staging" + / "experiment-install-cleanup" + ) + cleanup_artifact = cleanup_wrapper / "payload" / "artifact" + cleanup_artifact.mkdir(parents=True) + for directory_path in ( + cleanup_wrapper, + cleanup_wrapper / "payload", + cleanup_artifact, + ): + directory_path.chmod(0o700) + cleanup_canary = cleanup_artifact / "experiment.cue" + cleanup_canary.write_bytes(b"foreign cleanup canary\n") + cleanup_canary.chmod(0o600) + cleanup_before = fingerprint(cleanup_wrapper) + cleanup_result = cli( + cleanup_home, + "experiment", + "install", + str(direct_source), + ) + cleanup_after = fingerprint(cleanup_wrapper) + check( + "IST-BOUND-001", + foreign_result.returncode == 125 + and foreign_before == foreign_after + and bound_result.returncode == 125 + and bound_before == bound_after + and cleanup_result.returncode == 125 + and cleanup_before == cleanup_after, + "unknown, over-bound, or unproven cleanup residue is preserved", + ( + f"foreign={foreign_result.returncode}/{foreign_before != foreign_after} " + f"bound={bound_result.returncode}/{bound_before != bound_after} " + f"cleanup={cleanup_result.returncode}/{cleanup_before != cleanup_after}" + ), + ) + + tamper_home = new_home(root, "committed-tamper-home") + installed = cli(tamper_home, "experiment", "install", str(direct_source)) + envelope = tamper_home / "experiments" / "first-experiment" + link_detected = mode_detected = digest_detected = restored = False + tamper_detail = f"install={installed.returncode}" + if installed.returncode == 0 and envelope.is_dir(): + records = sorted((envelope / "records").glob("*.json")) + if records: + record = records[0] + outside_link = root / "outside-record-link" + os.link(record, outside_link) + linked_inspect = cli(tamper_home, "experiment", "inspect", "first-experiment") + link_detected = linked_inspect.returncode == 125 + outside_link.unlink() + restored_inspect = cli(tamper_home, "experiment", "inspect", "first-experiment") + restored = restored_inspect.returncode == 0 + envelope.chmod(0o700) + mode_inspect = cli(tamper_home, "experiment", "inspect", "first-experiment") + mode_detected = mode_inspect.returncode == 125 + envelope.chmod(0o500) + raw = record.read_bytes() + record.chmod(0o600) + record.write_bytes(bytes([raw[0] ^ 1]) + raw[1:]) + record.chmod(0o400) + digest_inspect = cli(tamper_home, "experiment", "inspect", "first-experiment") + digest_detected = digest_inspect.returncode == 125 + tamper_detail += ( + f" link={linked_inspect.returncode} restore={restored_inspect.returncode} " + f"mode={mode_inspect.returncode} digest={digest_inspect.returncode}" + ) + + layout_home = new_home(root, "changing-layout-home") + layout_install = cli(layout_home, "experiment", "install", str(direct_source)) + layout_target = layout_home / "experiments" / "first-experiment" + layout_changed = False + layout_rc: int | None = None + layout_error: BaseException | None = None + if STORE is not None and layout_install.returncode == 0: + original_directory_names = STORE._directory_names + + def change_after_enumeration(path: Path, purpose: str, maximum: int): + nonlocal layout_changed + names = original_directory_names(path, purpose, maximum) + if path == layout_target and not layout_changed: + layout_changed = True + layout_target.chmod(0o700) + foreign = layout_target / "foreign-after-enumeration" + foreign.write_bytes(b"foreign layout entry\n") + foreign.chmod(0o400) + layout_target.chmod(0o500) + return names + + STORE._directory_names = change_after_enumeration + try: + layout_rc, _, layout_error = store_inspect( + layout_home, + "first-experiment", + ) + finally: + STORE._directory_names = original_directory_names + check( + "IST-STATE-003", + installed.returncode == 0 + and link_detected + and restored + and mode_detected + and digest_detected + and layout_install.returncode == 0 + and layout_changed + and layout_rc == 125 + and layout_error is not None, + "inspect detects link, mode, byte, and concurrent layout drift", + f"{tamper_detail} layout={layout_install.returncode}/{layout_changed}/{layout_rc}/{layout_error!r}", + ) + + identical_home = new_home(root, "identical-concurrency-home") + first_pid, first_release, first_pause = start_paused_publication( + identical_home, + direct_source, + ) + second = start_cli(identical_home, direct_source) + identical_waited = wait_for_lock_block(second) + if first_release >= 0: + try: + os.write(first_release, b"1") + finally: + os.close(first_release) + first_rc = finish_child(first_pid) + second_result = finish_process(second) + second_value = json_object(second_result) + identical_inspect = cli( + identical_home, + "experiment", + "inspect", + "first-experiment", + ) + check( + "IST-CONC-001", + first_pause == 0 + and identical_waited + and first_rc == 0 + and second_result.returncode == 0 + and isinstance(second_value, dict) + and second_value.get("changed") is False + and identical_inspect.returncode == 0 + and not tuple((identical_home / "experiments" / ".staging").iterdir()), + "overlapping identical installs block at the store lock and publish once", + ( + f"pause={first_pause} waited={identical_waited} first={first_rc} " + f"second={second_result.returncode}/{second_value!r} " + f"inspect={identical_inspect.returncode}" + ), + ) + + conflict_home = new_home(root, "different-concurrency-home") + conflict_one = source_directory( + root, + "conflict-source-one", + command="winner-one", + ) + conflict_two = source_directory( + root, + "conflict-source-two", + command="winner-two", + ) + one_pid, one_release, one_pause = start_paused_publication( + conflict_home, + conflict_one, + ) + two = start_cli(conflict_home, conflict_two) + conflict_waited = wait_for_lock_block(two) + if one_release >= 0: + try: + os.write(one_release, b"1") + finally: + os.close(one_release) + one_rc = finish_child(one_pid) + two_result = finish_process(two) + conflict_inspect = cli( + conflict_home, + "experiment", + "inspect", + "first-experiment", + ) + conflict_artifact = ( + conflict_home + / "experiments" + / "first-experiment" + / "artifact" + / "experiment.cue" + ) + check( + "IST-CONC-002", + one_pause == 0 + and conflict_waited + and one_rc == 0 + and two_result.returncode == 1 + and conflict_inspect.returncode == 0 + and conflict_artifact.is_file() + and conflict_artifact.read_bytes() + == (conflict_one / "experiment.cue").read_bytes() + and not tuple((conflict_home / "experiments" / ".staging").iterdir()), + "overlapping different candidates serialize to the paused winner and one conflict", + ( + f"pause={one_pause} waited={conflict_waited} first={one_rc} " + f"second={two_result.returncode} inspect={conflict_inspect.returncode}" + ), + ) + + seam_home = new_home(root, "fault-seam-home") + observed_points: list[str] = [] + seam_rc, seam_value, seam_error = store_install( + seam_home, + direct_source, + fault=observed_points.append, + ) + crash_failures: list[str] = [] + for index, point in enumerate( + ( + "experiment artifact.after_write", + "experiment receipt.after_fsync", + "experiment envelope.after_noreplace", + "experiment store root.after_fsync", + ) + ): + crash_home = new_home(root, f"crash-home-{index}") + child_rc = hard_exit_install(crash_home, direct_source, point) + before_inspect_stage = fingerprint(crash_home / "experiments" / ".staging") + inspected = cli(crash_home, "experiment", "inspect", "first-experiment") + after_inspect_stage = fingerprint(crash_home / "experiments" / ".staging") + retried = cli(crash_home, "experiment", "install", str(direct_source)) + retry_value = json_object(retried) + committed_point = point in { + "experiment envelope.after_noreplace", + "experiment store root.after_fsync", + } + expected_changed = not committed_point + if not ( + child_rc == 99 + and inspected.returncode == (0 if committed_point else 1) + and before_inspect_stage == after_inspect_stage + and retried.returncode == 0 + and isinstance(retry_value, dict) + and retry_value.get("changed") is expected_changed + and not tuple((crash_home / "experiments" / ".staging").iterdir()) + ): + crash_failures.append( + f"{point}:child={child_rc}:inspect={inspected.returncode}:" + f"retry={retried.returncode}/{retry_value!r}:" + f"read_changed={before_inspect_stage != after_inspect_stage}" + ) + + preintent_home = new_home(root, "preintent-crash-home") + preintent_stage = preintent_home / "experiments" / ".staging" + preintent_child_rc = hard_exit_before_intent(preintent_home, direct_source) + preintent_before = fingerprint(preintent_stage) + preintent_inspect = cli( + preintent_home, + "experiment", + "inspect", + "first-experiment", + ) + preintent_after_inspect = fingerprint(preintent_stage) + preintent_retry = cli( + preintent_home, + "experiment", + "install", + str(direct_source), + ) + preintent_value = json_object(preintent_retry) + + raw_home = new_home(root, "raw-publication-crash-home") + raw_child_rc = hard_exit_after_raw_publication(raw_home, direct_source) + raw_stage = raw_home / "experiments" / ".staging" + raw_before_inspect = fingerprint(raw_stage) + raw_inspect = cli( + raw_home, + "experiment", + "inspect", + "first-experiment", + ) + raw_after_inspect = fingerprint(raw_stage) + raw_retry = cli(raw_home, "experiment", "install", str(direct_source)) + raw_retry_value = json_object(raw_retry) + raw_final_inspect = cli( + raw_home, + "experiment", + "inspect", + "first-experiment", + ) + + output_home = new_home(root, "result-output-home") + output_rc, _, output_error = module_main( + output_home, + ["experiment", "install", str(direct_source)], + BrokenOutput(), + ) + output_inspect = cli(output_home, "experiment", "inspect", "first-experiment") + output_retry = cli(output_home, "experiment", "install", str(direct_source)) + output_value = json_object(output_retry) + cleanup_crash_failures: list[str] = [] + for phase in ("handoff", "remove"): + cleanup_crash_home = new_home(root, f"cleanup-{phase}-crash-home") + cleanup_child_rc = hard_exit_during_cleanup( + cleanup_crash_home, + direct_source, + phase, + ) + cleanup_stage = cleanup_crash_home / "experiments" / ".staging" + cleanup_before_inspect = fingerprint(cleanup_stage) + cleanup_inspect = cli( + cleanup_crash_home, + "experiment", + "inspect", + "first-experiment", + ) + cleanup_after_inspect = fingerprint(cleanup_stage) + cleanup_retry = cli( + cleanup_crash_home, + "experiment", + "install", + str(direct_source), + ) + cleanup_retry_value = json_object(cleanup_retry) + if not ( + cleanup_child_rc == 99 + and cleanup_inspect.returncode == 0 + and cleanup_before_inspect == cleanup_after_inspect + and cleanup_retry.returncode == 0 + and isinstance(cleanup_retry_value, dict) + and cleanup_retry_value.get("changed") is False + and not tuple(cleanup_stage.iterdir()) + ): + cleanup_crash_failures.append( + f"{phase}:child={cleanup_child_rc}:inspect={cleanup_inspect.returncode}:" + f"read_changed={cleanup_before_inspect != cleanup_after_inspect}:" + f"retry={cleanup_retry.returncode}/{cleanup_retry_value!r}" + ) + check( + "IST-CRASH-001", + seam_rc == 0 + and isinstance(seam_value, dict) + and seam_error is None + and set(FAULT_POINTS) <= set(observed_points) + and not crash_failures + and output_rc == 125 + and output_error is None + and output_inspect.returncode == 0 + and output_retry.returncode == 0 + and isinstance(output_value, dict) + and output_value.get("changed") is False + and not cleanup_crash_failures + and preintent_child_rc == 99 + and preintent_inspect.returncode == 1 + and preintent_before == preintent_after_inspect + and preintent_retry.returncode == 0 + and isinstance(preintent_value, dict) + and preintent_value.get("changed") is True + and not tuple(preintent_stage.iterdir()) + and raw_child_rc == 99 + and raw_inspect.returncode == 125 + and raw_before_inspect == raw_after_inspect + and raw_retry.returncode == 0 + and isinstance(raw_retry_value, dict) + and raw_retry_value.get("changed") is False + and raw_final_inspect.returncode == 0 + and not tuple(raw_stage.iterdir()), + "fault seams preserve views, restart cleanup, and recover uncertain output", + ( + f"seam={seam_rc}/{seam_error!r} " + f"missing={sorted(set(FAULT_POINTS)-set(observed_points))!r} " + f"crashes={crash_failures[:3]!r} output={output_rc}/{output_error!r}/" + f"{output_inspect.returncode}/{output_retry.returncode}/{output_value!r} " + f"cleanup={cleanup_crash_failures[:2]!r} " + f"preintent={preintent_child_rc}/{preintent_inspect.returncode}/" + f"{preintent_before != preintent_after_inspect}/{preintent_retry.returncode}/" + f"{preintent_value!r} " + f"raw={raw_child_rc}/{raw_inspect.returncode}/" + f"{raw_before_inspect != raw_after_inspect}/{raw_retry.returncode}/" + f"{raw_retry_value!r}/{raw_final_inspect.returncode}" + ), + ) + + live_home = new_home(root, "selected-entry-live-home") + added = cli(live_home, "image", "add", "vendor.worker", SUBJECT) + added_value = json_object(added) + local_source = source_directory( + root, + "local-source", + catalog_name="vendor.worker", + ) + live_events: list[str] = [] + held_catalog = held_store = False + + def observe_locks(point: str) -> None: + nonlocal held_catalog, held_store + live_events.append(point) + if point == "experiment envelope.before_noreplace": + held_catalog = lock_is_blocked( + live_home / "state" / "locks" / "image-catalog.lock" + ) + held_store = lock_is_blocked( + live_home / "state" / "locks" / "experiments.lock" + ) + + live_rc, live_value, live_error = store_install( + live_home, + local_source, + fault=observe_locks, + ) + entry_digest = added_value.get("entryDigest") if isinstance(added_value, dict) else None + if isinstance(entry_digest, str): + removed = cli( + live_home, + "image", + "remove", + "vendor.worker", + "--expect", + entry_digest, + ) + else: + removed = subprocess.CompletedProcess([], 125, b"", b"missing entry digest") + store_before_retry = fingerprint(live_home / "experiments") + stale_retry = cli(live_home, "experiment", "install", str(local_source)) + store_after_retry = fingerprint(live_home / "experiments") + retained = cli(live_home, "experiment", "inspect", "first-experiment") + try: + catalog_index = live_events.index("experiment catalog lock.after_acquire") + store_index = live_events.index("experiment store lock.after_acquire") + publish_index = live_events.index("experiment envelope.before_noreplace") + order_ok = catalog_index < store_index < publish_index + except ValueError: + order_ok = False + + live_race_home = new_home(root, "selected-entry-removal-race-home") + race_added = cli( + live_race_home, + "image", + "add", + "vendor.worker", + SUBJECT, + ) + race_added_value = json_object(race_added) + race_entry_digest = ( + race_added_value.get("entryDigest") + if isinstance(race_added_value, dict) + else None + ) + race_install_pid = 0 + race_release = -1 + race_pause = -1 + race_remove: subprocess.Popen[bytes] | None = None + race_remove_waited = False + race_install_rc = 124 + race_remove_result = subprocess.CompletedProcess([], 125, b"", b"missing entry digest") + if isinstance(race_entry_digest, str): + race_install_pid, race_release, race_pause = start_paused_publication( + live_race_home, + local_source, + ) + race_remove = start_remove( + live_race_home, + "vendor.worker", + race_entry_digest, + ) + race_remove_waited = wait_for_lock_block(race_remove) + if race_release >= 0: + try: + os.write(race_release, b"1") + finally: + os.close(race_release) + if race_install_pid > 0: + race_install_rc = finish_child(race_install_pid) + if race_remove is not None: + race_remove_result = finish_process(race_remove) + race_retained = cli( + live_race_home, + "experiment", + "inspect", + "first-experiment", + ) + race_before_retry = fingerprint(live_race_home / "experiments") + race_stale_retry = cli( + live_race_home, + "experiment", + "install", + str(local_source), + ) + race_after_retry = fingerprint(live_race_home / "experiments") + check( + "IST-LIVE-001", + added.returncode == 0 + and live_rc == 0 + and isinstance(live_value, dict) + and live_error is None + and order_ok + and held_catalog + and held_store + and removed.returncode == 0 + and stale_retry.returncode == 1 + and store_before_retry == store_after_retry + and retained.returncode == 0 + and race_added.returncode == 0 + and race_pause == 0 + and race_remove_waited + and race_install_rc == 0 + and race_remove_result.returncode == 0 + and race_retained.returncode == 0 + and race_stale_retry.returncode == 1 + and race_before_retry == race_after_retry, + "selected-entry locks order correctly and serialize a real removal race", + ( + f"add={added.returncode} install={live_rc}/{live_error!r} events={live_events!r} " + f"held={held_catalog}/{held_store} remove={removed.returncode} " + f"retry={stale_retry.returncode}/{store_before_retry != store_after_retry} " + f"inspect={retained.returncode} race={race_added.returncode}/{race_pause}/" + f"{race_remove_waited}/{race_install_rc}/{race_remove_result.returncode}/" + f"{race_retained.returncode}/{race_stale_retry.returncode}/" + f"{race_before_retry != race_after_retry}" + ), + ) + + provenance_home = new_home(root, "provenance-snapshot-home") + provenance_add = cli( + provenance_home, + "image", + "add", + "vendor.worker", + SUBJECT, + ) + provenance_source = source_directory( + root, + "provenance-source", + catalog_name="vendor.worker", + ) + initial_check = cli( + provenance_home, + "experiment", + "check", + str(provenance_source), + ) + initial_check_value = json_object(initial_check) + initial_catalog = None + if isinstance(initial_check_value, dict): + catalog_value = initial_check_value.get("catalog") + if isinstance(catalog_value, dict): + initial_catalog = catalog_value + provenance_rc: int | None = None + provenance_value: dict[str, object] | None = None + provenance_error: BaseException | None = None + unrelated_result: subprocess.CompletedProcess[bytes] | None = None + provenance_record: dict[str, object] | None = None + if STORE is not None: + original_held_catalog = STORE._held_catalog_context + + @contextmanager + def mutate_unrelated_before_hold(home: Path, dependencies, fault): + nonlocal unrelated_result + unrelated_result = cli( + home, + "image", + "add", + "vendor.unrelated", + OTHER_SUBJECT, + ) + with original_held_catalog(home, dependencies, fault) as held: + yield held + + STORE._held_catalog_context = mutate_unrelated_before_hold + try: + provenance_rc, provenance_value, provenance_error = store_install( + provenance_home, + provenance_source, + ) + finally: + STORE._held_catalog_context = original_held_catalog + provenance_path = ( + provenance_home + / "experiments" + / "first-experiment" + / "records" + / "provenance.json" + ) + if provenance_path.is_file(): + try: + loaded_provenance = json.loads(provenance_path.read_bytes()) + except (OSError, UnicodeError, json.JSONDecodeError): + loaded_provenance = None + if isinstance(loaded_provenance, dict): + provenance_record = loaded_provenance + platform_home = new_home(root, "platform-home") + platform_source = source_directory(root, "platform-source") + platform_before = fingerprint(platform_home) + source_touched = False + original_platform = MODULE.sys.platform + original_lstat = MODULE.os.lstat + original_listdir = MODULE.os.listdir + original_open = MODULE.os.open + + def touches_source(value: object) -> bool: + try: + rendered = os.fsdecode(os.fspath(value)) + except TypeError: + return False + base = str(platform_source) + return rendered == base or rendered.startswith(base + os.sep) + + def observed_lstat(path: object, *args, **kwargs): + nonlocal source_touched + source_touched = source_touched or touches_source(path) + return original_lstat(path, *args, **kwargs) + + def observed_listdir(path: object = "."): + nonlocal source_touched + source_touched = source_touched or touches_source(path) + return original_listdir(path) + + def observed_open(path: object, flags: int, mode: int = 0o777, *, dir_fd=None): + nonlocal source_touched + source_touched = source_touched or touches_source(path) + return original_open(path, flags, mode, dir_fd=dir_fd) + + MODULE.sys.platform = "darwin" + MODULE.os.lstat = observed_lstat + MODULE.os.listdir = observed_listdir + MODULE.os.open = observed_open + try: + platform_rc, _, platform_error = module_main( + platform_home, + ["experiment", "install", str(platform_source)], + ) + finally: + MODULE.os.open = original_open + MODULE.os.listdir = original_listdir + MODULE.os.lstat = original_lstat + MODULE.sys.platform = original_platform + platform_after = fingerprint(platform_home) + check( + "IST-PLAT-001", + platform_rc == 125 + and platform_error is None + and not source_touched + and platform_before == platform_after, + "non-Linux install fails before caller-source access or persistent effect", + ( + f"rc={platform_rc} error={platform_error!r} " + f"source_touched={source_touched} changed={platform_before != platform_after}" + ), + ) + check( + "IST-PROV-001", + provenance_add.returncode == 0 + and initial_check.returncode == 0 + and isinstance(initial_catalog, dict) + and unrelated_result is not None + and unrelated_result.returncode == 0 + and provenance_rc == 0 + and isinstance(provenance_value, dict) + and provenance_error is None + and isinstance(provenance_record, dict) + and provenance_record.get("catalog") == initial_catalog, + "provenance retains the authorized initial resolution snapshot across unrelated catalog mutation", + ( + f"add={provenance_add.returncode} check={initial_check.returncode}/{initial_catalog!r} " + f"unrelated={None if unrelated_result is None else unrelated_result.returncode} " + f"install={provenance_rc}/{provenance_error!r} " + f"stored={None if provenance_record is None else provenance_record.get('catalog')!r}" + ), + ) + + planning = load_module( + REPO_ROOT / "scripts" / "experiment.py", + "agent_lab_experiment_store_adversarial", + ) + + cue_home = new_home(root, "preflight-cue-home") + cue_source = source_directory(root, "preflight-cue-source") + cue_path = cue_source / "experiment.cue" + cue_raw = cue_path.read_bytes() + cue_path.write_bytes( + cue_raw.replace( + b'\t\tcommand: ["serve"]\n', + b'\t\tcommand: ["serve"]\n\t\tunknownField: true\n', + 1, + ) + ) + cue_before = fingerprint(cue_home / "experiments") + cue_failure = cli(cue_home, "experiment", "install", str(cue_source)) + cue_after = fingerprint(cue_home / "experiments") + + contract_home = new_home(root, "preflight-contract-home") + contract_source = source_directory(root, "preflight-contract-source") + contract_before = fingerprint(contract_home / "experiments") + original_experiment_loader = STORE._experiment_module if STORE is not None else None + original_verify_contract = planning.verify_contract_snapshot + + def report_contract_drift(repo_root: Path, expected: dict[str, bytes]) -> None: + del repo_root, expected + raise planning.InfrastructureError("contract snapshot changed during validation") + + contract_rc: int | None = None + contract_error: BaseException | None = None + if STORE is not None: + STORE._experiment_module = lambda: planning + planning.verify_contract_snapshot = report_contract_drift + try: + contract_rc, _, contract_error = store_install( + contract_home, + contract_source, + ) + finally: + planning.verify_contract_snapshot = original_verify_contract + STORE._experiment_module = original_experiment_loader + contract_after = fingerprint(contract_home / "experiments") + + selected_home = new_home(root, "preflight-selected-home") + selected_add = cli( + selected_home, + "image", + "add", + "vendor.worker", + SUBJECT, + ) + selected_add_value = json_object(selected_add) + selected_digest = ( + selected_add_value.get("entryDigest") + if isinstance(selected_add_value, dict) + else None + ) + selected_source = source_directory( + root, + "preflight-selected-source", + catalog_name="vendor.worker", + ) + selected_before = fingerprint(selected_home / "experiments") + selected_remove: subprocess.CompletedProcess[bytes] | None = None + selected_rc: int | None = None + selected_error: BaseException | None = None + if STORE is not None and isinstance(selected_digest, str): + original_held_context = STORE._held_catalog_context + + @contextmanager + def remove_selected_before_hold(home: Path, dependencies, fault): + nonlocal selected_remove + selected_remove = cli( + home, + "image", + "remove", + "vendor.worker", + "--expect", + selected_digest, + ) + with original_held_context(home, dependencies, fault) as held: + yield held + + STORE._held_catalog_context = remove_selected_before_hold + try: + selected_rc, _, selected_error = store_install( + selected_home, + selected_source, + ) + finally: + STORE._held_catalog_context = original_held_context + selected_after = fingerprint(selected_home / "experiments") + check( + "IST-PREFLIGHT-001", + cue_raw != cue_path.read_bytes() + and cue_failure.returncode == 1 + and not cue_failure.stdout + and cue_before == cue_after + and contract_rc == 125 + and contract_error is not None + and contract_before == contract_after + and selected_add.returncode == 0 + and selected_remove is not None + and selected_remove.returncode == 0 + and selected_rc == 1 + and selected_error is not None + and selected_before == selected_after, + "CUE, contract, and selected-entry drift fail before store-visible effects", + ( + f"cue={cue_failure.returncode}/{cue_before != cue_after} " + f"contract={contract_rc}/{contract_error!r}/{contract_before != contract_after} " + f"selected={selected_add.returncode}/" + f"{None if selected_remove is None else selected_remove.returncode}/" + f"{selected_rc}/{selected_error!r}/{selected_before != selected_after}" + ), + ) + + snapshot_mutation_home = new_home(root, "snapshot-after-mutation-home") + snapshot_mutation_source = source_directory( + root, + "snapshot-after-mutation-source", + requested_name="snapshot-mutated", + ) + snapshot_mutation_path = snapshot_mutation_source / "experiment.cue" + snapshot_original = snapshot_mutation_path.read_bytes() + snapshot_changed = snapshot_original.replace(b"serve", b"mutat", 1) + original_authored_manifest = planning.authored_manifest + mutation_after_snapshot = False + + def mutate_after_snapshot(snapshot): + nonlocal mutation_after_snapshot + snapshot_mutation_path.write_bytes(snapshot_changed) + mutation_after_snapshot = True + return original_authored_manifest(snapshot) + + snapshot_mutation_rc: int | None = None + snapshot_mutation_error: BaseException | None = None + if STORE is not None: + STORE._experiment_module = lambda: planning + planning.authored_manifest = mutate_after_snapshot + try: + snapshot_mutation_rc, _, snapshot_mutation_error = store_install( + snapshot_mutation_home, + snapshot_mutation_source, + ) + finally: + planning.authored_manifest = original_authored_manifest + STORE._experiment_module = original_experiment_loader + stored_mutation_artifact = ( + snapshot_mutation_home + / "experiments" + / "snapshot-mutated" + / "artifact" + / "experiment.cue" + ) + + snapshot_deletion_home = new_home(root, "snapshot-after-deletion-home") + snapshot_deletion_source = source_directory( + root, + "snapshot-after-deletion-source", + requested_name="snapshot-deleted", + ) + snapshot_deletion_path = snapshot_deletion_source / "experiment.cue" + deletion_original = snapshot_deletion_path.read_bytes() + deletion_after_snapshot = False + + def delete_after_snapshot(snapshot): + nonlocal deletion_after_snapshot + snapshot_deletion_path.unlink() + deletion_after_snapshot = True + return original_authored_manifest(snapshot) + + snapshot_deletion_rc: int | None = None + snapshot_deletion_error: BaseException | None = None + if STORE is not None: + STORE._experiment_module = lambda: planning + planning.authored_manifest = delete_after_snapshot + try: + snapshot_deletion_rc, _, snapshot_deletion_error = store_install( + snapshot_deletion_home, + snapshot_deletion_source, + ) + finally: + planning.authored_manifest = original_authored_manifest + STORE._experiment_module = original_experiment_loader + stored_deletion_artifact = ( + snapshot_deletion_home + / "experiments" + / "snapshot-deleted" + / "artifact" + / "experiment.cue" + ) + + snapshot_race_home = new_home(root, "snapshot-during-read-home") + snapshot_race_source = source_directory( + root, + "snapshot-during-read-source", + requested_name="snapshot-raced", + ) + snapshot_race_path = snapshot_race_source / "experiment.cue" + race_original = snapshot_race_path.read_bytes() + race_changed = race_original.replace(b"serve", b"mutat", 1) + race_metadata = snapshot_race_path.stat() + snapshot_race_before = fingerprint(snapshot_race_home / "experiments") + original_planning_read = planning.os.read + race_mutated_during_read = False + + def mutate_during_read(descriptor: int, maximum: int) -> bytes: + nonlocal race_mutated_during_read + data = original_planning_read(descriptor, maximum) + try: + target = os.readlink(f"/proc/self/fd/{descriptor}") + except OSError: + target = "" + if ( + data + and not race_mutated_during_read + and target == str(snapshot_race_path) + ): + time.sleep(0.001) + snapshot_race_path.write_bytes(race_changed) + os.utime( + snapshot_race_path, + ns=(race_metadata.st_atime_ns, race_metadata.st_mtime_ns), + ) + race_mutated_during_read = True + return data + + snapshot_race_rc: int | None = None + snapshot_race_error: BaseException | None = None + if STORE is not None: + STORE._experiment_module = lambda: planning + planning.os.read = mutate_during_read + try: + snapshot_race_rc, _, snapshot_race_error = store_install( + snapshot_race_home, + snapshot_race_source, + ) + finally: + planning.os.read = original_planning_read + STORE._experiment_module = original_experiment_loader + snapshot_race_after = fingerprint(snapshot_race_home / "experiments") + race_final_metadata = snapshot_race_path.stat() + check( + "IST-SNAPSHOT-001", + snapshot_original != snapshot_changed + and mutation_after_snapshot + and snapshot_mutation_rc == 0 + and snapshot_mutation_error is None + and stored_mutation_artifact.is_file() + and stored_mutation_artifact.read_bytes() == snapshot_original + and deletion_after_snapshot + and snapshot_deletion_rc == 0 + and snapshot_deletion_error is None + and not snapshot_deletion_path.exists() + and stored_deletion_artifact.is_file() + and stored_deletion_artifact.read_bytes() == deletion_original + and race_original != race_changed + and race_mutated_during_read + and race_final_metadata.st_mtime_ns == race_metadata.st_mtime_ns + and race_final_metadata.st_ctime_ns != race_metadata.st_ctime_ns + and snapshot_race_rc == 125 + and snapshot_race_error is not None + and snapshot_race_before == snapshot_race_after, + "post-snapshot source drift cannot change bytes and during-read drift is uncertain", + ( + f"mutation={mutation_after_snapshot}/{snapshot_mutation_rc}/" + f"{snapshot_mutation_error!r}/" + f"{stored_mutation_artifact.is_file() and stored_mutation_artifact.read_bytes() == snapshot_original} " + f"deletion={deletion_after_snapshot}/{snapshot_deletion_rc}/" + f"{snapshot_deletion_error!r}/" + f"{stored_deletion_artifact.is_file() and stored_deletion_artifact.read_bytes() == deletion_original} " + f"during={race_mutated_during_read}/{snapshot_race_rc}/" + f"{snapshot_race_error!r}/{snapshot_race_before != snapshot_race_after}/" + f"mtime={race_final_metadata.st_mtime_ns == race_metadata.st_mtime_ns}/" + f"ctime={race_final_metadata.st_ctime_ns != race_metadata.st_ctime_ns}" + ), + ) + + config_home = new_home(root, "changing-config-home") + config_source = source_directory(root, "changing-config-source") + config_path = config_home / "config.json" + config_raw = config_path.read_bytes() + config_stage = config_home / "experiments" / ".staging" + config_target = config_home / "experiments" / "first-experiment" + config_changed = False + config_rc: int | None = None + config_error: BaseException | None = None + if STORE is not None: + original_prepare_stage = STORE._prepare_stage + + def change_config_after_staging(*args, **kwargs): + nonlocal config_changed + prepared = original_prepare_stage(*args, **kwargs) + config_path.write_bytes(config_raw + b" ") + config_changed = True + return prepared + + STORE._prepare_stage = change_config_after_staging + try: + config_rc, _, config_error = store_install(config_home, config_source) + finally: + STORE._prepare_stage = original_prepare_stage + config_final_absent = not config_target.exists() and not config_target.is_symlink() + config_wrapper_count = len(tuple(config_stage.iterdir())) + config_path.write_bytes(config_raw) + config_before_inspect = fingerprint(config_stage) + config_inspect_before = cli( + config_home, + "experiment", + "inspect", + "first-experiment", + ) + config_after_inspect = fingerprint(config_stage) + config_retry = cli( + config_home, + "experiment", + "install", + str(config_source), + ) + config_retry_value = json_object(config_retry) + config_inspect_after = cli( + config_home, + "experiment", + "inspect", + "first-experiment", + ) + check( + "IST-CONFIG-001", + config_changed + and config_rc == 125 + and config_error is not None + and config_final_absent + and config_wrapper_count == 1 + and config_inspect_before.returncode == 1 + and config_before_inspect == config_after_inspect + and config_retry.returncode == 0 + and isinstance(config_retry_value, dict) + and config_retry_value.get("changed") is True + and config_inspect_after.returncode == 0 + and not tuple(config_stage.iterdir()), + "configuration drift before publication is uncertain and its owned residue is restartable", + ( + f"changed={config_changed} install={config_rc}/{config_error!r} " + f"final_absent={config_final_absent} wrappers={config_wrapper_count} " + f"inspect={config_inspect_before.returncode}/" + f"{config_before_inspect != config_after_inspect} " + f"retry={config_retry.returncode}/{config_retry_value!r}/" + f"{config_inspect_after.returncode}" + ), + ) + + reader_home = new_home(root, "reader-tamper-home") + reader_install = cli( + reader_home, + "experiment", + "install", + str(direct_source), + ) + reader_envelope = reader_home / "experiments" / "first-experiment" + reader_failures: list[str] = [] + reader_paths = ( + "artifact/experiment.cue", + "records/decision.json", + "records/plan.json", + "records/provenance.json", + "records/install.json", + ) + if reader_install.returncode == 0: + for index, relative in enumerate(reader_paths): + path = reader_envelope / relative + raw = path.read_bytes() + link = root / f"reader-hardlink-{index}" + os.link(path, link) + linked_rc, _, linked_error = store_inspect( + reader_home, + "first-experiment", + ) + link.unlink() + restored_link_rc, _, restored_link_error = store_inspect( + reader_home, + "first-experiment", + ) + path.chmod(0o600) + path.write_bytes(bytes((raw[0] ^ 1,)) + raw[1:]) + path.chmod(0o400) + corrupt_before_retry = fingerprint(reader_envelope) + corrupt_rc, _, corrupt_error = store_inspect( + reader_home, + "first-experiment", + ) + retry_rc: int | None = 125 + retry_error: BaseException | None = RuntimeError("not receipt") + retry_unchanged = True + if relative == "records/install.json": + retry_rc, _, retry_error = store_install(reader_home, direct_source) + retry_unchanged = corrupt_before_retry == fingerprint(reader_envelope) + path.chmod(0o600) + path.write_bytes(raw) + path.chmod(0o400) + restored_rc, _, restored_error = store_inspect( + reader_home, + "first-experiment", + ) + if not ( + linked_rc == 125 + and linked_error is not None + and restored_link_rc == 0 + and restored_link_error is None + and corrupt_rc == 125 + and corrupt_error is not None + and ( + relative != "records/install.json" + or (retry_rc == 125 and retry_error is not None and retry_unchanged) + ) + and restored_rc == 0 + and restored_error is None + ): + reader_failures.append( + f"{relative}:link={linked_rc}/{restored_link_rc}:" + f"corrupt={corrupt_rc}:retry={retry_rc}/{retry_unchanged}:" + f"restore={restored_rc}" + ) + check( + "IST-READ-001", + reader_install.returncode == 0 and not reader_failures, + "every stored byte record is safe-opened, digest-bound, and non-overwritable", + f"install={reader_install.returncode} failures={reader_failures[:5]!r}", + ) + + fault_source = source_directory( + root, + "fault-matrix-source", + requested_name="fault-experiment", + ) + fault_data = (fault_source / "experiment.cue").read_bytes() + fault_fixture = FaultFixtureExperiment(fault_data, "fault-experiment") + fault_failures: list[str] = [] + fault_counts: dict[str, int] = {} + primitives = ( + "open", + "write", + "chmod", + "fsync", + "link_unnamed", + "rename_noreplace", + ) + if STORE is not None: + original_fault_loader = STORE._experiment_module + STORE._experiment_module = lambda: fault_fixture + try: + for primitive in primitives: + baseline_home = new_home(root, f"fault-{primitive}-baseline-home") + if primitive in {"link_unnamed", "rename_noreplace"}: + owner = STORE + attribute = f"_{primitive}" + else: + owner = STORE.os + attribute = primitive + original_primitive = getattr(owner, attribute) + observed_calls = 0 + + def observe_primitive(*args, _original=original_primitive, **kwargs): + nonlocal observed_calls + observed_calls += 1 + return _original(*args, **kwargs) + + setattr(owner, attribute, observe_primitive) + try: + baseline_rc, baseline_value, baseline_error = store_install( + baseline_home, + fault_source, + ) + finally: + setattr(owner, attribute, original_primitive) + fault_counts[primitive] = observed_calls + if not ( + baseline_rc == 0 + and isinstance(baseline_value, dict) + and baseline_error is None + and observed_calls > 0 + ): + fault_failures.append( + f"{primitive}:baseline={baseline_rc}/{baseline_error!r}/" + f"calls={observed_calls}" + ) + continue + for ordinal in range(1, observed_calls + 1): + injected_home = new_home( + root, + f"fault-{primitive}-{ordinal:02d}-home", + ) + injected_calls = 0 + injected_hit = False + + def inject_primitive(*args, _original=original_primitive, **kwargs): + nonlocal injected_calls, injected_hit + injected_calls += 1 + if injected_calls == ordinal: + injected_hit = True + raise OSError(errno.EIO, f"injected {primitive} failure") + return _original(*args, **kwargs) + + setattr(owner, attribute, inject_primitive) + try: + injected_rc, injected_value, injected_error = store_install( + injected_home, + fault_source, + ) + finally: + setattr(owner, attribute, original_primitive) + before_retry_rc, _, _ = store_inspect( + injected_home, + "fault-experiment", + ) + retry_rc, retry_value, retry_error = store_install( + injected_home, + fault_source, + ) + final_rc, _, final_error = store_inspect( + injected_home, + "fault-experiment", + ) + staging_empty = not tuple( + (injected_home / "experiments" / ".staging").iterdir() + ) + if not ( + injected_hit + and injected_rc == 125 + and injected_value is None + and injected_error is not None + and before_retry_rc in (0, 1, 125) + and retry_rc == 0 + and isinstance(retry_value, dict) + and retry_error is None + and final_rc == 0 + and final_error is None + and staging_empty + ): + fault_failures.append( + f"{primitive}[{ordinal}/{observed_calls}]:" + f"hit={injected_hit}:first={injected_rc}/{injected_error!r}:" + f"before={before_retry_rc}:retry={retry_rc}/{retry_error!r}:" + f"final={final_rc}/{final_error!r}:empty={staging_empty}" + ) + finally: + STORE._experiment_module = original_fault_loader + check( + "IST-FAULT-001", + STORE is not None + and set(fault_counts) == set(primitives) + and all(count > 0 for count in fault_counts.values()) + and not fault_failures, + "every store open/write/chmod/fsync/rename failure is uncertain and restartable", + f"counts={fault_counts!r} failures={fault_failures[:8]!r}", + ) + + expected = [ + "IST-STATE-001", + "IST-LOCK-001", + "IST-STATE-002", + "IST-BOUND-001", + "IST-STATE-003", + "IST-CONC-001", + "IST-CONC-002", + "IST-CRASH-001", + "IST-LIVE-001", + "IST-PLAT-001", + "IST-PROV-001", + "IST-PREFLIGHT-001", + "IST-SNAPSHOT-001", + "IST-CONFIG-001", + "IST-READ-001", + "IST-FAULT-001", + ] + if OBSERVED != expected: + print(f"INFRA install state assertion identity drift: {OBSERVED!r}", file=sys.stderr) + return 125 + print(f"SUMMARY assertions=16 expected=16 failures={FAILURES} infra={INFRA}") + if INFRA: + return 125 + return 0 if FAILURES == 0 else 1 + + +if __name__ == "__main__": + try: + raise SystemExit(main()) + except SystemExit: + raise + except BaseException as error: + print(f"INFRA install state harness failed: {error!r}", file=sys.stderr) + print( + f"SUMMARY assertions={len(OBSERVED)} expected=16 failures={FAILURES} infra=1" + ) + raise SystemExit(125) diff --git a/tests/experiment/install-store-cases.sh b/tests/experiment/install-store-cases.sh new file mode 100755 index 0000000..b5dc93a --- /dev/null +++ b/tests/experiment/install-store-cases.sh @@ -0,0 +1,748 @@ +#!/usr/bin/env bash +set -u -o pipefail + +repo_root="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." >/dev/null 2>&1 && pwd)" +agent_lab="$repo_root/scripts/agent-lab" +bounded_helper="$repo_root/tests/helpers/run-bounded.py" +fixture="$repo_root/tests/experiment/fixtures/directories/minimal" +runtime_manifest="$repo_root/packaging/agent-lab-local.manifest" +expected_count=13 +work="" +failures=0 +infrastructure=0 + +cleanup_work() { + local failed=0 + if [ -n "$work" ] && [ -e "$work" ]; then + find "$work" -type f -exec chmod u+rw {} + 2>/dev/null || failed=1 + find "$work" -depth -type d -exec chmod u+rwx {} + 2>/dev/null || failed=1 + find "$work" -type f -delete 2>/dev/null || failed=1 + find "$work" -type l -delete 2>/dev/null || failed=1 + find "$work" -depth -type d -exec rmdir {} + 2>/dev/null || failed=1 + [ ! -e "$work" ] || failed=1 + fi + return "$failed" +} + +finish() { + local assertions=0 + if [ -n "${observed:-}" ] && [ -f "$observed" ]; then + assertions="$(wc -l < "$observed")" + fi + if ! cleanup_work; then + infrastructure=1 + fi + trap - EXIT + printf 'SUMMARY assertions=%s expected=%s failures=%s infra=%s\n' \ + "$assertions" "$expected_count" "$failures" "$infrastructure" + if [ "$infrastructure" -ne 0 ]; then + exit 125 + fi + if [ "$failures" -ne 0 ]; then + exit 1 + fi + printf 'EXPERIMENT INSTALL STORE PASS\n' +} + +if ! work="$(mktemp -d)"; then + printf 'SUMMARY assertions=0 expected=%s failures=0 infra=1\n' "$expected_count" + exit 125 +fi +trap 'cleanup_work >/dev/null 2>&1 || true' EXIT +observed="$work/observed" +: > "$observed" + +if [ ! -x "$agent_lab" ] || [ ! -f "$bounded_helper" ] || [ ! -d "$fixture" ] \ + || [ ! -f "$runtime_manifest" ] || ! command -v jq >/dev/null 2>&1 \ + || ! command -v python3 >/dev/null 2>&1 \ + || [ ! -d "$repo_root/.cache/dev/tools/cue" ] \ + || [ ! -d "$repo_root/.cache/dev/tools/cedar" ]; then + printf 'INFRA install-store prerequisites are unavailable\n' >&2 + infrastructure=1 + finish +fi +if ! python3 -I -B "$bounded_helper" --self-test > "$work/bounded-self-test.out" 2> "$work/bounded-self-test.err"; then + printf 'INFRA bounded command helper self-test failed\n' >&2 + infrastructure=1 + finish +fi + +export AGENT_LAB_CUE_TOOL_DIR="$repo_root/.cache/dev/tools/cue" +export AGENT_LAB_CEDAR_TOOL_DIR="$repo_root/.cache/dev/tools/cedar" + +pass() { printf 'PASS %s %s\n' "$1" "$2"; printf '%s\n' "$1" >> "$observed"; } +fail() { printf 'FAIL %s %s\n' "$1" "$2"; printf '%s\n' "$1" >> "$observed"; failures=$((failures + 1)); } + +capture() { + local label="$1" + local status="$work/$label.status" + local status_line="" + shift + CAPTURE_OUT="$work/$label.out" + CAPTURE_ERR="$work/$label.err" + CAPTURE_RC=0 + find "$status" -delete 2>/dev/null || true + python3 -I -B "$bounded_helper" \ + --timeout 5 --status "$status" --stdout "$CAPTURE_OUT" --stderr "$CAPTURE_ERR" -- "$@" || CAPTURE_RC=$? + if [ -f "$status" ]; then + status_line="$(cat "$status")" + fi + if [ "$status_line" != "child:$CAPTURE_RC" ]; then + printf 'INFRA bounded command status is inconsistent: %s rc=%s status=%s\n' \ + "$label" "$CAPTURE_RC" "$status_line" >&2 + infrastructure=1 + CAPTURE_RC=125 + fi +} + +init_home() { + local home="$1" + capture "init-$(basename -- "$home")" "$agent_lab" --home "$home" init + if [ "$CAPTURE_RC" -ne 0 ]; then + printf 'INFRA temporary Agent Lab home initialization failed: %s\n' \ + "$(tr '\n' ' ' < "$CAPTURE_ERR")" >&2 + infrastructure=1 + finish + fi +} + +state_receipt() { + python3 -I -B - "$1" <<'PY' +from __future__ import annotations + +import hashlib +import json +import os +from pathlib import Path +import stat +import sys + +root = Path(sys.argv[1]) +if not os.path.lexists(root): + print("absent") + raise SystemExit(0) + +records: list[list[object]] = [] + +def visit(path: Path, relative: str) -> None: + metadata = path.lstat() + kind = "other" + content = "" + if stat.S_ISDIR(metadata.st_mode): + kind = "directory" + elif stat.S_ISREG(metadata.st_mode): + kind = "file" + descriptor = os.open(path, os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0)) + try: + chunks: list[bytes] = [] + remaining = metadata.st_size + 1 + while remaining: + chunk = os.read(descriptor, min(65_536, remaining)) + if not chunk: + break + chunks.append(chunk) + remaining -= len(chunk) + content = hashlib.sha256(b"".join(chunks)).hexdigest() + finally: + os.close(descriptor) + elif stat.S_ISLNK(metadata.st_mode): + kind = "symlink" + content = os.readlink(path) + records.append([ + relative, + kind, + stat.S_IMODE(metadata.st_mode), + metadata.st_uid, + metadata.st_gid, + metadata.st_nlink, + metadata.st_dev, + metadata.st_ino, + metadata.st_size, + metadata.st_mtime_ns, + metadata.st_ctime_ns, + content, + ]) + if kind == "directory" and relative != "cache": + for child in sorted(path.iterdir(), key=lambda item: os.fsencode(item.name)): + child_relative = child.name if relative == "." else f"{relative}/{child.name}" + visit(child, child_relative) + +visit(root, ".") +encoded = json.dumps(records, ensure_ascii=True, separators=(",", ":")).encode("ascii") +print("sha256:" + hashlib.sha256(encoded).hexdigest()) +PY +} + +verify_install_envelope() { + python3 -I -B - "$1" "$2" "$3" "$4" "$5" "$6" <<'PY' +from __future__ import annotations + +import hashlib +import json +import os +from pathlib import Path +import stat +import sys + +home = Path(sys.argv[1]) +name = sys.argv[2] +source = Path(sys.argv[3]) +checked_path = Path(sys.argv[4]) +decision_path = Path(sys.argv[5]) +result_path = Path(sys.argv[6]) +target = home / "experiments" / name + +def canonical(value: object) -> bytes: + return json.dumps(value, ensure_ascii=True, separators=(",", ":"), sort_keys=True).encode("ascii") + +def digest(data: bytes) -> str: + return "sha256:" + hashlib.sha256(data).hexdigest() + +def identity_digest(domain: bytes, value: object) -> str: + return digest(domain + canonical(value)) + +def strings(value: object) -> set[str]: + found: set[str] = set() + if isinstance(value, str): + found.add(value) + elif isinstance(value, dict): + for key, item in value.items(): + found.add(key) + found.update(strings(item)) + elif isinstance(value, list): + for item in value: + found.update(strings(item)) + return found + +expected_paths = { + "artifact", + "artifact/experiment.cue", + "records", + "records/decision.json", + "records/install.json", + "records/plan.json", + "records/provenance.json", +} +actual_paths = { + str(path.relative_to(target)) + for path in target.rglob("*") +} +assert actual_paths == expected_paths +for relative in (".", "artifact", "records"): + path = target if relative == "." else target / relative + metadata = path.lstat() + assert stat.S_ISDIR(metadata.st_mode) + assert stat.S_IMODE(metadata.st_mode) == 0o500 + assert metadata.st_uid == os.getuid() +for relative in expected_paths - {"artifact", "records"}: + metadata = (target / relative).lstat() + assert stat.S_ISREG(metadata.st_mode) + assert stat.S_IMODE(metadata.st_mode) == 0o400 + assert metadata.st_uid == os.getuid() and metadata.st_nlink == 1 + +artifact_bytes = (target / "artifact/experiment.cue").read_bytes() +assert artifact_bytes == (source / "experiment.cue").read_bytes() +checked = json.loads(checked_path.read_bytes()) +expected_plan_bytes = canonical(checked["plan"]) + b"\n" +plan_bytes = (target / "records/plan.json").read_bytes() +decision_bytes = (target / "records/decision.json").read_bytes() +provenance_bytes = (target / "records/provenance.json").read_bytes() +receipt_bytes = (target / "records/install.json").read_bytes() +assert plan_bytes == expected_plan_bytes +assert decision_bytes == decision_path.read_bytes() + +plan = json.loads(plan_bytes) +decision = json.loads(decision_bytes) +provenance = json.loads(provenance_bytes) +receipt = json.loads(receipt_bytes) +for raw, value in ((plan_bytes, plan), (decision_bytes, decision), (provenance_bytes, provenance), (receipt_bytes, receipt)): + assert raw == canonical(value) + b"\n" +assert decision["verdict"] == "permit" +plan_digest = identity_digest(b"agent-lab.experiment-plan.v1\0", plan) +assert decision["binding"]["planDigest"] == plan_digest +assert isinstance(provenance, dict) and isinstance(provenance.get("apiVersion"), str) +assert not any(item.startswith("/") for item in strings(provenance)) + +selected_entries: list[dict[str, object]] = [] +for member in plan["spec"]["members"]: + resolved = member["resolvedImage"] + if resolved["origin"] == "direct": + continue + selected_entries.append({ + "entryDigest": resolved["entryDigest"], + "generation": resolved["generation"], + "name": member["requestedSelector"]["catalogName"], + "origin": resolved["origin"], + "subject": resolved["subject"], + }) +selected_entries.sort(key=lambda item: (str(item["origin"]).encode(), str(item["name"]).encode())) + +identity = { + "authorizationDigest": decision["binding"]["authorizationDigest"], + "contractDigest": plan["contract"]["digest"], + "planDigest": decision["binding"]["planDigest"], + "selectedEntries": selected_entries, + "sourceDigest": decision["binding"]["sourceDigest"], +} +assert identity["sourceDigest"] == checked["source"]["digest"] +assert provenance["authorizationDigest"] == identity["authorizationDigest"] +assert provenance["contractDigest"] == identity["contractDigest"] +assert provenance["planDigest"] == identity["planDigest"] +assert provenance["selectedEntries"] == identity["selectedEntries"] +assert provenance["source"]["digest"] == identity["sourceDigest"] + +decision_digest = identity_digest(b"agent-lab.experiment-decision.v1\0", decision) +provenance_digest = identity_digest(b"agent-lab.experiment-provenance.v1\0", provenance) +records = { + "artifact/experiment.cue": { + "digest": digest(artifact_bytes), + "schema": "agent-lab/v0alpha1", + }, + "records/decision.json": { + "digest": decision_digest, + "schema": decision["apiVersion"], + }, + "records/plan.json": { + "digest": plan_digest, + "schema": plan["apiVersion"], + }, + "records/provenance.json": { + "digest": provenance_digest, + "schema": provenance["apiVersion"], + }, +} +installation_key = identity_digest(b"agent-lab.experiment-installation-key.v1\0", identity) +assert receipt == { + "apiVersion": "agent-lab.experiment-install/v0alpha1", + "identity": identity, + "installationKey": installation_key, + "kind": "ExperimentInstallationReceipt", + "name": name, + "records": records, +} +receipt_digest = identity_digest(b"agent-lab.experiment-install-receipt.v1\0", receipt) +assert len({plan_digest, decision_digest, provenance_digest, receipt_digest}) == 4 + +result_bytes = result_path.read_bytes() +result = json.loads(result_bytes) +assert result_bytes == canonical(result) + b"\n" +assert result.get("changed") is True +assert result.get("name") == name +assert result.get("installationKey") == installation_key +assert result.get("receiptDigest") == receipt_digest +PY +} + +missing_home="$work/missing-home" +missing_before="$(state_receipt "$missing_home")" || infrastructure=1 +capture missing-install "$agent_lab" --home "$missing_home" experiment install "$fixture" +missing_after="$(state_receipt "$missing_home")" || infrastructure=1 +if [ "$CAPTURE_RC" -eq 1 ] && [ ! -s "$CAPTURE_OUT" ] \ + && [ "$missing_before" = absent ] && [ "$missing_after" = absent ]; then + pass INST-HOME-001 "install requires an initialized home without creating one" +else + fail INST-HOME-001 "install requires an initialized home without creating one" +fi + +core_home="$work/core-home" +init_home "$core_home" +unknown_before="$(state_receipt "$core_home")" || infrastructure=1 +capture unknown-inspect "$agent_lab" --home "$core_home" experiment inspect missing-experiment +unknown_after="$(state_receipt "$core_home")" || infrastructure=1 +if [ "$CAPTURE_RC" -eq 1 ] && [ ! -s "$CAPTURE_OUT" ] \ + && [ "$unknown_before" = "$unknown_after" ]; then + pass INST-UNKNOWN-001 "inspect reports an unknown name without mutating the initialized home" +else + fail INST-UNKNOWN-001 "inspect reports an unknown name without mutating the initialized home" +fi + +name_home="$work/name-home" +init_home "$name_home" +name_source="$work/name-source" +name_63="aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" +mkdir "$name_source" +sed "s/first-experiment/$name_63/" "$fixture/experiment.cue" > "$name_source/experiment.cue" +capture name-check "$agent_lab" --home "$name_home" experiment check "$name_source" +name_check_rc="$CAPTURE_RC" +name_check_out="$CAPTURE_OUT" +name_check_err="$CAPTURE_ERR" +capture name-decision "$agent_lab" --home "$name_home" experiment authorize install "$name_source" +name_decision_rc="$CAPTURE_RC" +name_decision_out="$CAPTURE_OUT" +name_decision_err="$CAPTURE_ERR" +capture name-install "$agent_lab" --home "$name_home" experiment install "$name_source" +name_install_rc="$CAPTURE_RC" +name_install_out="$CAPTURE_OUT" +name_install_err="$CAPTURE_ERR" +capture name-inspect "$agent_lab" --home "$name_home" experiment inspect "$name_63" +name_inspect_rc="$CAPTURE_RC" +name_inspect_out="$CAPTURE_OUT" +name_inspect_err="$CAPTURE_ERR" +if [ "${#name_63}" -eq 63 ] \ + && [ "$name_check_rc" -eq 0 ] && [ ! -s "$name_check_err" ] \ + && jq -e --arg name "$name_63" '.plan.metadata.requestedName == $name' "$name_check_out" >/dev/null 2>&1 \ + && [ "$name_decision_rc" -eq 0 ] && [ ! -s "$name_decision_err" ] \ + && jq -e --arg name "$name_63" '.verdict == "permit" and .resource.requestedName == $name' "$name_decision_out" >/dev/null 2>&1 \ + && [ "$name_install_rc" -eq 0 ] && [ ! -s "$name_install_err" ] \ + && jq -e --arg name "$name_63" '.changed == true and .name == $name' "$name_install_out" >/dev/null 2>&1 \ + && [ "$name_inspect_rc" -eq 0 ] && [ ! -s "$name_inspect_err" ] \ + && jq -e --arg name "$name_63" '.state == "installed" and .name == $name' "$name_inspect_out" >/dev/null 2>&1; then + pass INST-NAME-001 "the maximum-length valid Experiment name checks, authorizes, installs, and inspects" +else + fail INST-NAME-001 "the maximum-length valid Experiment name checks, authorizes, installs, and inspects" +fi + +capture core-check "$agent_lab" --home "$core_home" experiment check "$fixture" +check_out="$CAPTURE_OUT" +check_rc="$CAPTURE_RC" +capture core-decision "$agent_lab" --home "$core_home" experiment authorize install "$fixture" +decision_out="$CAPTURE_OUT" +decision_rc="$CAPTURE_RC" +if [ "$check_rc" -ne 0 ] || [ "$decision_rc" -ne 0 ]; then + printf 'INFRA prerequisite check/authorization failed for the canonical fixture\n' >&2 + infrastructure=1 + finish +fi +capture core-install "$agent_lab" --home "$core_home" experiment install "$fixture" +core_result="$CAPTURE_OUT" +core_install_rc="$CAPTURE_RC" +if [ "$core_install_rc" -eq 0 ] && [ ! -s "$CAPTURE_ERR" ] \ + && jq -e '.changed == true and .name == "first-experiment" and (.installationKey | startswith("sha256:")) and (.receiptDigest | startswith("sha256:"))' \ + "$core_result" >/dev/null 2>&1; then + pass INST-PERMIT-001 "a fresh permit publishes one named installation and canonical identity result" +else + fail INST-PERMIT-001 "a fresh permit publishes one named installation and canonical identity result" +fi + +verify_install_envelope \ + "$core_home" first-experiment "$fixture" "$check_out" "$decision_out" "$core_result" \ + > "$work/verify-envelope.out" 2> "$work/verify-envelope.err" +verify_rc=$? +if [ "$verify_rc" -eq 0 ]; then + pass INST-RECEIPT-001 "stored exact bytes, modes, schemas, and independent hashes are receipt-bound" +else + fail INST-RECEIPT-001 "stored exact bytes, modes, schemas, and independent hashes are receipt-bound" +fi + +core_before_inspect="$(state_receipt "$core_home")" || infrastructure=1 +capture core-inspect "$agent_lab" --home "$core_home" experiment inspect first-experiment +core_inspect_rc="$CAPTURE_RC" +core_after_inspect="$(state_receipt "$core_home")" || infrastructure=1 +core_key="$(jq -r '.installationKey // empty' "$core_result" 2>/dev/null)" +core_receipt="$(jq -r '.receiptDigest // empty' "$core_result" 2>/dev/null)" +if [ "$core_inspect_rc" -eq 0 ] && [ ! -s "$CAPTURE_ERR" ] \ + && jq -e --arg key "$core_key" --arg receipt "$core_receipt" \ + '.name == "first-experiment" and .state == "installed" and .installationKey == $key and .receiptDigest == $receipt' \ + "$CAPTURE_OUT" >/dev/null 2>&1 \ + && [ "$core_before_inspect" = "$core_after_inspect" ]; then + pass INST-INSPECT-001 "inspect read-only verifies and reports the exact installed identity" +else + fail INST-INSPECT-001 "inspect read-only verifies and reports the exact installed identity" +fi + +core_before_retry="$(state_receipt "$core_home")" || infrastructure=1 +capture core-retry "$agent_lab" --home "$core_home" experiment install "$fixture" +core_after_retry="$(state_receipt "$core_home")" || infrastructure=1 +if [ "$CAPTURE_RC" -eq 0 ] && [ ! -s "$CAPTURE_ERR" ] \ + && jq -e --arg key "$core_key" --arg receipt "$core_receipt" \ + '.changed == false and .name == "first-experiment" and .installationKey == $key and .receiptDigest == $receipt' \ + "$CAPTURE_OUT" >/dev/null 2>&1 \ + && [ "$core_before_retry" = "$core_after_retry" ]; then + pass INST-RETRY-001 "exact retry returns changed false without rewriting the verified envelope" +else + fail INST-RETRY-001 "exact retry returns changed false without rewriting the verified envelope" +fi + +conflict_source="$work/conflict-source" +mkdir "$conflict_source" +sed 's/sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb/' \ + "$fixture/experiment.cue" > "$conflict_source/experiment.cue" +core_before_conflict="$(state_receipt "$core_home")" || infrastructure=1 +capture core-conflict "$agent_lab" --home "$core_home" experiment install "$conflict_source" +core_after_conflict="$(state_receipt "$core_home")" || infrastructure=1 +if [ "$CAPTURE_RC" -eq 1 ] && [ ! -s "$CAPTURE_OUT" ] \ + && [ "$core_before_conflict" = "$core_after_conflict" ]; then + pass INST-CONFLICT-001 "same name with a different installation identity never overwrites" +else + fail INST-CONFLICT-001 "same name with a different installation identity never overwrites" +fi + +effect_home="$work/effect-home" +init_home "$effect_home" +runtime_replica="$work/deny-runtime" +mkdir -p "$runtime_replica" +replica_ok=1 +while IFS= read -r runtime_name; do + if [ -z "$runtime_name" ] || [ ! -f "$repo_root/$runtime_name" ]; then + replica_ok=0 + continue + fi + mkdir -p "$runtime_replica/$(dirname -- "$runtime_name")" + cp "$repo_root/$runtime_name" "$runtime_replica/$runtime_name" || replica_ok=0 +done < "$runtime_manifest" +chmod 700 "$runtime_replica/scripts/agent-lab" 2>/dev/null || replica_ok=0 +if [ "$replica_ok" -eq 1 ]; then + sed 's/^permit (/forbid (/' "$repo_root/authorization/experiment/v0alpha1/operator.cedar" \ + > "$runtime_replica/authorization/experiment/v0alpha1/operator.cedar" +fi +capture deny-preview "$runtime_replica/scripts/agent-lab" --home "$effect_home" experiment authorize install "$fixture" +deny_preview_rc="$CAPTURE_RC" +deny_preview_out="$CAPTURE_OUT" +effect_before_deny="$(state_receipt "$effect_home")" || infrastructure=1 +capture deny-install "$runtime_replica/scripts/agent-lab" --home "$effect_home" experiment install "$fixture" +effect_after_deny="$(state_receipt "$effect_home")" || infrastructure=1 +if [ "$replica_ok" -eq 1 ] && [ "$deny_preview_rc" -eq 1 ] \ + && jq -e '.verdict == "deny"' "$deny_preview_out" >/dev/null 2>&1 \ + && [ "$CAPTURE_RC" -eq 1 ] && [ ! -s "$CAPTURE_OUT" ] \ + && [ "$effect_before_deny" = "$effect_after_deny" ]; then + pass INST-DENY-001 "a freshly evaluated Cedar denial leaves store and staging unchanged" +else + fail INST-DENY-001 "a freshly evaluated Cedar denial leaves store and staging unchanged" +fi + +forged="$work/forged-permit.json" +printf '%s\n' '{"verdict":"permit","installationKey":"sha256:ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"}' > "$forged" +forged_source="$work/forged-source" +mkdir "$forged_source" +cp "$fixture/experiment.cue" "$forged_source/experiment.cue" +cp "$forged" "$forged_source/decision.json" +effect_before_forge="$(state_receipt "$effect_home")" || infrastructure=1 +capture forged-option "$agent_lab" --home "$effect_home" experiment install "$fixture" --decision "$forged" +forged_option_rc="$CAPTURE_RC" +capture forged-source "$agent_lab" --home "$effect_home" experiment install "$forged_source" +forged_source_rc="$CAPTURE_RC" +effect_after_forge="$(state_receipt "$effect_home")" || infrastructure=1 +if [ "$forged_option_rc" -eq 2 ] && [ "$forged_source_rc" -eq 1 ] \ + && [ "$effect_before_forge" = "$effect_after_forge" ]; then + pass INST-FORGE-001 "caller-supplied permit data is neither an option nor accepted source authority" +else + fail INST-FORGE-001 "caller-supplied permit data is neither an option nor accepted source authority" +fi + +mkdir "$effect_home/images/catalog" +printf '%s\n' '{"corrupt":"local-catalog-must-not-be-opened"}' > "$effect_home/images/catalog/current.json" +chmod 600 "$effect_home/images/catalog/current.json" +canary_bin="$work/canary-bin" +canary_marks="$work/canary-marks" +mkdir "$canary_bin" "$canary_marks" +for command in docker git curl wget zip unzip buildah podman skopeo oras; do + printf '%s\n' '#!/bin/sh' 'set -eu' ': > "$CANARY_DIR/${0##*/}"' 'exit 97' > "$canary_bin/$command" + chmod 700 "$canary_bin/$command" + CANARY_DIR="$canary_marks" "$canary_bin/$command" >/dev/null 2>&1 || true +done +calibrated="$(find "$canary_marks" -type f | wc -l)" +find "$canary_marks" -type f -delete +catalog_before="$(state_receipt "$effect_home/images")" || infrastructure=1 +capture canary-install env -i \ + PATH="$canary_bin:/usr/bin:/bin" LANG=C LC_ALL=C CANARY_DIR="$canary_marks" \ + AGENT_LAB_CUE_TOOL_DIR="$AGENT_LAB_CUE_TOOL_DIR" \ + AGENT_LAB_CEDAR_TOOL_DIR="$AGENT_LAB_CEDAR_TOOL_DIR" \ + "$agent_lab" --home "$effect_home" experiment install "$fixture" +catalog_after="$(state_receipt "$effect_home/images")" || infrastructure=1 +if [ "$calibrated" -eq 10 ] && [ "$CAPTURE_RC" -eq 0 ] \ + && [ -z "$(find "$canary_marks" -type f -print -quit)" ] \ + && [ "$catalog_before" = "$catalog_after" ]; then + pass INST-NOEF-001 "direct installation ignores local catalog state and invokes no forbidden effect command" +else + fail INST-NOEF-001 "direct installation ignores local catalog state and invokes no forbidden effect command" +fi + +local_home="$work/local-home" +init_home "$local_home" +subject="registry.example/operator/worker@sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc" +capture local-add "$agent_lab" --home "$local_home" image add vendor.worker "$subject" +local_entry="$(jq -r '.entryDigest // empty' "$CAPTURE_OUT" 2>/dev/null)" +local_source="$work/local-source" +mkdir "$local_source" +sed 's#digestRef: "registry.example/team/coordinator@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"#catalogName: "vendor.worker"#' \ + "$fixture/experiment.cue" > "$local_source/experiment.cue" +capture local-install "$agent_lab" --home "$local_home" experiment install "$local_source" +local_install_rc="$CAPTURE_RC" +local_result="$CAPTURE_OUT" +local_key="$(jq -r '.installationKey // empty' "$local_result" 2>/dev/null)" +local_receipt="$(jq -r '.receiptDigest // empty' "$local_result" 2>/dev/null)" +capture local-remove "$agent_lab" --home "$local_home" image remove vendor.worker --expect "$local_entry" +local_remove_rc="$CAPTURE_RC" +local_before_retry="$(state_receipt "$local_home")" || infrastructure=1 +capture local-retry "$agent_lab" --home "$local_home" experiment install "$local_source" +local_retry_rc="$CAPTURE_RC" +capture local-inspect "$agent_lab" --home "$local_home" experiment inspect first-experiment +local_inspect_rc="$CAPTURE_RC" +local_after_retry="$(state_receipt "$local_home")" || infrastructure=1 +if [ "$local_install_rc" -eq 0 ] && [ "$local_remove_rc" -eq 0 ] \ + && [ "$local_retry_rc" -eq 1 ] && [ "$local_inspect_rc" -eq 0 ] \ + && jq -e --arg key "$local_key" --arg receipt "$local_receipt" \ + '.state == "installed" and .installationKey == $key and .receiptDigest == $receipt' \ + "$CAPTURE_OUT" >/dev/null 2>&1 \ + && [ "$local_before_retry" = "$local_after_retry" ]; then + pass INST-LOCAL-001 "removed local selector blocks retry while retained installation remains inspectable" +else + fail INST-LOCAL-001 "removed local selector blocks retry while retained installation remains inspectable" +fi + +installed_source="$work/installed-runtime-source" +installed_unavailable="$work/installed-runtime-source-unavailable" +installed_prefix="$work/installed-prefix" +installed_home="$work/installed-home" +installed_candidate="$work/installed-candidate" +installed_tools="$work/installed-pinned-tools" +installed_unrelated="$work/installed-unrelated" +installed_fixture_ok=1 +mkdir -p \ + "$installed_source/packaging" \ + "$installed_source/scripts" \ + "$installed_candidate" \ + "$installed_tools/cue" \ + "$installed_tools/cedar" \ + "$installed_unrelated" || installed_fixture_ok=0 +while IFS= read -r runtime_name; do + if [ -z "$runtime_name" ] || [ ! -f "$repo_root/$runtime_name" ]; then + installed_fixture_ok=0 + continue + fi + mkdir -p "$installed_source/$(dirname -- "$runtime_name")" || installed_fixture_ok=0 + cp "$repo_root/$runtime_name" "$installed_source/$runtime_name" || installed_fixture_ok=0 +done < "$runtime_manifest" +cp "$runtime_manifest" "$installed_source/packaging/agent-lab-local.manifest" \ + || installed_fixture_ok=0 +cp "$repo_root/scripts/install-local" "$repo_root/scripts/install-local.py" \ + "$installed_source/scripts/" || installed_fixture_ok=0 +cp "$fixture/experiment.cue" "$installed_candidate/experiment.cue" \ + || installed_fixture_ok=0 +cp -a "$repo_root/.cache/dev/tools/cue/." "$installed_tools/cue/" \ + || installed_fixture_ok=0 +cp -a "$repo_root/.cache/dev/tools/cedar/." "$installed_tools/cedar/" \ + || installed_fixture_ok=0 +chmod +x "$installed_source/scripts/install-local" "$installed_source/scripts/agent-lab" \ + || installed_fixture_ok=0 + +installed_bundle_rc=125 +installed_init_rc=125 +installed_first_rc=125 +installed_inspect_rc=125 +installed_retry_rc=125 +installed_prefix_before="unavailable" +installed_prefix_after="unavailable" +installed_home_after_first="unavailable" +installed_home_after_inspect="unavailable" +installed_home_after_retry="unavailable" +installed_bundle_out="$work/installed-bundle.out" +installed_bundle_err="$work/installed-bundle.err" +installed_init_err="$work/installed-init.err" +installed_first_out="$work/installed-first.out" +installed_first_err="$work/installed-first.err" +installed_inspect_out="$work/installed-inspect.out" +installed_inspect_err="$work/installed-inspect.err" +installed_retry_out="$work/installed-retry.out" +installed_retry_err="$work/installed-retry.err" +: > "$installed_bundle_out" +: > "$installed_bundle_err" +: > "$installed_init_err" +: > "$installed_first_out" +: > "$installed_first_err" +: > "$installed_inspect_out" +: > "$installed_inspect_err" +: > "$installed_retry_out" +: > "$installed_retry_err" + +capture_installed() { + local label="$1" + shift + capture "$label" env -i PATH=/usr/bin:/bin LANG=C LC_ALL=C \ + /bin/sh -c 'cd "$1" || exit 125; shift; exec "$@"' \ + agent-lab-installed "$installed_unrelated" \ + "$installed_prefix/bin/agent-lab" --home "$installed_home" "$@" +} + +if [ "$installed_fixture_ok" -eq 1 ]; then + capture installed-bundle env -i PATH=/usr/bin:/bin LANG=C LC_ALL=C \ + "$installed_source/scripts/install-local" --prefix "$installed_prefix" + installed_bundle_rc="$CAPTURE_RC" + installed_bundle_out="$CAPTURE_OUT" + installed_bundle_err="$CAPTURE_ERR" + installed_prefix_before="$(state_receipt "$installed_prefix")" || infrastructure=1 + if ! mv "$installed_source" "$installed_unavailable"; then + installed_fixture_ok=0 + infrastructure=1 + fi + + capture_installed installed-init init + installed_init_rc="$CAPTURE_RC" + installed_init_err="$CAPTURE_ERR" + if [ "$installed_init_rc" -eq 0 ]; then + cp -a "$installed_tools/cue/." "$installed_home/cache/tools/cue/" \ + || installed_fixture_ok=0 + cp -a "$installed_tools/cedar/." "$installed_home/cache/tools/cedar/" \ + || installed_fixture_ok=0 + if [ "$installed_fixture_ok" -ne 1 ]; then + infrastructure=1 + fi + fi + + capture_installed installed-first experiment install "$installed_candidate" + installed_first_rc="$CAPTURE_RC" + installed_first_out="$CAPTURE_OUT" + installed_first_err="$CAPTURE_ERR" + installed_home_after_first="$(state_receipt "$installed_home")" || infrastructure=1 + + capture_installed installed-inspect experiment inspect first-experiment + installed_inspect_rc="$CAPTURE_RC" + installed_inspect_out="$CAPTURE_OUT" + installed_inspect_err="$CAPTURE_ERR" + installed_home_after_inspect="$(state_receipt "$installed_home")" || infrastructure=1 + + capture_installed installed-retry experiment install "$installed_candidate" + installed_retry_rc="$CAPTURE_RC" + installed_retry_out="$CAPTURE_OUT" + installed_retry_err="$CAPTURE_ERR" + installed_home_after_retry="$(state_receipt "$installed_home")" || infrastructure=1 + installed_prefix_after="$(state_receipt "$installed_prefix")" || infrastructure=1 +else + infrastructure=1 +fi + +installed_key="$(jq -r '.installationKey // empty' "$installed_first_out" 2>/dev/null)" +installed_receipt="$(jq -r '.receiptDigest // empty' "$installed_first_out" 2>/dev/null)" +installed_target="$(readlink -f "$installed_prefix/bin/agent-lab" 2>/dev/null || true)" +if [ "$installed_fixture_ok" -eq 1 ] \ + && [ "$installed_bundle_rc" -eq 0 ] && [ ! -s "$installed_bundle_err" ] \ + && grep -Eq '^installed:[0-9a-f]{64}$' "$installed_bundle_out" \ + && [ "$installed_init_rc" -eq 0 ] && [ ! -s "$installed_init_err" ] \ + && [ "$installed_first_rc" -eq 0 ] && [ ! -s "$installed_first_err" ] \ + && jq -e '.changed == true and .name == "first-experiment"' \ + "$installed_first_out" >/dev/null 2>&1 \ + && [ "$installed_inspect_rc" -eq 0 ] && [ ! -s "$installed_inspect_err" ] \ + && jq -e --arg key "$installed_key" --arg receipt "$installed_receipt" \ + '.state == "installed" and .name == "first-experiment" and + .installationKey == $key and .receiptDigest == $receipt' \ + "$installed_inspect_out" >/dev/null 2>&1 \ + && [ "$installed_retry_rc" -eq 0 ] && [ ! -s "$installed_retry_err" ] \ + && jq -e --arg key "$installed_key" --arg receipt "$installed_receipt" \ + '.changed == false and .name == "first-experiment" and + .installationKey == $key and .receiptDigest == $receipt' \ + "$installed_retry_out" >/dev/null 2>&1 \ + && [ "$installed_home_after_first" = "$installed_home_after_inspect" ] \ + && [ "$installed_home_after_first" = "$installed_home_after_retry" ] \ + && [ "$installed_prefix_before" = "$installed_prefix_after" ] \ + && [ ! -e "$installed_source" ] && [ -d "$installed_unavailable" ] \ + && [ -z "$(find "$installed_prefix" -name __pycache__ -print -quit 2>/dev/null)" ] \ + && case "$installed_target" in + "$installed_prefix"/lib/agent-lab/releases/*/scripts/agent-lab) true ;; + *) false ;; + esac; then + pass INST-RUNTIME-001 "installed runtime remains source-independent through install, inspect, and exact retry" +else + fail INST-RUNTIME-001 "installed runtime remains source-independent through install, inspect, and exact retry" +fi + +expected="$work/expected" +printf '%s\n' \ + INST-HOME-001 INST-UNKNOWN-001 INST-NAME-001 INST-PERMIT-001 INST-RECEIPT-001 \ + INST-INSPECT-001 INST-RETRY-001 INST-CONFLICT-001 INST-DENY-001 \ + INST-FORGE-001 INST-NOEF-001 INST-LOCAL-001 INST-RUNTIME-001 > "$expected" +if ! cmp -s "$expected" "$observed"; then + printf 'INFRA install-store assertion identity drift\n' >&2 + infrastructure=1 +fi + +finish diff --git a/tests/experiment/local-config-cases.sh b/tests/experiment/local-config-cases.sh new file mode 100755 index 0000000..9ea19a6 --- /dev/null +++ b/tests/experiment/local-config-cases.sh @@ -0,0 +1,159 @@ +#!/usr/bin/env bash +set -euo pipefail + +repo_root="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." >/dev/null 2>&1 && pwd)" +work="" +cleanup_work() { + local failed=0 + if [ -n "$work" ] && [ -e "$work" ]; then + find "$work" -type f -delete 2>/dev/null || failed=1 + find "$work" -type l -delete 2>/dev/null || failed=1 + find "$work" -depth -type d -exec rmdir {} + 2>/dev/null || failed=1 + [ ! -e "$work" ] || failed=1 + fi + return "$failed" +} +if ! work="$(mktemp -d)"; then + printf 'SUMMARY assertions=0 expected=5 failures=0 infra=1\n' + exit 125 +fi +trap 'cleanup_work >/dev/null 2>&1 || true' EXIT +rc=0 +"$repo_root/scripts/agent-lab" --home "$work/home" init > "$work/out" 2> "$work/err" || rc=$? +failures=0 +lock_receipt_ok=false +if python3 - "$work/home" <<'PY' +from pathlib import Path +import json +import os +import stat +import sys + +home = Path(sys.argv[1]) +receipt = json.loads((home / "home.json").read_bytes()) +locks = receipt.get("locks") +expected = { + "imageCatalog": ( + "state/locks/image-catalog.lock", + "agent-lab.image-catalog-lock/v0alpha1", + ), + "experiments": ( + "state/locks/experiments.lock", + "agent-lab.experiments-lock/v0alpha1", + ), +} +if not isinstance(locks, dict) or set(locks) != set(expected): + raise SystemExit(1) +for key, (relative, schema) in expected.items(): + record = locks.get(key) + path = home / relative + metadata = path.lstat() + if ( + not isinstance(record, dict) + or set(record) != {"device", "inode", "path", "schema"} + or record != { + "device": metadata.st_dev, + "inode": metadata.st_ino, + "path": relative, + "schema": schema, + } + or not stat.S_ISREG(metadata.st_mode) + or stat.S_IMODE(metadata.st_mode) != 0o600 + or metadata.st_nlink != 1 + or path.read_bytes() != (schema + "\n").encode("ascii") + ): + raise SystemExit(1) +PY +then + lock_receipt_ok=true +fi +if [ "$rc" -eq 0 ] && [ -f "$work/home/home.json" ] && [ -f "$work/home/config.json" ] && $lock_receipt_ok; then + printf 'PASS CFG-001 init creates the explicit private home and bound locks\n' +else + printf 'FAIL CFG-001 init creates the explicit private home and bound locks\n' + failures=1 +fi + +before="$(find "$work/home" -printf '%P %m %y\n' | LC_ALL=C sort)" +rerun_rc=0 +"$repo_root/scripts/agent-lab" --home "$work/home" init > "$work/rerun.out" 2> "$work/rerun.err" || rerun_rc=$? +after="$(find "$work/home" -printf '%P %m %y\n' | LC_ALL=C sort)" +printf 'initialized\n' >> "$work/home/state/locks/image-catalog.lock" +initialized_rc=0 +"$repo_root/scripts/agent-lab" --home "$work/home" config check > "$work/initialized.out" 2> "$work/initialized.err" || initialized_rc=$? +mv "$work/home/state/locks/experiments.lock" "$work/replaced-experiments.lock" +printf 'agent-lab.experiments-lock/v0alpha1\n' > "$work/home/state/locks/experiments.lock" +chmod 600 "$work/home/state/locks/experiments.lock" +replacement_check_rc=0 +"$repo_root/scripts/agent-lab" --home "$work/home" config check > "$work/replacement-check.out" 2> "$work/replacement-check.err" || replacement_check_rc=$? +replacement_init_rc=0 +"$repo_root/scripts/agent-lab" --home "$work/home" init > "$work/replacement-init.out" 2> "$work/replacement-init.err" || replacement_init_rc=$? +fifo_home="$work/fifo-home" +"$repo_root/scripts/agent-lab" --home "$fifo_home" init > "$work/fifo-init.out" 2> "$work/fifo-init.err" +mv "$fifo_home/state/locks/experiments.lock" "$work/fifo-experiments.lock" +mkfifo -m 600 "$fifo_home/state/locks/experiments.lock" +fifo_check_rc=0 +timeout --signal=TERM --kill-after=1s 2s \ + "$repo_root/scripts/agent-lab" --home "$fifo_home" config check \ + > "$work/fifo-check.out" 2> "$work/fifo-check.err" || fifo_check_rc=$? +unlink "$fifo_home/state/locks/experiments.lock" +if [ "$rerun_rc" -eq 0 ] && [ "$before" = "$after" ] && grep -Fxq 'changed:false' "$work/rerun.out" && + [ "$initialized_rc" -eq 0 ] && [ "$replacement_check_rc" -eq 125 ] && [ "$replacement_init_rc" -eq 125 ] && + [ "$fifo_check_rc" -eq 125 ]; then + printf 'PASS CFG-002 exact init retry and bound-lock verification are fail-closed\n' +else + printf 'FAIL CFG-002 exact init retry and bound-lock verification are fail-closed\n' + failures=$((failures + 1)) +fi + +python3 - "$work/home/config.json" <<'PY' +from pathlib import Path +import json +import sys +path = Path(sys.argv[1]) +value = json.loads(path.read_text()) +value["paths"]["cache"] = "other-cache" +path.write_text(json.dumps(value, sort_keys=True, separators=(",", ":")) + "\n") +PY +drift_rc=0 +"$repo_root/scripts/agent-lab" --home "$work/home" config check > "$work/drift.out" 2> "$work/drift.err" || drift_rc=$? +if [ "$drift_rc" -eq 125 ] && [ ! -s "$work/drift.out" ]; then + printf 'PASS CFG-003 post-init configuration drift is infrastructure uncertainty\n' +else + printf 'FAIL CFG-003 post-init configuration drift is infrastructure uncertainty\n' + failures=$((failures + 1)) +fi + +absent="$work/absent" +absent_rc=0 +"$repo_root/scripts/agent-lab" --home "$absent" config check > "$work/absent.out" 2> "$work/absent.err" || absent_rc=$? +if [ "$absent_rc" -eq 1 ] && [ ! -e "$absent" ]; then + printf 'PASS CFG-004 config check does not initialize an absent home\n' +else + printf 'FAIL CFG-004 config check does not initialize an absent home\n' + failures=$((failures + 1)) +fi + +tool_home="$work/tool-home" +"$repo_root/scripts/agent-lab" --home "$tool_home" init >/dev/null +cp -a "$repo_root/.cache/dev/tools/cue/." "$tool_home/cache/tools/cue/" +cp -a "$repo_root/.cache/dev/tools/cedar/." "$tool_home/cache/tools/cedar/" +tools_rc=0 +"$repo_root/scripts/agent-lab" --home "$tool_home" tools provision > "$work/tools.out" 2> "$work/tools.err" || tools_rc=$? +if [ "$tools_rc" -eq 0 ] && grep -Fxq 'tools:ready' "$work/tools.out"; then + printf 'PASS TOOL-001 explicit provisioning verifies the pinned user tools\n' +else + printf 'FAIL TOOL-001 explicit provisioning verifies the pinned user tools\n' + failures=$((failures + 1)) +fi + +infrastructure=0 +if ! cleanup_work; then + infrastructure=1 +fi +trap - EXIT +printf 'SUMMARY assertions=5 expected=5 failures=%s infra=%s\n' "$failures" "$infrastructure" +if [ "$infrastructure" -ne 0 ]; then + exit 125 +fi +[ "$failures" -eq 0 ] diff --git a/tests/experiment/local-image-catalog-cases.sh b/tests/experiment/local-image-catalog-cases.sh new file mode 100755 index 0000000..ca803d1 --- /dev/null +++ b/tests/experiment/local-image-catalog-cases.sh @@ -0,0 +1,247 @@ +#!/usr/bin/env bash +set -u -o pipefail + +repo_root="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." >/dev/null 2>&1 && pwd)" +expected_count=76 +work="" +declare -a lane_pids=() +declare -a signal_descendants=() +current_lane_pid="" + +cleanup_work() { + local failed=0 + if [ -n "$work" ] && [ -e "$work" ]; then + find "$work" -type f -delete 2>/dev/null || failed=1 + find "$work" -type l -delete 2>/dev/null || failed=1 + find "$work" -depth -type d -exec rmdir {} + 2>/dev/null || failed=1 + [ ! -e "$work" ] || failed=1 + fi + return "$failed" +} + +collect_descendants() { + local parent_pid="$1" + local children="" + local children_file + local child + local known + local observed + for children_file in /proc/"$parent_pid"/task/[0-9]*/children; do + if [ ! -r "$children_file" ]; then + continue + fi + children="" + IFS= read -r children < "$children_file" || true + for child in $children; do + if [[ ! "$child" =~ ^[0-9]+$ ]]; then + continue + fi + known=0 + for observed in "${signal_descendants[@]}"; do + if [ "$observed" = "$child" ]; then + known=1 + break + fi + done + if [ "$known" -eq 0 ]; then + if ! kill -STOP -- "$child" 2>/dev/null; then + continue + fi + signal_descendants[${#signal_descendants[@]}]="$child" + fi + collect_descendants "$child" + done + done +} + +catalog_signal() { + local signal_number="$1" + local before + local index + trap '' HUP INT QUIT TERM + trap - EXIT + signal_descendants=() + for _ in 1 2 3 4 5 6 7 8; do + before="${#signal_descendants[@]}" + collect_descendants "$$" + if [ "${#signal_descendants[@]}" -eq "$before" ]; then + break + fi + done + for ((index = ${#signal_descendants[@]} - 1; index >= 0; index--)); do + kill -TERM -- "${signal_descendants[index]}" 2>/dev/null || true + done + for ((index = ${#signal_descendants[@]} - 1; index >= 0; index--)); do + kill -CONT -- "${signal_descendants[index]}" 2>/dev/null || true + done + exit $((128 + signal_number)) +} + +if ! work="$(mktemp -d)"; then + printf 'SUMMARY assertions=0 expected=%s failures=0 infra=1\n' "$expected_count" + exit 125 +fi +trap 'cleanup_work >/dev/null 2>&1 || true' EXIT +trap 'catalog_signal 1' HUP +trap 'catalog_signal 2' INT +trap 'catalog_signal 3' QUIT +trap 'catalog_signal 15' TERM + +subcases=( + "$repo_root/tests/image/catalog-cases.sh" + "$repo_root/tests/image/catalog-state-cases.py" + "$repo_root/tests/experiment/catalog-resolution-cases.sh" + "$repo_root/tests/image/catalog-mutation-cases.py" +) +expected="$work/expected" +observed="$work/observed" +: > "$observed" +printf '%s\n' \ + CAT-NAME-001 CAT-NAME-002 CAT-OCI-001 CAT-OCI-002 \ + CAT-ADD-001 CAT-ADD-002 CAT-ADD-003 CAT-ADD-004 CAT-NS-001 \ + CAT-CAS-001 CAT-CAS-002 CAT-CAS-003 CAT-CAS-004 \ + CAT-READ-001 CAT-READ-002 CAT-NOEF-001 CAT-CONC-001 CAT-CONC-002 \ + CAT-STATE-001 CAT-STATE-002 CAT-STATE-003 CAT-STATE-004 \ + CAT-STATE-005 CAT-STATE-006 CAT-STATE-007 CAT-STATE-008 \ + CAT-STATE-009 CAT-STATE-010 CAT-STATE-011 CAT-STATE-012 \ + CAT-STATE-013 CAT-STATE-014 CAT-STATE-015 CAT-STATE-016 CAT-STATE-018 CAT-STATE-017 \ + CAT-BOUND-001 CAT-BOUND-002 CAT-BOUND-003 CAT-BOUND-004 \ + CAT-CRASH-001 CAT-CRASH-002 CAT-CRASH-003 CAT-CRASH-004 CAT-CRASH-005 \ + CAT-CRASH-006 CAT-CRASH-007 CAT-CRASH-008 CAT-CRASH-009 CAT-CRASH-010 \ + CAT-CRASH-011 CAT-PLAT-001 \ + RES-ENTRY-001 RES-SNAP-001 RES-SNAP-002 RES-ENTRY-002 RES-ENTRY-003 \ + RES-ISOLATE-001 RES-STATE-001 RES-STATE-002 RES-STATE-003 RES-INPUT-001 \ + RES-SNAP-003 RES-AUTH-001 RES-INSTALL-001 RES-NOEF-001 \ + M-CAT-OCI-001 M-CAT-SHADOW-001 M-CAT-CAS-001 M-CAT-AUTH-001 M-RES-BIND-001 \ + M-CAT-NOEF-001 M-CAT-ADMIT-001 M-CAT-ATOM-001 M-CAT-DUR-001 M-CAT-STAGE-001 > "$expected" +declared_count="$(wc -l < "$expected")" +if [ "$declared_count" -ne "$expected_count" ]; then + printf 'INFRA catalog aggregate expected-count drift\n' >&2 + exit 125 +fi +infrastructure=0 +readonly lane_count=2 + +run_subcase() { + local index="$1" + local subcase="${subcases[$index]}" + local output="$work/subcase-$index.out" + local status="$work/subcase-$index.status" + local rc=0 + + if [ ! -f "$subcase" ]; then + printf 'INFRA required catalog subcase is missing: %s\n' "$subcase" > "$output" + printf '125\n' > "$status" + return 0 + fi + case "$subcase" in + *.py) + python3 -I -B "$subcase" > "$output" 2>&1 || rc=$? + ;; + *) + bash "$subcase" > "$output" 2>&1 || rc=$? + ;; + esac + printf '%s\n' "$rc" > "$status" +} + +run_lane() { + local lane="$1" + local index + local lane_subcases=() + case "$lane" in + 0) + lane_subcases=(1) + ;; + 1) + lane_subcases=(0 2 3) + ;; + *) + return 125 + ;; + esac + for index in "${lane_subcases[@]}"; do + run_subcase "$index" || return 125 + done + return 0 +} + +for ((lane = 0; lane < lane_count; lane++)); do + run_lane "$lane" & + lane_pids[lane]=$! +done +for lane in "${!lane_pids[@]}"; do + lane_rc=0 + current_lane_pid="${lane_pids[lane]}" + lane_pids[lane]="" + wait "$current_lane_pid" || lane_rc=$? + current_lane_pid="" + if [ "$lane_rc" -ne 0 ]; then + infrastructure=1 + fi +done + +failures=0 +for index in "${!subcases[@]}"; do + subcase="${subcases[$index]}" + output="$work/subcase-$index.out" + status="$work/subcase-$index.status" + rc=125 + if [ ! -f "$output" ]; then + printf 'INFRA catalog subcase output is missing: %s\n' "$subcase" >&2 + infrastructure=1 + continue + fi + if [ -f "$status" ] && cmp -s "$status" <(printf '0\n'); then + rc=0 + elif [ -f "$status" ] && cmp -s "$status" <(printf '1\n'); then + rc=1 + elif [ -f "$status" ] && cmp -s "$status" <(printf '125\n'); then + rc=125 + else + printf 'INFRA catalog subcase status is missing or invalid: %s\n' "$subcase" >&2 + rc=125 + infrastructure=1 + fi + awk '/^(PASS|FAIL) [A-Z0-9-]+ /' "$output" + awk '/^(PASS|FAIL) [A-Z0-9-]+ / {print $2}' "$output" >> "$observed" + subcase_assertions="$(awk '/^(PASS|FAIL) [A-Z0-9-]+ / {count++} END {print count + 0}' "$output")" + subcase_failures="$(awk '/^FAIL [A-Z0-9-]+ / {count++} END {print count + 0}' "$output")" + failures=$((failures + subcase_failures)) + summary="SUMMARY assertions=$subcase_assertions expected=$subcase_assertions failures=$subcase_failures infra=0" + summary_count="$(grep -Fxc "$summary" "$output" || true)" + all_summary_count="$(grep -c '^SUMMARY ' "$output" || true)" + if [ "$summary_count" -ne 1 ] || [ "$all_summary_count" -ne 1 ]; then + printf 'INFRA catalog subcase summary is absent or inconsistent: %s\n' "$subcase" >&2 + awk '!/^(PASS|FAIL) [A-Z0-9-]+ / {print}' "$output" >&2 + infrastructure=1 + elif { [ "$rc" -eq 0 ] && [ "$subcase_failures" -ne 0 ]; } \ + || { [ "$rc" -eq 1 ] && [ "$subcase_failures" -eq 0 ]; } \ + || { [ "$rc" -ne 0 ] && [ "$rc" -ne 1 ]; }; then + printf 'INFRA catalog subcase status is inconsistent: rc=%s path=%s\n' "$rc" "$subcase" >&2 + awk '!/^(PASS|FAIL) [A-Z0-9-]+ / {print}' "$output" >&2 + infrastructure=1 + fi +done + +assertions="$(wc -l < "$observed")" +if ! cmp -s "$expected" "$observed"; then + printf 'FAIL catalog aggregate assertion identity drift\n' >&2 + diff -u "$expected" "$observed" >&2 || true + failures=$((failures + 1)) +fi + +if ! cleanup_work; then + infrastructure=1 +fi +trap - EXIT + +printf 'SUMMARY assertions=%s expected=%s failures=%s infra=%s\n' \ + "$assertions" "$expected_count" "$failures" "$infrastructure" +if [ "$infrastructure" -ne 0 ]; then + exit 125 +fi +if [ "$failures" -ne 0 ]; then + exit 1 +fi +printf 'EXPERIMENT LOCAL IMAGE CATALOG PASS\n' diff --git a/tests/experiment/local-lifecycle-cases.sh b/tests/experiment/local-lifecycle-cases.sh new file mode 100755 index 0000000..dd3dd3a --- /dev/null +++ b/tests/experiment/local-lifecycle-cases.sh @@ -0,0 +1,234 @@ +#!/usr/bin/env bash +set -u -o pipefail + +repo_root="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." >/dev/null 2>&1 && pwd)" +subcases=( + "$repo_root/tests/install/local-install-cases.sh" + "$repo_root/tests/experiment/local-config-cases.sh" + "$repo_root/tests/experiment/local-image-catalog-cases.sh" + "$repo_root/tests/experiment/install-store-cases.sh" + "$repo_root/tests/experiment/install-state-cases.py" + "$repo_root/tests/experiment/install-integrity-cases.py" + "$repo_root/tests/experiment/install-mutation-cases.py" +) +expected_count=133 +work="" +readonly lane_count=3 +lane_pids=() +lifecycle_pid="$$" +lifecycle_pgid="" +lifecycle_sid="" +lifecycle_identity="$(ps -o pgid=,sid= -p "$lifecycle_pid" 2>/dev/null || true)" +read -r lifecycle_pgid lifecycle_sid <<< "$lifecycle_identity" + +cleanup_work() { + local failed=0 + if [ -n "$work" ] && [ -e "$work" ]; then + find "$work" -type f -delete 2>/dev/null || failed=1 + find "$work" -type l -delete 2>/dev/null || failed=1 + find "$work" -depth -type d -exec rmdir {} + 2>/dev/null || failed=1 + [ ! -e "$work" ] || failed=1 + fi + return "$failed" +} + +wait_lanes() { + local lane pid + local wait_infrastructure=0 + for lane in "${!lane_pids[@]}"; do + pid="${lane_pids[$lane]}" + [ -n "$pid" ] || continue + lane_pids[lane]="" + wait "$pid" 2>/dev/null || wait_infrastructure=1 + done + lane_pids=() + [ "$wait_infrastructure" -eq 0 ] +} + +signal_lanes() { + local pid + for pid in "${lane_pids[@]}"; do + [ -n "$pid" ] || continue + kill -TERM "$pid" 2>/dev/null || true + done +} + +handle_signal() { + local status="$1" + + trap '' HUP INT QUIT TERM + trap - EXIT + # A dedicated session can cancel its group without signaling an unrelated caller. + if [ "$lifecycle_pgid" = "$lifecycle_pid" ] && + [ "$lifecycle_sid" = "$lifecycle_pid" ]; then + kill -TERM -- "-$lifecycle_pgid" 2>/dev/null || true + else + signal_lanes + fi + # Descendants may ignore TERM; signal exits deliberately preserve the private work tree. + exit "$status" +} + +if ! work="$(mktemp -d)"; then + printf 'SUMMARY assertions=0 expected=%s failures=0 infra=1\n' "$expected_count" + exit 125 +fi +trap 'cleanup_work >/dev/null 2>&1 || true' EXIT +trap 'handle_signal 129' HUP +trap 'handle_signal 130' INT +trap 'handle_signal 131' QUIT +trap 'handle_signal 143' TERM + +expected="$work/expected" +observed="$work/observed" +printf '%s\n' \ + PKG-001 PKG-002 PKG-003 PKG-004 PKG-005 \ + CFG-001 CFG-002 CFG-003 CFG-004 TOOL-001 \ + CAT-NAME-001 CAT-NAME-002 CAT-OCI-001 CAT-OCI-002 \ + CAT-ADD-001 CAT-ADD-002 CAT-ADD-003 CAT-ADD-004 CAT-NS-001 \ + CAT-CAS-001 CAT-CAS-002 CAT-CAS-003 CAT-CAS-004 \ + CAT-READ-001 CAT-READ-002 CAT-NOEF-001 CAT-CONC-001 CAT-CONC-002 \ + CAT-STATE-001 CAT-STATE-002 CAT-STATE-003 CAT-STATE-004 \ + CAT-STATE-005 CAT-STATE-006 CAT-STATE-007 CAT-STATE-008 \ + CAT-STATE-009 CAT-STATE-010 CAT-STATE-011 CAT-STATE-012 \ + CAT-STATE-013 CAT-STATE-014 CAT-STATE-015 CAT-STATE-016 CAT-STATE-018 CAT-STATE-017 \ + CAT-BOUND-001 CAT-BOUND-002 CAT-BOUND-003 CAT-BOUND-004 \ + CAT-CRASH-001 CAT-CRASH-002 CAT-CRASH-003 CAT-CRASH-004 CAT-CRASH-005 \ + CAT-CRASH-006 CAT-CRASH-007 CAT-CRASH-008 CAT-CRASH-009 CAT-CRASH-010 \ + CAT-CRASH-011 CAT-PLAT-001 \ + RES-ENTRY-001 RES-SNAP-001 RES-SNAP-002 RES-ENTRY-002 RES-ENTRY-003 \ + RES-ISOLATE-001 RES-STATE-001 RES-STATE-002 RES-STATE-003 RES-INPUT-001 \ + RES-SNAP-003 RES-AUTH-001 RES-INSTALL-001 RES-NOEF-001 \ + M-CAT-OCI-001 M-CAT-SHADOW-001 M-CAT-CAS-001 M-CAT-AUTH-001 M-RES-BIND-001 \ + M-CAT-NOEF-001 M-CAT-ADMIT-001 M-CAT-ATOM-001 M-CAT-DUR-001 M-CAT-STAGE-001 \ + INST-HOME-001 INST-UNKNOWN-001 INST-NAME-001 INST-PERMIT-001 INST-RECEIPT-001 \ + INST-INSPECT-001 INST-RETRY-001 INST-CONFLICT-001 INST-DENY-001 \ + INST-FORGE-001 INST-NOEF-001 INST-LOCAL-001 INST-RUNTIME-001 \ + IST-STATE-001 IST-LOCK-001 IST-STATE-002 IST-BOUND-001 IST-STATE-003 \ + IST-CONC-001 IST-CONC-002 IST-CRASH-001 IST-LIVE-001 IST-PLAT-001 \ + IST-PROV-001 IST-PREFLIGHT-001 IST-SNAPSHOT-001 IST-CONFIG-001 IST-READ-001 \ + IST-FAULT-001 \ + IIN-CLEAN-001 IIN-CONTRACT-001 IIN-BUNDLE-001 IIN-LOCK-001 \ + IIN-SNAPSHOT-001 IIN-PREFLIGHT-001 IIN-PLAN-001 \ + M-STORE-AUTH-001 M-STORE-SOURCE-001 M-STORE-ATOM-001 M-STORE-RETRY-001 \ + M-STORE-DUR-001 M-STORE-LAYOUT-001 M-STORE-KEY-001 M-STORE-VERIFY-001 \ + M-STORE-LIVE-001 M-STORE-UNCERT-001 M-STORE-STAGE-001 > "$expected" +: > "$observed" + +run_subcase() { + local index="$1" + local subcase="${subcases[$index]}" + local output="$work/subcase-$index.out" + local status_file="$work/subcase-$index.status" + local rc + + if ! : > "$output"; then + printf '125\n' > "$status_file" 2>/dev/null || true + return 125 + fi + if [ ! -f "$subcase" ]; then + rc=125 + else + case "$subcase" in + *.py) + if python3 -I -B "$subcase" > "$output" 2>&1; then + rc=0 + else + rc=$? + fi + ;; + *) + if bash "$subcase" > "$output" 2>&1; then + rc=0 + else + rc=$? + fi + ;; + esac + fi + printf '%s\n' "$rc" > "$status_file" +} + +run_lane() { + local lane="$1" + local index + local lane_infrastructure=0 + + for index in "${!subcases[@]}"; do + [ $((index % lane_count)) -eq "$lane" ] || continue + run_subcase "$index" || lane_infrastructure=1 + done + [ "$lane_infrastructure" -eq 0 ] +} + +infrastructure=0 +for ((lane = 0; lane < lane_count; lane++)); do + run_lane "$lane" & + lane_pids+=("$!") +done +if ! wait_lanes; then + infrastructure=1 +fi + +failures=0 +for index in "${!subcases[@]}"; do + output="$work/subcase-$index.out" + status_file="$work/subcase-$index.status" + rc=125 + if [ ! -f "$status_file" ] || + [ "$(wc -l < "$status_file" 2>/dev/null)" -ne 1 ] || + ! IFS= read -r rc < "$status_file"; then + infrastructure=1 + rc=125 + fi + if [ ! -f "$output" ]; then + infrastructure=1 + continue + fi + awk '/^(PASS|FAIL) [A-Z0-9-]+ / {print}' "$output" + awk '/^(PASS|FAIL) [A-Z0-9-]+ / {print $2}' "$output" >> "$observed" + reported_assertions="$(awk '/^(PASS|FAIL) [A-Z0-9-]+ / {count++} END {print count + 0}' "$output")" + reported_failures="$(awk '/^FAIL [A-Z0-9-]+ / {count++} END {print count + 0}' "$output")" + failures=$((failures + reported_failures)) + expected_summary="SUMMARY assertions=$reported_assertions expected=$reported_assertions failures=$reported_failures infra=0" + matching_summaries="$(grep -Fxc "$expected_summary" "$output" || true)" + all_summaries="$(grep -c '^SUMMARY ' "$output" || true)" + if [ "$matching_summaries" -ne 1 ] || [ "$all_summaries" -ne 1 ]; then + infrastructure=1 + fi + case "$rc" in + 0) + if [ "$reported_failures" -ne 0 ]; then + infrastructure=1 + fi + ;; + 1) + if [ "$reported_failures" -eq 0 ]; then + infrastructure=1 + fi + ;; + *) + infrastructure=1 + ;; + esac +done + +assertions="$(wc -l < "$observed")" +if ! cmp -s "$expected" "$observed"; then + failures=$((failures + 1)) +fi + +if ! cleanup_work; then + infrastructure=1 +fi +trap - EXIT + +printf 'SUMMARY assertions=%s expected=%s failures=%s infra=%s\n' \ + "$assertions" "$expected_count" "$failures" "$infrastructure" +if [ "$infrastructure" -ne 0 ]; then + exit 125 +fi +if [ "$failures" -ne 0 ]; then + exit 1 +fi +printf 'EXPERIMENT LOCAL LIFECYCLE PASS\n' diff --git a/tests/experiment/source-adapter-cases.sh b/tests/experiment/source-adapter-cases.sh new file mode 100755 index 0000000..38f4ba8 --- /dev/null +++ b/tests/experiment/source-adapter-cases.sh @@ -0,0 +1,122 @@ +#!/usr/bin/env bash +set -u -o pipefail + +repo_root="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." >/dev/null 2>&1 && pwd)" +subcases=( + "$repo_root/tests/experiment/zip-intake-cases.sh" + "$repo_root/tests/experiment/zip-mutation-cases.py" + "$repo_root/tests/experiment/git-intake-cases.sh" + "$repo_root/tests/experiment/git-mutation-cases.py" +) +expected_count=83 +work="" + +cleanup_work() { + local failed=0 + if [ -n "$work" ] && [ -e "$work" ]; then + find "$work" -type f -delete 2>/dev/null || failed=1 + find "$work" -type l -delete 2>/dev/null || failed=1 + find "$work" -depth -type d -exec rmdir {} + 2>/dev/null || failed=1 + [ ! -e "$work" ] || failed=1 + fi + return "$failed" +} + +if ! work="$(mktemp -d)"; then + printf 'SUMMARY assertions=0 expected=%s failures=0 infra=1\n' "$expected_count" + exit 125 +fi +trap 'cleanup_work >/dev/null 2>&1 || true' EXIT + +expected="$work/expected" +observed="$work/observed" +printf '%s\n' \ + ZIP-001 ZIP-COMPAT-001 ZIP-USAGE-001 ZIP-PATH-001 ZIP-COUNT-001 ZIP-TYPE-001 ZIP-META-001 \ + ZIP-FLAG-001 ZIP-METHOD-001 ZIP-ZIP64-001 ZIP-HEADER-001 ZIP-CRC-001 \ + ZIP-LENGTH-001 ZIP-SIZE-001 ZIP-BOMB-001 ZIP-TRUNC-001 ZIP-TRAIL-001 \ + ZIP-SIZE-002 ZIP-READ-001 ZIP-READ-002 ZIP-DECODE-001 ZIP-DECODE-003 ZIP-DECODE-002 ZIP-TIMEOUT-001 \ + ZIP-OUTPUT-001 ZIP-NOEF-001 ZIP-AUTH-001 ZIP-INSTALL-001 ZIP-RETRY-001 \ + ZIP-DENY-001 ZIP-PLAT-001 ZIP-NOEF-002 ZIP-RUNTIME-001 \ + M-ZIP-WATCHDOG-001 M-ZIP-COUNT-001 M-ZIP-NAME-001 M-ZIP-TYPE-001 M-ZIP-FLAG-001 \ + M-ZIP-METHOD-001 M-ZIP-SIZE-001 M-ZIP-BOMB-001 M-ZIP-CRC-001 \ + M-ZIP-HEADER-001 \ + M-ZIP-EXTRACT-001 M-ZIP-IDENTITY-001 M-ZIP-AUTH-001 \ + GIT-CLI-001 GIT-USAGE-001 GIT-URL-001 GIT-OID-001 GIT-PLAT-001 \ + GIT-FIXTURE-001 GIT-PIN-001 GIT-COMMIT-001 GIT-ROOT-001 GIT-TYPE-001 \ + GIT-BLOB-001 GIT-DRIFT-001 GIT-AUTHORITY-001 GIT-CREDENTIAL-001 \ + GIT-REDIRECT-001 GIT-CONTENT-001 GIT-TIMEOUT-001 GIT-OUTPUT-001 \ + GIT-ACQUIRE-001 GIT-PGROUP-001 GIT-CLEANUP-001 GIT-TAXONOMY-001 \ + GIT-CHECK-001 GIT-AUTH-001 GIT-DENY-001 GIT-INSTALL-001 GIT-IDENTITY-001 \ + GIT-RETRY-001 GIT-ADAPTER-001 GIT-NOEF-001 GIT-RUNTIME-001 GIT-DIAG-001 \ + M-GIT-AUTHORITY-001 M-GIT-REF-001 M-GIT-BOUND-001 M-GIT-IDENTITY-001 \ + M-GIT-DOWNSTREAM-001 > "$expected" +: > "$observed" + +infrastructure=0 +failures=0 +for index in "${!subcases[@]}"; do + subcase="${subcases[$index]}" + output="$work/subcase-$index.out" + if [ ! -f "$subcase" ]; then + infrastructure=1 + continue + fi + case "$subcase" in + *.py) + if python3 -I -B "$subcase" > "$output" 2>&1; then + rc=0 + else + rc=$? + fi + ;; + *) + if bash "$subcase" > "$output" 2>&1; then + rc=0 + else + rc=$? + fi + ;; + esac + awk '/^(PASS|FAIL) [A-Z0-9-]+ / {print}' "$output" + awk '/^(PASS|FAIL) [A-Z0-9-]+ / {print $2}' "$output" >> "$observed" + reported_assertions="$(awk '/^(PASS|FAIL) [A-Z0-9-]+ / {count++} END {print count + 0}' "$output")" + reported_failures="$(awk '/^FAIL [A-Z0-9-]+ / {count++} END {print count + 0}' "$output")" + failures=$((failures + reported_failures)) + expected_summary="SUMMARY assertions=$reported_assertions expected=$reported_assertions failures=$reported_failures infra=0" + matching_summaries="$(grep -Fxc "$expected_summary" "$output" || true)" + all_summaries="$(grep -c '^SUMMARY ' "$output" || true)" + if [ "$matching_summaries" -ne 1 ] || [ "$all_summaries" -ne 1 ]; then + infrastructure=1 + fi + case "$rc" in + 0) + [ "$reported_failures" -eq 0 ] || infrastructure=1 + ;; + 1) + [ "$reported_failures" -ne 0 ] || infrastructure=1 + ;; + *) + infrastructure=1 + ;; + esac +done + +assertions="$(wc -l < "$observed")" +if ! cmp -s "$expected" "$observed"; then + failures=$((failures + 1)) +fi + +if ! cleanup_work; then + infrastructure=1 +fi +trap - EXIT + +printf 'SUMMARY assertions=%s expected=%s failures=%s infra=%s\n' \ + "$assertions" "$expected_count" "$failures" "$infrastructure" +if [ "$infrastructure" -ne 0 ]; then + exit 125 +fi +if [ "$failures" -ne 0 ]; then + exit 1 +fi +printf 'EXPERIMENT SOURCE ADAPTERS PASS\n' diff --git a/tests/experiment/zip-fixtures.py b/tests/experiment/zip-fixtures.py new file mode 100644 index 0000000..a21867b --- /dev/null +++ b/tests/experiment/zip-fixtures.py @@ -0,0 +1,470 @@ +#!/usr/bin/env python3 +"""Generate deterministic harmless ZIP intake fixtures for the shell contract.""" + +from __future__ import annotations + +from pathlib import Path +import stat +import struct +import sys +import warnings +import zipfile + + +LOCAL = b"PK\x03\x04" +CENTRAL = b"PK\x01\x02" +EOCD = b"PK\x05\x06" + + +def zip_bytes( + entries: list[tuple[str, bytes, int, int, bytes, bytes]], + *, + archive_comment: bytes = b"", +) -> bytes: + from io import BytesIO + + output = BytesIO() + with warnings.catch_warnings(): + warnings.simplefilter("ignore", UserWarning) + with zipfile.ZipFile(output, "w") as archive: + archive.comment = archive_comment + for name, data, method, mode, extra, comment in entries: + info = zipfile.ZipInfo(name, date_time=(1980, 1, 1, 0, 0, 0)) + info.compress_type = method + info.create_system = 3 + info.external_attr = mode << 16 + info.extra = extra + info.comment = comment + archive.writestr(info, data) + return output.getvalue() + + +def one( + name: str, + data: bytes, + method: int = zipfile.ZIP_STORED, + mode: int = stat.S_IFREG | 0o600, + extra: bytes = b"", + comment: bytes = b"", + archive_comment: bytes = b"", +) -> bytes: + return zip_bytes( + [(name, data, method, mode, extra, comment)], + archive_comment=archive_comment, + ) + + +def offsets(data: bytes) -> tuple[int, int, int]: + local = data.find(LOCAL) + central = data.rfind(CENTRAL) + eocd = data.rfind(EOCD) + if local != 0 or central < 0 or eocd < 0: + raise ValueError("fixture has no canonical ZIP records") + return local, central, eocd + + +def u16(data: bytearray, offset: int, value: int) -> None: + struct.pack_into(" None: + struct.pack_into(" bytes: + data = bytearray(base) + operation(data, *offsets(data)) + return bytes(data) + + +def insert_gap(base: bytes, gap: bytes) -> bytes: + data = bytearray(base) + _local, central, eocd = offsets(data) + data[central:central] = gap + eocd += len(gap) + u32(data, eocd + 16, central + len(gap)) + return bytes(data) + + +def append_to_deflate_payload(base: bytes, suffix: bytes) -> bytes: + data = bytearray(base) + local, central, eocd = offsets(data) + compressed_size = struct.unpack_from(" bytes: + data = bytearray(base) + local, central, eocd = offsets(data) + compressed_size = struct.unpack_from(" bytes: + if len(replacement) != len(b"experiment.cue"): + raise ValueError("replacement name must preserve record lengths") + data = bytearray(base) + local_offset, central_offset, _ = offsets(data) + if local: + data[local_offset + 30 : local_offset + 44] = replacement + if central: + data[central_offset + 46 : central_offset + 60] = replacement + return bytes(data) + + +def main() -> int: + if len(sys.argv) != 3: + return 2 + source = Path(sys.argv[1]).read_bytes() + root = Path(sys.argv[2]) + root.mkdir(parents=True, exist_ok=True) + + regular = stat.S_IFREG | 0o600 + stored = one("experiment.cue", source) + deflated = one("experiment.cue", source, zipfile.ZIP_DEFLATED) + fixtures: dict[str, bytes] = { + "stored.zip": stored, + "deflated.zip": deflated, + "wrong-case.zip": one("Experiment.cue", source), + "wrapper.zip": one("wrapper/experiment.cue", source), + "dotdot.zip": one("../experiment.cue", source), + "backslash.zip": one("folder\\experiment.cue", source), + "absolute.zip": one("/experiment.cue", source), + "drive.zip": one("C:/experiment.cue", source), + "unc.zip": one("//host/share/experiment.cue", source), + "nonascii.zip": one("experıment.cue", source), + "extra-entry.zip": zip_bytes( + [ + ("experiment.cue", source, zipfile.ZIP_STORED, regular, b"", b""), + ("metadata", b"harmless", zipfile.ZIP_STORED, regular, b"", b""), + ] + ), + "duplicate.zip": zip_bytes( + [ + ("experiment.cue", source, zipfile.ZIP_STORED, regular, b"", b""), + ("experiment.cue", source, zipfile.ZIP_STORED, regular, b"", b""), + ] + ), + "normalized-collision.zip": zip_bytes( + [ + ("experiment.cue", source, zipfile.ZIP_STORED, regular, b"", b""), + ("./experiment.cue", source, zipfile.ZIP_STORED, regular, b"", b""), + ] + ), + "slash-backslash-collision.zip": zip_bytes( + [ + ("experiment.cue", source, zipfile.ZIP_STORED, regular, b"", b""), + (".\\experiment.cue", source, zipfile.ZIP_STORED, regular, b"", b""), + ] + ), + "directory-type.zip": one("experiment.cue", source, mode=stat.S_IFDIR | 0o700), + "symlink-type.zip": one("experiment.cue", source, mode=stat.S_IFLNK | 0o777), + "fifo-type.zip": one("experiment.cue", source, mode=stat.S_IFIFO | 0o600), + "extra-field.zip": one("experiment.cue", source, extra=b"\xfe\xca\x00\x00"), + "file-comment.zip": one("experiment.cue", source, comment=b"metadata"), + "archive-comment.zip": one("experiment.cue", source, archive_comment=b"metadata"), + } + + fixtures["nul-name.zip"] = replace_names(stored, b"experiment.cu\x00") + fixtures["control-name.zip"] = replace_names(stored, b"experiment.cu\x01") + fixtures["encrypted.zip"] = mutate( + stored, + lambda data, local, central, _eocd: ( + u16(data, local + 6, struct.unpack_from("/dev/null 2>&1 && pwd)" +agent_lab="$repo_root/scripts/agent-lab" +bounded_helper="$repo_root/tests/helpers/run-bounded.py" +fixture="$repo_root/tests/experiment/fixtures/directories/minimal" +runtime_manifest="$repo_root/packaging/agent-lab-local.manifest" +expected_runtime="$repo_root/tests/install/fixtures/expected-runtime-files.txt" +expected_count=33 +work="" +failures=0 +infrastructure=0 +cleanup_work() { + local failed=0 + if [ -n "$work" ] && [ -e "$work" ]; then + find "$work" -type f -exec chmod u+rw {} + 2>/dev/null || failed=1 + find "$work" -depth -type d -exec chmod u+rwx {} + 2>/dev/null || failed=1 + find "$work" -type f -delete 2>/dev/null || failed=1 + find "$work" -type l -delete 2>/dev/null || failed=1 + find "$work" -depth -type d -exec rmdir {} + 2>/dev/null || failed=1 + [ ! -e "$work" ] || failed=1 + fi + return "$failed" +} + +if ! work="$(mktemp -d)"; then + printf 'SUMMARY assertions=0 expected=%s failures=0 infra=1\n' "$expected_count" + exit 125 +fi +trap 'cleanup_work >/dev/null 2>&1 || true' EXIT +if ! mkdir -p "$work/home" "$work/tmp"; then + printf 'SUMMARY assertions=0 expected=%s failures=0 infra=1\n' "$expected_count" + exit 125 +fi + +if [ ! -x "$agent_lab" ] || [ ! -f "$bounded_helper" ] || [ ! -d "$fixture" ] || + [ ! -f "$runtime_manifest" ] || [ ! -f "$expected_runtime" ] || + ! command -v jq >/dev/null 2>&1 || ! command -v python3 >/dev/null 2>&1; then + printf 'INFRA zip-intake prerequisites are unavailable\n' >&2 + printf 'SUMMARY assertions=0 expected=%s failures=0 infra=1\n' "$expected_count" + exit 125 +fi +if ! python3 -I -B "$bounded_helper" --self-test \ + > "$work/bounded-self-test.out" 2> "$work/bounded-self-test.err"; then + printf 'INFRA bounded command helper self-test failed\n' >&2 + printf 'SUMMARY assertions=0 expected=%s failures=0 infra=1\n' "$expected_count" + exit 125 +fi + +if ! python3 -I -B "$repo_root/tests/experiment/zip-fixtures.py" \ + "$fixture/experiment.cue" "$work"; then + printf 'SUMMARY assertions=0 expected=%s failures=0 infra=1\n' "$expected_count" + exit 125 +fi + +capture() { + local name="$1" + local status="$work/$name.status" + local status_line="" + shift + CAPTURE_RC=0 + find "$status" -delete 2>/dev/null || true + python3 -I -B "$bounded_helper" --timeout 5 --status "$status" \ + --stdout "$work/$name.out" --stderr "$work/$name.err" -- \ + env -i PATH="${CAPTURE_PATH:-/usr/bin:/bin}" HOME="$work/home" TMPDIR="$work/tmp" LC_ALL=C \ + CANARY_DIR="${CANARY_DIR:-$work/no-canary}" \ + AGENT_LAB_CUE_TOOL_DIR="${AGENT_LAB_CUE_TOOL_DIR:-$repo_root/.cache/dev/tools/cue}" \ + AGENT_LAB_CEDAR_TOOL_DIR="${AGENT_LAB_CEDAR_TOOL_DIR:-$repo_root/.cache/dev/tools/cedar}" \ + "$@" || CAPTURE_RC=$? + if [ -f "$status" ]; then + status_line="$(cat "$status")" + fi + if [ "$status_line" != "child:$CAPTURE_RC" ]; then + printf 'INFRA bounded command status is inconsistent: %s rc=%s status=%s\n' \ + "$name" "$CAPTURE_RC" "$status_line" >&2 + infrastructure=1 + CAPTURE_RC=125 + fi +} + +observed="$work/observed" +: > "$observed" +pass() { printf 'PASS %s %s\n' "$1" "$2"; printf '%s\n' "$1" >> "$observed"; } +fail() { printf 'FAIL %s %s\n' "$1" "$2"; printf '%s\n' "$1" >> "$observed"; failures=$((failures + 1)); } +init_home() { + local label="$1" home="$2" + capture "$label" "$agent_lab" --home "$home" init + if [ "$CAPTURE_RC" -ne 0 ]; then + printf 'INFRA temporary Agent Lab home initialization failed\n' >&2 + printf 'SUMMARY assertions=%s expected=%s failures=%s infra=1\n' \ + "$(wc -l < "$observed")" "$expected_count" "$failures" + exit 125 + fi +} +tree_fingerprint() { + python3 -I -B - "$1" <<'PY' +from hashlib import sha256 +import os +from pathlib import Path +import stat +import sys + +root = Path(sys.argv[1]) +records = [] +if root.exists(): + for path in sorted(root.rglob("*"), key=lambda item: os.fsencode(str(item.relative_to(root)))): + metadata = path.lstat() + relative = str(path.relative_to(root)) + if stat.S_ISREG(metadata.st_mode): + content = sha256(path.read_bytes()).hexdigest() + kind = "file" + elif stat.S_ISDIR(metadata.st_mode): + content = "" + kind = "directory" + elif stat.S_ISLNK(metadata.st_mode): + content = os.readlink(path) + kind = "symlink" + else: + content = "" + kind = "other" + records.append((relative, kind, stat.S_IMODE(metadata.st_mode), content)) +print(sha256(repr(records).encode("utf-8")).hexdigest()) +PY +} +expect_all_reject() { + local id="$1" code="$2" detail="$3" + shift 3 + local archive accepted=0 + for archive in "$@"; do + capture "reject-${archive%.zip}" "$agent_lab" experiment check --zip "$work/$archive" + if [ "$CAPTURE_RC" -ne 1 ] || [ -s "$work/reject-${archive%.zip}.out" ] || + [ "$(wc -l < "$work/reject-${archive%.zip}.err")" -ne 1 ] || + ! grep -Fq "FAIL Experiment manifest zip archive $code" \ + "$work/reject-${archive%.zip}.err"; then + accepted=1 + fi + done + if [ "$accepted" -eq 0 ]; then + pass "$id" "$detail" + else + fail "$id" "$detail" + fi +} + +capture directory "$agent_lab" experiment check "$fixture" +directory_rc="$CAPTURE_RC" +capture stored "$agent_lab" experiment check --zip "$work/stored.zip" +stored_rc="$CAPTURE_RC" +capture deflated "$agent_lab" experiment check --zip "$work/deflated.zip" +deflated_rc="$CAPTURE_RC" + +if [ "$directory_rc" -eq 0 ] && [ "$stored_rc" -eq 0 ] && [ "$deflated_rc" -eq 0 ] && + [ ! -s "$work/directory.err" ] && [ ! -s "$work/stored.err" ] && + [ ! -s "$work/deflated.err" ] && + [ "$(jq -cS '.plan' "$work/directory.out")" = "$(jq -cS '.plan' "$work/stored.out")" ] && + [ "$(jq -cS '.plan' "$work/directory.out")" = "$(jq -cS '.plan' "$work/deflated.out")" ] && + [ "$(jq -r '.source.digest' "$work/directory.out")" = "$(jq -r '.source.digest' "$work/stored.out")" ] && + [ "$(jq -r '.source.digest' "$work/directory.out")" = "$(jq -r '.source.digest' "$work/deflated.out")" ] && + jq -e '.source.kind == "zip" and (.source.archiveDigest | startswith("sha256:"))' \ + "$work/stored.out" >/dev/null 2>&1 && + jq -e '.source.kind == "zip" and (.source.archiveDigest | startswith("sha256:"))' \ + "$work/deflated.out" >/dev/null 2>&1; then + pass ZIP-001 "public zip check normalizes stored and deflated sources" +else + fail ZIP-001 "public zip check normalizes stored and deflated sources" +fi + +capture utf8-ascii "$agent_lab" experiment check --zip "$work/utf8-ascii.zip" +utf8_ascii_rc="$CAPTURE_RC" +capture creator-version "$agent_lab" experiment check --zip "$work/creator-version-45.zip" +creator_version_rc="$CAPTURE_RC" +capture deflate-level-hint "$agent_lab" experiment check --zip "$work/deflate-level-hint.zip" +deflate_level_hint_rc="$CAPTURE_RC" +capture dos-archive-file "$agent_lab" experiment check --zip "$work/dos-archive-file.zip" +dos_archive_file_rc="$CAPTURE_RC" +capture ntfs-archive-file "$agent_lab" experiment check --zip "$work/ntfs-archive-file.zip" +ntfs_archive_file_rc="$CAPTURE_RC" +capture vfat-archive-file "$agent_lab" experiment check --zip "$work/vfat-archive-file.zip" +vfat_archive_file_rc="$CAPTURE_RC" +if [ "$utf8_ascii_rc" -eq 0 ] && [ "$creator_version_rc" -eq 0 ] && + [ "$deflate_level_hint_rc" -eq 0 ] && [ "$dos_archive_file_rc" -eq 0 ] && + [ "$ntfs_archive_file_rc" -eq 0 ] && + [ "$vfat_archive_file_rc" -eq 0 ] && + [ ! -s "$work/utf8-ascii.err" ] && [ ! -s "$work/creator-version.err" ] && + [ ! -s "$work/deflate-level-hint.err" ] && [ ! -s "$work/dos-archive-file.err" ] && + [ ! -s "$work/ntfs-archive-file.err" ] && + [ ! -s "$work/vfat-archive-file.err" ] && + [ "$(jq -r '.source.digest' "$work/utf8-ascii.out")" = \ + "$(jq -r '.source.digest' "$work/directory.out")" ] && + [ "$(jq -r '.source.digest' "$work/creator-version.out")" = \ + "$(jq -r '.source.digest' "$work/directory.out")" ] && + [ "$(jq -r '.source.digest' "$work/deflate-level-hint.out")" = \ + "$(jq -r '.source.digest' "$work/directory.out")" ] && + [ "$(jq -r '.source.digest' "$work/dos-archive-file.out")" = \ + "$(jq -r '.source.digest' "$work/directory.out")" ] && + [ "$(jq -r '.source.digest' "$work/ntfs-archive-file.out")" = \ + "$(jq -r '.source.digest' "$work/directory.out")" ] && + [ "$(jq -r '.source.digest' "$work/vfat-archive-file.out")" = \ + "$(jq -r '.source.digest' "$work/directory.out")" ]; then + pass ZIP-COMPAT-001 "benign ZIP creator and compression metadata remain compatible" +else + fail ZIP-COMPAT-001 "benign ZIP creator and compression metadata remain compatible" +fi + +capture usage-check "$agent_lab" experiment check --zip +usage_check_rc="$CAPTURE_RC" +capture usage-authorize "$agent_lab" experiment authorize install --zip +usage_authorize_rc="$CAPTURE_RC" +capture usage-install "$agent_lab" experiment install --zip +usage_install_rc="$CAPTURE_RC" +if [ "$usage_check_rc" -eq 2 ] && [ "$usage_authorize_rc" -eq 2 ] && + [ "$usage_install_rc" -eq 2 ] && [ ! -s "$work/usage-check.out" ] && + [ ! -s "$work/usage-authorize.out" ] && [ ! -s "$work/usage-install.out" ]; then + pass ZIP-USAGE-001 "incomplete zip options are usage errors before source or home access" +else + fail ZIP-USAGE-001 "incomplete zip options are usage errors before source or home access" +fi + +expect_all_reject ZIP-PATH-001 ZIP-PATH "only the exact ASCII root member name is accepted" \ + wrong-case.zip wrapper.zip dotdot.zip backslash.zip absolute.zip drive.zip unc.zip \ + nul-name.zip control-name.zip nonascii.zip + +expect_all_reject ZIP-COUNT-001 ZIP-COUNT "the archive has exactly one member" \ + zero-count.zip eocd-count-mismatch.zip extra-entry.zip duplicate.zip \ + duplicate-encoding.zip normalized-collision.zip slash-backslash-collision.zip + +expect_all_reject ZIP-TYPE-001 ZIP-TYPE "the sole member is a regular file" \ + directory-type.zip symlink-type.zip fifo-type.zip dos-volume-label.zip \ + ntfs-device.zip ntfs-reparse.zip + +expect_all_reject ZIP-META-001 ZIP-META "member and archive metadata are closed" \ + extra-field.zip file-comment.zip archive-comment.zip + +expect_all_reject ZIP-FLAG-001 ZIP-FLAG "encrypted and descriptor-based members are refused" \ + encrypted.zip strong-encrypted.zip data-descriptor.zip stored-option-flag.zip + +expect_all_reject ZIP-METHOD-001 ZIP-METHOD "only stored and raw deflate members are accepted" \ + unsupported-method.zip + +expect_all_reject ZIP-ZIP64-001 ZIP-ZIP64 "ZIP64 and multidisk records are refused" \ + zip64-version.zip zip64-sentinel.zip multidisk.zip + +expect_all_reject ZIP-HEADER-001 ZIP-HEADER "central and local headers agree exactly" \ + central-signature.zip central-name-mismatch.zip central-flags-mismatch.zip \ + local-method-mismatch.zip local-crc-mismatch.zip local-size-mismatch.zip \ + eocd-offset-mismatch.zip eocd-size-mismatch.zip prefixed.zip duplicate-eocd.zip \ + concatenated.zip deflate-version-too-low.zip + +expect_all_reject ZIP-CRC-001 ZIP-CRC "member CRC is verified" \ + bad-crc.zip corrupt-payload.zip + +expect_all_reject ZIP-LENGTH-001 ZIP-LENGTH "declared and decoded lengths agree" \ + bad-length.zip payload-gap.zip + +expect_all_reject ZIP-SIZE-001 ZIP-SIZE "archive and expanded source bounds apply before planning" \ + archive-over.zip expanded-over.zip deflate-bomb.zip + +expect_all_reject ZIP-BOMB-001 ZIP-BOMB "declared-small deflate expansion stops at the source bound" \ + declared-small-large.zip + +expect_all_reject ZIP-TRUNC-001 ZIP-TRUNC "truncated records and deflate streams are refused" \ + missing-central.zip truncated-deflate.zip + +expect_all_reject ZIP-TRAIL-001 ZIP-TRAIL "bytes after the canonical archive end are refused" \ + trailing.zip deflate-unused-input.zip + +capture expanded-limit "$agent_lab" experiment check --zip "$work/expanded-limit.zip" +limit_digest="$(python3 -I -B - "$work/expanded-limit.cue" <<'PY' +from hashlib import sha256 +from pathlib import Path +import sys +data = Path(sys.argv[1]).read_bytes() +name = b"experiment.cue" +digest = sha256(b"agent-lab.experiment-tree.v1\0") +digest.update(len(name).to_bytes(4, "big")) +digest.update(name) +digest.update(len(data).to_bytes(8, "big")) +digest.update(data) +print("sha256:" + digest.hexdigest()) +PY +)" +if [ "$CAPTURE_RC" -eq 0 ] && [ ! -s "$work/expanded-limit.err" ] && + [ "$(wc -c < "$work/expanded-limit.cue")" -eq 262144 ] && + [ "$(jq -r '.source.digest' "$work/expanded-limit.out")" = "$limit_digest" ]; then + pass ZIP-SIZE-002 "the exact expanded source limit remains valid" +else + fail ZIP-SIZE-002 "the exact expanded source limit remains valid" +fi + +ln -s "$work/stored.zip" "$work/archive-link.zip" +mkdir "$work/archive-directory" +capture missing "$agent_lab" experiment check --zip "$work/missing.zip" +missing_rc="$CAPTURE_RC" +capture linked "$agent_lab" experiment check --zip "$work/archive-link.zip" +linked_rc="$CAPTURE_RC" +capture archive-directory "$agent_lab" experiment check --zip "$work/archive-directory" +directory_path_rc="$CAPTURE_RC" +if [ "$missing_rc" -eq 125 ] && [ "$linked_rc" -eq 125 ] && + [ "$directory_path_rc" -eq 125 ] && [ ! -s "$work/missing.out" ] && + [ ! -s "$work/linked.out" ] && [ ! -s "$work/archive-directory.out" ]; then + pass ZIP-READ-001 "unavailable or unsafe archive paths are infrastructure failures" +else + fail ZIP-READ-001 "unavailable or unsafe archive paths are infrastructure failures" +fi + +if python3 -I -B - "$repo_root/scripts/experiment.py" \ + "$work/expanded-limit.zip" "$work/archive-limit.zip" <<'PY' +from importlib.util import module_from_spec, spec_from_file_location +from pathlib import Path +import sys + +spec = spec_from_file_location("zip_read_probe", sys.argv[1]) +assert spec is not None and spec.loader is not None +module = module_from_spec(spec) +sys.modules[spec.name] = module +spec.loader.exec_module(module) +original_read = module.os.read +for archive_name in sys.argv[2:]: + archive = Path(archive_name) + changed = False + + def mutating_read(descriptor, size): + global changed + data = original_read(descriptor, size) + if data and not changed: + changed = True + with archive.open("ab") as stream: + stream.write(b"xx") + return data + + module.os.read = mutating_read + try: + module.read_zip_snapshot(str(archive)) + except module.InfrastructureError as error: + assert "ZIP-READ" in str(error) + else: + raise AssertionError("mid-read archive mutation was accepted") + assert changed +PY +then + pass ZIP-READ-002 "mid-read archive mutation is infrastructure uncertainty" +else + fail ZIP-READ-002 "mid-read archive mutation is infrastructure uncertainty" +fi + +if python3 -I -B - "$repo_root/scripts/experiment.py" "$work/deflated.zip" <<'PY' +from importlib.util import module_from_spec, spec_from_file_location +import sys + +spec = spec_from_file_location("zip_decode_probe", sys.argv[1]) +assert spec is not None and spec.loader is not None +module = module_from_spec(spec) +sys.modules[spec.name] = module +spec.loader.exec_module(module) + +def uncertain_decoder(*_args, **_kwargs): + raise RuntimeError("injected decoder uncertainty") + +module.zlib.decompressobj = uncertain_decoder +try: + module.read_zip_snapshot(sys.argv[2]) +except module.InfrastructureError as error: + assert "ZIP-DECODE" in str(error) +else: + raise AssertionError("unexpected decoder exception was accepted") +PY +then + pass ZIP-DECODE-001 "unexpected decoder exceptions are infrastructure uncertainty" +else + fail ZIP-DECODE-001 "unexpected decoder exceptions are infrastructure uncertainty" +fi + +if python3 -I -B - "$repo_root/scripts/experiment.py" "$work/deflated.zip" <<'PY' +from importlib.util import module_from_spec, spec_from_file_location +import sys + +spec = spec_from_file_location("zip_late_decode_probe", sys.argv[1]) +assert spec is not None and spec.loader is not None +module = module_from_spec(spec) +sys.modules[spec.name] = module +spec.loader.exec_module(module) +original_decompressobj = module.zlib.decompressobj + +class LateFaultDecoder: + def __init__(self): + self.delegate = original_decompressobj(-15) + + @property + def eof(self): + raise RuntimeError("injected late decoder uncertainty") + + @property + def unconsumed_tail(self): + return self.delegate.unconsumed_tail + + @property + def unused_data(self): + return self.delegate.unused_data + + def decompress(self, data, size): + return self.delegate.decompress(data, size) + + def flush(self, size): + return self.delegate.flush(size) + +module.zlib.decompressobj = lambda _window: LateFaultDecoder() +try: + module.read_zip_snapshot(sys.argv[2]) +except module.InfrastructureError as error: + assert "ZIP-DECODE" in str(error) +else: + raise AssertionError("late decoder exception was accepted") +PY +then + pass ZIP-DECODE-003 "late decoder exceptions are infrastructure uncertainty" +else + fail ZIP-DECODE-003 "late decoder exceptions are infrastructure uncertainty" +fi + +if python3 -I -B - "$repo_root/scripts/experiment.py" "$work/deflated.zip" <<'PY' +from importlib.util import module_from_spec, spec_from_file_location +import sys + +spec = spec_from_file_location("zip_clock_probe", sys.argv[1]) +assert spec is not None and spec.loader is not None +module = module_from_spec(spec) +sys.modules[spec.name] = module +spec.loader.exec_module(module) + +def uncertain_clock(): + raise RuntimeError("injected monotonic-clock uncertainty") + +module.time.monotonic = uncertain_clock +try: + module.read_zip_snapshot(sys.argv[2]) +except module.InfrastructureError as error: + assert "ZIP-DECODE" in str(error) +else: + raise AssertionError("initial decoder clock failure was accepted") +PY +then + pass ZIP-DECODE-002 "initial decoder clock failure is infrastructure uncertainty" +else + fail ZIP-DECODE-002 "initial decoder clock failure is infrastructure uncertainty" +fi + +if python3 -I -B - "$repo_root/scripts/experiment.py" "$work/deflated.zip" <<'PY' +from importlib.util import module_from_spec, spec_from_file_location +import sys + +spec = spec_from_file_location("zip_timeout_probe", sys.argv[1]) +assert spec is not None and spec.loader is not None +module = module_from_spec(spec) +sys.modules[spec.name] = module +spec.loader.exec_module(module) +ticks = iter((0.0, 10.0)) +module.time.monotonic = lambda: next(ticks, 10.0) +try: + module.read_zip_snapshot(sys.argv[2]) +except module.InfrastructureError as error: + assert "ZIP-TIMEOUT" in str(error) +else: + raise AssertionError("decoder deadline was accepted") +PY +then + pass ZIP-TIMEOUT-001 "decoder deadline uncertainty is bounded and classified" +else + fail ZIP-TIMEOUT-001 "decoder deadline uncertainty is bounded and classified" +fi + +if AGENT_LAB_CUE_TOOL_DIR="${AGENT_LAB_CUE_TOOL_DIR:-$repo_root/.cache/dev/tools/cue}" \ + python3 -I -B - "$repo_root/scripts/experiment.py" "$work/stored.zip" <<'PY' +from importlib.util import module_from_spec, spec_from_file_location +import io +import sys + +spec = spec_from_file_location("zip_output_probe", sys.argv[1]) +assert spec is not None and spec.loader is not None +module = module_from_spec(spec) +sys.modules[spec.name] = module +spec.loader.exec_module(module) + +class ShortBuffer: + def write(self, data): + return max(0, len(data) - 1) + + def flush(self): + return None + +class ShortOutput: + buffer = ShortBuffer() + + def flush(self): + return None + +module.sys.stdout = ShortOutput() +errors = io.StringIO() +module.sys.stderr = errors +try: + result = module.main(["experiment.py", "check-zip", sys.argv[2]]) +except SystemExit as error: + result = error.code +expected = "INFRA Experiment checked source output could not be written\n" +raise SystemExit(0 if result == 125 and errors.getvalue() == expected else 1) +PY +then + pass ZIP-OUTPUT-001 "partial checked output is infrastructure uncertainty" +else + fail ZIP-OUTPUT-001 "partial checked output is infrastructure uncertainty" +fi + +if python3 -I -B - "$repo_root/scripts/experiment.py" \ + "$work/wrapper.zip" "$work/encrypted.zip" "$work/bad-crc.zip" <<'PY' +from importlib.util import module_from_spec, spec_from_file_location +import contextlib +import io +import sys + +spec = spec_from_file_location("zip_noeffect_probe", sys.argv[1]) +assert spec is not None and spec.loader is not None +module = module_from_spec(spec) +sys.modules[spec.name] = module +spec.loader.exec_module(module) +reached = False + +def forbidden(_snapshot): + global reached + reached = True + raise AssertionError("downstream planning reached") + +module.authored_manifest = forbidden +for archive in sys.argv[2:]: + with contextlib.redirect_stderr(io.StringIO()): + try: + module.main(["experiment.py", "check-zip", archive]) + except SystemExit as error: + assert error.code == 1 + else: + raise AssertionError("hostile archive returned") +assert not reached +PY +then + pass ZIP-NOEF-001 "structural rejection occurs before common planning" +else + fail ZIP-NOEF-001 "structural rejection occurs before common planning" +fi + +capture directory-authorize "$agent_lab" experiment authorize install "$fixture" +directory_authorize_rc="$CAPTURE_RC" +capture zip-authorize "$agent_lab" experiment authorize install --zip "$work/stored.zip" +zip_authorize_rc="$CAPTURE_RC" +if [ "$directory_authorize_rc" -eq 0 ] && [ "$zip_authorize_rc" -eq 0 ] && + [ ! -s "$work/directory-authorize.err" ] && [ ! -s "$work/zip-authorize.err" ] && + cmp -s "$work/directory-authorize.out" "$work/zip-authorize.out" && + jq -e '.verdict == "permit" and (.binding.sourceDigest | startswith("sha256:"))' \ + "$work/zip-authorize.out" >/dev/null 2>&1; then + pass ZIP-AUTH-001 "zip authorization uses the directory decision path and identity" +else + fail ZIP-AUTH-001 "zip authorization uses the directory decision path and identity" +fi + +zip_home="$work/zip-home" +init_home zip-home-init "$zip_home" +capture zip-install "$agent_lab" --home "$zip_home" experiment install --zip "$work/stored.zip" +zip_install_rc="$CAPTURE_RC" +zip_target="$zip_home/experiments/first-experiment" +archive_digest="sha256:$(sha256sum "$work/stored.zip" | awk '{print $1}')" +archive_bytes="$(wc -c < "$work/stored.zip")" +source_identity="$(jq -r '.source.digest' "$work/directory.out")" +if [ "$zip_install_rc" -eq 0 ] && [ ! -s "$work/zip-install.err" ] && + jq -e '.changed == true and .name == "first-experiment"' \ + "$work/zip-install.out" >/dev/null 2>&1 && + cmp -s "$fixture/experiment.cue" "$zip_target/artifact/experiment.cue" && + jq -e --arg archive_digest "$archive_digest" --argjson archive_bytes "$archive_bytes" \ + --arg source_digest "$source_identity" \ + '.source.digest == $source_digest and + .transport == {archiveBytes: $archive_bytes, archiveDigest: $archive_digest, kind: "zip"}' \ + "$zip_target/records/provenance.json" >/dev/null 2>&1; then + pass ZIP-INSTALL-001 "zip install publishes the common artifact with closed archive provenance" +else + fail ZIP-INSTALL-001 "zip install publishes the common artifact with closed archive provenance" +fi + +retry_home="$work/retry-home" +init_home retry-home-init "$retry_home" +capture directory-first "$agent_lab" --home "$retry_home" experiment install "$fixture" +directory_first_rc="$CAPTURE_RC" +retry_receipt="$retry_home/experiments/first-experiment/records/install.json" +if [ "$directory_first_rc" -eq 0 ] && [ -f "$retry_receipt" ]; then + cp "$retry_receipt" "$work/retry-receipt-before" +else + : > "$work/retry-receipt-before" +fi +capture zip-retry "$agent_lab" --home "$retry_home" experiment install --zip "$work/deflated.zip" +zip_retry_rc="$CAPTURE_RC" +if [ "$directory_first_rc" -eq 0 ] && [ "$zip_retry_rc" -eq 0 ] && + [ ! -s "$work/directory-first.err" ] && [ ! -s "$work/zip-retry.err" ] && + jq -e '.changed == false and .name == "first-experiment"' \ + "$work/zip-retry.out" >/dev/null 2>&1 && + cmp -s "$work/retry-receipt-before" "$retry_receipt"; then + pass ZIP-RETRY-001 "equivalent zip retry preserves the directory installation receipt" +else + fail ZIP-RETRY-001 "equivalent zip retry preserves the directory installation receipt" +fi + +deny_home="$work/deny-home" +init_home deny-home-init "$deny_home" +deny_runtime="$work/deny-runtime" +deny_runtime_ok=0 +if cmp -s "$expected_runtime" "$runtime_manifest"; then + deny_runtime_ok=1 +fi +while IFS= read -r runtime_name; do + if [ -z "$runtime_name" ] || [ ! -f "$repo_root/$runtime_name" ]; then + deny_runtime_ok=0 + continue + fi + mkdir -p "$deny_runtime/$(dirname -- "$runtime_name")" + cp "$repo_root/$runtime_name" "$deny_runtime/$runtime_name" || deny_runtime_ok=0 +done < "$expected_runtime" +if [ "$deny_runtime_ok" -eq 1 ]; then + sed 's/^permit (/forbid (/' \ + "$repo_root/authorization/experiment/v0alpha1/operator.cedar" \ + > "$deny_runtime/authorization/experiment/v0alpha1/operator.cedar" +fi +deny_before="$(tree_fingerprint "$deny_home")" +capture deny-preview "$deny_runtime/scripts/agent-lab" --home "$deny_home" \ + experiment authorize install --zip "$work/stored.zip" +deny_preview_rc="$CAPTURE_RC" +deny_after_preview="$(tree_fingerprint "$deny_home")" +capture deny-install "$deny_runtime/scripts/agent-lab" --home "$deny_home" \ + experiment install --zip "$work/stored.zip" +deny_install_rc="$CAPTURE_RC" +deny_after_install="$(tree_fingerprint "$deny_home")" +if [ "$deny_runtime_ok" -eq 1 ] && [ "$deny_preview_rc" -eq 1 ] && + [ "$deny_install_rc" -eq 1 ] && [ ! -s "$work/deny-preview.err" ] && + [ ! -s "$work/deny-install.out" ] && + [ "$(wc -l < "$work/deny-install.err")" -eq 1 ] && + grep -Fxq 'FAIL Experiment fresh Experiment installation authorization denied' \ + "$work/deny-install.err" && + jq -e '.verdict == "deny"' "$work/deny-preview.out" >/dev/null 2>&1 && + [ "$deny_before" = "$deny_after_preview" ] && + [ "$deny_before" = "$deny_after_install" ]; then + pass ZIP-DENY-001 "fresh zip denial leaves the initialized home unchanged" +else + fail ZIP-DENY-001 "fresh zip denial leaves the initialized home unchanged" +fi + +if python3 -I -B - "$repo_root/scripts/agent-lab.py" <<'PY' +from importlib.util import module_from_spec, spec_from_file_location +import io +from pathlib import Path +import sys + +spec = spec_from_file_location("zip_platform_probe", sys.argv[1]) +assert spec is not None and spec.loader is not None +module = module_from_spec(spec) +sys.modules[spec.name] = module +spec.loader.exec_module(module) +reached = [] + +def forbidden(*_args, **_kwargs): + reached.append(True) + raise AssertionError("pre-acquisition platform guard was bypassed") + +module.sys.platform = "darwin" +module.load_config_receipt = forbidden +module.experiment_store_module = forbidden +errors = io.StringIO() +module.sys.stderr = errors +result = module.experiment_command( + Path("/unavailable-home"), ["install", "--zip", "/unavailable-archive"] +) +assert result == 125 +assert not reached +assert errors.getvalue() == "INFRA Agent Lab Experiment installation requires Linux\n" +PY +then + pass ZIP-PLAT-001 "non-Linux zip install stops before home and archive acquisition" +else + fail ZIP-PLAT-001 "non-Linux zip install stops before home and archive acquisition" +fi + +canary_bin="$work/canary-bin" +canary_marks="$work/canary-marks" +mkdir "$canary_bin" "$canary_marks" +for command in docker git curl wget zip unzip; do + printf '%s\n' '#!/bin/sh' 'set -eu' ': > "$CANARY_DIR/${0##*/}"' 'exit 97' \ + > "$canary_bin/$command" + chmod 700 "$canary_bin/$command" + CANARY_DIR="$canary_marks" "$canary_bin/$command" >/dev/null 2>&1 || true +done +canaries_calibrated="$(find "$canary_marks" -type f -printf '%f\n' | LC_ALL=C sort | tr '\n' ' ')" +find "$canary_marks" -type f -delete +noeffect_home="$work/noeffect-home" +init_home noeffect-home-init "$noeffect_home" +archive_before="$(sha256sum "$work/deflated.zip")" +CAPTURE_PATH="$canary_bin:/usr/bin:/bin" +CANARY_DIR="$canary_marks" +capture noeffect-check "$agent_lab" experiment check --zip "$work/deflated.zip" +noeffect_check_rc="$CAPTURE_RC" +capture noeffect-authorize "$agent_lab" experiment authorize install --zip "$work/deflated.zip" +noeffect_authorize_rc="$CAPTURE_RC" +capture noeffect-install "$agent_lab" --home "$noeffect_home" experiment install --zip "$work/deflated.zip" +noeffect_install_rc="$CAPTURE_RC" +unset CAPTURE_PATH CANARY_DIR +archive_after="$(sha256sum "$work/deflated.zip")" +if [ "$canaries_calibrated" = "curl docker git unzip wget zip " ] && + [ "$noeffect_check_rc" -eq 0 ] && [ "$noeffect_authorize_rc" -eq 0 ] && + [ "$noeffect_install_rc" -eq 0 ] && [ ! -s "$work/noeffect-check.err" ] && + [ ! -s "$work/noeffect-authorize.err" ] && [ ! -s "$work/noeffect-install.err" ] && + [ -z "$(find "$canary_marks" -type f -print -quit)" ] && + [ "$archive_before" = "$archive_after" ]; then + pass ZIP-NOEF-002 "zip intake invokes no archive tool, Git, downloader, or Docker command" +else + fail ZIP-NOEF-002 "zip intake invokes no archive tool, Git, downloader, or Docker command" +fi + +installed_source="$work/installed-source" +installed_unavailable="$work/installed-source-unavailable" +installed_prefix="$work/installed-prefix" +installed_home="$work/installed-home" +installed_unrelated="$work/installed-unrelated" +installed_tools="$work/installed-tools" +installed_ok=0 +if cmp -s "$expected_runtime" "$runtime_manifest"; then + installed_ok=1 +fi +mkdir -p "$installed_source/packaging" "$installed_source/scripts" \ + "$installed_unrelated" "$installed_tools/cue" "$installed_tools/cedar" || installed_ok=0 +while IFS= read -r runtime_name; do + if [ -z "$runtime_name" ] || [ ! -f "$repo_root/$runtime_name" ]; then + installed_ok=0 + continue + fi + mkdir -p "$installed_source/$(dirname -- "$runtime_name")" || installed_ok=0 + cp "$repo_root/$runtime_name" "$installed_source/$runtime_name" || installed_ok=0 +done < "$expected_runtime" +cp "$runtime_manifest" "$installed_source/packaging/agent-lab-local.manifest" || installed_ok=0 +cp "$repo_root/scripts/install-local" "$repo_root/scripts/install-local.py" \ + "$installed_source/scripts/" || installed_ok=0 +cp -a "$repo_root/.cache/dev/tools/cue/." "$installed_tools/cue/" || installed_ok=0 +cp -a "$repo_root/.cache/dev/tools/cedar/." "$installed_tools/cedar/" || installed_ok=0 +chmod +x "$installed_source/scripts/install-local" "$installed_source/scripts/agent-lab" || installed_ok=0 +capture installed-bundle "$installed_source/scripts/install-local" --prefix "$installed_prefix" +installed_bundle_rc="$CAPTURE_RC" +if [ "$installed_bundle_rc" -eq 0 ]; then + mv "$installed_source" "$installed_unavailable" || installed_ok=0 +fi +capture installed-init env -i PATH=/usr/bin:/bin LANG=C LC_ALL=C \ + /bin/sh -c 'cd "$1" || exit 125; shift; exec "$@"' agent-lab-installed \ + "$installed_unrelated" "$installed_prefix/bin/agent-lab" --home "$installed_home" init +installed_init_rc="$CAPTURE_RC" +if [ "$installed_init_rc" -eq 0 ]; then + cp -a "$installed_tools/cue/." "$installed_home/cache/tools/cue/" || installed_ok=0 + cp -a "$installed_tools/cedar/." "$installed_home/cache/tools/cedar/" || installed_ok=0 +fi +capture installed-check env -i PATH=/usr/bin:/bin LANG=C LC_ALL=C \ + /bin/sh -c 'cd "$1" || exit 125; shift; exec "$@"' agent-lab-installed \ + "$installed_unrelated" "$installed_prefix/bin/agent-lab" --home "$installed_home" \ + experiment check --zip "$work/stored.zip" +installed_check_rc="$CAPTURE_RC" +capture installed-authorize env -i PATH=/usr/bin:/bin LANG=C LC_ALL=C \ + /bin/sh -c 'cd "$1" || exit 125; shift; exec "$@"' agent-lab-installed \ + "$installed_unrelated" "$installed_prefix/bin/agent-lab" --home "$installed_home" \ + experiment authorize install --zip "$work/stored.zip" +installed_authorize_rc="$CAPTURE_RC" +capture installed-install env -i PATH=/usr/bin:/bin LANG=C LC_ALL=C \ + /bin/sh -c 'cd "$1" || exit 125; shift; exec "$@"' agent-lab-installed \ + "$installed_unrelated" "$installed_prefix/bin/agent-lab" --home "$installed_home" \ + experiment install --zip "$work/stored.zip" +installed_install_rc="$CAPTURE_RC" +if [ "$installed_ok" -eq 1 ] && [ "$installed_bundle_rc" -eq 0 ] && + [ "$installed_init_rc" -eq 0 ] && [ "$installed_check_rc" -eq 0 ] && + [ "$installed_authorize_rc" -eq 0 ] && [ "$installed_install_rc" -eq 0 ] && + [ ! -s "$work/installed-bundle.err" ] && [ ! -s "$work/installed-check.err" ] && + [ ! -s "$work/installed-authorize.err" ] && [ ! -s "$work/installed-install.err" ] && + jq -e '.source.kind == "zip"' "$work/installed-check.out" >/dev/null 2>&1 && + jq -e '.verdict == "permit"' "$work/installed-authorize.out" >/dev/null 2>&1 && + jq -e '.changed == true and .name == "first-experiment"' \ + "$work/installed-install.out" >/dev/null 2>&1 && + [ ! -e "$installed_source" ] && [ -d "$installed_unavailable" ] && + [ -z "$(find "$installed_prefix" -name __pycache__ -print -quit)" ]; then + pass ZIP-RUNTIME-001 "installed runtime handles zip intake without its source replica" +else + fail ZIP-RUNTIME-001 "installed runtime handles zip intake without its source replica" +fi + +expected="$work/expected" +printf '%s\n' \ + ZIP-001 ZIP-COMPAT-001 ZIP-USAGE-001 ZIP-PATH-001 ZIP-COUNT-001 ZIP-TYPE-001 ZIP-META-001 \ + ZIP-FLAG-001 ZIP-METHOD-001 ZIP-ZIP64-001 ZIP-HEADER-001 ZIP-CRC-001 \ + ZIP-LENGTH-001 ZIP-SIZE-001 ZIP-BOMB-001 ZIP-TRUNC-001 ZIP-TRAIL-001 \ + ZIP-SIZE-002 ZIP-READ-001 ZIP-READ-002 ZIP-DECODE-001 ZIP-DECODE-003 ZIP-DECODE-002 ZIP-TIMEOUT-001 \ + ZIP-OUTPUT-001 ZIP-NOEF-001 ZIP-AUTH-001 ZIP-INSTALL-001 ZIP-RETRY-001 \ + ZIP-DENY-001 ZIP-PLAT-001 ZIP-NOEF-002 ZIP-RUNTIME-001 > "$expected" +if ! cmp -s "$expected" "$observed"; then + printf 'INFRA assertion identity drift\n' >&2 + infrastructure=1 +fi +if ! cleanup_work; then + infrastructure=1 +fi +trap - EXIT +printf 'SUMMARY assertions=%s expected=%s failures=%s infra=%s\n' \ + "$expected_count" "$expected_count" "$failures" "$infrastructure" +[ "$infrastructure" -eq 0 ] || exit 125 +[ "$failures" -eq 0 ] diff --git a/tests/experiment/zip-mutation-cases.py b/tests/experiment/zip-mutation-cases.py new file mode 100644 index 0000000..95f8d31 --- /dev/null +++ b/tests/experiment/zip-mutation-cases.py @@ -0,0 +1,633 @@ +#!/usr/bin/env python3 +"""Private-copy sensitivity mutations for the bounded ZIP intake controls.""" + +from __future__ import annotations + +from hashlib import sha256 +from importlib.util import module_from_spec, spec_from_file_location +import os +from pathlib import Path +import shutil +import stat +import subprocess +import sys +import tempfile + + +EXPECTED = ( + "M-ZIP-WATCHDOG-001", + "M-ZIP-COUNT-001", + "M-ZIP-NAME-001", + "M-ZIP-TYPE-001", + "M-ZIP-FLAG-001", + "M-ZIP-METHOD-001", + "M-ZIP-SIZE-001", + "M-ZIP-BOMB-001", + "M-ZIP-CRC-001", + "M-ZIP-HEADER-001", + "M-ZIP-EXTRACT-001", + "M-ZIP-IDENTITY-001", + "M-ZIP-AUTH-001", +) + + +def load_module(path: Path, label: str): + spec = spec_from_file_location(label, path) + if spec is None or spec.loader is None: + raise RuntimeError(f"cannot load {path}") + module = module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +def replace_once(source: str, needle: str, replacement: str) -> str: + if source.count(needle) != 1: + raise RuntimeError("private mutation did not match exactly once") + return source.replace(needle, replacement, 1) + + +def rejected_with(module, archive: Path, code: str) -> bool: + try: + module.read_zip_snapshot(str(archive)) + except module.InvalidManifest as error: + return code in str(error) + return False + + +def accepted_by(module, archive: Path) -> bool: + try: + snapshot = module.read_zip_snapshot(str(archive)) + except (module.InvalidManifest, module.InfrastructureError): + return False + return isinstance(snapshot, module.SourceSnapshot) + + +def parser_mutation( + production: Path, + original: str, + private_source: Path, + baseline_module, + fixture: Path, + code: str, + assertion: str, + needle: str, + replacement: str, + marker: Path, +) -> bool: + if not rejected_with(baseline_module, fixture, code): + return False + private_source.write_text(replace_once(original, needle, replacement), encoding="utf-8") + marker.unlink(missing_ok=True) + os.environ["AGENT_LAB_ZIP_MUTATION_MARK"] = str(marker) + try: + mutant = load_module(private_source, "zip_mutant_" + assertion.lower().replace("-", "_")) + killed = accepted_by(mutant, fixture) + finally: + os.environ.pop("AGENT_LAB_ZIP_MUTATION_MARK", None) + return marker.is_file() and killed and sha256(production.read_bytes()).hexdigest() == sha256( + original.encode("utf-8") + ).hexdigest() + + +def bomb_mutation( + production: Path, + original: str, + private_source: Path, + baseline_module, + fixture: Path, + marker: Path, +) -> bool: + if not rejected_with(baseline_module, fixture, "ZIP-BOMB"): + return False + needle = " decoded = decoder.decompress(chunk, remaining)\n" + replacement = ( + " mutation_marker = os.environ.get(\"AGENT_LAB_ZIP_MUTATION_MARK\")\n" + " if mutation_marker is not None:\n" + " Path(mutation_marker).touch()\n" + " decoded = decoder.decompress(chunk)\n" + ) + private_source.write_text(replace_once(original, needle, replacement), encoding="utf-8") + marker.unlink(missing_ok=True) + os.environ["AGENT_LAB_ZIP_MUTATION_MARK"] = str(marker) + produced_sizes: list[int] = [] + rejected = False + original_decompressobj = baseline_module.zlib.decompressobj + try: + mutant = load_module(private_source, "zip_mutant_bomb") + + class RecordingDecoder: + def __init__(self): + self.delegate = original_decompressobj(-15) + + @property + def eof(self): + return self.delegate.eof + + @property + def unconsumed_tail(self): + return self.delegate.unconsumed_tail + + @property + def unused_data(self): + return self.delegate.unused_data + + def decompress(self, data, *args): + decoded = self.delegate.decompress(data, *args) + produced_sizes.append(len(decoded)) + return decoded + + def flush(self, size): + return self.delegate.flush(size) + + mutant.zlib.decompressobj = lambda _window: RecordingDecoder() + try: + mutant.read_zip_snapshot(str(fixture)) + except mutant.InvalidManifest as error: + rejected = "ZIP-BOMB" in str(error) + finally: + baseline_module.zlib.decompressobj = original_decompressobj + os.environ.pop("AGENT_LAB_ZIP_MUTATION_MARK", None) + return ( + marker.is_file() + and rejected + and max(produced_sizes, default=0) > baseline_module.MAX_SOURCE_BYTES + and baseline_module.zlib.decompressobj is original_decompressobj + and sha256(production.read_bytes()).hexdigest() + == sha256(original.encode("utf-8")).hexdigest() + ) + + +def run_command( + repo: Path, + root: Path, + label: str, + command: list[str], + environment: dict[str, str], +) -> subprocess.CompletedProcess[bytes]: + stdout = root / f"{label}.out" + stderr = root / f"{label}.err" + status = root / f"{label}.status" + bounded = subprocess.run( + [ + sys.executable, + "-I", + "-B", + str(repo / "tests/helpers/run-bounded.py"), + "--timeout", + "5", + "--status", + str(status), + "--stdout", + str(stdout), + "--stderr", + str(stderr), + "--", + "/usr/bin/env", + "-i", + *(f"{name}={value}" for name, value in sorted(environment.items())), + *command, + ], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + ) + try: + status_line = status.read_text(encoding="ascii") + captured_stdout = stdout.read_bytes() + captured_stderr = stderr.read_bytes() + except OSError as error: + raise RuntimeError("bounded command result is unavailable") from error + expected_status = f"child:{bounded.returncode}\n" + if ( + bounded.stdout + or bounded.stderr + or status_line != expected_status + or bounded.returncode == 125 + ): + raise RuntimeError("bounded command infrastructure failure") + return subprocess.CompletedProcess( + command, bounded.returncode, captured_stdout, captured_stderr + ) + + +def authorization_mutation(repo: Path, root: Path, archive: Path, marker: Path) -> bool: + runtime = root / "deny-runtime" + manifest = repo / "packaging/agent-lab-local.manifest" + expected_runtime = repo / "tests/install/fixtures/expected-runtime-files.txt" + if manifest.read_bytes() != expected_runtime.read_bytes(): + raise RuntimeError("runtime manifest differs from the independent expected list") + for raw in expected_runtime.read_text(encoding="utf-8").splitlines(): + if not raw: + raise RuntimeError("runtime manifest contains an empty path") + source = repo / raw + target = runtime / raw + target.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(source, target) + entrypoint = runtime / "scripts/agent-lab" + entrypoint.chmod(entrypoint.stat().st_mode | stat.S_IXUSR) + policy = runtime / "authorization/experiment/v0alpha1/operator.cedar" + policy_text = policy.read_text(encoding="utf-8") + policy.write_text( + replace_once(policy_text, "permit (", "forbid ("), + encoding="utf-8", + ) + environment = { + "PATH": "/usr/bin:/bin", + "HOME": str(root / "empty-home"), + "TMPDIR": str(root / "tmp"), + "LC_ALL": "C", + "AGENT_LAB_CUE_TOOL_DIR": str(repo / ".cache/dev/tools/cue"), + "AGENT_LAB_CEDAR_TOOL_DIR": str(repo / ".cache/dev/tools/cedar"), + } + Path(environment["HOME"]).mkdir() + Path(environment["TMPDIR"]).mkdir() + baseline_home = root / "baseline-home" + baseline_init = run_command( + repo, + root, + "baseline-init", + [str(entrypoint), "--home", str(baseline_home), "init"], + environment, + ) + baseline = run_command( + repo, + root, + "baseline-install", + [ + str(entrypoint), + "--home", + str(baseline_home), + "experiment", + "install", + "--zip", + str(archive), + ], + environment, + ) + if baseline_init.returncode != 0 or baseline.returncode != 1: + return False + + store = runtime / "scripts/experiment_store.py" + store_original = store.read_text(encoding="utf-8") + needle = " decision, status = experiment.authorize_plan(plan, snapshot.digest)\n" + replacement = ( + " decision, status = experiment.authorize_plan(plan, snapshot.digest)\n" + " mutation_marker = os.environ.get(\"AGENT_LAB_ZIP_MUTATION_MARK\")\n" + " if mutation_marker is not None:\n" + " Path(mutation_marker).touch()\n" + " decision = dict(decision)\n" + " decision[\"verdict\"] = \"permit\"\n" + " status = 0\n" + ) + store.write_text(replace_once(store_original, needle, replacement), encoding="utf-8") + marker.unlink(missing_ok=True) + environment["AGENT_LAB_ZIP_MUTATION_MARK"] = str(marker) + mutant_home = root / "mutant-home" + mutant_init = run_command( + repo, + root, + "mutant-init", + [str(entrypoint), "--home", str(mutant_home), "init"], + environment, + ) + mutant = run_command( + repo, + root, + "mutant-install", + [ + str(entrypoint), + "--home", + str(mutant_home), + "experiment", + "install", + "--zip", + str(archive), + ], + environment, + ) + installed = mutant_home / "experiments/first-experiment/records/install.json" + return ( + mutant_init.returncode == 0 + and mutant.returncode == 0 + and marker.is_file() + and installed.is_file() + and sha256((repo / "scripts/experiment_store.py").read_bytes()).hexdigest() + == sha256(store_original.encode("utf-8")).hexdigest() + ) + + +def main() -> int: + repo = Path(__file__).resolve().parents[2] + production = repo / "scripts/experiment.py" + original = production.read_text(encoding="utf-8") + failures = 0 + infrastructure = 0 + results: list[tuple[str, bool, str]] = [] + work_path = tempfile.mkdtemp(prefix="agent-lab-zip-mutations-") + work = Path(work_path) + try: + bounded_self_test = subprocess.run( + [ + sys.executable, + "-I", + "-B", + str(repo / "tests/helpers/run-bounded.py"), + "--self-test", + ], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + ) + if ( + bounded_self_test.returncode != 0 + or bounded_self_test.stdout + or bounded_self_test.stderr + ): + raise RuntimeError("bounded command helper self-test failed") + wrapper_self_test = work / "wrapper-self-test" + wrapper_self_test.mkdir() + wrapper_rejected_reserved_status = False + try: + run_command( + repo, + wrapper_self_test, + "reserved-status", + [sys.executable, "-I", "-B", "-c", "raise SystemExit(125)"], + {"PATH": "/usr/bin:/bin", "LC_ALL": "C"}, + ) + except RuntimeError: + wrapper_rejected_reserved_status = True + results.append( + ( + "M-ZIP-WATCHDOG-001", + wrapper_rejected_reserved_status, + "reserved watchdog status is infrastructure uncertainty", + ) + ) + fixtures = work / "fixtures" + generated = subprocess.run( + [ + sys.executable, + "-I", + "-B", + str(repo / "tests/experiment/zip-fixtures.py"), + str(repo / "tests/experiment/fixtures/directories/minimal/experiment.cue"), + str(fixtures), + ], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + timeout=5, + ) + if generated.returncode != 0 or generated.stdout or generated.stderr: + raise RuntimeError("private ZIP fixtures could not be generated") + baseline_module = load_module(production, "zip_mutation_baseline") + private_source = work / "experiment.py" + shutil.copy2(repo / "scripts/image_reference.py", work / "image_reference.py") + marker = work / "reached" + cases = ( + ( + "M-ZIP-COUNT-001", + "zero-count.zip", + "ZIP-COUNT", + " if disk_entries != 1 or total_entries != 1:\n", + ( + " if os.environ.get(\"AGENT_LAB_ZIP_MUTATION_MARK\") is not None:\n" + " Path(os.environ[\"AGENT_LAB_ZIP_MUTATION_MARK\"]).touch()\n" + " if False:\n" + ), + ), + ( + "M-ZIP-NAME-001", + "wrong-case.zip", + "ZIP-PATH", + ' if decoded_name != "experiment.cue":\n', + ( + " if os.environ.get(\"AGENT_LAB_ZIP_MUTATION_MARK\") is not None:\n" + " Path(os.environ[\"AGENT_LAB_ZIP_MUTATION_MARK\"]).touch()\n" + " if False:\n" + ), + ), + ( + "M-ZIP-TYPE-001", + "symlink-type.zip", + "ZIP-TYPE", + ( + " if (\n" + " create_system not in (0, 3, 10, 14)\n" + " or dos_attributes & 0x458\n" + " or (create_system == 3 and unix_type not in (0, stat.S_IFREG))\n" + " ):\n" + ), + ( + " if os.environ.get(\"AGENT_LAB_ZIP_MUTATION_MARK\") is not None:\n" + " Path(os.environ[\"AGENT_LAB_ZIP_MUTATION_MARK\"]).touch()\n" + " if False:\n" + ), + ), + ( + "M-ZIP-FLAG-001", + "encrypted.zip", + "ZIP-FLAG", + " if flags & ~allowed_flags:\n", + ( + " if os.environ.get(\"AGENT_LAB_ZIP_MUTATION_MARK\") is not None:\n" + " Path(os.environ[\"AGENT_LAB_ZIP_MUTATION_MARK\"]).touch()\n" + " if False:\n" + ), + ), + ( + "M-ZIP-METHOD-001", + "unsupported-deflate-method.zip", + "ZIP-METHOD", + " if method not in (0, 8):\n", + ( + " if os.environ.get(\"AGENT_LAB_ZIP_MUTATION_MARK\") is not None:\n" + " Path(os.environ[\"AGENT_LAB_ZIP_MUTATION_MARK\"]).touch()\n" + " if False:\n" + ), + ), + ( + "M-ZIP-SIZE-001", + "expanded-over.zip", + "ZIP-SIZE", + " if expanded_size > MAX_SOURCE_BYTES:\n", + ( + " if os.environ.get(\"AGENT_LAB_ZIP_MUTATION_MARK\") is not None:\n" + " Path(os.environ[\"AGENT_LAB_ZIP_MUTATION_MARK\"]).touch()\n" + " if False:\n" + ), + ), + ( + "M-ZIP-CRC-001", + "bad-crc.zip", + "ZIP-CRC", + " if (zlib.crc32(data) & 0xFFFFFFFF) != crc:\n", + ( + " if os.environ.get(\"AGENT_LAB_ZIP_MUTATION_MARK\") is not None:\n" + " Path(os.environ[\"AGENT_LAB_ZIP_MUTATION_MARK\"]).touch()\n" + " if False:\n" + ), + ), + ( + "M-ZIP-HEADER-001", + "central-signature.zip", + "ZIP-HEADER", + ' if central_signature != b"PK\\x01\\x02":\n', + ( + " if os.environ.get(\"AGENT_LAB_ZIP_MUTATION_MARK\") is not None:\n" + " Path(os.environ[\"AGENT_LAB_ZIP_MUTATION_MARK\"]).touch()\n" + " if False:\n" + ), + ), + ) + for assertion, fixture_name, code, needle, replacement in cases: + result = parser_mutation( + production, + original, + private_source, + baseline_module, + fixtures / fixture_name, + code, + assertion, + needle, + replacement, + marker, + ) + results.append((assertion, result, "private parser mutation is killed")) + if assertion == "M-ZIP-SIZE-001": + results.append( + ( + "M-ZIP-BOMB-001", + bomb_mutation( + production, + original, + private_source, + baseline_module, + fixtures / "declared-small-large.zip", + marker, + ), + "unbounded decoder mutation produces over-bound output", + ) + ) + + extraction = work / "caller-destination" + extraction.mkdir() + extraction_needle = " archive = _read_zip_archive_once(path)\n" + extraction_replacement = ( + " archive = _read_zip_archive_once(path)\n" + " mutation_marker = os.environ.get(\"AGENT_LAB_ZIP_MUTATION_MARK\")\n" + " if mutation_marker is not None:\n" + " Path(mutation_marker).touch()\n" + " __import__(\"zipfile\").ZipFile(\n" + " __import__(\"io\").BytesIO(archive)\n" + " ).extractall(Path(os.environ[\"AGENT_LAB_ZIP_MUTATION_DEST\"]))\n" + ) + private_source.write_text( + replace_once(original, extraction_needle, extraction_replacement), encoding="utf-8" + ) + marker.unlink(missing_ok=True) + os.environ["AGENT_LAB_ZIP_MUTATION_MARK"] = str(marker) + os.environ["AGENT_LAB_ZIP_MUTATION_DEST"] = str(extraction) + try: + mutant = load_module(private_source, "zip_mutant_extract") + mutant.read_zip_snapshot(str(fixtures / "stored.zip")) + finally: + os.environ.pop("AGENT_LAB_ZIP_MUTATION_MARK", None) + os.environ.pop("AGENT_LAB_ZIP_MUTATION_DEST", None) + results.append( + ( + "M-ZIP-EXTRACT-001", + marker.is_file() + and (extraction / "experiment.cue").is_file() + and sha256(production.read_bytes()).hexdigest() + == sha256(original.encode("utf-8")).hexdigest(), + "caller-destination extraction mutation is observable", + ) + ) + + identity_needle = ( + " return SourceSnapshot(\n" + " data=data,\n" + " digest=source_digest(data),\n" + " transport={\n" + " \"archiveBytes\": len(archive),\n" + ) + identity_replacement = ( + " return SourceSnapshot(\n" + " data=data,\n" + " digest=(\n" + " Path(os.environ[\"AGENT_LAB_ZIP_MUTATION_MARK\"]).touch()\n" + " or \"sha256:\" + hashlib.sha256(archive).hexdigest()\n" + " ),\n" + " transport={\n" + " \"archiveBytes\": len(archive),\n" + ) + private_source.write_text( + replace_once(original, identity_needle, identity_replacement), encoding="utf-8" + ) + marker.unlink(missing_ok=True) + os.environ["AGENT_LAB_ZIP_MUTATION_MARK"] = str(marker) + try: + mutant = load_module(private_source, "zip_mutant_identity") + directory = mutant.read_directory_snapshot( + str(repo / "tests/experiment/fixtures/directories/minimal") + ) + zipped = mutant.read_zip_snapshot(str(fixtures / "stored.zip")) + finally: + os.environ.pop("AGENT_LAB_ZIP_MUTATION_MARK", None) + results.append( + ( + "M-ZIP-IDENTITY-001", + marker.is_file() + and directory.digest != zipped.digest + and sha256(production.read_bytes()).hexdigest() + == sha256(original.encode("utf-8")).hexdigest(), + "archive-identity mutation breaks the cross-transport oracle", + ) + ) + + results.append( + ( + "M-ZIP-AUTH-001", + authorization_mutation(repo, work / "auth", fixtures / "stored.zip", marker), + "saved-denial bypass mutation changes the no-effect outcome", + ) + ) + except (OSError, RuntimeError, subprocess.SubprocessError) as error: + print(f"INFRA zip mutation harness {error}", file=sys.stderr) + infrastructure = 1 + finally: + try: + for path in work.rglob("*"): + try: + path.chmod(path.stat().st_mode | stat.S_IWUSR | stat.S_IXUSR) + except OSError: + pass + shutil.rmtree(work) + except OSError: + infrastructure = 1 + + observed = tuple(item[0] for item in results) + if observed != EXPECTED: + infrastructure = 1 + for assertion, passed, detail in results: + if passed: + print(f"PASS {assertion} {detail}") + else: + print(f"FAIL {assertion} {detail}") + failures += 1 + print( + f"SUMMARY assertions={len(results)} expected={len(EXPECTED)} " + f"failures={failures} infra={infrastructure}" + ) + if infrastructure: + return 125 + if failures: + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/helpers/run-bounded.py b/tests/helpers/run-bounded.py new file mode 100644 index 0000000..a257a61 --- /dev/null +++ b/tests/helpers/run-bounded.py @@ -0,0 +1,249 @@ +#!/usr/bin/env python3 +"""Run one test command with a bounded, fully reaped process group.""" + +from __future__ import annotations + +import argparse +from contextlib import ExitStack +import os +from pathlib import Path +import signal +import subprocess +import sys +import tempfile +import time + + +MAX_TIMEOUT_SECONDS = 5.0 +TERMINATION_GRACE_SECONDS = 1.0 +HANDLED_SIGNALS = (signal.SIGINT, signal.SIGHUP, signal.SIGTERM) +ACTIVE_PROCESS: subprocess.Popen[bytes] | None = None +STATUS_PATH: Path | None = None + + +def process_group_exists(group: int) -> bool: + try: + os.killpg(group, 0) + except ProcessLookupError: + return False + except PermissionError: + return True + return True + + +def terminate_process_group( + process: subprocess.Popen[bytes], + grace_seconds: float = TERMINATION_GRACE_SECONDS, +) -> bool: + group = process.pid + if process_group_exists(group): + try: + os.killpg(group, signal.SIGTERM) + except ProcessLookupError: + pass + deadline = time.monotonic() + grace_seconds + while process_group_exists(group) and time.monotonic() < deadline: + process.poll() + time.sleep(0.01) + if process_group_exists(group): + try: + os.killpg(group, signal.SIGKILL) + except ProcessLookupError: + pass + try: + process.wait(timeout=grace_seconds) + except subprocess.TimeoutExpired: + try: + os.killpg(group, signal.SIGKILL) + except ProcessLookupError: + pass + try: + process.wait(timeout=grace_seconds) + except subprocess.TimeoutExpired: + return False + deadline = time.monotonic() + grace_seconds + while process_group_exists(group) and time.monotonic() < deadline: + time.sleep(0.01) + return not process_group_exists(group) + + +def normalized_returncode(returncode: int) -> int: + if returncode < 0: + return 128 + abs(returncode) + return returncode + + +def execute_bounded( + command: list[str], + stdout_path: Path, + stderr_path: Path, + timeout_seconds: float, + grace_seconds: float = TERMINATION_GRACE_SECONDS, +) -> tuple[str, int]: + global ACTIVE_PROCESS + + process: subprocess.Popen[bytes] | None = None + try: + with ExitStack() as stack: + stdout = stack.enter_context(stdout_path.open("wb")) + stderr = stack.enter_context(stderr_path.open("wb")) + previous_mask = signal.pthread_sigmask(signal.SIG_BLOCK, HANDLED_SIGNALS) + try: + process = subprocess.Popen( + command, + stdin=subprocess.DEVNULL, + stdout=stdout, + stderr=stderr, + start_new_session=True, + ) + ACTIVE_PROCESS = process + finally: + signal.pthread_sigmask(signal.SIG_SETMASK, previous_mask) + try: + returncode = process.wait(timeout=timeout_seconds) + except subprocess.TimeoutExpired: + if not terminate_process_group(process, grace_seconds): + return "infra", 125 + return "infra", 125 + if process_group_exists(process.pid): + if not terminate_process_group(process, grace_seconds): + return "infra", 125 + return "infra", 125 + return "child", normalized_returncode(returncode) + except (OSError, subprocess.SubprocessError): + if process is not None and process_group_exists(process.pid): + terminate_process_group(process, grace_seconds) + return "infra", 125 + finally: + ACTIVE_PROCESS = None + + +def write_status(path: Path, kind: str, returncode: int) -> bool: + try: + path.write_text(f"{kind}:{returncode}\n", encoding="ascii") + except OSError: + return False + return True + + +def interrupted(signum: int, _frame: object) -> None: + process = ACTIVE_PROCESS + if process is not None: + terminate_process_group(process) + status = STATUS_PATH + if status is not None: + write_status(status, "infra", 125) + os._exit(128 + signum) + + +def descendant_fixture(pid_path: Path, *, hang_parent: bool) -> str: + parent_action = "time.sleep(30)" if hang_parent else "os._exit(0)" + return f""" +import os +from pathlib import Path +import signal +import time + +pid_path = Path({str(pid_path)!r}) +child = os.fork() +if child == 0: + signal.signal(signal.SIGTERM, signal.SIG_IGN) + pid_path.write_text(str(os.getpid()), encoding="ascii") + os.close(0) + os.close(1) + os.close(2) + time.sleep(30) +deadline = time.monotonic() + 1.0 +while not pid_path.exists() and time.monotonic() < deadline: + time.sleep(0.01) +{parent_action} +""" + + +def run_self_test() -> int: + try: + with tempfile.TemporaryDirectory(prefix="agent-lab-bounded-command-") as directory: + root = Path(directory) + for name, hang_parent, timeout_seconds in ( + ("normal-residual", False, 1.0), + ("timeout-residual", True, 0.2), + ): + pid_path = root / f"{name}.pid" + kind, returncode = execute_bounded( + [ + sys.executable, + "-I", + "-B", + "-c", + descendant_fixture(pid_path, hang_parent=hang_parent), + ], + root / f"{name}.out", + root / f"{name}.err", + timeout_seconds, + 0.1, + ) + if kind != "infra" or returncode != 125 or not pid_path.is_file(): + return 125 + try: + descendant = int(pid_path.read_text(encoding="ascii")) + os.kill(descendant, 0) + except ProcessLookupError: + continue + except (OSError, ValueError): + return 125 + try: + os.kill(descendant, signal.SIGKILL) + except ProcessLookupError: + pass + return 125 + except OSError: + return 125 + return 0 + + +def parse_arguments(argv: list[str]) -> argparse.Namespace: + parser = argparse.ArgumentParser(add_help=False) + parser.add_argument("--self-test", action="store_true") + parser.add_argument("--timeout", type=float, default=MAX_TIMEOUT_SECONDS) + parser.add_argument("--status") + parser.add_argument("--stdout") + parser.add_argument("--stderr") + parser.add_argument("command", nargs=argparse.REMAINDER) + return parser.parse_args(argv) + + +def main(argv: list[str]) -> int: + global STATUS_PATH + + arguments = parse_arguments(argv) + if arguments.self_test: + return run_self_test() + command = list(arguments.command) + if command and command[0] == "--": + command.pop(0) + if ( + not command + or arguments.status is None + or arguments.stdout is None + or arguments.stderr is None + or not 0 < arguments.timeout <= MAX_TIMEOUT_SECONDS + ): + return 125 + STATUS_PATH = Path(arguments.status) + kind, returncode = execute_bounded( + command, + Path(arguments.stdout), + Path(arguments.stderr), + arguments.timeout, + ) + if not write_status(STATUS_PATH, kind, returncode): + return 125 + return returncode + + +for handled_signal in HANDLED_SIGNALS: + signal.signal(handled_signal, interrupted) + + +if __name__ == "__main__": + raise SystemExit(main(sys.argv[1:])) diff --git a/tests/image/catalog-cases.sh b/tests/image/catalog-cases.sh new file mode 100755 index 0000000..6102dc2 --- /dev/null +++ b/tests/image/catalog-cases.sh @@ -0,0 +1,407 @@ +#!/usr/bin/env bash +set -u -o pipefail + +repo_root="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." >/dev/null 2>&1 && pwd)" +agent_lab="$repo_root/scripts/agent-lab" +bounded_helper="$repo_root/tests/helpers/run-bounded.py" +expected_count=18 +work="" +failures=0 +infrastructure=0 + +cleanup_work() { + local failed=0 + if [ -n "$work" ] && [ -e "$work" ]; then + find "$work" -type f -delete 2>/dev/null || failed=1 + find "$work" -type l -delete 2>/dev/null || failed=1 + find "$work" -depth -type d -exec rmdir {} + 2>/dev/null || failed=1 + [ ! -e "$work" ] || failed=1 + fi + return "$failed" +} + +finish() { + local assertions=0 + if [ -n "${observed:-}" ] && [ -f "$observed" ]; then + assertions="$(wc -l < "$observed")" + fi + if ! cleanup_work; then + infrastructure=1 + fi + trap - EXIT + printf 'SUMMARY assertions=%s expected=%s failures=%s infra=%s\n' \ + "$assertions" "$expected_count" "$failures" "$infrastructure" + if [ "$infrastructure" -ne 0 ]; then + exit 125 + fi + if [ "$failures" -ne 0 ]; then + exit 1 + fi +} + +if ! work="$(mktemp -d)"; then + printf 'SUMMARY assertions=0 expected=%s failures=0 infra=1\n' "$expected_count" + exit 125 +fi +trap 'cleanup_work >/dev/null 2>&1 || true' EXIT +observed="$work/observed" +: > "$observed" + +if [ ! -x "$agent_lab" ] || [ ! -f "$bounded_helper" ] || ! command -v jq >/dev/null 2>&1 || ! command -v python3 >/dev/null 2>&1; then + printf 'INFRA catalog public-contract prerequisites are unavailable\n' >&2 + infrastructure=1 + finish +fi +if ! python3 -I -B "$bounded_helper" --self-test > "$work/bounded-self-test.out" 2> "$work/bounded-self-test.err"; then + printf 'INFRA bounded command helper self-test failed\n' >&2 + infrastructure=1 + finish +fi + +pass() { printf 'PASS %s %s\n' "$1" "$2"; printf '%s\n' "$1" >> "$observed"; } +fail() { printf 'FAIL %s %s\n' "$1" "$2"; printf '%s\n' "$1" >> "$observed"; failures=$((failures + 1)); } +run_bounded() { + local output="$1" + local errors="$2" + local expectation="$3" + local status="${output}.status" + local rc=0 + local status_line="" + shift 3 + find "$status" -delete 2>/dev/null || true + python3 -I -B "$bounded_helper" \ + --timeout 5 --status "$status" --stdout "$output" --stderr "$errors" -- "$@" || rc=$? + if [ -f "$status" ]; then + status_line="$(cat "$status")" + fi + if [ "$status_line" != "child:$rc" ]; then + infrastructure=1 + rc=125 + else + case "$rc" in + 0|1) + ;; + 125) + [ "$expectation" = expected-125 ] || infrastructure=1 + ;; + *) + infrastructure=1 + ;; + esac + fi + return "$rc" +} +capture() { + CAPTURE_RC=0 + run_bounded "$work/stdout" "$work/stderr" normal "$@" || CAPTURE_RC=$? +} +subject_a="registry.example/operator/worker@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" +subject_b="registry.example/operator/other@sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" + +init_home() { + local home="$1" + capture "$agent_lab" --home "$home" init + if [ "$CAPTURE_RC" -ne 0 ]; then + printf 'INFRA temporary Agent Lab home initialization failed: %s\n' "$(tr '\n' ' ' < "$work/stderr")" >&2 + infrastructure=1 + finish + fi +} + +grammar_home="$work/grammar-home" +init_home "$grammar_home" +valid_names=( + "a.b" + "vendor-one.image-2" + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" +) +valid_names_ok=true +for name in "${valid_names[@]}"; do + capture "$agent_lab" --home "$grammar_home" image add "$name" "$subject_a" + if [ "$CAPTURE_RC" -ne 0 ] || ! jq -e '.changed == true and .generation == 1' "$work/stdout" >/dev/null 2>&1; then + valid_names_ok=false + fi +done +if $valid_names_ok; then + pass CAT-NAME-001 "valid minimum, hyphenated, and maximum-length names are accepted" +else + fail CAT-NAME-001 "valid minimum, hyphenated, and maximum-length names are accepted" +fi + +invalid_names=( + "Agent.image" + "vendor.Image" + "vendor.image.extra" + "vendor_1.image" + ".image" + "vendor." + "vendor.-image" + "vendor.image-" + "vendor..image" + "vendor--one.image" + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.image" + "vendor/one.image" + "vendor:image" + "vendor@one.image" + "vendor.é" + $'vendor.im\nage' +) +invalid_names_ok=true +capture "$agent_lab" --home "$grammar_home" image list --all +before_invalid="$(jq -r 'length' "$work/stdout" 2>/dev/null || printf invalid)" +for name in "${invalid_names[@]}"; do + capture "$agent_lab" --home "$grammar_home" image add "$name" "$subject_a" + if [ "$CAPTURE_RC" -ne 1 ] || [ -s "$work/stdout" ]; then + invalid_names_ok=false + fi +done +capture "$agent_lab" --home "$grammar_home" image list --all +after_invalid="$(jq -r 'length' "$work/stdout" 2>/dev/null || printf invalid)" +if $invalid_names_ok && [ "$before_invalid" = "$after_invalid" ]; then + pass CAT-NAME-002 "invalid ASCII, Unicode, separator, control, and length boundaries are rejected without mutation" +else + fail CAT-NAME-002 "invalid ASCII, Unicode, separator, control, and length boundaries are rejected without mutation" +fi + +oci_home="$work/oci-home" +init_home "$oci_home" +port_subject="registry.example:443/team/image@sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc" +capture "$agent_lab" --home "$oci_home" image add valid.port "$port_subject" +if [ "$CAPTURE_RC" -eq 0 ] && jq -e --arg subject "$port_subject" '.changed == true' "$work/stdout" >/dev/null 2>&1; then + capture "$agent_lab" --home "$oci_home" image inspect valid.port + if [ "$CAPTURE_RC" -eq 0 ] && jq -e --arg subject "$port_subject" '.subject == $subject' "$work/stdout" >/dev/null 2>&1; then + pass CAT-OCI-001 "the shared digest-reference grammar accepts a bounded registry port" + else + fail CAT-OCI-001 "the shared digest-reference grammar accepts a bounded registry port" + fi +else + fail CAT-OCI-001 "the shared digest-reference grammar accepts a bounded registry port" +fi + +digest="sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd" +overlong="$(printf 'a%.0s' {1..256})@$digest" +invalid_subjects=( + "registry.example/team/image:latest" + "$digest" + "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd" + "/tmp/image@$digest" + "user:password@registry.example/team/image@$digest" + "registry.example/team//image@$digest" + "registry.example/team/../image@$digest" + "registry.example/team/_image@$digest" + "registry.example/team/image@$digest?query=1" + "registry.example/team/image@sha256:DDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDD" + "$overlong" +) +invalid_subjects_ok=true +index=0 +for subject in "${invalid_subjects[@]}"; do + index=$((index + 1)) + capture "$agent_lab" --home "$oci_home" image add "invalid.case$index" "$subject" + if [ "$CAPTURE_RC" -ne 1 ] || [ -s "$work/stdout" ]; then + invalid_subjects_ok=false + fi +done +capture "$agent_lab" --home "$oci_home" image list --all +if $invalid_subjects_ok && jq -e 'length == 1 and .[0].name == "valid.port"' "$work/stdout" >/dev/null 2>&1; then + pass CAT-OCI-002 "mutable, bare, path-like, credentialed, ambiguous, and overlong subjects are rejected without mutation" +else + fail CAT-OCI-002 "mutable, bare, path-like, credentialed, ambiguous, and overlong subjects are rejected without mutation" +fi + +semantics_home="$work/semantics-home" +init_home "$semantics_home" +capture "$agent_lab" --home "$semantics_home" image add vendor.worker "$subject_a" +if [ "$CAPTURE_RC" -eq 0 ] && jq -e '.changed == true and .generation == 1 and (.entryDigest | test("^sha256:[0-9a-f]{64}$"))' "$work/stdout" >/dev/null 2>&1; then + active_entry="$(jq -r '.entryDigest' "$work/stdout")" + pass CAT-ADD-001 "first add publishes one generation-one immutable binding" +else + active_entry="sha256:$(printf '0%.0s' {1..64})" + fail CAT-ADD-001 "first add publishes one generation-one immutable binding" +fi + +entry_count_before="$(find "$semantics_home/images/catalog/entries" -maxdepth 1 -type f 2>/dev/null | wc -l)" +snapshot_count_before="$(find "$semantics_home/images/catalog/snapshots" -maxdepth 1 -type f 2>/dev/null | wc -l)" +capture "$agent_lab" --home "$semantics_home" image add vendor.worker "$subject_a" +entry_count_after="$(find "$semantics_home/images/catalog/entries" -maxdepth 1 -type f 2>/dev/null | wc -l)" +snapshot_count_after="$(find "$semantics_home/images/catalog/snapshots" -maxdepth 1 -type f 2>/dev/null | wc -l)" +if [ "$CAPTURE_RC" -eq 0 ] && jq -e --arg entry "$active_entry" '.changed == false and .entryDigest == $entry and .generation == 1' "$work/stdout" >/dev/null 2>&1 && + [ "$entry_count_before" = "$entry_count_after" ] && [ "$snapshot_count_before" = "$snapshot_count_after" ]; then + pass CAT-ADD-002 "same-subject retry is idempotent and publishes no records" +else + fail CAT-ADD-002 "same-subject retry is idempotent and publishes no records" +fi + +capture "$agent_lab" --home "$semantics_home" image add vendor.worker "$subject_b" +conflict_rc="$CAPTURE_RC" +capture "$agent_lab" --home "$semantics_home" image inspect vendor.worker +if [ "$conflict_rc" -eq 1 ] && [ "$CAPTURE_RC" -eq 0 ] && jq -e --arg subject "$subject_a" --arg entry "$active_entry" '.state == "active" and .subject == $subject and .entryDigest == $entry' "$work/stdout" >/dev/null 2>&1; then + pass CAT-ADD-003 "different-subject add conflicts without overwriting" +else + fail CAT-ADD-003 "different-subject add conflicts without overwriting" +fi + +capture "$agent_lab" --home "$semantics_home" image add vendor.second "$subject_a" +if [ "$CAPTURE_RC" -eq 0 ] && jq -e '.changed == true and .generation == 1' "$work/stdout" >/dev/null 2>&1; then + pass CAT-ADD-004 "distinct names may bind the same immutable subject" +else + fail CAT-ADD-004 "distinct names may bind the same immutable subject" +fi + +capture "$agent_lab" --home "$semantics_home" image add agent-lab.worker "$subject_a" +reserved_add_rc="$CAPTURE_RC" +capture "$agent_lab" --home "$semantics_home" image remove agent-lab.worker --expect "$active_entry" +reserved_remove_rc="$CAPTURE_RC" +if [ "$reserved_add_rc" -eq 1 ] && [ "$reserved_remove_rc" -eq 1 ]; then + pass CAT-NS-001 "release-owned names cannot be added, removed, or shadowed locally" +else + fail CAT-NS-001 "release-owned names cannot be added, removed, or shadowed locally" +fi + +stale="sha256:$(printf 'e%.0s' {1..64})" +capture "$agent_lab" --home "$semantics_home" image remove vendor.worker --expect "$stale" +stale_rc="$CAPTURE_RC" +capture "$agent_lab" --home "$semantics_home" image inspect vendor.worker +if [ "$stale_rc" -eq 1 ] && [ "$CAPTURE_RC" -eq 0 ] && jq -e --arg entry "$active_entry" '.state == "active" and .entryDigest == $entry and .generation == 1' "$work/stdout" >/dev/null 2>&1; then + pass CAT-CAS-001 "stale remove CAS changes no state" +else + fail CAT-CAS-001 "stale remove CAS changes no state" +fi + +capture "$agent_lab" --home "$semantics_home" image remove vendor.worker --expect "$active_entry" +if [ "$CAPTURE_RC" -eq 0 ] && jq -e --arg previous "$active_entry" '.changed == true and .generation == 2 and .state == "removed"' "$work/stdout" >/dev/null 2>&1; then + tombstone_entry="$(jq -r '.entryDigest' "$work/stdout")" + pass CAT-CAS-002 "exact CAS publishes a generation-two tombstone" +else + tombstone_entry="sha256:$(printf '0%.0s' {1..64})" + fail CAT-CAS-002 "exact CAS publishes a generation-two tombstone" +fi + +capture "$agent_lab" --home "$semantics_home" image remove vendor.worker --expect "$active_entry" +retry_rc="$CAPTURE_RC" +retry_output="$(cat "$work/stdout")" +capture "$agent_lab" --home "$semantics_home" image remove vendor.worker --expect "$tombstone_entry" +wrong_retry_rc="$CAPTURE_RC" +if [ "$retry_rc" -eq 0 ] && printf '%s' "$retry_output" | jq -e --arg entry "$tombstone_entry" '.changed == false and .entryDigest == $entry and .generation == 2' >/dev/null 2>&1 && + [ "$wrong_retry_rc" -eq 1 ]; then + pass CAT-CAS-003 "original-token retry is idempotent and every other tombstone token conflicts" +else + fail CAT-CAS-003 "original-token retry is idempotent and every other tombstone token conflicts" +fi + +capture "$agent_lab" --home "$semantics_home" image add vendor.worker "$subject_a" +same_reuse_rc="$CAPTURE_RC" +capture "$agent_lab" --home "$semantics_home" image add vendor.worker "$subject_b" +other_reuse_rc="$CAPTURE_RC" +if [ "$same_reuse_rc" -eq 1 ] && [ "$other_reuse_rc" -eq 1 ]; then + pass CAT-CAS-004 "a tombstoned v0 name cannot be reused or restored" +else + fail CAT-CAS-004 "a tombstoned v0 name cannot be reused or restored" +fi + +ordering_home="$work/ordering-home" +init_home "$ordering_home" +capture "$agent_lab" --home "$ordering_home" image add zeta.one "$subject_a" +capture "$agent_lab" --home "$ordering_home" image add alpha.two "$subject_b" +capture "$agent_lab" --home "$ordering_home" image list +if [ "$CAPTURE_RC" -eq 0 ] && [ "$(jq -c '[.[].name]' "$work/stdout" 2>/dev/null)" = '["alpha.two","zeta.one"]' ] && + [ "$(python3 -I -c 'import json,sys; print(json.dumps(json.load(sys.stdin),ensure_ascii=True,separators=(",",":"),sort_keys=True))' < "$work/stdout" 2>/dev/null)" = "$(tr -d '\n' < "$work/stdout")" ]; then + pass CAT-READ-001 "list output is canonical and byte-sorted" +else + fail CAT-READ-001 "list output is canonical and byte-sorted" +fi + +capture "$agent_lab" --home "$semantics_home" image list +active_list="$(cat "$work/stdout")" +capture "$agent_lab" --home "$semantics_home" image list --all +all_list="$(cat "$work/stdout")" +capture "$agent_lab" --home "$semantics_home" image inspect vendor.worker +if printf '%s' "$active_list" | jq -e 'all(.[]; .state == "active") and all(.[]; .name != "vendor.worker")' >/dev/null 2>&1 && + printf '%s' "$all_list" | jq -e 'any(.[]; .name == "vendor.worker" and .state == "removed")' >/dev/null 2>&1 && + [ "$CAPTURE_RC" -eq 0 ] && jq -e --arg entry "$tombstone_entry" '.state == "removed" and .entryDigest == $entry and .generation == 2' "$work/stdout" >/dev/null 2>&1; then + pass CAT-READ-002 "list and inspect distinguish active and removed state without repair" +else + fail CAT-READ-002 "list and inspect distinguish active and removed state without repair" +fi + +canary_home="$work/canary-home" +canary_bin="$work/canary-bin" +canary_marks="$work/canary-marks" +mkdir "$canary_bin" "$canary_marks" +init_home "$canary_home" +for command in docker git curl wget; do + printf '%s\n' '#!/bin/sh' 'set -eu' ': > "$CANARY_DIR/${0##*/}"' > "$canary_bin/$command" + chmod 700 "$canary_bin/$command" + CANARY_DIR="$canary_marks" "$canary_bin/$command" +done +calibrated="$(find "$canary_marks" -type f | wc -l)" +find "$canary_marks" -type f -delete +canary_rc=0 +run_bounded "$work/canary.out" "$work/canary.err" normal \ + env -i PATH="$canary_bin:/usr/bin:/bin" LANG=C LC_ALL=C CANARY_DIR="$canary_marks" \ + "$agent_lab" --home "$canary_home" image add noeffect.mapping "$subject_a" || canary_rc=$? +if [ "$calibrated" -eq 4 ] && [ "$canary_rc" -eq 0 ] && [ -z "$(find "$canary_marks" -type f -print -quit)" ]; then + pass CAT-NOEF-001 "calibrated Docker, Git, downloader, and network-tool canaries remain silent" +else + fail CAT-NOEF-001 "calibrated Docker, Git, downloader, and network-tool canaries remain silent" +fi + +concurrent_home="$work/concurrent-home" +init_home "$concurrent_home" +run_bounded "$work/race-first-1.out" "$work/race-first-1.err" normal \ + "$agent_lab" --home "$concurrent_home" image add race.first "$subject_a" & +pid_one=$! +run_bounded "$work/race-first-2.out" "$work/race-first-2.err" normal \ + "$agent_lab" --home "$concurrent_home" image add race.first "$subject_a" & +pid_two=$! +wait "$pid_one"; rc_one=$? +wait "$pid_two"; rc_two=$? +case "$rc_one:$rc_two" in + 0:0) + ;; + *) + [ "$rc_one" -eq 0 ] || [ "$rc_one" -eq 1 ] || infrastructure=1 + [ "$rc_two" -eq 0 ] || [ "$rc_two" -eq 1 ] || infrastructure=1 + ;; +esac +first_outcomes="$(jq -r '.changed' "$work/race-first-1.out" "$work/race-first-2.out" 2>/dev/null | LC_ALL=C sort | tr '\n' ' ')" +if [ "$rc_one" -eq 0 ] && [ "$rc_two" -eq 0 ] && [ "$first_outcomes" = "false true " ] && + [ "$(find "$concurrent_home/images/catalog/entries" -maxdepth 1 -type f | wc -l)" -eq 1 ] && + [ "$(find "$concurrent_home/images/catalog/snapshots" -maxdepth 1 -type f | wc -l)" -eq 1 ]; then + pass CAT-CONC-001 "concurrent first add linearizes once with one idempotent observer" +else + fail CAT-CONC-001 "concurrent first add linearizes once with one idempotent observer" +fi + +capture "$agent_lab" --home "$concurrent_home" image inspect race.first +race_entry="$(jq -r '.entryDigest // empty' "$work/stdout" 2>/dev/null)" +run_bounded "$work/race-add.out" "$work/race-add.err" normal \ + "$agent_lab" --home "$concurrent_home" image add race.first "$subject_a" & +pid_add=$! +run_bounded "$work/race-remove.out" "$work/race-remove.err" normal \ + "$agent_lab" --home "$concurrent_home" image remove race.first --expect "$race_entry" & +pid_remove=$! +wait "$pid_add"; rc_add=$? +wait "$pid_remove"; rc_remove=$? +[ "$rc_add" -eq 0 ] || [ "$rc_add" -eq 1 ] || infrastructure=1 +[ "$rc_remove" -eq 0 ] || [ "$rc_remove" -eq 1 ] || infrastructure=1 +capture "$agent_lab" --home "$concurrent_home" image inspect race.first +if [ "$rc_remove" -eq 0 ] && { [ "$rc_add" -eq 0 ] || [ "$rc_add" -eq 1 ]; } && + [ "$CAPTURE_RC" -eq 0 ] && jq -e '.state == "removed" and .generation == 2' "$work/stdout" >/dev/null 2>&1 && + [ "$(find "$concurrent_home/images/catalog/entries" -maxdepth 1 -type f | wc -l)" -eq 2 ]; then + pass CAT-CONC-002 "concurrent add and remove have one legal linearized tombstone outcome" +else + fail CAT-CONC-002 "concurrent add and remove have one legal linearized tombstone outcome" +fi + +expected="$work/expected" +printf '%s\n' \ + CAT-NAME-001 CAT-NAME-002 CAT-OCI-001 CAT-OCI-002 \ + CAT-ADD-001 CAT-ADD-002 CAT-ADD-003 CAT-ADD-004 CAT-NS-001 \ + CAT-CAS-001 CAT-CAS-002 CAT-CAS-003 CAT-CAS-004 \ + CAT-READ-001 CAT-READ-002 CAT-NOEF-001 CAT-CONC-001 CAT-CONC-002 > "$expected" +if ! cmp -s "$expected" "$observed"; then + printf 'INFRA catalog public assertion identity drift\n' >&2 + infrastructure=1 +fi +finish diff --git a/tests/image/catalog-mutation-cases.py b/tests/image/catalog-mutation-cases.py new file mode 100644 index 0000000..a488252 --- /dev/null +++ b/tests/image/catalog-mutation-cases.py @@ -0,0 +1,1047 @@ +#!/usr/bin/env python3 +"""Private-copy sensitivity mutations for the local image catalog.""" + +from __future__ import annotations + +from dataclasses import dataclass +import hashlib +from importlib.util import module_from_spec, spec_from_file_location +import json +import os +from pathlib import Path +import shutil +import signal +import stat +import subprocess +import sys +import tempfile +import time +from typing import Callable, NamedTuple + + +REPO_ROOT = Path(__file__).resolve().parents[2] +RUNTIME_MANIFEST = REPO_ROOT / "packaging" / "agent-lab-local.manifest" +SUBJECT = "registry.example/operator/worker@sha256:" + "a" * 64 +OTHER_SUBJECT = "registry.example/operator/other@sha256:" + "b" * 64 +STALE_ENTRY = "sha256:" + "e" * 64 +COMMAND_TIMEOUT_SECONDS = 5 + + +class InfrastructureError(Exception): + """The mutation or its isolated evidence could not be proved.""" + + +class ProbeResult(NamedTuple): + secure: bool + detail: str + + +Probe = Callable[[Path, Path, Path | None], ProbeResult] + + +@dataclass(frozen=True) +class Mutation: + assertion: str + path: str + old: str + new: str + probe: Probe + message: str + + +def sha256_bytes(data: bytes) -> str: + return hashlib.sha256(data).hexdigest() + + +def manifest_paths() -> tuple[str, ...]: + try: + raw = RUNTIME_MANIFEST.read_bytes() + text = raw.decode("utf-8") + except (OSError, UnicodeError) as error: + raise InfrastructureError("runtime manifest cannot be read exactly") from error + if not text.endswith("\n"): + raise InfrastructureError("runtime manifest lacks its final newline") + names = tuple(line for line in text.splitlines() if line) + if not names or len(names) != len(set(names)) or names != tuple(sorted(names)): + raise InfrastructureError("runtime manifest is empty, duplicated, or unordered") + for name in names: + path = Path(name) + if path.is_absolute() or ".." in path.parts or str(path) != name: + raise InfrastructureError(f"runtime manifest path is unsafe: {name}") + required = { + "scripts/agent-lab", + "scripts/agent-lab.py", + "scripts/experiment.py", + "scripts/image_catalog.py", + } + if not required.issubset(names): + raise InfrastructureError("runtime manifest omits a catalog runtime path") + return names + + +def file_identity(path: Path) -> tuple[str, int, int, str]: + try: + metadata = path.lstat() + data = path.read_bytes() + except OSError as error: + raise InfrastructureError(f"runtime path cannot be fingerprinted: {path}") from error + if not stat.S_ISREG(metadata.st_mode) or metadata.st_nlink != 1: + raise InfrastructureError(f"runtime path is not a single regular file: {path}") + return ( + "file", + stat.S_IMODE(metadata.st_mode), + len(data), + sha256_bytes(data), + ) + + +def runtime_fingerprint(root: Path, names: tuple[str, ...]) -> tuple[tuple[str, tuple[str, int, int, str]], ...]: + paths = ("packaging/agent-lab-local.manifest", *names) + return tuple((name, file_identity(root / name)) for name in paths) + + +def tree_fingerprint(root: Path) -> tuple[tuple[str, str, int, int, str], ...]: + if not root.exists() and not root.is_symlink(): + return () + values: list[tuple[str, str, int, int, str]] = [] + pending = [root] + while pending: + path = pending.pop() + try: + metadata = path.lstat() + except OSError as error: + raise InfrastructureError(f"probe state cannot be fingerprinted: {path}") from error + relative = "." if path == root else path.relative_to(root).as_posix() + mode = stat.S_IMODE(metadata.st_mode) + if stat.S_ISDIR(metadata.st_mode): + kind = "directory" + payload = "" + try: + pending.extend(sorted(path.iterdir(), reverse=True)) + except OSError as error: + raise InfrastructureError(f"probe directory cannot be listed: {path}") from error + elif stat.S_ISREG(metadata.st_mode): + kind = "file" + try: + payload = sha256_bytes(path.read_bytes()) + except OSError as error: + raise InfrastructureError(f"probe file cannot be read: {path}") from error + elif stat.S_ISLNK(metadata.st_mode): + kind = "symlink" + try: + payload = os.readlink(path) + except OSError as error: + raise InfrastructureError(f"probe symlink cannot be read: {path}") from error + else: + kind = "other" + payload = "" + values.append((relative, kind, mode, metadata.st_nlink, payload)) + return tuple(sorted(values)) + + +def copy_runtime(destination: Path, names: tuple[str, ...]) -> None: + for name in ("packaging/agent-lab-local.manifest", *names): + source = REPO_ROOT / name + target = destination / name + identity = file_identity(source) + try: + target.parent.mkdir(mode=0o700, parents=True, exist_ok=True) + shutil.copyfile(source, target) + os.chmod(target, identity[1]) + except OSError as error: + raise InfrastructureError(f"runtime path cannot be copied privately: {name}") from error + if file_identity(target) != identity: + raise InfrastructureError(f"private runtime copy differs: {name}") + + +def command_environment(extra: dict[str, str] | None = None) -> dict[str, str]: + environment = { + "PATH": "/usr/bin:/bin", + "LANG": "C", + "LC_ALL": "C", + } + if extra: + environment.update(extra) + return environment + + +def process_group_exists(group: int) -> bool: + try: + os.killpg(group, 0) + except ProcessLookupError: + return False + except PermissionError: + return True + return True + + +def terminate_process_group(process: subprocess.Popen[bytes]) -> bool: + group = process.pid + if process_group_exists(group): + try: + os.killpg(group, signal.SIGTERM) + except ProcessLookupError: + pass + deadline = time.monotonic() + 1.0 + while process_group_exists(group) and time.monotonic() < deadline: + time.sleep(0.01) + if process_group_exists(group): + try: + os.killpg(group, signal.SIGKILL) + except ProcessLookupError: + pass + try: + process.communicate(timeout=1) + except subprocess.TimeoutExpired: + try: + os.killpg(group, signal.SIGKILL) + except ProcessLookupError: + pass + process.communicate() + deadline = time.monotonic() + 1.0 + while process_group_exists(group) and time.monotonic() < deadline: + time.sleep(0.01) + return not process_group_exists(group) + + +def run_command( + arguments: list[str], + *, + environment: dict[str, str] | None = None, + timeout: int = COMMAND_TIMEOUT_SECONDS, +) -> subprocess.CompletedProcess[bytes]: + process: subprocess.Popen[bytes] | None = None + try: + process = subprocess.Popen( + arguments, + stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + env=command_environment(environment), + start_new_session=True, + ) + stdout, stderr = process.communicate(timeout=timeout) + if process_group_exists(process.pid): + cleaned = terminate_process_group(process) + if not cleaned: + raise InfrastructureError( + f"bounded probe command left an uncontained process group: {arguments[0]}" + ) + raise InfrastructureError( + f"bounded probe command left a descendant process: {arguments[0]}" + ) + return subprocess.CompletedProcess(arguments, process.returncode, stdout, stderr) + except subprocess.TimeoutExpired as error: + assert process is not None + cleaned = terminate_process_group(process) + if not cleaned: + raise InfrastructureError( + f"timed-out probe left an uncontained process group: {arguments[0]}" + ) from error + raise InfrastructureError(f"bounded probe command timed out: {arguments[0]}") from error + except (OSError, subprocess.SubprocessError) as error: + if process is not None and process.poll() is None: + terminate_process_group(process) + raise InfrastructureError(f"bounded probe command could not complete: {arguments[0]}") from error + + +def cli( + runtime: Path, + home: Path, + *arguments: str, + environment: dict[str, str] | None = None, +) -> subprocess.CompletedProcess[bytes]: + return run_command( + [ + sys.executable, + "-I", + "-B", + str(runtime / "scripts" / "agent-lab.py"), + "--home", + str(home), + *arguments, + ], + environment=environment, + ) + + +def initialized_home(runtime: Path, probe_root: Path, name: str) -> Path: + home = probe_root / name + completed = cli(runtime, home, "init") + if completed.returncode != 0 or completed.stdout != b"changed:true\n" or completed.stderr: + raise InfrastructureError( + "private runtime home initialization failed: " + + completed.stderr.decode("utf-8", errors="replace") + ) + return home + + +def json_object(completed: subprocess.CompletedProcess[bytes], purpose: str) -> dict[str, object]: + if completed.returncode != 0 or completed.stderr: + raise InfrastructureError( + f"{purpose} setup failed: " + completed.stderr.decode("utf-8", errors="replace") + ) + try: + value = json.loads(completed.stdout) + except (UnicodeError, json.JSONDecodeError) as error: + raise InfrastructureError(f"{purpose} returned malformed JSON") from error + if not isinstance(value, dict): + raise InfrastructureError(f"{purpose} returned a non-object") + return value + + +def add_mapping(runtime: Path, home: Path, name: str = "vendor.worker", subject: str = SUBJECT) -> dict[str, object]: + return json_object(cli(runtime, home, "image", "add", name, subject), "catalog add") + + +def marker_environment(marker: Path | None) -> dict[str, str] | None: + if marker is None: + return None + return {"AGENT_LAB_MUTATION_MARK": str(marker)} + + +def probe_invalid_oci(runtime: Path, probe_root: Path, marker: Path | None) -> ProbeResult: + home = initialized_home(runtime, probe_root, "oci-home") + before = tree_fingerprint(home) + invalid_subject = "registry.example/operator/worker@sha256:" + "A" * 64 + completed = cli( + runtime, + home, + "image", + "add", + "vendor.worker", + invalid_subject, + environment=marker_environment(marker), + ) + after = tree_fingerprint(home) + if completed.returncode not in (0, 1): + raise InfrastructureError(f"OCI grammar probe returned {completed.returncode}") + secure = completed.returncode == 1 and not completed.stdout and before == after + return ProbeResult(secure, f"rc={completed.returncode} changed={before != after}") + + +def probe_reserved_shadow(runtime: Path, probe_root: Path, marker: Path | None) -> ProbeResult: + module = load_catalog_module( + runtime, + f"agent_lab_image_catalog_shadow_mutation_{os.getpid()}_{id(probe_root)}", + ) + value = { + "apiVersion": module.ENTRY_API, + "generation": 1, + "name": "agent-lab.worker", + "previousEntryDigest": None, + "state": "active", + "subject": SUBJECT, + "subjectDigest": SUBJECT.rsplit("@", 1)[1], + } + if marker is not None: + os.environ["AGENT_LAB_MUTATION_MARK"] = str(marker) + try: + module._entry_schema(value) + except Exception as error: + if isinstance(error, getattr(module, "CatalogInfrastructure", ())): + return ProbeResult(True, "reserved local entry rejected") + raise InfrastructureError( + f"reserved-shadow probe raised an uncontained error: {error}" + ) from error + finally: + if marker is not None: + os.environ.pop("AGENT_LAB_MUTATION_MARK", None) + return ProbeResult(False, "reserved local entry accepted") + + +def probe_cas(runtime: Path, probe_root: Path, marker: Path | None) -> ProbeResult: + home = initialized_home(runtime, probe_root, "cas-home") + active = add_mapping(runtime, home) + active_digest = active.get("entryDigest") + if not isinstance(active_digest, str): + raise InfrastructureError("catalog add omitted its entry digest") + before = tree_fingerprint(home / "images") + completed = cli( + runtime, + home, + "image", + "remove", + "vendor.worker", + "--expect", + STALE_ENTRY, + environment=marker_environment(marker), + ) + inspected = cli(runtime, home, "image", "inspect", "vendor.worker") + if completed.returncode not in (0, 1) or inspected.returncode != 0: + raise InfrastructureError( + f"CAS probe returned remove={completed.returncode} inspect={inspected.returncode}" + ) + try: + record = json.loads(inspected.stdout) + except (UnicodeError, json.JSONDecodeError) as error: + raise InfrastructureError("CAS probe inspect returned malformed JSON") from error + after = tree_fingerprint(home / "images") + secure = ( + completed.returncode == 1 + and not completed.stdout + and isinstance(record, dict) + and record.get("state") == "active" + and record.get("entryDigest") == active_digest + and before == after + ) + return ProbeResult(secure, f"rc={completed.returncode} changed={before != after}") + + +def probe_symlink_authority(runtime: Path, probe_root: Path, marker: Path | None) -> ProbeResult: + home = initialized_home(runtime, probe_root, "authority-home") + add_mapping(runtime, home) + pointer = home / "images" / "catalog" / "current.json" + outside_pointer = probe_root / "outside-current.json" + try: + pointer.rename(outside_pointer) + os.symlink(outside_pointer, pointer) + except OSError as error: + raise InfrastructureError("authority pointer symlink could not be created") from error + outside_before = tree_fingerprint(outside_pointer) + completed = cli( + runtime, + home, + "image", + "list", + environment=marker_environment(marker), + ) + outside_after = tree_fingerprint(outside_pointer) + if completed.returncode not in (0, 125): + raise InfrastructureError(f"symlink authority probe returned {completed.returncode}") + secure = completed.returncode == 125 and not completed.stdout and outside_before == outside_after + return ProbeResult(secure, f"rc={completed.returncode} external_changed={outside_before != outside_after}") + + +def install_cue_fixture(home: Path) -> None: + source = REPO_ROOT / ".cache" / "dev" / "tools" / "cue" + destination = home / "cache" / "tools" / "cue" + if not source.is_dir(): + raise InfrastructureError("pinned CUE fixture is unavailable") + try: + shutil.copytree(source, destination, dirs_exist_ok=True) + except OSError as error: + raise InfrastructureError("pinned CUE fixture could not be privately copied") from error + + +def write_local_artifact(path: Path) -> None: + data = ( + 'package experiment\n\n' + 'experiment: {\n' + ' apiVersion: "agent-lab/v0alpha1"\n' + ' kind: "Experiment"\n' + ' metadata: name: "mutation-binding"\n' + ' spec: members: [{\n' + ' name: "worker"\n' + ' image: catalogName: "vendor.worker"\n' + ' }]\n' + '}\n' + ) + try: + path.mkdir(mode=0o700) + (path / "experiment.cue").write_text(data, encoding="utf-8") + os.chmod(path / "experiment.cue", 0o600) + except OSError as error: + raise InfrastructureError("binding probe artifact could not be created") from error + + +def probe_selected_binding(runtime: Path, probe_root: Path, marker: Path | None) -> ProbeResult: + home = initialized_home(runtime, probe_root, "binding-home") + install_cue_fixture(home) + active = add_mapping(runtime, home) + entry_digest = active.get("entryDigest") + if not isinstance(entry_digest, str): + raise InfrastructureError("binding probe setup omitted its entry digest") + artifact = probe_root / "binding-artifact" + write_local_artifact(artifact) + completed = cli( + runtime, + home, + "experiment", + "check", + str(artifact), + environment=marker_environment(marker), + ) + if completed.returncode != 0: + raise InfrastructureError(f"selected-binding probe returned {completed.returncode}") + try: + value = json.loads(completed.stdout) + resolved = value["plan"]["spec"]["members"][0]["resolvedImage"] + except (UnicodeError, json.JSONDecodeError, KeyError, TypeError, IndexError) as error: + return ProbeResult(False, f"selected binding is absent or malformed: {error}") + secure = resolved == { + "entryDigest": entry_digest, + "generation": 1, + "origin": "local", + "subject": SUBJECT, + } + return ProbeResult(secure, f"resolved={resolved!r}") + + +def make_canary(path: Path, name: str) -> None: + script = '#!/bin/sh\nset -eu\n: > "$CANARY_DIR/' + name + '"\n' + try: + path.write_text(script, encoding="utf-8") + os.chmod(path, 0o700) + except OSError as error: + raise InfrastructureError(f"forbidden-effect canary could not be created: {name}") from error + + +def probe_no_forbidden_effect(runtime: Path, probe_root: Path, marker: Path | None) -> ProbeResult: + home = initialized_home(runtime, probe_root, "no-effect-home") + canary_bin = probe_root / "canary-bin" + canary_marks = probe_root / "canary-marks" + canary_bin.mkdir(mode=0o700) + canary_marks.mkdir(mode=0o700) + names = ("docker", "git", "curl", "wget") + for name in names: + make_canary(canary_bin / name, name) + calibrated = run_command( + [str(canary_bin / name)], + environment={"CANARY_DIR": str(canary_marks)}, + ) + if calibrated.returncode != 0 or not (canary_marks / name).is_file(): + raise InfrastructureError(f"forbidden-effect canary did not calibrate: {name}") + for path in canary_marks.iterdir(): + path.unlink() + environment = { + "PATH": f"{canary_bin}:/usr/bin:/bin", + "CANARY_DIR": str(canary_marks), + } + if marker is not None: + environment["AGENT_LAB_MUTATION_MARK"] = str(marker) + completed = cli( + runtime, + home, + "image", + "add", + "vendor.worker", + SUBJECT, + environment=environment, + ) + if completed.returncode != 0: + raise InfrastructureError(f"no-effect probe catalog add returned {completed.returncode}") + fired = tuple(sorted(path.name for path in canary_marks.iterdir())) + return ProbeResult(not fired, f"forbidden_effects={fired!r}") + + +def probe_no_admission_authority( + runtime: Path, + probe_root: Path, + marker: Path | None, +) -> ProbeResult: + home = initialized_home(runtime, probe_root, "no-admission-authority-home") + completed = cli( + runtime, + home, + "image", + "add", + "vendor.worker", + SUBJECT, + environment=marker_environment(marker), + ) + result = json_object(completed, "catalog admission-boundary probe") + secure = ( + set(result) == {"changed", "entryDigest", "generation"} + and result.get("changed") is True + and type(result.get("generation")) is int + and result.get("generation") == 1 + and isinstance(result.get("entryDigest"), str) + ) + return ProbeResult(secure, f"result_keys={tuple(sorted(result))!r}") + + +def probe_unknown_staging(runtime: Path, probe_root: Path, marker: Path | None) -> ProbeResult: + home = initialized_home(runtime, probe_root, "staging-home") + wrapper = home / "images" / ".staging" / "foreign-wrapper" + try: + wrapper.mkdir(mode=0o700) + (wrapper / "unknown").write_bytes(b"foreign\n") + os.chmod(wrapper / "unknown", 0o600) + except OSError as error: + raise InfrastructureError("unknown staging fixture could not be created") from error + before = tree_fingerprint(wrapper) + completed = cli( + runtime, + home, + "image", + "add", + "vendor.worker", + SUBJECT, + environment=marker_environment(marker), + ) + after = tree_fingerprint(wrapper) + if completed.returncode not in (0, 125): + raise InfrastructureError(f"unknown-staging probe returned {completed.returncode}") + secure = ( + completed.returncode == 125 + and not completed.stdout + and before == after + and not (home / "images" / "catalog").exists() + ) + return ProbeResult(secure, f"rc={completed.returncode} foreign_changed={before != after}") + + +def load_catalog_module(runtime: Path, name: str): + path = runtime / "scripts" / "image_catalog.py" + spec = spec_from_file_location(name, path) + if spec is None or spec.loader is None: + raise InfrastructureError("private catalog module cannot be loaded") + module = module_from_spec(spec) + sys.modules[name] = module + previous = sys.dont_write_bytecode + sys.dont_write_bytecode = True + try: + spec.loader.exec_module(module) + except (ImportError, OSError, SyntaxError) as error: + raise InfrastructureError("private catalog module import failed") from error + finally: + sys.dont_write_bytecode = previous + return module + + +def probe_durability(runtime: Path, probe_root: Path, marker: Path | None) -> ProbeResult: + """Verify immutable record files are synced before current-pointer publication.""" + + home = initialized_home(runtime, probe_root, "durability-home") + add_mapping(runtime, home) + module = load_catalog_module(runtime, f"agent_lab_image_catalog_mutation_{os.getpid()}_{id(runtime)}") + events: list[tuple[str, str]] = [] + original_fsync = module.os.fsync + original_replace = module.os.replace + + def observed_fsync(descriptor: int) -> None: + try: + target = os.readlink(f"/proc/self/fd/{descriptor}") + except OSError: + target = f"fd:{descriptor}" + events.append(("fsync", target)) + original_fsync(descriptor) + + def observed_replace(source: os.PathLike[str] | str, target: os.PathLike[str] | str) -> None: + events.append(("replace", os.fspath(target))) + original_replace(source, target) + + module.os.fsync = observed_fsync + module.os.replace = observed_replace + if marker is not None: + os.environ["AGENT_LAB_MUTATION_MARK"] = str(marker) + try: + module.add_image(home, "vendor.second", OTHER_SUBJECT) + except Exception as error: + if isinstance(error, getattr(module, "CatalogReject", ())): + raise InfrastructureError(f"durability probe was rejected: {error}") from error + if isinstance(error, getattr(module, "CatalogInfrastructure", ())): + raise InfrastructureError(f"durability probe returned infrastructure failure: {error}") from error + raise InfrastructureError(f"durability probe raised an uncontained error: {error}") from error + finally: + module.os.fsync = original_fsync + module.os.replace = original_replace + if marker is not None: + os.environ.pop("AGENT_LAB_MUTATION_MARK", None) + + pointer_indexes = [ + index + for index, event in enumerate(events) + if event[0] == "replace" + and Path(event[1]) == home / "images" / "catalog" / "current.json" + ] + entry_syncs = [ + index + for index, event in enumerate(events) + if event[0] == "fsync" and event[1].endswith("/payload/entry.json") + ] + snapshot_syncs = [ + index + for index, event in enumerate(events) + if event[0] == "fsync" and event[1].endswith("/payload/snapshot.json") + ] + secure = ( + len(pointer_indexes) == 1 + and bool(entry_syncs) + and bool(snapshot_syncs) + and max(entry_syncs) < pointer_indexes[0] + and max(snapshot_syncs) < pointer_indexes[0] + ) + return ProbeResult( + secure, + f"entry_syncs={entry_syncs} snapshot_syncs={snapshot_syncs} pointer={pointer_indexes}", + ) + + +def probe_atomic_publication(runtime: Path, probe_root: Path, marker: Path | None) -> ProbeResult: + """Reject a producer that can expose a partial digest-addressed final record.""" + + home = initialized_home(runtime, probe_root, "atomic-publication-home") + add_mapping(runtime, home) + environment = {"AGENT_LAB_MUTATION_MARK": str(marker)} if marker is not None else None + attempted = cli( + runtime, + home, + "image", + "add", + "vendor.second", + OTHER_SUBJECT, + environment=environment, + ) + observed = cli(runtime, home, "image", "list") + try: + records = json.loads(observed.stdout) if observed.returncode == 0 else [] + except json.JSONDecodeError: + records = [] + secure = ( + attempted.returncode == 0 + and observed.returncode == 0 + and [record.get("name") for record in records] == ["vendor.second", "vendor.worker"] + ) + return ProbeResult( + secure, + f"add_rc={attempted.returncode} list_rc={observed.returncode} records={records!r}", + ) + + +def apply_mutation(runtime: Path, mutation: Mutation) -> None: + path = runtime / mutation.path + try: + source = path.read_text(encoding="utf-8") + except (OSError, UnicodeError) as error: + raise InfrastructureError(f"mutation source cannot be read: {mutation.path}") from error + occurrences = source.count(mutation.old) + if occurrences != 1: + raise InfrastructureError( + f"{mutation.assertion} replacement applicability is {occurrences}, expected exactly 1" + ) + mutated = source.replace(mutation.old, mutation.new, 1) + if mutated.count(mutation.new) != 1 or mutated == source: + raise InfrastructureError(f"{mutation.assertion} replacement result is ambiguous") + try: + path.write_text(mutated, encoding="utf-8") + except OSError as error: + raise InfrastructureError(f"{mutation.assertion} private source cannot be written") from error + if path.read_text(encoding="utf-8") != mutated: + raise InfrastructureError(f"{mutation.assertion} private source write was not exact") + + +def compile_mutation(runtime: Path, mutation: Mutation, cache: Path) -> None: + completed = run_command( + [ + sys.executable, + "-I", + "-X", + f"pycache_prefix={cache}", + "-m", + "py_compile", + str(runtime / mutation.path), + ] + ) + if completed.returncode != 0 or completed.stderr: + raise InfrastructureError( + f"{mutation.assertion} private mutation does not compile: " + + completed.stderr.decode("utf-8", errors="replace") + ) + + +# Each source rewrite must match exactly once in the copied runtime. The marker write proves the +# altered branch executed; it is confined to the mutation's temporary directory. +MUTATIONS: tuple[Mutation, ...] = ( + Mutation( + "M-CAT-OCI-001", + "scripts/image_reference.py", + ' and OCI_SUBJECT.fullmatch(value) is not None\n', + ( + ' and (\n' + ' __import__("pathlib").Path(\n' + ' __import__("os").environ["AGENT_LAB_MUTATION_MARK"]\n' + ' ).touch()\n' + ' or OCI_SUBJECT.fullmatch(value.lower()) is not None\n' + ' )\n' + ), + probe_invalid_oci, + "the invalid-OCI oracle detects an uppercase digest admitted by the shared parser", + ), + Mutation( + "M-CAT-SHADOW-001", + "scripts/image_catalog.py", + ( + ' or not isinstance(name, str)\n' + ' or name.startswith("agent-lab.")\n' + ' or not oci_subject(subject)\n' + ), + ( + ' or not isinstance(name, str)\n' + ' or (\n' + ' name.startswith("agent-lab.")\n' + ' and Path(os.environ["AGENT_LAB_MUTATION_MARK"]).touch() is not None\n' + ' )\n' + ' or not oci_subject(subject)\n' + ), + probe_reserved_shadow, + "the reserved-name oracle detects a local agent-lab.* entry admitted by its closed schema", + ), + Mutation( + "M-CAT-CAS-001", + "scripts/image_catalog.py", + ( + ' if prior["entryDigest"] != expected_entry_digest:\n' + ' _reject("remove compare-and-swap conflict")\n' + ' subject = prior["subject"]\n' + ' assert isinstance(subject, str)\n' + ' entry, entry_digest, snapshot, snapshot_digest, intent = _candidate_values(\n' + ' state,\n' + ' kind="remove",\n' + ' name=name,\n' + ' subject=subject,\n' + ' expected_entry_digest=expected_entry_digest,\n' + ' limits=limits,\n' + ' )\n' + ), + ( + ' Path(os.environ["AGENT_LAB_MUTATION_MARK"]).touch()\n' + ' subject = prior["subject"]\n' + ' assert isinstance(subject, str)\n' + ' entry, entry_digest, snapshot, snapshot_digest, intent = _candidate_values(\n' + ' state,\n' + ' kind="remove",\n' + ' name=name,\n' + ' subject=subject,\n' + ' expected_entry_digest=str(prior["entryDigest"]),\n' + ' limits=limits,\n' + ' )\n' + ), + probe_cas, + "the stale-token oracle detects removal with a substituted CAS token", + ), + Mutation( + "M-CAT-AUTH-001", + "scripts/image_catalog.py", + ( + 'def _read_file(path: Path, maximum: int, purpose: str) -> bytes:\n' + ' try:\n' + ' lexical = path.lstat()\n' + ), + ( + 'def _read_file(path: Path, maximum: int, purpose: str) -> bytes:\n' + ' if purpose == "local image current pointer" and path.is_symlink():\n' + ' Path(os.environ["AGENT_LAB_MUTATION_MARK"]).touch()\n' + ' return path.read_bytes()\n' + ' try:\n' + ' lexical = path.lstat()\n' + ), + probe_symlink_authority, + "the authority oracle detects a followed current-pointer symlink", + ), + Mutation( + "M-RES-BIND-001", + "scripts/experiment.py", + ' "entryDigest": record["entryDigest"],\n', + ( + ' "entryDigest": (\n' + ' Path(os.environ["AGENT_LAB_MUTATION_MARK"]).touch()\n' + ' or "sha256:" + "0" * 64\n' + ' ),\n' + ), + probe_selected_binding, + "the plan-binding oracle detects substitution of the selected entry digest", + ), + Mutation( + "M-CAT-NOEF-001", + "scripts/agent-lab.py", + ' result = catalog.add_image(home, argv[1], argv[2])\n', + ( + ' Path(os.environ["AGENT_LAB_MUTATION_MARK"]).touch()\n' + ' subprocess.run(("docker",), check=False)\n' + ' result = catalog.add_image(home, argv[1], argv[2])\n' + ), + probe_no_forbidden_effect, + "the calibrated no-effect oracle detects an injected Engine command", + ), + Mutation( + "M-CAT-ADMIT-001", + "scripts/image_catalog.py", + ' return {"changed": True, "entryDigest": entry_digest, "generation": 1}\n', + ( + ' Path(os.environ["AGENT_LAB_MUTATION_MARK"]).touch()\n' + ' return {\n' + ' "admitted": True,\n' + ' "changed": True,\n' + ' "entryDigest": entry_digest,\n' + ' "generation": 1,\n' + ' }\n' + ), + probe_no_admission_authority, + "the result-schema oracle detects catalog membership presented as admission authority", + ), + Mutation( + "M-CAT-ATOM-001", + "scripts/image_catalog.py", + ( + ' _fault(fault, f"{purpose}.before_noreplace")\n' + ' _rename_noreplace(source, target)\n' + ' _fault(fault, f"{purpose}.after_noreplace")\n' + ), + ( + ' _fault(fault, f"{purpose}.before_noreplace")\n' + ' Path(os.environ["AGENT_LAB_MUTATION_MARK"]).touch()\n' + ' descriptor = os.open(target, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)\n' + ' try:\n' + ' _write_all(descriptor, data[: max(1, len(data) // 2)])\n' + ' finally:\n' + ' os.close(descriptor)\n' + ' _fault(fault, f"{purpose}.after_noreplace")\n' + ), + probe_atomic_publication, + "the atomicity oracle detects a partial digest-addressed final record", + ), + Mutation( + "M-CAT-DUR-001", + "scripts/image_catalog.py", + ( + ' _write_all(descriptor, data)\n' + ' _fault(fault, f"{purpose}.before_fsync")\n' + ' os.fsync(descriptor)\n' + ' _fault(fault, f"{purpose}.after_fsync")\n' + ' metadata = os.fstat(descriptor)\n' + ), + ( + ' _write_all(descriptor, data)\n' + ' mutation_marker = os.environ.get("AGENT_LAB_MUTATION_MARK")\n' + ' if mutation_marker is not None:\n' + ' Path(mutation_marker).touch()\n' + ' _fault(fault, f"{purpose}.before_fsync")\n' + ' _fault(fault, f"{purpose}.after_fsync")\n' + ' metadata = os.fstat(descriptor)\n' + ), + probe_durability, + "the durability oracle detects publication without immutable-file fsync", + ), + Mutation( + "M-CAT-STAGE-001", + "scripts/image_catalog.py", + ( + ' if names != (CLEANUP_WRAPPER,):\n' + ' _infra("catalog staging root contains an unknown wrapper")\n' + ' cleanup = authority.staging / CLEANUP_WRAPPER\n' + ), + ( + ' if names != (CLEANUP_WRAPPER,):\n' + ' Path(os.environ["AGENT_LAB_MUTATION_MARK"]).touch()\n' + ' for name in names:\n' + ' _remove_owned_tree(authority.staging / name)\n' + ' _fsync_directory(authority.staging, "catalog staging root")\n' + ' return None\n' + ' cleanup = authority.staging / CLEANUP_WRAPPER\n' + ), + probe_unknown_staging, + "the staging oracle detects broad deletion of an unknown wrapper", + ), +) + + +def execute_mutation(root: Path, names: tuple[str, ...], mutation: Mutation) -> tuple[bool, str]: + runtime = root / "runtime" + copy_runtime(runtime, names) + copied = runtime_fingerprint(runtime, names) + copied_tree = tree_fingerprint(runtime) + pristine = mutation.probe(runtime, root / "pristine", None) + if not pristine.secure: + raise InfrastructureError( + f"{mutation.assertion} pristine probe is not GREEN: {pristine.detail}" + ) + if runtime_fingerprint(runtime, names) != copied: + raise InfrastructureError(f"{mutation.assertion} pristine probe changed its runtime copy") + if tree_fingerprint(runtime) != copied_tree: + raise InfrastructureError(f"{mutation.assertion} pristine probe changed runtime topology") + + apply_mutation(runtime, mutation) + compile_mutation(runtime, mutation, root / "pycache") + mutated = runtime_fingerprint(runtime, names) + mutated_tree = tree_fingerprint(runtime) + changed = [name for (name, before), (_, after) in zip(copied, mutated) if before != after] + if changed != [mutation.path]: + raise InfrastructureError( + f"{mutation.assertion} changed unexpected private runtime paths: {changed!r}" + ) + + marker = root / "mutation-reached" + result = mutation.probe(runtime, root / "mutant", marker) + if not marker.is_file(): + raise InfrastructureError(f"{mutation.assertion} did not prove its mutated path was reached") + if runtime_fingerprint(runtime, names) != mutated: + raise InfrastructureError(f"{mutation.assertion} mutant probe changed its runtime copy") + if tree_fingerprint(runtime) != mutated_tree: + raise InfrastructureError(f"{mutation.assertion} mutant probe changed runtime topology") + return not result.secure, result.detail + + +def main() -> int: + try: + names = manifest_paths() + shared_before = runtime_fingerprint(REPO_ROOT, names) + if not MUTATIONS: + raise InfrastructureError("catalog mutation declarations are unavailable") + expected = ( + "M-CAT-OCI-001", + "M-CAT-SHADOW-001", + "M-CAT-CAS-001", + "M-CAT-AUTH-001", + "M-RES-BIND-001", + "M-CAT-NOEF-001", + "M-CAT-ADMIT-001", + "M-CAT-ATOM-001", + "M-CAT-DUR-001", + "M-CAT-STAGE-001", + ) + if tuple(mutation.assertion for mutation in MUTATIONS) != expected: + raise InfrastructureError("catalog mutation assertion identity drift") + + failures = 0 + observed: list[str] = [] + for mutation in MUTATIONS: + try: + temporary = Path( + tempfile.mkdtemp( + prefix=f"agent-lab-{mutation.assertion.lower()}-", + dir="/tmp", + ) + ) + except OSError as error: + raise InfrastructureError( + f"{mutation.assertion} private mutation root is unavailable" + ) from error + try: + detected, detail = execute_mutation(temporary, names, mutation) + observed.append(mutation.assertion) + if detected: + print(f"PASS {mutation.assertion} {mutation.message}") + else: + failures += 1 + print(f"FAIL {mutation.assertion} {mutation.message} ({detail})") + finally: + cleanup_error: OSError | None = None + try: + shutil.rmtree(temporary) + except OSError as error: + cleanup_error = error + if runtime_fingerprint(REPO_ROOT, names) != shared_before: + raise InfrastructureError( + f"{mutation.assertion} changed the shared checkout runtime fingerprint" + ) + if cleanup_error is not None: + raise InfrastructureError( + f"{mutation.assertion} private mutation cleanup is uncertain" + ) from cleanup_error + if temporary.exists() or temporary.is_symlink(): + raise InfrastructureError( + f"{mutation.assertion} private mutation cleanup was incomplete" + ) + if tuple(observed) != expected: + raise InfrastructureError("catalog mutation execution identity drift") + print(f"SUMMARY assertions=10 expected=10 failures={failures} infra=0") + return 0 if failures == 0 else 1 + except InfrastructureError as error: + print(f"INFRA catalog mutation evidence: {error}", file=sys.stderr) + return 125 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/image/catalog-state-cases.py b/tests/image/catalog-state-cases.py new file mode 100755 index 0000000..2de19d3 --- /dev/null +++ b/tests/image/catalog-state-cases.py @@ -0,0 +1,1804 @@ +#!/usr/bin/env python3 +"""Adversarial local image-catalog filesystem and durability cases.""" + +from __future__ import annotations + +from contextlib import redirect_stderr, redirect_stdout +import hashlib +from importlib.util import module_from_spec, spec_from_file_location +import io +import json +import os +from pathlib import Path +import shutil +import stat +import subprocess +import sys +import tempfile +import time + + +REPO_ROOT = Path(__file__).resolve().parents[2] +AGENT_LAB = REPO_ROOT / "scripts" / "agent-lab" +AGENT_LAB_MODULE = REPO_ROOT / "scripts" / "agent-lab.py" +CATALOG_MODULE = REPO_ROOT / "scripts" / "image_catalog.py" +SUBJECT = "registry.example/operator/worker@sha256:" + "a" * 64 +OTHER_SUBJECT = "registry.example/operator/other@sha256:" + "b" * 64 +THIRD_SUBJECT = "registry.example/operator/third@sha256:" + "c" * 64 +ENTRY_DOMAIN = b"agent-lab.local-image-entry.v1\0" +SNAPSHOT_DOMAIN = b"agent-lab.local-image-snapshot.v1\0" + + +def canonical(value: object) -> bytes: + return json.dumps(value, ensure_ascii=True, separators=(",", ":"), sort_keys=True).encode() + + +def digest(domain: bytes, value: object) -> str: + return "sha256:" + hashlib.sha256(domain + canonical(value)).hexdigest() + + +def load_module(): + spec = spec_from_file_location("agent_lab_catalog_state", AGENT_LAB_MODULE) + if spec is None or spec.loader is None: + raise RuntimeError("Agent Lab application module cannot be loaded") + module = module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +MODULE = load_module() + + +def load_catalog_module(): + spec = spec_from_file_location("agent_lab_catalog_contract", CATALOG_MODULE) + if spec is None or spec.loader is None: + raise RuntimeError("Agent Lab catalog module cannot be loaded") + module = module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +CATALOG = load_catalog_module() +FAILURES = 0 +OBSERVED: list[str] = [] +CLI_CALLS = 0 + + +class MatrixInfrastructure(RuntimeError): + """Crash-matrix setup could not establish trusted fixture state.""" + + +def check(assertion: str, condition: bool, message: str, detail: str = "") -> None: + global FAILURES + OBSERVED.append(assertion) + if condition: + print(f"PASS {assertion} {message}") + else: + FAILURES += 1 + suffix = f" ({detail})" if detail else "" + print(f"FAIL {assertion} {message}{suffix}") + + +def cli(home: Path, *arguments: str, timeout: float = 5.0) -> subprocess.CompletedProcess[bytes]: + global CLI_CALLS + CLI_CALLS += 1 + environment = { + "PATH": "/usr/bin:/bin", + "LANG": "C", + "LC_ALL": "C", + } + return subprocess.run( + [str(AGENT_LAB), "--home", str(home), *arguments], + stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + env=environment, + timeout=timeout, + check=False, + ) + + +def module_image(home: Path, *arguments: str) -> tuple[int | None, str, str, BaseException | None]: + output = io.StringIO() + errors = io.StringIO() + try: + with redirect_stdout(output), redirect_stderr(errors): + result = MODULE.image_command(home, list(arguments)) + return result, output.getvalue(), errors.getvalue(), None + except BaseException as error: # The assertion records an uncontained production fault as RED. + return None, output.getvalue(), errors.getvalue(), error + + +def module_main(home: Path, arguments: list[str], output: io.TextIOBase | None = None) -> tuple[int | None, BaseException | None]: + stream = output if output is not None else io.StringIO() + errors = io.StringIO() + try: + with redirect_stdout(stream), redirect_stderr(errors): + result = MODULE.main(["--home", str(home), *arguments]) + return result, None + except BaseException as error: # The assertion records an uncontained production fault as RED. + return None, error + + +def new_home(root: Path, name: str) -> Path: + home = root / name + completed = cli(home, "init") + if completed.returncode != 0: + raise RuntimeError(f"temporary home init failed: {completed.stderr.decode(errors='replace')}") + return home + + +def add(home: Path, name: str = "vendor.worker", subject: str = SUBJECT) -> dict[str, object]: + completed = cli(home, "image", "add", name, subject) + if completed.returncode != 0: + raise RuntimeError(f"catalog setup add failed: {completed.stderr.decode(errors='replace')}") + value = json.loads(completed.stdout) + if not isinstance(value, dict): + raise RuntimeError("catalog setup add returned a non-object") + return value + + +def matrix_home(root: Path, name: str) -> Path: + home = root / name + output = io.StringIO() + errors = io.StringIO() + error: BaseException | None = None + returncode: int | None = None + try: + with redirect_stdout(output), redirect_stderr(errors): + returncode = MODULE.main(["--home", str(home), "init"]) + except BaseException as caught: # Setup uncertainty must remain infrastructure. + error = caught + if returncode != 0 or error is not None or errors.getvalue(): + detail = ( + f"init={returncode}:error={type(error).__name__ if error else 'none'}:" + f"stderr={errors.getvalue()!r}" + ) + raise MatrixInfrastructure(detail) from error + return home + + +def matrix_add(home: Path, name: str, subject: str) -> None: + returncode, output, errors, error = module_image(home, "add", name, subject) + try: + value = json.loads(output) if returncode == 0 and error is None else None + except json.JSONDecodeError: + value = None + if errors or not isinstance(value, dict) or value.get("changed") is not True: + detail = ( + f"add={returncode}:error={type(error).__name__ if error else 'none'}:" + f"stderr={errors!r}:value={value!r}" + ) + raise MatrixInfrastructure(detail) from error + + +def current_snapshot(home: Path) -> tuple[Path, dict[str, object], str]: + root = home / "images" / "catalog" + pointer = json.loads((root / "current.json").read_bytes()) + snapshot_digest = pointer["snapshotDigest"] + path = root / "snapshots" / f"{snapshot_digest[7:]}.json" + value = json.loads(path.read_bytes()) + return path, value, snapshot_digest + + +def stable_record(path: Path, maximum: int = 65536) -> bytes | None: + descriptor = -1 + no_follow = getattr(os, "O_NOFOLLOW", None) + if no_follow is None: + return None + try: + before = path.lstat() + if ( + not stat.S_ISREG(before.st_mode) + or stat.S_IMODE(before.st_mode) != 0o600 + or before.st_nlink != 1 + or before.st_uid != os.geteuid() + or before.st_size < 0 + or before.st_size > maximum + ): + return None + descriptor = os.open( + path, + os.O_RDONLY + | no_follow + | getattr(os, "O_CLOEXEC", 0) + | getattr(os, "O_NONBLOCK", 0), + ) + opened = os.fstat(descriptor) + signature = ( + opened.st_dev, + opened.st_ino, + opened.st_mode, + opened.st_nlink, + opened.st_uid, + opened.st_size, + opened.st_mtime_ns, + opened.st_ctime_ns, + ) + if signature != ( + before.st_dev, + before.st_ino, + before.st_mode, + before.st_nlink, + before.st_uid, + before.st_size, + before.st_mtime_ns, + before.st_ctime_ns, + ): + return None + chunks: list[bytes] = [] + observed = 0 + while True: + chunk = os.read(descriptor, min(65536, maximum + 1 - observed)) + if not chunk: + break + chunks.append(chunk) + observed += len(chunk) + if observed > maximum: + return None + after_descriptor = os.fstat(descriptor) + after_path = path.lstat() + after_signature = ( + after_descriptor.st_dev, + after_descriptor.st_ino, + after_descriptor.st_mode, + after_descriptor.st_nlink, + after_descriptor.st_uid, + after_descriptor.st_size, + after_descriptor.st_mtime_ns, + after_descriptor.st_ctime_ns, + ) + path_signature = ( + after_path.st_dev, + after_path.st_ino, + after_path.st_mode, + after_path.st_nlink, + after_path.st_uid, + after_path.st_size, + after_path.st_mtime_ns, + after_path.st_ctime_ns, + ) + if after_signature != signature or path_signature != signature or observed != opened.st_size: + return None + return b"".join(chunks) + except OSError: + return None + finally: + if descriptor >= 0: + try: + os.close(descriptor) + except OSError: + pass + + +def stored_active_names(home: Path) -> list[str] | None: + try: + root = home / "images" / "catalog" + pointer_raw = stable_record(root / "current.json") + if pointer_raw is None: + return None + pointer = json.loads(pointer_raw) + if ( + not isinstance(pointer, dict) + or set(pointer) != {"apiVersion", "snapshotDigest"} + or pointer.get("apiVersion") != "agent-lab.local-image-current/v0alpha1" + or pointer_raw != canonical(pointer) + b"\n" + ): + return None + snapshot_digest = pointer.get("snapshotDigest") + if ( + not isinstance(snapshot_digest, str) + or not snapshot_digest.startswith("sha256:") + or len(snapshot_digest) != 71 + or any(character not in "0123456789abcdef" for character in snapshot_digest[7:]) + ): + return None + snapshot_raw = stable_record( + root / "snapshots" / f"{snapshot_digest[7:]}.json", + maximum=262144, + ) + if snapshot_raw is None: + return None + snapshot = json.loads(snapshot_raw) + if ( + not isinstance(snapshot, dict) + or set(snapshot) + != {"apiVersion", "previousSnapshotDigest", "records", "revision"} + or snapshot.get("apiVersion") != "agent-lab.local-image-snapshot/v0alpha1" + or snapshot_raw != canonical(snapshot) + b"\n" + or digest(SNAPSHOT_DOMAIN, snapshot) != snapshot_digest + or not isinstance(snapshot.get("revision"), int) + or isinstance(snapshot.get("revision"), bool) + or int(snapshot["revision"]) < 1 + ): + return None + records = snapshot.get("records") + if not isinstance(records, dict): + return None + active: list[str] = [] + for name, projection in records.items(): + if ( + not isinstance(name, str) + or not isinstance(projection, dict) + or set(projection) != {"entryDigest", "generation", "state"} + ): + return None + entry_digest = projection.get("entryDigest") + if ( + not isinstance(entry_digest, str) + or not entry_digest.startswith("sha256:") + or len(entry_digest) != 71 + or any(character not in "0123456789abcdef" for character in entry_digest[7:]) + ): + return None + entry_raw = stable_record(root / "entries" / f"{entry_digest[7:]}.json") + if entry_raw is None: + return None + entry = json.loads(entry_raw) + if ( + not isinstance(entry, dict) + or set(entry) + != { + "apiVersion", + "generation", + "name", + "previousEntryDigest", + "state", + "subject", + "subjectDigest", + } + or entry.get("apiVersion") != "agent-lab.local-image-entry/v0alpha1" + or entry_raw != canonical(entry) + b"\n" + or digest(ENTRY_DOMAIN, entry) != entry_digest + or entry.get("name") != name + or entry.get("generation") != projection.get("generation") + or entry.get("state") != projection.get("state") + or not isinstance(entry.get("generation"), int) + or isinstance(entry.get("generation"), bool) + or int(entry["generation"]) < 1 + or entry.get("state") not in ("active", "removed") + or not isinstance(entry.get("subject"), str) + or entry.get("subjectDigest") != str(entry["subject"]).rsplit("@", 1)[-1] + ): + return None + if entry.get("state") == "active": + active.append(name) + return sorted(active, key=lambda item: item.encode("ascii")) + except (OSError, UnicodeError, ValueError, json.JSONDecodeError): + return None + + +def stored_oracle_sensitivity(home: Path, expected: list[str]) -> bool: + try: + snapshot_path, snapshot, _snapshot_digest = current_snapshot(home) + original = snapshot_path.read_bytes() + original_mode = stat.S_IMODE(snapshot_path.lstat().st_mode) + snapshot_path.chmod(0o644) + mode_rejected = stored_active_names(home) is None + snapshot_path.chmod(original_mode) + mutated = dict(snapshot) + revision = mutated.get("revision") + if not isinstance(revision, int) or isinstance(revision, bool): + return False + mutated["revision"] = revision + 1 + snapshot_path.write_bytes(canonical(mutated) + b"\n") + content_rejected = stored_active_names(home) is None + snapshot_path.write_bytes(original) + return ( + mode_rejected + and content_rejected + and stored_active_names(home) == expected + ) + except (OSError, UnicodeError, ValueError, json.JSONDecodeError): + return False + + +def matrix_infrastructure_sensitivity(root: Path, home: Path) -> bool: + original_main = MODULE.main + init_propagated = False + try: + MODULE.main = lambda _arguments: 125 + try: + matrix_home(root, "injected-infrastructure-home") + except MatrixInfrastructure: + init_propagated = True + finally: + MODULE.main = original_main + + original_image_command = MODULE.image_command + add_propagated = False + try: + MODULE.image_command = lambda _home, _arguments: 125 + try: + matrix_add(home, "vendor.infrastructure", THIRD_SUBJECT) + except MatrixInfrastructure: + add_propagated = True + finally: + MODULE.image_command = original_image_command + return init_propagated and add_propagated + + +def fingerprint(root: Path) -> tuple[tuple[str, str, int, int, str], ...]: + if not root.exists() and not root.is_symlink(): + return () + records: list[tuple[str, str, int, int, str]] = [] + paths = [root, *sorted(root.rglob("*"), key=lambda item: os.fsencode(str(item.relative_to(root))))] + for path in paths: + metadata = path.lstat() + relative = "." if path == root else str(path.relative_to(root)) + if stat.S_ISLNK(metadata.st_mode): + kind = "l" + identity = os.readlink(path) + elif stat.S_ISREG(metadata.st_mode): + kind = "f" + identity = hashlib.sha256(path.read_bytes()).hexdigest() + elif stat.S_ISDIR(metadata.st_mode): + kind = "d" + identity = "" + else: + kind = "o" + identity = "" + records.append((relative, kind, stat.S_IMODE(metadata.st_mode), metadata.st_nlink, identity)) + return tuple(records) + + +def write_record(root: Path, domain: bytes, value: dict[str, object]) -> Path: + record_digest = digest(domain, value) + path = root / f"{record_digest[7:]}.json" + path.write_bytes(canonical(value) + b"\n") + path.chmod(0o600) + return path + + +def hard_exit_add( + home: Path, + name: str, + subject: str, + point: str, + *, + limits=None, +) -> int: + pid = os.fork() + if pid == 0: + def stop_at(observed: str) -> None: + if observed == point: + os._exit(99) + + try: + keyword = {} if limits is None else {"limits": limits} + CATALOG.add_image(home, name, subject, fault=stop_at, **keyword) + except BaseException: + os._exit(98) + os._exit(97) + deadline = time.monotonic() + 5.0 + while time.monotonic() < deadline: + waited, status = os.waitpid(pid, os.WNOHANG) + if waited == pid: + return os.waitstatus_to_exitcode(status) + time.sleep(0.01) + os.kill(pid, 9) + os.waitpid(pid, 0) + return 124 + + +class BrokenOutput(io.StringIO): + def write(self, value: str) -> int: + raise OSError("injected result-output failure") + + +class HardExitOutput(io.StringIO): + def __init__(self, point: str) -> None: + super().__init__() + self.point = point + + def write(self, value: str) -> int: + if self.point == "result.before_write": + os._exit(99) + written = super().write(value) + if self.point == "result.after_write": + os._exit(99) + return written + + +def hard_exit_result(home: Path, point: str) -> int: + pid = os.fork() + if pid == 0: + output = HardExitOutput(point) + try: + with redirect_stdout(output), redirect_stderr(io.StringIO()): + MODULE.main(["--home", str(home), "image", "add", "vendor.worker", SUBJECT]) + except BaseException: + os._exit(98) + os._exit(97) + deadline = time.monotonic() + 5.0 + while time.monotonic() < deadline: + waited, status = os.waitpid(pid, os.WNOHANG) + if waited == pid: + return os.waitstatus_to_exitcode(status) + time.sleep(0.01) + os.kill(pid, 9) + os.waitpid(pid, 0) + return 124 + + +def hard_exit_cleanup( + home: Path, + name: str, + subject: str, + operation: str, +) -> int: + pid = os.fork() + if pid == 0: + original_unlink = Path.unlink + original_rmdir = Path.rmdir + + def stop_after_unlink(path: Path, *args, **kwargs) -> None: + original_unlink(path, *args, **kwargs) + if path.name == "intent.json": + os._exit(99) + + def stop_after_rmdir(path: Path, *args, **kwargs) -> None: + original_rmdir(path, *args, **kwargs) + if path.name == "payload": + os._exit(99) + + if operation == "intent-unlink": + Path.unlink = stop_after_unlink + elif operation == "payload-rmdir": + Path.rmdir = stop_after_rmdir + else: + os._exit(96) + try: + CATALOG.add_image(home, name, subject) + except BaseException: + os._exit(98) + os._exit(97) + deadline = time.monotonic() + 5.0 + while time.monotonic() < deadline: + waited, status = os.waitpid(pid, os.WNOHANG) + if waited == pid: + return os.waitstatus_to_exitcode(status) + time.sleep(0.01) + os.kill(pid, 9) + os.waitpid(pid, 0) + return 124 + + +def catalog_add_result(home: Path, name: str, subject: str, *, limits=None): + keyword = {} if limits is None else {"limits": limits} + try: + return 0, CATALOG.add_image(home, name, subject, **keyword), None + except CATALOG.CatalogReject as error: + return 1, None, error + except CATALOG.CatalogInfrastructure as error: + return 125, None, error + except BaseException as error: # The assertion records an uncontained production fault as RED. + return None, None, error + + +def traced_catalog_add(home: Path, name: str, subject: str): + targets: list[str] = [] + original_fsync = CATALOG.os.fsync + + def record_fsync(descriptor: int) -> None: + try: + targets.append(os.readlink(f"/proc/self/fd/{descriptor}")) + except OSError: + targets.append("") + original_fsync(descriptor) + + CATALOG.os.fsync = record_fsync + try: + rc, value, error = catalog_add_result(home, name, subject) + finally: + CATALOG.os.fsync = original_fsync + return rc, value, error, targets + + +def main() -> int: + with tempfile.TemporaryDirectory(prefix="agent-lab-catalog-state-") as directory: + root = Path(directory) + + missing_home = new_home(root, "missing-home") + add(missing_home) + committed = missing_home / "images" / "catalog" + saved_catalog = root / "saved-committed-catalog" + committed.rename(saved_catalog) + completed = cli(missing_home, "image", "list") + check( + "CAT-STATE-001", + completed.returncode == 125 and completed.stdout == b"" and not committed.exists(), + "a missing previously committed catalog is infrastructure uncertainty, not empty", + f"rc={completed.returncode} stdout={completed.stdout!r}", + ) + + history_home = new_home(root, "history-home") + add(history_home) + entries = history_home / "images" / "catalog" / "entries" + entries.rename(root / "removed-entry-history") + completed = cli(history_home, "image", "list") + check( + "CAT-STATE-002", + completed.returncode == 125 and completed.stdout == b"", + "missing immutable entry history fails closed", + f"rc={completed.returncode} stdout={completed.stdout!r}", + ) + + canonical_home = new_home(root, "canonical-home") + add(canonical_home) + snapshot_path, snapshot, _ = current_snapshot(canonical_home) + snapshot_path.write_text(json.dumps(snapshot, indent=2, sort_keys=True) + "\n", encoding="utf-8") + completed = cli(canonical_home, "image", "list") + check( + "CAT-STATE-003", + completed.returncode == 125, + "noncanonical snapshot bytes fail closed even when semantic digest fields match", + f"rc={completed.returncode}", + ) + + corrupt_home = new_home(root, "corrupt-home") + add(corrupt_home) + _, snapshot, _ = current_snapshot(corrupt_home) + records = snapshot.get("records") + if not isinstance(records, dict) or not isinstance(records.get("vendor.worker"), dict): + raise RuntimeError("catalog setup snapshot has an unexpected shape") + records["vendor.worker"]["state"] = "future" + corrupt_digest = digest(SNAPSHOT_DOMAIN, snapshot) + corrupt_path = corrupt_home / "images" / "catalog" / "snapshots" / f"{corrupt_digest[7:]}.json" + corrupt_path.write_bytes(canonical(snapshot) + b"\n") + (corrupt_home / "images" / "catalog" / "current.json").write_bytes( + canonical( + { + "apiVersion": "agent-lab.local-image-current/v0alpha1", + "snapshotDigest": corrupt_digest, + } + ) + + b"\n" + ) + completed = cli(corrupt_home, "image", "list") + check( + "CAT-STATE-004", + completed.returncode == 125 and completed.stdout == b"", + "closed record validation rejects an unknown state even under a recomputed snapshot digest", + f"rc={completed.returncode} stdout={completed.stdout!r}", + ) + + mode_home = new_home(root, "mode-home") + add(mode_home) + current = mode_home / "images" / "catalog" / "current.json" + current.chmod(0o644) + completed = cli(mode_home, "image", "list") + check( + "CAT-STATE-005", + completed.returncode == 125, + "wrong catalog record mode fails closed", + f"rc={completed.returncode}", + ) + + link_count_home = new_home(root, "link-count-home") + add(link_count_home) + current = link_count_home / "images" / "catalog" / "current.json" + os.link(current, root / "second-current-link") + completed = cli(link_count_home, "image", "list") + check( + "CAT-STATE-006", + completed.returncode == 125, + "multiply linked catalog authority fails closed", + f"rc={completed.returncode}", + ) + + source_home = new_home(root, "symlink-source-home") + add(source_home) + outside_catalog = root / "outside-catalog" + (source_home / "images" / "catalog").rename(outside_catalog) + symlink_home = new_home(root, "symlink-home") + os.symlink(outside_catalog, symlink_home / "images" / "catalog") + before = fingerprint(outside_catalog) + completed = cli(symlink_home, "image", "add", "vendor.second", OTHER_SUBJECT) + after = fingerprint(outside_catalog) + check( + "CAT-STATE-007", + completed.returncode == 125 and before == after, + "a symlinked initialized catalog is refused before external target mutation", + f"rc={completed.returncode} target_changed={before != after}", + ) + + lock_home = new_home(root, "lock-home") + add(lock_home) + lock_path = lock_home / "state" / "locks" / "image-catalog.lock" + saved_lock = root / "saved-image-catalog.lock" + lock_path.rename(saved_lock) + os.symlink(saved_lock, lock_path) + completed = cli(lock_home, "image", "list") + check( + "CAT-STATE-008", + completed.returncode == 125, + "replacement or symlink of the stable catalog lock fails closed", + f"rc={completed.returncode}", + ) + + staging_home = new_home(root, "staging-home") + foreign_wrapper = staging_home / "images" / ".staging" / "foreign-wrapper" + (foreign_wrapper / "payload").mkdir(parents=True) + (foreign_wrapper / "intent.json").write_text('{"owner":"foreign"}\n', encoding="utf-8") + before = fingerprint(foreign_wrapper) + completed = cli(staging_home, "image", "add", "vendor.worker", SUBJECT) + after = fingerprint(foreign_wrapper) + unsafe_cleanup_home = new_home(root, "unsafe-cleanup-home") + unsafe_cleanup = ( + unsafe_cleanup_home / "images" / ".staging" / "image-catalog-cleanup" + ) + unsafe_cleanup.mkdir(mode=0o700) + (unsafe_cleanup / "payload").write_bytes(b"not-a-directory\n") + (unsafe_cleanup / "payload").chmod(0o600) + cleanup_before = fingerprint(unsafe_cleanup) + cleanup_read = cli(unsafe_cleanup_home, "image", "list") + cleanup_retry = cli( + unsafe_cleanup_home, + "image", + "add", + "vendor.worker", + SUBJECT, + ) + cleanup_after = fingerprint(unsafe_cleanup) + check( + "CAT-STATE-009", + completed.returncode == 125 + and before == after + and not (staging_home / "images" / "catalog").exists() + and cleanup_read.returncode == 125 + and cleanup_read.stdout == b"" + and cleanup_retry.returncode == 125 + and cleanup_retry.stdout == b"" + and cleanup_before == cleanup_after, + "unknown or unsafe staging ownership is preserved and blocks reads and mutation", + ( + f"rc={completed.returncode} wrapper_changed={before != after} " + f"cleanup_read={cleanup_read.returncode} cleanup_retry={cleanup_retry.returncode} " + f"cleanup_changed={cleanup_before != cleanup_after}" + ), + ) + + pristine_home = new_home(root, "pristine-home") + before = fingerprint(pristine_home / "images") + completed = cli(pristine_home, "image", "list") + after = fingerprint(pristine_home / "images") + check( + "CAT-STATE-010", + completed.returncode == 0 and completed.stdout == b"[]\n" and before == after, + "pristine read returns canonical empty without initialization or cleanup", + f"rc={completed.returncode} changed={before != after} stdout={completed.stdout!r}", + ) + + schema_home = new_home(root, "schema-home") + add(schema_home) + entry_paths = list((schema_home / "images" / "catalog" / "entries").glob("*.json")) + snapshot_paths = list((schema_home / "images" / "catalog" / "snapshots").glob("*.json")) + schema_ok = len(entry_paths) == 1 and len(snapshot_paths) == 1 + for path in [*entry_paths, *snapshot_paths]: + raw = path.read_bytes() + try: + value = json.loads(raw) + except (UnicodeError, json.JSONDecodeError): + schema_ok = False + continue + api_version = value.get("apiVersion") if isinstance(value, dict) else None + schema_ok = ( + schema_ok + and isinstance(api_version, str) + and api_version.startswith("agent-lab.") + and api_version.endswith("/v0alpha1") + and raw == canonical(value) + b"\n" + ) + check( + "CAT-STATE-011", + schema_ok, + "immutable entry and snapshot records carry closed versioned canonical schemas", + ) + + truncated_home = new_home(root, "truncated-home") + add(truncated_home) + current = truncated_home / "images" / "catalog" / "current.json" + current.write_bytes(b'{"snapshotDigest":') + before = current.read_bytes() + completed = cli(truncated_home, "image", "inspect", "vendor.worker") + check( + "CAT-STATE-012", + completed.returncode == 125 and completed.stdout == b"" and current.read_bytes() == before, + "truncated authority is infrastructure uncertainty and read-only commands do not repair it", + f"rc={completed.returncode}", + ) + + orphan_home = new_home(root, "orphan-history-home") + add(orphan_home) + orphan_entry = { + "apiVersion": "agent-lab.local-image-entry/v0alpha1", + "generation": 2, + "name": "orphan.image", + "previousEntryDigest": "sha256:" + "e" * 64, + "state": "removed", + "subject": OTHER_SUBJECT, + "subjectDigest": OTHER_SUBJECT.rsplit("@", 1)[1], + } + orphan_path = write_record( + orphan_home / "images" / "catalog" / "entries", + ENTRY_DOMAIN, + orphan_entry, + ) + before = orphan_path.read_bytes() + completed = cli(orphan_home, "image", "list", "--all") + check( + "CAT-STATE-013", + completed.returncode == 125 and completed.stdout == b"" and orphan_path.read_bytes() == before, + "malformed unreachable tombstone history fails closed without repair", + f"rc={completed.returncode}", + ) + + conflict_home = new_home(root, "conflicting-history-home") + add(conflict_home) + conflicting_entry = { + "apiVersion": "agent-lab.local-image-entry/v0alpha1", + "generation": 1, + "name": "vendor.worker", + "previousEntryDigest": None, + "state": "active", + "subject": OTHER_SUBJECT, + "subjectDigest": OTHER_SUBJECT.rsplit("@", 1)[1], + } + conflict_path = write_record( + conflict_home / "images" / "catalog" / "entries", + ENTRY_DOMAIN, + conflicting_entry, + ) + before = conflict_path.read_bytes() + completed = cli(conflict_home, "image", "list", "--all") + check( + "CAT-STATE-014", + completed.returncode == 125 and completed.stdout == b"" and conflict_path.read_bytes() == before, + "conflicting unreachable generation history fails closed without repair", + f"rc={completed.returncode}", + ) + + marker_home = new_home(root, "marker-home") + add(marker_home) + marker_lock = marker_home / "state" / "locks" / "image-catalog.lock" + marker_lock.write_bytes(b"") + truncated = cli(marker_home, "image", "list") + moved_catalog = root / "marker-moved-catalog" + (marker_home / "images" / "catalog").rename(moved_catalog) + missing = cli(marker_home, "image", "list") + check( + "CAT-STATE-015", + truncated.returncode == 125 + and missing.returncode == 125 + and truncated.stdout == b"" + and missing.stdout == b"", + "truncated initialization evidence never turns an existing or lost catalog into pristine state", + f"truncated_rc={truncated.returncode} missing_rc={missing.returncode}", + ) + + replacement_home = new_home(root, "replacement-home") + add(replacement_home) + replacement_lock = replacement_home / "state" / "locks" / "image-catalog.lock" + lock_bytes = replacement_lock.read_bytes() + replacement_lock.rename(root / "original-image-catalog.lock") + replacement_lock.write_bytes(lock_bytes) + replacement_lock.chmod(0o600) + completed = cli(replacement_home, "image", "list") + check( + "CAT-STATE-016", + completed.returncode == 125 and completed.stdout == b"", + "the immutable home receipt detects a pre-invocation stable-lock replacement", + f"rc={completed.returncode}", + ) + + split_home = new_home(root, "split-lock-home") + original_flock = CATALOG.fcntl.flock + split_rejected = False + split_replaced = False + + def replace_after_flock(descriptor: int, operation: int) -> None: + nonlocal split_replaced + original_flock(descriptor, operation) + if operation in (CATALOG.fcntl.LOCK_SH, CATALOG.fcntl.LOCK_EX) and not split_replaced: + path = split_home / "state" / "locks" / "image-catalog.lock" + data = path.read_bytes() + path.rename(root / "split-original-image-catalog.lock") + path.write_bytes(data) + path.chmod(0o600) + split_replaced = True + + CATALOG.fcntl.flock = replace_after_flock + try: + CATALOG.list_images(split_home) + except CATALOG.CatalogInfrastructure: + split_rejected = True + finally: + CATALOG.fcntl.flock = original_flock + + transition_home = new_home(root, "marker-transition-home") + transition_authority = CATALOG._load_home(transition_home) + transition_lock = transition_home / "state" / "locks" / "image-catalog.lock" + original_open = CATALOG.os.open + transition_observed = None + transitioned = False + + def append_marker_before_open(path, flags, *args, **kwargs): + nonlocal transitioned + if Path(path) == transition_lock and flags & os.O_RDWR and not transitioned: + marker_descriptor = original_open( + path, + os.O_WRONLY | os.O_APPEND | getattr(os, "O_CLOEXEC", 0), + ) + try: + os.write(marker_descriptor, b"initialized\n") + os.fsync(marker_descriptor) + finally: + os.close(marker_descriptor) + transitioned = True + return original_open(path, flags, *args, **kwargs) + + CATALOG.os.open = append_marker_before_open + try: + with CATALOG._catalog_lock(transition_authority, exclusive=False) as descriptor: + transition_observed = CATALOG._lock_bytes(descriptor) + finally: + CATALOG.os.open = original_open + check( + "CAT-STATE-018", + split_rejected + and split_replaced + and transitioned + and transition_observed == CATALOG.LOCK_INITIALIZED, + "post-acquisition replacement is rejected while a valid same-inode marker transition is accepted", + ) + + nested_home = new_home(root, "nested-json-home") + add(nested_home) + nested_current = nested_home / "images" / "catalog" / "current.json" + nested_current.write_bytes(b"[" * 30_000 + b"0" + b"]" * 30_000 + b"\n") + completed = cli(nested_home, "image", "list") + check( + "CAT-STATE-017", + completed.returncode == 125 + and completed.stdout == b"" + and b"Traceback" not in completed.stderr, + "bounded deeply nested JSON is classified as infrastructure uncertainty without traceback", + f"rc={completed.returncode} stderr={completed.stderr[-120:]!r}", + ) + + names_home = new_home(root, "names-bound-home") + names_ok = True + for index in range(256): + rc, _, _, error = module_image(names_home, "add", f"v{index:03d}.image", SUBJECT) + if rc != 0 or error is not None: + names_ok = False + break + before = fingerprint(names_home / "images" / "catalog") + rc, _, _, error = module_image(names_home, "add", "v256.image", SUBJECT) + after = fingerprint(names_home / "images" / "catalog") + check( + "CAT-BOUND-001", + names_ok and rc == 1 and error is None and before == after, + "the 257th logical name is a stable refusal with no publication", + f"setup_ok={names_ok} rc={rc} error={error!r} changed={before != after}", + ) + + bytes_home = new_home(root, "bytes-bound-home") + add(bytes_home) + orphan = bytes_home / "images" / "catalog" / "entries" / ("f" * 64 + ".json") + with orphan.open("wb") as stream: + stream.truncate(67_108_865) + before = orphan.stat().st_size + completed = cli(bytes_home, "image", "list") + check( + "CAT-BOUND-002", + completed.returncode == 125 and orphan.exists() and orphan.stat().st_size == before, + "over-bound physical catalog state fails closed without broad deletion", + f"rc={completed.returncode}", + ) + + physical_limit_home = new_home(root, "physical-name-limit-home") + physical_limits = CATALOG.CatalogLimits(names=2) + catalog_add_result( + physical_limit_home, + "vendor.first", + SUBJECT, + limits=physical_limits, + ) + child_rc = hard_exit_add( + physical_limit_home, + "vendor.orphan", + OTHER_SUBJECT, + "catalog immutable entry.after_noreplace", + limits=physical_limits, + ) + before = fingerprint(physical_limit_home / "images" / "catalog") + third_rc, _, third_error = catalog_add_result( + physical_limit_home, + "vendor.third", + THIRD_SUBJECT, + limits=physical_limits, + ) + after = fingerprint(physical_limit_home / "images" / "catalog") + observed = cli(physical_limit_home, "image", "list") + check( + "CAT-BOUND-003", + child_rc == 99 + and third_rc == 1 + and isinstance(third_error, CATALOG.CatalogReject) + and before == after + and observed.returncode == 0 + and [item.get("name") for item in json.loads(observed.stdout)] == ["vendor.first"], + "a valid unreachable orphan consumes physical-name capacity before new publication", + ( + f"child_rc={child_rc} third_rc={third_rc} error={third_error!r} " + f"changed={before != after} list_rc={observed.returncode}" + ), + ) + + orphan_conflict_home = new_home(root, "orphan-generation-conflict-home") + add(orphan_conflict_home, "vendor.first", SUBJECT) + child_rc = hard_exit_add( + orphan_conflict_home, + "vendor.orphan", + OTHER_SUBJECT, + "catalog immutable entry.after_noreplace", + ) + before = fingerprint(orphan_conflict_home / "images" / "catalog") + conflict_rc, _, conflict_error = catalog_add_result( + orphan_conflict_home, + "vendor.orphan", + THIRD_SUBJECT, + ) + after = fingerprint(orphan_conflict_home / "images" / "catalog") + observed = cli(orphan_conflict_home, "image", "list") + check( + "CAT-BOUND-004", + child_rc == 99 + and conflict_rc == 1 + and isinstance(conflict_error, CATALOG.CatalogReject) + and before == after + and observed.returncode == 0 + and [item.get("name") for item in json.loads(observed.stdout)] == ["vendor.first"], + "a conflicting generation-one orphan is rejected before immutable publication", + ( + f"child_rc={child_rc} conflict_rc={conflict_rc} error={conflict_error!r} " + f"changed={before != after} list_rc={observed.returncode}" + ), + ) + + fsync_home = new_home(root, "fsync-home") + original_fsync = MODULE.os.fsync + record_fsync_failed = False + + def fail_first_fsync(descriptor: int) -> None: + nonlocal record_fsync_failed + target = os.readlink(f"/proc/self/fd/{descriptor}") + if not record_fsync_failed and "/payload/catalog/entries/" in target: + record_fsync_failed = True + raise OSError("injected fsync failure") + original_fsync(descriptor) + + MODULE.os.fsync = fail_first_fsync + try: + first_rc, _, _, first_error = module_image(fsync_home, "add", "vendor.worker", SUBJECT) + finally: + MODULE.os.fsync = original_fsync + retry = cli(fsync_home, "image", "add", "vendor.worker", SUBJECT) + staged = list((fsync_home / "images" / ".staging").iterdir()) + check( + "CAT-CRASH-001", + first_rc == 125 + and first_error is None + and record_fsync_failed + and retry.returncode == 0 + and json.loads(retry.stdout).get("changed") is True + and not staged, + "staged immutable-record fsync failure is contained and retry reconciles safely", + f"first_rc={first_rc} error={first_error!r} retry_rc={retry.returncode} staged={len(staged)}", + ) + + replace_home = new_home(root, "replace-home") + add(replace_home) + original_replace = MODULE.os.replace + replace_calls = 0 + + def fail_pointer_replace(source: os.PathLike[str] | str, target: os.PathLike[str] | str) -> None: + nonlocal replace_calls + final_pointer = replace_home / "images" / "catalog" / "current.json" + if Path(target) == final_pointer: + replace_calls += 1 + if replace_calls == 1: + raise OSError("injected pointer publication failure") + original_replace(source, target) + + MODULE.os.replace = fail_pointer_replace + try: + first_rc, _, _, first_error = module_image( + replace_home, + "add", + "vendor.second", + OTHER_SUBJECT, + ) + finally: + MODULE.os.replace = original_replace + retry = cli(replace_home, "image", "add", "vendor.second", OTHER_SUBJECT) + staged = list((replace_home / "images" / ".staging").iterdir()) + check( + "CAT-CRASH-002", + first_rc == 125 + and first_error is None + and retry.returncode == 0 + and json.loads(retry.stdout).get("changed") is True + and not staged, + "pointer publication failure leaves an uncommitted operation that retry proves and reconciles", + f"first_rc={first_rc} error={first_error!r} retry_rc={retry.returncode} staged={len(staged)}", + ) + + result_home = new_home(root, "result-home") + first_rc, first_error = module_main( + result_home, + ["image", "add", "vendor.worker", SUBJECT], + BrokenOutput(), + ) + retry = cli(result_home, "image", "add", "vendor.worker", SUBJECT) + retry_value = json.loads(retry.stdout) if retry.returncode == 0 else {} + result_exit_codes: list[int] = [] + result_retries: list[dict[str, object]] = [] + for index, point in enumerate(("result.before_write", "result.after_write")): + hard_exit_home = new_home(root, f"result-hard-exit-{index}") + result_exit_codes.append(hard_exit_result(hard_exit_home, point)) + completed = cli(hard_exit_home, "image", "add", "vendor.worker", SUBJECT) + result_retries.append(json.loads(completed.stdout) if completed.returncode == 0 else {}) + check( + "CAT-CRASH-003", + first_rc == 125 + and first_error is None + and retry.returncode == 0 + and retry_value.get("changed") is False + and result_exit_codes == [99, 99] + and [value.get("changed") for value in result_retries] == [False, False], + "failures and hard exits before or after result write retry as committed idempotent state", + ( + f"first_rc={first_rc} error={first_error!r} retry_rc={retry.returncode} " + f"retry={retry_value!r} exits={result_exit_codes!r} hard_retries={result_retries!r}" + ), + ) + + forged_home = new_home(root, "forged-stage-home") + victim = add(forged_home) + _, _, base_digest = current_snapshot(forged_home) + victim_digest = victim.get("entryDigest") + if not isinstance(victim_digest, str): + raise RuntimeError("catalog setup add omitted the active entry digest") + wrapper = forged_home / "images" / ".staging" / "image-catalog-operation" + wrapper.mkdir(mode=0o700) + forged_intent = { + "apiVersion": "agent-lab.local-image-intent/v0alpha1", + "baseSnapshotDigest": base_digest, + "bootstrap": False, + "candidateEntryDigest": victim_digest, + "candidateSnapshotDigest": "sha256:" + "f" * 64, + "entryPreexisting": False, + "expectedEntryDigest": None, + "kind": "add", + "name": "vendor.second", + "snapshotPreexisting": True, + "subject": OTHER_SUBJECT, + } + intent_path = wrapper / "intent.json" + intent_path.write_bytes(canonical(forged_intent) + b"\n") + intent_path.chmod(0o600) + before = fingerprint(forged_home / "images") + completed = cli(forged_home, "image", "add", "vendor.third", OTHER_SUBJECT) + after = fingerprint(forged_home / "images") + observed = cli(forged_home, "image", "list") + check( + "CAT-CRASH-004", + completed.returncode == 125 + and completed.stdout == b"" + and before == after + and observed.returncode == 125 + and observed.stdout == b"", + "unproven staged intent cannot delete committed immutable history or its own evidence", + ( + f"rc={completed.returncode} changed={before != after} " + f"list_rc={observed.returncode}" + ), + ) + + atomic_home = new_home(root, "atomic-record-home") + add(atomic_home) + exit_code = hard_exit_add( + atomic_home, + "vendor.second", + OTHER_SUBJECT, + "catalog immutable entry.before_noreplace", + ) + observed = cli(atomic_home, "image", "list") + observed_value = json.loads(observed.stdout) if observed.returncode == 0 else [] + retry = cli(atomic_home, "image", "add", "vendor.second", OTHER_SUBJECT) + check( + "CAT-CRASH-005", + exit_code == 99 + and observed.returncode == 0 + and [item.get("name") for item in observed_value] == ["vendor.worker"] + and retry.returncode == 0 + and json.loads(retry.stdout).get("changed") is True, + "hard exit at immutable-record no-replace publication exposes the old complete view and retries", + ( + f"child_rc={exit_code} list_rc={observed.returncode} " + f"list={observed_value!r} retry_rc={retry.returncode}" + ), + ) + + bootstrap_points = ( + "catalog intent.before_fsync", + "catalog intent.after_fsync", + "catalog intent wrapper.before_fsync", + "catalog intent wrapper.after_fsync", + "catalog intent publication.before_fsync", + "catalog intent publication.after_fsync", + "catalog staged entry.before_fsync", + "catalog staged entry.after_fsync", + "catalog staged snapshot.before_fsync", + "catalog staged snapshot.after_fsync", + "catalog staged pointer.before_fsync", + "catalog staged pointer.after_fsync", + "catalog staged pointer.before_replace", + "catalog staged pointer.after_replace", + "catalog staged entries.before_fsync", + "catalog staged entries.after_fsync", + "catalog staged snapshots.before_fsync", + "catalog staged snapshots.after_fsync", + "catalog staged root.before_fsync", + "catalog staged root.after_fsync", + "catalog staged payload.before_fsync", + "catalog staged payload.after_fsync", + "catalog staged wrapper.before_fsync", + "catalog staged wrapper.after_fsync", + "catalog staged operation.before_fsync", + "catalog staged operation.after_fsync", + "catalog marker.before_write", + "catalog marker.after_write", + "catalog marker.before_fsync", + "catalog marker.after_fsync", + "bootstrap.before_rename", + "bootstrap.after_rename", + "catalog bootstrap parent.before_fsync", + "catalog bootstrap parent.after_fsync", + "catalog staging cleanup.before_remove", + "catalog staging cleanup handoff.before_rename", + "catalog staging cleanup handoff.after_rename", + "catalog staging cleanup handoff.before_fsync", + "catalog staging cleanup handoff.after_fsync", + "catalog staging cleanup.after_remove", + "catalog staging cleanup.after_unlink", + "catalog staging cleanup.after_rmdir", + "catalog staging cleanup.before_fsync", + "catalog staging cleanup.after_fsync", + ) + later_points = ( + "catalog intent.before_fsync", + "catalog intent.after_fsync", + "catalog intent wrapper.before_fsync", + "catalog intent wrapper.after_fsync", + "catalog intent publication.before_fsync", + "catalog intent publication.after_fsync", + "catalog staged entry.before_fsync", + "catalog staged entry.after_fsync", + "catalog staged snapshot.before_fsync", + "catalog staged snapshot.after_fsync", + "catalog staged pointer.before_fsync", + "catalog staged pointer.after_fsync", + "catalog staged pointer.before_replace", + "catalog staged pointer.after_replace", + "catalog staged payload.before_fsync", + "catalog staged payload.after_fsync", + "catalog staged wrapper.before_fsync", + "catalog staged wrapper.after_fsync", + "catalog staged operation.before_fsync", + "catalog staged operation.after_fsync", + "catalog immutable entry.before_noreplace", + "catalog immutable entry.after_noreplace", + "catalog immutable entry history.before_fsync", + "catalog immutable entry history.after_fsync", + "catalog immutable snapshot.before_noreplace", + "catalog immutable snapshot.after_noreplace", + "catalog immutable snapshot history.before_fsync", + "catalog immutable snapshot history.after_fsync", + "pointer.before_replace", + "pointer.after_replace", + "catalog current pointer.before_fsync", + "catalog current pointer.after_fsync", + "catalog staging cleanup.before_remove", + "catalog staging cleanup handoff.before_rename", + "catalog staging cleanup handoff.after_rename", + "catalog staging cleanup handoff.before_fsync", + "catalog staging cleanup handoff.after_fsync", + "catalog staging cleanup.after_remove", + "catalog staging cleanup.after_unlink", + "catalog staging cleanup.after_rmdir", + "catalog staging cleanup.before_fsync", + "catalog staging cleanup.after_fsync", + ) + matrix_failures: list[str] = [] + matrix_oracle_home: Path | None = None + matrix_cli_start = CLI_CALLS + for index, point in enumerate(bootstrap_points): + home = matrix_home(root, f"bootstrap-crash-{index:02d}") + child_rc = hard_exit_add(home, "vendor.worker", SUBJECT, point) + before_retry = cli(home, "image", "list") + retry = cli(home, "image", "add", "vendor.worker", SUBJECT) + try: + before_records = json.loads(before_retry.stdout) if before_retry.returncode == 0 else None + retry_value = json.loads(retry.stdout) if retry.returncode == 0 else None + except json.JSONDecodeError: + before_records = retry_value = None + final_names = stored_active_names(home) + if final_names == ["vendor.worker"]: + matrix_oracle_home = home + if not ( + child_rc == 99 + and before_retry.returncode == 0 + and isinstance(before_records, list) + and len(before_records) in (0, 1) + and retry.returncode == 0 + and isinstance(retry_value, dict) + and retry_value.get("changed") in (True, False) + and final_names == ["vendor.worker"] + and not tuple((home / "images" / ".staging").iterdir()) + ): + matrix_failures.append( + f"bootstrap:{point}:child={child_rc}:before={before_retry.returncode}:" + f"retry={retry.returncode}:final={final_names!r}" + ) + for index, point in enumerate(later_points): + home = matrix_home(root, f"later-crash-{index:02d}") + matrix_add(home, "vendor.worker", SUBJECT) + child_rc = hard_exit_add(home, "vendor.second", OTHER_SUBJECT, point) + before_retry = cli(home, "image", "list") + retry = cli(home, "image", "add", "vendor.second", OTHER_SUBJECT) + try: + before_records = json.loads(before_retry.stdout) if before_retry.returncode == 0 else None + retry_value = json.loads(retry.stdout) if retry.returncode == 0 else None + except json.JSONDecodeError: + before_records = retry_value = None + before_names = ( + [record.get("name") for record in before_records] + if isinstance(before_records, list) + else None + ) + final_names = stored_active_names(home) + if final_names == ["vendor.second", "vendor.worker"]: + matrix_oracle_home = home + if not ( + child_rc == 99 + and before_retry.returncode == 0 + and before_names in (["vendor.worker"], ["vendor.second", "vendor.worker"]) + and retry.returncode == 0 + and isinstance(retry_value, dict) + and retry_value.get("changed") in (True, False) + and final_names == ["vendor.second", "vendor.worker"] + and not tuple((home / "images" / ".staging").iterdir()) + ): + matrix_failures.append( + f"later:{point}:child={child_rc}:before={before_retry.returncode}:" + f"retry={retry.returncode}:final={final_names!r}" + ) + matrix_cli_calls = CLI_CALLS - matrix_cli_start + if len(bootstrap_points) != 44 or len(later_points) != 42: + matrix_failures.append( + f"seam-counts:{len(bootstrap_points)}/{len(later_points)}:expected=44/42" + ) + expected_matrix_cli_calls = 172 + if matrix_cli_calls != expected_matrix_cli_calls: + matrix_failures.append( + f"cli-calls:{matrix_cli_calls}:expected={expected_matrix_cli_calls}" + ) + if matrix_oracle_home is None or not stored_oracle_sensitivity( + matrix_oracle_home, ["vendor.second", "vendor.worker"] + ): + matrix_failures.append("stored-oracle-sensitivity") + if matrix_oracle_home is None or not matrix_infrastructure_sensitivity( + root, matrix_oracle_home + ): + matrix_failures.append("matrix-infrastructure-sensitivity") + check( + "CAT-CRASH-006", + not matrix_failures, + "hard exits around every emitted stage, record, pointer, marker, and parent-fsync seam preserve old/new views and retry", + "; ".join(matrix_failures[:5]), + ) + + preintent_home = new_home(root, "preintent-crash-home") + add(preintent_home) + child_rc = hard_exit_add( + preintent_home, + "vendor.second", + OTHER_SUBJECT, + "catalog wrapper.after_create", + ) + before_retry = cli(preintent_home, "image", "list") + stage_before = fingerprint(preintent_home / "images" / ".staging") + retry = cli(preintent_home, "image", "add", "vendor.second", OTHER_SUBJECT) + stage_after = fingerprint(preintent_home / "images" / ".staging") + pristine_preintent_home = new_home(root, "pristine-preintent-crash-home") + pristine_child_rc = hard_exit_add( + pristine_preintent_home, + "vendor.worker", + SUBJECT, + "catalog wrapper.after_create", + ) + pristine_before = fingerprint(pristine_preintent_home / "images" / ".staging") + pristine_list = cli(pristine_preintent_home, "image", "list") + pristine_retry = cli( + pristine_preintent_home, + "image", + "add", + "vendor.worker", + SUBJECT, + ) + pristine_after = fingerprint(pristine_preintent_home / "images" / ".staging") + check( + "CAT-CRASH-007", + child_rc == 99 + and before_retry.returncode == 125 + and retry.returncode == 125 + and retry.stdout == b"" + and stage_before == stage_after + and pristine_child_rc == 99 + and pristine_list.returncode == 125 + and pristine_retry.returncode == 125 + and pristine_before == pristine_after, + "a pre-intent hard exit blocks reads and mutation without deleting residue", + ( + f"child_rc={child_rc} list_rc={before_retry.returncode} " + f"retry_rc={retry.returncode} changed={stage_before != stage_after} " + f"pristine_child={pristine_child_rc} pristine_list={pristine_list.returncode} " + f"pristine_retry={pristine_retry.returncode}" + ), + ) + + phase_home = new_home(root, "bootstrap-phase-home") + child_rc = hard_exit_add( + phase_home, + "vendor.worker", + SUBJECT, + "catalog marker.after_fsync", + ) + staged_catalog = ( + phase_home + / "images" + / ".staging" + / "image-catalog-operation" + / "payload" + / "catalog" + ) + staged_current = staged_catalog / "current.json" + staged_current_available = True + try: + shutil.copyfile(staged_current, staged_catalog / "current.next") + (staged_catalog / "current.next").chmod(0o600) + except FileNotFoundError: + staged_current_available = False + before = fingerprint(phase_home / "images") + retry = cli(phase_home, "image", "add", "vendor.worker", SUBJECT) + after = fingerprint(phase_home / "images") + + marker_phase_home = new_home(root, "initialized-incomplete-bootstrap-home") + marker_child = hard_exit_add( + marker_phase_home, + "vendor.worker", + SUBJECT, + "catalog marker.after_fsync", + ) + marker_payload = ( + marker_phase_home + / "images" + / ".staging" + / "image-catalog-operation" + / "payload" + ) + marker_payload_available = True + try: + shutil.rmtree(marker_payload) + except FileNotFoundError: + marker_payload_available = False + marker_before = fingerprint(marker_phase_home / "images") + marker_read = cli(marker_phase_home, "image", "list") + marker_retry = cli(marker_phase_home, "image", "add", "vendor.worker", SUBJECT) + marker_after = fingerprint(marker_phase_home / "images") + check( + "CAT-CRASH-008", + staged_current_available + and marker_payload_available + and child_rc == 99 + and retry.returncode == 125 + and retry.stdout == b"" + and before == after + and not (phase_home / "images" / "catalog").exists() + and marker_child == 99 + and marker_read.returncode == 125 + and marker_read.stdout == b"" + and marker_retry.returncode == 125 + and marker_retry.stdout == b"" + and marker_before == marker_after + and not (marker_phase_home / "images" / "catalog").exists(), + "phase-inconsistent bootstrap staging remains inert and cannot become committed authority", + ( + f"phase_fixture={staged_current_available} child_rc={child_rc} " + f"retry_rc={retry.returncode} changed={before != after} " + f"marker_fixture={marker_payload_available} " + f"marker_child={marker_child} marker_read={marker_read.returncode} " + f"marker_retry={marker_retry.returncode} marker_changed={marker_before != marker_after}" + ), + ) + + incomplete_home = new_home(root, "incomplete-later-phase-home") + add(incomplete_home) + child_rc = hard_exit_add( + incomplete_home, + "vendor.second", + OTHER_SUBJECT, + "catalog staged pointer.after_replace", + ) + incomplete_payload = ( + incomplete_home + / "images" + / ".staging" + / "image-catalog-operation" + / "payload" + ) + incomplete_entry = incomplete_payload / "entry.json" + incomplete_snapshot = incomplete_payload / "snapshot.json" + incomplete_payload_available = True + try: + incomplete_entry.unlink() + incomplete_snapshot.unlink() + except FileNotFoundError: + incomplete_payload_available = False + before = fingerprint(incomplete_home / "images") + retry = cli(incomplete_home, "image", "add", "vendor.second", OTHER_SUBJECT) + after = fingerprint(incomplete_home / "images") + check( + "CAT-CRASH-009", + incomplete_payload_available + and child_rc == 99 + and retry.returncode == 125 + and retry.stdout == b"" + and before == after, + "a later stage cannot claim a pointer phase without candidate record evidence", + ( + f"fixture={incomplete_payload_available} child_rc={child_rc} " + f"retry_rc={retry.returncode} changed={before != after}" + ), + ) + + marker_durable_home = new_home(root, "marker-durable-recovery-home") + marker_child = hard_exit_add( + marker_durable_home, + "vendor.worker", + SUBJECT, + "catalog marker.after_write", + ) + marker_rc, marker_value, marker_error, marker_targets = traced_catalog_add( + marker_durable_home, + "vendor.worker", + SUBJECT, + ) + marker_lock_target = str(marker_durable_home / "state" / "locks" / "image-catalog.lock") + marker_images_target = str(marker_durable_home / "images") + + bootstrap_durable_home = new_home(root, "bootstrap-durable-recovery-home") + bootstrap_child = hard_exit_add( + bootstrap_durable_home, + "vendor.worker", + SUBJECT, + "bootstrap.after_rename", + ) + bootstrap_rc, bootstrap_value, bootstrap_error, bootstrap_targets = traced_catalog_add( + bootstrap_durable_home, + "vendor.worker", + SUBJECT, + ) + + pointer_durable_home = new_home(root, "pointer-durable-recovery-home") + add(pointer_durable_home) + pointer_child = hard_exit_add( + pointer_durable_home, + "vendor.second", + OTHER_SUBJECT, + "pointer.after_replace", + ) + pointer_rc, pointer_value, pointer_error, pointer_targets = traced_catalog_add( + pointer_durable_home, + "vendor.second", + OTHER_SUBJECT, + ) + + cleanup_durable_home = new_home(root, "cleanup-durable-recovery-home") + add(cleanup_durable_home) + cleanup_rc, cleanup_value, cleanup_error, cleanup_targets = traced_catalog_add( + cleanup_durable_home, + "vendor.worker", + SUBJECT, + ) + marker_ordered = ( + marker_lock_target in marker_targets + and marker_images_target in marker_targets + and marker_targets.index(marker_lock_target) < marker_targets.index(marker_images_target) + ) + check( + "CAT-CRASH-010", + marker_child == 99 + and marker_rc == 0 + and isinstance(marker_value, dict) + and marker_value.get("changed") is False + and marker_error is None + and marker_ordered + and bootstrap_child == 99 + and bootstrap_rc == 0 + and isinstance(bootstrap_value, dict) + and bootstrap_value.get("changed") is False + and bootstrap_error is None + and str(bootstrap_durable_home / "images") in bootstrap_targets + and pointer_child == 99 + and pointer_rc == 0 + and isinstance(pointer_value, dict) + and pointer_value.get("changed") is False + and pointer_error is None + and str(pointer_durable_home / "images" / "catalog") in pointer_targets + and cleanup_rc == 0 + and isinstance(cleanup_value, dict) + and cleanup_value.get("changed") is False + and cleanup_error is None + and str(cleanup_durable_home / "images" / ".staging") in cleanup_targets, + "recovery completes marker, commit-parent, and cleanup-root durability before success", + ( + f"marker=({marker_child},{marker_rc},{marker_targets!r}) " + f"bootstrap=({bootstrap_child},{bootstrap_rc},{bootstrap_targets!r}) " + f"pointer=({pointer_child},{pointer_rc},{pointer_targets!r}) " + f"cleanup=({cleanup_rc},{cleanup_targets!r})" + ), + ) + + cleanup_unlink_home = new_home(root, "cleanup-intent-unlink-home") + add(cleanup_unlink_home) + committed_child = hard_exit_add( + cleanup_unlink_home, + "vendor.second", + OTHER_SUBJECT, + "pointer.after_replace", + ) + unlink_child = hard_exit_cleanup( + cleanup_unlink_home, + "vendor.second", + OTHER_SUBJECT, + "intent-unlink", + ) + unlink_read = cli(cleanup_unlink_home, "image", "list") + unlink_retry = cli( + cleanup_unlink_home, + "image", + "add", + "vendor.second", + OTHER_SUBJECT, + ) + + cleanup_rmdir_home = new_home(root, "cleanup-payload-rmdir-home") + add(cleanup_rmdir_home) + orphan_child = hard_exit_add( + cleanup_rmdir_home, + "vendor.second", + OTHER_SUBJECT, + "catalog immutable entry.after_noreplace", + ) + rmdir_child = hard_exit_cleanup( + cleanup_rmdir_home, + "vendor.third", + THIRD_SUBJECT, + "payload-rmdir", + ) + rmdir_read = cli(cleanup_rmdir_home, "image", "list") + rmdir_retry = cli( + cleanup_rmdir_home, + "image", + "add", + "vendor.third", + THIRD_SUBJECT, + ) + check( + "CAT-CRASH-011", + committed_child == 99 + and unlink_child == 99 + and unlink_read.returncode == 0 + and [item.get("name") for item in json.loads(unlink_read.stdout)] + == ["vendor.second", "vendor.worker"] + and unlink_retry.returncode == 0 + and json.loads(unlink_retry.stdout).get("changed") is False + and not tuple((cleanup_unlink_home / "images" / ".staging").iterdir()) + and orphan_child == 99 + and rmdir_child == 99 + and rmdir_read.returncode == 0 + and [item.get("name") for item in json.loads(rmdir_read.stdout)] + == ["vendor.worker"] + and rmdir_retry.returncode == 0 + and json.loads(rmdir_retry.stdout).get("changed") is True + and not tuple((cleanup_rmdir_home / "images" / ".staging").iterdir()), + "cleanup remains restartable across internal intent-unlink and payload-rmdir crashes", + ( + f"committed={committed_child} unlink={unlink_child} " + f"unlink_read={unlink_read.returncode} unlink_retry={unlink_retry.returncode} " + f"orphan={orphan_child} rmdir={rmdir_child} " + f"rmdir_read={rmdir_read.returncode} rmdir_retry={rmdir_retry.returncode}" + ), + ) + + platform_home = new_home(root, "platform-home") + before = fingerprint(platform_home) + original_platform = MODULE.sys.platform + MODULE.sys.platform = "darwin" + try: + platform_rc, platform_error = module_main( + platform_home, + ["image", "add", "vendor.worker", SUBJECT], + ) + finally: + MODULE.sys.platform = original_platform + after = fingerprint(platform_home) + check( + "CAT-PLAT-001", + platform_rc == 125 and platform_error is None and before == after, + "an injected non-Linux host refuses mutation before opening catalog state", + f"rc={platform_rc} error={platform_error!r} changed={before != after}", + ) + + expected = [ + "CAT-STATE-001", + "CAT-STATE-002", + "CAT-STATE-003", + "CAT-STATE-004", + "CAT-STATE-005", + "CAT-STATE-006", + "CAT-STATE-007", + "CAT-STATE-008", + "CAT-STATE-009", + "CAT-STATE-010", + "CAT-STATE-011", + "CAT-STATE-012", + "CAT-STATE-013", + "CAT-STATE-014", + "CAT-STATE-015", + "CAT-STATE-016", + "CAT-STATE-018", + "CAT-STATE-017", + "CAT-BOUND-001", + "CAT-BOUND-002", + "CAT-BOUND-003", + "CAT-BOUND-004", + "CAT-CRASH-001", + "CAT-CRASH-002", + "CAT-CRASH-003", + "CAT-CRASH-004", + "CAT-CRASH-005", + "CAT-CRASH-006", + "CAT-CRASH-007", + "CAT-CRASH-008", + "CAT-CRASH-009", + "CAT-CRASH-010", + "CAT-CRASH-011", + "CAT-PLAT-001", + ] + if OBSERVED != expected: + print(f"INFRA catalog state assertion identity drift: {OBSERVED!r}", file=sys.stderr) + return 125 + print(f"SUMMARY assertions=34 expected=34 failures={FAILURES} infra=0") + return 0 if FAILURES == 0 else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/install/fixtures/expected-runtime-files.txt b/tests/install/fixtures/expected-runtime-files.txt new file mode 100644 index 0000000..b62cbce --- /dev/null +++ b/tests/install/fixtures/expected-runtime-files.txt @@ -0,0 +1,16 @@ +authorization/experiment/v0alpha1/operator.cedar +authorization/experiment/v0alpha1/schema.cedarschema +catalog/experiment-images/v0alpha1.json +contracts/experiment/v0alpha1/cue.mod/module.cue +contracts/experiment/v0alpha1/plan.cue +contracts/experiment/v0alpha1/schema.cue +scripts/agent-lab +scripts/agent-lab.py +scripts/dev/cedar-tool.py +scripts/dev/cue-tool.py +scripts/experiment.py +scripts/experiment_store.py +scripts/image_catalog.py +scripts/image_reference.py +tools/cedar.lock +tools/cue.lock diff --git a/tests/install/local-install-cases.sh b/tests/install/local-install-cases.sh new file mode 100755 index 0000000..968c4d7 --- /dev/null +++ b/tests/install/local-install-cases.sh @@ -0,0 +1,106 @@ +#!/usr/bin/env bash +set -euo pipefail + +repo_root="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." >/dev/null 2>&1 && pwd)" +expected="$repo_root/tests/install/fixtures/expected-runtime-files.txt" +manifest="$repo_root/packaging/agent-lab-local.manifest" +installer="$repo_root/scripts/install-local" +work="" +cleanup_work() { + local failed=0 + if [ -n "$work" ] && [ -e "$work" ]; then + find "$work" -type f -delete 2>/dev/null || failed=1 + find "$work" -type l -delete 2>/dev/null || failed=1 + find "$work" -depth -type d -exec rmdir {} + 2>/dev/null || failed=1 + [ ! -e "$work" ] || failed=1 + fi + return "$failed" +} +if ! work="$(mktemp -d)"; then + printf 'SUMMARY assertions=0 expected=5 failures=0 infra=1\n' + exit 125 +fi +trap 'cleanup_work >/dev/null 2>&1 || true' EXIT +failures=0 +pass() { printf 'PASS %s %s\n' "$1" "$2"; } +fail() { printf 'FAIL %s %s\n' "$1" "$2"; failures=$((failures + 1)); } + +if [ -f "$manifest" ] && cmp -s "$expected" "$manifest"; then + pass PKG-001 "production runtime manifest matches the independent allowlist" +else + fail PKG-001 "production runtime manifest matches the independent allowlist" +fi + +if [ -x "$installer" ]; then + pass PKG-002 "local installer entrypoint exists" +else + fail PKG-002 "local installer entrypoint exists" +fi + +prefix="$work/prefix" +home="$work/agent-home" +install_rc=125 +version_rc=125 +if [ -f "$repo_root/scripts/install-local.py" ]; then + replica="$work/source-replica" + mkdir -p "$replica/packaging" "$replica/scripts" + while IFS= read -r name; do + mkdir -p "$replica/$(dirname -- "$name")" + cp "$repo_root/$name" "$replica/$name" + done < "$expected" + cp "$manifest" "$replica/packaging/agent-lab-local.manifest" + cp "$repo_root/scripts/install-local" "$repo_root/scripts/install-local.py" "$replica/scripts/" + chmod +x "$replica/scripts/install-local" "$replica/scripts/agent-lab" + install_rc=0 + "$replica/scripts/install-local" --prefix "$prefix" > "$work/install.out" 2> "$work/install.err" || install_rc=$? + cp -R "$repo_root/tests/experiment/fixtures/directories/minimal" "$work/artifact" + mv "$replica" "$work/source-unavailable" + mkdir "$work/unrelated" + version_rc=0 + (cd "$work/unrelated" && env -i PATH=/usr/bin:/bin "$prefix/bin/agent-lab" --home "$home" version) \ + > "$work/version.out" 2> "$work/version.err" || version_rc=$? +fi +if [ "$install_rc" -eq 0 ] && [ "$version_rc" -eq 0 ] && + grep -Fxq 'agent-lab v0alpha1' "$work/version.out" && [ ! -s "$work/version.err" ]; then + pass PKG-003 "installed CLI runs after its isolated source replica is unavailable" +else + fail PKG-003 "installed CLI runs after its isolated source replica is unavailable" +fi + +check_rc=125 +if [ "$install_rc" -eq 0 ] && [ "$version_rc" -eq 0 ]; then + "$prefix/bin/agent-lab" --home "$home" init > "$work/init.out" 2> "$work/init.err" + cp -a "$repo_root/.cache/dev/tools/cue/." "$home/cache/tools/cue/" + cp -a "$repo_root/.cache/dev/tools/cedar/." "$home/cache/tools/cedar/" + check_rc=0 + (cd "$work/unrelated" && env -i PATH=/usr/bin:/bin "$prefix/bin/agent-lab" --home "$home" experiment check "$work/artifact") \ + > "$work/check.out" 2> "$work/check.err" || check_rc=$? +fi +if [ "$check_rc" -eq 0 ] && [ ! -s "$work/check.err" ] && + jq -e '.source.kind == "directory" and .plan.kind == "RequestedExperimentPlan"' "$work/check.out" >/dev/null 2>&1; then + pass PKG-004 "installed Experiment check is independent of the source checkout" +else + fail PKG-004 "installed Experiment check is independent of the source checkout" + printf 'PKG-004 rc=%s stderr=%s\n' "$check_rc" "$(tr '\n' ' ' < "$work/check.err" 2>/dev/null || true)" +fi + +mkdir "$work/symlink-target" +ln -s "$work/symlink-target" "$work/symlink-prefix" +symlink_rc=0 +"$installer" --prefix "$work/symlink-prefix" > "$work/symlink.out" 2> "$work/symlink.err" || symlink_rc=$? +if [ "$symlink_rc" -eq 125 ] && [ -z "$(find "$work/symlink-target" -mindepth 1 -print -quit)" ]; then + pass PKG-005 "installer refuses a symlinked prefix before writes" +else + fail PKG-005 "installer refuses a symlinked prefix before writes" +fi + +infrastructure=0 +if ! cleanup_work; then + infrastructure=1 +fi +trap - EXIT +printf 'SUMMARY assertions=5 expected=5 failures=%s infra=%s\n' "$failures" "$infrastructure" +if [ "$infrastructure" -ne 0 ]; then + exit 125 +fi +[ "$failures" -eq 0 ] diff --git a/tests/security/fast.manifest b/tests/security/fast.manifest index bf29de7..23cebcc 100644 --- a/tests/security/fast.manifest +++ b/tests/security/fast.manifest @@ -50,8 +50,10 @@ suite docker-harness-contract tests/dev/docker-harness-cases.sh SUMMARY failures suite guard-command tests/guard/pretooluse-cases.sh SUMMARY failures=0 suite guard-mount tests/guard/cases.sh SUMMARY failures=0 suite config-authority tests/agent/config-guard.sh SUMMARY failures=0 -suite experiment-contract tests/experiment/contract-cases.sh SUMMARY failures=0 -suite experiment-authorization tests/experiment/authorization-cases.sh SUMMARY failures=0 +suite experiment-contract tests/experiment/contract-cases.sh EXPERIMENT CONTRACT PASS +suite experiment-authorization tests/experiment/authorization-cases.sh EXPERIMENT AUTHORIZATION PASS +suite experiment-local-lifecycle tests/experiment/local-lifecycle-cases.sh EXPERIMENT LOCAL LIFECYCLE PASS +suite experiment-source-adapters tests/experiment/source-adapter-cases.sh EXPERIMENT SOURCE ADAPTERS PASS suite config-matrix tests/agent/config-matrix.sh SUMMARY failures=0 suite allowlist-schema tests/agent/allowlist-cases.sh SUMMARY failures=0 suite image-volume-policy tests/agent/image-volume-policy-cases.sh SUMMARY failures=0