From fa90ca1bf61af3213e91a174d878db68fe2c85f3 Mon Sep 17 00:00:00 2001 From: ci Date: Wed, 2 Sep 2026 15:56:29 -0400 Subject: [PATCH 01/17] Add inactive local Git materializer --- README.md | 23 + RESTORE.md | 22 + .../local-git-materializer/v1/materialize.sh | 376 ++++++++++++ .../local-git-materializer/v1/protocol.jq | 308 ++++++++++ ci/required-files.txt | 5 + .../local-git-materializer-adapter.test.sh | 556 ++++++++++++++++++ 6 files changed, 1290 insertions(+) create mode 100755 adapters/local-git-materializer/v1/materialize.sh create mode 100644 adapters/local-git-materializer/v1/protocol.jq create mode 100755 scripts/test/local-git-materializer-adapter.test.sh diff --git a/README.md b/README.md index f423f55..8a3c429 100644 --- a/README.md +++ b/README.md @@ -245,6 +245,29 @@ main. The payload is offline and unqualified. It does not call GitHub or a CLI, use a credential, rerun or cancel work, dispatch a workflow, change a repository, grant authority or qualification, or activate a profile. +## Inactive local Git candidate materializer + +`adapters/local-git-materializer/v1/` implements the existing portable-core v2 +`core.forge.materialize-candidate.v2` capability without a Git forge. It reads one +exact, sanitized bare source repository and one contract-bound patch. It imports +reachable objects into a caller-disposable bare repository, applies the patch to a +scratch-only index, and returns a canonical receipt and validated stage result. + +The fixed `materialize` command accepts only caller-named physical source, +candidate, and scratch boundaries. It rejects worktrees, alternates, shallow or +partial repositories, replace or graft state, active hooks and filters, remote +configuration, unsafe paths, binary patches, symlinks, and submodules. It never +checks out a worktree or runs a transport command. Tests cover both SHA-1 and +SHA-256 object formats with disposable local fixtures. + +This PR lands only the inactive package payload. A later assembly PR may add a +manifest whose package reference points to this payload's durable commit on main. +GitHub and later GitLab change-request normalizers remain separate observation +inputs; they do not claim this materialization capability. The package is not +qualified, selected, installed, or activated. It reads no credential, contacts no +provider or real target during construction, and cannot push, publish, merge, or +grant authority. + ## The current default team You talk **only** to yshifu, in a Claude Code session. yshifu orchestrates the other roles diff --git a/RESTORE.md b/RESTORE.md index 70877ca..4e0ab0a 100644 --- a/RESTORE.md +++ b/RESTORE.md @@ -479,6 +479,28 @@ default-set wiring. The pure jq payload is offline and unqualified. It does not call GitHub, use a credential, rerun, cancel, or dispatch work, change a repository, grant authority or qualification, or activate a profile. +Restore the three paths in the manifest's inactive local Git materializer block, +then run: + +```sh +bash scripts/test/local-git-materializer-adapter.test.sh +``` + +This builds disposable SHA-1 and SHA-256 source repositories, validates a complete +portable-core v2 profile and stage request, and proves that a contract-bound patch +becomes a deterministic bare child commit and path-free receipt. The negative +matrix rejects moved identities, unsafe directories and paths, hooks, filters, +remotes, worktrees, alternates, shallow or partial repositories, replace state, +binary patches, symlinks, and submodules. It also checks that the source stays +unchanged and scratch is removed. + +This payload has no adapter manifest. A later assembly PR may bind the directory +tree through the payload's durable main commit. It is an inactive, local-only +materializer, not a GitHub or GitLab operation. Restoring it does not qualify an +adapter, select or activate a profile, read a credential, contact a provider or +real target during construction, or permit push, publish, merge, or another +external write. + --- ## 5. Smoke test — prove the rebuilt team is alive diff --git a/adapters/local-git-materializer/v1/materialize.sh b/adapters/local-git-materializer/v1/materialize.sh new file mode 100755 index 0000000..d978c53 --- /dev/null +++ b/adapters/local-git-materializer/v1/materialize.sh @@ -0,0 +1,376 @@ +#!/usr/bin/env bash +# shellcheck disable=SC2016 +set -euo pipefail + +emit_error() { + printf '%s\n' "${1:-E_RUNTIME}" >&2 + exit 1 +} + +[ "$#" -eq 6 ] || emit_error E_USAGE +script_path=${BASH_SOURCE[0]} +case "$script_path" in /*) ;; *) emit_error E_USAGE ;; esac +if [ "$1" = materialize ]; then + exec /usr/bin/env -i PATH="${PATH:-/usr/bin:/bin}" LC_ALL=C \ + /bin/bash "$script_path" __materialize_clean "$2" "$3" "$4" "$5" "$6" +fi +[ "$1" = __materialize_clean ] || emit_error E_USAGE +export LC_ALL=C +umask 077 +input_path=$2 +source_repository_id=$3 +source_git_dir=$4 +candidate_root=$5 +scratch_root=$6 + +case "$script_path:$input_path:$source_git_dir:$candidate_root:$scratch_root" in + /*:/*:/*:/*:/*) ;; + *) emit_error E_USAGE ;; +esac +[ -f "$script_path" ] && [ ! -L "$script_path" ] || emit_error E_PACKAGE +script_dir=$(CDPATH='' cd -P -- "${script_path%/*}" && pwd -P) || emit_error E_PACKAGE +repo_root=$(CDPATH='' cd -P -- "$script_dir/../../.." && pwd -P) || emit_error E_PACKAGE +protocol="$script_dir/protocol.jq" +core="$repo_root/scripts/core-contract.sh" +registry="$repo_root/core/v2/generation-registry.json" +for required in "$protocol" "$core" "$registry"; do + [ -f "$required" ] && [ ! -L "$required" ] || emit_error E_PACKAGE +done + +jq_bin=$(command -v jq 2>/dev/null) || emit_error E_DEPENDENCY +[ -f "$jq_bin" ] && [ ! -L "$jq_bin" ] && + [ "$($jq_bin --version 2>/dev/null)" = jq-1.6 ] || emit_error E_DEPENDENCY +generation=$($jq_bin -r '.[-1].generation_id' "$registry") || emit_error E_PACKAGE +modules="$repo_root/core/v2/generations/$generation/modules" +[ -d "$modules" ] && [ ! -L "$modules" ] || emit_error E_PACKAGE + +case "$source_repository_id" in + ''|*[!a-z0-9._:-]* ) emit_error E_REPOSITORY ;; +esac +[ "${#source_repository_id}" -le 128 ] || emit_error E_REPOSITORY + +physical_dir() { + local path=$1 actual + [ -d "$path" ] && [ ! -L "$path" ] || return 1 + actual=$(CDPATH='' cd -P -- "$path" && pwd -P) || return 1 + [ "$actual" = "$path" ] +} + +directory_mode() { + case "$(uname -s)" in + Darwin) /usr/bin/stat -f '%Lp' "$1" ;; + *) /usr/bin/stat -c '%a' "$1" ;; + esac +} + +directory_owner() { + case "$(uname -s)" in + Darwin) /usr/bin/stat -f '%u' "$1" ;; + *) /usr/bin/stat -c '%u' "$1" ;; + esac +} + +empty_private_dir() { + physical_dir "$1" && + [ "$(directory_mode "$1")" = 700 ] && + [ "$(directory_owner "$1")" = "$(id -u)" ] && + [ -z "$(find "$1" -mindepth 1 -print -quit)" ] +} + +overlaps() { + case "$1/" in "$2/"*) return 0 ;; esac + case "$2/" in "$1/"*) return 0 ;; esac + return 1 +} + +[ -f "$input_path" ] && [ ! -L "$input_path" ] || emit_error E_INPUT +physical_dir "$source_git_dir" || emit_error E_SOURCE_GIT +empty_private_dir "$candidate_root" || emit_error E_CANDIDATE_ROOT +empty_private_dir "$scratch_root" || emit_error E_SCRATCH_ROOT +if overlaps "$source_git_dir" "$candidate_root" || + overlaps "$source_git_dir" "$scratch_root" || + overlaps "$candidate_root" "$scratch_root"; then + emit_error E_BOUNDARY +fi + +run_root="$scratch_root/run" +staging_repo="$candidate_root/.staging.git" +final_repo="$candidate_root/repository.git" +success=0 +cleanup() { + /bin/rm -rf -- "$run_root" 2>/dev/null || : + if [ "$success" -ne 1 ]; then + /bin/rm -rf -- "$staging_repo" "$final_repo" 2>/dev/null || : + fi +} +trap cleanup EXIT +trap 'exit 129' HUP +trap 'exit 130' INT +trap 'exit 143' TERM +/bin/mkdir -m 700 "$run_root" || emit_error E_SCRATCH_ROOT +/bin/mkdir -m 500 "$run_root/no-hooks" || emit_error E_SCRATCH_ROOT + +input_snapshot="$run_root/input.json" +/bin/cp "$input_path" "$input_snapshot" || emit_error E_INPUT +/bin/chmod 0400 "$input_snapshot" || emit_error E_INPUT +input_canonical="$run_root/input.canonical" +"$jq_bin" -S -c . "$input_snapshot" > "$input_canonical" 2>/dev/null || emit_error E_INPUT +/usr/bin/cmp -s "$input_snapshot" "$input_canonical" || emit_error E_INPUT +"$jq_bin" -L "$modules" -e --arg command validate-input -f "$protocol" \ + "$input_snapshot" >/dev/null 2>&1 || emit_error E_CONTRACT + +sha_file() { + /usr/bin/shasum -a 256 "$1" | /usr/bin/awk '{print $1}' +} + +snapshot_pair() { + local selector=$1 output=$2 expected actual + "$jq_bin" -S -c "$selector.content" "$input_snapshot" > "$output" 2>/dev/null || + return 1 + expected=$("$jq_bin" -r "$selector.sha256" "$input_snapshot") || return 1 + actual=$(sha_file "$output") || return 1 + [ "$actual" = "$expected" ] +} + +profile_file="$run_root/profile.json" +resolved_file="$run_root/resolved-profile.json" +request_file="$run_root/stage-request.json" +snapshot_pair '.profile' "$profile_file" || emit_error E_DIGEST +snapshot_pair '.resolved_profile' "$resolved_file" || emit_error E_DIGEST +snapshot_pair '.stage_request' "$request_file" || emit_error E_DIGEST +manifest_files=() +manifest_count=$("$jq_bin" '.manifests | length' "$input_snapshot") || emit_error E_INPUT +manifest_index=0 +while [ "$manifest_index" -lt "$manifest_count" ]; do + manifest_file="$run_root/manifest.$manifest_index.json" + snapshot_pair ".manifests[$manifest_index]" "$manifest_file" || emit_error E_DIGEST + manifest_files+=("$manifest_file") + manifest_index=$((manifest_index + 1)) +done + +"$core" validate-profile-set "$profile_file" "$resolved_file" \ + "${manifest_files[@]}" >/dev/null 2>&1 || emit_error E_CORE_PROFILE +"$core" validate-document "$request_file" >/dev/null 2>&1 || emit_error E_CORE_REQUEST + +contract_file="$run_root/materialization-contract.json" +patch_file="$run_root/producer.patch" +"$jq_bin" -j -L "$modules" --arg command contract -f "$protocol" \ + "$input_snapshot" > "$contract_file" 2>/dev/null || emit_error E_CONTRACT +"$jq_bin" -j -L "$modules" --arg command patch -f "$protocol" \ + "$input_snapshot" > "$patch_file" 2>/dev/null || emit_error E_PATCH +contract_sha=$(sha_file "$contract_file") +patch_sha=$(sha_file "$patch_file") +expected_contract_sha=$("$jq_bin" -r ' + .stage_request.content.body.operation.arguments.materialization_contract.input_id as $id | + .payloads[] | select(.input_id == $id) | .sha256 +' "$input_snapshot") || emit_error E_DIGEST +expected_patch_sha=$("$jq_bin" -r \ + '.payloads[] | select(.input_id == "input.producer-patch") | .sha256' \ + "$input_snapshot") || emit_error E_DIGEST +[ "$contract_sha" = "$expected_contract_sha" ] && + [ "$patch_sha" = "$expected_patch_sha" ] || emit_error E_DIGEST +contract_canonical="$run_root/materialization-contract.canonical" +"$jq_bin" -S -c . "$contract_file" > "$contract_canonical" 2>/dev/null || emit_error E_CONTRACT +/usr/bin/cmp -s "$contract_file" "$contract_canonical" || emit_error E_CONTRACT + +patch_bytes=$(wc -c < "$patch_file" | /usr/bin/tr -d ' ') +max_patch_bytes=$("$jq_bin" -r '.max_patch_bytes' "$contract_file") || emit_error E_CONTRACT +[ "$patch_bytes" -gt 0 ] && [ "$patch_bytes" -le "$max_patch_bytes" ] || emit_error E_PATCH_LIMIT +patch_without_nul=$(LC_ALL=C /usr/bin/tr -d '\000' < "$patch_file" | wc -c | /usr/bin/tr -d ' ') +[ "$patch_without_nul" = "$patch_bytes" ] || emit_error E_BINARY_PATCH +if /usr/bin/grep -aEq '^(GIT binary patch|Binary files .+ differ)$' "$patch_file"; then + emit_error E_BINARY_PATCH +fi + +source_commit=$("$jq_bin" -r '.stage_request.content.body.target_revision.value.commit_id' \ + "$input_snapshot") || emit_error E_SOURCE_IDENTITY +source_algorithm=$("$jq_bin" -r '.stage_request.content.body.target_revision.value.hash_algorithm' \ + "$input_snapshot") || emit_error E_SOURCE_IDENTITY +request_repository_id=$("$jq_bin" -r '.stage_request.content.body.target_repository_id' \ + "$input_snapshot") || emit_error E_SOURCE_IDENTITY +source_input_id=$("$jq_bin" -r \ + '.stage_request.content.body.operation.arguments.source_tree_input_id' \ + "$input_snapshot") || emit_error E_SOURCE_IDENTITY +source_tree=$("$jq_bin" -r --arg id "$source_input_id" ' + .stage_request.content.body.inputs[] | select(.input_id == $id) | + .value.value.value.object_id +' "$input_snapshot") || emit_error E_SOURCE_IDENTITY +[ "$request_repository_id" = "$source_repository_id" ] || emit_error E_SOURCE_IDENTITY + +git_env=(/usr/bin/env -i HOME="$run_root" TMPDIR="$run_root" PATH=/usr/bin:/bin LC_ALL=C + GIT_CONFIG_NOSYSTEM=1 GIT_CONFIG_GLOBAL=/dev/null GIT_NO_REPLACE_OBJECTS=1 + GIT_NO_LAZY_FETCH=1 GIT_TERMINAL_PROMPT=0 GIT_OPTIONAL_LOCKS=0 + GIT_CONFIG_COUNT=1 GIT_CONFIG_KEY_0=core.hooksPath + GIT_CONFIG_VALUE_0="$run_root/no-hooks") + +git_dir() { + local directory=$1 + shift + "${git_env[@]}" /usr/bin/git --no-replace-objects --git-dir="$directory" "$@" +} + +source_config="$run_root/source-config" +git_dir "$source_git_dir" config --local --name-only --list --no-includes \ + > "$source_config" 2>/dev/null || emit_error E_SOURCE_GIT +while IFS= read -r config_key; do + case "$config_key" in + core.repositoryformatversion|core.filemode|core.bare|core.logallrefupdates|core.ignorecase|core.precomposeunicode|extensions.objectformat) ;; + '') ;; + *) emit_error E_SOURCE_CONFIG ;; + esac +done < "$source_config" +[ "$(git_dir "$source_git_dir" rev-parse --is-bare-repository 2>/dev/null)" = true ] || + emit_error E_SOURCE_WORKTREE +[ ! -e "$source_git_dir/commondir" ] && [ ! -e "$source_git_dir/shallow" ] && + [ ! -e "$source_git_dir/info/grafts" ] && + [ ! -e "$source_git_dir/objects/info/alternates" ] && + [ ! -d "$source_git_dir/refs/replace" ] && + [ -z "$(find "$source_git_dir" -type l -print -quit)" ] && + [ -z "$(find "$source_git_dir/objects/pack" -type f -name '*.promisor' -print -quit 2>/dev/null)" ] || + emit_error E_SOURCE_GIT +if find "$source_git_dir/hooks" -type f ! -name '*.sample' -print -quit 2>/dev/null | + /usr/bin/grep -q .; then + emit_error E_SOURCE_HOOK +fi +actual_algorithm=$(git_dir "$source_git_dir" rev-parse --show-object-format 2>/dev/null) || + emit_error E_SOURCE_GIT +[ "$actual_algorithm" = "$source_algorithm" ] || emit_error E_SOURCE_IDENTITY +git_dir "$source_git_dir" cat-file -e "$source_commit^{commit}" >/dev/null 2>&1 || + emit_error E_SOURCE_IDENTITY +actual_source_tree=$(git_dir "$source_git_dir" rev-parse "$source_commit^{tree}" 2>/dev/null) || + emit_error E_SOURCE_IDENTITY +[ "$actual_source_tree" = "$source_tree" ] || emit_error E_SOURCE_IDENTITY + +safe_repo_path() { + local value=$1 component lowered bytes + [ -n "$value" ] || return 1 + case "$value" in /*|*\\*) return 1 ;; esac + bytes=$(printf '%s' "$value" | wc -c | /usr/bin/tr -d ' ') + [ "$bytes" -le 4096 ] || return 1 + if printf '%s' "$value" | LC_ALL=C /usr/bin/grep -q '[[:cntrl:]]'; then return 1; fi + old_ifs=$IFS + IFS=/ + read -r -a path_components <<< "$value" + IFS=$old_ifs + for component in "${path_components[@]}"; do + [ -n "$component" ] && [ "$component" != . ] && [ "$component" != .. ] || return 1 + lowered=$(printf '%s' "$component" | /usr/bin/tr '[:upper:]' '[:lower:]') + [ "$lowered" != .git ] || return 1 + case "$component" in *.|*' ') return 1 ;; esac + done +} + +scan_tree() { + local repository=$1 tree=$2 output=$3 entry metadata mode type object path + : > "$output" + while IFS= read -r -d '' entry; do + metadata=${entry%%$'\t'*} + path=${entry#*$'\t'} + read -r mode type object <<< "$metadata" + [ -n "$object" ] || return 1 + case "$mode:$type" in 100644:blob|100755:blob) ;; *) return 1 ;; esac + safe_repo_path "$path" || return 1 + printf '%s\n' "$path" >> "$output" + done < <(git_dir "$repository" ls-tree -rz -r "$tree") +} + +source_paths="$run_root/source-paths" +scan_tree "$source_git_dir" "$source_tree" "$source_paths" || emit_error E_SOURCE_TREE + +"${git_env[@]}" /usr/bin/git init --bare --object-format="$source_algorithm" \ + "$staging_repo" >/dev/null 2>&1 || emit_error E_CANDIDATE_GIT +source_pack="$run_root/source.pack" +printf '%s\n' "$source_commit" | + git_dir "$source_git_dir" pack-objects --quiet --stdout --revs > "$source_pack" || + emit_error E_SOURCE_GIT +git_dir "$staging_repo" index-pack --stdin --fix-thin < "$source_pack" >/dev/null 2>&1 || + emit_error E_CANDIDATE_GIT +git_dir "$staging_repo" cat-file -e "$source_commit^{commit}" >/dev/null 2>&1 || + emit_error E_CANDIDATE_GIT + +index_file="$run_root/index" +GIT_INDEX_FILE="$index_file" git_dir "$staging_repo" read-tree "$source_tree" || + emit_error E_CANDIDATE_GIT +GIT_INDEX_FILE="$index_file" git_dir "$staging_repo" apply --cached --check \ + --whitespace=nowarn "$patch_file" >/dev/null 2>&1 || emit_error E_PATCH +GIT_INDEX_FILE="$index_file" git_dir "$staging_repo" apply --cached \ + --whitespace=nowarn "$patch_file" >/dev/null 2>&1 || emit_error E_PATCH +candidate_tree=$(GIT_INDEX_FILE="$index_file" git_dir "$staging_repo" write-tree 2>/dev/null) || + emit_error E_CANDIDATE_GIT +candidate_paths="$run_root/candidate-paths" +scan_tree "$staging_repo" "$candidate_tree" "$candidate_paths" || emit_error E_CANDIDATE_TREE + +changed_paths="$run_root/changed-paths" +: > "$changed_paths" +while IFS= read -r -d '' changed_path; do + safe_repo_path "$changed_path" || emit_error E_PATCH_PATH + printf '%s\n' "$changed_path" >> "$changed_paths" +done < <(git_dir "$staging_repo" diff-tree -r --name-only -z "$source_tree" "$candidate_tree") +LC_ALL=C /usr/bin/sort -u "$changed_paths" -o "$changed_paths" +changed_count=$(wc -l < "$changed_paths" | /usr/bin/tr -d ' ') +max_changed=$("$jq_bin" -r '.max_changed_paths' "$contract_file") || emit_error E_CONTRACT +[ "$changed_count" -le "$max_changed" ] || emit_error E_PATCH_LIMIT +while IFS= read -r changed_path; do + "$jq_bin" -e --arg path "$changed_path" '.allowed_paths | index($path) != null' \ + "$contract_file" >/dev/null || emit_error E_PATCH_SCOPE +done < "$changed_paths" +changed_paths_json="$run_root/changed-paths.json" +"$jq_bin" -Rsc 'split("\n") | map(select(length > 0))' "$changed_paths" | + "$jq_bin" -S -c . > "$changed_paths_json" || emit_error E_RUNTIME +changed_paths_sha=$(sha_file "$changed_paths_json") + +commit_time=$("$jq_bin" -r '.attempt.started_at' "$input_snapshot") || emit_error E_INPUT +candidate_commit=$(printf '%s\n' 'ystack local candidate' | + "${git_env[@]}" GIT_AUTHOR_NAME='ystack local materializer' \ + GIT_AUTHOR_EMAIL='materializer@example.invalid' GIT_COMMITTER_NAME='ystack local materializer' \ + GIT_COMMITTER_EMAIL='materializer@example.invalid' GIT_AUTHOR_DATE="$commit_time" \ + GIT_COMMITTER_DATE="$commit_time" /usr/bin/git --no-replace-objects \ + --git-dir="$staging_repo" commit-tree "$candidate_tree" -p "$source_commit") || + emit_error E_CANDIDATE_GIT +git_dir "$staging_repo" update-ref refs/heads/candidate "$candidate_commit" || + emit_error E_CANDIDATE_GIT +[ "$(git_dir "$staging_repo" rev-parse "$candidate_commit^" 2>/dev/null)" = "$source_commit" ] && + [ "$(git_dir "$staging_repo" rev-parse "$candidate_commit^{tree}" 2>/dev/null)" = "$candidate_tree" ] && + [ "$(git_dir "$staging_repo" rev-parse --is-bare-repository 2>/dev/null)" = true ] || + emit_error E_CANDIDATE_GIT +git_dir "$staging_repo" fsck --strict --no-progress >/dev/null 2>&1 || emit_error E_CANDIDATE_GIT + +receipt_file="$run_root/receipt.json" +"$jq_bin" -S -c -L "$modules" --arg command receipt \ + --arg source_repository_id "$source_repository_id" \ + --arg source_hash_algorithm "$source_algorithm" --arg source_commit "$source_commit" \ + --arg source_tree "$source_tree" --arg candidate_commit "$candidate_commit" \ + --arg candidate_tree "$candidate_tree" --arg changed_path_count "$changed_count" \ + --arg changed_paths_sha256 "$changed_paths_sha" -f "$protocol" "$input_snapshot" \ + > "$receipt_file" 2>/dev/null || emit_error E_RECEIPT +receipt_sha=$(sha_file "$receipt_file") +outcome=changed +[ "$candidate_tree" != "$source_tree" ] || outcome=no-change +result_file="$run_root/stage-result.json" +"$jq_bin" -S -c -L "$modules" --arg command stage-result \ + --arg receipt_sha256 "$receipt_sha" --arg outcome "$outcome" \ + -f "$protocol" "$input_snapshot" > "$result_file" 2>/dev/null || emit_error E_RESULT +"$core" validate-stage-run "$request_file" "$resolved_file" "$result_file" \ + >/dev/null 2>&1 || emit_error E_CORE_RESULT + +/bin/mv "$staging_repo" "$final_repo" || emit_error E_CANDIDATE_ROOT +response_file="$run_root/response.json" +"$jq_bin" -S -c -n --slurpfile result "$result_file" --rawfile receipt "$receipt_file" \ + --arg receipt_sha "$receipt_sha" ' + { + schema_version:1, + kind:"local_git_materialization_response", + stage_result:$result[0], + payloads:[{ + content_id:"candidate.materialization.receipt", + media_type:"application/json", + sha256:$receipt_sha, + data:$receipt + }], + authority:"none", + qualification:{state:"unavailable",reason_id:"adapter.unqualified"}, + effects:["caller-disposable-candidate-repository"] + } +' > "$response_file" || emit_error E_RESULT +success=1 +/bin/cat "$response_file" diff --git a/adapters/local-git-materializer/v1/protocol.jq b/adapters/local-git-materializer/v1/protocol.jq new file mode 100644 index 0000000..387b9f8 --- /dev/null +++ b/adapters/local-git-materializer/v1/protocol.jq @@ -0,0 +1,308 @@ +import "schema" as schema; +import "profile_graph" as profile; +import "stage_request" as request; + +def exact($required; $optional): + . as $value | + type == "object" and + ((keys_unsorted - ($required + $optional)) | length) == 0 and + all($required[]; . as $key | $value | has($key)); + +def pair($kind): + profile::document_pair_ok($kind); + +def path_ok: + type == "string" and utf8bytelength >= 1 and utf8bytelength <= 4096 and + (test("[\\x{0000}-\\x{001f}\\x{007f}-\\x{009f}]") | not) and + (startswith("/") | not) and (contains("\\") | not) and + (split("/") | + all(.[]; + . != "" and . != "." and . != ".." and + (ascii_downcase != ".git") and + (endswith(".") | not) and (endswith(" ") | not))); + +def attempt_ok: + exact( + ["attempt_id","attempt_number","result_id","started_at","finished_at", + "recorded_at"]; + []) and + (.attempt_id | schema::id_ok) and + (.attempt_number | schema::int_ok) and .attempt_number >= 1 and + (.result_id | schema::id_ok) and + (.started_at | schema::time_ok) and + (.finished_at | schema::time_ok) and + (.recorded_at | schema::time_ok) and + .started_at <= .finished_at and .finished_at <= .recorded_at; + +def payload_ok: + exact(["input_id","media_type","sha256","data"];[]) and + (.input_id | schema::id_ok) and + (.media_type | schema::media_type_ok) and + (.sha256 | schema::sha256_ok) and + (.data | type == "string" and utf8bytelength <= 2097152); + +def materialization_contract_ok: + exact( + ["schema_version","kind","allowed_paths","max_patch_bytes", + "max_changed_paths","allowed_modes","allow_binary_patch", + "allow_symlinks","allow_submodules","candidate_repository_kind"]; + []) and + .schema_version == 1 and .kind == "local_git_materialization_contract" and + (.allowed_paths | + type == "array" and length >= 1 and length <= 256 and + all(.[];path_ok) and . == sort and length == (unique | length)) and + (.max_patch_bytes | schema::int_ok) and + .max_patch_bytes >= 1 and .max_patch_bytes <= 1048576 and + (.max_changed_paths | schema::int_ok) and + .max_changed_paths >= 1 and .max_changed_paths <= 256 and + .max_changed_paths <= (.allowed_paths | length) and + .allowed_modes == ["100644","100755"] and + .allow_binary_patch == false and .allow_symlinks == false and + .allow_submodules == false and .candidate_repository_kind == "bare"; + +def input_content_ref($body; $input_id): + [$body.inputs[] | + select(.input_id == $input_id and .value.type == "artifact" and + .value.value.type == "content") | + .value.value.value] as $matches | + if ($matches | length) == 1 then $matches[0] else null end; + +def payload_for($input; $input_id): + [$input.payloads[] | select(.input_id == $input_id)] as $matches | + if ($matches | length) == 1 then $matches[0] else null end; + +def payload_matches_input($input; $input_id): + input_content_ref($input.stage_request.content.body;$input_id) as $ref | + payload_for($input;$input_id) as $payload | + $ref != null and $payload != null and + $payload.media_type == $ref.media_type and $payload.sha256 == $ref.sha256; + +def selected_binding($input): + [$input.resolved_profile.content.body.bindings[] | + select(.binding.binding_id == + $input.stage_request.content.body.operation.binding_id)]; + +def selected_manifest($input; $binding): + [$input.manifests[] | + select(profile::document_ref_for_pair(.) == $binding.binding.manifest_ref)]; + +def materializer_manifest_ok: + (.content | profile::adapter_manifest_self_ok) and + .content.id == "adapter.local-git-materializer.v1" and + .content.body.adapter_version == "v1" and + .content.body.offered_roles == ["forge"] and + .content.body.offered_execution_kinds == ["deterministic"] and + .content.body.offered_capabilities == ["core.forge.materialize-candidate.v2"] and + .content.body.offered_permissions == [ + "core.perm.candidate-repository.write.v2", + "core.perm.evidence.write.v1", + "core.perm.scratch.write.v1", + "core.perm.target.read.v1" + ] and + .content.body.offered_tools == [] and + (.content.body | has("config_contract_ref") | not); + +def core_relations_ok($input): + ($input.profile | pair("profile")) and + ($input.resolved_profile | pair("resolved_profile")) and + ($input.manifests | + type == "array" and length >= 1 and length <= 8 and + all(.[];pair("adapter_manifest")) and + (map(.content.id) | . == sort and length == (unique | length))) and + profile::profile_set_ok( + $input.profile;$input.resolved_profile;$input.manifests) and + ($input.stage_request | pair("stage_request")) and + ($input.stage_request.content | request::document_self_ok) and + request::stage_request_resolved_ref_ok( + $input.stage_request;$input.resolved_profile) and + request::stage_request_resolved_relation_ok( + $input.stage_request.content.body;$input.resolved_profile.content.body); + +def materializer_relations_ok($input): + $input.stage_request.content.body as $body | + selected_binding($input) as $bindings | + ($bindings | length) == 1 and + $bindings[0] as $binding | + selected_manifest($input;$binding) as $manifests | + ($manifests | length) == 1 and + ($manifests[0] | materializer_manifest_ok) and + profile::binding_manifest_graph_ok( + $binding.binding;$binding;$manifests[0]) and + $body.operation == { + role:"forge", + binding_id:$binding.binding.binding_id, + capability_id:"core.forge.materialize-candidate.v2", + permissions:[ + "core.perm.candidate-repository.write.v2", + "core.perm.evidence.write.v1", + "core.perm.scratch.write.v1", + "core.perm.target.read.v1" + ], + arguments:$body.operation.arguments + } and + $body.operation.arguments.network_mode == "deny" and + $body.target_revision.state == "present" and + $binding.binding.execution_kind == "deterministic" and + ($binding.binding | has("config_ref") | not) and + ($binding.binding | has("prompt_ref") | not) and + ($binding.binding | has("model_request") | not) and + $binding.binding.skill_refs == [] and $binding.binding.requested_tools == [] and + $input.attempt.started_at >= $body.requested_at; + +def payload_relations_ok($input): + $input.stage_request.content.body.operation.arguments as $arguments | + $arguments.materialization_contract.input_id as $contract_id | + ($input.payloads | + type == "array" and length == 2 and all(.[];payload_ok) and + (map(.input_id) | . == sort and length == (unique | length))) and + payload_matches_input($input;$contract_id) and + payload_matches_input($input;"input.producer-patch") and + (payload_for($input;$contract_id).media_type == "application/json") and + (payload_for($input;"input.producer-patch").media_type == "text/x-diff") and + (payload_for($input;$contract_id).data | + try fromjson catch null | materialization_contract_ok); + +def input_ok: + . as $input | + exact( + ["schema_version","kind","attempt","profile","resolved_profile", + "manifests","stage_request","payloads"]; + []) and + .schema_version == 1 and .kind == "local_git_materialization_input" and + (.attempt | attempt_ok) and + core_relations_ok($input) and materializer_relations_ok($input) and + payload_relations_ok($input); + +def document_ref($pair): profile::document_ref_for_pair($pair); + +def receipt: + . as $input | + ($ARGS.named.source_repository_id // "") as $source_repository_id | + ($ARGS.named.source_hash_algorithm // "") as $source_hash_algorithm | + ($ARGS.named.source_commit // "") as $source_commit | + ($ARGS.named.source_tree // "") as $source_tree | + ($ARGS.named.candidate_commit // "") as $candidate_commit | + ($ARGS.named.candidate_tree // "") as $candidate_tree | + ($ARGS.named.changed_path_count // "") as $changed_path_count | + ($ARGS.named.changed_paths_sha256 // "") as $changed_paths_sha256 | + if (($source_hash_algorithm == "sha1" and + ($source_commit | test("\\A[0-9a-f]{40}\\z")) and + ($source_tree | test("\\A[0-9a-f]{40}\\z")) and + ($candidate_commit | test("\\A[0-9a-f]{40}\\z")) and + ($candidate_tree | test("\\A[0-9a-f]{40}\\z"))) or + ($source_hash_algorithm == "sha256" and + ($source_commit | test("\\A[0-9a-f]{64}\\z")) and + ($source_tree | test("\\A[0-9a-f]{64}\\z")) and + ($candidate_commit | test("\\A[0-9a-f]{64}\\z")) and + ($candidate_tree | test("\\A[0-9a-f]{64}\\z")))) and + ($changed_paths_sha256 | schema::sha256_ok) and + ($changed_path_count | tonumber | schema::int_ok) then + ($input.stage_request.content.body.operation.arguments.materialization_contract.input_id) as $contract_id | + { + schema_version:1, + kind:"candidate_materialization_receipt", + adapter:{id:"adapter.local-git-materializer.v1",version:"v1",status:"inactive"}, + attempt:{ + attempt_id:$input.attempt.attempt_id, + attempt_number:$input.attempt.attempt_number + }, + request_ref:document_ref($input.stage_request), + resolved_profile_ref:document_ref($input.resolved_profile), + manifest_ref:(selected_binding($input)[0].binding.manifest_ref), + materialization_contract_ref: + input_content_ref($input.stage_request.content.body;$contract_id), + patch_ref: + input_content_ref($input.stage_request.content.body;"input.producer-patch"), + source:{ + repository_id:$source_repository_id, + hash_algorithm:$source_hash_algorithm, + commit_id:$source_commit, + tree_id:$source_tree + }, + candidate:{ + repository_kind:"bare", + hash_algorithm:$source_hash_algorithm, + commit_id:$candidate_commit, + tree_id:$candidate_tree, + parent_commit_id:$source_commit + }, + changed_paths:{ + count:($changed_path_count | tonumber), + sha256:$changed_paths_sha256 + } + } + else error("E_RECEIPT") end; + +def recorded($value; $source_ref): + {state:"recorded",value:$value,source_ref:$source_ref}; +def not_applicable: {state:"not-applicable"}; + +def stage_result: + . as $input | + ($ARGS.named.receipt_sha256 // "") as $receipt_sha256 | + ($ARGS.named.outcome // "") as $outcome | + request::expected_execution_projection( + $input.stage_request.content.body;$input.resolved_profile.content.body) as $projection | + { + content_id:"candidate.materialization.receipt", + media_type:"application/json", + sha256:$receipt_sha256 + } as $receipt_ref | + ($outcome == "changed" or $outcome == "no-change") as $outcome_ok | + if $outcome_ok and ($receipt_sha256 | schema::sha256_ok) and $projection != null then + { + schema_version:2, + kind:"stage_result", + id:$input.attempt.result_id, + body:{ + request_ref:document_ref($input.stage_request), + resolved_profile_ref:document_ref($input.resolved_profile), + attempt_id:$input.attempt.attempt_id, + attempt_number:$input.attempt.attempt_number, + reported_by:$projection.performer, + status:"completed", + outcome:{family:"change",value:$outcome}, + outputs:(if $outcome == "changed" then [{ + output_id:$input.stage_request.content.body.operation.arguments.candidate_output_id, + ref:$receipt_ref + }] else [] end), + diagnostics:[], + execution:{ + performer:$projection.performer, + actual_binding:$projection.actual_binding, + environment:$projection.environment, + used_capability:$projection.used_capability, + metadata:{ + kind:"deterministic", + provider:not_applicable, + model:not_applicable, + snapshot:not_applicable, + effort:not_applicable, + prompt:not_applicable, + skills:not_applicable, + tools:recorded([];$receipt_ref) + } + }, + evidence:[{ + evidence_id:"evidence.local-git-materialization", + kind:"deterministic", + verdict:"passed", + proof_ref:$receipt_ref + }], + started_at:$input.attempt.started_at, + finished_at:$input.attempt.finished_at, + recorded_at:$input.attempt.recorded_at + } + } + else error("E_RESULT") end; + +if $command == "validate-input" then input_ok +elif $command == "contract" then + . as $input | + payload_for( + $input; + $input.stage_request.content.body.operation.arguments.materialization_contract.input_id).data +elif $command == "patch" then payload_for(.;"input.producer-patch").data +elif $command == "receipt" then receipt +elif $command == "stage-result" then stage_result +else error("E_COMMAND") end diff --git a/ci/required-files.txt b/ci/required-files.txt index e93e1c9..a8890cf 100644 --- a/ci/required-files.txt +++ b/ci/required-files.txt @@ -236,3 +236,8 @@ scripts/test/default-github-forge-adapter.test.sh # Inactive GitHub Actions CI normalizer payload adapters/github-actions-ci/v1/normalize.jq scripts/test/default-github-actions-ci-adapter.test.sh + +# Inactive provider-neutral local Git candidate materializer +adapters/local-git-materializer/v1/materialize.sh +adapters/local-git-materializer/v1/protocol.jq +scripts/test/local-git-materializer-adapter.test.sh diff --git a/scripts/test/local-git-materializer-adapter.test.sh b/scripts/test/local-git-materializer-adapter.test.sh new file mode 100755 index 0000000..0f0d4bf --- /dev/null +++ b/scripts/test/local-git-materializer-adapter.test.sh @@ -0,0 +1,556 @@ +#!/usr/bin/env bash +# shellcheck disable=SC2016 +set -euo pipefail +export LC_ALL=C +umask 077 + +root=$(CDPATH='' cd -P -- "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd -P) +adapter="$root/adapters/local-git-materializer/v1/materialize.sh" +protocol="$root/adapters/local-git-materializer/v1/protocol.jq" +test_tmp_base=${TMPDIR:-/tmp} +tmp=$(/usr/bin/mktemp -d "${test_tmp_base%/}/ystack-local-materializer.XXXXXX") +tmp=$(CDPATH='' cd -P -- "$tmp" && pwd -P) +cleanup() { /bin/rm -rf -- "$tmp"; } +trap cleanup EXIT + +sha_file() { /usr/bin/shasum -a 256 "$1" | /usr/bin/awk '{print $1}'; } +platform=$(/usr/bin/uname -s):$(/usr/bin/uname -m) +case "$platform" in + Linux:x86_64) jq_asset=jq-linux64; jq_sha=af986793a515d500ab2d35f8d2aecd656e764504b789b66d7e1a0b727a124c44 ;; + Darwin:x86_64|Darwin:arm64) jq_asset=jq-osx-amd64; jq_sha=5c0a0a3ea600f302ee458b30317425dd9632d1ad8882259fcaf4e9b868b2b1ef ;; + *) printf 'FAIL: unsupported host %s\n' "$platform" >&2; exit 1 ;; +esac +jq_bin="${TMPDIR:-/tmp}/ystack-portable-core-jq16/$jq_asset" +[ -f "$jq_bin" ] && [ ! -L "$jq_bin" ] && [ "$(sha_file "$jq_bin")" = "$jq_sha" ] || { + printf '%s\n' 'FAIL: pinned jq 1.6 is required' >&2 + exit 1 +} +jq_cmd=("$jq_bin") +[ "$platform" != Darwin:arm64 ] || jq_cmd=(/usr/bin/arch -x86_64 "$jq_bin") +[ "$("${jq_cmd[@]}" --version)" = jq-1.6 ] || exit 1 +runtime_bin="$tmp/bin" +/bin/mkdir -m 700 "$runtime_bin" +if [ "$platform" = Darwin:arm64 ]; then + printf '%s\n' '#!/bin/bash' "exec /usr/bin/arch -x86_64 '$jq_bin' \"\$@\"" > "$runtime_bin/jq" +else + /bin/cp "$jq_bin" "$runtime_bin/jq" +fi +/bin/chmod 0555 "$runtime_bin/jq" +export PATH="$runtime_bin:/usr/bin:/bin" +generation=$("${jq_cmd[@]}" -r '.[-1].generation_id' "$root/core/v2/generation-registry.json") +modules="$root/core/v2/generations/$generation/modules" +core="$root/scripts/core-contract.sh" + +passed=0 +pass() { passed=$((passed + 1)); printf 'ok %s - %s\n' "$passed" "$1"; } +fail() { printf 'FAIL: %s\n' "$1" >&2; exit 1; } + +git_clean() { + /usr/bin/env -i HOME="$tmp/home" TMPDIR="$tmp" PATH=/usr/bin:/bin LC_ALL=C \ + GIT_CONFIG_NOSYSTEM=1 GIT_CONFIG_GLOBAL=/dev/null GIT_NO_REPLACE_OBJECTS=1 \ + GIT_NO_LAZY_FETCH=1 GIT_TERMINAL_PROMPT=0 GIT_OPTIONAL_LOCKS=0 \ + /usr/bin/git --no-replace-objects "$@" +} + +make_bare_source() { + local destination=$1 format=$2 mode=${3:-100644} source_blob tree commit + /bin/mkdir -m 700 "$destination" + git_clean init -q --bare --object-format="$format" "$destination" + source_blob=$(printf '%s\n' 'alpha' 'beta' | git_clean --git-dir="$destination" hash-object -w --stdin) + tree=$(printf '%s blob %s\tsource.txt\n' "$mode" "$source_blob" | + git_clean --git-dir="$destination" mktree) + commit=$(printf '%s\n' source | + /usr/bin/env -i HOME="$tmp/home" TMPDIR="$tmp" PATH=/usr/bin:/bin LC_ALL=C \ + GIT_CONFIG_NOSYSTEM=1 GIT_CONFIG_GLOBAL=/dev/null GIT_NO_REPLACE_OBJECTS=1 \ + GIT_AUTHOR_NAME=fixture GIT_AUTHOR_EMAIL=fixture@example.invalid \ + GIT_COMMITTER_NAME=fixture GIT_COMMITTER_EMAIL=fixture@example.invalid \ + GIT_AUTHOR_DATE=2000-01-01T00:00:00Z GIT_COMMITTER_DATE=2000-01-01T00:00:00Z \ + /usr/bin/git --no-replace-objects --git-dir="$destination" commit-tree "$tree") + git_clean --git-dir="$destination" update-ref refs/heads/main "$commit" + printf '%s %s\n' "$commit" "$tree" +} + +/bin/mkdir -m 700 "$tmp/home" +read -r source_commit source_tree < <(make_bare_source "$tmp/source.git" sha1) +source_fingerprint=$(find "$tmp/source.git" -type f -print0 | LC_ALL=C sort -z | + xargs -0 /usr/bin/shasum -a 256 | /usr/bin/shasum -a 256 | /usr/bin/awk '{print $1}') + +contract_file="$tmp/contract.json" +"${jq_cmd[@]}" -S -c -n '{ + schema_version:1,kind:"local_git_materialization_contract", + allowed_paths:["source.txt"],max_patch_bytes:65536,max_changed_paths:1, + allowed_modes:["100644","100755"],allow_binary_patch:false, + allow_symlinks:false,allow_submodules:false,candidate_repository_kind:"bare" +}' > "$contract_file" +patch_file="$tmp/change.patch" +printf '%s\n' \ + 'diff --git a/source.txt b/source.txt' \ + '--- a/source.txt' \ + '+++ b/source.txt' \ + '@@ -1,2 +1,3 @@' \ + ' alpha' \ + ' beta' \ + '+gamma' > "$patch_file" +contract_sha=$(sha_file "$contract_file") +patch_sha=$(sha_file "$patch_file") + +manifest_dir="$tmp/manifests" +/bin/mkdir -m 700 "$manifest_dir" +forge_manifest="$manifest_dir/forge.json" +"${jq_cmd[@]}" -L "$root/scripts/test" -S -c -n ' + import "portable-core-profile-graph-fixtures" as f; + def v2: walk(if type=="object" and has("schema_version") then .schema_version=2 else . end); + { + schema_version:2,kind:"adapter_manifest",id:"adapter.local-git-materializer.v1", + body:{adapter_version:"v1",package_ref:(f::blob("adapters/local-git-materializer/v1";"6") | + .location={kind:"root"} | .object_type="tree" | .mode="040000"), + offered_roles:["forge"],offered_execution_kinds:["deterministic"], + offered_capabilities:["core.forge.materialize-candidate.v2"], + offered_permissions:["core.perm.candidate-repository.write.v2", + "core.perm.evidence.write.v1","core.perm.scratch.write.v1", + "core.perm.target.read.v1"],offered_tools:[]}} + | v2 +' > "$forge_manifest" +for role in producer publisher reviewer verifier; do + "${jq_cmd[@]}" -L "$root/scripts/test" -S -c -n --arg role "$role" ' + import "portable-core-profile-graph-fixtures" as f; + def v2: walk(if type=="object" and has("schema_version") then .schema_version=2 else . end); + f::manifest($role) | v2 + ' > "$manifest_dir/$role.json" +done +manifest_shas=$( + "${jq_cmd[@]}" -S -c -n \ + --arg forge "$(sha_file "$forge_manifest")" \ + --arg producer "$(sha_file "$manifest_dir/producer.json")" \ + --arg publisher "$(sha_file "$manifest_dir/publisher.json")" \ + --arg reviewer "$(sha_file "$manifest_dir/reviewer.json")" \ + --arg verifier "$(sha_file "$manifest_dir/verifier.json")" \ + '{forge:$forge,producer:$producer,publisher:$publisher,reviewer:$reviewer,verifier:$verifier}' +) + +profile_file="$tmp/profile.json" +"${jq_cmd[@]}" -L "$root/scripts/test" -S -c -n --argjson shas "$manifest_shas" \ + --slurpfile forge "$forge_manifest" ' + import "portable-core-profile-graph-fixtures" as f; + def v2: walk(if type=="object" and has("schema_version") then .schema_version=2 else . end); + def forge_binding: { + binding_id:"binding.forge",role:"forge", + manifest_ref:{schema_version:2,kind:"adapter_manifest",id:$forge[0].id,sha256:$shas.forge}, + execution_kind:"deterministic",adapter_instance_id:"instance.forge", + principal_id:"principal.forge",execution_boundary_id:"boundary.forge", + authority_ref:f::scope("authority";"authority-forge";f::sha("5")), + package_ref:$forge[0].body.package_ref,skill_refs:[],requested_tools:[], + requested_capabilities:["core.forge.materialize-candidate.v2"], + requested_permissions:["core.perm.candidate-repository.write.v2", + "core.perm.evidence.write.v1","core.perm.scratch.write.v1", + "core.perm.target.read.v1"]}; + f::profile_doc($shas) | v2 | + .body.bindings += [forge_binding] | .body.bindings |= sort_by(.binding_id) +' > "$profile_file" +profile_sha=$(sha_file "$profile_file") + +resolved_file="$tmp/resolved.json" +"${jq_cmd[@]}" -L "$root/scripts/test" -S -c -n --argjson shas "$manifest_shas" \ + --slurpfile profile "$profile_file" --slurpfile forge "$forge_manifest" \ + --arg profile_sha "$profile_sha" ' + import "portable-core-profile-graph-fixtures" as f; + def v2: walk(if type=="object" and has("schema_version") then .schema_version=2 else . end); + f::resolved_profile_doc($profile[0];$profile_sha;$shas) | v2 | + .body.bindings |= map(if .binding.role=="forge" then + .adapter_implementation={id:$forge[0].id,version:"v1"} | + .manifest_source=f::source_value(f::blob("manifests/forge.json";"a");"canonical-json";$shas.forge) | + .package_source=f::source_value($forge[0].body.package_ref;"raw-bytes";f::sha("6")) | + .config_source={state:"absent"} | .prompt_source={state:"absent"} | + .skill_sources=[] | .tool_sources=[] + else . end) +' > "$resolved_file" +resolved_sha=$(sha_file "$resolved_file") + +request_file="$tmp/request.json" +"${jq_cmd[@]}" -L "$root/scripts/test" -S -c -n \ + --arg resolved_sha "$resolved_sha" --arg source_commit "$source_commit" \ + --arg source_tree "$source_tree" --arg contract_sha "$contract_sha" \ + --arg patch_sha "$patch_sha" ' + import "portable-core-stage-request-fixtures" as f; + def v2: walk(if type=="object" and has("schema_version") then .schema_version=2 else . end); + def revision: {repository_id:"fixture.target",hash_algorithm:"sha1",commit_id:$source_commit}; + def content($id;$media;$sha): {content_id:$id,media_type:$media,sha256:$sha}; + def named($id;$ref): {input_id:$id,value:{type:"artifact",value:{type:"content",value:$ref}}}; + f::request_doc("producer";$resolved_sha) | v2 | + .id="request.local-git-materializer" | .body.stage_id="stage.materialize" | + .body.target_repository_id="fixture.target" | + .body.target_revision={state:"present",value:revision} | + .body.source={state:"present",value:{type:"git-object",value:{revision:revision, + location:{kind:"root"},object_type:"tree",object_id:$source_tree,mode:"040000"}}} | + .body.base={state:"present",value:revision} | + .body.inputs=([ + f::named_content_input("finish";f::sha("1")), + named("input.materialize";content("payload-materialize";"application/json";$contract_sha)), + named("input.producer-patch";content("producer.patch";"text/x-diff";$patch_sha)), + {input_id:"input.source-tree",value:{type:"artifact",value:{type:"git-object",value:{ + revision:revision,location:{kind:"root"},object_type:"tree",object_id:$source_tree,mode:"040000"}}}}, + f::named_content_input("verify";f::sha("2"))] | sort_by(.input_id)) | + .body.operation={role:"forge",binding_id:"binding.forge", + capability_id:"core.forge.materialize-candidate.v2", + permissions:["core.perm.candidate-repository.write.v2","core.perm.evidence.write.v1", + "core.perm.scratch.write.v1","core.perm.target.read.v1"], + arguments:{source_tree_input_id:"input.source-tree",candidate_output_id:"candidate.repository", + materialization_contract:{ref:(f::scope("output-contract";"materialize";f::sha("3")) | + .subject_ref.value.value=content("payload-materialize";"application/json";$contract_sha)), + input_id:"input.materialize"},network_mode:"deny"}} | + .body.required_evidence_kinds=["deterministic"] +' > "$request_file" +request_sha=$(sha_file "$request_file") + +input_file="$tmp/input.json" +"${jq_cmd[@]}" -S -c -n --slurpfile profile "$profile_file" \ + --slurpfile resolved "$resolved_file" --slurpfile request "$request_file" \ + --slurpfile forge "$forge_manifest" --slurpfile producer "$manifest_dir/producer.json" \ + --slurpfile publisher "$manifest_dir/publisher.json" --slurpfile reviewer "$manifest_dir/reviewer.json" \ + --slurpfile verifier "$manifest_dir/verifier.json" --rawfile contract "$contract_file" \ + --rawfile patch "$patch_file" --argjson shas "$manifest_shas" \ + --arg profile_sha "$profile_sha" --arg resolved_sha "$resolved_sha" --arg request_sha "$request_sha" \ + --arg contract_sha "$contract_sha" --arg patch_sha "$patch_sha" ' + {schema_version:1,kind:"local_git_materialization_input", + attempt:{attempt_id:"attempt.materialize",attempt_number:1,result_id:"result.materialize", + started_at:"2026-08-30T00:00:01Z",finished_at:"2026-08-30T00:00:02Z", + recorded_at:"2026-08-30T00:00:03Z"}, + profile:{content:$profile[0],sha256:$profile_sha}, + resolved_profile:{content:$resolved[0],sha256:$resolved_sha}, + manifests:([ + {content:$forge[0],sha256:$shas.forge}, + {content:$producer[0],sha256:$shas.producer}, + {content:$publisher[0],sha256:$shas.publisher}, + {content:$reviewer[0],sha256:$shas.reviewer}, + {content:$verifier[0],sha256:$shas.verifier}] | sort_by(.content.id)), + stage_request:{content:$request[0],sha256:$request_sha}, + payloads:([ + {input_id:"input.materialize",media_type:"application/json",sha256:$contract_sha,data:$contract}, + {input_id:"input.producer-patch",media_type:"text/x-diff",sha256:$patch_sha,data:$patch}] + | sort_by(.input_id))} +' > "$input_file" + +for document in "$profile_file" "$resolved_file" "$forge_manifest" \ + "$manifest_dir/producer.json" "$manifest_dir/publisher.json" \ + "$manifest_dir/reviewer.json" "$manifest_dir/verifier.json"; do + "$core" validate-document "$document" || fail "core-document-${document##*/}" +done +"$core" validate-profile-set "$profile_file" "$resolved_file" \ + "$forge_manifest" "$manifest_dir/producer.json" "$manifest_dir/publisher.json" \ + "$manifest_dir/reviewer.json" "$manifest_dir/verifier.json" || fail core-profile-fixture +"$core" validate-document "$request_file" || fail core-request-fixture +"${jq_cmd[@]}" -L "$modules" -e --arg command validate-input -f "$protocol" \ + "$input_file" >/dev/null || fail protocol-fixture +pass 'core v2 input fixture validates' + +run_case() { + local name=$1 input=${2:-$input_file} source=${3:-$tmp/source.git} + local case_root="$tmp/case-$name" + local candidate="$case_root/candidate" scratch="$case_root/scratch" + /bin/mkdir -m 700 "$case_root" "$candidate" "$scratch" + PATH="$runtime_bin:/usr/bin:/bin" GH_TOKEN=must-not-read GITHUB_TOKEN=must-not-read \ + AWS_SECRET_ACCESS_KEY=must-not-read SSH_AUTH_SOCK=/must/not/read \ + "$adapter" materialize "$input" fixture.target "$source" "$candidate" "$scratch" \ + > "$case_root/out" 2> "$case_root/err" + printf '%s\n' "$case_root" +} + +case_root=$(run_case success) +[ ! -s "$case_root/err" ] || fail success-stderr +"${jq_cmd[@]}" -e ' + .schema_version==1 and .kind=="local_git_materialization_response" and + .authority=="none" and .qualification=={state:"unavailable",reason_id:"adapter.unqualified"} and + .effects==["caller-disposable-candidate-repository"] and + .stage_result.body.status=="completed" and .stage_result.body.outcome=={family:"change",value:"changed"} and + .stage_result.body.outputs[0].output_id=="candidate.repository" and + .payloads[0].content_id=="candidate.materialization.receipt" +' "$case_root/out" >/dev/null || fail success-response +receipt_file="$case_root/receipt" +"${jq_cmd[@]}" -j '.payloads[0].data' "$case_root/out" > "$receipt_file" +[ "$(sha_file "$receipt_file")" = "$("${jq_cmd[@]}" -r '.payloads[0].sha256' "$case_root/out")" ] || + fail receipt-digest +if /usr/bin/grep -Fq "$tmp" "$receipt_file" || + /usr/bin/grep -Eq 'must-not-read|GH_TOKEN|GITHUB_TOKEN|AWS_SECRET_ACCESS_KEY|SSH_AUTH_SOCK' \ + "$case_root/out"; then + fail receipt-leaked-local-or-credential-data +fi +candidate_repo="$case_root/candidate/repository.git" +[ "$(git_clean --git-dir="$candidate_repo" rev-parse --is-bare-repository)" = true ] || fail bare +candidate_commit=$(git_clean --git-dir="$candidate_repo" rev-parse refs/heads/candidate) +[ "$(git_clean --git-dir="$candidate_repo" rev-parse "$candidate_commit^")" = "$source_commit" ] || fail parent +git_clean --git-dir="$candidate_repo" show "$candidate_commit:source.txt" | + /usr/bin/grep -Fxq gamma || fail patch-content +[ -z "$(find "$case_root/scratch" -mindepth 1 -print -quit)" ] || fail scratch-clean +[ "$source_fingerprint" = "$(find "$tmp/source.git" -type f -print0 | LC_ALL=C sort -z | + xargs -0 /usr/bin/shasum -a 256 | /usr/bin/shasum -a 256 | /usr/bin/awk '{print $1}')" ] || + fail source-mutated +"${jq_cmd[@]}" -S -c '.stage_result' "$case_root/out" > "$case_root/result.json" +"$core" validate-stage-run "$request_file" "$resolved_file" "$case_root/result.json" || fail stage-result +pass 'materializes deterministic bare child and validates stage result' + +repeat_root=$(run_case repeat) +/usr/bin/cmp -s "$case_root/out" "$repeat_root/out" || fail deterministic-response +pass 'same exact input produces the same receipt and commit' + +expect_error() { + local name=$1 expected=$2 input=${3:-$input_file} source=${4:-$tmp/source.git} + local case_root="$tmp/error-$name" + local candidate="$case_root/candidate" scratch="$case_root/scratch" + /bin/mkdir -m 700 "$case_root" "$candidate" "$scratch" + if PATH="$runtime_bin:/usr/bin:/bin" "$adapter" materialize "$input" fixture.target \ + "$source" "$candidate" "$scratch" > "$case_root/out" 2> "$case_root/err"; then + fail "$name accepted" + fi + [ ! -s "$case_root/out" ] && [ "$(cat "$case_root/err")" = "$expected" ] || fail "$name error" + [ -z "$(find "$candidate" -mindepth 1 -print -quit)" ] || fail "$name candidate cleanup" + [ -z "$(find "$scratch" -mindepth 1 -print -quit)" ] || fail "$name scratch cleanup" + pass "$name" +} + +refresh_request_pair() { + local source=$1 destination=$2 request_snapshot="$tmp/request-refresh" + "${jq_cmd[@]}" -S -c '.stage_request.content' "$source" > "$request_snapshot" + "${jq_cmd[@]}" -S -c --arg sha "$(sha_file "$request_snapshot")" \ + '.stage_request.sha256=$sha' "$source" > "$destination" +} + +input_for_source() { + local source=$1 destination=$2 algorithm=$3 commit=$4 tree=$5 intermediate="$tmp/source-input.next" + "${jq_cmd[@]}" -S -c --arg algorithm "$algorithm" --arg commit "$commit" --arg tree "$tree" ' + .stage_request.content.body.target_revision.value |= + (.hash_algorithm=$algorithm | .commit_id=$commit) | + .stage_request.content.body.base.value |= + (.hash_algorithm=$algorithm | .commit_id=$commit) | + .stage_request.content.body.source.value.value.revision |= + (.hash_algorithm=$algorithm | .commit_id=$commit) | + .stage_request.content.body.source.value.value.object_id=$tree | + (.stage_request.content.body.inputs[] | select(.input_id=="input.source-tree") | + .value.value.value.revision) |= (.hash_algorithm=$algorithm | .commit_id=$commit) | + (.stage_request.content.body.inputs[] | select(.input_id=="input.source-tree") | + .value.value.value.object_id)=$tree + ' "$source" > "$intermediate" + refresh_request_pair "$intermediate" "$destination" +} + +input_with_patch() { + local source=$1 patch=$2 destination=$3 sha intermediate="$tmp/patch-input.next" + sha=$(sha_file "$patch") + "${jq_cmd[@]}" -S -c --rawfile patch "$patch" --arg sha "$sha" ' + (.payloads[] | select(.input_id=="input.producer-patch")) |= + (.data=$patch | .sha256=$sha) | + (.stage_request.content.body.inputs[] | select(.input_id=="input.producer-patch") | + .value.value.value.sha256)=$sha + ' "$source" > "$intermediate" + refresh_request_pair "$intermediate" "$destination" +} + +input_with_contract() { + local source=$1 contract=$2 destination=$3 sha intermediate="$tmp/contract-input.next" + sha=$(sha_file "$contract") + "${jq_cmd[@]}" -S -c --rawfile contract "$contract" --arg sha "$sha" ' + (.payloads[] | select(.input_id=="input.materialize")) |= + (.data=$contract | .sha256=$sha) | + (.stage_request.content.body.inputs[] | select(.input_id=="input.materialize") | + .value.value.value.sha256)=$sha | + .stage_request.content.body.operation.arguments.materialization_contract.ref.subject_ref.value.value.sha256=$sha + ' "$source" > "$intermediate" + refresh_request_pair "$intermediate" "$destination" +} + +bad_digest="$tmp/bad-digest.json" +"${jq_cmd[@]}" -S -c '.payloads[0].sha256=("0"*64)' "$input_file" > "$bad_digest" +expect_error bad-digest E_CONTRACT "$bad_digest" + +wrong_repository="$tmp/wrong-repository.json" +"${jq_cmd[@]}" -S -c '.stage_request.content.body.target_repository_id="other.target"' \ + "$input_file" > "$wrong_repository" +expect_error wrong-repository E_CONTRACT "$wrong_repository" + +missing_revision="$tmp/missing-revision.json" +"${jq_cmd[@]}" -S -c '.stage_request.content.body.target_revision.value.commit_id=("0"*40)' \ + "$input_file" > "$missing_revision.next" +refresh_request_pair "$missing_revision.next" "$missing_revision" +expect_error missing-revision E_CONTRACT "$missing_revision" + +/usr/bin/printf '%s\n' '/invalid/alternate' > "$tmp/source.git/objects/info/alternates" +expect_error alternates E_SOURCE_GIT +/bin/rm "$tmp/source.git/objects/info/alternates" + +git_clean --git-dir="$tmp/source.git" config filter.evil.smudge 'touch /tmp/must-not-run' +expect_error filter-config E_SOURCE_CONFIG +git_clean --git-dir="$tmp/source.git" config --unset-all filter.evil.smudge + +hook_marker="$tmp/hook-ran" +printf '%s\n' '#!/bin/sh' "touch '$hook_marker'" > "$tmp/source.git/hooks/post-checkout" +/bin/chmod 0755 "$tmp/source.git/hooks/post-checkout" +expect_error source-hook E_SOURCE_HOOK +[ ! -e "$hook_marker" ] || fail source-hook-ran +/bin/rm "$tmp/source.git/hooks/post-checkout" + +/usr/bin/touch "$tmp/source.git/shallow" +expect_error shallow E_SOURCE_GIT +/bin/rm "$tmp/source.git/shallow" + +/bin/mkdir -p "$tmp/source.git/refs/replace" +/usr/bin/touch "$tmp/source.git/refs/replace/0000000000000000000000000000000000000000" +expect_error replace-ref E_SOURCE_GIT +/bin/rm -rf "$tmp/source.git/refs/replace" + +normal_repo="$tmp/normal" +/bin/mkdir -m 700 "$normal_repo" +git_clean init -q "$normal_repo" +expect_error linked-worktree E_SOURCE_WORKTREE "$input_file" "$normal_repo/.git" + +promisor="$tmp/source.git/objects/pack/test.promisor" +/usr/bin/touch "$promisor" +expect_error promisor E_SOURCE_GIT +/bin/rm "$promisor" + +remote_source="$tmp/source-remote.git" +/bin/cp -R "$tmp/source.git" "$remote_source" +git_clean --git-dir="$remote_source" config remote.origin.url https://example.invalid/repo.git +expect_error remote-config E_SOURCE_CONFIG "$input_file" "$remote_source" + +symlink_source="$tmp/source-symlink.git" +read -r symlink_commit symlink_tree < <(make_bare_source "$symlink_source" sha1 120000) +symlink_input="$tmp/symlink-input.json" +input_for_source "$input_file" "$symlink_input" sha1 "$symlink_commit" "$symlink_tree" +expect_error source-symlink-mode E_SOURCE_TREE "$symlink_input" "$symlink_source" + +submodule_source="$tmp/source-submodule.git" +/bin/mkdir -m 700 "$submodule_source" +git_clean init -q --bare --object-format=sha1 "$submodule_source" +sub_blob=$(printf '%s\n' nested | git_clean --git-dir="$submodule_source" hash-object -w --stdin) +sub_tree=$(printf '100644 blob %s\tnested.txt\n' "$sub_blob" | + git_clean --git-dir="$submodule_source" mktree) +sub_commit=$(printf '%s\n' nested | + /usr/bin/env -i HOME="$tmp/home" TMPDIR="$tmp" PATH=/usr/bin:/bin LC_ALL=C \ + GIT_CONFIG_NOSYSTEM=1 GIT_CONFIG_GLOBAL=/dev/null GIT_NO_REPLACE_OBJECTS=1 \ + GIT_AUTHOR_NAME=fixture GIT_AUTHOR_EMAIL=fixture@example.invalid \ + GIT_COMMITTER_NAME=fixture GIT_COMMITTER_EMAIL=fixture@example.invalid \ + GIT_AUTHOR_DATE=2000-01-01T00:00:00Z GIT_COMMITTER_DATE=2000-01-01T00:00:00Z \ + /usr/bin/git --no-replace-objects --git-dir="$submodule_source" commit-tree "$sub_tree") +outer_tree=$(printf '160000 commit %s\tmodule\n' "$sub_commit" | + git_clean --git-dir="$submodule_source" mktree) +outer_commit=$(printf '%s\n' outer | + /usr/bin/env -i HOME="$tmp/home" TMPDIR="$tmp" PATH=/usr/bin:/bin LC_ALL=C \ + GIT_CONFIG_NOSYSTEM=1 GIT_CONFIG_GLOBAL=/dev/null GIT_NO_REPLACE_OBJECTS=1 \ + GIT_AUTHOR_NAME=fixture GIT_AUTHOR_EMAIL=fixture@example.invalid \ + GIT_COMMITTER_NAME=fixture GIT_COMMITTER_EMAIL=fixture@example.invalid \ + GIT_AUTHOR_DATE=2000-01-01T00:00:00Z GIT_COMMITTER_DATE=2000-01-01T00:00:00Z \ + /usr/bin/git --no-replace-objects --git-dir="$submodule_source" commit-tree "$outer_tree") +git_clean --git-dir="$submodule_source" update-ref refs/heads/main "$outer_commit" +submodule_input="$tmp/submodule-input.json" +input_for_source "$input_file" "$submodule_input" sha1 "$outer_commit" "$outer_tree" +expect_error source-submodule-mode E_SOURCE_TREE "$submodule_input" "$submodule_source" + +sha256_source="$tmp/source-sha256.git" +read -r sha256_commit sha256_tree < <(make_bare_source "$sha256_source" sha256) +sha256_input="$tmp/sha256-input.json" +input_for_source "$input_file" "$sha256_input" sha256 "$sha256_commit" "$sha256_tree" +sha256_root=$(run_case sha256 "$sha256_input" "$sha256_source") +[ ! -s "$sha256_root/err" ] && + [ "$(git_clean --git-dir="$sha256_root/candidate/repository.git" rev-parse --show-object-format)" = sha256 ] || + fail sha256-materialization +pass 'SHA-256 source and candidate identities remain exact' + +scope_contract="$tmp/scope-contract.json" +"${jq_cmd[@]}" -S -c '.allowed_paths=["other.txt"]' "$contract_file" > "$scope_contract" +scope_input="$tmp/scope-input.json" +input_with_contract "$input_file" "$scope_contract" "$scope_input" +expect_error patch-scope E_PATCH_SCOPE "$scope_input" + +binary_patch="$tmp/binary.patch" +printf '%s\n' 'GIT binary patch' 'literal 0' 'HcmV?d00001' > "$binary_patch" +binary_input="$tmp/binary-input.json" +input_with_patch "$input_file" "$binary_patch" "$binary_input" +expect_error binary-patch E_BINARY_PATCH "$binary_input" + +wrong_tree_input="$tmp/wrong-tree-input.json" +"${jq_cmd[@]}" -S -c ' + .stage_request.content.body.source.value.value.object_id=("0"*40) | + (.stage_request.content.body.inputs[] | select(.input_id=="input.source-tree") | + .value.value.value.object_id)=("0"*40) +' "$input_file" > "$wrong_tree_input.next" +refresh_request_pair "$wrong_tree_input.next" "$wrong_tree_input" +expect_error wrong-source-tree E_SOURCE_IDENTITY "$wrong_tree_input" + +mode_patch_repo="$tmp/mode-patch" +/bin/mkdir -m 700 "$mode_patch_repo" +git_clean init -q "$mode_patch_repo" +git_clean -C "$mode_patch_repo" config user.name fixture +git_clean -C "$mode_patch_repo" config user.email fixture@example.invalid +printf '%s\n' alpha beta > "$mode_patch_repo/source.txt" +git_clean -C "$mode_patch_repo" add source.txt +git_clean -C "$mode_patch_repo" commit -q -m source +/bin/ln -s outside "$mode_patch_repo/link" +git_clean -C "$mode_patch_repo" add link +symlink_patch="$tmp/symlink.patch" +git_clean -C "$mode_patch_repo" diff --cached --binary > "$symlink_patch" +git_clean -C "$mode_patch_repo" reset -q +/bin/rm "$mode_patch_repo/link" +symlink_contract="$tmp/symlink-contract.json" +"${jq_cmd[@]}" -S -c '.allowed_paths=["link"]' "$contract_file" > "$symlink_contract" +symlink_patch_contract_input="$tmp/symlink-patch-contract-input.json" +input_with_contract "$input_file" "$symlink_contract" "$symlink_patch_contract_input" +symlink_patch_input="$tmp/symlink-patch-input.json" +input_with_patch "$symlink_patch_contract_input" "$symlink_patch" "$symlink_patch_input" +expect_error patch-created-symlink E_CANDIDATE_TREE "$symlink_patch_input" + +git_clean -C "$mode_patch_repo" update-index --add --cacheinfo \ + "160000,$source_commit,module" +submodule_patch="$tmp/submodule.patch" +git_clean -C "$mode_patch_repo" diff --cached --binary > "$submodule_patch" +submodule_contract="$tmp/submodule-contract.json" +"${jq_cmd[@]}" -S -c '.allowed_paths=["module"]' "$contract_file" > "$submodule_contract" +submodule_patch_contract_input="$tmp/submodule-patch-contract-input.json" +input_with_contract "$input_file" "$submodule_contract" "$submodule_patch_contract_input" +submodule_patch_input="$tmp/submodule-patch-input.json" +input_with_patch "$submodule_patch_contract_input" "$submodule_patch" "$submodule_patch_input" +expect_error patch-created-submodule E_CANDIDATE_TREE "$submodule_patch_input" + +case_nonempty="$tmp/nonempty" +/bin/mkdir -m 700 "$case_nonempty" "$case_nonempty/candidate" "$case_nonempty/scratch" +/usr/bin/touch "$case_nonempty/candidate/existing" +if PATH="$runtime_bin:/usr/bin:/bin" "$adapter" materialize "$input_file" fixture.target \ + "$tmp/source.git" "$case_nonempty/candidate" "$case_nonempty/scratch" \ + > "$case_nonempty/out" 2> "$case_nonempty/err"; then fail nonempty-candidate; fi +[ "$(cat "$case_nonempty/err")" = E_CANDIDATE_ROOT ] || fail nonempty-candidate-error +pass 'non-empty candidate root rejected' + +overlap_candidate="$tmp/source.git/candidate-boundary" +overlap_scratch="$tmp/overlap-scratch" +/bin/mkdir -m 700 "$overlap_candidate" "$overlap_scratch" +if PATH="$runtime_bin:/usr/bin:/bin" "$adapter" materialize "$input_file" fixture.target \ + "$tmp/source.git" "$overlap_candidate" "$overlap_scratch" \ + > "$tmp/overlap.out" 2> "$tmp/overlap.err"; then fail overlapping-boundary; fi +[ "$(cat "$tmp/overlap.err")" = E_BOUNDARY ] || fail overlapping-boundary-error +/bin/rmdir "$overlap_candidate" +pass 'source, candidate, and scratch boundaries cannot overlap' + +outside="$tmp/outside-sentinel" +/usr/bin/printf '%s\n' unchanged > "$outside" +traversal_patch="$tmp/traversal.patch" +/usr/bin/printf '%s\n' 'diff --git a/../outside-sentinel b/../outside-sentinel' \ + '--- a/../outside-sentinel' '+++ b/../outside-sentinel' '@@ -1 +1 @@' '-unchanged' '+changed' \ + > "$traversal_patch" +traversal_input="$tmp/traversal-input.json" +input_with_patch "$input_file" "$traversal_patch" "$traversal_input" +expect_error traversal E_PATCH "$traversal_input" +[ "$(cat "$outside")" = unchanged ] || fail traversal-write + +symlink_boundary="$tmp/candidate-link" +/bin/ln -s "$tmp" "$symlink_boundary" +boundary_scratch="$tmp/boundary-scratch" +/bin/mkdir -m 700 "$boundary_scratch" +if PATH="$runtime_bin:/usr/bin:/bin" "$adapter" materialize "$input_file" fixture.target \ + "$tmp/source.git" "$symlink_boundary" "$boundary_scratch" \ + > "$tmp/boundary.out" 2> "$tmp/boundary.err"; then fail candidate-symlink; fi +[ "$(cat "$tmp/boundary.err")" = E_CANDIDATE_ROOT ] || fail candidate-symlink-error +pass 'symlink candidate boundary rejected' + +if /usr/bin/grep -Eq 'curl|wget|gh |glab |github[.]com|gitlab[.]com|git (clone|fetch|pull|push)' \ + "$adapter" "$protocol"; then fail network-command; fi +pass 'payload has no provider, transport, credential, authority or qualification path' + +printf 'local Git materializer: %s focused checks passed\n' "$passed" From 2bab3717f165e035014a7d5a8a5be28f5f732a0b Mon Sep 17 00:00:00 2001 From: ci Date: Thu, 3 Sep 2026 10:25:47 -0400 Subject: [PATCH 02/17] Close materializer source and response gaps --- .../local-git-materializer/v1/materialize.sh | 5 +++- .../local-git-materializer-adapter.test.sh | 30 +++++++++++++++++++ 2 files changed, 34 insertions(+), 1 deletion(-) diff --git a/adapters/local-git-materializer/v1/materialize.sh b/adapters/local-git-materializer/v1/materialize.sh index 09b5720..1ac1194 100755 --- a/adapters/local-git-materializer/v1/materialize.sh +++ b/adapters/local-git-materializer/v1/materialize.sh @@ -230,12 +230,15 @@ done < "$source_config" [ "$(git_dir "$source_git_dir" rev-parse --is-bare-repository 2>/dev/null)" = true ] || emit_error E_SOURCE_WORKTREE [ ! -e "$source_git_dir/commondir" ] && [ ! -e "$source_git_dir/shallow" ] && + [ -z "$(find "$source_git_dir/worktrees" -mindepth 1 -print -quit 2>/dev/null)" ] && [ ! -e "$source_git_dir/info/grafts" ] && [ ! -e "$source_git_dir/objects/info/alternates" ] && [ ! -d "$source_git_dir/refs/replace" ] && [ -z "$(find "$source_git_dir" -type l -print -quit)" ] && [ -z "$(find "$source_git_dir/objects/pack" -type f -name '*.promisor' -print -quit 2>/dev/null)" ] || emit_error E_SOURCE_GIT +[ -z "$(git_dir "$source_git_dir" for-each-ref --format='%(refname)' \ + refs/replace/ 2>/dev/null)" ] || emit_error E_SOURCE_GIT if find "$source_git_dir/hooks" -type f ! -name '*.sample' -print -quit 2>/dev/null | /usr/bin/grep -q .; then emit_error E_SOURCE_HOOK @@ -390,5 +393,5 @@ response_file="$run_root/response.json" effects:["caller-disposable-candidate-repository"] } ' > "$response_file" || emit_error E_RESULT +/bin/cat "$response_file" || emit_error E_RESULT success=1 -/bin/cat "$response_file" diff --git a/scripts/test/local-git-materializer-adapter.test.sh b/scripts/test/local-git-materializer-adapter.test.sh index eeff0a1..b2db439 100755 --- a/scripts/test/local-git-materializer-adapter.test.sh +++ b/scripts/test/local-git-materializer-adapter.test.sh @@ -416,11 +416,29 @@ expect_error shallow E_SOURCE_GIT expect_error replace-ref E_SOURCE_GIT /bin/rm -rf "$tmp/source.git/refs/replace" +packed_replace_source="$tmp/source-packed-replace.git" +/bin/cp -R "$tmp/source.git" "$packed_replace_source" +git_clean --git-dir="$packed_replace_source" update-ref \ + "refs/replace/$source_tree" "$source_tree" +git_clean --git-dir="$packed_replace_source" pack-refs --all --prune +/bin/rmdir "$packed_replace_source/refs/replace" 2>/dev/null || : +[ ! -d "$packed_replace_source/refs/replace" ] || fail packed-replace-fixture +expect_error packed-replace-ref E_SOURCE_GIT "$input_file" "$packed_replace_source" + normal_repo="$tmp/normal" /bin/mkdir -m 700 "$normal_repo" git_clean init -q "$normal_repo" expect_error linked-worktree E_SOURCE_WORKTREE "$input_file" "$normal_repo/.git" +linked_source="$tmp/source-with-worktree.git" +/bin/cp -R "$tmp/source.git" "$linked_source" +linked_path="$tmp/source-linked" +git_clean --git-dir="$linked_source" worktree add --detach "$linked_path" \ + "$source_commit" >/dev/null +[ -d "$linked_source/worktrees" ] || fail bare-linked-worktree-fixture +expect_error bare-linked-worktree-metadata E_SOURCE_GIT "$input_file" "$linked_source" +git_clean --git-dir="$linked_source" worktree remove --force "$linked_path" + promisor="$tmp/source.git/objects/pack/test.promisor" /usr/bin/touch "$promisor" expect_error promisor E_SOURCE_GIT @@ -548,6 +566,18 @@ if PATH="$runtime_bin:/usr/bin:/bin" "$adapter" materialize "$input_file" fixtur /bin/rmdir "$overlap_candidate" pass 'source, candidate, and scratch boundaries cannot overlap' +closed_output="$tmp/closed-output" +/bin/mkdir -m 700 "$closed_output" "$closed_output/candidate" "$closed_output/scratch" +if PATH="$runtime_bin:/usr/bin:/bin" "$adapter" materialize "$input_file" fixture.target \ + "$tmp/source.git" "$closed_output/candidate" "$closed_output/scratch" \ + >&- 2> "$closed_output/err"; then + fail closed-output-accepted +fi +[ -z "$(find "$closed_output/candidate" -mindepth 1 -print -quit)" ] && + [ -z "$(find "$closed_output/scratch" -mindepth 1 -print -quit)" ] || + fail closed-output-cleanup +pass 'response failure removes candidate and scratch state' + outside="$tmp/outside-sentinel" /usr/bin/printf '%s\n' unchanged > "$outside" traversal_patch="$tmp/traversal.patch" From 52fa387da7c0851b9bdadb37abe313b7f9b169e7 Mon Sep 17 00:00:00 2001 From: ci Date: Thu, 3 Sep 2026 11:34:56 -0400 Subject: [PATCH 03/17] Enforce materializer scratch and path bounds --- .../local-git-materializer/v1/materialize.sh | 25 ++++++++++++---- .../local-git-materializer-adapter.test.sh | 30 +++++++++++++++++++ 2 files changed, 50 insertions(+), 5 deletions(-) diff --git a/adapters/local-git-materializer/v1/materialize.sh b/adapters/local-git-materializer/v1/materialize.sh index 1ac1194..da7ec7d 100755 --- a/adapters/local-git-materializer/v1/materialize.sh +++ b/adapters/local-git-materializer/v1/materialize.sh @@ -91,6 +91,10 @@ overlaps() { } [ -f "$input_path" ] && [ ! -L "$input_path" ] || emit_error E_INPUT +input_bytes=$(/usr/bin/wc -c < "$input_path" | /usr/bin/tr -d ' ') || emit_error E_INPUT +case "$input_bytes" in ''|*[!0-9]*) emit_error E_INPUT ;; esac +[ "${#input_bytes}" -le 7 ] || emit_error E_INPUT +[ "$input_bytes" -le 8388608 ] || emit_error E_INPUT physical_dir "$source_git_dir" || emit_error E_SOURCE_GIT empty_private_dir "$candidate_root" || emit_error E_CANDIDATE_ROOT empty_private_dir "$scratch_root" || emit_error E_SCRATCH_ROOT @@ -155,9 +159,19 @@ while [ "$manifest_index" -lt "$manifest_count" ]; do manifest_index=$((manifest_index + 1)) done -"$core" validate-profile-set "$profile_file" "$resolved_file" \ - "${manifest_files[@]}" >/dev/null 2>&1 || emit_error E_CORE_PROFILE -"$core" validate-document "$request_file" >/dev/null 2>&1 || emit_error E_CORE_REQUEST +core_validate() { + local validation_id=$1 validation_root receipt_path + shift + validation_root="$run_root/core-$validation_id" + receipt_path="$run_root/core-$validation_id.receipt" + /bin/mkdir -m 700 "$validation_root" || return 1 + "$core" --accounted-validation "$validation_root" 536870912 "$@" \ + 3> "$receipt_path" >/dev/null 2>&1 +} + +core_validate profile validate-profile-set "$profile_file" "$resolved_file" \ + "${manifest_files[@]}" || emit_error E_CORE_PROFILE +core_validate request validate-document "$request_file" || emit_error E_CORE_REQUEST contract_file="$run_root/materialization-contract.json" patch_file="$run_root/producer.patch" @@ -256,6 +270,7 @@ safe_repo_path() { local value=$1 component lowered bytes [ -n "$value" ] || return 1 case "$value" in /*|*\\*) return 1 ;; esac + case "$value" in *$'\n'*|*$'\r'*) return 1 ;; esac bytes=$(printf '%s' "$value" | wc -c | /usr/bin/tr -d ' ') [ "$bytes" -le 4096 ] || return 1 if printf '%s' "$value" | LC_ALL=C /usr/bin/grep -q '[[:cntrl:]]'; then return 1; fi @@ -371,8 +386,8 @@ result_file="$run_root/stage-result.json" --arg receipt_json "$(<"$receipt_file")" \ --arg verified_receipt_json "$(<"$verified_receipt")" --arg outcome "$outcome" \ -f "$protocol" "$input_snapshot" > "$result_file" 2>/dev/null || emit_error E_RESULT -"$core" validate-stage-run "$request_file" "$resolved_file" "$result_file" \ - >/dev/null 2>&1 || emit_error E_CORE_RESULT +core_validate result validate-stage-run "$request_file" "$resolved_file" \ + "$result_file" || emit_error E_CORE_RESULT /bin/mv "$staging_repo" "$final_repo" || emit_error E_CANDIDATE_ROOT response_file="$run_root/response.json" diff --git a/scripts/test/local-git-materializer-adapter.test.sh b/scripts/test/local-git-materializer-adapter.test.sh index b2db439..455a5f1 100755 --- a/scripts/test/local-git-materializer-adapter.test.sh +++ b/scripts/test/local-git-materializer-adapter.test.sh @@ -392,6 +392,10 @@ missing_revision="$tmp/missing-revision.json" refresh_request_pair "$missing_revision.next" "$missing_revision" expect_error missing-revision E_CONTRACT "$missing_revision" +oversize_input="$tmp/oversize-input.json" +/bin/dd if=/dev/zero of="$oversize_input" bs=8388609 count=1 2>/dev/null +expect_error oversize-input E_INPUT "$oversize_input" + /usr/bin/printf '%s\n' '/invalid/alternate' > "$tmp/source.git/objects/info/alternates" expect_error alternates E_SOURCE_GIT /bin/rm "$tmp/source.git/objects/info/alternates" @@ -482,6 +486,26 @@ submodule_input="$tmp/submodule-input.json" input_for_source "$input_file" "$submodule_input" sha1 "$outer_commit" "$outer_tree" expect_error source-submodule-mode E_SOURCE_TREE "$submodule_input" "$submodule_source" +newline_source="$tmp/source-newline.git" +/bin/mkdir -m 700 "$newline_source" +git_clean init -q --bare --object-format=sha1 "$newline_source" +newline_blob=$(printf '%s\n' value | + git_clean --git-dir="$newline_source" hash-object -w --stdin) +newline_tree=$(printf '100644 blob %s\tfoo\nbar\0' "$newline_blob" | + git_clean --git-dir="$newline_source" mktree -z) +newline_commit=$(printf '%s\n' newline | + /usr/bin/env -i HOME="$tmp/home" TMPDIR="$tmp" PATH=/usr/bin:/bin LC_ALL=C \ + GIT_CONFIG_NOSYSTEM=1 GIT_CONFIG_GLOBAL=/dev/null GIT_NO_REPLACE_OBJECTS=1 \ + GIT_AUTHOR_NAME=fixture GIT_AUTHOR_EMAIL=fixture@example.invalid \ + GIT_COMMITTER_NAME=fixture GIT_COMMITTER_EMAIL=fixture@example.invalid \ + GIT_AUTHOR_DATE=2000-01-01T00:00:00Z GIT_COMMITTER_DATE=2000-01-01T00:00:00Z \ + /usr/bin/git --no-replace-objects --git-dir="$newline_source" \ + commit-tree "$newline_tree") +git_clean --git-dir="$newline_source" update-ref refs/heads/main "$newline_commit" +newline_input="$tmp/newline-input.json" +input_for_source "$input_file" "$newline_input" sha1 "$newline_commit" "$newline_tree" +expect_error source-newline-path E_SOURCE_TREE "$newline_input" "$newline_source" + sha256_source="$tmp/source-sha256.git" read -r sha256_commit sha256_tree < <(make_bare_source "$sha256_source" sha256) sha256_input="$tmp/sha256-input.json" @@ -601,6 +625,12 @@ pass 'symlink candidate boundary rejected' if /usr/bin/grep -Eq 'curl|wget|gh |glab |github[.]com|gitlab[.]com|git (clone|fetch|pull|push)' \ "$adapter" "$protocol"; then fail network-command; fi +if [ "$(/usr/bin/grep -Fc -- '--accounted-validation' "$adapter")" -ne 1 ] || + /usr/bin/grep -Eq '"\$core" (validate-document|validate-profile-set|validate-stage-run)' \ + "$adapter"; then + fail unaccounted-core-validation +fi +pass 'core validation is routed through caller-owned accounted scratch' pass 'payload has no provider, transport, credential, authority or qualification path' printf 'local Git materializer: %s focused checks passed\n' "$passed" From c143e07145a2d2d0eccdad99899a49b2a5b3860e Mon Sep 17 00:00:00 2001 From: ci Date: Thu, 3 Sep 2026 12:17:09 -0400 Subject: [PATCH 04/17] Close materializer path and time gaps --- .../local-git-materializer/v1/materialize.sh | 7 ++- .../local-git-materializer-adapter.test.sh | 49 +++++++++++++++++++ 2 files changed, 54 insertions(+), 2 deletions(-) diff --git a/adapters/local-git-materializer/v1/materialize.sh b/adapters/local-git-materializer/v1/materialize.sh index da7ec7d..f98ff5c 100755 --- a/adapters/local-git-materializer/v1/materialize.sh +++ b/adapters/local-git-materializer/v1/materialize.sh @@ -273,7 +273,10 @@ safe_repo_path() { case "$value" in *$'\n'*|*$'\r'*) return 1 ;; esac bytes=$(printf '%s' "$value" | wc -c | /usr/bin/tr -d ' ') [ "$bytes" -le 4096 ] || return 1 - if printf '%s' "$value" | LC_ALL=C /usr/bin/grep -q '[[:cntrl:]]'; then return 1; fi + printf '%s' "$value" | /usr/bin/iconv -f UTF-8 -t UTF-8 >/dev/null 2>&1 || return 1 + printf '%s' "$value" | + "$jq_bin" -Rse 'test("[\\x{0000}-\\x{001f}\\x{007f}-\\x{009f}]") | not' \ + >/dev/null 2>&1 || return 1 old_ifs=$IFS IFS=/ read -r -a path_components <<< "$value" @@ -348,7 +351,7 @@ changed_paths_sha=$(sha_file "$changed_paths_json") if [ "$candidate_tree" = "$source_tree" ]; then candidate_commit=$source_commit else - commit_time=$("$jq_bin" -r '.attempt.started_at' "$input_snapshot") || emit_error E_INPUT + commit_time=2000-01-01T00:00:00Z candidate_commit=$(printf '%s\n' 'ystack local candidate' | "${git_env[@]}" GIT_AUTHOR_NAME='ystack local materializer' \ GIT_AUTHOR_EMAIL='materializer@example.invalid' \ diff --git a/scripts/test/local-git-materializer-adapter.test.sh b/scripts/test/local-git-materializer-adapter.test.sh index 455a5f1..6dde4b5 100755 --- a/scripts/test/local-git-materializer-adapter.test.sh +++ b/scripts/test/local-git-materializer-adapter.test.sh @@ -305,6 +305,18 @@ repeat_root=$(run_case repeat) /usr/bin/cmp -s "$case_root/out" "$repeat_root/out" || fail deterministic-response pass 'same exact input produces the same receipt and commit' +wide_time_input="$tmp/wide-time-input.json" +"${jq_cmd[@]}" -S -c ' + .attempt.started_at="2100-01-01T00:00:01Z" | + .attempt.finished_at="2100-01-01T00:00:02Z" | + .attempt.recorded_at="2100-01-01T00:00:03Z" +' "$input_file" > "$wide_time_input" +wide_time_root=$(run_case wide-time "$wide_time_input") +[ ! -s "$wide_time_root/err" ] || fail wide-time-stderr +[ "$(git_clean --git-dir="$wide_time_root/candidate/repository.git" rev-parse refs/heads/candidate)" = \ + "$candidate_commit" ] || fail wide-time-candidate-identity +pass 'contract-valid timestamps outside Git date range keep deterministic candidate identity' + expect_error() { local name=$1 expected=$2 input=${3:-$input_file} source=${4:-$tmp/source.git} local case_root="$tmp/error-$name" @@ -506,6 +518,43 @@ newline_input="$tmp/newline-input.json" input_for_source "$input_file" "$newline_input" sha1 "$newline_commit" "$newline_tree" expect_error source-newline-path E_SOURCE_TREE "$newline_input" "$newline_source" +c1_source="$tmp/source-c1.git" +/bin/mkdir -m 700 "$c1_source" +git_clean init -q --bare --object-format=sha1 "$c1_source" +c1_blob=$(printf '%s\n' value | git_clean --git-dir="$c1_source" hash-object -w --stdin) +c1_tree=$(printf '100644 blob %s\tfoo\302\200bar\0' "$c1_blob" | + git_clean --git-dir="$c1_source" mktree -z) +c1_commit=$(printf '%s\n' c1-control | + /usr/bin/env -i HOME="$tmp/home" TMPDIR="$tmp" PATH=/usr/bin:/bin LC_ALL=C \ + GIT_CONFIG_NOSYSTEM=1 GIT_CONFIG_GLOBAL=/dev/null GIT_NO_REPLACE_OBJECTS=1 \ + GIT_AUTHOR_NAME=fixture GIT_AUTHOR_EMAIL=fixture@example.invalid \ + GIT_COMMITTER_NAME=fixture GIT_COMMITTER_EMAIL=fixture@example.invalid \ + GIT_AUTHOR_DATE=2000-01-01T00:00:00Z GIT_COMMITTER_DATE=2000-01-01T00:00:00Z \ + /usr/bin/git --no-replace-objects --git-dir="$c1_source" commit-tree "$c1_tree") +git_clean --git-dir="$c1_source" update-ref refs/heads/main "$c1_commit" +c1_input="$tmp/c1-input.json" +input_for_source "$input_file" "$c1_input" sha1 "$c1_commit" "$c1_tree" +expect_error source-c1-control-path E_SOURCE_TREE "$c1_input" "$c1_source" + +malformed_source="$tmp/source-malformed-utf8.git" +/bin/mkdir -m 700 "$malformed_source" +git_clean init -q --bare --object-format=sha1 "$malformed_source" +malformed_blob=$(printf '%s\n' value | + git_clean --git-dir="$malformed_source" hash-object -w --stdin) +malformed_tree=$(printf '100644 blob %s\tfoo\200bar\0' "$malformed_blob" | + git_clean --git-dir="$malformed_source" mktree -z) +malformed_commit=$(printf '%s\n' malformed-utf8 | + /usr/bin/env -i HOME="$tmp/home" TMPDIR="$tmp" PATH=/usr/bin:/bin LC_ALL=C \ + GIT_CONFIG_NOSYSTEM=1 GIT_CONFIG_GLOBAL=/dev/null GIT_NO_REPLACE_OBJECTS=1 \ + GIT_AUTHOR_NAME=fixture GIT_AUTHOR_EMAIL=fixture@example.invalid \ + GIT_COMMITTER_NAME=fixture GIT_COMMITTER_EMAIL=fixture@example.invalid \ + GIT_AUTHOR_DATE=2000-01-01T00:00:00Z GIT_COMMITTER_DATE=2000-01-01T00:00:00Z \ + /usr/bin/git --no-replace-objects --git-dir="$malformed_source" commit-tree "$malformed_tree") +git_clean --git-dir="$malformed_source" update-ref refs/heads/main "$malformed_commit" +malformed_input="$tmp/malformed-input.json" +input_for_source "$input_file" "$malformed_input" sha1 "$malformed_commit" "$malformed_tree" +expect_error source-malformed-utf8-path E_SOURCE_TREE "$malformed_input" "$malformed_source" + sha256_source="$tmp/source-sha256.git" read -r sha256_commit sha256_tree < <(make_bare_source "$sha256_source" sha256) sha256_input="$tmp/sha256-input.json" From e1f93354433bc420aad22bb89f67bf33eb07930b Mon Sep 17 00:00:00 2001 From: ci Date: Thu, 3 Sep 2026 13:52:05 -0400 Subject: [PATCH 05/17] Bound materializer source import --- README.md | 5 ++- RESTORE.md | 5 ++- .../local-git-materializer/v1/materialize.sh | 44 +++++++++++++++++-- .../local-git-materializer-adapter.test.sh | 40 +++++++++++++++++ 4 files changed, 87 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index e07784c..3aee02a 100644 --- a/README.md +++ b/README.md @@ -274,12 +274,15 @@ grant authority or qualification, or activate a profile. exact, sanitized bare source repository and one contract-bound patch. It imports reachable objects into a caller-disposable bare repository, applies the patch to a scratch-only index, and returns a canonical receipt and validated stage result. +Reachable source history is limited to 65,536 objects and 256 MiB of uncompressed +object data; the streamed pack is capped at the same byte limit. The fixed `materialize` command accepts only caller-named physical source, candidate, and scratch boundaries. It rejects worktrees, alternates, shallow or partial repositories, replace or graft state, active hooks and filters, remote configuration, unsafe paths, binary patches, symlinks, and submodules. It never -checks out a worktree or runs a transport command. Tests cover both SHA-1 and +inherits host Git templates, checks out a worktree, or runs a transport command. +Tests cover both SHA-1 and SHA-256 object formats with disposable local fixtures. This PR lands only the inactive package payload. A later assembly PR may add a diff --git a/RESTORE.md b/RESTORE.md index 1e9d078..8b6c495 100644 --- a/RESTORE.md +++ b/RESTORE.md @@ -520,8 +520,9 @@ portable-core v2 profile and stage request, and proves that a contract-bound pat becomes a deterministic bare child commit and path-free receipt. The negative matrix rejects moved identities, unsafe directories and paths, hooks, filters, remotes, worktrees, alternates, shallow or partial repositories, replace state, -binary patches, symlinks, and submodules. It also checks that the source stays -unchanged and scratch is removed. +binary patches, symlinks, submodules, host Git templates, and reachable source +history above the fixed 65,536-object or 256 MiB import budget. It also checks that +the source stays unchanged and scratch is removed. This payload has no adapter manifest. A later assembly PR may bind the directory tree through the payload's durable main commit. It is an inactive, local-only diff --git a/adapters/local-git-materializer/v1/materialize.sh b/adapters/local-git-materializer/v1/materialize.sh index f98ff5c..8deec9a 100755 --- a/adapters/local-git-materializer/v1/materialize.sh +++ b/adapters/local-git-materializer/v1/materialize.sh @@ -120,6 +120,7 @@ trap 'exit 130' INT trap 'exit 143' TERM /bin/mkdir -m 700 "$run_root" || emit_error E_SCRATCH_ROOT /bin/mkdir -m 500 "$run_root/no-hooks" || emit_error E_SCRATCH_ROOT +/bin/mkdir -m 500 "$run_root/empty-template" || emit_error E_SCRATCH_ROOT input_snapshot="$run_root/input.json" /bin/cp "$input_path" "$input_snapshot" || emit_error E_INPUT @@ -306,12 +307,47 @@ scan_tree() { source_paths="$run_root/source-paths" scan_tree "$source_git_dir" "$source_tree" "$source_paths" || emit_error E_SOURCE_TREE -"${git_env[@]}" /usr/bin/git init --bare --object-format="$source_algorithm" \ +empty_template="$run_root/empty-template" +source_import_byte_limit=268435456 +source_import_count_limit=65536 +source_objects="$run_root/source.objects" +if ! git_dir "$source_git_dir" rev-list --objects --no-object-names "$source_commit" | + /usr/bin/awk -v limit="$source_import_count_limit" \ + 'NR > limit { exit 2 } { print }' > "$source_objects"; then + emit_error E_SOURCE_LIMIT +fi +source_object_sizes="$run_root/source-object-sizes" +git_dir "$source_git_dir" cat-file \ + --batch-check='%(objectname) %(objecttype) %(objectsize)' \ + < "$source_objects" > "$source_object_sizes" || emit_error E_SOURCE_LIMIT +source_object_bytes=0 +while IFS=' ' read -r object_id object_type object_size extra; do + [ -z "$extra" ] || emit_error E_SOURCE_LIMIT + case "$source_algorithm" in + sha1) [[ "$object_id" =~ ^[0-9a-f]{40}$ ]] || emit_error E_SOURCE_LIMIT ;; + sha256) [[ "$object_id" =~ ^[0-9a-f]{64}$ ]] || emit_error E_SOURCE_LIMIT ;; + *) emit_error E_SOURCE_LIMIT ;; + esac + case "$object_type" in blob|tree|commit) ;; *) emit_error E_SOURCE_LIMIT ;; esac + case "$object_size" in ''|*[!0-9]*) emit_error E_SOURCE_LIMIT ;; esac + [ "${#object_size}" -le 9 ] && [ "$object_size" -le "$source_import_byte_limit" ] && + [ "$source_object_bytes" -le "$((source_import_byte_limit - object_size))" ] || + emit_error E_SOURCE_LIMIT + source_object_bytes=$((source_object_bytes + object_size)) +done < "$source_object_sizes" + +"${git_env[@]}" /usr/bin/git init --template="$empty_template" --bare \ + --object-format="$source_algorithm" \ "$staging_repo" >/dev/null 2>&1 || emit_error E_CANDIDATE_GIT source_pack="$run_root/source.pack" -printf '%s\n' "$source_commit" | - git_dir "$source_git_dir" pack-objects --quiet --stdout --revs > "$source_pack" || - emit_error E_SOURCE_GIT +source_pack_ceiling=$((source_import_byte_limit + 1)) +if ! git_dir "$source_git_dir" pack-objects --quiet --stdout < "$source_objects" | + /usr/bin/head -c "$source_pack_ceiling" > "$source_pack"; then + emit_error E_SOURCE_LIMIT +fi +source_pack_bytes=$(/usr/bin/wc -c < "$source_pack" | /usr/bin/tr -d ' ') || + emit_error E_SOURCE_LIMIT +[ "$source_pack_bytes" -le "$source_import_byte_limit" ] || emit_error E_SOURCE_LIMIT git_dir "$staging_repo" index-pack --stdin --fix-thin < "$source_pack" >/dev/null 2>&1 || emit_error E_CANDIDATE_GIT git_dir "$staging_repo" cat-file -e "$source_commit^{commit}" >/dev/null 2>&1 || diff --git a/scripts/test/local-git-materializer-adapter.test.sh b/scripts/test/local-git-materializer-adapter.test.sh index 6dde4b5..c14c404 100755 --- a/scripts/test/local-git-materializer-adapter.test.sh +++ b/scripts/test/local-git-materializer-adapter.test.sh @@ -317,6 +317,27 @@ wide_time_root=$(run_case wide-time "$wide_time_input") "$candidate_commit" ] || fail wide-time-candidate-identity pass 'contract-valid timestamps outside Git date range keep deterministic candidate identity' +host_template="$tmp/host-template" +empty_template="$tmp/empty-template" +/bin/mkdir -m 700 "$host_template" "$empty_template" "$host_template/hooks" +printf '%s\n' '#!/bin/sh' 'exit 1' > "$host_template/hooks/post-checkout" +/bin/chmod 0755 "$host_template/hooks/post-checkout" +host_template_repo="$tmp/host-template-repo.git" +empty_template_repo="$tmp/empty-template-repo.git" +/usr/bin/env -i HOME="$tmp/home" TMPDIR="$tmp" PATH=/usr/bin:/bin LC_ALL=C \ + GIT_CONFIG_NOSYSTEM=1 GIT_CONFIG_GLOBAL=/dev/null GIT_TEMPLATE_DIR="$host_template" \ + /usr/bin/git init -q --bare "$host_template_repo" +[ -x "$host_template_repo/hooks/post-checkout" ] || fail host-template-fixture +/usr/bin/env -i HOME="$tmp/home" TMPDIR="$tmp" PATH=/usr/bin:/bin LC_ALL=C \ + GIT_CONFIG_NOSYSTEM=1 GIT_CONFIG_GLOBAL=/dev/null GIT_TEMPLATE_DIR="$host_template" \ + /usr/bin/git init -q --template="$empty_template" --bare "$empty_template_repo" +[ ! -e "$empty_template_repo/hooks/post-checkout" ] || fail explicit-empty-template +/usr/bin/grep -Fq 'git init --template="$empty_template" --bare' "$adapter" || + fail materializer-empty-template-option +[ -z "$(find "$candidate_repo/hooks" -type f -print -quit 2>/dev/null)" ] || + fail candidate-template-content +pass 'explicit empty template prevents host Git template contamination' + expect_error() { local name=$1 expected=$2 input=${3:-$input_file} source=${4:-$tmp/source.git} local case_root="$tmp/error-$name" @@ -408,6 +429,25 @@ oversize_input="$tmp/oversize-input.json" /bin/dd if=/dev/zero of="$oversize_input" bs=8388609 count=1 2>/dev/null expect_error oversize-input E_INPUT "$oversize_input" +large_source="$tmp/source-large.git" +/bin/mkdir -m 700 "$large_source" +git_clean init -q --bare --object-format=sha1 "$large_source" +large_blob=$(/bin/dd if=/dev/zero bs=1048576 count=257 2>/dev/null | + git_clean --git-dir="$large_source" hash-object -w --stdin) +large_tree=$(printf '100644 blob %s\tsource.txt\n' "$large_blob" | + git_clean --git-dir="$large_source" mktree) +large_commit=$(printf '%s\n' large-source | + /usr/bin/env -i HOME="$tmp/home" TMPDIR="$tmp" PATH=/usr/bin:/bin LC_ALL=C \ + GIT_CONFIG_NOSYSTEM=1 GIT_CONFIG_GLOBAL=/dev/null GIT_NO_REPLACE_OBJECTS=1 \ + GIT_AUTHOR_NAME=fixture GIT_AUTHOR_EMAIL=fixture@example.invalid \ + GIT_COMMITTER_NAME=fixture GIT_COMMITTER_EMAIL=fixture@example.invalid \ + GIT_AUTHOR_DATE=2000-01-01T00:00:00Z GIT_COMMITTER_DATE=2000-01-01T00:00:00Z \ + /usr/bin/git --no-replace-objects --git-dir="$large_source" commit-tree "$large_tree") +git_clean --git-dir="$large_source" update-ref refs/heads/main "$large_commit" +large_input="$tmp/large-input.json" +input_for_source "$input_file" "$large_input" sha1 "$large_commit" "$large_tree" +expect_error source-import-budget E_SOURCE_LIMIT "$large_input" "$large_source" + /usr/bin/printf '%s\n' '/invalid/alternate' > "$tmp/source.git/objects/info/alternates" expect_error alternates E_SOURCE_GIT /bin/rm "$tmp/source.git/objects/info/alternates" From 8eae999a27ba907223a49e55c16d84fc1b2815f9 Mon Sep 17 00:00:00 2001 From: ci Date: Thu, 3 Sep 2026 15:21:07 -0400 Subject: [PATCH 06/17] Bound materializer tree validation --- README.md | 6 +- RESTORE.md | 6 +- .../local-git-materializer/v1/materialize.sh | 53 ++++++++++------ .../local-git-materializer-adapter.test.sh | 61 +++++++++++++++++++ 4 files changed, 105 insertions(+), 21 deletions(-) diff --git a/README.md b/README.md index 3aee02a..3e4543c 100644 --- a/README.md +++ b/README.md @@ -276,14 +276,16 @@ reachable objects into a caller-disposable bare repository, applies the patch to scratch-only index, and returns a canonical receipt and validated stage result. Reachable source history is limited to 65,536 objects and 256 MiB of uncompressed object data; the streamed pack is capped at the same byte limit. +Each tree scan is limited to 65,536 entries, 64 path components, and a 64 MiB +encoded listing, and validates UTF-8 once before its bounded built-in path walk. The fixed `materialize` command accepts only caller-named physical source, candidate, and scratch boundaries. It rejects worktrees, alternates, shallow or partial repositories, replace or graft state, active hooks and filters, remote configuration, unsafe paths, binary patches, symlinks, and submodules. It never inherits host Git templates, checks out a worktree, or runs a transport command. -Tests cover both SHA-1 and -SHA-256 object formats with disposable local fixtures. +An empty producer patch returns the explicit `no-change` result. Tests cover both +SHA-1 and SHA-256 object formats with disposable local fixtures. This PR lands only the inactive package payload. A later assembly PR may add a manifest whose package reference points to this payload's durable commit on main. diff --git a/RESTORE.md b/RESTORE.md index 8b6c495..067fb42 100644 --- a/RESTORE.md +++ b/RESTORE.md @@ -521,8 +521,10 @@ becomes a deterministic bare child commit and path-free receipt. The negative matrix rejects moved identities, unsafe directories and paths, hooks, filters, remotes, worktrees, alternates, shallow or partial repositories, replace state, binary patches, symlinks, submodules, host Git templates, and reachable source -history above the fixed 65,536-object or 256 MiB import budget. It also checks that -the source stays unchanged and scratch is removed. +history above the fixed 65,536-object or 256 MiB import budget. It proves empty-patch +`no-change` and rejects tree scans above 65,536 entries, 64 path components, or a +64 MiB encoded listing. It also checks that the source stays unchanged and scratch +is removed. This payload has no adapter manifest. A later assembly PR may bind the directory tree through the payload's durable main commit. It is an inactive, local-only diff --git a/adapters/local-git-materializer/v1/materialize.sh b/adapters/local-git-materializer/v1/materialize.sh index 8deec9a..4259131 100755 --- a/adapters/local-git-materializer/v1/materialize.sh +++ b/adapters/local-git-materializer/v1/materialize.sh @@ -198,7 +198,7 @@ contract_canonical="$run_root/materialization-contract.canonical" patch_bytes=$(wc -c < "$patch_file" | /usr/bin/tr -d ' ') max_patch_bytes=$("$jq_bin" -r '.max_patch_bytes' "$contract_file") || emit_error E_CONTRACT -[ "$patch_bytes" -gt 0 ] && [ "$patch_bytes" -le "$max_patch_bytes" ] || emit_error E_PATCH_LIMIT +[ "$patch_bytes" -le "$max_patch_bytes" ] || emit_error E_PATCH_LIMIT patch_without_nul=$(LC_ALL=C /usr/bin/tr -d '\000' < "$patch_file" | wc -c | /usr/bin/tr -d ' ') [ "$patch_without_nul" = "$patch_bytes" ] || emit_error E_BINARY_PATCH if /usr/bin/grep -aEq '^(GIT binary patch|Binary files .+ differ)$' "$patch_file"; then @@ -268,32 +268,45 @@ actual_source_tree=$(git_dir "$source_git_dir" rev-parse "$source_commit^{tree}" [ "$actual_source_tree" = "$source_tree" ] || emit_error E_SOURCE_IDENTITY safe_repo_path() { - local value=$1 component lowered bytes + local value=$1 component component_count old_ifs [ -n "$value" ] || return 1 + [ "${#value}" -le 4096 ] || return 1 case "$value" in /*|*\\*) return 1 ;; esac - case "$value" in *$'\n'*|*$'\r'*) return 1 ;; esac - bytes=$(printf '%s' "$value" | wc -c | /usr/bin/tr -d ' ') - [ "$bytes" -le 4096 ] || return 1 - printf '%s' "$value" | /usr/bin/iconv -f UTF-8 -t UTF-8 >/dev/null 2>&1 || return 1 - printf '%s' "$value" | - "$jq_bin" -Rse 'test("[\\x{0000}-\\x{001f}\\x{007f}-\\x{009f}]") | not' \ - >/dev/null 2>&1 || return 1 + [[ ! "$value" =~ [[:cntrl:]] ]] || return 1 + case "$value" in *$'\302'[$'\200'-$'\237']*) return 1 ;; esac old_ifs=$IFS IFS=/ read -r -a path_components <<< "$value" IFS=$old_ifs + component_count=0 for component in "${path_components[@]}"; do + component_count=$((component_count + 1)) + [ "$component_count" -le 64 ] || return 1 [ -n "$component" ] && [ "$component" != . ] && [ "$component" != .. ] || return 1 - lowered=$(printf '%s' "$component" | /usr/bin/tr '[:upper:]' '[:lower:]') - [ "$lowered" != .git ] || return 1 + case "$component" in .[gG][iI][tT]) return 1 ;; esac case "$component" in *.|*' ') return 1 ;; esac done } scan_tree() { - local repository=$1 tree=$2 output=$3 entry metadata mode type object path + local repository=$1 tree=$2 output=$3 raw_output entry metadata mode type object path + local raw_bytes entry_count tree_scan_byte_limit tree_scan_entry_limit tree_scan_ceiling + tree_scan_byte_limit=67108864 + tree_scan_entry_limit=65536 + tree_scan_ceiling=$((tree_scan_byte_limit + 1)) + raw_output="$output.raw" + if ! git_dir "$repository" ls-tree -rz -r "$tree" | + /usr/bin/head -c "$tree_scan_ceiling" > "$raw_output"; then + return 1 + fi + raw_bytes=$(/usr/bin/wc -c < "$raw_output" | /usr/bin/tr -d ' ') || return 1 + [ "$raw_bytes" -le "$tree_scan_byte_limit" ] || return 1 + /usr/bin/iconv -f UTF-8 -t UTF-8 "$raw_output" >/dev/null 2>&1 || return 1 : > "$output" + entry_count=0 while IFS= read -r -d '' entry; do + entry_count=$((entry_count + 1)) + [ "$entry_count" -le "$tree_scan_entry_limit" ] || return 1 metadata=${entry%%$'\t'*} path=${entry#*$'\t'} read -r mode type object <<< "$metadata" @@ -301,11 +314,13 @@ scan_tree() { case "$mode:$type" in 100644:blob|100755:blob) ;; *) return 1 ;; esac safe_repo_path "$path" || return 1 printf '%s\n' "$path" >> "$output" - done < <(git_dir "$repository" ls-tree -rz -r "$tree") + done < "$raw_output" + /bin/rm -f -- "$raw_output" } source_paths="$run_root/source-paths" scan_tree "$source_git_dir" "$source_tree" "$source_paths" || emit_error E_SOURCE_TREE +/bin/rm -f -- "$source_paths" empty_template="$run_root/empty-template" source_import_byte_limit=268435456 @@ -350,20 +365,24 @@ source_pack_bytes=$(/usr/bin/wc -c < "$source_pack" | /usr/bin/tr -d ' ') || [ "$source_pack_bytes" -le "$source_import_byte_limit" ] || emit_error E_SOURCE_LIMIT git_dir "$staging_repo" index-pack --stdin --fix-thin < "$source_pack" >/dev/null 2>&1 || emit_error E_CANDIDATE_GIT +/bin/rm -f -- "$source_pack" git_dir "$staging_repo" cat-file -e "$source_commit^{commit}" >/dev/null 2>&1 || emit_error E_CANDIDATE_GIT index_file="$run_root/index" GIT_INDEX_FILE="$index_file" git_dir "$staging_repo" read-tree "$source_tree" || emit_error E_CANDIDATE_GIT -GIT_INDEX_FILE="$index_file" git_dir "$staging_repo" apply --cached --check \ - --whitespace=nowarn "$patch_file" >/dev/null 2>&1 || emit_error E_PATCH -GIT_INDEX_FILE="$index_file" git_dir "$staging_repo" apply --cached \ - --whitespace=nowarn "$patch_file" >/dev/null 2>&1 || emit_error E_PATCH +if [ "$patch_bytes" -gt 0 ]; then + GIT_INDEX_FILE="$index_file" git_dir "$staging_repo" apply --cached --check \ + --whitespace=nowarn "$patch_file" >/dev/null 2>&1 || emit_error E_PATCH + GIT_INDEX_FILE="$index_file" git_dir "$staging_repo" apply --cached \ + --whitespace=nowarn "$patch_file" >/dev/null 2>&1 || emit_error E_PATCH +fi candidate_tree=$(GIT_INDEX_FILE="$index_file" git_dir "$staging_repo" write-tree 2>/dev/null) || emit_error E_CANDIDATE_GIT candidate_paths="$run_root/candidate-paths" scan_tree "$staging_repo" "$candidate_tree" "$candidate_paths" || emit_error E_CANDIDATE_TREE +/bin/rm -f -- "$candidate_paths" changed_paths="$run_root/changed-paths" : > "$changed_paths" diff --git a/scripts/test/local-git-materializer-adapter.test.sh b/scripts/test/local-git-materializer-adapter.test.sh index c14c404..75d8a82 100755 --- a/scripts/test/local-git-materializer-adapter.test.sh +++ b/scripts/test/local-git-materializer-adapter.test.sh @@ -409,6 +409,21 @@ input_with_contract() { refresh_request_pair "$intermediate" "$destination" } +empty_patch="$tmp/empty.patch" +: > "$empty_patch" +no_change_input="$tmp/no-change-input.json" +input_with_patch "$input_file" "$empty_patch" "$no_change_input" +no_change_root=$(run_case no-change "$no_change_input") +[ ! -s "$no_change_root/err" ] || fail no-change-stderr +"${jq_cmd[@]}" -e --arg commit "$source_commit" --arg tree "$source_tree" ' + .stage_result.body.status=="completed" and + .stage_result.body.outcome=={family:"change",value:"no-change"} and + (.payloads[0].data | fromjson | + .candidate.commit_id==$commit and .candidate.tree_id==$tree and + .changed_paths.count==0) +' "$no_change_root/out" >/dev/null || fail no-change-response +pass 'empty patch returns the canonical no-change result' + bad_digest="$tmp/bad-digest.json" "${jq_cmd[@]}" -S -c '.trust_context.verified_payloads[0].sha256=("0"*64)' \ "$input_file" > "$bad_digest" @@ -595,6 +610,52 @@ malformed_input="$tmp/malformed-input.json" input_for_source "$input_file" "$malformed_input" sha1 "$malformed_commit" "$malformed_tree" expect_error source-malformed-utf8-path E_SOURCE_TREE "$malformed_input" "$malformed_source" +many_paths_source="$tmp/source-many-paths.git" +/bin/mkdir -m 700 "$many_paths_source" +git_clean init -q --bare --object-format=sha1 "$many_paths_source" +many_paths_blob=$(printf '%s\n' value | + git_clean --git-dir="$many_paths_source" hash-object -w --stdin) +many_paths_tree=$(/usr/bin/awk -v object="$many_paths_blob" \ + 'BEGIN { for (i=1; i<=65537; i++) printf "100644 blob %s\tpath-%05d\n", object, i }' | + git_clean --git-dir="$many_paths_source" mktree) +many_paths_commit=$(printf '%s\n' many-paths | + /usr/bin/env -i HOME="$tmp/home" TMPDIR="$tmp" PATH=/usr/bin:/bin LC_ALL=C \ + GIT_CONFIG_NOSYSTEM=1 GIT_CONFIG_GLOBAL=/dev/null GIT_NO_REPLACE_OBJECTS=1 \ + GIT_AUTHOR_NAME=fixture GIT_AUTHOR_EMAIL=fixture@example.invalid \ + GIT_COMMITTER_NAME=fixture GIT_COMMITTER_EMAIL=fixture@example.invalid \ + GIT_AUTHOR_DATE=2000-01-01T00:00:00Z GIT_COMMITTER_DATE=2000-01-01T00:00:00Z \ + /usr/bin/git --no-replace-objects --git-dir="$many_paths_source" \ + commit-tree "$many_paths_tree") +git_clean --git-dir="$many_paths_source" update-ref refs/heads/main "$many_paths_commit" +many_paths_input="$tmp/many-paths-input.json" +input_for_source "$input_file" "$many_paths_input" sha1 \ + "$many_paths_commit" "$many_paths_tree" +expect_error source-tree-entry-limit E_SOURCE_TREE "$many_paths_input" "$many_paths_source" + +deep_path_source="$tmp/source-deep-path.git" +/bin/mkdir -m 700 "$deep_path_source" +git_clean init -q --bare --object-format=sha1 "$deep_path_source" +deep_path_blob=$(printf '%s\n' value | + git_clean --git-dir="$deep_path_source" hash-object -w --stdin) +deep_path_tree=$(printf '100644 blob %s\tleaf\n' "$deep_path_blob" | + git_clean --git-dir="$deep_path_source" mktree) +for _ in {1..64}; do + deep_path_tree=$(printf '040000 tree %s\td\n' "$deep_path_tree" | + git_clean --git-dir="$deep_path_source" mktree) +done +deep_path_commit=$(printf '%s\n' deep-path | + /usr/bin/env -i HOME="$tmp/home" TMPDIR="$tmp" PATH=/usr/bin:/bin LC_ALL=C \ + GIT_CONFIG_NOSYSTEM=1 GIT_CONFIG_GLOBAL=/dev/null GIT_NO_REPLACE_OBJECTS=1 \ + GIT_AUTHOR_NAME=fixture GIT_AUTHOR_EMAIL=fixture@example.invalid \ + GIT_COMMITTER_NAME=fixture GIT_COMMITTER_EMAIL=fixture@example.invalid \ + GIT_AUTHOR_DATE=2000-01-01T00:00:00Z GIT_COMMITTER_DATE=2000-01-01T00:00:00Z \ + /usr/bin/git --no-replace-objects --git-dir="$deep_path_source" \ + commit-tree "$deep_path_tree") +git_clean --git-dir="$deep_path_source" update-ref refs/heads/main "$deep_path_commit" +deep_path_input="$tmp/deep-path-input.json" +input_for_source "$input_file" "$deep_path_input" sha1 "$deep_path_commit" "$deep_path_tree" +expect_error source-tree-component-limit E_SOURCE_TREE "$deep_path_input" "$deep_path_source" + sha256_source="$tmp/source-sha256.git" read -r sha256_commit sha256_tree < <(make_bare_source "$sha256_source" sha256) sha256_input="$tmp/sha256-input.json" From 62b39cc1973195b46e2ca62279955ead979258ec Mon Sep 17 00:00:00 2001 From: ci Date: Thu, 3 Sep 2026 16:45:53 -0400 Subject: [PATCH 07/17] Harden materializer worker entry --- README.md | 2 +- RESTORE.md | 2 +- .../local-git-materializer/v1/materialize.sh | 16 +++- .../local-git-materializer-adapter.test.sh | 81 +++++++++++++++++++ 4 files changed, 97 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 3e4543c..ab7007d 100644 --- a/README.md +++ b/README.md @@ -276,7 +276,7 @@ reachable objects into a caller-disposable bare repository, applies the patch to scratch-only index, and returns a canonical receipt and validated stage result. Reachable source history is limited to 65,536 objects and 256 MiB of uncompressed object data; the streamed pack is capped at the same byte limit. -Each tree scan is limited to 65,536 entries, 64 path components, and a 64 MiB +Each tree scan is limited to 65,536 entries, 64 path components, and a 16 MiB encoded listing, and validates UTF-8 once before its bounded built-in path walk. The fixed `materialize` command accepts only caller-named physical source, diff --git a/RESTORE.md b/RESTORE.md index 067fb42..e26fbf6 100644 --- a/RESTORE.md +++ b/RESTORE.md @@ -523,7 +523,7 @@ remotes, worktrees, alternates, shallow or partial repositories, replace state, binary patches, symlinks, submodules, host Git templates, and reachable source history above the fixed 65,536-object or 256 MiB import budget. It proves empty-patch `no-change` and rejects tree scans above 65,536 entries, 64 path components, or a -64 MiB encoded listing. It also checks that the source stays unchanged and scratch +16 MiB encoded listing. It also checks that the source stays unchanged and scratch is removed. This payload has no adapter manifest. A later assembly PR may bind the directory diff --git a/adapters/local-git-materializer/v1/materialize.sh b/adapters/local-git-materializer/v1/materialize.sh index 4259131..d549667 100755 --- a/adapters/local-git-materializer/v1/materialize.sh +++ b/adapters/local-git-materializer/v1/materialize.sh @@ -1,5 +1,17 @@ -#!/usr/bin/env bash +#!/bin/bash -p # shellcheck disable=SC2016 + +clean_path=${PATH:-/usr/bin:/bin} +while IFS= builtin read -r inherited_function; do + builtin unset -f "$inherited_function" 2>/dev/null || : +done < <(builtin compgen -A function) +while IFS= builtin read -r exported_name; do + case "$exported_name" in PATH) ;; *) builtin unset "$exported_name" 2>/dev/null || : ;; esac +done < <(builtin compgen -e) +PATH=$clean_path +LC_ALL=C +export PATH LC_ALL + set -euo pipefail emit_error() { @@ -291,7 +303,7 @@ safe_repo_path() { scan_tree() { local repository=$1 tree=$2 output=$3 raw_output entry metadata mode type object path local raw_bytes entry_count tree_scan_byte_limit tree_scan_entry_limit tree_scan_ceiling - tree_scan_byte_limit=67108864 + tree_scan_byte_limit=16777216 tree_scan_entry_limit=65536 tree_scan_ceiling=$((tree_scan_byte_limit + 1)) raw_output="$output.raw" diff --git a/scripts/test/local-git-materializer-adapter.test.sh b/scripts/test/local-git-materializer-adapter.test.sh index 75d8a82..e9430cc 100755 --- a/scripts/test/local-git-materializer-adapter.test.sh +++ b/scripts/test/local-git-materializer-adapter.test.sh @@ -353,6 +353,27 @@ expect_error() { pass "$name" } +direct_case="$tmp/direct-clean-worker" +/bin/mkdir -m 700 "$direct_case" "$direct_case/candidate" "$direct_case/scratch" +/usr/bin/touch "$direct_case/candidate/occupied" +printf '%s\n' "/usr/bin/touch '$direct_case/bash-env-ran'" > "$direct_case/bash-env" +if ( + # shellcheck disable=SC2329 + find() { return 0; } + export -f find + PATH="$runtime_bin:/usr/bin:/bin" GH_TOKEN=must-not-read BASH_ENV="$direct_case/bash-env" \ + "$adapter" __materialize_clean "$input_file" fixture.target "$tmp/source.git" \ + "$direct_case/candidate" "$direct_case/scratch" \ + > "$direct_case/out" 2> "$direct_case/err" +); then + fail direct-clean-worker-accepted +fi +[ ! -s "$direct_case/out" ] && + [ "$(cat "$direct_case/err")" = E_CANDIDATE_ROOT ] && + [ -f "$direct_case/candidate/occupied" ] && [ ! -e "$direct_case/bash-env-ran" ] || + fail direct-clean-worker-sanitization +pass 'direct worker entry blocks startup files and strips hostile environment state' + refresh_request_pair() { local source=$1 destination=$2 request_snapshot="$tmp/request-refresh" "${jq_cmd[@]}" -S -c '.stage_request.content' "$source" > "$request_snapshot" @@ -463,6 +484,66 @@ large_input="$tmp/large-input.json" input_for_source "$input_file" "$large_input" sha1 "$large_commit" "$large_tree" expect_error source-import-budget E_SOURCE_LIMIT "$large_input" "$large_source" +many_objects_source="$tmp/source-many-objects.git" +/bin/mkdir -m 700 "$many_objects_source" +git_clean init -q --bare --object-format=sha1 "$many_objects_source" +/usr/bin/awk 'BEGIN { + for (i=1; i<=32768; i++) { + data=sprintf("blob-%05d",i) + print "blob" + printf "mark :%d\n",i + printf "data %d\n%s\n",length(data),data + } + print "commit refs/heads/main" + print "mark :40000" + print "author fixture 946684800 +0000" + print "committer fixture 946684800 +0000" + print "data 11" + print "count-limit" + for (i=1; i<=32768; i++) + printf "M 100644 :%d dir-%05d/file\n",i,i + print "" +}' | git_clean --git-dir="$many_objects_source" fast-import --quiet +many_objects_commit=$(git_clean --git-dir="$many_objects_source" rev-parse refs/heads/main) +many_objects_tree=$(git_clean --git-dir="$many_objects_source" \ + rev-parse "$many_objects_commit^{tree}") +many_objects_count=$(git_clean --git-dir="$many_objects_source" rev-list \ + --objects --no-object-names "$many_objects_commit" | /usr/bin/wc -l | /usr/bin/tr -d ' ') +[ "$many_objects_count" -gt 65536 ] || fail many-objects-fixture +many_objects_input="$tmp/many-objects-input.json" +input_for_source "$input_file" "$many_objects_input" sha1 \ + "$many_objects_commit" "$many_objects_tree" +expect_error source-import-object-count E_SOURCE_LIMIT \ + "$many_objects_input" "$many_objects_source" + +large_listing_source="$tmp/source-large-listing.git" +/bin/mkdir -m 700 "$large_listing_source" +git_clean init -q --bare --object-format=sha1 "$large_listing_source" +large_listing_blob=$(printf '%s\n' value | + git_clean --git-dir="$large_listing_source" hash-object -w --stdin) +large_listing_prefix=$(/usr/bin/awk 'BEGIN { for (i=1; i<=4088; i++) printf "a" }') +large_listing_tree=$(/usr/bin/awk -v object="$large_listing_blob" \ + -v prefix="$large_listing_prefix" 'BEGIN { + for (i=1; i<=4097; i++) + printf "100644 blob %s\t%s%05d\n",object,prefix,i + }' | git_clean --git-dir="$large_listing_source" mktree) +[ "$(git_clean --git-dir="$large_listing_source" cat-file -s "$large_listing_tree")" \ + -gt 16777216 ] || fail large-listing-fixture +large_listing_commit=$(printf '%s\n' large-listing | + /usr/bin/env -i HOME="$tmp/home" TMPDIR="$tmp" PATH=/usr/bin:/bin LC_ALL=C \ + GIT_CONFIG_NOSYSTEM=1 GIT_CONFIG_GLOBAL=/dev/null GIT_NO_REPLACE_OBJECTS=1 \ + GIT_AUTHOR_NAME=fixture GIT_AUTHOR_EMAIL=fixture@example.invalid \ + GIT_COMMITTER_NAME=fixture GIT_COMMITTER_EMAIL=fixture@example.invalid \ + GIT_AUTHOR_DATE=2000-01-01T00:00:00Z GIT_COMMITTER_DATE=2000-01-01T00:00:00Z \ + /usr/bin/git --no-replace-objects --git-dir="$large_listing_source" \ + commit-tree "$large_listing_tree") +git_clean --git-dir="$large_listing_source" update-ref refs/heads/main "$large_listing_commit" +large_listing_input="$tmp/large-listing-input.json" +input_for_source "$input_file" "$large_listing_input" sha1 \ + "$large_listing_commit" "$large_listing_tree" +expect_error source-tree-byte-limit E_SOURCE_TREE \ + "$large_listing_input" "$large_listing_source" + /usr/bin/printf '%s\n' '/invalid/alternate' > "$tmp/source.git/objects/info/alternates" expect_error alternates E_SOURCE_GIT /bin/rm "$tmp/source.git/objects/info/alternates" From f25170e97295483dd389a4fcfc890bb1b1c0ee71 Mon Sep 17 00:00:00 2001 From: ci Date: Thu, 3 Sep 2026 17:40:13 -0400 Subject: [PATCH 08/17] Bound materializer candidate writes --- README.md | 2 + RESTORE.md | 3 +- .../local-git-materializer/v1/materialize.sh | 53 ++++++++++++++++++- .../local-git-materializer-adapter.test.sh | 41 +++++++++++++- 4 files changed, 96 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index ab7007d..5fac9ea 100644 --- a/README.md +++ b/README.md @@ -278,6 +278,8 @@ Reachable source history is limited to 65,536 objects and 256 MiB of uncompresse object data; the streamed pack is capped at the same byte limit. Each tree scan is limited to 65,536 entries, 64 path components, and a 16 MiB encoded listing, and validates UTF-8 once before its bounded built-in path walk. +Before mutating the index, patch paths must already fit the contract and their +cumulative source blob sizes plus patch bytes must fit a 256 MiB candidate budget. The fixed `materialize` command accepts only caller-named physical source, candidate, and scratch boundaries. It rejects worktrees, alternates, shallow or diff --git a/RESTORE.md b/RESTORE.md index e26fbf6..9d4f11e 100644 --- a/RESTORE.md +++ b/RESTORE.md @@ -524,7 +524,8 @@ binary patches, symlinks, submodules, host Git templates, and reachable source history above the fixed 65,536-object or 256 MiB import budget. It proves empty-patch `no-change` and rejects tree scans above 65,536 entries, 64 path components, or a 16 MiB encoded listing. It also checks that the source stays unchanged and scratch -is removed. +is removed. A shared-large-blob fixture proves the 256 MiB pre-apply candidate +budget blocks path fan-out before Git writes changed blobs. This payload has no adapter manifest. A later assembly PR may bind the directory tree through the payload's durable main commit. It is an inactive, local-only diff --git a/adapters/local-git-materializer/v1/materialize.sh b/adapters/local-git-materializer/v1/materialize.sh index d549667..f37b8a6 100755 --- a/adapters/local-git-materializer/v1/materialize.sh +++ b/adapters/local-git-materializer/v1/materialize.sh @@ -363,6 +363,58 @@ while IFS=' ' read -r object_id object_type object_size extra; do source_object_bytes=$((source_object_bytes + object_size)) done < "$source_object_sizes" +max_changed=$("$jq_bin" -r '.max_changed_paths' "$contract_file") || emit_error E_CONTRACT +patch_paths="$run_root/patch-paths" +: > "$patch_paths" +if [ "$patch_bytes" -gt 0 ]; then + patch_numstat="$run_root/patch-numstat" + git_dir "$source_git_dir" apply --numstat -z --whitespace=nowarn \ + "$patch_file" > "$patch_numstat" 2>/dev/null || emit_error E_PATCH + exec 4< "$patch_numstat" + while IFS= builtin read -r -d '' patch_record <&4; do + case "$patch_record" in *$'\t'*$'\t'*) ;; *) emit_error E_PATCH ;; esac + patch_stat_tail=${patch_record#*$'\t'} + patch_stat_tail=${patch_stat_tail#*$'\t'} + if [ -n "$patch_stat_tail" ]; then + safe_repo_path "$patch_stat_tail" || emit_error E_PATCH_PATH + printf '%s\n' "$patch_stat_tail" >> "$patch_paths" + else + IFS= builtin read -r -d '' patch_old_path <&4 || emit_error E_PATCH + IFS= builtin read -r -d '' patch_new_path <&4 || emit_error E_PATCH + if ! safe_repo_path "$patch_old_path" || ! safe_repo_path "$patch_new_path"; then + emit_error E_PATCH_PATH + fi + printf '%s\n%s\n' "$patch_old_path" "$patch_new_path" >> "$patch_paths" + fi + done + exec 4<&- +fi +LC_ALL=C /usr/bin/sort -u "$patch_paths" -o "$patch_paths" +patch_path_count=$(/usr/bin/wc -l < "$patch_paths" | /usr/bin/tr -d ' ') +[ "$patch_path_count" -le "$max_changed" ] || emit_error E_PATCH_LIMIT +candidate_mutation_byte_limit=268435456 +candidate_source_bytes=0 +while IFS= read -r patch_path; do + "$jq_bin" -e --arg path "$patch_path" '.allowed_paths | index($path) != null' \ + "$contract_file" >/dev/null || emit_error E_PATCH_SCOPE + source_path_bytes=0 + if source_path_object=$(git_dir "$source_git_dir" rev-parse --verify \ + "$source_tree:$patch_path" 2>/dev/null); then + [ "$(git_dir "$source_git_dir" cat-file -t "$source_path_object" 2>/dev/null)" = blob ] || + emit_error E_PATCH_PATH + source_path_bytes=$(git_dir "$source_git_dir" cat-file -s \ + "$source_path_object" 2>/dev/null) || emit_error E_SOURCE_GIT + fi + case "$source_path_bytes" in ''|*[!0-9]*) emit_error E_SOURCE_GIT ;; esac + [ "${#source_path_bytes}" -le 9 ] && + [ "$source_path_bytes" -le "$candidate_mutation_byte_limit" ] && + [ "$candidate_source_bytes" -le "$((candidate_mutation_byte_limit - source_path_bytes))" ] || + emit_error E_CANDIDATE_LIMIT + candidate_source_bytes=$((candidate_source_bytes + source_path_bytes)) +done < "$patch_paths" +[ "$patch_bytes" -le "$((candidate_mutation_byte_limit - candidate_source_bytes))" ] || + emit_error E_CANDIDATE_LIMIT + "${git_env[@]}" /usr/bin/git init --template="$empty_template" --bare \ --object-format="$source_algorithm" \ "$staging_repo" >/dev/null 2>&1 || emit_error E_CANDIDATE_GIT @@ -404,7 +456,6 @@ while IFS= read -r -d '' changed_path; do done < <(git_dir "$staging_repo" diff-tree -r --name-only -z "$source_tree" "$candidate_tree") LC_ALL=C /usr/bin/sort -u "$changed_paths" -o "$changed_paths" changed_count=$(wc -l < "$changed_paths" | /usr/bin/tr -d ' ') -max_changed=$("$jq_bin" -r '.max_changed_paths' "$contract_file") || emit_error E_CONTRACT [ "$changed_count" -le "$max_changed" ] || emit_error E_PATCH_LIMIT while IFS= read -r changed_path; do "$jq_bin" -e --arg path "$changed_path" '.allowed_paths | index($path) != null' \ diff --git a/scripts/test/local-git-materializer-adapter.test.sh b/scripts/test/local-git-materializer-adapter.test.sh index e9430cc..931e76a 100755 --- a/scripts/test/local-git-materializer-adapter.test.sh +++ b/scripts/test/local-git-materializer-adapter.test.sh @@ -544,6 +544,45 @@ input_for_source "$input_file" "$large_listing_input" sha1 \ expect_error source-tree-byte-limit E_SOURCE_TREE \ "$large_listing_input" "$large_listing_source" +shared_source="$tmp/source-shared-large-blob.git" +/bin/mkdir -m 700 "$shared_source" +git_clean init -q --bare --object-format=sha1 "$shared_source" +shared_blob=$( + { printf 'alpha\n'; /bin/dd if=/dev/zero bs=1048576 count=8 2>/dev/null | + /usr/bin/tr '\000' a; } | + git_clean --git-dir="$shared_source" hash-object -w --stdin +) +shared_contract="$tmp/shared-contract.json" +"${jq_cmd[@]}" -S -c ' + .allowed_paths=([range(1;34) | "shared-" + tostring] | sort) | + .max_changed_paths=33 +' "$contract_file" > "$shared_contract" +shared_paths="$tmp/shared-paths" +"${jq_cmd[@]}" -r '.allowed_paths[]' "$shared_contract" > "$shared_paths" +shared_tree=$(while IFS= read -r path; do + printf '100644 blob %s\t%s\n' "$shared_blob" "$path" +done < "$shared_paths" | git_clean --git-dir="$shared_source" mktree) +shared_commit=$(printf '%s\n' shared-large-blob | + /usr/bin/env -i HOME="$tmp/home" TMPDIR="$tmp" PATH=/usr/bin:/bin LC_ALL=C \ + GIT_CONFIG_NOSYSTEM=1 GIT_CONFIG_GLOBAL=/dev/null GIT_NO_REPLACE_OBJECTS=1 \ + GIT_AUTHOR_NAME=fixture GIT_AUTHOR_EMAIL=fixture@example.invalid \ + GIT_COMMITTER_NAME=fixture GIT_COMMITTER_EMAIL=fixture@example.invalid \ + GIT_AUTHOR_DATE=2000-01-01T00:00:00Z GIT_COMMITTER_DATE=2000-01-01T00:00:00Z \ + /usr/bin/git --no-replace-objects --git-dir="$shared_source" commit-tree "$shared_tree") +git_clean --git-dir="$shared_source" update-ref refs/heads/main "$shared_commit" +shared_patch="$tmp/shared.patch" +while IFS= read -r path; do + printf '%s\n' "diff --git a/$path b/$path" "--- a/$path" "+++ b/$path" \ + '@@ -1 +1 @@' '-alpha' '+beta' +done < "$shared_paths" > "$shared_patch" +shared_source_input="$tmp/shared-source-input.json" +input_for_source "$input_file" "$shared_source_input" sha1 "$shared_commit" "$shared_tree" +shared_contract_input="$tmp/shared-contract-input.json" +input_with_contract "$shared_source_input" "$shared_contract" "$shared_contract_input" +shared_input="$tmp/shared-input.json" +input_with_patch "$shared_contract_input" "$shared_patch" "$shared_input" +expect_error candidate-mutation-budget E_CANDIDATE_LIMIT "$shared_input" "$shared_source" + /usr/bin/printf '%s\n' '/invalid/alternate' > "$tmp/source.git/objects/info/alternates" expect_error alternates E_SOURCE_GIT /bin/rm "$tmp/source.git/objects/info/alternates" @@ -841,7 +880,7 @@ traversal_patch="$tmp/traversal.patch" > "$traversal_patch" traversal_input="$tmp/traversal-input.json" input_with_patch "$input_file" "$traversal_patch" "$traversal_input" -expect_error traversal E_PATCH "$traversal_input" +expect_error traversal E_PATCH_PATH "$traversal_input" [ "$(cat "$outside")" = unchanged ] || fail traversal-write symlink_boundary="$tmp/candidate-link" From aa1dc6c7de3fe26f0ab7aafddc20f9c3c0044347 Mon Sep 17 00:00:00 2001 From: ci Date: Thu, 3 Sep 2026 18:27:53 -0400 Subject: [PATCH 09/17] Close materializer copy and tree gaps --- README.md | 3 +- RESTORE.md | 3 +- .../local-git-materializer/v1/materialize.sh | 17 +++++++-- .../local-git-materializer-adapter.test.sh | 37 +++++++++++++++++++ 4 files changed, 54 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 5fac9ea..e2e9942 100644 --- a/README.md +++ b/README.md @@ -284,7 +284,8 @@ cumulative source blob sizes plus patch bytes must fit a 256 MiB candidate budge The fixed `materialize` command accepts only caller-named physical source, candidate, and scratch boundaries. It rejects worktrees, alternates, shallow or partial repositories, replace or graft state, active hooks and filters, remote -configuration, unsafe paths, binary patches, symlinks, and submodules. It never +configuration, unsafe paths, binary or copy/rename patches, empty subtrees, +symlinks, and submodules. It never inherits host Git templates, checks out a worktree, or runs a transport command. An empty producer patch returns the explicit `no-change` result. Tests cover both SHA-1 and SHA-256 object formats with disposable local fixtures. diff --git a/RESTORE.md b/RESTORE.md index 9d4f11e..93d533e 100644 --- a/RESTORE.md +++ b/RESTORE.md @@ -521,7 +521,8 @@ becomes a deterministic bare child commit and path-free receipt. The negative matrix rejects moved identities, unsafe directories and paths, hooks, filters, remotes, worktrees, alternates, shallow or partial repositories, replace state, binary patches, symlinks, submodules, host Git templates, and reachable source -history above the fixed 65,536-object or 256 MiB import budget. It proves empty-patch +history above the fixed 65,536-object or 256 MiB import budget. Copy/rename patch +metadata and source trees containing empty subtrees also fail closed. It proves empty-patch `no-change` and rejects tree scans above 65,536 entries, 64 path components, or a 16 MiB encoded listing. It also checks that the source stays unchanged and scratch is removed. A shared-large-blob fixture proves the 256 MiB pre-apply candidate diff --git a/adapters/local-git-materializer/v1/materialize.sh b/adapters/local-git-materializer/v1/materialize.sh index f37b8a6..8d6b472 100755 --- a/adapters/local-git-materializer/v1/materialize.sh +++ b/adapters/local-git-materializer/v1/materialize.sh @@ -216,6 +216,11 @@ patch_without_nul=$(LC_ALL=C /usr/bin/tr -d '\000' < "$patch_file" | wc -c | /us if /usr/bin/grep -aEq '^(GIT binary patch|Binary files .+ differ)$' "$patch_file"; then emit_error E_BINARY_PATCH fi +if /usr/bin/grep -aEq \ + '^(copy from|copy to|rename from|rename to|similarity index|dissimilarity index) ' \ + "$patch_file"; then + emit_error E_PATCH +fi source_commit=$("$jq_bin" -r '.stage_request.content.body.target_revision.value.commit_id' \ "$input_snapshot") || emit_error E_SOURCE_IDENTITY @@ -301,19 +306,20 @@ safe_repo_path() { } scan_tree() { - local repository=$1 tree=$2 output=$3 raw_output entry metadata mode type object path + local repository=$1 tree=$2 output=$3 raw_output entry metadata mode type object path empty_tree local raw_bytes entry_count tree_scan_byte_limit tree_scan_entry_limit tree_scan_ceiling tree_scan_byte_limit=16777216 tree_scan_entry_limit=65536 tree_scan_ceiling=$((tree_scan_byte_limit + 1)) raw_output="$output.raw" - if ! git_dir "$repository" ls-tree -rz -r "$tree" | + if ! git_dir "$repository" ls-tree -rz -r -t "$tree" | /usr/bin/head -c "$tree_scan_ceiling" > "$raw_output"; then return 1 fi raw_bytes=$(/usr/bin/wc -c < "$raw_output" | /usr/bin/tr -d ' ') || return 1 [ "$raw_bytes" -le "$tree_scan_byte_limit" ] || return 1 /usr/bin/iconv -f UTF-8 -t UTF-8 "$raw_output" >/dev/null 2>&1 || return 1 + empty_tree=$(git_dir "$repository" hash-object -t tree --stdin "$output" entry_count=0 while IFS= read -r -d '' entry; do @@ -323,9 +329,12 @@ scan_tree() { path=${entry#*$'\t'} read -r mode type object <<< "$metadata" [ -n "$object" ] || return 1 - case "$mode:$type" in 100644:blob|100755:blob) ;; *) return 1 ;; esac safe_repo_path "$path" || return 1 - printf '%s\n' "$path" >> "$output" + case "$mode:$type" in + 100644:blob|100755:blob) printf '%s\n' "$path" >> "$output" ;; + 040000:tree) [ "$object" != "$empty_tree" ] || return 1 ;; + *) return 1 ;; + esac done < "$raw_output" /bin/rm -f -- "$raw_output" } diff --git a/scripts/test/local-git-materializer-adapter.test.sh b/scripts/test/local-git-materializer-adapter.test.sh index 931e76a..d9015dd 100755 --- a/scripts/test/local-git-materializer-adapter.test.sh +++ b/scripts/test/local-git-materializer-adapter.test.sh @@ -583,6 +583,20 @@ shared_input="$tmp/shared-input.json" input_with_patch "$shared_contract_input" "$shared_patch" "$shared_input" expect_error candidate-mutation-budget E_CANDIDATE_LIMIT "$shared_input" "$shared_source" +copy_contract="$tmp/copy-contract.json" +"${jq_cmd[@]}" -S -c \ + '.allowed_paths=["copy-1","source.txt"] | .max_changed_paths=1' \ + "$contract_file" > "$copy_contract" +copy_patch="$tmp/copy.patch" +printf '%s\n' 'diff --git a/source.txt b/copy-1' 'similarity index 99%' \ + 'copy from source.txt' 'copy to copy-1' '--- a/source.txt' '+++ b/copy-1' \ + '@@ -1 +1 @@' '-alpha' '+beta' > "$copy_patch" +copy_contract_input="$tmp/copy-contract-input.json" +input_with_contract "$input_file" "$copy_contract" "$copy_contract_input" +copy_input="$tmp/copy-input.json" +input_with_patch "$copy_contract_input" "$copy_patch" "$copy_input" +expect_error copy-metadata E_PATCH "$copy_input" + /usr/bin/printf '%s\n' '/invalid/alternate' > "$tmp/source.git/objects/info/alternates" expect_error alternates E_SOURCE_GIT /bin/rm "$tmp/source.git/objects/info/alternates" @@ -673,6 +687,29 @@ submodule_input="$tmp/submodule-input.json" input_for_source "$input_file" "$submodule_input" sha1 "$outer_commit" "$outer_tree" expect_error source-submodule-mode E_SOURCE_TREE "$submodule_input" "$submodule_source" +empty_subtree_source="$tmp/source-empty-subtree.git" +/bin/mkdir -m 700 "$empty_subtree_source" +git_clean init -q --bare --object-format=sha1 "$empty_subtree_source" +empty_subtree_blob=$(printf '%s\n' alpha beta | + git_clean --git-dir="$empty_subtree_source" hash-object -w --stdin) +empty_tree=$(git_clean --git-dir="$empty_subtree_source" mktree Date: Thu, 3 Sep 2026 19:17:29 -0400 Subject: [PATCH 10/17] Bound materializer source scans --- README.md | 5 +- RESTORE.md | 3 +- .../local-git-materializer/v1/materialize.sh | 46 ++++++++++++++++--- .../local-git-materializer-adapter.test.sh | 21 +++++++++ 4 files changed, 67 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index e2e9942..c8fce34 100644 --- a/README.md +++ b/README.md @@ -276,13 +276,16 @@ reachable objects into a caller-disposable bare repository, applies the patch to scratch-only index, and returns a canonical receipt and validated stage result. Reachable source history is limited to 65,536 objects and 256 MiB of uncompressed object data; the streamed pack is capped at the same byte limit. +The complete source filesystem inventory is capped at 65,536 entries and 8 MiB +before Git reads it. Each tree scan is limited to 65,536 entries, 64 path components, and a 16 MiB encoded listing, and validates UTF-8 once before its bounded built-in path walk. Before mutating the index, patch paths must already fit the contract and their cumulative source blob sizes plus patch bytes must fit a 256 MiB candidate budget. The fixed `materialize` command accepts only caller-named physical source, -candidate, and scratch boundaries. It rejects worktrees, alternates, shallow or +candidate, and scratch boundaries, with every path independently absolute. It +rejects worktrees, alternates, shallow or partial repositories, replace or graft state, active hooks and filters, remote configuration, unsafe paths, binary or copy/rename patches, empty subtrees, symlinks, and submodules. It never diff --git a/RESTORE.md b/RESTORE.md index 93d533e..2d97347 100644 --- a/RESTORE.md +++ b/RESTORE.md @@ -525,7 +525,8 @@ history above the fixed 65,536-object or 256 MiB import budget. Copy/rename patc metadata and source trees containing empty subtrees also fail closed. It proves empty-patch `no-change` and rejects tree scans above 65,536 entries, 64 path components, or a 16 MiB encoded listing. It also checks that the source stays unchanged and scratch -is removed. A shared-large-blob fixture proves the 256 MiB pre-apply candidate +is removed. The complete source filesystem scan is capped at 65,536 entries and +8 MiB. A shared-large-blob fixture proves the 256 MiB pre-apply candidate budget blocks path fan-out before Git writes changed blobs. This payload has no adapter manifest. A later assembly PR may bind the directory diff --git a/adapters/local-git-materializer/v1/materialize.sh b/adapters/local-git-materializer/v1/materialize.sh index 8d6b472..5ab803f 100755 --- a/adapters/local-git-materializer/v1/materialize.sh +++ b/adapters/local-git-materializer/v1/materialize.sh @@ -35,10 +35,10 @@ source_git_dir=$4 candidate_root=$5 scratch_root=$6 -case "$script_path:$input_path:$source_git_dir:$candidate_root:$scratch_root" in - /*:/*:/*:/*:/*) ;; - *) emit_error E_USAGE ;; -esac +for absolute_path in "$script_path" "$input_path" "$source_git_dir" \ + "$candidate_root" "$scratch_root"; do + case "$absolute_path" in /*) ;; *) emit_error E_USAGE ;; esac +done [ -f "$script_path" ] && [ ! -L "$script_path" ] || emit_error E_PACKAGE script_dir=$(CDPATH='' cd -P -- "${script_path%/*}" && pwd -P) || emit_error E_PACKAGE repo_root=$(CDPATH='' cd -P -- "$script_dir/../../.." && pwd -P) || emit_error E_PACKAGE @@ -249,6 +249,30 @@ git_dir() { "${git_env[@]}" /usr/bin/git --no-replace-objects --git-dir="$directory" "$@" } +source_inventory="$run_root/source-filesystem" +source_inventory_byte_limit=8388608 +source_inventory_entry_limit=65536 +source_inventory_ceiling=$((source_inventory_byte_limit + 1)) +if ! /usr/bin/find "$source_git_dir" -mindepth 1 -print0 | + /usr/bin/head -c "$source_inventory_ceiling" > "$source_inventory"; then + emit_error E_SOURCE_LIMIT +fi +source_inventory_bytes=$(/usr/bin/wc -c < "$source_inventory" | /usr/bin/tr -d ' ') || + emit_error E_SOURCE_LIMIT +[ "$source_inventory_bytes" -le "$source_inventory_byte_limit" ] || + emit_error E_SOURCE_LIMIT +source_inventory_entries=0 +while IFS= builtin read -r -d '' source_entry; do + source_inventory_entries=$((source_inventory_entries + 1)) + [ "$source_inventory_entries" -le "$source_inventory_entry_limit" ] || + emit_error E_SOURCE_LIMIT + case "$source_entry" in "$source_git_dir"/*) ;; *) emit_error E_SOURCE_GIT ;; esac + if [ -L "$source_entry" ] || { [ ! -f "$source_entry" ] && [ ! -d "$source_entry" ]; }; then + emit_error E_SOURCE_GIT + fi +done < "$source_inventory" +/bin/rm -f -- "$source_inventory" + source_config="$run_root/source-config" git_dir "$source_git_dir" config --local --name-only --list --no-includes \ > "$source_config" 2>/dev/null || emit_error E_SOURCE_GIT @@ -266,7 +290,6 @@ done < "$source_config" [ ! -e "$source_git_dir/info/grafts" ] && [ ! -e "$source_git_dir/objects/info/alternates" ] && [ ! -d "$source_git_dir/refs/replace" ] && - [ -z "$(find "$source_git_dir" -type l -print -quit)" ] && [ -z "$(find "$source_git_dir/objects/pack" -type f -name '*.promisor' -print -quit 2>/dev/null)" ] || emit_error E_SOURCE_GIT [ -z "$(git_dir "$source_git_dir" for-each-ref --format='%(refname)' \ @@ -458,11 +481,22 @@ scan_tree "$staging_repo" "$candidate_tree" "$candidate_paths" || emit_error E_C /bin/rm -f -- "$candidate_paths" changed_paths="$run_root/changed-paths" +changed_paths_raw="$run_root/changed-paths.raw" +changed_paths_byte_limit=2097152 +changed_paths_ceiling=$((changed_paths_byte_limit + 1)) +if ! git_dir "$staging_repo" diff-tree -r --name-only -z \ + "$source_tree" "$candidate_tree" | + /usr/bin/head -c "$changed_paths_ceiling" > "$changed_paths_raw"; then + emit_error E_CANDIDATE_GIT +fi +changed_paths_bytes=$(/usr/bin/wc -c < "$changed_paths_raw" | /usr/bin/tr -d ' ') || + emit_error E_CANDIDATE_GIT +[ "$changed_paths_bytes" -le "$changed_paths_byte_limit" ] || emit_error E_PATCH_LIMIT : > "$changed_paths" while IFS= read -r -d '' changed_path; do safe_repo_path "$changed_path" || emit_error E_PATCH_PATH printf '%s\n' "$changed_path" >> "$changed_paths" -done < <(git_dir "$staging_repo" diff-tree -r --name-only -z "$source_tree" "$candidate_tree") +done < "$changed_paths_raw" LC_ALL=C /usr/bin/sort -u "$changed_paths" -o "$changed_paths" changed_count=$(wc -l < "$changed_paths" | /usr/bin/tr -d ' ') [ "$changed_count" -le "$max_changed" ] || emit_error E_PATCH_LIMIT diff --git a/scripts/test/local-git-materializer-adapter.test.sh b/scripts/test/local-git-materializer-adapter.test.sh index d9015dd..e590d2b 100755 --- a/scripts/test/local-git-materializer-adapter.test.sh +++ b/scripts/test/local-git-materializer-adapter.test.sh @@ -374,6 +374,20 @@ fi fail direct-clean-worker-sanitization pass 'direct worker entry blocks startup files and strips hostile environment state' +relative_case="$tmp/relative-path" +/bin/mkdir -m 700 "$relative_case" "$relative_case/candidate" "$relative_case/scratch" +if ( + cd "$tmp" + PATH="$runtime_bin:/usr/bin:/bin" "$adapter" materialize 'relative:/input.json' \ + fixture.target "$tmp/source.git" "$relative_case/candidate" "$relative_case/scratch" \ + > "$relative_case/out" 2> "$relative_case/err" +); then + fail relative-input-path-accepted +fi +[ ! -s "$relative_case/out" ] && [ "$(cat "$relative_case/err")" = E_USAGE ] || + fail relative-input-path +pass 'each filesystem argument must be independently absolute' + refresh_request_pair() { local source=$1 destination=$2 request_snapshot="$tmp/request-refresh" "${jq_cmd[@]}" -S -c '.stage_request.content' "$source" > "$request_snapshot" @@ -940,4 +954,11 @@ fi pass 'core validation is routed through caller-owned accounted scratch' pass 'payload has no provider, transport, credential, authority or qualification path' +if /usr/bin/grep -Fq 'done < <(git_dir "$staging_repo" diff-tree' "$adapter" || + ! /usr/bin/grep -Fq 'diff-tree -r --name-only -z' "$adapter" || + ! /usr/bin/grep -Fq '> "$changed_paths_raw"' "$adapter"; then + fail changed-path-status-capture +fi +pass 'changed-path enumeration is captured before evidence processing' + printf 'local Git materializer: %s focused checks passed\n' "$passed" From ab26997df85aa02d08f88416f6009fbd3c881f29 Mon Sep 17 00:00:00 2001 From: ci Date: Thu, 3 Sep 2026 20:09:39 -0400 Subject: [PATCH 11/17] Preflight materializer resource bounds --- README.md | 10 +- RESTORE.md | 7 +- .../local-git-materializer/v1/materialize.sh | 104 +++++++++++++----- .../local-git-materializer-adapter.test.sh | 49 +++++++-- 4 files changed, 125 insertions(+), 45 deletions(-) diff --git a/README.md b/README.md index c8fce34..26cf748 100644 --- a/README.md +++ b/README.md @@ -276,10 +276,12 @@ reachable objects into a caller-disposable bare repository, applies the patch to scratch-only index, and returns a canonical receipt and validated stage result. Reachable source history is limited to 65,536 objects and 256 MiB of uncompressed object data; the streamed pack is capped at the same byte limit. -The complete source filesystem inventory is capped at 65,536 entries and 8 MiB -before Git reads it. -Each tree scan is limited to 65,536 entries, 64 path components, and a 16 MiB -encoded listing, and validates UTF-8 once before its bounded built-in path walk. +The complete source filesystem inventory is capped at 65,536 entries and 8 MiB, +and repository config is snapshotted at 1 MiB before parsing. +Each tree scan is limited to 65,536 entries, 1,024 tree objects, 64 path +components, and a 16 MiB encoded listing. Each commit or tree is size-checked +before a non-recursive tree step, and each step validates UTF-8 before its bounded +built-in path walk. Before mutating the index, patch paths must already fit the contract and their cumulative source blob sizes plus patch bytes must fit a 256 MiB candidate budget. diff --git a/RESTORE.md b/RESTORE.md index 2d97347..0e6183f 100644 --- a/RESTORE.md +++ b/RESTORE.md @@ -525,8 +525,11 @@ history above the fixed 65,536-object or 256 MiB import budget. Copy/rename patc metadata and source trees containing empty subtrees also fail closed. It proves empty-patch `no-change` and rejects tree scans above 65,536 entries, 64 path components, or a 16 MiB encoded listing. It also checks that the source stays unchanged and scratch -is removed. The complete source filesystem scan is capped at 65,536 entries and -8 MiB. A shared-large-blob fixture proves the 256 MiB pre-apply candidate +is removed. Input and config snapshots are stream-capped before parsing. The +complete source filesystem scan is capped at 65,536 entries and 8 MiB. Tree scans +are capped at 1,024 tree objects, with each object size-checked before non-recursive +expansion. A shared-large-blob +fixture proves the 256 MiB pre-apply candidate budget blocks path fan-out before Git writes changed blobs. This payload has no adapter manifest. A later assembly PR may bind the directory diff --git a/adapters/local-git-materializer/v1/materialize.sh b/adapters/local-git-materializer/v1/materialize.sh index 5ab803f..e59f7a2 100755 --- a/adapters/local-git-materializer/v1/materialize.sh +++ b/adapters/local-git-materializer/v1/materialize.sh @@ -135,7 +135,13 @@ trap 'exit 143' TERM /bin/mkdir -m 500 "$run_root/empty-template" || emit_error E_SCRATCH_ROOT input_snapshot="$run_root/input.json" -/bin/cp "$input_path" "$input_snapshot" || emit_error E_INPUT +input_copy_ceiling=8388609 +if ! /usr/bin/head -c "$input_copy_ceiling" "$input_path" > "$input_snapshot"; then + emit_error E_INPUT +fi +input_snapshot_bytes=$(/usr/bin/wc -c < "$input_snapshot" | /usr/bin/tr -d ' ') || + emit_error E_INPUT +[ "$input_snapshot_bytes" -le 8388608 ] || emit_error E_INPUT /bin/chmod 0400 "$input_snapshot" || emit_error E_INPUT input_canonical="$run_root/input.canonical" "$jq_bin" -S -c . "$input_snapshot" > "$input_canonical" 2>/dev/null || emit_error E_INPUT @@ -273,9 +279,22 @@ while IFS= builtin read -r -d '' source_entry; do done < "$source_inventory" /bin/rm -f -- "$source_inventory" +source_config_input="$source_git_dir/config" +[ -f "$source_config_input" ] && [ ! -L "$source_config_input" ] || + emit_error E_SOURCE_CONFIG +source_config_snapshot="$run_root/source-config.snapshot" +source_config_ceiling=1048577 +if ! /usr/bin/head -c "$source_config_ceiling" "$source_config_input" \ + > "$source_config_snapshot"; then + emit_error E_SOURCE_CONFIG +fi +source_config_bytes=$(/usr/bin/wc -c < "$source_config_snapshot" | /usr/bin/tr -d ' ') || + emit_error E_SOURCE_CONFIG +[ "$source_config_bytes" -le 1048576 ] || emit_error E_SOURCE_CONFIG source_config="$run_root/source-config" -git_dir "$source_git_dir" config --local --name-only --list --no-includes \ - > "$source_config" 2>/dev/null || emit_error E_SOURCE_GIT +"${git_env[@]}" /usr/bin/git config --file "$source_config_snapshot" \ + --name-only --list --no-includes > "$source_config" 2>/dev/null || + emit_error E_SOURCE_CONFIG while IFS= read -r config_key; do case "$config_key" in core.repositoryformatversion|core.filemode|core.bare|core.logallrefupdates|core.ignorecase|core.precomposeunicode|extensions.objectformat) ;; @@ -301,8 +320,12 @@ fi actual_algorithm=$(git_dir "$source_git_dir" rev-parse --show-object-format 2>/dev/null) || emit_error E_SOURCE_GIT [ "$actual_algorithm" = "$source_algorithm" ] || emit_error E_SOURCE_IDENTITY -git_dir "$source_git_dir" cat-file -e "$source_commit^{commit}" >/dev/null 2>&1 || +source_commit_type=$(git_dir "$source_git_dir" cat-file -t "$source_commit" 2>/dev/null) || emit_error E_SOURCE_IDENTITY +source_commit_size=$(git_dir "$source_git_dir" cat-file -s "$source_commit" 2>/dev/null) || + emit_error E_SOURCE_IDENTITY +[ "$source_commit_type" = commit ] && [ "$source_commit_size" -le 1048576 ] || + emit_error E_SOURCE_LIMIT actual_source_tree=$(git_dir "$source_git_dir" rev-parse "$source_commit^{tree}" 2>/dev/null) || emit_error E_SOURCE_IDENTITY [ "$actual_source_tree" = "$source_tree" ] || emit_error E_SOURCE_IDENTITY @@ -329,37 +352,58 @@ safe_repo_path() { } scan_tree() { - local repository=$1 tree=$2 output=$3 raw_output entry metadata mode type object path empty_tree - local raw_bytes entry_count tree_scan_byte_limit tree_scan_entry_limit tree_scan_ceiling + local repository=$1 tree=$2 output=$3 queue current_tree prefix raw_output + local entry metadata mode type object path full_path empty_tree tree_type tree_bytes + local raw_bytes entry_count tree_count total_bytes remaining + local tree_scan_byte_limit tree_scan_entry_limit tree_scan_tree_limit tree_scan_byte_limit=16777216 tree_scan_entry_limit=65536 - tree_scan_ceiling=$((tree_scan_byte_limit + 1)) - raw_output="$output.raw" - if ! git_dir "$repository" ls-tree -rz -r -t "$tree" | - /usr/bin/head -c "$tree_scan_ceiling" > "$raw_output"; then - return 1 - fi - raw_bytes=$(/usr/bin/wc -c < "$raw_output" | /usr/bin/tr -d ' ') || return 1 - [ "$raw_bytes" -le "$tree_scan_byte_limit" ] || return 1 - /usr/bin/iconv -f UTF-8 -t UTF-8 "$raw_output" >/dev/null 2>&1 || return 1 + tree_scan_tree_limit=1024 empty_tree=$(git_dir "$repository" hash-object -t tree --stdin "$queue" : > "$output" entry_count=0 - while IFS= read -r -d '' entry; do - entry_count=$((entry_count + 1)) - [ "$entry_count" -le "$tree_scan_entry_limit" ] || return 1 - metadata=${entry%%$'\t'*} - path=${entry#*$'\t'} - read -r mode type object <<< "$metadata" - [ -n "$object" ] || return 1 - safe_repo_path "$path" || return 1 - case "$mode:$type" in - 100644:blob|100755:blob) printf '%s\n' "$path" >> "$output" ;; - 040000:tree) [ "$object" != "$empty_tree" ] || return 1 ;; - *) return 1 ;; - esac - done < "$raw_output" - /bin/rm -f -- "$raw_output" + tree_count=0 + total_bytes=0 + while IFS=$'\t' read -r current_tree prefix; do + tree_count=$((tree_count + 1)) + [ "$tree_count" -le "$tree_scan_tree_limit" ] || return 1 + tree_type=$(git_dir "$repository" cat-file -t "$current_tree" 2>/dev/null) || return 1 + tree_bytes=$(git_dir "$repository" cat-file -s "$current_tree" 2>/dev/null) || return 1 + [ "$tree_type" = tree ] || return 1 + case "$tree_bytes" in ''|*[!0-9]*) return 1 ;; esac + [ "${#tree_bytes}" -le 8 ] && [ "$tree_bytes" -le "$tree_scan_byte_limit" ] || return 1 + [ -z "$prefix" ] || [ "$current_tree" != "$empty_tree" ] || return 1 + remaining=$((tree_scan_byte_limit - total_bytes)) + [ "$tree_bytes" -le "$remaining" ] || return 1 + raw_output="$output.raw" + if ! git_dir "$repository" ls-tree -z "$current_tree" | + /usr/bin/head -c "$((remaining + 1))" > "$raw_output"; then + return 1 + fi + raw_bytes=$(/usr/bin/wc -c < "$raw_output" | /usr/bin/tr -d ' ') || return 1 + [ "$raw_bytes" -le "$remaining" ] || return 1 + total_bytes=$((total_bytes + raw_bytes)) + /usr/bin/iconv -f UTF-8 -t UTF-8 "$raw_output" >/dev/null 2>&1 || return 1 + while IFS= read -r -d '' entry; do + entry_count=$((entry_count + 1)) + [ "$entry_count" -le "$tree_scan_entry_limit" ] || return 1 + metadata=${entry%%$'\t'*} + path=${entry#*$'\t'} + read -r mode type object <<< "$metadata" + [ -n "$object" ] || return 1 + full_path=$path + [ -z "$prefix" ] || full_path="$prefix/$path" + safe_repo_path "$full_path" || return 1 + case "$mode:$type" in + 100644:blob|100755:blob) printf '%s\n' "$full_path" >> "$output" ;; + 040000:tree) printf '%s\t%s\n' "$object" "$full_path" >> "$queue" ;; + *) return 1 ;; + esac + done < "$raw_output" + done < "$queue" + /bin/rm -f -- "$queue" "$raw_output" } source_paths="$run_root/source-paths" diff --git a/scripts/test/local-git-materializer-adapter.test.sh b/scripts/test/local-git-materializer-adapter.test.sh index e590d2b..3b79532 100755 --- a/scripts/test/local-git-materializer-adapter.test.sh +++ b/scripts/test/local-git-materializer-adapter.test.sh @@ -507,16 +507,15 @@ git_clean init -q --bare --object-format=sha1 "$many_objects_source" print "blob" printf "mark :%d\n",i printf "data %d\n%s\n",length(data),data + print "commit refs/heads/main" + printf "mark :%d\n",40000+i + print "author fixture 946684800 +0000" + print "committer fixture 946684800 +0000" + print "data 5" + print "count" + if (i>1) printf "from :%d\n",40000+i-1 + printf "M 100644 :%d file\n\n",i } - print "commit refs/heads/main" - print "mark :40000" - print "author fixture 946684800 +0000" - print "committer fixture 946684800 +0000" - print "data 11" - print "count-limit" - for (i=1; i<=32768; i++) - printf "M 100644 :%d dir-%05d/file\n",i,i - print "" }' | git_clean --git-dir="$many_objects_source" fast-import --quiet many_objects_commit=$(git_clean --git-dir="$many_objects_source" rev-parse refs/heads/main) many_objects_tree=$(git_clean --git-dir="$many_objects_source" \ @@ -827,6 +826,30 @@ deep_path_input="$tmp/deep-path-input.json" input_for_source "$input_file" "$deep_path_input" sha1 "$deep_path_commit" "$deep_path_tree" expect_error source-tree-component-limit E_SOURCE_TREE "$deep_path_input" "$deep_path_source" +many_trees_source="$tmp/source-many-trees.git" +/bin/mkdir -m 700 "$many_trees_source" +git_clean init -q --bare --object-format=sha1 "$many_trees_source" +many_trees_blob=$(printf '%s\n' value | + git_clean --git-dir="$many_trees_source" hash-object -w --stdin) +many_trees_leaf=$(printf '100644 blob %s\tfile\n' "$many_trees_blob" | + git_clean --git-dir="$many_trees_source" mktree) +many_trees_root=$(/usr/bin/awk -v tree="$many_trees_leaf" \ + 'BEGIN { for (i=1; i<=1024; i++) printf "040000 tree %s\tdir-%04d\n",tree,i }' | + git_clean --git-dir="$many_trees_source" mktree) +many_trees_commit=$(printf '%s\n' many-trees | + /usr/bin/env -i HOME="$tmp/home" TMPDIR="$tmp" PATH=/usr/bin:/bin LC_ALL=C \ + GIT_CONFIG_NOSYSTEM=1 GIT_CONFIG_GLOBAL=/dev/null GIT_NO_REPLACE_OBJECTS=1 \ + GIT_AUTHOR_NAME=fixture GIT_AUTHOR_EMAIL=fixture@example.invalid \ + GIT_COMMITTER_NAME=fixture GIT_COMMITTER_EMAIL=fixture@example.invalid \ + GIT_AUTHOR_DATE=2000-01-01T00:00:00Z GIT_COMMITTER_DATE=2000-01-01T00:00:00Z \ + /usr/bin/git --no-replace-objects --git-dir="$many_trees_source" \ + commit-tree "$many_trees_root") +git_clean --git-dir="$many_trees_source" update-ref refs/heads/main "$many_trees_commit" +many_trees_input="$tmp/many-trees-input.json" +input_for_source "$input_file" "$many_trees_input" sha1 \ + "$many_trees_commit" "$many_trees_root" +expect_error source-tree-object-limit E_SOURCE_TREE "$many_trees_input" "$many_trees_source" + sha256_source="$tmp/source-sha256.git" read -r sha256_commit sha256_tree < <(make_bare_source "$sha256_source" sha256) sha256_input="$tmp/sha256-input.json" @@ -961,4 +984,12 @@ if /usr/bin/grep -Fq 'done < <(git_dir "$staging_repo" diff-tree' "$adapter" || fi pass 'changed-path enumeration is captured before evidence processing' +if /usr/bin/grep -Fq '/bin/cp "$input_path"' "$adapter" || + ! /usr/bin/grep -Fq 'input_copy_ceiling=8388609' "$adapter" || + ! /usr/bin/grep -Fq 'source_config_ceiling=1048577' "$adapter" || + /usr/bin/grep -Fq 'ls-tree -rz -r' "$adapter"; then + fail preparse-resource-bounds +fi +pass 'input, config, and tree bounds precede copying or recursive parsing' + printf 'local Git materializer: %s focused checks passed\n' "$passed" From 3885f5b273dd300e10ec1ffff94e82138fc82a2a Mon Sep 17 00:00:00 2001 From: ci Date: Thu, 3 Sep 2026 20:54:07 -0400 Subject: [PATCH 12/17] Support directory file transitions --- .../local-git-materializer/v1/materialize.sh | 12 +++-- .../local-git-materializer-adapter.test.sh | 49 +++++++++++++++++++ 2 files changed, 58 insertions(+), 3 deletions(-) diff --git a/adapters/local-git-materializer/v1/materialize.sh b/adapters/local-git-materializer/v1/materialize.sh index e59f7a2..a24b801 100755 --- a/adapters/local-git-materializer/v1/materialize.sh +++ b/adapters/local-git-materializer/v1/materialize.sh @@ -476,10 +476,16 @@ while IFS= read -r patch_path; do source_path_bytes=0 if source_path_object=$(git_dir "$source_git_dir" rev-parse --verify \ "$source_tree:$patch_path" 2>/dev/null); then - [ "$(git_dir "$source_git_dir" cat-file -t "$source_path_object" 2>/dev/null)" = blob ] || - emit_error E_PATCH_PATH - source_path_bytes=$(git_dir "$source_git_dir" cat-file -s \ + source_path_type=$(git_dir "$source_git_dir" cat-file -t \ "$source_path_object" 2>/dev/null) || emit_error E_SOURCE_GIT + case "$source_path_type" in + blob) + source_path_bytes=$(git_dir "$source_git_dir" cat-file -s \ + "$source_path_object" 2>/dev/null) || emit_error E_SOURCE_GIT + ;; + tree) source_path_bytes=0 ;; + *) emit_error E_PATCH_PATH ;; + esac fi case "$source_path_bytes" in ''|*[!0-9]*) emit_error E_SOURCE_GIT ;; esac [ "${#source_path_bytes}" -le 9 ] && diff --git a/scripts/test/local-git-materializer-adapter.test.sh b/scripts/test/local-git-materializer-adapter.test.sh index 3b79532..d321cbf 100755 --- a/scripts/test/local-git-materializer-adapter.test.sh +++ b/scripts/test/local-git-materializer-adapter.test.sh @@ -610,6 +610,55 @@ copy_input="$tmp/copy-input.json" input_with_patch "$copy_contract_input" "$copy_patch" "$copy_input" expect_error copy-metadata E_PATCH "$copy_input" +directory_file_source="$tmp/source-directory-file.git" +/bin/mkdir -m 700 "$directory_file_source" +git_clean init -q --bare --object-format=sha1 "$directory_file_source" +directory_file_blob=$(printf '%s\n' value | + git_clean --git-dir="$directory_file_source" hash-object -w --stdin) +directory_file_child=$(printf '100644 blob %s\tfile\n' "$directory_file_blob" | + git_clean --git-dir="$directory_file_source" mktree) +directory_file_tree=$(printf '040000 tree %s\tdir\n' "$directory_file_child" | + git_clean --git-dir="$directory_file_source" mktree) +directory_file_commit=$(printf '%s\n' directory-file | + /usr/bin/env -i HOME="$tmp/home" TMPDIR="$tmp" PATH=/usr/bin:/bin LC_ALL=C \ + GIT_CONFIG_NOSYSTEM=1 GIT_CONFIG_GLOBAL=/dev/null GIT_NO_REPLACE_OBJECTS=1 \ + GIT_AUTHOR_NAME=fixture GIT_AUTHOR_EMAIL=fixture@example.invalid \ + GIT_COMMITTER_NAME=fixture GIT_COMMITTER_EMAIL=fixture@example.invalid \ + GIT_AUTHOR_DATE=2000-01-01T00:00:00Z GIT_COMMITTER_DATE=2000-01-01T00:00:00Z \ + /usr/bin/git --no-replace-objects --git-dir="$directory_file_source" \ + commit-tree "$directory_file_tree") +git_clean --git-dir="$directory_file_source" update-ref refs/heads/main "$directory_file_commit" +directory_file_contract="$tmp/directory-file-contract.json" +"${jq_cmd[@]}" -S -c \ + '.allowed_paths=["dir","dir/file"] | .max_changed_paths=2' \ + "$contract_file" > "$directory_file_contract" +directory_file_patch="$tmp/directory-file.patch" +printf '%s\n' 'diff --git a/dir/file b/dir/file' 'deleted file mode 100644' \ + '--- a/dir/file' '+++ /dev/null' '@@ -1 +0,0 @@' '-value' \ + 'diff --git a/dir b/dir' 'new file mode 100644' '--- /dev/null' '+++ b/dir' \ + '@@ -0,0 +1 @@' '+replacement' > "$directory_file_patch" +directory_file_source_input="$tmp/directory-file-source-input.json" +input_for_source "$input_file" "$directory_file_source_input" sha1 \ + "$directory_file_commit" "$directory_file_tree" +directory_file_contract_input="$tmp/directory-file-contract-input.json" +input_with_contract "$directory_file_source_input" "$directory_file_contract" \ + "$directory_file_contract_input" +directory_file_input="$tmp/directory-file-input.json" +input_with_patch "$directory_file_contract_input" "$directory_file_patch" \ + "$directory_file_input" +directory_file_root=$(run_case directory-file "$directory_file_input" "$directory_file_source") +[ ! -s "$directory_file_root/err" ] || fail directory-file-stderr +directory_file_candidate="$directory_file_root/candidate/repository.git" +directory_file_candidate_commit=$(git_clean --git-dir="$directory_file_candidate" \ + rev-parse refs/heads/candidate) +if [ "$(git_clean --git-dir="$directory_file_candidate" show \ + "$directory_file_candidate_commit:dir")" != replacement ] || + git_clean --git-dir="$directory_file_candidate" cat-file -e \ + "$directory_file_candidate_commit:dir/file" 2>/dev/null; then + fail directory-file-result +fi +pass 'directory-to-file transition preserves preflight accounting and applies cleanly' + /usr/bin/printf '%s\n' '/invalid/alternate' > "$tmp/source.git/objects/info/alternates" expect_error alternates E_SOURCE_GIT /bin/rm "$tmp/source.git/objects/info/alternates" From 619ae263c55361c290d8a64aa6d5e60c272b0f2b Mon Sep 17 00:00:00 2001 From: ci Date: Fri, 4 Sep 2026 07:15:19 -0400 Subject: [PATCH 13/17] Bound materializer object closure --- README.md | 4 +- RESTORE.md | 5 +- .../local-git-materializer/v1/materialize.sh | 54 ++-- .../v1/object-closure.c | 255 ++++++++++++++++++ ci/required-files.txt | 1 + .../local-git-materializer-adapter.test.sh | 39 ++- 6 files changed, 325 insertions(+), 33 deletions(-) create mode 100644 adapters/local-git-materializer/v1/object-closure.c diff --git a/README.md b/README.md index 26cf748..8eb4543 100644 --- a/README.md +++ b/README.md @@ -287,7 +287,9 @@ cumulative source blob sizes plus patch bytes must fit a 256 MiB candidate budge The fixed `materialize` command accepts only caller-named physical source, candidate, and scratch boundaries, with every path independently absolute. It -rejects worktrees, alternates, shallow or +also receives the execution boundary's compiled `ystack-object-closure-v1` helper, +whose source is private to this adapter package. +It rejects worktrees, alternates, shallow or partial repositories, replace or graft state, active hooks and filters, remote configuration, unsafe paths, binary or copy/rename patches, empty subtrees, symlinks, and submodules. It never diff --git a/RESTORE.md b/RESTORE.md index 0e6183f..b835bb6 100644 --- a/RESTORE.md +++ b/RESTORE.md @@ -508,7 +508,7 @@ default-set wiring. The pure jq payload is offline and unqualified. It does not call GitHub, use a credential, rerun, cancel, or dispatch work, change a repository, grant authority or qualification, or activate a profile. -Restore the three paths in the manifest's inactive local Git materializer block, +Restore the four paths in the manifest's inactive local Git materializer block, then run: ```sh @@ -531,6 +531,9 @@ are capped at 1,024 tree objects, with each object size-checked before non-recur expansion. A shared-large-blob fixture proves the 256 MiB pre-apply candidate budget blocks path fan-out before Git writes changed blobs. +The test compiles the private object-closure helper with strict warnings. It proves +that an oversized historical tree and an oversized packed-refs file fail before +recursive traversal or parsing. This payload has no adapter manifest. A later assembly PR may bind the directory tree through the payload's durable main commit. It is an inactive, local-only diff --git a/adapters/local-git-materializer/v1/materialize.sh b/adapters/local-git-materializer/v1/materialize.sh index a24b801..59a2620 100755 --- a/adapters/local-git-materializer/v1/materialize.sh +++ b/adapters/local-git-materializer/v1/materialize.sh @@ -19,12 +19,12 @@ emit_error() { exit 1 } -[ "$#" -eq 6 ] || emit_error E_USAGE +[ "$#" -eq 7 ] || emit_error E_USAGE script_path=${BASH_SOURCE[0]} case "$script_path" in /*) ;; *) emit_error E_USAGE ;; esac if [ "$1" = materialize ]; then exec /usr/bin/env -i PATH="${PATH:-/usr/bin:/bin}" LC_ALL=C \ - /bin/bash "$script_path" __materialize_clean "$2" "$3" "$4" "$5" "$6" + /bin/bash "$script_path" __materialize_clean "$2" "$3" "$4" "$5" "$6" "$7" fi [ "$1" = __materialize_clean ] || emit_error E_USAGE export LC_ALL=C @@ -34,11 +34,16 @@ source_repository_id=$3 source_git_dir=$4 candidate_root=$5 scratch_root=$6 +closure_helper=$7 for absolute_path in "$script_path" "$input_path" "$source_git_dir" \ - "$candidate_root" "$scratch_root"; do + "$candidate_root" "$scratch_root" "$closure_helper"; do case "$absolute_path" in /*) ;; *) emit_error E_USAGE ;; esac done +[ -f "$closure_helper" ] && [ -x "$closure_helper" ] && [ ! -L "$closure_helper" ] || + emit_error E_DEPENDENCY +[ "$("$closure_helper" version 2>/dev/null)" = ystack-object-closure-v1 ] || + emit_error E_DEPENDENCY [ -f "$script_path" ] && [ ! -L "$script_path" ] || emit_error E_PACKAGE script_dir=$(CDPATH='' cd -P -- "${script_path%/*}" && pwd -P) || emit_error E_PACKAGE repo_root=$(CDPATH='' cd -P -- "$script_dir/../../.." && pwd -P) || emit_error E_PACKAGE @@ -311,8 +316,21 @@ done < "$source_config" [ ! -d "$source_git_dir/refs/replace" ] && [ -z "$(find "$source_git_dir/objects/pack" -type f -name '*.promisor' -print -quit 2>/dev/null)" ] || emit_error E_SOURCE_GIT -[ -z "$(git_dir "$source_git_dir" for-each-ref --format='%(refname)' \ - refs/replace/ 2>/dev/null)" ] || emit_error E_SOURCE_GIT +packed_refs="$source_git_dir/packed-refs" +if [ -e "$packed_refs" ]; then + [ -f "$packed_refs" ] && [ ! -L "$packed_refs" ] || emit_error E_SOURCE_GIT + packed_refs_snapshot="$run_root/packed-refs" + if ! /usr/bin/head -c 1048577 "$packed_refs" > "$packed_refs_snapshot"; then + emit_error E_SOURCE_LIMIT + fi + packed_refs_bytes=$(/usr/bin/wc -c < "$packed_refs_snapshot" | /usr/bin/tr -d ' ') || + emit_error E_SOURCE_LIMIT + [ "$packed_refs_bytes" -le 1048576 ] || emit_error E_SOURCE_LIMIT + if /usr/bin/grep -aEq '^[0-9a-f]{40} refs/replace/|^[0-9a-f]{64} refs/replace/' \ + "$packed_refs_snapshot"; then + emit_error E_SOURCE_GIT + fi +fi if find "$source_git_dir/hooks" -type f ! -name '*.sample' -print -quit 2>/dev/null | /usr/bin/grep -q .; then emit_error E_SOURCE_HOOK @@ -412,32 +430,10 @@ scan_tree "$source_git_dir" "$source_tree" "$source_paths" || emit_error E_SOURC empty_template="$run_root/empty-template" source_import_byte_limit=268435456 -source_import_count_limit=65536 source_objects="$run_root/source.objects" -if ! git_dir "$source_git_dir" rev-list --objects --no-object-names "$source_commit" | - /usr/bin/awk -v limit="$source_import_count_limit" \ - 'NR > limit { exit 2 } { print }' > "$source_objects"; then +"${git_env[@]}" "$closure_helper" walk "$source_git_dir" \ + "$source_algorithm" "$source_commit" "$source_objects" 2>/dev/null || emit_error E_SOURCE_LIMIT -fi -source_object_sizes="$run_root/source-object-sizes" -git_dir "$source_git_dir" cat-file \ - --batch-check='%(objectname) %(objecttype) %(objectsize)' \ - < "$source_objects" > "$source_object_sizes" || emit_error E_SOURCE_LIMIT -source_object_bytes=0 -while IFS=' ' read -r object_id object_type object_size extra; do - [ -z "$extra" ] || emit_error E_SOURCE_LIMIT - case "$source_algorithm" in - sha1) [[ "$object_id" =~ ^[0-9a-f]{40}$ ]] || emit_error E_SOURCE_LIMIT ;; - sha256) [[ "$object_id" =~ ^[0-9a-f]{64}$ ]] || emit_error E_SOURCE_LIMIT ;; - *) emit_error E_SOURCE_LIMIT ;; - esac - case "$object_type" in blob|tree|commit) ;; *) emit_error E_SOURCE_LIMIT ;; esac - case "$object_size" in ''|*[!0-9]*) emit_error E_SOURCE_LIMIT ;; esac - [ "${#object_size}" -le 9 ] && [ "$object_size" -le "$source_import_byte_limit" ] && - [ "$source_object_bytes" -le "$((source_import_byte_limit - object_size))" ] || - emit_error E_SOURCE_LIMIT - source_object_bytes=$((source_object_bytes + object_size)) -done < "$source_object_sizes" max_changed=$("$jq_bin" -r '.max_changed_paths' "$contract_file") || emit_error E_CONTRACT patch_paths="$run_root/patch-paths" diff --git a/adapters/local-git-materializer/v1/object-closure.c b/adapters/local-git-materializer/v1/object-closure.c new file mode 100644 index 0000000..6f3e5c5 --- /dev/null +++ b/adapters/local-git-materializer/v1/object-closure.c @@ -0,0 +1,255 @@ +#define _POSIX_C_SOURCE 200809L + +/* Private exception boundary: https://github.com/yihanzhu/ystack/pull/229#issuecomment-5539240046 */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifndef O_CLOEXEC +#define O_CLOEXEC 0 +#endif +#ifndef O_NOFOLLOW +#define O_NOFOLLOW 0 +#endif + +#define OBJECT_MAX 65536U +#define BYTE_MAX 268435456ULL +#define COMMIT_MAX 1048576ULL +#define TREE_MAX 16777216ULL +#define TABLE_SIZE 131071U + +struct item { char oid[65]; }; +struct set_slot { char oid[65]; int used; }; + +static void fail(const char *code) +{ + (void)fprintf(stderr, "%s\n", code); + exit(1); +} + +static void limit_child(void) +{ + struct rlimit cpu = { 120U, 120U }; + if (setrlimit(RLIMIT_CPU, &cpu) != 0) _exit(126); +#if !defined(__APPLE__) + struct rlimit memory = { 536870912U, 536870912U }; + if (setrlimit(RLIMIT_DATA, &memory) != 0) _exit(126); +#if defined(RLIMIT_AS) + if (setrlimit(RLIMIT_AS, &memory) != 0) _exit(126); +#endif +#endif +} + +static uint64_t hash_oid(const char *value) +{ + uint64_t hash = 1469598103934665603ULL; + while (*value != '\0') { + hash ^= (unsigned char)*value++; + hash *= 1099511628211ULL; + } + return hash; +} + +static int valid_oid(const char *value, size_t length) +{ + size_t index; + if (strlen(value) != length) return 0; + for (index = 0; index < length; index++) { + if (!((value[index] >= '0' && value[index] <= '9') || + (value[index] >= 'a' && value[index] <= 'f'))) return 0; + } + return 1; +} + +static int remember(struct set_slot *set, const char *oid) +{ + size_t slot = (size_t)(hash_oid(oid) % TABLE_SIZE); + size_t start = slot; + do { + if (!set[slot].used) { + set[slot].used = 1; + (void)strcpy(set[slot].oid, oid); + return 1; + } + if (strcmp(set[slot].oid, oid) == 0) return 0; + slot = (slot + 1U) % TABLE_SIZE; + } while (slot != start); + fail("E_SOURCE_LIMIT"); + return 0; +} + +static void enqueue(struct item *queue, size_t *length, struct set_slot *set, + const char *oid, size_t oid_length) +{ + if (!valid_oid(oid, oid_length)) fail("E_SOURCE_GIT"); + if (!remember(set, oid)) return; + if (*length >= OBJECT_MAX) fail("E_SOURCE_LIMIT"); + (void)strcpy(queue[*length].oid, oid); + *length += 1U; +} + +static void command(FILE *input, const char *name, const char *oid) +{ + if (fprintf(input, "%s %s\nflush\n", name, oid) < 0 || fflush(input) != 0) + fail("E_SOURCE_GIT"); +} + +static void read_header(FILE *output, const char *expected, char *type, + unsigned long long *size) +{ + char *line = NULL; + size_t capacity = 0U; + char oid[65]; + char extra; + ssize_t length = getline(&line, &capacity, output); + if (length <= 0 || (size_t)length > 160U || + sscanf(line, "%64s %15s %llu %c", oid, type, size, &extra) != 3 || + strcmp(oid, expected) != 0) { + free(line); + fail("E_SOURCE_GIT"); + } + free(line); +} + +static unsigned char *read_contents(FILE *input, FILE *output, const char *oid, + const char *expected_type, + unsigned long long expected_size) +{ + char type[16]; + unsigned long long size; + unsigned char *body; + command(input, "contents", oid); + read_header(output, oid, type, &size); + if (strcmp(type, expected_type) != 0 || size != expected_size || size > SIZE_MAX) + fail("E_SOURCE_GIT"); + body = malloc((size_t)size + 1U); + if (body == NULL) fail("E_SOURCE_LIMIT"); + if (fread(body, 1U, (size_t)size, output) != (size_t)size || fgetc(output) != '\n') { + free(body); + fail("E_SOURCE_GIT"); + } + body[size] = '\0'; + return body; +} + +static void parse_commit(unsigned char *body, size_t size, struct item *queue, + size_t *length, struct set_slot *set, size_t oid_length) +{ + char *cursor = (char *)body; + char *end = (char *)body + size; + int tree_seen = 0; + while (cursor < end) { + char *newline = memchr(cursor, '\n', (size_t)(end - cursor)); + size_t line_length; + if (newline == NULL) fail("E_SOURCE_GIT"); + line_length = (size_t)(newline - cursor); + if (line_length == 0U) break; + if (line_length == oid_length + 5U && memcmp(cursor, "tree ", 5U) == 0) { + char oid[65]; + if (tree_seen) fail("E_SOURCE_GIT"); + (void)memcpy(oid, cursor + 5, oid_length); oid[oid_length] = '\0'; + enqueue(queue, length, set, oid, oid_length); tree_seen = 1; + } else if (line_length == oid_length + 7U && memcmp(cursor, "parent ", 7U) == 0) { + char oid[65]; + (void)memcpy(oid, cursor + 7, oid_length); oid[oid_length] = '\0'; + enqueue(queue, length, set, oid, oid_length); + } + cursor = newline + 1; + } + if (!tree_seen) fail("E_SOURCE_GIT"); +} + +static void parse_tree(unsigned char *body, size_t size, struct item *queue, + size_t *length, struct set_slot *set, size_t oid_length) +{ + size_t cursor = 0U; + while (cursor < size) { + size_t mode_start = cursor; + size_t mode_length; + size_t name_start; + char oid[65]; + while (cursor < size && body[cursor] != ' ') cursor++; + if (cursor == size || cursor == mode_start) fail("E_SOURCE_GIT"); + mode_length = cursor - mode_start; + name_start = ++cursor; + while (cursor < size && body[cursor] != '\0') cursor++; + if (cursor == size || cursor == name_start) fail("E_SOURCE_GIT"); + cursor++; + if (size - cursor < oid_length) fail("E_SOURCE_GIT"); + if (!(mode_length == 6U && memcmp(body + mode_start, "160000", 6U) == 0)) { + size_t index; + for (index = 0; index < oid_length; index++) + (void)sprintf(oid + index * 2U, "%02x", body[cursor + index]); + oid[oid_length * 2U] = '\0'; + enqueue(queue, length, set, oid, oid_length * 2U); + } + cursor += oid_length; + } +} + +int main(int argc, char **argv) +{ + int to_child[2], from_child[2], output_fd; + pid_t child; + FILE *input, *output; + struct item *queue; + struct set_slot *set; + size_t length = 0U, cursor = 0U, oid_text_length, oid_raw_length; + unsigned long long total = 0U; + int status; + if (argc == 2 && strcmp(argv[1], "version") == 0) { + (void)puts("ystack-object-closure-v1"); + return 0; + } + if (argc != 6 || strcmp(argv[1], "walk") != 0) fail("E_USAGE"); + if (strcmp(argv[3], "sha1") == 0) { oid_text_length = 40U; oid_raw_length = 20U; } + else if (strcmp(argv[3], "sha256") == 0) { oid_text_length = 64U; oid_raw_length = 32U; } + else fail("E_USAGE"); + queue = calloc(OBJECT_MAX, sizeof(*queue)); set = calloc(TABLE_SIZE, sizeof(*set)); + if (queue == NULL || set == NULL) fail("E_SOURCE_LIMIT"); + output_fd = open(argv[5], O_WRONLY | O_CREAT | O_EXCL | O_CLOEXEC | O_NOFOLLOW, 0600); + if (output_fd < 0 || pipe(to_child) != 0 || pipe(from_child) != 0) fail("E_SOURCE_GIT"); + child = fork(); + if (child < 0) fail("E_SOURCE_GIT"); + if (child == 0) { + limit_child(); + (void)dup2(to_child[0], STDIN_FILENO); (void)dup2(from_child[1], STDOUT_FILENO); + close(to_child[1]); close(from_child[0]); + execl("/usr/bin/git", "/usr/bin/git", "--no-replace-objects", "--git-dir", argv[2], + "cat-file", "--batch-command", "--buffer", (char *)NULL); + _exit(127); + } + close(to_child[0]); close(from_child[1]); + input = fdopen(to_child[1], "w"); output = fdopen(from_child[0], "r"); + if (input == NULL || output == NULL) fail("E_SOURCE_GIT"); + enqueue(queue, &length, set, argv[4], oid_text_length); + while (cursor < length) { + char type[16]; unsigned long long size; unsigned char *body = NULL; + const char *oid = queue[cursor++].oid; + command(input, "info", oid); read_header(output, oid, type, &size); + if (size > BYTE_MAX - total) fail("E_SOURCE_LIMIT"); total += size; + if (strcmp(type, "commit") == 0) { + if (size > COMMIT_MAX) fail("E_SOURCE_LIMIT"); + body = read_contents(input, output, oid, type, size); + parse_commit(body, (size_t)size, queue, &length, set, oid_text_length); + } else if (strcmp(type, "tree") == 0) { + if (size > TREE_MAX) fail("E_SOURCE_LIMIT"); + body = read_contents(input, output, oid, type, size); + parse_tree(body, (size_t)size, queue, &length, set, oid_raw_length); + } else if (strcmp(type, "blob") != 0) fail("E_SOURCE_GIT"); + free(body); + if (dprintf(output_fd, "%s\n", oid) < 0) fail("E_SOURCE_GIT"); + } + (void)fclose(input); (void)fclose(output); (void)close(output_fd); + if (waitpid(child, &status, 0) < 0 || !WIFEXITED(status) || WEXITSTATUS(status) != 0) + fail("E_SOURCE_GIT"); + free(queue); free(set); return 0; +} diff --git a/ci/required-files.txt b/ci/required-files.txt index 8a54f89..c7dac9b 100644 --- a/ci/required-files.txt +++ b/ci/required-files.txt @@ -254,6 +254,7 @@ scripts/test/default-github-actions-ci-adapter.test.sh # Inactive provider-neutral local Git candidate materializer adapters/local-git-materializer/v1/materialize.sh +adapters/local-git-materializer/v1/object-closure.c scripts/test/local-git-materializer-adapter.test.sh # Inactive Claude Code producer normalizer payload diff --git a/scripts/test/local-git-materializer-adapter.test.sh b/scripts/test/local-git-materializer-adapter.test.sh index d321cbf..be83b93 100755 --- a/scripts/test/local-git-materializer-adapter.test.sh +++ b/scripts/test/local-git-materializer-adapter.test.sh @@ -6,6 +6,7 @@ umask 077 root=$(CDPATH='' cd -P -- "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd -P) adapter="$root/adapters/local-git-materializer/v1/materialize.sh" +closure_source="$root/adapters/local-git-materializer/v1/object-closure.c" protocol="$root/adapters/local-git-materializer/v1/protocol.jq" test_tmp_base=${TMPDIR:-/tmp} tmp=$(/usr/bin/mktemp -d "${test_tmp_base%/}/ystack-local-materializer.XXXXXX") @@ -36,6 +37,10 @@ else /bin/cp "$jq_bin" "$runtime_bin/jq" fi /bin/chmod 0555 "$runtime_bin/jq" +/usr/bin/cc -std=c11 -Wall -Wextra -Werror -O2 "$closure_source" \ + -o "$runtime_bin/object-closure" +/bin/chmod 0555 "$runtime_bin/object-closure" +closure_helper="$runtime_bin/object-closure" export PATH="$runtime_bin:/usr/bin:/bin" generation=$(/usr/bin/sed -n \ "s/^PORTABLE_CORE_GENERATION='\(g-[0-9a-f]\{64\}\)'$/\1/p" \ @@ -264,6 +269,7 @@ run_case() { PATH="$runtime_bin:/usr/bin:/bin" GH_TOKEN=must-not-read GITHUB_TOKEN=must-not-read \ AWS_SECRET_ACCESS_KEY=must-not-read SSH_AUTH_SOCK=/must/not/read \ "$adapter" materialize "$input" fixture.target "$source" "$candidate" "$scratch" \ + "$closure_helper" \ > "$case_root/out" 2> "$case_root/err" printf '%s\n' "$case_root" } @@ -344,7 +350,8 @@ expect_error() { local candidate="$case_root/candidate" scratch="$case_root/scratch" /bin/mkdir -m 700 "$case_root" "$candidate" "$scratch" if PATH="$runtime_bin:/usr/bin:/bin" "$adapter" materialize "$input" fixture.target \ - "$source" "$candidate" "$scratch" > "$case_root/out" 2> "$case_root/err"; then + "$source" "$candidate" "$scratch" "$closure_helper" \ + > "$case_root/out" 2> "$case_root/err"; then fail "$name accepted" fi [ ! -s "$case_root/out" ] && [ "$(cat "$case_root/err")" = "$expected" ] || fail "$name error" @@ -363,7 +370,7 @@ if ( export -f find PATH="$runtime_bin:/usr/bin:/bin" GH_TOKEN=must-not-read BASH_ENV="$direct_case/bash-env" \ "$adapter" __materialize_clean "$input_file" fixture.target "$tmp/source.git" \ - "$direct_case/candidate" "$direct_case/scratch" \ + "$direct_case/candidate" "$direct_case/scratch" "$closure_helper" \ > "$direct_case/out" 2> "$direct_case/err" ); then fail direct-clean-worker-accepted @@ -380,6 +387,7 @@ if ( cd "$tmp" PATH="$runtime_bin:/usr/bin:/bin" "$adapter" materialize 'relative:/input.json' \ fixture.target "$tmp/source.git" "$relative_case/candidate" "$relative_case/scratch" \ + "$closure_helper" \ > "$relative_case/out" 2> "$relative_case/err" ); then fail relative-input-path-accepted @@ -557,6 +565,23 @@ input_for_source "$input_file" "$large_listing_input" sha1 \ expect_error source-tree-byte-limit E_SOURCE_TREE \ "$large_listing_input" "$large_listing_source" +historical_blob=$(printf '%s\n' alpha beta | + git_clean --git-dir="$large_listing_source" hash-object -w --stdin) +historical_tip_tree=$(printf '100644 blob %s\tsource.txt\n' "$historical_blob" | + git_clean --git-dir="$large_listing_source" mktree) +historical_tip=$(printf '%s\n' historical-tip | + /usr/bin/env -i HOME="$tmp/home" TMPDIR="$tmp" PATH=/usr/bin:/bin LC_ALL=C \ + GIT_CONFIG_NOSYSTEM=1 GIT_CONFIG_GLOBAL=/dev/null GIT_NO_REPLACE_OBJECTS=1 \ + GIT_AUTHOR_NAME=fixture GIT_AUTHOR_EMAIL=fixture@example.invalid \ + GIT_COMMITTER_NAME=fixture GIT_COMMITTER_EMAIL=fixture@example.invalid \ + GIT_AUTHOR_DATE=2000-01-01T00:00:00Z GIT_COMMITTER_DATE=2000-01-01T00:00:00Z \ + /usr/bin/git --no-replace-objects --git-dir="$large_listing_source" \ + commit-tree "$historical_tip_tree" -p "$large_listing_commit") +git_clean --git-dir="$large_listing_source" update-ref refs/heads/main "$historical_tip" +historical_input="$tmp/historical-input.json" +input_for_source "$input_file" "$historical_input" sha1 "$historical_tip" "$historical_tip_tree" +expect_error oversized-historical-tree E_SOURCE_LIMIT "$historical_input" "$large_listing_source" + shared_source="$tmp/source-shared-large-blob.git" /bin/mkdir -m 700 "$shared_source" git_clean init -q --bare --object-format=sha1 "$shared_source" @@ -692,6 +717,12 @@ git_clean --git-dir="$packed_replace_source" pack-refs --all --prune [ ! -d "$packed_replace_source/refs/replace" ] || fail packed-replace-fixture expect_error packed-replace-ref E_SOURCE_GIT "$input_file" "$packed_replace_source" +large_packed_refs_source="$tmp/source-large-packed-refs.git" +/bin/cp -R "$tmp/source.git" "$large_packed_refs_source" +/usr/bin/awk 'BEGIN { for (i=0; i<524289; i++) print "#" }' \ + > "$large_packed_refs_source/packed-refs" +expect_error packed-refs-limit E_SOURCE_LIMIT "$input_file" "$large_packed_refs_source" + normal_repo="$tmp/normal" /bin/mkdir -m 700 "$normal_repo" git_clean init -q "$normal_repo" @@ -969,6 +1000,7 @@ case_nonempty="$tmp/nonempty" /usr/bin/touch "$case_nonempty/candidate/existing" if PATH="$runtime_bin:/usr/bin:/bin" "$adapter" materialize "$input_file" fixture.target \ "$tmp/source.git" "$case_nonempty/candidate" "$case_nonempty/scratch" \ + "$closure_helper" \ > "$case_nonempty/out" 2> "$case_nonempty/err"; then fail nonempty-candidate; fi [ "$(cat "$case_nonempty/err")" = E_CANDIDATE_ROOT ] || fail nonempty-candidate-error pass 'non-empty candidate root rejected' @@ -978,6 +1010,7 @@ overlap_scratch="$tmp/overlap-scratch" /bin/mkdir -m 700 "$overlap_candidate" "$overlap_scratch" if PATH="$runtime_bin:/usr/bin:/bin" "$adapter" materialize "$input_file" fixture.target \ "$tmp/source.git" "$overlap_candidate" "$overlap_scratch" \ + "$closure_helper" \ > "$tmp/overlap.out" 2> "$tmp/overlap.err"; then fail overlapping-boundary; fi [ "$(cat "$tmp/overlap.err")" = E_BOUNDARY ] || fail overlapping-boundary-error /bin/rmdir "$overlap_candidate" @@ -987,6 +1020,7 @@ closed_output="$tmp/closed-output" /bin/mkdir -m 700 "$closed_output" "$closed_output/candidate" "$closed_output/scratch" if PATH="$runtime_bin:/usr/bin:/bin" "$adapter" materialize "$input_file" fixture.target \ "$tmp/source.git" "$closed_output/candidate" "$closed_output/scratch" \ + "$closure_helper" \ >&- 2> "$closed_output/err"; then fail closed-output-accepted fi @@ -1012,6 +1046,7 @@ boundary_scratch="$tmp/boundary-scratch" /bin/mkdir -m 700 "$boundary_scratch" if PATH="$runtime_bin:/usr/bin:/bin" "$adapter" materialize "$input_file" fixture.target \ "$tmp/source.git" "$symlink_boundary" "$boundary_scratch" \ + "$closure_helper" \ > "$tmp/boundary.out" 2> "$tmp/boundary.err"; then fail candidate-symlink; fi [ "$(cat "$tmp/boundary.err")" = E_CANDIDATE_ROOT ] || fail candidate-symlink-error pass 'symlink candidate boundary rejected' From 2df4dd2d2f9bd71d0b1e815a93c2e03ed3967346 Mon Sep 17 00:00:00 2001 From: ci Date: Fri, 4 Sep 2026 07:44:39 -0400 Subject: [PATCH 14/17] Fix GCC indentation diagnostic --- adapters/local-git-materializer/v1/object-closure.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/adapters/local-git-materializer/v1/object-closure.c b/adapters/local-git-materializer/v1/object-closure.c index 6f3e5c5..c4ab722 100644 --- a/adapters/local-git-materializer/v1/object-closure.c +++ b/adapters/local-git-materializer/v1/object-closure.c @@ -235,7 +235,8 @@ int main(int argc, char **argv) char type[16]; unsigned long long size; unsigned char *body = NULL; const char *oid = queue[cursor++].oid; command(input, "info", oid); read_header(output, oid, type, &size); - if (size > BYTE_MAX - total) fail("E_SOURCE_LIMIT"); total += size; + if (size > BYTE_MAX - total) fail("E_SOURCE_LIMIT"); + total += size; if (strcmp(type, "commit") == 0) { if (size > COMMIT_MAX) fail("E_SOURCE_LIMIT"); body = read_contents(input, output, oid, type, size); From 8459a0a9629fb509203c31810fbce21275ea9c90 Mon Sep 17 00:00:00 2001 From: ci Date: Fri, 4 Sep 2026 08:33:22 -0400 Subject: [PATCH 15/17] Reject uppercase packed replacement refs --- adapters/local-git-materializer/v1/materialize.sh | 2 +- scripts/test/local-git-materializer-adapter.test.sh | 12 ++++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/adapters/local-git-materializer/v1/materialize.sh b/adapters/local-git-materializer/v1/materialize.sh index 59a2620..a571011 100755 --- a/adapters/local-git-materializer/v1/materialize.sh +++ b/adapters/local-git-materializer/v1/materialize.sh @@ -326,7 +326,7 @@ if [ -e "$packed_refs" ]; then packed_refs_bytes=$(/usr/bin/wc -c < "$packed_refs_snapshot" | /usr/bin/tr -d ' ') || emit_error E_SOURCE_LIMIT [ "$packed_refs_bytes" -le 1048576 ] || emit_error E_SOURCE_LIMIT - if /usr/bin/grep -aEq '^[0-9a-f]{40} refs/replace/|^[0-9a-f]{64} refs/replace/' \ + if /usr/bin/grep -aEq '^[0-9A-Fa-f]{40} refs/replace/|^[0-9A-Fa-f]{64} refs/replace/' \ "$packed_refs_snapshot"; then emit_error E_SOURCE_GIT fi diff --git a/scripts/test/local-git-materializer-adapter.test.sh b/scripts/test/local-git-materializer-adapter.test.sh index be83b93..3d88c5c 100755 --- a/scripts/test/local-git-materializer-adapter.test.sh +++ b/scripts/test/local-git-materializer-adapter.test.sh @@ -717,6 +717,18 @@ git_clean --git-dir="$packed_replace_source" pack-refs --all --prune [ ! -d "$packed_replace_source/refs/replace" ] || fail packed-replace-fixture expect_error packed-replace-ref E_SOURCE_GIT "$input_file" "$packed_replace_source" +uppercase_packed_replace_source="$tmp/source-uppercase-packed-replace.git" +/bin/cp -R "$packed_replace_source" "$uppercase_packed_replace_source" +/usr/bin/awk '/^[0-9a-f]+ / {$1=toupper($1)} {print}' \ + "$uppercase_packed_replace_source/packed-refs" \ + > "$uppercase_packed_replace_source/packed-refs.upper" +/bin/mv "$uppercase_packed_replace_source/packed-refs.upper" \ + "$uppercase_packed_replace_source/packed-refs" +git_clean --git-dir="$uppercase_packed_replace_source" show-ref --verify \ + "refs/replace/$source_tree" >/dev/null || fail uppercase-packed-replace-fixture +expect_error uppercase-packed-replace-ref E_SOURCE_GIT "$input_file" \ + "$uppercase_packed_replace_source" + large_packed_refs_source="$tmp/source-large-packed-refs.git" /bin/cp -R "$tmp/source.git" "$large_packed_refs_source" /usr/bin/awk 'BEGIN { for (i=0; i<524289; i++) print "#" }' \ From 6b6664de296132e86e69eae95bc3129c85517f8e Mon Sep 17 00:00:00 2001 From: ci Date: Fri, 4 Sep 2026 09:23:03 -0400 Subject: [PATCH 16/17] Preserve inherited resource ceilings --- .../local-git-materializer/v1/object-closure.c | 17 ++++++++++++----- .../test/local-git-materializer-adapter.test.sh | 10 ++++++++++ 2 files changed, 22 insertions(+), 5 deletions(-) diff --git a/adapters/local-git-materializer/v1/object-closure.c b/adapters/local-git-materializer/v1/object-closure.c index c4ab722..00cb3c6 100644 --- a/adapters/local-git-materializer/v1/object-closure.c +++ b/adapters/local-git-materializer/v1/object-closure.c @@ -35,15 +35,22 @@ static void fail(const char *code) exit(1); } +static void cap_limit(int resource, rlim_t ceiling) +{ + struct rlimit limit; + if (getrlimit(resource, &limit) != 0) _exit(126); + if (limit.rlim_cur > ceiling) limit.rlim_cur = ceiling; + if (limit.rlim_max > ceiling) limit.rlim_max = ceiling; + if (setrlimit(resource, &limit) != 0) _exit(126); +} + static void limit_child(void) { - struct rlimit cpu = { 120U, 120U }; - if (setrlimit(RLIMIT_CPU, &cpu) != 0) _exit(126); + cap_limit(RLIMIT_CPU, 120U); #if !defined(__APPLE__) - struct rlimit memory = { 536870912U, 536870912U }; - if (setrlimit(RLIMIT_DATA, &memory) != 0) _exit(126); + cap_limit(RLIMIT_DATA, 536870912U); #if defined(RLIMIT_AS) - if (setrlimit(RLIMIT_AS, &memory) != 0) _exit(126); + cap_limit(RLIMIT_AS, 536870912U); #endif #endif } diff --git a/scripts/test/local-git-materializer-adapter.test.sh b/scripts/test/local-git-materializer-adapter.test.sh index 3d88c5c..12ca676 100755 --- a/scripts/test/local-git-materializer-adapter.test.sh +++ b/scripts/test/local-git-materializer-adapter.test.sh @@ -87,6 +87,16 @@ read -r source_commit source_tree < <(make_bare_source "$tmp/source.git" sha1) source_fingerprint=$(find "$tmp/source.git" -type f -print0 | LC_ALL=C sort -z | xargs -0 /usr/bin/shasum -a 256 | /usr/bin/shasum -a 256 | /usr/bin/awk '{print $1}') +limited_closure="$tmp/limited-closure" +( + ulimit -S -t 1 + ulimit -H -t 1 + "$closure_helper" walk "$tmp/source.git" sha1 "$source_commit" "$limited_closure" +) +[ "$(/usr/bin/wc -l < "$limited_closure" | /usr/bin/tr -d ' ')" -eq 3 ] || + fail inherited-resource-ceiling +pass 'stricter inherited resource ceilings remain valid' + contract_file="$tmp/contract.json" "${jq_cmd[@]}" -S -c -n '{ schema_version:1,kind:"local_git_materialization_contract", From 7239500f87d53f957fb48e9f5198196e5f6a7df2 Mon Sep 17 00:00:00 2001 From: ci Date: Fri, 4 Sep 2026 10:09:48 -0400 Subject: [PATCH 17/17] Reject root boundary overlap --- adapters/local-git-materializer/v1/materialize.sh | 3 +++ scripts/test/local-git-materializer-adapter.test.sh | 2 ++ 2 files changed, 5 insertions(+) diff --git a/adapters/local-git-materializer/v1/materialize.sh b/adapters/local-git-materializer/v1/materialize.sh index a571011..9fc0cbe 100755 --- a/adapters/local-git-materializer/v1/materialize.sh +++ b/adapters/local-git-materializer/v1/materialize.sh @@ -102,6 +102,9 @@ empty_private_dir() { } overlaps() { + if [ "$1" = / ] || [ "$2" = / ]; then + return 0 + fi case "$1/" in "$2/"*) return 0 ;; esac case "$2/" in "$1/"*) return 0 ;; esac return 1 diff --git a/scripts/test/local-git-materializer-adapter.test.sh b/scripts/test/local-git-materializer-adapter.test.sh index 12ca676..1caea09 100755 --- a/scripts/test/local-git-materializer-adapter.test.sh +++ b/scripts/test/local-git-materializer-adapter.test.sh @@ -406,6 +406,8 @@ fi fail relative-input-path pass 'each filesystem argument must be independently absolute' +expect_error root-source-boundary E_BOUNDARY "$input_file" / + refresh_request_pair() { local source=$1 destination=$2 request_snapshot="$tmp/request-refresh" "${jq_cmd[@]}" -S -c '.stage_request.content' "$source" > "$request_snapshot"