From f4be1a97aa5e44fcf0f1390fd8007c9ea2e01601 Mon Sep 17 00:00:00 2001 From: ci Date: Tue, 1 Sep 2026 09:53:46 -0400 Subject: [PATCH 1/5] Add inactive canonical state scanner --- orchestrator/v1/scan-state.sh | 159 +++++++++ orchestrator/v1/state-scanner.jq | 202 ++++++++++++ .../test/orchestrator-state-scanner.test.sh | 307 ++++++++++++++++++ 3 files changed, 668 insertions(+) create mode 100755 orchestrator/v1/scan-state.sh create mode 100644 orchestrator/v1/state-scanner.jq create mode 100755 scripts/test/orchestrator-state-scanner.test.sh diff --git a/orchestrator/v1/scan-state.sh b/orchestrator/v1/scan-state.sh new file mode 100755 index 0000000..8a93226 --- /dev/null +++ b/orchestrator/v1/scan-state.sh @@ -0,0 +1,159 @@ +#!/bin/bash +set -uo pipefail +export LC_ALL=C +umask 077 + +emit_error() { + case "${1:-}" in + E_USAGE|E_RUNTIME|E_LIMIT|E_PARSE|E_CANONICAL|E_SHAPE|E_RELATION|E_STALE) + /usr/bin/printf '%s\n' "$1" >&2 + ;; + *) /usr/bin/printf '%s\n' E_RUNTIME >&2 ;; + esac + exit 1 +} + +sha256_path() { + /usr/bin/shasum -a 256 "$1" | /usr/bin/awk '{print $1}' +} + +sha256_line() { + /usr/bin/printf '%s\n' "$1" | /usr/bin/shasum -a 256 | + /usr/bin/awk '{print $1}' +} + +verify_hash() { + [ -f "$2" ] && [ ! -L "$2" ] && + [ "$(sha256_path "$2")" = "$1" ] +} + +verify_core() { + verify_hash bdb5def832e8e611bba8a7b30a2aae95ea4f2701c44b198cf51cd3dfd9ff88f3 \ + "$wrapper" && + verify_hash f55b697716dc13a6d2c71bde7769493b3f4b091fd7a94d3280c5d417974df3a1 \ + "$registry" && + verify_hash 8d1d02d36ac7ada778f05248f9413062b3fc251499914c15d79f003bbd009ade \ + "$modules/schema.jq" && + verify_hash c00f9cfbe88df5cb1dbcfbead61288ff7d68684d43d095e74f26e7820f0d7207 \ + "$modules/profile_graph.jq" && + verify_hash 6572a6ecbac332dc9c4a8ef35acd1feebdc2e8aab04941fc0b756f3a5cbcf29e \ + "$modules/stage_request.jq" && + verify_hash 8e49c2c091f1bbe525f7499e3fca072f6916a14d5bb34adbf121439e8ca2d281 \ + "$modules/result_facts.jq" && + verify_hash ed992f26761d08e3c3f5ab57eda9bcd771ad59e3aebeb02643de88844184d2d3 \ + "$modules/result_truth.jq" +} + +[ "$#" -eq 4 ] && [ "$1" = scan ] || emit_error E_USAGE +expected_repository_id=$2 +expected_commit_id=$3 +input=$4 +[[ "$expected_repository_id" =~ ^[a-z0-9][a-z0-9._:-]{0,127}$ ]] || + emit_error E_USAGE +[[ "$expected_commit_id" =~ ^[0-9a-f]{40}$ ]] || + [[ "$expected_commit_id" =~ ^[0-9a-f]{64}$ ]] || emit_error E_USAGE + +source_path=${BASH_SOURCE[0]} +case "$source_path" in /*) ;; *) source_path="$(pwd -P)/$source_path" ;; esac +[ -f "$source_path" ] && [ ! -L "$source_path" ] || emit_error E_RUNTIME +source_dir=$(CDPATH='' cd -P -- "${source_path%/*}" 2>/dev/null && pwd -P) || + emit_error E_RUNTIME +source_path="$source_dir/${source_path##*/}" +[ "$source_path" = "$source_dir/scan-state.sh" ] || emit_error E_RUNTIME +repo=$(CDPATH='' cd -P -- "$source_dir/../.." 2>/dev/null && pwd -P) || + emit_error E_RUNTIME +[ "$source_dir" = "$repo/orchestrator/v1" ] || emit_error E_RUNTIME + +generation=g-392d20099dfa99872764009b268c8871914b4dbc0da467ec346baa921818ae3e +modules="$repo/core/v2/generations/$generation/modules" +program="$source_dir/state-scanner.jq" +registry="$repo/core/v2/generation-registry.json" +wrapper="$repo/scripts/core-contract.sh" +for required_dir in "$repo" "$repo/orchestrator" "$source_dir" "$repo/core" \ + "$repo/core/v2" "$repo/core/v2/generations" "${modules%/modules}" "$modules"; do + [ -d "$required_dir" ] && [ ! -L "$required_dir" ] || emit_error E_RUNTIME +done +verify_hash 7210dbdf53ce07a3d7274732d22cf2f53245a1968dbd54ba7c8a6b77b1c4b831 "$program" || emit_error E_RUNTIME +verify_core || emit_error E_STALE + +[ -f "$input" ] && [ ! -L "$input" ] || emit_error E_RUNTIME +jq_bin=$(command -v jq 2>/dev/null) || emit_error E_RUNTIME +case "$jq_bin" in /*) ;; *) emit_error E_RUNTIME ;; esac +[ -f "$jq_bin" ] && [ -x "$jq_bin" ] && [ ! -L "$jq_bin" ] && + [ "$($jq_bin --version 2>/dev/null)" = jq-1.6 ] || emit_error E_RUNTIME + +scratch=$(/usr/bin/mktemp -d "${TMPDIR:-/tmp}/ystack-state-scan.XXXXXX" 2>/dev/null) || + emit_error E_RUNTIME +cleanup() { /bin/rm -rf -- "$scratch" >/dev/null 2>&1 || :; } +signal_exit() { trap - EXIT HUP INT TERM; cleanup; exit 1; } +trap cleanup EXIT +trap signal_exit HUP INT TERM + +raw="$scratch/raw.json" +/bin/dd if="$input" of="$raw" bs=1048577 count=1 2>/dev/null || emit_error E_RUNTIME +raw_size=$(/usr/bin/wc -c < "$raw" | /usr/bin/tr -d ' ') || emit_error E_RUNTIME +[ "$raw_size" -le 1048576 ] || emit_error E_LIMIT +bom=$(/usr/bin/od -An -tx1 -N3 "$raw" 2>/dev/null | /usr/bin/tr -d ' \n') || + emit_error E_RUNTIME +[ "$bom" != efbbbf ] || emit_error E_PARSE +"$jq_bin" . "$raw" >/dev/null 2>&1 || emit_error E_PARSE +[ "$("$jq_bin" -s 'length' "$raw" 2>/dev/null)" -eq 1 ] || emit_error E_PARSE +canonical="$scratch/canonical.json" +"$jq_bin" -S -c . "$raw" > "$canonical" 2>/dev/null || emit_error E_PARSE +/usr/bin/cmp -s "$raw" "$canonical" || emit_error E_CANONICAL + +"$jq_bin" -e ' + def depth: + if type == "array" then (if length == 0 then 1 else 1 + ([.[]|depth]|max) end) + elif type == "object" then (if length == 0 then 1 else 1 + ([.[]|depth]|max) end) + else 1 end; + def members: + if type == "array" then length + ([.[]|members]|add // 0) + elif type == "object" then (keys_unsorted|length) + ([.[]|members]|add // 0) + else 0 end; + def strings_ok: + if type == "array" then all(.[];strings_ok) + elif type == "object" then + all(keys_unsorted[];utf8bytelength <= 8192) and all(.[];strings_ok) + elif type == "string" then utf8bytelength <= 8192 else true end; + depth <= 32 and members <= 16384 and strings_ok +' "$raw" >/dev/null 2>&1 || emit_error E_LIMIT + +result=$("$jq_bin" -L "$modules" -S -c -r \ + --arg expected_repository_id "$expected_repository_id" \ + --arg expected_commit_id "$expected_commit_id" \ + -f "$program" "$raw" 2>/dev/null) || emit_error E_RUNTIME +case "$result" in + E_SHAPE|E_RELATION|E_STALE) emit_error "$result" ;; + \{*) ;; + *) emit_error E_RUNTIME ;; +esac + +item_count=$("$jq_bin" -r '.body.items | length' "$raw" 2>/dev/null) || + emit_error E_RUNTIME +i=0 +while [ "$i" -lt "$item_count" ]; do + for pair_name in request resolved_profile; do + expected=$("$jq_bin" -r ".body.items[$i].$pair_name.sha256 // empty" "$raw") || + emit_error E_PARSE + content=$("$jq_bin" -S -c ".body.items[$i].$pair_name.content" "$raw") || + emit_error E_PARSE + [ -n "$expected" ] && [ "$(sha256_line "$content")" = "$expected" ] || + emit_error E_RELATION + done + if [ "$("$jq_bin" -r ".body.items[$i].latest_result.state // empty" "$raw")" = present ]; then + expected=$("$jq_bin" -r ".body.items[$i].latest_result.value.sha256 // empty" "$raw") || + emit_error E_PARSE + content=$("$jq_bin" -S -c ".body.items[$i].latest_result.value.content" "$raw") || + emit_error E_PARSE + [ -n "$expected" ] && [ "$(sha256_line "$content")" = "$expected" ] || + emit_error E_RELATION + fi + i=$((i + 1)) +done + +verify_hash 7210dbdf53ce07a3d7274732d22cf2f53245a1968dbd54ba7c8a6b77b1c4b831 "$program" || emit_error E_RUNTIME +verify_core || emit_error E_STALE +/usr/bin/printf '%s\n' "$result" +trap - EXIT HUP INT TERM +cleanup diff --git a/orchestrator/v1/state-scanner.jq b/orchestrator/v1/state-scanner.jq new file mode 100644 index 0000000..adc7ef3 --- /dev/null +++ b/orchestrator/v1/state-scanner.jq @@ -0,0 +1,202 @@ +import "schema" as schema; +import "profile_graph" as profile; +import "stage_request" as request; +import "result_truth" as result; + +def pair_shape($kind): + schema::exact_fields(["content","sha256"];[]) and + (.content | schema::envelope_ok($kind)) and + (.sha256 | schema::sha256_ok); + +def present_shape(value_ok): + (schema::exact_fields(["state"];[]) and .state == "absent") or + (schema::exact_fields(["state","value"];[]) and .state == "present" and + (.value | value_ok)); + +def core_contract_shape: + schema::exact_fields( + ["generation_id_sha256","package_ref","semantic_identity"];[]) and + (.generation_id_sha256 | schema::sha256_ok) and + (.package_ref | schema::content_ref_ok) and + (.semantic_identity | schema::id_ok); + +def attempt_value_shape: + schema::exact_fields( + ["attempt_id","attempt_number","deadline_at","request_ref","state"];[]) and + (.attempt_id | schema::id_ok) and + (.attempt_number | schema::int_ok) and .attempt_number >= 1 and + (.deadline_at | schema::time_ok) and + (.request_ref | schema::document_ref_kind_ok("stage_request")) and + (.state == "dispatched" or .state == "started"); + +def item_shape: + schema::exact_fields( + ["attempt","latest_result","request","resolved_profile","retry_limit"];[]) and + (.request | pair_shape("stage_request")) and + (.resolved_profile | pair_shape("resolved_profile")) and + (.latest_result | present_shape(pair_shape("stage_result"))) and + (.attempt | present_shape(attempt_value_shape)) and + (.retry_limit | schema::int_ok) and + .retry_limit >= 1 and .retry_limit <= 10; + +def snapshot_shape: + schema::exact_fields(["body","id","kind","schema_version"];[]) and + .schema_version == 1 and .kind == "orchestrator_state_snapshot" and + (.id | schema::id_ok) and + (.body | + schema::exact_fields( + ["core_contract","items","observed_at","source_revision"];[]) and + (.core_contract | core_contract_shape) and + (.source_revision | schema::git_revision_ref_ok) and + (.observed_at | schema::time_ok) and + (.items | type == "array" and length >= 1 and length <= 64 and + all(.[];item_shape))); + +def expected_core: + { + generation_id_sha256: + "6f6acbbd0cf40ab3c913328d6c0070635424ffe920bcdb900fbd0718345d7137", + package_ref:{ + content_id:"core-contract-package.v2", + media_type:"application/vnd.ystack.core-contract+json", + sha256:"005431c5c7e3a39dc3ab75dfcafd0f09359331667fdcacb140514a4384592716" + }, + semantic_identity:"core.contracts.v2" + }; + +def stage_key: + .request.content.body | + [.initiative_id,.workflow_id,.stage_id,.task_class_id]; + +def request_ref: + .request | profile::document_ref_for_pair(.); + +def item_relation($source; $observed_at): + . as $item | + ($item.request.content | request::document_self_ok) and + ($item.resolved_profile.content | profile::resolved_profile_self_ok) and + request::stage_request_resolved_ref_ok( + $item.request;$item.resolved_profile) and + request::stage_request_resolved_relation_ok( + $item.request.content.body;$item.resolved_profile.content.body) and + $item.request.content.body.target_repository_id == $source.repository_id and + $item.request.content.body.requested_at <= $observed_at and + (if $item.latest_result.state == "present" then + $item.attempt.state == "absent" and + result::stage_run_ok( + $item.request;$item.resolved_profile;$item.latest_result.value) and + $item.latest_result.value.content.body.recorded_at <= $observed_at and + $item.latest_result.value.content.body.attempt_number <= $item.retry_limit + elif $item.attempt.state == "present" then + $item.attempt.value.request_ref == ($item | request_ref) and + $item.attempt.value.attempt_number <= $item.retry_limit and + $item.request.content.body.requested_at <= $item.attempt.value.deadline_at + else true + end); + +def set_relations: + .body.items as $items | + ($items | map(stage_key)) as $keys | + ($keys == ($keys | sort)) and + (($keys | length) == ($keys | unique | length)) and + (($items | map(.request | profile::document_ref_for_pair(.))) as $refs | + ($refs | length) == ($refs | unique | length)) and + ([ $items[] | + if .attempt.state == "present" then .attempt.value.attempt_id + elif .latest_result.state == "present" then + .latest_result.value.content.body.attempt_id + else empty end ] as $attempt_ids | + ($attempt_ids | length) == ($attempt_ids | unique | length)); + +def source_reason($item): + if $item.latest_result.state == "present" and + ($item.latest_result.value.content.body | has("reason")) + then {state:"present",value:$item.latest_result.value.content.body.reason.reason_id} + else {state:"absent"} + end; + +def attempt_number($item): + if $item.latest_result.state == "present" then + $item.latest_result.value.content.body.attempt_number + elif $item.attempt.state == "present" then $item.attempt.value.attempt_number + else 0 + end; + +def target_moved($item; $source): + $item.request.content.body.target_revision.state == "present" and + $item.request.content.body.target_revision.value != $source and + ($item.latest_result.state == "absent" or + ($item.latest_result.value.content.body.status != "completed" and + $item.latest_result.value.content.body.status != "skipped")); + +def classification($item; $source; $observed_at): + ($item.latest_result.value.content.body.status // null) as $status | + (attempt_number($item)) as $attempt_number | + (source_reason($item)) as $source_reason | + (if target_moved($item;$source) then + ["stale","refresh-stage-inputs","scanner.target-revision-moved"] + elif $status == "completed" then + ["terminal","none","scanner.stage-completed"] + elif $status == "skipped" then + ["terminal","none","scanner.stage-skipped"] + elif $status == "stale" then + ["stale","refresh-stage-inputs","scanner.stage-stale"] + elif $status == "blocked" then + ["blocked","resolve-stage-blocker","scanner.stage-blocked"] + elif $status == "failed" or $status == "cancelled" then + if $attempt_number < $item.retry_limit then + ["retryable","retry-stage","scanner.stage-" + $status] + else ["blocked","operator-reconcile","scanner.retry-limit-reached"] + end + elif $item.attempt.state == "present" and + $item.attempt.value.deadline_at <= $observed_at then + ["stranded","recover-stranded-attempt","scanner.attempt-deadline-reached"] + elif $item.attempt.state == "present" then + ["pending","wait-for-attempt","scanner.attempt-in-flight"] + else ["pending","dispatch-stage","scanner.no-attempt"] + end) as $decision | + { + stage_key:{ + initiative_id:$item.request.content.body.initiative_id, + workflow_id:$item.request.content.body.workflow_id, + stage_id:$item.request.content.body.stage_id, + task_class_id:$item.request.content.body.task_class_id + }, + class:$decision[0], + recovery:{ + action:$decision[1], + reason_id:$decision[2], + source_reason_id:$source_reason, + attempt_number:$attempt_number, + retry_limit:$item.retry_limit + } + }; + +. as $snapshot | +if (snapshot_shape | not) then "E_SHAPE" +elif $snapshot.body.core_contract != expected_core then "E_STALE" +elif $snapshot.body.source_revision.repository_id != $expected_repository_id or + $snapshot.body.source_revision.commit_id != $expected_commit_id then "E_STALE" +elif (all($snapshot.body.items[]; + item_relation( + $snapshot.body.source_revision;$snapshot.body.observed_at)) | not) +then "E_RELATION" +elif ($snapshot | set_relations | not) then "E_RELATION" +else { + schema_version:1, + kind:"orchestrator_state_observation", + id:$snapshot.id, + body:{ + activation_state:"inactive", + authority_effect:"none", + mode:"observation-only", + core_contract:$snapshot.body.core_contract, + source_revision:$snapshot.body.source_revision, + observed_at:$snapshot.body.observed_at, + classifications:[ + $snapshot.body.items[] | + classification(.;$snapshot.body.source_revision;$snapshot.body.observed_at) + ] + } +} +end diff --git a/scripts/test/orchestrator-state-scanner.test.sh b/scripts/test/orchestrator-state-scanner.test.sh new file mode 100755 index 0000000..dd1cc7c --- /dev/null +++ b/scripts/test/orchestrator-state-scanner.test.sh @@ -0,0 +1,307 @@ +#!/bin/bash +# shellcheck disable=SC2016 +set -euo pipefail +export LC_ALL=C + +root=$(CDPATH='' cd -P -- "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd -P) +scanner="$root/orchestrator/v1/scan-state.sh" +fixtures="$root/scripts/test" +tmp=$(/usr/bin/mktemp -d "${TMPDIR:-/tmp}/ystack-state-scanner-test.XXXXXX") +cleanup() { /bin/rm -rf -- "$tmp"; } +trap cleanup EXIT + +sha_file() { /usr/bin/shasum -a 256 "$1" | /usr/bin/awk '{print $1}'; } +sha_line() { + /usr/bin/printf '%s\n' "$1" | /usr/bin/shasum -a 256 | + /usr/bin/awk '{print $1}' +} + +platform=$(/usr/bin/uname -s):$(/usr/bin/uname -m) +case "$platform" in + Darwin:*) jq_asset=jq-osx-amd64; jq_sha=5c0a0a3ea600f302ee458b30317425dd9632d1ad8882259fcaf4e9b868b2b1ef ;; + Linux:x86_64) jq_asset=jq-linux64; jq_sha=af986793a515d500ab2d35f8d2aecd656e764504b789b66d7e1a0b727a124c44 ;; + *) /usr/bin/printf 'FAIL: unsupported host %s\n' "$platform" >&2; exit 1 ;; +esac +system_jq=$(command -v jq) +jq_source=$system_jq +if [ "$($system_jq --version 2>/dev/null)" != jq-1.6 ]; then + jq_source="${TMPDIR:-/tmp}/ystack-portable-core-jq16/$jq_asset" + [ -f "$jq_source" ] && [ "$(sha_file "$jq_source")" = "$jq_sha" ] || { + /usr/bin/printf '%s\n' 'FAIL: verified jq 1.6 required' >&2 + exit 1 + } +fi +/bin/mkdir -m 700 "$tmp/bin" +/bin/cp "$jq_source" "$tmp/bin/jq" +/bin/chmod 0555 "$tmp/bin/jq" +export PATH="$tmp/bin:/usr/bin:/bin" +jq_bin="$tmp/bin/jq" +[ "$($jq_bin --version)" = jq-1.6 ] || { + /usr/bin/printf '%s\n' 'FAIL: jq identity' >&2 + exit 1 +} + +passed=0 +failed=0 +pass() { passed=$((passed + 1)); } +fail() { /usr/bin/printf 'FAIL: %s\n' "$1" >&2; failed=$((failed + 1)); } + +expect_class() { + local name=$1 snapshot=$2 commit=$3 class=$4 action=$5 reason=$6 output + if ! output=$("$scanner" scan repo.example "$commit" "$snapshot" 2>"$tmp/$name.err"); then + fail "$name returned $(<"$tmp/$name.err")" + return + fi + if /usr/bin/printf '%s\n' "$output" | "$jq_bin" -e -S -c \ + --arg class "$class" --arg action "$action" --arg reason "$reason" ' + .schema_version == 1 and .kind == "orchestrator_state_observation" and + .body.activation_state == "inactive" and + .body.authority_effect == "none" and .body.mode == "observation-only" and + (.body.classifications | length) == 1 and + .body.classifications[0].class == $class and + .body.classifications[0].recovery.action == $action and + .body.classifications[0].recovery.reason_id == $reason and + ((tojson | test("grant|approval|qualification|publish|schedule|wake")) | not) + ' >/dev/null && + [ "$output" = "$(/usr/bin/printf '%s\n' "$output" | "$jq_bin" -S -c .)" ]; then + pass + else + fail "$name produced the wrong observation" + fi +} + +expect_error() { + local name=$1 expected=$2 snapshot=$3 commit=${4:-1111111111111111111111111111111111111111} + local output status + set +e + output=$("$scanner" scan repo.example "$commit" "$snapshot" 2>"$tmp/$name.err") + status=$? + set -e + if [ "$status" -ne 0 ] && [ -z "$output" ] && + [ "$(<"$tmp/$name.err")" = "$expected" ]; then + pass + else + fail "$name expected $expected" + fi +} + +resolved="$tmp/resolved.json" +"$jq_bin" -L "$fixtures" -S -c -n ' + import "portable-core-profile-graph-fixtures" as profile; + def v2: walk(if type == "object" and has("schema_version") + then .schema_version=2 else . end); + def forge_binding($shas): { + binding_id:"binding.forge",role:"forge", + manifest_ref:{schema_version:2,kind:"adapter_manifest",id:"manifest.forge",sha256:$shas.forge}, + execution_kind:"deterministic",adapter_instance_id:"instance.forge", + principal_id:"principal.forge",execution_boundary_id:"boundary.forge", + authority_ref:profile::scope("authority";"authority-forge";("5"*64)), + package_ref:profile::blob("packages/forge.bin";"6"),skill_refs:[],requested_tools:[], + requested_capabilities:["core.forge.materialize-candidate.v2"], + requested_permissions:["core.perm.candidate-repository.write.v2", + "core.perm.evidence.write.v1","core.perm.scratch.write.v1","core.perm.target.read.v1"] + }; + {forge:("0"*64),producer:("a"*64),publisher:("b"*64), + reviewer:("c"*64),verifier:("d"*64)} as $shas | + (profile::profile_doc($shas) | v2 | + .body.profile_version="v2" | + .body.bindings += [forge_binding($shas)] | + .body.bindings |= sort_by(.binding_id)) as $profile | + profile::resolved_profile_doc($profile;("e"*64);$shas) | v2 | + .body.bindings |= map( + if .binding.role == "forge" then + .adapter_implementation.version="v2" | + .manifest_source=profile::source_value( + profile::blob("manifests/forge.json";"a");"canonical-json";$shas.forge) + else . end) +' >"$resolved" +resolved_sha=$(sha_file "$resolved") + +request="$tmp/request.json" +"$jq_bin" -L "$fixtures" -S -c -n --arg resolved_sha "$resolved_sha" ' + import "portable-core-stage-request-fixtures" as request; + def v2: walk(if type == "object" and has("schema_version") + then .schema_version=2 else . end); + request::request_doc("producer";$resolved_sha) | v2 +' >"$request" +request_sha=$(sha_file "$request") + +for flavor in completed skipped stale blocked failed cancelled; do + result_file="$tmp/result-$flavor.json" + "$jq_bin" -L "$fixtures" -S -c -n \ + --slurpfile request "$request" --slurpfile resolved "$resolved" \ + --arg request_sha "$request_sha" --arg resolved_sha "$resolved_sha" \ + --arg flavor "$flavor" ' + import "portable-core-result-truth-fixtures" as result; + def v2: walk(if type == "object" and has("schema_version") + then .schema_version=2 else . end); + (if $flavor == "completed" then + result::completed_result_doc($request[0];$request_sha;$resolved[0];$resolved_sha) + elif $flavor == "skipped" then + result::skipped_result_doc($request[0];$request_sha;$resolved[0];$resolved_sha) + elif $flavor == "stale" then + result::stale_result_doc($request[0];$request_sha;$resolved[0];$resolved_sha) + elif $flavor == "blocked" then + result::blocked_result_doc($request[0];$request_sha;$resolved[0];$resolved_sha) + elif $flavor == "failed" then + result::failed_result_doc($request[0];$request_sha;$resolved[0];$resolved_sha) + else result::cancelled_result_doc($request[0];$request_sha;$resolved[0];$resolved_sha) + end) | v2 + ' >"$result_file" +done + +make_snapshot() { + local destination=$1 result_path=$2 attempt_state=$3 deadline=$4 retry_limit=$5 commit=$6 + local result_args=(--slurpfile result /dev/null) + if [ "$result_path" != absent ]; then result_args=(--slurpfile result "$result_path"); fi + "$jq_bin" -S -c -n --slurpfile request "$request" --slurpfile resolved "$resolved" \ + "${result_args[@]}" --arg request_sha "$request_sha" --arg resolved_sha "$resolved_sha" \ + --arg attempt_state "$attempt_state" --arg deadline "$deadline" \ + --argjson retry_limit "$retry_limit" --arg commit "$commit" ' + def pair($docs;$sha): {content:$docs[0],sha256:$sha}; + def doc_ref($pair): + {schema_version:2,kind:$pair.content.kind,id:$pair.content.id,sha256:$pair.sha256}; + (pair($request;$request_sha)) as $request_pair | + (pair($resolved;$resolved_sha)) as $resolved_pair | + { + schema_version:1, + kind:"orchestrator_state_snapshot", + id:"snapshot.example", + body:{ + core_contract:{ + generation_id_sha256:"6f6acbbd0cf40ab3c913328d6c0070635424ffe920bcdb900fbd0718345d7137", + package_ref:{content_id:"core-contract-package.v2", + media_type:"application/vnd.ystack.core-contract+json", + sha256:"005431c5c7e3a39dc3ab75dfcafd0f09359331667fdcacb140514a4384592716"}, + semantic_identity:"core.contracts.v2" + }, + items:[{ + attempt:(if $attempt_state == "absent" then {state:"absent"} else + {state:"present",value:{attempt_id:"attempt.pending",attempt_number:1, + deadline_at:$deadline,request_ref:doc_ref($request_pair),state:$attempt_state}} end), + latest_result:(if ($result | length) == 1 then + {state:"present",value:pair($result;("0"*64))} + else {state:"absent"} end), + request:$request_pair, + resolved_profile:$resolved_pair, + retry_limit:$retry_limit + }], + observed_at:"2026-08-30T00:10:00Z", + source_revision:{repository_id:"repo.example",hash_algorithm:"sha1",commit_id:$commit} + } + } + ' >"$destination" + if [ "$result_path" != absent ]; then + local result_sha + result_sha=$(sha_file "$result_path") + "$jq_bin" -S -c --arg sha "$result_sha" \ + '.body.items[0].latest_result.value.sha256=$sha' "$destination" >"$destination.next" + /bin/mv "$destination.next" "$destination" + fi +} + +commit_one=1111111111111111111111111111111111111111 +commit_two=2222222222222222222222222222222222222222 +pending="$tmp/pending.json" +make_snapshot "$pending" absent absent 2026-08-30T00:20:00Z 2 "$commit_one" +expect_class pending "$pending" "$commit_one" pending dispatch-stage scanner.no-attempt + +inflight="$tmp/inflight.json" +make_snapshot "$inflight" absent started 2026-08-30T00:20:00Z 2 "$commit_one" +expect_class in-flight "$inflight" "$commit_one" pending wait-for-attempt scanner.attempt-in-flight + +stranded="$tmp/stranded.json" +make_snapshot "$stranded" absent dispatched 2026-08-30T00:10:00Z 2 "$commit_one" +expect_class stranded "$stranded" "$commit_one" stranded recover-stranded-attempt scanner.attempt-deadline-reached + +for spec in \ + 'completed terminal none scanner.stage-completed 2' \ + 'skipped terminal none scanner.stage-skipped 2' \ + 'stale stale refresh-stage-inputs scanner.stage-stale 2' \ + 'blocked blocked resolve-stage-blocker scanner.stage-blocked 2' \ + 'failed retryable retry-stage scanner.stage-failed 2' \ + 'cancelled retryable retry-stage scanner.stage-cancelled 2' \ + 'failed blocked operator-reconcile scanner.retry-limit-reached 1'; do + read -r flavor expected_class action reason retry_limit <<<"$spec" + snapshot="$tmp/$flavor-$retry_limit.json" + make_snapshot "$snapshot" "$tmp/result-$flavor.json" absent \ + 2026-08-30T00:20:00Z "$retry_limit" "$commit_one" + expect_class "$flavor-$expected_class-$retry_limit" "$snapshot" "$commit_one" \ + "$expected_class" "$action" "$reason" +done + +moved="$tmp/moved.json" +make_snapshot "$moved" absent absent 2026-08-30T00:20:00Z 2 "$commit_two" +expect_class moved-request "$moved" "$commit_two" stale refresh-stage-inputs scanner.target-revision-moved + +completed_moved="$tmp/completed-moved.json" +make_snapshot "$completed_moved" "$tmp/result-completed.json" absent \ + 2026-08-30T00:20:00Z 2 "$commit_two" +expect_class immutable-terminal "$completed_moved" "$commit_two" terminal none scanner.stage-completed + +bad_core="$tmp/bad-core.json" +"$jq_bin" -S -c '.body.core_contract.semantic_identity="core.contracts.v9"' \ + "$pending" >"$bad_core" +expect_error stale-core E_STALE "$bad_core" +expect_error stale-snapshot E_STALE "$pending" "$commit_two" + +bad_sha="$tmp/bad-sha.json" +"$jq_bin" -S -c '.body.items[0].request.sha256=("0"*64)' "$pending" >"$bad_sha" +expect_error content-ref-hash E_RELATION "$bad_sha" + +ambiguous="$tmp/ambiguous.json" +"$jq_bin" -S -c --slurpfile attempt "$inflight" \ + '.body.items[0].attempt=$attempt[0].body.items[0].attempt' \ + "$tmp/completed-2.json" >"$ambiguous" +expect_error ambiguous-current-and-terminal E_RELATION "$ambiguous" + +time_travel="$tmp/time-travel.json" +"$jq_bin" -S -c '.body.observed_at="2026-08-29T23:59:59Z"' "$pending" >"$time_travel" +expect_error observation-before-request E_RELATION "$time_travel" + +duplicate="$tmp/duplicate.json" +"$jq_bin" -S -c '.body.items += [.body.items[0]]' "$pending" >"$duplicate" +expect_error duplicate-stage E_RELATION "$duplicate" + +unordered="$tmp/unordered.json" +"$jq_bin" -S -c ' + .body.items=[(.body.items[0] | .request.content.body.stage_id="stage.z"), + (.body.items[0] | .request.content.body.stage_id="stage.a")] +' "$pending" >"$unordered.raw" +for index in 0 1; do + content=$("$jq_bin" -S -c ".body.items[$index].request.content" "$unordered.raw") + digest=$(sha_line "$content") + "$jq_bin" -S -c --argjson index "$index" --arg digest "$digest" \ + '.body.items[$index].request.sha256=$digest' "$unordered.raw" >"$unordered.next" + /bin/mv "$unordered.next" "$unordered.raw" +done +/bin/mv "$unordered.raw" "$unordered" +expect_error unordered-stage E_RELATION "$unordered" + +too_many="$tmp/too-many.json" +"$jq_bin" -S -c '.body.items=[range(0;65) as $n | {}]' \ + "$pending" >"$too_many" +expect_error item-bound E_SHAPE "$too_many" +oversize="$tmp/oversize.json" +/bin/dd if=/dev/zero of="$oversize" bs=1048577 count=1 2>/dev/null +expect_error byte-bound E_LIMIT "$oversize" + +pretty="$tmp/pretty.json" +"$jq_bin" . "$pending" >"$pretty" +expect_error noncanonical E_CANONICAL "$pretty" +duplicate_key="$tmp/duplicate-key.json" +{ + /usr/bin/printf '%s' '{"body":null,' + /usr/bin/printf '%s' "$(<"$pending")" | /usr/bin/cut -c2- +} >"$duplicate_key" +expect_error duplicate-key E_CANONICAL "$duplicate_key" + +link="$tmp/input-link.json" +/bin/ln -s "$pending" "$link" +expect_error symlink-input E_RUNTIME "$link" + +if [ "$failed" -ne 0 ]; then + /usr/bin/printf 'orchestrator state scanner: %s passed, %s failed\n' "$passed" "$failed" >&2 + exit 1 +fi +/usr/bin/printf 'orchestrator state scanner: %s/%s checks passed\n' "$passed" "$passed" From 3b998c69a4592002b68c4ad528332e754921df81 Mon Sep 17 00:00:00 2001 From: ci Date: Tue, 1 Sep 2026 11:35:44 -0400 Subject: [PATCH 2/5] Harden canonical state scanner provenance --- orchestrator/v1/scan-state.sh | 164 +--------- orchestrator/v1/state-scanner-driver.sh | 187 +++++++++++ orchestrator/v1/state-scanner-launcher.sh | 295 +++++++++++++++++ orchestrator/v1/state-scanner.jq | 257 +++++++++++---- .../test/orchestrator-state-scanner.test.sh | 304 ++++++++++++++++-- 5 files changed, 969 insertions(+), 238 deletions(-) create mode 100755 orchestrator/v1/state-scanner-driver.sh create mode 100755 orchestrator/v1/state-scanner-launcher.sh diff --git a/orchestrator/v1/scan-state.sh b/orchestrator/v1/scan-state.sh index 8a93226..b36a589 100755 --- a/orchestrator/v1/scan-state.sh +++ b/orchestrator/v1/scan-state.sh @@ -1,159 +1,11 @@ #!/bin/bash -set -uo pipefail -export LC_ALL=C -umask 077 -emit_error() { - case "${1:-}" in - E_USAGE|E_RUNTIME|E_LIMIT|E_PARSE|E_CANONICAL|E_SHAPE|E_RELATION|E_STALE) - /usr/bin/printf '%s\n' "$1" >&2 - ;; - *) /usr/bin/printf '%s\n' E_RUNTIME >&2 ;; - esac - exit 1 -} - -sha256_path() { - /usr/bin/shasum -a 256 "$1" | /usr/bin/awk '{print $1}' -} - -sha256_line() { - /usr/bin/printf '%s\n' "$1" | /usr/bin/shasum -a 256 | - /usr/bin/awk '{print $1}' -} - -verify_hash() { - [ -f "$2" ] && [ ! -L "$2" ] && - [ "$(sha256_path "$2")" = "$1" ] -} - -verify_core() { - verify_hash bdb5def832e8e611bba8a7b30a2aae95ea4f2701c44b198cf51cd3dfd9ff88f3 \ - "$wrapper" && - verify_hash f55b697716dc13a6d2c71bde7769493b3f4b091fd7a94d3280c5d417974df3a1 \ - "$registry" && - verify_hash 8d1d02d36ac7ada778f05248f9413062b3fc251499914c15d79f003bbd009ade \ - "$modules/schema.jq" && - verify_hash c00f9cfbe88df5cb1dbcfbead61288ff7d68684d43d095e74f26e7820f0d7207 \ - "$modules/profile_graph.jq" && - verify_hash 6572a6ecbac332dc9c4a8ef35acd1feebdc2e8aab04941fc0b756f3a5cbcf29e \ - "$modules/stage_request.jq" && - verify_hash 8e49c2c091f1bbe525f7499e3fca072f6916a14d5bb34adbf121439e8ca2d281 \ - "$modules/result_facts.jq" && - verify_hash ed992f26761d08e3c3f5ab57eda9bcd771ad59e3aebeb02643de88844184d2d3 \ - "$modules/result_truth.jq" -} - -[ "$#" -eq 4 ] && [ "$1" = scan ] || emit_error E_USAGE -expected_repository_id=$2 -expected_commit_id=$3 -input=$4 -[[ "$expected_repository_id" =~ ^[a-z0-9][a-z0-9._:-]{0,127}$ ]] || - emit_error E_USAGE -[[ "$expected_commit_id" =~ ^[0-9a-f]{40}$ ]] || - [[ "$expected_commit_id" =~ ^[0-9a-f]{64}$ ]] || emit_error E_USAGE - -source_path=${BASH_SOURCE[0]} -case "$source_path" in /*) ;; *) source_path="$(pwd -P)/$source_path" ;; esac -[ -f "$source_path" ] && [ ! -L "$source_path" ] || emit_error E_RUNTIME -source_dir=$(CDPATH='' cd -P -- "${source_path%/*}" 2>/dev/null && pwd -P) || - emit_error E_RUNTIME -source_path="$source_dir/${source_path##*/}" -[ "$source_path" = "$source_dir/scan-state.sh" ] || emit_error E_RUNTIME -repo=$(CDPATH='' cd -P -- "$source_dir/../.." 2>/dev/null && pwd -P) || - emit_error E_RUNTIME -[ "$source_dir" = "$repo/orchestrator/v1" ] || emit_error E_RUNTIME - -generation=g-392d20099dfa99872764009b268c8871914b4dbc0da467ec346baa921818ae3e -modules="$repo/core/v2/generations/$generation/modules" -program="$source_dir/state-scanner.jq" -registry="$repo/core/v2/generation-registry.json" -wrapper="$repo/scripts/core-contract.sh" -for required_dir in "$repo" "$repo/orchestrator" "$source_dir" "$repo/core" \ - "$repo/core/v2" "$repo/core/v2/generations" "${modules%/modules}" "$modules"; do - [ -d "$required_dir" ] && [ ! -L "$required_dir" ] || emit_error E_RUNTIME -done -verify_hash 7210dbdf53ce07a3d7274732d22cf2f53245a1968dbd54ba7c8a6b77b1c4b831 "$program" || emit_error E_RUNTIME -verify_core || emit_error E_STALE - -[ -f "$input" ] && [ ! -L "$input" ] || emit_error E_RUNTIME -jq_bin=$(command -v jq 2>/dev/null) || emit_error E_RUNTIME -case "$jq_bin" in /*) ;; *) emit_error E_RUNTIME ;; esac -[ -f "$jq_bin" ] && [ -x "$jq_bin" ] && [ ! -L "$jq_bin" ] && - [ "$($jq_bin --version 2>/dev/null)" = jq-1.6 ] || emit_error E_RUNTIME - -scratch=$(/usr/bin/mktemp -d "${TMPDIR:-/tmp}/ystack-state-scan.XXXXXX" 2>/dev/null) || - emit_error E_RUNTIME -cleanup() { /bin/rm -rf -- "$scratch" >/dev/null 2>&1 || :; } -signal_exit() { trap - EXIT HUP INT TERM; cleanup; exit 1; } -trap cleanup EXIT -trap signal_exit HUP INT TERM - -raw="$scratch/raw.json" -/bin/dd if="$input" of="$raw" bs=1048577 count=1 2>/dev/null || emit_error E_RUNTIME -raw_size=$(/usr/bin/wc -c < "$raw" | /usr/bin/tr -d ' ') || emit_error E_RUNTIME -[ "$raw_size" -le 1048576 ] || emit_error E_LIMIT -bom=$(/usr/bin/od -An -tx1 -N3 "$raw" 2>/dev/null | /usr/bin/tr -d ' \n') || - emit_error E_RUNTIME -[ "$bom" != efbbbf ] || emit_error E_PARSE -"$jq_bin" . "$raw" >/dev/null 2>&1 || emit_error E_PARSE -[ "$("$jq_bin" -s 'length' "$raw" 2>/dev/null)" -eq 1 ] || emit_error E_PARSE -canonical="$scratch/canonical.json" -"$jq_bin" -S -c . "$raw" > "$canonical" 2>/dev/null || emit_error E_PARSE -/usr/bin/cmp -s "$raw" "$canonical" || emit_error E_CANONICAL - -"$jq_bin" -e ' - def depth: - if type == "array" then (if length == 0 then 1 else 1 + ([.[]|depth]|max) end) - elif type == "object" then (if length == 0 then 1 else 1 + ([.[]|depth]|max) end) - else 1 end; - def members: - if type == "array" then length + ([.[]|members]|add // 0) - elif type == "object" then (keys_unsorted|length) + ([.[]|members]|add // 0) - else 0 end; - def strings_ok: - if type == "array" then all(.[];strings_ok) - elif type == "object" then - all(keys_unsorted[];utf8bytelength <= 8192) and all(.[];strings_ok) - elif type == "string" then utf8bytelength <= 8192 else true end; - depth <= 32 and members <= 16384 and strings_ok -' "$raw" >/dev/null 2>&1 || emit_error E_LIMIT - -result=$("$jq_bin" -L "$modules" -S -c -r \ - --arg expected_repository_id "$expected_repository_id" \ - --arg expected_commit_id "$expected_commit_id" \ - -f "$program" "$raw" 2>/dev/null) || emit_error E_RUNTIME -case "$result" in - E_SHAPE|E_RELATION|E_STALE) emit_error "$result" ;; - \{*) ;; - *) emit_error E_RUNTIME ;; +entrypoint=${BASH_SOURCE[0]} +case "$entrypoint" in + /*) ;; + *) entrypoint="$PWD/$entrypoint" ;; esac - -item_count=$("$jq_bin" -r '.body.items | length' "$raw" 2>/dev/null) || - emit_error E_RUNTIME -i=0 -while [ "$i" -lt "$item_count" ]; do - for pair_name in request resolved_profile; do - expected=$("$jq_bin" -r ".body.items[$i].$pair_name.sha256 // empty" "$raw") || - emit_error E_PARSE - content=$("$jq_bin" -S -c ".body.items[$i].$pair_name.content" "$raw") || - emit_error E_PARSE - [ -n "$expected" ] && [ "$(sha256_line "$content")" = "$expected" ] || - emit_error E_RELATION - done - if [ "$("$jq_bin" -r ".body.items[$i].latest_result.state // empty" "$raw")" = present ]; then - expected=$("$jq_bin" -r ".body.items[$i].latest_result.value.sha256 // empty" "$raw") || - emit_error E_PARSE - content=$("$jq_bin" -S -c ".body.items[$i].latest_result.value.content" "$raw") || - emit_error E_PARSE - [ -n "$expected" ] && [ "$(sha256_line "$content")" = "$expected" ] || - emit_error E_RELATION - fi - i=$((i + 1)) -done - -verify_hash 7210dbdf53ce07a3d7274732d22cf2f53245a1968dbd54ba7c8a6b77b1c4b831 "$program" || emit_error E_RUNTIME -verify_core || emit_error E_STALE -/usr/bin/printf '%s\n' "$result" -trap - EXIT HUP INT TERM -cleanup +launcher="${entrypoint%/*}/state-scanner-launcher.sh" +requested_tmp=${TMPDIR:-/tmp} +/usr/bin/env -i LC_ALL=C PATH=/usr/bin:/bin TMPDIR="$requested_tmp" \ + /bin/bash "$launcher" "$@" diff --git a/orchestrator/v1/state-scanner-driver.sh b/orchestrator/v1/state-scanner-driver.sh new file mode 100755 index 0000000..59f0500 --- /dev/null +++ b/orchestrator/v1/state-scanner-driver.sh @@ -0,0 +1,187 @@ +#!/bin/bash +set -uo pipefail +export LC_ALL=C +umask 077 + +emit_error() { + case "${1:-}" in + E_USAGE|E_RUNTIME|E_LIMIT|E_PARSE|E_CANONICAL|E_SHAPE|E_RELATION|E_STALE) + /usr/bin/printf '%s\n' "$1" >&2 + ;; + *) /usr/bin/printf '%s\n' E_RUNTIME >&2 ;; + esac + exit 1 +} + +sha256_path() { + /usr/bin/shasum -a 256 "$1" | /usr/bin/awk '{print $1}' +} + +sha256_line() { + /usr/bin/printf '%s\n' "$1" | /usr/bin/shasum -a 256 | + /usr/bin/awk '{print $1}' +} + +verify_hash() { + [ -f "$2" ] && [ ! -L "$2" ] && + [ "$(sha256_path "$2")" = "$1" ] +} + +[ "$#" -eq 8 ] && [ "$1" = run ] || emit_error E_USAGE +expected_repository_id=$2 +expected_commit_id=$3 +runtime=$4 +input=$5 +evaluator=$6 +evaluator_sha256=$7 +snapshot_sha256=$8 + +self=${BASH_SOURCE[0]} +case "$self" in /*) ;; *) emit_error E_RUNTIME ;; esac +self_dir=$(CDPATH='' cd -P -- "${self%/*}" 2>/dev/null && pwd -P) || + emit_error E_RUNTIME +self="$self_dir/${self##*/}" +[ "$self" = "$runtime/driver.sh" ] && [ "$self_dir" = "$runtime" ] || + emit_error E_RUNTIME +[ -d "$runtime" ] && [ ! -L "$runtime" ] || emit_error E_RUNTIME +runtime_parent=${runtime%/*} +case "$input" in "$runtime_parent"/input.json) ;; *) emit_error E_RUNTIME ;; esac +case "$evaluator" in "$runtime"/evaluator.json) ;; *) emit_error E_RUNTIME ;; esac + +generation=g-392d20099dfa99872764009b268c8871914b4dbc0da467ec346baa921818ae3e +modules="$runtime/core/v2/generations/$generation/modules" +program="$runtime/program.jq" +jq_bin="$runtime/jq" +work="$runtime_parent/work" +verify_runtime() { + verify_hash 8838c85aae5a2ed9ada659ae1a13c5cf8f561463789d1d5d9f28370d479f6c80 \ + "$program" && + verify_hash f55b697716dc13a6d2c71bde7769493b3f4b091fd7a94d3280c5d417974df3a1 \ + "$runtime/core/v2/generation-registry.json" && + verify_hash 65eb40b9afb9b4f1d809ed66d0f2ca625f656c34e856cedcde9cbbde857f0f0a \ + "$runtime/core/v2/generations/$generation/contracts.jq" && + verify_hash db87c6e97e93dc2a6eebd83087878c04f5528badc620d57fc9d883694e2ac28b \ + "$runtime/core/v2/generations/$generation/core-ingress.sh" && + verify_hash c00f9cfbe88df5cb1dbcfbead61288ff7d68684d43d095e74f26e7820f0d7207 \ + "$modules/profile_graph.jq" && + verify_hash 8e49c2c091f1bbe525f7499e3fca072f6916a14d5bb34adbf121439e8ca2d281 \ + "$modules/result_facts.jq" && + verify_hash ed992f26761d08e3c3f5ab57eda9bcd771ad59e3aebeb02643de88844184d2d3 \ + "$modules/result_truth.jq" && + verify_hash 8d1d02d36ac7ada778f05248f9413062b3fc251499914c15d79f003bbd009ade \ + "$modules/schema.jq" && + verify_hash 6572a6ecbac332dc9c4a8ef35acd1feebdc2e8aab04941fc0b756f3a5cbcf29e \ + "$modules/stage_request.jq" && + verify_hash bdb5def832e8e611bba8a7b30a2aae95ea4f2701c44b198cf51cd3dfd9ff88f3 \ + "$runtime/scripts/core-contract.sh" && + [ -f "$jq_bin" ] && [ -x "$jq_bin" ] && [ ! -L "$jq_bin" ] && + { [ "$(sha256_path "$jq_bin")" = \ + af986793a515d500ab2d35f8d2aecd656e764504b789b66d7e1a0b727a124c44 ] || + [ "$(sha256_path "$jq_bin")" = \ + 5c0a0a3ea600f302ee458b30317425dd9632d1ad8882259fcaf4e9b868b2b1ef ]; } +} + +verify_runtime || emit_error E_STALE +[ -d "$work" ] && [ ! -L "$work" ] || emit_error E_RUNTIME +[ -f "$input" ] && [ ! -L "$input" ] && + [ "$(sha256_path "$input")" = "$snapshot_sha256" ] || emit_error E_RUNTIME +[ -f "$evaluator" ] && [ ! -L "$evaluator" ] && + [ "$(sha256_path "$evaluator")" = "$evaluator_sha256" ] || emit_error E_RUNTIME + +raw_size=$(/usr/bin/wc -c < "$input" | /usr/bin/tr -d ' ') || emit_error E_RUNTIME +[ "$raw_size" -le 1048576 ] || emit_error E_LIMIT +bom=$(/usr/bin/od -An -tx1 -N3 "$input" 2>/dev/null | /usr/bin/tr -d ' \n') || + emit_error E_RUNTIME +[ "$bom" != efbbbf ] || emit_error E_PARSE +"$jq_bin" . "$input" >/dev/null 2>&1 || emit_error E_PARSE +[ "$("$jq_bin" -s 'length' "$input" 2>/dev/null)" -eq 1 ] || emit_error E_PARSE +"$jq_bin" -S -c . "$input" > "$work/input-canonical.json" 2>/dev/null || + emit_error E_PARSE +/usr/bin/cmp -s "$input" "$work/input-canonical.json" || emit_error E_CANONICAL +"$jq_bin" -S -c . "$evaluator" > "$work/evaluator-canonical.json" 2>/dev/null || + emit_error E_RUNTIME +/usr/bin/cmp -s "$evaluator" "$work/evaluator-canonical.json" || emit_error E_RUNTIME + +"$jq_bin" -e ' + def depth: + if type == "array" then (if length == 0 then 1 else 1 + ([.[]|depth]|max) end) + elif type == "object" then (if length == 0 then 1 else 1 + ([.[]|depth]|max) end) + else 1 end; + def members: + if type == "array" then length + ([.[]|members]|add // 0) + elif type == "object" then (keys_unsorted|length) + ([.[]|members]|add // 0) + else 0 end; + def strings_ok: + if type == "array" then all(.[];strings_ok) + elif type == "object" then + all(keys_unsorted[];utf8bytelength <= 8192) and all(.[];strings_ok) + elif type == "string" then utf8bytelength <= 8192 else true end; + depth <= 32 and members <= 16384 and strings_ok +' "$input" >/dev/null 2>&1 || emit_error E_LIMIT + +item_count=$("$jq_bin" -r ' + .body.items | if type == "array" then length else 0 end +' "$input" 2>/dev/null) || emit_error E_RUNTIME +: >"$work/item-sha-lines" +i=0 +while [ "$i" -lt "$item_count" ]; do + content=$("$jq_bin" -S -c ".body.items[$i]" "$input") || emit_error E_RUNTIME + sha256_line "$content" >>"$work/item-sha-lines" || emit_error E_RUNTIME + i=$((i + 1)) +done +"$jq_bin" -R -s -c 'split("\n")[:-1]' "$work/item-sha-lines" \ + >"$work/item-shas.json" 2>/dev/null || emit_error E_RUNTIME + +result=$("$jq_bin" -L "$modules" -S -c -r \ + --arg scanner_operation scan \ + --arg expected_repository_id "$expected_repository_id" \ + --arg expected_commit_id "$expected_commit_id" \ + --arg snapshot_sha256 "$snapshot_sha256" \ + --arg evaluator_sha256 "$evaluator_sha256" \ + --slurpfile evaluator_docs "$evaluator" \ + --slurpfile item_sha_docs "$work/item-shas.json" \ + --slurpfile snapshot_docs /dev/null --slurpfile candidate_docs /dev/null \ + -f "$program" "$input" 2>/dev/null) || emit_error E_RUNTIME +case "$result" in E_SHAPE|E_RELATION|E_STALE) emit_error "$result" ;; esac + +/usr/bin/printf '%s\n' "$result" > "$work/candidate.json" +"$jq_bin" -S -c . "$work/candidate.json" > "$work/candidate-canonical.json" \ + 2>/dev/null || emit_error E_RUNTIME +/usr/bin/cmp -s "$work/candidate.json" "$work/candidate-canonical.json" || + emit_error E_RUNTIME + +i=0 +while [ "$i" -lt "$item_count" ]; do + for pair_name in request resolved_profile; do + expected=$("$jq_bin" -r ".body.items[$i].$pair_name.sha256 // empty" "$input") || + emit_error E_RUNTIME + content=$("$jq_bin" -S -c ".body.items[$i].$pair_name.content" "$input") || + emit_error E_RUNTIME + [ -n "$expected" ] && [ "$(sha256_line "$content")" = "$expected" ] || + emit_error E_RELATION + done + if [ "$("$jq_bin" -r ".body.items[$i].latest_result.state" "$input")" = present ]; then + expected=$("$jq_bin" -r ".body.items[$i].latest_result.value.sha256" "$input") || + emit_error E_RUNTIME + content=$("$jq_bin" -S -c ".body.items[$i].latest_result.value.content" "$input") || + emit_error E_RUNTIME + [ "$(sha256_line "$content")" = "$expected" ] || emit_error E_RELATION + fi + i=$((i + 1)) +done + +"$jq_bin" -n -e -L "$modules" \ + --arg scanner_operation validate-observation \ + --arg expected_repository_id "$expected_repository_id" \ + --arg expected_commit_id "$expected_commit_id" \ + --arg snapshot_sha256 "$snapshot_sha256" \ + --arg evaluator_sha256 "$evaluator_sha256" \ + --slurpfile evaluator_docs "$evaluator" \ + --slurpfile item_sha_docs "$work/item-shas.json" \ + --slurpfile snapshot_docs "$input" \ + --slurpfile candidate_docs "$work/candidate.json" \ + -f "$program" >/dev/null 2>&1 || emit_error E_RUNTIME +verify_runtime || emit_error E_STALE +[ "$(sha256_path "$input")" = "$snapshot_sha256" ] && + [ "$(sha256_path "$evaluator")" = "$evaluator_sha256" ] || emit_error E_RUNTIME +/bin/cat "$work/candidate.json" diff --git a/orchestrator/v1/state-scanner-launcher.sh b/orchestrator/v1/state-scanner-launcher.sh new file mode 100755 index 0000000..e73beed --- /dev/null +++ b/orchestrator/v1/state-scanner-launcher.sh @@ -0,0 +1,295 @@ +#!/bin/bash +# shellcheck disable=SC2016 +set -uo pipefail +export LC_ALL=C +umask 077 + +emit_error() { + case "${1:-}" in + E_USAGE|E_RUNTIME|E_LIMIT|E_PARSE|E_CANONICAL|E_SHAPE|E_RELATION|E_STALE) + /usr/bin/printf '%s\n' "$1" >&2 + ;; + *) /usr/bin/printf '%s\n' E_RUNTIME >&2 ;; + esac + exit 1 +} + +sha256_path() { + /usr/bin/shasum -a 256 "$1" | /usr/bin/awk '{print $1}' +} + +sha256_line() { + /usr/bin/printf '%s\n' "$1" | /usr/bin/shasum -a 256 | + /usr/bin/awk '{print $1}' +} + +snapshot_file() { + /usr/bin/perl -MFcntl=:DEFAULT,:mode -e ' + my ($source,$target,$limit,$mode)=@ARGV; + sysopen(my $in,$source,O_RDONLY|O_NOFOLLOW) or exit 40; + my @stat=stat($in); + @stat && S_ISREG($stat[2]) or exit 40; + $stat[7] <= $limit or exit 42; + sysopen(my $out,$target,O_WRONLY|O_CREAT|O_EXCL|O_NOFOLLOW,oct($mode)) + or exit 40; + my $total=0; + while (1) { + my $read=sysread($in,my $buffer,65536); + defined($read) or exit 40; + last if $read == 0; + $total += $read; + $total <= $limit or exit 42; + my $offset=0; + while ($offset < $read) { + my $written=syswrite($out,$buffer,$read-$offset,$offset); + defined($written) && $written > 0 or exit 40; + $offset += $written; + } + } + close($in) or exit 40; + close($out) or exit 40; + chmod(oct($mode),$target) == 1 or exit 40; + ' "$1" "$2" "$3" "$4" +} + +snapshot_expected() { + local expected=$1 source=$2 target=$3 mode=$4 status=0 + snapshot_file "$source" "$target" 16777216 "$mode" || status=$? + [ "$status" -eq 0 ] && [ "$(sha256_path "$target")" = "$expected" ] +} + +[ "$#" -eq 4 ] && [ "$1" = scan ] || emit_error E_USAGE +expected_repository_id=$2 +expected_commit_id=$3 +input=$4 +[[ "$expected_repository_id" =~ ^[a-z0-9][a-z0-9._:-]{0,127}$ ]] || + emit_error E_USAGE +[[ "$expected_commit_id" =~ ^[0-9a-f]{40}$ ]] || + [[ "$expected_commit_id" =~ ^[0-9a-f]{64}$ ]] || emit_error E_USAGE + +self=${BASH_SOURCE[0]} +case "$self" in /*) ;; *) self="$(pwd -P)/$self" ;; esac +[ -f "$self" ] && [ ! -L "$self" ] || emit_error E_RUNTIME +source_dir=$(CDPATH='' cd -P -- "${self%/*}" 2>/dev/null && pwd -P) || + emit_error E_RUNTIME +self="$source_dir/${self##*/}" +[ "$self" = "$source_dir/state-scanner-launcher.sh" ] || emit_error E_RUNTIME +repo=$(CDPATH='' cd -P -- "$source_dir/../.." 2>/dev/null && pwd -P) || + emit_error E_RUNTIME +[ "$source_dir" = "$repo/orchestrator/v1" ] || emit_error E_RUNTIME + +platform=$(/usr/bin/uname -s):$(/usr/bin/uname -m) +case "$platform" in + Linux:x86_64) + host_os=linux; host_arch=x86_64; jq_arch=x86_64; execution_mode=native + jq_asset=jq-linux64 + jq_sha=af986793a515d500ab2d35f8d2aecd656e764504b789b66d7e1a0b727a124c44 + ;; + Darwin:x86_64) + host_os=darwin; host_arch=x86_64; jq_arch=x86_64; execution_mode=native + jq_asset=jq-osx-amd64 + jq_sha=5c0a0a3ea600f302ee458b30317425dd9632d1ad8882259fcaf4e9b868b2b1ef + ;; + Darwin:arm64) + host_os=darwin; host_arch=arm64; jq_arch=x86_64; execution_mode=rosetta + jq_asset=jq-osx-amd64 + jq_sha=5c0a0a3ea600f302ee458b30317425dd9632d1ad8882259fcaf4e9b868b2b1ef + ;; + *) emit_error E_RUNTIME ;; +esac + +jq_source='' +for candidate in "${TMPDIR:-/tmp}/ystack-portable-core-jq16/$jq_asset" "/usr/bin/jq"; do + if [ -f "$candidate" ] && [ ! -L "$candidate" ] && + [ "$(sha256_path "$candidate")" = "$jq_sha" ]; then + jq_source=$candidate + break + fi +done +[ -n "$jq_source" ] || emit_error E_RUNTIME + +scratch=$(/usr/bin/mktemp -d "${TMPDIR:-/tmp}/ystack-state-scan.XXXXXX" 2>/dev/null) || + emit_error E_RUNTIME +scratch=$(CDPATH='' cd -P -- "$scratch" 2>/dev/null && pwd -P) || emit_error E_RUNTIME +cleanup() { /bin/rm -rf -- "$scratch" >/dev/null 2>&1 || :; } +signal_exit() { trap - EXIT HUP INT TERM; cleanup; exit 1; } +trap cleanup EXIT +trap signal_exit HUP INT TERM +runtime="$scratch/runtime" +/bin/mkdir -m 0700 "$runtime" "$runtime/core" "$runtime/core/v2" \ + "$runtime/core/v2/generations" "$runtime/scripts" "$scratch/work" || + emit_error E_RUNTIME +generation=g-392d20099dfa99872764009b268c8871914b4dbc0da467ec346baa921818ae3e +generation_runtime="$runtime/core/v2/generations/$generation" +/bin/mkdir -m 0700 "$generation_runtime" "$generation_runtime/modules" || + emit_error E_RUNTIME + +snapshot_file "$source_dir/scan-state.sh" "$runtime/bootstrap.sh" 1048576 0400 || + emit_error E_RUNTIME +bootstrap_sha=$(sha256_path "$runtime/bootstrap.sh") || emit_error E_RUNTIME +snapshot_file "$self" "$runtime/launcher.sh" 1048576 0400 || emit_error E_RUNTIME +launcher_sha=$(sha256_path "$runtime/launcher.sh") || emit_error E_RUNTIME +snapshot_expected daaf761762722730d61e882f97794478afd5a157bab5e556be4e7169dbc4cc04 \ + "$source_dir/state-scanner-driver.sh" "$runtime/driver.sh" 0400 || + emit_error E_STALE +snapshot_expected 8838c85aae5a2ed9ada659ae1a13c5cf8f561463789d1d5d9f28370d479f6c80 \ + "$source_dir/state-scanner.jq" "$runtime/program.jq" 0400 || emit_error E_STALE +snapshot_expected f55b697716dc13a6d2c71bde7769493b3f4b091fd7a94d3280c5d417974df3a1 \ + "$repo/core/v2/generation-registry.json" \ + "$runtime/core/v2/generation-registry.json" 0400 || emit_error E_STALE +snapshot_expected 65eb40b9afb9b4f1d809ed66d0f2ca625f656c34e856cedcde9cbbde857f0f0a \ + "$repo/core/v2/generations/$generation/contracts.jq" \ + "$generation_runtime/contracts.jq" 0400 || emit_error E_STALE +snapshot_expected db87c6e97e93dc2a6eebd83087878c04f5528badc620d57fc9d883694e2ac28b \ + "$repo/core/v2/generations/$generation/core-ingress.sh" \ + "$generation_runtime/core-ingress.sh" 0400 || emit_error E_STALE +for member in \ + 'profile_graph.jq c00f9cfbe88df5cb1dbcfbead61288ff7d68684d43d095e74f26e7820f0d7207' \ + 'result_facts.jq 8e49c2c091f1bbe525f7499e3fca072f6916a14d5bb34adbf121439e8ca2d281' \ + 'result_truth.jq ed992f26761d08e3c3f5ab57eda9bcd771ad59e3aebeb02643de88844184d2d3' \ + 'schema.jq 8d1d02d36ac7ada778f05248f9413062b3fc251499914c15d79f003bbd009ade' \ + 'stage_request.jq 6572a6ecbac332dc9c4a8ef35acd1feebdc2e8aab04941fc0b756f3a5cbcf29e'; do + read -r name digest <<<"$member" + snapshot_expected "$digest" \ + "$repo/core/v2/generations/$generation/modules/$name" \ + "$generation_runtime/modules/$name" 0400 || emit_error E_STALE +done +snapshot_expected bdb5def832e8e611bba8a7b30a2aae95ea4f2701c44b198cf51cd3dfd9ff88f3 \ + "$repo/scripts/core-contract.sh" "$runtime/scripts/core-contract.sh" 0400 || + emit_error E_STALE +snapshot_expected "$jq_sha" "$jq_source" "$runtime/jq" 0500 || emit_error E_RUNTIME +bash_sha=$(sha256_path /bin/bash) || emit_error E_RUNTIME +snapshot_file "$input" "$scratch/input.json" 1048576 0400 || { + status=$? + [ "$status" -eq 42 ] && emit_error E_LIMIT + emit_error E_RUNTIME +} +snapshot_sha=$(sha256_path "$scratch/input.json") || emit_error E_RUNTIME + +"$runtime/jq" -S -c -n \ + --arg bootstrap_sha "$bootstrap_sha" --arg launcher_sha "$launcher_sha" \ + --arg driver_sha daaf761762722730d61e882f97794478afd5a157bab5e556be4e7169dbc4cc04 \ + --arg program_sha 8838c85aae5a2ed9ada659ae1a13c5cf8f561463789d1d5d9f28370d479f6c80 \ + --arg jq_sha "$jq_sha" --arg bash_sha "$bash_sha" \ + --arg host_os "$host_os" --arg host_arch "$host_arch" \ + --arg jq_arch "$jq_arch" --arg execution_mode "$execution_mode" ' + def ref($id;$media;$sha): {content_id:$id,media_type:$media,sha256:$sha}; + { + schema_version:1,kind:"orchestrator_state_scanner_evaluator", + id:"orchestrator.state-scanner.v1", + body:{ + core_contract:{ + generation_id_sha256:"6f6acbbd0cf40ab3c913328d6c0070635424ffe920bcdb900fbd0718345d7137", + package_ref:ref("core-contract-package.v2"; + "application/vnd.ystack.core-contract+json"; + "005431c5c7e3a39dc3ab75dfcafd0f09359331667fdcacb140514a4384592716"), + semantic_identity:"core.contracts.v2" + }, + core_closure:[ + {path:"core/v2/generation-registry.json",sha256:"f55b697716dc13a6d2c71bde7769493b3f4b091fd7a94d3280c5d417974df3a1"}, + {path:"core/v2/generations/g-392d20099dfa99872764009b268c8871914b4dbc0da467ec346baa921818ae3e/contracts.jq",sha256:"65eb40b9afb9b4f1d809ed66d0f2ca625f656c34e856cedcde9cbbde857f0f0a"}, + {path:"core/v2/generations/g-392d20099dfa99872764009b268c8871914b4dbc0da467ec346baa921818ae3e/core-ingress.sh",sha256:"db87c6e97e93dc2a6eebd83087878c04f5528badc620d57fc9d883694e2ac28b"}, + {path:"core/v2/generations/g-392d20099dfa99872764009b268c8871914b4dbc0da467ec346baa921818ae3e/modules/profile_graph.jq",sha256:"c00f9cfbe88df5cb1dbcfbead61288ff7d68684d43d095e74f26e7820f0d7207"}, + {path:"core/v2/generations/g-392d20099dfa99872764009b268c8871914b4dbc0da467ec346baa921818ae3e/modules/result_facts.jq",sha256:"8e49c2c091f1bbe525f7499e3fca072f6916a14d5bb34adbf121439e8ca2d281"}, + {path:"core/v2/generations/g-392d20099dfa99872764009b268c8871914b4dbc0da467ec346baa921818ae3e/modules/result_truth.jq",sha256:"ed992f26761d08e3c3f5ab57eda9bcd771ad59e3aebeb02643de88844184d2d3"}, + {path:"core/v2/generations/g-392d20099dfa99872764009b268c8871914b4dbc0da467ec346baa921818ae3e/modules/schema.jq",sha256:"8d1d02d36ac7ada778f05248f9413062b3fc251499914c15d79f003bbd009ade"}, + {path:"core/v2/generations/g-392d20099dfa99872764009b268c8871914b4dbc0da467ec346baa921818ae3e/modules/stage_request.jq",sha256:"6572a6ecbac332dc9c4a8ef35acd1feebdc2e8aab04941fc0b756f3a5cbcf29e"}, + {path:"scripts/core-contract.sh",sha256:"bdb5def832e8e611bba8a7b30a2aae95ea4f2701c44b198cf51cd3dfd9ff88f3"} + ], + bootstrap_ref:ref("orchestrator-state-scanner-bootstrap.v1";"text/x-shellscript";$bootstrap_sha), + launcher_ref:ref("orchestrator-state-scanner-launcher.v1";"text/x-shellscript";$launcher_sha), + driver_ref:ref("orchestrator-state-scanner-driver.v1";"text/x-shellscript";$driver_sha), + program_ref:ref("orchestrator-state-scanner-program.v1";"text/x-jq";$program_sha), + runtime:{ + host_os:$host_os,host_architecture:$host_arch,jq_architecture:$jq_arch, + execution_mode:$execution_mode, + jq_ref:ref("jq-runtime.v1";"application/x-executable";$jq_sha), + shell_ref:ref("bash-runtime";"application/x-executable";$bash_sha) + } + } + } +' > "$runtime/evaluator.json" 2>/dev/null || emit_error E_RUNTIME +evaluator_sha=$(sha256_path "$runtime/evaluator.json") || emit_error E_RUNTIME + +verify_private_core() { + [ "$(sha256_path "$runtime/core/v2/generation-registry.json")" = \ + f55b697716dc13a6d2c71bde7769493b3f4b091fd7a94d3280c5d417974df3a1 ] && + [ "$(sha256_path "$generation_runtime/contracts.jq")" = \ + 65eb40b9afb9b4f1d809ed66d0f2ca625f656c34e856cedcde9cbbde857f0f0a ] && + [ "$(sha256_path "$generation_runtime/core-ingress.sh")" = \ + db87c6e97e93dc2a6eebd83087878c04f5528badc620d57fc9d883694e2ac28b ] && + [ "$(sha256_path "$generation_runtime/modules/profile_graph.jq")" = \ + c00f9cfbe88df5cb1dbcfbead61288ff7d68684d43d095e74f26e7820f0d7207 ] && + [ "$(sha256_path "$generation_runtime/modules/result_facts.jq")" = \ + 8e49c2c091f1bbe525f7499e3fca072f6916a14d5bb34adbf121439e8ca2d281 ] && + [ "$(sha256_path "$generation_runtime/modules/result_truth.jq")" = \ + ed992f26761d08e3c3f5ab57eda9bcd771ad59e3aebeb02643de88844184d2d3 ] && + [ "$(sha256_path "$generation_runtime/modules/schema.jq")" = \ + 8d1d02d36ac7ada778f05248f9413062b3fc251499914c15d79f003bbd009ade ] && + [ "$(sha256_path "$generation_runtime/modules/stage_request.jq")" = \ + 6572a6ecbac332dc9c4a8ef35acd1feebdc2e8aab04941fc0b756f3a5cbcf29e ] && + [ "$(sha256_path "$runtime/scripts/core-contract.sh")" = \ + bdb5def832e8e611bba8a7b30a2aae95ea4f2701c44b198cf51cd3dfd9ff88f3 ] +} + +output="$scratch/output.json" +error="$scratch/error" +/usr/bin/env -i LC_ALL=C PATH=/usr/bin:/bin TMPDIR="$scratch" \ + /bin/bash "$runtime/driver.sh" run \ + "$expected_repository_id" "$expected_commit_id" "$runtime" \ + "$scratch/input.json" "$runtime/evaluator.json" "$evaluator_sha" \ + "$snapshot_sha" >"$output" 2>"$error" +status=$? +if [ "$status" -ne 0 ]; then + [ ! -s "$output" ] || emit_error E_RUNTIME + driver_error=$(/bin/cat "$error" 2>/dev/null) || emit_error E_RUNTIME + case "$driver_error" in + E_RUNTIME|E_LIMIT|E_PARSE|E_CANONICAL|E_SHAPE|E_RELATION|E_STALE) + emit_error "$driver_error" + ;; + *) emit_error E_RUNTIME ;; + esac +fi +[ ! -s "$error" ] || emit_error E_RUNTIME + +item_count=$("$runtime/jq" -r '.body.items | length' "$scratch/input.json") || + emit_error E_RUNTIME +i=0 +while [ "$i" -lt "$item_count" ]; do + content=$("$runtime/jq" -S -c ".body.items[$i]" "$scratch/input.json") || + emit_error E_RUNTIME + [ "$(sha256_line "$content")" = \ + "$("$runtime/jq" -r ".[$i]" "$scratch/work/item-shas.json")" ] || + emit_error E_RUNTIME + i=$((i + 1)) +done + +verify_private_core || emit_error E_STALE +"$runtime/jq" -n -e -L "$generation_runtime/modules" \ + --arg scanner_operation validate-observation \ + --arg expected_repository_id "$expected_repository_id" \ + --arg expected_commit_id "$expected_commit_id" \ + --arg snapshot_sha256 "$snapshot_sha" \ + --arg evaluator_sha256 "$evaluator_sha" \ + --slurpfile evaluator_docs "$runtime/evaluator.json" \ + --slurpfile item_sha_docs "$scratch/work/item-shas.json" \ + --slurpfile snapshot_docs "$scratch/input.json" \ + --slurpfile candidate_docs "$output" \ + -f "$runtime/program.jq" >/dev/null 2>&1 || emit_error E_RUNTIME +verify_private_core || emit_error E_STALE +[ "$(sha256_path "$runtime/bootstrap.sh")" = "$bootstrap_sha" ] && +[ "$(sha256_path "$runtime/launcher.sh")" = "$launcher_sha" ] && +[ "$(sha256_path "$runtime/driver.sh")" = \ + daaf761762722730d61e882f97794478afd5a157bab5e556be4e7169dbc4cc04 ] && +[ "$(sha256_path "$runtime/program.jq")" = \ + 8838c85aae5a2ed9ada659ae1a13c5cf8f561463789d1d5d9f28370d479f6c80 ] && +[ "$(sha256_path "$runtime/jq")" = "$jq_sha" ] && +[ "$(sha256_path /bin/bash)" = "$bash_sha" ] && +[ "$(sha256_path "$scratch/input.json")" = "$snapshot_sha" ] && +[ "$(sha256_path "$runtime/evaluator.json")" = "$evaluator_sha" ] || + emit_error E_RUNTIME +if ! /bin/cat "$output" 2>"$scratch/delivery-error"; then + emit_error E_RUNTIME +fi +trap - EXIT HUP INT TERM +cleanup diff --git a/orchestrator/v1/state-scanner.jq b/orchestrator/v1/state-scanner.jq index adc7ef3..6ccbc1c 100644 --- a/orchestrator/v1/state-scanner.jq +++ b/orchestrator/v1/state-scanner.jq @@ -13,12 +13,81 @@ def present_shape(value_ok): (schema::exact_fields(["state","value"];[]) and .state == "present" and (.value | value_ok)); -def core_contract_shape: - schema::exact_fields( - ["generation_id_sha256","package_ref","semantic_identity"];[]) and - (.generation_id_sha256 | schema::sha256_ok) and - (.package_ref | schema::content_ref_ok) and - (.semantic_identity | schema::id_ok); +def expected_core: + { + generation_id_sha256: + "6f6acbbd0cf40ab3c913328d6c0070635424ffe920bcdb900fbd0718345d7137", + package_ref:{ + content_id:"core-contract-package.v2", + media_type:"application/vnd.ystack.core-contract+json", + sha256:"005431c5c7e3a39dc3ab75dfcafd0f09359331667fdcacb140514a4384592716" + }, + semantic_identity:"core.contracts.v2" + }; + +def expected_core_closure: + [ + {path:"core/v2/generation-registry.json", + sha256:"f55b697716dc13a6d2c71bde7769493b3f4b091fd7a94d3280c5d417974df3a1"}, + {path:"core/v2/generations/g-392d20099dfa99872764009b268c8871914b4dbc0da467ec346baa921818ae3e/contracts.jq", + sha256:"65eb40b9afb9b4f1d809ed66d0f2ca625f656c34e856cedcde9cbbde857f0f0a"}, + {path:"core/v2/generations/g-392d20099dfa99872764009b268c8871914b4dbc0da467ec346baa921818ae3e/core-ingress.sh", + sha256:"db87c6e97e93dc2a6eebd83087878c04f5528badc620d57fc9d883694e2ac28b"}, + {path:"core/v2/generations/g-392d20099dfa99872764009b268c8871914b4dbc0da467ec346baa921818ae3e/modules/profile_graph.jq", + sha256:"c00f9cfbe88df5cb1dbcfbead61288ff7d68684d43d095e74f26e7820f0d7207"}, + {path:"core/v2/generations/g-392d20099dfa99872764009b268c8871914b4dbc0da467ec346baa921818ae3e/modules/result_facts.jq", + sha256:"8e49c2c091f1bbe525f7499e3fca072f6916a14d5bb34adbf121439e8ca2d281"}, + {path:"core/v2/generations/g-392d20099dfa99872764009b268c8871914b4dbc0da467ec346baa921818ae3e/modules/result_truth.jq", + sha256:"ed992f26761d08e3c3f5ab57eda9bcd771ad59e3aebeb02643de88844184d2d3"}, + {path:"core/v2/generations/g-392d20099dfa99872764009b268c8871914b4dbc0da467ec346baa921818ae3e/modules/schema.jq", + sha256:"8d1d02d36ac7ada778f05248f9413062b3fc251499914c15d79f003bbd009ade"}, + {path:"core/v2/generations/g-392d20099dfa99872764009b268c8871914b4dbc0da467ec346baa921818ae3e/modules/stage_request.jq", + sha256:"6572a6ecbac332dc9c4a8ef35acd1feebdc2e8aab04941fc0b756f3a5cbcf29e"}, + {path:"scripts/core-contract.sh", + sha256:"bdb5def832e8e611bba8a7b30a2aae95ea4f2701c44b198cf51cd3dfd9ff88f3"} + ]; + +def ref_shape($content_id; $media_type): + schema::content_ref_ok and + .content_id == $content_id and .media_type == $media_type; + +def evaluator_shape: + schema::exact_fields(["body","id","kind","schema_version"];[]) and + .schema_version == 1 and + .kind == "orchestrator_state_scanner_evaluator" and + .id == "orchestrator.state-scanner.v1" and + (.body | + schema::exact_fields( + ["bootstrap_ref","core_closure","core_contract","driver_ref", + "launcher_ref","program_ref","runtime"];[]) and + .core_contract == expected_core and + .core_closure == expected_core_closure and + (.bootstrap_ref | + ref_shape("orchestrator-state-scanner-bootstrap.v1";"text/x-shellscript")) and + (.launcher_ref | + ref_shape("orchestrator-state-scanner-launcher.v1";"text/x-shellscript")) and + (.driver_ref | + ref_shape("orchestrator-state-scanner-driver.v1";"text/x-shellscript")) and + (.program_ref | + ref_shape("orchestrator-state-scanner-program.v1";"text/x-jq")) and + (.runtime | + schema::exact_fields( + ["execution_mode","host_architecture","host_os","jq_architecture", + "jq_ref","shell_ref"];[]) and + (.jq_ref | ref_shape("jq-runtime.v1";"application/x-executable")) and + (.shell_ref | ref_shape("bash-runtime";"application/x-executable")) and + ((.host_os == "linux" and .host_architecture == "x86_64" and + .jq_architecture == "x86_64" and .execution_mode == "native" and + .jq_ref.sha256 == + "af986793a515d500ab2d35f8d2aecd656e764504b789b66d7e1a0b727a124c44") or + (.host_os == "darwin" and .host_architecture == "x86_64" and + .jq_architecture == "x86_64" and .execution_mode == "native" and + .jq_ref.sha256 == + "5c0a0a3ea600f302ee458b30317425dd9632d1ad8882259fcaf4e9b868b2b1ef") or + (.host_os == "darwin" and .host_architecture == "arm64" and + .jq_architecture == "x86_64" and .execution_mode == "rosetta" and + .jq_ref.sha256 == + "5c0a0a3ea600f302ee458b30317425dd9632d1ad8882259fcaf4e9b868b2b1ef")))); def attempt_value_shape: schema::exact_fields( @@ -40,29 +109,27 @@ def item_shape: .retry_limit >= 1 and .retry_limit <= 10; def snapshot_shape: + . as $snapshot | schema::exact_fields(["body","id","kind","schema_version"];[]) and .schema_version == 1 and .kind == "orchestrator_state_snapshot" and (.id | schema::id_ok) and (.body | schema::exact_fields( - ["core_contract","items","observed_at","source_revision"];[]) and - (.core_contract | core_contract_shape) and + ["core_contract","items","observed_at","snapshot_contract", + "source_revision"];[]) and + (.core_contract | type == "object") and (.source_revision | schema::git_revision_ref_ok) and (.observed_at | schema::time_ok) and - (.items | type == "array" and length >= 1 and length <= 64 and - all(.[];item_shape))); - -def expected_core: - { - generation_id_sha256: - "6f6acbbd0cf40ab3c913328d6c0070635424ffe920bcdb900fbd0718345d7137", - package_ref:{ - content_id:"core-contract-package.v2", - media_type:"application/vnd.ystack.core-contract+json", - sha256:"005431c5c7e3a39dc3ab75dfcafd0f09359331667fdcacb140514a4384592716" - }, - semantic_identity:"core.contracts.v2" - }; + (.items | type == "array" and length <= 64 and all(.[];item_shape)) and + (.snapshot_contract | + schema::exact_fields( + ["completeness","declared_item_count","maximum_item_count", + "schema_identity"];[]) and + .completeness == "complete" and + .schema_identity == "orchestrator.state-snapshot.v1" and + .maximum_item_count == 64 and + (.declared_item_count | schema::int_ok) and + .declared_item_count == ($snapshot.body.items | length))); def stage_key: .request.content.body | @@ -94,8 +161,8 @@ def item_relation($source; $observed_at): else true end); -def set_relations: - .body.items as $items | +def set_relations($snapshot): + $snapshot.body.items as $items | ($items | map(stage_key)) as $keys | ($keys == ($keys | sort)) and (($keys | length) == ($keys | unique | length)) and @@ -124,28 +191,46 @@ def attempt_number($item): def target_moved($item; $source): $item.request.content.body.target_revision.state == "present" and - $item.request.content.body.target_revision.value != $source and - ($item.latest_result.state == "absent" or - ($item.latest_result.value.content.body.status != "completed" and - $item.latest_result.value.content.body.status != "skipped")); + $item.request.content.body.target_revision.value != $source; + +def snapshot_ref($snapshot; $snapshot_sha): + { + schema_identity:$snapshot.body.snapshot_contract.schema_identity, + kind:$snapshot.kind, + id:$snapshot.id, + sha256:$snapshot_sha + }; -def classification($item; $source; $observed_at): +def evaluator_ref($evaluator_sha): + { + content_id:"orchestrator-state-scanner-evaluator.v1", + media_type:"application/vnd.ystack.orchestrator-state-scanner-evaluator+json", + sha256:$evaluator_sha + }; + +def item_ref($item_sha): + {schema_identity:"orchestrator.state-item.v1",sha256:$item_sha}; + +def classification( + $item; $item_sha; $source; $observed_at; $snapshot_ref; $evaluator_ref): ($item.latest_result.value.content.body.status // null) as $status | (attempt_number($item)) as $attempt_number | (source_reason($item)) as $source_reason | - (if target_moved($item;$source) then - ["stale","refresh-stage-inputs","scanner.target-revision-moved"] - elif $status == "completed" then + (if $status == "completed" then ["terminal","none","scanner.stage-completed"] elif $status == "skipped" then ["terminal","none","scanner.stage-skipped"] + elif $status == "cancelled" then + ["terminal","none","scanner.stage-cancelled"] + elif target_moved($item;$source) then + ["stale","refresh-stage-inputs","scanner.target-revision-moved"] elif $status == "stale" then ["stale","refresh-stage-inputs","scanner.stage-stale"] elif $status == "blocked" then ["blocked","resolve-stage-blocker","scanner.stage-blocked"] - elif $status == "failed" or $status == "cancelled" then + elif $status == "failed" then if $attempt_number < $item.retry_limit then - ["retryable","retry-stage","scanner.stage-" + $status] + ["retryable","retry-stage","scanner.stage-failed"] else ["blocked","operator-reconcile","scanner.retry-limit-reached"] end elif $item.attempt.state == "present" and @@ -163,40 +248,88 @@ def classification($item; $source; $observed_at): task_class_id:$item.request.content.body.task_class_id }, class:$decision[0], + provenance:{ + snapshot_ref:$snapshot_ref, + evaluator_ref:$evaluator_ref, + item_ref:item_ref($item_sha), + request_ref:($item.request | profile::document_ref_for_pair(.)), + resolved_profile_ref: + ($item.resolved_profile | profile::document_ref_for_pair(.)), + latest_result_ref: + (if $item.latest_result.state == "present" then + {state:"present",value: + ($item.latest_result.value | profile::document_ref_for_pair(.))} + else {state:"absent"} end), + active_attempt:$item.attempt + }, recovery:{ action:$decision[1], reason_id:$decision[2], - source_reason_id:$source_reason, + source_reason:$source_reason, attempt_number:$attempt_number, retry_limit:$item.retry_limit } }; -. as $snapshot | -if (snapshot_shape | not) then "E_SHAPE" -elif $snapshot.body.core_contract != expected_core then "E_STALE" -elif $snapshot.body.source_revision.repository_id != $expected_repository_id or - $snapshot.body.source_revision.commit_id != $expected_commit_id then "E_STALE" -elif (all($snapshot.body.items[]; - item_relation( - $snapshot.body.source_revision;$snapshot.body.observed_at)) | not) -then "E_RELATION" -elif ($snapshot | set_relations | not) then "E_RELATION" -else { - schema_version:1, - kind:"orchestrator_state_observation", - id:$snapshot.id, - body:{ - activation_state:"inactive", - authority_effect:"none", - mode:"observation-only", - core_contract:$snapshot.body.core_contract, - source_revision:$snapshot.body.source_revision, - observed_at:$snapshot.body.observed_at, - classifications:[ - $snapshot.body.items[] | - classification(.;$snapshot.body.source_revision;$snapshot.body.observed_at) - ] - } -} +def observation($snapshot; $snapshot_sha; $item_shas; $evaluator; $evaluator_sha): + (snapshot_ref($snapshot;$snapshot_sha)) as $snapshot_ref | + (evaluator_ref($evaluator_sha)) as $evaluator_ref | + { + schema_version:1, + kind:"orchestrator_state_observation", + id:$snapshot.id, + body:{ + activation_state:"inactive", + authority_effect:"none", + mode:"observation-only", + core_contract:$snapshot.body.core_contract, + source_revision:$snapshot.body.source_revision, + observed_at:$snapshot.body.observed_at, + snapshot_ref:$snapshot_ref, + evaluator:{content:$evaluator,sha256:$evaluator_sha}, + classifications:[ + range(0;($snapshot.body.items | length)) as $index | + classification( + $snapshot.body.items[$index];$item_shas[$index]; + $snapshot.body.source_revision;$snapshot.body.observed_at; + $snapshot_ref;$evaluator_ref) + ] + } + }; + +def evaluate($snapshot; $snapshot_sha; $item_shas; $evaluator; $evaluator_sha): + if (($snapshot | snapshot_shape) | not) then "E_SHAPE" + elif $snapshot.body.core_contract != expected_core then "E_STALE" + elif ($item_shas | type) != "array" or + ($item_shas | length) != ($snapshot.body.items | length) or + (all($item_shas[];schema::sha256_ok) | not) then "E_RELATION" + elif ($snapshot_sha | schema::sha256_ok | not) or + ($evaluator_sha | schema::sha256_ok | not) or + (($evaluator | evaluator_shape) | not) then "E_STALE" + elif $snapshot.body.source_revision.repository_id != $expected_repository_id or + $snapshot.body.source_revision.commit_id != $expected_commit_id then "E_STALE" + elif (all($snapshot.body.items[]; + item_relation( + $snapshot.body.source_revision;$snapshot.body.observed_at)) | not) + then "E_RELATION" + elif (set_relations($snapshot) | not) then "E_RELATION" + else observation($snapshot;$snapshot_sha;$item_shas;$evaluator;$evaluator_sha) + end; + +if $scanner_operation == "scan" then + if ($evaluator_docs | length) != 1 or ($item_sha_docs | length) != 1 + then "E_STALE" + else evaluate( + .;$snapshot_sha256;$item_sha_docs[0];$evaluator_docs[0];$evaluator_sha256) + end +elif $scanner_operation == "validate-observation" then + if ($snapshot_docs | length) != 1 or ($evaluator_docs | length) != 1 or + ($candidate_docs | length) != 1 or ($item_sha_docs | length) != 1 then false + else + evaluate( + $snapshot_docs[0];$snapshot_sha256;$item_sha_docs[0]; + $evaluator_docs[0];$evaluator_sha256) as $expected | + ($expected | type) == "object" and $candidate_docs[0] == $expected + end +else false end diff --git a/scripts/test/orchestrator-state-scanner.test.sh b/scripts/test/orchestrator-state-scanner.test.sh index dd1cc7c..0b81925 100755 --- a/scripts/test/orchestrator-state-scanner.test.sh +++ b/scripts/test/orchestrator-state-scanner.test.sh @@ -1,5 +1,5 @@ #!/bin/bash -# shellcheck disable=SC2016 +# shellcheck disable=SC2016,SC2329 set -euo pipefail export LC_ALL=C @@ -18,19 +18,37 @@ sha_line() { platform=$(/usr/bin/uname -s):$(/usr/bin/uname -m) case "$platform" in - Darwin:*) jq_asset=jq-osx-amd64; jq_sha=5c0a0a3ea600f302ee458b30317425dd9632d1ad8882259fcaf4e9b868b2b1ef ;; - Linux:x86_64) jq_asset=jq-linux64; jq_sha=af986793a515d500ab2d35f8d2aecd656e764504b789b66d7e1a0b727a124c44 ;; + Darwin:x86_64) + jq_asset=jq-osx-amd64; jq_sha=5c0a0a3ea600f302ee458b30317425dd9632d1ad8882259fcaf4e9b868b2b1ef + expected_host_arch=x86_64; expected_execution_mode=native + ;; + Darwin:arm64) + jq_asset=jq-osx-amd64; jq_sha=5c0a0a3ea600f302ee458b30317425dd9632d1ad8882259fcaf4e9b868b2b1ef + expected_host_arch=arm64; expected_execution_mode=rosetta + ;; + Linux:x86_64) + jq_asset=jq-linux64; jq_sha=af986793a515d500ab2d35f8d2aecd656e764504b789b66d7e1a0b727a124c44 + expected_host_arch=x86_64; expected_execution_mode=native + ;; *) /usr/bin/printf 'FAIL: unsupported host %s\n' "$platform" >&2; exit 1 ;; esac -system_jq=$(command -v jq) -jq_source=$system_jq -if [ "$($system_jq --version 2>/dev/null)" != jq-1.6 ]; then - jq_source="${TMPDIR:-/tmp}/ystack-portable-core-jq16/$jq_asset" - [ -f "$jq_source" ] && [ "$(sha_file "$jq_source")" = "$jq_sha" ] || { - /usr/bin/printf '%s\n' 'FAIL: verified jq 1.6 required' >&2 - exit 1 - } -fi +jq_source='' +for candidate in "${TMPDIR:-/tmp}/ystack-portable-core-jq16/$jq_asset" \ + /usr/bin/jq "$(command -v jq)"; do + if [ -f "$candidate" ] && [ ! -L "$candidate" ] && + [ "$(sha_file "$candidate")" = "$jq_sha" ]; then + jq_source=$candidate + break + fi +done +[ -n "$jq_source" ] || { + /usr/bin/printf '%s\n' 'FAIL: verified architecture-bound jq 1.6 required' >&2 + exit 1 +} +case "$platform" in + Darwin:*) /usr/bin/file "$jq_source" | /usr/bin/grep -Fq x86_64 ;; + Linux:x86_64) /usr/bin/file "$jq_source" | /usr/bin/grep -Eq 'x86-64|x86_64' ;; +esac /bin/mkdir -m 700 "$tmp/bin" /bin/cp "$jq_source" "$tmp/bin/jq" /bin/chmod 0555 "$tmp/bin/jq" @@ -47,21 +65,63 @@ pass() { passed=$((passed + 1)); } fail() { /usr/bin/printf 'FAIL: %s\n' "$1" >&2; failed=$((failed + 1)); } expect_class() { - local name=$1 snapshot=$2 commit=$3 class=$4 action=$5 reason=$6 output + local name=$1 snapshot=$2 commit=$3 class=$4 action=$5 reason=$6 + local output snapshot_sha item_content item_sha + snapshot_sha=$(sha_file "$snapshot") + item_content=$("$jq_bin" -S -c '.body.items[0]' "$snapshot") + item_sha=$(sha_line "$item_content") if ! output=$("$scanner" scan repo.example "$commit" "$snapshot" 2>"$tmp/$name.err"); then fail "$name returned $(<"$tmp/$name.err")" return fi if /usr/bin/printf '%s\n' "$output" | "$jq_bin" -e -S -c \ - --arg class "$class" --arg action "$action" --arg reason "$reason" ' + --arg class "$class" --arg action "$action" --arg reason "$reason" \ + --arg snapshot_sha "$snapshot_sha" --arg item_sha "$item_sha" \ + --arg host_arch "$expected_host_arch" \ + --arg execution_mode "$expected_execution_mode" --slurpfile snapshot "$snapshot" ' .schema_version == 1 and .kind == "orchestrator_state_observation" and + (.body | keys) == ["activation_state","authority_effect","classifications", + "core_contract","evaluator","mode","observed_at","snapshot_ref", + "source_revision"] and .body.activation_state == "inactive" and .body.authority_effect == "none" and .body.mode == "observation-only" and + .body.snapshot_ref == {schema_identity:"orchestrator.state-snapshot.v1", + kind:"orchestrator_state_snapshot",id:$snapshot[0].id,sha256:$snapshot_sha} and + .body.evaluator.sha256 == + .body.classifications[0].provenance.evaluator_ref.sha256 and + (.body.evaluator.content.body.bootstrap_ref.sha256 | + test("^[0-9a-f]{64}$")) and + .body.evaluator.content.body.program_ref.sha256 == + "8838c85aae5a2ed9ada659ae1a13c5cf8f561463789d1d5d9f28370d479f6c80" and + .body.evaluator.content.body.driver_ref.sha256 == + "daaf761762722730d61e882f97794478afd5a157bab5e556be4e7169dbc4cc04" and + .body.evaluator.content.body.runtime.host_architecture == $host_arch and + .body.evaluator.content.body.runtime.execution_mode == $execution_mode and + (.body.evaluator.content.body.core_closure | length) == 9 and (.body.classifications | length) == 1 and + (.body.classifications[0] | keys) == ["class","provenance","recovery","stage_key"] and + (.body.classifications[0].provenance | keys) == ["active_attempt","evaluator_ref", + "item_ref","latest_result_ref","request_ref","resolved_profile_ref","snapshot_ref"] and + (.body.classifications[0].recovery | keys) == ["action","attempt_number", + "reason_id","retry_limit","source_reason"] and + .body.classifications[0].provenance.snapshot_ref == .body.snapshot_ref and + .body.classifications[0].provenance.item_ref == + {schema_identity:"orchestrator.state-item.v1",sha256:$item_sha} and + .body.classifications[0].provenance.request_ref.sha256 == + $snapshot[0].body.items[0].request.sha256 and + .body.classifications[0].provenance.resolved_profile_ref.sha256 == + $snapshot[0].body.items[0].resolved_profile.sha256 and + .body.classifications[0].provenance.active_attempt == + $snapshot[0].body.items[0].attempt and + .body.classifications[0].provenance.latest_result_ref == + (if $snapshot[0].body.items[0].latest_result.state == "present" then + {state:"present",value:{schema_version:2,kind:"stage_result", + id:$snapshot[0].body.items[0].latest_result.value.content.id, + sha256:$snapshot[0].body.items[0].latest_result.value.sha256}} + else {state:"absent"} end) and .body.classifications[0].class == $class and .body.classifications[0].recovery.action == $action and - .body.classifications[0].recovery.reason_id == $reason and - ((tojson | test("grant|approval|qualification|publish|schedule|wake")) | not) + .body.classifications[0].recovery.reason_id == $reason ' >/dev/null && [ "$output" = "$(/usr/bin/printf '%s\n' "$output" | "$jq_bin" -S -c .)" ]; then pass @@ -187,6 +247,8 @@ make_snapshot() { retry_limit:$retry_limit }], observed_at:"2026-08-30T00:10:00Z", + snapshot_contract:{completeness:"complete",declared_item_count:1, + maximum_item_count:64,schema_identity:"orchestrator.state-snapshot.v1"}, source_revision:{repository_id:"repo.example",hash_algorithm:"sha1",commit_id:$commit} } } @@ -205,6 +267,71 @@ commit_two=2222222222222222222222222222222222222222 pending="$tmp/pending.json" make_snapshot "$pending" absent absent 2026-08-30T00:20:00Z 2 "$commit_one" expect_class pending "$pending" "$commit_one" pending dispatch-stage scanner.no-attempt +baseline_output=$("$scanner" scan repo.example "$commit_one" "$pending") +malicious_path="$tmp/malicious-path" +/bin/mkdir "$malicious_path" +{ + /usr/bin/printf '%s\n' '#!/bin/bash' + /usr/bin/printf ': > "%s"\n' "$tmp/path-jq-ran" + /usr/bin/printf '%s\n' 'exit 0' +} >"$malicious_path/jq" +/bin/chmod 0500 "$malicious_path/jq" +path_output=$(PATH="$malicious_path:/usr/bin:/bin" \ + "$scanner" scan repo.example "$commit_one" "$pending") +if [ "$path_output" = "$baseline_output" ] && [ ! -e "$tmp/path-jq-ran" ]; then + pass +else + fail 'writable PATH runtime is ignored' +fi + +poison_dir="$tmp/perl-poison" +/bin/mkdir "$poison_dir" +{ + /usr/bin/printf '%s\n' 'package ScannerPoison;' + /usr/bin/printf 'BEGIN { open(my $fh, ">", "%s") or die; print {$fh} "ran\\n"; close($fh) or die; }\n' \ + "$tmp/perl-poison-ran" + /usr/bin/printf '%s\n' '1;' +} >"$poison_dir/ScannerPoison.pm" +compgen() { /usr/bin/printf '%s\n' LC_ALL PATH PWD SHLVL TMPDIR; } +export -f compgen +poison_output=$(PERL5LIB="$poison_dir" PERL5OPT=-MScannerPoison \ + "$scanner" scan repo.example "$commit_one" "$pending" 2>"$tmp/perl-poison.err") +unset -f compgen +if [ "$poison_output" = "$baseline_output" ] && [ ! -s "$tmp/perl-poison.err" ] && + [ ! -e "$tmp/perl-poison-ran" ]; then + pass +else + fail 'ambient Perl options are removed before snapshotting' +fi + +set +e +"$scanner" scan repo.example "$commit_one" "$pending" \ + >&- 2>"$tmp/closed-output.err" +closed_output_status=$? +set -e +if [ "$closed_output_status" -ne 0 ] && + [ "$(<"$tmp/closed-output.err")" = E_RUNTIME ]; then + pass +else + fail 'failed output delivery cannot return success' +fi + +empty="$tmp/empty.json" +"$jq_bin" -S -c ' + .id="snapshot.empty" | .body.items=[] | + .body.snapshot_contract.declared_item_count=0 +' "$pending" >"$empty" +if empty_output=$("$scanner" scan repo.example "$commit_one" "$empty") && + /usr/bin/printf '%s\n' "$empty_output" | "$jq_bin" -e \ + --arg sha "$(sha_file "$empty")" ' + .id == "snapshot.empty" and .body.classifications == [] and + .body.snapshot_ref.sha256 == $sha and + .body.evaluator.content.kind == "orchestrator_state_scanner_evaluator" + ' >/dev/null; then + pass +else + fail 'empty complete snapshot' +fi inflight="$tmp/inflight.json" make_snapshot "$inflight" absent started 2026-08-30T00:20:00Z 2 "$commit_one" @@ -220,7 +347,7 @@ for spec in \ 'stale stale refresh-stage-inputs scanner.stage-stale 2' \ 'blocked blocked resolve-stage-blocker scanner.stage-blocked 2' \ 'failed retryable retry-stage scanner.stage-failed 2' \ - 'cancelled retryable retry-stage scanner.stage-cancelled 2' \ + 'cancelled terminal none scanner.stage-cancelled 2' \ 'failed blocked operator-reconcile scanner.retry-limit-reached 1'; do read -r flavor expected_class action reason retry_limit <<<"$spec" snapshot="$tmp/$flavor-$retry_limit.json" @@ -239,12 +366,48 @@ make_snapshot "$completed_moved" "$tmp/result-completed.json" absent \ 2026-08-30T00:20:00Z 2 "$commit_two" expect_class immutable-terminal "$completed_moved" "$commit_two" terminal none scanner.stage-completed +cancelled_moved="$tmp/cancelled-moved.json" +make_snapshot "$cancelled_moved" "$tmp/result-cancelled.json" absent \ + 2026-08-30T00:20:00Z 2 "$commit_two" +expect_class immutable-cancellation "$cancelled_moved" "$commit_two" terminal none scanner.stage-cancelled + +other_pending="$tmp/pending-other.json" +"$jq_bin" -S -c \ + '.body.items[0].request.content.body.requested_by.principal_id="principal.other"' \ + "$pending" >"$other_pending.raw" +other_content=$("$jq_bin" -S -c '.body.items[0].request.content' "$other_pending.raw") +other_request_sha=$(sha_line "$other_content") +"$jq_bin" -S -c --arg sha "$other_request_sha" \ + '.body.items[0].request.sha256=$sha' "$other_pending.raw" >"$other_pending" +other_output=$("$scanner" scan repo.example "$commit_one" "$other_pending") +if [ "$baseline_output" != "$other_output" ] && + [ "$(/usr/bin/printf '%s\n' "$other_output" | "$jq_bin" -r \ + '.body.classifications[0].provenance.request_ref.sha256')" = "$other_request_sha" ] && + [ "$(/usr/bin/printf '%s\n' "$other_output" | "$jq_bin" -r \ + '.body.snapshot_ref.sha256')" = "$(sha_file "$other_pending")" ]; then + pass +else + fail 'distinct snapshots retain distinct provenance' +fi + bad_core="$tmp/bad-core.json" "$jq_bin" -S -c '.body.core_contract.semantic_identity="core.contracts.v9"' \ "$pending" >"$bad_core" expect_error stale-core E_STALE "$bad_core" expect_error stale-snapshot E_STALE "$pending" "$commit_two" +truncated="$tmp/truncated.json" +"$jq_bin" -S -c '.body.snapshot_contract.declared_item_count=0' \ + "$pending" >"$truncated" +expect_error incomplete-count E_SHAPE "$truncated" +partial="$tmp/partial.json" +"$jq_bin" -S -c '.body.snapshot_contract.completeness="partial"' \ + "$pending" >"$partial" +expect_error partial-page E_SHAPE "$partial" +paged="$tmp/paged.json" +"$jq_bin" -S -c '.body.snapshot_contract.page_cursor="next"' "$pending" >"$paged" +expect_error ambiguous-page-field E_SHAPE "$paged" + bad_sha="$tmp/bad-sha.json" "$jq_bin" -S -c '.body.items[0].request.sha256=("0"*64)' "$pending" >"$bad_sha" expect_error content-ref-hash E_RELATION "$bad_sha" @@ -260,13 +423,17 @@ time_travel="$tmp/time-travel.json" expect_error observation-before-request E_RELATION "$time_travel" duplicate="$tmp/duplicate.json" -"$jq_bin" -S -c '.body.items += [.body.items[0]]' "$pending" >"$duplicate" +"$jq_bin" -S -c ' + .body.items += [.body.items[0]] | + .body.snapshot_contract.declared_item_count=2 +' "$pending" >"$duplicate" expect_error duplicate-stage E_RELATION "$duplicate" unordered="$tmp/unordered.json" "$jq_bin" -S -c ' .body.items=[(.body.items[0] | .request.content.body.stage_id="stage.z"), - (.body.items[0] | .request.content.body.stage_id="stage.a")] + (.body.items[0] | .request.content.body.stage_id="stage.a")] | + .body.snapshot_contract.declared_item_count=2 ' "$pending" >"$unordered.raw" for index in 0 1; do content=$("$jq_bin" -S -c ".body.items[$index].request.content" "$unordered.raw") @@ -279,7 +446,10 @@ done expect_error unordered-stage E_RELATION "$unordered" too_many="$tmp/too-many.json" -"$jq_bin" -S -c '.body.items=[range(0;65) as $n | {}]' \ +"$jq_bin" -S -c ' + .body.items=[range(0;65) as $n | {}] | + .body.snapshot_contract.declared_item_count=65 +' \ "$pending" >"$too_many" expect_error item-bound E_SHAPE "$too_many" oversize="$tmp/oversize.json" @@ -300,6 +470,100 @@ link="$tmp/input-link.json" /bin/ln -s "$pending" "$link" expect_error symlink-input E_RUNTIME "$link" +copy_root="$tmp/scanner-copy" +copy_modules="$copy_root/core/v2/generations/g-392d20099dfa99872764009b268c8871914b4dbc0da467ec346baa921818ae3e/modules" +/bin/mkdir -p "$copy_root/orchestrator/v1" "$copy_modules" "$copy_root/scripts" +/bin/cp "$root/orchestrator/v1/scan-state.sh" \ + "$root/orchestrator/v1/state-scanner-launcher.sh" \ + "$root/orchestrator/v1/state-scanner-driver.sh" \ + "$root/orchestrator/v1/state-scanner.jq" "$copy_root/orchestrator/v1/" +/bin/cp "$root/core/v2/generation-registry.json" "$copy_root/core/v2/" +/bin/cp "$root/core/v2/generations/g-392d20099dfa99872764009b268c8871914b4dbc0da467ec346baa921818ae3e/contracts.jq" \ + "$root/core/v2/generations/g-392d20099dfa99872764009b268c8871914b4dbc0da467ec346baa921818ae3e/core-ingress.sh" \ + "$copy_root/core/v2/generations/g-392d20099dfa99872764009b268c8871914b4dbc0da467ec346baa921818ae3e/" +/bin/cp "$root/core/v2/generations/g-392d20099dfa99872764009b268c8871914b4dbc0da467ec346baa921818ae3e/modules/"*.jq \ + "$copy_modules/" +/bin/cp "$root/scripts/core-contract.sh" "$copy_root/scripts/" +/bin/chmod 0500 "$copy_root/orchestrator/v1/scan-state.sh" +copy_scanner="$copy_root/orchestrator/v1/scan-state.sh" +race_tmp="$tmp/race-tmp" +/bin/mkdir -p "$race_tmp/ystack-portable-core-jq16" +/bin/cp "$jq_source" "$race_tmp/ystack-portable-core-jq16/$jq_asset" +/bin/chmod 0500 "$race_tmp/ystack-portable-core-jq16/$jq_asset" + +program_copy="$copy_root/orchestrator/v1/state-scanner.jq" +/bin/cp "$program_copy" "$tmp/program-saved" +/usr/bin/printf '\n' >>"$program_copy" +set +e +tampered_output=$(TMPDIR="$race_tmp" "$copy_scanner" scan \ + repo.example "$commit_one" "$pending" 2>"$tmp/tampered-program.err") +tampered_status=$? +set -e +if [ "$tampered_status" -ne 0 ] && [ -z "$tampered_output" ] && + [ "$(<"$tmp/tampered-program.err")" = E_STALE ]; then + pass +else + fail 'same-inode program mutation fails closed' +fi +/bin/cp "$tmp/program-saved" "$program_copy" + +/bin/mv "$program_copy" "$tmp/program-real" +/bin/ln -s "$tmp/program-real" "$program_copy" +set +e +symlink_output=$(TMPDIR="$race_tmp" "$copy_scanner" scan \ + repo.example "$commit_one" "$pending" 2>"$tmp/symlink-program.err") +symlink_status=$? +set -e +if [ "$symlink_status" -ne 0 ] && [ -z "$symlink_output" ] && + [ "$(<"$tmp/symlink-program.err")" = E_STALE ]; then + pass +else + fail 'swapped program symlink fails closed' +fi +/bin/rm "$program_copy" +/bin/mv "$tmp/program-real" "$program_copy" + +race_cache="$race_tmp/ystack-portable-core-jq16/$jq_asset" +/bin/cp "$program_copy" "$tmp/program-race-saved" +/bin/cp "$race_cache" "$tmp/jq-race-saved" +TMPDIR="$race_tmp" "$copy_scanner" scan repo.example "$commit_one" "$pending" \ + >"$tmp/race-output" 2>"$tmp/race-error" & +race_pid=$! +runtime_ready='' +for _ in {1..200}; do + for candidate in "$race_tmp"/ystack-state-scan.*/runtime; do + if [ -f "$candidate/program.jq" ] && [ -f "$candidate/jq" ]; then + runtime_ready=$candidate + break 2 + fi + done + /bin/sleep 0.01 +done +if [ -n "$runtime_ready" ] && kill -0 "$race_pid" 2>/dev/null; then + /usr/bin/printf '\n' >>"$program_copy" + /bin/chmod 0700 "$race_cache" + /usr/bin/printf '\000' >>"$race_cache" + program_size=$(/usr/bin/wc -c <"$tmp/program-race-saved" | /usr/bin/tr -d ' ') + jq_size=$(/usr/bin/wc -c <"$tmp/jq-race-saved" | /usr/bin/tr -d ' ') + /bin/dd if="$tmp/program-race-saved" of="$program_copy" bs=65536 conv=notrunc 2>/dev/null + /usr/bin/truncate -s "$program_size" "$program_copy" + /bin/dd if="$tmp/jq-race-saved" of="$race_cache" bs=65536 conv=notrunc 2>/dev/null + /usr/bin/truncate -s "$jq_size" "$race_cache" + /bin/chmod 0500 "$race_cache" +else + fail 'private runtime was not observable before completion' +fi +set +e +wait "$race_pid" +race_status=$? +set -e +if [ "$race_status" -eq 0 ] && [ ! -s "$tmp/race-error" ] && + [ "$(<"$tmp/race-output")" = "$baseline_output" ]; then + pass +else + fail 'private snapshot survives source swap-restore' +fi + if [ "$failed" -ne 0 ]; then /usr/bin/printf 'orchestrator state scanner: %s passed, %s failed\n' "$passed" "$failed" >&2 exit 1 From 6bc83adb5de4d9aafce97a7bed63cb2b68888284 Mon Sep 17 00:00:00 2001 From: ci Date: Wed, 2 Sep 2026 06:50:19 -0400 Subject: [PATCH 3/5] Document inactive canonical state scanner --- README.md | 18 ++++++++++++++++++ RESTORE.md | 14 ++++++++++++++ ci/required-files.txt | 7 +++++++ 3 files changed, 39 insertions(+) diff --git a/README.md b/README.md index 5c57cf1..e4f7972 100644 --- a/README.md +++ b/README.md @@ -183,6 +183,24 @@ references. The package stays inactive, stores nothing, grants no authority, and performs no candidate, credential, network, adapter, publish, deploy, or external write. +## Inactive canonical state scanner + +`orchestrator/v1/scan-state.sh` reads one bounded canonical snapshot that binds an +explicit Git repository and commit and carries canonical stage request, resolved +profile, attempt, and result records. Run it with the same repository and commit +identities: + +```text +orchestrator/v1/scan-state.sh scan REPOSITORY_ID COMMIT_ID SNAPSHOT.json +``` + +It emits a deterministic canonical observation that classifies each stage as +terminal, stale, blocked, retryable, stranded, or pending, with a recovery action +and reason. Those fields are messages for later recovery work, not commands. The +scanner is inactive and observation only: it does not deliver events, schedule, +dispatch, retry, reconcile, write state, use a credential or network, activate a +profile, or touch a target. + ## The current default team You talk **only** to yshifu, in a Claude Code session. yshifu orchestrates the other roles diff --git a/RESTORE.md b/RESTORE.md index 2764659..061c911 100644 --- a/RESTORE.md +++ b/RESTORE.md @@ -422,6 +422,20 @@ launcher is not self-attested. It does not read proof bytes, establish proof truth, qualify a workflow, store evidence, grant authority, activate a profile, run a candidate or adapter, or perform a network or external-write action. +Restore the five paths in the manifest's inactive canonical state scanner block +from the same commit. With the same pinned, architecture-bound jq 1.6 runtime used +by the portable core, run: + +```sh +bash scripts/test/orchestrator-state-scanner.test.sh +``` + +This checks bounded canonical snapshots, exact repository and commit binding, +deterministic pending and stranded classifications, recovery reasons, private +runtime snapshots, and fail-closed input handling. The scanner remains inactive +and observation only. It does not deliver or retry events, reconcile or write +state, use a credential or network, activate a profile, or touch a target. + --- ## 5. Smoke test — prove the rebuilt team is alive diff --git a/ci/required-files.txt b/ci/required-files.txt index 118a0e9..915c905 100644 --- a/ci/required-files.txt +++ b/ci/required-files.txt @@ -217,3 +217,10 @@ scripts/test/control-evidence-integrity.test.sh # Inactive Control foundation policy roll-up control/v1/control-policy-set.json scripts/test/control-foundation-rollup.test.sh + +# Inactive canonical orchestrator state scanner +orchestrator/v1/scan-state.sh +orchestrator/v1/state-scanner-launcher.sh +orchestrator/v1/state-scanner-driver.sh +orchestrator/v1/state-scanner.jq +scripts/test/orchestrator-state-scanner.test.sh From 2c8e6e4f867d28901aedb594f74d1e2c59af6d46 Mon Sep 17 00:00:00 2001 From: ci Date: Wed, 2 Sep 2026 07:22:01 -0400 Subject: [PATCH 4/5] Sync scanner paths with portable core guard --- scripts/test/portable-core-schema.test.sh | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/scripts/test/portable-core-schema.test.sh b/scripts/test/portable-core-schema.test.sh index 00e6c74..6bfe76c 100755 --- a/scripts/test/portable-core-schema.test.sh +++ b/scripts/test/portable-core-schema.test.sh @@ -761,7 +761,11 @@ v2_activation_path_ok() { README.md|RESTORE.md|ci/required-files.txt|\ control/v1/control-policy-set.json|\ core/v2/generation-registry.json|\ + orchestrator/v1/state-scanner-driver.sh|\ + orchestrator/v1/state-scanner-launcher.sh|\ + orchestrator/v1/state-scanner.jq|\ scripts/core-contract.sh|scripts/lib/profile-resolution.sh|\ + scripts/test/orchestrator-state-scanner.test.sh|\ scripts/test/portable-core-schema.test.sh|\ scripts/test/portable-core-v2-fake-forge.test.sh) ;; *) v2_generation_path_ok "$1" || return 1 ;; @@ -772,6 +776,7 @@ schema_import_path_ok() { local import_path="$1" local test_path case "$import_path" in + orchestrator/v1/state-scanner.jq) ;; scripts/test/portable-core-*) test_path="${import_path#scripts/test/}" case "$test_path" in */*) return 1 ;; esac @@ -842,8 +847,12 @@ printf '%s\n' \ control/v1/control-policy-set.json \ core/v2/generation-registry.json \ "core/v2/generations/$schema_v2_generation/core-ingress.sh" \ + orchestrator/v1/state-scanner-driver.sh \ + orchestrator/v1/state-scanner-launcher.sh \ + orchestrator/v1/state-scanner.jq \ scripts/core-contract.sh \ scripts/lib/profile-resolution.sh \ + scripts/test/orchestrator-state-scanner.test.sh \ scripts/test/portable-core-schema.test.sh \ scripts/test/portable-core-v2-fake-forge.test.sh > \ "$schema_v2_expected_live_hits" @@ -890,8 +899,12 @@ printf '%s\n' \ "core/v2/generations/$schema_v2_generation/modules/result_truth.jq" \ "core/v2/generations/$schema_v2_generation/modules/schema.jq" \ "core/v2/generations/$schema_v2_generation/modules/stage_request.jq" \ + orchestrator/v1/state-scanner-driver.sh \ + orchestrator/v1/state-scanner-launcher.sh \ + orchestrator/v1/state-scanner.jq \ scripts/core-contract.sh \ scripts/lib/profile-resolution.sh \ + scripts/test/orchestrator-state-scanner.test.sh \ scripts/test/portable-core-schema.test.sh \ scripts/test/portable-core-v2-fake-forge.test.sh > "$schema_v2_allowed_paths" schema_v2_injected_source="$schema_test_tmp/v2-injected-source" From 123d1778c14974e4b1d663a3a8d45d9a0c96e182 Mon Sep 17 00:00:00 2001 From: ci Date: Wed, 2 Sep 2026 08:11:23 -0400 Subject: [PATCH 5/5] Stream scanner document hashes without argv --- orchestrator/v1/state-scanner-driver.sh | 2 +- orchestrator/v1/state-scanner-launcher.sh | 8 +-- .../test/orchestrator-state-scanner.test.sh | 70 ++++++++++++++++++- 3 files changed, 72 insertions(+), 8 deletions(-) diff --git a/orchestrator/v1/state-scanner-driver.sh b/orchestrator/v1/state-scanner-driver.sh index 59f0500..86a486d 100755 --- a/orchestrator/v1/state-scanner-driver.sh +++ b/orchestrator/v1/state-scanner-driver.sh @@ -18,7 +18,7 @@ sha256_path() { } sha256_line() { - /usr/bin/printf '%s\n' "$1" | /usr/bin/shasum -a 256 | + builtin printf '%s\n' "$1" | /usr/bin/shasum -a 256 | /usr/bin/awk '{print $1}' } diff --git a/orchestrator/v1/state-scanner-launcher.sh b/orchestrator/v1/state-scanner-launcher.sh index e73beed..5a7f351 100755 --- a/orchestrator/v1/state-scanner-launcher.sh +++ b/orchestrator/v1/state-scanner-launcher.sh @@ -19,7 +19,7 @@ sha256_path() { } sha256_line() { - /usr/bin/printf '%s\n' "$1" | /usr/bin/shasum -a 256 | + builtin printf '%s\n' "$1" | /usr/bin/shasum -a 256 | /usr/bin/awk '{print $1}' } @@ -129,7 +129,7 @@ snapshot_file "$source_dir/scan-state.sh" "$runtime/bootstrap.sh" 1048576 0400 | bootstrap_sha=$(sha256_path "$runtime/bootstrap.sh") || emit_error E_RUNTIME snapshot_file "$self" "$runtime/launcher.sh" 1048576 0400 || emit_error E_RUNTIME launcher_sha=$(sha256_path "$runtime/launcher.sh") || emit_error E_RUNTIME -snapshot_expected daaf761762722730d61e882f97794478afd5a157bab5e556be4e7169dbc4cc04 \ +snapshot_expected 3f0b14cddd27ef7638b3227159af686defa5f5662c893096cc6711a692d57d1a \ "$source_dir/state-scanner-driver.sh" "$runtime/driver.sh" 0400 || emit_error E_STALE snapshot_expected 8838c85aae5a2ed9ada659ae1a13c5cf8f561463789d1d5d9f28370d479f6c80 \ @@ -168,7 +168,7 @@ snapshot_sha=$(sha256_path "$scratch/input.json") || emit_error E_RUNTIME "$runtime/jq" -S -c -n \ --arg bootstrap_sha "$bootstrap_sha" --arg launcher_sha "$launcher_sha" \ - --arg driver_sha daaf761762722730d61e882f97794478afd5a157bab5e556be4e7169dbc4cc04 \ + --arg driver_sha 3f0b14cddd27ef7638b3227159af686defa5f5662c893096cc6711a692d57d1a \ --arg program_sha 8838c85aae5a2ed9ada659ae1a13c5cf8f561463789d1d5d9f28370d479f6c80 \ --arg jq_sha "$jq_sha" --arg bash_sha "$bash_sha" \ --arg host_os "$host_os" --arg host_arch "$host_arch" \ @@ -280,7 +280,7 @@ verify_private_core || emit_error E_STALE [ "$(sha256_path "$runtime/bootstrap.sh")" = "$bootstrap_sha" ] && [ "$(sha256_path "$runtime/launcher.sh")" = "$launcher_sha" ] && [ "$(sha256_path "$runtime/driver.sh")" = \ - daaf761762722730d61e882f97794478afd5a157bab5e556be4e7169dbc4cc04 ] && + 3f0b14cddd27ef7638b3227159af686defa5f5662c893096cc6711a692d57d1a ] && [ "$(sha256_path "$runtime/program.jq")" = \ 8838c85aae5a2ed9ada659ae1a13c5cf8f561463789d1d5d9f28370d479f6c80 ] && [ "$(sha256_path "$runtime/jq")" = "$jq_sha" ] && diff --git a/scripts/test/orchestrator-state-scanner.test.sh b/scripts/test/orchestrator-state-scanner.test.sh index 0b81925..3d0869f 100755 --- a/scripts/test/orchestrator-state-scanner.test.sh +++ b/scripts/test/orchestrator-state-scanner.test.sh @@ -12,8 +12,8 @@ trap cleanup EXIT sha_file() { /usr/bin/shasum -a 256 "$1" | /usr/bin/awk '{print $1}'; } sha_line() { - /usr/bin/printf '%s\n' "$1" | /usr/bin/shasum -a 256 | - /usr/bin/awk '{print $1}' + builtin printf '%s\n' "$1" | /usr/bin/shasum -a 256 | + /usr/bin/awk '{print $1}' } platform=$(/usr/bin/uname -s):$(/usr/bin/uname -m) @@ -94,7 +94,7 @@ expect_class() { .body.evaluator.content.body.program_ref.sha256 == "8838c85aae5a2ed9ada659ae1a13c5cf8f561463789d1d5d9f28370d479f6c80" and .body.evaluator.content.body.driver_ref.sha256 == - "daaf761762722730d61e882f97794478afd5a157bab5e556be4e7169dbc4cc04" and + "3f0b14cddd27ef7638b3227159af686defa5f5662c893096cc6711a692d57d1a" and .body.evaluator.content.body.runtime.host_architecture == $host_arch and .body.evaluator.content.body.runtime.execution_mode == $execution_mode and (.body.evaluator.content.body.core_closure | length) == 9 and @@ -268,6 +268,70 @@ pending="$tmp/pending.json" make_snapshot "$pending" absent absent 2026-08-30T00:20:00Z 2 "$commit_one" expect_class pending "$pending" "$commit_one" pending dispatch-stage scanner.no-attempt baseline_output=$("$scanner" scan repo.example "$commit_one" "$pending") + +hash_helpers_builtin=true +for hash_script in \ + "$root/orchestrator/v1/state-scanner-driver.sh" \ + "$root/orchestrator/v1/state-scanner-launcher.sh"; do + hash_body=$(/usr/bin/sed -n '/^sha256_line() {$/,/^}$/p' "$hash_script") + if [[ "$hash_body" != *"builtin printf '%s\n' \"\$1\""* ]] || + [[ "$hash_body" == *'/usr/bin/printf'* ]]; then + hash_helpers_builtin=false + fi +done +if [ "$hash_helpers_builtin" = true ]; then + pass +else + fail 'document hashing must stream through builtin printf' +fi + +large_request="$tmp/request-large.json" +"$jq_bin" -S -c ' + def digest($n): (("0" * 64) + ($n | tostring))[-64:]; + def long_id($prefix;$n): + ($n | tostring) as $suffix | + $prefix + ("x" * (128 - ($prefix | length) - ($suffix | length))) + $suffix; + def media_type: "application/" + ("x" * 115); + def content($prefix;$n): + {content_id:long_id($prefix;$n),media_type:media_type,sha256:digest($n)}; + def gate($n): { + purpose:"gate-requirement", + decision_record_ref:content("decision.";$n), + subject_ref:{ + type:"artifact", + value:{type:"content",value:content("subject.";$n)} + }, + scope_sha256:digest($n) + }; + def evidence($n): { + stage_result_ref:{ + schema_version:2, + kind:"stage_result", + id:long_id("result.";$n), + sha256:digest($n) + }, + evidence_id:long_id("evidence.";$n) + }; + .body.risk.required_gate_refs=[range(0;256) | gate(.)] | + .body.prior_evidence_refs=[range(0;256) | evidence(.)] +' "$request" >"$large_request" +large_request_sha=$(sha_file "$large_request") +large_snapshot="$tmp/snapshot-large.json" +"$jq_bin" -S -c --slurpfile request "$large_request" \ + --arg request_sha "$large_request_sha" ' + .body.items[0].request={content:$request[0],sha256:$request_sha} + ' "$pending" >"$large_snapshot" +large_request_size=$(/usr/bin/wc -c <"$large_request" | /usr/bin/tr -d ' ') +large_snapshot_size=$(/usr/bin/wc -c <"$large_snapshot" | /usr/bin/tr -d ' ') +if [ "$large_request_size" -gt 131072 ] && + [ "$large_snapshot_size" -gt "$large_request_size" ] && + [ "$large_snapshot_size" -le 1048576 ]; then + expect_class large-valid-request "$large_snapshot" "$commit_one" \ + pending dispatch-stage scanner.no-attempt +else + fail 'large valid request did not cross the per-argument threshold' +fi + malicious_path="$tmp/malicious-path" /bin/mkdir "$malicious_path" {