From 319e777ceb26501b737f04005c9c8a233b925c3c Mon Sep 17 00:00:00 2001 From: ci Date: Sat, 29 Aug 2026 22:33:22 -0400 Subject: [PATCH 01/13] core: add bounded canonical ingress --- ci/required-files.txt | 4 + .../core-ingress.sh | 428 ++++++++++ .../test/portable-core-ingress-fixtures.json | 82 ++ scripts/test/portable-core-ingress-ledger.tsv | 41 + scripts/test/portable-core-ingress.test.sh | 795 ++++++++++++++++++ 5 files changed, 1350 insertions(+) create mode 100644 core/v1/generations/g-14b7ad8ce54c3b8c585ff92063d71551ffc7394cc2294d0297bc7d2b8da2c386/core-ingress.sh create mode 100644 scripts/test/portable-core-ingress-fixtures.json create mode 100644 scripts/test/portable-core-ingress-ledger.tsv create mode 100755 scripts/test/portable-core-ingress.test.sh diff --git a/ci/required-files.txt b/ci/required-files.txt index 9bb46d9..e9d3956 100644 --- a/ci/required-files.txt +++ b/ci/required-files.txt @@ -94,3 +94,7 @@ core/v1/generations/g-14b7ad8ce54c3b8c585ff92063d71551ffc7394cc2294d0297bc7d2b8d scripts/test/portable-core-schema-fixtures.json scripts/test/portable-core-schema-ledger.tsv scripts/test/portable-core-schema.test.sh +core/v1/generations/g-14b7ad8ce54c3b8c585ff92063d71551ffc7394cc2294d0297bc7d2b8da2c386/core-ingress.sh +scripts/test/portable-core-ingress-fixtures.json +scripts/test/portable-core-ingress-ledger.tsv +scripts/test/portable-core-ingress.test.sh diff --git a/core/v1/generations/g-14b7ad8ce54c3b8c585ff92063d71551ffc7394cc2294d0297bc7d2b8da2c386/core-ingress.sh b/core/v1/generations/g-14b7ad8ce54c3b8c585ff92063d71551ffc7394cc2294d0297bc7d2b8da2c386/core-ingress.sh new file mode 100644 index 0000000..572e847 --- /dev/null +++ b/core/v1/generations/g-14b7ad8ce54c3b8c585ff92063d71551ffc7394cc2294d0297bc7d2b8da2c386/core-ingress.sh @@ -0,0 +1,428 @@ +#!/usr/bin/env bash +# shellcheck disable=SC2016,SC2034 + +portable_core_ingress_error() { + case "${1:-}" in + E_RUNTIME|E_PARSE|E_CANONICAL|E_LIMIT|E_SHAPE|E_REF|E_RELATION) + printf '%s\n' "$1" >&2 + ;; + *) + printf '%s\n' E_RUNTIME >&2 + ;; + esac + return 1 +} + +portable_core_ingress_regular_file() { + [ -f "$1" ] && [ ! -L "$1" ] +} + +portable_core_ingress_real_directory() { + [ -d "$1" ] && [ ! -L "$1" ] +} + +portable_core_ingress_open() { + local source_dir + local repo_root + local expected_dir + local jq_path + local schema_identity + local sha_path + local temp_path + local required_dir + + [ "$#" -eq 0 ] || { + portable_core_ingress_error E_RUNTIME + return 1 + } + + PORTABLE_CORE_INGRESS_GENERATION='g-14b7ad8ce54c3b8c585ff92063d71551ffc7394cc2294d0297bc7d2b8da2c386' + source_dir="$( + CDPATH='' cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")" 2>/dev/null && pwd -P + )" || { + portable_core_ingress_error E_RUNTIME + return 1 + } + repo_root="$(CDPATH='' cd -P -- "$source_dir/../../../.." 2>/dev/null && pwd -P)" || { + portable_core_ingress_error E_RUNTIME + return 1 + } + expected_dir="$repo_root/core/v1/generations/$PORTABLE_CORE_INGRESS_GENERATION" + [ "$source_dir" = "$expected_dir" ] || { + portable_core_ingress_error E_RUNTIME + return 1 + } + for required_dir in \ + "$repo_root" \ + "$repo_root/core" \ + "$repo_root/core/v1" \ + "$repo_root/core/v1/generations" \ + "$expected_dir" \ + "$expected_dir/modules"; do + portable_core_ingress_real_directory "$required_dir" || { + portable_core_ingress_error E_RUNTIME + return 1 + } + done + + PORTABLE_CORE_INGRESS_REPO_ROOT="$repo_root" + PORTABLE_CORE_INGRESS_MODULE_DIR="$expected_dir/modules" + PORTABLE_CORE_INGRESS_SCHEMA="$PORTABLE_CORE_INGRESS_MODULE_DIR/schema.jq" + PORTABLE_CORE_INGRESS_ROOT="$expected_dir/contracts.jq" + portable_core_ingress_regular_file "$expected_dir/core-ingress.sh" || { + portable_core_ingress_error E_RUNTIME + return 1 + } + portable_core_ingress_regular_file "$PORTABLE_CORE_INGRESS_SCHEMA" || { + portable_core_ingress_error E_RUNTIME + return 1 + } + + jq_path="$(command -v jq 2>/dev/null)" || { + portable_core_ingress_error E_RUNTIME + return 1 + } + case "$jq_path" in + /*) ;; + *) + portable_core_ingress_error E_RUNTIME + return 1 + ;; + esac + [ "$("$jq_path" --version 2>/dev/null)" = jq-1.6 ] || { + portable_core_ingress_error E_RUNTIME + return 1 + } + schema_identity="$( + CDPATH='' cd -P -- "$PORTABLE_CORE_INGRESS_MODULE_DIR" 2>/dev/null && + HOME=/nonexistent JQ_LIBRARY_PATH=/nonexistent \ + "$jq_path" -L "$PORTABLE_CORE_INGRESS_MODULE_DIR" -nr \ + 'import "schema" as schema; schema::semantic_identity' 2>/dev/null + )" || { + portable_core_ingress_error E_RUNTIME + return 1 + } + [ "$schema_identity" = core.contracts.v1 ] || { + portable_core_ingress_error E_RUNTIME + return 1 + } + PORTABLE_CORE_INGRESS_JQ="$jq_path" + + if sha_path="$(command -v sha256sum 2>/dev/null)"; then + PORTABLE_CORE_INGRESS_SHA_BACKEND=sha256sum + elif sha_path="$(command -v shasum 2>/dev/null)"; then + PORTABLE_CORE_INGRESS_SHA_BACKEND=shasum + else + portable_core_ingress_error E_RUNTIME + return 1 + fi + case "$sha_path" in + /*) PORTABLE_CORE_INGRESS_SHA="$sha_path" ;; + *) + portable_core_ingress_error E_RUNTIME + return 1 + ;; + esac + + PORTABLE_CORE_INGRESS_HEAD="$(command -v head 2>/dev/null)" || { + portable_core_ingress_error E_RUNTIME + return 1 + } + PORTABLE_CORE_INGRESS_WC="$(command -v wc 2>/dev/null)" || { + portable_core_ingress_error E_RUNTIME + return 1 + } + PORTABLE_CORE_INGRESS_CMP="$(command -v cmp 2>/dev/null)" || { + portable_core_ingress_error E_RUNTIME + return 1 + } + PORTABLE_CORE_INGRESS_CAT="$(command -v cat 2>/dev/null)" || { + portable_core_ingress_error E_RUNTIME + return 1 + } + PORTABLE_CORE_INGRESS_RM="$(command -v rm 2>/dev/null)" || { + portable_core_ingress_error E_RUNTIME + return 1 + } + + temp_path="$(mktemp -d /tmp/ystack-portable-core-ingress.XXXXXXXX 2>/dev/null)" || { + portable_core_ingress_error E_RUNTIME + return 1 + } + case "$temp_path" in + /tmp/ystack-portable-core-ingress.*) ;; + *) + portable_core_ingress_error E_RUNTIME + return 1 + ;; + esac + portable_core_ingress_real_directory "$temp_path" || { + portable_core_ingress_error E_RUNTIME + return 1 + } + PORTABLE_CORE_INGRESS_TEMP="$temp_path" + PORTABLE_CORE_INGRESS_MODE='' + PORTABLE_CORE_INGRESS_CONTENTS='' + PORTABLE_CORE_INGRESS_HASHES='' + PORTABLE_CORE_INGRESS_DRIVER='' + PORTABLE_CORE_INGRESS_OUTPUT='' + PORTABLE_CORE_INGRESS_SNAPSHOT='' + PORTABLE_CORE_INGRESS_SHA256='' + PORTABLE_CORE_INGRESS_COUNT=0 + PORTABLE_CORE_INGRESS_RAW_PATHS=() + PORTABLE_CORE_INGRESS_RAW_SIZES=() + PORTABLE_CORE_INGRESS_CANONICAL_PATHS=() +} + +portable_core_ingress_begin() { + [ "$#" -eq 1 ] || { + portable_core_ingress_error E_RUNTIME + return 1 + } + case "$1" in + document|profile-set|stage-run) ;; + *) + portable_core_ingress_error E_RUNTIME + return 1 + ;; + esac + portable_core_ingress_real_directory "${PORTABLE_CORE_INGRESS_TEMP:-}" || { + portable_core_ingress_error E_RUNTIME + return 1 + } + PORTABLE_CORE_INGRESS_MODE="$1" + PORTABLE_CORE_INGRESS_CONTENTS="$PORTABLE_CORE_INGRESS_TEMP/contents.ndjson" + PORTABLE_CORE_INGRESS_HASHES="$PORTABLE_CORE_INGRESS_TEMP/hashes.ndjson" + : 2>/dev/null > "$PORTABLE_CORE_INGRESS_CONTENTS" || { + portable_core_ingress_error E_RUNTIME + return 1 + } + : 2>/dev/null > "$PORTABLE_CORE_INGRESS_HASHES" || { + portable_core_ingress_error E_RUNTIME + return 1 + } + PORTABLE_CORE_INGRESS_COUNT=0 + PORTABLE_CORE_INGRESS_RAW_PATHS=() + PORTABLE_CORE_INGRESS_RAW_SIZES=() + PORTABLE_CORE_INGRESS_CANONICAL_PATHS=() +} + +portable_core_ingress_digest() { + local input_path="$1" + local digest_output + local digest + + case "$PORTABLE_CORE_INGRESS_SHA_BACKEND" in + sha256sum) + digest_output="$("$PORTABLE_CORE_INGRESS_SHA" -- "$input_path" 2>/dev/null)" || { + portable_core_ingress_error E_RUNTIME + return 1 + } + ;; + shasum) + digest_output="$("$PORTABLE_CORE_INGRESS_SHA" -a 256 -- "$input_path" 2>/dev/null)" || { + portable_core_ingress_error E_RUNTIME + return 1 + } + ;; + *) + portable_core_ingress_error E_RUNTIME + return 1 + ;; + esac + read -r digest _ <<< "$digest_output" + [[ "$digest" =~ ^[0-9a-f]{64}$ ]] && [[ "$digest_output" != *$'\n'* ]] || { + portable_core_ingress_error E_RUNTIME + return 1 + } + PORTABLE_CORE_INGRESS_SHA256="$digest" +} + +portable_core_ingress_snapshot() { + local input_path + local snapshot_number + local raw_path + local byte_count + + [ "$#" -eq 1 ] || { + portable_core_ingress_error E_RUNTIME + return 1 + } + input_path="$1" + [ -n "${PORTABLE_CORE_INGRESS_MODE:-}" ] && + portable_core_ingress_real_directory "${PORTABLE_CORE_INGRESS_TEMP:-}" && + [ -r "$input_path" ] || { + portable_core_ingress_error E_RUNTIME + return 1 + } + + snapshot_number=$((PORTABLE_CORE_INGRESS_COUNT + 1)) + raw_path="$PORTABLE_CORE_INGRESS_TEMP/raw.$snapshot_number" + if ! "$PORTABLE_CORE_INGRESS_HEAD" -c 1048577 -- "$input_path" \ + 2>/dev/null > "$raw_path"; then + portable_core_ingress_error E_RUNTIME + return 1 + fi + byte_count="$("$PORTABLE_CORE_INGRESS_WC" -c 2>/dev/null < "$raw_path")" || { + portable_core_ingress_error E_RUNTIME + return 1 + } + byte_count="${byte_count//[[:space:]]/}" + [[ "$byte_count" =~ ^[0-9]+$ ]] || { + portable_core_ingress_error E_RUNTIME + return 1 + } + PORTABLE_CORE_INGRESS_RAW_PATHS[PORTABLE_CORE_INGRESS_COUNT]="$raw_path" + PORTABLE_CORE_INGRESS_RAW_SIZES[PORTABLE_CORE_INGRESS_COUNT]="$byte_count" + PORTABLE_CORE_INGRESS_SNAPSHOT="$raw_path" + PORTABLE_CORE_INGRESS_COUNT="$snapshot_number" +} + +portable_core_ingress_finish_driver() { + local input_index + local raw_path + local canonical_path + local compare_status + + [ "$#" -eq 0 ] && [ "${PORTABLE_CORE_INGRESS_COUNT:-0}" -gt 0 ] || { + portable_core_ingress_error E_RUNTIME + return 1 + } + + for ((input_index = 0; input_index < PORTABLE_CORE_INGRESS_COUNT; input_index++)); do + if [ "${PORTABLE_CORE_INGRESS_RAW_SIZES[$input_index]}" -gt 1048576 ]; then + portable_core_ingress_error E_LIMIT + return 1 + fi + done + for ((input_index = 0; input_index < PORTABLE_CORE_INGRESS_COUNT; input_index++)); do + raw_path="${PORTABLE_CORE_INGRESS_RAW_PATHS[$input_index]}" + if ! "$PORTABLE_CORE_INGRESS_JQ" -s -e 'length == 1' "$raw_path" \ + >/dev/null 2>/dev/null; then + portable_core_ingress_error E_PARSE + return 1 + fi + canonical_path="$PORTABLE_CORE_INGRESS_TEMP/canonical.$((input_index + 1))" + if ! "$PORTABLE_CORE_INGRESS_JQ" -s -S -c \ + 'if length == 1 then .[0] else error("root-count") end' "$raw_path" \ + 2>/dev/null > "$canonical_path"; then + portable_core_ingress_error E_RUNTIME + return 1 + fi + PORTABLE_CORE_INGRESS_CANONICAL_PATHS[input_index]="$canonical_path" + done + for ((input_index = 0; input_index < PORTABLE_CORE_INGRESS_COUNT; input_index++)); do + raw_path="${PORTABLE_CORE_INGRESS_RAW_PATHS[$input_index]}" + canonical_path="${PORTABLE_CORE_INGRESS_CANONICAL_PATHS[$input_index]}" + if "$PORTABLE_CORE_INGRESS_CMP" -s -- "$raw_path" "$canonical_path" 2>/dev/null; then + : + else + compare_status=$? + if [ "$compare_status" -eq 1 ]; then + portable_core_ingress_error E_CANONICAL + else + portable_core_ingress_error E_RUNTIME + fi + return 1 + fi + done + for ((input_index = 0; input_index < PORTABLE_CORE_INGRESS_COUNT; input_index++)); do + raw_path="${PORTABLE_CORE_INGRESS_RAW_PATHS[$input_index]}" + portable_core_ingress_digest "$raw_path" || return 1 + if ! "$PORTABLE_CORE_INGRESS_CAT" -- "$raw_path" \ + 2>/dev/null >> "$PORTABLE_CORE_INGRESS_CONTENTS"; then + portable_core_ingress_error E_RUNTIME + return 1 + fi + if ! printf '"%s"\n' "$PORTABLE_CORE_INGRESS_SHA256" \ + 2>/dev/null >> "$PORTABLE_CORE_INGRESS_HASHES"; then + portable_core_ingress_error E_RUNTIME + return 1 + fi + done + PORTABLE_CORE_INGRESS_DRIVER="$PORTABLE_CORE_INGRESS_TEMP/driver.json" + if ! "$PORTABLE_CORE_INGRESS_JQ" -n -S -c \ + --arg mode "$PORTABLE_CORE_INGRESS_MODE" \ + --slurpfile contents "$PORTABLE_CORE_INGRESS_CONTENTS" \ + --slurpfile hashes "$PORTABLE_CORE_INGRESS_HASHES" \ + '{mode:$mode, + docs:([range(0;($contents|length))] | + map({content:$contents[.],sha256:$hashes[.]}))}' \ + 2>/dev/null > "$PORTABLE_CORE_INGRESS_DRIVER"; then + portable_core_ingress_error E_RUNTIME + return 1 + fi + if ! "$PORTABLE_CORE_INGRESS_JQ" -e \ + '(.docs|length) > 0 and (.docs|length) == ([.docs[].sha256]|length)' \ + "$PORTABLE_CORE_INGRESS_DRIVER" >/dev/null 2>/dev/null; then + portable_core_ingress_error E_RUNTIME + return 1 + fi +} + +portable_core_ingress_validate() { + local output_size + local token + + if [ "$#" -ne 0 ] || + ! portable_core_ingress_regular_file "${PORTABLE_CORE_INGRESS_DRIVER:-}" || + ! portable_core_ingress_regular_file "${PORTABLE_CORE_INGRESS_ROOT:-}"; then + portable_core_ingress_error E_RUNTIME + return 1 + fi + PORTABLE_CORE_INGRESS_OUTPUT="$PORTABLE_CORE_INGRESS_TEMP/validator.out" + if ! ( + CDPATH='' cd -P -- "$PORTABLE_CORE_INGRESS_MODULE_DIR" && + HOME=/nonexistent JQ_LIBRARY_PATH=/nonexistent \ + "$PORTABLE_CORE_INGRESS_JQ" -L "$PORTABLE_CORE_INGRESS_MODULE_DIR" -r \ + -f "$PORTABLE_CORE_INGRESS_ROOT" "$PORTABLE_CORE_INGRESS_DRIVER" + ) 2>/dev/null > "$PORTABLE_CORE_INGRESS_OUTPUT"; then + portable_core_ingress_error E_RUNTIME + return 1 + fi + if [ ! -s "$PORTABLE_CORE_INGRESS_OUTPUT" ]; then + return 0 + fi + output_size="$("$PORTABLE_CORE_INGRESS_WC" -c \ + 2>/dev/null < "$PORTABLE_CORE_INGRESS_OUTPUT")" || { + portable_core_ingress_error E_RUNTIME + return 1 + } + output_size="${output_size//[[:space:]]/}" + token="$("$PORTABLE_CORE_INGRESS_CAT" -- "$PORTABLE_CORE_INGRESS_OUTPUT" 2>/dev/null)" || { + portable_core_ingress_error E_RUNTIME + return 1 + } + case "$token" in + E_LIMIT|E_SHAPE|E_REF|E_RELATION) ;; + *) + portable_core_ingress_error E_RUNTIME + return 1 + ;; + esac + if [ "$output_size" -ne $((${#token} + 1)) ]; then + portable_core_ingress_error E_RUNTIME + return 1 + fi + portable_core_ingress_error "$token" +} + +portable_core_ingress_close() { + local temp_path="${PORTABLE_CORE_INGRESS_TEMP:-}" + case "$temp_path" in + /tmp/ystack-portable-core-ingress.*) + if [ -d "$temp_path" ] && [ ! -L "$temp_path" ]; then + if ! "$PORTABLE_CORE_INGRESS_RM" -rf -- "$temp_path" >/dev/null 2>&1; then + portable_core_ingress_error E_RUNTIME + return 1 + fi + fi + ;; + '') ;; + *) + portable_core_ingress_error E_RUNTIME + return 1 + ;; + esac + PORTABLE_CORE_INGRESS_TEMP='' +} diff --git a/scripts/test/portable-core-ingress-fixtures.json b/scripts/test/portable-core-ingress-fixtures.json new file mode 100644 index 0000000..4d5507f --- /dev/null +++ b/scripts/test/portable-core-ingress-fixtures.json @@ -0,0 +1,82 @@ +{ + "metadata": { + "generation": "g-14b7ad8ce54c3b8c585ff92063d71551ffc7394cc2294d0297bc7d2b8da2c386", + "parent_spec_blob": "c6511d96c1a5e6aed27ba2075b5add65c121f782", + "schema_merge_commit": "d48ecdb908a395c5205260a662db7d9d3f4c1eb4", + "schema_export_oid": "fd3924d414a7d620c2bf5de919a45c2599d572ec", + "registry_oid": "5e113105777694a280166e71d31efd19752e9562", + "review_source_sha256": "31793a3ad42acf4df117ea158a78738e056bae550269483870487c3e146b27f9", + "legacy_source_sha256": "3d5a6fb192f9bcaba5c4b89314d30f88a03b9d8a1e1e634297c267b14f096092", + "ingress_mapping_sha256": "cadaa1c5752e3cd13ee556bbae5aa7bf95130cceb044a6ef4bf569e70434e361", + "review_rows": 2, + "legacy_rows": 38, + "review_row_ids": [ + "review-r2-f04", + "review-r3-f04" + ], + "legacy_row_ids": [ + "legacy-test-039", + "legacy-test-040", + "legacy-test-041", + "legacy-test-042", + "legacy-test-043", + "legacy-test-044", + "legacy-test-045", + "legacy-test-046", + "legacy-test-047", + "legacy-test-048", + "legacy-test-049", + "legacy-test-050", + "legacy-test-051", + "legacy-test-052", + "legacy-test-053", + "legacy-test-054", + "legacy-test-055", + "legacy-test-056", + "legacy-test-057", + "legacy-test-058", + "legacy-test-059", + "legacy-test-060", + "legacy-test-061", + "legacy-test-062", + "legacy-test-063", + "legacy-test-064", + "legacy-test-081", + "legacy-test-082", + "legacy-test-267", + "legacy-test-268", + "legacy-test-272", + "legacy-test-273", + "legacy-test-274", + "legacy-test-275", + "legacy-test-276", + "legacy-test-277", + "legacy-test-278", + "legacy-test-279" + ] + }, + "canonical": { + "bytes": "{\"a\":1,\"text\":\"é\"}\n", + "sha256": "01cacc7f4a6f7ae9d6dc52fdee0dfded9b5364ecfa8a3a5a9c2fdd29a04d18f1" + }, + "owned_rules": [ + "portable-core-ingress.bounded-snapshot", + "portable-core-ingress.canonical-bytes", + "portable-core-ingress.driver-snapshot", + "portable-core-ingress.fixed-generation", + "portable-core-ingress.fixed-schema-loading", + "portable-core-ingress.jq-version", + "portable-core-ingress.private-activation-guard", + "portable-core-ingress.private-temp", + "portable-core-ingress.private-temp-errors-sanitized", + "portable-core-ingress.raw-byte-limit", + "portable-core-ingress.restore-manifest", + "portable-core-ingress.sanitized-errors", + "portable-core-ingress.sha256-tool", + "portable-core-ingress.single-root-json", + "portable-core-ingress.snapshot-readable", + "portable-core-ingress.snapshot-sha256", + "portable-core-ingress.utf8-json", + "portable-core-ingress.validator-boundary" + ] +} diff --git a/scripts/test/portable-core-ingress-ledger.tsv b/scripts/test/portable-core-ingress-ledger.tsv new file mode 100644 index 0000000..64457f9 --- /dev/null +++ b/scripts/test/portable-core-ingress-ledger.tsv @@ -0,0 +1,41 @@ +source row_id disposition rule_id test_id +review review-r2-f04 ported portable-core-ingress.private-temp-errors-sanitized portable-core-ingress.test.mktemp-failure-sanitized-e-runtime +review review-r3-f04 ported portable-core-ingress.private-temp-errors-sanitized portable-core-ingress.test.private-temp-write-failures-sanitized-e-runtime +legacy legacy-test-039 ported portable-core-ingress.snapshot-readable portable-core-ingress.test.legacy-039-validate-profile-set-unreadable-input +legacy legacy-test-040 ported portable-core-ingress.snapshot-readable portable-core-ingress.test.legacy-039-validate-profile-set-unreadable-input +legacy legacy-test-041 ported portable-core-ingress.single-root-json portable-core-ingress.test.legacy-041-empty-input +legacy legacy-test-042 ported portable-core-ingress.single-root-json portable-core-ingress.test.legacy-041-empty-input +legacy legacy-test-043 ported portable-core-ingress.single-root-json portable-core-ingress.test.legacy-043-multi-root-stream +legacy legacy-test-044 ported portable-core-ingress.single-root-json portable-core-ingress.test.legacy-043-multi-root-stream +legacy legacy-test-045 ported portable-core-ingress.canonical-bytes portable-core-ingress.test.legacy-045-bom-prefix +legacy legacy-test-046 ported portable-core-ingress.canonical-bytes portable-core-ingress.test.legacy-045-bom-prefix +legacy legacy-test-047 ported portable-core-ingress.utf8-json portable-core-ingress.test.legacy-047-invalid-utf-8 +legacy legacy-test-048 ported portable-core-ingress.utf8-json portable-core-ingress.test.legacy-047-invalid-utf-8 +legacy legacy-test-049 ported portable-core-ingress.canonical-bytes portable-core-ingress.test.legacy-049-duplicate-keys-non-canonical +legacy legacy-test-050 ported portable-core-ingress.canonical-bytes portable-core-ingress.test.legacy-049-duplicate-keys-non-canonical +legacy legacy-test-051 ported portable-core-ingress.canonical-bytes portable-core-ingress.test.legacy-051-alternate-whitespace-non-canonical +legacy legacy-test-052 ported portable-core-ingress.canonical-bytes portable-core-ingress.test.legacy-051-alternate-whitespace-non-canonical +legacy legacy-test-053 ported portable-core-ingress.canonical-bytes portable-core-ingress.test.legacy-053-alternate-escaping-non-canonical +legacy legacy-test-054 ported portable-core-ingress.canonical-bytes portable-core-ingress.test.legacy-053-alternate-escaping-non-canonical +legacy legacy-test-055 ported portable-core-ingress.canonical-bytes portable-core-ingress.test.legacy-055-missing-final-lf-non-canonical +legacy legacy-test-056 ported portable-core-ingress.canonical-bytes portable-core-ingress.test.legacy-055-missing-final-lf-non-canonical +legacy legacy-test-057 ported portable-core-ingress.canonical-bytes portable-core-ingress.test.legacy-057-extra-final-lf-non-canonical +legacy legacy-test-058 ported portable-core-ingress.canonical-bytes portable-core-ingress.test.legacy-057-extra-final-lf-non-canonical +legacy legacy-test-059 ported portable-core-ingress.canonical-bytes portable-core-ingress.test.legacy-059-unsorted-keys-non-canonical +legacy legacy-test-060 ported portable-core-ingress.canonical-bytes portable-core-ingress.test.legacy-059-unsorted-keys-non-canonical +legacy legacy-test-061 ported portable-core-ingress.raw-byte-limit portable-core-ingress.test.legacy-061-at-exact-1-048-576-byte-boundary-is-still-just-a-shape-failure-not-e-limit +legacy legacy-test-062 ported portable-core-ingress.raw-byte-limit portable-core-ingress.test.legacy-061-at-exact-1-048-576-byte-boundary-is-still-just-a-shape-failure-not-e-limit +legacy legacy-test-063 ported portable-core-ingress.raw-byte-limit portable-core-ingress.test.legacy-063-one-byte-over-the-1-048-576-limit +legacy legacy-test-064 ported portable-core-ingress.raw-byte-limit portable-core-ingress.test.legacy-063-one-byte-over-the-1-048-576-limit +legacy legacy-test-081 ported portable-core-ingress.sanitized-errors portable-core-ingress.test.legacy-081-sanitized-input-diagnostics +legacy legacy-test-082 ported portable-core-ingress.sanitized-errors portable-core-ingress.test.legacy-081-sanitized-input-diagnostics +legacy legacy-test-267 ported portable-core-ingress.jq-version portable-core-ingress.test.legacy-267-non-1-6-jq-on-path-is-rejected +legacy legacy-test-268 ported portable-core-ingress.jq-version portable-core-ingress.test.legacy-267-non-1-6-jq-on-path-is-rejected +legacy legacy-test-272 ported portable-core-ingress.sha256-tool portable-core-ingress.test.legacy-272-missing-sha-tool-to-e-runtime +legacy legacy-test-273 ported portable-core-ingress.sha256-tool portable-core-ingress.test.legacy-272-missing-sha-tool-to-e-runtime +legacy legacy-test-274 replaced-by portable-core-ingress.sha256-tool portable-core-ingress.test.legacy-274-sha256-backend-portability +legacy legacy-test-275 ported portable-core-ingress.private-temp portable-core-ingress.test.legacy-275-mktemp-failure-sanitized +legacy legacy-test-276 ported portable-core-ingress.private-temp portable-core-ingress.test.legacy-275-mktemp-failure-sanitized +legacy legacy-test-277 ported portable-core-ingress.private-temp portable-core-ingress.test.legacy-275-mktemp-failure-sanitized +legacy legacy-test-278 ported portable-core-ingress.private-temp portable-core-ingress.test.legacy-278-unmutated-mktemp-path-still-succeeds +legacy legacy-test-279 ported portable-core-ingress.private-temp portable-core-ingress.test.legacy-278-unmutated-mktemp-path-still-succeeds diff --git a/scripts/test/portable-core-ingress.test.sh b/scripts/test/portable-core-ingress.test.sh new file mode 100755 index 0000000..e555cf4 --- /dev/null +++ b/scripts/test/portable-core-ingress.test.sh @@ -0,0 +1,795 @@ +#!/usr/bin/env bash +# shellcheck disable=SC2016,SC2034 +set -euo pipefail + +ingress_repo="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd -P)" +ingress_generation='g-14b7ad8ce54c3b8c585ff92063d71551ffc7394cc2294d0297bc7d2b8da2c386' +ingress_product="$ingress_repo/core/v1/generations/$ingress_generation/core-ingress.sh" +ingress_schema="$ingress_repo/core/v1/generations/$ingress_generation/modules/schema.jq" +ingress_fixture="$ingress_repo/scripts/test/portable-core-ingress-fixtures.json" +ingress_ledger="$ingress_repo/scripts/test/portable-core-ingress-ledger.tsv" +ingress_manifest="$ingress_repo/ci/required-files.txt" +ingress_host_path="$PATH" +ingress_tmp="$(mktemp -d "${TMPDIR:-/tmp}/ystack-portable-ingress-test.XXXXXX")" +ingress_download='' + +cleanup() { + if [ -n "${PORTABLE_CORE_INGRESS_TEMP:-}" ]; then + portable_core_ingress_close >/dev/null 2>&1 || true + fi + if [ -n "$ingress_download" ] && [ -f "$ingress_download" ]; then + rm -f -- "$ingress_download" + fi + rm -rf -- "$ingress_tmp" +} +trap cleanup EXIT + +sha256_path() { + if command -v sha256sum >/dev/null 2>&1; then + sha256sum "$1" | awk '{print $1}' + else + shasum -a 256 "$1" | awk '{print $1}' + fi +} + +ingress_platform="$(uname -s):$(uname -m)" +case "$ingress_platform" in + Linux:x86_64) + ingress_asset='jq-linux64' + ingress_asset_sha256='af986793a515d500ab2d35f8d2aecd656e764504b789b66d7e1a0b727a124c44' + ;; + Darwin:x86_64|Darwin:arm64) + ingress_asset='jq-osx-amd64' + ingress_asset_sha256='5c0a0a3ea600f302ee458b30317425dd9632d1ad8882259fcaf4e9b868b2b1ef' + ;; + *) + echo "FAIL: unsupported jq 1.6 proof platform: $ingress_platform" >&2 + exit 1 + ;; +esac + +ingress_cache="${TMPDIR:-/tmp}/ystack-portable-core-jq16" +mkdir -p "$ingress_cache" +ingress_jq="$ingress_cache/$ingress_asset" +if [ ! -f "$ingress_jq" ] || + [ "$(sha256_path "$ingress_jq")" != "$ingress_asset_sha256" ]; then + ingress_download="$(mktemp "$ingress_cache/.jq-1.6.XXXXXX")" + curl --proto '=https' --tlsv1.2 -fsSL \ + "https://github.com/jqlang/jq/releases/download/jq-1.6/$ingress_asset" \ + -o "$ingress_download" + [ "$(sha256_path "$ingress_download")" = "$ingress_asset_sha256" ] || { + echo 'FAIL: jq 1.6 release asset digest mismatch' >&2 + exit 1 + } + chmod 0555 "$ingress_download" + mv "$ingress_download" "$ingress_jq" + ingress_download='' +fi +[ "$(sha256_path "$ingress_jq")" = "$ingress_asset_sha256" ] && + [ "$("$ingress_jq" --version)" = jq-1.6 ] || { + echo 'FAIL: pinned jq 1.6 identity check failed' >&2 + exit 1 + } + +ingress_proof_bin="$ingress_tmp/proof-bin" +mkdir -p "$ingress_proof_bin" +ln -s "$ingress_jq" "$ingress_proof_bin/jq" +PATH="$ingress_proof_bin:$ingress_host_path" +export PATH + +# shellcheck source=/dev/null +source "$ingress_product" + +ingress_failures=0 +ingress_direct_total=0 +ingress_direct_passed=0 +ingress_runtime_total=0 +ingress_runtime_passed=0 +ingress_guard_total=0 +ingress_guard_passed=0 +ingress_seen_rules="$ingress_tmp/seen-rules" +ingress_seen_tests="$ingress_tmp/seen-tests" +: > "$ingress_seen_rules" +: > "$ingress_seen_tests" + +fail_case() { + echo "FAIL: $1" >&2 + ingress_failures=$((ingress_failures + 1)) +} + +mark_rule() { + printf '%s\n' "$1" >> "$ingress_seen_rules" +} + +mark_test() { + printf '%s\n' "$1" >> "$ingress_seen_tests" +} + +expect_failure() { + local case_id="$1" + local expected="$2" + shift 2 + local stdout_file="$ingress_tmp/$case_id.stdout" + local stderr_file="$ingress_tmp/$case_id.stderr" + local actual + ingress_direct_total=$((ingress_direct_total + 1)) + if "$@" >"$stdout_file" 2>"$stderr_file"; then + fail_case "$case_id unexpectedly succeeded" + return + fi + actual="$(cat "$stderr_file")" + if [ ! -s "$stdout_file" ] && [ "$actual" = "$expected" ] && + [ "$(wc -c < "$stderr_file" | tr -d ' ')" -eq $((${#expected} + 1)) ]; then + ingress_direct_passed=$((ingress_direct_passed + 1)) + else + fail_case "$case_id expected exact $expected" + fi +} + +expect_success() { + local case_id="$1" + shift + local stdout_file="$ingress_tmp/$case_id.stdout" + local stderr_file="$ingress_tmp/$case_id.stderr" + ingress_direct_total=$((ingress_direct_total + 1)) + if "$@" >"$stdout_file" 2>"$stderr_file" && + [ ! -s "$stdout_file" ] && [ ! -s "$stderr_file" ]; then + ingress_direct_passed=$((ingress_direct_passed + 1)) + else + fail_case "$case_id expected silent success" + fi +} + +start_ingress() { + portable_core_ingress_open && portable_core_ingress_begin "${1:-document}" +} + +stop_ingress() { + portable_core_ingress_close +} + +canonical_file="$ingress_tmp/canonical.json" +"$ingress_jq" -j '.canonical.bytes' "$ingress_fixture" > "$canonical_file" +expected_digest="$("$ingress_jq" -r '.canonical.sha256' "$ingress_fixture")" + +if start_ingress document && + [ "$PORTABLE_CORE_INGRESS_GENERATION" = "$ingress_generation" ] && + [ "$PORTABLE_CORE_INGRESS_SCHEMA" = "$ingress_schema" ] && + [ "$PORTABLE_CORE_INGRESS_MODULE_DIR" = "${ingress_schema%/*}" ]; then + ingress_direct_total=$((ingress_direct_total + 1)) + ingress_direct_passed=$((ingress_direct_passed + 1)) +else + ingress_direct_total=$((ingress_direct_total + 1)) + fail_case 'fixed generation and schema paths' +fi +mark_rule portable-core-ingress.fixed-generation +mark_rule portable-core-ingress.fixed-schema-loading + +expect_success canonical-snapshot portable_core_ingress_snapshot "$canonical_file" +expect_success canonical-driver portable_core_ingress_finish_driver +if [ "$PORTABLE_CORE_INGRESS_SHA256" = "$expected_digest" ] && + cmp -s "$canonical_file" "$PORTABLE_CORE_INGRESS_SNAPSHOT"; then + ingress_direct_total=$((ingress_direct_total + 1)) + ingress_direct_passed=$((ingress_direct_passed + 1)) +else + ingress_direct_total=$((ingress_direct_total + 1)) + fail_case 'snapshot digest and bytes differ' +fi +mark_rule portable-core-ingress.snapshot-sha256 +if "$ingress_jq" -e --arg digest "$expected_digest" \ + '.mode == "document" and (.docs|length) == 1 and + .docs[0].content == {a:1,text:"é"} and .docs[0].sha256 == $digest' \ + "$PORTABLE_CORE_INGRESS_DRIVER" >/dev/null; then + ingress_direct_total=$((ingress_direct_total + 1)) + ingress_direct_passed=$((ingress_direct_passed + 1)) +else + ingress_direct_total=$((ingress_direct_total + 1)) + fail_case 'driver does not bind the accepted snapshot and digest' +fi +mark_rule portable-core-ingress.driver-snapshot +mark_rule portable-core-ingress.bounded-snapshot +mark_test portable-core-ingress.test.legacy-278-unmutated-mktemp-path-still-succeeds +stop_ingress +mark_rule portable-core-ingress.private-temp + +empty_file="$ingress_tmp/empty.json" +multi_file="$ingress_tmp/multi.json" +bom_file="$ingress_tmp/bom.json" +utf8_file="$ingress_tmp/invalid-utf8.json" +duplicate_file="$ingress_tmp/duplicate.json" +whitespace_file="$ingress_tmp/whitespace.json" +escape_file="$ingress_tmp/escape.json" +no_lf_file="$ingress_tmp/no-lf.json" +extra_lf_file="$ingress_tmp/extra-lf.json" +unsorted_file="$ingress_tmp/unsorted.json" +: > "$empty_file" +printf '{}\n{}\n' > "$multi_file" +printf '\357\273\277{}\n' > "$bom_file" +printf '\200\n' > "$utf8_file" +printf '{"a":1,"a":2}\n' > "$duplicate_file" +printf '{ "a": 1 }\n' > "$whitespace_file" +printf '{"x":"\\u0061"}\n' > "$escape_file" +printf '{}' > "$no_lf_file" +printf '{}\n\n' > "$extra_lf_file" +printf '{"b":1,"a":2}\n' > "$unsorted_file" + +start_ingress document +expect_failure unreadable-input E_RUNTIME portable_core_ingress_snapshot \ + "$ingress_tmp/distinctive-missing-input-SECRET.json" +mark_test portable-core-ingress.test.legacy-039-validate-profile-set-unreadable-input +stop_ingress + +document_finish_failure() { + local case_id="$1" + local expected="$2" + local input_file="$3" + start_ingress document + portable_core_ingress_snapshot "$input_file" + expect_failure "$case_id" "$expected" portable_core_ingress_finish_driver + stop_ingress +} + +document_finish_failure empty-input E_PARSE "$empty_file" +mark_test portable-core-ingress.test.legacy-041-empty-input +document_finish_failure multi-root E_PARSE "$multi_file" +mark_test portable-core-ingress.test.legacy-043-multi-root-stream +document_finish_failure bom-prefix E_CANONICAL "$bom_file" +mark_test portable-core-ingress.test.legacy-045-bom-prefix +document_finish_failure invalid-utf8 E_PARSE "$utf8_file" +mark_test portable-core-ingress.test.legacy-047-invalid-utf-8 +document_finish_failure duplicate-keys E_CANONICAL "$duplicate_file" +mark_test portable-core-ingress.test.legacy-049-duplicate-keys-non-canonical +document_finish_failure alternate-whitespace E_CANONICAL "$whitespace_file" +mark_test portable-core-ingress.test.legacy-051-alternate-whitespace-non-canonical +document_finish_failure alternate-escaping E_CANONICAL "$escape_file" +mark_test portable-core-ingress.test.legacy-053-alternate-escaping-non-canonical +document_finish_failure missing-final-lf E_CANONICAL "$no_lf_file" +mark_test portable-core-ingress.test.legacy-055-missing-final-lf-non-canonical +document_finish_failure extra-final-lf E_CANONICAL "$extra_lf_file" +mark_test portable-core-ingress.test.legacy-057-extra-final-lf-non-canonical +document_finish_failure unsorted-keys E_CANONICAL "$unsorted_file" +mark_test portable-core-ingress.test.legacy-059-unsorted-keys-non-canonical +mark_rule portable-core-ingress.snapshot-readable +mark_rule portable-core-ingress.single-root-json +mark_rule portable-core-ingress.utf8-json +mark_rule portable-core-ingress.canonical-bytes +mark_rule portable-core-ingress.sanitized-errors +mark_test portable-core-ingress.test.legacy-081-sanitized-input-diagnostics + +python3 - "$ingress_tmp/at-limit.json" "$ingress_tmp/over-limit.json" <<'PY' +import sys +prefix = b'{"x":"' +suffix = b'"}\n' +limit = 1048576 +payload = prefix + (b'a' * (limit - len(prefix) - len(suffix))) + suffix +open(sys.argv[1], 'wb').write(payload) +open(sys.argv[2], 'wb').write(payload + b'x') +PY +[ "$(wc -c < "$ingress_tmp/at-limit.json" | tr -d ' ')" -eq 1048576 ] +[ "$(wc -c < "$ingress_tmp/over-limit.json" | tr -d ' ')" -eq 1048577 ] + +start_ingress document +expect_success at-limit-snapshot portable_core_ingress_snapshot "$ingress_tmp/at-limit.json" +expect_success at-limit-driver portable_core_ingress_finish_driver +stop_ingress +start_ingress document +portable_core_ingress_snapshot "$ingress_tmp/over-limit.json" +expect_failure over-limit E_LIMIT portable_core_ingress_finish_driver +stop_ingress +mark_rule portable-core-ingress.raw-byte-limit +mark_test portable-core-ingress.test.legacy-063-one-byte-over-the-1-048-576-limit + +start_ingress profile-set +portable_core_ingress_snapshot "$empty_file" +portable_core_ingress_snapshot "$ingress_tmp/over-limit.json" +expect_failure multi-limit-before-parse E_LIMIT portable_core_ingress_finish_driver +stop_ingress +start_ingress profile-set +portable_core_ingress_snapshot "$whitespace_file" +portable_core_ingress_snapshot "$empty_file" +expect_failure multi-parse-before-canonical E_PARSE portable_core_ingress_finish_driver +stop_ingress +start_ingress profile-set +portable_core_ingress_snapshot "$ingress_tmp/over-limit.json" +expect_failure multi-runtime-before-limit E_RUNTIME portable_core_ingress_snapshot \ + "$ingress_tmp/missing-later-input-SECRET.json" +stop_ingress + +snapshot_original="$ingress_tmp/snapshot-original.json" +printf '{"v":1}\n' > "$snapshot_original" +snapshot_original_digest="$(sha256_path "$snapshot_original")" +start_ingress document +expect_success one-bounded-read portable_core_ingress_snapshot "$snapshot_original" +printf '{"v":2}\n' > "$snapshot_original" +expect_success finish-preserved-snapshot portable_core_ingress_finish_driver +if "$ingress_jq" -e --arg digest "$snapshot_original_digest" \ + '.docs == [{content:{v:1},sha256:$digest}]' \ + "$PORTABLE_CORE_INGRESS_DRIVER" >/dev/null; then + ingress_direct_total=$((ingress_direct_total + 1)) + ingress_direct_passed=$((ingress_direct_passed + 1)) +else + ingress_direct_total=$((ingress_direct_total + 1)) + fail_case 'driver reread mutable caller input' +fi +stop_ingress + +runtime_failure() { + local case_id="$1" + local expected="$2" + shift 2 + ingress_runtime_total=$((ingress_runtime_total + 1)) + expect_failure "$case_id" "$expected" "$@" + if [ "$(cat "$ingress_tmp/$case_id.stderr")" = "$expected" ]; then + ingress_runtime_passed=$((ingress_runtime_passed + 1)) + fi +} + +start_ingress document +real_head="$PORTABLE_CORE_INGRESS_HEAD" +head_log="$ingress_tmp/head-arguments" +logged_head="$ingress_tmp/logged-head" +printf '%s\n' '#!/bin/sh' \ + "printf '%s\\n' \"\$*\" >> \"$head_log\"" \ + "exec \"$real_head\" \"\$@\"" > "$logged_head" +chmod +x "$logged_head" +PORTABLE_CORE_INGRESS_HEAD="$logged_head" +portable_core_ingress_snapshot "$canonical_file" +portable_core_ingress_finish_driver +if [ "$(wc -l < "$head_log" | tr -d ' ')" -eq 1 ] && + [ "$(cat "$head_log")" = "-c 1048577 -- $canonical_file" ]; then + ingress_direct_total=$((ingress_direct_total + 1)) + ingress_direct_passed=$((ingress_direct_passed + 1)) +else + ingress_direct_total=$((ingress_direct_total + 1)) + fail_case 'snapshot was not one bounded 1,048,577-byte read' +fi +stop_ingress + +start_ingress document +head_failure="$ingress_tmp/head-failure" +printf '%s\n' '#!/bin/sh' 'printf "%s\\n" "head SECRET path" >&2' 'exit 1' > "$head_failure" +chmod +x "$head_failure" +PORTABLE_CORE_INGRESS_HEAD="$head_failure" +runtime_failure bounded-read-failure E_RUNTIME portable_core_ingress_snapshot "$canonical_file" +stop_ingress +start_ingress document +wc_failure="$ingress_tmp/wc-failure" +printf '%s\n' '#!/bin/sh' 'printf "%s\\n" "wc SECRET path" >&2' 'exit 1' > "$wc_failure" +chmod +x "$wc_failure" +PORTABLE_CORE_INGRESS_WC="$wc_failure" +runtime_failure byte-count-failure E_RUNTIME portable_core_ingress_snapshot "$canonical_file" +stop_ingress + +fake_jq_bin="$ingress_tmp/fake-jq-bin" +mkdir -p "$fake_jq_bin" +printf '%s\n' '#!/bin/sh' \ + 'if [ "${1:-}" = "--version" ]; then printf "%s\\n" jq-1.7; exit 0; fi' \ + 'exit 99' > "$fake_jq_bin/jq" +chmod +x "$fake_jq_bin/jq" +runtime_failure non-jq-1-6 E_RUNTIME env PATH="$fake_jq_bin:$ingress_host_path" \ + bash -c 'source "$1"; portable_core_ingress_open' _ "$ingress_product" +mark_rule portable-core-ingress.jq-version +mark_test portable-core-ingress.test.legacy-267-non-1-6-jq-on-path-is-rejected + +make_isolated_bin() { + local destination="$1" + local tool + local tool_path + mkdir -p "$destination" + ln -s "$ingress_jq" "$destination/jq" + for tool in dirname head wc cmp cat rm mktemp; do + tool_path="$(command -v "$tool")" + ln -s "$tool_path" "$destination/$tool" + done +} + +no_sha_bin="$ingress_tmp/no-sha-bin" +make_isolated_bin "$no_sha_bin" +runtime_failure missing-sha E_RUNTIME env PATH="$no_sha_bin" \ + /bin/bash -c 'source "$1"; portable_core_ingress_open' _ "$ingress_product" +mark_test portable-core-ingress.test.legacy-272-missing-sha-tool-to-e-runtime + +mktemp_fail_bin="$ingress_tmp/mktemp-fail-bin" +make_isolated_bin "$mktemp_fail_bin" +rm "$mktemp_fail_bin/mktemp" +printf '%s\n' '#!/bin/sh' \ + 'printf "%s\\n" "mktemp SECRET /private/caller/path" >&2' \ + 'exit 1' > "$mktemp_fail_bin/mktemp" +chmod +x "$mktemp_fail_bin/mktemp" +sha_real="$(command -v shasum || command -v sha256sum)" +case "${sha_real##*/}" in + shasum) ln -s "$sha_real" "$mktemp_fail_bin/shasum" ;; + sha256sum) ln -s "$sha_real" "$mktemp_fail_bin/sha256sum" ;; +esac +runtime_failure mktemp-failure E_RUNTIME env PATH="$mktemp_fail_bin" \ + /bin/bash -c 'source "$1"; portable_core_ingress_open' _ "$ingress_product" +mark_rule portable-core-ingress.private-temp-errors-sanitized +mark_test portable-core-ingress.test.mktemp-failure-sanitized-e-runtime +mark_test portable-core-ingress.test.legacy-275-mktemp-failure-sanitized + +sha256_bin="$ingress_tmp/sha256-bin" +make_isolated_bin "$sha256_bin" +case "${sha_real##*/}" in + shasum) + printf '%s\n' '#!/bin/sh' \ + "exec \"$sha_real\" -a 256 \"\${2}\"" > "$sha256_bin/sha256sum" + ;; + sha256sum) + printf '%s\n' '#!/bin/sh' \ + "exec \"$sha_real\" -- \"\${2}\"" > "$sha256_bin/sha256sum" + ;; +esac +chmod +x "$sha256_bin/sha256sum" +PATH="$sha256_bin" portable_core_ingress_open +portable_core_ingress_begin document +portable_core_ingress_snapshot "$canonical_file" +portable_core_ingress_finish_driver +sha256_backend_digest="$PORTABLE_CORE_INGRESS_SHA256" +sha256_backend_name="$PORTABLE_CORE_INGRESS_SHA_BACKEND" +portable_core_ingress_close + +shasum_bin="$ingress_tmp/shasum-bin" +make_isolated_bin "$shasum_bin" +case "${sha_real##*/}" in + shasum) ln -s "$sha_real" "$shasum_bin/shasum" ;; + sha256sum) + printf '%s\n' '#!/bin/sh' \ + "exec \"$sha_real\" -- \"\${4}\"" > "$shasum_bin/shasum" + chmod +x "$shasum_bin/shasum" + ;; +esac +PATH="$shasum_bin" portable_core_ingress_open +portable_core_ingress_begin document +portable_core_ingress_snapshot "$canonical_file" +portable_core_ingress_finish_driver +shasum_backend_digest="$PORTABLE_CORE_INGRESS_SHA256" +shasum_backend_name="$PORTABLE_CORE_INGRESS_SHA_BACKEND" +portable_core_ingress_close +if [ "$sha256_backend_name" = sha256sum ] && [ "$shasum_backend_name" = shasum ] && + [ "$sha256_backend_digest" = "$expected_digest" ] && + [ "$shasum_backend_digest" = "$expected_digest" ]; then + ingress_direct_total=$((ingress_direct_total + 1)) + ingress_direct_passed=$((ingress_direct_passed + 1)) +else + ingress_direct_total=$((ingress_direct_total + 1)) + fail_case 'selected SHA-256 backends disagree' +fi +mark_rule portable-core-ingress.sha256-tool +mark_test portable-core-ingress.test.legacy-274-sha256-backend-portability + +start_ingress document +sha_failure="$ingress_tmp/sha-failure" +printf '%s\n' '#!/bin/sh' 'printf "%s\\n" "sha SECRET path" >&2' 'exit 1' > "$sha_failure" +chmod +x "$sha_failure" +PORTABLE_CORE_INGRESS_SHA="$sha_failure" +portable_core_ingress_snapshot "$canonical_file" +runtime_failure sha-command-failure E_RUNTIME portable_core_ingress_finish_driver +stop_ingress +start_ingress document +sha_empty="$ingress_tmp/sha-empty" +printf '%s\n' '#!/bin/sh' 'exit 0' > "$sha_empty" +chmod +x "$sha_empty" +PORTABLE_CORE_INGRESS_SHA="$sha_empty" +portable_core_ingress_snapshot "$canonical_file" +runtime_failure sha-empty-digest E_RUNTIME portable_core_ingress_finish_driver +stop_ingress +start_ingress document +sha_multiline="$ingress_tmp/sha-multiline" +printf '%s\n' '#!/bin/sh' \ + 'printf "%s\\n%s\\n" "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa file" "extra"' \ + > "$sha_multiline" +chmod +x "$sha_multiline" +PORTABLE_CORE_INGRESS_SHA="$sha_multiline" +portable_core_ingress_snapshot "$canonical_file" +runtime_failure sha-multiline-digest E_RUNTIME portable_core_ingress_finish_driver +stop_ingress + +start_ingress document +jq_canonical_fail="$ingress_tmp/jq-canonical-fail" +printf '%s\n' '#!/bin/sh' \ + 'case " $* " in *" -s -e "*) exit 0 ;; esac' \ + 'printf "%s\\n" "jq SECRET path" >&2' \ + 'exit 1' > "$jq_canonical_fail" +chmod +x "$jq_canonical_fail" +PORTABLE_CORE_INGRESS_JQ="$jq_canonical_fail" +portable_core_ingress_snapshot "$canonical_file" +runtime_failure canonicalizer-failure E_RUNTIME portable_core_ingress_finish_driver +stop_ingress + +start_ingress document +cmp_failure="$ingress_tmp/cmp-failure" +printf '%s\n' '#!/bin/sh' 'printf "%s\\n" "cmp SECRET path" >&2' 'exit 2' > "$cmp_failure" +chmod +x "$cmp_failure" +PORTABLE_CORE_INGRESS_CMP="$cmp_failure" +portable_core_ingress_snapshot "$canonical_file" +runtime_failure cmp-operational-failure E_RUNTIME portable_core_ingress_finish_driver +stop_ingress + +write_failure_case() { + local case_id="$1" + local target_kind="$2" + start_ingress document + case "$target_kind" in + raw) mkdir "$PORTABLE_CORE_INGRESS_TEMP/raw.1" ;; + canonical) mkdir "$PORTABLE_CORE_INGRESS_TEMP/canonical.1" ;; + contents) + rm "$PORTABLE_CORE_INGRESS_CONTENTS" + mkdir "$PORTABLE_CORE_INGRESS_CONTENTS" + ;; + hashes) + rm "$PORTABLE_CORE_INGRESS_HASHES" + mkdir "$PORTABLE_CORE_INGRESS_HASHES" + ;; + esac + if [ "$target_kind" = raw ]; then + runtime_failure "$case_id" E_RUNTIME portable_core_ingress_snapshot "$canonical_file" + else + portable_core_ingress_snapshot "$canonical_file" + runtime_failure "$case_id" E_RUNTIME portable_core_ingress_finish_driver + fi + stop_ingress +} + +portable_core_ingress_open +mkdir "$PORTABLE_CORE_INGRESS_TEMP/contents.ndjson" +runtime_failure contents-truncate-failure E_RUNTIME portable_core_ingress_begin document +stop_ingress +portable_core_ingress_open +mkdir "$PORTABLE_CORE_INGRESS_TEMP/hashes.ndjson" +runtime_failure hashes-truncate-failure E_RUNTIME portable_core_ingress_begin document +stop_ingress +write_failure_case raw-write-failure raw +write_failure_case canonical-write-failure canonical +write_failure_case contents-append-failure contents +write_failure_case hashes-append-failure hashes + +start_ingress document +cleanup_temp="$PORTABLE_CORE_INGRESS_TEMP" +cleanup_real_rm="$PORTABLE_CORE_INGRESS_RM" +cleanup_fail="$ingress_tmp/cleanup-fail" +printf '%s\n' '#!/bin/sh' 'printf "%s\\n" "rm SECRET path" >&2' 'exit 1' > "$cleanup_fail" +chmod +x "$cleanup_fail" +PORTABLE_CORE_INGRESS_RM="$cleanup_fail" +runtime_failure cleanup-failure E_RUNTIME portable_core_ingress_close +PORTABLE_CORE_INGRESS_RM="$cleanup_real_rm" +"$cleanup_real_rm" -rf -- "$cleanup_temp" +PORTABLE_CORE_INGRESS_TEMP='' + +start_ingress document +portable_core_ingress_snapshot "$canonical_file" +mkdir "$PORTABLE_CORE_INGRESS_TEMP/driver.json" +runtime_failure driver-write-failure E_RUNTIME portable_core_ingress_finish_driver +stop_ingress +mark_test portable-core-ingress.test.private-temp-write-failures-sanitized-e-runtime + +package_copy="$ingress_tmp/package-copy" +package_generation_dir="$package_copy/core/v1/generations/$ingress_generation" +mkdir -p "$package_generation_dir/modules" +cp "$ingress_product" "$package_generation_dir/core-ingress.sh" +cp "$ingress_schema" "$package_generation_dir/modules/schema.jq" +package_product="$package_generation_dir/core-ingress.sh" +package_root="$package_generation_dir/contracts.jq" + +printf '%s\n' 'import "schema" as schema;' \ + 'if schema::semantic_identity == "core.contracts.v1" then "E_SHAPE" else error("identity") end' \ + > "$package_root" +# shellcheck source=/dev/null +source "$package_product" +start_ingress document +portable_core_ingress_snapshot "$ingress_tmp/at-limit.json" +portable_core_ingress_finish_driver +expect_failure at-limit-reaches-validator E_SHAPE portable_core_ingress_validate +stop_ingress +mark_test portable-core-ingress.test.legacy-061-at-exact-1-048-576-byte-boundary-is-still-just-a-shape-failure-not-e-limit + +printf '%s\n' 'import "schema" as schema;' \ + 'if schema::semantic_identity == "core.contracts.v1" then empty else error("identity") end' \ + > "$package_root" +poison_home="$ingress_tmp/poison-home" +poison_cwd="$ingress_tmp/poison-cwd" +mkdir -p "$poison_home/.jq" "$poison_cwd" +printf '%s\n' 'def semantic_identity: "poison";' > "$poison_home/.jq/schema.jq" +printf '%s\n' 'def semantic_identity: "poison";' > "$poison_cwd/schema.jq" +( + cd "$poison_cwd" + HOME="$poison_home" JQ_LIBRARY_PATH="$poison_cwd" start_ingress document + portable_core_ingress_snapshot "$canonical_file" + portable_core_ingress_finish_driver + expect_success fixed-module-validator portable_core_ingress_validate + stop_ingress +) + +printf '%s\n' 'error("validator SECRET /private/path")' > "$package_root" +start_ingress document +portable_core_ingress_snapshot "$canonical_file" +portable_core_ingress_finish_driver +runtime_failure validator-nonzero E_RUNTIME portable_core_ingress_validate +stop_ingress +printf '%s\n' '["E_SHAPE","E_REF"][]' > "$package_root" +start_ingress document +portable_core_ingress_snapshot "$canonical_file" +portable_core_ingress_finish_driver +runtime_failure validator-extra-output E_RUNTIME portable_core_ingress_validate +stop_ingress +printf '%s\n' '"UNKNOWN"' > "$package_root" +start_ingress document +portable_core_ingress_snapshot "$canonical_file" +portable_core_ingress_finish_driver +runtime_failure validator-unknown-token E_RUNTIME portable_core_ingress_validate +stop_ingress +printf '%s\n' '"E_SHAPE\n"' > "$package_root" +start_ingress document +portable_core_ingress_snapshot "$canonical_file" +portable_core_ingress_finish_driver +runtime_failure validator-extra-newline E_RUNTIME portable_core_ingress_validate +stop_ingress +printf '%s\n' 'empty' > "$package_root" +start_ingress document +portable_core_ingress_snapshot "$canonical_file" +portable_core_ingress_finish_driver +mkdir "$PORTABLE_CORE_INGRESS_TEMP/validator.out" +runtime_failure validator-output-write E_RUNTIME portable_core_ingress_validate +stop_ingress + +root_target="$ingress_tmp/root-target.jq" +printf '%s\n' 'empty' > "$root_target" +rm "$package_root" +ln -s "$root_target" "$package_root" +start_ingress document +portable_core_ingress_snapshot "$canonical_file" +portable_core_ingress_finish_driver +runtime_failure validator-root-symlink E_RUNTIME portable_core_ingress_validate +stop_ingress +rm "$package_root" +printf '%s\n' 'empty' > "$package_root" + +mv "$package_generation_dir/modules/schema.jq" "$package_generation_dir/modules/schema-real.jq" +ln -s schema-real.jq "$package_generation_dir/modules/schema.jq" +runtime_failure schema-symlink E_RUNTIME env PATH="$ingress_proof_bin:$ingress_host_path" \ + /bin/bash -c 'source "$1"; portable_core_ingress_open' _ "$package_product" +rm "$package_generation_dir/modules/schema.jq" +mv "$package_generation_dir/modules/schema-real.jq" "$package_generation_dir/modules/schema.jq" + +mv "$package_generation_dir/modules" "$package_generation_dir/modules-real" +ln -s modules-real "$package_generation_dir/modules" +runtime_failure module-directory-symlink E_RUNTIME env PATH="$ingress_proof_bin:$ingress_host_path" \ + /bin/bash -c 'source "$1"; portable_core_ingress_open' _ "$package_product" +rm "$package_generation_dir/modules" +mv "$package_generation_dir/modules-real" "$package_generation_dir/modules" + +mv "$package_product" "$package_generation_dir/core-ingress-real.sh" +ln -s core-ingress-real.sh "$package_product" +runtime_failure ingress-library-symlink E_RUNTIME env PATH="$ingress_proof_bin:$ingress_host_path" \ + /bin/bash -c 'source "$1"; portable_core_ingress_open' _ "$package_product" +mark_rule portable-core-ingress.validator-boundary + +# shellcheck source=/dev/null +source "$ingress_product" + +guard_paths="$ingress_tmp/generation-files" +find "$ingress_repo/core/v1/generations/$ingress_generation" -type f -print | + sed "s#^$ingress_repo/##" | LC_ALL=C sort > "$guard_paths" +expected_generation_files="core/v1/generations/$ingress_generation/core-ingress.sh +core/v1/generations/$ingress_generation/modules/schema.jq" +if [ "$(cat "$guard_paths")" = "$expected_generation_files" ] && + [ ! -e "$ingress_repo/scripts/core-contract.sh" ] && + [ ! -e "$ingress_repo/core/v1/generations/$ingress_generation/contracts.jq" ] && + [ -z "$(find "$ingress_repo/core/v1/generations/$ingress_generation" -type l -print -quit)" ]; then + ingress_guard_total=$((ingress_guard_total + 1)) + ingress_guard_passed=$((ingress_guard_passed + 1)) +else + ingress_guard_total=$((ingress_guard_total + 1)) + fail_case 'private generation guard' +fi +if [ "$(git -C "$ingress_repo" ls-tree HEAD \ + "core/v1/generations/$ingress_generation/modules/schema.jq" | awk '{print $3}')" = \ + fd3924d414a7d620c2bf5de919a45c2599d572ec ] && + [ "$(git -C "$ingress_repo" ls-tree HEAD core/v1/generation-registry.json | awk '{print $3}')" = \ + 5e113105777694a280166e71d31efd19752e9562 ]; then + ingress_guard_total=$((ingress_guard_total + 1)) + ingress_guard_passed=$((ingress_guard_passed + 1)) +else + ingress_guard_total=$((ingress_guard_total + 1)) + fail_case 'schema G3 export or registry OID moved' +fi +mark_rule portable-core-ingress.private-activation-guard + +required_paths="core/v1/generations/$ingress_generation/core-ingress.sh +scripts/test/portable-core-ingress-fixtures.json +scripts/test/portable-core-ingress-ledger.tsv +scripts/test/portable-core-ingress.test.sh" +manifest_ok=true +while IFS= read -r required_path; do + [ "$(grep -Fxc "$required_path" "$ingress_manifest" || true)" -eq 1 ] && + [ -f "$ingress_repo/$required_path" ] || manifest_ok=false +done <<< "$required_paths" +if [ "$manifest_ok" = true ]; then + ingress_guard_total=$((ingress_guard_total + 1)) + ingress_guard_passed=$((ingress_guard_passed + 1)) +else + ingress_guard_total=$((ingress_guard_total + 1)) + fail_case 'restore manifest coverage' +fi +manifest_base="$ingress_tmp/base-manifest" +manifest_prefix="$ingress_tmp/current-manifest-prefix" +git -C "$ingress_repo" show d48ecdb908a395c5205260a662db7d9d3f4c1eb4:ci/required-files.txt \ + > "$manifest_base" +head -n "$(wc -l < "$manifest_base" | tr -d ' ')" "$ingress_manifest" > "$manifest_prefix" +if cmp -s "$manifest_base" "$manifest_prefix" && + [ "$(tail -n 4 "$ingress_manifest")" = "$required_paths" ]; then + ingress_guard_total=$((ingress_guard_total + 1)) + ingress_guard_passed=$((ingress_guard_passed + 1)) +else + ingress_guard_total=$((ingress_guard_total + 1)) + fail_case 'restore manifest is not an exact append' +fi +mark_rule portable-core-ingress.restore-manifest + +mark_test portable-core-ingress.test.mktemp-failure-sanitized-e-runtime +mark_test portable-core-ingress.test.private-temp-write-failures-sanitized-e-runtime + +review_digest="$("$ingress_jq" -r '.metadata.review_source_sha256' "$ingress_fixture")" +legacy_digest="$("$ingress_jq" -r '.metadata.legacy_source_sha256' "$ingress_fixture")" +mapping_digest="$("$ingress_jq" -r '.metadata.ingress_mapping_sha256' "$ingress_fixture")" +if [ "$review_digest" != 31793a3ad42acf4df117ea158a78738e056bae550269483870487c3e146b27f9 ] || + [ "$legacy_digest" != 3d5a6fb192f9bcaba5c4b89314d30f88a03b9d8a1e1e634297c267b14f096092 ] || + [ "$(sha256_path "$ingress_ledger")" != "$mapping_digest" ]; then + fail_case 'ledger checksum mismatch' +fi +"$ingress_jq" -r '.metadata.review_row_ids[]' "$ingress_fixture" | + LC_ALL=C sort > "$ingress_tmp/source-review-ids" +"$ingress_jq" -r '.metadata.legacy_row_ids[]' "$ingress_fixture" | + LC_ALL=C sort > "$ingress_tmp/source-legacy-ids" +awk -F '\t' 'NR > 1 && $1 == "review" {print $2}' "$ingress_ledger" | + LC_ALL=C sort > "$ingress_tmp/local-review-ids" +awk -F '\t' 'NR > 1 && $1 == "legacy" {print $2}' "$ingress_ledger" | + LC_ALL=C sort > "$ingress_tmp/local-legacy-ids" +cmp -s "$ingress_tmp/source-review-ids" "$ingress_tmp/local-review-ids" || + fail_case 'review ledger row set mismatch' +cmp -s "$ingress_tmp/source-legacy-ids" "$ingress_tmp/local-legacy-ids" || + fail_case 'legacy ledger row set mismatch' + +expected_rules="$ingress_tmp/expected-rules" +"$ingress_jq" -r '.owned_rules[]' "$ingress_fixture" | LC_ALL=C sort > "$expected_rules" +LC_ALL=C sort -u "$ingress_seen_rules" > "$ingress_tmp/seen-rules.sorted" +cmp -s "$expected_rules" "$ingress_tmp/seen-rules.sorted" || + fail_case 'owned rule inventory does not match executed proof' +tail -n +2 "$ingress_ledger" | cut -f5 | LC_ALL=C sort -u > "$ingress_tmp/expected-tests" +LC_ALL=C sort -u "$ingress_seen_tests" > "$ingress_tmp/seen-tests.sorted" +cmp -s "$ingress_tmp/expected-tests" "$ingress_tmp/seen-tests.sorted" || + fail_case 'ledger test IDs do not match executed proof' + +ingress_review_total="$(awk -F '\t' 'NR > 1 && $1 == "review" {n++} END {print n+0}' "$ingress_ledger")" +ingress_legacy_total="$(awk -F '\t' 'NR > 1 && $1 == "legacy" {n++} END {print n+0}' "$ingress_ledger")" +ingress_review_accounted="$(awk -F '\t' ' + NR == FNR {seen[$1]=1; next} + FNR > 1 && $1 == "review" && ($5 in seen) {n++} + END {print n+0} +' "$ingress_tmp/seen-tests.sorted" "$ingress_ledger")" +ingress_legacy_accounted="$(awk -F '\t' ' + NR == FNR {seen[$1]=1; next} + FNR > 1 && $1 == "legacy" && ($5 in seen) {n++} + END {print n+0} +' "$ingress_tmp/seen-tests.sorted" "$ingress_ledger")" +[ "$ingress_review_total" -eq 2 ] && [ "$ingress_legacy_total" -eq 38 ] || + fail_case 'ingress ledger denominators' +[ "$ingress_review_accounted" -eq "$ingress_review_total" ] || + fail_case 'review ledger execution accounting' +[ "$ingress_legacy_accounted" -eq "$ingress_legacy_total" ] || + fail_case 'legacy ledger execution accounting' + +ingress_owned_total="$(wc -l < "$expected_rules" | tr -d ' ')" +ingress_owned_passed="$ingress_owned_total" +if [ "$ingress_failures" -ne 0 ]; then + ingress_owned_passed=0 +fi + +printf 'owned rules: %s/%s\n' "$ingress_owned_passed" "$ingress_owned_total" +printf 'direct cases: %s/%s\n' "$ingress_direct_passed" "$ingress_direct_total" +printf 'runtime/error cases: %s/%s\n' "$ingress_runtime_passed" "$ingress_runtime_total" +printf 'activation/restore cases: %s/%s\n' "$ingress_guard_passed" "$ingress_guard_total" +printf 'review findings accounted for: %s/%s\n' "$ingress_review_accounted" "$ingress_review_total" +printf 'legacy assertions accounted for: %s/%s\n' "$ingress_legacy_accounted" "$ingress_legacy_total" +printf 'failures: %s\n' "$ingress_failures" + +[ "$ingress_failures" -eq 0 ] From 60af033fe06df01ac838395b355229016cd93714 Mon Sep 17 00:00:00 2001 From: ci Date: Sat, 29 Aug 2026 22:37:25 -0400 Subject: [PATCH 02/13] test: make ingress proof shallow-safe --- .../test/portable-core-ingress-fixtures.json | 2 ++ scripts/test/portable-core-ingress.test.sh | 20 +++++++++---------- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/scripts/test/portable-core-ingress-fixtures.json b/scripts/test/portable-core-ingress-fixtures.json index 4d5507f..fb70580 100644 --- a/scripts/test/portable-core-ingress-fixtures.json +++ b/scripts/test/portable-core-ingress-fixtures.json @@ -8,6 +8,8 @@ "review_source_sha256": "31793a3ad42acf4df117ea158a78738e056bae550269483870487c3e146b27f9", "legacy_source_sha256": "3d5a6fb192f9bcaba5c4b89314d30f88a03b9d8a1e1e634297c267b14f096092", "ingress_mapping_sha256": "cadaa1c5752e3cd13ee556bbae5aa7bf95130cceb044a6ef4bf569e70434e361", + "prior_manifest_lines": 96, + "prior_manifest_sha256": "1dd31b6a65e37e5d441400002ee97ec1a3cd9fc76d8d95b577f3af596903c752", "review_rows": 2, "legacy_rows": 38, "review_row_ids": [ diff --git a/scripts/test/portable-core-ingress.test.sh b/scripts/test/portable-core-ingress.test.sh index e555cf4..5009078 100755 --- a/scripts/test/portable-core-ingress.test.sh +++ b/scripts/test/portable-core-ingress.test.sh @@ -258,10 +258,9 @@ mark_test portable-core-ingress.test.legacy-081-sanitized-input-diagnostics python3 - "$ingress_tmp/at-limit.json" "$ingress_tmp/over-limit.json" <<'PY' import sys -prefix = b'{"x":"' -suffix = b'"}\n' -limit = 1048576 -payload = prefix + (b'a' * (limit - len(prefix) - len(suffix))) + suffix +items = ([b'a' * 8192] * 127) + [b'b' * 7806] +payload = b'["' + b'","'.join(items) + b'"]\n' +assert len(payload) == 1048576 open(sys.argv[1], 'wb').write(payload) open(sys.argv[2], 'wb').write(payload + b'x') PY @@ -571,7 +570,9 @@ package_product="$package_generation_dir/core-ingress.sh" package_root="$package_generation_dir/contracts.jq" printf '%s\n' 'import "schema" as schema;' \ - 'if schema::semantic_identity == "core.contracts.v1" then "E_SHAPE" else error("identity") end' \ + 'if (.docs[0].content | schema::parsed_limits_ok | not) then "E_LIMIT"' \ + 'elif (.docs[0].content | schema::document_envelope_ok | not) then "E_SHAPE"' \ + 'else empty end' \ > "$package_root" # shellcheck source=/dev/null source "$package_product" @@ -710,12 +711,11 @@ else ingress_guard_total=$((ingress_guard_total + 1)) fail_case 'restore manifest coverage' fi -manifest_base="$ingress_tmp/base-manifest" manifest_prefix="$ingress_tmp/current-manifest-prefix" -git -C "$ingress_repo" show d48ecdb908a395c5205260a662db7d9d3f4c1eb4:ci/required-files.txt \ - > "$manifest_base" -head -n "$(wc -l < "$manifest_base" | tr -d ' ')" "$ingress_manifest" > "$manifest_prefix" -if cmp -s "$manifest_base" "$manifest_prefix" && +manifest_base_lines="$("$ingress_jq" -r '.metadata.prior_manifest_lines' "$ingress_fixture")" +manifest_base_digest="$("$ingress_jq" -r '.metadata.prior_manifest_sha256' "$ingress_fixture")" +head -n "$manifest_base_lines" "$ingress_manifest" > "$manifest_prefix" +if [ "$(sha256_path "$manifest_prefix")" = "$manifest_base_digest" ] && [ "$(tail -n 4 "$ingress_manifest")" = "$required_paths" ]; then ingress_guard_total=$((ingress_guard_total + 1)) ingress_guard_passed=$((ingress_guard_passed + 1)) From d25a2d5ecedd6a403824f6a55b0461ea9401bc02 Mon Sep 17 00:00:00 2001 From: ci Date: Sat, 29 Aug 2026 22:42:23 -0400 Subject: [PATCH 03/13] fix: enforce strict ingress phases --- .../core-ingress.sh | 8 ++ .../test/portable-core-ingress-fixtures.json | 1 + scripts/test/portable-core-ingress.test.sh | 112 +++++++++++++++++- 3 files changed, 116 insertions(+), 5 deletions(-) diff --git a/core/v1/generations/g-14b7ad8ce54c3b8c585ff92063d71551ffc7394cc2294d0297bc7d2b8da2c386/core-ingress.sh b/core/v1/generations/g-14b7ad8ce54c3b8c585ff92063d71551ffc7394cc2294d0297bc7d2b8da2c386/core-ingress.sh index 572e847..437a698 100644 --- a/core/v1/generations/g-14b7ad8ce54c3b8c585ff92063d71551ffc7394cc2294d0297bc7d2b8da2c386/core-ingress.sh +++ b/core/v1/generations/g-14b7ad8ce54c3b8c585ff92063d71551ffc7394cc2294d0297bc7d2b8da2c386/core-ingress.sh @@ -283,6 +283,7 @@ portable_core_ingress_finish_driver() { local raw_path local canonical_path local compare_status + local json_token_pattern [ "$#" -eq 0 ] && [ "${PORTABLE_CORE_INGRESS_COUNT:-0}" -gt 0 ] || { portable_core_ingress_error E_RUNTIME @@ -295,6 +296,7 @@ portable_core_ingress_finish_driver() { return 1 fi done + json_token_pattern='(?:[ \t\r\n]+|[\[\]{}:,]|"(?:[^"\\\x00-\x1f]|\\(?:["\\/bfnrt]|u[0-9A-Fa-f]{4}))*"|(?/dev/null 2>/dev/null; then + portable_core_ingress_error E_PARSE + return 1 + fi canonical_path="$PORTABLE_CORE_INGRESS_TEMP/canonical.$((input_index + 1))" if ! "$PORTABLE_CORE_INGRESS_JQ" -s -S -c \ 'if length == 1 then .[0] else error("root-count") end' "$raw_path" \ diff --git a/scripts/test/portable-core-ingress-fixtures.json b/scripts/test/portable-core-ingress-fixtures.json index fb70580..1bc3073 100644 --- a/scripts/test/portable-core-ingress-fixtures.json +++ b/scripts/test/portable-core-ingress-fixtures.json @@ -64,6 +64,7 @@ "owned_rules": [ "portable-core-ingress.bounded-snapshot", "portable-core-ingress.canonical-bytes", + "portable-core-ingress.command-routes", "portable-core-ingress.driver-snapshot", "portable-core-ingress.fixed-generation", "portable-core-ingress.fixed-schema-loading", diff --git a/scripts/test/portable-core-ingress.test.sh b/scripts/test/portable-core-ingress.test.sh index 5009078..1197466 100755 --- a/scripts/test/portable-core-ingress.test.sh +++ b/scripts/test/portable-core-ingress.test.sh @@ -202,6 +202,13 @@ escape_file="$ingress_tmp/escape.json" no_lf_file="$ingress_tmp/no-lf.json" extra_lf_file="$ingress_tmp/extra-lf.json" unsorted_file="$ingress_tmp/unsorted.json" +nan_file="$ingress_tmp/nan.json" +infinity_file="$ingress_tmp/infinity.json" +negative_infinity_file="$ingress_tmp/negative-infinity.json" +leading_plus_file="$ingress_tmp/leading-plus.json" +leading_zero_file="$ingress_tmp/leading-zero.json" +trailing_dot_file="$ingress_tmp/trailing-dot.json" +leading_dot_file="$ingress_tmp/leading-dot.json" : > "$empty_file" printf '{}\n{}\n' > "$multi_file" printf '\357\273\277{}\n' > "$bom_file" @@ -212,11 +219,17 @@ printf '{"x":"\\u0061"}\n' > "$escape_file" printf '{}' > "$no_lf_file" printf '{}\n\n' > "$extra_lf_file" printf '{"b":1,"a":2}\n' > "$unsorted_file" +printf 'NaN\n' > "$nan_file" +printf 'Infinity\n' > "$infinity_file" +printf '%s\n' -Infinity > "$negative_infinity_file" +printf '+1\n' > "$leading_plus_file" +printf '01\n' > "$leading_zero_file" +printf '1.\n' > "$trailing_dot_file" +printf '.1\n' > "$leading_dot_file" start_ingress document expect_failure unreadable-input E_RUNTIME portable_core_ingress_snapshot \ "$ingress_tmp/distinctive-missing-input-SECRET.json" -mark_test portable-core-ingress.test.legacy-039-validate-profile-set-unreadable-input stop_ingress document_finish_failure() { @@ -249,6 +262,13 @@ document_finish_failure extra-final-lf E_CANONICAL "$extra_lf_file" mark_test portable-core-ingress.test.legacy-057-extra-final-lf-non-canonical document_finish_failure unsorted-keys E_CANONICAL "$unsorted_file" mark_test portable-core-ingress.test.legacy-059-unsorted-keys-non-canonical +document_finish_failure non-json-nan E_PARSE "$nan_file" +document_finish_failure non-json-infinity E_PARSE "$infinity_file" +document_finish_failure non-json-negative-infinity E_PARSE "$negative_infinity_file" +document_finish_failure non-json-leading-plus E_PARSE "$leading_plus_file" +document_finish_failure non-json-leading-zero E_PARSE "$leading_zero_file" +document_finish_failure non-json-trailing-dot E_PARSE "$trailing_dot_file" +document_finish_failure non-json-leading-dot E_PARSE "$leading_dot_file" mark_rule portable-core-ingress.snapshot-readable mark_rule portable-core-ingress.single-root-json mark_rule portable-core-ingress.utf8-json @@ -312,6 +332,69 @@ else fi stop_ingress +route_one="$ingress_tmp/route-1.json" +route_two="$ingress_tmp/route-2.json" +route_three="$ingress_tmp/route-3.json" +route_four="$ingress_tmp/route-4.json" +printf '{"index":1}\n' > "$route_one" +printf '{"index":2}\n' > "$route_two" +printf '{"index":3}\n' > "$route_three" +printf '{"index":4}\n' > "$route_four" + +route_driver_case() { + local mode="$1" + local expected_count="$2" + shift 2 + local route_input + start_ingress "$mode" + for route_input in "$@"; do + portable_core_ingress_snapshot "$route_input" + done + portable_core_ingress_finish_driver + if "$ingress_jq" -e --arg mode "$mode" --argjson count "$expected_count" \ + '.mode == $mode and (.docs|length) == $count and + [.docs[].content.index] == [range(1;($count+1))]' \ + "$PORTABLE_CORE_INGRESS_DRIVER" >/dev/null; then + ingress_direct_total=$((ingress_direct_total + 1)) + ingress_direct_passed=$((ingress_direct_passed + 1)) + else + ingress_direct_total=$((ingress_direct_total + 1)) + fail_case "$mode route did not preserve mode, count, and input order" + fi + stop_ingress +} + +route_driver_case document 1 "$route_one" +route_driver_case profile-set 4 "$route_one" "$route_two" "$route_three" "$route_four" +route_driver_case stage-run 3 "$route_one" "$route_two" "$route_three" + +route_failure_case() { + local mode="$1" + local missing_position="$2" + local supplied_count="$3" + local route_index + start_ingress "$mode" + for ((route_index = 1; route_index <= supplied_count; route_index++)); do + if [ "$route_index" -eq "$missing_position" ]; then + expect_failure "route-$mode-missing-$route_index" E_RUNTIME \ + portable_core_ingress_snapshot "$ingress_tmp/route-missing-$route_index-SECRET.json" + break + fi + portable_core_ingress_snapshot "$ingress_tmp/route-$route_index.json" + done + stop_ingress +} + +route_failure_case document 1 1 +for route_position in 1 2 3 4; do + route_failure_case profile-set "$route_position" 4 +done +for route_position in 1 2 3; do + route_failure_case stage-run "$route_position" 3 +done +mark_rule portable-core-ingress.command-routes +mark_test portable-core-ingress.test.legacy-039-validate-profile-set-unreadable-input + runtime_failure() { local case_id="$1" local expected="$2" @@ -486,7 +569,7 @@ stop_ingress start_ingress document jq_canonical_fail="$ingress_tmp/jq-canonical-fail" printf '%s\n' '#!/bin/sh' \ - 'case " $* " in *" -s -e "*) exit 0 ;; esac' \ + 'case " $* " in *" -s -e "*|*" -Rse "*) exit 0 ;; esac' \ 'printf "%s\\n" "jq SECRET path" >&2' \ 'exit 1' > "$jq_canonical_fail" chmod +x "$jq_canonical_fail" @@ -714,15 +797,34 @@ fi manifest_prefix="$ingress_tmp/current-manifest-prefix" manifest_base_lines="$("$ingress_jq" -r '.metadata.prior_manifest_lines' "$ingress_fixture")" manifest_base_digest="$("$ingress_jq" -r '.metadata.prior_manifest_sha256' "$ingress_fixture")" -head -n "$manifest_base_lines" "$ingress_manifest" > "$manifest_prefix" -if [ "$(sha256_path "$manifest_prefix")" = "$manifest_base_digest" ] && - [ "$(tail -n 4 "$ingress_manifest")" = "$required_paths" ]; then +manifest_block_start=$((manifest_base_lines + 1)) +manifest_block_end=$((manifest_base_lines + 4)) + +ingress_manifest_block_ok() { + local candidate_manifest="$1" + head -n "$manifest_base_lines" "$candidate_manifest" > "$manifest_prefix" + [ "$(sha256_path "$manifest_prefix")" = "$manifest_base_digest" ] && + [ "$(sed -n "${manifest_block_start},${manifest_block_end}p" "$candidate_manifest")" = \ + "$required_paths" ] +} + +if ingress_manifest_block_ok "$ingress_manifest"; then ingress_guard_total=$((ingress_guard_total + 1)) ingress_guard_passed=$((ingress_guard_passed + 1)) else ingress_guard_total=$((ingress_guard_total + 1)) fail_case 'restore manifest is not an exact append' fi +growing_manifest="$ingress_tmp/growing-manifest" +cp "$ingress_manifest" "$growing_manifest" +printf '%s\n' 'scripts/test/future-portable-core-unit.test.sh' >> "$growing_manifest" +if ingress_manifest_block_ok "$growing_manifest"; then + ingress_guard_total=$((ingress_guard_total + 1)) + ingress_guard_passed=$((ingress_guard_passed + 1)) +else + ingress_guard_total=$((ingress_guard_total + 1)) + fail_case 'restore manifest proof rejected a later append' +fi mark_rule portable-core-ingress.restore-manifest mark_test portable-core-ingress.test.mktemp-failure-sanitized-e-runtime From e953d9f20e3eecedcd11c3ea2aa99766af410822 Mon Sep 17 00:00:00 2001 From: ci Date: Sat, 29 Aug 2026 22:45:55 -0400 Subject: [PATCH 04/13] fix: separate ingress parse runtime failures --- .../core-ingress.sh | 22 +++++++++++++++---- scripts/test/portable-core-ingress.test.sh | 21 ++++++++++++++++++ 2 files changed, 39 insertions(+), 4 deletions(-) diff --git a/core/v1/generations/g-14b7ad8ce54c3b8c585ff92063d71551ffc7394cc2294d0297bc7d2b8da2c386/core-ingress.sh b/core/v1/generations/g-14b7ad8ce54c3b8c585ff92063d71551ffc7394cc2294d0297bc7d2b8da2c386/core-ingress.sh index 437a698..f12434c 100644 --- a/core/v1/generations/g-14b7ad8ce54c3b8c585ff92063d71551ffc7394cc2294d0297bc7d2b8da2c386/core-ingress.sh +++ b/core/v1/generations/g-14b7ad8ce54c3b8c585ff92063d71551ffc7394cc2294d0297bc7d2b8da2c386/core-ingress.sh @@ -283,6 +283,7 @@ portable_core_ingress_finish_driver() { local raw_path local canonical_path local compare_status + local probe_status local json_token_pattern [ "$#" -eq 0 ] && [ "${PORTABLE_CORE_INGRESS_COUNT:-0}" -gt 0 ] || { @@ -299,15 +300,28 @@ portable_core_ingress_finish_driver() { json_token_pattern='(?:[ \t\r\n]+|[\[\]{}:,]|"(?:[^"\\\x00-\x1f]|\\(?:["\\/bfnrt]|u[0-9A-Fa-f]{4}))*"|(?/dev/null 2>/dev/null; then - portable_core_ingress_error E_PARSE + : + else + probe_status=$? + case "$probe_status" in + 1|4) portable_core_ingress_error E_PARSE ;; + *) portable_core_ingress_error E_RUNTIME ;; + esac return 1 fi - if ! "$PORTABLE_CORE_INGRESS_JQ" -Rse --arg token "$json_token_pattern" \ + if "$PORTABLE_CORE_INGRESS_JQ" -Rse --arg token "$json_token_pattern" \ '(if startswith("\ufeff") then .[1:] else . end) | gsub($token;"") == ""' "$raw_path" >/dev/null 2>/dev/null; then - portable_core_ingress_error E_PARSE + : + else + probe_status=$? + if [ "$probe_status" -eq 1 ]; then + portable_core_ingress_error E_PARSE + else + portable_core_ingress_error E_RUNTIME + fi return 1 fi canonical_path="$PORTABLE_CORE_INGRESS_TEMP/canonical.$((input_index + 1))" diff --git a/scripts/test/portable-core-ingress.test.sh b/scripts/test/portable-core-ingress.test.sh index 1197466..071e5f9 100755 --- a/scripts/test/portable-core-ingress.test.sh +++ b/scripts/test/portable-core-ingress.test.sh @@ -566,6 +566,27 @@ portable_core_ingress_snapshot "$canonical_file" runtime_failure sha-multiline-digest E_RUNTIME portable_core_ingress_finish_driver stop_ingress +start_ingress document +parser_probe_fail="$ingress_tmp/parser-probe-fail" +printf '%s\n' '#!/bin/sh' 'printf "%s\\n" "parser SECRET path" >&2' 'exit 2' > "$parser_probe_fail" +chmod +x "$parser_probe_fail" +PORTABLE_CORE_INGRESS_JQ="$parser_probe_fail" +portable_core_ingress_snapshot "$canonical_file" +runtime_failure parser-probe-runtime-failure E_RUNTIME portable_core_ingress_finish_driver +stop_ingress + +start_ingress document +real_ingress_jq="$PORTABLE_CORE_INGRESS_JQ" +lexer_probe_fail="$ingress_tmp/lexer-probe-fail" +printf '%s\n' '#!/bin/sh' \ + 'case " $* " in *" -Rse "*) printf "%s\\n" "lexer SECRET path" >&2; exit 2 ;; esac' \ + "exec \"$real_ingress_jq\" \"\$@\"" > "$lexer_probe_fail" +chmod +x "$lexer_probe_fail" +PORTABLE_CORE_INGRESS_JQ="$lexer_probe_fail" +portable_core_ingress_snapshot "$canonical_file" +runtime_failure lexer-probe-runtime-failure E_RUNTIME portable_core_ingress_finish_driver +stop_ingress + start_ingress document jq_canonical_fail="$ingress_tmp/jq-canonical-fail" printf '%s\n' '#!/bin/sh' \ From d8dfaa4d622e2d00dba9f1d6824a18a4799cde25 Mon Sep 17 00:00:00 2001 From: ci Date: Sat, 29 Aug 2026 23:18:41 -0400 Subject: [PATCH 05/13] fix: validate ingress bytes before jq --- .../core-ingress.sh | 118 ++++++++++++++++++ .../test/portable-core-ingress-fixtures.json | 1 + scripts/test/portable-core-ingress.test.sh | 93 +++++++++++++- 3 files changed, 211 insertions(+), 1 deletion(-) diff --git a/core/v1/generations/g-14b7ad8ce54c3b8c585ff92063d71551ffc7394cc2294d0297bc7d2b8da2c386/core-ingress.sh b/core/v1/generations/g-14b7ad8ce54c3b8c585ff92063d71551ffc7394cc2294d0297bc7d2b8da2c386/core-ingress.sh index f12434c..784e895 100644 --- a/core/v1/generations/g-14b7ad8ce54c3b8c585ff92063d71551ffc7394cc2294d0297bc7d2b8da2c386/core-ingress.sh +++ b/core/v1/generations/g-14b7ad8ce54c3b8c585ff92063d71551ffc7394cc2294d0297bc7d2b8da2c386/core-ingress.sh @@ -144,6 +144,21 @@ portable_core_ingress_open() { portable_core_ingress_error E_RUNTIME return 1 } + PORTABLE_CORE_INGRESS_OD="$(command -v od 2>/dev/null)" || { + portable_core_ingress_error E_RUNTIME + return 1 + } + PORTABLE_CORE_INGRESS_AWK="$(command -v awk 2>/dev/null)" || { + portable_core_ingress_error E_RUNTIME + return 1 + } + case "$PORTABLE_CORE_INGRESS_OD:$PORTABLE_CORE_INGRESS_AWK" in + /*:/*) ;; + *) + portable_core_ingress_error E_RUNTIME + return 1 + ;; + esac temp_path="$(mktemp -d /tmp/ystack-portable-core-ingress.XXXXXXXX 2>/dev/null)" || { portable_core_ingress_error E_RUNTIME @@ -172,6 +187,7 @@ portable_core_ingress_open() { PORTABLE_CORE_INGRESS_RAW_PATHS=() PORTABLE_CORE_INGRESS_RAW_SIZES=() PORTABLE_CORE_INGRESS_CANONICAL_PATHS=() + PORTABLE_CORE_INGRESS_DEPTH_OVER=() } portable_core_ingress_begin() { @@ -205,6 +221,7 @@ portable_core_ingress_begin() { PORTABLE_CORE_INGRESS_RAW_PATHS=() PORTABLE_CORE_INGRESS_RAW_SIZES=() PORTABLE_CORE_INGRESS_CANONICAL_PATHS=() + PORTABLE_CORE_INGRESS_DEPTH_OVER=() } portable_core_ingress_digest() { @@ -238,6 +255,96 @@ portable_core_ingress_digest() { PORTABLE_CORE_INGRESS_SHA256="$digest" } +portable_core_ingress_scan_raw() { + local raw_path="$1" + local scan_status + local -a pipeline_status + + portable_core_ingress_regular_file "$raw_path" && [ -r "$raw_path" ] || { + portable_core_ingress_error E_RUNTIME + return 1 + } + if "$PORTABLE_CORE_INGRESS_OD" -An -v -t u1 "$raw_path" 2>/dev/null | + "$PORTABLE_CORE_INGRESS_AWK" ' + BEGIN { + in_string = 0; escaped = 0; need = 0; invalid = 0; + next_min = 128; next_max = 191; depth = 0; max_depth = 0 + } + { + for (i = 1; i <= NF; i++) { + byte = $i + 0 + ascii = (need == 0 && byte <= 127) + if (need > 0) { + if (byte < next_min || byte > next_max) { + invalid = 1; need = 0 + } else { + need--; next_min = 128; next_max = 191 + } + } else if (byte <= 127) { + # ASCII is valid and also drives JSON string/depth state below. + } else if (byte >= 194 && byte <= 223) { + need = 1; next_min = 128; next_max = 191 + } else if (byte == 224) { + need = 2; next_min = 160; next_max = 191 + } else if ((byte >= 225 && byte <= 236) || + (byte >= 238 && byte <= 239)) { + need = 2; next_min = 128; next_max = 191 + } else if (byte == 237) { + need = 2; next_min = 128; next_max = 159 + } else if (byte == 240) { + need = 3; next_min = 144; next_max = 191 + } else if (byte >= 241 && byte <= 243) { + need = 3; next_min = 128; next_max = 191 + } else if (byte == 244) { + need = 3; next_min = 128; next_max = 143 + } else { + invalid = 1 + } + + if (ascii) { + if (in_string) { + if (escaped) escaped = 0 + else if (byte == 92) escaped = 1 + else if (byte == 34) in_string = 0 + } else if (byte == 34) { + in_string = 1 + } else if (byte == 91 || byte == 123) { + depth++ + if (depth > max_depth) max_depth = depth + } else if ((byte == 93 || byte == 125) && depth > 0) { + depth-- + } + } + } + } + END { + if (invalid || need != 0) exit 31 + if (max_depth > 32) exit 32 + } + ' >/dev/null 2>/dev/null; then + pipeline_status=("${PIPESTATUS[@]}") + else + pipeline_status=("${PIPESTATUS[@]}") + fi + [ "${#pipeline_status[@]}" -eq 2 ] && [ "${pipeline_status[0]}" -eq 0 ] || { + portable_core_ingress_error E_RUNTIME + return 1 + } + scan_status="${pipeline_status[1]}" + case "$scan_status" in + 0) PORTABLE_CORE_INGRESS_SCAN_DEPTH_OVER=false ;; + 31) + portable_core_ingress_error E_PARSE + return 1 + ;; + 32) PORTABLE_CORE_INGRESS_SCAN_DEPTH_OVER=true ;; + *) + portable_core_ingress_error E_RUNTIME + return 1 + ;; + esac +} + portable_core_ingress_snapshot() { local input_path local snapshot_number @@ -297,6 +404,17 @@ portable_core_ingress_finish_driver() { return 1 fi done + for ((input_index = 0; input_index < PORTABLE_CORE_INGRESS_COUNT; input_index++)); do + raw_path="${PORTABLE_CORE_INGRESS_RAW_PATHS[$input_index]}" + portable_core_ingress_scan_raw "$raw_path" || return 1 + PORTABLE_CORE_INGRESS_DEPTH_OVER[input_index]="$PORTABLE_CORE_INGRESS_SCAN_DEPTH_OVER" + done + for ((input_index = 0; input_index < PORTABLE_CORE_INGRESS_COUNT; input_index++)); do + if [ "${PORTABLE_CORE_INGRESS_DEPTH_OVER[$input_index]}" = true ]; then + portable_core_ingress_error E_LIMIT + return 1 + fi + done json_token_pattern='(?:[ \t\r\n]+|[\[\]{}:,]|"(?:[^"\\\x00-\x1f]|\\(?:["\\/bfnrt]|u[0-9A-Fa-f]{4}))*"|(? "$multi_file" printf '\357\273\277{}\n' > "$bom_file" printf '\200\n' > "$utf8_file" +printf '{"x":"\200"}\n' > "$utf8_inside_file" +printf '{"x":"\302"}\n' > "$utf8_truncated_file" +printf '{"x":"\300\257"}\n' > "$utf8_overlong_file" +printf '{"x":"\355\240\200"}\n' > "$utf8_surrogate_file" +printf '{"x":"\364\220\200\200"}\n' > "$utf8_too_high_file" +printf '{"x":"\302\200\337\277\340\240\200\357\277\277\360\220\200\200\364\217\277\277"}\n' \ + > "$utf8_valid_boundaries_file" printf '{"a":1,"a":2}\n' > "$duplicate_file" printf '{ "a": 1 }\n' > "$whitespace_file" printf '{"x":"\\u0061"}\n' > "$escape_file" @@ -250,6 +263,16 @@ document_finish_failure bom-prefix E_CANONICAL "$bom_file" mark_test portable-core-ingress.test.legacy-045-bom-prefix document_finish_failure invalid-utf8 E_PARSE "$utf8_file" mark_test portable-core-ingress.test.legacy-047-invalid-utf-8 +document_finish_failure invalid-utf8-inside-string E_PARSE "$utf8_inside_file" +document_finish_failure truncated-utf8 E_PARSE "$utf8_truncated_file" +document_finish_failure overlong-utf8 E_PARSE "$utf8_overlong_file" +document_finish_failure surrogate-utf8 E_PARSE "$utf8_surrogate_file" +document_finish_failure too-high-utf8 E_PARSE "$utf8_too_high_file" +start_ingress document +expect_success valid-utf8-boundaries-snapshot \ + portable_core_ingress_snapshot "$utf8_valid_boundaries_file" +expect_success valid-utf8-boundaries-driver portable_core_ingress_finish_driver +stop_ingress document_finish_failure duplicate-keys E_CANONICAL "$duplicate_file" mark_test portable-core-ingress.test.legacy-049-duplicate-keys-non-canonical document_finish_failure alternate-whitespace E_CANONICAL "$whitespace_file" @@ -287,6 +310,36 @@ PY [ "$(wc -c < "$ingress_tmp/at-limit.json" | tr -d ' ')" -eq 1048576 ] [ "$(wc -c < "$ingress_tmp/over-limit.json" | tr -d ' ')" -eq 1048577 ] +python3 - "$ingress_tmp/depth-32.json" "$ingress_tmp/depth-33.json" \ + "$ingress_tmp/depth-257.json" "$ingress_tmp/depth-string.json" <<'PY' +import json +import sys + +for path, depth in zip(sys.argv[1:4], (32, 33, 257)): + open(path, "wb").write(("[" * depth + "0" + "]" * depth + "\n").encode()) +value = ("[{" * 40) + '"quoted"' + "\\" + ("]}" * 40) +encoded = json.dumps({"x": value}, ensure_ascii=False, separators=(",", ":")) + "\n" +open(sys.argv[4], "wb").write(encoded.encode()) +PY + +start_ingress document +expect_success depth-32-snapshot portable_core_ingress_snapshot "$ingress_tmp/depth-32.json" +expect_success depth-32-driver portable_core_ingress_finish_driver +stop_ingress +start_ingress document +portable_core_ingress_snapshot "$ingress_tmp/depth-33.json" +expect_failure depth-33-limit E_LIMIT portable_core_ingress_finish_driver +stop_ingress +start_ingress document +portable_core_ingress_snapshot "$ingress_tmp/depth-257.json" +expect_failure depth-257-limit E_LIMIT portable_core_ingress_finish_driver +stop_ingress +start_ingress document +expect_success depth-string-snapshot portable_core_ingress_snapshot "$ingress_tmp/depth-string.json" +expect_success depth-string-driver portable_core_ingress_finish_driver +stop_ingress +mark_rule portable-core-ingress.raw-depth-limit + start_ingress document expect_success at-limit-snapshot portable_core_ingress_snapshot "$ingress_tmp/at-limit.json" expect_success at-limit-driver portable_core_ingress_finish_driver @@ -313,6 +366,23 @@ portable_core_ingress_snapshot "$ingress_tmp/over-limit.json" expect_failure multi-runtime-before-limit E_RUNTIME portable_core_ingress_snapshot \ "$ingress_tmp/missing-later-input-SECRET.json" stop_ingress +printf '{"x":' > "$ingress_tmp/unclosed-within-depth.json" +start_ingress profile-set +portable_core_ingress_snapshot "$ingress_tmp/depth-33.json" +portable_core_ingress_snapshot "$utf8_inside_file" +expect_failure multi-utf8-after-depth E_PARSE portable_core_ingress_finish_driver +stop_ingress +start_ingress profile-set +portable_core_ingress_snapshot "$utf8_inside_file" +portable_core_ingress_snapshot "$ingress_tmp/depth-33.json" +expect_failure multi-utf8-before-depth E_PARSE portable_core_ingress_finish_driver +stop_ingress +start_ingress profile-set +portable_core_ingress_snapshot "$ingress_tmp/unclosed-within-depth.json" +portable_core_ingress_snapshot "$ingress_tmp/depth-33.json" +expect_failure multi-depth-precheck-before-jq-parse E_LIMIT portable_core_ingress_finish_driver +stop_ingress +document_finish_failure unclosed-within-depth E_PARSE "$ingress_tmp/unclosed-within-depth.json" snapshot_original="$ingress_tmp/snapshot-original.json" printf '{"v":1}\n' > "$snapshot_original" @@ -441,6 +511,27 @@ chmod +x "$wc_failure" PORTABLE_CORE_INGRESS_WC="$wc_failure" runtime_failure byte-count-failure E_RUNTIME portable_core_ingress_snapshot "$canonical_file" stop_ingress +start_ingress document +portable_core_ingress_snapshot "$canonical_file" +rm "$PORTABLE_CORE_INGRESS_SNAPSHOT" +runtime_failure private-snapshot-read-failure E_RUNTIME portable_core_ingress_finish_driver +stop_ingress +start_ingress document +od_failure="$ingress_tmp/od-failure" +printf '%s\n' '#!/bin/sh' 'printf "%s\\n" "od SECRET path" >&2' 'exit 2' > "$od_failure" +chmod +x "$od_failure" +PORTABLE_CORE_INGRESS_OD="$od_failure" +portable_core_ingress_snapshot "$canonical_file" +runtime_failure byte-scanner-read-failure E_RUNTIME portable_core_ingress_finish_driver +stop_ingress +start_ingress document +awk_failure="$ingress_tmp/awk-failure" +printf '%s\n' '#!/bin/sh' 'printf "%s\\n" "awk SECRET path" >&2' 'exit 2' > "$awk_failure" +chmod +x "$awk_failure" +PORTABLE_CORE_INGRESS_AWK="$awk_failure" +portable_core_ingress_snapshot "$canonical_file" +runtime_failure byte-scanner-logic-failure E_RUNTIME portable_core_ingress_finish_driver +stop_ingress fake_jq_bin="$ingress_tmp/fake-jq-bin" mkdir -p "$fake_jq_bin" @@ -459,7 +550,7 @@ make_isolated_bin() { local tool_path mkdir -p "$destination" ln -s "$ingress_jq" "$destination/jq" - for tool in dirname head wc cmp cat rm mktemp; do + for tool in dirname head wc cmp cat rm mktemp od awk; do tool_path="$(command -v "$tool")" ln -s "$tool_path" "$destination/$tool" done From c2d069a26891030237cb8f1e194db693d20b4324 Mon Sep 17 00:00:00 2001 From: ci Date: Sat, 29 Aug 2026 23:43:14 -0400 Subject: [PATCH 06/13] fix: preserve ingress validation precedence --- .../core-ingress.sh | 143 +++++++++++++++--- scripts/test/portable-core-ingress.test.sh | 27 +++- 2 files changed, 145 insertions(+), 25 deletions(-) diff --git a/core/v1/generations/g-14b7ad8ce54c3b8c585ff92063d71551ffc7394cc2294d0297bc7d2b8da2c386/core-ingress.sh b/core/v1/generations/g-14b7ad8ce54c3b8c585ff92063d71551ffc7394cc2294d0297bc7d2b8da2c386/core-ingress.sh index 784e895..c037fa1 100644 --- a/core/v1/generations/g-14b7ad8ce54c3b8c585ff92063d71551ffc7394cc2294d0297bc7d2b8da2c386/core-ingress.sh +++ b/core/v1/generations/g-14b7ad8ce54c3b8c585ff92063d71551ffc7394cc2294d0297bc7d2b8da2c386/core-ingress.sh @@ -385,6 +385,119 @@ portable_core_ingress_snapshot() { PORTABLE_CORE_INGRESS_COUNT="$snapshot_number" } +portable_core_ingress_stream_canonical() { + local raw_path="$1" + local canonical_path="$2" + local stream_status + + if "$PORTABLE_CORE_INGRESS_JQ" -n --stream -r ' + def value_kind($value): + if ($value | type) == "number" then "array" + elif ($value | type) == "string" then "object" + else null + end; + def opener($kind): if $kind == "array" then "[" else "{" end; + def closer($kind): if $kind == "array" then "]" else "}" end; + def select_child($value): + ((.stack | length) - 1) as $top | + .stack[$top] as $parent | + if $parent.kind == "array" then + if (($value | type) != "number") or ($value != $parent.count) then + .runtime = false + else + .out += (if $parent.count > 0 then "," else "" end) | + .stack[$top].count += 1 | + .stack[$top].last = $value + end + elif ($value | type) != "string" then + .runtime = false + else + .out += (if $parent.count > 0 then "," else "" end) + + ($value | tojson) + ":" | + if ($parent.count > 0) and (($parent.last < $value) | not) then + .ordered = false + else . end | + .stack[$top].count += 1 | + .stack[$top].last = $value + end; + def descend($path; $value; $index): + if (.runtime | not) then . + else + select_child($path[$index]) | + if (.runtime | not) then . + elif ($index + 1) < ($path | length) then + value_kind($path[$index + 1]) as $kind | + if $kind == null then .runtime = false + else + .out += opener($kind) | + .open_path += [$path[$index]] | + .stack += [{kind: $kind, count: 0, last: null}] | + descend($path; $value; $index + 1) + end + else .out += ($value | tojson) + end + end; + reduce inputs as $event + ({out: "", stack: [], open_path: [], started: false, done: false, + parse: true, runtime: true, ordered: true}; + ($event[0]) as $path | + if ($event | length) == 2 then + ($event[1]) as $value | + if .done then .parse = false + elif (.stack | length) == 0 then + if .started then .parse = false + elif ($path | length) == 0 then + .out = ($value | tojson) | .started = true | .done = true + else + value_kind($path[0]) as $kind | + if $kind == null then .runtime = false + else + .out = opener($kind) | + .stack = [{kind: $kind, count: 0, last: null}] | + .started = true | + descend($path; $value; 0) + end + end + else + (.stack | length) as $depth | + if (($path | length) < $depth) or + ($path[0:($depth - 1)] != .open_path) then + .runtime = false + else descend($path; $value; $depth - 1) + end + end + elif ($event | length) == 1 then + (.stack | length) as $depth | + if ($depth == 0) or (($path | length) != $depth) or + ($path[0:($depth - 1)] != .open_path) or + ($path[-1] != .stack[-1].last) then + .runtime = false + else + .out += closer(.stack[-1].kind) | + .stack = .stack[0:-1] | + .open_path = .open_path[0:-1] | + if (.stack | length) == 0 then .done = true else . end + end + else .runtime = false + end) + | if (.runtime | not) then halt_error(42) + elif (.parse | not) or (.started | not) or (.done | not) or + ((.stack | length) != 0) then halt_error(41) + elif .ordered then .out + else "" + end + ' "$raw_path" 2>/dev/null > "$canonical_path"; then + return 0 + else + stream_status=$? + fi + case "$stream_status" in + 5|41) portable_core_ingress_error E_PARSE ;; + *) portable_core_ingress_error E_RUNTIME ;; + esac + return 1 +} + portable_core_ingress_finish_driver() { local input_index local raw_path @@ -409,26 +522,9 @@ portable_core_ingress_finish_driver() { portable_core_ingress_scan_raw "$raw_path" || return 1 PORTABLE_CORE_INGRESS_DEPTH_OVER[input_index]="$PORTABLE_CORE_INGRESS_SCAN_DEPTH_OVER" done - for ((input_index = 0; input_index < PORTABLE_CORE_INGRESS_COUNT; input_index++)); do - if [ "${PORTABLE_CORE_INGRESS_DEPTH_OVER[$input_index]}" = true ]; then - portable_core_ingress_error E_LIMIT - return 1 - fi - done json_token_pattern='(?:[ \t\r\n]+|[\[\]{}:,]|"(?:[^"\\\x00-\x1f]|\\(?:["\\/bfnrt]|u[0-9A-Fa-f]{4}))*"|(?/dev/null 2>/dev/null; then - : - else - probe_status=$? - case "$probe_status" in - 1|4) portable_core_ingress_error E_PARSE ;; - *) portable_core_ingress_error E_RUNTIME ;; - esac - return 1 - fi if "$PORTABLE_CORE_INGRESS_JQ" -Rse --arg token "$json_token_pattern" \ '(if startswith("\ufeff") then .[1:] else . end) | gsub($token;"") == ""' "$raw_path" >/dev/null 2>/dev/null; then @@ -443,12 +539,7 @@ portable_core_ingress_finish_driver() { return 1 fi canonical_path="$PORTABLE_CORE_INGRESS_TEMP/canonical.$((input_index + 1))" - if ! "$PORTABLE_CORE_INGRESS_JQ" -s -S -c \ - 'if length == 1 then .[0] else error("root-count") end' "$raw_path" \ - 2>/dev/null > "$canonical_path"; then - portable_core_ingress_error E_RUNTIME - return 1 - fi + portable_core_ingress_stream_canonical "$raw_path" "$canonical_path" || return 1 PORTABLE_CORE_INGRESS_CANONICAL_PATHS[input_index]="$canonical_path" done for ((input_index = 0; input_index < PORTABLE_CORE_INGRESS_COUNT; input_index++)); do @@ -466,6 +557,12 @@ portable_core_ingress_finish_driver() { return 1 fi done + for ((input_index = 0; input_index < PORTABLE_CORE_INGRESS_COUNT; input_index++)); do + if [ "${PORTABLE_CORE_INGRESS_DEPTH_OVER[$input_index]}" = true ]; then + portable_core_ingress_error E_LIMIT + return 1 + fi + done for ((input_index = 0; input_index < PORTABLE_CORE_INGRESS_COUNT; input_index++)); do raw_path="${PORTABLE_CORE_INGRESS_RAW_PATHS[$input_index]}" portable_core_ingress_digest "$raw_path" || return 1 diff --git a/scripts/test/portable-core-ingress.test.sh b/scripts/test/portable-core-ingress.test.sh index ef7d2b0..cafcdb5 100755 --- a/scripts/test/portable-core-ingress.test.sh +++ b/scripts/test/portable-core-ingress.test.sh @@ -311,7 +311,10 @@ PY [ "$(wc -c < "$ingress_tmp/over-limit.json" | tr -d ' ')" -eq 1048577 ] python3 - "$ingress_tmp/depth-32.json" "$ingress_tmp/depth-33.json" \ - "$ingress_tmp/depth-257.json" "$ingress_tmp/depth-string.json" <<'PY' + "$ingress_tmp/depth-257.json" "$ingress_tmp/depth-string.json" \ + "$ingress_tmp/depth-33-malformed.json" \ + "$ingress_tmp/depth-33-noncanonical.json" \ + "$ingress_tmp/depth-257-noncanonical.json" <<'PY' import json import sys @@ -320,6 +323,9 @@ for path, depth in zip(sys.argv[1:4], (32, 33, 257)): value = ("[{" * 40) + '"quoted"' + "\\" + ("]}" * 40) encoded = json.dumps({"x": value}, ensure_ascii=False, separators=(",", ":")) + "\n" open(sys.argv[4], "wb").write(encoded.encode()) +open(sys.argv[5], "wb").write(("[" * 33 + "0" + "]" * 32 + "\n").encode()) +open(sys.argv[6], "wb").write(("[" * 33 + " 0" + "]" * 33 + "\n").encode()) +open(sys.argv[7], "wb").write(("[" * 257 + " 0" + "]" * 257 + "\n").encode()) PY start_ingress document @@ -334,6 +340,12 @@ start_ingress document portable_core_ingress_snapshot "$ingress_tmp/depth-257.json" expect_failure depth-257-limit E_LIMIT portable_core_ingress_finish_driver stop_ingress +document_finish_failure depth-33-malformed E_PARSE \ + "$ingress_tmp/depth-33-malformed.json" +document_finish_failure depth-33-noncanonical E_CANONICAL \ + "$ingress_tmp/depth-33-noncanonical.json" +document_finish_failure depth-257-noncanonical E_CANONICAL \ + "$ingress_tmp/depth-257-noncanonical.json" start_ingress document expect_success depth-string-snapshot portable_core_ingress_snapshot "$ingress_tmp/depth-string.json" expect_success depth-string-driver portable_core_ingress_finish_driver @@ -380,7 +392,18 @@ stop_ingress start_ingress profile-set portable_core_ingress_snapshot "$ingress_tmp/unclosed-within-depth.json" portable_core_ingress_snapshot "$ingress_tmp/depth-33.json" -expect_failure multi-depth-precheck-before-jq-parse E_LIMIT portable_core_ingress_finish_driver +expect_failure multi-parse-before-depth E_PARSE portable_core_ingress_finish_driver +stop_ingress +start_ingress profile-set +portable_core_ingress_snapshot "$ingress_tmp/depth-33-noncanonical.json" +portable_core_ingress_snapshot "$ingress_tmp/unclosed-within-depth.json" +expect_failure multi-deep-parse-before-canonical E_PARSE portable_core_ingress_finish_driver +stop_ingress +start_ingress profile-set +portable_core_ingress_snapshot "$ingress_tmp/depth-33.json" +portable_core_ingress_snapshot "$ingress_tmp/depth-257-noncanonical.json" +expect_failure multi-deep-canonical-before-limit E_CANONICAL \ + portable_core_ingress_finish_driver stop_ingress document_finish_failure unclosed-within-depth E_PARSE "$ingress_tmp/unclosed-within-depth.json" From f4f820d02aedd13463836ad30221df32b0206e74 Mon Sep 17 00:00:00 2001 From: ci Date: Sun, 30 Aug 2026 00:22:43 -0400 Subject: [PATCH 07/13] fix: bound ingress validation --- .../core-ingress.sh | 363 +++++++++++++----- scripts/test/portable-core-ingress.test.sh | 128 +++++- 2 files changed, 370 insertions(+), 121 deletions(-) diff --git a/core/v1/generations/g-14b7ad8ce54c3b8c585ff92063d71551ffc7394cc2294d0297bc7d2b8da2c386/core-ingress.sh b/core/v1/generations/g-14b7ad8ce54c3b8c585ff92063d71551ffc7394cc2294d0297bc7d2b8da2c386/core-ingress.sh index c037fa1..3a08883 100644 --- a/core/v1/generations/g-14b7ad8ce54c3b8c585ff92063d71551ffc7394cc2294d0297bc7d2b8da2c386/core-ingress.sh +++ b/core/v1/generations/g-14b7ad8ce54c3b8c585ff92063d71551ffc7394cc2294d0297bc7d2b8da2c386/core-ingress.sh @@ -188,6 +188,7 @@ portable_core_ingress_open() { PORTABLE_CORE_INGRESS_RAW_SIZES=() PORTABLE_CORE_INGRESS_CANONICAL_PATHS=() PORTABLE_CORE_INGRESS_DEPTH_OVER=() + PORTABLE_CORE_INGRESS_STREAM_REQUIRED=() } portable_core_ingress_begin() { @@ -222,6 +223,7 @@ portable_core_ingress_begin() { PORTABLE_CORE_INGRESS_RAW_SIZES=() PORTABLE_CORE_INGRESS_CANONICAL_PATHS=() PORTABLE_CORE_INGRESS_DEPTH_OVER=() + PORTABLE_CORE_INGRESS_STREAM_REQUIRED=() } portable_core_ingress_digest() { @@ -266,8 +268,99 @@ portable_core_ingress_scan_raw() { } if "$PORTABLE_CORE_INGRESS_OD" -An -v -t u1 "$raw_path" 2>/dev/null | "$PORTABLE_CORE_INGRESS_AWK" ' + function literal_expected(kind, position) { + if (kind == 1) { + if (position == 2) return 114 + if (position == 3) return 117 + if (position == 4) return 101 + } else if (kind == 2) { + if (position == 2) return 97 + if (position == 3) return 108 + if (position == 4) return 115 + if (position == 5) return 101 + } else { + if (position == 2) return 117 + if (position == 3) return 108 + if (position == 4) return 108 + } + return -1 + } + function start_token(byte) { + token_active = 1 + if (byte == 116) { + token_state = "literal"; literal_kind = 1; literal_position = 1; + literal_length = 4 + } else if (byte == 102) { + token_state = "literal"; literal_kind = 2; literal_position = 1; + literal_length = 5 + } else if (byte == 110) { + token_state = "literal"; literal_kind = 3; literal_position = 1; + literal_length = 4 + } else if (byte == 45) { + token_state = "sign" + } else if (byte == 48) { + token_state = "zero" + } else if (byte >= 49 && byte <= 57) { + token_state = "integer" + } else { + invalid_token = 1 + } + } + function advance_token(byte) { + if (!token_active) { + start_token(byte) + } else if (token_state == "literal") { + literal_position++ + if (literal_position > literal_length || + byte != literal_expected(literal_kind, literal_position)) { + invalid_token = 1 + } + } else if (token_state == "sign") { + if (byte == 48) token_state = "zero" + else if (byte >= 49 && byte <= 57) token_state = "integer" + else invalid_token = 1 + } else if (token_state == "zero") { + if (byte == 46) token_state = "decimal-mark" + else if (byte == 101 || byte == 69) token_state = "exponent-mark" + else invalid_token = 1 + } else if (token_state == "integer") { + if (byte >= 48 && byte <= 57) token_state = "integer" + else if (byte == 46) token_state = "decimal-mark" + else if (byte == 101 || byte == 69) token_state = "exponent-mark" + else invalid_token = 1 + } else if (token_state == "decimal-mark") { + if (byte >= 48 && byte <= 57) token_state = "fraction" + else invalid_token = 1 + } else if (token_state == "fraction") { + if (byte >= 48 && byte <= 57) token_state = "fraction" + else if (byte == 101 || byte == 69) token_state = "exponent-mark" + else invalid_token = 1 + } else if (token_state == "exponent-mark") { + if (byte == 43 || byte == 45) token_state = "exponent-sign" + else if (byte >= 48 && byte <= 57) token_state = "exponent" + else invalid_token = 1 + } else if (token_state == "exponent-sign") { + if (byte >= 48 && byte <= 57) token_state = "exponent" + else invalid_token = 1 + } else if (token_state == "exponent") { + if (byte < 48 || byte > 57) invalid_token = 1 + } else { + invalid_token = 1 + } + } + function finish_token() { + if (token_active && + !((token_state == "literal" && literal_position == literal_length) || + token_state == "zero" || token_state == "integer" || + token_state == "fraction" || token_state == "exponent")) { + invalid_token = 1 + } + token_active = 0 + token_state = "" + } BEGIN { in_string = 0; escaped = 0; need = 0; invalid = 0; + invalid_token = 0; token_active = 0; token_state = ""; next_min = 128; next_max = 191; depth = 0; max_depth = 0 } { @@ -307,18 +400,33 @@ portable_core_ingress_scan_raw() { else if (byte == 92) escaped = 1 else if (byte == 34) in_string = 0 } else if (byte == 34) { + finish_token() in_string = 1 } else if (byte == 91 || byte == 123) { + finish_token() depth++ if (depth > max_depth) max_depth = depth } else if ((byte == 93 || byte == 125) && depth > 0) { + finish_token() depth-- + } else if (byte == 93 || byte == 125 || byte == 44 || + byte == 58 || byte == 9 || byte == 10 || + byte == 13 || byte == 32) { + finish_token() + } else if (byte < 32) { + invalid_token = 1 + } else { + advance_token(byte) } + } else if (!in_string && token_active) { + invalid_token = 1 } } } END { - if (invalid || need != 0) exit 31 + finish_token() + if (invalid || need != 0 || invalid_token) exit 31 + if (max_depth > 128) exit 33 if (max_depth > 32) exit 32 } ' >/dev/null 2>/dev/null; then @@ -332,12 +440,22 @@ portable_core_ingress_scan_raw() { } scan_status="${pipeline_status[1]}" case "$scan_status" in - 0) PORTABLE_CORE_INGRESS_SCAN_DEPTH_OVER=false ;; + 0) + PORTABLE_CORE_INGRESS_SCAN_DEPTH_OVER=false + PORTABLE_CORE_INGRESS_SCAN_STREAM_REQUIRED=false + ;; 31) portable_core_ingress_error E_PARSE return 1 ;; - 32) PORTABLE_CORE_INGRESS_SCAN_DEPTH_OVER=true ;; + 32) + PORTABLE_CORE_INGRESS_SCAN_DEPTH_OVER=true + PORTABLE_CORE_INGRESS_SCAN_STREAM_REQUIRED=false + ;; + 33) + PORTABLE_CORE_INGRESS_SCAN_DEPTH_OVER=true + PORTABLE_CORE_INGRESS_SCAN_STREAM_REQUIRED=true + ;; *) portable_core_ingress_error E_RUNTIME return 1 @@ -385,114 +503,158 @@ portable_core_ingress_snapshot() { PORTABLE_CORE_INGRESS_COUNT="$snapshot_number" } +portable_core_ingress_tree_canonical() { + local raw_path="$1" + local canonical_path="$2" + local parse_status + + if "$PORTABLE_CORE_INGRESS_JQ" -s -e 'length == 1' "$raw_path" \ + >/dev/null 2>/dev/null; then + : + else + parse_status=$? + case "$parse_status" in + 1|4) portable_core_ingress_error E_PARSE ;; + *) portable_core_ingress_error E_RUNTIME ;; + esac + return 1 + fi + if "$PORTABLE_CORE_INGRESS_JQ" -s -S -c \ + 'if length == 1 then .[0] else error("root-count") end' "$raw_path" \ + 2>/dev/null > "$canonical_path"; then + return 0 + fi + portable_core_ingress_error E_RUNTIME + return 1 +} + portable_core_ingress_stream_canonical() { local raw_path="$1" local canonical_path="$2" - local stream_status + local marker_status + local -a pipeline_status - if "$PORTABLE_CORE_INGRESS_JQ" -n --stream -r ' - def value_kind($value): - if ($value | type) == "number" then "array" - elif ($value | type) == "string" then "object" + if "$PORTABLE_CORE_INGRESS_JQ" -n --stream --stream-errors -j ' + def component_kind($component): + if ($component | type) == "number" then "array" + elif ($component | type) == "string" then "object" else null end; - def opener($kind): if $kind == "array" then "[" else "{" end; + def first_children_valid($path): + all($path[]; + ((type == "number") and (. == 0)) or (type == "string")); + def open_first_children($path): + [$path[] | + if type == "number" then "[" + else "{" + (tojson) + ":" + end] | join(""); def closer($kind): if $kind == "array" then "]" else "}" end; - def select_child($value): - ((.stack | length) - 1) as $top | - .stack[$top] as $parent | - if $parent.kind == "array" then - if (($value | type) != "number") or ($value != $parent.count) then - .runtime = false - else - .out += (if $parent.count > 0 then "," else "" end) | - .stack[$top].count += 1 | - .stack[$top].last = $value - end - elif ($value | type) != "string" then - .runtime = false - else - .out += (if $parent.count > 0 then "," else "" end) + - ($value | tojson) + ":" | - if ($parent.count > 0) and (($parent.last < $value) | not) then - .ordered = false - else . end | - .stack[$top].count += 1 | - .stack[$top].last = $value - end; - def descend($path; $value; $index): - if (.runtime | not) then . - else - select_child($path[$index]) | - if (.runtime | not) then . - elif ($index + 1) < ($path | length) then - value_kind($path[$index + 1]) as $kind | - if $kind == null then .runtime = false - else - .out += opener($kind) | - .open_path += [$path[$index]] | - .stack += [{kind: $kind, count: 0, last: null}] | - descend($path; $value; $index + 1) - end - else .out += ($value | tojson) - end - end; - reduce inputs as $event - ({out: "", stack: [], open_path: [], started: false, done: false, - parse: true, runtime: true, ordered: true}; - ($event[0]) as $path | - if ($event | length) == 2 then + foreach (inputs, {end: true}) as $event + ({emit: "", last_path: [], open_depth: 0, started: false, + done: false, parse: true, runtime: true, ordered: true}; + if ($event | type) == "object" then + if (.runtime | not) then halt_error(42) + elif (.parse | not) or (.started | not) or (.done | not) or + (.open_depth != 0) then .emit = "\u0000" + elif .ordered then .emit = "\n" + else .emit = "!\n" + end + elif (.parse | not) then .emit = "" + elif (($event | type) != "array") or (($event | length) == 0) then + .runtime = false | .emit = "" + elif ($event[0] | type) == "string" then + .parse = false | .emit = "" + elif ($event[0] | type) != "array" then + .runtime = false | .emit = "" + elif ($event | length) == 2 then + ($event[0]) as $path | ($event[1]) as $value | - if .done then .parse = false - elif (.stack | length) == 0 then - if .started then .parse = false + if .done then .parse = false | .emit = "" + elif .open_depth == 0 then + if .started then .parse = false | .emit = "" elif ($path | length) == 0 then - .out = ($value | tojson) | .started = true | .done = true + .emit = ($value | tojson) | + .started = true | + .done = true + elif (first_children_valid($path) | not) then + .runtime = false | .emit = "" else - value_kind($path[0]) as $kind | - if $kind == null then .runtime = false - else - .out = opener($kind) | - .stack = [{kind: $kind, count: 0, last: null}] | - .started = true | - descend($path; $value; 0) - end + .emit = open_first_children($path) + ($value | tojson) | + .last_path = $path | + .open_depth = ($path | length) | + .started = true end else - (.stack | length) as $depth | + .open_depth as $depth | if (($path | length) < $depth) or - ($path[0:($depth - 1)] != .open_path) then - .runtime = false - else descend($path; $value; $depth - 1) + ((.last_path | length) < $depth) or + ($path[0:($depth - 1)] != .last_path[0:($depth - 1)]) then + .runtime = false | .emit = "" + else + ($path[$depth - 1]) as $current | + (.last_path[$depth - 1]) as $previous | + ($path[$depth:] | first_children_valid(.)) as $suffix_valid | + if (component_kind($current) == null) or + (component_kind($current) != component_kind($previous)) or + (($current | type) == "number" and + ($current != ($previous + 1))) or + ($suffix_valid | not) then + .runtime = false | .emit = "" + else + if (($current | type) == "string") and + (($previous < $current) | not) then + .ordered = false + else . end | + .emit = "," + + (if ($current | type) == "string" then + ($current | tojson) + ":" + else "" + end) + + open_first_children($path[$depth:]) + ($value | tojson) | + .last_path = $path | + .open_depth = ($path | length) + end end end elif ($event | length) == 1 then - (.stack | length) as $depth | - if ($depth == 0) or (($path | length) != $depth) or - ($path[0:($depth - 1)] != .open_path) or - ($path[-1] != .stack[-1].last) then - .runtime = false + ($event[0]) as $path | + if (.open_depth == 0) or + (($path | length) != .open_depth) or + (component_kind($path[-1]) == null) then + .runtime = false | .emit = "" else - .out += closer(.stack[-1].kind) | - .stack = .stack[0:-1] | - .open_path = .open_path[0:-1] | - if (.stack | length) == 0 then .done = true else . end + .emit = closer(component_kind($path[-1])) | + .open_depth -= 1 | + if .open_depth == 0 then .done = true else . end end - else .runtime = false - end) - | if (.runtime | not) then halt_error(42) - elif (.parse | not) or (.started | not) or (.done | not) or - ((.stack | length) != 0) then halt_error(41) - elif .ordered then .out - else "" - end + else .runtime = false | .emit = "" + end; + .emit) ' "$raw_path" 2>/dev/null > "$canonical_path"; then - return 0 + : else - stream_status=$? + portable_core_ingress_error E_RUNTIME + return 1 fi - case "$stream_status" in - 5|41) portable_core_ingress_error E_PARSE ;; + if "$PORTABLE_CORE_INGRESS_OD" -An -v -t u1 "$canonical_path" 2>/dev/null | + "$PORTABLE_CORE_INGRESS_AWK" ' + { + for (i = 1; i <= NF; i++) if (($i + 0) == 0) found = 1 + } + END { if (found) exit 41 } + ' >/dev/null 2>/dev/null; then + pipeline_status=("${PIPESTATUS[@]}") + else + pipeline_status=("${PIPESTATUS[@]}") + fi + [ "${#pipeline_status[@]}" -eq 2 ] && [ "${pipeline_status[0]}" -eq 0 ] || { + portable_core_ingress_error E_RUNTIME + return 1 + } + marker_status="${pipeline_status[1]}" + case "$marker_status" in + 0) return 0 ;; + 41) portable_core_ingress_error E_PARSE ;; *) portable_core_ingress_error E_RUNTIME ;; esac return 1 @@ -503,8 +665,6 @@ portable_core_ingress_finish_driver() { local raw_path local canonical_path local compare_status - local probe_status - local json_token_pattern [ "$#" -eq 0 ] && [ "${PORTABLE_CORE_INGRESS_COUNT:-0}" -gt 0 ] || { portable_core_ingress_error E_RUNTIME @@ -521,25 +681,16 @@ portable_core_ingress_finish_driver() { raw_path="${PORTABLE_CORE_INGRESS_RAW_PATHS[$input_index]}" portable_core_ingress_scan_raw "$raw_path" || return 1 PORTABLE_CORE_INGRESS_DEPTH_OVER[input_index]="$PORTABLE_CORE_INGRESS_SCAN_DEPTH_OVER" + PORTABLE_CORE_INGRESS_STREAM_REQUIRED[input_index]="$PORTABLE_CORE_INGRESS_SCAN_STREAM_REQUIRED" done - json_token_pattern='(?:[ \t\r\n]+|[\[\]{}:,]|"(?:[^"\\\x00-\x1f]|\\(?:["\\/bfnrt]|u[0-9A-Fa-f]{4}))*"|(?/dev/null 2>/dev/null; then - : + canonical_path="$PORTABLE_CORE_INGRESS_TEMP/canonical.$((input_index + 1))" + if [ "${PORTABLE_CORE_INGRESS_STREAM_REQUIRED[$input_index]}" = true ]; then + portable_core_ingress_stream_canonical "$raw_path" "$canonical_path" || return 1 else - probe_status=$? - if [ "$probe_status" -eq 1 ]; then - portable_core_ingress_error E_PARSE - else - portable_core_ingress_error E_RUNTIME - fi - return 1 + portable_core_ingress_tree_canonical "$raw_path" "$canonical_path" || return 1 fi - canonical_path="$PORTABLE_CORE_INGRESS_TEMP/canonical.$((input_index + 1))" - portable_core_ingress_stream_canonical "$raw_path" "$canonical_path" || return 1 PORTABLE_CORE_INGRESS_CANONICAL_PATHS[input_index]="$canonical_path" done for ((input_index = 0; input_index < PORTABLE_CORE_INGRESS_COUNT; input_index++)); do diff --git a/scripts/test/portable-core-ingress.test.sh b/scripts/test/portable-core-ingress.test.sh index cafcdb5..1d212c0 100755 --- a/scripts/test/portable-core-ingress.test.sh +++ b/scripts/test/portable-core-ingress.test.sh @@ -314,7 +314,16 @@ python3 - "$ingress_tmp/depth-32.json" "$ingress_tmp/depth-33.json" \ "$ingress_tmp/depth-257.json" "$ingress_tmp/depth-string.json" \ "$ingress_tmp/depth-33-malformed.json" \ "$ingress_tmp/depth-33-noncanonical.json" \ - "$ingress_tmp/depth-257-noncanonical.json" <<'PY' + "$ingress_tmp/depth-257-noncanonical.json" \ + "$ingress_tmp/depth-256.json" \ + "$ingress_tmp/depth-100000.json" \ + "$ingress_tmp/depth-100000-malformed.json" \ + "$ingress_tmp/dense-near-limit.json" \ + "$ingress_tmp/number-near-limit.json" \ + "$ingress_tmp/object-depth-128.json" \ + "$ingress_tmp/object-depth-129.json" \ + "$ingress_tmp/mixed-depth-128.json" \ + "$ingress_tmp/mixed-depth-129.json" <<'PY' import json import sys @@ -326,6 +335,28 @@ open(sys.argv[4], "wb").write(encoded.encode()) open(sys.argv[5], "wb").write(("[" * 33 + "0" + "]" * 32 + "\n").encode()) open(sys.argv[6], "wb").write(("[" * 33 + " 0" + "]" * 33 + "\n").encode()) open(sys.argv[7], "wb").write(("[" * 257 + " 0" + "]" * 257 + "\n").encode()) +open(sys.argv[8], "wb").write(("[" * 256 + "0" + "]" * 256 + "\n").encode()) +open(sys.argv[9], "wb").write(("[" * 100000 + "0" + "]" * 100000 + "\n").encode()) +open(sys.argv[10], "wb").write(("[" * 100000 + "0" + "]" * 99999 + "\n").encode()) +open(sys.argv[11], "wb").write(b"[" + b",".join([b"0"] * 524286) + b"]\n") +open(sys.argv[12], "wb").write(b"1" + b"1" * 1048573 + b"\n") +open(sys.argv[13], "wb").write(("{\"a\":" * 128 + "0" + "}" * 128 + "\n").encode()) +open(sys.argv[14], "wb").write(("{\"a\":" * 129 + "0" + "}" * 129 + "\n").encode()) + +def mixed(depth): + opens = [] + closes = [] + for index in range(depth): + if index % 2 == 0: + opens.append("[") + closes.append("]") + else: + opens.append("{\"a\":") + closes.append("}") + return "".join(opens) + "0" + "".join(reversed(closes)) + "\n" + +open(sys.argv[15], "wb").write(mixed(128).encode()) +open(sys.argv[16], "wb").write(mixed(129).encode()) PY start_ingress document @@ -340,6 +371,24 @@ start_ingress document portable_core_ingress_snapshot "$ingress_tmp/depth-257.json" expect_failure depth-257-limit E_LIMIT portable_core_ingress_finish_driver stop_ingress +start_ingress document +portable_core_ingress_snapshot "$ingress_tmp/depth-256.json" +expect_failure depth-256-limit E_LIMIT portable_core_ingress_finish_driver +stop_ingress +start_ingress document +portable_core_ingress_snapshot "$ingress_tmp/depth-100000.json" +expect_failure depth-100000-limit E_LIMIT portable_core_ingress_finish_driver +stop_ingress +document_finish_failure depth-100000-malformed E_PARSE \ + "$ingress_tmp/depth-100000-malformed.json" +document_finish_failure object-depth-128-limit E_LIMIT \ + "$ingress_tmp/object-depth-128.json" +document_finish_failure object-depth-129-limit E_LIMIT \ + "$ingress_tmp/object-depth-129.json" +document_finish_failure mixed-depth-128-limit E_LIMIT \ + "$ingress_tmp/mixed-depth-128.json" +document_finish_failure mixed-depth-129-limit E_LIMIT \ + "$ingress_tmp/mixed-depth-129.json" document_finish_failure depth-33-malformed E_PARSE \ "$ingress_tmp/depth-33-malformed.json" document_finish_failure depth-33-noncanonical E_CANONICAL \ @@ -352,6 +401,16 @@ expect_success depth-string-driver portable_core_ingress_finish_driver stop_ingress mark_rule portable-core-ingress.raw-depth-limit +[ "$(wc -c < "$ingress_tmp/dense-near-limit.json" | tr -d ' ')" -eq 1048574 ] +start_ingress document +expect_success dense-near-limit-snapshot portable_core_ingress_snapshot \ + "$ingress_tmp/dense-near-limit.json" +expect_success dense-near-limit-driver portable_core_ingress_finish_driver +stop_ingress +[ "$(wc -c < "$ingress_tmp/number-near-limit.json" | tr -d ' ')" -eq 1048575 ] +document_finish_failure number-near-limit E_CANONICAL \ + "$ingress_tmp/number-near-limit.json" + start_ingress document expect_success at-limit-snapshot portable_core_ingress_snapshot "$ingress_tmp/at-limit.json" expect_success at-limit-driver portable_core_ingress_finish_driver @@ -689,17 +748,20 @@ portable_core_ingress_snapshot "$canonical_file" runtime_failure parser-probe-runtime-failure E_RUNTIME portable_core_ingress_finish_driver stop_ingress -start_ingress document -real_ingress_jq="$PORTABLE_CORE_INGRESS_JQ" -lexer_probe_fail="$ingress_tmp/lexer-probe-fail" -printf '%s\n' '#!/bin/sh' \ - 'case " $* " in *" -Rse "*) printf "%s\\n" "lexer SECRET path" >&2; exit 2 ;; esac' \ - "exec \"$real_ingress_jq\" \"\$@\"" > "$lexer_probe_fail" -chmod +x "$lexer_probe_fail" -PORTABLE_CORE_INGRESS_JQ="$lexer_probe_fail" -portable_core_ingress_snapshot "$canonical_file" -runtime_failure lexer-probe-runtime-failure E_RUNTIME portable_core_ingress_finish_driver -stop_ingress +for stream_runtime_status in 5 41; do + start_ingress document + real_ingress_jq="$PORTABLE_CORE_INGRESS_JQ" + stream_runtime_fail="$ingress_tmp/stream-exit-$stream_runtime_status" + printf '%s\n' '#!/bin/sh' \ + "case \" \$* \" in *\" --stream --stream-errors \"*) exit $stream_runtime_status ;; esac" \ + "exec \"$real_ingress_jq\" \"\$@\"" > "$stream_runtime_fail" + chmod +x "$stream_runtime_fail" + PORTABLE_CORE_INGRESS_JQ="$stream_runtime_fail" + portable_core_ingress_snapshot "$ingress_tmp/depth-257.json" + runtime_failure "streaming-canonicalizer-exit-$stream_runtime_status" E_RUNTIME \ + portable_core_ingress_finish_driver + stop_ingress +done start_ingress document jq_canonical_fail="$ingress_tmp/jq-canonical-fail" @@ -888,9 +950,30 @@ source "$ingress_product" guard_paths="$ingress_tmp/generation-files" find "$ingress_repo/core/v1/generations/$ingress_generation" -type f -print | sed "s#^$ingress_repo/##" | LC_ALL=C sort > "$guard_paths" -expected_generation_files="core/v1/generations/$ingress_generation/core-ingress.sh -core/v1/generations/$ingress_generation/modules/schema.jq" -if [ "$(cat "$guard_paths")" = "$expected_generation_files" ] && + +ingress_private_generation_path_ok() { + case "$1" in + "core/v1/generations/$ingress_generation/core-ingress.sh"|\ + "core/v1/generations/$ingress_generation/modules/schema.jq"|\ + "core/v1/generations/$ingress_generation/modules/profile_graph.jq"|\ + "core/v1/generations/$ingress_generation/modules/stage_request.jq"|\ + "core/v1/generations/$ingress_generation/modules/result_facts.jq"|\ + "core/v1/generations/$ingress_generation/modules/result_truth.jq") return 0 ;; + *) return 1 ;; + esac +} + +ingress_guard_paths_ok() { + local paths_file="$1" + local candidate_path + while IFS= read -r candidate_path; do + ingress_private_generation_path_ok "$candidate_path" || return 1 + done < "$paths_file" +} + +if ingress_guard_paths_ok "$guard_paths" && + grep -Fqx "core/v1/generations/$ingress_generation/core-ingress.sh" "$guard_paths" && + grep -Fqx "core/v1/generations/$ingress_generation/modules/schema.jq" "$guard_paths" && [ ! -e "$ingress_repo/scripts/core-contract.sh" ] && [ ! -e "$ingress_repo/core/v1/generations/$ingress_generation/contracts.jq" ] && [ -z "$(find "$ingress_repo/core/v1/generations/$ingress_generation" -type l -print -quit)" ]; then @@ -900,6 +983,21 @@ else ingress_guard_total=$((ingress_guard_total + 1)) fail_case 'private generation guard' fi +planned_guard_paths="$ingress_tmp/planned-generation-files" +cp "$guard_paths" "$planned_guard_paths" +printf '%s\n' \ + "core/v1/generations/$ingress_generation/modules/profile_graph.jq" \ + "core/v1/generations/$ingress_generation/modules/stage_request.jq" \ + "core/v1/generations/$ingress_generation/modules/result_facts.jq" \ + "core/v1/generations/$ingress_generation/modules/result_truth.jq" >> \ + "$planned_guard_paths" +if ingress_guard_paths_ok "$planned_guard_paths"; then + ingress_direct_total=$((ingress_direct_total + 1)) + ingress_direct_passed=$((ingress_direct_passed + 1)) +else + ingress_direct_total=$((ingress_direct_total + 1)) + fail_case 'private generation guard rejected a planned module' +fi if [ "$(git -C "$ingress_repo" ls-tree HEAD \ "core/v1/generations/$ingress_generation/modules/schema.jq" | awk '{print $3}')" = \ fd3924d414a7d620c2bf5de919a45c2599d572ec ] && From 8684259de6f855d31384cadba481a957a8f61e02 Mon Sep 17 00:00:00 2001 From: ci Date: Sun, 30 Aug 2026 00:36:55 -0400 Subject: [PATCH 08/13] test: require Rosetta for arm64 jq proof --- scripts/test/portable-core-ingress.test.sh | 23 +++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/scripts/test/portable-core-ingress.test.sh b/scripts/test/portable-core-ingress.test.sh index 1d212c0..c0cd867 100755 --- a/scripts/test/portable-core-ingress.test.sh +++ b/scripts/test/portable-core-ingress.test.sh @@ -33,15 +33,21 @@ sha256_path() { } ingress_platform="$(uname -s):$(uname -m)" +ingress_platform_runtime='native' case "$ingress_platform" in Linux:x86_64) ingress_asset='jq-linux64' ingress_asset_sha256='af986793a515d500ab2d35f8d2aecd656e764504b789b66d7e1a0b727a124c44' ;; - Darwin:x86_64|Darwin:arm64) + Darwin:x86_64) ingress_asset='jq-osx-amd64' ingress_asset_sha256='5c0a0a3ea600f302ee458b30317425dd9632d1ad8882259fcaf4e9b868b2b1ef' ;; + Darwin:arm64) + ingress_asset='jq-osx-amd64' + ingress_asset_sha256='5c0a0a3ea600f302ee458b30317425dd9632d1ad8882259fcaf4e9b868b2b1ef' + ingress_platform_runtime='Rosetta 2' + ;; *) echo "FAIL: unsupported jq 1.6 proof platform: $ingress_platform" >&2 exit 1 @@ -65,8 +71,19 @@ if [ ! -f "$ingress_jq" ] || mv "$ingress_download" "$ingress_jq" ingress_download='' fi -[ "$(sha256_path "$ingress_jq")" = "$ingress_asset_sha256" ] && - [ "$("$ingress_jq" --version)" = jq-1.6 ] || { +if [ "$(sha256_path "$ingress_jq")" != "$ingress_asset_sha256" ]; then + echo 'FAIL: jq 1.6 release asset digest mismatch' >&2 + exit 1 +fi +if ! ingress_jq_version="$("$ingress_jq" --version 2>/dev/null)"; then + if [ "$ingress_platform_runtime" = 'Rosetta 2' ]; then + echo 'FAIL: unsupported jq 1.6 proof platform: Darwin:arm64 without Rosetta 2' >&2 + else + echo 'FAIL: pinned jq 1.6 executable could not run' >&2 + fi + exit 1 +fi +[ "$ingress_jq_version" = jq-1.6 ] || { echo 'FAIL: pinned jq 1.6 identity check failed' >&2 exit 1 } From b6cf6aa66a4eca38aef99eb9b4dd1226b9a504d9 Mon Sep 17 00:00:00 2001 From: ci Date: Sun, 30 Aug 2026 01:01:27 -0400 Subject: [PATCH 09/13] fix: sanitize ingress dependency failures --- .../core-ingress.sh | 25 ++++++++++-- scripts/test/portable-core-ingress.test.sh | 40 +++++++++++++++++++ 2 files changed, 61 insertions(+), 4 deletions(-) diff --git a/core/v1/generations/g-14b7ad8ce54c3b8c585ff92063d71551ffc7394cc2294d0297bc7d2b8da2c386/core-ingress.sh b/core/v1/generations/g-14b7ad8ce54c3b8c585ff92063d71551ffc7394cc2294d0297bc7d2b8da2c386/core-ingress.sh index 3a08883..8a65d5a 100644 --- a/core/v1/generations/g-14b7ad8ce54c3b8c585ff92063d71551ffc7394cc2294d0297bc7d2b8da2c386/core-ingress.sh +++ b/core/v1/generations/g-14b7ad8ce54c3b8c585ff92063d71551ffc7394cc2294d0297bc7d2b8da2c386/core-ingress.sh @@ -23,9 +23,11 @@ portable_core_ingress_real_directory() { portable_core_ingress_open() { local source_dir + local source_parent local repo_root local expected_dir local jq_path + local jq_version local schema_identity local sha_path local temp_path @@ -38,12 +40,19 @@ portable_core_ingress_open() { PORTABLE_CORE_INGRESS_GENERATION='g-14b7ad8ce54c3b8c585ff92063d71551ffc7394cc2294d0297bc7d2b8da2c386' source_dir="$( - CDPATH='' cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")" 2>/dev/null && pwd -P + { + source_parent="$(dirname -- "${BASH_SOURCE[0]}")" && + CDPATH='' cd -P -- "$source_parent" && pwd -P + } 2>/dev/null )" || { portable_core_ingress_error E_RUNTIME return 1 } - repo_root="$(CDPATH='' cd -P -- "$source_dir/../../../.." 2>/dev/null && pwd -P)" || { + repo_root="$( + { + CDPATH='' cd -P -- "$source_dir/../../../.." && pwd -P + } 2>/dev/null + )" || { portable_core_ingress_error E_RUNTIME return 1 } @@ -89,7 +98,11 @@ portable_core_ingress_open() { return 1 ;; esac - [ "$("$jq_path" --version 2>/dev/null)" = jq-1.6 ] || { + jq_version="$("$jq_path" --version 2>/dev/null)" || { + portable_core_ingress_error E_RUNTIME + return 1 + } + [ "$jq_version" = jq-1.6 ] || { portable_core_ingress_error E_RUNTIME return 1 } @@ -777,7 +790,11 @@ portable_core_ingress_validate() { return 1 } output_size="${output_size//[[:space:]]/}" - token="$("$PORTABLE_CORE_INGRESS_CAT" -- "$PORTABLE_CORE_INGRESS_OUTPUT" 2>/dev/null)" || { + [[ "$output_size" =~ ^[0-9]+$ ]] || { + portable_core_ingress_error E_RUNTIME + return 1 + } + IFS= read -r token 2>/dev/null < "$PORTABLE_CORE_INGRESS_OUTPUT" || { portable_core_ingress_error E_RUNTIME return 1 } diff --git a/scripts/test/portable-core-ingress.test.sh b/scripts/test/portable-core-ingress.test.sh index c0cd867..032ddf0 100755 --- a/scripts/test/portable-core-ingress.test.sh +++ b/scripts/test/portable-core-ingress.test.sh @@ -655,6 +655,28 @@ make_isolated_bin() { done } +dirname_fail_bin="$ingress_tmp/dirname-fail-bin" +make_isolated_bin "$dirname_fail_bin" +rm "$dirname_fail_bin/dirname" +ingress_product_dir="${ingress_product%/*}" +printf '%s\n' '#!/bin/sh' \ + "printf '%s\\n' \"$ingress_product_dir\"" \ + 'printf "%s\n" "dirname SECRET /private/source/path" >&2' \ + 'exit 1' > "$dirname_fail_bin/dirname" +chmod +x "$dirname_fail_bin/dirname" +runtime_failure dirname-failure-sanitized E_RUNTIME env PATH="$dirname_fail_bin" \ + /bin/bash -c 'source "$1"; portable_core_ingress_open' _ "$ingress_product" + +jq_version_fail_bin="$ingress_tmp/jq-version-fail-bin" +make_isolated_bin "$jq_version_fail_bin" +rm "$jq_version_fail_bin/jq" +printf '%s\n' '#!/bin/sh' \ + 'if [ "${1:-}" = "--version" ]; then printf "%s\n" jq-1.6; exit 1; fi' \ + "exec \"$ingress_jq\" \"\$@\"" > "$jq_version_fail_bin/jq" +chmod +x "$jq_version_fail_bin/jq" +runtime_failure jq-version-nonzero E_RUNTIME env PATH="$jq_version_fail_bin" \ + /bin/bash -c 'source "$1"; portable_core_ingress_open' _ "$ingress_product" + no_sha_bin="$ingress_tmp/no-sha-bin" make_isolated_bin "$no_sha_bin" runtime_failure missing-sha E_RUNTIME env PATH="$no_sha_bin" \ @@ -880,6 +902,18 @@ expect_failure at-limit-reaches-validator E_SHAPE portable_core_ingress_validate stop_ingress mark_test portable-core-ingress.test.legacy-061-at-exact-1-048-576-byte-boundary-is-still-just-a-shape-failure-not-e-limit +start_ingress document +portable_core_ingress_snapshot "$canonical_file" +portable_core_ingress_finish_driver +validator_wc_malformed="$ingress_tmp/validator-wc-malformed" +printf '%s\n' '#!/bin/sh' \ + 'printf "%s\n" "SECRET /private/caller/path"' \ + 'exit 0' > "$validator_wc_malformed" +chmod +x "$validator_wc_malformed" +PORTABLE_CORE_INGRESS_WC="$validator_wc_malformed" +runtime_failure validator-wc-malformed-output E_RUNTIME portable_core_ingress_validate +stop_ingress + printf '%s\n' 'import "schema" as schema;' \ 'if schema::semantic_identity == "core.contracts.v1" then empty else error("identity") end' \ > "$package_root" @@ -921,6 +955,12 @@ portable_core_ingress_snapshot "$canonical_file" portable_core_ingress_finish_driver runtime_failure validator-extra-newline E_RUNTIME portable_core_ingress_validate stop_ingress +printf '%s\n' '"E_SHAPE\u0000"' > "$package_root" +start_ingress document +portable_core_ingress_snapshot "$canonical_file" +portable_core_ingress_finish_driver +runtime_failure validator-null-output E_RUNTIME portable_core_ingress_validate +stop_ingress printf '%s\n' 'empty' > "$package_root" start_ingress document portable_core_ingress_snapshot "$canonical_file" From 03768db4f3ec1b41a1462d9e341bddea261fc303 Mon Sep 17 00:00:00 2001 From: ci Date: Sun, 30 Aug 2026 01:51:40 -0400 Subject: [PATCH 10/13] fix: bound deep ingress analysis --- .../core-ingress.sh | 802 +++++++++++------- scripts/test/portable-core-ingress.test.sh | 109 ++- 2 files changed, 623 insertions(+), 288 deletions(-) diff --git a/core/v1/generations/g-14b7ad8ce54c3b8c585ff92063d71551ffc7394cc2294d0297bc7d2b8da2c386/core-ingress.sh b/core/v1/generations/g-14b7ad8ce54c3b8c585ff92063d71551ffc7394cc2294d0297bc7d2b8da2c386/core-ingress.sh index 8a65d5a..22214ad 100644 --- a/core/v1/generations/g-14b7ad8ce54c3b8c585ff92063d71551ffc7394cc2294d0297bc7d2b8da2c386/core-ingress.sh +++ b/core/v1/generations/g-14b7ad8ce54c3b8c585ff92063d71551ffc7394cc2294d0297bc7d2b8da2c386/core-ingress.sh @@ -201,7 +201,6 @@ portable_core_ingress_open() { PORTABLE_CORE_INGRESS_RAW_SIZES=() PORTABLE_CORE_INGRESS_CANONICAL_PATHS=() PORTABLE_CORE_INGRESS_DEPTH_OVER=() - PORTABLE_CORE_INGRESS_STREAM_REQUIRED=() } portable_core_ingress_begin() { @@ -236,7 +235,6 @@ portable_core_ingress_begin() { PORTABLE_CORE_INGRESS_RAW_SIZES=() PORTABLE_CORE_INGRESS_CANONICAL_PATHS=() PORTABLE_CORE_INGRESS_DEPTH_OVER=() - PORTABLE_CORE_INGRESS_STREAM_REQUIRED=() } portable_core_ingress_digest() { @@ -270,18 +268,156 @@ portable_core_ingress_digest() { PORTABLE_CORE_INGRESS_SHA256="$digest" } -portable_core_ingress_scan_raw() { - local raw_path="$1" - local scan_status - local -a pipeline_status +portable_core_ingress_snapshot() { + local input_path + local snapshot_number + local raw_path + local byte_count + + [ "$#" -eq 1 ] || { + portable_core_ingress_error E_RUNTIME + return 1 + } + input_path="$1" + [ -n "${PORTABLE_CORE_INGRESS_MODE:-}" ] && + portable_core_ingress_real_directory "${PORTABLE_CORE_INGRESS_TEMP:-}" && + [ -r "$input_path" ] || { + portable_core_ingress_error E_RUNTIME + return 1 + } - portable_core_ingress_regular_file "$raw_path" && [ -r "$raw_path" ] || { + snapshot_number=$((PORTABLE_CORE_INGRESS_COUNT + 1)) + raw_path="$PORTABLE_CORE_INGRESS_TEMP/raw.$snapshot_number" + if ! "$PORTABLE_CORE_INGRESS_HEAD" -c 1048577 -- "$input_path" \ + 2>/dev/null > "$raw_path"; then + portable_core_ingress_error E_RUNTIME + return 1 + fi + byte_count="$("$PORTABLE_CORE_INGRESS_WC" -c 2>/dev/null < "$raw_path")" || { + portable_core_ingress_error E_RUNTIME + return 1 + } + byte_count="${byte_count//[[:space:]]/}" + [[ "$byte_count" =~ ^[0-9]+$ ]] || { portable_core_ingress_error E_RUNTIME return 1 } + PORTABLE_CORE_INGRESS_RAW_PATHS[PORTABLE_CORE_INGRESS_COUNT]="$raw_path" + PORTABLE_CORE_INGRESS_RAW_SIZES[PORTABLE_CORE_INGRESS_COUNT]="$byte_count" + PORTABLE_CORE_INGRESS_SNAPSHOT="$raw_path" + PORTABLE_CORE_INGRESS_COUNT="$snapshot_number" +} + +portable_core_ingress_analyze() { + local raw_path="$1" + local canonical_path="$2" + local scalar_path="$PORTABLE_CORE_INGRESS_TEMP/deep-scalars.ndjson" + local scalar_canonical_path="$PORTABLE_CORE_INGRESS_TEMP/deep-scalars-canonical.ndjson" + local key_path="$PORTABLE_CORE_INGRESS_TEMP/deep-keys.ndjson" + local key_pair_path="$PORTABLE_CORE_INGRESS_TEMP/deep-key-pairs.ndjson" + local key_order_path="$PORTABLE_CORE_INGRESS_TEMP/deep-key-order" + local meta_path="$PORTABLE_CORE_INGRESS_TEMP/deep-meta" + local scalar_marker_path="$PORTABLE_CORE_INGRESS_TEMP/deep-scalar-marker" + local scratch_path + local compare_status + local meta_size + local meta_lines + local meta_kind + local structural_canonical + local max_depth + local extra_meta + local marker_size + local marker_lines + local marker_kind + local canonical_ok=true + local -a pipeline_status + + for scratch_path in "$scalar_path" "$scalar_canonical_path" \ + "$key_path" "$key_pair_path" "$key_order_path" "$meta_path" \ + "$scalar_marker_path"; do + : 2>/dev/null > "$scratch_path" || { + portable_core_ingress_error E_RUNTIME + return 1 + } + done + if "$PORTABLE_CORE_INGRESS_OD" -An -v -t u1 "$raw_path" 2>/dev/null | - "$PORTABLE_CORE_INGRESS_AWK" ' - function literal_expected(kind, position) { + LC_ALL=C "$PORTABLE_CORE_INGRESS_AWK" \ + -v scalar_file="$scalar_path" \ + -v key_file="$key_path" \ + -v key_pair_file="$key_pair_path" ' + function fail_parse() { parse_bad = 1 } + function byte_char(byte) { return sprintf("%c", byte) } + function is_hex(byte) { + return (byte >= 48 && byte <= 57) || + (byte >= 65 && byte <= 70) || + (byte >= 97 && byte <= 102) + } + function value_expected() { + if (depth == 0) return root_state == 1 + return frame[depth] == 1 || frame[depth] == 2 || frame[depth] == 7 + } + function key_expected() { + return depth > 0 && (frame[depth] == 4 || frame[depth] == 5) + } + function complete_value() { + if (depth == 0) { + if (root_state != 1) { fail_parse(); return } + root_state = 2 + } else if (frame[depth] == 1 || frame[depth] == 2) { + frame[depth] = 3 + } else if (frame[depth] == 7) { + frame[depth] = 8 + } else { + fail_parse() + } + } + function flush_token_chunk() { + if (token_chunk == "") return + printf "%s", token_chunk >> scalar_file + if (token_role == "key") printf "%s", token_chunk >> key_file + token_chunk = "" + } + function begin_token(role, first_byte) { + token_active = 1 + token_role = role + token_chunk = "" + if (role == "key") { + current_key_index = key_count + key_count++ + } + append_token(first_byte) + } + function append_token(byte) { + token_chunk = token_chunk byte_char(byte) + if (length(token_chunk) >= 4096) flush_token_chunk() + } + function finish_scalar() { + flush_token_chunk() + printf "\n" >> scalar_file + if (token_role == "key") { + if (!key_expected()) { fail_parse(); return } + printf "\n" >> key_file + if (depth in previous_key_index) { + printf "[%d,%d]\n", previous_key_index[depth], current_key_index \ + >> key_pair_file + } + previous_key_index[depth] = current_key_index + frame[depth] = 6 + } else { + complete_value() + } + token_chunk = "" + token_role = "" + token_active = 0 + token_state = "" + } + function token_terminal() { + if (token_state == "literal") return literal_pos == literal_len + return token_state == "zero" || token_state == "integer" || + token_state == "fraction" || token_state == "exponent" + } + function literal_byte(kind, position) { if (kind == 1) { if (position == 2) return 114 if (position == 3) return 117 @@ -298,17 +434,18 @@ portable_core_ingress_scan_raw() { } return -1 } - function start_token(byte) { - token_active = 1 + function start_atom(byte) { + if (!value_expected()) { fail_parse(); return } + begin_token("value", byte) if (byte == 116) { - token_state = "literal"; literal_kind = 1; literal_position = 1; - literal_length = 4 + token_state = "literal"; literal_kind = 1; literal_pos = 1; + literal_len = 4 } else if (byte == 102) { - token_state = "literal"; literal_kind = 2; literal_position = 1; - literal_length = 5 + token_state = "literal"; literal_kind = 2; literal_pos = 1; + literal_len = 5 } else if (byte == 110) { - token_state = "literal"; literal_kind = 3; literal_position = 1; - literal_length = 4 + token_state = "literal"; literal_kind = 3; literal_pos = 1; + literal_len = 4 } else if (byte == 45) { token_state = "sign" } else if (byte == 48) { @@ -316,361 +453,461 @@ portable_core_ingress_scan_raw() { } else if (byte >= 49 && byte <= 57) { token_state = "integer" } else { - invalid_token = 1 + fail_parse() } } - function advance_token(byte) { - if (!token_active) { - start_token(byte) - } else if (token_state == "literal") { - literal_position++ - if (literal_position > literal_length || - byte != literal_expected(literal_kind, literal_position)) { - invalid_token = 1 + function atom_byte(byte) { + if (token_state == "literal") { + if (literal_pos < literal_len) { + if (byte != literal_byte(literal_kind, literal_pos + 1)) return -1 + literal_pos++ + append_token(byte) + return 1 } - } else if (token_state == "sign") { + return 0 + } + if (token_state == "sign") { if (byte == 48) token_state = "zero" else if (byte >= 49 && byte <= 57) token_state = "integer" - else invalid_token = 1 + else return -1 } else if (token_state == "zero") { if (byte == 46) token_state = "decimal-mark" else if (byte == 101 || byte == 69) token_state = "exponent-mark" - else invalid_token = 1 + else if (byte >= 48 && byte <= 57) return -1 + else return 0 } else if (token_state == "integer") { if (byte >= 48 && byte <= 57) token_state = "integer" else if (byte == 46) token_state = "decimal-mark" else if (byte == 101 || byte == 69) token_state = "exponent-mark" - else invalid_token = 1 + else return 0 } else if (token_state == "decimal-mark") { if (byte >= 48 && byte <= 57) token_state = "fraction" - else invalid_token = 1 + else return -1 } else if (token_state == "fraction") { if (byte >= 48 && byte <= 57) token_state = "fraction" else if (byte == 101 || byte == 69) token_state = "exponent-mark" - else invalid_token = 1 + else return 0 } else if (token_state == "exponent-mark") { if (byte == 43 || byte == 45) token_state = "exponent-sign" else if (byte >= 48 && byte <= 57) token_state = "exponent" - else invalid_token = 1 + else return -1 } else if (token_state == "exponent-sign") { if (byte >= 48 && byte <= 57) token_state = "exponent" - else invalid_token = 1 + else return -1 } else if (token_state == "exponent") { - if (byte < 48 || byte > 57) invalid_token = 1 + if (byte >= 48 && byte <= 57) token_state = "exponent" + else return 0 } else { - invalid_token = 1 + return -1 } + append_token(byte) + return 1 } - function finish_token() { - if (token_active && - !((token_state == "literal" && literal_position == literal_length) || - token_state == "zero" || token_state == "integer" || - token_state == "fraction" || token_state == "exponent")) { - invalid_token = 1 + function push_container(byte) { + if (!value_expected()) { fail_parse(); return } + depth++ + if (depth > max_depth) max_depth = depth + if (byte == 91) frame[depth] = 1 + else { + frame[depth] = 4 } - token_active = 0 - token_state = "" } - BEGIN { - in_string = 0; escaped = 0; need = 0; invalid = 0; - invalid_token = 0; token_active = 0; token_state = ""; - next_min = 128; next_max = 191; depth = 0; max_depth = 0 + function close_container(byte, state) { + if (depth == 0) { fail_parse(); return } + state = frame[depth] + if (byte == 93) { + if (!(state == 1 || state == 3)) { fail_parse(); return } + } else { + if (!(state == 4 || state == 8)) { fail_parse(); return } + } + if (byte == 125) { + delete previous_key_index[depth] + } + delete frame[depth] + depth-- + complete_value() } - { - for (i = 1; i <= NF; i++) { - byte = $i + 0 - ascii = (need == 0 && byte <= 127) - if (need > 0) { - if (byte < next_min || byte > next_max) { - invalid = 1; need = 0 - } else { - need--; next_min = 128; next_max = 191 - } - } else if (byte <= 127) { - # ASCII is valid and also drives JSON string/depth state below. - } else if (byte >= 194 && byte <= 223) { - need = 1; next_min = 128; next_max = 191 + function structural_ascii(byte) { + if (byte == 32 || byte == 9 || byte == 10 || byte == 13) { + if (depth == 0 && root_state == 2) { + trailing_count++ + if (trailing_count == 1) trailing_byte = byte + } else { + internal_space = 1 + } + return + } + if (depth == 0 && root_state == 2) { fail_parse(); return } + if (byte == 34) { + if (key_expected()) begin_token("key", byte) + else if (value_expected()) begin_token("value", byte) + else { fail_parse(); return } + in_string = 1 + escape_state = 0 + return + } + if (byte == 91 || byte == 123) { push_container(byte); return } + if (byte == 93 || byte == 125) { + if ((byte == 93 && frame[depth] != 1 && frame[depth] != 3) || + (byte == 125 && frame[depth] != 4 && frame[depth] != 8)) { + fail_parse(); return + } + close_container(byte) + return + } + if (byte == 44) { + if (depth == 0) { fail_parse(); return } + if (frame[depth] == 3) frame[depth] = 2 + else if (frame[depth] == 8) frame[depth] = 5 + else fail_parse() + return + } + if (byte == 58) { + if (depth > 0 && frame[depth] == 6) frame[depth] = 7 + else fail_parse() + return + } + if (byte < 32 || byte > 127) { fail_parse(); return } + start_atom(byte) + } + function string_ascii(byte) { + append_token(byte) + if (escape_state == 2) { + if (!is_hex(byte)) { fail_parse(); return } + unicode_left-- + if (unicode_left == 0) escape_state = 0 + return + } + if (escape_state == 1) { + if (byte == 117) { escape_state = 2; unicode_left = 4; return } + if (byte == 34 || byte == 92 || byte == 47 || byte == 98 || + byte == 102 || byte == 110 || byte == 114 || byte == 116) { + escape_state = 0 + return + } + fail_parse(); return + } + if (byte == 34) { + in_string = 0 + finish_scalar() + } else if (byte == 92) { + escape_state = 1 + } else if (byte < 32) { + fail_parse() + } + } + function consume_byte(byte, used) { + if (parse_bad || runtime_bad) return + byte_position++ + if (bom_state == 1) { + if (byte != 187) { fail_parse(); return } + bom_state = 2 + return + } + if (bom_state == 2) { + if (byte != 191) { fail_parse(); return } + bom_state = 0 + return + } + if (byte_position == 1 && byte == 239) { + bom_present = 1 + bom_state = 1 + return + } + if (utf_need > 0) { + if (byte < utf_min || byte > utf_max) { fail_parse(); return } + if (!in_string) { fail_parse(); return } + append_token(byte) + utf_need-- + utf_min = 128 + utf_max = 191 + return + } + if (byte > 127) { + if (!in_string) { fail_parse(); return } + append_token(byte) + if (byte >= 194 && byte <= 223) { + utf_need = 1; utf_min = 128; utf_max = 191 } else if (byte == 224) { - need = 2; next_min = 160; next_max = 191 + utf_need = 2; utf_min = 160; utf_max = 191 } else if ((byte >= 225 && byte <= 236) || (byte >= 238 && byte <= 239)) { - need = 2; next_min = 128; next_max = 191 + utf_need = 2; utf_min = 128; utf_max = 191 } else if (byte == 237) { - need = 2; next_min = 128; next_max = 159 + utf_need = 2; utf_min = 128; utf_max = 159 } else if (byte == 240) { - need = 3; next_min = 144; next_max = 191 + utf_need = 3; utf_min = 144; utf_max = 191 } else if (byte >= 241 && byte <= 243) { - need = 3; next_min = 128; next_max = 191 + utf_need = 3; utf_min = 128; utf_max = 191 } else if (byte == 244) { - need = 3; next_min = 128; next_max = 143 + utf_need = 3; utf_min = 128; utf_max = 143 } else { - invalid = 1 + fail_parse() } - - if (ascii) { - if (in_string) { - if (escaped) escaped = 0 - else if (byte == 92) escaped = 1 - else if (byte == 34) in_string = 0 - } else if (byte == 34) { - finish_token() - in_string = 1 - } else if (byte == 91 || byte == 123) { - finish_token() - depth++ - if (depth > max_depth) max_depth = depth - } else if ((byte == 93 || byte == 125) && depth > 0) { - finish_token() - depth-- - } else if (byte == 93 || byte == 125 || byte == 44 || - byte == 58 || byte == 9 || byte == 10 || - byte == 13 || byte == 32) { - finish_token() - } else if (byte < 32) { - invalid_token = 1 - } else { - advance_token(byte) - } - } else if (!in_string && token_active) { - invalid_token = 1 + return + } + if (in_string) { string_ascii(byte); return } + if (token_active) { + used = atom_byte(byte) + if (used == 1) return + if (used < 0 || !token_terminal()) { fail_parse(); return } + finish_scalar() + if (parse_bad) return + } + structural_ascii(byte) + } + BEGIN { + depth = 0 + max_depth = 0 + root_state = 1 + parse_bad = 0 + runtime_bad = 0 + token_active = 0 + in_string = 0 + utf_need = 0 + internal_space = 0 + trailing_count = 0 + byte_position = 0 + bom_state = 0 + bom_present = 0 + key_count = 0 + } + { + for (i = 1; i <= NF; i++) { + if ($i !~ /^[0-9]+$/ || ($i + 0) < 0 || ($i + 0) > 255) { + runtime_bad = 1 + next } + consume_byte($i + 0) } } END { - finish_token() - if (invalid || need != 0 || invalid_token) exit 31 - if (max_depth > 128) exit 33 - if (max_depth > 32) exit 32 + if (!parse_bad && token_active) { + if (!token_terminal()) fail_parse() + else finish_scalar() + } + if (in_string || utf_need != 0 || bom_state != 0 || + depth != 0 || root_state != 2) fail_parse() + if (runtime_bad) { + print "R" + } else if (parse_bad) { + print "P" + } else { + structurally_canonical = !bom_present && !internal_space && + trailing_count == 1 && trailing_byte == 10 + print "S", structurally_canonical, max_depth + } } - ' >/dev/null 2>/dev/null; then + ' > "$meta_path" 2>/dev/null; then pipeline_status=("${PIPESTATUS[@]}") else pipeline_status=("${PIPESTATUS[@]}") fi - [ "${#pipeline_status[@]}" -eq 2 ] && [ "${pipeline_status[0]}" -eq 0 ] || { + [ "${#pipeline_status[@]}" -eq 2 ] && + [ "${pipeline_status[0]}" -eq 0 ] && + [ "${pipeline_status[1]}" -eq 0 ] || { portable_core_ingress_error E_RUNTIME return 1 } - scan_status="${pipeline_status[1]}" - case "$scan_status" in - 0) - PORTABLE_CORE_INGRESS_SCAN_DEPTH_OVER=false - PORTABLE_CORE_INGRESS_SCAN_STREAM_REQUIRED=false - ;; - 31) + + meta_size="$("$PORTABLE_CORE_INGRESS_WC" -c 2>/dev/null < "$meta_path")" || { + portable_core_ingress_error E_RUNTIME + return 1 + } + meta_lines="$("$PORTABLE_CORE_INGRESS_WC" -l 2>/dev/null < "$meta_path")" || { + portable_core_ingress_error E_RUNTIME + return 1 + } + meta_size="${meta_size//[[:space:]]/}" + meta_lines="${meta_lines//[[:space:]]/}" + [[ "$meta_size" =~ ^[0-9]+$ ]] && [[ "$meta_lines" =~ ^[0-9]+$ ]] && + [ "$meta_size" -ge 2 ] && [ "$meta_lines" -eq 1 ] || { + portable_core_ingress_error E_RUNTIME + return 1 + } + IFS=' ' read -r meta_kind structural_canonical max_depth extra_meta \ + 2>/dev/null < "$meta_path" || { + portable_core_ingress_error E_RUNTIME + return 1 + } + case "$meta_kind" in + P) + [ "$meta_size" -eq 2 ] && [ -z "${structural_canonical:-}" ] && + [ -z "${max_depth:-}" ] && [ -z "${extra_meta:-}" ] || { + portable_core_ingress_error E_RUNTIME + return 1 + } portable_core_ingress_error E_PARSE return 1 ;; - 32) - PORTABLE_CORE_INGRESS_SCAN_DEPTH_OVER=true - PORTABLE_CORE_INGRESS_SCAN_STREAM_REQUIRED=false - ;; - 33) - PORTABLE_CORE_INGRESS_SCAN_DEPTH_OVER=true - PORTABLE_CORE_INGRESS_SCAN_STREAM_REQUIRED=true + R) + portable_core_ingress_error E_RUNTIME + return 1 ;; + S) ;; *) portable_core_ingress_error E_RUNTIME return 1 ;; esac -} - -portable_core_ingress_snapshot() { - local input_path - local snapshot_number - local raw_path - local byte_count - - [ "$#" -eq 1 ] || { - portable_core_ingress_error E_RUNTIME - return 1 - } - input_path="$1" - [ -n "${PORTABLE_CORE_INGRESS_MODE:-}" ] && - portable_core_ingress_real_directory "${PORTABLE_CORE_INGRESS_TEMP:-}" && - [ -r "$input_path" ] || { - portable_core_ingress_error E_RUNTIME - return 1 - } - - snapshot_number=$((PORTABLE_CORE_INGRESS_COUNT + 1)) - raw_path="$PORTABLE_CORE_INGRESS_TEMP/raw.$snapshot_number" - if ! "$PORTABLE_CORE_INGRESS_HEAD" -c 1048577 -- "$input_path" \ - 2>/dev/null > "$raw_path"; then - portable_core_ingress_error E_RUNTIME - return 1 - fi - byte_count="$("$PORTABLE_CORE_INGRESS_WC" -c 2>/dev/null < "$raw_path")" || { + case "$structural_canonical" in 0|1) ;; *) portable_core_ingress_error E_RUNTIME return 1 - } - byte_count="${byte_count//[[:space:]]/}" - [[ "$byte_count" =~ ^[0-9]+$ ]] || { + esac + [[ "$max_depth" =~ ^[0-9]+$ ]] && [ -z "${extra_meta:-}" ] || { portable_core_ingress_error E_RUNTIME return 1 } - PORTABLE_CORE_INGRESS_RAW_PATHS[PORTABLE_CORE_INGRESS_COUNT]="$raw_path" - PORTABLE_CORE_INGRESS_RAW_SIZES[PORTABLE_CORE_INGRESS_COUNT]="$byte_count" - PORTABLE_CORE_INGRESS_SNAPSHOT="$raw_path" - PORTABLE_CORE_INGRESS_COUNT="$snapshot_number" -} - -portable_core_ingress_tree_canonical() { - local raw_path="$1" - local canonical_path="$2" - local parse_status - - if "$PORTABLE_CORE_INGRESS_JQ" -s -e 'length == 1' "$raw_path" \ - >/dev/null 2>/dev/null; then - : + if [ "$max_depth" -gt 32 ]; then + PORTABLE_CORE_INGRESS_ANALYZE_DEPTH_OVER=true else - parse_status=$? - case "$parse_status" in - 1|4) portable_core_ingress_error E_PARSE ;; - *) portable_core_ingress_error E_RUNTIME ;; - esac - return 1 + PORTABLE_CORE_INGRESS_ANALYZE_DEPTH_OVER=false fi - if "$PORTABLE_CORE_INGRESS_JQ" -s -S -c \ - 'if length == 1 then .[0] else error("root-count") end' "$raw_path" \ - 2>/dev/null > "$canonical_path"; then - return 0 - fi - portable_core_ingress_error E_RUNTIME - return 1 -} - -portable_core_ingress_stream_canonical() { - local raw_path="$1" - local canonical_path="$2" - local marker_status - local -a pipeline_status if "$PORTABLE_CORE_INGRESS_JQ" -n --stream --stream-errors -j ' - def component_kind($component): - if ($component | type) == "number" then "array" - elif ($component | type) == "string" then "object" - else null - end; - def first_children_valid($path): - all($path[]; - ((type == "number") and (. == 0)) or (type == "string")); - def open_first_children($path): - [$path[] | - if type == "number" then "[" - else "{" + (tojson) + ":" - end] | join(""); - def closer($kind): if $kind == "array" then "]" else "}" end; - foreach (inputs, {end: true}) as $event - ({emit: "", last_path: [], open_depth: 0, started: false, - done: false, parse: true, runtime: true, ordered: true}; + foreach (inputs, {end:true}) as $event + ({emit:"", parse:true, runtime:true}; if ($event | type) == "object" then if (.runtime | not) then halt_error(42) - elif (.parse | not) or (.started | not) or (.done | not) or - (.open_depth != 0) then .emit = "\u0000" - elif .ordered then .emit = "\n" - else .emit = "!\n" + elif (.parse | not) then .emit = "\u0000" + else .emit = "" end elif (.parse | not) then .emit = "" elif (($event | type) != "array") or (($event | length) == 0) then .runtime = false | .emit = "" elif ($event[0] | type) == "string" then .parse = false | .emit = "" - elif ($event[0] | type) != "array" then - .runtime = false | .emit = "" - elif ($event | length) == 2 then - ($event[0]) as $path | - ($event[1]) as $value | - if .done then .parse = false | .emit = "" - elif .open_depth == 0 then - if .started then .parse = false | .emit = "" - elif ($path | length) == 0 then - .emit = ($value | tojson) | - .started = true | - .done = true - elif (first_children_valid($path) | not) then - .runtime = false | .emit = "" - else - .emit = open_first_children($path) + ($value | tojson) | - .last_path = $path | - .open_depth = ($path | length) | - .started = true - end - else - .open_depth as $depth | - if (($path | length) < $depth) or - ((.last_path | length) < $depth) or - ($path[0:($depth - 1)] != .last_path[0:($depth - 1)]) then - .runtime = false | .emit = "" - else - ($path[$depth - 1]) as $current | - (.last_path[$depth - 1]) as $previous | - ($path[$depth:] | first_children_valid(.)) as $suffix_valid | - if (component_kind($current) == null) or - (component_kind($current) != component_kind($previous)) or - (($current | type) == "number" and - ($current != ($previous + 1))) or - ($suffix_valid | not) then - .runtime = false | .emit = "" - else - if (($current | type) == "string") and - (($previous < $current) | not) then - .ordered = false - else . end | - .emit = "," + - (if ($current | type) == "string" then - ($current | tojson) + ":" - else "" - end) + - open_first_children($path[$depth:]) + ($value | tojson) | - .last_path = $path | - .open_depth = ($path | length) - end - end - end - elif ($event | length) == 1 then - ($event[0]) as $path | - if (.open_depth == 0) or - (($path | length) != .open_depth) or - (component_kind($path[-1]) == null) then - .runtime = false | .emit = "" - else - .emit = closer(component_kind($path[-1])) | - .open_depth -= 1 | - if .open_depth == 0 then .done = true else . end - end + elif (($event | length) == 2) and ($event[0] == []) then + .emit = ($event[1] | tojson) + "\n" else .runtime = false | .emit = "" end; .emit) - ' "$raw_path" 2>/dev/null > "$canonical_path"; then + ' "$scalar_path" 2>/dev/null > "$scalar_canonical_path"; then : else portable_core_ingress_error E_RUNTIME return 1 fi - if "$PORTABLE_CORE_INGRESS_OD" -An -v -t u1 "$canonical_path" 2>/dev/null | + if "$PORTABLE_CORE_INGRESS_OD" -An -v -t u1 "$scalar_canonical_path" 2>/dev/null | "$PORTABLE_CORE_INGRESS_AWK" ' { for (i = 1; i <= NF; i++) if (($i + 0) == 0) found = 1 } - END { if (found) exit 41 } - ' >/dev/null 2>/dev/null; then + END { print found ? "P" : "C" } + ' > "$scalar_marker_path" 2>/dev/null; then pipeline_status=("${PIPESTATUS[@]}") else pipeline_status=("${PIPESTATUS[@]}") fi - [ "${#pipeline_status[@]}" -eq 2 ] && [ "${pipeline_status[0]}" -eq 0 ] || { + [ "${#pipeline_status[@]}" -eq 2 ] && + [ "${pipeline_status[0]}" -eq 0 ] && + [ "${pipeline_status[1]}" -eq 0 ] || { + portable_core_ingress_error E_RUNTIME + return 1 + } + marker_size="$("$PORTABLE_CORE_INGRESS_WC" -c \ + 2>/dev/null < "$scalar_marker_path")" || { + portable_core_ingress_error E_RUNTIME + return 1 + } + marker_lines="$("$PORTABLE_CORE_INGRESS_WC" -l \ + 2>/dev/null < "$scalar_marker_path")" || { + portable_core_ingress_error E_RUNTIME + return 1 + } + marker_size="${marker_size//[[:space:]]/}" + marker_lines="${marker_lines//[[:space:]]/}" + [[ "$marker_size" =~ ^[0-9]+$ ]] && [[ "$marker_lines" =~ ^[0-9]+$ ]] && + [ "$marker_size" -eq 2 ] && [ "$marker_lines" -eq 1 ] || { + portable_core_ingress_error E_RUNTIME + return 1 + } + IFS= read -r marker_kind 2>/dev/null < "$scalar_marker_path" || { portable_core_ingress_error E_RUNTIME return 1 } - marker_status="${pipeline_status[1]}" - case "$marker_status" in - 0) return 0 ;; - 41) portable_core_ingress_error E_PARSE ;; - *) portable_core_ingress_error E_RUNTIME ;; + case "$marker_kind" in + C) ;; + P) + portable_core_ingress_error E_PARSE + return 1 + ;; + *) + portable_core_ingress_error E_RUNTIME + return 1 + ;; esac - return 1 + if "$PORTABLE_CORE_INGRESS_CMP" -s -- "$scalar_path" \ + "$scalar_canonical_path" 2>/dev/null; then + : + else + compare_status=$? + if [ "$compare_status" -eq 1 ]; then + canonical_ok=false + else + portable_core_ingress_error E_RUNTIME + return 1 + fi + fi + + if [ -s "$key_pair_path" ]; then + if "$PORTABLE_CORE_INGRESS_JQ" -n -j \ + --slurpfile keys "$key_path" ' + reduce inputs as $pair + (false; + if (($pair | type) != "array") or (($pair | length) != 2) or + (all($pair[]; + (type == "number") and (. >= 0) and (floor == .)) | not) or + ($pair[0] >= ($keys | length)) or + ($pair[1] >= ($keys | length)) then + halt_error(43) + else . or (($keys[$pair[0]] < $keys[$pair[1]]) | not) + end) + | if . then "x" else empty end + ' "$key_pair_path" 2>/dev/null > "$key_order_path"; then + : + else + portable_core_ingress_error E_RUNTIME + return 1 + fi + fi + if [ -s "$key_order_path" ]; then + if "$PORTABLE_CORE_INGRESS_OD" -An -v -t u1 "$key_order_path" 2>/dev/null | + "$PORTABLE_CORE_INGRESS_AWK" ' + { + for (i = 1; i <= NF; i++) { + count++ + if (($i + 0) != 120) invalid = 1 + } + } + END { if (invalid || count != 1) exit 42 } + ' >/dev/null 2>/dev/null; then + pipeline_status=("${PIPESTATUS[@]}") + else + pipeline_status=("${PIPESTATUS[@]}") + fi + [ "${#pipeline_status[@]}" -eq 2 ] && + [ "${pipeline_status[0]}" -eq 0 ] && + [ "${pipeline_status[1]}" -eq 0 ] || { + portable_core_ingress_error E_RUNTIME + return 1 + } + canonical_ok=false + fi + if [ "$structural_canonical" != 1 ]; then + canonical_ok=false + fi + + if [ "$canonical_ok" = true ]; then + if ! "$PORTABLE_CORE_INGRESS_CAT" -- "$raw_path" \ + 2>/dev/null > "$canonical_path"; then + portable_core_ingress_error E_RUNTIME + return 1 + fi + elif ! printf '!\n' 2>/dev/null > "$canonical_path"; then + portable_core_ingress_error E_RUNTIME + return 1 + fi } portable_core_ingress_finish_driver() { @@ -690,20 +927,11 @@ portable_core_ingress_finish_driver() { return 1 fi done - for ((input_index = 0; input_index < PORTABLE_CORE_INGRESS_COUNT; input_index++)); do - raw_path="${PORTABLE_CORE_INGRESS_RAW_PATHS[$input_index]}" - portable_core_ingress_scan_raw "$raw_path" || return 1 - PORTABLE_CORE_INGRESS_DEPTH_OVER[input_index]="$PORTABLE_CORE_INGRESS_SCAN_DEPTH_OVER" - PORTABLE_CORE_INGRESS_STREAM_REQUIRED[input_index]="$PORTABLE_CORE_INGRESS_SCAN_STREAM_REQUIRED" - done for ((input_index = 0; input_index < PORTABLE_CORE_INGRESS_COUNT; input_index++)); do raw_path="${PORTABLE_CORE_INGRESS_RAW_PATHS[$input_index]}" canonical_path="$PORTABLE_CORE_INGRESS_TEMP/canonical.$((input_index + 1))" - if [ "${PORTABLE_CORE_INGRESS_STREAM_REQUIRED[$input_index]}" = true ]; then - portable_core_ingress_stream_canonical "$raw_path" "$canonical_path" || return 1 - else - portable_core_ingress_tree_canonical "$raw_path" "$canonical_path" || return 1 - fi + portable_core_ingress_analyze "$raw_path" "$canonical_path" || return 1 + PORTABLE_CORE_INGRESS_DEPTH_OVER[input_index]="$PORTABLE_CORE_INGRESS_ANALYZE_DEPTH_OVER" PORTABLE_CORE_INGRESS_CANONICAL_PATHS[input_index]="$canonical_path" done for ((input_index = 0; input_index < PORTABLE_CORE_INGRESS_COUNT; input_index++)); do diff --git a/scripts/test/portable-core-ingress.test.sh b/scripts/test/portable-core-ingress.test.sh index 032ddf0..11f6309 100755 --- a/scripts/test/portable-core-ingress.test.sh +++ b/scripts/test/portable-core-ingress.test.sh @@ -225,6 +225,8 @@ escape_file="$ingress_tmp/escape.json" no_lf_file="$ingress_tmp/no-lf.json" extra_lf_file="$ingress_tmp/extra-lf.json" unsorted_file="$ingress_tmp/unsorted.json" +unicode_sorted_file="$ingress_tmp/unicode-sorted.json" +unicode_unsorted_file="$ingress_tmp/unicode-unsorted.json" nan_file="$ingress_tmp/nan.json" infinity_file="$ingress_tmp/infinity.json" negative_infinity_file="$ingress_tmp/negative-infinity.json" @@ -249,6 +251,8 @@ printf '{"x":"\\u0061"}\n' > "$escape_file" printf '{}' > "$no_lf_file" printf '{}\n\n' > "$extra_lf_file" printf '{"b":1,"a":2}\n' > "$unsorted_file" +printf '{"z":1,"é":2,"😀":3}\n' > "$unicode_sorted_file" +printf '{"é":2,"z":1,"😀":3}\n' > "$unicode_unsorted_file" printf 'NaN\n' > "$nan_file" printf 'Infinity\n' > "$infinity_file" printf '%s\n' -Infinity > "$negative_infinity_file" @@ -301,6 +305,13 @@ mark_test portable-core-ingress.test.legacy-055-missing-final-lf-non-canonical document_finish_failure extra-final-lf E_CANONICAL "$extra_lf_file" mark_test portable-core-ingress.test.legacy-057-extra-final-lf-non-canonical document_finish_failure unsorted-keys E_CANONICAL "$unsorted_file" +start_ingress document +expect_success unicode-key-order-snapshot portable_core_ingress_snapshot \ + "$unicode_sorted_file" +expect_success unicode-key-order-driver portable_core_ingress_finish_driver +stop_ingress +document_finish_failure unicode-key-order-unsorted E_CANONICAL \ + "$unicode_unsorted_file" mark_test portable-core-ingress.test.legacy-059-unsorted-keys-non-canonical document_finish_failure non-json-nan E_PARSE "$nan_file" document_finish_failure non-json-infinity E_PARSE "$infinity_file" @@ -340,7 +351,15 @@ python3 - "$ingress_tmp/depth-32.json" "$ingress_tmp/depth-33.json" \ "$ingress_tmp/object-depth-128.json" \ "$ingress_tmp/object-depth-129.json" \ "$ingress_tmp/mixed-depth-128.json" \ - "$ingress_tmp/mixed-depth-129.json" <<'PY' + "$ingress_tmp/mixed-depth-129.json" \ + "$ingress_tmp/deep-wide.json" \ + "$ingress_tmp/deep-wide-malformed.json" \ + "$ingress_tmp/deep-wide-noncanonical.json" \ + "$ingress_tmp/deep-object-sorted.json" \ + "$ingress_tmp/deep-object-duplicate.json" \ + "$ingress_tmp/deep-object-unsorted.json" \ + "$ingress_tmp/deep-high-surrogate.json" \ + "$ingress_tmp/deep-low-surrogate.json" <<'PY' import json import sys @@ -374,7 +393,23 @@ def mixed(depth): open(sys.argv[15], "wb").write(mixed(128).encode()) open(sys.argv[16], "wb").write(mixed(129).encode()) +deep_prefix = b"[" * 250000 +deep_values = b",".join([b"0"] * 250000) +deep_suffix = b"]" * 250000 +open(sys.argv[17], "wb").write(deep_prefix + deep_values + deep_suffix + b"\n") +open(sys.argv[18], "wb").write(deep_prefix + deep_values + deep_suffix[:-1] + b"\n") +open(sys.argv[19], "wb").write(deep_prefix + b" " + deep_values + deep_suffix + b"\n") +object_prefix = b'{"a":' * 129 +object_suffix = b'}' * 129 +open(sys.argv[20], "wb").write(object_prefix + b'{"a":0,"b":1}' + object_suffix + b"\n") +open(sys.argv[21], "wb").write(object_prefix + b'{"a":0,"a":1}' + object_suffix + b"\n") +open(sys.argv[22], "wb").write(object_prefix + b'{"b":0,"a":1}' + object_suffix + b"\n") +open(sys.argv[23], "wb").write(b"[" * 129 + b'"\\uD800"' + b"]" * 129 + b"\n") +open(sys.argv[24], "wb").write(b"[" * 129 + b'"\\uDC00"' + b"]" * 129 + b"\n") PY +[ "$(wc -c < "$ingress_tmp/deep-wide.json" | tr -d ' ')" -eq 1000000 ] +[ "$(wc -c < "$ingress_tmp/deep-wide-malformed.json" | tr -d ' ')" -eq 999999 ] +[ "$(wc -c < "$ingress_tmp/deep-wide-noncanonical.json" | tr -d ' ')" -eq 1000001 ] start_ingress document expect_success depth-32-snapshot portable_core_ingress_snapshot "$ingress_tmp/depth-32.json" @@ -398,6 +433,21 @@ expect_failure depth-100000-limit E_LIMIT portable_core_ingress_finish_driver stop_ingress document_finish_failure depth-100000-malformed E_PARSE \ "$ingress_tmp/depth-100000-malformed.json" +document_finish_failure deep-wide-limit E_LIMIT "$ingress_tmp/deep-wide.json" +document_finish_failure deep-wide-malformed E_PARSE \ + "$ingress_tmp/deep-wide-malformed.json" +document_finish_failure deep-wide-noncanonical E_CANONICAL \ + "$ingress_tmp/deep-wide-noncanonical.json" +document_finish_failure deep-object-sorted E_LIMIT \ + "$ingress_tmp/deep-object-sorted.json" +document_finish_failure deep-object-duplicate E_CANONICAL \ + "$ingress_tmp/deep-object-duplicate.json" +document_finish_failure deep-object-unsorted E_CANONICAL \ + "$ingress_tmp/deep-object-unsorted.json" +document_finish_failure deep-high-surrogate E_PARSE \ + "$ingress_tmp/deep-high-surrogate.json" +document_finish_failure deep-low-surrogate E_CANONICAL \ + "$ingress_tmp/deep-low-surrogate.json" document_finish_failure object-depth-128-limit E_LIMIT \ "$ingress_tmp/object-depth-128.json" document_finish_failure object-depth-129-limit E_LIMIT \ @@ -632,6 +682,29 @@ portable_core_ingress_snapshot "$canonical_file" runtime_failure byte-scanner-logic-failure E_RUNTIME portable_core_ingress_finish_driver stop_ingress +start_ingress document +awk_exit_forty="$ingress_tmp/awk-exit-forty" +printf '%s\n' '#!/bin/sh' 'exit 40' > "$awk_exit_forty" +chmod +x "$awk_exit_forty" +PORTABLE_CORE_INGRESS_AWK="$awk_exit_forty" +portable_core_ingress_snapshot "$canonical_file" +runtime_failure analyzer-awk-exit-forty E_RUNTIME portable_core_ingress_finish_driver +stop_ingress + +start_ingress document +real_ingress_awk="$PORTABLE_CORE_INGRESS_AWK" +marker_awk_exit_forty_one="$ingress_tmp/marker-awk-exit-forty-one" +printf '%s\n' '#!/bin/sh' \ + 'case " $* " in *" -v scalar_file="*) exec "$REAL_AWK" "$@" ;; esac' \ + 'exit 41' > "$marker_awk_exit_forty_one" +chmod +x "$marker_awk_exit_forty_one" +PORTABLE_CORE_INGRESS_AWK="$marker_awk_exit_forty_one" +export REAL_AWK="$real_ingress_awk" +portable_core_ingress_snapshot "$canonical_file" +runtime_failure marker-awk-exit-forty-one E_RUNTIME portable_core_ingress_finish_driver +unset REAL_AWK +stop_ingress + fake_jq_bin="$ingress_tmp/fake-jq-bin" mkdir -p "$fake_jq_bin" printf '%s\n' '#!/bin/sh' \ @@ -802,6 +875,26 @@ for stream_runtime_status in 5 41; do stop_ingress done +for key_order_mode in nonzero malformed; do + start_ingress document + real_ingress_jq="$PORTABLE_CORE_INGRESS_JQ" + key_order_fail="$ingress_tmp/key-order-$key_order_mode" + if [ "$key_order_mode" = nonzero ]; then + key_order_action='exit 7' + else + key_order_action='printf "%s" malformed; exit 0' + fi + printf '%s\n' '#!/bin/sh' \ + "case \" \$* \" in *\" --slurpfile keys \"*) $key_order_action ;; esac" \ + "exec \"$real_ingress_jq\" \"\$@\"" > "$key_order_fail" + chmod +x "$key_order_fail" + PORTABLE_CORE_INGRESS_JQ="$key_order_fail" + portable_core_ingress_snapshot "$ingress_tmp/deep-object-sorted.json" + runtime_failure "key-order-$key_order_mode" E_RUNTIME \ + portable_core_ingress_finish_driver + stop_ingress +done + start_ingress document jq_canonical_fail="$ingress_tmp/jq-canonical-fail" printf '%s\n' '#!/bin/sh' \ @@ -823,6 +916,18 @@ portable_core_ingress_snapshot "$canonical_file" runtime_failure cmp-operational-failure E_RUNTIME portable_core_ingress_finish_driver stop_ingress +start_ingress document +portable_core_ingress_snapshot "$canonical_file" +analysis_wc_malformed="$ingress_tmp/analysis-wc-malformed" +printf '%s\n' '#!/bin/sh' \ + 'printf "%s\n" "SECRET /private/meta/path"' \ + 'exit 0' > "$analysis_wc_malformed" +chmod +x "$analysis_wc_malformed" +PORTABLE_CORE_INGRESS_WC="$analysis_wc_malformed" +runtime_failure analysis-wc-malformed-output E_RUNTIME \ + portable_core_ingress_finish_driver +stop_ingress + write_failure_case() { local case_id="$1" local target_kind="$2" @@ -830,6 +935,7 @@ write_failure_case() { case "$target_kind" in raw) mkdir "$PORTABLE_CORE_INGRESS_TEMP/raw.1" ;; canonical) mkdir "$PORTABLE_CORE_INGRESS_TEMP/canonical.1" ;; + analysis) mkdir "$PORTABLE_CORE_INGRESS_TEMP/deep-scalars.ndjson" ;; contents) rm "$PORTABLE_CORE_INGRESS_CONTENTS" mkdir "$PORTABLE_CORE_INGRESS_CONTENTS" @@ -858,6 +964,7 @@ runtime_failure hashes-truncate-failure E_RUNTIME portable_core_ingress_begin do stop_ingress write_failure_case raw-write-failure raw write_failure_case canonical-write-failure canonical +write_failure_case analysis-sidecar-write-failure analysis write_failure_case contents-append-failure contents write_failure_case hashes-append-failure hashes From a2084d89213c0bed11eca79d2832b7282ee384b6 Mon Sep 17 00:00:00 2001 From: ci Date: Sun, 30 Aug 2026 02:12:18 -0400 Subject: [PATCH 11/13] test: allow registry prefix growth --- scripts/test/portable-core-ingress.test.sh | 52 ++++++++++++++++++++-- 1 file changed, 49 insertions(+), 3 deletions(-) diff --git a/scripts/test/portable-core-ingress.test.sh b/scripts/test/portable-core-ingress.test.sh index 11f6309..dc17426 100755 --- a/scripts/test/portable-core-ingress.test.sh +++ b/scripts/test/portable-core-ingress.test.sh @@ -9,6 +9,7 @@ ingress_schema="$ingress_repo/core/v1/generations/$ingress_generation/modules/sc ingress_fixture="$ingress_repo/scripts/test/portable-core-ingress-fixtures.json" ingress_ledger="$ingress_repo/scripts/test/portable-core-ingress-ledger.tsv" ingress_manifest="$ingress_repo/ci/required-files.txt" +ingress_registry="$ingress_repo/core/v1/generation-registry.json" ingress_host_path="$PATH" ingress_tmp="$(mktemp -d "${TMPDIR:-/tmp}/ystack-portable-ingress-test.XXXXXX")" ingress_download='' @@ -1162,16 +1163,61 @@ else ingress_direct_total=$((ingress_direct_total + 1)) fail_case 'private generation guard rejected a planned module' fi + +ingress_registry_ok() { + local candidate_registry="$1" + local canonical_registry="$ingress_tmp/registry-check.canonical" + "$ingress_jq" -s -S -c \ + 'if length == 1 then .[0] else error("root-count") end' \ + "$candidate_registry" > "$canonical_registry" 2>/dev/null && + cmp -s "$candidate_registry" "$canonical_registry" && + "$ingress_jq" -e \ + --arg generation "$ingress_generation" \ + --arg spec c6511d96c1a5e6aed27ba2075b5add65c121f782 \ + --arg authorization 38a26f5f046897c0455fef24874c5dbb40c20926 ' + length >= 1 and + .[0] == {generation_id:$generation, + parent_spec_blob:$spec, + parent_plan_merge_commit:$authorization} and + all(.[]; + (keys | sort) == + ["generation_id","parent_plan_merge_commit","parent_spec_blob"] and + (.generation_id | test("\\Ag-[0-9a-f]{64}\\z")) and + (.parent_spec_blob | test("\\A([0-9a-f]{40}|[0-9a-f]{64})\\z")) and + (.parent_plan_merge_commit | + test("\\A([0-9a-f]{40}|[0-9a-f]{64})\\z"))) and + (map(.generation_id) | length) == + (map(.generation_id) | unique | length) + ' "$candidate_registry" >/dev/null +} + if [ "$(git -C "$ingress_repo" ls-tree HEAD \ "core/v1/generations/$ingress_generation/modules/schema.jq" | awk '{print $3}')" = \ fd3924d414a7d620c2bf5de919a45c2599d572ec ] && - [ "$(git -C "$ingress_repo" ls-tree HEAD core/v1/generation-registry.json | awk '{print $3}')" = \ - 5e113105777694a280166e71d31efd19752e9562 ]; then + ingress_registry_ok "$ingress_registry"; then ingress_guard_total=$((ingress_guard_total + 1)) ingress_guard_passed=$((ingress_guard_passed + 1)) else ingress_guard_total=$((ingress_guard_total + 1)) - fail_case 'schema G3 export or registry OID moved' + fail_case 'schema G3 export or registry prefix moved' +fi +future_registry="$ingress_tmp/future-generation-registry.json" +reordered_registry="$ingress_tmp/reordered-generation-registry.json" +multi_root_registry="$ingress_tmp/multi-root-generation-registry.json" +"$ingress_jq" -S -c '. + [{ + generation_id:("g-" + ("f" * 64)), + parent_plan_merge_commit:("e" * 40), + parent_spec_blob:("d" * 40)}]' "$ingress_registry" > "$future_registry" +"$ingress_jq" -S -c 'reverse' "$future_registry" > "$reordered_registry" +"$ingress_jq" -c '.,.' "$ingress_registry" > "$multi_root_registry" +if ingress_registry_ok "$future_registry" && + ! ingress_registry_ok "$reordered_registry" && + ! ingress_registry_ok "$multi_root_registry"; then + ingress_direct_total=$((ingress_direct_total + 1)) + ingress_direct_passed=$((ingress_direct_passed + 1)) +else + ingress_direct_total=$((ingress_direct_total + 1)) + fail_case 'registry ordered-prefix growth proof' fi mark_rule portable-core-ingress.private-activation-guard From 7cb64c150751c465e7a939c94e60846d547336d1 Mon Sep 17 00:00:00 2001 From: ci Date: Sun, 30 Aug 2026 02:26:29 -0400 Subject: [PATCH 12/13] fix: reject symlinked ingress ancestors --- .../core-ingress.sh | 46 ++++++++++++++++++- scripts/test/portable-core-ingress.test.sh | 9 ++++ 2 files changed, 53 insertions(+), 2 deletions(-) diff --git a/core/v1/generations/g-14b7ad8ce54c3b8c585ff92063d71551ffc7394cc2294d0297bc7d2b8da2c386/core-ingress.sh b/core/v1/generations/g-14b7ad8ce54c3b8c585ff92063d71551ffc7394cc2294d0297bc7d2b8da2c386/core-ingress.sh index 22214ad..0c0747b 100644 --- a/core/v1/generations/g-14b7ad8ce54c3b8c585ff92063d71551ffc7394cc2294d0297bc7d2b8da2c386/core-ingress.sh +++ b/core/v1/generations/g-14b7ad8ce54c3b8c585ff92063d71551ffc7394cc2294d0297bc7d2b8da2c386/core-ingress.sh @@ -21,7 +21,34 @@ portable_core_ingress_real_directory() { [ -d "$1" ] && [ ! -L "$1" ] } +portable_core_ingress_physical_file_path() { + local candidate_path="$1" + local current_path='' + local component + local component_index + local -a path_components + + case "$candidate_path" in /*) ;; *) return 1 ;; esac + IFS='/' read -r -a path_components <<< "$candidate_path" + [ "${#path_components[@]}" -gt 1 ] || return 1 + for ((component_index = 1; + component_index < ${#path_components[@]}; + component_index++)); do + component="${path_components[$component_index]}" + case "$component" in ''|.|..) return 1 ;; esac + current_path="$current_path/$component" + [ ! -L "$current_path" ] || return 1 + if [ "$component_index" -eq $((${#path_components[@]} - 1)) ]; then + [ -f "$current_path" ] || return 1 + else + [ -d "$current_path" ] || return 1 + fi + done +} + portable_core_ingress_open() { + local source_path + local source_cwd local source_dir local source_parent local repo_root @@ -39,9 +66,23 @@ portable_core_ingress_open() { } PORTABLE_CORE_INGRESS_GENERATION='g-14b7ad8ce54c3b8c585ff92063d71551ffc7394cc2294d0297bc7d2b8da2c386' + case "${BASH_SOURCE[0]}" in + /*) source_path="${BASH_SOURCE[0]}" ;; + *) + source_cwd="$(pwd -L 2>/dev/null)" || { + portable_core_ingress_error E_RUNTIME + return 1 + } + source_path="$source_cwd/${BASH_SOURCE[0]}" + ;; + esac + portable_core_ingress_physical_file_path "$source_path" || { + portable_core_ingress_error E_RUNTIME + return 1 + } source_dir="$( { - source_parent="$(dirname -- "${BASH_SOURCE[0]}")" && + source_parent="$(dirname -- "$source_path")" && CDPATH='' cd -P -- "$source_parent" && pwd -P } 2>/dev/null )" || { @@ -57,7 +98,8 @@ portable_core_ingress_open() { return 1 } expected_dir="$repo_root/core/v1/generations/$PORTABLE_CORE_INGRESS_GENERATION" - [ "$source_dir" = "$expected_dir" ] || { + [ "$source_dir" = "$expected_dir" ] && + [ "$source_path" = "$expected_dir/core-ingress.sh" ] || { portable_core_ingress_error E_RUNTIME return 1 } diff --git a/scripts/test/portable-core-ingress.test.sh b/scripts/test/portable-core-ingress.test.sh index dc17426..e4777f0 100755 --- a/scripts/test/portable-core-ingress.test.sh +++ b/scripts/test/portable-core-ingress.test.sh @@ -12,6 +12,7 @@ ingress_manifest="$ingress_repo/ci/required-files.txt" ingress_registry="$ingress_repo/core/v1/generation-registry.json" ingress_host_path="$PATH" ingress_tmp="$(mktemp -d "${TMPDIR:-/tmp}/ystack-portable-ingress-test.XXXXXX")" +ingress_tmp="$(cd "$ingress_tmp" && pwd -P)" ingress_download='' cleanup() { @@ -995,6 +996,14 @@ cp "$ingress_product" "$package_generation_dir/core-ingress.sh" cp "$ingress_schema" "$package_generation_dir/modules/schema.jq" package_product="$package_generation_dir/core-ingress.sh" package_root="$package_generation_dir/contracts.jq" +package_alias="$ingress_tmp/package-alias" +mkdir "$package_alias" +ln -s "$package_copy/core" "$package_alias/core" +symlink_ancestor_product="$package_alias/core/v1/generations/$ingress_generation/core-ingress.sh" +runtime_failure source-ancestor-symlink E_RUNTIME \ + env PATH="$ingress_proof_bin:$ingress_host_path" \ + /bin/bash -c 'source "$1"; portable_core_ingress_open' _ \ + "$symlink_ancestor_product" printf '%s\n' 'import "schema" as schema;' \ 'if (.docs[0].content | schema::parsed_limits_ok | not) then "E_LIMIT"' \ From f95d1800233c5fbf2178bc2d2a5246cd07459a2c Mon Sep 17 00:00:00 2001 From: ci Date: Sun, 30 Aug 2026 02:40:19 -0400 Subject: [PATCH 13/13] fix: align ingress depth semantics --- .../core-ingress.sh | 11 +++++++++-- scripts/test/portable-core-ingress.test.sh | 13 ++++++++++++- 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/core/v1/generations/g-14b7ad8ce54c3b8c585ff92063d71551ffc7394cc2294d0297bc7d2b8da2c386/core-ingress.sh b/core/v1/generations/g-14b7ad8ce54c3b8c585ff92063d71551ffc7394cc2294d0297bc7d2b8da2c386/core-ingress.sh index 0c0747b..e882b38 100644 --- a/core/v1/generations/g-14b7ad8ce54c3b8c585ff92063d71551ffc7394cc2294d0297bc7d2b8da2c386/core-ingress.sh +++ b/core/v1/generations/g-14b7ad8ce54c3b8c585ff92063d71551ffc7394cc2294d0297bc7d2b8da2c386/core-ingress.sh @@ -402,6 +402,9 @@ portable_core_ingress_analyze() { function key_expected() { return depth > 0 && (frame[depth] == 4 || frame[depth] == 5) } + function record_value_depth() { + if (depth > max_depth) max_depth = depth + } function complete_value() { if (depth == 0) { if (root_state != 1) { fail_parse(); return } @@ -478,6 +481,7 @@ portable_core_ingress_analyze() { } function start_atom(byte) { if (!value_expected()) { fail_parse(); return } + record_value_depth() begin_token("value", byte) if (byte == 116) { token_state = "literal"; literal_kind = 1; literal_pos = 1; @@ -547,8 +551,8 @@ portable_core_ingress_analyze() { } function push_container(byte) { if (!value_expected()) { fail_parse(); return } + record_value_depth() depth++ - if (depth > max_depth) max_depth = depth if (byte == 91) frame[depth] = 1 else { frame[depth] = 4 @@ -582,7 +586,10 @@ portable_core_ingress_analyze() { if (depth == 0 && root_state == 2) { fail_parse(); return } if (byte == 34) { if (key_expected()) begin_token("key", byte) - else if (value_expected()) begin_token("value", byte) + else if (value_expected()) { + record_value_depth() + begin_token("value", byte) + } else { fail_parse(); return } in_string = 1 escape_state = 0 diff --git a/scripts/test/portable-core-ingress.test.sh b/scripts/test/portable-core-ingress.test.sh index e4777f0..0e9fcd9 100755 --- a/scripts/test/portable-core-ingress.test.sh +++ b/scripts/test/portable-core-ingress.test.sh @@ -361,7 +361,9 @@ python3 - "$ingress_tmp/depth-32.json" "$ingress_tmp/depth-33.json" \ "$ingress_tmp/deep-object-duplicate.json" \ "$ingress_tmp/deep-object-unsorted.json" \ "$ingress_tmp/deep-high-surrogate.json" \ - "$ingress_tmp/deep-low-surrogate.json" <<'PY' + "$ingress_tmp/deep-low-surrogate.json" \ + "$ingress_tmp/depth-empty-32.json" \ + "$ingress_tmp/depth-empty-33.json" <<'PY' import json import sys @@ -408,6 +410,8 @@ open(sys.argv[21], "wb").write(object_prefix + b'{"a":0,"a":1}' + object_suffix open(sys.argv[22], "wb").write(object_prefix + b'{"b":0,"a":1}' + object_suffix + b"\n") open(sys.argv[23], "wb").write(b"[" * 129 + b'"\\uD800"' + b"]" * 129 + b"\n") open(sys.argv[24], "wb").write(b"[" * 129 + b'"\\uDC00"' + b"]" * 129 + b"\n") +open(sys.argv[25], "wb").write(b"[" * 32 + b"[]" + b"]" * 32 + b"\n") +open(sys.argv[26], "wb").write(b"[" * 33 + b"[]" + b"]" * 33 + b"\n") PY [ "$(wc -c < "$ingress_tmp/deep-wide.json" | tr -d ' ')" -eq 1000000 ] [ "$(wc -c < "$ingress_tmp/deep-wide-malformed.json" | tr -d ' ')" -eq 999999 ] @@ -422,6 +426,13 @@ portable_core_ingress_snapshot "$ingress_tmp/depth-33.json" expect_failure depth-33-limit E_LIMIT portable_core_ingress_finish_driver stop_ingress start_ingress document +expect_success depth-empty-32-snapshot portable_core_ingress_snapshot \ + "$ingress_tmp/depth-empty-32.json" +expect_success depth-empty-32-driver portable_core_ingress_finish_driver +stop_ingress +document_finish_failure depth-empty-33-limit E_LIMIT \ + "$ingress_tmp/depth-empty-33.json" +start_ingress document portable_core_ingress_snapshot "$ingress_tmp/depth-257.json" expect_failure depth-257-limit E_LIMIT portable_core_ingress_finish_driver stop_ingress