diff --git a/.config/daily-only.filterset b/.config/daily-only.filterset index a3cc15177f5..789638d491d 100644 --- a/.config/daily-only.filterset +++ b/.config/daily-only.filterset @@ -29,7 +29,7 @@ package(=fractal) and binary(=integration) and test(/^boot_canaries::/) and not package(=fractal) and binary(=integration) and test(/^fleet::/) and not test(/^fleet::(action_coverage_|transport_prefer_|fleet_workflows_have_no_path_filter$|fixture_port_reservation_excludes_other_processes_until_released$)/) # Seeded convergence worlds: three to five nodes and an oracle, 300-599 steps, CPU-bound, 11-30 s -# each: about 265 s per run for these 14 seeds (hetz2, 2026-10-24). Approved by Nathan +# each: about 265 s per run for these 14 seeds (a build host, 2026-10-24). Approved by Nathan # (2026-10-24). Kept per PR: seeds 1, 2 and 34, seed_1_converges_on_a_real_disk, and the # suite's checks (a_seed_runs_the_same_every_time, the_checks_catch_*), so every PR still runs # three worlds and the oracle's own sabotage checks. All 17 seeds run daily. diff --git a/.config/nextest.toml b/.config/nextest.toml index 9ee943c4855..a58b7314f2d 100644 --- a/.config/nextest.toml +++ b/.config/nextest.toml @@ -25,7 +25,7 @@ filter = ''' ''' priority = 100 -# Then the tests of 15 to 75 seconds that started late. A warm hetz2 run of main f1d0414f +# Then the tests of 15 to 75 seconds that started late. A warm build-host run of main f1d0414f # (2026-10-25, 16 threads, nextest 429 s for 4924 test-seconds) started these 55 tests 30 to # 400 seconds in, so the run ended on a tail of 20 to 30 second tests (smallclaims' sync worker # tests started at 400 s) while most threads were idle. Each is named exactly, so a renamed test @@ -143,30 +143,23 @@ retries = { count = 2, backoff = "fixed", delay = "30s", jitter = false } slow-timeout = { period = "60s", terminate-after = 10 } status-level = "fail" final-status-level = "flaky" -# The gate covers fractal, fractal-runtime, fractal-drivers and the other shared crates, and the st2 test -# files. It leaves out st2-only code: st2's own unit tests (none of its remaining modules is -# used by st3), its catalog, eval-run, resync, nomad-survival, event and resource-profile -# supervisor tests, and fractal-drivers' catalog, resync and resource-profile supervisor modules. -# `transport_isolation` needs real systemd user scopes; the daily isolation-vm lane runs it in a -# NixOS VM. The cost check and the load test (`daemon_cost`, `daemon_load`) run in their own +# The gate covers the current workspace. It leaves out the retained fractal-drivers +# catalog library modules. Current scope controls +# need real systemd user scopes and run in the daily isolation VM. +# The cost check and the load test (`daemon_cost`, `daemon_load`) run in their own # perf jobs (scripts/ci-perf). Only the in-memory targets-file linkage unit and the cost # check's route coverage and route-inventory units also run here: they start no daemon or load, # and a new route without a cost probe then fails in its own pull request. See docs/ci.md. # The outbound-only real-network timing check also runs once in perf-cost. Its strict # convergence and redundant-connection windows stay separate from required deterministic # backoff and inbound/route interruption controls. -# scripts/ci-nextest preserves the retained binary selection in one workspace invocation. -# Its filters control execution; Cargo still compiles excluded workspace test targets. +# scripts/ci-nextest runs the current workspace in one invocation. Its filters control +# execution; Cargo still compiles test targets containing excluded test cases. default-filter = ''' not ( ((package(=fractal) or package(=fractal-daemon) or package(=fractal-api-client-v0) or package(=fractal-api-mailbox) or package(=fractal-attention) or package(=fractal-delivery) or package(=fractal-missions) or package(=fractal-seats) or package(=fractal-evidence) or package(=fractal-github) or package(=fractal-seats-base) or package(=fractal-delivery-base) or package(=fractal-missions-base) or package(=fractal-github-base) or package(=fractal-secrets) or package(=fractal-api-core) or package(=fractal-reconcile) or package(=fractal-peer) or package(=fractal-store)) and test(/^messaging_faults::link_minutes$/)) - or (package(=fractal-legacy-runner) and ( - kind(lib) or kind(bin) - or binary(=event_e2e) or binary(=resource_profile_supervisor_e2e) - or test(/^(catalog_[^:]*|eval_run_e2e|nomad_survival|resync[^:]*|supervisor_auto_archive|transport_isolation)::/) - )) or (package(=fractal-drivers) and kind(lib) - and test(/^(catalog[^:]*|resync|resource_profile_supervisor)::/)) + and test(/^catalog[^:]*::/)) or ((package(=fractal) or package(=fractal-daemon) or package(=fractal-api-client-v0) or package(=fractal-api-mailbox) or package(=fractal-attention) or package(=fractal-delivery) or package(=fractal-missions) or package(=fractal-seats) or package(=fractal-evidence) or package(=fractal-github) or package(=fractal-seats-base) or package(=fractal-delivery-base) or package(=fractal-missions-base) or package(=fractal-github-base) or package(=fractal-secrets) or package(=fractal-api-core) or package(=fractal-reconcile) or package(=fractal-peer) or package(=fractal-store)) and test(/^daemon_(cost|load)::/) and not test(=daemon_load::targets_come_from_the_targets_file_and_describe_without_gating) and not test(=daemon_cost::the_cost_check_covers_every_route) diff --git a/.github/clippy-baseline.json b/.github/clippy-baseline.json index 18a29cd1a99..4af728e3898 100644 --- a/.github/clippy-baseline.json +++ b/.github/clippy-baseline.json @@ -10,13 +10,8 @@ "clippy::collapsible_if": 5, "clippy::items_after_test_module": 2, "clippy::needless_borrow": 2, - "clippy::too_many_arguments": 12, - "clippy::useless_format": 1, - "clippy::while_let_loop": 1 - }, - "fractal-legacy-runner": { - "clippy::obfuscated_if_else": 1, - "clippy::too_many_arguments": 2 + "clippy::too_many_arguments": 9, + "clippy::useless_format": 1 }, "fractal": { "clippy::collapsible_if": 4, @@ -37,9 +32,6 @@ "clippy::large_enum_variant": 1, "clippy::manual_is_multiple_of": 1 }, - "fractal-client-codegen": { - "clippy::useless_format": 2 - }, "fractal-feed": { "clippy::filter_map_bool_then": 1, "clippy::if_same_then_else": 1, diff --git a/.github/clippy-splits.json b/.github/clippy-splits.json index 63ced68ccd2..3349f5c5378 100644 --- a/.github/clippy-splits.json +++ b/.github/clippy-splits.json @@ -24,7 +24,6 @@ "fractal-github-issue-component": "st2-github-issue-component", "fractal-github-pr-component": "st2-github-pr-component", "fractal-kernel": "st3-kernel", - "fractal-legacy-runner": "st2", "fractal-migrate": "st3-migrate", "fractal-missions": "st3-missions", "fractal-missions-base": "st3-missions-base", diff --git a/.github/fleet-compat-baseline.json b/.github/fleet-compat-baseline.json index fd31bdd08f1..58a363a056c 100644 --- a/.github/fleet-compat-baseline.json +++ b/.github/fleet-compat-baseline.json @@ -1,16 +1,16 @@ { - "commit": "943445316e3738a1fc1ee58f676334b7ec684bf4", - "tag": "v0.3.23", + "commit": "6a0b06ca0e7f436dbb58dcf2a40a91aa47c1b454", + "tag": "v0.3.25", "why": "Rolling previous-published-release compatibility baseline; advance after each stable release.", "exception_reason": null, "bundles": { "x86_64-unknown-linux-gnu": { - "url": "https://github.com/compoundingtech/fractal/releases/download/v0.3.23/smalltalk-x86_64-unknown-linux-gnu.tar.gz", - "sha256": "f09c386e0ed328d95a55366621eeb1e2de15883650261a7010b974148cde49f4" + "url": "https://github.com/compoundingtech/fractal/releases/download/v0.3.25/smalltalk-x86_64-unknown-linux-gnu.tar.gz", + "sha256": "7254d3e7cb910a5629d191eb841e24aa8dd500b46b74cf2b025be1e4d6fde8b2" }, "aarch64-apple-darwin": { - "url": "https://github.com/compoundingtech/fractal/releases/download/v0.3.23/smalltalk-aarch64-apple-darwin.tar.gz", - "sha256": "8057d3f504edc373a21d838f66a17eb5847c49735ecb352f74f092aa14ca8f30" + "url": "https://github.com/compoundingtech/fractal/releases/download/v0.3.25/smalltalk-aarch64-apple-darwin.tar.gz", + "sha256": "9244062d6751d9e706439c37072b97388bd1f2bfb8501ae09c2e0838c4709987" } } } diff --git a/.github/messaging-compat-baseline.json b/.github/messaging-compat-baseline.json index fd31bdd08f1..58a363a056c 100644 --- a/.github/messaging-compat-baseline.json +++ b/.github/messaging-compat-baseline.json @@ -1,16 +1,16 @@ { - "commit": "943445316e3738a1fc1ee58f676334b7ec684bf4", - "tag": "v0.3.23", + "commit": "6a0b06ca0e7f436dbb58dcf2a40a91aa47c1b454", + "tag": "v0.3.25", "why": "Rolling previous-published-release compatibility baseline; advance after each stable release.", "exception_reason": null, "bundles": { "x86_64-unknown-linux-gnu": { - "url": "https://github.com/compoundingtech/fractal/releases/download/v0.3.23/smalltalk-x86_64-unknown-linux-gnu.tar.gz", - "sha256": "f09c386e0ed328d95a55366621eeb1e2de15883650261a7010b974148cde49f4" + "url": "https://github.com/compoundingtech/fractal/releases/download/v0.3.25/smalltalk-x86_64-unknown-linux-gnu.tar.gz", + "sha256": "7254d3e7cb910a5629d191eb841e24aa8dd500b46b74cf2b025be1e4d6fde8b2" }, "aarch64-apple-darwin": { - "url": "https://github.com/compoundingtech/fractal/releases/download/v0.3.23/smalltalk-aarch64-apple-darwin.tar.gz", - "sha256": "8057d3f504edc373a21d838f66a17eb5847c49735ecb352f74f092aa14ca8f30" + "url": "https://github.com/compoundingtech/fractal/releases/download/v0.3.25/smalltalk-aarch64-apple-darwin.tar.gz", + "sha256": "9244062d6751d9e706439c37072b97388bd1f2bfb8501ae09c2e0838c4709987" } } } diff --git a/.github/workflows/daily-long.yml b/.github/workflows/daily-long.yml index 06ecac19c83..556212376f1 100644 --- a/.github/workflows/daily-long.yml +++ b/.github/workflows/daily-long.yml @@ -50,8 +50,21 @@ jobs: export GIT_CONFIG_VALUE_0 fi head=$(git rev-parse HEAD) - rm -f "$(git rev-parse --git-path shallow)" - git fetch --quiet --refetch --no-tags origin '+refs/heads/*:refs/remotes/origin/*' '+refs/tags/*:refs/tags/*' '^refs/tags/evidence/*' "$head" + shallow=$(git rev-parse --git-path shallow) + errors=$(mktemp) + fetch() { + status=0 + git fetch --quiet --refetch --no-tags origin '+refs/heads/*:refs/remotes/origin/*' '+refs/tags/*:refs/tags/*' '^refs/tags/evidence/*' "$head" 2> "$errors" || status=$? + cat "$errors" >&2 + return "$status" + } + transport() { grep -Eq 'HTTP (408|429|5[0-9][0-9])|RPC failed|remote end hung up|early EOF|Connection (timed out|reset)|Operation timed out' "$errors"; } + [ ! -f "$shallow" ] || mv "$shallow" "$shallow.full-fetch" + if ! fetch && ! { transport && echo 'full history fetch: transport failure; trying once more' >&2 && fetch; }; then + [ ! -f "$shallow.full-fetch" ] || mv "$shallow.full-fetch" "$shallow" + exit 1 + fi + rm -f "$shallow.full-fetch" git fsck --connectivity-only --no-dangling --no-progress - name: Use the runner's own caches run: | @@ -234,8 +247,21 @@ jobs: export GIT_CONFIG_VALUE_0 fi head=$(git rev-parse HEAD) - rm -f "$(git rev-parse --git-path shallow)" - git fetch --quiet --refetch --no-tags origin '+refs/heads/*:refs/remotes/origin/*' '+refs/tags/*:refs/tags/*' '^refs/tags/evidence/*' "$head" + shallow=$(git rev-parse --git-path shallow) + errors=$(mktemp) + fetch() { + status=0 + git fetch --quiet --refetch --no-tags origin '+refs/heads/*:refs/remotes/origin/*' '+refs/tags/*:refs/tags/*' '^refs/tags/evidence/*' "$head" 2> "$errors" || status=$? + cat "$errors" >&2 + return "$status" + } + transport() { grep -Eq 'HTTP (408|429|5[0-9][0-9])|RPC failed|remote end hung up|early EOF|Connection (timed out|reset)|Operation timed out' "$errors"; } + [ ! -f "$shallow" ] || mv "$shallow" "$shallow.full-fetch" + if ! fetch && ! { transport && echo 'full history fetch: transport failure; trying once more' >&2 && fetch; }; then + [ ! -f "$shallow.full-fetch" ] || mv "$shallow.full-fetch" "$shallow" + exit 1 + fi + rm -f "$shallow.full-fetch" git fsck --connectivity-only --no-dangling --no-progress - name: Use the runner's own caches run: | @@ -415,8 +441,21 @@ jobs: export GIT_CONFIG_VALUE_0 fi head=$(git rev-parse HEAD) - rm -f "$(git rev-parse --git-path shallow)" - git fetch --quiet --refetch --no-tags origin '+refs/heads/*:refs/remotes/origin/*' '+refs/tags/*:refs/tags/*' '^refs/tags/evidence/*' "$head" + shallow=$(git rev-parse --git-path shallow) + errors=$(mktemp) + fetch() { + status=0 + git fetch --quiet --refetch --no-tags origin '+refs/heads/*:refs/remotes/origin/*' '+refs/tags/*:refs/tags/*' '^refs/tags/evidence/*' "$head" 2> "$errors" || status=$? + cat "$errors" >&2 + return "$status" + } + transport() { grep -Eq 'HTTP (408|429|5[0-9][0-9])|RPC failed|remote end hung up|early EOF|Connection (timed out|reset)|Operation timed out' "$errors"; } + [ ! -f "$shallow" ] || mv "$shallow" "$shallow.full-fetch" + if ! fetch && ! { transport && echo 'full history fetch: transport failure; trying once more' >&2 && fetch; }; then + [ ! -f "$shallow.full-fetch" ] || mv "$shallow.full-fetch" "$shallow" + exit 1 + fi + rm -f "$shallow.full-fetch" git fsck --connectivity-only --no-dangling --no-progress - name: Use the runner's own caches run: | @@ -585,8 +624,21 @@ jobs: export GIT_CONFIG_VALUE_0 fi head=$(git rev-parse HEAD) - rm -f "$(git rev-parse --git-path shallow)" - git fetch --quiet --refetch --no-tags origin '+refs/heads/*:refs/remotes/origin/*' '+refs/tags/*:refs/tags/*' '^refs/tags/evidence/*' "$head" + shallow=$(git rev-parse --git-path shallow) + errors=$(mktemp) + fetch() { + status=0 + git fetch --quiet --refetch --no-tags origin '+refs/heads/*:refs/remotes/origin/*' '+refs/tags/*:refs/tags/*' '^refs/tags/evidence/*' "$head" 2> "$errors" || status=$? + cat "$errors" >&2 + return "$status" + } + transport() { grep -Eq 'HTTP (408|429|5[0-9][0-9])|RPC failed|remote end hung up|early EOF|Connection (timed out|reset)|Operation timed out' "$errors"; } + [ ! -f "$shallow" ] || mv "$shallow" "$shallow.full-fetch" + if ! fetch && ! { transport && echo 'full history fetch: transport failure; trying once more' >&2 && fetch; }; then + [ ! -f "$shallow.full-fetch" ] || mv "$shallow.full-fetch" "$shallow" + exit 1 + fi + rm -f "$shallow.full-fetch" git fsck --connectivity-only --no-dangling --no-progress - name: Use the runner's own caches run: | @@ -755,8 +807,21 @@ jobs: export GIT_CONFIG_VALUE_0 fi head=$(git rev-parse HEAD) - rm -f "$(git rev-parse --git-path shallow)" - git fetch --quiet --refetch --no-tags origin '+refs/heads/*:refs/remotes/origin/*' '+refs/tags/*:refs/tags/*' '^refs/tags/evidence/*' "$head" + shallow=$(git rev-parse --git-path shallow) + errors=$(mktemp) + fetch() { + status=0 + git fetch --quiet --refetch --no-tags origin '+refs/heads/*:refs/remotes/origin/*' '+refs/tags/*:refs/tags/*' '^refs/tags/evidence/*' "$head" 2> "$errors" || status=$? + cat "$errors" >&2 + return "$status" + } + transport() { grep -Eq 'HTTP (408|429|5[0-9][0-9])|RPC failed|remote end hung up|early EOF|Connection (timed out|reset)|Operation timed out' "$errors"; } + [ ! -f "$shallow" ] || mv "$shallow" "$shallow.full-fetch" + if ! fetch && ! { transport && echo 'full history fetch: transport failure; trying once more' >&2 && fetch; }; then + [ ! -f "$shallow.full-fetch" ] || mv "$shallow.full-fetch" "$shallow" + exit 1 + fi + rm -f "$shallow.full-fetch" git fsck --connectivity-only --no-dangling --no-progress - name: Use the runner's own caches run: | @@ -916,8 +981,21 @@ jobs: export GIT_CONFIG_VALUE_0 fi head=$(git rev-parse HEAD) - rm -f "$(git rev-parse --git-path shallow)" - git fetch --quiet --refetch --no-tags origin '+refs/heads/*:refs/remotes/origin/*' '+refs/tags/*:refs/tags/*' '^refs/tags/evidence/*' "$head" + shallow=$(git rev-parse --git-path shallow) + errors=$(mktemp) + fetch() { + status=0 + git fetch --quiet --refetch --no-tags origin '+refs/heads/*:refs/remotes/origin/*' '+refs/tags/*:refs/tags/*' '^refs/tags/evidence/*' "$head" 2> "$errors" || status=$? + cat "$errors" >&2 + return "$status" + } + transport() { grep -Eq 'HTTP (408|429|5[0-9][0-9])|RPC failed|remote end hung up|early EOF|Connection (timed out|reset)|Operation timed out' "$errors"; } + [ ! -f "$shallow" ] || mv "$shallow" "$shallow.full-fetch" + if ! fetch && ! { transport && echo 'full history fetch: transport failure; trying once more' >&2 && fetch; }; then + [ ! -f "$shallow.full-fetch" ] || mv "$shallow.full-fetch" "$shallow" + exit 1 + fi + rm -f "$shallow.full-fetch" git fsck --connectivity-only --no-dangling --no-progress - name: Use the runner's own caches run: | @@ -1067,8 +1145,21 @@ jobs: export GIT_CONFIG_VALUE_0 fi head=$(git rev-parse HEAD) - rm -f "$(git rev-parse --git-path shallow)" - git fetch --quiet --refetch --no-tags origin '+refs/heads/*:refs/remotes/origin/*' '+refs/tags/*:refs/tags/*' '^refs/tags/evidence/*' "$head" + shallow=$(git rev-parse --git-path shallow) + errors=$(mktemp) + fetch() { + status=0 + git fetch --quiet --refetch --no-tags origin '+refs/heads/*:refs/remotes/origin/*' '+refs/tags/*:refs/tags/*' '^refs/tags/evidence/*' "$head" 2> "$errors" || status=$? + cat "$errors" >&2 + return "$status" + } + transport() { grep -Eq 'HTTP (408|429|5[0-9][0-9])|RPC failed|remote end hung up|early EOF|Connection (timed out|reset)|Operation timed out' "$errors"; } + [ ! -f "$shallow" ] || mv "$shallow" "$shallow.full-fetch" + if ! fetch && ! { transport && echo 'full history fetch: transport failure; trying once more' >&2 && fetch; }; then + [ ! -f "$shallow.full-fetch" ] || mv "$shallow.full-fetch" "$shallow" + exit 1 + fi + rm -f "$shallow.full-fetch" git fsck --connectivity-only --no-dangling --no-progress - name: Use the runner's own caches run: | @@ -1241,8 +1332,21 @@ jobs: export GIT_CONFIG_VALUE_0 fi head=$(git rev-parse HEAD) - rm -f "$(git rev-parse --git-path shallow)" - git fetch --quiet --refetch --no-tags origin '+refs/heads/*:refs/remotes/origin/*' '+refs/tags/*:refs/tags/*' '^refs/tags/evidence/*' "$head" + shallow=$(git rev-parse --git-path shallow) + errors=$(mktemp) + fetch() { + status=0 + git fetch --quiet --refetch --no-tags origin '+refs/heads/*:refs/remotes/origin/*' '+refs/tags/*:refs/tags/*' '^refs/tags/evidence/*' "$head" 2> "$errors" || status=$? + cat "$errors" >&2 + return "$status" + } + transport() { grep -Eq 'HTTP (408|429|5[0-9][0-9])|RPC failed|remote end hung up|early EOF|Connection (timed out|reset)|Operation timed out' "$errors"; } + [ ! -f "$shallow" ] || mv "$shallow" "$shallow.full-fetch" + if ! fetch && ! { transport && echo 'full history fetch: transport failure; trying once more' >&2 && fetch; }; then + [ ! -f "$shallow.full-fetch" ] || mv "$shallow.full-fetch" "$shallow" + exit 1 + fi + rm -f "$shallow.full-fetch" git fsck --connectivity-only --no-dangling --no-progress - name: Name compatible snapshot recipes env: @@ -1364,8 +1468,21 @@ jobs: export GIT_CONFIG_VALUE_0 fi head=$(git rev-parse HEAD) - rm -f "$(git rev-parse --git-path shallow)" - git fetch --quiet --refetch --no-tags origin '+refs/heads/*:refs/remotes/origin/*' '+refs/tags/*:refs/tags/*' '^refs/tags/evidence/*' "$head" + shallow=$(git rev-parse --git-path shallow) + errors=$(mktemp) + fetch() { + status=0 + git fetch --quiet --refetch --no-tags origin '+refs/heads/*:refs/remotes/origin/*' '+refs/tags/*:refs/tags/*' '^refs/tags/evidence/*' "$head" 2> "$errors" || status=$? + cat "$errors" >&2 + return "$status" + } + transport() { grep -Eq 'HTTP (408|429|5[0-9][0-9])|RPC failed|remote end hung up|early EOF|Connection (timed out|reset)|Operation timed out' "$errors"; } + [ ! -f "$shallow" ] || mv "$shallow" "$shallow.full-fetch" + if ! fetch && ! { transport && echo 'full history fetch: transport failure; trying once more' >&2 && fetch; }; then + [ ! -f "$shallow.full-fetch" ] || mv "$shallow.full-fetch" "$shallow" + exit 1 + fi + rm -f "$shallow.full-fetch" git fsck --connectivity-only --no-dangling --no-progress - name: Name compatible snapshot recipes env: @@ -1445,8 +1562,21 @@ jobs: export GIT_CONFIG_VALUE_0 fi head=$(git rev-parse HEAD) - rm -f "$(git rev-parse --git-path shallow)" - git fetch --quiet --refetch --no-tags origin '+refs/heads/*:refs/remotes/origin/*' '+refs/tags/*:refs/tags/*' '^refs/tags/evidence/*' "$head" + shallow=$(git rev-parse --git-path shallow) + errors=$(mktemp) + fetch() { + status=0 + git fetch --quiet --refetch --no-tags origin '+refs/heads/*:refs/remotes/origin/*' '+refs/tags/*:refs/tags/*' '^refs/tags/evidence/*' "$head" 2> "$errors" || status=$? + cat "$errors" >&2 + return "$status" + } + transport() { grep -Eq 'HTTP (408|429|5[0-9][0-9])|RPC failed|remote end hung up|early EOF|Connection (timed out|reset)|Operation timed out' "$errors"; } + [ ! -f "$shallow" ] || mv "$shallow" "$shallow.full-fetch" + if ! fetch && ! { transport && echo 'full history fetch: transport failure; trying once more' >&2 && fetch; }; then + [ ! -f "$shallow.full-fetch" ] || mv "$shallow.full-fetch" "$shallow" + exit 1 + fi + rm -f "$shallow.full-fetch" git fsck --connectivity-only --no-dangling --no-progress - name: Install Nix uses: DeterminateSystems/determinate-nix-action@v3 @@ -1516,8 +1646,21 @@ jobs: export GIT_CONFIG_VALUE_0 fi head=$(git rev-parse HEAD) - rm -f "$(git rev-parse --git-path shallow)" - git fetch --quiet --refetch --no-tags origin '+refs/heads/*:refs/remotes/origin/*' '+refs/tags/*:refs/tags/*' '^refs/tags/evidence/*' "$head" + shallow=$(git rev-parse --git-path shallow) + errors=$(mktemp) + fetch() { + status=0 + git fetch --quiet --refetch --no-tags origin '+refs/heads/*:refs/remotes/origin/*' '+refs/tags/*:refs/tags/*' '^refs/tags/evidence/*' "$head" 2> "$errors" || status=$? + cat "$errors" >&2 + return "$status" + } + transport() { grep -Eq 'HTTP (408|429|5[0-9][0-9])|RPC failed|remote end hung up|early EOF|Connection (timed out|reset)|Operation timed out' "$errors"; } + [ ! -f "$shallow" ] || mv "$shallow" "$shallow.full-fetch" + if ! fetch && ! { transport && echo 'full history fetch: transport failure; trying once more' >&2 && fetch; }; then + [ ! -f "$shallow.full-fetch" ] || mv "$shallow.full-fetch" "$shallow" + exit 1 + fi + rm -f "$shallow.full-fetch" git fsck --connectivity-only --no-dangling --no-progress - name: Install Nix uses: DeterminateSystems/determinate-nix-action@v3 diff --git a/.github/workflows/debug-assertions-daily.yml b/.github/workflows/debug-assertions-daily.yml index 0c07159bb20..e3c2050c306 100644 --- a/.github/workflows/debug-assertions-daily.yml +++ b/.github/workflows/debug-assertions-daily.yml @@ -49,8 +49,21 @@ jobs: export GIT_CONFIG_VALUE_0 fi head=$(git rev-parse HEAD) - rm -f "$(git rev-parse --git-path shallow)" - git fetch --quiet --refetch --no-tags origin '+refs/heads/*:refs/remotes/origin/*' '+refs/tags/*:refs/tags/*' '^refs/tags/evidence/*' "$head" + shallow=$(git rev-parse --git-path shallow) + errors=$(mktemp) + fetch() { + status=0 + git fetch --quiet --refetch --no-tags origin '+refs/heads/*:refs/remotes/origin/*' '+refs/tags/*:refs/tags/*' '^refs/tags/evidence/*' "$head" 2> "$errors" || status=$? + cat "$errors" >&2 + return "$status" + } + transport() { grep -Eq 'HTTP (408|429|5[0-9][0-9])|RPC failed|remote end hung up|early EOF|Connection (timed out|reset)|Operation timed out' "$errors"; } + [ ! -f "$shallow" ] || mv "$shallow" "$shallow.full-fetch" + if ! fetch && ! { transport && echo 'full history fetch: transport failure; trying once more' >&2 && fetch; }; then + [ ! -f "$shallow.full-fetch" ] || mv "$shallow.full-fetch" "$shallow" + exit 1 + fi + rm -f "$shallow.full-fetch" git fsck --connectivity-only --no-dangling --no-progress - name: Use the runner's own caches run: | diff --git a/.github/workflows/fleet.yml b/.github/workflows/fleet.yml index 121ca223e49..d24e79b7916 100644 --- a/.github/workflows/fleet.yml +++ b/.github/workflows/fleet.yml @@ -1156,8 +1156,21 @@ jobs: export GIT_CONFIG_VALUE_0 fi head=$(git rev-parse HEAD) - rm -f "$(git rev-parse --git-path shallow)" - git fetch --quiet --refetch --no-tags origin '+refs/heads/*:refs/remotes/origin/*' '+refs/tags/*:refs/tags/*' '^refs/tags/evidence/*' "$head" + shallow=$(git rev-parse --git-path shallow) + errors=$(mktemp) + fetch() { + status=0 + git fetch --quiet --refetch --no-tags origin '+refs/heads/*:refs/remotes/origin/*' '+refs/tags/*:refs/tags/*' '^refs/tags/evidence/*' "$head" 2> "$errors" || status=$? + cat "$errors" >&2 + return "$status" + } + transport() { grep -Eq 'HTTP (408|429|5[0-9][0-9])|RPC failed|remote end hung up|early EOF|Connection (timed out|reset)|Operation timed out' "$errors"; } + [ ! -f "$shallow" ] || mv "$shallow" "$shallow.full-fetch" + if ! fetch && ! { transport && echo 'full history fetch: transport failure; trying once more' >&2 && fetch; }; then + [ ! -f "$shallow.full-fetch" ] || mv "$shallow.full-fetch" "$shallow" + exit 1 + fi + rm -f "$shallow.full-fetch" git fsck --connectivity-only --no-dangling --no-progress - name: Test release and PR classification safety run: python3 scripts/release_notes_test.py @@ -1374,8 +1387,21 @@ jobs: export GIT_CONFIG_VALUE_0 fi head=$(git rev-parse HEAD) - rm -f "$(git rev-parse --git-path shallow)" - git fetch --quiet --refetch --no-tags origin '+refs/heads/*:refs/remotes/origin/*' '+refs/tags/*:refs/tags/*' '^refs/tags/evidence/*' "$head" + shallow=$(git rev-parse --git-path shallow) + errors=$(mktemp) + fetch() { + status=0 + git fetch --quiet --refetch --no-tags origin '+refs/heads/*:refs/remotes/origin/*' '+refs/tags/*:refs/tags/*' '^refs/tags/evidence/*' "$head" 2> "$errors" || status=$? + cat "$errors" >&2 + return "$status" + } + transport() { grep -Eq 'HTTP (408|429|5[0-9][0-9])|RPC failed|remote end hung up|early EOF|Connection (timed out|reset)|Operation timed out' "$errors"; } + [ ! -f "$shallow" ] || mv "$shallow" "$shallow.full-fetch" + if ! fetch && ! { transport && echo 'full history fetch: transport failure; trying once more' >&2 && fetch; }; then + [ ! -f "$shallow.full-fetch" ] || mv "$shallow.full-fetch" "$shallow" + exit 1 + fi + rm -f "$shallow.full-fetch" git fsck --connectivity-only --no-dangling --no-progress if: needs.evidence-gate-genie-freshness.outputs.skip != 'true' && needs.evidence-gate-genie-freshness.outputs.refuse != 'true' - name: Use the runner's own caches @@ -1420,7 +1446,7 @@ jobs: env: MEGAREPO_STORE: "${{ runner.temp }}/megarepo" run: | - nix run "github:overengineeringstudio/effect-utils/$(jq -r '.nodes["effect-utils"].locked.rev' flake.lock)#megarepo" -- apply --worktree-mode commit --git-protocol https + python3 scripts/prepare-pnpm-source-inputs.py apply printf 'MEGAREPO_STORE=%s\n' "$MEGAREPO_STORE" >> "$GITHUB_ENV" if: needs.evidence-gate-genie-freshness.outputs.skip != 'true' && needs.evidence-gate-genie-freshness.outputs.refuse != 'true' - name: Check license coverage, runner selection and generated files @@ -1533,8 +1559,21 @@ jobs: export GIT_CONFIG_VALUE_0 fi head=$(git rev-parse HEAD) - rm -f "$(git rev-parse --git-path shallow)" - git fetch --quiet --refetch --no-tags origin '+refs/heads/*:refs/remotes/origin/*' '+refs/tags/*:refs/tags/*' '^refs/tags/evidence/*' "$head" + shallow=$(git rev-parse --git-path shallow) + errors=$(mktemp) + fetch() { + status=0 + git fetch --quiet --refetch --no-tags origin '+refs/heads/*:refs/remotes/origin/*' '+refs/tags/*:refs/tags/*' '^refs/tags/evidence/*' "$head" 2> "$errors" || status=$? + cat "$errors" >&2 + return "$status" + } + transport() { grep -Eq 'HTTP (408|429|5[0-9][0-9])|RPC failed|remote end hung up|early EOF|Connection (timed out|reset)|Operation timed out' "$errors"; } + [ ! -f "$shallow" ] || mv "$shallow" "$shallow.full-fetch" + if ! fetch && ! { transport && echo 'full history fetch: transport failure; trying once more' >&2 && fetch; }; then + [ ! -f "$shallow.full-fetch" ] || mv "$shallow.full-fetch" "$shallow" + exit 1 + fi + rm -f "$shallow.full-fetch" git fsck --connectivity-only --no-dangling --no-progress if: needs.evidence-gate-ci-script-tests.outputs.skip != 'true' && needs.evidence-gate-ci-script-tests.outputs.refuse != 'true' - name: Use the runner's own caches @@ -1579,7 +1618,7 @@ jobs: env: MEGAREPO_STORE: "${{ runner.temp }}/megarepo" run: | - nix run "github:overengineeringstudio/effect-utils/$(jq -r '.nodes["effect-utils"].locked.rev' flake.lock)#megarepo" -- apply --worktree-mode commit --git-protocol https + python3 scripts/prepare-pnpm-source-inputs.py apply printf 'MEGAREPO_STORE=%s\n' "$MEGAREPO_STORE" >> "$GITHUB_ENV" if: needs.evidence-gate-ci-script-tests.outputs.skip != 'true' && needs.evidence-gate-ci-script-tests.outputs.refuse != 'true' - name: Run the CI scripts' self-tests on their declared inputs @@ -1696,8 +1735,21 @@ jobs: export GIT_CONFIG_VALUE_0 fi head=$(git rev-parse HEAD) - rm -f "$(git rev-parse --git-path shallow)" - git fetch --quiet --refetch --no-tags origin '+refs/heads/*:refs/remotes/origin/*' '+refs/tags/*:refs/tags/*' '^refs/tags/evidence/*' "$head" + shallow=$(git rev-parse --git-path shallow) + errors=$(mktemp) + fetch() { + status=0 + git fetch --quiet --refetch --no-tags origin '+refs/heads/*:refs/remotes/origin/*' '+refs/tags/*:refs/tags/*' '^refs/tags/evidence/*' "$head" 2> "$errors" || status=$? + cat "$errors" >&2 + return "$status" + } + transport() { grep -Eq 'HTTP (408|429|5[0-9][0-9])|RPC failed|remote end hung up|early EOF|Connection (timed out|reset)|Operation timed out' "$errors"; } + [ ! -f "$shallow" ] || mv "$shallow" "$shallow.full-fetch" + if ! fetch && ! { transport && echo 'full history fetch: transport failure; trying once more' >&2 && fetch; }; then + [ ! -f "$shallow.full-fetch" ] || mv "$shallow.full-fetch" "$shallow" + exit 1 + fi + rm -f "$shallow.full-fetch" git fsck --connectivity-only --no-dangling --no-progress - name: Install Nix uses: DeterminateSystems/determinate-nix-action@v3 @@ -1716,7 +1768,7 @@ jobs: env: MEGAREPO_STORE: "${{ runner.temp }}/megarepo" run: | - nix run "github:overengineeringstudio/effect-utils/$(jq -r '.nodes["effect-utils"].locked.rev' flake.lock)#megarepo" -- apply --worktree-mode commit --git-protocol https + python3 scripts/prepare-pnpm-source-inputs.py apply printf 'MEGAREPO_STORE=%s\n' "$MEGAREPO_STORE" >> "$GITHUB_ENV" if: needs.evidence-gate-typescript-client.outputs.skip != 'true' && needs.evidence-gate-typescript-client.outputs.refuse != 'true' - uses: actions/setup-node@v4 @@ -1852,8 +1904,21 @@ jobs: export GIT_CONFIG_VALUE_0 fi head=$(git rev-parse HEAD) - rm -f "$(git rev-parse --git-path shallow)" - git fetch --quiet --refetch --no-tags origin '+refs/heads/*:refs/remotes/origin/*' '+refs/tags/*:refs/tags/*' '^refs/tags/evidence/*' "$head" + shallow=$(git rev-parse --git-path shallow) + errors=$(mktemp) + fetch() { + status=0 + git fetch --quiet --refetch --no-tags origin '+refs/heads/*:refs/remotes/origin/*' '+refs/tags/*:refs/tags/*' '^refs/tags/evidence/*' "$head" 2> "$errors" || status=$? + cat "$errors" >&2 + return "$status" + } + transport() { grep -Eq 'HTTP (408|429|5[0-9][0-9])|RPC failed|remote end hung up|early EOF|Connection (timed out|reset)|Operation timed out' "$errors"; } + [ ! -f "$shallow" ] || mv "$shallow" "$shallow.full-fetch" + if ! fetch && ! { transport && echo 'full history fetch: transport failure; trying once more' >&2 && fetch; }; then + [ ! -f "$shallow.full-fetch" ] || mv "$shallow.full-fetch" "$shallow" + exit 1 + fi + rm -f "$shallow.full-fetch" git fsck --connectivity-only --no-dangling --no-progress - name: Install Nix uses: DeterminateSystems/determinate-nix-action@v3 @@ -1958,8 +2023,21 @@ jobs: export GIT_CONFIG_VALUE_0 fi head=$(git rev-parse HEAD) - rm -f "$(git rev-parse --git-path shallow)" - git fetch --quiet --refetch --no-tags origin '+refs/heads/*:refs/remotes/origin/*' '+refs/tags/*:refs/tags/*' '^refs/tags/evidence/*' "$head" + shallow=$(git rev-parse --git-path shallow) + errors=$(mktemp) + fetch() { + status=0 + git fetch --quiet --refetch --no-tags origin '+refs/heads/*:refs/remotes/origin/*' '+refs/tags/*:refs/tags/*' '^refs/tags/evidence/*' "$head" 2> "$errors" || status=$? + cat "$errors" >&2 + return "$status" + } + transport() { grep -Eq 'HTTP (408|429|5[0-9][0-9])|RPC failed|remote end hung up|early EOF|Connection (timed out|reset)|Operation timed out' "$errors"; } + [ ! -f "$shallow" ] || mv "$shallow" "$shallow.full-fetch" + if ! fetch && ! { transport && echo 'full history fetch: transport failure; trying once more' >&2 && fetch; }; then + [ ! -f "$shallow.full-fetch" ] || mv "$shallow.full-fetch" "$shallow" + exit 1 + fi + rm -f "$shallow.full-fetch" git fsck --connectivity-only --no-dangling --no-progress - name: Install Nix uses: DeterminateSystems/determinate-nix-action@v3 @@ -2068,8 +2146,21 @@ jobs: export GIT_CONFIG_VALUE_0 fi head=$(git rev-parse HEAD) - rm -f "$(git rev-parse --git-path shallow)" - git fetch --quiet --refetch --no-tags origin '+refs/heads/*:refs/remotes/origin/*' '+refs/tags/*:refs/tags/*' '^refs/tags/evidence/*' "$head" + shallow=$(git rev-parse --git-path shallow) + errors=$(mktemp) + fetch() { + status=0 + git fetch --quiet --refetch --no-tags origin '+refs/heads/*:refs/remotes/origin/*' '+refs/tags/*:refs/tags/*' '^refs/tags/evidence/*' "$head" 2> "$errors" || status=$? + cat "$errors" >&2 + return "$status" + } + transport() { grep -Eq 'HTTP (408|429|5[0-9][0-9])|RPC failed|remote end hung up|early EOF|Connection (timed out|reset)|Operation timed out' "$errors"; } + [ ! -f "$shallow" ] || mv "$shallow" "$shallow.full-fetch" + if ! fetch && ! { transport && echo 'full history fetch: transport failure; trying once more' >&2 && fetch; }; then + [ ! -f "$shallow.full-fetch" ] || mv "$shallow.full-fetch" "$shallow" + exit 1 + fi + rm -f "$shallow.full-fetch" git fsck --connectivity-only --no-dangling --no-progress - name: Use the runner's own caches run: | @@ -2253,8 +2344,21 @@ jobs: export GIT_CONFIG_VALUE_0 fi head=$(git rev-parse HEAD) - rm -f "$(git rev-parse --git-path shallow)" - git fetch --quiet --refetch --no-tags origin '+refs/heads/*:refs/remotes/origin/*' '+refs/tags/*:refs/tags/*' '^refs/tags/evidence/*' "$head" + shallow=$(git rev-parse --git-path shallow) + errors=$(mktemp) + fetch() { + status=0 + git fetch --quiet --refetch --no-tags origin '+refs/heads/*:refs/remotes/origin/*' '+refs/tags/*:refs/tags/*' '^refs/tags/evidence/*' "$head" 2> "$errors" || status=$? + cat "$errors" >&2 + return "$status" + } + transport() { grep -Eq 'HTTP (408|429|5[0-9][0-9])|RPC failed|remote end hung up|early EOF|Connection (timed out|reset)|Operation timed out' "$errors"; } + [ ! -f "$shallow" ] || mv "$shallow" "$shallow.full-fetch" + if ! fetch && ! { transport && echo 'full history fetch: transport failure; trying once more' >&2 && fetch; }; then + [ ! -f "$shallow.full-fetch" ] || mv "$shallow.full-fetch" "$shallow" + exit 1 + fi + rm -f "$shallow.full-fetch" git fsck --connectivity-only --no-dangling --no-progress - name: Use the runner's own caches run: | @@ -2477,8 +2581,21 @@ jobs: export GIT_CONFIG_VALUE_0 fi head=$(git rev-parse HEAD) - rm -f "$(git rev-parse --git-path shallow)" - git fetch --quiet --refetch --no-tags origin '+refs/heads/*:refs/remotes/origin/*' '+refs/tags/*:refs/tags/*' '^refs/tags/evidence/*' "$head" + shallow=$(git rev-parse --git-path shallow) + errors=$(mktemp) + fetch() { + status=0 + git fetch --quiet --refetch --no-tags origin '+refs/heads/*:refs/remotes/origin/*' '+refs/tags/*:refs/tags/*' '^refs/tags/evidence/*' "$head" 2> "$errors" || status=$? + cat "$errors" >&2 + return "$status" + } + transport() { grep -Eq 'HTTP (408|429|5[0-9][0-9])|RPC failed|remote end hung up|early EOF|Connection (timed out|reset)|Operation timed out' "$errors"; } + [ ! -f "$shallow" ] || mv "$shallow" "$shallow.full-fetch" + if ! fetch && ! { transport && echo 'full history fetch: transport failure; trying once more' >&2 && fetch; }; then + [ ! -f "$shallow.full-fetch" ] || mv "$shallow.full-fetch" "$shallow" + exit 1 + fi + rm -f "$shallow.full-fetch" git fsck --connectivity-only --no-dangling --no-progress - name: Use the runner's own caches run: | @@ -2698,8 +2815,21 @@ jobs: export GIT_CONFIG_VALUE_0 fi head=$(git rev-parse HEAD) - rm -f "$(git rev-parse --git-path shallow)" - git fetch --quiet --refetch --no-tags origin '+refs/heads/*:refs/remotes/origin/*' '+refs/tags/*:refs/tags/*' '^refs/tags/evidence/*' "$head" + shallow=$(git rev-parse --git-path shallow) + errors=$(mktemp) + fetch() { + status=0 + git fetch --quiet --refetch --no-tags origin '+refs/heads/*:refs/remotes/origin/*' '+refs/tags/*:refs/tags/*' '^refs/tags/evidence/*' "$head" 2> "$errors" || status=$? + cat "$errors" >&2 + return "$status" + } + transport() { grep -Eq 'HTTP (408|429|5[0-9][0-9])|RPC failed|remote end hung up|early EOF|Connection (timed out|reset)|Operation timed out' "$errors"; } + [ ! -f "$shallow" ] || mv "$shallow" "$shallow.full-fetch" + if ! fetch && ! { transport && echo 'full history fetch: transport failure; trying once more' >&2 && fetch; }; then + [ ! -f "$shallow.full-fetch" ] || mv "$shallow.full-fetch" "$shallow" + exit 1 + fi + rm -f "$shallow.full-fetch" git fsck --connectivity-only --no-dangling --no-progress - name: Use the runner's own caches run: | @@ -2919,8 +3049,21 @@ jobs: export GIT_CONFIG_VALUE_0 fi head=$(git rev-parse HEAD) - rm -f "$(git rev-parse --git-path shallow)" - git fetch --quiet --refetch --no-tags origin '+refs/heads/*:refs/remotes/origin/*' '+refs/tags/*:refs/tags/*' '^refs/tags/evidence/*' "$head" + shallow=$(git rev-parse --git-path shallow) + errors=$(mktemp) + fetch() { + status=0 + git fetch --quiet --refetch --no-tags origin '+refs/heads/*:refs/remotes/origin/*' '+refs/tags/*:refs/tags/*' '^refs/tags/evidence/*' "$head" 2> "$errors" || status=$? + cat "$errors" >&2 + return "$status" + } + transport() { grep -Eq 'HTTP (408|429|5[0-9][0-9])|RPC failed|remote end hung up|early EOF|Connection (timed out|reset)|Operation timed out' "$errors"; } + [ ! -f "$shallow" ] || mv "$shallow" "$shallow.full-fetch" + if ! fetch && ! { transport && echo 'full history fetch: transport failure; trying once more' >&2 && fetch; }; then + [ ! -f "$shallow.full-fetch" ] || mv "$shallow.full-fetch" "$shallow" + exit 1 + fi + rm -f "$shallow.full-fetch" git fsck --connectivity-only --no-dangling --no-progress - name: Use the runner's own caches run: | @@ -3136,8 +3279,21 @@ jobs: export GIT_CONFIG_VALUE_0 fi head=$(git rev-parse HEAD) - rm -f "$(git rev-parse --git-path shallow)" - git fetch --quiet --refetch --no-tags origin '+refs/heads/*:refs/remotes/origin/*' '+refs/tags/*:refs/tags/*' '^refs/tags/evidence/*' "$head" + shallow=$(git rev-parse --git-path shallow) + errors=$(mktemp) + fetch() { + status=0 + git fetch --quiet --refetch --no-tags origin '+refs/heads/*:refs/remotes/origin/*' '+refs/tags/*:refs/tags/*' '^refs/tags/evidence/*' "$head" 2> "$errors" || status=$? + cat "$errors" >&2 + return "$status" + } + transport() { grep -Eq 'HTTP (408|429|5[0-9][0-9])|RPC failed|remote end hung up|early EOF|Connection (timed out|reset)|Operation timed out' "$errors"; } + [ ! -f "$shallow" ] || mv "$shallow" "$shallow.full-fetch" + if ! fetch && ! { transport && echo 'full history fetch: transport failure; trying once more' >&2 && fetch; }; then + [ ! -f "$shallow.full-fetch" ] || mv "$shallow.full-fetch" "$shallow" + exit 1 + fi + rm -f "$shallow.full-fetch" git fsck --connectivity-only --no-dangling --no-progress - name: Use the runner's own caches run: | @@ -3345,8 +3501,21 @@ jobs: export GIT_CONFIG_VALUE_0 fi head=$(git rev-parse HEAD) - rm -f "$(git rev-parse --git-path shallow)" - git fetch --quiet --refetch --no-tags origin '+refs/heads/*:refs/remotes/origin/*' '+refs/tags/*:refs/tags/*' '^refs/tags/evidence/*' "$head" + shallow=$(git rev-parse --git-path shallow) + errors=$(mktemp) + fetch() { + status=0 + git fetch --quiet --refetch --no-tags origin '+refs/heads/*:refs/remotes/origin/*' '+refs/tags/*:refs/tags/*' '^refs/tags/evidence/*' "$head" 2> "$errors" || status=$? + cat "$errors" >&2 + return "$status" + } + transport() { grep -Eq 'HTTP (408|429|5[0-9][0-9])|RPC failed|remote end hung up|early EOF|Connection (timed out|reset)|Operation timed out' "$errors"; } + [ ! -f "$shallow" ] || mv "$shallow" "$shallow.full-fetch" + if ! fetch && ! { transport && echo 'full history fetch: transport failure; trying once more' >&2 && fetch; }; then + [ ! -f "$shallow.full-fetch" ] || mv "$shallow.full-fetch" "$shallow" + exit 1 + fi + rm -f "$shallow.full-fetch" git fsck --connectivity-only --no-dangling --no-progress - name: Use the runner's own caches run: | @@ -3658,8 +3827,21 @@ jobs: export GIT_CONFIG_VALUE_0 fi head=$(git rev-parse HEAD) - rm -f "$(git rev-parse --git-path shallow)" - git fetch --quiet --refetch --no-tags origin '+refs/heads/*:refs/remotes/origin/*' '+refs/tags/*:refs/tags/*' '^refs/tags/evidence/*' "$head" + shallow=$(git rev-parse --git-path shallow) + errors=$(mktemp) + fetch() { + status=0 + git fetch --quiet --refetch --no-tags origin '+refs/heads/*:refs/remotes/origin/*' '+refs/tags/*:refs/tags/*' '^refs/tags/evidence/*' "$head" 2> "$errors" || status=$? + cat "$errors" >&2 + return "$status" + } + transport() { grep -Eq 'HTTP (408|429|5[0-9][0-9])|RPC failed|remote end hung up|early EOF|Connection (timed out|reset)|Operation timed out' "$errors"; } + [ ! -f "$shallow" ] || mv "$shallow" "$shallow.full-fetch" + if ! fetch && ! { transport && echo 'full history fetch: transport failure; trying once more' >&2 && fetch; }; then + [ ! -f "$shallow.full-fetch" ] || mv "$shallow.full-fetch" "$shallow" + exit 1 + fi + rm -f "$shallow.full-fetch" git fsck --connectivity-only --no-dangling --no-progress - name: Take the signer scripts from the base commit id: base @@ -3795,8 +3977,21 @@ jobs: export GIT_CONFIG_VALUE_0 fi head=$(git rev-parse HEAD) - rm -f "$(git rev-parse --git-path shallow)" - git fetch --quiet --refetch --no-tags origin '+refs/heads/*:refs/remotes/origin/*' '+refs/tags/*:refs/tags/*' '^refs/tags/evidence/*' "$head" + shallow=$(git rev-parse --git-path shallow) + errors=$(mktemp) + fetch() { + status=0 + git fetch --quiet --refetch --no-tags origin '+refs/heads/*:refs/remotes/origin/*' '+refs/tags/*:refs/tags/*' '^refs/tags/evidence/*' "$head" 2> "$errors" || status=$? + cat "$errors" >&2 + return "$status" + } + transport() { grep -Eq 'HTTP (408|429|5[0-9][0-9])|RPC failed|remote end hung up|early EOF|Connection (timed out|reset)|Operation timed out' "$errors"; } + [ ! -f "$shallow" ] || mv "$shallow" "$shallow.full-fetch" + if ! fetch && ! { transport && echo 'full history fetch: transport failure; trying once more' >&2 && fetch; }; then + [ ! -f "$shallow.full-fetch" ] || mv "$shallow.full-fetch" "$shallow" + exit 1 + fi + rm -f "$shallow.full-fetch" git fsck --connectivity-only --no-dangling --no-progress - name: Take the signer scripts from the base commit id: base @@ -3950,6 +4145,7 @@ jobs: "nix/fractal-web*.nix", "scripts/ci-fractal-web*", "scripts/ci-web-shell*", + "scripts/prepare-pnpm-source-inputs.py", "scripts/ci-typescript-client*", ".github/workflows/fleet.yml*", ".github/workflows/workspace-ci.ts", @@ -4153,6 +4349,7 @@ jobs: "flake.lock", "scripts/ci-fractal-web*", "scripts/ci-web-shell*", + "scripts/prepare-pnpm-source-inputs.py", "nix/web-shell*", ".github/workflows/fleet.yml*", ".github/workflows/workspace-ci.ts", @@ -4307,8 +4504,21 @@ jobs: export GIT_CONFIG_VALUE_0 fi head=$(git rev-parse HEAD) - rm -f "$(git rev-parse --git-path shallow)" - git fetch --quiet --refetch --no-tags origin '+refs/heads/*:refs/remotes/origin/*' '+refs/tags/*:refs/tags/*' '^refs/tags/evidence/*' "$head" + shallow=$(git rev-parse --git-path shallow) + errors=$(mktemp) + fetch() { + status=0 + git fetch --quiet --refetch --no-tags origin '+refs/heads/*:refs/remotes/origin/*' '+refs/tags/*:refs/tags/*' '^refs/tags/evidence/*' "$head" 2> "$errors" || status=$? + cat "$errors" >&2 + return "$status" + } + transport() { grep -Eq 'HTTP (408|429|5[0-9][0-9])|RPC failed|remote end hung up|early EOF|Connection (timed out|reset)|Operation timed out' "$errors"; } + [ ! -f "$shallow" ] || mv "$shallow" "$shallow.full-fetch" + if ! fetch && ! { transport && echo 'full history fetch: transport failure; trying once more' >&2 && fetch; }; then + [ ! -f "$shallow.full-fetch" ] || mv "$shallow.full-fetch" "$shallow" + exit 1 + fi + rm -f "$shallow.full-fetch" git fsck --connectivity-only --no-dangling --no-progress - name: Install Nix uses: DeterminateSystems/determinate-nix-action@v3 @@ -4328,7 +4538,7 @@ jobs: env: MEGAREPO_STORE: "${{ runner.temp }}/megarepo" run: | - nix run "github:overengineeringstudio/effect-utils/$(jq -r '.nodes["effect-utils"].locked.rev' flake.lock)#megarepo" -- apply --worktree-mode commit --git-protocol https + python3 scripts/prepare-pnpm-source-inputs.py apply printf 'MEGAREPO_STORE=%s\n' "$MEGAREPO_STORE" >> "$GITHUB_ENV" - name: Check web shell identity invariants run: python3 scripts/ci-web-shell-test @@ -4336,7 +4546,7 @@ jobs: id: web-shell-key shell: bash env: - CI_WEB_SHELL_SOURCE_HASH: ${{ hashFiles('nix/web-shell.nix', 'nix/web-shell-inputs.json', 'scripts/ci-web-shell', 'scripts/ci-web-shell-inputs.jq') }} + CI_WEB_SHELL_SOURCE_HASH: ${{ hashFiles('nix/web-shell.nix', 'nix/web-shell-inputs.json', 'scripts/ci-web-shell', 'scripts/ci-web-shell-inputs.jq', 'scripts/prepare-pnpm-source-inputs.py') }} run: bash scripts/ci-web-shell key - name: Restore the pnpm store id: pnpm-store diff --git a/.github/workflows/ios-landing-screenshots.yml b/.github/workflows/ios-landing-screenshots.yml index fa6bcb21fbe..13f5233b0e2 100644 --- a/.github/workflows/ios-landing-screenshots.yml +++ b/.github/workflows/ios-landing-screenshots.yml @@ -43,7 +43,7 @@ jobs: env: MEGAREPO_STORE: "${{ runner.temp }}/megarepo" run: | - nix run "github:overengineeringstudio/effect-utils/$(jq -r '.nodes["effect-utils"].locked.rev' flake.lock)#megarepo" -- apply --worktree-mode commit --git-protocol https + python3 scripts/prepare-pnpm-source-inputs.py apply printf 'MEGAREPO_STORE=%s\n' "$MEGAREPO_STORE" >> "$GITHUB_ENV" - name: Install locked workspace dependencies env: diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml index a5d1a6e63d3..ef1fed2e765 100644 --- a/.github/workflows/macos.yml +++ b/.github/workflows/macos.yml @@ -52,8 +52,21 @@ jobs: export GIT_CONFIG_VALUE_0 fi head=$(git rev-parse HEAD) - rm -f "$(git rev-parse --git-path shallow)" - git fetch --quiet --refetch --no-tags origin '+refs/heads/*:refs/remotes/origin/*' '+refs/tags/*:refs/tags/*' '^refs/tags/evidence/*' "$head" + shallow=$(git rev-parse --git-path shallow) + errors=$(mktemp) + fetch() { + status=0 + git fetch --quiet --refetch --no-tags origin '+refs/heads/*:refs/remotes/origin/*' '+refs/tags/*:refs/tags/*' '^refs/tags/evidence/*' "$head" 2> "$errors" || status=$? + cat "$errors" >&2 + return "$status" + } + transport() { grep -Eq 'HTTP (408|429|5[0-9][0-9])|RPC failed|remote end hung up|early EOF|Connection (timed out|reset)|Operation timed out' "$errors"; } + [ ! -f "$shallow" ] || mv "$shallow" "$shallow.full-fetch" + if ! fetch && ! { transport && echo 'full history fetch: transport failure; trying once more' >&2 && fetch; }; then + [ ! -f "$shallow.full-fetch" ] || mv "$shallow.full-fetch" "$shallow" + exit 1 + fi + rm -f "$shallow.full-fetch" git fsck --connectivity-only --no-dangling --no-progress - name: Use the runner's own caches run: | diff --git a/.github/workflows/main-upkeep.yml b/.github/workflows/main-upkeep.yml index e43fdc98ffe..141a584698a 100644 --- a/.github/workflows/main-upkeep.yml +++ b/.github/workflows/main-upkeep.yml @@ -177,8 +177,21 @@ jobs: export GIT_CONFIG_VALUE_0 fi head=$(git rev-parse HEAD) - rm -f "$(git rev-parse --git-path shallow)" - git fetch --quiet --refetch --no-tags origin '+refs/heads/*:refs/remotes/origin/*' '+refs/tags/*:refs/tags/*' '^refs/tags/evidence/*' "$head" + shallow=$(git rev-parse --git-path shallow) + errors=$(mktemp) + fetch() { + status=0 + git fetch --quiet --refetch --no-tags origin '+refs/heads/*:refs/remotes/origin/*' '+refs/tags/*:refs/tags/*' '^refs/tags/evidence/*' "$head" 2> "$errors" || status=$? + cat "$errors" >&2 + return "$status" + } + transport() { grep -Eq 'HTTP (408|429|5[0-9][0-9])|RPC failed|remote end hung up|early EOF|Connection (timed out|reset)|Operation timed out' "$errors"; } + [ ! -f "$shallow" ] || mv "$shallow" "$shallow.full-fetch" + if ! fetch && ! { transport && echo 'full history fetch: transport failure; trying once more' >&2 && fetch; }; then + [ ! -f "$shallow.full-fetch" ] || mv "$shallow.full-fetch" "$shallow" + exit 1 + fi + rm -f "$shallow.full-fetch" git fsck --connectivity-only --no-dangling --no-progress if: success() && (steps.cargo-probe.outputs.cache-hit != 'true' || steps.nix-probe.outputs.cache-hit != 'true') && (success()) - name: Use the runner's own caches @@ -392,8 +405,21 @@ jobs: export GIT_CONFIG_VALUE_0 fi head=$(git rev-parse HEAD) - rm -f "$(git rev-parse --git-path shallow)" - git fetch --quiet --refetch --no-tags origin '+refs/heads/*:refs/remotes/origin/*' '+refs/tags/*:refs/tags/*' '^refs/tags/evidence/*' "$head" + shallow=$(git rev-parse --git-path shallow) + errors=$(mktemp) + fetch() { + status=0 + git fetch --quiet --refetch --no-tags origin '+refs/heads/*:refs/remotes/origin/*' '+refs/tags/*:refs/tags/*' '^refs/tags/evidence/*' "$head" 2> "$errors" || status=$? + cat "$errors" >&2 + return "$status" + } + transport() { grep -Eq 'HTTP (408|429|5[0-9][0-9])|RPC failed|remote end hung up|early EOF|Connection (timed out|reset)|Operation timed out' "$errors"; } + [ ! -f "$shallow" ] || mv "$shallow" "$shallow.full-fetch" + if ! fetch && ! { transport && echo 'full history fetch: transport failure; trying once more' >&2 && fetch; }; then + [ ! -f "$shallow.full-fetch" ] || mv "$shallow.full-fetch" "$shallow" + exit 1 + fi + rm -f "$shallow.full-fetch" git fsck --connectivity-only --no-dangling --no-progress if: success() && (steps.cargo-probe.outputs.cache-hit != 'true' || steps.nix-probe.outputs.cache-hit != 'true') && (success()) - name: Use the runner's own caches @@ -585,8 +611,21 @@ jobs: export GIT_CONFIG_VALUE_0 fi head=$(git rev-parse HEAD) - rm -f "$(git rev-parse --git-path shallow)" - git fetch --quiet --refetch --no-tags origin '+refs/heads/*:refs/remotes/origin/*' '+refs/tags/*:refs/tags/*' '^refs/tags/evidence/*' "$head" + shallow=$(git rev-parse --git-path shallow) + errors=$(mktemp) + fetch() { + status=0 + git fetch --quiet --refetch --no-tags origin '+refs/heads/*:refs/remotes/origin/*' '+refs/tags/*:refs/tags/*' '^refs/tags/evidence/*' "$head" 2> "$errors" || status=$? + cat "$errors" >&2 + return "$status" + } + transport() { grep -Eq 'HTTP (408|429|5[0-9][0-9])|RPC failed|remote end hung up|early EOF|Connection (timed out|reset)|Operation timed out' "$errors"; } + [ ! -f "$shallow" ] || mv "$shallow" "$shallow.full-fetch" + if ! fetch && ! { transport && echo 'full history fetch: transport failure; trying once more' >&2 && fetch; }; then + [ ! -f "$shallow.full-fetch" ] || mv "$shallow.full-fetch" "$shallow" + exit 1 + fi + rm -f "$shallow.full-fetch" git fsck --connectivity-only --no-dangling --no-progress if: success() && (steps.cargo-probe.outputs.cache-hit != 'true' || steps.nix-probe.outputs.cache-hit != 'true') && (success()) - name: Use the runner's own caches @@ -762,8 +801,21 @@ jobs: export GIT_CONFIG_VALUE_0 fi head=$(git rev-parse HEAD) - rm -f "$(git rev-parse --git-path shallow)" - git fetch --quiet --refetch --no-tags origin '+refs/heads/*:refs/remotes/origin/*' '+refs/tags/*:refs/tags/*' '^refs/tags/evidence/*' "$head" + shallow=$(git rev-parse --git-path shallow) + errors=$(mktemp) + fetch() { + status=0 + git fetch --quiet --refetch --no-tags origin '+refs/heads/*:refs/remotes/origin/*' '+refs/tags/*:refs/tags/*' '^refs/tags/evidence/*' "$head" 2> "$errors" || status=$? + cat "$errors" >&2 + return "$status" + } + transport() { grep -Eq 'HTTP (408|429|5[0-9][0-9])|RPC failed|remote end hung up|early EOF|Connection (timed out|reset)|Operation timed out' "$errors"; } + [ ! -f "$shallow" ] || mv "$shallow" "$shallow.full-fetch" + if ! fetch && ! { transport && echo 'full history fetch: transport failure; trying once more' >&2 && fetch; }; then + [ ! -f "$shallow.full-fetch" ] || mv "$shallow.full-fetch" "$shallow" + exit 1 + fi + rm -f "$shallow.full-fetch" git fsck --connectivity-only --no-dangling --no-progress if: success() && (steps.nix-probe.outputs.cache-hit != 'true') && (success()) - name: Use the runner's own caches @@ -808,7 +860,7 @@ jobs: env: MEGAREPO_STORE: "${{ runner.temp }}/megarepo" run: | - nix run "github:overengineeringstudio/effect-utils/$(jq -r '.nodes["effect-utils"].locked.rev' flake.lock)#megarepo" -- apply --worktree-mode commit --git-protocol https + python3 scripts/prepare-pnpm-source-inputs.py apply printf 'MEGAREPO_STORE=%s\n' "$MEGAREPO_STORE" >> "$GITHUB_ENV" if: success() && (steps.nix-probe.outputs.cache-hit != 'true') && (success()) - name: Build missing cache contents @@ -895,8 +947,21 @@ jobs: export GIT_CONFIG_VALUE_0 fi head=$(git rev-parse HEAD) - rm -f "$(git rev-parse --git-path shallow)" - git fetch --quiet --refetch --no-tags origin '+refs/heads/*:refs/remotes/origin/*' '+refs/tags/*:refs/tags/*' '^refs/tags/evidence/*' "$head" + shallow=$(git rev-parse --git-path shallow) + errors=$(mktemp) + fetch() { + status=0 + git fetch --quiet --refetch --no-tags origin '+refs/heads/*:refs/remotes/origin/*' '+refs/tags/*:refs/tags/*' '^refs/tags/evidence/*' "$head" 2> "$errors" || status=$? + cat "$errors" >&2 + return "$status" + } + transport() { grep -Eq 'HTTP (408|429|5[0-9][0-9])|RPC failed|remote end hung up|early EOF|Connection (timed out|reset)|Operation timed out' "$errors"; } + [ ! -f "$shallow" ] || mv "$shallow" "$shallow.full-fetch" + if ! fetch && ! { transport && echo 'full history fetch: transport failure; trying once more' >&2 && fetch; }; then + [ ! -f "$shallow.full-fetch" ] || mv "$shallow.full-fetch" "$shallow" + exit 1 + fi + rm -f "$shallow.full-fetch" git fsck --connectivity-only --no-dangling --no-progress if: success() && (steps.cargo-probe.outputs.cache-hit != 'true' || steps.nix-probe.outputs.cache-hit != 'true') && (success()) - name: Use the runner's own caches @@ -1049,7 +1114,7 @@ jobs: env: MEGAREPO_STORE: "${{ runner.temp }}/megarepo" run: | - nix run "github:overengineeringstudio/effect-utils/$(jq -r '.nodes["effect-utils"].locked.rev' flake.lock)#megarepo" -- apply --worktree-mode commit --git-protocol https + python3 scripts/prepare-pnpm-source-inputs.py apply printf 'MEGAREPO_STORE=%s\n' "$MEGAREPO_STORE" >> "$GITHUB_ENV" - uses: actions/setup-node@v4 with: @@ -1111,13 +1176,13 @@ jobs: env: MEGAREPO_STORE: "${{ runner.temp }}/megarepo" run: | - nix run "github:overengineeringstudio/effect-utils/$(jq -r '.nodes["effect-utils"].locked.rev' flake.lock)#megarepo" -- apply --worktree-mode commit --git-protocol https + python3 scripts/prepare-pnpm-source-inputs.py apply printf 'MEGAREPO_STORE=%s\n' "$MEGAREPO_STORE" >> "$GITHUB_ENV" - name: Identify the web shell inputs id: web-shell-key shell: bash env: - CI_WEB_SHELL_SOURCE_HASH: ${{ hashFiles('nix/web-shell.nix', 'nix/web-shell-inputs.json', 'scripts/ci-web-shell', 'scripts/ci-web-shell-inputs.jq') }} + CI_WEB_SHELL_SOURCE_HASH: ${{ hashFiles('nix/web-shell.nix', 'nix/web-shell-inputs.json', 'scripts/ci-web-shell', 'scripts/ci-web-shell-inputs.jq', 'scripts/prepare-pnpm-source-inputs.py') }} run: bash scripts/ci-web-shell key - name: Probe the pnpm store id: pnpm-store @@ -1210,8 +1275,21 @@ jobs: export GIT_CONFIG_VALUE_0 fi head=$(git rev-parse HEAD) - rm -f "$(git rev-parse --git-path shallow)" - git fetch --quiet --refetch --no-tags origin '+refs/heads/*:refs/remotes/origin/*' '+refs/tags/*:refs/tags/*' '^refs/tags/evidence/*' "$head" + shallow=$(git rev-parse --git-path shallow) + errors=$(mktemp) + fetch() { + status=0 + git fetch --quiet --refetch --no-tags origin '+refs/heads/*:refs/remotes/origin/*' '+refs/tags/*:refs/tags/*' '^refs/tags/evidence/*' "$head" 2> "$errors" || status=$? + cat "$errors" >&2 + return "$status" + } + transport() { grep -Eq 'HTTP (408|429|5[0-9][0-9])|RPC failed|remote end hung up|early EOF|Connection (timed out|reset)|Operation timed out' "$errors"; } + [ ! -f "$shallow" ] || mv "$shallow" "$shallow.full-fetch" + if ! fetch && ! { transport && echo 'full history fetch: transport failure; trying once more' >&2 && fetch; }; then + [ ! -f "$shallow.full-fetch" ] || mv "$shallow.full-fetch" "$shallow" + exit 1 + fi + rm -f "$shallow.full-fetch" git fsck --connectivity-only --no-dangling --no-progress - name: Use the runner's own caches run: | @@ -2367,8 +2445,21 @@ jobs: export GIT_CONFIG_VALUE_0 fi head=$(git rev-parse HEAD) - rm -f "$(git rev-parse --git-path shallow)" - git fetch --quiet --refetch --no-tags origin '+refs/heads/*:refs/remotes/origin/*' '+refs/tags/*:refs/tags/*' '^refs/tags/evidence/*' "$head" + shallow=$(git rev-parse --git-path shallow) + errors=$(mktemp) + fetch() { + status=0 + git fetch --quiet --refetch --no-tags origin '+refs/heads/*:refs/remotes/origin/*' '+refs/tags/*:refs/tags/*' '^refs/tags/evidence/*' "$head" 2> "$errors" || status=$? + cat "$errors" >&2 + return "$status" + } + transport() { grep -Eq 'HTTP (408|429|5[0-9][0-9])|RPC failed|remote end hung up|early EOF|Connection (timed out|reset)|Operation timed out' "$errors"; } + [ ! -f "$shallow" ] || mv "$shallow" "$shallow.full-fetch" + if ! fetch && ! { transport && echo 'full history fetch: transport failure; trying once more' >&2 && fetch; }; then + [ ! -f "$shallow.full-fetch" ] || mv "$shallow.full-fetch" "$shallow" + exit 1 + fi + rm -f "$shallow.full-fetch" git fsck --connectivity-only --no-dangling --no-progress - name: Test release and PR classification safety run: python3 scripts/release_notes_test.py @@ -2417,8 +2508,21 @@ jobs: export GIT_CONFIG_VALUE_0 fi head=$(git rev-parse HEAD) - rm -f "$(git rev-parse --git-path shallow)" - git fetch --quiet --refetch --no-tags origin '+refs/heads/*:refs/remotes/origin/*' '+refs/tags/*:refs/tags/*' '^refs/tags/evidence/*' "$head" + shallow=$(git rev-parse --git-path shallow) + errors=$(mktemp) + fetch() { + status=0 + git fetch --quiet --refetch --no-tags origin '+refs/heads/*:refs/remotes/origin/*' '+refs/tags/*:refs/tags/*' '^refs/tags/evidence/*' "$head" 2> "$errors" || status=$? + cat "$errors" >&2 + return "$status" + } + transport() { grep -Eq 'HTTP (408|429|5[0-9][0-9])|RPC failed|remote end hung up|early EOF|Connection (timed out|reset)|Operation timed out' "$errors"; } + [ ! -f "$shallow" ] || mv "$shallow" "$shallow.full-fetch" + if ! fetch && ! { transport && echo 'full history fetch: transport failure; trying once more' >&2 && fetch; }; then + [ ! -f "$shallow.full-fetch" ] || mv "$shallow.full-fetch" "$shallow" + exit 1 + fi + rm -f "$shallow.full-fetch" git fsck --connectivity-only --no-dangling --no-progress if: needs.proof-evidence-gate-genie-freshness.outputs.skip != 'true' && needs.proof-evidence-gate-genie-freshness.outputs.refuse != 'true' - name: Use the runner's own caches @@ -2463,7 +2567,7 @@ jobs: env: MEGAREPO_STORE: "${{ runner.temp }}/megarepo" run: | - nix run "github:overengineeringstudio/effect-utils/$(jq -r '.nodes["effect-utils"].locked.rev' flake.lock)#megarepo" -- apply --worktree-mode commit --git-protocol https + python3 scripts/prepare-pnpm-source-inputs.py apply printf 'MEGAREPO_STORE=%s\n' "$MEGAREPO_STORE" >> "$GITHUB_ENV" if: needs.proof-evidence-gate-genie-freshness.outputs.skip != 'true' && needs.proof-evidence-gate-genie-freshness.outputs.refuse != 'true' - name: Check license coverage, runner selection and generated files @@ -2576,8 +2680,21 @@ jobs: export GIT_CONFIG_VALUE_0 fi head=$(git rev-parse HEAD) - rm -f "$(git rev-parse --git-path shallow)" - git fetch --quiet --refetch --no-tags origin '+refs/heads/*:refs/remotes/origin/*' '+refs/tags/*:refs/tags/*' '^refs/tags/evidence/*' "$head" + shallow=$(git rev-parse --git-path shallow) + errors=$(mktemp) + fetch() { + status=0 + git fetch --quiet --refetch --no-tags origin '+refs/heads/*:refs/remotes/origin/*' '+refs/tags/*:refs/tags/*' '^refs/tags/evidence/*' "$head" 2> "$errors" || status=$? + cat "$errors" >&2 + return "$status" + } + transport() { grep -Eq 'HTTP (408|429|5[0-9][0-9])|RPC failed|remote end hung up|early EOF|Connection (timed out|reset)|Operation timed out' "$errors"; } + [ ! -f "$shallow" ] || mv "$shallow" "$shallow.full-fetch" + if ! fetch && ! { transport && echo 'full history fetch: transport failure; trying once more' >&2 && fetch; }; then + [ ! -f "$shallow.full-fetch" ] || mv "$shallow.full-fetch" "$shallow" + exit 1 + fi + rm -f "$shallow.full-fetch" git fsck --connectivity-only --no-dangling --no-progress if: needs.proof-evidence-gate-ci-script-tests.outputs.skip != 'true' && needs.proof-evidence-gate-ci-script-tests.outputs.refuse != 'true' - name: Use the runner's own caches @@ -2622,7 +2739,7 @@ jobs: env: MEGAREPO_STORE: "${{ runner.temp }}/megarepo" run: | - nix run "github:overengineeringstudio/effect-utils/$(jq -r '.nodes["effect-utils"].locked.rev' flake.lock)#megarepo" -- apply --worktree-mode commit --git-protocol https + python3 scripts/prepare-pnpm-source-inputs.py apply printf 'MEGAREPO_STORE=%s\n' "$MEGAREPO_STORE" >> "$GITHUB_ENV" if: needs.proof-evidence-gate-ci-script-tests.outputs.skip != 'true' && needs.proof-evidence-gate-ci-script-tests.outputs.refuse != 'true' - name: Run the CI scripts' self-tests on their declared inputs @@ -2746,7 +2863,7 @@ jobs: env: MEGAREPO_STORE: "${{ runner.temp }}/megarepo" run: | - nix run "github:overengineeringstudio/effect-utils/$(jq -r '.nodes["effect-utils"].locked.rev' flake.lock)#megarepo" -- apply --worktree-mode commit --git-protocol https + python3 scripts/prepare-pnpm-source-inputs.py apply printf 'MEGAREPO_STORE=%s\n' "$MEGAREPO_STORE" >> "$GITHUB_ENV" if: needs.proof-evidence-gate-typescript-client.outputs.skip != 'true' && needs.proof-evidence-gate-typescript-client.outputs.refuse != 'true' - uses: actions/setup-node@v4 @@ -2882,8 +2999,21 @@ jobs: export GIT_CONFIG_VALUE_0 fi head=$(git rev-parse HEAD) - rm -f "$(git rev-parse --git-path shallow)" - git fetch --quiet --refetch --no-tags origin '+refs/heads/*:refs/remotes/origin/*' '+refs/tags/*:refs/tags/*' '^refs/tags/evidence/*' "$head" + shallow=$(git rev-parse --git-path shallow) + errors=$(mktemp) + fetch() { + status=0 + git fetch --quiet --refetch --no-tags origin '+refs/heads/*:refs/remotes/origin/*' '+refs/tags/*:refs/tags/*' '^refs/tags/evidence/*' "$head" 2> "$errors" || status=$? + cat "$errors" >&2 + return "$status" + } + transport() { grep -Eq 'HTTP (408|429|5[0-9][0-9])|RPC failed|remote end hung up|early EOF|Connection (timed out|reset)|Operation timed out' "$errors"; } + [ ! -f "$shallow" ] || mv "$shallow" "$shallow.full-fetch" + if ! fetch && ! { transport && echo 'full history fetch: transport failure; trying once more' >&2 && fetch; }; then + [ ! -f "$shallow.full-fetch" ] || mv "$shallow.full-fetch" "$shallow" + exit 1 + fi + rm -f "$shallow.full-fetch" git fsck --connectivity-only --no-dangling --no-progress - name: Install Nix uses: DeterminateSystems/determinate-nix-action@v3 @@ -2988,8 +3118,21 @@ jobs: export GIT_CONFIG_VALUE_0 fi head=$(git rev-parse HEAD) - rm -f "$(git rev-parse --git-path shallow)" - git fetch --quiet --refetch --no-tags origin '+refs/heads/*:refs/remotes/origin/*' '+refs/tags/*:refs/tags/*' '^refs/tags/evidence/*' "$head" + shallow=$(git rev-parse --git-path shallow) + errors=$(mktemp) + fetch() { + status=0 + git fetch --quiet --refetch --no-tags origin '+refs/heads/*:refs/remotes/origin/*' '+refs/tags/*:refs/tags/*' '^refs/tags/evidence/*' "$head" 2> "$errors" || status=$? + cat "$errors" >&2 + return "$status" + } + transport() { grep -Eq 'HTTP (408|429|5[0-9][0-9])|RPC failed|remote end hung up|early EOF|Connection (timed out|reset)|Operation timed out' "$errors"; } + [ ! -f "$shallow" ] || mv "$shallow" "$shallow.full-fetch" + if ! fetch && ! { transport && echo 'full history fetch: transport failure; trying once more' >&2 && fetch; }; then + [ ! -f "$shallow.full-fetch" ] || mv "$shallow.full-fetch" "$shallow" + exit 1 + fi + rm -f "$shallow.full-fetch" git fsck --connectivity-only --no-dangling --no-progress - name: Install Nix uses: DeterminateSystems/determinate-nix-action@v3 @@ -3104,8 +3247,21 @@ jobs: export GIT_CONFIG_VALUE_0 fi head=$(git rev-parse HEAD) - rm -f "$(git rev-parse --git-path shallow)" - git fetch --quiet --refetch --no-tags origin '+refs/heads/*:refs/remotes/origin/*' '+refs/tags/*:refs/tags/*' '^refs/tags/evidence/*' "$head" + shallow=$(git rev-parse --git-path shallow) + errors=$(mktemp) + fetch() { + status=0 + git fetch --quiet --refetch --no-tags origin '+refs/heads/*:refs/remotes/origin/*' '+refs/tags/*:refs/tags/*' '^refs/tags/evidence/*' "$head" 2> "$errors" || status=$? + cat "$errors" >&2 + return "$status" + } + transport() { grep -Eq 'HTTP (408|429|5[0-9][0-9])|RPC failed|remote end hung up|early EOF|Connection (timed out|reset)|Operation timed out' "$errors"; } + [ ! -f "$shallow" ] || mv "$shallow" "$shallow.full-fetch" + if ! fetch && ! { transport && echo 'full history fetch: transport failure; trying once more' >&2 && fetch; }; then + [ ! -f "$shallow.full-fetch" ] || mv "$shallow.full-fetch" "$shallow" + exit 1 + fi + rm -f "$shallow.full-fetch" git fsck --connectivity-only --no-dangling --no-progress - name: Use the runner's own caches run: | @@ -3296,8 +3452,21 @@ jobs: export GIT_CONFIG_VALUE_0 fi head=$(git rev-parse HEAD) - rm -f "$(git rev-parse --git-path shallow)" - git fetch --quiet --refetch --no-tags origin '+refs/heads/*:refs/remotes/origin/*' '+refs/tags/*:refs/tags/*' '^refs/tags/evidence/*' "$head" + shallow=$(git rev-parse --git-path shallow) + errors=$(mktemp) + fetch() { + status=0 + git fetch --quiet --refetch --no-tags origin '+refs/heads/*:refs/remotes/origin/*' '+refs/tags/*:refs/tags/*' '^refs/tags/evidence/*' "$head" 2> "$errors" || status=$? + cat "$errors" >&2 + return "$status" + } + transport() { grep -Eq 'HTTP (408|429|5[0-9][0-9])|RPC failed|remote end hung up|early EOF|Connection (timed out|reset)|Operation timed out' "$errors"; } + [ ! -f "$shallow" ] || mv "$shallow" "$shallow.full-fetch" + if ! fetch && ! { transport && echo 'full history fetch: transport failure; trying once more' >&2 && fetch; }; then + [ ! -f "$shallow.full-fetch" ] || mv "$shallow.full-fetch" "$shallow" + exit 1 + fi + rm -f "$shallow.full-fetch" git fsck --connectivity-only --no-dangling --no-progress - name: Use the runner's own caches run: | @@ -3527,8 +3696,21 @@ jobs: export GIT_CONFIG_VALUE_0 fi head=$(git rev-parse HEAD) - rm -f "$(git rev-parse --git-path shallow)" - git fetch --quiet --refetch --no-tags origin '+refs/heads/*:refs/remotes/origin/*' '+refs/tags/*:refs/tags/*' '^refs/tags/evidence/*' "$head" + shallow=$(git rev-parse --git-path shallow) + errors=$(mktemp) + fetch() { + status=0 + git fetch --quiet --refetch --no-tags origin '+refs/heads/*:refs/remotes/origin/*' '+refs/tags/*:refs/tags/*' '^refs/tags/evidence/*' "$head" 2> "$errors" || status=$? + cat "$errors" >&2 + return "$status" + } + transport() { grep -Eq 'HTTP (408|429|5[0-9][0-9])|RPC failed|remote end hung up|early EOF|Connection (timed out|reset)|Operation timed out' "$errors"; } + [ ! -f "$shallow" ] || mv "$shallow" "$shallow.full-fetch" + if ! fetch && ! { transport && echo 'full history fetch: transport failure; trying once more' >&2 && fetch; }; then + [ ! -f "$shallow.full-fetch" ] || mv "$shallow.full-fetch" "$shallow" + exit 1 + fi + rm -f "$shallow.full-fetch" git fsck --connectivity-only --no-dangling --no-progress - name: Use the runner's own caches run: | @@ -3755,8 +3937,21 @@ jobs: export GIT_CONFIG_VALUE_0 fi head=$(git rev-parse HEAD) - rm -f "$(git rev-parse --git-path shallow)" - git fetch --quiet --refetch --no-tags origin '+refs/heads/*:refs/remotes/origin/*' '+refs/tags/*:refs/tags/*' '^refs/tags/evidence/*' "$head" + shallow=$(git rev-parse --git-path shallow) + errors=$(mktemp) + fetch() { + status=0 + git fetch --quiet --refetch --no-tags origin '+refs/heads/*:refs/remotes/origin/*' '+refs/tags/*:refs/tags/*' '^refs/tags/evidence/*' "$head" 2> "$errors" || status=$? + cat "$errors" >&2 + return "$status" + } + transport() { grep -Eq 'HTTP (408|429|5[0-9][0-9])|RPC failed|remote end hung up|early EOF|Connection (timed out|reset)|Operation timed out' "$errors"; } + [ ! -f "$shallow" ] || mv "$shallow" "$shallow.full-fetch" + if ! fetch && ! { transport && echo 'full history fetch: transport failure; trying once more' >&2 && fetch; }; then + [ ! -f "$shallow.full-fetch" ] || mv "$shallow.full-fetch" "$shallow" + exit 1 + fi + rm -f "$shallow.full-fetch" git fsck --connectivity-only --no-dangling --no-progress - name: Use the runner's own caches run: | @@ -3983,8 +4178,21 @@ jobs: export GIT_CONFIG_VALUE_0 fi head=$(git rev-parse HEAD) - rm -f "$(git rev-parse --git-path shallow)" - git fetch --quiet --refetch --no-tags origin '+refs/heads/*:refs/remotes/origin/*' '+refs/tags/*:refs/tags/*' '^refs/tags/evidence/*' "$head" + shallow=$(git rev-parse --git-path shallow) + errors=$(mktemp) + fetch() { + status=0 + git fetch --quiet --refetch --no-tags origin '+refs/heads/*:refs/remotes/origin/*' '+refs/tags/*:refs/tags/*' '^refs/tags/evidence/*' "$head" 2> "$errors" || status=$? + cat "$errors" >&2 + return "$status" + } + transport() { grep -Eq 'HTTP (408|429|5[0-9][0-9])|RPC failed|remote end hung up|early EOF|Connection (timed out|reset)|Operation timed out' "$errors"; } + [ ! -f "$shallow" ] || mv "$shallow" "$shallow.full-fetch" + if ! fetch && ! { transport && echo 'full history fetch: transport failure; trying once more' >&2 && fetch; }; then + [ ! -f "$shallow.full-fetch" ] || mv "$shallow.full-fetch" "$shallow" + exit 1 + fi + rm -f "$shallow.full-fetch" git fsck --connectivity-only --no-dangling --no-progress - name: Use the runner's own caches run: | @@ -4206,8 +4414,21 @@ jobs: export GIT_CONFIG_VALUE_0 fi head=$(git rev-parse HEAD) - rm -f "$(git rev-parse --git-path shallow)" - git fetch --quiet --refetch --no-tags origin '+refs/heads/*:refs/remotes/origin/*' '+refs/tags/*:refs/tags/*' '^refs/tags/evidence/*' "$head" + shallow=$(git rev-parse --git-path shallow) + errors=$(mktemp) + fetch() { + status=0 + git fetch --quiet --refetch --no-tags origin '+refs/heads/*:refs/remotes/origin/*' '+refs/tags/*:refs/tags/*' '^refs/tags/evidence/*' "$head" 2> "$errors" || status=$? + cat "$errors" >&2 + return "$status" + } + transport() { grep -Eq 'HTTP (408|429|5[0-9][0-9])|RPC failed|remote end hung up|early EOF|Connection (timed out|reset)|Operation timed out' "$errors"; } + [ ! -f "$shallow" ] || mv "$shallow" "$shallow.full-fetch" + if ! fetch && ! { transport && echo 'full history fetch: transport failure; trying once more' >&2 && fetch; }; then + [ ! -f "$shallow.full-fetch" ] || mv "$shallow.full-fetch" "$shallow" + exit 1 + fi + rm -f "$shallow.full-fetch" git fsck --connectivity-only --no-dangling --no-progress - name: Use the runner's own caches run: | @@ -4421,8 +4642,21 @@ jobs: export GIT_CONFIG_VALUE_0 fi head=$(git rev-parse HEAD) - rm -f "$(git rev-parse --git-path shallow)" - git fetch --quiet --refetch --no-tags origin '+refs/heads/*:refs/remotes/origin/*' '+refs/tags/*:refs/tags/*' '^refs/tags/evidence/*' "$head" + shallow=$(git rev-parse --git-path shallow) + errors=$(mktemp) + fetch() { + status=0 + git fetch --quiet --refetch --no-tags origin '+refs/heads/*:refs/remotes/origin/*' '+refs/tags/*:refs/tags/*' '^refs/tags/evidence/*' "$head" 2> "$errors" || status=$? + cat "$errors" >&2 + return "$status" + } + transport() { grep -Eq 'HTTP (408|429|5[0-9][0-9])|RPC failed|remote end hung up|early EOF|Connection (timed out|reset)|Operation timed out' "$errors"; } + [ ! -f "$shallow" ] || mv "$shallow" "$shallow.full-fetch" + if ! fetch && ! { transport && echo 'full history fetch: transport failure; trying once more' >&2 && fetch; }; then + [ ! -f "$shallow.full-fetch" ] || mv "$shallow.full-fetch" "$shallow" + exit 1 + fi + rm -f "$shallow.full-fetch" git fsck --connectivity-only --no-dangling --no-progress - name: Use the runner's own caches run: | @@ -4736,8 +4970,21 @@ jobs: export GIT_CONFIG_VALUE_0 fi head=$(git rev-parse HEAD) - rm -f "$(git rev-parse --git-path shallow)" - git fetch --quiet --refetch --no-tags origin '+refs/heads/*:refs/remotes/origin/*' '+refs/tags/*:refs/tags/*' '^refs/tags/evidence/*' "$head" + shallow=$(git rev-parse --git-path shallow) + errors=$(mktemp) + fetch() { + status=0 + git fetch --quiet --refetch --no-tags origin '+refs/heads/*:refs/remotes/origin/*' '+refs/tags/*:refs/tags/*' '^refs/tags/evidence/*' "$head" 2> "$errors" || status=$? + cat "$errors" >&2 + return "$status" + } + transport() { grep -Eq 'HTTP (408|429|5[0-9][0-9])|RPC failed|remote end hung up|early EOF|Connection (timed out|reset)|Operation timed out' "$errors"; } + [ ! -f "$shallow" ] || mv "$shallow" "$shallow.full-fetch" + if ! fetch && ! { transport && echo 'full history fetch: transport failure; trying once more' >&2 && fetch; }; then + [ ! -f "$shallow.full-fetch" ] || mv "$shallow.full-fetch" "$shallow" + exit 1 + fi + rm -f "$shallow.full-fetch" git fsck --connectivity-only --no-dangling --no-progress - name: Take the signer scripts from the base commit id: base @@ -4875,8 +5122,21 @@ jobs: export GIT_CONFIG_VALUE_0 fi head=$(git rev-parse HEAD) - rm -f "$(git rev-parse --git-path shallow)" - git fetch --quiet --refetch --no-tags origin '+refs/heads/*:refs/remotes/origin/*' '+refs/tags/*:refs/tags/*' '^refs/tags/evidence/*' "$head" + shallow=$(git rev-parse --git-path shallow) + errors=$(mktemp) + fetch() { + status=0 + git fetch --quiet --refetch --no-tags origin '+refs/heads/*:refs/remotes/origin/*' '+refs/tags/*:refs/tags/*' '^refs/tags/evidence/*' "$head" 2> "$errors" || status=$? + cat "$errors" >&2 + return "$status" + } + transport() { grep -Eq 'HTTP (408|429|5[0-9][0-9])|RPC failed|remote end hung up|early EOF|Connection (timed out|reset)|Operation timed out' "$errors"; } + [ ! -f "$shallow" ] || mv "$shallow" "$shallow.full-fetch" + if ! fetch && ! { transport && echo 'full history fetch: transport failure; trying once more' >&2 && fetch; }; then + [ ! -f "$shallow.full-fetch" ] || mv "$shallow.full-fetch" "$shallow" + exit 1 + fi + rm -f "$shallow.full-fetch" git fsck --connectivity-only --no-dangling --no-progress - name: Take the signer scripts from the base commit id: base @@ -5030,6 +5290,7 @@ jobs: "nix/fractal-web*.nix", "scripts/ci-fractal-web*", "scripts/ci-web-shell*", + "scripts/prepare-pnpm-source-inputs.py", "scripts/ci-typescript-client*", ".github/workflows/fleet.yml*", ".github/workflows/workspace-ci.ts", @@ -5233,6 +5494,7 @@ jobs: "flake.lock", "scripts/ci-fractal-web*", "scripts/ci-web-shell*", + "scripts/prepare-pnpm-source-inputs.py", "nix/web-shell*", ".github/workflows/fleet.yml*", ".github/workflows/workspace-ci.ts", @@ -5401,7 +5663,7 @@ jobs: env: MEGAREPO_STORE: "${{ runner.temp }}/megarepo" run: | - nix run "github:overengineeringstudio/effect-utils/$(jq -r '.nodes["effect-utils"].locked.rev' flake.lock)#megarepo" -- apply --worktree-mode commit --git-protocol https + python3 scripts/prepare-pnpm-source-inputs.py apply printf 'MEGAREPO_STORE=%s\n' "$MEGAREPO_STORE" >> "$GITHUB_ENV" - name: Check web shell identity invariants run: python3 scripts/ci-web-shell-test @@ -5409,7 +5671,7 @@ jobs: id: web-shell-key shell: bash env: - CI_WEB_SHELL_SOURCE_HASH: ${{ hashFiles('nix/web-shell.nix', 'nix/web-shell-inputs.json', 'scripts/ci-web-shell', 'scripts/ci-web-shell-inputs.jq') }} + CI_WEB_SHELL_SOURCE_HASH: ${{ hashFiles('nix/web-shell.nix', 'nix/web-shell-inputs.json', 'scripts/ci-web-shell', 'scripts/ci-web-shell-inputs.jq', 'scripts/prepare-pnpm-source-inputs.py') }} run: bash scripts/ci-web-shell key - name: Restore the pnpm store id: pnpm-store diff --git a/.github/workflows/perf-load.ts b/.github/workflows/perf-load.ts index 017f774980a..5e72dc48a6b 100644 --- a/.github/workflows/perf-load.ts +++ b/.github/workflows/perf-load.ts @@ -127,7 +127,7 @@ python3 scripts/ci-perf-cache pack`, ], }, // The same hog run on a macOS runner, under `taskpolicy -b`: the hog's priority on a Mac. It also - // keeps the compiled test binary, which is what runs on a fleet Mac (builds happen on hetz2 and + // keeps the compiled test binary, which is what runs on a fleet Mac (builds happen on a build host and // CI only). It builds cold and keeps no cache. 'perf-load-hog-macos': { 'runs-on': macosRunner, diff --git a/.github/workflows/perf.yml b/.github/workflows/perf.yml index cfbfb1a0470..0d835b0361c 100644 --- a/.github/workflows/perf.yml +++ b/.github/workflows/perf.yml @@ -153,8 +153,21 @@ jobs: export GIT_CONFIG_VALUE_0 fi head=$(git rev-parse HEAD) - rm -f "$(git rev-parse --git-path shallow)" - git fetch --quiet --refetch --no-tags origin '+refs/heads/*:refs/remotes/origin/*' '+refs/tags/*:refs/tags/*' '^refs/tags/evidence/*' "$head" + shallow=$(git rev-parse --git-path shallow) + errors=$(mktemp) + fetch() { + status=0 + git fetch --quiet --refetch --no-tags origin '+refs/heads/*:refs/remotes/origin/*' '+refs/tags/*:refs/tags/*' '^refs/tags/evidence/*' "$head" 2> "$errors" || status=$? + cat "$errors" >&2 + return "$status" + } + transport() { grep -Eq 'HTTP (408|429|5[0-9][0-9])|RPC failed|remote end hung up|early EOF|Connection (timed out|reset)|Operation timed out' "$errors"; } + [ ! -f "$shallow" ] || mv "$shallow" "$shallow.full-fetch" + if ! fetch && ! { transport && echo 'full history fetch: transport failure; trying once more' >&2 && fetch; }; then + [ ! -f "$shallow.full-fetch" ] || mv "$shallow.full-fetch" "$shallow" + exit 1 + fi + rm -f "$shallow.full-fetch" git fsck --connectivity-only --no-dangling --no-progress - name: Name compatible snapshot recipes env: @@ -272,8 +285,21 @@ jobs: export GIT_CONFIG_VALUE_0 fi head=$(git rev-parse HEAD) - rm -f "$(git rev-parse --git-path shallow)" - git fetch --quiet --refetch --no-tags origin '+refs/heads/*:refs/remotes/origin/*' '+refs/tags/*:refs/tags/*' '^refs/tags/evidence/*' "$head" + shallow=$(git rev-parse --git-path shallow) + errors=$(mktemp) + fetch() { + status=0 + git fetch --quiet --refetch --no-tags origin '+refs/heads/*:refs/remotes/origin/*' '+refs/tags/*:refs/tags/*' '^refs/tags/evidence/*' "$head" 2> "$errors" || status=$? + cat "$errors" >&2 + return "$status" + } + transport() { grep -Eq 'HTTP (408|429|5[0-9][0-9])|RPC failed|remote end hung up|early EOF|Connection (timed out|reset)|Operation timed out' "$errors"; } + [ ! -f "$shallow" ] || mv "$shallow" "$shallow.full-fetch" + if ! fetch && ! { transport && echo 'full history fetch: transport failure; trying once more' >&2 && fetch; }; then + [ ! -f "$shallow.full-fetch" ] || mv "$shallow.full-fetch" "$shallow" + exit 1 + fi + rm -f "$shallow.full-fetch" git fsck --connectivity-only --no-dangling --no-progress - name: Name compatible snapshot recipes env: @@ -348,8 +374,21 @@ jobs: export GIT_CONFIG_VALUE_0 fi head=$(git rev-parse HEAD) - rm -f "$(git rev-parse --git-path shallow)" - git fetch --quiet --refetch --no-tags origin '+refs/heads/*:refs/remotes/origin/*' '+refs/tags/*:refs/tags/*' '^refs/tags/evidence/*' "$head" + shallow=$(git rev-parse --git-path shallow) + errors=$(mktemp) + fetch() { + status=0 + git fetch --quiet --refetch --no-tags origin '+refs/heads/*:refs/remotes/origin/*' '+refs/tags/*:refs/tags/*' '^refs/tags/evidence/*' "$head" 2> "$errors" || status=$? + cat "$errors" >&2 + return "$status" + } + transport() { grep -Eq 'HTTP (408|429|5[0-9][0-9])|RPC failed|remote end hung up|early EOF|Connection (timed out|reset)|Operation timed out' "$errors"; } + [ ! -f "$shallow" ] || mv "$shallow" "$shallow.full-fetch" + if ! fetch && ! { transport && echo 'full history fetch: transport failure; trying once more' >&2 && fetch; }; then + [ ! -f "$shallow.full-fetch" ] || mv "$shallow.full-fetch" "$shallow" + exit 1 + fi + rm -f "$shallow.full-fetch" git fsck --connectivity-only --no-dangling --no-progress - name: Install Nix uses: DeterminateSystems/determinate-nix-action@v3 diff --git a/.github/workflows/public-repo.yml b/.github/workflows/public-repo.yml index 5b778e27c93..9fd69648d9b 100644 --- a/.github/workflows/public-repo.yml +++ b/.github/workflows/public-repo.yml @@ -48,8 +48,21 @@ jobs: export GIT_CONFIG_VALUE_0 fi head=$(git rev-parse HEAD) - rm -f "$(git rev-parse --git-path shallow)" - git fetch --quiet --refetch --no-tags origin '+refs/heads/*:refs/remotes/origin/*' '+refs/tags/*:refs/tags/*' '^refs/tags/evidence/*' "$head" + shallow=$(git rev-parse --git-path shallow) + errors=$(mktemp) + fetch() { + status=0 + git fetch --quiet --refetch --no-tags origin '+refs/heads/*:refs/remotes/origin/*' '+refs/tags/*:refs/tags/*' '^refs/tags/evidence/*' "$head" 2> "$errors" || status=$? + cat "$errors" >&2 + return "$status" + } + transport() { grep -Eq 'HTTP (408|429|5[0-9][0-9])|RPC failed|remote end hung up|early EOF|Connection (timed out|reset)|Operation timed out' "$errors"; } + [ ! -f "$shallow" ] || mv "$shallow" "$shallow.full-fetch" + if ! fetch && ! { transport && echo 'full history fetch: transport failure; trying once more' >&2 && fetch; }; then + [ ! -f "$shallow.full-fetch" ] || mv "$shallow.full-fetch" "$shallow" + exit 1 + fi + rm -f "$shallow.full-fetch" git fsck --connectivity-only --no-dangling --no-progress - name: Test changelog requirement policy run: node --test apps/fractal-web/scripts/ci/changelog-policy.unit.test.mjs diff --git a/.github/workflows/release-daily.yml b/.github/workflows/release-daily.yml index 09633056f0e..3aca655f38c 100644 --- a/.github/workflows/release-daily.yml +++ b/.github/workflows/release-daily.yml @@ -46,8 +46,21 @@ jobs: export GIT_CONFIG_VALUE_0 fi head=$(git rev-parse HEAD) - rm -f "$(git rev-parse --git-path shallow)" - git fetch --quiet --refetch --no-tags origin '+refs/heads/*:refs/remotes/origin/*' '+refs/tags/*:refs/tags/*' '^refs/tags/evidence/*' "$head" + shallow=$(git rev-parse --git-path shallow) + errors=$(mktemp) + fetch() { + status=0 + git fetch --quiet --refetch --no-tags origin '+refs/heads/*:refs/remotes/origin/*' '+refs/tags/*:refs/tags/*' '^refs/tags/evidence/*' "$head" 2> "$errors" || status=$? + cat "$errors" >&2 + return "$status" + } + transport() { grep -Eq 'HTTP (408|429|5[0-9][0-9])|RPC failed|remote end hung up|early EOF|Connection (timed out|reset)|Operation timed out' "$errors"; } + [ ! -f "$shallow" ] || mv "$shallow" "$shallow.full-fetch" + if ! fetch && ! { transport && echo 'full history fetch: transport failure; trying once more' >&2 && fetch; }; then + [ ! -f "$shallow.full-fetch" ] || mv "$shallow.full-fetch" "$shallow" + exit 1 + fi + rm -f "$shallow.full-fetch" git fsck --connectivity-only --no-dangling --no-progress - name: Publish the verified archives of main when it changed run: 'scripts/release-fractal-daily ${TAG:+--tag "$TAG"}' diff --git a/.github/workflows/release-fractal.yml b/.github/workflows/release-fractal.yml index a60c9703ce3..06d67d0cba6 100644 --- a/.github/workflows/release-fractal.yml +++ b/.github/workflows/release-fractal.yml @@ -417,7 +417,7 @@ jobs: env: MEGAREPO_STORE: "${{ runner.temp }}/megarepo" run: | - nix run "github:overengineeringstudio/effect-utils/$(jq -r '.nodes["effect-utils"].locked.rev' flake.lock)#megarepo" -- apply --worktree-mode commit --git-protocol https + python3 scripts/prepare-pnpm-source-inputs.py apply printf 'MEGAREPO_STORE=%s\n' "$MEGAREPO_STORE" >> "$GITHUB_ENV" - name: Build the offline embed and capture four posters env: @@ -661,8 +661,21 @@ jobs: export GIT_CONFIG_VALUE_0 fi head=$(git rev-parse HEAD) - rm -f "$(git rev-parse --git-path shallow)" - git fetch --quiet --refetch --no-tags origin '+refs/heads/*:refs/remotes/origin/*' '+refs/tags/*:refs/tags/*' '^refs/tags/evidence/*' "$head" + shallow=$(git rev-parse --git-path shallow) + errors=$(mktemp) + fetch() { + status=0 + git fetch --quiet --refetch --no-tags origin '+refs/heads/*:refs/remotes/origin/*' '+refs/tags/*:refs/tags/*' '^refs/tags/evidence/*' "$head" 2> "$errors" || status=$? + cat "$errors" >&2 + return "$status" + } + transport() { grep -Eq 'HTTP (408|429|5[0-9][0-9])|RPC failed|remote end hung up|early EOF|Connection (timed out|reset)|Operation timed out' "$errors"; } + [ ! -f "$shallow" ] || mv "$shallow" "$shallow.full-fetch" + if ! fetch && ! { transport && echo 'full history fetch: transport failure; trying once more' >&2 && fetch; }; then + [ ! -f "$shallow.full-fetch" ] || mv "$shallow.full-fetch" "$shallow" + exit 1 + fi + rm -f "$shallow.full-fetch" git fsck --connectivity-only --no-dangling --no-progress - uses: actions/download-artifact@v4 with: @@ -702,8 +715,21 @@ jobs: export GIT_CONFIG_VALUE_0 fi head=$(git rev-parse HEAD) - rm -f "$(git rev-parse --git-path shallow)" - git fetch --quiet --refetch --no-tags origin '+refs/heads/*:refs/remotes/origin/*' '+refs/tags/*:refs/tags/*' '^refs/tags/evidence/*' "$head" + shallow=$(git rev-parse --git-path shallow) + errors=$(mktemp) + fetch() { + status=0 + git fetch --quiet --refetch --no-tags origin '+refs/heads/*:refs/remotes/origin/*' '+refs/tags/*:refs/tags/*' '^refs/tags/evidence/*' "$head" 2> "$errors" || status=$? + cat "$errors" >&2 + return "$status" + } + transport() { grep -Eq 'HTTP (408|429|5[0-9][0-9])|RPC failed|remote end hung up|early EOF|Connection (timed out|reset)|Operation timed out' "$errors"; } + [ ! -f "$shallow" ] || mv "$shallow" "$shallow.full-fetch" + if ! fetch && ! { transport && echo 'full history fetch: transport failure; trying once more' >&2 && fetch; }; then + [ ! -f "$shallow.full-fetch" ] || mv "$shallow.full-fetch" "$shallow" + exit 1 + fi + rm -f "$shallow.full-fetch" git fsck --connectivity-only --no-dangling --no-progress - uses: actions/download-artifact@v4 with: diff --git a/.github/workflows/storybook-deploy.yml b/.github/workflows/storybook-deploy.yml index 6546f536691..451b8e5b6de 100644 --- a/.github/workflows/storybook-deploy.yml +++ b/.github/workflows/storybook-deploy.yml @@ -60,7 +60,7 @@ jobs: env: MEGAREPO_STORE: "${{ runner.temp }}/megarepo" run: | - nix run "github:overengineeringstudio/effect-utils/$(jq -r '.nodes["effect-utils"].locked.rev' flake.lock)#megarepo" -- apply --worktree-mode commit --git-protocol https + python3 scripts/prepare-pnpm-source-inputs.py apply printf 'MEGAREPO_STORE=%s\n' "$MEGAREPO_STORE" >> "$GITHUB_ENV" if: steps.credentials.outputs.enabled == 'true' - name: Install the standalone deployment stack diff --git a/.github/workflows/workspace-ci.ts b/.github/workflows/workspace-ci.ts index d0864ee2a61..8dcf202de87 100644 --- a/.github/workflows/workspace-ci.ts +++ b/.github/workflows/workspace-ci.ts @@ -209,8 +209,12 @@ export const afterPickRunner = { needs: [pickRunnerJobId], if: '${{ !cancelled() * record up to the GC cap, and `fetch-depth: 0` fetches every tag: 5,000 of them added 81.5 MiB and 16 s to a full fetch * (measured 2026-10-19), mostly source objects of tested commits that no branch holds. So check out one commit, then fetch the * rest with a negative refspec (git 2.29 or later). Deepening the shallow clone (`--unshallow`) took 21 s against 11 s for a - * full fetch (2026-10-20, from hetz2): the server cannot reuse its packs for it. So the shallow marker goes and the history is + * full fetch (2026-10-20, from a build host): the server cannot reuse its packs for it. So the shallow marker goes and the history is * fetched as a fresh clone would (`--refetch`, git 2.36 or later), then checked: the whole history is here or the step fails. + * The marker is only moved aside: a fetch that fails installs no pack and updates no ref, so putting the marker back leaves the + * one-commit checkout exactly as it was, never a checkout without the marker whose parents are missing (a 408 did that to a merge + * group on 2026-11-03). git does not retry a dropped transfer, and GitHub answers a slow pack negotiation with HTTP 408, so a + * transport failure, and only one, is tried once more; any other failure ends the step at once. * The job token is used only when checkout did not persist its own. */ export const fullHistoryFetchScript = `if [ -z "$(git config --local --get-regexp '^http\\..*\\.extraheader$' || true)" ]; then @@ -219,8 +223,21 @@ export const fullHistoryFetchScript = `if [ -z "$(git config --local --get-regex export GIT_CONFIG_VALUE_0 fi head=$(git rev-parse HEAD) -rm -f "$(git rev-parse --git-path shallow)" -git fetch --quiet --refetch --no-tags origin '+refs/heads/*:refs/remotes/origin/*' '+refs/tags/*:refs/tags/*' '^refs/tags/evidence/*' "$head" +shallow=$(git rev-parse --git-path shallow) +errors=$(mktemp) +fetch() { + status=0 + git fetch --quiet --refetch --no-tags origin '+refs/heads/*:refs/remotes/origin/*' '+refs/tags/*:refs/tags/*' '^refs/tags/evidence/*' "$head" 2> "$errors" || status=$? + cat "$errors" >&2 + return "$status" +} +transport() { grep -Eq 'HTTP (408|429|5[0-9][0-9])|RPC failed|remote end hung up|early EOF|Connection (timed out|reset)|Operation timed out' "$errors"; } +[ ! -f "$shallow" ] || mv "$shallow" "$shallow.full-fetch" +if ! fetch && ! { transport && echo 'full history fetch: transport failure; trying once more' >&2 && fetch; }; then + [ ! -f "$shallow.full-fetch" ] || mv "$shallow.full-fetch" "$shallow" + exit 1 +fi +rm -f "$shallow.full-fetch" git fsck --connectivity-only --no-dangling --no-progress` /** @@ -457,7 +474,7 @@ export const testArchiveProducer = { } /** - * isolation-vm's work: the retained st2 controls and current-runtime scope proofs need a real systemd user manager, which the + * isolation-vm's work: the current-runtime scope proofs need a real systemd user manager, which the * runner image lacks, so a NixOS VM runs the shared producer's prebuilt test binary and compiles nothing; a * second VM runs the sekrets gateway. Workspace CI and the daily long-lane run share these steps. */ @@ -839,7 +856,7 @@ export const pnpmStoreEnv = { pnpm_config_store_dir: '${{ runner.temp }}/pnpm-st export const megarepoApplyStep = { name: 'Materialize megarepo members', env: { MEGAREPO_STORE: '${{ runner.temp }}/megarepo' }, - run: `nix run "github:overengineeringstudio/effect-utils/$(jq -r '.nodes["effect-utils"].locked.rev' flake.lock)#megarepo" -- apply --worktree-mode commit --git-protocol https + run: `python3 scripts/prepare-pnpm-source-inputs.py apply printf 'MEGAREPO_STORE=%s\\n' "$MEGAREPO_STORE" >> "$GITHUB_ENV"`, } as const @@ -868,7 +885,7 @@ export const fractalWebShellKeyStep = { id: 'web-shell-key', shell: 'bash', env: { - CI_WEB_SHELL_SOURCE_HASH: "${{ hashFiles('nix/web-shell.nix', 'nix/web-shell-inputs.json', 'scripts/ci-web-shell', 'scripts/ci-web-shell-inputs.jq') }}", + CI_WEB_SHELL_SOURCE_HASH: "${{ hashFiles('nix/web-shell.nix', 'nix/web-shell-inputs.json', 'scripts/ci-web-shell', 'scripts/ci-web-shell-inputs.jq', 'scripts/prepare-pnpm-source-inputs.py') }}", }, run: 'bash scripts/ci-web-shell key', } as const diff --git a/.gitignore b/.gitignore index d54d7b50d93..41cb2b6585c 100644 --- a/.gitignore +++ b/.gitignore @@ -43,6 +43,7 @@ packages/fractal-sdk/node_modules/ # Derived from apps/fractal-web/.changelog/ by scripts/changelog-registry.mjs; never committed (it would conflict). /apps/fractal-web/.changelog/registry.generated.ts +/apps/fractal-web/.changelog/registry.generated.ts.*.tmp # Local Alchemy cache/state metadata; remote Cloudflare state is authoritative. /infra/storybook/.alchemy/ diff --git a/.st/evidence-skip-policy.json b/.st/evidence-skip-policy.json index b63111f7040..6557636ad10 100644 --- a/.st/evidence-skip-policy.json +++ b/.st/evidence-skip-policy.json @@ -22,7 +22,6 @@ "clients", "components", "crates", - "docs/st2", "docs/st3", "docs/vrs", "evals", @@ -34,9 +33,7 @@ "plugins", "scripts", "slo", - "src", "templates", - "tests", "vendor", "wit" ], @@ -192,23 +189,21 @@ "claude-channel/plugins/st2-channel/.mcp.json", "clients/swift/FractalClient/Sources/FractalClient", "clients/typescript/fractal-client", + "clients/typescript/fractal-declarations/fixtures", "clients/typescript/fractal-scenarios/recordings/fleet-mid-refactor-proof.script.json", "clients/typescript/fractal-scenarios/scripts/recording-script.schema.json", "clients/typescript/fractal-views/tests/fixtures", "clippy.toml", "components", "crates", - "docs/st2/invariants.md", "docs/st3/action-coverage.json", "docs/st3/client-v0", "docs/st3/invariants.md", "docs/st3/onboarding-guide.md", "docs/st3/operational-state/cli-commands.json", "docs/st3/schema.md", - "docs/vrs/requirements.md", "examples/client-tui", "examples/fractal", - "examples/native/agent-codex.kdl", "fixtures", "flake.lock", "flake.nix", @@ -228,7 +223,6 @@ "scripts/ci-mail-redelivery-canaries", "scripts/ci-nextest", "scripts/ci-nextest-flaky", - "scripts/ci-nextest-targets", "scripts/ci-perf-cache", "scripts/ci-rustc-wrapper", "scripts/ci-rustc-wrapper-test", @@ -268,8 +262,6 @@ "scripts/script_process.py", "scripts/sqlite_large_values.py", "slo/targets.toml", - "src", - "tests", "vendor/libghostty-vt-sys", "wit" ], @@ -383,6 +375,7 @@ "claude-channel/.claude-plugin/marketplace.json", "claude-channel/plugins/st2-channel/.claude-plugin/plugin.json", "claude-channel/plugins/st2-channel/.mcp.json", + "clients/typescript/fractal-declarations/fixtures", "clients/typescript/fractal-scenarios/recordings/fleet-mid-refactor-proof.script.json", "clients/typescript/fractal-scenarios/scripts/recording-script.schema.json", "components/github-issue/Cargo.toml", @@ -436,17 +429,14 @@ "crates/fractal-wire", "crates/pty-launch-environment.rs", "crates/smallclaims", - "docs/st2/invariants.md", "docs/st3/action-coverage.json", "docs/st3/client-v0", "docs/st3/invariants.md", "docs/st3/onboarding-guide.md", "docs/st3/operational-state/cli-commands.json", "docs/st3/schema.md", - "docs/vrs/requirements.md", "examples/client-tui", "examples/fractal", - "examples/native/agent-codex.kdl", "fixtures", "flake.lock", "flake.nix", @@ -466,7 +456,6 @@ "scripts/ci-mail-redelivery-canaries", "scripts/ci-nextest", "scripts/ci-nextest-flaky", - "scripts/ci-nextest-targets", "scripts/ci-perf-cache", "scripts/ci-rustc-wrapper", "scripts/ci-rustc-wrapper-test", @@ -612,6 +601,7 @@ "apps/fractal-web/src/folders/core.mts", "apps/ios", "clients/typescript", + "crates/fractal-client-codegen/templates/declarations.ts.in", "crates/fractal-tui/tests/fixtures/quality-hour.json", "docs/st3/client-v0/fixtures", "fixtures/clients", @@ -892,8 +882,7 @@ "packages/fractal-ui/.gitignore", "pnpm-lock.yaml", "pnpm-workspace.yaml", - "scripts", - "src/main.rs" + "scripts" ], "kinds": { "observed": { @@ -1068,7 +1057,8 @@ "scripts/ci-typescript-client", "scripts/ci-web-shell", "scripts/ci-web-shell-inputs.jq", - "scripts/ci-web-shell-test" + "scripts/ci-web-shell-test", + "scripts/prepare-pnpm-source-inputs.py" ], "kinds": { "wrapper": { diff --git a/.st/lane-bursts.json b/.st/lane-bursts.json new file mode 100644 index 00000000000..d1f6cbbf39b --- /dev/null +++ b/.st/lane-bursts.json @@ -0,0 +1,5 @@ +{ + "version": 1, + "about": "What scripts/fractal-test asks its seat's lane to be raised to for one run (docs/st3/seat-lanes.md, Bursts), by what the run builds. Set from measurements: a full run's Rust checks reached 24G in an 8-core lane (2026-11-02, on a build host); the per-seat sampler (2026-11-03) refines it.", + "build": {"cores": 8, "memory": "24G"} +} diff --git a/.st/required-lanes.json b/.st/required-lanes.json index 1df2723cb35..d2299c1e99d 100644 --- a/.st/required-lanes.json +++ b/.st/required-lanes.json @@ -38,7 +38,8 @@ "scripts/ci-local-web", "scripts/ci-web-shell", "scripts/ci-web-shell-inputs.jq", - "scripts/ci-web-shell-test" + "scripts/ci-web-shell-test", + "scripts/prepare-pnpm-source-inputs.py" ], "tree_inputs": [ ".buckconfig", @@ -90,7 +91,8 @@ "scripts/ci-typescript-client", "scripts/ci-web-shell", "scripts/ci-web-shell-inputs.jq", - "scripts/ci-web-shell-test" + "scripts/ci-web-shell-test", + "scripts/prepare-pnpm-source-inputs.py" ], "ci_inputs": [ ".github/workflows/fleet.yml", @@ -119,6 +121,7 @@ "apps/ios/package.json", "buck2/toolchains/BUCK", "clients/typescript/fractal-client/package.json", + "clients/typescript/fractal-declarations/package.json", "clients/typescript/fractal-scenarios/package.json", "clients/typescript/fractal-views/package.json", "flake.lock", @@ -172,8 +175,7 @@ "packages/fractal-ui/.gitignore", "pnpm-lock.yaml", "pnpm-workspace.yaml", - "scripts", - "src/main.rs" + "scripts" ] }, "linux-gate": { @@ -211,7 +213,6 @@ "scripts/ci-mail-redelivery-canaries", "scripts/ci-nextest", "scripts/ci-nextest-flaky", - "scripts/ci-nextest-targets", "scripts/ci-perf-cache", "scripts/ci-rustc-wrapper", "scripts/ci-rustc-wrapper-test", @@ -262,7 +263,6 @@ "clients", "components", "crates", - "docs/st2", "docs/st3", "docs/vrs", "evals", @@ -274,9 +274,7 @@ "plugins", "scripts", "slo", - "src", "templates", - "tests", "vendor", "wit" ], @@ -312,7 +310,6 @@ "scripts/ci-mail-redelivery-canaries", "scripts/ci-nextest", "scripts/ci-nextest-flaky", - "scripts/ci-nextest-targets", "scripts/ci-perf-cache", "scripts/ci-rust-test-result", "scripts/ci-rustc-wrapper", @@ -364,7 +361,6 @@ "clients", "components", "crates", - "docs/st2", "docs/st3", "docs/vrs", "evals", @@ -376,9 +372,7 @@ "plugins", "scripts", "slo", - "src", "templates", - "tests", "vendor", "wit" ], @@ -404,6 +398,7 @@ "definition_paths": [ "apps/ios/package.json", "clients/typescript/fractal-client/package.json", + "clients/typescript/fractal-declarations/package.json", "clients/typescript/fractal-views/package.json", "flake.lock", "flake.nix", @@ -415,7 +410,8 @@ "pnpm-source-inputs.json", "pnpm-workspace.yaml", "scripts/ci-nix-fixed-outputs", - "scripts/ci-typescript-client" + "scripts/ci-typescript-client", + "scripts/prepare-pnpm-source-inputs.py" ], "tree_inputs": [ "apps/fractal-web/package.json", @@ -423,6 +419,7 @@ "apps/fractal-web/src/folders/core.mts", "apps/ios", "clients/typescript", + "crates/fractal-client-codegen/templates/declarations.ts.in", "crates/fractal-tui/tests/fixtures/quality-hour.json", "docs/st3/client-v0/fixtures", "fixtures/clients", @@ -499,7 +496,6 @@ "scripts/ci-mail-redelivery-canaries", "scripts/ci-nextest", "scripts/ci-nextest-flaky", - "scripts/ci-nextest-targets", "scripts/ci-perf-cache", "scripts/ci-rustc-wrapper", "scripts/ci-rustc-wrapper-test", @@ -552,23 +548,21 @@ "claude-channel/plugins/st2-channel/.mcp.json", "clients/swift/FractalClient/Sources/FractalClient", "clients/typescript/fractal-client", + "clients/typescript/fractal-declarations/fixtures", "clients/typescript/fractal-scenarios/recordings/fleet-mid-refactor-proof.script.json", "clients/typescript/fractal-scenarios/scripts/recording-script.schema.json", "clients/typescript/fractal-views/tests/fixtures", "clippy.toml", "components", "crates", - "docs/st2/invariants.md", "docs/st3/action-coverage.json", "docs/st3/client-v0", "docs/st3/invariants.md", "docs/st3/onboarding-guide.md", "docs/st3/operational-state/cli-commands.json", "docs/st3/schema.md", - "docs/vrs/requirements.md", "examples/client-tui", "examples/fractal", - "examples/native/agent-codex.kdl", "fixtures", "flake.lock", "flake.nix", @@ -588,7 +582,6 @@ "scripts/ci-mail-redelivery-canaries", "scripts/ci-nextest", "scripts/ci-nextest-flaky", - "scripts/ci-nextest-targets", "scripts/ci-perf-cache", "scripts/ci-rustc-wrapper", "scripts/ci-rustc-wrapper-test", @@ -628,8 +621,6 @@ "scripts/script_process.py", "scripts/sqlite_large_values.py", "slo/targets.toml", - "src", - "tests", "vendor/libghostty-vt-sys", "wit" ] @@ -666,7 +657,6 @@ "scripts/ci-mail-redelivery-canaries", "scripts/ci-nextest", "scripts/ci-nextest-flaky", - "scripts/ci-nextest-targets", "scripts/ci-perf-cache", "scripts/ci-rustc-wrapper", "scripts/ci-rustc-wrapper-test", @@ -714,6 +704,7 @@ "claude-channel/.claude-plugin/marketplace.json", "claude-channel/plugins/st2-channel/.claude-plugin/plugin.json", "claude-channel/plugins/st2-channel/.mcp.json", + "clients/typescript/fractal-declarations/fixtures", "clients/typescript/fractal-scenarios/recordings/fleet-mid-refactor-proof.script.json", "clients/typescript/fractal-scenarios/scripts/recording-script.schema.json", "components/github-issue/Cargo.toml", @@ -767,17 +758,14 @@ "crates/fractal-wire", "crates/pty-launch-environment.rs", "crates/smallclaims", - "docs/st2/invariants.md", "docs/st3/action-coverage.json", "docs/st3/client-v0", "docs/st3/invariants.md", "docs/st3/onboarding-guide.md", "docs/st3/operational-state/cli-commands.json", "docs/st3/schema.md", - "docs/vrs/requirements.md", "examples/client-tui", "examples/fractal", - "examples/native/agent-codex.kdl", "fixtures", "flake.lock", "flake.nix", @@ -797,7 +785,6 @@ "scripts/ci-mail-redelivery-canaries", "scripts/ci-nextest", "scripts/ci-nextest-flaky", - "scripts/ci-nextest-targets", "scripts/ci-perf-cache", "scripts/ci-rustc-wrapper", "scripts/ci-rustc-wrapper-test", diff --git a/.st/rust-test-inputs.json b/.st/rust-test-inputs.json index 68cbdd693b6..dcfe33f7109 100644 --- a/.st/rust-test-inputs.json +++ b/.st/rust-test-inputs.json @@ -11,7 +11,6 @@ "scripts/ci-linux", "scripts/ci-nextest", "scripts/ci-nextest-flaky", - "scripts/ci-nextest-targets", "scripts/ci-rustc-wrapper", "scripts/rust-test-keys", "scripts/fractal_test_process.py" @@ -45,8 +44,7 @@ "examples/fractal", "fixtures", "scripts", - "slo", - "src/main.rs" + "slo" ] }, "fractal-api-client-v0": { @@ -112,11 +110,15 @@ "clients/typescript/fractal-client/Client.generated.ts", "clients/typescript/fractal-client/Models.generated.ts", "clients/typescript/fractal-client/Schema.generated.ts", + "clients/typescript/fractal-declarations/fixtures/daily-review.kdl", + "clients/typescript/fractal-declarations/index.ts", "crates/fractal-client/src/contract.rs", "crates/fractal-client/src/generated.rs", "crates/fractal-client/src/lib.rs", + "crates/fractal/tests/fixtures/validate-revisions.kdl", "docs/st3/client-v0/schemas/client-v0.schema.json", - "docs/st3/client-v0/schemas/operations.json" + "docs/st3/client-v0/schemas/operations.json", + "fixtures/clients/person-pins-v2.json" ] }, "fractal-client-tui": { @@ -140,8 +142,7 @@ "fixtures/clients/transcripts/native-omp-run.json", "fixtures/clients/transcripts/noise.json", "fixtures/clients/transcripts/omp-parity.expected.json", - "fixtures/clients/transcripts/omp-parity.json", - "src" + "fixtures/clients/transcripts/omp-parity.json" ] }, "fractal-daemon": { @@ -213,8 +214,7 @@ ".github/workflows/daily-long.yml", ".github/workflows/fleet.yml", ".github/workflows/perf.yml", - "scripts/ci-test-paths.rs", - "src/main.rs" + "scripts/ci-test-paths.rs" ] }, "fractal-github-issue-component": { @@ -239,19 +239,6 @@ "crates/fractal-daemon/hooks/pi-channel.ts" ] }, - "fractal-legacy-runner": { - "extra": [ - "AGENTS.md", - "crates", - "docs/st2/invariants.md", - "docs/vrs", - "examples/native/agent-claude.kdl", - "examples/native/agent-codex.kdl", - "scripts/ci-test-paths.rs", - "scripts/conflict-census-test", - "templates/bus.st2.md" - ] - }, "fractal-migrate": { "extra": [] }, @@ -344,6 +331,7 @@ "fractal-store": { "extra": [ ".github/workflows/perf.yml", + "AGENTS.md", "crates/fractal-daemon/fixtures/native-records/omp-parity.jsonl", "crates/fractal-daemon/fixtures/omp-admin-events.jsonl", "crates/fractal-daemon/fixtures/omp-native-events.jsonl", @@ -380,6 +368,12 @@ "docs/st3/client-v0/fixtures/terminal-screen.json", "fixtures/clients", "fixtures/scenarios/fleet-mid-refactor", + "fixtures/scenarios/fleet-mid-refactor/attention.json", + "fixtures/scenarios/fleet-mid-refactor/conversation.json", + "fixtures/scenarios/fleet-mid-refactor/details.json", + "fixtures/scenarios/fleet-mid-refactor/roster.json", + "fixtures/scenarios/fleet-mid-refactor/sync.json", + "fixtures/scenarios/fleet-mid-refactor/terminal.json", "scripts/ci-test-paths.rs", "slo" ] diff --git a/CLAUDE.md b/CLAUDE.md index 9db6baf386c..74e31fca967 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -2,8 +2,8 @@ Read [README.md](README.md) before changing the product surface. -Read [docs/st2/invariants.md](docs/st2/invariants.md) before changing st2 lifecycle, messaging, teardown, or presence. -Each st2 invariant names its proof. Preserve that proof while st2 remains in this repository. +Read [native runtime invariants](docs/st3/invariants.md) before changing native delivery, +harness state, context, diagnostics, stop ordering or status-line chaining. Preserve each row's proof. Read [docs/st3/README.md](docs/st3/README.md) before changing Fractal claims, missions, replication, runtime ownership, or recovery. `st skill` prints how an agent seat uses Fractal at runtime. diff --git a/Cargo.lock b/Cargo.lock index 1f7df256fa3..b613e4b9de0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1661,6 +1661,7 @@ dependencies = [ "fractal-api-mailbox", "fractal-drivers", "fractal-runtime", + "fractal-schema", "futures-util", "hex", "http-body-util", @@ -1782,6 +1783,7 @@ name = "fractal-client-codegen" version = "0.1.0" dependencies = [ "anyhow", + "fractal-store", "serde_json", "sha2 0.10.9", ] @@ -2156,30 +2158,6 @@ dependencies = [ "walkdir", ] -[[package]] -name = "fractal-legacy-runner" -version = "0.1.0" -dependencies = [ - "agent-spec", - "anyhow", - "clap", - "clap_complete", - "fractal-build-stamp", - "fractal-drivers", - "fractal-resource-protocol", - "fractal-resource-providers", - "fractal-resource-wasip2", - "fractal-wire", - "kdl", - "libc", - "parking_lot", - "serde", - "serde_json", - "sha2 0.10.9", - "tempfile", - "tracing", -] - [[package]] name = "fractal-migrate" version = "0.1.0" @@ -5877,6 +5855,7 @@ dependencies = [ "serde", "serde_json", "sha2 0.10.9", + "smallclaims", "tempfile", "tokio", "toml 0.9.12+spec-1.1.0", diff --git a/Cargo.toml b/Cargo.toml index ec4249c5537..270cdb20cbe 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,7 +1,6 @@ -# The root remains a package while the legacy fractal-legacy-runner runner is present. -# Shared release identity lives in workspace.package so retiring fractal-legacy-runner does not -# change the current binaries' release stamps. +# Release identity is shared by the current workspace packages. [workspace] +resolver = "3" members = [ "examples/client-tui", "components/github-issue", @@ -58,7 +57,6 @@ members = [ ] default-members = [ "examples/client-tui", - ".", "crates/agent-spec", "crates/smallclaims", "crates/fractal-build-stamp", @@ -187,80 +185,3 @@ opt-level = 3 debug = 0 debug-assertions = true overflow-checks = true - -[package] -name = "fractal-legacy-runner" -version.workspace = true -edition = "2024" -description = "Harness-agnostic runner: reconcile a unified catalog+inbox folder of rendered agent specs, keep their ptys running, and deliver messages by moving inbox files." -license = "MIT" -# Every file in tests/ is a module of one test binary; see tests/integration.rs. -autotests = false - -[[bin]] -name = "st2" -path = "src/main.rs" - -[lib] -name = "fractal_legacy_runner" -path = "src/lib.rs" - -[[test]] -name = "integration" -path = "tests/integration.rs" - -# These set process-wide environment variables, so each stays a test binary of its own. -[[test]] -name = "agent_resource" -path = "tests/agent_resource.rs" - -[[test]] -name = "driver_expansion" -path = "tests/driver_expansion.rs" - -[[test]] -name = "event_e2e" -path = "tests/event_e2e.rs" - -[[test]] -name = "resource_profile_supervisor_e2e" -path = "tests/resource_profile_supervisor_e2e.rs" - -[dependencies] -# Sync by design: the runner is I/O-light (shell-outs to `pty` plus a -# folder watch + a sleep), so no tokio. Deps grow per milestone; M0 needs only parse + CLI. -agent-spec = { path = "crates/agent-spec" } -# The build identity: only the binary packages depend on it (see crates/fractal-build-stamp). -fractal-build-stamp = { path = "crates/fractal-build-stamp" } -fractal-drivers = { path = "crates/fractal-drivers" } -anyhow = "1" -clap = { version = "4", features = ["derive"] } -clap_complete = "4" -kdl = "6" -libc = "0.2" -serde = { version = "1", features = ["derive"] } -serde_json = "1" -sha2 = "0.10" -fractal-resource-protocol = { package = "fractal-resource-protocol", path = "crates/fractal-resource-protocol" } -fractal-resource-providers = { path = "crates/fractal-resource-providers", optional = true } -fractal-resource-wasip2 = { path = "crates/fractal-resource-wasip2", optional = true, features = ["runtime"] } -fractal-wire = { package = "fractal-wire", path = "crates/fractal-wire" } -tempfile = "3" -tracing = "0.1" - -[dev-dependencies] -libc = "0.2" -parking_lot = "0.12" - -[features] -# Sandbox wasm resource-profile resolvers (`agent-spec/profile_wasm`). Off by default: the -# runner binary carries no wasmtime unless a build opts in. -wasm-resolver = ["agent-spec/wasm-resolver", "fractal-drivers/wasm-resolver"] -# Typed Component Model resource providers. This is the sole feature that admits Wasmtime for -# observation; default members and the default fractal-legacy-runner binary remain Wasmtime-free. -wasip2-provider-runtime = [ - "wasm-resolver", - "dep:fractal-resource-providers", - "dep:fractal-resource-wasip2", - "fractal-drivers/wasip2-provider-runtime", -] diff --git a/README.md b/README.md index f9a3ffb5378..113140c6abf 100644 --- a/README.md +++ b/README.md @@ -25,8 +25,9 @@ fractal ``` Supported platforms are Linux x86_64 with glibc 2.35+ (Ubuntu 22.04+) and macOS Apple -Silicon with macOS 15+. The installer needs `curl`, `tar` and a SHA256 checker; macOS -also needs Python 3. Harness CLIs and their accounts are separate. If none is +Silicon with macOS 15+. The installer needs `curl`, `tar` and a SHA256 checker; +Python 3 is also required on macOS and when verifying an existing ownership receipt. +Harness CLIs and their accounts are separate. If none is installed, `fractal` still opens; install a supported harness and run `fractal setup` later. For Claude, first complete its native welcome, account or API-key confirmation and permission-bypass consent as the same user and profile. Successful auth-status or @@ -180,4 +181,3 @@ Run `pnpm check:fractal-coverage` in `bun apps/fractal-web/scripts/replay-transcript.ts DIRECTORY...` to audit saved timeline pages. [Contributing](docs/development.md#contributing) covers source builds, checks, and repository layout. -This repository also carries the previous generation; see [its guide](README.st2.md). diff --git a/README.st2.md b/README.st2.md deleted file mode 100644 index 3fefcfa64da..00000000000 --- a/README.st2.md +++ /dev/null @@ -1,774 +0,0 @@ -# st2 legacy guide - -This guide describes the st2 runtime during the Fractal migration. - -st2 runs a declarative network of Codex, Claude, and pi agents from one catalog. It owns process -reconciliation, native messages, normalized terminal DING delivery, presence, durable context, -workspace materialization, and explicit teardown. - -Canonical KDL is the authoring interface. Publish one explicit Agent Spec transactionally with -`st2 agent publish`; st2 does not compile human intent into declarations. - -## Product intent and implementation contract - -Read the [vision](docs/vrs/vision.md), [requirements](docs/vrs/requirements.md), and -[specification](docs/vrs/spec.md) before changing product behavior. Update `docs/vrs/spec.md` with -implementation changes. Alex must approve changes to `docs/vrs/vision.md` or -`docs/vrs/requirements.md`. - -## Install - -Prerequisites: - -- Rust and Cargo; -- `pty` on `PATH` with `pty run --unset-env` support; -- at least one supported harness on `PATH`: `codex`, `claude`, or `pi`; - (`codex` is admitted by an exact version allowlist; `pi` is not version-pinned, but the shipped pi - extension is type-checked against a pinned pi release by `nix flake check`); -- Git when a declaration materializes workspace files; -- Bash and `jq` on `PATH` when lifecycle hooks are enabled. - -From a checkout (`scripts/install` installs st2 together with st3 and the `st` symlink): - -```sh -cargo install --path . --locked -st2 --help -pty --help -st2 hooks install -st2 hooks verify -st2 claude-channel install -st2 claude-channel status -``` - -When upgrading, deploy and activate the compatible `pty` before this version of -`st2`. An older `pty` may silently ignore the unknown `--unset-env` option and -still launch the agent without persisting the removal. The initial environment -can therefore look correct while a later restart reintroduces the caller's -ambient value. The Nix input and development shell pin the compatible artifact; -Cargo installs rely on the operator to satisfy this runtime prerequisite. - -The standard catalog is: - -```text -${XDG_STATE_HOME:-$HOME/.local/state}/st2/default/catalog -``` - -Every catalog-aware command accepts `--catalog`; otherwise st2 uses `$CATALOG`, then that standard -location. - -### A catalog may declare its session registry - -A catalog's tasks live in `/pty` unless the catalog says otherwise. To put several catalogs -in one host-wide `pty` registry — so any viewer enumerates every session without knowing which -catalog produced it — declare it in `/catalog.kdl`: - -```kdl -catalog { - pty-root "/run/agents/pty" -} -``` - -`pty-root` accepts `$VAR`/`$CATALOG`; a relative value anchors at the catalog root. The resolution -order is an exported `PTY_ROOT` (a deliberate override, used by `st2 eval` for a short socket path), -then this declaration, then `/pty`. A catalog that declares nothing is unaffected. - -Prefer the declaration over exporting `PTY_ROOT` into readers: a reader that misses the export -resolves a different registry and reports live agents as dead. When adopting it on a host whose -systemd unit was installed with an ambient `PTY_ROOT`, reinstall the unit without one -(`st2 service install`) — the export still wins, and leaving it pins the supervisor to the old -registry while everything else follows the catalog. - -### A catalog may declare how long retirement is kept - -`st2 up` archives retired, quiescent seats itself, so a long-lived catalog stays bounded without -anyone remembering the chore. The grace period is `archive-after`, declared beside the registry: - -```kdl -catalog { - pty-root "/run/agents/pty" - archive-after "7d" -} -``` - -It accepts the usual duration spellings (`90`, `30m`, `12h`, `7d`; a bare number is seconds) and -defaults to `7d`. `archive-after "0"` disables the automatic step without disabling -`st2 catalog archive`. A value st2 cannot parse fails `st2 validate` rather than falling back to the -default: archiving on a clock the operator did not write is the one outcome this setting exists to -prevent. - -Each pass archives at most 25 seats, so a catalog holding hundreds of retirements drains over -several passes, and it never queues for the authoring lock — a pass blocked behind -`st2 catalog apply` would stall every live agent's reconciliation, and a due seat is still due next -pass. Because st2 records no timestamp for a desired-state edit, the clock starts when the -supervisor first observes the retirement; it keeps that observation in -`/.st2/retired-observed.json`, never in the spec, and drops a seat's row as soon as it -stops being retired — so un-retiring and retiring again serves a fresh grace period. - -Lifecycle hooks are installed only by the explicit `st2 hooks install` command. The installer -publishes an immutable content-addressed set, then atomically selects it with a receipt. `st2 up` -verifies its own immutable set for Codex launches; any local workspace render that actually -references `$ST_HOOKS` verifies that set before writing. Selecting a successor does not invalidate -an older running binary's installed set during cutover. Hook-free materialization does not require -an installed set. These checks never create, refresh, or rewrite hooks. To select this binary's -exact hook set when the installed and candidate builds are older or cannot be ordered, use `st2 -hooks install --replace`. `st2 hooks verify-own` is the read-only cutover probe for an installed -binary that may no longer be selected. - -`ST_HOOKS` overrides the machine-local hook root for installation, verification, and managed tasks. -During materialization, hook commands such as `$ST_HOOKS/codex-stop.sh` resolve to the invoking -binary's immutable set, so rendered settings are versioned without embedding a machine-specific -root in the declaration. - -The hooks have a small operational purpose: session-start restores durable context and exposes the -current inbox; pre-compact preserves a recovery breadcrumb when no context was written; stop and -failure hooks surface newly arrived work or a harness failure. They fail open so hook trouble does -not prevent the harness from starting or stopping. - -`st2 claude-channel install` publishes the Claude Code marketplace and plugin embedded in the st2 -binary. It registers the plugin for the current user and installs one machine policy fragment with -`sudo`. The policy approves the stable `st2-channel@st2` identity. The plugin starts the st2 MCP -server through the managed task's `PATH`, `CATALOG`, and `ST_AGENT`; it writes no `.mcp.json` file -into a product workspace. Re-running the command updates the marketplace and reinstalls the exact -embedded plugin. `st2 claude-channel status` verifies all four parts. `uninstall` removes only the -st2 user state and its policy fragment. Installation is optional. When the plugin is absent, the -native driver passes an inline MCP declaration and uses Claude's development channel. Claude can -show a confirmation prompt on that path, so install the plugin for unattended agents. Claude treats -the managed plugin list as an allowlist. Administrators must include any other approved channel -plugins in their managed policy. - -The same immutable set also carries `pi-channel.ts`. pi has no hook mechanism of its own — an -extension is where a pi session exposes that surface — so st2 ships one and `st2 driver pi-session` -splices it into the launch from the set this binary verified. A declaration never names it, and a -host whose selected set predates it is told to run `st2 hooks install` rather than launching -without a channel. That extension carries the pi equivalent of the session-start hook: st2 composes -the same restored working state, boot ritual, and unread-inbox listing and puts it on the channel's -opening frame, and the extension waits for that frame before the session's first turn, so a -restarted pi agent boots knowing what it was doing. A managed pi seat also runs with `PI_OFFLINE=1` -and `PI_SKIP_VERSION_CHECK=1` unless its declaration sets them, so a supervised agent does not -self-update or make its boot latency depend on the network. - -## Author a native agent - -Start from the maintained [Codex](examples/native/agent-codex.kdl), -[Claude](examples/native/agent-claude.kdl), or [pi](examples/native/agent-pi.kdl) -declaration: - -```sh -export CATALOG="${XDG_STATE_HOME:-$HOME/.local/state}/st2/default/catalog" -bundle="$(mktemp -d)" -mkdir -p "$bundle/assets" -cp examples/native/agent-codex.kdl "$bundle/agent.kdl" -cp ./composed-AGENTS.md "$bundle/assets/AGENTS.md" -${EDITOR:-vi} "$bundle/agent.kdl" -input_sha256="$(st2 agent digest --bundle "$bundle")" -st2 agent publish --catalog "$CATALOG" --bundle "$bundle" \ - --input-sha256 "$input_sha256" --expect-absent --json -``` - -Replace ``, ``, ``, and ``. Include every file referenced by -`copy` in the bundle. For a later declaration-only update, publish `agent.kdl` with the current -declaration's SHA-256 via `--spec ... --expect-sha256 HEX`; sibling assets and state are preserved. -Bind either operation to the exact captured source with the SHA-256 returned by -`st2 agent digest`. - -To prepare and apply a complete declaration-plane replacement without copying -runtime state or workspaces: - -```sh -st2 catalog snapshot --catalog "$CATALOG" --output ./prepared --json -# Edit/render ./prepared. Retain rootSha256 from snapshot as the incumbent CAS, -# and afterRootSha256 from diff as the exact desired-input CAS. -st2 catalog diff --catalog "$CATALOG" --prepared ./prepared \ - --expect-sha256 --json -st2 catalog apply --catalog "$CATALOG" --prepared ./prepared \ - --input-sha256 --expect-sha256 --json -``` - -If the incumbent declarations cannot be parsed or must remain opaque to the -current parser, bind a one-time repair to their exact structural bytes instead: - -```sh -st2 catalog snapshot --catalog "$CATALOG" --output ./invalid-preimage \ - --raw-preimage --json -# Produce a fully valid ./prepared directory from that capture. -st2 catalog digest --catalog "$CATALOG" --prepared ./prepared --json -st2 catalog apply --catalog "$CATALOG" --prepared ./prepared \ - --input-sha256 --expect-sha256 \ - --raw-preimage --json -``` - -Raw-preimage mode has its own hash and receipt schemas. It makes no semantic -assertion about the live bytes—including validity, profiles, the catalog -envelope, or effective PTY root—while still fully validating the prepared and -applied catalogs. The caller-supplied raw-domain digest is the exact live -precondition. This is a byte-oriented transaction, not a validation bypass or -migration-policy engine. - -To publish that exact snapshot as a new, absent catalog: - -```sh -st2 catalog bootstrap --catalog "$NEW_CATALOG" --prepared ./prepared \ - --input-sha256 --json -``` - -Ordinary `catalog apply` is policy-free. It rejects state/control content, -symlinks, unprojected workspace facts, catalog-local/default PTY roots, and -effective PTY-root changes. Raw-preimage apply replaces those live semantic -checks with its exact byte-domain CAS. Bootstrap is a separate create-only -declaration transaction, not an apply mode. It atomically publishes absence or -the complete catalog, initializes its persistent lock and generation before -visibility, and never reads or writes the external PTY registry. Process -adoption and PTY-root migration remain separate because that registry has -independent producers. A crash during apply leaves a durable marker and -content-addressed stage. `st2 catalog apply --catalog "$CATALOG" --resume --json` -resumes without the original prepared source. Snapshots own the complete -bounded `_templates` library and empty canonical per-agent `.workspace` -directory facts, but never traverse, hash, copy, or delete workspace content. - -A retired declaration is runtime teardown only: it keeps its spec and `resources/` byte-identical -and stays reversible, so a long-lived catalog accumulates retired identities. Archival is the -pressure valve that keeps the live plane bounded: - -```sh -st2 catalog archive --catalog "$CATALOG" --all-retired --dry-run --json -st2 catalog archive --catalog "$CATALOG" --identity --json -st2 catalog unarchive --catalog "$CATALOG" --json -``` - -Archival moves the whole identity directory from `agents//` to -`.st2/archive//` under the exclusive catalog-authoring lock, in one generation -commit, as a same-filesystem rename — the archive root is a child of the catalog root, so no bytes -are copied and no partial bundle can exist. `.st2` is control space at any depth, so an archived -declaration is undiscoverable rather than filtered, and the whole-catalog transaction never projects -it. A tombstone beside the moved directory keeps the identity traceable as one `archived` row in -`st2 catalog graph --json`. - -Eligibility fails closed against the local host only, because another host's runtime records are not -observable from here: the declaration must sit at its canonical path, be retired in either spelling, -have no live or dead record for any declared task (the rule `st2 doctor` already applies to -retirement), and be named as `supervisor` by no declaration that stays behind. `--identity` refuses -the whole run if any named identity is ineligible; `--all-retired` reports the ineligible ones and -archives the rest. `st2 catalog unarchive` is the exact reverse move. - -`st2 up` applies exactly this gate on its own once a retirement outlives `archive-after`, so these -verbs are for the seats you do not want to wait out and for putting one back. - -The compact declaration shape is: - -```kdl -agent "" { - host "" - workspace "" - resource "work" uri="github-issue://example/project/123" reason="release work item" - // Optional metadata: - // role "worker" - // supervisor "" - // name "Release worker" - // description "Owns release preparation and verification." - argv "codex" "--dangerously-bypass-approvals-and-sandbox" "--dangerously-bypass-hook-trust" "" - ding - - render { - copy "assets/AGENTS.md" "AGENTS.md" - json-upsert ".codex/hooks.json" #""" -{"hooks":{"SessionStart":[{"hooks":[{"type":"command","command":"$ST_HOOKS/codex-session-start.sh","timeout":5}]}]}} -"""# - git-exclude "AGENTS.md" ".codex/hooks.json" - } -} -``` - -For a zero-interruption migration, add `lifecycle "adopt-only"` to the compact -agent or to an explicit `pty`/`exec` task. st2 adopts that task when its current -generation is alive. If the generation is dead or absent, st2 reports the task -as `held` and does not remove or launch that task. A compact `ding` is derived -from the generated agent task: while that agent is held, st2 suppresses a -missing DING and stops an exact generated DING proved live: - -```kdl -agent "" { - host "" - workspace "" - lifecycle "adopt-only" - argv "codex" "" -} -``` - -This is a fence, not a restart policy. After inspecting or recovering the -original generation, deliberately change the lifecycle back to `"service"` (or -remove the field) to authorize ordinary absent launch and dead replacement. -Whole-agent desired state remains an explicit, separate lifecycle instruction -and takes precedence over task lifecycle. To keep an agent in the catalog while -running none of its tasks: - -```sh -st2 --catalog agent desired-state suspended \ - --reason "Waiting for capacity" -st2 --catalog agent desired-state running -st2 --catalog agent desired-state retired \ - --reason "Mission complete" -``` - -The canonical KDL is `desired-state "suspended" reason="..."`. Running is -canonically omitted. New suspended and retired states require a bounded reason; -legacy `retired #true` remains readable. Suspension stops the agent and its -derived DING without deleting its declaration, inbox, context, or Resources. -Resume uses ordinary `service`, `adopt-only`, and `keep` behavior. Nix-owned -declarations must be changed at their Nix source. - -`resource` binds an agent-local semantic name to an exact RFC 3986 absolute URI. The URI scheme -selects an open, downstream-owned Resource profile; st2 preserves URI bytes without normalization. -It neither registers schemes, owns profile schemas, nor resolves targets. -Binding order is irrelevant and names must be unique within the agent: - -```kdl -resource "work" uri="github-issue://example/project/123" reason="release work item" -resource "source" uri="worktree://github.com/example/project/change" reason="primary checkout" -resource "delivery" uri="ding://host/agent" reason="notification channel for this agent" -``` - -The envelope is `name` + `uri` + a required human-facing `reason`, plus an optional -`inactive-reason` that preserves a retired binding without deleting it. It carries no -access, readiness, or lifecycle policy, and URI possession conveys no authority. A Resource URI may -be referenced by any number of agent declarations. Resource-only declaration edits do not stop, -replace, or relaunch a live task. Resource profiles and resolvers remain opaque to st2; catalog -readers use the public `agent-spec` crate to inspect the bindings, and `st2 resource ls|read` -projects them for one agent. - -The positional agent value is the stable automation identity. Optional `name` and `description` -fields are presentation only; they never route messages, select tasks, or rename durable state. -Mutate a catalog-owned KDL declaration through the constrained commands: - -```sh -st2 rename "Release worker" -st2 describe "Owns release preparation and verification." -st2 rename --clear -st2 resource add --uri --reason "" -st2 resource remove -st2 resource rename -``` - -These commands preserve unrelated KDL bytes and serialize local writers through the persistent -shared `.st2/catalog-authoring.lock`. They refuse TOML, JSON, and -explicitly `meta { managed-by "nix" }` targets. Nix generators must emit that marker before the -compatible st2 binary is activated. In the trusted single-operator fleet, caller-supplied -`ST_AGENT` limits an invocation to itself or declared descendants; it is a guardrail rather than -authentication, and absence selects the operator path. The sibling `/name` convention -is hard-retired and ignored. - -`argv` launches its first value directly with the remaining values as arguments. It resolves a bare -program such as `codex` through the task environment's `PATH`, preserves argument boundaries, and -does not introduce a shell. Use `command #"..."#` instead when the task intentionally needs shell -syntax such as pipelines, redirects, or variable expansion; `command` continues to run under -`sh -c`. A runnable task must declare exactly one of `argv` or `command`. - -### Scheduled work is coming soon, not implemented - -st2 does not parse or run scheduled entries today. The intended direction starts with a declarative -DING targeted at the containing agent; scheduled PTY or exec work may be considered later. This is a -non-functional preview, and `st2 validate` correctly rejects it today: - -```kdl -agent "" { - // Current implemented fields and tasks go here. - - // FUTURE ONLY — reserved and rejected by the current contract. - schedule "local-health" { - every "2h" - ding "Run the local health check." - } -} -``` - -st2 provides `CATALOG`, flat native `ST_ROOT`, local `PTY_ROOT`, `ST_AGENT`, and `ST_HOOKS` to the -task. The complete st2-managed overlay is also persisted in PTY metadata, so a manual `pty restart` -retains those values. Declarations should not contain machine-specific install paths. - -## Validate and materialize - -Gate the declaration before starting anything: - -```sh -st2 hooks verify -st2 validate --catalog "$CATALOG" --strict -st2 up --catalog "$CATALOG" --host --materialize-only -``` - -Validation always checks the whole synced catalog structurally. External workspace and task-path -presence is checked only for the selected run host, which defaults to the local short hostname; use -`--host ` to validate another machine's local paths. Supervisors may be declared as either a -bare identity or the fully-qualified `.` bus id. - -Materialization simulates all content operations before writing. It refuses any real change to a -Git-tracked target, including a mode-only change; byte-identical content with the declared mode is -accepted. Inspect the declared targets and keep generated overlays untracked. Detection invokes -`git` and fails closed if the executable is unavailable or a workspace that appears to be a worktree -cannot be inspected. - -Each content directive accepts `executable=#true`. st2 applies exact mode `0755` when the property is -true and exact mode `0644` when it is absent or false. Materialization repairs a wrong existing mode -even when the file bytes already match. A `copy` source mode does not affect the destination mode. -An operation whose bytes and mode already match is not reported as materialized. - -Inline content uses the existing `file` directive. st2 writes the decoded KDL string without adding -or removing a newline. A blank line before a multiline string's closing delimiter encodes one final -newline: - -```kdl -file ".st2/bin/probe" executable=#true { - content #""" -#!/bin/sh -printf 'ready\n' - -"""# -} -``` - -Use `ensure-line` when a workspace-owned text file must import a catalog-owned contract: - -```kdl -copy "_templates/CONTRACT.md" ".st2/CONTRACT.md" -ensure-line "CLAUDE.md" "@.st2/CONTRACT.md" -ensure-line "AGENTS.md" "@.st2/CONTRACT.md" -``` - -Each harness loads the import from its native contract file. A later boot-prompt rewrite cannot -silently remove the contract-loading instruction. On 2026-09-01, Alex stated that Codex supports -`@` imports in `AGENTS.md`. This Codex behavior is undocumented and was not source-verified for this -change. - -`ensure-line` searches a UTF-8 file for one exact full line. An existing match causes no write. -If the line is absent, st2 preserves all existing bytes, adds a separator newline when necessary, -and appends the declared line with a final newline. st2 creates a missing target in a non-Git -workspace or when Git does not track that path. - -For a Git-tracked target, `ensure-line` is a verifier. st2 accepts an exact existing line but refuses -to add a missing line. The refusal occurs before any render operation writes to the workspace. The -repository owner must add and commit the line outside st2. This rule prevents materialization from -leaving a customer repository dirty or changing a repository that assistants can only read. - -`git-exclude` is advisory. `copy`, `file`, `json-upsert`, and `ensure-line` are boot-gating. - -## Run - -On headless Linux, install the systemd user service: - -```sh -st2 service install --catalog "$CATALOG" --host -st2 service status -``` - -Each task runs in its own transient scope, so restarting the supervisor does not kill live agents. - -On macOS, use a manual one-shot reconcile: - -```sh -st2 up --catalog "$CATALOG" --host --once -``` - -There is intentionally no resident macOS service path. - -For a shortest-path change to one exact task, render only its owning agent and reconcile only that -task in a bounded pass: - -```sh -st2 up --catalog "$CATALOG" --host --once --task -``` - -Unknown, ambiguous, and wrong-host task selectors refuse before workspace writes or PTY inspection. -A selected generated DING that is dead or absent is reported as `held`, because starting its -canonical agent would broaden the exact-task operation. Explicit sibling tasks remain independently -selectable. - -`st2 doctor` accepts the absence of a live host lock as the normal manual/`--once` mode. For a -resident `st2 up` deployment, use `st2 doctor --require-supervisor` to make a missing loop fail the -health check. A stale lock left by a dead supervisor is always a failure. The underlying -non-interactive `pty list --json` runtime probe is bounded; a wedged client becomes an explicit -doctor failure instead of hanging the health check. Its short outer deadline is containment for a -wedged runtime, not a fleet-scalability mechanism. - -For a foreground supervisor on any host: - -```sh -st2 up --catalog "$CATALOG" --host -``` - -### Typed task inventory (diagnostic only) - -Use the typed inventory instead of parsing `doctor` prose: - -```sh -st2 tasks --catalog "$CATALOG" --host --json -``` - -The `st2.task-inventory.v3` envelope joins the selected host's desired PTY and exec tasks to -read-only runtime evidence. Each row keeps the declared `residencyPolicy` separate from nullable -host-local `runtimeResidency`. A complete observation exits zero. Catalog parse errors, declaration -drift during observation, duplicate runtime IDs, malformed residency records, timeouts, malformed -output, PID reuse, and otherwise unprovable generations emit `complete: false` and exit non-zero. -Missing runtime rows become `absent` only when the corresponding backend observation is complete; -uncertainty remains `indeterminate`. - -Every runtime has a tagged `resourceTarget`. A live Linux process reports -`{"type":"linuxCgroupV2","path":"/..."}` from its exact unified -`/proc//cgroup` membership; a live Darwin process reports -`{"type":"darwinProcessTree","rootPid":...}` as a best-effort tree root. All -other cases report `{"type":"unavailable","reason":"..."}` from a bounded -reason set. These are per-observation locators, not identity: consumers keep -using `runtimeId` as the stable task key and rediscover the target on each -sample. - -A PTY root positively absent at admission is not passed to `pty` and remains absent; an absent exec -state root likewise remains absent. If an admitted PTY root is concurrently removed, the result is -incomplete because its filesystem identity changed, but the external `pty list` implementation may -recreate its registry before st2 can detect the race. Observation never rewrites an existing exec PID -record. It also does not serialize catalog or runtime writers, reconcile tasks, or authorize a -control-plane cutover. Consumers that require a zero-write boundary under concurrent root deletion -or a transactional declaration boundary need a separate protocol. - -### Staged control-plane replacement gate - -`st2 up` is a replaceable control plane, not the lifetime owner of an agent. Stopping it normally -or killing it must leave every agent running. `st2 down` and declaration retirement are separate, -explicit lifecycle actions and must never be used merely to replace the control plane. - -Every staged recovery or binary cutover must retain a pre-stop receipt of each agent's stable task -identity, PID, and process-creation identity. Install the verified replacement binary atomically -while those tasks continue running, restart the control plane, and do not accept the host until its -first reconcile proves all of the following: - -- every pre-existing agent is still usable with the same PID and creation identity; -- the replacement reports those agents as adopted and does not launch or duplicate them; -- only genuinely missing declared work is launched; -- explicit teardown remains the only path that stops an agent. - -The executable gate drives the real st2 binary and both PTY and exec backends through normal stop, -forced kill, atomic binary replacement, adoption, a live-task heartbeat, and a duplicate-boot -receipt: - -```sh -cargo test --test integration --all-features nomad_survival:: -``` - -## Messages, DING, status, and context - -Inside a managed task, `CATALOG`, `ST_ROOT`, and `ST_AGENT` are already set: - -```sh -st2 status "$ST_AGENT" --set available -st2 message send --subject "work" -m "Please handle the scoped task." -st2 message ls -st2 message read -st2 message reply -m "Handled." -st2 message archive -st2 agents --json --enrich -st2 agents --identity "$ST_AGENT" --json -st2 context read --full -``` - -The roster includes suspended and retired declarations instead of silently conflating desired -lifecycle with runtime presence. Both JSON shapes keep stable `identity` separate from optional -`name` and `description`, and contain the compatibility `retired` projection, `desiredState`, -`desiredStateReason`, plus the declaration's ordered `resources` descriptors. `--enrich` -additionally supplies `lastActivity` and `inbox`. Human output prints the same presentation fields -as separate columns and appends the non-running state and rationale. -`--identity .` selects exactly one qualified Agent Spec or fails, -so external harness hooks can read its current name and description without parsing KDL or relying -on a duplicate state file. - -For a catalog-backed agent, every native bus operation resolves the same agent directory used by -the roster: presence is `/status`, while unread messages, archive receipts, and context -live under `/resources/`. The flat `/` layout remains only as the -intentional catalog-less fallback used by isolated folder evals. In a catalog-backed root, -`st2 message ls` rejects an absent identity; recovery inspection of a deliberately orphaned flat -box must be explicit with `st2 message ls --orphan` (and optionally `--archive`). - -Services use a separate declared request surface; they do not borrow an Agent -Spec identity. Declare the endpoint without creating a task: - -```kdl -// /principals/host-a/example-ci/principal.kdl -principal "example-ci" host="host-a" -``` - -Then publish once, let the addressed agent reply from its normal inbox, and -observe the typed result. Bodies are JSON and `--tag` is a repeatable -`key=value` map: - -```sh -st2 request send host-a.repair-agent \ - --as host-a.example-ci \ - --idempotency-key 'escalate:repo#7:abc' \ - --tag kind=example-ci.escalation \ - -m '{"candidate":"abc"}' --json - -st2 request read --json - -st2 request reply \ - --tag outcome=needs-human \ - -m '{"outcome":"needs-human"}' --json - -st2 request status --as host-a.example-ci \ - --idempotency-key 'escalate:repo#7:abc' --json -``` - -The key atomically reserves one message filename and exact envelope before -publication. Exact retries return that filename with `deduplicated: true`; a -different request under the same key fails. Replies route only to the declared -principal's canonical `resources/inbox`, never a flat orphan mailbox. - -Adopters should cut directly to the native layout. Before launching a migrated identity, install and -verify hooks, validate and materialize its hand-authored declaration, stop any predecessor transport, -and decide how any unread predecessor backlog will be archived or forwarded. Never run predecessor -and native DING owners concurrently for the same identity. - -A pi agent does not use DING. Its messages are delivered natively into the live session by the -channel extension st2 injects, which calls pi's own message API and reads pi's own idle proof, so -no screen is inspected on that path. Declare it with a typed `pi {}` driver or with -`deliver "pi-channel"`. If the declaration also carries `ding`, native delivery wins and st2 omits -the sidecar. - -Native DING watches the recipient inbox and delivers a normalized notice: - -```text -[DING] new st2 message: [id:] (from ); check your inbox -``` - -Consumers must key on the `[DING]` prefix and stable id, not descriptive words. Every maintained -harness that uses DING uses the same transport: normalize untrusted fields into bounded, -single-line printable text, positively identify an empty current Codex or Claude composer, and send one bracketed-paste -sequence without Return. The sidecar then observes for a short bounded window and sends a separate -bare Return only after two immediately adjacent inspections show the exact notice in a positively -idle composer. A human draft, active turn, modal, changed composer, unreadable screen, command -timeout, or unknown renderer defers submission. Once a paste command starts, the sidecar retains -ownership and retries by inspection only, so a timeout cannot duplicate the paste. This measured -screen heuristic is fail-closed for the maintained renderer versions but is not an evented TUI -contract; renderer changes can defer delivery and remain an explicit design gap. - -Agents must declare `busy` before actively executing work and return to `available` only when -yielding or ready for new work, but `busy` never suppresses DING. Fresh `dnd` is the only delivery -hold. Each status record keeps the state on its first line. Version 1 writers add `v1 ` on -the second line. New readers use that origin timestamp and accept a legacy bare state. Old readers -keep using the first line and file mtime. A live session owner refreshes non-DND presence every five -minutes. This changes the replicated bytes and stays below the 15-minute stale limit. A legacy -`dnd` upgrades once with its existing mtime, then remains unchanged. A malformed versioned record or -a timestamp more than 60 seconds in the future reads as `unknown` without an mtime fallback. Version -1 status contributes its origin timestamp to `lastActivity`. New arrivals remain FIFO. -Same-filename archive receipts shadow and clean -restored inbox duplicates. Failed or uncertain PTY operations retain the notice for safe retry. Unsafe -delivery retries use a bounded backoff, so an active composer cannot make the sidecar spawn a fresh -PTY probe on every inbox poll. On start -or restart, the sidecar first adopts an exact staged recovery/backlog notice when present, then sends -one generic check-inbox recovery DING if unread work remains; it does not replay every backlog -message. - -Both DING and the catalog supervisor use filesystem events only as an early-wake optimization. -Create, modify, rename, and remove events wake them; read/open access events do not. Their own inbox -and catalog reads therefore cannot self-trigger a Linux inotify loop, and the bounded timer remains -the correctness fallback. - -## Cleanup - -Explicit teardown is the only operation that ends declared tasks: - -```sh -st2 down --catalog "$CATALOG" --host -``` - -On Linux, remove the supervisor service after teardown when it is no longer wanted: - -```sh -st2 service uninstall -``` - -## Command surface - -```text -ls, up, down, validate, doctor -message, ding, agents, status, context, resource, rename, describe -env, pty, shell, pretrust -hooks, service, claude-channel, eval -agent digest, agent publish -catalog bootstrap, catalog snapshot, catalog apply, catalog archive, catalog unarchive -completions -``` - -`st2 completions ` emits completions from the live command tree. No generated completion or -manpage tree is committed. - -## Clean-room verification - -The test suite builds a temporary `PATH` containing only the current `st2` binary, required Git, -and `pty`, `codex`, and `claude` shims. It installs and verifies a scratch hook receipt, instantiates -both maintained hand-authored KDL examples in fresh Git workspaces, validates and materializes them -twice, and verifies the help/doctor/native authoring surface without a predecessor transport binary: - -```sh -cargo test --test integration --all-features native_only:: -``` - -Run the complete local gate with: - -```sh -cargo clippy --all-targets --all-features -- -D warnings -cargo test --all-targets --all-features -``` - -The linker can exhaust a small temporary filesystem during a complete gate. A space or quota error -is an environment failure, not a compile failure. Set `CARGO_TARGET_DIR` to a filesystem with enough -space, reduce parallel jobs or debug information if necessary, and rerun the same gate. Use -`cargo clean --target-dir ` to remove the temporary build artifacts after the run. - -## Eval contract - -The sole canonical agent contract is -[`AGENT-SPEC.md`](https://github.com/compoundingtech/evals/blob/main/AGENT-SPEC.md) in the evals -repository. Eval corpus definitions, execution/readiness evidence, authorization, and results belong -to that repository; st2 does not duplicate or pin its ledgers here. - -An eval that exercises production Agent Specs opts in explicitly: - -```kdl -eval { - copy "./fixture" - run "publish-canonical-team" { command "..." } - canonical-agents - message { from "requester"; to "evalhost.supervisor"; content "./task.md" } - max-timeout "300s" - judges { /* held-out checks */ } -} -``` - -`copy` and deterministic `run` steps populate the hermetic temporary catalog first. -`canonical-agents` then recursively discovers the catalog and projects declarations resolved to the -eval host. Explicit `identity` and `host` fields remain authoritative independent of organizational -placement; each declaration parent remains its native state/resource anchor. The directive is -mutually exclusive with compact `team` / `agent` declarations, so the local Agent Spec vector is the -sole authority for launch, kickoff routing, supervision, logs, and teardown. Strict catalog -validation, fleet-unique nonempty task runtime IDs, and warning-free local materialization all finish -before an agent task starts; backend launch errors are fatal. Remote-host declarations remain inert. -Native inbox/archive paths are frozen from the admitted local vector, so later catalog mutation -cannot redirect eval traffic. A multi-agent team completes only after the existing worker-report ordering. -For a singleton, the requester inbox is snapshotted before kickoff; only a newly appearing -interviewer reply at-or-after the exact kickoff receipt completes it. Canonical completion gates the -verdict. Without the directive, Agent Spec-shaped files inside a fixture remain inert and compact -evals retain their flat bus and completion semantics. - -`st2 agent publish --catalog ROOT (--spec FILE | --bundle DIR) --input-sha256 HEX -(--expect-absent | --expect-sha256 HEX) [--managed-by MARKER]` is the single-agent declaration -writer. Replacing an incumbent that carries `meta { managed-by "nix" }` requires -`--managed-by nix`, because that publication rewrites bytes the Nix projection owns; an assertion -is admitted only when it names exactly the marker the incumbent carries. Creating a declaration -and republishing byte-identical bytes need no assertion. -`st2 catalog apply --catalog ROOT -(--prepared DIR --input-sha256 INPUT_HEX --expect-sha256 ROOT_HEX [--raw-preimage] | --resume)` is the complete -declaration-plane writer. Each admits the complete prospective catalog under a -compare-and-swap lock before making one atomic change. -`st2 catalog digest --catalog ROOT --prepared DIR` computes the exact desired -projection digest consumed by apply. Raw-preimage repair uses it for the fully -validated successor while binding the opaque incumbent through the separate -raw-domain digest. Ordinary valid-catalog workflows reuse `afterRootSha256` -from the required policy-inspection diff instead. -`st2 catalog bootstrap --catalog ROOT --prepared DIR --input-sha256 ROOT_HEX` -is the create-only writer for an absent catalog. An exact completed replay is -`unchanged`; any different or incomplete existing target fails closed. diff --git a/apps/fractal-web/.changelog/2026-11-01-sidebar-agents-tab.ts b/apps/fractal-web/.changelog/2026-11-01-sidebar-agents-tab.ts new file mode 100644 index 00000000000..aed893dab1b --- /dev/null +++ b/apps/fractal-web/.changelog/2026-11-01-sidebar-agents-tab.ts @@ -0,0 +1,7 @@ +import type { ChangelogEntrySource } from '../src/changelog/contract.ts' + +export default { + title: 'Read agent status in the sidebar', + summary: 'The sidebar now labels its Agents view with an attention marker before the tab name, equal-width tabs, and full touch targets. Agent rows use shared activity glyphs and show a diamond when they need you or a question mark for a harness question. Press G then A to open Agents, or the question-mark key to see navigation shortcuts even when the sidebar is collapsed.', + kind: 'feature', +} satisfies ChangelogEntrySource diff --git a/apps/fractal-web/.changelog/2026-11-02-embed-landmark-groups.ts b/apps/fractal-web/.changelog/2026-11-02-embed-landmark-groups.ts new file mode 100644 index 00000000000..400bd8c14b1 --- /dev/null +++ b/apps/fractal-web/.changelog/2026-11-02-embed-landmark-groups.ts @@ -0,0 +1,7 @@ +import type { ChangelogEntrySource } from '../src/changelog/contract.ts' + +export default { + title: 'Keep repeated conversation controls accessible', + summary: 'Work logs and message composers are named groups within their workspace rather than repeated page landmarks. Pages showing multiple product embeds retain their labels and form behavior without duplicate landmarks.', + kind: 'fix', +} satisfies ChangelogEntrySource diff --git a/apps/fractal-web/.changelog/2026-11-02-served-prompt-cards.ts b/apps/fractal-web/.changelog/2026-11-02-served-prompt-cards.ts new file mode 100644 index 00000000000..b6075291740 --- /dev/null +++ b/apps/fractal-web/.changelog/2026-11-02-served-prompt-cards.ts @@ -0,0 +1,7 @@ +import type { ChangelogEntrySource } from '../src/changelog/contract.ts' + +export default { + title: 'Read question cards from served attention', + summary: 'Prompt cards now show the calls and options Fractal serves with the open attention row instead of inferring them from conversation history. Missing, incomplete or unsupported prompt details explain that the answer belongs in the agent’s terminal.', + kind: 'fix', +} satisfies ChangelogEntrySource diff --git a/apps/fractal-web/.changelog/2026-11-02-transcript-grammar-deferral.ts b/apps/fractal-web/.changelog/2026-11-02-transcript-grammar-deferral.ts new file mode 100644 index 00000000000..9686c6523ff --- /dev/null +++ b/apps/fractal-web/.changelog/2026-11-02-transcript-grammar-deferral.ts @@ -0,0 +1,7 @@ +import type { ChangelogEntrySource } from '../src/changelog/contract.ts' + +export default { + title: 'Load transcript syntax coloring on demand', + summary: 'Conversation prose and code remain readable immediately, while syntax-coloring grammars load only when a supported code block is rendered. Code controls, keyboard focus, and wrap state stay in place during loading.', + kind: 'fix', +} satisfies ChangelogEntrySource diff --git a/apps/fractal-web/scripts/devnet/INTEGRATOR.md b/apps/fractal-web/scripts/devnet/INTEGRATOR.md index 965dc38f2fd..271697b389e 100644 --- a/apps/fractal-web/scripts/devnet/INTEGRATOR.md +++ b/apps/fractal-web/scripts/devnet/INTEGRATOR.md @@ -79,6 +79,18 @@ when adopting the script to reuse the existing member key. The production build helper and generated launchers remain deployment-owned inputs, not replacements for the separate launcher scripts. +Failure attribution builds each probe in a fresh temporary +`live-r3-probe-*` namespace under `FRACTAL_BUILD_ROOT`. Probe outputs are removed +after the helper returns; empty snapshot namespaces are also removed, without +deleting leftover snapshots that might still hold leases. Existing outputs are +never reused: ignored private/vendor overlays can change without changing the +commit, and a raw build receipt can exist before host validation finishes. +A probe passes only when the full build-and-validation helper exits successfully. +Once a failing prefix is found, the integrator ejects that PR and reintegrates +later accepted heads before rebuilding. +Tree-cache identities include the integrator version, so upgrading the attribution +logic does not replay an older unattributed failure. + Deployment overlays can import `edgeOrTailnetAdmission` from `server/host.mts` and pass their existing edge admission as `edge`. With `origin: undefined`, it returns that exact policy. With a published HTTPS origin and a nonempty login allowlist, @@ -163,14 +175,20 @@ until its replacement is Ready and the edge cuts over. A previous registration is stopped first only when Devnet status positively reports it is not running or its lifecycle is starting/terminal. Missing or failed observations never authorize stopping a server; Devnet decides whether replacement can proceed. -If replacement fails, Devnet owns server recovery. The integrator restores the -previous build pointer only if the edge confirms that previous revision serves; -if the candidate serves or the observation is inconclusive, the pointer stays -unchanged. Neither generation is stopped or relaunched, and the original error -is propagated. In particular, `ReplacementRecoveryRequired` needs operator -recovery: the integrator never runs `--abandon-replace`. If replacement succeeds -but subsequent publication verification fails, the integrator stops the new -server and relaunches the previous build as before. +After a nonzero replacement exit, the integrator verifies the published candidate +build identity; neither the exit code nor stderr text establishes whether the +switch committed. If the candidate serves, activation succeeds and the tick +records the last 4000 stderr characters in `replaceWarning` (or the command error +if stderr is empty). The current pointer is preserved and normal retention runs. +Otherwise Devnet owns server recovery. The integrator restores the previous build +pointer only if the edge confirms that previous revision serves; an inconclusive +observation leaves the pointer unchanged. Neither generation is stopped or +relaunched, and the original error is propagated with the last 4000 diagnostic +characters in the tick's `detail`. `ReplacementRecoveryRequired` remains a Devnet +recovery diagnostic, not a success/failure discriminator; the integrator never runs +`--abandon-replace`. If replacement exits successfully but subsequent publication +verification fails, the integrator stops the new server and relaunches the previous +build as before. After confirmed activation, the integrator keeps the current build and one previous rollback build and deletes older build directories. `integrator-activation.json` diff --git a/apps/fractal-web/scripts/devnet/integrator.py b/apps/fractal-web/scripts/devnet/integrator.py index e5f00f40def..009a5ea3c90 100755 --- a/apps/fractal-web/scripts/devnet/integrator.py +++ b/apps/fractal-web/scripts/devnet/integrator.py @@ -1,9 +1,9 @@ #!/usr/bin/env python3 """Integrate open Fractal PRs and publish an immutable production build. -Version 10: isolate host publication and reject executable candidate artifact types. +Version 12: attribute failures using fresh, validated probe builds. """ -import datetime, fcntl, hashlib, json, math, os, pathlib, re, shutil, subprocess, time +import contextlib, datetime, fcntl, hashlib, json, math, os, pathlib, re, shutil, subprocess, tempfile, time import sys SCRIPT_DIR = pathlib.Path(__file__).resolve().parent sys.path.insert(0, str(SCRIPT_DIR)) @@ -50,7 +50,7 @@ def required_input(name): except (ValueError, TypeError) as error: raise SystemExit(f'invalid integrator trust configuration: {error}') PR_METADATA_FIELDS = 'number,headRefName,headRefOid,baseRefName,baseRefOid,changedFiles,isDraft,author,isCrossRepository,headRepositoryOwner' -INTEGRATOR_VERSION = 10 +INTEGRATOR_VERSION = 12 DEPENDENCY_SANDBOX_VERSION = candidate_sandbox.DEPENDENCY_POLICY try: SELECTION_BUDGET_SECONDS = float(os.environ.get('FRACTAL_INTEGRATOR_SELECTION_BUDGET_SECONDS', '60')) @@ -90,7 +90,7 @@ def atomic_json(path, value): def candidate_key(private_hash): # Tree identity excludes ephemeral merge/stamp commit timestamps. - return hashlib.sha256((text('rev-parse', 'HEAD^{tree}') + private_hash).encode()).hexdigest() + return hashlib.sha256((str(INTEGRATOR_VERSION) + ':' + text('rev-parse', 'HEAD^{tree}') + private_hash).encode()).hexdigest() def publish_skips(row): if not DRY_RUN: @@ -407,7 +407,19 @@ def prepare_dependencies(): def probe_revision(revision): git('checkout','--detach',revision) prepare_dependencies() - return build('live-r3-probe').returncode == 0 + # A raw receipt does not prove host validation completed, and a prior output + # for this commit may have used different ignored private/vendor overlays. + # Keep publication's immutable-output guard and build every probe afresh. + with tempfile.TemporaryDirectory(prefix='live-r3-probe-', + dir=required_input('FRACTAL_BUILD_ROOT')) as output: + name=pathlib.Path(output).name + try: + return build(name).returncode == 0 + finally: + # The helper removes its snapshots. Remove only an empty namespace; + # never delete a leftover snapshot that might still hold a lease. + with contextlib.suppress(OSError): + (pathlib.Path(required_input('FRACTAL_BUILD_SNAPSHOT_ROOT'))/name).rmdir() def build_candidate(row, private_hash, checkpoints): cache=L/'live-r3-tree-cache.json' @@ -646,7 +658,7 @@ def registration_not_ready(cleanenv): except (OSError, ValueError, KeyError, TypeError, AttributeError, subprocess.SubprocessError): return False -def activate(previous): +def activate(previous, row): current=os.path.realpath(P/'current') sha=json.loads(candidate_sandbox.read_regular(P/'current/build-receipt.json'))['buildIdentity']['rev'] cleanenv={k:v for k,v in os.environ.items() if k not in ('PTY_SESSION','ST_AGENT')} @@ -662,16 +674,22 @@ def activate(previous): try: if previous and registration_not_ready(cleanenv): swap_server(cleanenv) - subprocess.run(['devnet','up','--replace',purpose,'--ready-path','/','--',str(P/'current/launch.sh'),purpose,str(P/'current')],cwd=C/'apps/fractal-web',env=cleanenv,check=True) + subprocess.run(['devnet','up','--replace',purpose,'--ready-path','/','--',str(P/'current/launch.sh'),purpose,str(P/'current')],cwd=C/'apps/fractal-web',env=cleanenv,text=True,stderr=subprocess.PIPE,check=True) replacement_completed=True subprocess.run(['devnet','url',purpose,'--remote','--wait'],cwd=C/'apps/fractal-web',env=cleanenv,check=True,timeout=120) url=current_server_url(cleanenv,sha) if not url: raise RuntimeError('published edge does not serve candidate build identity') - except Exception: - # Failed replacement may already have committed; reconcile before moving the pointer. + except Exception as error: + # Exit status cannot distinguish a failed switch from failed retirement. + # Only the published candidate identity confirms replacement success. if not replacement_completed: - if not current_server_url(cleanenv,sha) and previous and previous!=current: + url=current_server_url(cleanenv,sha) + if url and isinstance(error,subprocess.CalledProcessError): + row['replaceWarning']=(error.stderr or str(error))[-4000:] + recovery_retain_builds(previous) + return url + if not url and previous and previous!=current: rollback_sha=json.loads(candidate_sandbox.read_regular(pathlib.Path(previous)/'build-receipt.json'))['buildIdentity']['rev'] if current_server_url(cleanenv,rollback_sha): recovery_select_build(rollback_sha) @@ -845,7 +863,7 @@ def priority(p): row.update(sha=old['sha'],included=old['included'],skipped=skipped,build='skipped',reason='unchanged',selectionComplete=True) previous=recovery_previous() recovery_select_build(row['sha']) - row['url']=activate(previous) + row['url']=activate(previous, row) return git('checkout','-B','live-r3',base) overlay=C/'apps/fractal-web/private' @@ -905,7 +923,7 @@ def priority(p): atomic_json(state,{'signature':signature,'sha':row['sha'],'included':row['included'],'skipped':row['skipped'],'pinnedBase':row['pinnedBase']}) return recovery_select_build(row['sha']) - row['url']=activate(previous) + row['url']=activate(previous, row) row['build']='built' atomic_json(state,{'signature':signature,'sha':row['sha'],'included':row['included'],'skipped':row['skipped'],'pinnedBase':row['pinnedBase']}) except SelectionDeadline as error: diff --git a/apps/fractal-web/scripts/devnet/integrator_selection_test.py b/apps/fractal-web/scripts/devnet/integrator_selection_test.py index 3febb3e293a..40358aff8d4 100755 --- a/apps/fractal-web/scripts/devnet/integrator_selection_test.py +++ b/apps/fractal-web/scripts/devnet/integrator_selection_test.py @@ -57,6 +57,108 @@ def activation_builds(self): self.module.recovery_select_build('b' * 40) return str(self.module.P / 'builds' / ('a' * 40)) + def test_existing_main_probe_ejects_breaking_pr_and_keeps_later_pr(self): + main, broken, later = ('a' * 40, 'b' * 40, 'c' * 40) + build_root = self.root / 'build-root' + receipt = build_root / 'live-r3-probe' / 'builds' / main / 'build-receipt.json' + receipt.parent.mkdir(parents=True) + receipt.write_text(json.dumps({'buildIdentity': {'rev': main}})) + current = {'sha': later, 'tree': 'full'} + builds = [] + + def git(*args, **kwargs): + if args[0] == 'checkout': + current['sha'] = args[-1] + if args[-1] == main: + current['tree'] = 'reduced' + elif args[0] == 'merge': + current['sha'] = later + return subprocess.CompletedProcess([], 0) + + def build(name): + builds.append((name, current['sha'])) + # The old shared namespace collides; fresh probes really compile. + ok = ((name == 'live-r3' and current['tree'] == 'reduced') or + (name.startswith('live-r3-probe-') and current['sha'] == main)) + return subprocess.CompletedProcess([], 0 if ok else 1) + + row = {'included': [{'pr': 10}, {'pr': 11}], 'skipped': []} + checkpoints = [{'sha': main, 'pr': None}, + {'sha': broken, 'pr': 10, 'head': 'broken-head'}, + {'sha': later, 'pr': 11, 'head': 'later-head'}] + with patch.dict(os.environ, FRACTAL_BUILD_ROOT=str(build_root), + FRACTAL_BUILD_SNAPSHOT_ROOT=str(self.root / 'snapshots')), \ + patch.object(self.module, 'git', side_effect=git) as commands, \ + patch.object(self.module, 'text', side_effect=lambda *args: current['sha']), \ + patch.object(self.module, 'candidate_key', side_effect=lambda _: current['tree']), \ + patch.object(self.module, 'prepare_dependencies'), \ + patch.object(self.module, 'stamp'), \ + patch.object(self.module, 'retain_main_registry'), \ + patch.object(self.module, 'build', side_effect=build): + self.assertTrue(self.module.build_candidate(row, 'overlay', checkpoints)) + self.assertEqual(row['included'], [{'pr': 11}]) + self.assertEqual(row['skipped'], [{'pr': 10, 'reason': 'build-break', 'files': []}]) + self.assertEqual([sha for _, sha in builds], [later, main, broken, later]) + self.assertEqual(builds[0][0], 'live-r3') + self.assertEqual(builds[-1][0], 'live-r3') + self.assertTrue(all(name.startswith('live-r3-probe-') for name, _ in builds[1:3])) + self.assertNotEqual(builds[1][0], builds[2][0]) + self.assertTrue(all(not (build_root / name).exists() for name, _ in builds[1:3])) + self.assertIn(('merge', '--no-ff', '--no-edit', 'later-head'), + [call.args for call in commands.call_args_list]) + cached = json.loads((self.module.L / 'live-r3-tree-cache.json').read_text()) + self.assertEqual(cached['full']['ejected'], 10) + self.assertEqual(cached['full']['included'], [{'pr': 11}]) + + def test_changed_overlay_rebuilds_main_and_does_not_eject_a_pr(self): + sha = 'a' * 40 + build_root = self.root / 'build-root' + receipt = build_root / 'live-r3-probe' / 'builds' / sha / 'build-receipt.json' + receipt.parent.mkdir(parents=True) + receipt.write_text(json.dumps({'buildIdentity': {'rev': sha}, 'overlay': 'old'})) + with patch.dict(os.environ, FRACTAL_BUILD_ROOT=str(build_root), + FRACTAL_BUILD_SNAPSHOT_ROOT=str(self.root / 'snapshots')), \ + patch.object(self.module, 'git'), \ + patch.object(self.module, 'prepare_dependencies'), \ + patch.object(self.module, 'build', return_value=subprocess.CompletedProcess([], 1)) as build: + # The same git revision with today's overlay now fails to build. + self.assertIsNone(self.module.first_break([sha, 'b' * 40], self.module.probe_revision)) + build.assert_called_once() + self.assertTrue(build.call_args.args[0].startswith('live-r3-probe-')) + + def test_raw_receipt_from_interrupted_validation_is_not_reused(self): + sha = 'a' * 40 + build_root = self.root / 'build-root' + receipt = build_root / 'live-r3-probe' / 'builds' / sha / 'build-receipt.json' + receipt.parent.mkdir(parents=True) + receipt.write_text(json.dumps({'buildIdentity': {'rev': sha}})) + names = [] + + def build(name): + names.append(name) + output = build_root / name / 'builds' / sha + output.mkdir(parents=True) + (output / 'build-receipt.json').write_text(receipt.read_text()) + # Promotion produced a matching raw receipt, but host validation failed. + return subprocess.CompletedProcess([], 1) + + with patch.dict(os.environ, FRACTAL_BUILD_ROOT=str(build_root), + FRACTAL_BUILD_SNAPSHOT_ROOT=str(self.root / 'snapshots')), \ + patch.object(self.module, 'git'), \ + patch.object(self.module, 'prepare_dependencies'), \ + patch.object(self.module, 'build', side_effect=build): + self.assertFalse(self.module.probe_revision(sha)) + self.assertFalse(self.module.probe_revision(sha)) + self.assertEqual(len(set(names)), 2) + self.assertTrue(all(not (build_root / name).exists() for name in names)) + self.assertTrue(receipt.exists()) + + def test_tree_cache_identity_changes_with_integrator_version(self): + with patch.object(self.module, 'text', return_value='tree'): + old = self.module.candidate_key('overlay') + with patch.object(self.module, 'INTEGRATOR_VERSION', self.module.INTEGRATOR_VERSION + 1): + self.assertNotEqual(self.module.candidate_key('overlay'), old) + def test_activation_replaces_serving_generation_without_down(self): previous = self.activation_builds() for rollback, observations in ( @@ -67,7 +169,7 @@ def test_activation_replaces_serving_generation_without_down(self): patch.object(self.module, 'registration_not_ready', return_value=False), \ patch.object(self.module.subprocess, 'run') as run, \ patch.object(self.module, 'recovery_retain_builds') as retain: - self.assertEqual(self.module.activate(rollback), 'https://new.invalid') + self.assertEqual(self.module.activate(rollback, {}), 'https://new.invalid') commands = [call.args[0] for call in run.call_args_list] self.assertEqual(commands, [ ['devnet', 'up', '--replace', 'test', '--ready-path', '/', '--', @@ -82,7 +184,7 @@ def test_activation_clears_nonserving_registration_before_replace(self): patch.object(self.module, 'registration_not_ready', return_value=True), \ patch.object(self.module.subprocess, 'run') as run, \ patch.object(self.module, 'recovery_retain_builds'): - self.module.activate(previous) + self.module.activate(previous, {}) self.assertEqual([call.args[0][:3] for call in run.call_args_list], [['devnet', 'down', 'test'], ['devnet', 'up', '--replace'], ['devnet', 'url', 'test']]) @@ -99,7 +201,7 @@ def test_failed_replace_preserves_devnet_recovery_and_restores_build_pointer(sel patch.object(self.module.subprocess, 'run', side_effect=error) as run, \ patch.object(self.module, 'recovery_retain_builds') as retain: with self.assertRaises(subprocess.CalledProcessError) as raised: - self.module.activate(previous) + self.module.activate(previous, {}) self.assertIs(raised.exception, error) self.assertEqual(run.call_count, 1) self.assertEqual(run.call_args.args[0][:3], ['devnet', 'up', '--replace']) @@ -114,7 +216,7 @@ def test_publication_failure_after_replace_still_relaunches_previous_build(self) patch.object(self.module.subprocess, 'run') as run, \ patch.object(self.module, 'recovery_retain_builds') as retain: with self.assertRaisesRegex(RuntimeError, 'previous build rolled back'): - self.module.activate(previous) + self.module.activate(previous, {}) self.assertEqual([call.args[0][:3] for call in run.call_args_list], [['devnet', 'up', '--replace'], ['devnet', 'url', 'test'], ['devnet', 'down', 'test'], ['devnet', 'up', 'test']]) @@ -153,31 +255,41 @@ def test_inconclusive_registration_probe_does_not_stop_server(self): patch.object(self.module.subprocess, 'run', side_effect=[status, self.response({}), self.response({})]) as run, \ patch.object(self.module, 'recovery_retain_builds'): - self.module.activate(previous) + self.module.activate(previous, {}) self.assertEqual([call.args[0][:3] for call in run.call_args_list], [['devnet', 'status', 'test'], ['devnet', 'up', '--replace'], ['devnet', 'url', 'test']]) - def test_failed_replace_preserves_candidate_or_unknown_publication_pointer(self): + def test_nonzero_replace_classifies_only_by_published_candidate_identity(self): previous = self.activation_builds() - for message in ('old generation retirement failed', 'ReplacementRecoveryRequired'): - for published in ('https://candidate.invalid', None): - error = subprocess.CalledProcessError(1, ['devnet', 'up', '--replace'], stderr=message) - observations = [None, published] + ([] if published else [None]) - with self.subTest(error=message, published=published), \ - patch.object(self.module, 'registration_not_ready', return_value=False), \ - patch.object(self.module, 'current_server_url', side_effect=observations) as probe, \ - patch.object(self.module.subprocess, 'run', side_effect=error) as run, \ - patch.object(self.module, 'recovery_retain_builds') as retain: - with self.assertRaises(subprocess.CalledProcessError) as raised: - self.module.activate(previous) - self.assertIs(raised.exception, error) - self.assertEqual(run.call_count, 1) - self.assertEqual(probe.call_args_list[1].args[1], 'b' * 40) - if not published: - self.assertEqual(probe.call_args_list[2].args[1], 'a' * 40) - self.assertEqual((self.module.P / 'current').resolve().name, 'b' * 40) - retain.assert_not_called() + for code in (1, 3): + for message in ('old generation retirement failed', 'ReplacementRecoveryRequired', ''): + for published in ('https://candidate.invalid', None): + row = {} + error = subprocess.CalledProcessError(code, ['devnet', 'up', '--replace'], + stderr=message + 'x' * 4100 if message else '') + observations = [None, published] + ([] if published else [None]) + with self.subTest(code=code, error=message, published=published), \ + patch.object(self.module, 'registration_not_ready', return_value=False), \ + patch.object(self.module, 'current_server_url', side_effect=observations) as probe, \ + patch.object(self.module.subprocess, 'run', side_effect=error) as run, \ + patch.object(self.module, 'recovery_retain_builds') as retain: + if published: + self.assertEqual(self.module.activate(previous, row), published) + self.assertEqual(row, {'replaceWarning': (error.stderr or str(error))[-4000:]}) + retain.assert_called_once_with(previous) + else: + with self.assertRaises(subprocess.CalledProcessError) as raised: + self.module.activate(previous, row) + self.assertIs(raised.exception, error) + self.assertEqual(row, {}) + self.assertEqual(probe.call_args_list[2].args[1], 'a' * 40) + retain.assert_not_called() + self.assertEqual(run.call_count, 1) + self.assertEqual(run.call_args.kwargs['stderr'], subprocess.PIPE) + self.assertTrue(run.call_args.kwargs['text']) + self.assertEqual(probe.call_args_list[1].args[1], 'b' * 40) + self.assertEqual((self.module.P / 'current').resolve().name, 'b' * 40) def test_miss_fetches_complete_paths_then_hit_does_not_call_api(self): with patch.object(self.module, 'run', side_effect=self.responses()) as run: @@ -308,6 +420,33 @@ def observed(row): self.assertIsNone(self.module.SELECTION_DEADLINE) self.assertFalse(self.module.C.exists()) + def test_failed_replace_stderr_tail_reaches_tick_receipt(self): + previous = self.activation_builds() + info = self.module.C / '.git/info' + info.mkdir(parents=True) + (info / 'exclude').write_text('') + diagnostic = 'prelude' + 'x' * 4100 + 'strict PTY inventory incomplete' + error = subprocess.CalledProcessError(3, ['devnet', 'up', '--replace'], + stderr=diagnostic) + with patch.object(self.module, 'run', return_value=self.response([])), \ + patch.object(self.module, 'prepare_clone', + side_effect=lambda: self.module.activate(previous, {})), \ + patch.object(self.module, 'registration_not_ready', return_value=False), \ + patch.object(self.module, 'current_server_url', + side_effect=[None, None, 'https://old.invalid']), \ + patch.object(self.module.subprocess, 'run', side_effect=error) as run, \ + patch.object(self.module, 'observe_served'), \ + patch.object(self.module.candidate_sandbox, 'validate_configuration'), \ + contextlib.redirect_stdout(io.StringIO()): + self.module.tick() + row = json.loads((self.module.L / 'live-r3.jsonl').read_text()) + self.assertEqual(row['build'], 'failed') + self.assertEqual(row['detail'], diagnostic[-4000:]) + self.assertNotIn('replaceWarning', row) + self.assertEqual(run.call_args.kwargs['stderr'], subprocess.PIPE) + self.assertTrue(run.call_args.kwargs['text']) + self.assertEqual((self.module.P / 'current').resolve(), Path(previous)) + def test_clone_config_refusal_is_preserved_in_tick_receipt(self): subprocess.run(['git', 'init', '-q', str(self.module.C)], check=True) config = self.module.C / '.git/config' diff --git a/apps/fractal-web/scripts/replay-e2e.mjs b/apps/fractal-web/scripts/replay-e2e.mjs index 5a69a103ed5..4911b1490ce 100644 --- a/apps/fractal-web/scripts/replay-e2e.mjs +++ b/apps/fractal-web/scripts/replay-e2e.mjs @@ -5,6 +5,7 @@ import { performance } from 'node:perf_hooks' import { fileURLToPath } from 'node:url' import { createServer } from 'vite' import { loadWorld } from '@fractal/scenarios' +import { collectResponse } from './response-wait.mjs' // Run in the web Nix shell: its Playwright driver and Chromium are also used by the kit gate. const started = performance.now() @@ -182,7 +183,9 @@ try { assert(elapsed <= offset, `Replay clock already passed requested offset ${offset}: ${elapsed}`) await page.clock.runFor(offset - elapsed) } - const paneResponse = page.waitForResponse(response => + // The pane's module can load at any point after navigation; it is collected from here and bounded + // only where the click needs it, so a slow setup cannot reject the wait before it is awaited. + const paneResponse = collectResponse(page, response => new URL(response.url()).pathname.endsWith('/ConversationPane.tsx')) await page.goto(`${origin}/?scenario=${scenario}&open=missions:detail`) await roster.waitFor({ state: 'visible' }) @@ -203,7 +206,7 @@ try { await agent.click() assert(new URL(page.url()).pathname.startsWith('/w/agent/'), 'agent selection must navigate in-app') assert.equal(new URL(page.url()).searchParams.get('scenario'), scenario) - await (await paneResponse).finished() + await (await paneResponse.within(10_000, 'ConversationPane.tsx after the agent click')).finished() const pane = page.getByTestId('conversation-history-host') const transcript = page.getByTestId('transcript-scroll') while (!await pane.isVisible() || !await transcript.getByTestId('transcript-empty').isVisible()) { diff --git a/apps/fractal-web/scripts/response-wait.mjs b/apps/fractal-web/scripts/response-wait.mjs new file mode 100644 index 00000000000..77868c49a71 --- /dev/null +++ b/apps/fractal-web/scripts/response-wait.mjs @@ -0,0 +1,27 @@ +// A page response collected from the moment this is called, with no timer until something waits on it. +// page.waitForResponse starts its timeout when it is created, so one created before a long setup and +// awaited after it can reject first, and that unhandled rejection ends the whole browser run (#4244). +export function collectResponse(page, matches) { + let onResponse + const response = new Promise(resolve => { + onResponse = candidate => { + if (!matches(candidate)) return + page.off('response', onResponse) + resolve(candidate) + } + page.on('response', onResponse) + }) + return { + // The collected response, or a rejection `ms` after this call when it has not arrived. + within(ms, what) { + let deadline + const late = new Promise((_, reject) => { + deadline = setTimeout(() => { + page.off('response', onResponse) + reject(new Error(`${what} did not arrive within ${ms} ms`)) + }, ms) + }) + return Promise.race([response, late]).finally(() => clearTimeout(deadline)) + }, + } +} diff --git a/apps/fractal-web/scripts/response-wait.node-test.mjs b/apps/fractal-web/scripts/response-wait.node-test.mjs new file mode 100644 index 00000000000..493cd41266f --- /dev/null +++ b/apps/fractal-web/scripts/response-wait.node-test.mjs @@ -0,0 +1,49 @@ +import assert from 'node:assert/strict' +import { EventEmitter } from 'node:events' +import { test } from 'node:test' +import { collectResponse } from './response-wait.mjs' + +const pane = { url: () => 'http://localhost:5173/src/web/ConversationPane.tsx' } +const other = { url: () => 'http://localhost:5173/src/web/Sidebar.tsx' } +const isPane = response => new URL(response.url()).pathname.endsWith('/ConversationPane.tsx') + +test('a wait created before a setup longer than its bound neither rejects early nor misses a later response', async t => { + t.mock.timers.enable({ apis: ['setTimeout'] }) + const page = new EventEmitter() + const rejections = [] + const note = reason => rejections.push(reason) + process.on('unhandledRejection', note) + t.after(() => process.off('unhandledRejection', note)) + const wait = collectResponse(page, isPane) + t.mock.timers.tick(30_000) // setup takes three times the bound + await new Promise(setImmediate) + const loaded = wait.within(10_000, 'ConversationPane.tsx') + page.emit('response', other) + t.mock.timers.tick(9_000) + page.emit('response', pane) + assert.equal(await loaded, pane) + await new Promise(setImmediate) + assert.deepEqual(rejections, []) + assert.equal(page.listenerCount('response'), 0) +}) + +test('a response that arrives before anything waits is kept', async t => { + t.mock.timers.enable({ apis: ['setTimeout'] }) + const page = new EventEmitter() + const wait = collectResponse(page, isPane) + page.emit('response', pane) + t.mock.timers.tick(30_000) + assert.equal(await wait.within(10_000, 'ConversationPane.tsx'), pane) +}) + +test('a missing response still fails at the bound, counted from the wait', async t => { + t.mock.timers.enable({ apis: ['setTimeout'] }) + const page = new EventEmitter() + const wait = collectResponse(page, isPane) + t.mock.timers.tick(30_000) + const loaded = wait.within(10_000, 'ConversationPane.tsx') + page.emit('response', other) + t.mock.timers.tick(10_000) + await assert.rejects(loaded, /ConversationPane\.tsx did not arrive within 10000 ms/) + assert.equal(page.listenerCount('response'), 0) +}) diff --git a/apps/fractal-web/src/conversation/alerts/AgentAlerts.integration.test.tsx b/apps/fractal-web/src/conversation/alerts/AgentAlerts.integration.test.tsx index cfb046cab70..8985307335b 100644 --- a/apps/fractal-web/src/conversation/alerts/AgentAlerts.integration.test.tsx +++ b/apps/fractal-web/src/conversation/alerts/AgentAlerts.integration.test.tsx @@ -8,7 +8,8 @@ import type * as Native from '@fractal/client/schema' vi.mock('@stylexjs/stylex', () => import('../../testStylex.ts')) vi.mock('../../data/react.tsx', () => ({ useAttention: vi.fn(), useDataSource: vi.fn() })) -import { AgentAlerts, waitingCalls } from './AgentAlerts.tsx' +import { AgentAlerts } from './AgentAlerts.tsx' +import { Option } from 'effect' const seat = 'agent/example/atlas' @@ -34,8 +35,8 @@ const prompting = (episode: string, answers: readonly string[], callId: string): }) as unknown as Native.Attention const asking = (callId: string, labels: readonly string[]) => ({ - _tag: 'ToolCall', id: `item/${callId}`, callId, name: 'AskUserQuestion', callSeen: true, - input: { questions: [{ question: 'Which one?', header: 'Pick', multiSelect: false, options: labels.map((label) => ({ label, description: '' })) }] }, + call_id: callId, tool: 'AskUserQuestion', input_complete: true, + input: JSON.stringify({ questions: [{ question: 'Which one?', header: 'Pick', multiSelect: false, options: labels.map((label) => ({ label, description: '' })) }] }), }) let host: HTMLDivElement @@ -62,8 +63,8 @@ describe('AgentAlerts, pressed', () => { answered.push({ episode: (alert as unknown as { episode: string }).episode, answer, sent }) return { _tag: 'Sent' } as const } - const show = (alert: Native.Attention, items: Parameters[0]) => - act(async () => root.render()) + const show = (alert: Native.Attention, calls: Native.AttentionPromptCall[]) => + act(async () => root.render()) await show(prompting('episode-1', ['answer', 'deny'], 'q1'), [asking('q1', ['Red', 'Blue'])]) expect(button('Send')!.disabled).toBe(true) @@ -86,9 +87,10 @@ describe('AgentAlerts, pressed', () => { it('allows the one waiting call it shows, by pressing Allow', async () => { const answered: string[] = [] - const alert = { ...prompting('episode-3', ['allow', 'deny'], 'c1'), detail: 'Claude asks to use Bash' } as Native.Attention + const alert = { ...prompting('episode-3', ['allow', 'deny'], 'c1'), detail: 'Claude asks to use Bash', + presentation: { prompt: { calls: Option.some([{ call_id: 'c1', tool: 'Bash', input: '{"command":"make"}', input_complete: true }]), allow: true, answers: ['allow', 'deny'] } } } as unknown as Native.Attention await act(async () => root.render( - { answered.push(`${answer}:${sent}`); return { _tag: 'Sent' } }} />, )) await press('Allow') diff --git a/apps/fractal-web/src/conversation/alerts/AgentAlerts.tsx b/apps/fractal-web/src/conversation/alerts/AgentAlerts.tsx index 2af8c4f0e4a..ceeef58dc3a 100644 --- a/apps/fractal-web/src/conversation/alerts/AgentAlerts.tsx +++ b/apps/fractal-web/src/conversation/alerts/AgentAlerts.tsx @@ -1,19 +1,19 @@ import type * as Native from '@fractal/client/schema' import * as stylex from '@stylexjs/stylex' import * as React from 'react' +import { Option } from 'effect' import { alertConversations, alertsIn, answerLabel, - offeredAnswers, - pendingCalls, + servedAnswers, + servedPrompt, pendingQuestions, promptAnswers, promptAlertKey, promptCalls, questionAnswer, - type PendingCall, } from '../../../../../clients/typescript/fractal-views/homeView.ts' import { geometryVars as g, spaceVars as s } from '../../../../../packages/fractal-ui/src/assistant-ui/composition-tokens.stylex.ts' import { useAttention, useDataSource } from '../../data/react.tsx' @@ -22,15 +22,6 @@ import { tokens } from '../../ui-compat/tokens.stylex.ts' import { AlertButton } from './AlertButton.tsx' import type { AlertAction, AlertActionResult } from './alert-action-port.ts' import { AnswerAlert } from './AnswerAlert.tsx' - -/** What the strip reads of a conversation's items: its tool calls, joined with their results. */ -export type ConversationCall = { readonly _tag: string; readonly id: string } & Partial<{ - readonly callId: string - readonly name: string - readonly input: unknown - readonly result: unknown - readonly callSeen: boolean -}> import type { PromptAnswerResult } from './prompt-answer-port.ts' const styles = stylex.create({ @@ -84,16 +75,6 @@ export const promptAlerts = (items: readonly Native.Attention[], agentRef: strin export const conversationAlerts = (items: readonly Native.Attention[], agentRef: string): readonly Native.Attention[] => alertsIn(items, agentRef, undefined) -/** The tool calls a conversation shows with no result yet, in full, as st3-views reads them. */ -export const waitingCalls = (items: readonly ConversationCall[]): PendingCall[] => - pendingCalls( - items.flatMap((item) => - item._tag === 'ToolCall' && item.callSeen === true && item.result === undefined && item.callId !== undefined - ? [{ id: item.id, type: 'tool_call' as const, body: { call_id: item.callId, name: item.name ?? 'tool', arguments: item.input } }] - : [], - ), - ) - type Answer = (alert: Native.Attention, answer: string, sent: string) => Promise type Act = (alert: Native.Attention, action: AlertAction) => Promise @@ -106,12 +87,10 @@ type Act = (alert: Native.Attention, action: AlertAction) => Promise { @@ -119,7 +98,7 @@ export const AgentAlerts = ({ const one = (alert: Native.Attention) => alert.attention_kind === 'harness-prompt' // A new prompt or revision behind the same alert starts with nothing chosen. - ? + ? : const [first, ...rest] = alerts return ( @@ -141,21 +120,20 @@ export const AgentAlerts = ({ const PromptAlert = ({ alert, - calls, onAnswer, }: { readonly alert: Native.Attention - readonly calls: readonly PendingCall[] readonly onAnswer?: Answer }): React.ReactNode => { const prompt = promptAnswers(alert) - // Only the exact call the prompt asks about, when the daemon names it. - const shown = prompt === null ? calls : promptCalls(prompt, calls) + const card = alert.presentation?.prompt + const served = servedPrompt({ presentation: card === undefined ? undefined : { prompt: { ...card, calls: Option.getOrNull(card.calls) } } }) + const shown = prompt === null || served.calls === null ? [] : promptCalls(prompt, served.calls) const [chosen, setChosen] = React.useState([]) const [status, setStatus] = React.useState() const [sending, setSending] = React.useState(false) - const offered = prompt === null || onAnswer === undefined ? [] : offeredAnswers(prompt, calls) - const questions = pendingQuestions(shown) + const offered = prompt === null || onAnswer === undefined ? [] : servedAnswers(prompt, served) + const questions = pendingQuestions(shown.filter(call => call.complete === true)) const gated = prompt?.answers.find((answer) => answer === 'allow' || answer === 'approve' || answer === 'answer') const sendable = questions === null ? null : questionAnswer(questions, chosen) const choose = (question: number, option: number, multi: boolean) => @@ -181,8 +159,8 @@ const PromptAlert = ({
◆ {alert.title}
{alert.detail ?
{alert.detail}
: null} - {prompt === null ? ( -
Answer it in the seat's terminal.
+ {prompt === null || served.calls === null ? ( +
{served.calls === null ? `${served.why} ` : ''}Answer it in the agent's terminal.
) : questions !== null && gated === 'answer' ? ( questions.map((question, index) => (
@@ -213,7 +191,7 @@ const PromptAlert = ({
{prompt.call_id == null ? 'Fractal does not say which call this asks about: answer it in the seat\'s terminal.' - : 'What it would let through is not in the conversation yet: answer it in the seat\'s terminal.'} + : 'The prompt details are incomplete or unsupported: answer it in the agent\'s terminal.'}
) : null} {offered.length > 0 ? ( @@ -234,16 +212,13 @@ const PromptAlert = ({ /** The agent's prompt alerts from the active data source, answered through its prompt port. */ export const LiveAgentAlerts = ({ agentRef, - items, ux, }: { readonly agentRef: string - readonly items: readonly ConversationCall[] readonly ux?: UxTelemetry }): React.ReactNode => { const source = useDataSource() const attention = useAttention() - const calls = React.useMemo(() => waitingCalls(items), [items]) const alerts = React.useMemo( () => (attention._tag === 'Observed' ? conversationAlerts(attention.value, agentRef) : []), [attention, agentRef], @@ -289,5 +264,5 @@ export const LiveAgentAlerts = ({ answer: sent, }) } - return + return } diff --git a/apps/fractal-web/src/conversation/alerts/AgentAlerts.unit.test.tsx b/apps/fractal-web/src/conversation/alerts/AgentAlerts.unit.test.tsx index 8163fe23306..216afd526b7 100644 --- a/apps/fractal-web/src/conversation/alerts/AgentAlerts.unit.test.tsx +++ b/apps/fractal-web/src/conversation/alerts/AgentAlerts.unit.test.tsx @@ -2,6 +2,7 @@ import { renderToStaticMarkup } from 'react-dom/server' import { describe, expect, it, vi } from 'vitest' import { ClientError, type FractalClient } from '@fractal/client' import type * as Native from '@fractal/client/schema' +import { Option } from 'effect' // Node rendering exercises the real composition, not the CSS compiler. vi.mock('@stylexjs/stylex', () => ({ @@ -13,7 +14,7 @@ vi.mock('@stylexjs/stylex', () => ({ })) vi.mock('../../data/react.tsx', () => ({ useAttention: vi.fn(), useDataSource: vi.fn() })) -import { AgentAlerts, promptAlerts, waitingCalls } from './AgentAlerts.tsx' +import { AgentAlerts, promptAlerts } from './AgentAlerts.tsx' import { gatewayPromptAnswers } from './prompt-answer-port.ts' const seat = 'agent/example/atlas' @@ -42,11 +43,11 @@ const alert = (id: string, answers: readonly string[], detail = 'Claude asks to : {}, }) as unknown as Native.Attention -const call = (callId: string, name: string, input: unknown, result?: unknown) => - ({ _tag: 'ToolCall', id: `item/${callId}`, callId, name, input, callSeen: true, ...(result === undefined ? {} : { result }) }) +const call = (call_id: string, tool: string, input: unknown, input_complete = true) => + ({ call_id, tool, input: JSON.stringify(input), input_complete }) -const render = (alerts: readonly Native.Attention[], items: Parameters[0]) => - renderToStaticMarkup( ({ _tag: 'Sent' })} />) +const render = (alerts: readonly Native.Attention[], calls: Native.AttentionPromptCall[], allow = true) => + renderToStaticMarkup( ({ ...alert, presentation: { prompt: { calls: Option.some(calls), allow, answers: [] } } }) as unknown as Native.Attention)} onAnswer={async () => ({ _tag: 'Sent' })} />) describe('AgentAlerts', () => { it('shows only the open prompt alerts of this conversation', () => { @@ -60,7 +61,6 @@ describe('AgentAlerts', () => { it('shows the one waiting call in full beside allow and deny', () => { const html = render([alert('attention/one', ['allow', 'deny'], undefined, 'c1')], [ - call('done', 'Bash', { command: 'ls' }, { status: 'ok' }), call('c1', 'Bash', { command: 'rm -rf build\necho done' }), ]) expect(html).toContain('command: rm -rf build') @@ -88,7 +88,7 @@ describe('AgentAlerts', () => { const none = render([alert('attention/one', ['allow', 'deny'], undefined, 'c1')], []) expect(none).not.toContain('>Allow<') expect(none).toContain('>Deny<') - expect(none).toContain('not in the conversation yet') + expect(none).toContain('incomplete or unsupported') // A prompt that names no call is only denied here, even beside a lone waiting call. for (const waiting of [[call('c1', 'Bash', { command: 'make' })], [call('c1', 'Bash', { command: 'make' }), call('c2', 'Write', { file_path: 'notes.md' })]]) { const unnamed = render([alert('attention/one', ['allow', 'deny'])], waiting) @@ -120,7 +120,7 @@ describe('AgentAlerts', () => { const screen = render([alert('attention/mcp', ['use', 'use-all', 'skip'], 'New MCP server found in this project: probe-echo.')], []) for (const label of ['Use it', 'Use all', 'Skip']) expect(screen).toContain(`>${label}<`) const terminal = render([alert('attention/t', [], 'Codex asks a question')], []) - expect(terminal).toContain('Answer it in the seat's terminal.') + expect(terminal).toContain('Answer it in the agent's terminal.') }) it('offers a trust prompt\'s own answers, and a denied call allowed once, for its kind or dismissed', () => { @@ -129,6 +129,18 @@ describe('AgentAlerts', () => { const denied = render([alert('attention/denied', ['approve-once', 'approve-kind', 'dismiss'], 'Auto mode denied a Bash call.')], []) for (const label of ['Allow once', 'Allow this kind', 'Dismiss']) expect(denied).toContain(`>${label}<`) }) + + it('withholds gated answers and explains missing, incomplete or unsupported served prompts', () => { + const item = alert('attention/q', ['answer', 'deny'], undefined, 'q') + for (const tool of ['AskUserQuestion', 'request_user_input']) { + const html = render([item], [call('q', tool, { questions: [{ question: 'Pick?', options: [{ label: 'One' }] }] }, false)], false) + expect(html).not.toContain('>Send<') + expect(html).toContain('incomplete or unsupported') + } + expect(render([item], [call('q', 'request_user_input', { questions: [] })])).toContain('incomplete or unsupported') + expect(render([alert('a', ['allow', 'deny'], undefined, 'c')], [call('c', 'Bash', { command: 'make' })], false)).not.toContain('>Allow<') + expect(renderToStaticMarkup()).toContain('Answer it in the agent's terminal.') + }) }) describe('gatewayPromptAnswers', () => { diff --git a/apps/fractal-web/src/conversation/alerts/AnswerAlert.unit.test.tsx b/apps/fractal-web/src/conversation/alerts/AnswerAlert.unit.test.tsx index 00afdb0512b..3ca3345f2da 100644 --- a/apps/fractal-web/src/conversation/alerts/AnswerAlert.unit.test.tsx +++ b/apps/fractal-web/src/conversation/alerts/AnswerAlert.unit.test.tsx @@ -225,7 +225,7 @@ describe('AnswerAlert', () => { it('opens the first of several alerts and folds the rest under their titles', () => { const html = renderToStaticMarkup( - ({ _tag: 'Sent' })} />, + ({ _tag: 'Sent' })} />, ) expect(html).toContain('◆ 2 alerts') expect(html.indexOf('First?')).toBeLessThan(html.indexOf(' act(async () => root.render()) + const render = () => act(async () => root.render()) await render() expect(container.textContent).toContain('◆ 2 alerts') @@ -66,3 +66,19 @@ it('times each answered alert until the strip no longer shows it, and drops refu act(() => root.unmount()) container.remove() }) + +it('removes a served prompt card when its authoritative attention row closes', async () => { + vi.stubGlobal('IS_REACT_ACT_ENVIRONMENT', true) + data.attention = [{ ...ask('attention/question'), attention_kind: 'harness-prompt', title: 'Choose a color' }] + const container = document.createElement('div') + document.body.append(container) + const root = createRoot(container) + const render = () => act(async () => root.render()) + await render() + expect(container.textContent).toContain('Choose a color') + data.attention = [{ ...data.attention[0] as Native.Attention, state: 'resolved' }] + await render() + expect(container.querySelector('[data-testid="conversation-alerts"]')).toBeNull() + await act(async () => root.unmount()) + container.remove() +}) diff --git a/apps/fractal-web/src/embed/Embed.integration.test.tsx b/apps/fractal-web/src/embed/Embed.integration.test.tsx index bb228d03cae..141ae00eb70 100644 --- a/apps/fractal-web/src/embed/Embed.integration.test.tsx +++ b/apps/fractal-web/src/embed/Embed.integration.test.tsx @@ -2,6 +2,8 @@ import { act } from 'react' import { afterEach, beforeEach, expect, it, vi } from 'vitest' import type { Mock } from 'vitest' +import { createRoot } from 'react-dom/client' +import { Embed } from './Embed.tsx' vi.mock('@stylexjs/stylex', () => import('../testStylex.ts')) // The same adapter replacement as vite.embed.config.ts, without changing the live app tests. @@ -89,3 +91,30 @@ it('uses the narrow container inside a desktop window and keeps agent navigation expect(network).not.toHaveBeenCalled() } finally { await act(async () => dispose()); host.remove() } }) + +it('keeps repeated embed work logs and composers as named groups, not duplicate landmarks', async () => { + const host = document.createElement('div') + document.body.append(host) + const root = createRoot(host) + try { + await act(async () => root.render(<>)) + await vi.waitFor(async () => { + await act(async () => {}) + expect(host.querySelectorAll('[role="group"][aria-label^="Message composer"]')).toHaveLength(2) + }) + const logs = host.querySelectorAll('[data-testid="work-log"]') + const composers = host.querySelectorAll('[role="group"][aria-label^="Message composer"]') + expect(logs.length).toBeGreaterThan(1) + expect(composers).toHaveLength(2) + for (const composer of composers) { + const form = composer.querySelector('form') + expect(form).not.toBeNull() + expect(form?.getAttribute('role')).toBeNull() + expect(form?.getAttribute('aria-label')).toBeNull() + } + for (const element of [...logs, ...composers]) { + expect(element.getAttribute('role')).toBe('group') + expect(element.getAttribute('aria-label')).toBeTruthy() + } + } finally { await act(async () => root.unmount()); host.remove() } +}) diff --git a/apps/fractal-web/src/shell/sidebar/StatusIcon.tsx b/apps/fractal-web/src/shell/sidebar/StatusIcon.tsx index 07405871541..1b6857c8d6e 100644 --- a/apps/fractal-web/src/shell/sidebar/StatusIcon.tsx +++ b/apps/fractal-web/src/shell/sidebar/StatusIcon.tsx @@ -5,21 +5,9 @@ import { tokens } from '../../ui-compat/tokens.stylex.ts' import type { Agent, Feed } from '../../data/source.ts' -/** Accessible labels and visual tones for observed agent statuses. */ -export const statuses = { - working: { label: 'Working', tone: 'green' }, - waiting: { label: 'Needs you', tone: 'amber' }, - idle: { label: 'Idle', tone: 'gray' }, - pending: { label: 'Pending', tone: 'gray' }, - stale: { label: 'Stale observation', tone: 'gray' }, - offline: { label: 'Offline', tone: 'gray' }, - ended: { label: 'Ended', tone: 'gray' }, - suspended: { label: 'Suspended', tone: 'gray' }, - retired: { label: 'Retired', tone: 'gray' }, - unobserved: { label: 'Not observed', tone: 'gray' }, -} as const -/** Sidebar status derived from an agent observation and fleet freshness. */ -export type AgentStatus = keyof typeof statuses +export { statuses } from '../../../../../packages/fractal-ui/src/assistant-ui/sidebar/model.ts' +export type { AgentStatus } from '../../../../../packages/fractal-ui/src/assistant-ui/sidebar/model.ts' +import { statuses, type AgentStatus } from '../../../../../packages/fractal-ui/src/assistant-ui/sidebar/model.ts' /** Resolves status without treating unavailable or stale observations as live state. */ export const agentStatus = ({ agent, @@ -43,10 +31,11 @@ export const agentStatus = ({ : !agent.connected ? 'offline' : agent.activity === 'errored' - ? 'stale' + ? 'failed' : agent.activity const paths: Record = { + failed: <>, working: ( <> @@ -106,6 +95,7 @@ const styles = stylex.create({ icon: { display: 'inline-flex', flexShrink: 0, color: tokens['--ds-gray-900'] }, green: { color: tokens['--ds-green-900'] }, amber: { color: tokens['--ds-amber-900'] }, + red: { color: tokens['--ds-red-900'] }, spinner: { animationName: spin, animationDuration: '1s', @@ -130,6 +120,7 @@ export const StatusIcon = ({ styles.icon, statuses[status].tone === 'green' && styles.green, statuses[status].tone === 'amber' && styles.amber, + statuses[status].tone === 'red' && styles.red, )} > = { waiting: 0, + failed: 0, working: 1, pending: 2, idle: 3, diff --git a/apps/fractal-web/src/shell/sidebar/state.ts b/apps/fractal-web/src/shell/sidebar/state.ts index 35efde5ad9e..e2947264197 100644 --- a/apps/fractal-web/src/shell/sidebar/state.ts +++ b/apps/fractal-web/src/shell/sidebar/state.ts @@ -27,6 +27,11 @@ const filters = Atom.family((gateway: string) => defaultValue: defaultFilters, }), ) +export const SidebarTab = Schema.Literals(['agents', 'alerts', 'missions', 'fleet']) +export type SidebarTab = typeof SidebarTab.Type +const tab = Atom.family((gateway: string) => persistedAtom({ + key: `${gateway}:sidebar.tab`, schema: SidebarTab, defaultValue: 'agents', +})) const collapsed = Atom.family((gateway: string) => persistedAtom({ key: `${gateway}:sidebar.collapsedFolders`, @@ -38,6 +43,7 @@ const collapsed = Atom.family((gateway: string) => export const sidebarState = (gateway: string) => ({ filters: filters(gateway), collapsed: collapsed(gateway), + tab: tab(gateway), }) /** Cache the subscribed atoms themselves: a weak-cached wrapper could be collected between renders. */ const fixtureFilters = Atom.family((_gateway: string) => @@ -46,8 +52,12 @@ const fixtureFilters = Atom.family((_gateway: string) => const fixtureCollapsed = Atom.family((_gateway: string) => Atom.make([]).pipe(Atom.keepAlive), ) +const fixtureTab = Atom.family((_gateway: string) => + Atom.make('agents').pipe(Atom.keepAlive), +) /** Story registries are isolated, including filters and collapse; no writes to live browser state. */ export const fixtureSidebarState = (gateway: string) => ({ filters: fixtureFilters(gateway), collapsed: fixtureCollapsed(gateway), + tab: fixtureTab(gateway), }) diff --git a/apps/fractal-web/src/web/ComposerImages.stories.tsx b/apps/fractal-web/src/web/ComposerImages.stories.tsx index 484c5b7f89e..6ad917ea0f2 100644 --- a/apps/fractal-web/src/web/ComposerImages.stories.tsx +++ b/apps/fractal-web/src/web/ComposerImages.stories.tsx @@ -264,7 +264,7 @@ export const AllStates: Story = { const labels = ['Oversize image refusal', 'Image upload in progress', 'Several staged images', 'Sent image'] const sections = labels.map(name => canvas.getByRole('region', { name })) for (const [index, section] of sections.entries()) { - await expect(within(section).getByRole('form', { name: `Message composer, ${labels[index]}` })).toBeVisible() + await expect(within(section).getByRole('group', { name: `Message composer, ${labels[index]}` })).toBeVisible() } await refuseOversize(sections[0]!) await showUploading(sections[1]!) diff --git a/apps/fractal-web/src/web/ConversationPane.composition.integration.test.tsx b/apps/fractal-web/src/web/ConversationPane.composition.integration.test.tsx index e8105001b27..ba683fbc143 100644 --- a/apps/fractal-web/src/web/ConversationPane.composition.integration.test.tsx +++ b/apps/fractal-web/src/web/ConversationPane.composition.integration.test.tsx @@ -61,7 +61,7 @@ vi.mock('../data/react.tsx', async () => { useNowSnapshot: () => source.now, useGrants: () => ({ actions: 'ungranted', messageSend: source.sendable ? 'granted' : 'ungranted', attachmentUpload: source.sendable ? 'granted' : 'ungranted', terminalInput: 'ungranted', terminalControl: 'ungranted' }), usePausedNow: () => undefined, - useAttention: () => ({ _tag: 'Waiting' }), + useAttention: () => ({ _tag: 'Observed', freshness: 'live', value: [] }), }) }) @@ -175,6 +175,16 @@ const mount = async (ux?: UxTelemetry, strictMode = false) => { const text = () => container.textContent ?? '' describe('ConversationPane composition activation', () => { + it('does not create a question card from a running raw call without open attention', async () => { + source.feed = { _tag: 'Observed', freshness: 'live', value: { + items: [{ _tag: 'ToolCall', id: 'raw-question', callId: 'q1', name: 'AskUserQuestion', status: 'running', callSeen: true, + input: { questions: [{ question: 'Raw only?', options: [{ label: 'One' }] }] }, at: at(1) }], + hasOlder: false, observation: { empty: false }, + } } + await mount() + expect(container.querySelector('[data-testid="conversation-alerts"]')).toBeNull() + }) + it('keeps the parent composer mounted while child tabs remain read-only', async () => { source.feed = { _tag: 'Observed', freshness: 'live', value: { items: scenario, hasOlder: false, observation: { empty: false }, @@ -448,7 +458,7 @@ describe('ConversationPane composition activation', () => { expect(source.composerProps.every(props => props.readingColumn === true)).toBe(true) expect(source.composerProps.every(props => props.placeholder === undefined)).toBe(true) expect(source.composerProps.every(props => props.conversationLabel === 'Example Agent')).toBe(true) - expect(container.querySelector('form')?.getAttribute('aria-label')).toBe('Message composer, Example Agent') + expect(container.querySelector('form')?.parentElement?.getAttribute('aria-label')).toBe('Message composer, Example Agent') }) it('keeps the source-owned scroll command stable through settlement, echo and Retry until a new Send', async () => { @@ -914,7 +924,7 @@ describe('ConversationPane composition activation', () => { await mount(undefined, strictMode) const stop = container.querySelector('[data-testid="run-stop"]')! const message = container.querySelector('textarea[aria-label="Message"]')! - const composer = container.querySelector('form[aria-label="Message composer, Example Agent"]')! + const composer = container.querySelector('[role="group"][aria-label="Message composer, Example Agent"] form')! expect(message.disabled).toBe(!sendGranted) stop.focus() expect(document.activeElement).toBe(stop) diff --git a/apps/fractal-web/src/web/ConversationPane.tsx b/apps/fractal-web/src/web/ConversationPane.tsx index f8ed6cfff85..d37ebf557e5 100644 --- a/apps/fractal-web/src/web/ConversationPane.tsx +++ b/apps/fractal-web/src/web/ConversationPane.tsx @@ -250,7 +250,7 @@ const ConversationContent = React.memo(function ConversationContent({ agentRef, />}
{frozen === undefined ? null : } - {!readOnly && } + {!readOnly && } {/* An unreadable conversation keeps its composer and draft; the binding names why sending waits. */} {!readOnly &&
{ if (!(event.target instanceof HTMLTextAreaElement)) return diff --git a/apps/fractal-web/src/web/LiveAgentWorkspace.graph.integration.test.ts b/apps/fractal-web/src/web/LiveAgentWorkspace.graph.integration.test.ts index f25a25934b6..4be303c3286 100644 --- a/apps/fractal-web/src/web/LiveAgentWorkspace.graph.integration.test.ts +++ b/apps/fractal-web/src/web/LiveAgentWorkspace.graph.integration.test.ts @@ -89,7 +89,11 @@ describe('conversation eager graph boundary', () => { expect(graph.files).not.toContain(resolve(kit, 'src/assistant-ui/index.ts')) expect(graph.files).not.toContain(resolve(kit, 'src/assistant-ui/workbench/Workbench.tsx')) expect(graph.files).not.toContain(resolve(kit, 'src/assistant-ui/composition/DiffPanel.tsx')) - expect(graph.dependencies.some(specifier => /^refractor(?:\/|$)/.test(specifier))).toBe(true) + expect(graph.dependencies.some(specifier => /^refractor(?:\/|$)/.test(specifier))).toBe(false) + expect(graph.files).not.toContain(resolve(kit, 'src/assistant-ui/composition/syntaxGrammar.ts')) + const withGrammars = eagerGraph(resolve(web, 'ConversationPane.tsx'), true) + expect(withGrammars.files).toContain(resolve(kit, 'src/assistant-ui/composition/syntaxGrammar.ts')) + expect(withGrammars.dependencies).toContain('refractor/core') }) it('reaches the real diff viewer only through the on-demand Changes inspector', () => { @@ -97,7 +101,11 @@ describe('conversation eager graph boundary', () => { expect(graph.files).toContain(resolve(web, 'capturedChanges.ts')) expect(graph.files).toContain(resolve(kit, 'src/assistant-ui/composition/DiffPanel.tsx')) expect(graph.files).toContain(resolve(kit, 'src/assistant-ui/composition/Markdown.tsx')) - expect(graph.dependencies.some(specifier => /^refractor(?:\/|$)/.test(specifier))).toBe(true) + expect(graph.dependencies.some(specifier => /^refractor(?:\/|$)/.test(specifier))).toBe(false) + expect(graph.files).not.toContain(resolve(kit, 'src/assistant-ui/composition/syntaxGrammar.ts')) + const withGrammars = eagerGraph(resolve(web, 'ChangesInspector.tsx'), true) + expect(withGrammars.files).toContain(resolve(kit, 'src/assistant-ui/composition/syntaxGrammar.ts')) + expect(withGrammars.dependencies).toContain('refractor/core') expect(graph.files).not.toContain(resolve(web, 'ConversationPane.tsx')) expect(graph.files).not.toContain(resolve(kit, 'src/assistant-ui/index.ts')) expect(graph.files).not.toContain(resolve(kit, 'src/assistant-ui/workbench/Workbench.tsx')) diff --git a/apps/fractal-web/src/web/LiveAgentWorkspace.phone.integration.test.tsx b/apps/fractal-web/src/web/LiveAgentWorkspace.phone.integration.test.tsx index 0569117a379..c8033a43c37 100644 --- a/apps/fractal-web/src/web/LiveAgentWorkspace.phone.integration.test.tsx +++ b/apps/fractal-web/src/web/LiveAgentWorkspace.phone.integration.test.tsx @@ -176,9 +176,13 @@ it('switches Thread and Changes with arrow keys and connects the selected tab to expect(tabs[1]!.getAttribute('aria-selected')).toBe('true') expect(document.activeElement).toBe(tabs[1]) expect(host.querySelector('[aria-label="Captured changes"]')).not.toBeNull() - expect(document.getElementById(tabs[1]!.getAttribute('aria-controls')!)).toBe(host.querySelector('[role="tabpanel"]')) + const panel = host.querySelector(`[role="tabpanel"][aria-labelledby="${tabs[1]!.id}"]`) + expect(panel).not.toBeNull() + expect(document.getElementById(tabs[1]!.getAttribute('aria-controls')!)).toBe(panel) await act(async () => tabs[1]!.dispatchEvent(new KeyboardEvent('keydown', { key: 'ArrowLeft', bubbles: true, cancelable: true }))) expect(tabs[0]!.getAttribute('aria-selected')).toBe('true') + expect(panel!.getAttribute('aria-labelledby')).toBe(tabs[0]!.id) + expect(document.getElementById(tabs[0]!.getAttribute('aria-controls')!)).toBe(panel) expect(host.querySelector('[aria-label="Captured changes"]')).toBeNull() }) @@ -206,7 +210,7 @@ it('retains the draft, scroll position and WorkspaceBody viewport store across t await act(async () => window.dispatchEvent(new Event('resize'))) expect(host.querySelector('[aria-label="Expand sidebar"]') !== null).toBe(width < 900) expect(host.querySelector('[aria-label="Message agent"]')).toBe(field) - expect(host.querySelector('[role="tablist"]') !== null).toBe(width < 900) + expect(host.querySelector('[role="tablist"][aria-label="Workspace views"]') !== null).toBe(width < 900) expect(field.value).toBe('Unsent rotation draft') expect(host.querySelector('[data-testid="retained-conversation"]')).toBe(history) expect(history.scrollTop).toBe(672) diff --git a/apps/fractal-web/src/web/LiveAgentWorkspace.tsx b/apps/fractal-web/src/web/LiveAgentWorkspace.tsx index 18654d15074..f7a6273cf51 100644 --- a/apps/fractal-web/src/web/LiveAgentWorkspace.tsx +++ b/apps/fractal-web/src/web/LiveAgentWorkspace.tsx @@ -50,7 +50,8 @@ import { FolderAttentionSignal } from './AttentionControls.tsx' import { liveShowRows, liveSidebarTree } from './liveSidebarTree.ts' import { RosterLoading, RosterStaleNotice, RosterUnavailable, SidebarEmptyState } from './SidebarEmptyState.tsx' import { useAgentsLastObservedAt } from './lastObserved.tsx' -import { ResponsiveAgentSidebar, sidebarMinimumWidth, useSidebarShortcut } from './ResponsiveAgentSidebar.tsx' +import { ResponsiveAgentSidebar, sidebarMinimumWidth, useSidebarShortcut, useSidebarNavigation } from './ResponsiveAgentSidebar.tsx' +import { SidebarTabs } from '../../../../packages/fractal-ui/src/assistant-ui/sidebar/SidebarTabs.tsx' import { SidebarChromeFooter } from './SidebarChromeFooter.tsx' import { syncLine } from '../../../../packages/fractal-ui/src/assistant-ui/sync-presentation/sync-line.ts' import { RosterDecodeNotice, shownRosterRefs } from './LiveRoster.tsx' @@ -454,6 +455,13 @@ function Workspace({ ux, onSelectConversation, dimensions, navigation = browserN const folderSource = useDataSource() const sidebarSettings = (folderSource.mode === 'fixtures' ? fixtureSidebarState : sidebarState)(folderSource.gateway ?? folderSource.mode) const [filters, setFilters] = useAtom(sidebarSettings.filters) + const [sidebarTab, setSidebarTab] = useAtom(sidebarSettings.tab) + const showAgents = React.useCallback(() => { + setSidebarTab('agents') + if (narrow) agentsDrawer.current?.open() + else setCollapsed(false) + }, [setSidebarTab, narrow, setCollapsed]) + const shortcutHelp = useSidebarNavigation(showAgents) const machineMemory = useMachineMemory() const asking = useAskingAgents() // The query is the only Show state; live gateways start from this browser's remembered machines. @@ -576,7 +584,7 @@ function Workspace({ ux, onSelectConversation, dimensions, navigation = browserN
{call.detail ?? (call.status === 'running' ? 'No output received yet.' : 'No output recorded.')}
- const sidebarBody = <> + const sidebarBody = setSidebarTab('agents')} tabs={[{ id: 'agents', label: 'Agents', questionCount: agents.filter(agent => agent.ask._tag === 'Known' && agent.ask.value === 'question').length, needsYouCount: needsYou.size }]}> rosterRef.current?.querySelector('[role="row"]')?.focus()} /> {fleet._tag === 'Unavailable' ? : fleet._tag !== 'Observed' ? null @@ -606,7 +614,7 @@ function Workspace({ ux, onSelectConversation, dimensions, navigation = browserN open({ ref: MISSIONS_REF, presentation: 'detail' })} /> open({ ref: 'monitor/quota', presentation: 'detail' })} /> - + // The panes stay outside changing shell providers, whose context propagation would scan every retained turn. // Only the stable header DOM host and narrow geometry cross that boundary. return ( @@ -615,6 +623,7 @@ function Workspace({ ux, onSelectConversation, dimensions, navigation = browserN setResourcePanel((value) => ({ ...value, ...change })) }}> setPanelFraction(value / viewport) }}>
+ {shortcutHelp} {narrow ? null : undefined} onToggleCollapsed={() => setCollapsed(value => !value)} onWidthChange={setDragWidth} onWidthCommit={value => { setRatio(value / viewport); setDragWidth(undefined) }} onWidthReset={() => setRatio(256 / viewport)} headerActions={}>{sidebarBody}} { setDiffOpen(false); setResourcesAgent(undefined) }} dock={resourcesDockOpen ?
: dockedInspector !== undefined ? ( }> diff --git a/apps/fractal-web/src/web/Markdown.defer.integration.test.tsx b/apps/fractal-web/src/web/Markdown.defer.integration.test.tsx new file mode 100644 index 00000000000..8152a46ead7 --- /dev/null +++ b/apps/fractal-web/src/web/Markdown.defer.integration.test.tsx @@ -0,0 +1,57 @@ +// @vitest-environment jsdom +import * as React from 'react' +import { act } from 'react' +import { createRoot } from 'react-dom/client' +import { expect, it, vi } from 'vitest' +import { Markdown } from '../../../../packages/fractal-ui/src/assistant-ui/composition/Markdown.tsx' + +vi.mock('@stylexjs/stylex', () => import('../testStylex.ts')) +const grammar = vi.hoisted(() => { + let release!: () => void + let loaded!: () => void + const ready = new Promise(resolve => { release = resolve }) + const finished = new Promise(resolve => { loaded = resolve }) + return { ready, release, finished, loaded, requested: false } +}) +vi.mock('../../../../packages/fractal-ui/src/assistant-ui/composition/syntaxGrammar', async importOriginal => { + grammar.requested = true + await grammar.ready + const actual = await importOriginal() + grammar.loaded() + return actual +}) + +it('keeps readable prose, code, focus and wrap state while the real grammar module loads', async () => { + vi.stubGlobal('IS_REACT_ACT_ENVIRONMENT', true) + const container = document.createElement('div') + document.body.append(container) + const root = createRoot(container) + try { + await act(async () => root.render(\n```'} />)) + expect(grammar.requested).toBe(false) + expect(container.querySelector('h2')?.textContent).toBe('Ready now') + expect(container.querySelector('strong')?.textContent).toBe('prose') + expect(container.querySelector('code')?.textContent).toBe('literal