diff --git a/.github/workflows/_ci-relay.yml b/.github/workflows/_ci-relay.yml index a498336ae98..4155c334ede 100644 --- a/.github/workflows/_ci-relay.yml +++ b/.github/workflows/_ci-relay.yml @@ -54,6 +54,7 @@ jobs: with: path: | target/ci/buzz-relay + target/ci/buzz target/ci/git-credential-nostr target/ci/backend-integration-tests.tar.zst target/ci/postgres-tests.tar.zst @@ -84,7 +85,7 @@ jobs: env: RUSTC_WRAPPER: sccache run: | - cargo build --profile ci -p buzz-relay -p git-credential-nostr + cargo build --profile ci -p buzz-relay -p git-credential-nostr -p buzz-cli cargo nextest archive \ --cargo-profile ci \ -p buzz-db \ @@ -115,6 +116,7 @@ jobs: with: path: | target/ci/buzz-relay + target/ci/buzz target/ci/git-credential-nostr target/ci/backend-integration-tests.tar.zst target/ci/postgres-tests.tar.zst @@ -125,6 +127,7 @@ jobs: name: desktop-e2e-relay path: | target/ci/buzz-relay + target/ci/buzz target/ci/git-credential-nostr target/ci/backend-integration-tests.tar.zst target/ci/postgres-tests.tar.zst @@ -199,12 +202,13 @@ jobs: name: desktop-e2e-relay path: target/ci - name: Restore relay executable permission - run: chmod +x ./target/ci/buzz-relay + run: chmod +x ./target/ci/buzz-relay ./target/ci/buzz - name: PostgreSQL-backed tests env: BUZZ_POSTGRES_ADMIN_URL: postgres://buzz:${{ env.BUZZ_TEST_POSTGRES_PASSWORD }}@localhost:5432/postgres PGPASSWORD: ${{ env.BUZZ_TEST_POSTGRES_PASSWORD }} run: | + export BUZZ_TEST_BIN_DIR="$PWD/target/ci" scripts/postgres-test-run.sh \ --archive-file target/ci/postgres-tests.tar.zst diff --git a/Cargo.lock b/Cargo.lock index dd15b2c2227..0c383d65808 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1119,6 +1119,7 @@ dependencies = [ "clap", "diffy", "dirs", + "fs2", "hex", "infer", "nostr 0.44.7", @@ -8756,9 +8757,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.45" +version = "0.23.46" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d41d731c7d2f962d1ccc364cec258de3c0e93b38c2fb3ba97ac74513048d634" +checksum = "48e13bd8c0e9365c43cfa5c9e8f9ad49d3c8444926c9aac819e0e4dc503c8fdf" dependencies = [ "aws-lc-rs", "log", diff --git a/FORK_PATCHES.md b/FORK_PATCHES.md index 0d3693992e6..3d98511a9dd 100644 --- a/FORK_PATCHES.md +++ b/FORK_PATCHES.md @@ -596,7 +596,7 @@ New module: `false`. ## `desktop/src-tauri/Cargo.lock` -Direct Desktop gethostname dependency without version changes +Direct Desktop gethostname dependency without version changes Pin rustls 0.23.46 for the release, resolving RUSTSEC-2026-0346 through RUSTSEC-2026-0349 without unrelated dependency changes. New module: `false`. @@ -2617,10 +2617,11 @@ New module: `false`. ## `.github/workflows/_ci-relay.yml` -Integrate existing reynholm/buzz main 33f54de2 without retiring fork patches +Integrate existing reynholm/buzz main 33f54de2 without retiring fork patches Build, cache and transfer the CLI executable alongside relay artifacts for managed recovery tests in the PostgreSQL lane. New module: `false`. +- Required symbol: `export BUZZ_TEST_BIN_DIR="$PWD/target/ci"` - Verify: `just ci` ## `.github/workflows/_ci-rust.yml` @@ -2665,7 +2666,7 @@ New module: `false`. ## `Cargo.lock` -Integrate existing reynholm/buzz main 33f54de2 without retiring fork patches +Integrate existing reynholm/buzz main 33f54de2 without retiring fork patches Pin rustls 0.23.46 for the release, resolving RUSTSEC-2026-0346 through RUSTSEC-2026-0349 without unrelated dependency changes. New module: `false`. @@ -2673,11 +2674,13 @@ New module: `false`. ## `Justfile` -Integrate existing reynholm/buzz main 33f54de2 without retiring fork patches +Integrate existing reynholm/buzz main 33f54de2 without retiring fork patches Select the infra-free demo echo cancellation regression in the required nextest unit lane. New module: `false`. +- Required symbol: `test(=mesh_boot::tests::demo_echo_keeps_inflight_read_across_ticks_and_still_drains)` - Verify: `just ci` +- Verify: `just test-unit` ## `bin/.uv-0.12.21.pkg` @@ -2777,10 +2780,11 @@ New module: `false`. ## `crates/buzz-acp/src/lib.rs` -Integrate existing reynholm/buzz main 33f54de2 without retiring fork patches +Integrate existing reynholm/buzz main 33f54de2 without retiring fork patches Preserve the heartbeat collection call with a textual drift guard; full live-service-loop behavioral mutation coverage is not claimed. New module: `false`. +- Required symbol: `pool.queue_finished_background_work(&mut queue);` - Verify: `just ci` ## `crates/buzz-acp/src/pool.rs` @@ -2993,11 +2997,12 @@ New module: `false`. ## `crates/buzz-db/src/runtime/tests/thread_window_postgres_tests.rs` -Integrate existing reynholm/buzz main 33f54de2 without retiring fork patches +Integrate existing reynholm/buzz main 33f54de2 without retiring fork patches Verify production prebuild migration reaches the latest embedded migration while preserving the ingestion witness/index checks. New module: `false`. - Verify: `just ci` +- Verify: `scripts/postgres-test-run.sh -p buzz-db --lib --tests` ## `crates/buzz-db/src/store/community.rs` @@ -5174,11 +5179,14 @@ New module: `false`. ## `scripts/run-tests.sh` -Integrate existing reynholm/buzz main 33f54de2 without retiring fork patches +Integrate existing reynholm/buzz main 33f54de2 without retiring fork patches Keep the cargo-test fallback selecting the same demo echo receive-retention regression as nextest. New module: `false`. +- Required symbol: `buzz-relay demo echo receive retention` - Verify: `just ci` +- Verify: `bash -n scripts/run-tests.sh` +- Verify: `just test-unit` ## `.config/nextest.toml` @@ -5371,3 +5379,432 @@ New module: `true`. - Required symbol: `publish_linux.py --tag "$TAG" --artifacts linux-assets --publish` - Required symbol: `TRIGGERING_ACTOR: ${{ github.triggering_actor }}` - Verify: `python3 -m unittest discover -s scripts/fork/tests -v` + +## `TESTING.md` + +Document production-writer recovery, replica-floor premises, authenticated CLI/relay regression commands and evidence boundaries. + +New module: `false`. + +- Verify: `BUZZ_TEST_BIN_DIR="$PWD/target/debug" cargo test -p buzz-relay --lib handlers::ingest::postgres_tests::managed_delivery_tests:: -- --ignored --test-threads=1` + +## `crates/buzz-acp/TESTING.md` + +Document original-session collection, bounded ordinary preemption and bundled agent/MCP/CLI delivery runtime fixtures. + +New module: `false`. + +- Verify: `env -u BUZZ_ACP_SESSION_POLICY -u BUZZ_ACP_ALLOWED_RESPOND_TO -u BUZZ_ACP_LAZY_POOL -u BUZZ_ACP_IDLE_POOL_SLEEP cargo test -p buzz-acp background -- --test-threads=1` + +## `crates/buzz-acp/src/acp/process_tracking_tests.rs` + +Inherited integration inventory: real ACP adapter regressions distinguish resident providers, active task children and unrelated sessions. + +New module: `true`. + +- Required symbol: `prompt_work_is_tracked_under_worker_and_resident_provider` +- Verify: `env -u BUZZ_ACP_SESSION_POLICY -u BUZZ_ACP_ALLOWED_RESPOND_TO -u BUZZ_ACP_LAZY_POOL -u BUZZ_ACP_IDLE_POOL_SLEEP cargo test -p buzz-acp process_tracking_tests -- --test-threads=1` + +## `crates/buzz-acp/src/isolated_execution.rs` + +Inherited integration inventory: fresh-process isolated prompt executor with cancellation, deadlines and bounded shutdown; module retention does not assert a new background-delivery invocation. + +New module: `false`. + +- Required symbol: `pub(crate) async fn execute(` +- Required symbol: `agent.acp.shutdown().await;` +- Verify: `env -u BUZZ_ACP_SESSION_POLICY -u BUZZ_ACP_ALLOWED_RESPOND_TO -u BUZZ_ACP_LAZY_POOL -u BUZZ_ACP_IDLE_POOL_SLEEP cargo test -p buzz-acp --lib -- --test-threads=1` + +## `crates/buzz-acp/src/pool/background_work.rs` + +Retain collection authority in the original worker/session, poll child exit independently of pool sleep, and compose stable-key final-report instructions. + +New module: `true`. + +- Required symbol: `pub(crate) struct RuntimeHold` +- Required symbol: `pub fn queue_finished_background_work` +- Required symbol: `pub(super) fn followup_prompt` +- Invocation: `crates/buzz-acp/src/pool.rs` → `followup_prompt`; exact call `background_work::followup_prompt(batch.channel_id, continuation, &ctx.cwd)`; behavior test `background_managed_delivery_key_stays_on_collection_across_error_and_new_message`; verify `env -u BUZZ_ACP_SESSION_POLICY -u BUZZ_ACP_ALLOWED_RESPOND_TO -u BUZZ_ACP_LAZY_POOL -u BUZZ_ACP_IDLE_POOL_SLEEP cargo test -p buzz-acp background_managed_delivery_key_stays_on_collection_across_error_and_new_message -- --test-threads=1` +- Verify: `env -u BUZZ_ACP_SESSION_POLICY -u BUZZ_ACP_ALLOWED_RESPOND_TO -u BUZZ_ACP_LAZY_POOL -u BUZZ_ACP_IDLE_POOL_SLEEP cargo test -p buzz-acp background_work_tests -- --test-threads=1` + +## `crates/buzz-acp/src/pool/background_work_tests.rs` + +Production-dispatch local-process regressions for route/session retention, collection progress under repeated preemption, cancellation, rotation and lost-task cleanup. + +New module: `true`. + +- Required symbol: `background_collection_completes_under_repeated_ordinary_preemption` +- Required symbol: `background_explicit_cancel_after_ordinary_control_never_resumes_collection` +- Verify: `env -u BUZZ_ACP_SESSION_POLICY -u BUZZ_ACP_ALLOWED_RESPOND_TO -u BUZZ_ACP_LAZY_POOL -u BUZZ_ACP_IDLE_POOL_SLEEP cargo test -p buzz-acp background_work_tests -- --test-threads=1` + +## `crates/buzz-acp/src/pool/delivery_runtime_tests.rs` + +Run actual bundled agent, MCP and CLI processes against a signature-checking mock relay: application-error replay, lost acknowledgments, process restart and concurrency; not real-relay evidence. + +New module: `true`. + +- Required symbol: `managed_runtime_bundled_agent_publication_then_application_error_retries_one_event` +- Required symbol: `managed_runtime_killed_after_relay_acceptance_replays_on_new_process` +- Verify: `BUZZ_TEST_BIN_DIR="$PWD/target/debug" env -u BUZZ_ACP_SESSION_POLICY -u BUZZ_ACP_ALLOWED_RESPOND_TO -u BUZZ_ACP_LAZY_POOL -u BUZZ_ACP_IDLE_POOL_SLEEP cargo test -p buzz-acp managed_runtime_ -- --ignored --test-threads=1` + +## `crates/buzz-acp/src/pool/pi_prompt_tests.rs` + +Inherited integration inventory: Pi ACP system-prompt metadata, workspace skill installation and restore compatibility fixtures. + +New module: `false`. + +- Required symbol: `pi_composed_prompt_uses_meta_without_capability_negotiation` +- Required symbol: `real_pi_preserves_buzz_prompt_and_launch_skills_on_restore` +- Verify: `env -u BUZZ_ACP_SESSION_POLICY -u BUZZ_ACP_ALLOWED_RESPOND_TO -u BUZZ_ACP_LAZY_POOL -u BUZZ_ACP_IDLE_POOL_SLEEP cargo test -p buzz-acp pi_prompt_tests -- --test-threads=1` + +## `crates/buzz-acp/src/process_tree.rs` + +Inherited integration inventory: bounded process snapshots and resident-helper classification keep live task children out of idle-pool teardown. + +New module: `true`. + +- Required symbol: `pub fn snapshot()` +- Required symbol: `pub fn blocking_pids(` +- Required symbol: `pub fn is_resident_helper(` +- Invocation: `crates/buzz-acp/src/acp.rs` → `snapshot`; exact call `let tree = crate::process_tree::snapshot()?;`; behavior test `prompt_work_is_tracked_under_worker_and_resident_provider`; verify `env -u BUZZ_ACP_SESSION_POLICY -u BUZZ_ACP_ALLOWED_RESPOND_TO -u BUZZ_ACP_LAZY_POOL -u BUZZ_ACP_IDLE_POOL_SLEEP cargo test -p buzz-acp prompt_work_is_tracked_under_worker_and_resident_provider -- --test-threads=1` +- Verify: `env -u BUZZ_ACP_SESSION_POLICY -u BUZZ_ACP_ALLOWED_RESPOND_TO -u BUZZ_ACP_LAZY_POOL -u BUZZ_ACP_IDLE_POOL_SLEEP cargo test -p buzz-acp process_tree -- --test-threads=1` + +## `crates/buzz-acp/src/process_tree/snapshot_tests.rs` + +Inherited integration inventory: process-list timeout/output bounds, malformed snapshots and zombie exclusion tests. + +New module: `true`. + +- Required symbol: `snapshot_rejects_partial_output_and_ignores_zombies` +- Required symbol: `listing_timeout_kills_a_stalled_command` +- Verify: `env -u BUZZ_ACP_SESSION_POLICY -u BUZZ_ACP_ALLOWED_RESPOND_TO -u BUZZ_ACP_LAZY_POOL -u BUZZ_ACP_IDLE_POOL_SLEEP cargo test -p buzz-acp process_tree -- --test-threads=1` + +## `crates/buzz-acp/src/queue/background.rs` + +Keep background collection separate from ordinary messages with stable delivery UUID, original thread/session ownership, finite retries and bounded ordinary preemption. + +New module: `true`. + +- Required symbol: `pub(crate) struct RuntimeContinuation` +- Required symbol: `pub ordinary_preempted: bool` +- Required symbol: `background_managed_delivery_key_stays_on_collection_across_error_and_new_message` +- Invocation: `crates/buzz-acp/src/queue.rs` → `enqueue_continuation`; exact call `self.enqueue_continuation(batch.scope, continuation);`; behavior test `background_collection_completes_under_repeated_ordinary_preemption`; verify `env -u BUZZ_ACP_SESSION_POLICY -u BUZZ_ACP_ALLOWED_RESPOND_TO -u BUZZ_ACP_LAZY_POOL -u BUZZ_ACP_IDLE_POOL_SLEEP cargo test -p buzz-acp background_collection_completes_under_repeated_ordinary_preemption -- --test-threads=1` +- Verify: `env -u BUZZ_ACP_SESSION_POLICY -u BUZZ_ACP_ALLOWED_RESPOND_TO -u BUZZ_ACP_LAZY_POOL -u BUZZ_ACP_IDLE_POOL_SLEEP cargo test -p buzz-acp background -- --test-threads=1` + +## `crates/buzz-cli/Cargo.toml` + +Add filesystem locking and temporary-fixture dependencies for durable managed-report journals. + +New module: `false`. + +- Required symbol: `fs2 = "0.4"` +- Verify: `cargo test -p buzz-cli managed_ -- --test-threads=1` + +## `crates/buzz-cli/TESTING.md` + +Specify managed publisher scope, journal durability and retention, recovery opt-in/upgrade requirements and real versus mock-relay checks. + +New module: `false`. + +- Verify: `cargo test -p buzz-cli managed_ -- --test-threads=1` +- Verify: `BUZZ_TEST_BIN_DIR="$PWD/target/debug" cargo test -p buzz-relay --lib handlers::ingest::postgres_tests::managed_delivery_tests:: -- --ignored --test-threads=1` + +## `crates/buzz-cli/src/commands/delivery.rs` + +Lock, validate and fsync a bounded original-report journal before submission; refresh signed recovery stamps and persist only verified canonical receipts while preserving original evidence. + +New module: `true`. + +- Required symbol: `pub(super) async fn deliver(` +- Required symbol: `fn acknowledgment(` +- Required symbol: `fn durable_directory(` +- Required symbol: `recovery_candidate` +- Required symbol: `canonical_event` +- Invocation: `crates/buzz-cli/src/commands/messages.rs` → `deliver`; exact call `super::delivery::deliver( + client, + &channel, + &reply_to, + &content, + &delivery_key, + journal_dir.as_deref(), + mentions, + )`; behavior test `managed_same_key_changed_text_returns_first_signed_delivery`; verify `cargo test -p buzz-cli managed_same_key_changed_text_returns_first_signed_delivery -- --test-threads=1` +- Invocation: `crates/buzz-cli/src/commands/delivery.rs` → `submit_event`; exact call `write(&dir, &journal)?; + let result = client + .submit_event(submission)`; behavior test `managed_same_key_changed_text_returns_first_signed_delivery`; verify `cargo test -p buzz-cli managed_same_key_changed_text_returns_first_signed_delivery -- --test-threads=1` +- Verify: `cargo test -p buzz-cli managed_ -- --test-threads=1` +- Verify: `BUZZ_TEST_BIN_DIR="$PWD/target/debug" cargo test -p buzz-relay --lib handlers::ingest::postgres_tests::managed_delivery_tests:: -- --ignored --test-threads=1` + +## `crates/buzz-cli/src/commands/messages.rs` + +Expose messages deliver and preserve original route, stable UUID and signed callback mentions without suppressing ordinary progress or explicit edits. + +New module: `false`. + +- Required symbol: `MessagesCmd::Deliver {` +- Required symbol: `pub(super) fn event_mention_pubkeys(` +- Required symbol: `async fn resolve_content_mentions(` +- Invocation: `crates/buzz-cli/src/commands/delivery.rs` → `event_mention_pubkeys`; exact call `messages::event_mention_pubkeys(&chosen)`; behavior test `managed_recovery_mention_history_survives_lost_ack`; verify `BUZZ_TEST_BIN_DIR="$PWD/target/debug" cargo test -p buzz-relay --lib managed_recovery_mention_history_survives_lost_ack -- --ignored --test-threads=1` +- Verify: `cargo test -p buzz-cli managed_ -- --test-threads=1` +- Verify: `cargo test -p buzz-cli mention -- --test-threads=1` + +## `crates/buzz-cli/src/commands/mod.rs` + +Wire the managed delivery module into the CLI message dispatcher. + +New module: `false`. + +- Required symbol: `mod delivery;` +- Verify: `cargo test -p buzz-cli managed_ -- --test-threads=1` + +## `crates/buzz-cli/src/delivery_tests.rs` + +Production CLI regressions for same-key replay, namespace/route integrity, concurrency, durable ancestor links, bounded retention/retries and legacy wrapper rejection. + +New module: `true`. + +- Required symbol: `managed_same_key_changed_text_returns_first_signed_delivery` +- Required symbol: `managed_expired_journal_never_treats_legacy_wrapper_ack_as_delivery` +- Required symbol: `managed_existing_nested_directory_requires_ancestor_link_sync_before_publication` +- Verify: `cargo test -p buzz-cli managed_ -- --test-threads=1` + +## `crates/buzz-cli/src/error.rs` + +Expose managed-delivery pending as a safe same-key retry while preserving existing ambiguous unjournaled-send errors. + +New module: `false`. + +- Required symbol: `ManagedDeliveryPending` +- Verify: `cargo test -p buzz-cli managed_ -- --test-threads=1` + +## `crates/buzz-cli/src/lib.rs` + +Define messages deliver arguments and compile production-publisher regression tests. + +New module: `false`. + +- Required symbol: `Deliver {` +- Required symbol: `mod delivery_tests;` +- Verify: `cargo test -p buzz-cli managed_ -- --test-threads=1` + +## `crates/buzz-core/src/kind.rs` + +Reserve ephemeral managed recovery kind 20003 for signed canonical election; upgraded relays route it through shared ingest. + +New module: `false`. + +- Required symbol: `KIND_MANAGED_DELIVERY_RECOVERY: u32 = 20003` +- Invocation: `crates/buzz-relay/src/handlers/event.rs` → `is_ephemeral`; exact call `if is_ephemeral(kind_u32) + && kind_u32 != buzz_core::kind::KIND_NIP43_LEAVE_REQUEST + && kind_u32 != buzz_core::kind::KIND_MANAGED_DELIVERY_RECOVERY`; behavior test `managed_ws_original_ack_correlates_after_recovery_wins`; verify `BUZZ_TEST_BIN_DIR="$PWD/target/debug" cargo test -p buzz-relay --lib managed_ws_original_ack_correlates_after_recovery_wins -- --ignored --test-threads=1` +- Verify: `BUZZ_TEST_BIN_DIR="$PWD/target/debug" cargo test -p buzz-relay --lib handlers::ingest::postgres_tests::managed_delivery_tests:: -- --ignored --test-threads=1` + +## `crates/buzz-core/src/lib.rs` + +Export the shared signed managed-report recovery protocol. + +New module: `false`. + +- Required symbol: `pub mod managed_delivery;` +- Verify: `cargo test -p buzz-core managed_delivery -- --test-threads=1` + +## `crates/buzz-core/src/managed_delivery.rs` + +Validate author-signed original report and compact candidate stamp, immutable body/tags/route, canonical UUID and current wrapper/candidate timestamps. + +New module: `true`. + +- Required symbol: `pub struct RecoveryRequest` +- Required symbol: `pub struct ReportStamp` +- Required symbol: `pub fn delivery_key(` +- Required symbol: `pub fn same_report(` +- Required symbol: `pub fn decode(` +- Invocation: `crates/buzz-relay/src/handlers/ingest.rs` → `decode`; exact call `buzz_core::managed_delivery::RecoveryRequest::decode( + &event, + nostr::Timestamp::now().as_secs(), + )`; behavior test `managed_old_absent_report_recovers_with_production_writer`; verify `BUZZ_TEST_BIN_DIR="$PWD/target/debug" cargo test -p buzz-relay --lib managed_old_absent_report_recovers_with_production_writer -- --ignored --test-threads=1` +- Verify: `cargo test -p buzz-core managed_delivery -- --test-threads=1` + +## `crates/buzz-db/src/error.rs` + +Return an explicit canonical delivery conflict for UUID reuse with another report or route. + +New module: `false`. + +- Required symbol: `ManagedDeliveryConflict` +- Verify: `BUZZ_TEST_BIN_DIR="$PWD/target/debug" cargo test -p buzz-relay --lib handlers::ingest::postgres_tests::managed_delivery_tests:: -- --ignored --test-threads=1` + +## `crates/buzz-db/src/store/event.rs` + +Route ordinary marked kind-9 inserts through the same atomic canonical election as recovery while retaining transactional thread metadata and the production commit-time floor. + +New module: `false`. + +- Required symbol: `crate::store::managed_delivery::insert(` +- Invocation: `crates/buzz-db/src/store/event.rs` → `insert`; exact call `crate::store::managed_delivery::insert( + pool, + community_id, + event, + channel, + thread_meta, + None, + )`; behavior test `managed_recovery_and_original_race_share_one_election`; verify `BUZZ_TEST_BIN_DIR="$PWD/target/debug" cargo test -p buzz-relay --lib managed_recovery_and_original_race_share_one_election -- --ignored --test-threads=1` +- Verify: `BUZZ_TEST_BIN_DIR="$PWD/target/debug" cargo test -p buzz-relay --lib handlers::ingest::postgres_tests::managed_delivery_tests:: -- --ignored --test-threads=1` + +## `crates/buzz-db/src/store/managed_delivery.rs` + +Elect once per community/author/UUID under a transaction advisory lock; adopt accepted or soft-deleted originals, otherwise commit candidate, thread counters, callback mentions and durable binding together. + +New module: `true`. + +- Required symbol: `pub async fn insert(` +- Required symbol: `pub async fn recover_managed_delivery(` +- Required symbol: `pg_advisory_xact_lock` +- Required symbol: `INSERT INTO managed_deliveries` +- Invocation: `crates/buzz-relay/src/handlers/ingest.rs` → `recover_managed_delivery`; exact call `.recover_managed_delivery( + tenant.community(), + &event, + channel, + thread_params, + &request.original, + )`; behavior test `managed_old_accepted_report_recovers_with_production_writer`; verify `BUZZ_TEST_BIN_DIR="$PWD/target/debug" cargo test -p buzz-relay --lib managed_old_accepted_report_recovers_with_production_writer -- --ignored --test-threads=1` +- Invocation: `crates/buzz-db/src/store/managed_delivery.rs` → `insert_mentions_in_transaction`; exact call `crate::runtime::insert_mentions_in_transaction( + &mut tx, + community, + &result.0.event, + Some(channel), + )`; behavior test `managed_recovery_mention_history_survives_lost_ack`; verify `BUZZ_TEST_BIN_DIR="$PWD/target/debug" cargo test -p buzz-relay --lib managed_recovery_mention_history_survives_lost_ack -- --ignored --test-threads=1` +- Verify: `BUZZ_TEST_BIN_DIR="$PWD/target/debug" cargo test -p buzz-relay --lib handlers::ingest::postgres_tests::managed_delivery_tests:: -- --ignored --test-threads=1` + +## `crates/buzz-db/src/store/mod.rs` + +Register the canonical managed-delivery storage module. + +New module: `false`. + +- Required symbol: `pub mod managed_delivery;` +- Verify: `BUZZ_TEST_BIN_DIR="$PWD/target/debug" cargo test -p buzz-relay --lib handlers::ingest::postgres_tests::managed_delivery_tests:: -- --ignored --test-threads=1` + +## `crates/buzz-db/tests/observability_source.rs` + +Repair stale aggregate-cardinality and bounded-helper assumptions while retaining typed Authorization/EventWrite checkout and direct-pool bypass guards. + +New module: `false`. + +- Required symbol: `p0_pool_acquisitions_use_typed_operation_pairs_without_other` +- Verify: `cargo test -p buzz-db --test observability_source` + +## `crates/buzz-relay/src/handlers/ingest/postgres_tests/managed_delivery_tests.rs` + +Use the armed production writer and authenticated CLI/router to test absent/accepted/deleted recovery, races, WS acknowledgment correlation, escaped-size limits and callback Home Mentions after lost acknowledgments. + +New module: `true`. + +- Required symbol: `managed_recovery_and_original_race_share_one_election` +- Required symbol: `managed_ws_original_ack_correlates_after_recovery_wins` +- Required symbol: `managed_recovery_mention_history_survives_lost_ack` +- Required symbol: `buzz.created_at_floor` +- Verify: `BUZZ_TEST_BIN_DIR="$PWD/target/debug" cargo test -p buzz-relay --lib handlers::ingest::postgres_tests::managed_delivery_tests:: -- --ignored --test-threads=1` + +## `crates/buzz-relay/src/handlers/ingest/timestamp.rs` + +Keep the future ceiling and ordinary age checks; permit old self-marked reports to acknowledge retained acceptance while absent old inserts remain subject to the 960-second writer floor. + +New module: `true`. + +- Required symbol: `pub(super) fn validate(` +- Required symbol: `fn managed_report(` +- Required symbol: `managed_timestamp_exception_preserves_future_and_marker_authority_boundaries` +- Invocation: `crates/buzz-relay/src/handlers/ingest.rs` → `validate`; exact call `timestamp::validate(&event, &auth, nostr::Timestamp::now().as_secs())?;`; behavior test `managed_old_report_still_enforces_signature_route_membership_and_archive`; verify `BUZZ_TEST_BIN_DIR="$PWD/target/debug" cargo test -p buzz-relay --lib managed_old_report_still_enforces_signature_route_membership_and_archive -- --ignored --test-threads=1` +- Verify: `cargo test -p buzz-relay --lib timestamp::tests -- --test-threads=1` +- Verify: `BUZZ_TEST_BIN_DIR="$PWD/target/debug" cargo test -p buzz-relay --lib handlers::ingest::postgres_tests::managed_delivery_tests:: -- --ignored --test-threads=1` + +## `docs/superpowers/plans/2026-10-14-background-result-atomic-recovery.md` + +Record the owner-approved atomic canonical recovery plan and preserved replica-floor safety contract. + +New module: `true`. + +- Verify: `python3 scripts/fork/sync.py validate` + +## `docs/superpowers/plans/2026-10-14-background-result-delivery.md` + +Record original-session collection and durable managed publication implementation/verification plan. + +New module: `true`. + +- Verify: `python3 scripts/fork/sync.py validate` + +## `docs/superpowers/reports/2026-10-14-background-result-delivery.md` + +Preserve implementation scope, actual validation receipts, mutation evidence and remaining runtime/acceptance limits. + +New module: `true`. + +- Verify: `python3 scripts/fork/sync.py validate` + +## `docs/superpowers/specs/2026-10-14-background-result-delivery-task.md` + +Inherited integration inventory: agreed background collection/publication requirements and explicit authorization boundaries; registered as task input, not newly authored implementation. + +New module: `true`. + +- Verify: `python3 scripts/fork/sync.py validate` + +## `migrations/0056_managed_report_delivery.sql` + +Create bounded canonical delivery bindings independent of event redaction with community write fencing and per-author completion-key uniqueness. + +New module: `true`. + +- Required symbol: `CREATE TABLE managed_deliveries` +- Required symbol: `PRIMARY KEY (community_id, author, delivery_key)` +- Required symbol: `SELECT attach_community_write_fence('managed_deliveries');` +- Verify: `BUZZ_TEST_BIN_DIR="$PWD/target/debug" cargo test -p buzz-relay --lib handlers::ingest::postgres_tests::managed_delivery_tests:: -- --ignored --test-threads=1` + +## `desktop/src/shared/api/relayReconnectReplay.test.mjs` + +Keep channel replay lookback coupled to the shared relay timestamp helper and production ingest wiring plus the unchanged DB floor/clock margin. + +New module: `false`. + +- Required symbol: `channel replay lookback stays coupled to relay and DB source constants` +- Verify: `pnpm -C desktop test` + +## `scripts/check-postgres-test-discovery.py` + +Resolve conventional out-of-line Rust tests inside inline postgres_tests modules, retaining ancestor discovery context. + +New module: `false`. + +- Required symbol: `inline_ranges = module_ranges(source)` +- Verify: `bash scripts/test-postgres-test-discovery.sh` + +## `scripts/test-postgres-test-discovery.sh` + +Exercise nested file and directory Rust modules in the PostgreSQL discovery contract. + +New module: `false`. + +- Required symbol: `nested_out_of_line_recovery_test` +- Verify: `bash scripts/test-postgres-test-discovery.sh` + +## `deny.toml` + +Track the owner-approved RUSTSEC-2024-0449 exception for fork-v0.5.26-4 only, with YAML replacement issue19 and mandatory reassessment before another release. Other advisory policies remain unchanged. + +New module: `false`. + +- Required symbol: `RUSTSEC-2024-0449` +- Required symbol: `https://github.com/reynholm/buzz/issues/19` +- Verify: `cargo-deny check` + +## `crates/buzz-relay/src/mesh_boot.rs` + +Preserve one in-flight demo echo receive across drain ticks; retain bounded explicit shutdown and the existing validated frame/Goodbye handling. + +New module: `false`. + +- Required symbol: `tokio::pin!(receive);` +- Required symbol: `demo_echo_keeps_inflight_read_across_ticks_and_still_drains` +- Verify: `cargo test -p buzz-relay -- --test-threads=1` diff --git a/Justfile b/Justfile index c448027b680..0ce9308fb9e 100644 --- a/Justfile +++ b/Justfile @@ -478,6 +478,10 @@ test-unit: # `tests::postgres_tests::` stays in the PostgreSQL lane. cargo nextest run -p buzz-relay --lib --bin buzz-relay \ -E 'test(/^api::admin::/) + test(/^handlers::channel_authz::/) + test(/^handlers::moderation_authz::/) + test(/^handlers::side_effects::tests::/) + test(/^storage_sweep::tests::/) + test(/^nip_fi_core::tests::/) + test(/^nip_fi_http::tests::/) + test(/^nip_fi_config::tests::/) + test(/^readiness::tests::/) + test(/^router::tests::/) + test(/^api::parse_query_tests::/) + test(/^api::git::transport::off_mode_precedence_tests::/) + test(/^audio::join::tests::/) + test(/^audio::handler::tests::/) + test(/^nip_fi_gate::tests::/) + test(/^nip_fi_session::tests::/) + test(/^nip_fi_shadow::tests::/) + test(/^nip_fi_shadow_session::tests::/) + test(=state::tests::neither_a_confirmed_inactive_community_nor_a_failed_lookup_admits_the_socket) + test(=handlers::req::tests::timed_out_historical_read_deregisters_before_closed) + test(=handlers::req::tests::superseded_timeout_leaves_replacement_intact) + test(=handlers::req::tests::search_claim_retires_live_and_yields_to_replacement) + test(=handlers::req::tests::concurrent_claims_and_stale_teardowns_keep_the_last_owner) + test(=handlers::req::tests::timeout_closed_is_emitted_before_a_replacement_can_claim) + test(=handlers::req::tests::revoke_then_replacement_keeps_replacement_whole) + test(=handlers::req::tests::claims_after_connection_cleanup_are_refused) + test(=handlers::req::tests::dropped_terminal_frame_cancels_connection) + test(=handlers::req::tests::revoke_dropped_terminal_frame_cancels_connection) + (kind(bin) & (test(/^tests::/) + test(/^composition_tests::/) + test(/^env_filter_tests::/)) - test(/^tests::postgres_tests::/))' + # Ordinary drain ticks must not discard a partially read mesh frame. + # Infra-free; keep the cargo fallback in scripts/run-tests.sh in step. + cargo nextest run -p buzz-relay --lib \ + -E 'test(=mesh_boot::tests::demo_echo_keeps_inflight_read_across_ticks_and_still_drains)' # Note on audio::join::tests scope: the full suite is infra-free (no DB, # no Redis). The infra-free audio/FI regression witnesses — bootstrap # ordering barrier, CommitConfirmed arm, pending-close invisibility, diff --git a/TESTING.md b/TESTING.md index d939265c414..e70f5ac66b2 100644 --- a/TESTING.md +++ b/TESTING.md @@ -2,6 +2,28 @@ ## Automated Tests +For managed final-report recovery, explicitly run the ignored production-ingest +and standalone-CLI HTTP checks against an isolated schema-compatible Postgres +and Redis. The HTTP check uses real NIP-98 verification and replay protection: + +```sh +cargo build -p buzz-cli +BUZZ_TEST_BIN_DIR="$PWD/target/debug" \ + cargo test -p buzz-relay managed_old_ -- --ignored --test-threads=1 +cargo test -p buzz-relay managed_timestamp_exception -- --test-threads=1 +``` + +Set `BUZZ_TEST_DATABASE_URL` and `REDIS_URL` to the isolated services first. +These checks use `Db::new` and production migrations, assert the 960-second +writer floor and cover absent/accepted reports, earlier accepted candidates, +soft-deleted originals, candidate renewal, current authority and concurrent +original/recovery elections. Historical acceptance is explicitly seeded through +a separate fixture pool; tested recovery never disarms production connections. +Kind 20003 is a signed recovery command over the existing event bridge. The relay +atomically elects one signed kind-9 report and returns its canonical signed event. +A coordinated relay upgrade must drain old writers before enabling recovery; +unknown-server wrapper acknowledgment is not report confirmation. + ```bash just test-unit # unit tests — no infrastructure needed just test # unit + integration (starts Docker if needed) diff --git a/crates/buzz-acp/TESTING.md b/crates/buzz-acp/TESTING.md index c30938f691a..ce6ee0b5d45 100644 --- a/crates/buzz-acp/TESTING.md +++ b/crates/buzz-acp/TESTING.md @@ -22,7 +22,7 @@ cargo test -p buzz-acp ``` Managed agent sessions may already export harness options. Clear them when -running the package suite: three CLI parsing tests assert the unset defaults, +running the package suite: CLI parsing tests assert the unset defaults, and inherited values would change the inputs those tests exercise. Running the package serially also avoids scheduling flakes in existing short-deadline tests. @@ -30,6 +30,7 @@ package serially also avoids scheduling flakes in existing short-deadline tests. env -u BUZZ_ACP_ALLOWED_RESPOND_TO \ -u BUZZ_ACP_LAZY_POOL \ -u BUZZ_ACP_IDLE_POOL_SLEEP \ + -u BUZZ_ACP_SESSION_POLICY \ cargo test -p buzz-acp -- --test-threads=1 ``` @@ -45,8 +46,13 @@ BUZZ_TEST_PI_ACP=/absolute/buzz-pi-acp/dist/index.js \ On Unix, the `background` and `process_` test filters exercise real local ACP adapters and child processes. They cover provider residency, direct and nested work, bounded process enumeration, queue serialization, retry exhaustion, -session rotation, and completion while a result is being delivered. They do not -require provider credentials or an external relay. +session rotation, and completion while a result is being delivered. Marker-controlled +ACP adapters exercise `dispatch_pending` and result handling for collection before +and after new messages, backoff across successful ordinary turns, steer/interrupt, +original Channel/Thread/DM routes, generation replacement and channel revocation. +A retry-exhaustion test drives real ACP failures and captures the signed notice; +it advances queue retry eligibility directly to avoid waiting the full backoff. +They do not require provider credentials or an external relay. For a human check, build `buzz-acp` and use the existing relay/auth/adapter configuration with `--lazy-pool --idle-pool-sleep 2 --agents 2`: @@ -54,21 +60,47 @@ configuration with `--lazy-pool --idle-pool-sleep 2 --agents 2`: 1. Ask the agent to start a background task lasting at least ten seconds and finish its turn before the task exits. The worker must stay alive beyond the two-second idle bound, then collect the result once in the original thread. -2. Send a message while collection is running. It must follow the configured - queue/steer policy without a concurrent prompt in that conversation. -3. Repeat with `--idle-pool-sleep 0`, and without `--lazy-pool`. Completion must +2. Send a message after the child exits but before collection starts, while + collection is running, and during a failed collection's retry backoff. Repeat + with queue, steer, and interrupt. Ready collection runs first; collection + backoff permits ordinary messages without resetting its retry budget or + deadline. An interrupted collection resumes in the original session. + One ordinary cancel is permitted per collection; after that, ordinary + Interrupt/OwnerInterrupt and fallback-Steer queue behind its protected retry. + Native steer also queues during collection rather than extending its deadline. + Keep sending messages during the retry: collection must finish, then all + retained messages must reach a foreground turn. Explicit owner/observer + cancel and owner rotate still stop a protected retry. +3. Under `--session-policy channel`, send the new message in a different thread + while the child is still running. Its result must still be collected in the + first thread. Repeat with `--session-policy thread` and a DM. +4. Repeat with `--idle-pool-sleep 0`, and without `--lazy-pool`. Completion must still trigger collection. With Claude and no MCP servers, an ordinary turn must leave the worker available for other channels. -4. Repeat with `--max-turns-per-session 1`: collection must reuse the original +5. Repeat with `--max-turns-per-session 1`: collection must reuse the original session before rotation. In a separate run, send owner `!rotate` while the child is running: its exit must not trigger collection in a replacement - session. + session. Repeat with owner `!cancel` and removal from the channel: no old + collection may resume. Repeated collection errors must end with a failure + notice in the original thread and leave the worker available. + +The harness holds at most one observed process group's collection per worker. +Ready collections use original arrival age for FIFO among collections and take +precedence over ordinary batches. Their retry counters and deadlines are +separate from ordinary messages. Holds protect original worker/session ownership +until success, terminal failure, or explicit invalidation. This is an in-process +collection guarantee; successful ACP completion alone does not confirm publication. Process tracking follows children under the worker and resident providers; detached/reparented processes are outside that topology. ## Git bootstrap +The credential fixture requires Git 2.46 or newer for the `authtype` credential +protocol. Ensure the test subprocess PATH selects a compatible Git. Older Apple +Git can complete signing checks but fail `git credential fill` with a username +prompt error; that is a toolchain limitation, not a relay dependency. + `cargo test -p buzz-acp --test git_bootstrap` starts the actual harness with a probe adapter, runs real signed commits/tags and scoped credential resolution, and verifies key cleanup on startup failure and SIGTERM. No relay is contacted. @@ -86,3 +118,53 @@ Goose and uses its provider to invoke the native developer shell. Both operate only on temporary local repositories, verify commit/tag signatures and identity, check unrelated-remote credential scoping, and assert keyfile removal. They do not replace authenticated relay clone/push/readback testing. + +## Managed background final reports + +A `RuntimeContinuation` receives a random delivery UUID when its background +origin is recorded. Queue retries, ordinary message interleaving and healthy +steer/interrupt preserve it. A fresh observed group receives a fresh UUID. Only +collection prompts supply the key, original channel/parent and an explicitly +quoted workspace `.buzz-deliveries` path in `buzz messages deliver`. Progress +uses ordinary `messages send`; corrections use `messages edit`. Callback +mentions can use `--mention` or ordinary member-name mentions. Pending managed +publication must retry the same key/route/journal, including after ACP +application errors; changing the summary returns the first signed report. + +```sh +cargo build -p buzz-cli -p buzz-agent -p buzz-dev-mcp +BUZZ_TEST_BIN_DIR="$PWD/target/debug" \ + env -u BUZZ_ACP_ALLOWED_RESPOND_TO -u BUZZ_ACP_LAZY_POOL \ + -u BUZZ_ACP_IDLE_POOL_SLEEP -u BUZZ_ACP_SESSION_POLICY \ + cargo test -p buzz-acp managed_runtime_ -- --ignored --test-threads=1 +``` + +These four checks exercise actual CLI processes and actual bundled agent + MCP +shell, production collection dispatch and error handling, a model error after +publication, same-key replay with changed text, lost response bodies, process +kill after relay acceptance, concurrent callers and a workspace name containing +spaces/apostrophes. Readback counts unique verified signed events in a **local +mock relay** that follows the real relay's duplicate acknowledgment contract. +They do not use a real authenticated relay/provider or attest human testing. +The bundled-agent fixture seeds a preserved continuation in the original ACP +session; the separate background lifecycle tests exercise actual child groups. + +The persistent CLI journal recovers an already prepared signed event when the +same key/route is explicitly invoked after restart. ACP's queue, provider +session, group observation and ownership token remain in-memory; harness death +does not reconstruct an unjournaled lost result or automatically replay the +journal. Neither a successful ACP prompt nor arbitrary shell/MCP publication +proves managed delivery. The managed path covers kind-9 channel/thread reports, +including DM channel membership, only when the provider calls the supplied +command. See `crates/buzz-cli/TESTING.md` for bounded attempts, seven-day confirmed +retention, capacity/corruption handling and filesystem durability limits. + +For human smoke testing with the native app and an authorized local relay, +start a task with a real background child. Send another-thread traffic while it +runs and while collection backs off. Make collection publish through the +supplied command, then cause an ACP application error. Let the same session +retry with a changed summary: the app and CLI readback must show one original +report/event ID, in the original thread, with its callback mention. A second +background group must produce a second report/key. Explicit cancel/rotate and +membership removal still invalidate collection according to the lifecycle +runbook above. Agent fixtures cannot mark this human check complete. diff --git a/crates/buzz-acp/src/lib.rs b/crates/buzz-acp/src/lib.rs index 273e1c45cb5..a32345ec912 100644 --- a/crates/buzz-acp/src/lib.rs +++ b/crates/buzz-acp/src/lib.rs @@ -1620,6 +1620,7 @@ fn handle_relay_observer_control_event( keys: &nostr::Keys, event: nostr::Event, pool: &mut AgentPool, + queue: &mut EventQueue, observer: Option<&observer::ObserverHandle>, owner_pubkey_hex: &str, event_publisher: RelayEventPublisher, @@ -1663,7 +1664,7 @@ fn handle_relay_observer_control_event( let command_type = payload.get("type").and_then(|value| value.as_str()); match command_type { Some("cancel_turn") => { - handle_cancel_turn_control(&payload, pool, observer); + handle_cancel_turn_control(&payload, pool, queue, observer); } Some("switch_model") => { handle_switch_model_control(&payload, pool, observer); @@ -1824,10 +1825,12 @@ fn emit_project_owner_control_result( ); } -/// Handle a `cancel_turn` control frame: signal the in-flight task to cancel. +/// Cancel the unambiguous active turn, revoking its background authority even +/// when a prior ordinary control already consumed the one-shot sender. fn handle_cancel_turn_control( payload: &serde_json::Value, pool: &mut AgentPool, + queue: &mut EventQueue, observer: Option<&observer::ObserverHandle>, ) { let Some(channel_id) = payload @@ -1841,10 +1844,23 @@ fn handle_cancel_turn_control( let status = if pool.channel_control_is_ambiguous(channel_id) { "ambiguous_target" - } else if signal_in_flight_task(pool, channel_id, ControlSignal::Cancel) { - "sent" } else { - "no_active_turn" + // Use only the active task's actual scope. A channel-only frame cannot + // choose among sibling threads, or invent a scope when genuinely idle. + let active_scope = pool + .task_map() + .values() + .find(|meta| meta.channel_id == Some(channel_id)) + .and_then(|meta| meta.scope.clone()); + let revoked = active_scope + .as_ref() + .is_some_and(|scope| cancel_background_collection_for_scope(pool, queue, scope)); + let sent = signal_in_flight_task(pool, channel_id, ControlSignal::Cancel); + if sent || revoked { + "sent" + } else { + "no_active_turn" + } }; if let Some(observer) = observer { observer.emit( @@ -3235,6 +3251,7 @@ async fn run_harness( &config.keys, event, &mut pool, + &mut queue, observer.as_ref(), owner_hex, relay.event_publisher(), @@ -3436,12 +3453,15 @@ async fn run_harness( .await, &buzz_event.event, ); + let background_pending = cancel_background_collection_for_scope( + &mut pool, &mut queue, &scope, + ); let fired = signal_in_flight_task_for_scope( &mut pool, &scope, ControlSignal::Cancel, ); - if !fired { + if !fired && !background_pending { tracing::warn!( channel_id = %buzz_event.channel_id, scope = %scope.telemetry_label(), @@ -3488,6 +3508,7 @@ async fn run_harness( .await, &buzz_event.event, ); + queue.remove_continuation(&scope); let fired = signal_in_flight_task_for_scope( &mut pool, &scope, @@ -3500,7 +3521,6 @@ async fn run_harness( "!rotate received — cancelling in-flight turn and rotating session" ); } else { - queue.remove_continuation(&scope); let invalidated = pool.invalidate_scope_session(&scope); tracing::info!( @@ -4305,6 +4325,17 @@ fn signal_in_flight_task( .find(|m| m.channel_id == Some(channel_id)); if let Some(meta) = entry { + if matches!(mode, ControlSignal::Steer | ControlSignal::Interrupt) + && meta + .recoverable_batch + .as_ref() + .and_then(|batch| batch.continuation.as_ref()) + .is_some_and(|continuation| continuation.ordinary_preempted) + { + // Keep queued traffic for the next foreground turn. Repeated ordinary + // cancellation must not reset the collection's deadline indefinitely. + return false; + } if let Some(tx) = meta.control_tx.take() { tracing::info!(channel = %channel_id, ?mode, "control signal sent to in-flight task"); let _ = tx.send(mode); @@ -4314,6 +4345,35 @@ fn signal_in_flight_task( false } +/// Revoke background collection independently of the task's one-shot sender. +/// Clearing its recoverable token also suppresses a late cancellation result; +/// an idle scope with no background work retains its existing session. +fn cancel_background_collection_for_scope( + pool: &mut AgentPool, + queue: &mut EventQueue, + scope: &scope::SessionScope, +) -> bool { + if !queue.has_pending_continuation(scope) && !pool.has_background_collection_for_scope(scope) { + return false; + } + queue.remove_continuation(scope); + pool.invalidate_scope_session(scope); + for meta in pool + .task_map_mut() + .values_mut() + .filter(|meta| meta.scope.as_ref() == Some(scope)) + { + if meta + .recoverable_batch + .as_ref() + .is_some_and(|batch| batch.continuation.is_some()) + { + meta.recoverable_batch = None; + } + } + true +} + /// Send a control signal to the in-flight task for one exact session scope. /// /// The scope-precise counterpart of [`signal_in_flight_task`]: mid-turn @@ -4333,6 +4393,15 @@ fn signal_in_flight_task_for_scope( .find(|m| m.scope.as_ref() == Some(scope)); if let Some(meta) = entry { + if matches!(mode, ControlSignal::Steer | ControlSignal::Interrupt) + && meta + .recoverable_batch + .as_ref() + .and_then(|batch| batch.continuation.as_ref()) + .is_some_and(|continuation| continuation.ordinary_preempted) + { + return false; + } if let Some(tx) = meta.control_tx.take() { tracing::info!( channel = %scope.channel_id(), @@ -4378,6 +4447,17 @@ fn try_native_steer( be: queue::BatchEvent, steer_ack_tx: &mpsc::UnboundedSender, ) -> bool { + if pool.task_map().values().any(|meta| { + meta.scope.as_ref() == Some(&scope) + && meta + .recoverable_batch + .as_ref() + .is_some_and(|batch| batch.continuation.is_some()) + }) { + // Native steers can repeatedly extend a turn's deadline. Collections + // use the bounded ordinary cancel path, then queue subsequent traffic. + return false; + } let channel_id = scope.channel_id(); // Build the steer body: framing strings come from // `queue::native_steer_framing()` (Eva's drift-proof requirement — @@ -4891,7 +4971,22 @@ fn dispatch_pending( tracing::debug!(agent = agent.index, channel = %channel_id, scope = %scope.telemetry_label(), affinity_hit, "agent_claimed"); // Assign ownership before spawning and before cloning the retry token. - let owner_generation = pool.record_scope_owner(scope.clone(), agent.index); + // The pending collection's generation remains authoritative across + // serialized ordinary turns in its original worker/session. + let owner_generation = if agent.has_background_collection() { + agent + .background_context + .as_ref() + .filter(|hold| hold.scope == scope) + .and_then(|hold| { + hold.continuation + .as_ref() + .map(|continuation| continuation.generation) + }) + } else { + None + } + .unwrap_or_else(|| pool.record_scope_owner(scope.clone(), agent.index)); agent .state .set_scope_owner_generation(scope.clone(), owner_generation); @@ -5083,6 +5178,27 @@ fn handle_prompt_result( ) -> LoopAction { let before = pool.task_map().len(); let agent_index = result.agent.index; + // Continuations always have a recovery token at dispatch. Explicit owner + // cancellation can revoke it while the ordinary control is still draining. + let collection_cancelled = result + .batch + .as_ref() + .is_some_and(|batch| batch.continuation.is_some()) + && pool + .task_map() + .values() + .any(|meta| meta.agent_index == agent_index && meta.recoverable_batch.is_none()); + if collection_cancelled { + result.batch = None; + result.agent.finish_background_collection(); + } + let collecting = pool.task_map().values().any(|meta| { + meta.agent_index == agent_index + && meta + .recoverable_batch + .as_ref() + .is_some_and(|batch| batch.continuation.is_some()) + }); let successful_steer_deliveries = pool .task_map() .values() @@ -5244,6 +5360,14 @@ fn handle_prompt_result( } } + if collecting && !matches!(result.outcome, PromptOutcome::Ok(_)) { + if let PromptSource::Channel(scope) = &result.source { + if !queue.has_pending_continuation(scope) { + result.agent.finish_background_collection(); + } + } + } + match &result.source { PromptSource::Channel(scope) => queue.mark_complete(scope.clone()), PromptSource::Heartbeat => *heartbeat_in_flight = false, @@ -5508,6 +5632,7 @@ fn recover_panicked_agent( return; }; let i = meta.agent_index; + pool.clear_runtime_hold(i); // Requeue BEFORE mark_complete (same rationale as handle_prompt_result). if let Some(batch) = meta.recoverable_batch { @@ -6563,7 +6688,12 @@ mod owner_control_command_tests { "channelId": ch.to_string(), "modelId": "new-model", "requestId": "pick-1", }); - handle_cancel_turn_control(&payload, &mut pool, Some(&observer)); + handle_cancel_turn_control( + &payload, + &mut pool, + &mut EventQueue::new(DedupMode::Queue), + Some(&observer), + ); handle_switch_model_control(&payload, &mut pool, Some(&observer)); let results = observer.snapshot(); assert_eq!(results.len(), 2); @@ -6614,9 +6744,12 @@ mod owner_control_command_tests { "channelId": ch.to_string(), "modelId": "new-model", "requestId": "pick-1", }); match &signal { - ControlSignal::Cancel => { - handle_cancel_turn_control(&payload, &mut pool, Some(&observer)) - } + ControlSignal::Cancel => handle_cancel_turn_control( + &payload, + &mut pool, + &mut EventQueue::new(DedupMode::Queue), + Some(&observer), + ), _ => handle_switch_model_control(&payload, &mut pool, Some(&observer)), } assert_eq!(rx.await.unwrap(), signal); diff --git a/crates/buzz-acp/src/pool.rs b/crates/buzz-acp/src/pool.rs index 400b0a1788a..0c7d2124c70 100644 --- a/crates/buzz-acp/src/pool.rs +++ b/crates/buzz-acp/src/pool.rs @@ -45,7 +45,7 @@ use crate::relay::{ChannelInfo, RestClient}; use crate::scope::SessionScope; #[cfg(all(test, unix))] -mod background_work_tests; +pub(crate) mod background_work_tests; /// Window within which agent activity before a hard-cap death qualifies /// the turn as "recently active" (eligible for requeue instead of dead-letter). @@ -1086,8 +1086,9 @@ impl AgentPool { .unwrap_or(false) }); if let Some(i) = idx { - // A real prompt for this session replaces the pending follow-up. - let hold = self.runtime_holds.remove(&i); + // Keep pool-side authority visible while this worker is checked + // out, so a later explicit control can still revoke collection. + let hold = self.runtime_holds.get(&i).cloned(); let mut agent = self.agents[i].take()?; agent.background_context = hold; return Some(agent); @@ -2424,7 +2425,7 @@ fn send_prompt_result( batch: Option, ) { agent.acp.clear_steer_rx(); - if !matches!(outcome, PromptOutcome::Ok(_)) { + if !matches!(outcome, PromptOutcome::Ok(_)) && !agent.has_background_collection() { if let Some(context) = &mut agent.background_context { context.continuation = None; } @@ -2467,16 +2468,7 @@ pub async fn run_prompt_task( .as_ref() .and_then(|batch| { batch.continuation.as_ref().map(|continuation| { - format!( - "{}\nChannel: {}\nReply thread: {}", - background_work::FOLLOWUP_PROMPT, - batch.channel_id, - continuation - .thread_tags - .root_event_id - .as_deref() - .unwrap_or("none") - ) + background_work::followup_prompt(batch.channel_id, continuation, &ctx.cwd) }) }) .or(prompt_text); @@ -3303,7 +3295,12 @@ pub async fn run_prompt_task( { Ok(stop_reason) => { log_stop_reason(&source, &stop_reason); - agent.state.invalidate(&source); + if !matches!(control_signal, ControlSignal::Steer | ControlSignal::Interrupt) + || !agent.has_background_collection() + { + agent.state.invalidate(&source); + agent.finish_background_collection(); + } let retry_batch = requeue_cancelled_batch(&ctx, control_signal, batch); @@ -3451,6 +3448,15 @@ pub async fn run_prompt_task( agent.state.heartbeat_standing_context_sent = true; } + if batch + .as_ref() + .is_some_and(|batch| batch.continuation.is_some()) + { + // The original collection succeeded. Newly launched children, + // if any, form a fresh observed group in this same destination. + agent.background_context = None; + agent.remember_background_context(&source, batch.as_ref(), &session_id); + } let background_running = agent.background_work_running(&source); let should_rotate = matches!( stop_reason, @@ -12517,3 +12523,6 @@ done"# #[cfg(all(test, unix))] #[path = "pool/pi_prompt_tests.rs"] mod pi_prompt_tests; + +#[cfg(all(test, unix))] +mod delivery_runtime_tests; diff --git a/crates/buzz-acp/src/pool/background_work.rs b/crates/buzz-acp/src/pool/background_work.rs index 4a17f17b41c..928acdc8168 100644 --- a/crates/buzz-acp/src/pool/background_work.rs +++ b/crates/buzz-acp/src/pool/background_work.rs @@ -5,12 +5,40 @@ use crate::queue::{EventQueue, RuntimeContinuation}; pub(super) const FOLLOWUP_PROMPT: &str = "The background child process you left running has exited. Collect its result and continue the work in the same Buzz thread. Post the update there. Do not start a second copy of the task."; -/// An idle worker with live background work; an invalidated origin never resumes. -#[derive(Debug)] +/// Shell arguments stay literal even when workspace names contain apostrophes. +fn shell_argument(value: &str) -> String { + format!("'{}'", value.replace('\'', "'\\''")) +} + +pub(super) fn followup_prompt( + channel: Uuid, + continuation: &RuntimeContinuation, + cwd: &str, +) -> String { + let journal = std::path::Path::new(cwd).join(".buzz-deliveries"); + let parent = continuation + .thread_tags + .parent_event_id + .as_deref() + .or(continuation.thread_tags.root_event_id.as_deref()); + let command = parent.map(|parent| format!( + "buzz messages deliver --delivery-key {} --channel {} --reply-to {} --journal-dir {} --content ", + shell_argument(&continuation.delivery_key.to_string()), + shell_argument(&channel.to_string()), + shell_argument(parent), + shell_argument(&journal.to_string_lossy()), + )).unwrap_or_else(|| "Original reply event is unavailable; do not invent a destination or publish a report elsewhere.".into()); + format!("{FOLLOWUP_PROMPT}\nChannel: {channel}\nReply thread: {}\nPublish the final report using this managed command (replace with properly quoted text or use --content - and stdin):\n{command}\nKeep this delivery key, original route and journal directory for every retry, including after an application error. An unconfirmed managed delivery can be safely retried with the SAME key; never replace it with a fresh key or bare messages send. Replays return the first report even if the summary changes. Preserve callback notifications with --mention or ordinary @mentions. Use messages send for separate progress and messages edit for an explicit correction. Managed delivery does not cover other publishing tools.", continuation.thread_tags.root_event_id.as_deref().unwrap_or("none")) +} + +/// A worker with live or pending background work; an invalidated origin never resumes. +#[derive(Clone, Debug)] pub(crate) struct RuntimeHold { pub scope: SessionScope, pub blocking_pids: HashSet, pub continuation: Option, + /// Child exit has been queued; keep its session protected until collection ends. + pub collection_pending: bool, } /// Whether a queued completion can still use its original provider session. @@ -27,10 +55,16 @@ impl OwnedAgent { batch: Option<&FlushBatch>, session_id: &str, ) { + if self.has_background_collection() { + return; + } self.background_context = source.scope().map(|scope| RuntimeHold { scope: scope.clone(), blocking_pids: HashSet::new(), + collection_pending: false, continuation: Some(RuntimeContinuation { + delivery_key: Uuid::new_v4(), + ordinary_preempted: false, worker_index: self.index, session_id: session_id.to_owned(), generation: self @@ -45,15 +79,36 @@ impl OwnedAgent { }); } + pub(crate) fn has_background_collection(&self) -> bool { + self.background_context.as_ref().is_some_and(|hold| { + hold.continuation.is_some() + && (hold.collection_pending || !hold.blocking_pids.is_empty()) + }) + } + + /// Stop retaining an original collection after success or terminal failure. + pub(crate) fn finish_background_collection(&mut self) { + if let Some(hold) = &mut self.background_context { + hold.continuation = None; + hold.collection_pending = false; + // An empty completed hold must not turn into another collection. + hold.blocking_pids.clear(); + } + } + pub(super) fn background_work_running(&mut self, source: &PromptSource) -> bool { if source.scope().is_none() { return false; } let children = self.acp.background_child_pids(); if let (Some(hold), Some(children)) = (&mut self.background_context, &children) { - hold.blocking_pids = children.clone(); + // Retain evidence of observed work that just exited until the pool + // queues it; an ordinary turn must not rotate in this gap. + if !children.is_empty() { + hold.blocking_pids = children.clone(); + } } - children.is_none_or(|children| !children.is_empty()) + self.has_background_collection() || children.is_none_or(|children| !children.is_empty()) } } @@ -70,6 +125,12 @@ impl AgentPool { !self.runtime_holds.is_empty() } + pub(crate) fn has_background_collection_for_scope(&self, scope: &SessionScope) -> bool { + self.runtime_holds + .values() + .any(|hold| hold.scope == *scope && hold.continuation.is_some()) + } + pub(super) fn runtime_claim_allowed(&self, index: usize, scope: Option<&SessionScope>) -> bool { self.runtime_holds .get(&index) @@ -121,7 +182,13 @@ impl AgentPool { match agent.acp.background_child_pids() { // Work seen at prompt completion may exit while publishing metrics. // Keep that completion pending so the poll still queues collection. - Some(children) if children.is_empty() && hold.blocking_pids.is_empty() => return, + Some(children) + if children.is_empty() + && hold.blocking_pids.is_empty() + && !hold.collection_pending => + { + return + } Some(children) => hold.blocking_pids = children, None => {} // Unknown state must not authorize killing live work. } @@ -129,6 +196,10 @@ impl AgentPool { !self.continuation_matches(agent, &hold.scope, continuation) }) { hold.continuation = None; + hold.collection_pending = false; + if hold.blocking_pids.is_empty() { + return; + } } self.set_runtime_hold(agent.index, hold); } @@ -149,13 +220,24 @@ impl AgentPool { } continue; } - let Some(hold) = self.runtime_holds.remove(&index) else { + let Some(hold) = self.runtime_holds.get(&index) else { continue; }; - if let Some(continuation) = hold.continuation { - if let Some(Some(agent)) = self.agents.get(index) { - if self.continuation_matches(agent, &hold.scope, &continuation) { - queue.enqueue_continuation(hold.scope, continuation); + let valid = hold.continuation.as_ref().is_some_and(|continuation| { + self.agents[index].as_ref().is_some_and(|agent| { + self.continuation_matches(agent, &hold.scope, continuation) + }) + }); + if !valid { + self.runtime_holds.remove(&index); + continue; + } + if let Some(hold) = self.runtime_holds.get_mut(&index) { + hold.blocking_pids.clear(); + if !hold.collection_pending { + if let Some(continuation) = &hold.continuation { + queue.enqueue_continuation(hold.scope.clone(), continuation.clone()); + hold.collection_pending = true; } } } diff --git a/crates/buzz-acp/src/pool/background_work_tests.rs b/crates/buzz-acp/src/pool/background_work_tests.rs index 1ddcdd3160c..24fe02b56fd 100644 --- a/crates/buzz-acp/src/pool/background_work_tests.rs +++ b/crates/buzz-acp/src/pool/background_work_tests.rs @@ -53,6 +53,7 @@ async fn background_hold_cannot_be_claimed_by_another_cached_session() { scope: a.clone(), blocking_pids: HashSet::from([123]), continuation: None, + collection_pending: false, }, ); assert!( @@ -93,6 +94,7 @@ async fn background_hold_does_not_arm_an_unserviceable_fork_deadline() { scope: b, blocking_pids: HashSet::from([123]), continuation: None, + collection_pending: false, }, ); assert!(matches!( @@ -112,27 +114,56 @@ async fn background_hold_does_not_arm_an_unserviceable_fork_deadline() { // The child exits only when the test creates a marker. This exercises actual // process enumeration without scheduler-dependent sleep durations. const ADAPTER: &str = r#" -import json, subprocess, sys +import json, os, subprocess, sys, threading, time child = None +cancelled = threading.Event() +write_lock = threading.Lock() +def respond(req, result=None, error=None): + reply = {'jsonrpc': '2.0', 'id': req['id']} + reply['error' if error else 'result'] = error if error else result + with write_lock: + print(json.dumps(reply), flush=True) +def prompt(req): + text = str(req['params']['prompt']) + collecting = 'background child process' in text + if collecting: + open(os.path.join(os.path.dirname(sys.argv[1]), 'collection-started'), 'w').close() + while os.path.exists(os.path.join(os.path.dirname(sys.argv[1]), 'block-collection')) and not cancelled.is_set(): + time.sleep(.01) + if os.path.exists(os.path.join(os.path.dirname(sys.argv[1]), 'fail-collection')): + respond(req, error={'code': -32000, 'message': 'controlled collection failure'}) + return + if collecting and cancelled.is_set(): + open(os.path.join(os.path.dirname(sys.argv[1]), 'cancel-received'), 'w').close() + while os.path.exists(os.path.join(os.path.dirname(sys.argv[1]), 'hold-cancel-ack')): + time.sleep(.01) + next_group = os.path.join(os.path.dirname(sys.argv[1]), 'next-group') + if collecting and os.path.exists(next_group): + os.remove(next_group) + program = 'import os,sys,time\nwhile not os.path.exists(sys.argv[1]): time.sleep(.01)' + subprocess.Popen([sys.executable, '-c', program, os.path.join(os.path.dirname(sys.argv[1]), 'finish-next')]) + respond(req, {'stopReason': 'cancelled' if cancelled.is_set() else 'end_turn'}) for line in sys.stdin: req = json.loads(line) with open(sys.argv[2], 'a') as log: log.write(line) if req['method'] == 'initialize': - result = {'protocolVersion': 1, 'agentCapabilities': {}} + respond(req, {'protocolVersion': 1, 'agentCapabilities': {}}) elif req['method'] == 'session/new': - result = {'sessionId': 'original-session'} + respond(req, {'sessionId': 'original-session'}) elif req['method'] == 'session/prompt': if child is None: program = 'import os,sys,time\nwhile not os.path.exists(sys.argv[1]): time.sleep(.01)' child = subprocess.Popen([sys.executable, '-c', program, sys.argv[1]]) - result = {'stopReason': 'end_turn'} - else: - result = {} - print(json.dumps({'jsonrpc': '2.0', 'id': req['id'], 'result': result}), flush=True) + cancelled.clear() + threading.Thread(target=prompt, args=(req,)).start() + elif req['method'] == 'session/cancel': + cancelled.set() + elif 'id' in req: + respond(req, {}) "#; -async fn started_background_turn( +pub(crate) async fn started_background_turn( max_turns: u32, ) -> ( AgentPool, @@ -140,6 +171,27 @@ async fn started_background_turn( SessionScope, tempfile::TempDir, Arc, +) { + started_background_turn_in_scope( + max_turns, + SessionScope::Conversation { + channel_id: Uuid::new_v4(), + }, + false, + ) + .await +} + +async fn started_background_turn_in_scope( + max_turns: u32, + scope: SessionScope, + dm: bool, +) -> ( + AgentPool, + EventQueue, + SessionScope, + tempfile::TempDir, + Arc, ) { let dir = tempfile::tempdir().unwrap(); let mut agent = worker(0).await; @@ -158,9 +210,6 @@ async fn started_background_turn( .await .unwrap(); agent.acp.initialize().await.unwrap(); - let scope = SessionScope::Conversation { - channel_id: Uuid::new_v4(), - }; let mut pool = AgentPool::from_slots(vec![Some(agent), Some(worker(1).await)]); let event = EventBuilder::new(Kind::Custom(9), "start background work") .sign_with_keys(&Keys::generate()) @@ -192,6 +241,24 @@ async fn started_background_turn( .await; } }); + if dm { + ctx.channel_info.cache.write().unwrap().insert( + scope.channel_id(), + PromptChannelInfo { + name: "DM".into(), + channel_type: "dm".into(), + description: None, + project: None, + }, + ); + ctx.channel_info.projects.write().unwrap().insert( + scope.channel_id(), + CachedProjectInfo { + value: None, + fetched_at: std::time::Instant::now(), + }, + ); + } ctx.max_turns_per_session = max_turns; let ctx = Arc::new(ctx); assert_eq!( @@ -225,11 +292,22 @@ async fn started_background_turn( (pool, queue, scope, dir, ctx) } -async fn finish_child(pool: &mut AgentPool, queue: &mut EventQueue, dir: &tempfile::TempDir) { +pub(crate) async fn finish_child( + pool: &mut AgentPool, + queue: &mut EventQueue, + dir: &tempfile::TempDir, +) { std::fs::write(dir.path().join("finish"), "done").unwrap(); timeout(Duration::from_secs(5), async { - while pool.has_runtime_hold() { + loop { pool.queue_finished_background_work(queue); + if pool + .runtime_holds + .get(&0) + .is_none_or(|hold| hold.blocking_pids.is_empty()) + { + break; + } tokio::time::sleep(Duration::from_millis(20)).await; } }) @@ -422,3 +500,649 @@ async fn observed_background_exit_during_result_delivery_still_queues_collection pool.queue_finished_background_work(&mut queue); assert!(queue.flush_next().unwrap().continuation.is_some()); } + +pub(crate) fn push_new_message(queue: &mut EventQueue, scope: &SessionScope) { + let event = EventBuilder::new(Kind::Custom(9), "unrelated new request") + .tags([nostr::Tag::parse(["e", &"b".repeat(64), "", "root"]).unwrap()]) + .sign_with_keys(&Keys::generate()) + .unwrap(); + queue.push(QueuedEvent { + channel_id: scope.channel_id(), + scope: scope.clone(), + event, + prompt_tag: "test".into(), + received_at: std::time::Instant::now(), + edit: None, + }); +} + +pub(crate) fn dispatch( + pool: &mut AgentPool, + queue: &mut EventQueue, + ctx: &Arc, +) -> usize { + crate::dispatch_pending(pool, queue, ctx, &mut tokio::time::Instant::now(), None).len() +} + +async fn return_success(pool: &mut AgentPool, queue: &mut EventQueue) { + let result = timeout(Duration::from_secs(10), pool.result_rx.recv()) + .await + .unwrap() + .unwrap(); + assert!(matches!( + result.outcome, + PromptOutcome::Ok(StopReason::EndTurn) + )); + apply_result(pool, queue, result, None, &HashSet::new()); +} + +pub(crate) fn apply_result( + pool: &mut AgentPool, + queue: &mut EventQueue, + result: PromptResult, + rest: Option<&RestClient>, + removed: &HashSet, +) { + let config = crate::build_mcp_servers_tests::test_config(); + let mut crash_history: Vec<_> = (0..pool.agents.len()) + .map(|_| crate::SlotCircuit { + crash_times: Vec::new(), + open_until: None, + respawn_in_flight: false, + }) + .collect(); + let (respawn_tx, _) = mpsc::channel(8); + let mut respawn_tasks = tokio::task::JoinSet::new(); + crate::handle_prompt_result( + pool, + queue, + &config, + result, + &mut false, + removed, + &mut crash_history, + &respawn_tx, + &mut respawn_tasks, + None, + rest, + ); + assert!( + respawn_tasks.is_empty(), + "healthy adapter must stay reusable" + ); +} + +fn last_prompt(dir: &tempfile::TempDir) -> serde_json::Value { + std::fs::read_to_string(dir.path().join("requests")) + .unwrap() + .lines() + .map(|line| serde_json::from_str::(line).unwrap()) + .rfind(|r| r["method"] == "session/prompt") + .unwrap() +} + +#[tokio::test] +async fn background_ready_collection_survives_message_in_channel_thread_and_dm() { + for (scope, dm) in [ + ( + SessionScope::Conversation { + channel_id: Uuid::new_v4(), + }, + false, + ), + (scope(), false), + ( + SessionScope::Conversation { + channel_id: Uuid::new_v4(), + }, + true, + ), + ] { + let (mut pool, mut queue, scope, dir, ctx) = + started_background_turn_in_scope(0, scope, dm).await; + let original = pool.runtime_holds[&0] + .continuation + .as_ref() + .unwrap() + .thread_tags + .clone(); + finish_child(&mut pool, &mut queue, &dir).await; + push_new_message(&mut queue, &scope); + assert_eq!(dispatch(&mut pool, &mut queue, &ctx), 1); + assert!( + pool.task_map + .values() + .next() + .unwrap() + .recoverable_batch + .as_ref() + .unwrap() + .continuation + .is_some(), + "ready collection must precede the new request" + ); + return_success(&mut pool, &mut queue).await; + let prompt = last_prompt(&dir); + assert_eq!(prompt["params"]["sessionId"], "original-session"); + assert!(prompt["params"]["prompt"][0]["text"] + .as_str() + .unwrap() + .contains(original.root_event_id.as_ref().unwrap())); + assert_eq!(dispatch(&mut pool, &mut queue, &ctx), 1); + return_success(&mut pool, &mut queue).await; + assert!(!queue.has_undispatched_work()); + assert!(!pool.has_runtime_hold()); + } +} + +#[tokio::test] +async fn background_exit_survives_already_queued_message() { + let (mut pool, mut queue, scope, dir, ctx) = started_background_turn(0).await; + push_new_message(&mut queue, &scope); + finish_child(&mut pool, &mut queue, &dir).await; + assert_eq!(dispatch(&mut pool, &mut queue, &ctx), 1); + assert!(pool + .task_map + .values() + .next() + .unwrap() + .recoverable_batch + .as_ref() + .unwrap() + .continuation + .is_some()); + return_success(&mut pool, &mut queue).await; + assert!(last_prompt(&dir)["params"]["prompt"][0]["text"] + .as_str() + .unwrap() + .contains("background child process")); +} + +#[tokio::test] +async fn background_live_child_keeps_original_thread_after_other_thread_message() { + let (mut pool, mut queue, scope, dir, ctx) = started_background_turn(0).await; + let original = pool.runtime_holds[&0] + .continuation + .as_ref() + .unwrap() + .thread_tags + .clone(); + push_new_message(&mut queue, &scope); + assert_eq!(dispatch(&mut pool, &mut queue, &ctx), 1); + return_success(&mut pool, &mut queue).await; + finish_child(&mut pool, &mut queue, &dir).await; + assert_eq!(dispatch(&mut pool, &mut queue, &ctx), 1); + return_success(&mut pool, &mut queue).await; + let prompt = last_prompt(&dir); + let text = prompt["params"]["prompt"][0]["text"].as_str().unwrap(); + assert!( + text.contains(original.root_event_id.as_ref().unwrap()), + "{text}" + ); + assert!( + !text.contains(&"b".repeat(64)), + "another thread must not redirect collection" + ); +} + +#[tokio::test] +async fn background_backoff_message_cannot_rotate_original_session_at_max_turns_one() { + let (mut pool, mut queue, scope, dir, ctx) = started_background_turn(1).await; + finish_child(&mut pool, &mut queue, &dir).await; + let batch = queue.flush_next().unwrap(); + assert!(queue.requeue(batch).is_none()); + queue.mark_complete(&scope); + push_new_message(&mut queue, &scope); + assert_eq!(dispatch(&mut pool, &mut queue, &ctx), 1); + return_success(&mut pool, &mut queue).await; + assert_eq!( + pool.agents[0] + .as_ref() + .unwrap() + .state + .sessions + .get(&scope) + .map(String::as_str), + Some("original-session"), + "rotation must wait for collection even after child exit" + ); + assert!(queue.has_undispatched_work()); + assert!(queue.next_retry_deadline().is_some()); +} + +async fn wait_marker(dir: &tempfile::TempDir, name: &str) { + timeout(Duration::from_secs(5), async { + while !dir.path().join(name).exists() { + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .unwrap(); +} + +#[tokio::test] +async fn background_collection_resumes_original_session_after_steer_and_interrupt() { + for signal in [ControlSignal::Steer, ControlSignal::Interrupt] { + let (mut pool, mut queue, scope, dir, ctx) = started_background_turn(0).await; + finish_child(&mut pool, &mut queue, &dir).await; + std::fs::write(dir.path().join("block-collection"), "wait").unwrap(); + assert_eq!(dispatch(&mut pool, &mut queue, &ctx), 1); + wait_marker(&dir, "collection-started").await; + push_new_message(&mut queue, &scope); + assert_eq!( + dispatch(&mut pool, &mut queue, &ctx), + 0, + "same scope serializes" + ); + assert!(crate::signal_in_flight_task_for_scope( + &mut pool, + &scope, + signal.clone() + )); + let result = timeout(Duration::from_secs(10), pool.result_rx.recv()) + .await + .unwrap() + .unwrap(); + assert!(matches!(result.outcome, PromptOutcome::Cancelled)); + assert_eq!( + result.agent.state.sessions.get(&scope).map(String::as_str), + Some("original-session"), + "ordinary interruption must not invalidate the collection's original session" + ); + assert!(result.batch.as_ref().unwrap().continuation.is_some()); + apply_result(&mut pool, &mut queue, result, None, &HashSet::new()); + std::fs::remove_file(dir.path().join("block-collection")).unwrap(); + assert_eq!(dispatch(&mut pool, &mut queue, &ctx), 1); + assert!(pool + .task_map + .values() + .next() + .unwrap() + .recoverable_batch + .as_ref() + .unwrap() + .continuation + .is_some()); + return_success(&mut pool, &mut queue).await; + assert_eq!(last_prompt(&dir)["params"]["sessionId"], "original-session"); + assert_eq!(dispatch(&mut pool, &mut queue, &ctx), 1); + return_success(&mut pool, &mut queue).await; + } +} + +#[tokio::test] +async fn background_collection_completes_under_repeated_ordinary_preemption() { + for mode in [ + crate::config::MultipleEventHandling::Interrupt, + crate::config::MultipleEventHandling::Steer, + ] { + let (mut pool, mut queue, scope, dir, ctx) = started_background_turn(0).await; + finish_child(&mut pool, &mut queue, &dir).await; + std::fs::write(dir.path().join("block-collection"), "wait").unwrap(); + assert_eq!(dispatch(&mut pool, &mut queue, &ctx), 1); + wait_marker(&dir, "collection-started").await; + assert!(crate::signal_in_flight_task_for_scope( + &mut pool, + &scope, + ControlSignal::Interrupt + )); + let result = timeout(Duration::from_secs(10), pool.result_rx.recv()) + .await + .unwrap() + .unwrap(); + assert!(matches!(result.outcome, PromptOutcome::Cancelled)); + apply_result(&mut pool, &mut queue, result, None, &HashSet::new()); + std::fs::remove_file(dir.path().join("collection-started")).unwrap(); + assert_eq!(dispatch(&mut pool, &mut queue, &ctx), 1); + wait_marker(&dir, "collection-started").await; + let (ack_tx, _ack_rx) = tokio::sync::mpsc::unbounded_channel(); + let native_event = EventBuilder::new(Kind::Custom(9), "native steering must queue") + .sign_with_keys(&Keys::generate()) + .unwrap(); + let received_at = std::time::Instant::now(); + assert!(queue.push(QueuedEvent { + channel_id: scope.channel_id(), + scope: scope.clone(), + event: native_event.clone(), + prompt_tag: "test".into(), + received_at, + edit: None + })); + assert!(!crate::try_native_steer( + &mut pool, + &mut queue, + scope.clone(), + crate::queue::BatchEvent { + event: native_event, + prompt_tag: "test".into(), + received_at, + edit: None + }, + &ack_tx + )); + for i in 0..8 { + let event = EventBuilder::new(Kind::Custom(9), format!("ordinary message {i}")) + .sign_with_keys(&Keys::generate()) + .unwrap(); + let author = event.pubkey.to_hex(); + let id = event.id.to_hex(); + let received_at = std::time::Instant::now(); + let accepted = queue.push(QueuedEvent { + channel_id: scope.channel_id(), + scope: scope.clone(), + event: event.clone(), + prompt_tag: "test".into(), + received_at, + edit: None, + }); + crate::QueuedNormalListenerEvent { + accepted, + scope: scope.clone(), + effective_author: author, + reaction_target_id: id, + steer_event: crate::queue::BatchEvent { + event, + prompt_tag: "test".into(), + received_at, + edit: None, + }, + } + .steer_or_interrupt(mode, None, &mut pool, &mut queue, &ack_tx); + assert!( + pool.task_map.values().next().unwrap().control_tx.is_some(), + "repeated ordinary traffic must give collection an execution opportunity" + ); + assert_eq!(dispatch(&mut pool, &mut queue, &ctx), 0); + } + std::fs::remove_file(dir.path().join("block-collection")).unwrap(); + return_success(&mut pool, &mut queue).await; + assert_eq!(last_prompt(&dir)["params"]["sessionId"], "original-session"); + assert_eq!(dispatch(&mut pool, &mut queue, &ctx), 1); + return_success(&mut pool, &mut queue).await; + let prompt = last_prompt(&dir).to_string(); + for i in 0..8 { + assert!(prompt.contains(&format!("ordinary message {i}"))); + } + assert!(prompt.contains("native steering must queue")); + } +} + +#[tokio::test] +async fn background_generation_replacement_and_channel_revocation_never_resume_collection() { + for revoked in [false, true] { + let (mut pool, mut queue, scope, dir, ctx) = started_background_turn(0).await; + finish_child(&mut pool, &mut queue, &dir).await; + if revoked { + queue.drain_channel(scope.channel_id()); + pool.invalidate_channel_sessions(scope.channel_id()); + } else { + pool.record_scope_owner(scope.clone(), 1); + } + assert_eq!(dispatch(&mut pool, &mut queue, &ctx), 0); + pool.queue_finished_background_work(&mut queue); + assert!(!pool.has_runtime_hold()); + assert!(!queue.has_undispatched_work()); + let requests = std::fs::read_to_string(dir.path().join("requests")).unwrap(); + assert_eq!( + requests + .lines() + .filter(|line| line.contains("session/prompt")) + .count(), + 1, + "revoked or replaced ownership must never submit collection" + ); + } +} + +#[tokio::test] +async fn background_collection_blocks_heartbeat_and_idle_sleep_in_each_pool_mode() { + for (lazy, idle_secs) in [(false, 0), (true, 0), (true, 2)] { + let (mut pool, mut queue, _scope, dir, ctx) = started_background_turn(0).await; + // Keep only the original worker so heartbeat cannot use another slot. + pool.agents.pop(); + finish_child(&mut pool, &mut queue, &dir).await; + assert!(pool.try_claim(None).is_none()); + let now = tokio::time::Instant::now(); + assert!(!crate::idle_pool_sleep_due( + lazy, + now - Duration::from_secs(60), + now, + Duration::from_secs(idle_secs), + queue.has_in_flight(), + false, + queue.has_undispatched_work(), + false, + pool.has_runtime_hold() + )); + assert_eq!(dispatch(&mut pool, &mut queue, &ctx), 1); + return_success(&mut pool, &mut queue).await; + assert!(pool.try_claim(None).is_some()); + assert!(!queue.has_undispatched_work()); + } +} + +#[tokio::test] +async fn background_success_can_start_a_new_group_without_repeating_old_collection() { + let (mut pool, mut queue, scope, dir, ctx) = started_background_turn(1).await; + let original = pool.runtime_holds[&0] + .continuation + .as_ref() + .unwrap() + .thread_tags + .clone(); + let original_key = pool.runtime_holds[&0] + .continuation + .as_ref() + .unwrap() + .delivery_key; + finish_child(&mut pool, &mut queue, &dir).await; + std::fs::write(dir.path().join("next-group"), "start").unwrap(); + assert_eq!(dispatch(&mut pool, &mut queue, &ctx), 1); + return_success(&mut pool, &mut queue).await; + assert!(pool.has_runtime_hold(), "new child group retains residency"); + assert_ne!( + pool.runtime_holds[&0] + .continuation + .as_ref() + .unwrap() + .delivery_key, + original_key, + "new groups cannot suppress one another's report" + ); + assert!(!queue.has_undispatched_work(), "old collection completed"); + assert_eq!( + pool.agents[0] + .as_ref() + .unwrap() + .state + .sessions + .get(&scope) + .map(String::as_str), + Some("original-session") + ); + std::fs::write(dir.path().join("finish-next"), "done").unwrap(); + finish_child(&mut pool, &mut queue, &dir).await; + assert_eq!(dispatch(&mut pool, &mut queue, &ctx), 1); + return_success(&mut pool, &mut queue).await; + assert!(!pool.has_runtime_hold()); + assert!(!queue.has_undispatched_work()); + assert!( + !pool.agents[0] + .as_ref() + .unwrap() + .state + .sessions + .contains_key(&scope), + "rotation follows the final group collection" + ); + let requests = std::fs::read_to_string(dir.path().join("requests")).unwrap(); + assert_eq!( + requests + .lines() + .filter(|line| line.contains("session/prompt")) + .count(), + 3 + ); + assert!(last_prompt(&dir)["params"]["prompt"][0]["text"] + .as_str() + .unwrap() + .contains(original.root_event_id.as_ref().unwrap())); +} + +#[tokio::test] +async fn background_explicit_cancel_after_ordinary_control_never_resumes_collection() { + for ordinary in [ControlSignal::Steer, ControlSignal::Interrupt] { + let (mut pool, mut queue, scope, dir, ctx) = started_background_turn(0).await; + finish_child(&mut pool, &mut queue, &dir).await; + std::fs::write(dir.path().join("block-collection"), "wait").unwrap(); + std::fs::write(dir.path().join("hold-cancel-ack"), "wait").unwrap(); + assert_eq!(dispatch(&mut pool, &mut queue, &ctx), 1); + wait_marker(&dir, "collection-started").await; + push_new_message(&mut queue, &scope); + assert!(crate::signal_in_flight_task_for_scope( + &mut pool, &scope, ordinary, + )); + wait_marker(&dir, "cancel-received").await; + // The ordinary control already consumed the one-shot. The later + // explicit cancel must still revoke the checked-out collection. + assert!(crate::cancel_background_collection_for_scope( + &mut pool, &mut queue, &scope + )); + assert!(!crate::signal_in_flight_task_for_scope( + &mut pool, + &scope, + ControlSignal::Cancel, + )); + std::fs::remove_file(dir.path().join("hold-cancel-ack")).unwrap(); + let result = timeout(Duration::from_secs(10), pool.result_rx.recv()) + .await + .unwrap() + .unwrap(); + assert!(matches!(result.outcome, PromptOutcome::Cancelled)); + apply_result(&mut pool, &mut queue, result, None, &HashSet::new()); + assert!(!queue.has_pending_continuation(&scope), "explicit cancellation must suppress the returned continuation, without a stale-session failure notice"); + assert!(!pool.has_runtime_hold()); + std::fs::remove_file(dir.path().join("block-collection")).unwrap(); + assert_eq!(dispatch(&mut pool, &mut queue, &ctx), 1); + assert!( + !pool.task_map.values().any(|meta| meta + .recoverable_batch + .as_ref() + .is_some_and(|batch| batch.continuation.is_some())), + "explicit cancel after an ordinary control must never revive collection" + ); + return_success(&mut pool, &mut queue).await; + assert!( + !crate::cancel_background_collection_for_scope(&mut pool, &mut queue, &scope), + "genuinely idle cancel remains a no-op" + ); + assert!(pool.agents[0] + .as_ref() + .unwrap() + .state + .sessions + .contains_key(&scope)); + } +} + +#[tokio::test] +async fn background_lost_prompt_task_releases_pool_side_hold_before_respawn() { + let (mut pool, mut queue, scope, dir, ctx) = started_background_turn(0).await; + finish_child(&mut pool, &mut queue, &dir).await; + std::fs::write(dir.path().join("block-collection"), "wait").unwrap(); + assert_eq!(dispatch(&mut pool, &mut queue, &ctx), 1); + wait_marker(&dir, "collection-started").await; + pool.join_set.abort_all(); + let error = loop { + if let Err(error) = pool.join_set.join_next().await.unwrap() { + break error; + } + }; + let mut config = crate::build_mcp_servers_tests::test_config(); + config.agent_command = "true".into(); + config.agent_args.clear(); + let mut crash_history: Vec<_> = (0..pool.agents.len()) + .map(|_| crate::SlotCircuit { + crash_times: Vec::new(), + open_until: None, + respawn_in_flight: false, + }) + .collect(); + let (respawn_tx, _) = mpsc::channel(8); + let mut respawn_tasks = tokio::task::JoinSet::new(); + crate::recover_panicked_agent( + &mut pool, + &mut queue, + &config, + error, + &mut false, + &HashSet::new(), + &mut HashMap::new(), + &mut crash_history, + &respawn_tx, + &mut respawn_tasks, + None, + ); + assert!( + !pool.has_runtime_hold(), + "a lost worker must not retain background residency during circuit cooldown" + ); + assert!( + queue.has_pending_continuation(&scope), + "existing failure recovery remains queued" + ); + respawn_tasks.abort_all(); +} + +#[tokio::test] +async fn background_observer_cancel_after_ordinary_control_never_resumes_collection() { + for ordinary in [ControlSignal::Steer, ControlSignal::Interrupt] { + let (mut pool, mut queue, scope, dir, ctx) = + started_background_turn_in_scope(0, scope(), false).await; + finish_child(&mut pool, &mut queue, &dir).await; + std::fs::write(dir.path().join("block-collection"), "wait").unwrap(); + std::fs::write(dir.path().join("hold-cancel-ack"), "wait").unwrap(); + assert_eq!(dispatch(&mut pool, &mut queue, &ctx), 1); + wait_marker(&dir, "collection-started").await; + push_new_message(&mut queue, &scope); + assert!(crate::signal_in_flight_task_for_scope( + &mut pool, &scope, ordinary + )); + wait_marker(&dir, "cancel-received").await; + let observer = observer::ObserverHandle::in_process(); + let payload = serde_json::json!({"channelId": scope.channel_id().to_string(), "requestId": "stop-collection"}); + crate::handle_cancel_turn_control(&payload, &mut pool, &mut queue, Some(&observer)); + assert_eq!( + observer.snapshot()[0].payload["status"], + "sent", + "explicit observer cancel must revoke collection even without a sender" + ); + std::fs::remove_file(dir.path().join("hold-cancel-ack")).unwrap(); + let result = timeout(Duration::from_secs(10), pool.result_rx.recv()) + .await + .unwrap() + .unwrap(); + assert!(matches!(result.outcome, PromptOutcome::Cancelled)); + apply_result(&mut pool, &mut queue, result, None, &HashSet::new()); + assert!(!queue.has_pending_continuation(&scope)); + assert!(!pool.has_runtime_hold()); + std::fs::remove_file(dir.path().join("block-collection")).unwrap(); + assert_eq!(dispatch(&mut pool, &mut queue, &ctx), 1); + assert!(!pool.task_map.values().any(|meta| meta + .recoverable_batch + .as_ref() + .is_some_and(|batch| batch.continuation.is_some()))); + return_success(&mut pool, &mut queue).await; + crate::handle_cancel_turn_control(&payload, &mut pool, &mut queue, Some(&observer)); + assert_eq!( + observer.snapshot().last().unwrap().payload["status"], + "no_active_turn" + ); + assert!(pool.agents[0] + .as_ref() + .unwrap() + .state + .sessions + .contains_key(&scope)); + } +} diff --git a/crates/buzz-acp/src/pool/delivery_runtime_tests.rs b/crates/buzz-acp/src/pool/delivery_runtime_tests.rs new file mode 100644 index 00000000000..ccc3d4ccad9 --- /dev/null +++ b/crates/buzz-acp/src/pool/delivery_runtime_tests.rs @@ -0,0 +1,547 @@ +//! Opt-in production runtime checks: real bundled ACP agent, MCP shell and CLI, +//! with deterministic local model and signature-checking mock relay readback. +use super::*; +use crate::queue::{EventQueue, RuntimeContinuation}; +use nostr::{Event, EventBuilder, Kind, Tag}; +use std::{ + path::PathBuf, + sync::atomic::{AtomicBool, AtomicUsize, Ordering}, +}; +use tokio::io::{AsyncReadExt, AsyncWriteExt}; + +async fn request(socket: &mut tokio::net::TcpStream) -> (String, serde_json::Value, Vec) { + let mut bytes = Vec::new(); + let mut buf = [0; 8192]; + let end = loop { + let n = socket.read(&mut buf).await.unwrap(); + assert_ne!(n, 0); + bytes.extend_from_slice(&buf[..n]); + assert!(bytes.len() < 2_000_000); + if let Some(end) = bytes.windows(4).position(|s| s == b"\r\n\r\n") { + break end + 4; + } + }; + let headers = String::from_utf8_lossy(&bytes[..end]); + let route = headers + .lines() + .next() + .unwrap() + .split_whitespace() + .nth(1) + .unwrap() + .to_string(); + let length: usize = headers + .lines() + .find_map(|line| { + line.to_lowercase() + .strip_prefix("content-length:") + .map(|s| s.trim().parse().unwrap()) + }) + .unwrap(); + assert!(length < 1_900_000); + while bytes.len() < end + length { + let n = socket.read(&mut buf).await.unwrap(); + assert_ne!(n, 0); + bytes.extend_from_slice(&buf[..n]); + } + let body = bytes[end..end + length].to_vec(); + (route, serde_json::from_slice(&body).unwrap(), body) +} + +async fn response(socket: &mut tokio::net::TcpStream, status: &str, body: serde_json::Value) { + let body = body.to_string(); + let response = format!("HTTP/1.1 {status}\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", body.len()); + let _ = socket.write_all(response.as_bytes()).await; +} + +struct MockRelay { + url: String, + keys: nostr::Keys, + channel: Uuid, + parent: Event, + events: Arc>>, + bodies: Arc>>>, + losses: Arc, + hold: Arc, + accepted: Arc, + task: tokio::task::JoinHandle<()>, +} + +impl MockRelay { + async fn start() -> Self { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let url = format!("http://{}", listener.local_addr().unwrap()); + let keys = nostr::Keys::generate(); + let channel = Uuid::new_v4(); + let parent = EventBuilder::new(Kind::Custom(9), "original task") + .tag(Tag::parse(["h", &channel.to_string()]).unwrap()) + .sign_with_keys(&keys) + .unwrap(); + let membership = EventBuilder::new(Kind::Custom(39002), "") + .tags([ + Tag::parse(["d", &channel.to_string()]).unwrap(), + Tag::parse(["p", &keys.public_key().to_hex()]).unwrap(), + ]) + .allow_self_tagging() + .sign_with_keys(&keys) + .unwrap(); + let events = Arc::new(std::sync::Mutex::new(Vec::::new())); + let bodies = Arc::new(std::sync::Mutex::new(Vec::new())); + let losses = Arc::new(AtomicUsize::new(0)); + let hold = Arc::new(AtomicBool::new(false)); + let accepted = Arc::new(tokio::sync::Notify::new()); + let (events_task, bodies_task, losses_task, hold_task, accepted_task, parent_task) = ( + events.clone(), + bodies.clone(), + losses.clone(), + hold.clone(), + accepted.clone(), + parent.clone(), + ); + let task = tokio::spawn(async move { + while let Ok((mut socket, _)) = listener.accept().await { + let (route, value, body) = request(&mut socket).await; + if route == "/query" { + let filter = &value[0]; + let reply = if filter["kinds"][0] == 39002 { + serde_json::json!([membership]) + } else if filter["ids"][0] == parent_task.id.to_hex() { + serde_json::json!([parent_task]) + } else { + serde_json::json!(*events_task.lock().unwrap()) + }; + response(&mut socket, "200 OK", reply).await; + } else if route == "/events" { + let event: Event = serde_json::from_value(value).unwrap(); + event.verify().unwrap(); + assert!(event + .tags + .iter() + .any(|tag| tag.as_slice() == ["h", &channel.to_string()])); + let duplicate = { + let mut events = events_task.lock().unwrap(); + let duplicate = events.iter().any(|e| e.id == event.id); + if !duplicate { + events.push(event.clone()); + } + duplicate + }; + bodies_task.lock().unwrap().push(body); + accepted_task.notify_one(); + while hold_task.load(Ordering::SeqCst) { + tokio::time::sleep(Duration::from_millis(10)).await; + } + if losses_task + .fetch_update(Ordering::SeqCst, Ordering::SeqCst, |n| n.checked_sub(1)) + .is_ok() + { + // Stored event, successful HTTP header, then lost response body. + let _ = socket.write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 4096\r\nConnection: close\r\n\r\n{").await; + } else { + response(&mut socket, "200 OK", serde_json::json!({"accepted":true,"event_id":event.id.to_hex(),"message":if duplicate {"duplicate:"} else {"saved"}})).await; + } + } else { + panic!("unexpected mock relay route {route}"); + } + } + }); + Self { + url, + keys, + channel, + parent, + events, + bodies, + losses, + hold, + accepted, + task, + } + } + + fn command( + &self, + binary: &std::path::Path, + dir: &std::path::Path, + key: Uuid, + content: &str, + ) -> tokio::process::Command { + let mut command = tokio::process::Command::new(binary); + command + .env_remove("BUZZ_AUTH_TAG") + .arg("--relay") + .arg(&self.url) + .arg("--private-key") + .arg(self.keys.secret_key().to_secret_hex()) + .args([ + "messages", + "deliver", + "--delivery-key", + &key.to_string(), + "--channel", + &self.channel.to_string(), + "--reply-to", + &self.parent.id.to_hex(), + "--journal-dir", + ]) + .arg(dir) + .args(["--content", content]); + command.kill_on_drop(true); + command + } + + async fn readback(&self) -> Vec { + let events: Vec = reqwest::Client::new() + .post(format!("{}/query", self.url)) + .json(&serde_json::json!([{"kinds":[9],"#h":[self.channel.to_string()]}])) + .send() + .await + .unwrap() + .json() + .await + .unwrap(); + for event in &events { + event.verify().unwrap(); + } + eprintln!( + "mock-relay verified readback: {}", + serde_json::to_string(&events).unwrap() + ); + events + } +} + +impl Drop for MockRelay { + fn drop(&mut self) { + self.task.abort(); + } +} + +fn binaries() -> PathBuf { + PathBuf::from( + std::env::var("BUZZ_TEST_BIN_DIR").expect("set BUZZ_TEST_BIN_DIR to built binaries"), + ) +} + +#[tokio::test] +#[ignore = "requires rebuilt buzz CLI, buzz-agent and buzz-dev-mcp; BUZZ_TEST_BIN_DIR"] +async fn managed_runtime_body_loss_and_process_restart_replay_saved_bytes() { + let relay = MockRelay::start().await; + let dir = tempfile::tempdir().unwrap(); + let key = Uuid::new_v4(); + relay.losses.store(3, Ordering::SeqCst); + let first = relay + .command(&binaries().join("buzz"), dir.path(), key, "first") + .output() + .await + .unwrap(); + assert_eq!( + first.status.code(), + Some(2), + "{}", + String::from_utf8_lossy(&first.stderr) + ); + assert!(String::from_utf8_lossy(&first.stderr).contains("managed_delivery_pending")); + let retry = relay + .command( + &binaries().join("buzz"), + dir.path(), + key, + "changed on restart", + ) + .output() + .await + .unwrap(); + assert!( + retry.status.success(), + "{}", + String::from_utf8_lossy(&retry.stderr) + ); + let events = relay.readback().await; + assert_eq!(events.len(), 1); + assert_eq!(events[0].content, "first"); + let bodies = relay.bodies.lock().unwrap(); + assert_eq!(bodies.len(), 4); + assert!(bodies.iter().all(|body| body == &bodies[0])); +} + +#[tokio::test] +#[ignore = "requires rebuilt buzz CLI; BUZZ_TEST_BIN_DIR"] +async fn managed_runtime_killed_after_relay_acceptance_replays_on_new_process() { + let relay = MockRelay::start().await; + let dir = tempfile::tempdir().unwrap(); + let key = Uuid::new_v4(); + relay.hold.store(true, Ordering::SeqCst); + let mut first = relay + .command( + &binaries().join("buzz"), + dir.path(), + key, + "saved before crash", + ) + .spawn() + .unwrap(); + timeout(Duration::from_secs(10), relay.accepted.notified()) + .await + .unwrap(); + let journal = std::fs::read_to_string(dir.path().join("journal.json")).unwrap(); + assert!(journal.contains(&relay.events.lock().unwrap()[0].id.to_hex())); + assert!(journal.contains("\"confirmed_at\":null")); + first.kill().await.unwrap(); + first.wait().await.unwrap(); + relay.hold.store(false, Ordering::SeqCst); + let retry = relay + .command( + &binaries().join("buzz"), + dir.path(), + key, + "changed after crash", + ) + .output() + .await + .unwrap(); + assert!( + retry.status.success(), + "{}", + String::from_utf8_lossy(&retry.stderr) + ); + assert_eq!(relay.readback().await.len(), 1); + let bodies = relay.bodies.lock().unwrap(); + assert_eq!(bodies.len(), 2); + assert_eq!(bodies[0], bodies[1]); +} + +#[tokio::test] +#[ignore = "requires rebuilt buzz CLI; BUZZ_TEST_BIN_DIR"] +async fn managed_runtime_concurrent_cli_processes_share_event_and_lock() { + let relay = MockRelay::start().await; + let dir = tempfile::tempdir().unwrap(); + let key = Uuid::new_v4(); + let mut a = relay.command( + &binaries().join("buzz"), + dir.path(), + key, + "first concurrent", + ); + let mut b = relay.command( + &binaries().join("buzz"), + dir.path(), + key, + "second concurrent", + ); + let (a, b) = tokio::join!(a.output(), b.output()); + assert!(a.unwrap().status.success()); + assert!(b.unwrap().status.success()); + assert_eq!(relay.readback().await.len(), 1); + assert_eq!(relay.bodies.lock().unwrap().len(), 1); +} + +#[tokio::test] +#[ignore = "requires rebuilt buzz-agent and buzz-dev-mcp embedding managed CLI; BUZZ_TEST_BIN_DIR"] +async fn managed_runtime_bundled_agent_publication_then_application_error_retries_one_event() { + let relay = MockRelay::start().await; + let dir = tempfile::tempdir().unwrap(); + let workspace = dir.path().join("owner's workspace"); + std::fs::create_dir(&workspace).unwrap(); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let model_url = format!("http://{}", listener.local_addr().unwrap()); + let model = tokio::spawn(async move { + let mut commands = Vec::new(); + for round in 0..4 { + let (mut socket, _) = listener.accept().await.unwrap(); + let (_, value, _) = request(&mut socket).await; + if round == 1 { + assert!( + value["messages"].to_string().contains("accepted"), + "publication tool must have completed before model error" + ); + response(&mut socket, "400 Bad Request", serde_json::json!({"error":{"message":"application failure after final report publication","type":"invalid_request_error","code":"test_application_error"}})).await; + continue; + } + let (message, reason) = if round == 0 || round == 2 { + let name = value["tools"] + .as_array() + .unwrap() + .iter() + .find_map(|tool| { + tool["function"]["name"] + .as_str() + .filter(|name| name.ends_with("__shell")) + }) + .unwrap(); + let prompt = value["messages"] + .as_array() + .unwrap() + .iter() + .rev() + .filter_map(|m| m["content"].as_str()) + .find(|text| text.contains("buzz messages deliver --delivery-key")) + .unwrap(); + let line = prompt + .lines() + .find(|line| line.starts_with("buzz messages deliver --delivery-key")) + .unwrap(); + commands.push(line.to_string()); + let text = if round == 0 { + "first final report" + } else { + "changed final report after retry" + }; + let command = format!( + "env -u BUZZ_AUTH_TAG {}", + line.replace("", &format!("'{text}'")) + ); + ( + serde_json::json!({"role":"assistant","content":null,"tool_calls":[{"id":format!("publish-{round}"),"type":"function","function":{"name":name,"arguments":serde_json::json!({"command":command}).to_string()}}]}), + "tool_calls", + ) + } else { + ( + serde_json::json!({"role":"assistant","content":"done"}), + "stop", + ) + }; + response(&mut socket, "200 OK", serde_json::json!({"id":"managed-test","object":"chat.completion","model":"probe","choices":[{"index":0,"message":message,"finish_reason":reason}]})).await; + } + assert_eq!(commands.len(), 2); + assert_eq!(commands[0], commands[1]); + }); + let mut config = crate::build_mcp_servers_tests::test_config(); + config.keys = relay.keys.clone(); + config.relay_url = relay.url.clone(); + config.mcp_command = binaries() + .join("buzz-dev-mcp") + .to_string_lossy() + .into_owned(); + let env = vec![ + ("BUZZ_AGENT_PROVIDER".into(), "openai".into()), + ("OPENAI_COMPAT_BASE_URL".into(), model_url), + ("OPENAI_COMPAT_API_KEY".into(), "test".into()), + ("OPENAI_COMPAT_MODEL".into(), "probe".into()), + ]; + let mut agent = OwnedAgent { + index: 0, + acp: AcpClient::spawn( + binaries().join("buzz-agent").to_str().unwrap(), + &[], + &env, + false, + ) + .await + .unwrap(), + state: SessionState::default(), + model_capabilities: None, + desired_model: None, + model_overridden: false, + desired_model_request_id: None, + desired_model_pending_ack: false, + startup_effort: None, + agent_name: "delivery-test".into(), + goose_system_prompt_supported: None, + protocol_version: 2, + background_context: None, + }; + agent.acp.initialize().await.unwrap(); + let session = agent + .acp + .session_new_full( + workspace.to_str().unwrap(), + crate::build_mcp_servers(&config), + None, + None, + ) + .await + .unwrap(); + let scope = SessionScope::Conversation { + channel_id: relay.channel, + }; + agent + .state + .sessions + .insert(scope.clone(), session.session_id.clone()); + let mut pool = AgentPool::from_slots(vec![Some(agent)]); + let generation = pool.record_scope_owner(scope.clone(), 0); + pool.agents[0] + .as_mut() + .unwrap() + .state + .set_scope_owner_generation(scope.clone(), generation); + let continuation = RuntimeContinuation { + delivery_key: Uuid::new_v4(), + ordinary_preempted: false, + worker_index: 0, + session_id: session.session_id, + generation, + thread_tags: ThreadTags { + root_event_id: Some(relay.parent.id.to_hex()), + parent_event_id: Some(relay.parent.id.to_hex()), + ..Default::default() + }, + received_at: std::time::Instant::now(), + }; + let mut queue = EventQueue::new(DedupMode::Queue); + queue.enqueue_continuation(scope.clone(), continuation); + let mut ctx = tests::make_prompt_context_no_owner(); + ctx.cwd = workspace.to_string_lossy().into_owned(); + ctx.agent_keys = relay.keys.clone(); + ctx.memory_enabled = false; + ctx.context_message_limit = 0; + ctx.rest_client.base_url = relay.url.clone(); + ctx.channel_info = ChannelInfoResolver::new(HashMap::new(), ctx.rest_client.clone()); + ctx.channel_info.cache.write().unwrap().insert( + relay.channel, + PromptChannelInfo { + name: "reports".into(), + channel_type: "stream".into(), + description: None, + project: None, + }, + ); + ctx.channel_info.projects.write().unwrap().insert( + relay.channel, + CachedProjectInfo { + value: None, + fetched_at: std::time::Instant::now(), + }, + ); + let ctx = Arc::new(ctx); + assert_eq!( + background_work_tests::dispatch(&mut pool, &mut queue, &ctx), + 1 + ); + let result = timeout(Duration::from_secs(30), pool.rx_and_join_set().0.recv()) + .await + .unwrap() + .unwrap(); + assert!(matches!( + result.outcome, + PromptOutcome::Error(AcpError::AgentError { .. }) + )); + background_work_tests::apply_result(&mut pool, &mut queue, result, None, &HashSet::new()); + assert_eq!(relay.readback().await.len(), 1); + tokio::time::sleep_until(queue.next_retry_deadline().unwrap().into()).await; + assert_eq!( + background_work_tests::dispatch(&mut pool, &mut queue, &ctx), + 1 + ); + let result = timeout(Duration::from_secs(30), pool.rx_and_join_set().0.recv()) + .await + .unwrap() + .unwrap(); + assert!(matches!( + result.outcome, + PromptOutcome::Ok(StopReason::EndTurn) + )); + background_work_tests::apply_result(&mut pool, &mut queue, result, None, &HashSet::new()); + model.await.unwrap(); + let events = relay.readback().await; + assert_eq!(events.len(), 1); + assert_eq!(events[0].content, "first final report"); + events[0].verify().unwrap(); + assert_eq!(relay.bodies.lock().unwrap().len(), 1); + let journal = std::fs::read_to_string(workspace.join(".buzz-deliveries/journal.json")).unwrap(); + assert!(journal.contains(&events[0].id.to_hex())); + for agent in pool.agents.iter_mut().flatten() { + agent.acp.shutdown().await; + } +} diff --git a/crates/buzz-acp/src/process_tree.rs b/crates/buzz-acp/src/process_tree.rs index b4bb1b09ac4..21b8bfc8748 100644 --- a/crates/buzz-acp/src/process_tree.rs +++ b/crates/buzz-acp/src/process_tree.rs @@ -50,6 +50,7 @@ pub fn is_resident_helper(command: &str) -> bool { /// Parse one `ps -axo pid=,ppid=,command=` line. /// /// Returns `(pid, ppid, command)`. +#[cfg(any(unix, test))] pub fn parse_ps_line(line: &str) -> Option<(u32, u32, String)> { let trimmed = line.trim(); if trimmed.is_empty() { diff --git a/crates/buzz-acp/src/queue.rs b/crates/buzz-acp/src/queue.rs index d2d3dc5dd88..b02b8cf586c 100644 --- a/crates/buzz-acp/src/queue.rs +++ b/crates/buzz-acp/src/queue.rs @@ -241,6 +241,9 @@ pub struct FlushBatch { /// ``` pub struct EventQueue { continuations: HashMap, + continuation_retry_after: HashMap, + continuation_retry_counts: HashMap, + in_flight_continuations: HashSet, queues: HashMap>, in_flight_scopes: HashSet, /// Per-scope deadline for auto-expiring stuck in-flight entries. @@ -288,6 +291,9 @@ impl EventQueue { pub fn new(dedup_mode: DedupMode) -> Self { Self { continuations: HashMap::new(), + continuation_retry_after: HashMap::new(), + continuation_retry_counts: HashMap::new(), + in_flight_continuations: HashSet::new(), queues: HashMap::new(), in_flight_scopes: HashSet::new(), in_flight_deadlines: HashMap::new(), @@ -358,7 +364,6 @@ impl EventQueue { } let channel_id = event.channel_id; let scope = event.scope.clone(); - self.remove_continuation(&scope); let queue = self.queues.entry(scope.clone()).or_default(); // Enforce per-scope depth cap: drop oldest in this partition. if queue.len() >= MAX_PENDING_PER_SCOPE { @@ -440,6 +445,7 @@ impl EventQueue { auto-releasing; {lost_events} dispatched event(s) orphaned" ); self.in_flight_scopes.remove(&scope); + self.in_flight_continuations.remove(&scope); self.in_flight_deadlines.remove(&scope); self.in_flight_reply_threads.remove(&scope); // Recover any withheld goose-native steer events for the expired @@ -450,6 +456,12 @@ impl EventQueue { self.recover_withheld_for_expired_scope(&scope); } + // Ready collection precedes ordinary traffic so continuous messages + // cannot starve it. A collection in backoff leaves messages eligible. + if let Some(batch) = self.flush_continuation() { + return Some(batch); + } + // Find the scope whose head event has the oldest received_at, // excluding in-flight scopes and throttled scopes. let scope = self @@ -495,7 +507,7 @@ impl EventQueue { cancel_reason, }); } - None => return self.flush_continuation(), + None => return None, } } }; @@ -557,10 +569,18 @@ impl EventQueue { .iter() .filter(|(scope, _)| { !self.in_flight_scopes.contains(*scope) - && (self.queues.get(*scope).is_some_and(|q| !q.is_empty()) - || self.continuations.contains_key(*scope)) + && self.queues.get(*scope).is_some_and(|q| !q.is_empty()) }) .map(|(_, &deadline)| deadline) + .chain( + self.continuation_retry_after + .iter() + .filter(|(scope, _)| { + !self.in_flight_scopes.contains(*scope) + && self.continuations.contains_key(*scope) + }) + .map(|(_, &deadline)| deadline), + ) .min() } @@ -576,11 +596,20 @@ impl EventQueue { /// Also cleans up any already-expired `retry_after` entry. pub fn mark_complete(&mut self, scope: K) { let scope = scope.into_scope(); + let collecting = self.in_flight_continuations.remove(&scope); self.in_flight_scopes.remove(&scope); self.in_flight_deadlines.remove(&scope); self.in_flight_batch_sizes.remove(&scope); self.in_flight_reply_threads.remove(&scope); let now = Instant::now(); + if collecting { + // Retried/cancelled collection remains independently pending. + // Ordinary message completion never touches this retry budget. + if !self.continuations.contains_key(&scope) { + self.remove_continuation(&scope); + } + return; + } match self.retry_after.get(&scope) { // Active throttle → scope was requeued; keep retry_counts intact. Some(&deadline) if deadline > now => {} @@ -603,6 +632,14 @@ impl EventQueue { self.in_flight_deadlines.remove(scope); self.in_flight_batch_sizes.remove(scope); self.in_flight_reply_threads.remove(scope); + self.in_flight_continuations.remove(scope); + if self + .continuation_retry_after + .get(scope) + .is_some_and(|deadline| *deadline <= Instant::now()) + { + self.continuation_retry_after.remove(scope); + } if self .retry_after .get(scope) @@ -632,12 +669,14 @@ impl EventQueue { pub fn requeue(&mut self, batch: FlushBatch) -> Option { let channel_id = batch.channel_id; let scope = batch.scope.clone(); - if batch.continuation.is_some() && self.has_pending_scope(&scope) { - // A newly accepted real message supersedes the failed continuation. - return None; - } + let collecting = batch.continuation.is_some(); let attempt = { - let count = self.retry_counts.entry(scope.clone()).or_insert(0); + let counts = if collecting { + &mut self.continuation_retry_counts + } else { + &mut self.retry_counts + }; + let count = counts.entry(scope.clone()).or_insert(0); *count += 1; *count }; @@ -651,6 +690,10 @@ impl EventQueue { MAX_RETRIES, batch.events.len(), ); + if collecting { + self.remove_continuation(&scope); + return Some(batch); + } self.retry_counts.remove(&scope); // Also clear retry_after so fresh traffic on this scope isn't // throttled by stale backoff from the discarded poison batch. @@ -682,7 +725,8 @@ impl EventQueue { if let Some(continuation) = batch.continuation { self.enqueue_continuation(scope.clone(), continuation); - self.retry_after.insert(scope, Instant::now() + delay); + self.continuation_retry_after + .insert(scope, Instant::now() + delay); return None; } @@ -792,8 +836,10 @@ impl EventQueue { /// the generic queue — they are stored separately and merged by /// `flush_next()`. No retry throttle, no backoff. pub fn requeue_as_cancelled(&mut self, batch: FlushBatch, reason: CancelReason) { - if batch.continuation.is_some() { - return; // A control message supersedes background collection. + if let Some(mut continuation) = batch.continuation { + continuation.ordinary_preempted = true; + self.enqueue_continuation(batch.scope, continuation); + return; } let scope = batch.scope.clone(); let entry = self.cancelled_batches.entry(scope.clone()).or_default(); @@ -830,6 +876,7 @@ impl EventQueue { auto-releasing; {lost_events} dispatched event(s) orphaned" ); self.in_flight_scopes.remove(&scope); + self.in_flight_continuations.remove(&scope); self.in_flight_deadlines.remove(&scope); self.in_flight_reply_threads.remove(&scope); // Symmetric with the flush_next expiry block: recover withheld @@ -949,6 +996,10 @@ impl EventQueue { pub fn drain_channel(&mut self, channel_id: Uuid) -> Vec { self.continuations .retain(|scope, _| scope.channel_id() != channel_id); + self.continuation_retry_after + .retain(|s, _| s.channel_id() != channel_id); + self.continuation_retry_counts + .retain(|s, _| s.channel_id() != channel_id); // Channel-wide cleanup must find and clear EVERY child thread scope for // this channel, not just the conversation scope. let mut ids: Vec = Vec::new(); @@ -1199,12 +1250,16 @@ impl EventQueue { pub fn compact_expired_state(&mut self) { let now = Instant::now(); self.retry_after.retain(|_, deadline| *deadline > now); + self.continuation_retry_after + .retain(|_, deadline| *deadline > now); + self.continuation_retry_counts.retain(|scope, _| { + self.continuations.contains_key(scope) || self.in_flight_continuations.contains(scope) + }); // Remove retry_counts for channels with no active throttle, no // queued events, AND no in-flight prompt — they completed their // retry cycle and are truly idle. self.retry_counts.retain(|scope, _| { self.retry_after.contains_key(scope) - || self.continuations.contains_key(scope) || self.queues.get(scope).is_some_and(|q| !q.is_empty()) || self.in_flight_scopes.contains(scope) }); diff --git a/crates/buzz-acp/src/queue/background.rs b/crates/buzz-acp/src/queue/background.rs index 828da7d74ae..4da2e160555 100644 --- a/crates/buzz-acp/src/queue/background.rs +++ b/crates/buzz-acp/src/queue/background.rs @@ -1,10 +1,14 @@ -//! Background completions share the event queue's in-flight and retry lifecycle. +//! Background collections share scope serialization, with independent retries. use super::*; /// Authority and reply destination of a background task's original session. #[derive(Clone, Debug)] pub(crate) struct RuntimeContinuation { + /// Stable random identity of this group's managed final report. + pub delivery_key: Uuid, + /// One ordinary cancellation is allowed; its retry must get an execution window. + pub ordinary_preempted: bool, pub worker_index: usize, pub session_id: String, pub generation: u64, @@ -31,22 +35,23 @@ impl FlushBatch { } impl EventQueue { - /// Enqueue once; a real queued message already supersedes the continuation. + /// Keep one collection per scope independently of pending messages. pub(crate) fn enqueue_continuation( &mut self, scope: SessionScope, continuation: RuntimeContinuation, ) { - if !self.has_pending_scope(&scope) { - self.continuations.entry(scope).or_insert(continuation); - } + self.continuations.entry(scope).or_insert(continuation); } pub(crate) fn remove_continuation(&mut self, scope: &SessionScope) { - if self.continuations.remove(scope).is_some() { - self.retry_after.remove(scope); - self.retry_counts.remove(scope); - } + self.continuations.remove(scope); + self.continuation_retry_after.remove(scope); + self.continuation_retry_counts.remove(scope); + } + + pub(crate) fn has_pending_continuation(&self, scope: &SessionScope) -> bool { + self.continuations.contains_key(scope) } pub(super) fn flush_continuation(&mut self) -> Option { @@ -57,13 +62,14 @@ impl EventQueue { .filter(|(scope, _)| { !self.in_flight_scopes.contains(*scope) && self - .retry_after + .continuation_retry_after .get(*scope) .is_none_or(|deadline| *deadline <= now) }) .min_by_key(|(_, continuation)| continuation.received_at) .map(|(scope, _)| scope.clone())?; let continuation = self.continuations.remove(&scope)?; + self.in_flight_continuations.insert(scope.clone()); self.in_flight_scopes.insert(scope.clone()); self.in_flight_deadlines .insert(scope.clone(), now + self.in_flight_deadline); @@ -87,7 +93,7 @@ impl EventQueue { self.continuations.keys().any(|scope| { !self.in_flight_scopes.contains(scope) && self - .retry_after + .continuation_retry_after .get(scope) .is_none_or(|deadline| *deadline <= now) }) @@ -101,6 +107,8 @@ mod tests { fn continuation() -> RuntimeContinuation { RuntimeContinuation { + delivery_key: Uuid::new_v4(), + ordinary_preempted: false, worker_index: 0, session_id: "original".into(), generation: 1, @@ -164,13 +172,15 @@ mod tests { assert!(queue.has_undispatched_work()); assert!(!queue.has_flushable_work()); assert!(queue.next_retry_deadline().is_some()); - queue.retry_after.insert(scope.clone(), Instant::now()); + queue + .continuation_retry_after + .insert(scope.clone(), Instant::now()); } } } #[test] - fn real_message_supersedes_a_retrying_continuation_without_backoff() { + fn background_retry_survives_successful_message_without_blocking_it() { let scope = SessionScope::Conversation { channel_id: Uuid::new_v4(), }; @@ -184,7 +194,70 @@ mod tests { assert!(batch.continuation.is_none()); assert_eq!(batch.events.len(), 1); queue.mark_complete(&scope); - assert!(!queue.has_undispatched_work()); + assert!( + queue.has_undispatched_work(), + "ordinary success must keep collection" + ); + assert!(!queue.has_flushable_work(), "collection keeps its backoff"); + assert!(queue.next_retry_deadline().is_some()); + } + + #[test] + fn ready_background_collection_wins_over_pending_messages() { + let scope = SessionScope::Conversation { + channel_id: Uuid::new_v4(), + }; + let mut queue = EventQueue::new(DedupMode::Queue); + queue.enqueue_continuation(scope.clone(), continuation()); + queue.push(event(&scope)); + assert!(queue.flush_next().unwrap().continuation.is_some()); + queue.mark_complete(&scope); + assert_eq!(queue.flush_next().unwrap().events.len(), 1); + } + + #[test] + fn background_exit_is_retained_when_message_was_queued_first() { + let scope = SessionScope::Conversation { + channel_id: Uuid::new_v4(), + }; + let mut queue = EventQueue::new(DedupMode::Queue); + queue.push(event(&scope)); + queue.enqueue_continuation(scope.clone(), continuation()); + assert!(queue.flush_next().unwrap().continuation.is_some()); + } + + #[test] + fn background_collection_is_preserved_by_cancel_merge() { + for reason in [CancelReason::Steer, CancelReason::Interrupt] { + let scope = SessionScope::Conversation { + channel_id: Uuid::new_v4(), + }; + let mut queue = EventQueue::new(DedupMode::Queue); + queue.enqueue_continuation(scope.clone(), continuation()); + let batch = queue.flush_next().unwrap(); + queue.push(event(&scope)); + queue.requeue_as_cancelled(batch, reason); + queue.mark_complete(&scope); + assert!(queue.flush_next().unwrap().continuation.is_some()); + } + } + + #[test] + fn background_failure_is_retained_when_new_message_waits() { + let scope = SessionScope::Conversation { + channel_id: Uuid::new_v4(), + }; + let mut queue = EventQueue::new(DedupMode::Queue); + queue.enqueue_continuation(scope.clone(), continuation()); + let batch = queue.flush_next().unwrap(); + queue.push(event(&scope)); + queue.requeue(batch); + queue.mark_complete(&scope); + let message = queue.flush_next().unwrap(); + assert!(message.continuation.is_none()); + queue.mark_complete(&scope); + assert!(queue.has_undispatched_work()); + assert!(queue.next_retry_deadline().is_some()); } #[test] @@ -213,7 +286,9 @@ mod tests { if attempt == MAX_RETRIES { assert!(failed.is_some()); } else { - queue.retry_after.insert(scope.clone(), Instant::now()); + queue + .continuation_retry_after + .insert(scope.clone(), Instant::now()); let waiting = queue.flush_next().unwrap(); queue.requeue_preserve_timestamps(waiting); queue.release_unstarted(&scope); @@ -224,4 +299,167 @@ mod tests { } } } + + #[cfg(unix)] + #[tokio::test] + async fn background_failed_collection_keeps_finite_budget_across_messages_and_posts_notice() { + use crate::pool::background_work_tests::{ + apply_result, dispatch, finish_child, push_new_message, started_background_turn, + }; + use tokio::io::{AsyncBufReadExt, AsyncReadExt, AsyncWriteExt, BufReader}; + let (mut pool, mut queue, scope, dir, ctx) = started_background_turn(0).await; + finish_child(&mut pool, &mut queue, &dir).await; + let original = queue.continuations[&scope].thread_tags.clone(); + std::fs::write(dir.path().join("fail-collection"), "fail").unwrap(); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let mut rest = ctx.rest_client.clone(); + rest.base_url = format!("http://{}", listener.local_addr().unwrap()); + for attempt in 0..=MAX_RETRIES { + assert_eq!(dispatch(&mut pool, &mut queue, &ctx), 1); + let result = + tokio::time::timeout(Duration::from_secs(10), pool.rx_and_join_set().0.recv()) + .await + .unwrap() + .unwrap(); + assert!(matches!( + result.outcome, + crate::pool::PromptOutcome::Error(crate::acp::AcpError::AgentError { .. }) + )); + apply_result(&mut pool, &mut queue, result, Some(&rest), &HashSet::new()); + if attempt == MAX_RETRIES { + assert!(!queue.has_undispatched_work()); + assert!( + !pool.has_runtime_hold(), + "terminal collection must release original session" + ); + assert!(pool.try_claim(None).is_some()); + break; + } + let deadline = queue.next_retry_deadline().unwrap(); + assert!(deadline > Instant::now()); + push_new_message(&mut queue, &scope); + assert_eq!(dispatch(&mut pool, &mut queue, &ctx), 1); + let message = + tokio::time::timeout(Duration::from_secs(10), pool.rx_and_join_set().0.recv()) + .await + .unwrap() + .unwrap(); + assert!(matches!(message.outcome, crate::pool::PromptOutcome::Ok(_))); + apply_result(&mut pool, &mut queue, message, None, &HashSet::new()); + assert_eq!( + queue.next_retry_deadline(), + Some(deadline), + "ordinary success cannot clear collection backoff" + ); + // Advance eligibility at the queue seam; real child/ACP requests + // and handle_prompt_result still execute on every retry. + queue + .continuation_retry_after + .insert(scope.clone(), Instant::now()); + } + let notice: nostr::Event = tokio::time::timeout(Duration::from_secs(3), async { + let (socket, _) = listener.accept().await.unwrap(); + let mut reader = BufReader::new(socket); + let mut line = String::new(); + reader.read_line(&mut line).await.unwrap(); + assert_eq!(line, "POST /events HTTP/1.1\r\n"); + let mut content_length = None; + for _ in 0..64 { + line.clear(); + assert_ne!(reader.read_line(&mut line).await.unwrap(), 0); + if line == "\r\n" { + break; + } + if let Some(value) = line.to_ascii_lowercase().strip_prefix("content-length:") { + content_length = Some(value.trim().parse::().unwrap()); + } + } + let size = content_length.unwrap(); + assert!(size < 65536); + let mut body = vec![0; size]; + reader.read_exact(&mut body).await.unwrap(); + reader + .get_mut() + .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\nConnection: close\r\n\r\n{}") + .await + .unwrap(); + serde_json::from_slice(&body).unwrap() + }) + .await + .unwrap(); + notice.verify().unwrap(); + assert!(notice.content.contains("multiple retries")); + assert_eq!(crate::queue::routing_thread_tags(¬ice, None), original); + assert!(notice + .tags + .iter() + .any(|tag| tag.as_slice() == ["h", &scope.channel_id().to_string()])); + } + #[cfg(unix)] + #[tokio::test] + async fn background_managed_delivery_key_stays_on_collection_across_error_and_new_message() { + use crate::pool::background_work_tests::{ + apply_result, dispatch, finish_child, push_new_message, started_background_turn, + }; + let (mut pool, mut queue, scope, dir, ctx) = started_background_turn(0).await; + finish_child(&mut pool, &mut queue, &dir).await; + std::fs::write(dir.path().join("fail-collection"), "fail").unwrap(); + let original = queue.continuations[&scope].thread_tags.clone(); + assert_eq!(dispatch(&mut pool, &mut queue, &ctx), 1); + let result = tokio::time::timeout(Duration::from_secs(10), pool.rx_and_join_set().0.recv()) + .await + .unwrap() + .unwrap(); + assert!(matches!( + result.outcome, + crate::pool::PromptOutcome::Error(crate::acp::AcpError::AgentError { .. }) + )); + apply_result(&mut pool, &mut queue, result, None, &HashSet::new()); + push_new_message(&mut queue, &scope); + assert_eq!(dispatch(&mut pool, &mut queue, &ctx), 1); + let result = tokio::time::timeout(Duration::from_secs(10), pool.rx_and_join_set().0.recv()) + .await + .unwrap() + .unwrap(); + apply_result(&mut pool, &mut queue, result, None, &HashSet::new()); + std::fs::remove_file(dir.path().join("fail-collection")).unwrap(); + queue + .continuation_retry_after + .insert(scope.clone(), Instant::now()); + assert_eq!(dispatch(&mut pool, &mut queue, &ctx), 1); + let result = tokio::time::timeout(Duration::from_secs(10), pool.rx_and_join_set().0.recv()) + .await + .unwrap() + .unwrap(); + apply_result(&mut pool, &mut queue, result, None, &HashSet::new()); + let requests: Vec = std::fs::read_to_string(dir.path().join("requests")) + .unwrap() + .lines() + .map(|line| serde_json::from_str(line).unwrap()) + .collect(); + let prompts: Vec<&str> = requests + .iter() + .filter(|r| r["method"] == "session/prompt") + .map(|r| r["params"]["prompt"][0]["text"].as_str().unwrap()) + .collect(); + assert_eq!(prompts.len(), 4); + let key = |prompt: &str| -> String { + prompt + .split("--delivery-key ") + .nth(1) + .expect("collection must receive managed delivery command") + .split_whitespace() + .next() + .unwrap() + .trim_matches('\'') + .to_string() + }; + assert!(Uuid::parse_str(&key(prompts[1])).is_ok()); + assert_eq!(key(prompts[1]), key(prompts[3])); + assert!(!prompts[0].contains("--delivery-key ")); + assert!(!prompts[2].contains("--delivery-key ")); + assert!(prompts[3].contains(&scope.channel_id().to_string())); + assert!(prompts[3].contains(original.parent_event_id.as_deref().unwrap())); + assert!(prompts[3].contains("--journal-dir ")); + } } diff --git a/crates/buzz-cli/Cargo.toml b/crates/buzz-cli/Cargo.toml index 59d1bb2cee6..714abf6728e 100644 --- a/crates/buzz-cli/Cargo.toml +++ b/crates/buzz-cli/Cargo.toml @@ -86,6 +86,9 @@ rustls = { version = "0.23", default-features = false, features = ["ring", "std" # Random number generation — full jitter for exponential backoff in with_retry rand = { workspace = true } +# Cross-process advisory lock for the durable final-report journal +fs2 = "0.4" + [dev-dependencies] # Scratch files for channel-templates.json fixtures in tests tempfile = "3" diff --git a/crates/buzz-cli/TESTING.md b/crates/buzz-cli/TESTING.md index b7fa06d2031..abfe69b7a57 100644 --- a/crates/buzz-cli/TESTING.md +++ b/crates/buzz-cli/TESTING.md @@ -625,3 +625,124 @@ buzz channels delete --channel "$FORUM_ID" | jq . | 60 | `notes ls` | ☐ | Own, --author all, --tag, --limit | | 61 | `notes rm` | ☐ | Delete→get 404, double-delete idempotent, missing slug → NotFound | | 62 | `users set-status` | ☐ | Text+emoji, text only, emoji-only (`--text ""`), `--clear`, `--clear` + `--text` → exit 1 | + +## Managed final-report delivery + +`messages deliver` journals the first signed kind-9 event before publishing it: + +```sh +DELIVERY_KEY=$(uuidgen) +JOURNAL_DIR="$PWD/.buzz-deliveries" +buzz messages deliver --delivery-key "$DELIVERY_KEY" --channel "$CHANNEL_ID" \ + --reply-to "$ROOT_EVENT_ID" --journal-dir "$JOURNAL_DIR" --content '@Callback final report' +buzz messages deliver --delivery-key "$DELIVERY_KEY" --channel "$CHANNEL_ID" \ + --reply-to "$ROOT_EVENT_ID" --journal-dir "$JOURNAL_DIR" --content 'changed summary' +``` + +Both commands return the first event ID. Read back with `messages get` or +`messages thread`: exactly one report bears `["buzz-delivery", DELIVERY_KEY]`, +and its content and original thread stay unchanged. `--mention ` and +automatic member-name/nostr-URI mentions work as for `messages send` and remain +part of the first signed event. Use `messages edit` for an explicit correction; +use ordinary `messages send` for separate progress. Two fresh keys produce two +reports even with identical text. Reusing a key with another channel or parent +returns a conflict; a parent from another channel is also refused. + +The default directory is `.buzz-deliveries` in the caller's working directory. +`--journal-dir` overrides `BUZZ_DELIVERY_JOURNAL_DIR`. Use an absolute persistent +directory across shell working-directory changes and restarts. ACP continuation +prompts supply their workspace directory explicitly. Namespace identity includes +the normalized relay URL, signer public key and delivery UUID; records bind the +original channel and parent. Membership is queried before each managed +submission and before returning a locally confirmed delivery. The relay also +enforces authorization on each HTTP request. Revoked access never redirects a +report to another channel or DM. + +A lost/invalid acknowledgment leaves the signed event in the journal. +`managed_delivery_pending`, exit 2, `retryable:true` means retry the **same +managed command/key/route/journal**; it does not authorize a fresh `messages +send` or key. A valid acknowledgment must say `accepted:true` and name the saved +event ID. The real relay's `accepted:true, message:"duplicate:"` response is a +successful replay. Unjournaled `messages send` retains its existing +`delivery_unknown, retryable:false` ambiguity contract. + +Late recovery requires an updated relay with atomic managed-report election. +For a report older than 600 seconds, the CLI journals a fresh signed copy that +preserves the original author, content and exact tags, then submits a signed +kind-20003 recovery command containing both events. An aged pending candidate +may be renewed again; the original evidence remains unchanged. The relay uses +one writer transaction to return a previously accepted report or insert one +current candidate and bind it to the completion key. The 960-second DB floor +remains active. Soft-deleted reports are acknowledged without resurrection. + +The CLI validates and durably saves the chosen signed event before returning. +Recovered unaccepted reports can have a new event ID/date, with the same content, +mentions and original thread. A lost acknowledgment can return an earlier +candidate; callers must use the returned canonical ID. Journal v1 is migrated to +v2 under its existing lock. The record retains original evidence plus the latest candidate and canonical +timestamp/ID/signature; immutable content/tags are reconstructed and verified. Existing size/attempt limits +still apply. Old CLI versions reject v2 journals instead of publishing through +an incompatible path. Unknown relays cannot publish the nested candidate as a +kind-9 message, and ordinary wrapper acceptance is not sufficient confirmation. + +Upgrade all relay writers and drain old in-flight writes before enabling this +recovery contract: mixed old writers do not participate in the canonical-key +lock. Each completion UUID must remain unique; the server retains its binding +until community deletion, independently of the journal's seven-day confirmed +retention. Key reuse with another body or route conflicts. Direct unmarked +progress and edit operations retain their separate behavior. + +Each record permits eight managed submissions, each using the existing maximum +of three transport attempts (24 HTTP publication requests at most). Attempts +are saved before sending, so a crash can consume an attempt without publishing. +Exhaustion is explicit and preserves the signed evidence. Records are capped at +1024, 512 KiB each and 16 MiB serialized total. Capacity rejects new records; +unconfirmed/terminal records never silently expire or get evicted. Confirmed +records can expire after seven days; afterward this command no longer promises +idempotency for that old key. There is one lock file and one bounded snapshot, +with a fixed replacement file, and a 30-second bounded lock wait. New Unix +directories use 0700 and journal files use 0600. Snapshot replacement fsyncs both +file and directory before sending. Each invocation fsyncs every ancestor of the +journal directory, including existing links a concurrent creator may have +exposed before syncing. Paths deeper than 128 ancestors fail visibly. +Use a filesystem supporting advisory locks, atomic replacement and directory +fsync; unsupported filesystem/platform errors fail visibly before publication. +The durability workflow is exercised on macOS/Unix; Windows and network +filesystems have not been validated. + +This covers cooperating standalone CLI and embedded CLI callers of +`messages deliver`. Other MCP/shell publishers do not share the journal. State +survives process restart only while this persistent volume survives. There is +no startup replay, no recovery of unjournaled provider output and no general +exactly-once guarantee. Recover a saved event by explicitly repeating its key +and route against the same journal. Keep pending/terminal records for inspection; +do not delete them to make a new send appear safe. + +Automated local checks: + +```sh +cargo test -p buzz-cli managed_ -- --test-threads=1 +cargo build -p buzz-cli -p buzz-agent -p buzz-dev-mcp +BUZZ_TEST_BIN_DIR="$PWD/target/debug" \ + env -u BUZZ_ACP_ALLOWED_RESPOND_TO -u BUZZ_ACP_LAZY_POOL \ + -u BUZZ_ACP_IDLE_POOL_SLEEP -u BUZZ_ACP_SESSION_POLICY \ + cargo test -p buzz-acp managed_runtime_ -- --ignored --test-threads=1 +``` + +Those runtime checks use actual bundled agent/MCP/CLI processes, a deterministic +local model and a signature-checking **mock relay**, including publication +followed by ACP application error/retry, response-body loss, process kill after +relay acceptance and concurrent CLI processes. They are not evidence of a real +authenticated relay or human smoke test. + +Human checks against an authorized local relay: run the commands above and +inspect event readback; repeat with a fresh key; send progress and edit the +managed report; run two same-key invocations concurrently; restart the CLI and +replay with changed text. Then use an authorized owner account to remove the +publisher's membership and repeat the original command: expect exit 3 and no +new publication or alternate route. Check the app shows one final report in the +original thread and preserves the intended callback mention. A crash/power-loss +check requires a disposable persisted journal and a relay/proxy boundary that +can stop the publisher after acceptance; preserve the journal and confirm +readback by event ID before recovery. Do not infer real power-loss durability +from process-kill tests alone. diff --git a/crates/buzz-cli/src/commands/delivery.rs b/crates/buzz-cli/src/commands/delivery.rs new file mode 100644 index 00000000000..3a5ae619a17 --- /dev/null +++ b/crates/buzz-cli/src/commands/delivery.rs @@ -0,0 +1,524 @@ +//! Durable final-report publication. The lock protects the entire bounded snapshot. +use super::messages; +use crate::{client::BuzzClient, error::CliError, validate}; +use buzz_core::managed_delivery::{same_report, RecoveryRequest, ReportStamp, RECEIPT_PREFIX}; +use fs2::FileExt; +use nostr::{Event, EventBuilder, Kind, Tag}; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; +use std::{ + collections::BTreeMap, + fs::{self, File, OpenOptions}, + io::{Read, Write}, + path::{Path, PathBuf}, + time::{Duration, SystemTime, UNIX_EPOCH}, +}; +use tokio::time::Instant; + +const MAX_RECORDS: usize = 1024; +const MAX_RECORD_BYTES: usize = 512 * 1024; +const MAX_JOURNAL_BYTES: usize = 16 * 1024 * 1024; +const MAX_ATTEMPTS: u8 = 8; +const RETENTION_SECS: u64 = 7 * 24 * 60 * 60; + +#[derive(Default, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct Journal { + version: u8, + entries: BTreeMap, +} + +#[derive(Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct Record { + relay: String, + signer: String, + key: String, + channel: String, + parent: String, + event: Event, + #[serde(default, skip_serializing_if = "Option::is_none")] + recovery_candidate: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + canonical_event: Option, + attempts: u8, + confirmed_at: Option, + last_error: Option, +} + +fn error(detail: impl std::fmt::Display) -> CliError { + CliError::Other(format!("managed delivery journal: {detail}")) +} + +fn now() -> Result { + Ok(SystemTime::now() + .duration_since(UNIX_EPOCH) + .map_err(error)? + .as_secs()) +} + +fn relay_namespace(client: &BuzzClient) -> Result { + let mut url = + url::Url::parse(&crate::client::normalize_relay_url(client.relay_url())).map_err(error)?; + if !matches!(url.scheme(), "http" | "https") + || url.host_str().is_none() + || !url.username().is_empty() + || url.password().is_some() + || url.query().is_some() + || url.fragment().is_some() + { + return Err(CliError::Usage( + "managed delivery requires an HTTP(S) relay URL without credentials, query or fragment" + .into(), + )); + } + let path = url.path().trim_end_matches('/').to_owned(); + url.set_path(&path); + Ok(url.as_str().trim_end_matches('/').to_owned()) +} + +fn namespace(relay: &str, signer: &str, key: &str) -> Result { + let bytes = serde_json::to_vec(&(relay, signer, key)).map_err(error)?; + Ok(hex::encode(Sha256::digest(bytes))) +} + +fn private_file(path: &Path) -> Result { + let mut options = OpenOptions::new(); + options.create(true).read(true).write(true).truncate(false); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + options.mode(0o600); + } + options.open(path).map_err(error) +} + +fn sync_directory(path: &Path) -> Result<(), CliError> { + File::open(path) + .and_then(|dir| dir.sync_all()) + .map_err(error) +} + +fn durable_directory(dir: &Path) -> Result<(), CliError> { + let mut missing = Vec::new(); + let mut ancestor = dir; + while !ancestor.is_dir() { + if missing.len() >= 128 { + return Err(error("journal directory nesting exceeds 128 levels")); + } + missing.push(ancestor); + ancestor = ancestor + .parent() + .ok_or_else(|| error("journal directory has no existing ancestor"))?; + } + for directory in missing.into_iter().rev() { + let builder = fs::DirBuilder::new(); + #[cfg(unix)] + let builder = { + use std::os::unix::fs::DirBuilderExt; + let mut builder = builder; + builder.mode(0o700); + builder + }; + match builder.create(directory) { + Ok(()) => {} + Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists && directory.is_dir() => {} + Err(e) => return Err(error(e)), + } + } + // A concurrent creator may expose the whole chain before syncing any link. + // Every publisher must persist every ancestor link before it can send. + for (depth, parent) in dir.ancestors().skip(1).enumerate() { + if depth >= 128 { + return Err(error("journal directory nesting exceeds 128 levels")); + } + sync_directory(parent)?; + } + Ok(()) +} + +async fn lock(dir: &Path) -> Result { + durable_directory(dir)?; + let file = private_file(&dir.join("journal.lock"))?; + let deadline = Instant::now() + Duration::from_secs(30); + loop { + match file.try_lock_exclusive() { + Ok(()) => return Ok(file), + Err(e) if e.kind() == std::io::ErrorKind::WouldBlock && Instant::now() < deadline => { + tokio::time::sleep(Duration::from_millis(25)).await + } + Err(e) if e.kind() == std::io::ErrorKind::WouldBlock => { + return Err(CliError::ManagedDeliveryPending( + "journal is busy; retry the same managed key and original route".into(), + )) + } + Err(e) => return Err(error(format!("could not acquire journal lock: {e}"))), + } + } +} + +fn read(dir: &Path) -> Result { + let file = match File::open(dir.join("journal.json")) { + Ok(file) => file, + Err(e) if e.kind() == std::io::ErrorKind::NotFound => { + return Ok(Journal { + version: 2, + ..Default::default() + }) + } + Err(e) => return Err(error(e)), + }; + if file.metadata().map_err(error)?.len() > MAX_JOURNAL_BYTES as u64 { + return Err(error("journal exceeds 16 MiB")); + } + let mut bytes = Vec::new(); + file.take(MAX_JOURNAL_BYTES as u64 + 1) + .read_to_end(&mut bytes) + .map_err(error)?; + if bytes.len() > MAX_JOURNAL_BYTES { + return Err(error("journal exceeds 16 MiB")); + } + let mut journal: Journal = serde_json::from_slice(&bytes).map_err(error)?; + if !matches!(journal.version, 1 | 2) || journal.entries.len() > MAX_RECORDS { + return Err(error("unsupported version or record limit exceeded")); + } + for (id, record) in &journal.entries { + if serde_json::to_vec(record).map_err(error)?.len() > MAX_RECORD_BYTES { + return Err(error("record exceeds 512 KiB")); + } + record.event.verify().map_err(error)?; + for extra in [ + record.recovery_candidate.as_ref(), + record.canonical_event.as_ref(), + ] + .into_iter() + .flatten() + { + let restored = extra.restore(&record.event).map_err(error)?; + if !same_report(&record.event, &restored) { + return Err(error( + "recovery/canonical report changed the original payload or route", + )); + } + } + let markers = buzz_core::nip10::parse_thread_markers_from_parts( + record.event.tags.iter().map(|t| t.as_slice()), + ); + let tag = |name: &str, value: &str| { + record + .event + .tags + .iter() + .filter(|t| t.as_slice().first().map(String::as_str) == Some(name)) + .map(|t| t.as_slice()) + .collect::>() + == vec![&[name.to_owned(), value.to_owned()][..]] + }; + if record.attempts > MAX_ATTEMPTS + || record.event.kind.as_u16() != 9 + || record.event.pubkey.to_hex() != record.signer + || namespace(&record.relay, &record.signer, &record.key)? != *id + || uuid::Uuid::parse_str(&record.key) + .map_err(error)? + .to_string() + != record.key + || uuid::Uuid::parse_str(&record.channel) + .map_err(error)? + .to_string() + != record.channel + || markers.reply.as_deref() != Some(&record.parent) + || !tag("h", &record.channel) + || !tag("buzz-delivery", &record.key) + { + return Err(error("record identity, signature or route is invalid")); + } + validate::validate_content_size(&record.event.content)?; + validate::validate_hex64(&record.parent)?; + } + journal.version = 2; + Ok(journal) +} + +fn write(dir: &Path, journal: &Journal) -> Result<(), CliError> { + let bytes = serde_json::to_vec(journal).map_err(error)?; + if journal.entries.len() > MAX_RECORDS || bytes.len() > MAX_JOURNAL_BYTES { + return Err(error( + "journal capacity reached; pending records are retained", + )); + } + if journal + .entries + .values() + .any(|r| serde_json::to_vec(r).map_or(true, |b| b.len() > MAX_RECORD_BYTES)) + { + return Err(error("record exceeds 512 KiB")); + } + // A fixed temporary name is safe under the one lock, including crash residue. + let temporary = dir.join("journal.pending"); + let mut file = private_file(&temporary)?; + file.set_len(0).map_err(error)?; + file.write_all(&bytes).map_err(error)?; + file.sync_all().map_err(error)?; + fs::rename(temporary, dir.join("journal.json")).map_err(error)?; + sync_directory(dir) +} + +async fn check_membership(client: &BuzzClient, channel: &str) -> Result<(), CliError> { + let raw = client + .query(&serde_json::json!({"kinds":[39002],"#d":[channel],"limit":1})) + .await?; + let events: Vec = serde_json::from_str(&raw).map_err(error)?; + let event = events + .first() + .ok_or_else(|| error("current channel membership is unavailable"))?; + event.verify().map_err(error)?; + if events.len() != 1 + || event.kind.as_u16() != 39002 + || !event + .tags + .iter() + .any(|tag| tag.as_slice() == ["d", channel]) + { + return Err(error( + "membership snapshot does not identify the original channel", + )); + } + let signer = client.keys().public_key().to_hex(); + if !event.tags.iter().any(|tag| { + let parts = tag.as_slice(); + parts.first().map(String::as_str) == Some("p") && parts.get(1) == Some(&signer) + }) { + return Err(CliError::Auth("original channel membership was revoked; delivery is retained and will not be rerouted".into())); + } + Ok(()) +} + +fn acknowledgment( + raw: &str, + original: &Event, + recovery: bool, +) -> Result<(serde_json::Value, Event), CliError> { + let response: serde_json::Value = serde_json::from_str(raw).map_err(error)?; + if response["accepted"] != true { + return Err(error("relay did not accept the managed report")); + } + let message = response["message"].as_str().unwrap_or(""); + let chosen = if let Some(encoded) = message.strip_prefix(RECEIPT_PREFIX) { + let event: Event = serde_json::from_str(encoded).map_err(error)?; + event.verify().map_err(error)?; + if !same_report(original, &event) { + return Err(error( + "canonical receipt changed the original signed report", + )); + } + event + } else if !recovery { + original.clone() + } else { + return Err(error( + "relay does not support atomic recovery; signed evidence retained", + )); + }; + if response["event_id"] != chosen.id.to_hex() { + return Err(error("relay did not acknowledge the canonical event ID")); + } + Ok(( + serde_json::json!({"event_id":chosen.id.to_hex(),"accepted":true,"message":if recovery {"managed report recovered"} else {"managed report acknowledged"},"mention_pubkeys":messages::event_mention_pubkeys(&chosen)}), + chosen, + )) +} + +#[allow(clippy::too_many_arguments)] +pub(super) async fn deliver( + client: &BuzzClient, + channel: &str, + parent: &str, + content: &str, + key: &str, + directory: Option<&Path>, + mentions: Vec, +) -> Result { + let key = uuid::Uuid::parse_str(key) + .map_err(|_| CliError::Usage("--delivery-key must be a UUID".into()))? + .to_string(); + let channel = validate::parse_uuid(channel)?.to_string(); + let parent = validate::parse_event_id(parent)?.to_hex(); + let dir: PathBuf = directory + .map(Path::to_owned) + .unwrap_or_else(|| PathBuf::from(".buzz-deliveries")); + let dir = if dir.is_absolute() { + dir + } else { + std::env::current_dir().map_err(error)?.join(dir) + }; + let relay = relay_namespace(client)?; + let signer = client.keys().public_key().to_hex(); + let id = namespace(&relay, &signer, &key)?; + let _lock = lock(&dir).await?; + let mut journal = read(&dir)?; + let timestamp = now()?; + journal.entries.retain(|_, record| { + record + .confirmed_at + .is_none_or(|confirmed| timestamp.saturating_sub(confirmed) < RETENTION_SECS) + }); + if let Some(record) = journal.entries.get(&id) { + if record.channel != channel || record.parent != parent { + return Err(CliError::Conflict( + "delivery key is already bound to another original route".into(), + )); + } + } + check_membership(client, &channel).await?; + if !journal.entries.contains_key(&id) { + if journal.entries.len() >= MAX_RECORDS { + return Err(error( + "1024-record capacity reached; pending records are retained", + )); + } + let parent_event = messages::fetch_event(client, &parent).await?; + if messages::channel_id_from_event(&parent, &parent_event)?.to_string() != channel { + return Err(CliError::Conflict( + "reply parent belongs to another channel".into(), + )); + } + let event = messages::prepare_message( + client, + messages::SendMessageParams { + channel_id: channel.clone(), + content: content.into(), + kind: Some(9), + reply_to: Some(parent.clone()), + broadcast: false, + files: vec![], + mentions, + }, + Some(&key), + ) + .await?; + journal.entries.insert( + id.clone(), + Record { + relay, + signer, + key: key.clone(), + channel, + parent, + event, + recovery_candidate: None, + canonical_event: None, + attempts: 0, + confirmed_at: None, + last_error: None, + }, + ); + write(&dir, &journal)?; + } + let record = journal + .entries + .get_mut(&id) + .ok_or_else(|| error("record disappeared"))?; + if record.confirmed_at.is_some() { + let chosen = match record.canonical_event.as_ref() { + Some(stamp) => stamp.restore(&record.event).map_err(error)?, + None => record.event.clone(), + }; + return Ok( + serde_json::json!({"event_id":chosen.id.to_hex(),"accepted":true,"message":"managed delivery already confirmed","mention_pubkeys":messages::event_mention_pubkeys(&chosen)}), + ); + } + if record.attempts >= MAX_ATTEMPTS { + return Err(error(format!("terminal: delivery {key} exhausted {MAX_ATTEMPTS} managed submissions; signed evidence retained in {}", dir.display()))); + } + let original = record.event.clone(); + let recovery_now = now()?; + let recovery = recovery_now.saturating_sub(original.created_at.as_secs()) > 600; + let submission = if recovery { + if record + .recovery_candidate + .as_ref() + .is_none_or(|e| recovery_now.saturating_sub(e.created_at) > 600) + { + let signed = EventBuilder::new(original.kind, &original.content) + .tags(original.tags.clone()) + .allow_self_tagging() + .sign_with_keys(client.keys()) + .map_err(error)?; + record.recovery_candidate = Some(ReportStamp::from(&signed)); + } + let candidate = record + .recovery_candidate + .clone() + .ok_or_else(|| error("recovery candidate disappeared"))?; + let request = RecoveryRequest { + original: original.clone(), + candidate, + }; + let body = serde_json::to_string(&request).map_err(error)?; + if body.len() > buzz_core::managed_delivery::MAX_RECOVERY_BYTES { + return Err(error( + "recovery payload exceeds protocol limit; original retained", + )); + } + client.sign_event( + EventBuilder::new( + Kind::Custom(buzz_core::kind::KIND_MANAGED_DELIVERY_RECOVERY as u16), + body, + ) + .tag(Tag::parse(["h", &record.channel]).map_err(error)?), + )? + } else { + original.clone() + }; + if serde_json::to_vec(&submission).map_err(error)?.len() > 1024 * 1024 { + return Err(error( + "recovery envelope exceeds transport limit; original retained", + )); + } + record.attempts += 1; + write(&dir, &journal)?; + let result = client + .submit_event(submission) + .await + .and_then(|raw| acknowledgment(&raw, &original, recovery)); + let record = journal + .entries + .get_mut(&id) + .ok_or_else(|| error("record disappeared"))?; + match result { + Ok((output, chosen)) => { + record.canonical_event = + (chosen.id != record.event.id).then(|| ReportStamp::from(&chosen)); + record.confirmed_at = Some(now()?); + record.last_error = None; + write(&dir, &journal)?; + Ok(output) + } + Err(e) => { + record.last_error = Some(e.to_string().chars().take(2048).collect()); + let state = if record.attempts == MAX_ATTEMPTS { + "terminal" + } else { + "unconfirmed; retry this same key and route" + }; + write(&dir, &journal)?; + let detail = format!( + "{state}: delivery {key}: {e}; signed event retained in {}", + dir.display() + ); + if matches!( + e, + CliError::Relay { + status: 401 | 403, + .. + } | CliError::Auth(_) + ) { + Err(e) + } else if state == "terminal" { + Err(error(detail)) + } else { + Err(CliError::ManagedDeliveryPending(detail)) + } + } + } +} diff --git a/crates/buzz-cli/src/commands/messages.rs b/crates/buzz-cli/src/commands/messages.rs index f80f928d316..03bb77cef7f 100644 --- a/crates/buzz-cli/src/commands/messages.rs +++ b/crates/buzz-cli/src/commands/messages.rs @@ -63,7 +63,10 @@ fn thread_ref_from_parent_tags( /// - Nested reply: `root` is the parent's own root marker; `parent` is unchanged. /// /// Ensures CLI-sent replies thread correctly using the same NIP-10 logic. -async fn fetch_event(client: &BuzzClient, event_id: &str) -> Result { +pub(super) async fn fetch_event( + client: &BuzzClient, + event_id: &str, +) -> Result { let filter = serde_json::json!({ "ids": [event_id], "limit": 1 }); let raw = client.query(&filter).await?; let events: serde_json::Value = serde_json::from_str(&raw) @@ -94,7 +97,10 @@ fn thread_ref_from_event(event_id: &str, event: &serde_json::Value) -> Result Result { +pub(super) fn channel_id_from_event( + event_id: &str, + event: &serde_json::Value, +) -> Result { let tags = event .get("tags") .and_then(|tags| tags.as_array()) @@ -278,7 +284,7 @@ fn missing_members(mentions: &[String], members: &[String]) -> Vec { .collect() } -fn event_mention_pubkeys(event: &nostr::Event) -> Vec { +pub(super) fn event_mention_pubkeys(event: &nostr::Event) -> Vec { event .tags .iter() @@ -608,10 +614,27 @@ pub struct SendMessageParams { pub mentions: Vec, } -pub async fn cmd_send_message( +pub async fn cmd_send_message(client: &BuzzClient, p: SendMessageParams) -> Result<(), CliError> { + let event = prepare_message(client, p, None).await?; + let emitted_mentions = event_mention_pubkeys(&event); + let resp = client.submit_event(event).await?; + let mut output: serde_json::Value = serde_json::from_str(&normalize_write_response(&resp)) + .unwrap_or_else(|_| serde_json::json!({ "response": resp })); + if let Some(object) = output.as_object_mut() { + object.insert( + "mention_pubkeys".into(), + serde_json::json!(emitted_mentions), + ); + } + println!("{output}"); + Ok(()) +} + +pub(super) async fn prepare_message( client: &BuzzClient, mut p: SendMessageParams, -) -> Result<(), CliError> { + delivery_key: Option<&str>, +) -> Result { // Allow '-' to read content from stdin. This keeps callers from having to // jam shell-metacharacter-heavy text (backticks, $vars, etc.) through argv // quoting — the source of countless self-inflicted command-substitution @@ -742,19 +765,16 @@ pub async fn cmd_send_message( } }; + let builder = if let Some(key) = delivery_key { + builder.tag( + nostr::Tag::parse(["buzz-delivery", key]) + .map_err(|e| CliError::Other(e.to_string()))?, + ) + } else { + builder + }; let event = client.sign_event(builder)?; - let emitted_mentions = event_mention_pubkeys(&event); - let resp = client.submit_event(event).await?; - let mut output: serde_json::Value = serde_json::from_str(&normalize_write_response(&resp)) - .unwrap_or_else(|_| serde_json::json!({ "response": resp })); - if let Some(object) = output.as_object_mut() { - object.insert( - "mention_pubkeys".into(), - serde_json::json!(emitted_mentions), - ); - } - println!("{output}"); - Ok(()) + Ok(event) } pub struct SendDiffParams { @@ -937,6 +957,27 @@ pub async fn dispatch( ) -> Result<(), CliError> { use crate::MessagesCmd; match cmd { + MessagesCmd::Deliver { + channel, + reply_to, + content, + delivery_key, + journal_dir, + mentions, + } => { + let output = super::delivery::deliver( + client, + &channel, + &reply_to, + &content, + &delivery_key, + journal_dir.as_deref(), + mentions, + ) + .await?; + println!("{output}"); + Ok(()) + } MessagesCmd::Send { channel, content, diff --git a/crates/buzz-cli/src/commands/mod.rs b/crates/buzz-cli/src/commands/mod.rs index fed35ec201e..59e418e51a5 100644 --- a/crates/buzz-cli/src/commands/mod.rs +++ b/crates/buzz-cli/src/commands/mod.rs @@ -1,6 +1,7 @@ pub mod agents; pub mod channel_templates; pub mod channels; +mod delivery; pub mod dms; pub mod emoji; pub mod feed; diff --git a/crates/buzz-cli/src/delivery_tests.rs b/crates/buzz-cli/src/delivery_tests.rs new file mode 100644 index 00000000000..6538167407d --- /dev/null +++ b/crates/buzz-cli/src/delivery_tests.rs @@ -0,0 +1,712 @@ +//! Production CLI regressions against a signature-checking local mock relay. +use super::*; +use axum::{extract::State, response::IntoResponse, routing::post, Json, Router}; +use nostr::{Event, EventBuilder, Kind, Tag}; +use std::sync::{Arc, Mutex}; + +#[derive(Clone)] +struct Relay { + keys: Keys, + channel: String, + parent: Event, + member: Arc>, + ack: Arc>, + membership_channel: Arc>>, + extra_members: Arc>>, + events: Arc>>, + bodies: Arc>>>, +} + +async fn query( + State(r): State, + Json(filter): Json, +) -> Json { + let filter = &filter[0]; + if filter["kinds"][0] == 39002 { + let membership_channel = r + .membership_channel + .lock() + .unwrap() + .clone() + .unwrap_or_else(|| filter["#d"][0].as_str().unwrap().to_string()); + let mut tags = if *r.member.lock().unwrap() { + vec![ + Tag::parse(["d", &membership_channel]).unwrap(), + Tag::parse(["p", &r.keys.public_key().to_hex()]).unwrap(), + ] + } else { + vec![Tag::parse(["d", &membership_channel]).unwrap()] + }; + if *r.member.lock().unwrap() { + tags.extend( + r.extra_members + .lock() + .unwrap() + .iter() + .map(|pubkey| Tag::parse(["p", pubkey]).unwrap()), + ); + } + let e = EventBuilder::new(Kind::Custom(39002), "") + .tags(tags) + .allow_self_tagging() + .sign_with_keys(&r.keys) + .unwrap(); + return Json(serde_json::json!([e])); + } + if filter["kinds"][0] == 0 { + let e = EventBuilder::new(Kind::Metadata, r#"{"name":"Callback"}"#) + .sign_with_keys(&r.keys) + .unwrap(); + return Json(serde_json::json!([e])); + } + if filter["ids"][0] == r.parent.id.to_hex() { + return Json(serde_json::json!([r.parent])); + } + let events = r.events.lock().unwrap(); + Json(serde_json::json!(events + .iter() + .filter(|e| filter["ids"] + .as_array() + .is_none_or(|ids| ids.iter().any(|id| id == &e.id.to_hex()))) + .collect::>())) +} + +async fn publish(State(r): State, body: axum::body::Bytes) -> axum::response::Response { + let event: Event = serde_json::from_slice(&body).unwrap(); + event.verify().unwrap(); + r.bodies.lock().unwrap().push(body.to_vec()); + let mut events = r.events.lock().unwrap(); + let duplicate = events.iter().any(|e| e.id == event.id); + let ack = *r.ack.lock().unwrap(); + if !duplicate && ack != "rejected" { + events.push(event.clone()); + } + if ack == "malformed" { + return "{broken ack".into_response(); + } + match ack { + "wrong-id" => { + Json(serde_json::json!({"accepted":true,"event_id":"b".repeat(64),"message":"saved"})) + } + "rejected" => Json( + serde_json::json!({"accepted":false,"event_id":event.id.to_hex(),"message":"rejected"}), + ), + _ => Json( + serde_json::json!({"accepted":true,"event_id":event.id.to_hex(),"message":if duplicate {"duplicate:"} else {"saved"}}), + ), + }.into_response() +} + +async fn fixture() -> (Relay, String, tempfile::TempDir) { + fixture_with_keys(Keys::generate()).await +} + +async fn fixture_with_keys(keys: Keys) -> (Relay, String, tempfile::TempDir) { + let channel = "123e4567-e89b-12d3-a456-426614174000".to_string(); + let parent = EventBuilder::new(Kind::Custom(9), "original task") + .tag(Tag::parse(["h", &channel]).unwrap()) + .sign_with_keys(&keys) + .unwrap(); + let relay = Relay { + keys, + channel, + parent, + member: Arc::new(Mutex::new(true)), + ack: Arc::new(Mutex::new("ok")), + membership_channel: Arc::new(Mutex::new(None)), + extra_members: Arc::new(Mutex::new(vec![])), + events: Arc::new(Mutex::new(vec![])), + bodies: Arc::new(Mutex::new(vec![])), + }; + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let url = format!("http://{}", listener.local_addr().unwrap()); + let app = Router::new() + .route("/query", post(query)) + .route("/events", post(publish)) + .with_state(relay.clone()); + tokio::spawn(async move { + axum::serve(listener, app).await.unwrap(); + }); + (relay, url, tempfile::tempdir().unwrap()) +} + +async fn deliver(r: &Relay, url: &str, dir: &std::path::Path, key: &str, content: &str) -> i32 { + let cli = match parse_args([ + "buzz", + "messages", + "deliver", + "--delivery-key", + key, + "--journal-dir", + dir.to_str().unwrap(), + "--channel", + &r.channel, + "--reply-to", + &r.parent.id.to_hex(), + "--content", + content, + ]) { + Ok(cli) => cli, + Err(_) => return 1, + }; + let Cmd::Messages(command) = cli.command else { + panic!("expected messages command"); + }; + let client = BuzzClient::new(url.into(), r.keys.clone(), None, None).unwrap(); + match commands::messages::dispatch(command, &client, &OutputFormat::Json).await { + Ok(()) => 0, + Err(e) => { + eprintln!("{e}"); + error::exit_code(&e) + } + } +} + +#[tokio::test] +async fn managed_same_key_changed_text_returns_first_signed_delivery() { + let (r, url, dir) = fixture().await; + let key = Uuid::new_v4().to_string(); + assert_eq!(deliver(&r, &url, dir.path(), &key, "first").await, 0); + assert_eq!(deliver(&r, &url, dir.path(), &key, "changed").await, 0); + let events = r.events.lock().unwrap(); + assert_eq!(events.len(), 1); + assert_eq!(events[0].content, "first"); + assert!(events[0] + .tags + .iter() + .any(|tag| tag.as_slice() == ["buzz-delivery", &key])); + assert_eq!(r.bodies.lock().unwrap().len(), 1); +} + +#[tokio::test] +async fn managed_equal_text_distinct_keys_have_distinct_signed_events() { + let (r, url, dir) = fixture().await; + for _ in 0..2 { + assert_eq!( + deliver(&r, &url, dir.path(), &Uuid::new_v4().to_string(), "same").await, + 0 + ); + } + let events = r.events.lock().unwrap(); + assert_eq!(events.len(), 2); + assert_ne!(events[0].id, events[1].id); +} + +#[tokio::test] +async fn managed_concurrent_publishers_share_first_event() { + let (r, url, dir) = fixture().await; + let key = Uuid::new_v4().to_string(); + let (a, b) = tokio::join!( + deliver(&r, &url, dir.path(), &key, "first"), + deliver(&r, &url, dir.path(), &key, "second") + ); + assert_eq!((a, b), (0, 0)); + assert_eq!(r.events.lock().unwrap().len(), 1); + assert_eq!(r.bodies.lock().unwrap().len(), 1); +} + +#[tokio::test] +async fn managed_wrong_ack_keeps_signed_event_for_new_invocation() { + let (r, url, dir) = fixture().await; + let key = Uuid::new_v4().to_string(); + *r.ack.lock().unwrap() = "wrong-id"; + assert_ne!(deliver(&r, &url, dir.path(), &key, "original").await, 0); + *r.ack.lock().unwrap() = "ok"; + assert_eq!(deliver(&r, &url, dir.path(), &key, "new text").await, 0); + let bodies = r.bodies.lock().unwrap(); + assert_eq!(bodies.len(), 2); + assert_eq!(bodies[0], bodies[1]); + assert_eq!(r.events.lock().unwrap().len(), 1); +} + +#[tokio::test] +async fn managed_revoked_access_refuses_pending_and_confirmed_replay() { + let (r, url, dir) = fixture().await; + let key = Uuid::new_v4().to_string(); + assert_eq!(deliver(&r, &url, dir.path(), &key, "first").await, 0); + *r.member.lock().unwrap() = false; + assert_eq!(deliver(&r, &url, dir.path(), &key, "changed").await, 3); + assert_eq!( + deliver(&r, &url, dir.path(), &Uuid::new_v4().to_string(), "new").await, + 3 + ); + assert_eq!(r.events.lock().unwrap().len(), 1); +} + +#[tokio::test] +async fn managed_mentions_progress_and_explicit_edit_remain_independent() { + let (r, url, dir) = fixture().await; + let key = Uuid::new_v4().to_string(); + let client = BuzzClient::new(url.clone(), r.keys.clone(), None, None).unwrap(); + commands::messages::cmd_send_message( + &client, + commands::messages::SendMessageParams { + channel_id: r.channel.clone(), + content: "progress".into(), + kind: None, + reply_to: Some(r.parent.id.to_hex()), + broadcast: false, + files: vec![], + mentions: vec![], + }, + ) + .await + .unwrap(); + assert_eq!( + deliver(&r, &url, dir.path(), &key, "@Callback final").await, + 0 + ); + let event = r.events.lock().unwrap()[1].clone(); + assert!(event + .tags + .iter() + .any(|tag| tag.as_slice() == ["p", &r.keys.public_key().to_hex()])); + commands::messages::cmd_edit_message(&client, &event.id.to_hex(), "correction") + .await + .unwrap(); + assert_eq!( + deliver(&r, &url, dir.path(), &key, "another final").await, + 0 + ); + let events = r.events.lock().unwrap(); + assert_eq!(events.len(), 3); + assert_eq!(events[0].content, "progress"); + assert_eq!(events[2].kind.as_u16(), 40003); +} + +#[tokio::test] +async fn managed_pending_attempt_budget_is_finite_and_retains_evidence() { + let (r, url, dir) = fixture().await; + let key = Uuid::new_v4().to_string(); + *r.ack.lock().unwrap() = "wrong-id"; + for _ in 0..7 { + assert_eq!(deliver(&r, &url, dir.path(), &key, "first").await, 2); + } + assert_eq!(deliver(&r, &url, dir.path(), &key, "changed").await, 4); + assert_eq!( + deliver(&r, &url, dir.path(), &key, "changed again").await, + 4 + ); + assert_eq!(r.bodies.lock().unwrap().len(), 8); + assert_eq!(r.events.lock().unwrap().len(), 1); + assert!(std::fs::read_to_string(dir.path().join("journal.json")) + .unwrap() + .contains("first")); +} + +#[tokio::test] +async fn managed_route_conflict_does_not_publish_or_reroute() { + let (r, url, dir) = fixture().await; + let key = Uuid::new_v4().to_string(); + assert_eq!(deliver(&r, &url, dir.path(), &key, "first").await, 0); + let mut changed = r.clone(); + changed.channel = Uuid::new_v4().to_string(); + assert_eq!( + deliver(&changed, &url, dir.path(), &key, "changed").await, + 5 + ); + assert_eq!( + deliver( + &changed, + &url, + dir.path(), + &Uuid::new_v4().to_string(), + "new" + ) + .await, + 5 + ); + assert_eq!(r.events.lock().unwrap().len(), 1); +} + +#[tokio::test] +async fn managed_corrupt_or_oversized_journal_never_re_signs() { + let (r, url, dir) = fixture().await; + let key = Uuid::new_v4().to_string(); + assert_eq!(deliver(&r, &url, dir.path(), &key, "first").await, 0); + let path = dir.path().join("journal.json"); + let original = std::fs::read(&path).unwrap(); + std::fs::write(&path, b"broken").unwrap(); + assert_eq!(deliver(&r, &url, dir.path(), &key, "second").await, 4); + assert_eq!(std::fs::read(&path).unwrap(), b"broken"); + let file = std::fs::File::create(&path).unwrap(); + file.set_len(16 * 1024 * 1024 + 1).unwrap(); + assert_eq!(deliver(&r, &url, dir.path(), &key, "third").await, 4); + std::fs::write(&path, original).unwrap(); + let mut value: serde_json::Value = + serde_json::from_slice(&std::fs::read(&path).unwrap()).unwrap(); + value["entries"] + .as_object_mut() + .unwrap() + .values_mut() + .next() + .unwrap()["event"]["content"] = "signature corruption".into(); + std::fs::write(&path, value.to_string()).unwrap(); + assert_eq!(deliver(&r, &url, dir.path(), &key, "fourth").await, 4); + assert_eq!(r.events.lock().unwrap().len(), 1); +} + +#[tokio::test] +async fn managed_confirmed_expiry_is_explicit_boundary_pending_never_expires() { + let (r, url, dir) = fixture().await; + let key = Uuid::new_v4().to_string(); + assert_eq!(deliver(&r, &url, dir.path(), &key, "first").await, 0); + let path = dir.path().join("journal.json"); + let mut value: serde_json::Value = + serde_json::from_slice(&std::fs::read(&path).unwrap()).unwrap(); + value["entries"] + .as_object_mut() + .unwrap() + .values_mut() + .next() + .unwrap()["confirmed_at"] = 1.into(); + std::fs::write(&path, value.to_string()).unwrap(); + assert_eq!( + deliver(&r, &url, dir.path(), &key, "after retention").await, + 0 + ); + assert_eq!(r.events.lock().unwrap().len(), 2); + value = serde_json::from_slice(&std::fs::read(&path).unwrap()).unwrap(); + value["entries"] + .as_object_mut() + .unwrap() + .values_mut() + .next() + .unwrap()["confirmed_at"] = serde_json::Value::Null; + std::fs::write(&path, value.to_string()).unwrap(); + assert_eq!( + deliver(&r, &url, dir.path(), &key, "pending retry").await, + 0 + ); + assert_eq!(r.events.lock().unwrap().len(), 2); + let bodies = r.bodies.lock().unwrap(); + assert_eq!(bodies[1], bodies[2]); +} + +#[tokio::test] +async fn managed_full_pending_journal_rejects_new_keys_without_evicting_or_growing_locks() { + use sha2::{Digest, Sha256}; + let (r, url, dir) = fixture().await; + let original_key = Uuid::new_v4().to_string(); + assert_eq!( + deliver(&r, &url, dir.path(), &original_key, "first").await, + 0 + ); + let path = dir.path().join("journal.json"); + let mut value: serde_json::Value = + serde_json::from_slice(&std::fs::read(&path).unwrap()).unwrap(); + let template = value["entries"] + .as_object() + .unwrap() + .values() + .next() + .unwrap() + .clone(); + for _ in 1..1024 { + let key = Uuid::new_v4().to_string(); + let mut record = template.clone(); + let tags: Vec = r.events.lock().unwrap()[0] + .tags + .iter() + .filter(|t| t.as_slice()[0] != "buzz-delivery") + .cloned() + .collect(); + let event = EventBuilder::new(Kind::Custom(9), "retained pending") + .tags(tags) + .tag(Tag::parse(["buzz-delivery", &key]).unwrap()) + .sign_with_keys(&r.keys) + .unwrap(); + record["key"] = key.clone().into(); + record["event"] = serde_json::to_value(event).unwrap(); + record["confirmed_at"] = serde_json::Value::Null; + let namespace = serde_json::to_vec(&(&url, r.keys.public_key().to_hex(), &key)).unwrap(); + let id = hex::encode(Sha256::digest(namespace)); + value["entries"].as_object_mut().unwrap().insert(id, record); + } + std::fs::write(&path, value.to_string()).unwrap(); + assert_eq!( + deliver( + &r, + &url, + dir.path(), + &Uuid::new_v4().to_string(), + "overflow" + ) + .await, + 4 + ); + assert_eq!( + deliver(&r, &url, dir.path(), &original_key, "first replay").await, + 0 + ); + let after: serde_json::Value = serde_json::from_slice(&std::fs::read(&path).unwrap()).unwrap(); + assert_eq!(after["entries"].as_object().unwrap().len(), 1024); + assert_eq!(r.events.lock().unwrap().len(), 1); + assert!(std::fs::read_dir(dir.path()).unwrap().count() <= 3); +} + +#[tokio::test] +async fn managed_membership_snapshot_from_another_channel_cannot_authorize_delivery() { + let (r, url, dir) = fixture().await; + *r.membership_channel.lock().unwrap() = Some(Uuid::new_v4().to_string()); + assert_eq!( + deliver(&r, &url, dir.path(), &Uuid::new_v4().to_string(), "first").await, + 4 + ); + assert!(r.events.lock().unwrap().is_empty()); +} + +#[cfg(unix)] +#[tokio::test] +async fn managed_new_journal_directories_and_files_are_private() { + use std::os::unix::fs::PermissionsExt; + let (r, url, dir) = fixture().await; + let nested = dir.path().join("parent/new journal"); + assert_eq!( + deliver( + &r, + &url, + &nested, + &Uuid::new_v4().to_string(), + "private final report" + ) + .await, + 0 + ); + assert_eq!( + std::fs::metadata(&nested).unwrap().permissions().mode() & 0o777, + 0o700 + ); + for file in std::fs::read_dir(&nested).unwrap() { + assert_eq!( + file.unwrap().metadata().unwrap().permissions().mode() & 0o777, + 0o600 + ); + } +} + +#[tokio::test(start_paused = true)] +async fn managed_busy_journal_lock_has_bounded_safe_retry_wait() { + use fs2::FileExt; + let (r, url, dir) = fixture().await; + let held = std::fs::File::create(dir.path().join("journal.lock")).unwrap(); + held.lock_exclusive().unwrap(); + let result = tokio::time::timeout( + std::time::Duration::from_secs(31), + deliver(&r, &url, dir.path(), &Uuid::new_v4().to_string(), "first"), + ) + .await; + assert_eq!(result.unwrap(), 2); + assert!(r.events.lock().unwrap().is_empty()); +} + +#[tokio::test] +async fn managed_malformed_and_rejected_ack_remain_pending_until_valid_ack() { + let (r, url, dir) = fixture().await; + let key = Uuid::new_v4().to_string(); + *r.ack.lock().unwrap() = "rejected"; + assert_eq!(deliver(&r, &url, dir.path(), &key, "first").await, 2); + assert!(r.events.lock().unwrap().is_empty()); + *r.ack.lock().unwrap() = "malformed"; + assert_eq!(deliver(&r, &url, dir.path(), &key, "changed").await, 2); + *r.ack.lock().unwrap() = "ok"; + assert_eq!( + deliver(&r, &url, dir.path(), &key, "changed again").await, + 0 + ); + assert_eq!(r.events.lock().unwrap().len(), 1); + let bodies = r.bodies.lock().unwrap(); + assert_eq!(bodies.len(), 3); + assert!(bodies.iter().all(|b| b == &bodies[0])); +} + +#[tokio::test] +async fn managed_oversized_record_and_changed_saved_namespace_or_route_fail_closed() { + let (r, url, dir) = fixture().await; + let key = Uuid::new_v4().to_string(); + assert_eq!(deliver(&r, &url, dir.path(), &key, "first").await, 0); + let path = dir.path().join("journal.json"); + let original: serde_json::Value = + serde_json::from_slice(&std::fs::read(&path).unwrap()).unwrap(); + for field in ["last_error", "relay", "parent", "attempts"] { + let mut value = original.clone(); + let record = value["entries"] + .as_object_mut() + .unwrap() + .values_mut() + .next() + .unwrap(); + record[field] = match field { + "last_error" => "x".repeat(512 * 1024).into(), + "relay" => "https://other.invalid".into(), + "parent" => "b".repeat(64).into(), + _ => 9.into(), + }; + std::fs::write(&path, value.to_string()).unwrap(); + assert_eq!(deliver(&r, &url, dir.path(), &key, "changed").await, 4); + } + assert_eq!(r.events.lock().unwrap().len(), 1); +} + +#[tokio::test] +async fn managed_key_namespace_separates_signers_and_relays() { + let (r, url, dir) = fixture().await; + let key = Uuid::new_v4().to_string(); + assert_eq!(deliver(&r, &url, dir.path(), &key, "first signer").await, 0); + let mut second_signer = r.clone(); + second_signer.keys = Keys::generate(); + r.extra_members + .lock() + .unwrap() + .push(second_signer.keys.public_key().to_hex()); + assert_eq!( + deliver(&second_signer, &url, dir.path(), &key, "second signer").await, + 0 + ); + assert_eq!(r.events.lock().unwrap().len(), 2); + let (other_relay, other_url, _) = fixture_with_keys(r.keys.clone()).await; + assert_eq!( + deliver(&other_relay, &other_url, dir.path(), &key, "other relay").await, + 0 + ); + assert_eq!(other_relay.events.lock().unwrap().len(), 1); + let journal: serde_json::Value = + serde_json::from_slice(&std::fs::read(dir.path().join("journal.json")).unwrap()).unwrap(); + assert_eq!(journal["entries"].as_object().unwrap().len(), 3); +} + +#[tokio::test] +async fn managed_explicit_callback_mention_preserves_unresolved_presentation_name() { + let (r, url, dir) = fixture().await; + let client = BuzzClient::new(url, r.keys.clone(), None, None).unwrap(); + let cli = parse_args([ + "buzz", + "messages", + "deliver", + "--delivery-key", + &Uuid::new_v4().to_string(), + "--journal-dir", + dir.path().to_str().unwrap(), + "--channel", + &r.channel, + "--reply-to", + &r.parent.id.to_hex(), + "--content", + "@Unknown callback", + "--mention", + &r.keys.public_key().to_hex(), + ]) + .unwrap(); + let Cmd::Messages(command) = cli.command else { + panic!("expected messages command"); + }; + commands::messages::dispatch(command, &client, &OutputFormat::Json) + .await + .unwrap(); + let events = r.events.lock().unwrap(); + assert_eq!(events.len(), 1); + assert!(events[0] + .tags + .iter() + .any(|tag| tag.as_slice() == ["p", &r.keys.public_key().to_hex()])); +} + +#[cfg(unix)] +#[tokio::test] +async fn managed_existing_directory_requires_parent_link_sync_before_publication() { + use std::os::unix::fs::PermissionsExt; + let (r, url, dir) = fixture().await; + let parent = dir.path().join("parent"); + let journal = parent.join("journal"); + std::fs::create_dir_all(&journal).unwrap(); + std::fs::set_permissions(&parent, std::fs::Permissions::from_mode(0o100)).unwrap(); + let result = deliver( + &r, + &url, + &journal, + &Uuid::new_v4().to_string(), + "must not publish without durable directory link", + ) + .await; + std::fs::set_permissions(&parent, std::fs::Permissions::from_mode(0o700)).unwrap(); + assert_eq!(result, 4); + assert!(r.events.lock().unwrap().is_empty()); +} + +#[cfg(unix)] +#[tokio::test] +async fn managed_existing_nested_directory_requires_ancestor_link_sync_before_publication() { + use std::os::unix::fs::PermissionsExt; + let (r, url, dir) = fixture().await; + let ancestor = dir.path().join("ancestor"); + let journal = ancestor.join("parent/journal"); + // A concurrent creator can expose the entire chain before syncing its links. + std::fs::create_dir_all(&journal).unwrap(); + std::fs::set_permissions(&ancestor, std::fs::Permissions::from_mode(0o100)).unwrap(); + let result = deliver( + &r, + &url, + &journal, + &Uuid::new_v4().to_string(), + "must not publish without durable ancestor links", + ) + .await; + std::fs::set_permissions(&ancestor, std::fs::Permissions::from_mode(0o700)).unwrap(); + assert_eq!(result, 4); + assert!(r.events.lock().unwrap().is_empty()); +} + +#[tokio::test] +async fn managed_expired_journal_never_treats_legacy_wrapper_ack_as_delivery() { + let (r, url, dir) = fixture().await; + let key = Uuid::new_v4().to_string(); + assert_eq!(deliver(&r, &url, dir.path(), &key, "first").await, 0); + let path = dir.path().join("journal.json"); + let mut journal: serde_json::Value = + serde_json::from_slice(&std::fs::read(&path).unwrap()).unwrap(); + let record = journal["entries"] + .as_object_mut() + .unwrap() + .values_mut() + .next() + .unwrap(); + let first: Event = serde_json::from_value(record["event"].clone()).unwrap(); + let old = EventBuilder::new(first.kind, &first.content) + .tags(first.tags.clone()) + .allow_self_tagging() + .custom_created_at(nostr::Timestamp::from( + nostr::Timestamp::now().as_secs() - 3600, + )) + .sign_with_keys(&r.keys) + .unwrap(); + record["event"] = serde_json::to_value(&old).unwrap(); + record["confirmed_at"] = serde_json::Value::Null; + std::fs::write(&path, journal.to_string()).unwrap(); + r.events.lock().unwrap().clear(); + assert_eq!( + deliver(&r, &url, dir.path(), &key, "changed after restart").await, + 2 + ); + let saved: serde_json::Value = serde_json::from_slice(&std::fs::read(&path).unwrap()).unwrap(); + let record = saved["entries"] + .as_object() + .unwrap() + .values() + .next() + .unwrap(); + let original: Event = serde_json::from_value(record["event"].clone()).unwrap(); + let stamp: buzz_core::managed_delivery::ReportStamp = + serde_json::from_value(record["recovery_candidate"].clone()).unwrap(); + let candidate = stamp.restore(&original).unwrap(); + assert_eq!(original, old); + assert!(buzz_core::managed_delivery::same_report( + &original, &candidate + )); + assert!(record["confirmed_at"].is_null()); + assert!(r + .events + .lock() + .unwrap() + .iter() + .all(|e| e.kind.as_u16() == 20003)); +} diff --git a/crates/buzz-cli/src/error.rs b/crates/buzz-cli/src/error.rs index 2edcd6aa9d4..af92f647f6e 100644 --- a/crates/buzz-cli/src/error.rs +++ b/crates/buzz-cli/src/error.rs @@ -39,6 +39,11 @@ pub enum CliError { #[error("delivery unknown: {0}")] DeliveryUnknown(String), + /// A managed attempt is pending or its journal is busy. Retrying the same + /// key, route and journal is safe; never replace it with a fresh send. + #[error("{0}")] + ManagedDeliveryPending(String), + /// Catch-all for unexpected failures #[error("{0}")] Other(String), @@ -79,6 +84,7 @@ pub fn is_retryable_error(e: &CliError) -> bool { || net_err.is_decode() } CliError::Relay { status, .. } => matches!(status, 429 | 502 | 503 | 504), + CliError::ManagedDeliveryPending(_) => true, CliError::DeliveryUnknown(_) => false, _ => false, } @@ -102,7 +108,7 @@ pub fn exit_code(e: &CliError) -> i32 { CliError::Key(_) => 3, CliError::Conflict(_) => 5, CliError::NotFound(_) => 1, - CliError::DeliveryUnknown(_) => 2, + CliError::DeliveryUnknown(_) | CliError::ManagedDeliveryPending(_) => 2, CliError::Other(_) => 4, } } @@ -125,6 +131,7 @@ pub fn print_error(e: &CliError) { CliError::Conflict(_) => "conflict", CliError::NotFound(_) => "not_found", CliError::DeliveryUnknown(_) => "delivery_unknown", + CliError::ManagedDeliveryPending(_) => "managed_delivery_pending", CliError::Other(_) => "error", }; let obj = serde_json::json!({ diff --git a/crates/buzz-cli/src/lib.rs b/crates/buzz-cli/src/lib.rs index c2f6b7e089c..0f08626810a 100644 --- a/crates/buzz-cli/src/lib.rs +++ b/crates/buzz-cli/src/lib.rs @@ -408,6 +408,27 @@ buzz agents archived" #[derive(Subcommand)] pub enum MessagesCmd { + /// Durably publish one final report per delivery UUID; retries return the first report + Deliver { + /// Stable background group UUID supplied by the harness + #[arg(long)] + delivery_key: String, + /// Original channel UUID + #[arg(long)] + channel: String, + /// Original parent event ID + #[arg(long)] + reply_to: String, + /// Final report text; '-' reads stdin on the first delivery + #[arg(long)] + content: String, + /// Persistent journal directory (default: .buzz-deliveries in the working directory) + #[arg(long, env = "BUZZ_DELIVERY_JOURNAL_DIR")] + journal_dir: Option, + /// Pubkey to notify (hex or npub; repeatable), with the same mention rules as send + #[arg(long = "mention")] + mentions: Vec, + }, /// Send a message to a channel #[command( after_help = "Examples:\n buzz messages send --channel --content \"hello\"\n buzz messages send --channel --content \"@alice check this\"\n echo \"hello from stdin\" | buzz messages send --channel --content -" @@ -2443,6 +2464,7 @@ mod tests { names(&cmd, "messages"), vec![ "delete", + "deliver", "edit", "get", "search", @@ -2587,7 +2609,7 @@ mod tests { ("feed", 1), ("issues", 6), ("media", 1), - ("messages", 8), + ("messages", 9), ("pack", 2), ("patches", 4), ("pr", 5), @@ -2786,3 +2808,6 @@ mod tests { ); } } + +#[cfg(test)] +mod delivery_tests; diff --git a/crates/buzz-core/src/kind.rs b/crates/buzz-core/src/kind.rs index 32d8959fb69..6ebda6538dd 100644 --- a/crates/buzz-core/src/kind.rs +++ b/crates/buzz-core/src/kind.rs @@ -467,6 +467,8 @@ pub const KIND_PRESENCE_UPDATE: u32 = 20001; pub const KIND_PAIRING: u32 = 24134; /// Ephemeral: typing indicator for a channel. pub const KIND_TYPING_INDICATOR: u32 = 20002; +/// Ephemeral command: elect one author-signed managed report; never fan out the wrapper. +pub const KIND_MANAGED_DELIVERY_RECOVERY: u32 = 20003; /// Ephemeral: owner-scoped encrypted agent observer telemetry and control frame. pub const KIND_AGENT_OBSERVER_FRAME: u32 = 24200; /// Ephemeral: huddle emoji reaction burst. Channel-scoped to the ephemeral diff --git a/crates/buzz-core/src/lib.rs b/crates/buzz-core/src/lib.rs index e7669c874b0..b0f07eb3057 100644 --- a/crates/buzz-core/src/lib.rs +++ b/crates/buzz-core/src/lib.rs @@ -24,6 +24,8 @@ pub mod git_perms; pub mod invite; /// Buzz kind number registry — custom event type constants. pub mod kind; +/// Managed background report recovery wire contract. +pub mod managed_delivery; /// Network utilities — SSRF-safe IP classification. pub mod network; /// NIP-10 thread-marker parsing — shared `root`/`reply` marker resolver. diff --git a/crates/buzz-core/src/managed_delivery.rs b/crates/buzz-core/src/managed_delivery.rs new file mode 100644 index 00000000000..d7f740892d3 --- /dev/null +++ b/crates/buzz-core/src/managed_delivery.rs @@ -0,0 +1,195 @@ +//! Author-signed managed report recovery over the Nostr event bridge. +use nostr::Event; +use serde::{Deserialize, Serialize}; +use uuid::Uuid; + +/// Prefix of a signed canonical report in a standard Nostr OK message. +pub const RECEIPT_PREFIX: &str = "managed-delivery:"; +/// Maximum serialized recovery payload size. +pub const MAX_RECOVERY_BYTES: usize = 512 * 1024; + +/// Saved original evidence and a newly timestamped copy, both author-signed. +#[derive(Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct RecoveryRequest { + /// The first journaled report, whose body and route are immutable. + pub original: Event, + /// A current signed report that may be inserted only if no report won yet. + pub candidate: ReportStamp, +} + +/// Signed fields that change during recovery; body and route remain in the original. +#[derive(Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ReportStamp { + /// Signed creation timestamp. + pub created_at: u64, + /// Canonical event ID. + pub id: String, + /// Author Schnorr signature. + pub sig: String, +} + +impl From<&Event> for ReportStamp { + fn from(event: &Event) -> Self { + Self { + created_at: event.created_at.as_secs(), + id: event.id.to_hex(), + sig: hex::encode(event.sig.serialize()), + } + } +} + +impl ReportStamp { + /// Reconstruct and verify the signed event against its immutable original fields. + pub fn restore(&self, original: &Event) -> Result { + let mut value = serde_json::to_value(original).map_err(|e| e.to_string())?; + value["created_at"] = self.created_at.into(); + value["id"] = self.id.clone().into(); + value["sig"] = self.sig.clone().into(); + let event: Event = serde_json::from_value(value).map_err(|e| e.to_string())?; + event.verify().map_err(|e| e.to_string())?; + Ok(event) + } +} + +/// Extract a single canonical delivery UUID from a kind-9 report. +pub fn delivery_key(event: &Event) -> Option { + if event.kind.as_u16() != 9 { + return None; + } + let mut tags = event + .tags + .iter() + .filter(|t| t.as_slice()[0] == "buzz-delivery"); + let tag = tags.next()?.as_slice(); + if tags.next().is_some() || tag.len() != 2 { + return None; + } + let key = Uuid::parse_str(&tag[1]).ok()?; + (key.to_string() == tag[1]).then_some(key) +} + +/// Compare immutable signed report fields; only time, ID and signature may differ. +pub fn same_report(first: &Event, other: &Event) -> bool { + first.kind.as_u16() == 9 + && first.kind == other.kind + && first.pubkey == other.pubkey + && first.tags == other.tags + && first.content == other.content + && delivery_key(first).is_some() +} + +impl RecoveryRequest { + /// Verify the wrapper and both reports before passing the candidate to normal ingest. + pub fn decode(wrapper: &Event, now: u64) -> Result { + if wrapper.kind.as_u16() != crate::kind::KIND_MANAGED_DELIVERY_RECOVERY as u16 + || wrapper.content.len() > MAX_RECOVERY_BYTES + || wrapper.created_at.as_secs().abs_diff(now) > 900 + { + return Err("invalid recovery envelope or timestamp".into()); + } + wrapper.verify().map_err(|e| e.to_string())?; + let request: Self = serde_json::from_str(&wrapper.content).map_err(|e| e.to_string())?; + request.original.verify().map_err(|e| e.to_string())?; + let candidate = request.candidate.restore(&request.original)?; + if !same_report(&request.original, &candidate) + || wrapper.pubkey != request.original.pubkey + || request.candidate.created_at.abs_diff(now) > 900 + || request.original.created_at.as_secs().saturating_sub(now) > 900 + { + return Err( + "recovery must preserve the verified original report and current candidate".into(), + ); + } + let channels = |event: &Event| { + event + .tags + .iter() + .filter(|t| t.as_slice()[0] == "h") + .map(|t| t.as_slice().to_vec()) + .collect::>() + }; + let original_channels = channels(&request.original); + if original_channels.len() != 1 + || original_channels[0].len() != 2 + || Uuid::parse_str(&original_channels[0][1]).is_err() + || channels(wrapper) != original_channels + { + return Err("recovery wrapper must name the original channel".into()); + } + Ok(request) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use nostr::{EventBuilder, Keys, Kind, Tag, Timestamp}; + + #[test] + fn recovery_checks_signed_evidence_and_immutable_payload() { + let keys = Keys::generate(); + let now = Timestamp::now().as_secs(); + let channel = Uuid::new_v4().to_string(); + let tags = vec![ + Tag::parse(["h", &channel]).unwrap(), + Tag::parse(["buzz-delivery", &Uuid::new_v4().to_string()]).unwrap(), + ]; + let make = |content: &str, timestamp: u64| { + EventBuilder::new(Kind::Custom(9), content) + .tags(tags.clone()) + .custom_created_at(Timestamp::from(timestamp)) + .sign_with_keys(&keys) + .unwrap() + }; + let original = make("first", now - 3600); + let wrapper = |original: Event, candidate: Event, timestamp: u64| { + EventBuilder::new( + Kind::Custom(crate::kind::KIND_MANAGED_DELIVERY_RECOVERY as u16), + serde_json::to_string(&RecoveryRequest { + original, + candidate: ReportStamp::from(&candidate), + }) + .unwrap(), + ) + .tag(Tag::parse(["h", &channel]).unwrap()) + .custom_created_at(Timestamp::from(timestamp)) + .sign_with_keys(&keys) + .unwrap() + }; + assert!( + RecoveryRequest::decode(&wrapper(original.clone(), make("first", now), now), now) + .is_ok() + ); + assert!(RecoveryRequest::decode( + &wrapper(original.clone(), make("changed", now), now), + now + ) + .is_err()); + assert!(RecoveryRequest::decode( + &wrapper(original.clone(), make("first", now - 901), now), + now + ) + .is_err()); + assert!(RecoveryRequest::decode( + &wrapper(original.clone(), make("first", now + 901), now), + now + ) + .is_err()); + assert!(RecoveryRequest::decode( + &wrapper(original.clone(), make("first", now), now + 901), + now + ) + .is_err()); + let mut forged = original.clone(); + forged.content.push('!'); + assert!(RecoveryRequest::decode(&wrapper(forged, make("first", now), now), now).is_err()); + let other = Keys::generate(); + let other_event = EventBuilder::new(original.kind, &original.content) + .tags(original.tags.clone()) + .sign_with_keys(&other) + .unwrap(); + assert!(RecoveryRequest::decode(&wrapper(original, other_event, now), now).is_err()); + } +} diff --git a/crates/buzz-db/src/error.rs b/crates/buzz-db/src/error.rs index 4f15cc2b247..f2cd459bcf3 100644 --- a/crates/buzz-db/src/error.rs +++ b/crates/buzz-db/src/error.rs @@ -21,6 +21,10 @@ pub enum DbError { #[error("ephemeral events (kind {0}) must not be stored")] EphemeralEventRejected(u16), + /// A completion key was reused with a different immutable report. + #[error("managed delivery conflict: {0}")] + ManagedDeliveryConflict(String), + /// The requested channel does not exist. #[error("channel not found: {0}")] ChannelNotFound(uuid::Uuid), diff --git a/crates/buzz-db/src/runtime/migration.rs b/crates/buzz-db/src/runtime/migration.rs index 47e112d7998..0e564eff15b 100644 --- a/crates/buzz-db/src/runtime/migration.rs +++ b/crates/buzz-db/src/runtime/migration.rs @@ -705,7 +705,7 @@ mod postgres_tests { let mut migrations: Vec<_> = MIGRATOR.iter().collect(); migrations.sort_by_key(|migration| migration.version); - assert_eq!(migrations.len(), 55); + assert_eq!(migrations.len(), 56); assert_eq!(migrations[48].version, 49); assert_eq!(migrations[49].version, 50); assert_eq!(migrations[50].version, 51); @@ -713,6 +713,15 @@ mod postgres_tests { assert_eq!(migrations[52].version, 53); assert_eq!(migrations[53].version, 54); assert_eq!(migrations[54].version, 55); + assert_eq!(migrations[55].version, 56); + assert!(migrations[55] + .sql + .as_str() + .contains("CREATE TABLE managed_deliveries")); + assert!(migrations[55] + .sql + .as_str() + .contains("attach_community_write_fence")); assert!(migrations[48] .sql .as_str() @@ -2048,7 +2057,9 @@ mod postgres_tests { let mut expected_fences = migration.fence_attachments.clone(); expected_fences.remove("product_feedback"); expected_fences.remove("rate_limit_violations"); - expected_fences.extend(["artifact_heads", "artifact_revisions"].map(str::to_owned)); + expected_fences.extend( + ["artifact_heads", "artifact_revisions", "managed_deliveries"].map(str::to_owned), + ); assert_eq!( expected_fences, schema.fence_attachments, "write-fence attachment targets differ after recovery policy" @@ -2480,14 +2491,15 @@ mod postgres_tests { .await .expect("connect migrated probe database"); MIGRATOR - .run_to(55, &migrated) + .run_to(56, &migrated) .await - .expect("apply migrations 1-55"); + .expect("apply migrations 1-56"); for table in [ "relay_admin_actions", "relay_admin_outbox", "relay_operator_audit", + "managed_deliveries", ] { assert_eq!( columns(&desired, table).await, diff --git a/crates/buzz-db/src/runtime/tests/thread_window_postgres_tests.rs b/crates/buzz-db/src/runtime/tests/thread_window_postgres_tests.rs index 1904bf6363d..8ac57b1a3d9 100644 --- a/crates/buzz-db/src/runtime/tests/thread_window_postgres_tests.rs +++ b/crates/buzz-db/src/runtime/tests/thread_window_postgres_tests.rs @@ -311,7 +311,17 @@ async fn migration_schema_thread_window_prebuild_does_not_queue_behind_writer() production_result.is_ok(), "production migrator must preserve ingestion progress: {production_result:?}" ); - assert_eq!(version, 55); + let migrations = sqlx::migrate::Migrator::new( + std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../../migrations"), + ) + .await + .unwrap(); + let expected_version = migrations + .iter() + .map(|migration| migration.version) + .max() + .unwrap(); + assert_eq!(version, expected_version); assert_eq!(final_oid, oid, "prebuild must not be replaced"); assert_eq!(count, 4, "all writer witnesses must persist"); } diff --git a/crates/buzz-db/src/store/deletion.rs b/crates/buzz-db/src/store/deletion.rs index de10f048d71..e807c58fd70 100644 --- a/crates/buzz-db/src/store/deletion.rs +++ b/crates/buzz-db/src/store/deletion.rs @@ -89,6 +89,7 @@ pub const EXPECTED_SCOPED_TABLES: &[&str] = &[ "events", "git_repo_names", "join_policy_acceptances", + "managed_deliveries", "moderation_actions", "moderation_reports", "parameterized_event_watermarks", @@ -120,6 +121,7 @@ pub const PURGE_SCOPED_TABLES: &[&str] = &[ "api_tokens", "channel_members", "thread_metadata", + "managed_deliveries", "moderation_actions", "workflows", "event_mentions", diff --git a/crates/buzz-db/src/store/event.rs b/crates/buzz-db/src/store/event.rs index 9025c2bfd43..3f70526d8e2 100644 --- a/crates/buzz-db/src/store/event.rs +++ b/crates/buzz-db/src/store/event.rs @@ -1746,6 +1746,19 @@ pub async fn insert_event_with_thread_metadata( ) -> Result<(StoredEvent, bool)> { use crate::store::replaceable::event_replacement_lock_key; + if let Some(channel) = channel_id { + if buzz_core::managed_delivery::delivery_key(event).is_some() { + return crate::store::managed_delivery::insert( + pool, + community_id, + event, + channel, + thread_meta, + None, + ) + .await; + } + } let connection = crate::observability::acquire_writer( pool, crate::observability::WriterOperation::EventWrite, diff --git a/crates/buzz-db/src/store/managed_delivery.rs b/crates/buzz-db/src/store/managed_delivery.rs new file mode 100644 index 00000000000..15265d4bc2a --- /dev/null +++ b/crates/buzz-db/src/store/managed_delivery.rs @@ -0,0 +1,123 @@ +//! Atomic canonical report selection on the guarded production writer. +use buzz_core::{ + managed_delivery::{delivery_key, same_report}, + CommunityId, StoredEvent, +}; +use chrono::{DateTime, Utc}; +use nostr::Event; +use sha2::{Digest, Sha256}; +use sqlx::{PgPool, Row}; +use uuid::Uuid; + +use crate::{ + error::{DbError, Result}, + store::event::{ + insert_event_with_thread_metadata_tx, row_to_stored_event, ThreadMetadataParams, + }, + Db, +}; + +/// Elect a report, adopting historical acceptance before inserting a fresh candidate. +pub async fn insert( + pool: &PgPool, + community: CommunityId, + candidate: &Event, + channel: Uuid, + metadata: Option>, + original: Option<&Event>, +) -> Result<(StoredEvent, bool)> { + let key = delivery_key(candidate) + .ok_or_else(|| DbError::InvalidData("missing managed delivery key".into()))?; + let original = original.unwrap_or(candidate); + if !same_report(original, candidate) { + return Err(DbError::InvalidData( + "managed recovery changed the report".into(), + )); + } + let author = candidate.pubkey.to_bytes(); + let mut hash = Sha256::new(); + hash.update(b"buzz-managed-delivery-v1"); + hash.update(community.as_uuid().as_bytes()); + hash.update(author); + hash.update(key.as_bytes()); + let mut lock_bytes = [0u8; 8]; + lock_bytes.copy_from_slice(&hash.finalize()[..8]); + let connection = crate::observability::acquire_writer( + pool, + crate::observability::WriterOperation::EventWrite, + ) + .await?; + let mut tx = sqlx::Transaction::begin(connection, None).await?; + sqlx::query("SELECT pg_advisory_xact_lock($1)") + .bind(i64::from_be_bytes(lock_bytes)) + .execute(&mut *tx) + .await?; + let bound = sqlx::query("SELECT event, received_at, channel_id FROM managed_deliveries WHERE community_id=$1 AND author=$2 AND delivery_key=$3") + .bind(community.as_uuid()).bind(author.as_slice()).bind(key).fetch_optional(&mut *tx).await?; + if let Some(row) = bound { + let chosen: Event = serde_json::from_value(row.try_get("event")?)?; + let bound_channel: Uuid = row.try_get("channel_id")?; + if bound_channel != channel || !same_report(&chosen, candidate) { + return Err(DbError::ManagedDeliveryConflict( + "delivery key is bound to a different report or route".into(), + )); + } + let stored = + StoredEvent::with_received_at(chosen, row.try_get("received_at")?, Some(channel), true); + tx.commit().await?; + return Ok((stored, false)); + } + // Includes soft-deleted rows: previous acceptance must never resurrect a report. + let historical = sqlx::query("SELECT id, pubkey, created_at, kind, tags, content, sig, received_at, channel_id FROM events WHERE community_id=$1 AND id=$2") + .bind(community.as_uuid()).bind(original.id.as_bytes().as_slice()).fetch_optional(&mut *tx).await?; + let result = if let Some(row) = historical { + let stored = row_to_stored_event(row)? + .ok_or_else(|| DbError::InvalidData("invalid historical report".into()))?; + if stored.channel_id != Some(channel) || !same_report(&stored.event, candidate) { + return Err(DbError::InvalidData( + "historical report does not match recovery".into(), + )); + } + (stored, false) + } else { + insert_event_with_thread_metadata_tx(&mut tx, community, candidate, Some(channel), metadata) + .await? + }; + if result.1 { + crate::runtime::insert_mentions_in_transaction( + &mut tx, + community, + &result.0.event, + Some(channel), + ) + .await?; + } + let received_at: DateTime = result.0.received_at; + sqlx::query("INSERT INTO managed_deliveries (community_id, author, delivery_key, channel_id, event, received_at) VALUES ($1,$2,$3,$4,$5,$6)") + .bind(community.as_uuid()).bind(author.as_slice()).bind(key).bind(channel) + .bind(serde_json::to_value(&result.0.event)?).bind(received_at).execute(&mut *tx).await?; + tx.commit().await?; + Ok(result) +} + +impl Db { + /// Recover an expired report using one guarded transaction and a shared completion lock. + pub async fn recover_managed_delivery( + &self, + community: CommunityId, + candidate: &Event, + channel: Uuid, + metadata: Option>, + original: &Event, + ) -> Result<(StoredEvent, bool)> { + insert( + self.pool(), + community, + candidate, + channel, + metadata, + Some(original), + ) + .await + } +} diff --git a/crates/buzz-db/src/store/mod.rs b/crates/buzz-db/src/store/mod.rs index 79f0f078ab2..614e6845b83 100644 --- a/crates/buzz-db/src/store/mod.rs +++ b/crates/buzz-db/src/store/mod.rs @@ -24,6 +24,8 @@ pub mod event; pub mod feed; /// Git repository name registry (NIP-34 kind:30617). pub mod git_repo; +/// Atomic managed report election. +pub mod managed_delivery; /// Community moderation: reports, bans/timeouts, audit actions. pub mod moderation; /// Deployment-global operator-listener mention registrations and delivery queues. diff --git a/crates/buzz-db/tests/observability_source.rs b/crates/buzz-db/tests/observability_source.rs index 832b05c56f2..64851615331 100644 --- a/crates/buzz-db/tests/observability_source.rs +++ b/crates/buzz-db/tests/observability_source.rs @@ -354,8 +354,15 @@ fn p0_pool_acquisitions_use_typed_operation_pairs_without_other() { .expect("restriction state must precede full ban reads") .0; assert!(restriction_state.contains("WriterOperation::Authorization")); - assert!(restriction_state.contains("fetch_optional(&mut *connection)")); + assert!(restriction_state.contains("acquire_writer(")); + // Cardinality is independent of the typed-connection invariant: aggregate + // queries return one row even when no restrictions match. + assert!( + restriction_state.contains("fetch_optional(&mut *connection)") + || restriction_state.contains("fetch_one(&mut *connection)") + ); assert!(!restriction_state.contains("fetch_optional(pool)")); + assert!(!restriction_state.contains("fetch_one(pool)")); let community_store = include_str!("../src/store/community.rs"); let ensure_community = community_store @@ -524,6 +531,17 @@ fn p0_pool_acquisitions_use_typed_operation_pairs_without_other() { .0; assert!(lease_stats.contains("WriterOperation::Maintenance")); assert!(lease_stats.contains("fetch_one(&mut *connection)")); + let bounded_lease_sql = deletion + .split_once("async fn bounded_serving_lease_sql(") + .expect("deletion store must expose bounded serving-lease SQL") + .1 + .split_once("/// Acquire a durable, expiring lease") + .expect("bounded serving-lease helper must precede acquisition") + .0; + assert!(bounded_lease_sql.contains("acquire_writer_until(")); + assert!(bounded_lease_sql.contains("WriterOperation::EventWrite")); + assert!(bounded_lease_sql.contains("operation(&mut connection)")); + assert!(!bounded_lease_sql.contains("self.pool.acquire()")); for (start, end) in [ ( "pub async fn acquire_serving_write_lease", @@ -553,10 +571,23 @@ fn p0_pool_acquisitions_use_typed_operation_pairs_without_other() { .split_once(end) .unwrap_or_else(|| panic!("serving-write seam {start} must precede {end}")) .0; - assert!( - function.contains("WriterOperation::EventWrite"), - "serving-write seam {start} must be event-write attributed" - ); + if matches!( + start, + "pub async fn acquire_serving_write_lease" + | "pub async fn release_serving_write_lease" + | "pub async fn verify_serving_write_lease" + ) { + assert!( + function.contains("self.bounded_serving_lease_sql("), + "serving-write seam {start} must use the attributed bounded helper" + ); + assert!(function.contains("serving_lease_transaction(connection)")); + } else { + assert!( + function.contains("WriterOperation::EventWrite"), + "serving-write seam {start} must be event-write attributed" + ); + } assert!(!function.contains("self.pool.begin().await")); assert!(!function.contains(".execute(&self.pool)")); assert!(!function.contains(".fetch_one(&self.pool)")); diff --git a/crates/buzz-relay/src/handlers/event.rs b/crates/buzz-relay/src/handlers/event.rs index 5b7b257c199..7b9260d4527 100644 --- a/crates/buzz-relay/src/handlers/event.rs +++ b/crates/buzz-relay/src/handlers/event.rs @@ -740,7 +740,10 @@ pub async fn handle_event(event: Event, conn: Arc, state: Arc, state: Arc Result { + let recovery = if event_kind_u32(&event) == buzz_core::kind::KIND_MANAGED_DELIVERY_RECOVERY { + if event.pubkey != *auth.pubkey() { + return Err(IngestError::AuthFailed( + "invalid: recovery author does not match authenticated identity".into(), + )); + } + Some( + buzz_core::managed_delivery::RecoveryRequest::decode( + &event, + nostr::Timestamp::now().as_secs(), + ) + .map_err(|e| IngestError::Rejected(format!("invalid: {e}")))?, + ) + } else { + None + }; + let event = match recovery.as_ref() { + Some(request) => request + .candidate + .restore(&request.original) + .map_err(IngestError::Rejected)?, + None => event, + }; let event_id_hex = event.id.to_hex(); let kind_u32 = event_kind_u32(&event); debug!(event_id = %event_id_hex, kind = kind_u32, "ingest_event"); @@ -2446,14 +2470,9 @@ async fn ingest_event_inner( }); } - const MAX_TIMESTAMP_DRIFT_SECS: i64 = 900; // ±15 minutes let now = chrono::Utc::now().timestamp(); let event_ts = event.created_at.as_secs() as i64; - if (event_ts - now).abs() > MAX_TIMESTAMP_DRIFT_SECS { - return Err(IngestError::Rejected( - "invalid: event timestamp too far from server time".into(), - )); - } + timestamp::validate(&event, &auth, nostr::Timestamp::now().as_secs())?; // kind:40100 canvas events carry a tighter future ceiling — see // `validate_canvas_future_timestamp` for the rationale and invariant. @@ -3420,16 +3439,32 @@ async fn ingest_event_inner( } } else { let thread_params = thread_meta.as_ref().map(|m| m.as_params()); - match state - .db - .insert_event_with_thread_metadata( - tenant.community(), - &event, - channel_id, - thread_params, - ) - .await - { + let write_result = if let Some(request) = recovery.as_ref() { + let channel = channel_id.ok_or_else(|| { + IngestError::Rejected("invalid: recovery requires original channel".into()) + })?; + state + .db + .recover_managed_delivery( + tenant.community(), + &event, + channel, + thread_params, + &request.original, + ) + .await + } else { + state + .db + .insert_event_with_thread_metadata( + tenant.community(), + &event, + channel_id, + thread_params, + ) + .await + }; + match write_result { Ok(result) => result, Err(e) => { // Compensate: if we pre-created a channel for kind:9007, @@ -3445,6 +3480,9 @@ async fn ingest_event_inner( state.invalidate_channel_deleted(tenant); } return Err(match e { + buzz_db::DbError::ManagedDeliveryConflict(message) => { + IngestError::CanvasConflict(format!("conflict: {message}")) + } buzz_db::DbError::AuthEventRejected => { IngestError::Rejected("invalid: AUTH events cannot be stored".into()) } @@ -3454,11 +3492,38 @@ async fn ingest_event_inner( } }; + let managed_receipt = if buzz_core::managed_delivery::delivery_key(&event).is_some() { + format!( + "{}{}", + buzz_core::managed_delivery::RECEIPT_PREFIX, + serde_json::to_string(&stored_event.event) + .map_err(|e| IngestError::Internal(e.to_string()))? + ) + } else { + String::new() + }; if !was_inserted { + if !managed_receipt.is_empty() { + if let Some(channel) = channel_id { + emit( + tracer, + TraceAction::WriteDuplicate { + msg_id: msg_id_label(stored_event.event.id.as_bytes()), + channel: channel_label(channel), + claimed_community: claimed_community_from_event(&event), + }, + state_for_request(tenant, auth.pubkey()), + ); + } + } return Ok(IngestResult { - event_id: event_id_hex, + event_id: stored_event.event.id.to_hex(), accepted: true, - message: "duplicate:".into(), + message: if managed_receipt.is_empty() { + "duplicate:".into() + } else { + managed_receipt + }, }); } @@ -3533,14 +3598,15 @@ async fn ingest_event_inner( info!(event_id = %event_id_hex, kind = kind_u32, "Event ingested via pipeline"); Ok(IngestResult { - event_id: event_id_hex, + event_id: stored_event.event.id.to_hex(), accepted: true, - message: String::new(), + message: managed_receipt, }) } #[cfg(test)] mod postgres_tests { + mod managed_delivery_tests; use std::sync::Mutex; use super::*; diff --git a/crates/buzz-relay/src/handlers/ingest/postgres_tests/managed_delivery_tests.rs b/crates/buzz-relay/src/handlers/ingest/postgres_tests/managed_delivery_tests.rs new file mode 100644 index 00000000000..10dc0c15a27 --- /dev/null +++ b/crates/buzz-relay/src/handlers/ingest/postgres_tests/managed_delivery_tests.rs @@ -0,0 +1,1105 @@ +//! Recovery regressions use the production writer constructor and its commit-time floor. +//! A separate unguarded fixture pool is used only to seed events accepted before the outage. +use super::*; +use buzz_core::managed_delivery::ReportStamp; +use buzz_db::channel::{ChannelType, ChannelVisibility}; +use nostr::{Keys, Tag, Timestamp}; + +#[tokio::test] +#[ignore = "requires Postgres and Redis"] +async fn managed_old_absent_report_recovers_with_production_writer() { + old_report_recovery(false, false).await; +} + +#[tokio::test] +#[ignore = "requires Postgres and Redis"] +async fn managed_old_accepted_report_recovers_with_production_writer() { + old_report_recovery(true, false).await; +} + +#[tokio::test] +#[ignore = "requires Postgres and Redis"] +async fn managed_old_deleted_original_never_resurrects() { + old_report_recovery(true, true).await; +} + +async fn old_report_recovery(already_accepted: bool, deleted: bool) { + let state = authenticated_http_state().await; + let pool = state.db.pool().clone(); + let host = format!("managed-recovery-{}.test", Uuid::new_v4().simple()); + let community = state + .db + .ensure_configured_community(&host) + .await + .unwrap() + .id; + let tenant = TenantContext::resolved(community, &host); + let author = Keys::generate(); + let channel = Uuid::new_v4(); + state + .db + .create_channel_with_id( + community, + channel, + "managed recovery", + ChannelType::Stream, + ChannelVisibility::Open, + None, + author.public_key().as_bytes(), + None, + ) + .await + .unwrap(); + let tracer: Arc = Arc::new(VecTracer::default()); + let auth = || IngestAuth::Http { + pubkey: author.public_key(), + scopes: vec![Scope::MessagesWrite], + auth_method: HttpAuthMethod::Nip98, + }; + let root = EventBuilder::new(Kind::Custom(9), "original task") + .tag(Tag::parse(["h", &channel.to_string()]).unwrap()) + .sign_with_keys(&author) + .unwrap(); + ingest_event_inner(&state, &tracer, &tenant, root.clone(), auth()) + .await + .unwrap(); + { + let key = Uuid::new_v4().to_string(); + let event = EventBuilder::new(Kind::Custom(9), "original final report") + .custom_created_at(Timestamp::from(Timestamp::now().as_secs() - 3600)) + .tags([ + Tag::parse(["h", &channel.to_string()]).unwrap(), + Tag::parse(["e", &root.id.to_hex(), "", "root"]).unwrap(), + Tag::parse(["e", &root.id.to_hex(), "", "reply"]).unwrap(), + Tag::parse(["buzz-delivery", &key]).unwrap(), + ]) + .sign_with_keys(&author) + .unwrap(); + if already_accepted { + seed_previously_accepted_event(&state, &tracer, &tenant, event.clone(), auth()).await; + } + if !already_accepted { + let rejected = + ingest_event_inner(&state, &tracer, &tenant, event.clone(), auth()).await; + assert!( + matches!(rejected, Err(IngestError::Internal(message)) if message.contains("replica-fence floor")) + ); + let count: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM managed_deliveries WHERE community_id=$1 AND author=$2 AND delivery_key=$3") + .bind(community.as_uuid()).bind(author.public_key().to_bytes().as_slice()) + .bind(Uuid::parse_str(&key).unwrap()).fetch_one(&pool).await.unwrap(); + assert_eq!( + count, 0, + "deferred floor failure must roll back canonical binding" + ); + } + if deleted { + sqlx::query("UPDATE events SET deleted_at=now() WHERE community_id=$1 AND id=$2") + .bind(community.as_uuid()) + .bind(event.id.as_bytes().as_slice()) + .execute(&pool) + .await + .unwrap(); + } + let candidate = EventBuilder::new(event.kind, &event.content) + .tags(event.tags.clone()) + .allow_self_tagging() + .sign_with_keys(&author) + .unwrap(); + let recovery = EventBuilder::new( + Kind::Custom(20003), + serde_json::json!({"original":event,"candidate":ReportStamp::from(&candidate)}) + .to_string(), + ) + .tag(Tag::parse(["h", &channel.to_string()]).unwrap()) + .sign_with_keys(&author) + .unwrap(); + let receipt = ingest_event_inner(&state, &tracer, &tenant, recovery.clone(), auth()) + .await + .expect("recovery must elect one report without disarming the writer floor"); + let chosen = if already_accepted { &event } else { &candidate }; + assert!(receipt.accepted); + assert_eq!(receipt.event_id, chosen.id.to_hex()); + let counts: (i32, i32) = sqlx::query_as("SELECT reply_count, descendant_count FROM thread_metadata WHERE community_id=$1 AND event_id=$2") + .bind(community.as_uuid()).bind(root.id.as_bytes().as_slice()).fetch_one(&pool).await.unwrap(); + let replay = ingest_event_inner(&state, &tracer, &tenant, recovery, auth()) + .await + .unwrap(); + assert!(replay.accepted); + assert!(replay.message.starts_with("managed-delivery:")); + let stored = state + .db + .get_event_by_id(community, chosen.id.as_bytes()) + .await + .unwrap(); + if deleted { + assert!(stored.is_none(), "deleted report must remain hidden"); + } else { + assert_eq!(&stored.unwrap().event, chosen); + } + let next_candidate = EventBuilder::new(candidate.kind, &candidate.content) + .tags(candidate.tags.clone()) + .allow_self_tagging() + .custom_created_at(Timestamp::from(Timestamp::now().as_secs() + 1)) + .sign_with_keys(&author) + .unwrap(); + let next_wrapper = EventBuilder::new( + Kind::Custom(20003), + serde_json::json!({"original":event,"candidate":ReportStamp::from(&next_candidate)}) + .to_string(), + ) + .tag(Tag::parse(["h", &channel.to_string()]).unwrap()) + .sign_with_keys(&author) + .unwrap(); + let (a, b) = tokio::join!( + ingest_event_inner(&state, &tracer, &tenant, next_wrapper.clone(), auth()), + ingest_event_inner(&state, &tracer, &tenant, next_wrapper, auth()) + ); + assert_eq!(a.unwrap().event_id, chosen.id.to_hex()); + assert_eq!(b.unwrap().event_id, chosen.id.to_hex()); + let report_count: i64 = + sqlx::query_scalar("SELECT COUNT(*) FROM events WHERE community_id=$1 AND tags @> $2") + .bind(community.as_uuid()) + .bind(serde_json::json!([["buzz-delivery", key]])) + .fetch_one(&pool) + .await + .unwrap(); + assert_eq!(report_count, 1); + let replay_counts: (i32, i32) = sqlx::query_as("SELECT reply_count, descendant_count FROM thread_metadata WHERE community_id=$1 AND event_id=$2") + .bind(community.as_uuid()).bind(root.id.as_bytes().as_slice()).fetch_one(&pool).await.unwrap(); + assert_eq!( + counts, replay_counts, + "duplicate replay must not increment thread counters" + ); + } +} + +async fn authenticated_http_state() -> Arc { + let mut config = crate::config::Config::for_test(); + config.database_url = crate::test_support::database_url(); + config.redis_url = std::env::var("REDIS_URL").expect("local test Redis"); + config.require_auth_token = true; + config.require_relay_membership = false; + let db = buzz_db::Db::new(&buzz_db::DbConfig { + database_url: config.database_url.clone(), + max_connections: 4, + ..Default::default() + }) + .await + .unwrap(); + if std::env::var("BUZZ_TEST_SCHEMA_MODE").as_deref() != Ok("desired") { + buzz_db::migration::run_migrations(db.pool()).await.unwrap(); + } + let pool = db.pool().clone(); + let floor: String = sqlx::query_scalar("SELECT current_setting('buzz.created_at_floor')") + .fetch_one(&pool) + .await + .unwrap(); + assert_eq!( + floor, "960", + "the recovery path must retain the production writer floor" + ); + let redis = deadpool_redis::Config::from_url(&config.redis_url) + .create_pool(Some(deadpool_redis::Runtime::Tokio1)) + .unwrap(); + let pubsub = Arc::new( + buzz_pubsub::PubSubManager::new(&config.redis_url, redis.clone()) + .await + .unwrap(), + ); + let audit = buzz_audit::AuditService::new(pool.clone()); + let auth = buzz_auth::AuthService::new(config.auth.clone()); + let search = buzz_search::SearchService::new(pool.clone()); + let workflows = Arc::new(buzz_workflow::WorkflowEngine::new( + db.clone(), + buzz_workflow::WorkflowConfig::default(), + )); + let media = buzz_media::MediaStorage::new(&config.media).unwrap(); + let (state, _) = crate::state::AppState::new( + config, + db, + redis, + audit, + pubsub, + auth, + search, + workflows, + Keys::generate(), + media, + ); + Arc::new(state) +} + +// Historical setup models an already-committed report, not a recovery write. +// Never use this unarmed pool for the operation under test. +async fn seed_previously_accepted_event( + production: &crate::state::AppState, + tracer: &Arc, + tenant: &TenantContext, + event: nostr::Event, + auth: IngestAuth, +) { + let db_url = &production.config.database_url; + let historical_pool = sqlx::PgPool::connect(db_url).await.unwrap(); + let historical_state = build_canvas_ingest_state(db_url, &historical_pool).await; + ingest_event_inner(&historical_state, tracer, tenant, event.clone(), auth) + .await + .unwrap(); + // Simulate an event accepted before deployment of the canonical-key ledger. + sqlx::query( + "DELETE FROM managed_deliveries WHERE community_id=$1 AND author=$2 AND delivery_key=$3", + ) + .bind(tenant.community().as_uuid()) + .bind(event.pubkey.to_bytes().as_slice()) + .bind(buzz_core::managed_delivery::delivery_key(&event).unwrap()) + .execute(&historical_pool) + .await + .unwrap(); + historical_pool.close().await; +} + +#[tokio::test] +#[ignore = "requires Postgres and Redis"] +async fn managed_old_report_still_enforces_signature_route_membership_and_archive() { + let state = authenticated_http_state().await; + let host = format!("managed-authority-{}.test", Uuid::new_v4().simple()); + let community = state + .db + .ensure_configured_community(&host) + .await + .unwrap() + .id; + let tenant = TenantContext::resolved(community, &host); + let author = Keys::generate(); + let owner = Keys::generate(); + let channel = Uuid::new_v4(); + state + .db + .create_channel_with_id( + community, + channel, + "private managed recovery", + ChannelType::Stream, + ChannelVisibility::Private, + None, + author.public_key().as_bytes(), + None, + ) + .await + .unwrap(); + let tracer: Arc = Arc::new(VecTracer::default()); + let auth = |keys: &Keys| IngestAuth::Http { + pubkey: keys.public_key(), + scopes: vec![Scope::MessagesWrite], + auth_method: HttpAuthMethod::Nip98, + }; + let make_event = |parent: Option| { + let mut tags = vec![ + Tag::parse(["h", &channel.to_string()]).unwrap(), + Tag::parse(["buzz-delivery", &Uuid::new_v4().to_string()]).unwrap(), + ]; + if let Some(parent) = parent { + tags.push(Tag::parse(["e", &parent, "", "reply"]).unwrap()); + } + EventBuilder::new(Kind::Custom(9), "old managed report") + .custom_created_at(Timestamp::from(Timestamp::now().as_secs() - 3600)) + .tags(tags) + .sign_with_keys(&author) + .unwrap() + }; + let future = EventBuilder::new(Kind::Custom(9), "ordinary future message") + .custom_created_at(Timestamp::from(Timestamp::now().as_secs() + 3600)) + .tag(Tag::parse(["h", &channel.to_string()]).unwrap()) + .sign_with_keys(&author) + .unwrap(); + assert!( + matches!( + ingest_event_inner(&state, &tracer, &tenant, future, auth(&author)).await, + Err(IngestError::Rejected(message)) if message.contains("timestamp too far") + ), + "ordinary message must traverse shared timestamp admission before storage" + ); + let event = make_event(None); + let mut invalid = event.clone(); + invalid.content.push_str("tampered"); + assert!( + ingest_event_inner(&state, &tracer, &tenant, invalid, auth(&author)) + .await + .is_err() + ); + assert!( + ingest_event_inner(&state, &tracer, &tenant, event.clone(), auth(&owner)) + .await + .is_err() + ); + assert!(ingest_event_inner( + &state, + &tracer, + &tenant, + make_event(Some("a".repeat(64))), + auth(&author) + ) + .await + .is_err()); + seed_previously_accepted_event(&state, &tracer, &tenant, event.clone(), auth(&author)).await; + state + .db + .add_member( + community, + channel, + owner.public_key().as_bytes(), + buzz_db::channel::MemberRole::Owner, + Some(author.public_key().as_bytes()), + ) + .await + .unwrap(); + state + .db + .remove_member( + community, + channel, + author.public_key().as_bytes(), + owner.public_key().as_bytes(), + ) + .await + .unwrap(); + state.invalidate_membership(&tenant, channel, author.public_key().as_bytes()); + assert!( + ingest_event_inner(&state, &tracer, &tenant, event.clone(), auth(&author)) + .await + .is_err(), + "even an accepted old event must pass current access before deduplication" + ); + state + .db + .add_member( + community, + channel, + author.public_key().as_bytes(), + buzz_db::channel::MemberRole::Owner, + Some(owner.public_key().as_bytes()), + ) + .await + .unwrap(); + state.invalidate_membership(&tenant, channel, author.public_key().as_bytes()); + state.db.archive_channel(community, channel).await.unwrap(); + let result = ingest_event_inner(&state, &tracer, &tenant, event, auth(&author)).await; + assert!(matches!(result, Err(IngestError::Rejected(message)) if message.contains("archived"))); +} + +#[tokio::test] +#[ignore = "requires Postgres, Redis and built CLI in BUZZ_TEST_BIN_DIR"] +async fn managed_old_absent_journal_recovers_via_actual_cli_and_authenticated_relay() { + old_journal_recovery(false, false, "original durable report").await; +} + +#[tokio::test] +#[ignore = "requires Postgres, Redis and built CLI in BUZZ_TEST_BIN_DIR"] +async fn managed_old_accepted_journal_recovers_via_actual_cli_and_authenticated_relay() { + old_journal_recovery(true, false, "original durable report").await; +} + +#[tokio::test] +#[ignore = "requires Postgres, Redis and built CLI in BUZZ_TEST_BIN_DIR"] +async fn managed_old_lost_candidate_ack_recovers_after_another_restart() { + old_journal_recovery(false, true, "original durable report").await; +} + +#[tokio::test] +#[ignore = "requires Postgres, Redis and built CLI in BUZZ_TEST_BIN_DIR"] +async fn managed_old_maximum_escaped_report_recovers() { + for content in [ + "\"".repeat(65536), + "\\".repeat(65536), + "\u{0001}".repeat(65536), + ] { + old_journal_recovery(false, false, &content).await; + } +} + +async fn old_journal_recovery(already_accepted: bool, accepted_candidate: bool, content: &str) { + use sha2::{Digest, Sha256}; + let state = authenticated_http_state().await; + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let host = listener.local_addr().unwrap().to_string(); + let relay = format!("http://{host}"); + let community = state + .db + .ensure_configured_community(&host) + .await + .unwrap() + .id; + let tenant = TenantContext::resolved(community, &host); + let author = Keys::generate(); + let channel = Uuid::new_v4(); + state + .db + .create_channel_with_id( + community, + channel, + "live managed recovery", + ChannelType::Stream, + ChannelVisibility::Private, + None, + author.public_key().as_bytes(), + None, + ) + .await + .unwrap(); + crate::handlers::side_effects::emit_group_discovery_events(&tenant, &state, channel) + .await + .unwrap(); + let root = EventBuilder::new(Kind::Custom(9), "original task") + .tag(Tag::parse(["h", &channel.to_string()]).unwrap()) + .sign_with_keys(&author) + .unwrap(); + let tracer: Arc = Arc::new(VecTracer::default()); + ingest_event_inner( + &state, + &tracer, + &tenant, + root.clone(), + IngestAuth::Http { + pubkey: author.public_key(), + scopes: vec![Scope::MessagesWrite], + auth_method: HttpAuthMethod::Nip98, + }, + ) + .await + .unwrap(); + let router = crate::router::build_router(state.clone()); + let (shutdown_tx, shutdown_rx) = tokio::sync::oneshot::channel(); + let server = tokio::spawn(async move { + axum::serve(listener, router) + .with_graceful_shutdown(async { + let _ = shutdown_rx.await; + }) + .await + .unwrap(); + }); + let cli = + std::path::PathBuf::from(std::env::var("BUZZ_TEST_BIN_DIR").expect("built CLI directory")) + .join("buzz"); + { + let dir = tempfile::tempdir().unwrap(); + let key = Uuid::new_v4().to_string(); + let signer = author.public_key().to_hex(); + let event = EventBuilder::new(Kind::Custom(9), content) + .custom_created_at(Timestamp::from(Timestamp::now().as_secs() - 3600)) + .tags([ + Tag::parse(["h", &channel.to_string()]).unwrap(), + Tag::parse(["e", &root.id.to_hex(), "", "root"]).unwrap(), + Tag::parse(["e", &root.id.to_hex(), "", "reply"]).unwrap(), + Tag::parse(["buzz-delivery", &key]).unwrap(), + ]) + .sign_with_keys(&author) + .unwrap(); + if already_accepted { + seed_previously_accepted_event( + &state, + &tracer, + &tenant, + event.clone(), + IngestAuth::Http { + pubkey: author.public_key(), + scopes: vec![Scope::MessagesWrite], + auth_method: HttpAuthMethod::Nip98, + }, + ) + .await; + } + let aged_candidate = EventBuilder::new(event.kind, &event.content) + .tags(event.tags.clone()) + .allow_self_tagging() + .custom_created_at(Timestamp::from(event.created_at.as_secs() + 60)) + .sign_with_keys(&author) + .unwrap(); + if accepted_candidate { + let historical_pool = sqlx::PgPool::connect(&state.config.database_url) + .await + .unwrap(); + let historical_state = + build_canvas_ingest_state(&state.config.database_url, &historical_pool).await; + ingest_event_inner( + &historical_state, + &tracer, + &tenant, + aged_candidate.clone(), + IngestAuth::Http { + pubkey: author.public_key(), + scopes: vec![Scope::MessagesWrite], + auth_method: HttpAuthMethod::Nip98, + }, + ) + .await + .unwrap(); + historical_pool.close().await; + } + let id = hex::encode(Sha256::digest( + serde_json::to_vec(&(&relay, &signer, &key)).unwrap(), + )); + let mut journal = serde_json::json!({"version":1,"entries":{id.clone():{"relay":relay,"signer":signer,"key":key,"channel":channel.to_string(),"parent":root.id.to_hex(),"event":event,"attempts":1,"confirmed_at":null,"last_error":null}}}); + // Candidate renewal must preserve the original evidence. + if accepted_candidate { + journal["version"] = 2.into(); + journal["entries"][&id]["recovery_candidate"] = + serde_json::to_value(ReportStamp::from(&aged_candidate)).unwrap(); + } + std::fs::write(dir.path().join("journal.json"), journal.to_string()).unwrap(); + let mut elected_id = None; + for _ in 0..2 { + let output = tokio::process::Command::new(&cli) + .env_remove("BUZZ_AUTH_TAG") + .env("BUZZ_PRIVATE_KEY", author.secret_key().to_secret_hex()) + .args([ + "--relay", + &relay, + "messages", + "deliver", + "--delivery-key", + &key, + "--channel", + &channel.to_string(), + "--reply-to", + &root.id.to_hex(), + "--journal-dir", + dir.path().to_str().unwrap(), + "--content", + "changed summary after restart", + ]) + .output() + .await + .unwrap(); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + let receipt: serde_json::Value = serde_json::from_slice(&output.stdout).unwrap(); + let id = receipt["event_id"].as_str().unwrap().to_owned(); + if accepted_candidate { + assert_eq!(id, aged_candidate.id.to_hex()); + } else if already_accepted { + assert_eq!(id, event.id.to_hex()); + } else { + assert_ne!(id, event.id.to_hex()); + } + if let Some(first) = &elected_id { + assert_eq!(first, &id); + } else { + elected_id = Some(id); + } + } + let saved: serde_json::Value = + serde_json::from_slice(&std::fs::read(dir.path().join("journal.json")).unwrap()) + .unwrap(); + assert!(saved["entries"][&id]["confirmed_at"].is_u64()); + assert_eq!( + serde_json::from_value::(saved["entries"][&id]["event"].clone()).unwrap(), + event + ); + let stamp: ReportStamp = + serde_json::from_value(saved["entries"][&id]["recovery_candidate"].clone()).unwrap(); + let renewed = stamp.restore(&event).unwrap(); + assert!( + Timestamp::now() + .as_secs() + .saturating_sub(renewed.created_at.as_secs()) + < 600 + ); + assert_ne!(renewed.id, aged_candidate.id); + let elected_id = nostr::EventId::from_hex(&elected_id.unwrap()).unwrap(); + let stored = state + .db + .get_event_by_id(community, elected_id.as_bytes()) + .await + .unwrap() + .unwrap(); + assert!(buzz_core::managed_delivery::same_report( + &event, + &stored.event + )); + let count: i64 = + sqlx::query_scalar("SELECT COUNT(*) FROM events WHERE community_id=$1 AND id=$2") + .bind(community.as_uuid()) + .bind(elected_id.as_bytes().as_slice()) + .fetch_one(state.db.pool()) + .await + .unwrap(); + assert_eq!(count, 1); + } + shutdown_tx.send(()).unwrap(); + server.await.unwrap(); +} + +#[tokio::test] +#[ignore = "requires Postgres and Redis"] +async fn managed_recovery_and_original_race_share_one_election() { + let state = authenticated_http_state().await; + let host = format!("managed-race-{}.test", Uuid::new_v4().simple()); + let community = state + .db + .ensure_configured_community(&host) + .await + .unwrap() + .id; + let tenant = TenantContext::resolved(community, &host); + let author = Keys::generate(); + let channel = Uuid::new_v4(); + state + .db + .create_channel_with_id( + community, + channel, + "managed race", + ChannelType::Stream, + ChannelVisibility::Private, + None, + author.public_key().as_bytes(), + None, + ) + .await + .unwrap(); + let tracer: Arc = Arc::new(VecTracer::default()); + let auth = || IngestAuth::Http { + pubkey: author.public_key(), + scopes: vec![Scope::MessagesWrite], + auth_method: HttpAuthMethod::Nip98, + }; + let root = EventBuilder::new(Kind::Custom(9), "root") + .tag(Tag::parse(["h", &channel.to_string()]).unwrap()) + .sign_with_keys(&author) + .unwrap(); + ingest_event_inner(&state, &tracer, &tenant, root.clone(), auth()) + .await + .unwrap(); + for _ in 0..8 { + let key = Uuid::new_v4().to_string(); + let original = EventBuilder::new(Kind::Custom(9), "same final report") + .tags([ + Tag::parse(["h", &channel.to_string()]).unwrap(), + Tag::parse(["e", &root.id.to_hex(), "", "root"]).unwrap(), + Tag::parse(["e", &root.id.to_hex(), "", "reply"]).unwrap(), + Tag::parse(["buzz-delivery", &key]).unwrap(), + ]) + .sign_with_keys(&author) + .unwrap(); + let wrap = |offset: u64| { + let candidate = EventBuilder::new(original.kind, &original.content) + .tags(original.tags.clone()) + .custom_created_at(Timestamp::from(original.created_at.as_secs() + offset)) + .sign_with_keys(&author) + .unwrap(); + EventBuilder::new( + Kind::Custom(20003), + serde_json::json!({"original":original,"candidate":ReportStamp::from(&candidate)}) + .to_string(), + ) + .tag(Tag::parse(["h", &channel.to_string()]).unwrap()) + .sign_with_keys(&author) + .unwrap() + }; + let (a, b, c) = tokio::join!( + ingest_event_inner(&state, &tracer, &tenant, original.clone(), auth()), + ingest_event_inner(&state, &tracer, &tenant, wrap(1), auth()), + ingest_event_inner(&state, &tracer, &tenant, wrap(2), auth()) + ); + let a = a.unwrap(); + let b = b.unwrap(); + let c = c.unwrap(); + assert_eq!(a.event_id, b.event_id); + assert_eq!(b.event_id, c.event_id); + let count: i64 = + sqlx::query_scalar("SELECT COUNT(*) FROM events WHERE community_id=$1 AND tags @> $2") + .bind(community.as_uuid()) + .bind(serde_json::json!([["buzz-delivery", key]])) + .fetch_one(state.db.pool()) + .await + .unwrap(); + assert_eq!(count, 1); + let changed = EventBuilder::new(original.kind, "changed report") + .tags(original.tags.clone()) + .sign_with_keys(&author) + .unwrap(); + assert!( + ingest_event_inner(&state, &tracer, &tenant, changed, auth()) + .await + .is_err() + ); + } + let replies: i32 = sqlx::query_scalar( + "SELECT reply_count FROM thread_metadata WHERE community_id=$1 AND event_id=$2", + ) + .bind(community.as_uuid()) + .bind(root.id.as_bytes().as_slice()) + .fetch_one(state.db.pool()) + .await + .unwrap(); + assert_eq!(replies, 8, "one thread contribution per elected report"); +} + +#[tokio::test] +#[ignore = "requires Postgres and Redis"] +async fn managed_ws_original_ack_correlates_after_recovery_wins() { + use futures_util::{SinkExt, StreamExt}; + use tokio_tungstenite::tungstenite::Message; + async fn next_json( + ws: &mut tokio_tungstenite::WebSocketStream< + tokio_tungstenite::MaybeTlsStream, + >, + ) -> serde_json::Value { + tokio::time::timeout(std::time::Duration::from_secs(5), async { + loop { + let msg = ws.next().await.unwrap().unwrap(); + if msg.is_text() { + return serde_json::from_str(msg.to_text().unwrap()).unwrap(); + } + } + }) + .await + .expect("WS response deadline") + } + let state = authenticated_http_state().await; + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let host = listener.local_addr().unwrap().to_string(); + let ws_url = format!("ws://{host}/"); + let community = state + .db + .ensure_configured_community(&host) + .await + .unwrap() + .id; + let tenant = TenantContext::resolved(community, &host); + let author = Keys::generate(); + let channel = Uuid::new_v4(); + state + .db + .create_channel_with_id( + community, + channel, + "managed ws", + ChannelType::Stream, + ChannelVisibility::Private, + None, + author.public_key().as_bytes(), + None, + ) + .await + .unwrap(); + let tracer: Arc = Arc::new(VecTracer::default()); + let auth = || IngestAuth::Http { + pubkey: author.public_key(), + scopes: vec![Scope::MessagesWrite], + auth_method: HttpAuthMethod::Nip98, + }; + let root = EventBuilder::new(Kind::Custom(9), "root") + .tag(Tag::parse(["h", &channel.to_string()]).unwrap()) + .sign_with_keys(&author) + .unwrap(); + ingest_event_inner(&state, &tracer, &tenant, root.clone(), auth()) + .await + .unwrap(); + let original = EventBuilder::new(Kind::Custom(9), "first report") + .tags([ + Tag::parse(["h", &channel.to_string()]).unwrap(), + Tag::parse(["e", &root.id.to_hex(), "", "root"]).unwrap(), + Tag::parse(["e", &root.id.to_hex(), "", "reply"]).unwrap(), + Tag::parse(["buzz-delivery", &Uuid::new_v4().to_string()]).unwrap(), + ]) + .sign_with_keys(&author) + .unwrap(); + let candidate = EventBuilder::new(original.kind, &original.content) + .tags(original.tags.clone()) + .custom_created_at(Timestamp::from(original.created_at.as_secs() + 1)) + .sign_with_keys(&author) + .unwrap(); + let wrapper = EventBuilder::new( + Kind::Custom(20003), + serde_json::json!({"original":original,"candidate":ReportStamp::from(&candidate)}) + .to_string(), + ) + .tag(Tag::parse(["h", &channel.to_string()]).unwrap()) + .sign_with_keys(&author) + .unwrap(); + let elected = ingest_event_inner(&state, &tracer, &tenant, wrapper.clone(), auth()) + .await + .unwrap(); + assert_eq!(elected.event_id, candidate.id.to_hex()); + let router = crate::router::build_router(state); + let (shutdown_tx, shutdown_rx) = tokio::sync::oneshot::channel(); + let server = tokio::spawn(async move { + axum::serve(listener, router) + .with_graceful_shutdown(async { + let _ = shutdown_rx.await; + }) + .await + .unwrap(); + }); + let (mut ws, _) = tokio_tungstenite::connect_async(&ws_url).await.unwrap(); + let challenge = loop { + let message = next_json(&mut ws).await; + if message[0] == "AUTH" { + break message[1].as_str().unwrap().to_owned(); + } + }; + let auth_event = EventBuilder::auth(challenge, nostr::RelayUrl::parse(&ws_url).unwrap()) + .sign_with_keys(&author) + .unwrap(); + ws.send(Message::Text( + serde_json::json!(["AUTH", auth_event]).to_string().into(), + )) + .await + .unwrap(); + loop { + let message = next_json(&mut ws).await; + if message[0] == "OK" && message[1] == auth_event.id.to_hex() { + assert_eq!(message[2], true); + break; + } + } + ws.send(Message::Text( + serde_json::json!(["EVENT", original]).to_string().into(), + )) + .await + .unwrap(); + let receipt = loop { + let message = next_json(&mut ws).await; + if message[0] == "OK" { + break message; + } + }; + assert_eq!( + receipt[1], + original.id.to_hex(), + "WS OK must correlate with the submitted original" + ); + assert_eq!(receipt[2], true); + let canonical: nostr::Event = serde_json::from_str( + receipt[3] + .as_str() + .unwrap() + .strip_prefix(buzz_core::managed_delivery::RECEIPT_PREFIX) + .unwrap(), + ) + .unwrap(); + canonical.verify().unwrap(); + assert_eq!(canonical.id, candidate.id); + ws.send(Message::Text( + serde_json::json!(["EVENT", wrapper]).to_string().into(), + )) + .await + .unwrap(); + let receipt = loop { + let message = next_json(&mut ws).await; + if message[0] == "OK" { + break message; + } + }; + assert_eq!(receipt[1], wrapper.id.to_hex()); + assert_eq!(receipt[2], true); + let canonical: nostr::Event = serde_json::from_str( + receipt[3] + .as_str() + .unwrap() + .strip_prefix(buzz_core::managed_delivery::RECEIPT_PREFIX) + .unwrap(), + ) + .unwrap(); + canonical.verify().unwrap(); + assert_eq!(canonical.id, candidate.id); + ws.close(None).await.unwrap(); + shutdown_tx.send(()).unwrap(); + server.await.unwrap(); +} + +#[tokio::test] +#[ignore = "requires Postgres, Redis and built CLI in BUZZ_TEST_BIN_DIR"] +async fn managed_recovery_mention_history_survives_lost_ack() { + use sha2::{Digest, Sha256}; + let state = authenticated_http_state().await; + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let host = listener.local_addr().unwrap().to_string(); + let relay = format!("http://{host}"); + let community = state + .db + .ensure_configured_community(&host) + .await + .unwrap() + .id; + let tenant = TenantContext::resolved(community, &host); + let author = Keys::generate(); + let recipient = Keys::generate(); + let channel = Uuid::new_v4(); + state + .db + .create_channel_with_id( + community, + channel, + "managed mentions", + ChannelType::Stream, + ChannelVisibility::Private, + None, + author.public_key().as_bytes(), + None, + ) + .await + .unwrap(); + state + .db + .add_member( + community, + channel, + recipient.public_key().as_bytes(), + buzz_db::channel::MemberRole::Member, + Some(author.public_key().as_bytes()), + ) + .await + .unwrap(); + crate::handlers::side_effects::emit_group_discovery_events(&tenant, &state, channel) + .await + .unwrap(); + let tracer: Arc = Arc::new(VecTracer::default()); + let auth = || IngestAuth::Http { + pubkey: author.public_key(), + scopes: vec![Scope::MessagesWrite], + auth_method: HttpAuthMethod::Nip98, + }; + let root = EventBuilder::new(Kind::Custom(9), "original task") + .tag(Tag::parse(["h", &channel.to_string()]).unwrap()) + .sign_with_keys(&author) + .unwrap(); + ingest_event_inner(&state, &tracer, &tenant, root.clone(), auth()) + .await + .unwrap(); + let key = Uuid::new_v4().to_string(); + let original = EventBuilder::new(Kind::Custom(9), "callback final report") + .tags([ + Tag::parse(["h", &channel.to_string()]).unwrap(), + Tag::parse(["e", &root.id.to_hex(), "", "root"]).unwrap(), + Tag::parse(["e", &root.id.to_hex(), "", "reply"]).unwrap(), + Tag::parse(["buzz-delivery", &key]).unwrap(), + Tag::parse(["p", &recipient.public_key().to_hex()]).unwrap(), + ]) + .custom_created_at(Timestamp::from(Timestamp::now().as_secs() - 3600)) + .sign_with_keys(&author) + .unwrap(); + let candidate = EventBuilder::new(original.kind, &original.content) + .tags(original.tags.clone()) + .sign_with_keys(&author) + .unwrap(); + let wrapper = EventBuilder::new( + Kind::Custom(20003), + serde_json::json!({"original":original,"candidate":ReportStamp::from(&candidate)}) + .to_string(), + ) + .tag(Tag::parse(["h", &channel.to_string()]).unwrap()) + .sign_with_keys(&author) + .unwrap(); + // The server commits, but the journal below deliberately has no local ACK. + let receipt = ingest_event_inner(&state, &tracer, &tenant, wrapper, auth()) + .await + .unwrap(); + assert_eq!(receipt.event_id, candidate.id.to_hex()); + let mentions = state + .db + .query_feed_mentions( + community, + recipient.public_key().as_bytes(), + &[channel], + None, + 10, + ) + .await + .unwrap(); + assert_eq!( + mentions.len(), + 1, + "recovered callback must appear in Home Mentions" + ); + assert_eq!(mentions[0].event.id, candidate.id); + let mut query = buzz_db::EventQuery::for_community(community); + query.channel_id = Some(channel); + query.p_tag_hex = Some(recipient.public_key().to_hex()); + query.kinds = Some(vec![9]); + assert_eq!( + state.db.query_events(&query).await.unwrap()[0].event.id, + candidate.id + ); + let router = crate::router::build_router(state.clone()); + let (shutdown_tx, shutdown_rx) = tokio::sync::oneshot::channel(); + let server = tokio::spawn(async move { + axum::serve(listener, router) + .with_graceful_shutdown(async { + let _ = shutdown_rx.await; + }) + .await + .unwrap(); + }); + let dir = tempfile::tempdir().unwrap(); + let signer = author.public_key().to_hex(); + let id = hex::encode(Sha256::digest( + serde_json::to_vec(&(&relay, &signer, &key)).unwrap(), + )); + let journal = serde_json::json!({"version":2,"entries":{id.clone():{"relay":relay,"signer":signer,"key":key,"channel":channel.to_string(),"parent":root.id.to_hex(),"event":original,"recovery_candidate":ReportStamp::from(&candidate),"attempts":1,"confirmed_at":null,"last_error":null}}}); + std::fs::write(dir.path().join("journal.json"), journal.to_string()).unwrap(); + let cli = std::path::PathBuf::from(std::env::var("BUZZ_TEST_BIN_DIR").unwrap()).join("buzz"); + let output = tokio::process::Command::new(cli) + .env_remove("BUZZ_AUTH_TAG") + .env("BUZZ_PRIVATE_KEY", author.secret_key().to_secret_hex()) + .args([ + "--relay", + &relay, + "messages", + "deliver", + "--delivery-key", + &key, + "--channel", + &channel.to_string(), + "--reply-to", + &root.id.to_hex(), + "--journal-dir", + dir.path().to_str().unwrap(), + "--content", + "changed retry summary", + ]) + .output() + .await + .unwrap(); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + let replay: serde_json::Value = serde_json::from_slice(&output.stdout).unwrap(); + assert_eq!(replay["event_id"], candidate.id.to_hex()); + assert_eq!( + replay["mention_pubkeys"], + serde_json::json!([recipient.public_key().to_hex()]) + ); + let mentions = state + .db + .query_feed_mentions( + community, + recipient.public_key().as_bytes(), + &[channel], + None, + 10, + ) + .await + .unwrap(); + assert_eq!(mentions.len(), 1); + assert_eq!(mentions[0].event.id, candidate.id); + let history = state.db.query_events(&query).await.unwrap(); + assert_eq!(history.len(), 1); + assert_eq!(history[0].event.id, candidate.id); + let count: i64 = sqlx::query_scalar( + "SELECT COUNT(*) FROM event_mentions WHERE community_id=$1 AND event_id=$2", + ) + .bind(community.as_uuid()) + .bind(candidate.id.as_bytes().as_slice()) + .fetch_one(state.db.pool()) + .await + .unwrap(); + assert_eq!(count, 1, "ACK-loss replay must not duplicate mention rows"); + shutdown_tx.send(()).unwrap(); + server.await.unwrap(); +} diff --git a/crates/buzz-relay/src/handlers/ingest/timestamp.rs b/crates/buzz-relay/src/handlers/ingest/timestamp.rs new file mode 100644 index 00000000000..7a566d0ebc7 --- /dev/null +++ b/crates/buzz-relay/src/handlers/ingest/timestamp.rs @@ -0,0 +1,99 @@ +//! Timestamp admission for ordinary events and signed managed-report recovery. +use super::{IngestAuth, IngestError}; +use buzz_core::kind::KIND_STREAM_MESSAGE; +use nostr::Event; + +fn managed_report(event: &Event, auth: &IngestAuth) -> bool { + if event.kind.as_u16() != KIND_STREAM_MESSAGE as u16 || event.pubkey != *auth.pubkey() { + return false; + } + let mut markers = event + .tags + .iter() + .map(|tag| tag.as_slice()) + .filter(|tag| tag.first().is_some_and(|name| name == "buzz-delivery")); + let Some(marker) = markers.next() else { + return false; + }; + marker.len() == 2 + && markers.next().is_none() + && uuid::Uuid::parse_str(&marker[1]).is_ok_and(|key| key.to_string() == marker[1]) +} + +pub(super) fn validate(event: &Event, auth: &IngestAuth, now: u64) -> Result<(), IngestError> { + const MAX_DRIFT: u64 = 900; + let timestamp = event.created_at.as_secs(); + // Old marked reports can acknowledge prior acceptance. New old rows still + // fail the commit-time DB floor; full recovery elects a current candidate + // through the managed recovery command. Current authorization remains required. + if timestamp.saturating_sub(now) > MAX_DRIFT + || (now.saturating_sub(timestamp) > MAX_DRIFT && !managed_report(event, auth)) + { + return Err(IngestError::Rejected( + "invalid: event timestamp too far from server time".into(), + )); + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use buzz_auth::Scope; + use nostr::{EventBuilder, Keys, Kind, Tag, Timestamp}; + + #[test] + fn managed_timestamp_exception_preserves_future_and_marker_authority_boundaries() { + let signer = Keys::generate(); + let other = Keys::generate(); + let key = "9937de50-bed6-41c7-8d7b-ee7b9db9e6ba"; + let canonical = vec![vec!["buzz-delivery".to_string(), key.to_string()]]; + let malformed = [ + vec![], + vec![vec!["buzz-delivery".into(), "invalid".into()]], + vec![vec!["buzz-delivery".into(), key.to_uppercase()]], + vec![vec!["buzz-delivery".into(), key.into(), "extra".into()]], + vec![ + vec!["buzz-delivery".into(), key.into()], + vec!["buzz-delivery".into(), uuid::Uuid::new_v4().to_string()], + ], + ]; + let event = |timestamp, kind, tags: Vec>| { + EventBuilder::new(Kind::Custom(kind), "report") + .custom_created_at(Timestamp::from(timestamp)) + .tags(tags.into_iter().map(|tag| Tag::parse(tag).unwrap())) + .sign_with_keys(&signer) + .unwrap() + }; + let auth = |keys: &Keys| IngestAuth::Http { + pubkey: keys.public_key(), + scopes: vec![Scope::MessagesWrite], + auth_method: super::super::HttpAuthMethod::Nip98, + }; + let now = 2_000_000; + for (timestamp, tags, allowed) in [ + (now - 900, vec![], true), + (now - 901, vec![], false), + (now - 3600, canonical.clone(), true), + (0, canonical.clone(), true), + (now + 900, canonical.clone(), true), + (now + 901, canonical.clone(), false), + (u64::MAX, canonical.clone(), false), + ] { + assert_eq!( + validate(&event(timestamp, 9, tags), &auth(&signer), now).is_ok(), + allowed + ); + } + for tags in malformed { + assert!(validate(&event(now - 3600, 9, tags), &auth(&signer), now).is_err()); + } + assert!(validate( + &event(now - 3600, 1, canonical.clone()), + &auth(&signer), + now + ) + .is_err()); + assert!(validate(&event(now - 3600, 9, canonical), &auth(&other), now).is_err()); + } +} diff --git a/crates/buzz-relay/src/mesh_boot.rs b/crates/buzz-relay/src/mesh_boot.rs index 63a346258f5..52ff344ca1f 100644 --- a/crates/buzz-relay/src/mesh_boot.rs +++ b/crates/buzz-relay/src/mesh_boot.rs @@ -421,24 +421,38 @@ pub(crate) async fn run_demo_echo( tracing::info!(%session_id, %peer, "mesh demo echo: session open"); let mut drain_tick = tokio::time::interval(std::time::Duration::from_millis(100)); loop { - let frame = tokio::select! { - _ = drain_tick.tick() => { - if shutting_down.load(Ordering::Relaxed) { - if let Some(community_id) = stream.community_id() { - if let Err(e) = stream.send_goodbye(community_id, GoodbyeReason::Draining).await { - tracing::warn!(%session_id, "mesh demo echo: draining goodbye failed: {e}"); - } else { - tracing::info!(%session_id, "mesh demo echo: sent draining goodbye"); + let frame = { + // Framing and fence validation may already have consumed a frame + // before yielding. Poll the same receive across ordinary drain + // ticks; only actual shutdown may abandon it and close the stream. + let receive = stream.recv_validated(&directory); + tokio::pin!(receive); + loop { + tokio::select! { + frame = &mut receive => break Some(frame), + _ = drain_tick.tick() => { + if shutting_down.load(Ordering::Relaxed) { + break None; } - } else { - let _ = stream.finish(); - tracing::info!(%session_id, "mesh demo echo: drain before community latch — closing"); } - return; } - continue; } - frame = stream.recv_validated(&directory) => frame, + }; + let Some(frame) = frame else { + if let Some(community_id) = stream.community_id() { + if let Err(e) = stream + .send_goodbye(community_id, GoodbyeReason::Draining) + .await + { + tracing::warn!(%session_id, "mesh demo echo: draining goodbye failed: {e}"); + } else { + tracing::info!(%session_id, "mesh demo echo: sent draining goodbye"); + } + } else { + let _ = stream.finish(); + tracing::info!(%session_id, "mesh demo echo: drain before community latch — closing"); + } + return; }; match frame { Ok(Some(ReliableFrame::Data(payload))) => { @@ -696,6 +710,69 @@ mod tests { MeshStream::new(Box::new(StubSend), Box::new(StubRecv)) } + #[tokio::test(start_paused = true)] + async fn demo_echo_keeps_inflight_read_across_ticks_and_still_drains() { + struct PendingRecv { + starts: Arc, + entered: Arc, + } + impl StreamRecvHalf for PendingRecv { + fn recv_frame(&mut self) -> BoxFuture<'_, Result, MeshError>> { + Box::pin(async move { + // A framing read may already have consumed bytes before it + // waits for the rest. Replacing this future loses that state. + self.starts.fetch_add(1, Ordering::SeqCst); + self.entered.notify_one(); + std::future::pending().await + }) + } + } + let starts = Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let entered = Arc::new(tokio::sync::Notify::new()); + let pool = deadpool_redis::Config::from_url("redis://127.0.0.1:1") + .create_pool(Some(deadpool_redis::Runtime::Tokio1)) + .unwrap(); + let fenced = FencedHeader { + session_id: uuid::Uuid::new_v4(), + generation: 1, + owner_runtime_id: rid(1), + }; + let stream = MeshStream::new( + Box::new(StubSend), + Box::new(PendingRecv { + starts: Arc::clone(&starts), + entered: Arc::clone(&entered), + }), + ); + let shutting_down = Arc::new(AtomicBool::new(false)); + let task = tokio::spawn(run_demo_echo( + SessionDirectory::new(pool), + ReliableInbound { + fenced, + from: rid(2), + stream: crate::tunnel::reliable::ReliableMeshStream::new_inbound(fenced, stream), + }, + Arc::clone(&shutting_down), + )); + entered.notified().await; + for _ in 0..3 { + tokio::time::advance(std::time::Duration::from_millis(100)).await; + tokio::task::yield_now().await; + assert!(!task.is_finished()); + } + let read_starts = starts.load(Ordering::SeqCst); + shutting_down.store(true, Ordering::Relaxed); + tokio::time::advance(std::time::Duration::from_millis(100)).await; + tokio::time::timeout(std::time::Duration::from_secs(1), task) + .await + .expect("shutdown must interrupt the pending read") + .unwrap(); + assert_eq!( + read_starts, 1, + "drain polling must preserve the receive future" + ); + } + fn rid(byte: u8) -> RuntimeId { RuntimeId([byte; 32]) } diff --git a/deny.toml b/deny.toml index d3c5fcd4bc7..a69d78a0af4 100644 --- a/deny.toml +++ b/deny.toml @@ -1,5 +1,9 @@ [advisories] ignore = [ + # Release-only approval: Andrey B., 2026-10-15, Buzz event + # 14da571fb65bf9ab1946c2be8ca6c3a621709f07a5ac5423e49fe6c64894c028. + # Reassess before the next release; remove after the YAML replacement. + { id = "RUSTSEC-2024-0449", reason = "unsafe-libyaml unmaintained; no safe upgrade. Approved only for fork-v0.5.26-4; replacement tracked in https://github.com/reynholm/buzz/issues/19" }, # instant 0.1.13 — unmaintained crate. Transitive dep: nostr → instant. # Will be resolved when nostr crate updates its dependencies. { id = "RUSTSEC-2024-0384", reason = "transitive dep via nostr; no upstream fix available" }, diff --git a/desktop/src-tauri/Cargo.lock b/desktop/src-tauri/Cargo.lock index c761ab3e67f..843c11a65ee 100644 --- a/desktop/src-tauri/Cargo.lock +++ b/desktop/src-tauri/Cargo.lock @@ -9063,9 +9063,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.45" +version = "0.23.46" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d41d731c7d2f962d1ccc364cec258de3c0e93b38c2fb3ba97ac74513048d634" +checksum = "48e13bd8c0e9365c43cfa5c9e8f9ad49d3c8444926c9aac819e0e4dc503c8fdf" dependencies = [ "aws-lc-rs", "log", diff --git a/desktop/src/shared/api/relayReconnectReplay.test.mjs b/desktop/src/shared/api/relayReconnectReplay.test.mjs index 7c7d96f44c1..000433a17ee 100644 --- a/desktop/src/shared/api/relayReconnectReplay.test.mjs +++ b/desktop/src/shared/api/relayReconnectReplay.test.mjs @@ -105,16 +105,17 @@ function numericList(source, pattern) { } test("channel replay lookback stays coupled to relay and DB source constants", async () => { - const [ingest, fence] = await Promise.all([ + const [ingest, timestamp, fence] = await Promise.all([ readFile("../crates/buzz-relay/src/handlers/ingest.rs", "utf8"), + readFile("../crates/buzz-relay/src/handlers/ingest/timestamp.rs", "utf8"), readFile("../crates/buzz-db/src/runtime/replica_fence.rs", "utf8"), ]); assert.match( - ingest, - new RegExp( - `MAX_TIMESTAMP_DRIFT_SECS: i64 = ${RELAY_INGEST_FUTURE_TOLERANCE_SECS}`, - ), + timestamp, + new RegExp(`MAX_DRIFT: u64 = ${RELAY_INGEST_FUTURE_TOLERANCE_SECS}`), ); + assert.match(timestamp, /timestamp\.saturating_sub\(now\) > MAX_DRIFT/); + assert.match(ingest, /timestamp::validate\s*\(/); assert.match( fence, new RegExp(`CREATED_AT_FLOOR_SECS: i64 = ${DB_CREATED_AT_FLOOR_SECS}`), diff --git a/docs/superpowers/plans/2026-10-14-background-result-atomic-recovery.md b/docs/superpowers/plans/2026-10-14-background-result-atomic-recovery.md new file mode 100644 index 00000000000..1c17ffa35ad --- /dev/null +++ b/docs/superpowers/plans/2026-10-14-background-result-atomic-recovery.md @@ -0,0 +1,24 @@ +# Atomic recovery of managed background reports + +Owner decision: Buzz event `e5539b60bf6d93287d8c94c11d8ec97748bfc13e900fd00cc4b225fd25a05a2c` selects option 1 after the published tradeoffs. Continue implementation in the existing worktree and obtain Astron's review; no GitHub publication. Earlier inherited mesh/source-assertion gate decisions remain open. + +The original same-signed-bytes requirement is amended only for recovery: the first report content and route remain immutable, but an unaccepted expired report may become a freshly timestamped author-signed message. An accepted report, including one subsequently deleted, must never be recreated. The production DB floor stays armed. This design follows VISION's relay-owned access boundary and Nostr-first operations. + +1. Add ephemeral Nostr recovery kind 20003. Content is `{original: Event, candidate: ReportStamp}`. The stamp contains only timestamp/ID/signature and reconstructs immutable fields from the original; both reconstructed events must be verified author-signed kind 9, with identical author, tags and content, and one canonical delivery UUID. Candidate has a current timestamp. The wrapper is author-signed, current, channel-scoped and bounded. Unknown servers cannot publish the nested candidate as a message. Route this command through shared ingest in both HTTP and WS; never fan out or store the wrapper. The selected candidate receives every ordinary message authorization, restriction, archive and thread check. +2. Add a community/author/UUID canonical delivery ledger. Transaction advisory lock serializes normal marked message writes and recovery. Under that lock: return an existing canonical record; otherwise adopt an existing original ID from the writer including soft-deleted rows; otherwise insert candidate + thread metadata + canonical record atomically. Store the signed canonical event for acknowledgment after redaction/retention. Reject key reuse across content/tags/routes. No event-floor bypass, replica read, or query-then-write gap. Attach the community deletion write fence and inventory/purge contract; maintain migration and desired schema parity. No automatic ledger expiry: one row per elected report, purged with its community. Coordinated relay upgrade is required; old writer processes do not participate in the lock. +3. Return the chosen signed event in the standard Nostr acknowledgment message, using `managed-delivery:` plus serialized Event; `event_id` names it. Canonical state drives duplicate receipts; only a newly inserted candidate fans out. Keep normal send/edit semantics outside the marker protocol. +4. CLI retains original signed event, optional recovery candidate stamp and canonical stamp in its bounded journal. Migrate v1 records safely. Before network use, persist a candidate identical except timestamp/id/signature; refresh an aged candidate without changing original evidence. Use recovery when original age exceeds 600 seconds, leaving margin to the 900/960 admission/storage limits. Validate canonical signature, author, UUID, content and exact tags against the original; persist confirmation before returning. Wrapper acknowledgments without this signed receipt fail visibly, including old-server ordinary acknowledgments. Normal timely v1-compatible publication remains supported. Keep original namespace, membership, limits, attempts and lock unchanged. + +Execution: implement inline in this session using TDD. Review design/implementation with a finite independent checker and Astron; do not ask for the already-selected contract again. + +Verification focus: + +- RED/GREEN guarded writer recovery for absent/accepted hour-old originals and real CLI/NIP98 HTTP. +- Concurrent wrappers with different fresh candidates, a normal original racing recovery, lost acknowledgment followed by another candidate, aged candidate renewal, first body/route preservation, soft-deleted originals and tenant/author/key isolation. +- Reject forged/mismatched envelopes, candidate payload/route changes, malformed/multiple UUIDs, future candidate/wrapper, insufficient scope, archived/revoked channels and unknown-server receipts. +- Existing CLI/ACP actual-process workflows continue; progress/edit and fresh keys remain independent. +- Full touched package suites, production-constructor DB tests, fmt/clippy/build and required `just test`; report inherited failures without claiming a full green gate. + +Self-review: safety depends on all managed writes taking the same writer lock before insertion, not on negative readback. The command has no new HTTP endpoint and no server signing authority. Journal preservation precedes candidate publication, and DB election precedes success. Deferred floor failure rolls back the binding as well. Existing canceled/observer/collection policies are unchanged by this B2 work. + +Independent review corrections: full candidate copies exceeded existing admitted-report bounds through JSON escaping. Use compact verified stamps in wire and journal, 512 KiB recovery content /1 MiB HTTP envelope bounds, while retaining 512 KiB record/16 MiB journal. The canonical receipt stays a full signed event. WS correlation must handle the normal original losing election, not only wrapper submissions. Both failures have actual-process RED/GREEN regressions; maximum-size coverage includes quote, backslash and U+0001 bodies. diff --git a/docs/superpowers/plans/2026-10-14-background-result-delivery.md b/docs/superpowers/plans/2026-10-14-background-result-delivery.md new file mode 100644 index 00000000000..cc1d64ad392 --- /dev/null +++ b/docs/superpowers/plans/2026-10-14-background-result-delivery.md @@ -0,0 +1,65 @@ +# Background result delivery implementation plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox syntax for tracking. + +**Goal:** Preserve background collection when new messages arrive (A), then provide durable idempotent final-report publication (B). + +**Architecture:** Collection obligations remain separate from ordinary queue batches, retain their original session and route, and have independent bounded retries. A managed CLI publisher journals one signed event per stable completion key before sending it and reuses it across retries and restarts. A succeeds independently of B. + +**Tech Stack:** Rust, Tokio, existing ACP queue/pool, Nostr SDK and CLI HTTP transport, filesystem journal and advisory locks. + +**Spec:** `docs/superpowers/specs/2026-10-14-background-result-delivery-task.md` + +## Global Constraints + +- Implement A first, independently reviewable; B is a separate change. +- Read root AGENTS.md, VISION.md, VISION_AGENT.md, VISION_REMOTE_AGENTS.md, TESTING.md and affected package TESTING.md before editing. +- No GitHub publication, PR or push. Work remains local. +- No unsafe or new production unwrap/expect; document public API. +- Preserve explicit cancel/rotate and membership revocation; never deliver a revoked channel's report elsewhere. +- Do not change detached/reparented child tracking or introduce unrelated refactors. +- Clean ACP environment variables (`BUZZ_ACP_ALLOWED_RESPOND_TO`, `BUZZ_ACP_LAZY_POOL`, `BUZZ_ACP_IDLE_POOL_SLEEP`, `BUZZ_ACP_SESSION_POLICY`) and run the complete affected package suite serially, plus fmt and clippy. The fourth variable was discovered as inherited input contamination during baseline. +- Activate Hermit; prepend `/Users/reynholm/.local/bin` to PATH to use the owner's identity guard. Before any commit verify user.name is global Andrey Bruhanov, user.email matches global configuration, and commit.gpgsign is empty; commit with `-s`. + +## Review Focus + +- New traffic during retry backoff cannot erase delivery or borrow its retry budget (Task 1). +- Channel-scoped messages from another thread cannot reroute a live child's result (Task 1). +- Interrupt cancellation and automatic rotation after child exit cannot invalidate pending collection (Task 1). +- Two concurrent publisher processes or response loss cannot construct distinct signed events for one key (Task 2). +- Revoked access, corrupt/full journal and expiry cannot silently drop a pending publication or route it elsewhere (Task 2). + +--- + +### Task 1: Preserve and schedule original background collection (A) + +**Files:** Modify `crates/buzz-acp/src/queue.rs`, `queue/background.rs`, `pool/background_work.rs`, `pool.rs`, `lib.rs`; extend `pool/background_work_tests.rs`, queue tests and `crates/buzz-acp/TESTING.md` as needed. Keep auxiliary changes within ACP. + +**Interfaces:** Consumes existing RuntimeContinuation ownership/session/generation/thread tags. Produces preserved continuation lifecycle used unchanged by B; report exact changed signatures. Retain FIFO by ready age for continuations before ordinary messages where practical; ordinary messages may run during collection backoff. Independent continuation retry counts/deadlines must survive message completion. Preserve original RuntimeHold across ordinary turns. Hold the original session until collection succeeds or fails terminally; explicit control actions invalidate it. + +- [x] Write falsifiable production-seam regressions: pending-before-message, message-before-child-exit, backoff-survives-message, interrupted-collection-resumes, live-child-route-survives-other-thread, max-turns=1-pending-collection-protects-session, generation/revocation clears work, and finite terminal retries. Exercise `dispatch_pending` with real local ACP child processes controlled by marker files. Cover queue/steer/interrupt, Channel/Thread/DM and sleep modes using existing test infrastructure. +- [x] Run new regressions and record expected missing-behavior failures before implementation. Also establish complete package baseline. +- [x] Implement minimal queue and ownership lifecycle changes. Separate ordinary message retry state from continuation retry state. Do not silently supersede on push/requeue/requeue_as_cancelled. Ready collection wins over continuous new traffic, while backoff permits message dispatch. Terminal collection failure remains visible through the existing notice path. +- [x] Run `env -u BUZZ_ACP_ALLOWED_RESPOND_TO -u BUZZ_ACP_LAZY_POOL -u BUZZ_ACP_IDLE_POOL_SLEEP -u BUZZ_ACP_SESSION_POLICY cargo test -p buzz-acp -- --test-threads=1`, `cargo fmt --all -- --check`, and `cargo clippy -p buzz-acp --all-targets -- -D warnings`. Record exact HEAD and outcomes, real workflow limitations, and human reproduction steps. +- [x] Self-review and commit only A files with verified guarded identity and DCO. Write implementation report to supplied report path; do not spawn reviewers or helpers. + +### Task 2: Durable managed final-report delivery (B) + +**Files:** Add focused journal/publisher module under `crates/buzz-cli/src/commands/`; wire `lib.rs`, `commands/mod.rs`, `commands/messages.rs`, `Cargo.toml` and lockfile if required. Add stable key to ACP `RuntimeContinuation` and prompt construction, tests and package guides. Add production-seam CLI and ACP/bundled-agent integration regressions. + +**Interfaces:** Consumes A's preserved continuation. Produces a stable random completion delivery ID allocated once when background group origin is recorded, retained through queue/prompt retries. Managed command: `buzz messages deliver --delivery-key KEY --channel UUID --reply-to EVENT --content TEXT`, with an explicit optional journal directory for tests/operators and ordinary explicit/automatic mention support for callback notifications. Existing `messages send` stays independent for progress; `messages edit` stays independent for corrections. Prompt supplies this final-report command and the original route. Default journal is persistent under agent's Buzz workspace; document override and actual coverage. Bind the delivery key into the signed event (a tag), so two keys with identical content in the same second produce two distinct event IDs. + +**Delivery contract (describe in report before implementation):** Identity namespaces include normalized relay URL, signer pubkey and random group key; entry binds original channel/reply destination. Atomically journal the first signed event before publish, then lock and replay the same event. Changed content for the same key returns the original delivery; changed route conflicts. Check current channel membership before every publication/ack replay and use ordinary Nostr `/events` publication. Journal retains unconfirmed signed events, bounded to 1024 records and 8 publication attempts; terminal failure is explicit and retains evidence. Confirmed entries may expire after 7 days; idempotency is guaranteed only while retained. Unconfirmed entries never silently expire to permit re-signing. Reject at capacity rather than evict pending data. Lock files do not grow per key without bound. Validate saved events and file sizes; atomic replacement and fsync cover restart between relay acceptance and local acknowledgment. No general exactly-once claim for bypassing shell/MCP publishers. + +- [x] Write production publisher regressions and observe RED: changed text same key yields one event, equal text distinct keys yields two, concurrent attempts one event, body-loss retry preserves bytes, restart after relay acceptance reuses event, progress/edit unaffected, revoked channel refuses, bounds/corruption fail visibly. +- [x] Implement managed command and journal. Reuse `BuzzClient::sign_event` and `submit_event`; do not add relay endpoints. Use fs2 locking (already used by bundled agent) or equally robust existing facilities; cap journal/record bytes and attempts. Validate `accepted` and event identity instead of treating arbitrary successful HTTP as acknowledgment. +- [x] Wire stable completion key and managed final command into ACP continuation prompt after A's task gate. Ensure unrelated messages never receive/reuse its key. Add a deterministic local model/ACP flow that invokes the actual production publisher, then returns application error, and retries with a single visible signed event in local mock-relay readback. Clearly distinguish mock from real relay evidence. +- [x] Run complete `buzz-cli`, `buzz-acp` and any modified `buzz-agent` package suites, fmt and clippy. Run `just ci` only if preparing a PR (no PR currently authorized). Provide precise human smoke-test commands and evidence limits. +- [x] Self-review and commit B separately with guarded identity/DCO. Write report and return commit IDs, outcomes and limitations. Do not spawn helpers or reviewers. + + +## Current handoff + +A: `cabc051c3a4ad7343db3e1d87df6740362447540`. +B: `33966afed44c35067f344a503fdf02f1d12d7489`, followed by B1 durability fix `c07ce95adc6b82dc87aef28a4d67c16b43ae9460`. +All local implementation checks completed; final whole-branch Astride24 review-master review is pending. No GitHub publication or thread completion marker. diff --git a/docs/superpowers/reports/2026-10-14-background-result-delivery.md b/docs/superpowers/reports/2026-10-14-background-result-delivery.md new file mode 100644 index 00000000000..277c0008d5d --- /dev/null +++ b/docs/superpowers/reports/2026-10-14-background-result-delivery.md @@ -0,0 +1,200 @@ +# Background result delivery — local implementation report + +Specification: [task](../specs/2026-10-14-background-result-delivery-task.md). Plan: [implementation](../plans/2026-10-14-background-result-delivery.md). + +**Current status: requested local implementation complete.** A2 is committed as `59fe7b69817395933225c7b91e02dcaecabba8e8`, owner-approved atomic B2 as `d33222b6680531bf6e895ec41799bde458c4a71f`, and B3/DB source guards as `30f52e7dfb3821116ba658e642deb04fd577333e`. Astron addressed review returned PASS with all prior important findings fixed. Fresh required `just test` completed exit0 on unchanged production/test source; full affected DB141/0 and relay1448/0 passed without filtering. The reviewer separately reproduced the unchanged intermittent mesh504; that failing run is retained as a known verification limitation, not substituted by the passing run. No GitHub publication, deployment or human acceptance is claimed. + +Repository: `reynholm/buzz`; origin was fetched before work. Base: `e72a377e5fd3d552e0e30dc523535d9a79cf1e99`. Branch: `fork/background-result-delivery`, isolated worktree `/Users/reynholm/.buzz/REPOS/buzz-background-result-delivery`. No GitHub publication, PR or push was performed. + +## A: preserve collection + +Commit: `cabc051c3a4ad7343db3e1d87df6740362447540`. Ordinary traffic cannot replace the pending collection or its original scope/session/generation/thread route. Collection has separate bounded retry state; ready collection takes precedence, while backoff permits ordinary traffic. Automatic rotation retains the original session until collection ends. Steer/Interrupt preserve collection; explicit owner cancel/rotate, unambiguous observer cancellation and membership revocation invalidate it. Lost prompt tasks release retained authority. + +The implementation changes `queue.rs`, `queue/background.rs`, `pool.rs`, `pool/background_work.rs` and `lib.rs` under `crates/buzz-acp/src`, with real-process regressions in `pool/background_work_tests.rs` and package documentation. Marker-controlled local ACP adapters exercise production dispatch/result handling, real children, Channel/Thread/DM routing, queue/steer/interrupt, backoff/finite failure, eager/lazy/sleep modes, max-turn rotation, owner/observer cancellation and access/generation fences. + +A final complete package run passed **1096 tests, 0 failed, 3 ignored**. Formatting and ACP Clippy passed. Independent task review passed after fixing a reproduced explicit-cancel race and lost-task cleanup. Exact receipts: `.superpowers/sdd/2026-10-14-background-result-delivery/task-1-report.md`, `stage-a-final-complete.log`, `stage-a-final-fmt.log`, `stage-a-final-clippy.log`, `task-1-fix-2-review.md`. Reviewed file hashes were independently matched to the commit; tests ran on that matching pre-commit source, not on a future SHA. + +## B: managed final-report delivery + +B implementation commit: `33966afed44c35067f344a503fdf02f1d12d7489`. Review fix: `c07ce95adc6b82dc87aef28a4d67c16b43ae9460`. Local B implementation is committed, but whole-feature verification is incomplete. Astron whole-branch review reported A2/B2; A2 is fixed and B2 remains open after scoped re-review. + +One random UUID is allocated per observed background group and retained in `RuntimeContinuation`. A collection prompt supplies `buzz messages deliver`, original channel/parent and a quoted absolute persistent journal path. The signed `buzz-delivery` tag distinguishes different keys even with equal content/time. Identity is normalized relay URL + signer public key + UUID. This does not identify internal provider tasks that ACP never exposes. + +The managed CLI stores the first signed event and original route before submitting through the existing Nostr publisher. A fixed advisory lock serializes cooperating processes; atomic replacement and file/directory fsync preserve the event across restart. Repeating the key with changed text returns the first event; changing route conflicts. Current signed channel membership is checked before submission or confirmed replay. Ordinary progress send and explicit edit remain independent; mentions belong to the first signed event. + +The journal is bounded to 1024 records, 512 KiB per record and 16 MiB serialized total. Each record permits eight managed submissions, each with up to three existing transport attempts. The lock wait is bounded to 30 seconds. Pending/terminal records are retained; capacity/corruption/attempt exhaustion fail visibly. Confirmed entries may expire after seven days; expiry, manual deletion or volume destruction ends idempotency. Pending errors are structured `managed_delivery_pending`, exit 2, `retryable:true`: replay exactly the same key/route/journal, rather than creating another key/send. + +Source responsibility: private `crates/buzz-cli/src/commands/delivery.rs` owns locking, bounded journal, validation, access checks, replay and acknowledgments. CLI parser/errors/dependency/message preparation wire the managed command. ACP continuation state/prompt/tests carry the key and original route. Package guides describe command use and native smoke checks. + +Four real local runtime scenarios use actual CLI processes and rebuilt embedded MCP/bundled agent: publication followed by model HTTP 400 and ACP AgentError/retry; response-body loss and process restart; SIGKILL after relay acceptance before local acknowledgment; concurrent publishers. The mock relay verifies signed event readback and counts one unique event for one key. These are mock-relay/model workflows, not authenticated relay/provider or human app evidence. Detailed source fingerprints, RED/GREEN failures, resource/access/corruption/mention cases and full logs are indexed in `.superpowers/sdd/2026-10-14-background-result-delivery/task-2-report.md`. + +Initial B-source validation before the ancestor-link review fix (all logs in the artifact directory above): + +| Check | Result | Receipt | +| --- | --- | --- | +| Complete CLI | 512 passed, 0 failed; 1 ignored doctest | `stage-b-final-cli-complete.log` | +| Complete ACP | 1097 passed, 0 failed; 7 ignored | `stage-b-final-acp-complete.log` | +| Complete embedded MCP | 80 passed, 0 failed | `stage-b-final-mcp-complete.log` | +| Explicit built-binary managed workflows | 4 passed, 0 failed | `stage-b-final-runtime.log` | +| Format / CLI+ACP+MCP Clippy / rebuilt CLI+MCP | exit 0 | `stage-b-final-fmt.log`, `stage-b-final-clippy.log`, `stage-b-final-build.log` | + +The four managed runtime tests are included among ACP's seven default-lane ignored tests and were explicitly exercised separately. CLI baseline was 493/0; final CLI adds 19 managed regressions. Prior failed command-catalog checks and a reproduced directory-creation fsync race are retained in the report; both are fixed. Final source seal: `8af029b4906a5e8a80cff75af485eebdf22660d290e43d431bdcd477f9007686`. Controller verified all 15 source file hashes. The complete ACP run started before final CLI-only corrections; exact ACP diff/file hashes stayed identical, so its result applies to the final ACP bytes. + +## Reproduce verification + +Activate Hermit and run complete affected packages serially. ACP requires all four harness overrides cleared and modern Git for the existing credential fixture: + +```sh +. ./bin/activate-hermit +cargo test -p buzz-cli -- --test-threads=1 +PATH=/Users/reynholm/.buzz/.scratch/M1-git-toolchain/install/bin:$PATH \ + env -u BUZZ_ACP_ALLOWED_RESPOND_TO -u BUZZ_ACP_LAZY_POOL \ + -u BUZZ_ACP_IDLE_POOL_SLEEP -u BUZZ_ACP_SESSION_POLICY \ + cargo test -p buzz-acp -- --test-threads=1 +cargo test -p buzz-dev-mcp -- --test-threads=1 +cargo fmt --all -- --check +cargo clippy -p buzz-cli -p buzz-acp -p buzz-dev-mcp --all-targets -- -D warnings +cargo build -p buzz-cli -p buzz-agent -p buzz-dev-mcp +BUZZ_TEST_BIN_DIR="$PWD/target/debug" \ + env -u BUZZ_ACP_ALLOWED_RESPOND_TO -u BUZZ_ACP_LAZY_POOL \ + -u BUZZ_ACP_IDLE_POOL_SLEEP -u BUZZ_ACP_SESSION_POLICY \ + cargo test -p buzz-acp managed_runtime_ -- --ignored --test-threads=1 --nocapture +``` + +No PR is being prepared, so `just ci`/PR gates are not represented as completed. Before A2/B2 review corrections, relay/db/auth production code was unchanged. B2 below adds a narrowly scoped shared-relay timestamp admission policy; database and auth code remain unchanged. + +## Human smoke checks and evidence limits + +The exact commands and expected event IDs/tags, same-key/fresh-key/concurrent runs, independent progress/edit and access revocation are in [CLI testing](../../../crates/buzz-cli/TESTING.md#managed-final-report-delivery). Use an existing authorized channel/root and an absolute persistent journal. Repeat the same key with changed summary and read back exactly one original report; a fresh key gives another. Remove publisher membership using an authorized owner and expect exit 3 with no redirected/new report. Check the native app shows original-thread delivery and callback mention. + +For [ACP testing](../../../crates/buzz-acp/TESTING.md#managed-background-final-reports), launch a real background child; interleave another-thread traffic while live, ready, collecting and backing off. Publish via the supplied command, cause application error, and retry in the same session with changed summary. Expect one signed final report. Repeat with a second group and expect another key/report. Verify Channel/Thread/DM, Queue/Steer/Interrupt, eager/lazy, sleep 0/2, max-turns 1, explicit cancel/rotate and membership removal as in the lifecycle runbook. Repeated collection failures must reach a visible finite terminal outcome. + +No human smoke confirmation or real authenticated relay/provider test is claimed. Process kill/lost response verifies ordering and recovery boundaries, not hardware power loss. Directory fsync/permissions were exercised on macOS/Unix; Windows/network filesystem durability is unverified and unsupported operations fail visibly. Detached/reparented-child tracking is unchanged. Provider results not journaled before loss, vanished persistent storage, harness queue/session resurrection and automatic startup replay are outside coverage. Arbitrary shell/MCP publication bypasses remain outside the managed-command guarantee. + +## Rulings made during execution + +1. Execute the user-approved task without another planning approval: the user explicitly instructed implementation. If wrong, local changes are reversible. +2. Explicitly prepend the owner-provided Git identity guard because the harness PATH omitted it. Effective identity/signing is checked before each commit. If wrong, commit attribution would be invalid. +3. Clear inherited `BUZZ_ACP_SESSION_POLICY` in verification, in addition to the three documented overrides; keep production defaults unchanged. If wrong, a deployment-input failure could be concealed. Original failing baseline is retained. +4. Use the existing scratch Git 2.49 for unchanged credential fixtures, without global install/configuration changes; actual source commits retain the owner guard. If wrong, an integration defect could be masked. Original Apple Git failures and matching modern-Git fixture results are retained. +5. Include the existing observer `cancel_turn` path for the same active-collection cancellation race. Preserve unambiguous active scope, sibling rejection and idle behavior. If wrong, Stop scope could broaden; production guard regressions constrain it. + +## Final review + +Internal A review: passed. Internal B review identified IMPORTANT B1, documented below; its complete final verdict was not preserved. Owner corrected the reviewer to Astron. Whole-branch review at `904bc0a5`: FAIL with A2/B2. Scoped V2 re-review: FAIL, A2 fixed and B2 partially fixed. Those earlier rounds were superseded by owner-approved atomic B2 and the addressed B3 PASS below; the fresh required gate subsequently completed exit0. + + +## Review finding B1 and final verified source + +The preserved internal B review finding concerns concurrent **nested** directory creation: after a creator exposes `/stable/new-parent/journal` but before it fsyncs `/stable`, another publisher can observe the existing chain. Syncing only `new-parent` leaves its own link unpersisted when publication begins. A process kill does not demonstrate the ensuing hardware-loss scenario; the filesystem ordering defect is nevertheless reachable before a send. + +`managed_existing_nested_directory_requires_ancestor_link_sync_before_publication` exercises actual CLI parser/dispatch with a real filesystem and signature-checking mock relay. It makes the higher ancestor execute-only while the immediate journal parent remains readable. Before the fix the command published successfully (exit 0 instead of 4): RED 0 passed / 1 failed, exit 101 (`stage-b-ancestor-sync-red.log`). After the fix it refuses publication before any event: GREEN 1 passed / 0 failed (`stage-b-ancestor-sync-green.log`). Every invocation now fsyncs all ancestor directories, even an already existing chain, with a 128-ancestor bound. No ACP source changed. + +The post-fix verification commands used the reproduction commands above on parent HEAD `33966afed44c35067f344a503fdf02f1d12d7489` plus the exact later committed fix. All logs below are under the same evidence directory: + +| Check | Actual result | Receipt | +| --- | --- | --- | +| Complete CLI | 513 passed / 0 failed; one ignored doctest | `stage-b-post-ancestor-cli-complete.log` | +| Complete embedded MCP | 80 passed / 0 failed | `stage-b-post-ancestor-mcp-complete.log` | +| Format, CLI+ACP+MCP Clippy, rebuilt CLI+agent+MCP | exit 0 | `stage-b-post-ancestor-fmt.log`, `stage-b-post-ancestor-clippy.log`, `stage-b-post-ancestor-build.log` | +| Explicit actual-process managed runtime scenarios | 4 passed / 0 failed | `stage-b-post-ancestor-runtime.log` | +| Complete ACP | Earlier 1097 passed / 0 failed / 7 ignored applies to byte-identical ACP source; not rerun for this CLI-only fix | `stage-b-final-acp-complete.log`, source manifests | + +Final B source seal relative to A: `370ad223eb08da963e6a63af2ccc7ebac03c28f69ccc7a9e355d116a3bc2a87a`. Files are listed in `stage-b-post-ancestor-source-manifest.json`; `stage-b-post-ancestor-reviewed.diff` is the sealed diff. All committed source hashes were checked against this manifest. The package guide was clarified after the checks; only documentation changed after those source checks. Final source commits are A `cabc051c`, B `33966afe`, B1 fix `c07ce95a`. The plan and this report are recorded separately. No new real-relay, hardware-loss or human-testing claim follows from these checks. + + +## Astron review A2/B2 corrections + +Reviewer report: `/Users/reynholm/.buzz/RESEARCH/BACKGROUND_RESULT_DELIVERY_REVIEW_ASTRON_2026_10_14.md`, reviewed base `e72a377e` through `904bc0a5`. Both findings were independently verified; A1, lost-task cleanup, observer cancellation and B1 remained fixed. Astride24 stopped without a verdict after the owner corrected the reviewer name to Astron. + +**A2:** A collection's token permits one ordinary cancellation. `requeue_as_cancelled` records that cancellation in `RuntimeContinuation.ordinary_preempted`; both exact-scope and unambiguous-channel signal paths then defer ordinary Steer/Interrupt, while explicit controls remain allowed. Native steering queues during all collection attempts, preventing a repeatedly extended deadline. A protected retry completes in its original session, then queued traffic runs; errors retain their existing separate finite retry budget/backoff. A fresh background group starts with a fresh false flag. No independent cancellation counter or unbounded table was added. + +Marker-controlled `background_collection_completes_under_repeated_ordinary_preemption` failed before changes (RED 0/1, exit101, `A2_RED.log`). Its first post-fix fixture incorrectly expected recoverable metadata on a normal Drop-mode foreground turn; that test-only assumption was replaced by assertions on the actual subsequent wire prompt. `A2_GREEN.log` retains this initial fixture failure. Final background regression group:31/0 (`A2_BACKGROUND_GREEN.log`). The new test exercises actual listener fallback/Interrupt paths, eight following ordinary events, native-steer admission refusal, collection completion and subsequent message delivery. Complete ACP:1098/0, seven ignored (`A2_ACP_COMPLETE.log`). Exact six ACP source hashes are sealed in `A2_ACP_TESTED_SOURCE.json` and remained unchanged after this run; package documentation was clarified separately. + +**B2 first proposal — insufficient, superseded by the production-writer finding below:** Preserve the saved signed event rather than re-signing an uncertain delivery. Shared relay ingest now permits a past timestamp beyond900 seconds only for a signature-verified self-published kind9 with exactly one two-element `buzz-delivery` tag whose UUID is canonical. The future900-second limit remains; unsigned saturating comparisons also reject extreme future timestamps safely. All subsequent current scope, membership, restriction, archive, signature/content and thread checks still run before storage/deduplication. There is no early-success bypass, new HTTP endpoint, schema migration, journal format change or UUID-based relay deduplication. Ordinary old messages and malformed markers remain refused. + +This is an explicit sender opt-in to historical timestamp admission: the marker does not prove a background task, a prior attempt or a durable journal. A currently authorized publisher can choose a backdated managed message. The event preserves its historical ordering. Older relays retain their fifteen-minute window; this recovery fix requires an updated relay. It was exercised locally, not deployed to the user's relay. Existing eight-attempt and retention limits remain in force. + +The initial shared-ingest B2 RED used isolated schema-compatible Postgres with an unarmed `Db::from_pool`, not the production writer: an hour-old saved event failed timestamp admission (0/1, exit101, `B2_RED.log`). Final ignored real-ingest cases cover an absent old event, an already accepted old event, duplicate thread-counter behavior, invalid signature/author/ancestry, revoked membership and archive. An additional case runs separate actual CLI processes against the **real HTTP router with NIP98 verification, real Redis replay protection and real Postgres**, restores pending journals in both remote states and verifies the exact original event, one stored ID and durable local acknowledgment. This is not a mock HTTP relay, but the storage setup omits the production floor guard and cannot prove production late insertion. It remains an in-process local server fixture, not a deployed provider/app or human acceptance test. Original journals are seeded as explicit crash/outage states; the already accepted case passes through actual ingest. + +Initial three explicit integration cases on the unarmed fixture:3/0 (`B2_FINAL_REAL_RELAY.log`). Fixed-clock production timestamp boundary test:1/0 (`B2_BOUNDARIES.log`). Four bundled-agent/MCP/CLI mock-runtime scenarios remain4/0 (`A2_B2_RUNTIME.log`). Final fmt/clippy exit0 (`A2_B2_FINAL_FMT.log`, `A2_B2_FINAL_CLIPPY.log`); all five binaries rebuilt (`A2_B2_BUILD.log`). Tests ran on parent904bc0a5 plus the exact sealed working fixes, not on a future commit SHA. `A2_B2_FINAL_SOURCE.json` records the final source; only the package guide's attempt-budget wording was clarified afterward. + +**Inherited verification failure:** two complete relay-package attempts returned1411 passed /1 failed /367 ignored in the library, stopping before later targets (`B2_RELAY_COMPLETE.log`, `B2_RELAY_COMPLETE_RECHECK.log`). The failing existing test is `api::mesh_demo::tests::demo_join_forwarded_arm_round_trips_echo`, returning504 instead of200. It also fails in an isolated module and exact-test process; one earlier exact run passed. Disabling Redis makes the fixture silently skip and is not accepted as verification. The five mesh implementation/fixture files are unchanged from basee72a377e. Source-supported likely cause: `run_demo_echo` selects a100ms tick against `recv_validated`, which consumes a frame then awaits Redis; cancellation can lose that frame. No cancellation trace was instrumented, so this remains a likely cause rather than an observed mechanism. No unrelated mesh fix was included. + +A separate full-target remainder run explicitly skipped only that test:1411 library +27 binary +9 boot-lifecycle =1447 passed,0 failed,371 ignored,1 filtered out (`B2_RELAY_REMAINDER.log`). This is **not** a green full relay package. The mandatory repository `just test` was also launched against newly created task-owned standard dev services; it finished exit1 with two failed steps: `buzz-db tests` and `workspace integration tests`, both at the unchanged source assertion `crates/buzz-db/tests/observability_source.rs:357`. `DB_OBSERVABILITY_BASELINE.json` verifies the test and moderation source are byte-identical to base, where the required `fetch_optional` is already absent. Feature completion remains unset pending B2 and owner decisions about inherited gates. + +Additional ruling: choose narrow relay admission over readback followed by re-signing, because a negative read cannot fence an in-flight original publication and does not deliver a saved absent expired event. Cost if wrong: managed publishers can opt into past-dated admission; current authority, signature and future-limit regressions constrain that policy. Documentation and this report make the changed policy and required relay upgrade explicit. + + +## V2 re-review: production writer exposes remaining B2 + +Astron report: `/Users/reynholm/.buzz/RESEARCH/BACKGROUND_RESULT_DELIVERY_REREVIEW_ASTRON_2026_10_14.md`, STATUS FAIL. Reviewer independently confirms A2 fixed, earlier A1/cleanup/observer/B1 preserved, and B2 partially fixed. Its own full ACP run passed 1098/0, seven ignored. The first V2 manifest contains 13/13 files; the report's B2 conclusion follows production code and SQL, not a reviewer DB run. + +`Db::new` arms `buzz.created_at_floor=960` on every writer connection (`crates/buzz-db/src/runtime/mod.rs:676`), matching relay startup at `main.rs:308`. Migration 0021 rejects a new old channel row at deferred COMMIT. Already-stored events use `ON CONFLICT DO NOTHING`, so they can recover without inserting; absent old events cannot. Disabling this guard would invalidate the replica-fence ordering premise and is not a valid fix. + +Recovery fixtures now use `Db::new` and assert the production GUC. Accepted historical events are seeded separately through an unarmed fixture pool to represent a pre-outage committed state; all tested recovery uses the guarded writer. Tests for accepted and absent states are split so a failure cannot hide the accepted branch. The first fixture edit had a duplicate pool-reference compile error; it is retained in `B2_PRODUCTION_WRITER_COMPILE_ATTEMPT.log` and was corrected before measuring behavior. + +Owner contract question published as Buzz event `7b8cb24e0a9124786c98611c264d04f6fd88e177a0de290a3401153653e36519`: choose full atomic delivery-key recovery that may create fresh signed bytes only for an unaccepted report, or explicitly bounded recovery with visible terminal expiry. No answer or waiver is inferred from silence. No database guard change, blind re-signing, GitHub publication or root completion reaction has been made. + +Corrected production-writer run: **3 passed, 2 failed**, exit101 (`B2_PRODUCTION_WRITER_RED.log`). Accepted-event shared ingest and actual CLI/HTTP/NIP98 recovery pass, as do current signature/route/membership/archive checks. Both absent-event cases fail: shared ingest exposes the deferred floor exception, and actual CLI returns `managed_delivery_pending` after HTTP500 while preserving the signed event. This confirms the remaining B2 defect without disabling the guard. No full green relay claim follows. + +A2 local commit attribution and contents were verified after committing: `59fe7b69817395933225c7b91e02dcaecabba8e8`, configured owner author/committer, required DCO, no signing. All six committed ACP source hashes match `A2_ACP_TESTED_SOURCE.json` and the 1098/0 package run. The package guide is the seventh committed file. B2 source and corrected reports remain working changes, deliberately uncommitted pending the contract decision. + + +## Owner-approved atomic B2 recovery + +Owner decision `e5539b60bf6d93287d8c94c11d8ec97748bfc13e900fd00cc4b225fd25a05a2c` selects option 1 from the published tradeoffs. This authorizes fresh signed bytes only for an unaccepted report while preserving original author, content and exact route/tags. Implementation plan: `docs/superpowers/plans/2026-10-14-background-result-atomic-recovery.md`. + +New ephemeral Nostr command kind 20003 carries a verified original and current candidate. Shared ingest verifies all three signatures and immutable fields, then sends the candidate through the ordinary kind-9 scope, membership, restriction, archive, content and ancestry gates. HTTP uses the existing bridge; WS excludes this command from generic ephemeral fanout and retains its effect permit. WS OK correlates with the outer request ID, while its `managed-delivery:` message contains the canonical author-signed report; HTTP `event_id` names that report. No relay signing or new HTTP endpoint is introduced. + +Normal marked publication and recovery acquire the same transaction advisory lock by community/author/UUID. The guarded writer returns an existing canonical binding or adopts a historical original, including soft-deleted rows, before inserting a current candidate. Candidate, thread metadata and signed canonical ledger row commit together. A deferred floor error rolls back the binding too. Only newly inserted candidates fan out or contribute counters. Different immutable payload/routes conflict. Migration 0056, desired schema and community deletion expected/purge inventories include the ledger and its universal write fence. One bounded signed-event row is retained per elected report until community purge; ordinary soft deletion does not reopen election. + +CLI journal v1 migrates under its lock to v2. Original evidence, a refreshed candidate and a differing canonical event remain bounded by the existing record/journal limits. The command is used once original age exceeds 600 seconds, with fresh-clock candidate renewal. Before publication the candidate is journaled and fsynced; after a signature/identity/immutable-payload-checked receipt, confirmation is persisted before returning. Lost acknowledgment can return an earlier candidate. Old-server wrapper acknowledgments are refused. Timely normal publication remains compatible with the prior relay, but full recovery needs a coordinated writer upgrade and drained old requests. Keys must not be reused across completion groups. + +Initial protocol RED: unknown-kind refusal, 0/1 (`B2_ATOMIC_COMMAND_RED.log`). The first GREEN attempt exposed missing test schema (`B2_ATOMIC_COMMAND_FIRST_GREEN.log`). Updating the existing desired-state fixture through pgschema failed on inherited partition PK alterations (`B2_ATOMIC_SCHEMA.log`), atomically rolling back. Switched angle to a new task-owned `buzz_atomic` database bootstrapped by the actual production migration runner; no guard or schema premise bypass. Intermediate tests exposed an outdated same-ID expectation and a test-only string borrowing compile error; both are retained in intermediate logs rather than hidden. + +Eight explicit migrated production-writer cases passed (`B2_ATOMIC_REAL_RACE.log`): absent/accepted hour-old reports, deleted-original adoption, lost acknowledgment of an earlier candidate, renewal after another restart, actual standalone CLI/NIP98/HTTP and current authority boundaries, plus eight three-way original/distinct-candidate races with one event and one thread-counter contribution each. Historical rows and journals are seeded outage snapshots, not actual elapsed-hour or hardware-crash experiments. Full core/CLI checks: CLI 514/0, core 264/0 plus two doctests; one ignored CLI doctest (`B2_ATOMIC_CORE_CLI_FULL.log`). All-target core/DB/CLI/relay clippy exit0 (`B2_ATOMIC_CLIPPY.log`). Final rebuilt-binary runs and full gates follow below. + +The first full DB package found two change-related migration expectations (count 55 and missing new fence target); updated these existing parity guards to include migration 56 and the new ledger. The live desired/migrated schema parity test now covers ledger columns/indexes. Full relay still fails the unchanged mesh echo (1411/1, 372 ignored, `B2_ATOMIC_RELAY_FULL.log`); this is not a green complete relay package. No scope expansion to unrelated mesh or DB observability fixes has been authorized. + + +## Internal atomic-recovery review corrections + +Finite independent code review found two IMPORTANT regressions in the initial atomic implementation: WS ordinary-original ACK used the elected candidate ID, and duplicate full-event payloads exceeded the recovery cap for previously admitted quote-heavy 64 KiB reports. Original scenarios were reproduced through actual NIP42 WS (`B2_WS_ACK_RED.log`, 0/1) and actual CLI/NIP98 HTTP (`B2_SIZE_RED.log`, 0/1). + +Both are fixed: all managed WS OK messages correlate the submitted request ID while the signed receipt names the canonical report. Wire/journal candidates use `ReportStamp {created_at,id,sig}`, reconstructing the original immutable body/tags/author and verifying the resulting event. This removes duplicate body copies without increasing the 512 KiB record/16 MiB journal bounds. Recovery content is capped at512 KiB and full HTTP envelope at1 MiB; the canonical receipt remains a full signed Event. Legacy v1 fixtures contain no new fields, and v2 candidate renewal remains separately covered. + +Final explicit run:10/0 (`B2_STAMP_WS_FINAL.log`), including actual WS original and wrapper submissions, three maximum-size bodies (quotes/backslashes/U+0001), true legacy v1 recovery, lost previous-candidate ack and the prior guarded-writer races/authority cases. These use production migrations and `Db::new`; no floor bypass. All-target clippy passed for core/DB/CLI/relay (`B2_STAMP_CLIPPY.log`). Independent targeted re-review returned PASS by reading and serialization probes; it did not run Rust tests. Its probe confirmed maximum-size bodies fit all bounds. Full reviewer receipt will be preserved with the final packet. + +Live desired-state versus production migration schema parity:1/0 (`B2_ATOMIC_SCHEMA_PARITY.log`), now covering the canonical ledger columns/indexes in addition to existing admin tables. Complete DB library after updating change-related migration expectations:138/0,391 ignored. Complete DB package remains red solely at the unchanged observability source assertion (2/1 source tests, `B2_ATOMIC_DB_FULL_V2.log`). This is a baseline limitation, not a full-green claim. + + +Final compact-source checks: complete CLI514/0 and core264/0 plus2 doctests (`B2_STAMP_CORE_CLI_FULL.log`); full relay **1448/0**,378 ignored (`B2_STAMP_RELAY_FULL.log`), including the formerly failing unchanged mesh test. Earlier failures remain evidence of an intermittent baseline test, but no test was filtered from this successful complete run. Freshly rebuilt bundled agent/MCP/CLI workflows4/0 (`B2_STAMP_RUNTIME.log`), build exit0 (`B2_STAMP_BUILD_ALL.log`), fmt/clippy exit0. The final explicit guarded recovery run also asserts a rejected expired ordinary write leaves no canonical binding, then recovers through the current candidate (`B2_STAMP_GUARD_FINAL.log`). + +The broad pre-commit `just test` finished exit1 with the same two inherited failed steps: DB package and workspace integration at `observability_source.rs:357`. Its earlier core/CLI stages ran before compact-stamp review corrections, so it is not attributed as final stamp-source evidence; a post-commit required gate is planned. Updated migration expectations pass; the unchanged moderation/test hashes remain those in `DB_OBSERVABILITY_BASELINE.json`. No source-assertion fix or gate waiver is inferred from the owner's B2 contract decision. + + +## Astron B3 correction and complete affected packages + +Whole-feature review on `d33222b6680531bf6e895ec41799bde458c4a71f` returned FAIL with IMPORTANT B3, while confirming B2 and the earlier lifecycle/durability corrections fixed. Report: `/Users/reynholm/.buzz/RESEARCH/BACKGROUND_RESULT_DELIVERY_ATOMIC_REVIEW_ASTRON_2026_10_14.md`. Its own complete relay 1448/0 was green; DB retained the source assertion failure. This review was read before the source freeze ended. + +**B3:** fresh recovery insertion bypassed the normal wrapper's mention index. A signed `p` tag remained in the report, but historical `#p` and Home Mentions join `event_mentions` and missed it. New `managed_recovery_mention_history_survives_lost_ack` exercises a private channel with an actual recipient member, a one-hour-old pending original and a current recovery candidate. It failed with zero Home Mentions entries before the fix (`B3_MENTIONS_RED.log`, 0/1, exit101). `insert_mentions_in_transaction` now runs for a newly inserted canonical report before the ledger insert and transaction commit. Failure propagates and rolls back event/thread/mentions/binding together. Existing binding replay remains non-inserting; no deployed-state backfill is claimed. + +GREEN `B3_MENTIONS_GREEN.log`: 11/0 explicit production `Db::new`, PostgreSQL/Redis recovery cases. The new case verifies Home Mentions and historical `#p`, then replays an unconfirmed journal through a separate actual CLI process and NIP98 HTTP relay. It checks the same signed canonical ID, preserved mention recipient and one index row after the lost-ACK retry. Historical time/crash state is seeded, not an actual hour of downtime. No native-app or human acceptance claim follows. + +**Required DB source guards:** the first stale assertion assumed `restriction_state` uses `fetch_optional`, but its unchanged aggregate query uses `fetch_one`. The revised guard requires typed Authorization acquisition and rejects pool bypass while accepting either result cardinality. Once that assertion passed, the complete test exposed a second stale assumption: acquire/release/verify serving leases delegate to the existing bounded helper instead of carrying inline EventWrite text. The guard now verifies the helper's bounded attributed checkout and connection forwarding plus those exact callsites/transaction constructor; renew and is_serving_active retain direct EventWrite requirements. Production moderation and lease behavior is unchanged. `DB_OBSERVABILITY_BASELINE.json` and `B3_LEASE_GUARD_BASELINE.json` preserve base equality evidence. No test was skipped or owner waiver assumed. + +Complete affected packages on parent `d33222b6` plus this sealed source: DB **141/0** (138 library and 3 source guards; 391 ignored), relay **1448/0** (1412 library, 27 binary, 9 boot; 379 ignored), zero doctest failures (`B3_FULL_DB_RELAY_V2.log`, exit0). The first full attempt retained the newly exposed stale lease assertion (`B3_FULL_DB_RELAY.log`, exit101). Final fmt/all-target DB+relay clippy exit0 (`B3_FMT_V2.log`, `B3_CLIPPY_V2.log`). Finite independent source review confirms B3 and the revised guards, with twelve source mutations rejected; it ran no Rust tests. + +The earlier post-commit broad gate ended exit1 at the stale source guards (`B2_POST_COMMIT_JUST_TEST.log`). B3 edits began while it was still running, so it is mixed-state evidence, not final B3 verification. The fresh required `just test` subsequently completed exit0 on stable B3 production/test source (`B3_JUST_TEST.log`, `B3_JUST_TEST_RECEIPT.json`). External addressed review returned PASS as recorded below. + +Additional human check: send a managed final report with an explicit callback `--mention`, preserve its pending journal while the first submission cannot reach the relay, then recover after the original expires. The original thread must contain one report; the recipient's Home Mentions and historical `#p` readback must include its canonical ID. Replay the same journal/key again and expect that same ID and one mention entry. Use disposable local infrastructure and an authorized recipient account; do not infer this check was performed by a human from the automated regression. + + +## Final local result and verification limits + +Astron addressed review at `30f52e7dfb3821116ba658e642deb04fd577333e` returned **PASS** for B3/source guards and preserved all earlier fixed finding statuses; a fresh independent verifier confirmed it. Report: `/Users/reynholm/.buzz/RESEARCH/BACKGROUND_RESULT_DELIVERY_B3_REVIEW_ASTRON_2026_10_14.md`. Reviewer-owned DB141/0 is separate from author evidence. Its relay library run failed the previously observed, unchanged mesh demo echo (1411/1, HTTP504); the full command stopped before subsequent targets. No new causal connection to the feature was confirmed, and no successful retry is substituted for that failure. + +Author's complete affected packages remain DB141/0,391ignored and relay1448/0,379ignored without filters (`B3_FULL_DB_RELAY_V2.log`). Fresh required `just test` now ends **exit0**, including DB package and workspace integration; CLI514/0, complete ACP1098/0,7ignored and embedded MCP80/0 passed within the script. The script also runs an additional ACP library pass. `B3_JUST_TEST_RECEIPT.json` records the checked start/completion HEADs and unchanged production/test hashes; the local B3 commit occurred during the run without changing compiled source. Report-only edits do not claim another test run. Earlier failed or mixed-state runs remain preserved. + +All requested local implementation stages and addressed review corrections are delivered; intermittent mesh behavior remains an explicitly recorded baseline test limitation outside the feature paths. No mesh fix, skip or waiver is claimed. Before any later PR, perform the repository's `just ci` and human-confirmed smoke gates; no PR/push is authorized here. Full expired recovery requires coordinated relay/writer upgrade with migration0056, original persistent journal and unique delivery keys. Arbitrary shell/MCP publications, loss of the journal/provider output, hardware power loss, native-app acceptance and deployed-relay behavior retain the limits described above. diff --git a/migrations/0056_managed_report_delivery.sql b/migrations/0056_managed_report_delivery.sql new file mode 100644 index 00000000000..9bacfb83d6e --- /dev/null +++ b/migrations/0056_managed_report_delivery.sql @@ -0,0 +1,11 @@ +-- One canonical signed report per author and completion key. Independent of event redaction. +CREATE TABLE managed_deliveries ( + community_id UUID NOT NULL REFERENCES communities(id), + author BYTEA NOT NULL CHECK (length(author) = 32), + delivery_key UUID NOT NULL, + channel_id UUID NOT NULL, + event JSONB NOT NULL CHECK (pg_column_size(event) <= 524288), + received_at TIMESTAMPTZ NOT NULL, + PRIMARY KEY (community_id, author, delivery_key) +); +SELECT attach_community_write_fence('managed_deliveries'); diff --git a/schema/schema.sql b/schema/schema.sql index fb4b06fb31f..f04731976f9 100644 --- a/schema/schema.sql +++ b/schema/schema.sql @@ -2058,3 +2058,15 @@ SELECT attach_community_write_fence('artifact_revisions'); -- The relay does not expire events. Any future row retention or partition -- retirement must skip payloads referenced by `artifact_heads.event_id` -- (NIP-AR: expiring earlier revisions MUST NOT remove the current revision). + +-- One canonical signed report per author and completion key. Independent of event redaction. +CREATE TABLE managed_deliveries ( + community_id UUID NOT NULL REFERENCES communities(id), + author BYTEA NOT NULL CHECK (length(author) = 32), + delivery_key UUID NOT NULL, + channel_id UUID NOT NULL, + event JSONB NOT NULL CHECK (pg_column_size(event) <= 524288), + received_at TIMESTAMPTZ NOT NULL, + PRIMARY KEY (community_id, author, delivery_key) +); +SELECT attach_community_write_fence('managed_deliveries'); diff --git a/scripts/check-postgres-test-discovery.py b/scripts/check-postgres-test-discovery.py index 449c984848b..414758e1edf 100755 --- a/scripts/check-postgres-test-discovery.py +++ b/scripts/check-postgres-test-discovery.py @@ -184,7 +184,9 @@ def integration_binary_is_postgres(path: Path) -> bool: ) -def standard_module_paths(parent_source: Path, module_name: str) -> list[Path]: +def standard_module_paths( + parent_source: Path, module_name: str, ancestors: list[str] +) -> list[Path]: """Resolve Rust's conventional out-of-line module source paths.""" root = crate_root(parent_source) is_crate_root = parent_source.name in {"lib.rs", "main.rs", "mod.rs"} @@ -200,6 +202,7 @@ def standard_module_paths(parent_source: Path, module_name: str) -> list[Path]: if is_crate_root else parent_source.parent / parent_source.stem ) + base = base.joinpath(*ancestors) return [base / f"{module_name}.rs", base / module_name / "mod.rs"] @@ -215,6 +218,7 @@ def out_of_line_module_index(files: list[Path]) -> dict[Path, list[str]]: for parent_source in sorted(context_files): source = parent_source.read_text(encoding="utf-8") sanitized = sanitize_rust(source) + inline_ranges = module_ranges(source) if "path" in source: for attribute in PATH_ATTRIBUTE.finditer(sanitized): @@ -232,11 +236,16 @@ def out_of_line_module_index(files: list[Path]) -> dict[Path, list[str]]: for module in OUT_OF_LINE_MODULE.finditer(sanitized): module_name = module.group("name") - for candidate in standard_module_paths(parent_source, module_name): + ancestors = [ + name for start, end, name in inline_ranges + if start < module.start() < end + ] + for candidate in standard_module_paths(parent_source, module_name, ancestors): if candidate.is_file(): module_names = names.setdefault(candidate.resolve(), []) - if module_name not in module_names: - module_names.append(module_name) + for name in ancestors + [module_name]: + if name not in module_names: + module_names.append(name) return names @@ -278,7 +287,7 @@ def file_has_postgres_lane_test( if ( ("postgres" in reason_lower or "postgresql" in reason_lower) and not EXTERNAL_INFRA.search(reason) - and not any(name.startswith("external_infra") for name in modules) + and not any(name.startswith("external_infra") for name in modules + external_modules) and ( any(name.endswith("postgres_tests") for name in modules + external_modules) or integration_binary_is_postgres(path) @@ -345,7 +354,9 @@ def validate_file( any(name.endswith("postgres_tests") for name in modules + external_modules) or integration_binary_is_postgres(path) ) - in_external_module = any(name.startswith("external_infra") for name in modules) + in_external_module = any( + name.startswith("external_infra") for name in modules + external_modules + ) needs_external_infra = bool(EXTERNAL_INFRA.search(reason)) if mentions_redis and not mentions_postgres and in_postgres_lane and not in_external_module: diff --git a/scripts/fork/patches.json b/scripts/fork/patches.json index a084ee29f6f..68c077e4407 100644 --- a/scripts/fork/patches.json +++ b/scripts/fork/patches.json @@ -840,7 +840,7 @@ }, { "path": "desktop/src-tauri/Cargo.lock", - "responsibility": "Direct Desktop gethostname dependency without version changes", + "responsibility": "Direct Desktop gethostname dependency without version changes Pin rustls 0.23.46 for the release, resolving RUSTSEC-2026-0346 through RUSTSEC-2026-0349 without unrelated dependency changes.", "is_new": false, "required_symbols": [], "invocation_seams": [], @@ -4390,9 +4390,11 @@ }, { "path": ".github/workflows/_ci-relay.yml", - "responsibility": "Integrate existing reynholm/buzz main 33f54de2 without retiring fork patches", + "responsibility": "Integrate existing reynholm/buzz main 33f54de2 without retiring fork patches Build, cache and transfer the CLI executable alongside relay artifacts for managed recovery tests in the PostgreSQL lane.", "is_new": false, - "required_symbols": [], + "required_symbols": [ + "export BUZZ_TEST_BIN_DIR=\"$PWD/target/ci\"" + ], "invocation_seams": [], "verification_commands": [ "just ci" @@ -4450,7 +4452,7 @@ }, { "path": "Cargo.lock", - "responsibility": "Integrate existing reynholm/buzz main 33f54de2 without retiring fork patches", + "responsibility": "Integrate existing reynholm/buzz main 33f54de2 without retiring fork patches Pin rustls 0.23.46 for the release, resolving RUSTSEC-2026-0346 through RUSTSEC-2026-0349 without unrelated dependency changes.", "is_new": false, "required_symbols": [], "invocation_seams": [], @@ -4460,12 +4462,15 @@ }, { "path": "Justfile", - "responsibility": "Integrate existing reynholm/buzz main 33f54de2 without retiring fork patches", + "responsibility": "Integrate existing reynholm/buzz main 33f54de2 without retiring fork patches Select the infra-free demo echo cancellation regression in the required nextest unit lane.", "is_new": false, - "required_symbols": [], + "required_symbols": [ + "test(=mesh_boot::tests::demo_echo_keeps_inflight_read_across_ticks_and_still_drains)" + ], "invocation_seams": [], "verification_commands": [ - "just ci" + "just ci", + "just test-unit" ] }, { @@ -4590,9 +4595,11 @@ }, { "path": "crates/buzz-acp/src/lib.rs", - "responsibility": "Integrate existing reynholm/buzz main 33f54de2 without retiring fork patches", + "responsibility": "Integrate existing reynholm/buzz main 33f54de2 without retiring fork patches Preserve the heartbeat collection call with a textual drift guard; full live-service-loop behavioral mutation coverage is not claimed.", "is_new": false, - "required_symbols": [], + "required_symbols": [ + "pool.queue_finished_background_work(&mut queue);" + ], "invocation_seams": [], "verification_commands": [ "just ci" @@ -4860,12 +4867,13 @@ }, { "path": "crates/buzz-db/src/runtime/tests/thread_window_postgres_tests.rs", - "responsibility": "Integrate existing reynholm/buzz main 33f54de2 without retiring fork patches", + "responsibility": "Integrate existing reynholm/buzz main 33f54de2 without retiring fork patches Verify production prebuild migration reaches the latest embedded migration while preserving the ingestion witness/index checks.", "is_new": false, "required_symbols": [], "invocation_seams": [], "verification_commands": [ - "just ci" + "just ci", + "scripts/postgres-test-run.sh -p buzz-db --lib --tests" ] }, { @@ -7585,12 +7593,16 @@ }, { "path": "scripts/run-tests.sh", - "responsibility": "Integrate existing reynholm/buzz main 33f54de2 without retiring fork patches", + "responsibility": "Integrate existing reynholm/buzz main 33f54de2 without retiring fork patches Keep the cargo-test fallback selecting the same demo echo receive-retention regression as nextest.", "is_new": false, - "required_symbols": [], + "required_symbols": [ + "buzz-relay demo echo receive retention" + ], "invocation_seams": [], "verification_commands": [ - "just ci" + "just ci", + "bash -n scripts/run-tests.sh", + "just test-unit" ] }, { @@ -7855,6 +7867,606 @@ "verification_commands": [ "python3 -m unittest discover -s scripts/fork/tests -v" ] + }, + { + "path": "TESTING.md", + "responsibility": "Document production-writer recovery, replica-floor premises, authenticated CLI/relay regression commands and evidence boundaries.", + "is_new": false, + "required_symbols": [], + "invocation_seams": [], + "verification_commands": [ + "BUZZ_TEST_BIN_DIR=\"$PWD/target/debug\" cargo test -p buzz-relay --lib handlers::ingest::postgres_tests::managed_delivery_tests:: -- --ignored --test-threads=1" + ] + }, + { + "path": "crates/buzz-acp/TESTING.md", + "responsibility": "Document original-session collection, bounded ordinary preemption and bundled agent/MCP/CLI delivery runtime fixtures.", + "is_new": false, + "required_symbols": [], + "invocation_seams": [], + "verification_commands": [ + "env -u BUZZ_ACP_SESSION_POLICY -u BUZZ_ACP_ALLOWED_RESPOND_TO -u BUZZ_ACP_LAZY_POOL -u BUZZ_ACP_IDLE_POOL_SLEEP cargo test -p buzz-acp background -- --test-threads=1" + ] + }, + { + "path": "crates/buzz-acp/src/acp/process_tracking_tests.rs", + "responsibility": "Inherited integration inventory: real ACP adapter regressions distinguish resident providers, active task children and unrelated sessions.", + "is_new": true, + "required_symbols": [ + "prompt_work_is_tracked_under_worker_and_resident_provider" + ], + "invocation_seams": [], + "verification_commands": [ + "env -u BUZZ_ACP_SESSION_POLICY -u BUZZ_ACP_ALLOWED_RESPOND_TO -u BUZZ_ACP_LAZY_POOL -u BUZZ_ACP_IDLE_POOL_SLEEP cargo test -p buzz-acp process_tracking_tests -- --test-threads=1" + ] + }, + { + "path": "crates/buzz-acp/src/isolated_execution.rs", + "responsibility": "Inherited integration inventory: fresh-process isolated prompt executor with cancellation, deadlines and bounded shutdown; module retention does not assert a new background-delivery invocation.", + "is_new": false, + "required_symbols": [ + "pub(crate) async fn execute(", + "agent.acp.shutdown().await;" + ], + "invocation_seams": [], + "verification_commands": [ + "env -u BUZZ_ACP_SESSION_POLICY -u BUZZ_ACP_ALLOWED_RESPOND_TO -u BUZZ_ACP_LAZY_POOL -u BUZZ_ACP_IDLE_POOL_SLEEP cargo test -p buzz-acp --lib -- --test-threads=1" + ] + }, + { + "path": "crates/buzz-acp/src/pool/background_work.rs", + "responsibility": "Retain collection authority in the original worker/session, poll child exit independently of pool sleep, and compose stable-key final-report instructions.", + "is_new": true, + "required_symbols": [ + "pub(crate) struct RuntimeHold", + "pub fn queue_finished_background_work", + "pub(super) fn followup_prompt" + ], + "invocation_seams": [ + { + "caller_path": "crates/buzz-acp/src/pool.rs", + "callee_symbol": "followup_prompt", + "invocation": "background_work::followup_prompt(batch.channel_id, continuation, &ctx.cwd)", + "behavior_test": "background_managed_delivery_key_stays_on_collection_across_error_and_new_message", + "verification_command": "env -u BUZZ_ACP_SESSION_POLICY -u BUZZ_ACP_ALLOWED_RESPOND_TO -u BUZZ_ACP_LAZY_POOL -u BUZZ_ACP_IDLE_POOL_SLEEP cargo test -p buzz-acp background_managed_delivery_key_stays_on_collection_across_error_and_new_message -- --test-threads=1", + "coverage": "Stable managed key and original route in the real collection prompt; bundled process coverage is a mock-relay workflow." + } + ], + "verification_commands": [ + "env -u BUZZ_ACP_SESSION_POLICY -u BUZZ_ACP_ALLOWED_RESPOND_TO -u BUZZ_ACP_LAZY_POOL -u BUZZ_ACP_IDLE_POOL_SLEEP cargo test -p buzz-acp background_work_tests -- --test-threads=1" + ] + }, + { + "path": "crates/buzz-acp/src/pool/background_work_tests.rs", + "responsibility": "Production-dispatch local-process regressions for route/session retention, collection progress under repeated preemption, cancellation, rotation and lost-task cleanup.", + "is_new": true, + "required_symbols": [ + "background_collection_completes_under_repeated_ordinary_preemption", + "background_explicit_cancel_after_ordinary_control_never_resumes_collection" + ], + "invocation_seams": [], + "verification_commands": [ + "env -u BUZZ_ACP_SESSION_POLICY -u BUZZ_ACP_ALLOWED_RESPOND_TO -u BUZZ_ACP_LAZY_POOL -u BUZZ_ACP_IDLE_POOL_SLEEP cargo test -p buzz-acp background_work_tests -- --test-threads=1" + ] + }, + { + "path": "crates/buzz-acp/src/pool/delivery_runtime_tests.rs", + "responsibility": "Run actual bundled agent, MCP and CLI processes against a signature-checking mock relay: application-error replay, lost acknowledgments, process restart and concurrency; not real-relay evidence.", + "is_new": true, + "required_symbols": [ + "managed_runtime_bundled_agent_publication_then_application_error_retries_one_event", + "managed_runtime_killed_after_relay_acceptance_replays_on_new_process" + ], + "invocation_seams": [], + "verification_commands": [ + "BUZZ_TEST_BIN_DIR=\"$PWD/target/debug\" env -u BUZZ_ACP_SESSION_POLICY -u BUZZ_ACP_ALLOWED_RESPOND_TO -u BUZZ_ACP_LAZY_POOL -u BUZZ_ACP_IDLE_POOL_SLEEP cargo test -p buzz-acp managed_runtime_ -- --ignored --test-threads=1" + ] + }, + { + "path": "crates/buzz-acp/src/pool/pi_prompt_tests.rs", + "responsibility": "Inherited integration inventory: Pi ACP system-prompt metadata, workspace skill installation and restore compatibility fixtures.", + "is_new": false, + "required_symbols": [ + "pi_composed_prompt_uses_meta_without_capability_negotiation", + "real_pi_preserves_buzz_prompt_and_launch_skills_on_restore" + ], + "invocation_seams": [], + "verification_commands": [ + "env -u BUZZ_ACP_SESSION_POLICY -u BUZZ_ACP_ALLOWED_RESPOND_TO -u BUZZ_ACP_LAZY_POOL -u BUZZ_ACP_IDLE_POOL_SLEEP cargo test -p buzz-acp pi_prompt_tests -- --test-threads=1" + ] + }, + { + "path": "crates/buzz-acp/src/process_tree.rs", + "responsibility": "Inherited integration inventory: bounded process snapshots and resident-helper classification keep live task children out of idle-pool teardown.", + "is_new": true, + "required_symbols": [ + "pub fn snapshot()", + "pub fn blocking_pids(", + "pub fn is_resident_helper(" + ], + "invocation_seams": [ + { + "caller_path": "crates/buzz-acp/src/acp.rs", + "callee_symbol": "snapshot", + "invocation": "let tree = crate::process_tree::snapshot()?;", + "behavior_test": "prompt_work_is_tracked_under_worker_and_resident_provider", + "verification_command": "env -u BUZZ_ACP_SESSION_POLICY -u BUZZ_ACP_ALLOWED_RESPOND_TO -u BUZZ_ACP_LAZY_POOL -u BUZZ_ACP_IDLE_POOL_SLEEP cargo test -p buzz-acp prompt_work_is_tracked_under_worker_and_resident_provider -- --test-threads=1", + "coverage": "Inherited production process classification exercised with real local ACP children; no new runtime mutation claim." + } + ], + "verification_commands": [ + "env -u BUZZ_ACP_SESSION_POLICY -u BUZZ_ACP_ALLOWED_RESPOND_TO -u BUZZ_ACP_LAZY_POOL -u BUZZ_ACP_IDLE_POOL_SLEEP cargo test -p buzz-acp process_tree -- --test-threads=1" + ] + }, + { + "path": "crates/buzz-acp/src/process_tree/snapshot_tests.rs", + "responsibility": "Inherited integration inventory: process-list timeout/output bounds, malformed snapshots and zombie exclusion tests.", + "is_new": true, + "required_symbols": [ + "snapshot_rejects_partial_output_and_ignores_zombies", + "listing_timeout_kills_a_stalled_command" + ], + "invocation_seams": [], + "verification_commands": [ + "env -u BUZZ_ACP_SESSION_POLICY -u BUZZ_ACP_ALLOWED_RESPOND_TO -u BUZZ_ACP_LAZY_POOL -u BUZZ_ACP_IDLE_POOL_SLEEP cargo test -p buzz-acp process_tree -- --test-threads=1" + ] + }, + { + "path": "crates/buzz-acp/src/queue/background.rs", + "responsibility": "Keep background collection separate from ordinary messages with stable delivery UUID, original thread/session ownership, finite retries and bounded ordinary preemption.", + "is_new": true, + "required_symbols": [ + "pub(crate) struct RuntimeContinuation", + "pub ordinary_preempted: bool", + "background_managed_delivery_key_stays_on_collection_across_error_and_new_message" + ], + "invocation_seams": [ + { + "caller_path": "crates/buzz-acp/src/queue.rs", + "callee_symbol": "enqueue_continuation", + "invocation": "self.enqueue_continuation(batch.scope, continuation);", + "behavior_test": "background_collection_completes_under_repeated_ordinary_preemption", + "verification_command": "env -u BUZZ_ACP_SESSION_POLICY -u BUZZ_ACP_ALLOWED_RESPOND_TO -u BUZZ_ACP_LAZY_POOL -u BUZZ_ACP_IDLE_POOL_SLEEP cargo test -p buzz-acp background_collection_completes_under_repeated_ordinary_preemption -- --test-threads=1", + "coverage": "A2_RED.log and A2_BACKGROUND_GREEN.log preserve behavioral RED/GREEN for bounded ordinary interruption; text seam mutation is separate evidence." + } + ], + "verification_commands": [ + "env -u BUZZ_ACP_SESSION_POLICY -u BUZZ_ACP_ALLOWED_RESPOND_TO -u BUZZ_ACP_LAZY_POOL -u BUZZ_ACP_IDLE_POOL_SLEEP cargo test -p buzz-acp background -- --test-threads=1" + ] + }, + { + "path": "crates/buzz-cli/Cargo.toml", + "responsibility": "Add filesystem locking and temporary-fixture dependencies for durable managed-report journals.", + "is_new": false, + "required_symbols": [ + "fs2 = \"0.4\"" + ], + "invocation_seams": [], + "verification_commands": [ + "cargo test -p buzz-cli managed_ -- --test-threads=1" + ] + }, + { + "path": "crates/buzz-cli/TESTING.md", + "responsibility": "Specify managed publisher scope, journal durability and retention, recovery opt-in/upgrade requirements and real versus mock-relay checks.", + "is_new": false, + "required_symbols": [], + "invocation_seams": [], + "verification_commands": [ + "cargo test -p buzz-cli managed_ -- --test-threads=1", + "BUZZ_TEST_BIN_DIR=\"$PWD/target/debug\" cargo test -p buzz-relay --lib handlers::ingest::postgres_tests::managed_delivery_tests:: -- --ignored --test-threads=1" + ] + }, + { + "path": "crates/buzz-cli/src/commands/delivery.rs", + "responsibility": "Lock, validate and fsync a bounded original-report journal before submission; refresh signed recovery stamps and persist only verified canonical receipts while preserving original evidence.", + "is_new": true, + "required_symbols": [ + "pub(super) async fn deliver(", + "fn acknowledgment(", + "fn durable_directory(", + "recovery_candidate", + "canonical_event" + ], + "invocation_seams": [ + { + "caller_path": "crates/buzz-cli/src/commands/messages.rs", + "callee_symbol": "deliver", + "invocation": "super::delivery::deliver(\n client,\n &channel,\n &reply_to,\n &content,\n &delivery_key,\n journal_dir.as_deref(),\n mentions,\n )", + "behavior_test": "managed_same_key_changed_text_returns_first_signed_delivery", + "verification_command": "cargo test -p buzz-cli managed_same_key_changed_text_returns_first_signed_delivery -- --test-threads=1", + "coverage": "Actual parsed messages deliver dispatcher; signature-checking mock-relay test." + }, + { + "caller_path": "crates/buzz-cli/src/commands/delivery.rs", + "callee_symbol": "submit_event", + "invocation": "write(&dir, &journal)?;\n let result = client\n .submit_event(submission)", + "behavior_test": "managed_same_key_changed_text_returns_first_signed_delivery", + "verification_command": "cargo test -p buzz-cli managed_same_key_changed_text_returns_first_signed_delivery -- --test-threads=1", + "coverage": "Actual CLI publisher must submit once and same-key replay must return the first signed ID and stored body." + } + ], + "verification_commands": [ + "cargo test -p buzz-cli managed_ -- --test-threads=1", + "BUZZ_TEST_BIN_DIR=\"$PWD/target/debug\" cargo test -p buzz-relay --lib handlers::ingest::postgres_tests::managed_delivery_tests:: -- --ignored --test-threads=1" + ] + }, + { + "path": "crates/buzz-cli/src/commands/messages.rs", + "responsibility": "Expose messages deliver and preserve original route, stable UUID and signed callback mentions without suppressing ordinary progress or explicit edits.", + "is_new": false, + "required_symbols": [ + "MessagesCmd::Deliver {", + "pub(super) fn event_mention_pubkeys(", + "async fn resolve_content_mentions(" + ], + "invocation_seams": [ + { + "caller_path": "crates/buzz-cli/src/commands/delivery.rs", + "callee_symbol": "event_mention_pubkeys", + "invocation": "messages::event_mention_pubkeys(&chosen)", + "behavior_test": "managed_recovery_mention_history_survives_lost_ack", + "verification_command": "BUZZ_TEST_BIN_DIR=\"$PWD/target/debug\" cargo test -p buzz-relay --lib managed_recovery_mention_history_survives_lost_ack -- --ignored --test-threads=1", + "coverage": "Actual CLI stdout must return the elected signed recipient in mention_pubkeys after unconfirmed recovery replay; stored p-tags alone are insufficient." + } + ], + "verification_commands": [ + "cargo test -p buzz-cli managed_ -- --test-threads=1", + "cargo test -p buzz-cli mention -- --test-threads=1" + ] + }, + { + "path": "crates/buzz-cli/src/commands/mod.rs", + "responsibility": "Wire the managed delivery module into the CLI message dispatcher.", + "is_new": false, + "required_symbols": [ + "mod delivery;" + ], + "invocation_seams": [], + "verification_commands": [ + "cargo test -p buzz-cli managed_ -- --test-threads=1" + ] + }, + { + "path": "crates/buzz-cli/src/delivery_tests.rs", + "responsibility": "Production CLI regressions for same-key replay, namespace/route integrity, concurrency, durable ancestor links, bounded retention/retries and legacy wrapper rejection.", + "is_new": true, + "required_symbols": [ + "managed_same_key_changed_text_returns_first_signed_delivery", + "managed_expired_journal_never_treats_legacy_wrapper_ack_as_delivery", + "managed_existing_nested_directory_requires_ancestor_link_sync_before_publication" + ], + "invocation_seams": [], + "verification_commands": [ + "cargo test -p buzz-cli managed_ -- --test-threads=1" + ] + }, + { + "path": "crates/buzz-cli/src/error.rs", + "responsibility": "Expose managed-delivery pending as a safe same-key retry while preserving existing ambiguous unjournaled-send errors.", + "is_new": false, + "required_symbols": [ + "ManagedDeliveryPending" + ], + "invocation_seams": [], + "verification_commands": [ + "cargo test -p buzz-cli managed_ -- --test-threads=1" + ] + }, + { + "path": "crates/buzz-cli/src/lib.rs", + "responsibility": "Define messages deliver arguments and compile production-publisher regression tests.", + "is_new": false, + "required_symbols": [ + "Deliver {", + "mod delivery_tests;" + ], + "invocation_seams": [], + "verification_commands": [ + "cargo test -p buzz-cli managed_ -- --test-threads=1" + ] + }, + { + "path": "crates/buzz-core/src/kind.rs", + "responsibility": "Reserve ephemeral managed recovery kind 20003 for signed canonical election; upgraded relays route it through shared ingest.", + "is_new": false, + "required_symbols": [ + "KIND_MANAGED_DELIVERY_RECOVERY: u32 = 20003" + ], + "invocation_seams": [ + { + "caller_path": "crates/buzz-relay/src/handlers/event.rs", + "callee_symbol": "is_ephemeral", + "invocation": "if is_ephemeral(kind_u32)\n && kind_u32 != buzz_core::kind::KIND_NIP43_LEAVE_REQUEST\n && kind_u32 != buzz_core::kind::KIND_MANAGED_DELIVERY_RECOVERY", + "behavior_test": "managed_ws_original_ack_correlates_after_recovery_wins", + "verification_command": "BUZZ_TEST_BIN_DIR=\"$PWD/target/debug\" cargo test -p buzz-relay --lib managed_ws_original_ack_correlates_after_recovery_wins -- --ignored --test-threads=1", + "coverage": "Real WS handler keeps recovery out of ephemeral fanout and correlates original EVENT acknowledgment after another canonical event wins." + } + ], + "verification_commands": [ + "BUZZ_TEST_BIN_DIR=\"$PWD/target/debug\" cargo test -p buzz-relay --lib handlers::ingest::postgres_tests::managed_delivery_tests:: -- --ignored --test-threads=1" + ] + }, + { + "path": "crates/buzz-core/src/lib.rs", + "responsibility": "Export the shared signed managed-report recovery protocol.", + "is_new": false, + "required_symbols": [ + "pub mod managed_delivery;" + ], + "invocation_seams": [], + "verification_commands": [ + "cargo test -p buzz-core managed_delivery -- --test-threads=1" + ] + }, + { + "path": "crates/buzz-core/src/managed_delivery.rs", + "responsibility": "Validate author-signed original report and compact candidate stamp, immutable body/tags/route, canonical UUID and current wrapper/candidate timestamps.", + "is_new": true, + "required_symbols": [ + "pub struct RecoveryRequest", + "pub struct ReportStamp", + "pub fn delivery_key(", + "pub fn same_report(", + "pub fn decode(" + ], + "invocation_seams": [ + { + "caller_path": "crates/buzz-relay/src/handlers/ingest.rs", + "callee_symbol": "decode", + "invocation": "buzz_core::managed_delivery::RecoveryRequest::decode(\n &event,\n nostr::Timestamp::now().as_secs(),\n )", + "behavior_test": "managed_old_absent_report_recovers_with_production_writer", + "verification_command": "BUZZ_TEST_BIN_DIR=\"$PWD/target/debug\" cargo test -p buzz-relay --lib managed_old_absent_report_recovers_with_production_writer -- --ignored --test-threads=1", + "coverage": "B2_ATOMIC_COMMAND_RED.log and B2_STAMP_GUARD_FINAL.log preserve real shared-ingest recovery RED/GREEN with armed writer floor." + } + ], + "verification_commands": [ + "cargo test -p buzz-core managed_delivery -- --test-threads=1" + ] + }, + { + "path": "crates/buzz-db/src/error.rs", + "responsibility": "Return an explicit canonical delivery conflict for UUID reuse with another report or route.", + "is_new": false, + "required_symbols": [ + "ManagedDeliveryConflict" + ], + "invocation_seams": [], + "verification_commands": [ + "BUZZ_TEST_BIN_DIR=\"$PWD/target/debug\" cargo test -p buzz-relay --lib handlers::ingest::postgres_tests::managed_delivery_tests:: -- --ignored --test-threads=1" + ] + }, + { + "path": "crates/buzz-db/src/store/event.rs", + "responsibility": "Route ordinary marked kind-9 inserts through the same atomic canonical election as recovery while retaining transactional thread metadata and the production commit-time floor.", + "is_new": false, + "required_symbols": [ + "crate::store::managed_delivery::insert(" + ], + "invocation_seams": [ + { + "caller_path": "crates/buzz-db/src/store/event.rs", + "callee_symbol": "insert", + "invocation": "crate::store::managed_delivery::insert(\n pool,\n community_id,\n event,\n channel,\n thread_meta,\n None,\n )", + "behavior_test": "managed_recovery_and_original_race_share_one_election", + "verification_command": "BUZZ_TEST_BIN_DIR=\"$PWD/target/debug\" cargo test -p buzz-relay --lib managed_recovery_and_original_race_share_one_election -- --ignored --test-threads=1", + "coverage": "Ordinary kind-9 writer participates in recovery election; text fingerprint cannot alone prove lock/transaction behavior." + } + ], + "verification_commands": [ + "BUZZ_TEST_BIN_DIR=\"$PWD/target/debug\" cargo test -p buzz-relay --lib handlers::ingest::postgres_tests::managed_delivery_tests:: -- --ignored --test-threads=1" + ] + }, + { + "path": "crates/buzz-db/src/store/managed_delivery.rs", + "responsibility": "Elect once per community/author/UUID under a transaction advisory lock; adopt accepted or soft-deleted originals, otherwise commit candidate, thread counters, callback mentions and durable binding together.", + "is_new": true, + "required_symbols": [ + "pub async fn insert(", + "pub async fn recover_managed_delivery(", + "pg_advisory_xact_lock", + "INSERT INTO managed_deliveries" + ], + "invocation_seams": [ + { + "caller_path": "crates/buzz-relay/src/handlers/ingest.rs", + "callee_symbol": "recover_managed_delivery", + "invocation": ".recover_managed_delivery(\n tenant.community(),\n &event,\n channel,\n thread_params,\n &request.original,\n )", + "behavior_test": "managed_old_accepted_report_recovers_with_production_writer", + "verification_command": "BUZZ_TEST_BIN_DIR=\"$PWD/target/debug\" cargo test -p buzz-relay --lib managed_old_accepted_report_recovers_with_production_writer -- --ignored --test-threads=1", + "coverage": "Recovery must adopt the previously accepted original instead of inserting a second current candidate; ordinary insertion is not an equivalent replacement. Deleted-original adoption is independently exercised." + }, + { + "caller_path": "crates/buzz-db/src/store/managed_delivery.rs", + "callee_symbol": "insert_mentions_in_transaction", + "invocation": "crate::runtime::insert_mentions_in_transaction(\n &mut tx,\n community,\n &result.0.event,\n Some(channel),\n )", + "behavior_test": "managed_recovery_mention_history_survives_lost_ack", + "verification_command": "BUZZ_TEST_BIN_DIR=\"$PWD/target/debug\" cargo test -p buzz-relay --lib managed_recovery_mention_history_survives_lost_ack -- --ignored --test-threads=1", + "coverage": "B3_MENTIONS_RED.log and B3_MENTIONS_GREEN.log preserve callback Home Mentions RED/GREEN; mentions commit with elected report." + } + ], + "verification_commands": [ + "BUZZ_TEST_BIN_DIR=\"$PWD/target/debug\" cargo test -p buzz-relay --lib handlers::ingest::postgres_tests::managed_delivery_tests:: -- --ignored --test-threads=1" + ] + }, + { + "path": "crates/buzz-db/src/store/mod.rs", + "responsibility": "Register the canonical managed-delivery storage module.", + "is_new": false, + "required_symbols": [ + "pub mod managed_delivery;" + ], + "invocation_seams": [], + "verification_commands": [ + "BUZZ_TEST_BIN_DIR=\"$PWD/target/debug\" cargo test -p buzz-relay --lib handlers::ingest::postgres_tests::managed_delivery_tests:: -- --ignored --test-threads=1" + ] + }, + { + "path": "crates/buzz-db/tests/observability_source.rs", + "responsibility": "Repair stale aggregate-cardinality and bounded-helper assumptions while retaining typed Authorization/EventWrite checkout and direct-pool bypass guards.", + "is_new": false, + "required_symbols": [ + "p0_pool_acquisitions_use_typed_operation_pairs_without_other" + ], + "invocation_seams": [], + "verification_commands": [ + "cargo test -p buzz-db --test observability_source" + ] + }, + { + "path": "crates/buzz-relay/src/handlers/ingest/postgres_tests/managed_delivery_tests.rs", + "responsibility": "Use the armed production writer and authenticated CLI/router to test absent/accepted/deleted recovery, races, WS acknowledgment correlation, escaped-size limits and callback Home Mentions after lost acknowledgments.", + "is_new": true, + "required_symbols": [ + "managed_recovery_and_original_race_share_one_election", + "managed_ws_original_ack_correlates_after_recovery_wins", + "managed_recovery_mention_history_survives_lost_ack", + "buzz.created_at_floor" + ], + "invocation_seams": [], + "verification_commands": [ + "BUZZ_TEST_BIN_DIR=\"$PWD/target/debug\" cargo test -p buzz-relay --lib handlers::ingest::postgres_tests::managed_delivery_tests:: -- --ignored --test-threads=1" + ] + }, + { + "path": "crates/buzz-relay/src/handlers/ingest/timestamp.rs", + "responsibility": "Keep the future ceiling and ordinary age checks; permit old self-marked reports to acknowledge retained acceptance while absent old inserts remain subject to the 960-second writer floor.", + "is_new": true, + "required_symbols": [ + "pub(super) fn validate(", + "fn managed_report(", + "managed_timestamp_exception_preserves_future_and_marker_authority_boundaries" + ], + "invocation_seams": [ + { + "caller_path": "crates/buzz-relay/src/handlers/ingest.rs", + "callee_symbol": "validate", + "invocation": "timestamp::validate(&event, &auth, nostr::Timestamp::now().as_secs())?;", + "behavior_test": "managed_old_report_still_enforces_signature_route_membership_and_archive", + "verification_command": "BUZZ_TEST_BIN_DIR=\"$PWD/target/debug\" cargo test -p buzz-relay --lib managed_old_report_still_enforces_signature_route_membership_and_archive -- --ignored --test-threads=1", + "coverage": "A signed ordinary future message must fail shared ingest with timestamp admission rejection before storage, in addition to old-report authority guards." + } + ], + "verification_commands": [ + "cargo test -p buzz-relay --lib timestamp::tests -- --test-threads=1", + "BUZZ_TEST_BIN_DIR=\"$PWD/target/debug\" cargo test -p buzz-relay --lib handlers::ingest::postgres_tests::managed_delivery_tests:: -- --ignored --test-threads=1" + ] + }, + { + "path": "docs/superpowers/plans/2026-10-14-background-result-atomic-recovery.md", + "responsibility": "Record the owner-approved atomic canonical recovery plan and preserved replica-floor safety contract.", + "is_new": true, + "required_symbols": [], + "invocation_seams": [], + "verification_commands": [ + "python3 scripts/fork/sync.py validate" + ] + }, + { + "path": "docs/superpowers/plans/2026-10-14-background-result-delivery.md", + "responsibility": "Record original-session collection and durable managed publication implementation/verification plan.", + "is_new": true, + "required_symbols": [], + "invocation_seams": [], + "verification_commands": [ + "python3 scripts/fork/sync.py validate" + ] + }, + { + "path": "docs/superpowers/reports/2026-10-14-background-result-delivery.md", + "responsibility": "Preserve implementation scope, actual validation receipts, mutation evidence and remaining runtime/acceptance limits.", + "is_new": true, + "required_symbols": [], + "invocation_seams": [], + "verification_commands": [ + "python3 scripts/fork/sync.py validate" + ] + }, + { + "path": "docs/superpowers/specs/2026-10-14-background-result-delivery-task.md", + "responsibility": "Inherited integration inventory: agreed background collection/publication requirements and explicit authorization boundaries; registered as task input, not newly authored implementation.", + "is_new": true, + "required_symbols": [], + "invocation_seams": [], + "verification_commands": [ + "python3 scripts/fork/sync.py validate" + ] + }, + { + "path": "migrations/0056_managed_report_delivery.sql", + "responsibility": "Create bounded canonical delivery bindings independent of event redaction with community write fencing and per-author completion-key uniqueness.", + "is_new": true, + "required_symbols": [ + "CREATE TABLE managed_deliveries", + "PRIMARY KEY (community_id, author, delivery_key)", + "SELECT attach_community_write_fence('managed_deliveries');" + ], + "invocation_seams": [], + "verification_commands": [ + "BUZZ_TEST_BIN_DIR=\"$PWD/target/debug\" cargo test -p buzz-relay --lib handlers::ingest::postgres_tests::managed_delivery_tests:: -- --ignored --test-threads=1" + ] + }, + { + "path": "desktop/src/shared/api/relayReconnectReplay.test.mjs", + "responsibility": "Keep channel replay lookback coupled to the shared relay timestamp helper and production ingest wiring plus the unchanged DB floor/clock margin.", + "is_new": false, + "required_symbols": [ + "channel replay lookback stays coupled to relay and DB source constants" + ], + "invocation_seams": [], + "verification_commands": [ + "pnpm -C desktop test" + ] + }, + { + "path": "scripts/check-postgres-test-discovery.py", + "responsibility": "Resolve conventional out-of-line Rust tests inside inline postgres_tests modules, retaining ancestor discovery context.", + "is_new": false, + "required_symbols": [ + "inline_ranges = module_ranges(source)" + ], + "invocation_seams": [], + "verification_commands": [ + "bash scripts/test-postgres-test-discovery.sh" + ] + }, + { + "path": "scripts/test-postgres-test-discovery.sh", + "responsibility": "Exercise nested file and directory Rust modules in the PostgreSQL discovery contract.", + "is_new": false, + "required_symbols": [ + "nested_out_of_line_recovery_test" + ], + "invocation_seams": [], + "verification_commands": [ + "bash scripts/test-postgres-test-discovery.sh" + ] + }, + { + "path": "deny.toml", + "responsibility": "Track the owner-approved RUSTSEC-2024-0449 exception for fork-v0.5.26-4 only, with YAML replacement issue19 and mandatory reassessment before another release. Other advisory policies remain unchanged.", + "is_new": false, + "required_symbols": [ + "RUSTSEC-2024-0449", + "https://github.com/reynholm/buzz/issues/19" + ], + "invocation_seams": [], + "verification_commands": [ + "cargo-deny check" + ] + }, + { + "path": "crates/buzz-relay/src/mesh_boot.rs", + "responsibility": "Preserve one in-flight demo echo receive across drain ticks; retain bounded explicit shutdown and the existing validated frame/Goodbye handling.", + "is_new": false, + "required_symbols": [ + "tokio::pin!(receive);", + "demo_echo_keeps_inflight_read_across_ticks_and_still_drains" + ], + "invocation_seams": [], + "verification_commands": [ + "cargo test -p buzz-relay -- --test-threads=1" + ] } ] } diff --git a/scripts/run-tests.sh b/scripts/run-tests.sh index 84061935ca5..f374eefe4a5 100755 --- a/scripts/run-tests.sh +++ b/scripts/run-tests.sh @@ -300,6 +300,9 @@ run_unit_tests() { cargo test -p buzz-relay --lib "$name" -- --exact --nocapture done + run_test_step "buzz-relay demo echo receive retention" \ + cargo test -p buzz-relay --lib mesh_boot::tests::demo_echo_keeps_inflight_read_across_ticks_and_still_drains -- --exact --nocapture + run_test_step "buzz-relay binary tests" \ cargo test -p buzz-relay --bin buzz-relay -- --nocapture } diff --git a/scripts/test-postgres-test-discovery.sh b/scripts/test-postgres-test-discovery.sh index 8b6b6c32183..75595f033b4 100755 --- a/scripts/test-postgres-test-discovery.sh +++ b/scripts/test-postgres-test-discovery.sh @@ -43,6 +43,27 @@ mod postgres_tests; #[cfg(test)] mod standard; + +#[cfg(test)] +mod nested { + mod postgres_tests { + mod recovery; + mod directory; + } +} +RS + +mkdir -p "$fixture_root/src/nested/postgres_tests/directory" +cat >"$fixture_root/src/nested/postgres_tests/recovery.rs" <<'RS' +#[test] +#[ignore = "requires PostgreSQL and Redis"] +fn nested_out_of_line_recovery_test() {} +RS + +cat >"$fixture_root/src/nested/postgres_tests/directory/mod.rs" <<'RS' +#[test] +#[ignore = "requires PostgreSQL"] +fn nested_directory_module_test() {} RS cat >"$fixture_root/src/out_of_line.rs" <<'RS' @@ -64,6 +85,7 @@ RS python3 "$checker" "$fixture_root" python3 "$checker" "$fixture_root/src/out_of_line.rs" python3 "$checker" "$fixture_root/src/standard/postgres_tests.rs" +python3 "$checker" "$fixture_root/src/nested/postgres_tests/recovery.rs" packages="$("$repo_root/scripts/postgres-test-packages.sh" "$fixture_root")" if [[ "$packages" != "postgres-discovery-fixture" ]]; then @@ -125,6 +147,26 @@ grep -q "postgres_only_test_cannot_hide_under_external_infra" \ "$fixture_root/external-postgres.out" rm "$fixture_root/src/external_postgres_only.rs" +mkdir -p "$fixture_root/src/external_parent/postgres_tests/external_infra_tests" +cat >"$fixture_root/src/external_parent.rs" <<'RS' +mod postgres_tests { + mod external_infra_tests { + mod cases; + } +} +RS +cat >"$fixture_root/src/external_parent/postgres_tests/external_infra_tests/cases.rs" <<'RS' +#[test] +#[ignore = "requires PostgreSQL"] +fn external_ancestor_cannot_hide_postgres_test() {} +RS +if python3 "$checker" "$fixture_root" >"$fixture_root/external-ancestor.out" 2>&1; then + echo "expected an out-of-line PostgreSQL test under external infra to fail" >&2 + exit 1 +fi +grep -q "external_ancestor_cannot_hide_postgres_test" "$fixture_root/external-ancestor.out" +rm -r "$fixture_root/src/external_parent" "$fixture_root/src/external_parent.rs" + cat >"$fixture_root/src/bare_ignore.rs" <<'RS' #[cfg(test)] mod postgres_tests {