test: private-mesh overlay functional harness

.agents/skills/overlay-functional/SKILL.md

@@ -0,0 +1,23 @@
+---
+name: overlay-functional
+description: >-
+  Run or fix the private-mesh overlay harness (Tor chutney-equivalent,
+  i2pd netid, cjdns TUN). Use when a PR is labeled overlay-functional, a
+  ship version-bump needs that job, or scripts/overlay-functional is red.
+  Do not treat this harness as the default PR pin.
+---
+
+# Overlay functional harness
+
+Owner: [`docs/overlay-functional.md`](../../../docs/overlay-functional.md).
+Default `cargo test` is the PR pin ([`TESTING.md`](../../../TESTING.md)).
+Fake SOCKS / control / SAM stay in unlabeled product PRs. Label this
+harness and every ship version-bump PR.
+
+When the labeled job is red, reproduce locally. Do not push and wait on
+GitHub Actions as the inner loop.
+
+```bash
+export CARGO_TARGET_DIR="${CARGO_TARGET_DIR:-$PWD/target/dev}"
+./scripts/overlay-functional/run.sh <filter>
+```

.agents/skills/ship-pr/SKILL.md

@@ -82,6 +82,7 @@ Suite and budgets: [`TESTING.md`](../../../TESTING.md).
 | After Green | `cargo test --workspace --quiet` |
 | After Refactor, before commit | Local CI except coverage (below) |
 | Core functional CI red | [core-functional skill](../core-functional/SKILL.md) |
+| Overlay functional CI red | [overlay-functional skill](../overlay-functional/SKILL.md) |
 | Never local | `./scripts/coverage.sh`, `nix build .#rbitcoin-musl`, host IBD |
 
 Redirect every command above (stdout and stderr to `/tmp`, print `EXIT`,
@@ -120,6 +121,9 @@ not operator zips.
 
 Label `core-functional` when the PR touches that harness, and on every ship
 version-bump PR. Do not label ordinary net or RPC PRs.
+Label `overlay-functional` when the PR touches that harness, and on every
+ship version-bump PR. Poll with `--interest overlay-functional`. It is not
+a required check.
 Label `nixos-module-runtime` when the NixOS module VM test should run (not
 eval). Poll with `--interest nixos-module-runtime`. It is not a required
 check.

.github/workflows/ci.yml

@@ -127,6 +127,8 @@ jobs:
         run: ./scripts/ci-os-smoke.test.sh
       - name: pr-checks-watch self-test
         run: ./scripts/pr-checks-watch.test.sh
+      - name: overlay-functional runner self-test
+        run: ./scripts/overlay-functional/run.sh.test.sh
       - name: cargo test
         run: cargo test --workspace
 

.github/workflows/overlay-functional.yml

@@ -0,0 +1,82 @@
+# Overlay functional gate: nightly + dispatch + ship PRs + labels.
+# Unlabeled non-ship PRs do not start the heavy job. Default cargo test
+# never calls run.sh.
+name: overlay-functional
+
+on:
+  schedule:
+    - cron: "42 6 * * *"
+  workflow_dispatch:
+  pull_request:
+    types: [opened, synchronize, reopened, labeled, unlabeled]
+
+concurrency:
+  group: overlay-functional-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+permissions:
+  contents: read
+
+jobs:
+  detect:
+    name: detect-ship
+    runs-on: ubuntu-latest
+    outputs:
+      ship: ${{ steps.v.outputs.ship }}
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        if: github.event_name == 'pull_request'
+        with:
+          persist-credentials: false
+      - name: Classify ship vs .99
+        id: v
+        run: |
+          set -euo pipefail
+          if [[ "${{ github.event_name }}" != "pull_request" ]]; then
+            echo "ship=false" >> "$GITHUB_OUTPUT"
+            exit 0
+          fi
+          kind="$(./scripts/release-gate.sh --kind)"
+          echo "kind=$kind" >> "$GITHUB_OUTPUT"
+          if [[ "$kind" == ship ]]; then
+            echo "ship=true" >> "$GITHUB_OUTPUT"
+          else
+            echo "ship=false" >> "$GITHUB_OUTPUT"
+          fi
+
+  overlay-functional:
+    name: overlay-functional
+    needs: [detect]
+    runs-on: ubuntu-latest
+    timeout-minutes: 45
+    if: >-
+      always() &&
+      needs.detect.result == 'success' &&
+      (github.event_name != 'pull_request' ||
+       needs.detect.outputs.ship == 'true' ||
+       contains(github.event.pull_request.labels.*.name, 'overlay-functional') ||
+       contains(github.event.pull_request.labels.*.name, 'release'))
+    env:
+      CARGO_TARGET_DIR: target/dev
+      RUSTFLAGS: -D warnings
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          persist-credentials: false
+      - uses: cachix/install-nix-action@13d8dd58da0234aa297dedd986986ccb8e7f3e24 # v31.11.1
+        with:
+          github_access_token: ${{ github.token }}
+          enable_kvm: false
+      - uses: nix-community/cache-nix-action@7df957e333c1e5da7721f60227dbba6d06080569 # v7
+        with:
+          primary-key: overlay-functional-${{ runner.os }}-${{ hashFiles('flake.lock') }}
+          restore-prefixes-first-match: overlay-functional-${{ runner.os }}-
+          gc-max-store-size-linux: 4G
+      - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2
+        with:
+          shared-key: overlay-functional
+          workspaces: ". -> target/dev"
+      - name: TUN
+        run: sudo modprobe tun
+      - name: overlay journeys
+        run: nix develop .#overlayFunctional --command ./scripts/overlay-functional/run.sh

OPERATOR.md

@@ -488,7 +488,10 @@ Omit ADDR for `127.0.0.1:7656`. Failed HELLO / `SESSION CREATE` is a start
 error. Unset: I2P rows may still load from `peers` v2 but are not dialed.
 `--only-net i2p` without `--i2p-sam` is a start error. `--i2p-accept-incoming`
 creates a persistent local destination (`{datadir}/i2p/p2p.priv`, 0600) and
-`STREAM FORWARD`s to the P2P bind. With `--listen=0` that is a start error
+`STREAM FORWARD`s to the P2P bind. That destination is published as
+`{52}.b32.i2p:0` (SAM 3.1 has no ports; Core refuses any other I2P port)
+on `getnetworkinfo.localaddresses` and in addrv2, including with
+`--no-discover`. With `--listen=0` that is a start error
 unless `--listen-onion` provides a loopback accept. NixOS:
 `services.rbitcoin.i2p.sam` / `i2p.acceptIncoming`; the unit `After`/`Wants`
 `i2pd.service` when SAM is set. Do not start i2pd from this module.

TESTING.md

@@ -18,6 +18,10 @@ Electrum TCP, Esplora HTTP, BIP324 P2P). Bitcoin Core’s Python functional
 suite is a **nightly / ship / `core-functional` label** oracle. It is **not**
 a required PR check and **not** required on PRs that merely touch net or RPC
 (too slow). Nightly Core is not a license to delete in-tree tests.
+Unlabeled PRs stay cargo-only for overlays too: live Tor / i2pd / cjdns
+meshes are the **`overlay-functional`** labeled / nightly / ship job
+([`docs/overlay-functional.md`](docs/overlay-functional.md)), not default
+`cargo test`.
 
 When adding or folding a pin:
 
@@ -497,6 +501,10 @@ suite. A red labeled run is reproduced locally with
 `./scripts/core-functional/run.sh <failing.py>` until that script passes
 ([`docs/core-functional.md`](docs/core-functional.md)); do not push-and-wait
 on that job as the inner loop.
+Label **`overlay-functional`** on harness PRs and on **ship** version-bump
+PRs. It is **not** a required PR check. A red labeled run is reproduced
+locally with `./scripts/overlay-functional/run.sh <filter>`
+([`docs/overlay-functional.md`](docs/overlay-functional.md)).
 
 ```bash
 python3 scripts/core-functional/check_inventory.py
@@ -511,4 +519,6 @@ python3 scripts/core-functional/check_inventory.py
 # cargo test stages Core JSON from the submodule:
 ./scripts/core-functional/init-submodule.sh
 ./scripts/core-functional/sync-core-fixtures.sh --check
+./scripts/overlay-functional/run.sh.test.sh
+./scripts/overlay-functional/run.sh --list
 ```

crates/rbitcoin-net/src/ephemeral.rs

@@ -121,7 +121,15 @@ pub fn spawn_isolated_broadcast_loop(
                 Err(tokio::sync::broadcast::error::RecvError::Lagged(_)) => continue,
                 Err(tokio::sync::broadcast::error::RecvError::Closed) => break,
             };
-            let Some(tx) = mp.get_tx(&txid) else {
+            // `get_tx` refuses the reactor thread. This task is a tokio worker.
+            let tx = match mp.try_get_tx(&txid) {
+                Some(tx) => Some(tx),
+                None => {
+                    tokio::time::sleep(Duration::from_millis(10)).await;
+                    mp.try_get_tx(&txid)
+                }
+            };
+            let Some(tx) = tx else {
                 continue;
             };
             isolated_broadcast_known_tx(&dialer, &addrman, magic, &user_agent, txid, tx).await;
@@ -414,6 +422,30 @@ mod tests {
         let _ = std::fs::remove_dir_all(dir);
     }
 
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+    async fn isolated_broadcast_loop_reads_mempool_off_reactor() {
+        let (dir, hub) = crate::chain::tiny_regtest_hub_labeled("iso-reactor");
+        let mp = MempoolHub::open(dir.join("mp"), Arc::clone(&hub.query)).unwrap();
+        mp.set_isolated_broadcast(true);
+        let am = Arc::new(Mutex::new(AddrMan::new()));
+        let h = spawn_isolated_broadcast_loop(
+            mp.clone(),
+            Dialer::Direct,
+            am,
+            Magic::REGTEST,
+            "/rbitcoin:test/".into(),
+        );
+        tokio::time::sleep(Duration::from_millis(20)).await;
+        mp.mark_local_origin(dummy_tx().compute_txid());
+        tokio::time::sleep(Duration::from_millis(80)).await;
+        assert!(
+            !h.is_finished(),
+            "broadcast task must not panic reading the mempool on a tokio worker"
+        );
+        h.abort();
+        let _ = std::fs::remove_dir_all(dir);
+    }
+
     #[tokio::test]
     async fn ephemeral_broadcast_loop_skips_unknown_txid() {
         let (dir, hub) = crate::chain::tiny_regtest_hub_labeled("iso-skip");