From 7689c05da09325119b88daed7bc00459a0c4d288 Mon Sep 17 00:00:00 2001 From: Warm Beer Date: Tue, 19 May 2026 19:34:58 +0200 Subject: [PATCH 01/18] feat(dht)!: add trust quarantine thresholds Add close-group quarantine below 0.20 with natural readmission at 0.45. Keep 0.35 as lazy swap eligibility, avoid quarantined peers in automatic lookups, and keep explicit sends unblocked. BREAKING CHANGE: AdaptiveDhtConfig now includes quarantine_threshold and quarantine_readmit_threshold fields. --- README.md | 8 +- docs/ROUTING_TABLE_DESIGN.md | 125 ++++++++--------- docs/SECURITY_MODEL.md | 35 ++--- docs/trust-signals-api.md | 31 +++-- src/adaptive/dht.rs | 106 +++++++++++++-- src/dht/core_engine.rs | 255 +++++++++++++++++++++++++++++++++++ src/dht_network_manager.rs | 222 ++++++++++++++++++++++++++++-- src/network.rs | 16 ++- tests/sybil_protection.rs | 7 +- tests/trust_flow.rs | 2 + 10 files changed, 683 insertions(+), 124 deletions(-) diff --git a/README.md b/README.md index 3627a7e3..a2d7396e 100644 --- a/README.md +++ b/README.md @@ -34,7 +34,7 @@ Key design decisions are documented in [docs/adr/](docs/adr/): - **DHT (Distributed Hash Table)**: Peer phonebook and routing with geographic awareness - **QUIC Transport**: High-performance networking with saorsa-transport - **Post-Quantum Cryptography**: Future-ready cryptographic algorithms (ML-DSA-65, ML-KEM-768) -- **Trust System**: Response-rate scoring with time decay and binary peer blocking +- **Trust System**: Response-rate scoring with time decay, lazy swap-out, and close-group quarantine ## Quick Start @@ -80,7 +80,7 @@ saorsa-core does **not** replicate application data. saorsa-node: 1. **Network Layer**: QUIC-based P2P networking with automatic NAT traversal (saorsa-transport 0.26) 2. **DHT**: Kademlia-based peer phonebook with geographic awareness -3. **Trust System**: Response-rate scoring with time decay and binary peer blocking +3. **Trust System**: Response-rate scoring with time decay, lazy swap-out, and close-group quarantine ### Cryptographic Architecture @@ -159,11 +159,11 @@ Saorsa Core implements defense-in-depth security designed for adversarial decent | Protection | Implementation | |------------|----------------| -| **Node Monitoring** | Automatic eviction after 3 consecutive failures | +| **Node Monitoring** | Trust-score quarantine for bad close-group peers | | **Reputation System** | Response-rate scoring with time decay | | **Sybil Resistance** | IP diversity limits (/64: 1, /48: 3, /32: 10, ASN: 20) | | **Geographic Diversity** | Regional diversity in routing | -| **Routing Validation** | Trust-based peer blocking and eviction | +| **Routing Validation** | Trust-based swap-out and close-group quarantine | ### Anti-Centralization diff --git a/docs/ROUTING_TABLE_DESIGN.md b/docs/ROUTING_TABLE_DESIGN.md index e2fdc1c8..a592da91 100644 --- a/docs/ROUTING_TABLE_DESIGN.md +++ b/docs/ROUTING_TABLE_DESIGN.md @@ -56,9 +56,11 @@ All parameters are configurable. Values below are a reference profile used for l | `IPV4_SUBNET_MASK` | Prefix length for IPv4 subnet grouping | `/24` | | `IPV6_SUBNET_MASK` | Prefix length for IPv6 subnet grouping | `/48` | | `TRUST_PROTECTION_THRESHOLD` | Trust score above which a peer resists swap-closer eviction | `0.7` | -| `BLOCK_THRESHOLD` | Trust score below which a peer is evicted and blocked | `0.15` | -| `EMA_ALPHA` | EMA smoothing factor — weight of each new observation (higher = faster response) | `0.3` | -| `DECAY_LAMBDA` | Per-second exponential decay rate toward neutral (0.5) | `4.198e-6` | +| `SWAP_THRESHOLD` | Trust score below which a peer is eligible for replacement when a better candidate needs the slot | `0.35` | +| `QUARANTINE_THRESHOLD` | Trust score below which automatic lookup/dial paths avoid the peer, and close-group peers are evicted immediately | `0.20` | +| `QUARANTINE_READMIT_THRESHOLD` | Trust score a quarantined peer must recover to before normal admission accepts it again | `0.45` | +| `EMA_ALPHA` | EMA smoothing factor — weight of each new observation (higher = faster response) | `0.124` | +| `DECAY_LAMBDA` | Per-second exponential decay rate toward neutral (0.5) | `1.394e-5` | | `SELF_LOOKUP_INTERVAL` | Periodic self-lookup cadence (maintenance phase only; bootstrap self-lookups run back-to-back with no interval) | random in `[5 min, 10 min]` | | `BUCKET_REFRESH_INTERVAL` | Periodic refresh cadence for stale k-buckets, randomised per cycle to break startup-lockstep across the network | random in `[7.5 min, 12.5 min]` | | `STALE_BUCKET_THRESHOLD` | Duration after which a bucket without activity is considered stale | `1 hour` | @@ -90,24 +92,22 @@ score = (1 - EMA_ALPHA)^W * score + (1 - (1 - EMA_ALPHA)^W) * observation This is equivalent to applying the unit-weight blend step `W` times when `W` is a positive integer, and extends naturally to fractional weights without ambiguity. -**Decay tuning**: `DECAY_LAMBDA = 4.198e-6` is tuned so that a peer experiencing ~3 evenly-spaced failures per day converges to the block threshold (0.15). The worst possible score (0.0) decays back above `BLOCK_THRESHOLD` in ~1 day. Derivation: at steady state with T = 28800 s between events, `0.15 = 0.5·(1 − d) / (1 − 0.7·d)` → `d = 0.8861` → `λ = −ln(0.8861) / 28800 ≈ 4.198e-6`. +**Decay tuning**: `DECAY_LAMBDA = 1.394e-5` is tuned so that a peer experiencing ~3 evenly-spaced failures per day converges to the swap threshold (0.35). The worst possible score (0.0) decays back above `SWAP_THRESHOLD` in ~1 day, and above the quarantine readmit threshold (0.45) after roughly 46 hours. -**Failures to block** (consecutive negative events from neutral 0.5 to below `BLOCK_THRESHOLD` 0.15, ignoring decay): +**Failures to cross thresholds** (consecutive unit-weight negative events from neutral 0.5, ignoring decay): -| Event weight | Events to block | Effective failures | -|---|---|---| -| `1.0` (internal event) | 4 | 4 | -| `2.0` | 2 | 4 | -| `3.0` | 2 | 6 | -| `5.0` (`MAX_CONSUMER_WEIGHT`) | 1 | 5 | +| Threshold | Unit failures | +|---|---| +| `SWAP_THRESHOLD` (`0.35`) | 3 | +| `QUARANTINE_THRESHOLD` (`0.20`) | 7 | -Note: time decay between events works in the peer's favor — in practice, more events may be needed if failures are spread over time. Core only records penalties (no internal success events), so the only counterforce is time decay. Higher weights are slightly less efficient per unit weight due to EMA non-linearity: at lower scores, each successive failure has diminishing marginal impact. The "Effective failures" column shows total weight applied (events × weight), not a count of equivalent unit-weight events. +Note: time decay between events works in the peer's favor — in practice, more events may be needed if failures are spread over time. Core only records penalties (no internal success events), so the only counterforce is time decay. Higher weights are slightly less efficient per unit weight due to EMA non-linearity: at lower scores, each successive failure has diminishing marginal impact. Parameter safety constraints (MUST hold): 1. `IP_EXACT_LIMIT >= 1`. 2. `IP_SUBNET_LIMIT >= 1`. -3. `TRUST_PROTECTION_THRESHOLD > BLOCK_THRESHOLD`. +3. `TRUST_PROTECTION_THRESHOLD > SWAP_THRESHOLD > QUARANTINE_THRESHOLD`. 4. `ALPHA >= 1`. 5. `LIVE_THRESHOLD > max(SELF_LOOKUP_INTERVAL)` (peers touched by self-lookup must not oscillate between live and stale between consecutive cycles; at reference values: 15 min > 10 min). The 5-minute margin at reference values is sufficient for typical network latencies (sub-second RTTs). Operators in high-latency environments (satellite, Tor overlay) SHOULD increase `LIVE_THRESHOLD` proportionally. 6. `STALE_REVALIDATION_TIMEOUT > 0`. @@ -129,7 +129,7 @@ Note: `K_BUCKET_SIZE` values below 4 produce degenerate behavior (single-peer ro 4. **Address requirement**: A `NodeInfo` with an empty address list MUST NOT be admitted to the routing table. 5. **Authenticated membership**: Only peers that have completed transport-level authentication are eligible for routing table insertion. Unauthenticated peers MUST NOT enter `LocalRT`. 6. **IP diversity**: No enforcement scope (per-bucket or routing-neighborhood) may exceed `IP_EXACT_LIMIT` nodes per exact IP or `IP_SUBNET_LIMIT` nodes per subnet, except via explicit loopback or testnet overrides. -7. **Trust blocking**: Peers with `TrustScore(self, P) < BLOCK_THRESHOLD` MUST be evicted from the routing table and MUST NOT be re-admitted until their trust score recovers above `BLOCK_THRESHOLD`. +7. **Trust quarantine**: Peers with `TrustScore(self, P) < QUARANTINE_THRESHOLD` MUST be skipped by local lookup result selection, FIND_NODE responses, and automatic lookup/dial candidate selection. If such a peer is in the K-closest-to-self set, it MUST be evicted and quarantined until `TrustScore(self, P) >= QUARANTINE_READMIT_THRESHOLD`. 8. **Trust protection (staleness-gated)**: A peer with `TrustScore(self, P) >= TRUST_PROTECTION_THRESHOLD` **AND** `last_seen` within `LIVE_THRESHOLD` MUST NOT be evicted by swap-closer admission. A peer whose `last_seen` exceeds `LIVE_THRESHOLD` receives no trust protection regardless of score — stale peers MUST NOT hold slots against live candidates. 9. **Deterministic distance**: `Distance(A, B)` is symmetric, deterministic, and consistent across all nodes. Two nodes compute the same distance between the same pair of keys. 10. **Atomic admission**: IP diversity checks, capacity checks, swap-closer evictions, trust score reads, and insertion MUST execute within a single write-locked critical section to prevent TOCTOU races. All `TrustScore` queries during admission (steps 4, 8) MUST occur while the routing table write lock is held. @@ -184,7 +184,7 @@ When a candidate peer `P` with `NodeInfo` and IP address `candidate_ip` is prese 1. **Self-check**: If `P.id == self.id`, reject. 2. **Address check**: If `P.addresses` is empty, reject. 3. **Authentication check**: If `P` has not completed transport-level authentication, reject. -4. **Trust block check**: If `TrustScore(self, P) < BLOCK_THRESHOLD`, reject. +4. **Trust quarantine check**: If `TrustScore(self, P) < QUARANTINE_THRESHOLD`, reject. If `P` was previously quarantined, reject until `TrustScore(self, P) >= QUARANTINE_READMIT_THRESHOLD`. 5. **Update short-circuit**: If `P` already exists in `KBucket(BucketIndex(self, P))`, merge addresses (Section 6.3), refresh `last_seen`, move `P` to tail, and return. The peer already holds its slot — IP diversity and capacity checks are skipped. 6. **Loopback check**: If `candidate_ip` is loopback and loopback is disallowed, reject. If loopback is allowed, skip all IP diversity checks (step 7–8) and proceed directly to step 9. 7. **Non-IP transport bypass**: If `P` has no IP-based address (e.g., Bluetooth, LoRa), skip IP diversity checks and proceed directly to step 9. @@ -209,7 +209,7 @@ When a candidate peer `P` with `NodeInfo` and IP address `candidate_ip` is prese - Stale peers in `KBucket(bucket_idx)` (bucket-level contention). - Stale routing-neighborhood violators identified in step 8c (if any). Release the write lock **once**, ping all collected stale peers in parallel (bounded by `STALE_REVALIDATION_TIMEOUT`), then re-acquire the write lock **once** and **re-evaluate the following checks** against the current routing table state: - - Trust block check (step 4): `TrustScore` may have changed during the unlocked window. + - Trust quarantine check (step 4): `TrustScore` may have changed during the unlocked window. - Per-bucket IP diversity (step 8b): bucket composition may have changed. - Routing-neighborhood IP diversity (step 8c): K-closest set may have changed. - Capacity pre-check (this step): slots may have been filled by concurrent admissions. @@ -252,29 +252,27 @@ When an IP diversity limit is exceeded and a candidate `P` contends for a slot: Rationale: swap-closer prefers geographically closer peers (lower XOR distance) while protecting long-lived, recently-seen, well-trusted peers from displacement by unproven newcomers from the same subnet. A peer that has not been seen within `LIVE_THRESHOLD` loses trust protection regardless of its score — it may have silently departed, and holding its slot against a live candidate degrades routing table quality. -### 7.4 Blocked Peer Handling - -When any interaction records a trust failure and `TrustScore(self, P)` drops below `BLOCK_THRESHOLD`: +### 7.4 Quarantined Peer Handling -1. Remove `P` from `LocalRT(self)`. -2. Disconnect `P` at the transport layer. - 2a. Cancel all in-flight RPCs to or from `P`. Cancelled operations do not record trust events — the eviction/blocking decision has already been made, and partial responses from a blocked peer should not influence trust state. The mechanism for distinguishing cancellation from genuine failure is an implementation choice, but MUST prevent cancelled RPCs from recording trust events. -3. Silently drop any incoming DHT messages from `P`. -4. Do not re-admit `P` until `TrustScore(self, P) >= BLOCK_THRESHOLD`. +When any interaction records a trust failure and `TrustScore(self, P)` drops below `QUARANTINE_THRESHOLD`: -Blocking is enforced at both the transport and routing table layers. API consumers can rely on `LocalRT` membership as the trust gate. +1. If `P` is in the K-closest-to-self set, remove `P` from `LocalRT(self)` and emit `PeerRemoved`. +2. Keep `P`'s trust record. Do not reset trust on eviction. +3. Mark `P` as quarantined if it was evicted from the close group. +4. Do not re-admit quarantined `P` until `TrustScore(self, P) >= QUARANTINE_READMIT_THRESHOLD`. +5. If `P` is not in the K-closest-to-self set, it may remain in the routing table, but local lookup result selection, FIND_NODE responses, and automatic lookup/dial paths MUST avoid it while `TrustScore(self, P) < QUARANTINE_THRESHOLD`. -Transport-level enforcement: the transport layer MUST query `TrustScore(self, P)` at authentication time and reject the connection if the score is below `BLOCK_THRESHOLD`. The transport MUST NOT rely solely on a cached block list, as peers may recover above `BLOCK_THRESHOLD` via time decay (see re-admission path below). The check occurs after the peer's identity is established but before allocating application-layer resources (buffers, session state, routing table interaction). The transport layer MUST also refuse outbound dials to blocked peers. +Quarantine is a routing-table and automatic lookup policy. It is not a blanket transport-level block for explicit user-initiated sends. -Re-admission path: a blocked peer can only re-enter when its trust score recovers above `BLOCK_THRESHOLD` through time-decay toward neutral AND the peer is rediscovered through normal network activity: +Re-admission path: a quarantined peer can only re-enter when its trust score recovers above `QUARANTINE_READMIT_THRESHOLD` through time-decay toward neutral AND the peer is rediscovered through normal network activity: -1. Peer `P` is returned in a `FIND_NODE` response from another peer during a lookup. -2. Local node checks `TrustScore(self, P)`. If still below `BLOCK_THRESHOLD`, `P` is silently skipped (not dialed). -3. If trust has recovered above `BLOCK_THRESHOLD`, local node dials `P`, authentication completes, and the standard admission flow (Section 7.1) applies. +1. Peer `P` is returned in a `FIND_NODE` response from another peer during a lookup, or connects through the normal authenticated peer path. +2. Local node checks `TrustScore(self, P)`. If still below `QUARANTINE_READMIT_THRESHOLD` for a quarantined peer, `P` is skipped/rejected. +3. If trust has recovered to `QUARANTINE_READMIT_THRESHOLD`, the standard admission flow (Section 7.1) applies. -A blocked peer cannot trigger its own re-admission — it requires third-party discovery after trust recovery. +No manual probing is required. Natural rediscovery plus trust decay is the temporary-ban mechanism. -Implementations SHOULD bound trust record storage for peers not in the routing table. The specific mechanism (LRU eviction, TTL-based expiry, score-at-neutral garbage collection) is an implementation choice. Unbounded accumulation of trust records for blocked or departed peers is a memory leak. +Implementations SHOULD bound trust record storage for peers not in the routing table. The specific mechanism (LRU eviction, TTL-based expiry, score-at-neutral garbage collection) is an implementation choice. Unbounded accumulation of trust records for quarantined or departed peers is a memory leak. ### 7.5 Stale Peer Revalidation on Admission Contention @@ -339,7 +337,7 @@ Algorithm: 2. Include self in `best_nodes` (self competes on distance but is never queried). 3. Mark self as "queried" to prevent self-RPC. 4. Loop (up to `MAX_LOOKUP_ITERATIONS`): - a. Select up to `ALPHA` unqueried peers from `best_nodes`, nearest first. Skip any peer with `TrustScore(self, peer) < BLOCK_THRESHOLD` (the peer may have been blocked since it entered `best_nodes`). + a. Select up to `ALPHA` unqueried peers from `best_nodes`, nearest first. Skip any peer that is below `QUARANTINE_THRESHOLD` or is quarantined and still below `QUARANTINE_READMIT_THRESHOLD`. Such peers are also excluded from local lookup results and FIND_NODE responses. b. Query each in parallel with `FIND_NODE(K)`. c. For each failed query, record trust penalty (`ConnectionFailed`/`ConnectionTimeout`). Successful responses are the expected baseline and do not generate trust events. d. For each response, accept at most `MAX_PEERS_PER_RESPONSE` peers (closest to `K` first; additional entries are silently dropped). Merge accepted peers into `best_nodes`, deduplicating by `PeerId`. @@ -351,7 +349,7 @@ Properties: - **Per-lookup isolation**: Each invocation of `find_closest_nodes_network` maintains its own `best_nodes` set, queried set, and top-K convergence state. Concurrent lookups (e.g., a self-lookup and a consumer-triggered lookup running simultaneously) do not share or interfere with each other's state. They may independently query the same remote peers and independently record trust outcomes. - Makes network requests: MUST NOT be called from within DHT request handlers (deadlock risk). - Trust recording: each RPC outcome is fed to the trust subsystem. -- Blocked peers: silently excluded from query candidates (they are not in `LocalRT`). +- Quarantined peers: silently excluded from automatic query candidates until trust recovers. ## 9. Routing Table Maintenance @@ -395,7 +393,7 @@ The routing table MUST emit events on membership changes to allow consumers to r | Event | Trigger | |---|---| | `PeerAdded(PeerId)` | New peer inserted into routing table | -| `PeerRemoved(PeerId)` | Peer evicted, blocked, or departed | +| `PeerRemoved(PeerId)` | Peer evicted, quarantined, or departed | | `KClosestPeersChanged { old, new, added, removed }` | Composition of the `K_BUCKET_SIZE`-closest peers to self changed | | `BootstrapComplete { num_peers }` | Bootstrap process finished (routing table stabilized or timeout reached) | @@ -411,7 +409,7 @@ Events MUST be emitted reliably for every routing table mutation. Consumers MAY Peers are detected as departed through: -1. **RPC failure**: Failed outbound RPC records trust failure. If trust drops below `BLOCK_THRESHOLD`, peer is evicted (Section 7.4). +1. **RPC failure**: Failed outbound RPC records trust failure. If trust drops below `QUARANTINE_THRESHOLD` and the peer is in the close group, it is evicted and quarantined (Section 7.4). 2. **Iterative lookup feedback**: Network lookups record success/failure per queried peer. 3. **Self-lookup refresh**: Periodic self-lookups discover that a previously-close peer is no longer returned by the network. 4. **Stale peer revalidation**: When a new candidate contends for a full bucket, all stale peers (not seen within `LIVE_THRESHOLD`) in that bucket are pinged. Non-responders are evicted immediately (Section 7.5). @@ -433,7 +431,7 @@ All paths converge on the same admission flow (Section 7.1), ensuring consistent ### 10.3 Automatic Re-Bootstrap -When `routing_table_size()` drops below `AUTO_REBOOTSTRAP_THRESHOLD` (e.g., due to mass blocking or network partition), the node MUST automatically trigger the bootstrap process (Section 11.1 steps 2–7). This prevents permanent isolation when the routing table is depleted. +When `routing_table_size()` drops below `AUTO_REBOOTSTRAP_THRESHOLD` (e.g., due to mass quarantine/departure or network partition), the node MUST automatically trigger the bootstrap process (Section 11.1 steps 2–7). This prevents permanent isolation when the routing table is depleted. Re-bootstrap follows the same flow as cold start: dial bootstrap peers, perform self-lookup, refresh buckets, emit `BootstrapComplete`. The close group cache is not reloaded (it reflects the state that led to depletion). A minimum cooldown of `REBOOTSTRAP_COOLDOWN` (reference: 5 minutes) MUST elapse between consecutive re-bootstrap attempts to prevent bootstrap node overload during persistent partitions. Re-bootstrap MAY fire multiple times if the routing table repeatedly drops below the threshold, subject to the cooldown. @@ -500,7 +498,7 @@ Defenses: An attacker attempts to insert malicious entries via `FIND_NODE` responses: 1. **No blind insertion**: Peers returned by `FIND_NODE` are not automatically added. They must be dialed, authenticated, and pass the admission flow. -2. **Trust baseline**: New peers start at neutral trust (0.5), well above `BLOCK_THRESHOLD` (0.15) but below `TRUST_PROTECTION_THRESHOLD` (0.7). They must demonstrate good behavior to earn protection. +2. **Trust baseline**: New peers start at neutral trust (0.5), above `SWAP_THRESHOLD` (0.35) and `QUARANTINE_THRESHOLD` (0.20), but below `TRUST_PROTECTION_THRESHOLD` (0.7). They must demonstrate good behavior to earn protection. 3. **IP diversity gates**: Even if an attacker can authenticate many identities, IP diversity limits prevent flooding. ## 13. Consumer API @@ -527,12 +525,12 @@ Consumers MUST NOT: - Directly read or write k-bucket contents. - Bypass IP diversity or trust checks when admitting peers. -- Remove peers from the routing table (that is owned by the trust/blocking subsystem). +- Remove peers from the routing table (that is owned by the trust/quarantine subsystem). - Manipulate trust scores directly — all trust mutations flow through `report_trust_event`. Consumers MAY: -- Report trust events via `report_trust_event` to reward or penalize peers based on application-level outcomes, which may indirectly cause routing table changes (eviction on block, trust protection gain/loss). +- Report trust events via `report_trust_event` to reward or penalize peers based on application-level outcomes, which may indirectly cause routing table changes (close-group quarantine, lazy swap eligibility, trust protection gain/loss). - Query `peer_trust` to make trust-informed decisions (e.g., preferring higher-trust peers for data retrieval). - Request network lookups to discover new peers (which may be admitted to the routing table as a side effect). @@ -589,14 +587,15 @@ All events — internal and consumer-reported — follow the same path through t 3. **Weight resolution**: Internal events have implicit weight `1.0`. Consumer events use their caller-specified weight (after validation/clamping). 4. **EMA update**: The trust engine applies time decay, then blends the observation using the EMA model (Section 4). Positive events use observation `1.0`, negative events use `0.0`. The weight scales influence via the continuous formula `score = (1 - EMA_ALPHA)^W * score + (1 - (1 - EMA_ALPHA)^W) * observation`, which generalizes naturally to fractional weights. At reference values (`EMA_ALPHA = 0.3`), a single weight-1.0 failure moves a neutral peer's score from 0.5 to 0.35; a single weight-5.0 failure moves it from 0.5 to ~0.08. 5. **Threshold checks**: - a. **Block check**: If `TrustScore(self, P)` dropped below `BLOCK_THRESHOLD`, trigger the blocked peer handling flow (Section 7.4) — peer is evicted from the routing table, disconnected, and blocked. - b. **Protection evaluation**: If `TrustScore(self, P)` crossed `TRUST_PROTECTION_THRESHOLD` in either direction, the peer's swap-closer protection status changes accordingly (Section 7.3). + a. **Quarantine check**: If `TrustScore(self, P)` dropped below `QUARANTINE_THRESHOLD`, local lookup results, FIND_NODE responses, and automatic lookup/dial paths avoid the peer. If it is in the K-closest-to-self set, trigger quarantine handling (Section 7.4). + b. **Swap eligibility**: If `TrustScore(self, P)` dropped below `SWAP_THRESHOLD`, the peer is eligible for lazy replacement when a better candidate needs its slot. + c. **Protection evaluation**: If `TrustScore(self, P)` crossed `TRUST_PROTECTION_THRESHOLD` in either direction, the peer's swap-closer protection status changes accordingly (Section 7.3). #### Consumer Reporting Invariants 1. **Unified model**: All events (internal and consumer-reported) are processed by the same EMA scoring model. There is no separate scoring path for consumer events. The trust score is a single value derived from the weighted history of all events, with time decay toward neutral. 2. **Weight as severity**: A consumer event with weight `W` has the same EMA impact as `W` consecutive internal events of the same category (exact for integer `W`, continuously interpolated for fractional `W` via the generalized blend formula in Section 4). Weight `1.0` is equivalent to a single internal event; weight `5.0` is equivalent to five. -3. **Bounded weight**: A single consumer event's weight is capped at `MAX_CONSUMER_WEIGHT`. At reference values (`EMA_ALPHA = 0.3`, `MAX_CONSUMER_WEIGHT = 5.0`), a single maximum-weight failure moves a neutral peer from 0.5 to ~0.08 — enough to cross `BLOCK_THRESHOLD` (0.15) in one event. This is intentional: with the penalty-only model, a severe application-level failure should be able to immediately block a neutral peer. +3. **Bounded weight**: A single consumer event's weight is capped at `MAX_CONSUMER_WEIGHT`. At reference values (`EMA_ALPHA = 0.124`, `MAX_CONSUMER_WEIGHT = 5.0`), a single maximum-weight failure moves a neutral peer to ~0.26 — enough to make it swap-eligible but not enough by itself to quarantine it. Severe repeated failures can still push a peer below `QUARANTINE_THRESHOLD`. 4. **Natural decay**: Because consumer events flow through the EMA, their influence decays over time just like internal events. A penalty reported last week has less influence on the current score than a penalty reported today. A peer that was penalized but then goes idle will drift back toward neutral (0.5). 5. **Idempotent path**: Reporting a trust event for a peer not in the routing table is valid. The trust engine maintains scores independently of routing table membership (a peer can have a trust record without being in `LocalRT`). 6. **No direct score manipulation**: Consumers cannot set a trust score to an arbitrary value. Scores are derived exclusively from the weighted EMA of all events plus time decay. @@ -659,10 +658,10 @@ Use this list to find design flaws before coding: - After a long offline period, the close group cache may contain departed peers. Mitigation: warm restart dials cached peers and falls back to bootstrap if they are unreachable. Self-lookup then refreshes the neighborhood. 9. **Consumer trust event flooding**: - - A misbehaving or buggy consumer could flood `report_trust_event` with `ApplicationFailure(MAX_CONSUMER_WEIGHT)` events, rapidly blocking many peers and depleting the routing table. Mitigation: `MAX_CONSUMER_WEIGHT` caps per-event influence, and the EMA's smoothing factor limits how far a single event can move the score — even at maximum weight, the score change is bounded by EMA dynamics, not by the weight alone. The consumer is a trusted local process. If rate limiting is needed in the future, it can be added at the `report_trust_event` interface without changing the scoring model. For v1, the consumer is assumed to report events honestly and at a reasonable rate. + - A misbehaving or buggy consumer could flood `report_trust_event` with `ApplicationFailure(MAX_CONSUMER_WEIGHT)` events, rapidly quarantining close-group peers and depleting the trusted close group. Mitigation: `MAX_CONSUMER_WEIGHT` caps per-event influence, and the EMA's smoothing factor limits how far a single event can move the score — even at maximum weight, the score change is bounded by EMA dynamics, not by the weight alone. The consumer is a trusted local process. If rate limiting is needed in the future, it can be added at the `report_trust_event` interface without changing the scoring model. For v1, the consumer is assumed to report events honestly and at a reasonable rate. 10. **Internal vs consumer event divergence**: - - Core only records penalties (connection failures), so a peer that is DHT-reachable but never receives consumer rewards will gradually drift below neutral as occasional connection hiccups accumulate. Consumer `ApplicationSuccess` events are the only way to push trust above neutral, preventing free-riding on bare connectivity. A peer that is reachable but serves bad data will be blocked even faster since there are no internal success events to counteract consumer-reported failures. + - Core only records penalties (connection failures), so a peer that is DHT-reachable but never receives consumer rewards will gradually drift below neutral as occasional connection hiccups accumulate. Consumer `ApplicationSuccess` events are the only way to push trust above neutral, preventing free-riding on bare connectivity. A peer that is reachable but serves bad data will become swap-eligible or quarantined faster since there are no internal success events to counteract consumer-reported failures. 11. **Consumer reward inflation**: - A consumer could report `ApplicationSuccess(MAX_CONSUMER_WEIGHT)` for every interaction, inflating a peer's trust toward 1.0. Because all events flow through EMA, the score asymptotically approaches 1.0 but the smoothing factor limits the rate. This is acceptable: the consumer is a trusted local process, and inflating trust simply means the peer gains stronger protection. If the peer later misbehaves, subsequent failures (internal or consumer-reported) will pull the score back down, and time decay ensures idle peers drift toward neutral. @@ -697,8 +696,8 @@ Each scenario should assert exact expected outcomes and state transitions. 3. **Empty address rejection**: - Candidate with zero addresses. Rejected with error. Routing table unchanged. -4. **Blocked peer rejection**: - - Peer with `TrustScore < BLOCK_THRESHOLD`. Rejected. Not in routing table. +4. **Quarantined peer rejection**: + - Peer with `TrustScore < QUARANTINE_THRESHOLD`, or previously quarantined peer with trust below `QUARANTINE_READMIT_THRESHOLD`. Rejected. Not in routing table. 5. **Bucket-full rejection (no stale peers)**: - Bucket at `K_BUCKET_SIZE` capacity, candidate cannot swap-closer, all incumbent peers have `last_seen` within `LIVE_THRESHOLD`. Stale revalidation finds no candidates. Rejected with "bucket at capacity." Routing table unchanged. @@ -799,17 +798,18 @@ Each scenario should assert exact expected outcomes and state transitions. 35. **KClosestPeersChanged event emission**: - Insert a peer into a bucket that affects the K-closest-to-self set. `KClosestPeersChanged` emitted with correct old and new sets. Insert a peer into a distant bucket that does NOT affect the K-closest set. `KClosestPeersChanged` is NOT emitted. Verify at-most-once semantics: a single admission with multiple swaps emits the event at most once. -36. **Blocked peer eviction**: - - Peer trust drops below 0.15 after failed interaction. Peer is immediately removed from routing table and disconnected. +36. **Close-group quarantine eviction**: + - K-closest peer trust drops below 0.20 after failed interaction. Peer is immediately removed from routing table, its trust record is retained, and `PeerRemoved` is emitted. -37. **Blocked peer inbound connection rejected**: - - Blocked peer initiates inbound connection. Transport identifies peer during authentication, checks trust score, rejects connection. No resources allocated, no routing table interaction. +37. **Quarantined peer inbound admission rejected**: + - Quarantined peer initiates inbound connection. Transport authenticates the peer, the normal routing-table admission path checks trust, and admission is rejected until trust reaches 0.45. -38. **Blocked peer skipped in lookup results**: - - Blocked peer appears in `FIND_NODE` response. Local node checks trust, finds it below `BLOCK_THRESHOLD`. Peer silently skipped — not dialed. +38. **Quarantined peer skipped in lookup results**: + - Quarantined peer appears in `FIND_NODE` response from a node that does not quarantine it. Local node checks trust, finds it below the applicable quarantine/readmit threshold. Peer silently skipped — not dialed. + - If the local node is answering a `FIND_NODE` request, quarantined peers are omitted from the response entirely. -39. **Blocked peer re-admission via lookup discovery after trust recovery**: - - Previously blocked peer's trust decays back above `BLOCK_THRESHOLD`. Peer appears in `FIND_NODE` response. Local node dials, authenticates, and admits through normal admission flow. +39. **Quarantined peer re-admission via lookup discovery after trust recovery**: + - Previously quarantined peer's trust decays back to at least `QUARANTINE_READMIT_THRESHOLD`. Peer appears in `FIND_NODE` response. Local node dials, authenticates, and admits through normal admission flow. ### Bootstrap Tests @@ -832,7 +832,7 @@ Each scenario should assert exact expected outcomes and state transitions. - Close group cache loaded. Cached peers dialed. Self-lookup and bucket refreshes complete. `BootstrapComplete` emitted. Event fires exactly once regardless of cold/warm path. 46. **Auto re-bootstrap on routing table depletion**: - - All peers blocked or departed. `routing_table_size()` drops below `AUTO_REBOOTSTRAP_THRESHOLD`. Bootstrap process automatically triggered. Bootstrap peers dialed, self-lookup runs. Routing table repopulated. `BootstrapComplete` emitted. + - Peers are quarantined or departed. `routing_table_size()` drops below `AUTO_REBOOTSTRAP_THRESHOLD`. Bootstrap process automatically triggered. Bootstrap peers dialed, self-lookup runs. Routing table repopulated. `BootstrapComplete` emitted. ### Security Tests @@ -851,19 +851,20 @@ Each scenario should assert exact expected outcomes and state transitions. 51. **Unauthenticated peer rejected**: - Peer returned by `FIND_NODE` but not yet authenticated. Not admitted to routing table. Must complete handshake first. -52. **Blocked peer messages dropped**: - - Peer below block threshold sends DHT message. Message silently dropped. No routing table interaction. +52. **Quarantined peer avoided by automatic lookup**: + - Peer below quarantine threshold appears as a candidate. Automatic lookup skips it and spends no alpha slot dialing it. + - Local lookup and FIND_NODE response construction do not return the peer as a candidate. ### Consumer Trust Reporting Tests 53. **Consumer reward improves trust**: - Peer starts at neutral trust (0.5). Consumer reports `ApplicationSuccess(1.0)`. Trust score increases above 0.5 (exact value determined by EMA smoothing factor). Peer remains in routing table. -54. **Consumer penalty degrades trust to blocking**: - - Peer starts at neutral trust (0.5). Consumer reports repeated `ApplicationFailure(3.0)` events. Trust score decreases with each event. After sufficient events, score drops below `BLOCK_THRESHOLD` (0.15). Peer is evicted from routing table and blocked (Section 7.4). +54. **Consumer penalty degrades trust to quarantine**: + - Peer starts at neutral trust (0.5). Consumer reports repeated `ApplicationFailure(3.0)` events. Trust score decreases with each event. After sufficient events, score drops below `QUARANTINE_THRESHOLD` (0.20). If the peer is in the K-closest set, it is evicted and quarantined (Section 7.4). -55. **Consumer penalty triggers blocking and eviction**: - - Peer is in routing table with trust slightly above `BLOCK_THRESHOLD`. Consumer reports `ApplicationFailure(weight)` sufficient to push score below `BLOCK_THRESHOLD`. Peer is immediately evicted from routing table, disconnected at transport layer, and blocked from re-admission. `PeerRemoved` event emitted. +55. **Consumer penalty triggers close-group quarantine**: + - Peer is in the K-closest set with trust slightly above `QUARANTINE_THRESHOLD`. Consumer reports `ApplicationFailure(weight)` sufficient to push score below `QUARANTINE_THRESHOLD`. Peer is immediately evicted from routing table and quarantined from re-admission until trust reaches `QUARANTINE_READMIT_THRESHOLD`. `PeerRemoved` event emitted. 56. **Consumer event for peer not in routing table**: - Peer has no routing table entry. Consumer reports `ApplicationFailure(2.0)`. Trust engine records the event and updates the EMA score (decreases from neutral 0.5). Routing table is unchanged. If the peer later attempts admission, the recorded low trust may cause rejection (Section 7.1 step 4). diff --git a/docs/SECURITY_MODEL.md b/docs/SECURITY_MODEL.md index 06b4bf0f..3df1a9c2 100644 --- a/docs/SECURITY_MODEL.md +++ b/docs/SECURITY_MODEL.md @@ -80,32 +80,37 @@ pub struct NodeLivenessState { - Periodic health pings (configurable interval) - Validation responses -### Eviction Criteria +### Routing-Table Trust Criteria -Nodes are automatically evicted when any threshold is exceeded: +Trust affects routing-table membership in two stages: -| Eviction Reason | Default Threshold | Configuration | +| Reason | Default Threshold | Configuration | |-----------------|-------------------|---------------| -| Consecutive Failures | 3 failures | `max_consecutive_failures` | -| Low Trust Score | < 0.15 | `min_trust_threshold` | -| Close Group Rejection | Consensus | BFT threshold | +| Lazy swap eligibility | < 0.35 | `swap_threshold` | +| Close-group quarantine / lookup avoidance | < 0.20 | `quarantine_threshold` | +| Quarantine readmission | >= 0.45 | `quarantine_readmit_threshold` | | Staleness | Configurable | `stale_timeout` | -### Eviction Manager +Peers outside the K-closest set are not globally evicted solely for low trust. +They are omitted from local lookup results, FIND_NODE responses, and automatic +lookup paths below the quarantine threshold, and can be lazily replaced when +better candidates need the slot. -The `EvictionManager` coordinates all eviction decisions: +### Quarantine Reasons + +Routing-table quarantine decisions are represented by events such as: ```rust -pub enum EvictionReason { - ConsecutiveFailures(u32), // Communication failures - LowTrust(String), // EigenTrust score below threshold - FailedAttestation, // Data challenge failure - CloseGroupRejection, // Consensus-based removal - Stale, // No activity timeout +pub enum QuarantineReason { + LowTrustCloseGroup, // Close-group trust below threshold + Stale, // No activity timeout during revalidation + IdentityMismatch, // Address authenticated as another peer } ``` -**Recovery Mechanism:** A single successful interaction resets the consecutive failure counter, allowing nodes to recover from transient issues. +**Recovery Mechanism:** Quarantined peers recover by trust decay toward neutral. +They are not manually probed; they must be rediscovered through the normal +admission path after reaching the readmission threshold. --- diff --git a/docs/trust-signals-api.md b/docs/trust-signals-api.md index 9cd36923..bf842fae 100644 --- a/docs/trust-signals-api.md +++ b/docs/trust-signals-api.md @@ -11,9 +11,9 @@ responsibility via `TrustEvent::ApplicationSuccess`. The trust system enables: - **Sybil resistance**: Malicious nodes are downscored automatically -- **Binary blocking**: Peers below the block threshold are evicted and rejected -- **Self-healing**: Time decay moves blocked peers back toward neutral over days -- **Live eviction**: Peers below trust threshold are evicted from the routing table immediately +- **Close-group quarantine**: K-closest peers below the quarantine threshold are evicted +- **Self-healing**: Time decay moves quarantined peers back toward neutral over days +- **Lazy swap-out**: Low-trust peers outside the close group are replaced when better candidates arrive ## Quick Start @@ -76,23 +76,34 @@ are not rewarded. Note: Peer disconnects are normal connection lifecycle — they do not affect trust. -## Peer Blocking +## Trust Thresholds -Peers whose trust score falls below `block_threshold` are: -- **Evicted** from the DHT routing table (via EvictionManager) -- **Blocked** from sending DHT messages (silently dropped) -- **Rejected** from re-entering the routing table on reconnect +The routing table uses three trust thresholds: + +- `swap_threshold` (`0.35` by default): peers below this score are eligible + for replacement when a better candidate needs the slot. +- `quarantine_threshold` (`0.20` by default): peers below this score are + skipped by lookup result selection and automatic lookup/dial paths. If such + a peer is currently in the K-closest-to-self set, it is evicted immediately + and quarantined. +- `quarantine_readmit_threshold` (`0.45` by default): a quarantined peer can + only re-enter through normal discovery/admission after its decayed trust + reaches this score. ```rust use saorsa_core::AdaptiveDhtConfig; let config = AdaptiveDhtConfig { - block_threshold: 0.15, // Block peers below 15% trust + swap_threshold: 0.35, + quarantine_threshold: 0.20, + quarantine_readmit_threshold: 0.45, ..Default::default() }; ``` -DHT routing uses pure Kademlia XOR distance — trust does not influence peer selection order. +Raw DHT routing uses Kademlia XOR distance. Local lookup results, FIND_NODE +responses, and automatic network lookups avoid quarantined peers so known-bad +contacts do not consume query slots or get handed out as lookup candidates. ## Architecture diff --git a/src/adaptive/dht.rs b/src/adaptive/dht.rs index 2e34e30c..70ec6852 100644 --- a/src/adaptive/dht.rs +++ b/src/adaptive/dht.rs @@ -31,6 +31,13 @@ use std::sync::Arc; /// Default trust score threshold below which a peer is eligible for swap-out const DEFAULT_SWAP_THRESHOLD: f64 = 0.35; +/// Default trust score threshold below which close-group peers are evicted +/// immediately and all peers are avoided by automatic lookup/dial paths. +const DEFAULT_QUARANTINE_THRESHOLD: f64 = 0.20; + +/// Default trust score a quarantined peer must recover to before readmission. +const DEFAULT_QUARANTINE_READMIT_THRESHOLD: f64 = 0.45; + /// Maximum weight multiplier per single consumer-reported event. /// Caps the influence of any single consumer event on the EMA. const MAX_CONSUMER_WEIGHT: f64 = 5.0; @@ -41,15 +48,25 @@ const MAX_CONSUMER_WEIGHT: f64 = 5.0; pub struct AdaptiveDhtConfig { /// Trust score below which a peer becomes eligible for swap-out from /// the routing table when a better candidate is available. - /// Peers are NOT immediately evicted. + /// Peers are not immediately evicted by this threshold alone. /// Default: 0.35 pub swap_threshold: f64, + /// Trust score below which automatic lookup/dial paths avoid a peer, and + /// K-closest peers are evicted immediately into temporary quarantine. + /// Default: 0.20 + pub quarantine_threshold: f64, + /// Trust score a quarantined peer must decay back to before normal + /// discovery/admission can accept it again. + /// Default: 0.45 + pub quarantine_readmit_threshold: f64, } impl Default for AdaptiveDhtConfig { fn default() -> Self { Self { swap_threshold: DEFAULT_SWAP_THRESHOLD, + quarantine_threshold: DEFAULT_QUARANTINE_THRESHOLD, + quarantine_readmit_threshold: DEFAULT_QUARANTINE_READMIT_THRESHOLD, } } } @@ -57,9 +74,11 @@ impl Default for AdaptiveDhtConfig { impl AdaptiveDhtConfig { /// Validate that all config values are within acceptable ranges. /// - /// Returns `Err` if `swap_threshold` is outside `[0.0, 0.5)` or is NaN. + /// Returns `Err` if a threshold is outside its safe range or is NaN. /// Values >= 0.5 (neutral trust) would make all unknown peers immediately - /// swap-eligible since they start at neutral (0.5). + /// swap/quarantine eligible since they start at neutral (0.5). The readmit + /// threshold must also stay below neutral because quarantined peers recover + /// by decay toward neutral, not by active probing. pub fn validate(&self) -> crate::error::P2pResult<()> { if !(0.0..0.5).contains(&self.swap_threshold) || self.swap_threshold.is_nan() { return Err(crate::error::P2PError::Validation( @@ -70,6 +89,37 @@ impl AdaptiveDhtConfig { .into(), )); } + if !(0.0..0.5).contains(&self.quarantine_threshold) || self.quarantine_threshold.is_nan() { + return Err(crate::error::P2PError::Validation( + format!( + "quarantine_threshold must be in [0.0, 0.5), got {}", + self.quarantine_threshold + ) + .into(), + )); + } + if !(0.0..0.5).contains(&self.quarantine_readmit_threshold) + || self.quarantine_readmit_threshold.is_nan() + { + return Err(crate::error::P2PError::Validation( + format!( + "quarantine_readmit_threshold must be in [0.0, 0.5), got {}", + self.quarantine_readmit_threshold + ) + .into(), + )); + } + if self.quarantine_threshold > 0.0 + && self.quarantine_readmit_threshold < self.quarantine_threshold + { + return Err(crate::error::P2PError::Validation( + format!( + "quarantine_readmit_threshold ({}) must be >= quarantine_threshold ({})", + self.quarantine_readmit_threshold, self.quarantine_threshold + ) + .into(), + )); + } Ok(()) } } @@ -143,12 +193,13 @@ impl AdaptiveDHT { /// This creates the `TrustEngine` and the `DhtNetworkManager` with the /// trust engine injected. Call [`start`](Self::start) to begin DHT /// operations. Trust scores are computed live — low-trust peers are - /// swapped out when better candidates arrive. + /// swapped out when better candidates arrive, and bad close-group peers + /// are quarantined immediately. /// /// # Errors /// - /// Returns an error if `swap_threshold` is not in `[0.0, 0.5)` or if - /// the underlying `DhtNetworkManager` fails to initialise. + /// Returns an error if any trust threshold is invalid or if the underlying + /// `DhtNetworkManager` fails to initialise. pub async fn new( transport: Arc, mut dht_config: DhtNetworkConfig, @@ -157,6 +208,8 @@ impl AdaptiveDHT { adaptive_config.validate()?; dht_config.swap_threshold = adaptive_config.swap_threshold; + dht_config.quarantine_threshold = adaptive_config.quarantine_threshold; + dht_config.quarantine_readmit_threshold = adaptive_config.quarantine_readmit_threshold; let trust_engine = Arc::new(TrustEngine::new()); @@ -183,9 +236,10 @@ impl AdaptiveDHT { /// to [`MAX_CONSUMER_WEIGHT`]. Zero or negative weights are silently /// ignored (no-op). /// - /// Trust scores are updated immediately but low-trust peers are not - /// evicted — they remain in the routing table until a better candidate - /// arrives and triggers a swap-out. + /// Trust scores are updated immediately. Peers below the quarantine + /// threshold are avoided by lookup result selection and automatic + /// lookup/dial paths, and K-closest peers below that threshold are + /// evicted into temporary quarantine. pub async fn report_trust_event(&self, peer_id: &PeerId, event: TrustEvent) { match event { TrustEvent::ApplicationSuccess(weight) | TrustEvent::ApplicationFailure(weight) => { @@ -208,6 +262,7 @@ impl AdaptiveDHT { ); } } + self.dht_manager.enforce_trust_quarantine(peer_id).await; } /// Get the current trust score for a peer (synchronous). @@ -243,7 +298,8 @@ impl AdaptiveDHT { /// Start the DHT manager. /// /// Trust scores are computed live — no background tasks needed. - /// Low-trust peers are swapped out when better candidates arrive. + /// Low-trust peers are swapped out when better candidates arrive; close + /// peers below the quarantine threshold are evicted immediately. pub async fn start(&self) -> Result<()> { Arc::clone(&self.dht_manager).start().await } @@ -325,6 +381,11 @@ mod tests { fn test_adaptive_dht_config_defaults() { let config = AdaptiveDhtConfig::default(); assert!((config.swap_threshold - DEFAULT_SWAP_THRESHOLD).abs() < f64::EPSILON); + assert!((config.quarantine_threshold - DEFAULT_QUARANTINE_THRESHOLD).abs() < f64::EPSILON); + assert!( + (config.quarantine_readmit_threshold - DEFAULT_QUARANTINE_READMIT_THRESHOLD).abs() + < f64::EPSILON + ); } #[test] @@ -342,6 +403,7 @@ mod tests { ] { let config = AdaptiveDhtConfig { swap_threshold: bad, + ..Default::default() }; assert!( config.validate().is_err(), @@ -355,6 +417,7 @@ mod tests { for &good in &[0.0, 0.15, 0.49] { let config = AdaptiveDhtConfig { swap_threshold: good, + ..Default::default() }; assert!( config.validate().is_ok(), @@ -363,6 +426,29 @@ mod tests { } } + #[test] + fn test_quarantine_threshold_validation() { + let valid = AdaptiveDhtConfig { + quarantine_threshold: 0.20, + quarantine_readmit_threshold: 0.45, + ..Default::default() + }; + assert!(valid.validate().is_ok()); + + let invalid_readmit = AdaptiveDhtConfig { + quarantine_threshold: 0.20, + quarantine_readmit_threshold: 0.10, + ..Default::default() + }; + assert!(invalid_readmit.validate().is_err()); + + let unreachable_readmit = AdaptiveDhtConfig { + quarantine_readmit_threshold: 0.50, + ..Default::default() + }; + assert!(unreachable_readmit.validate().is_err()); + } + // ========================================================================= // Integration tests: full trust signal flow // ========================================================================= diff --git a/src/dht/core_engine.rs b/src/dht/core_engine.rs index fbea9742..2acab1e0 100644 --- a/src/dht/core_engine.rs +++ b/src/dht/core_engine.rs @@ -203,6 +203,14 @@ const LIVE_THRESHOLD: Duration = Duration::from_secs(900); // 15 minutes #[allow(dead_code)] const DEFAULT_SWAP_THRESHOLD: f64 = 0.35; +/// Default trust score below which a close-group peer is quarantined. +#[allow(dead_code)] +const DEFAULT_QUARANTINE_THRESHOLD: f64 = 0.20; + +/// Default trust score required before a quarantined peer can be admitted again. +#[allow(dead_code)] +const DEFAULT_QUARANTINE_READMIT_THRESHOLD: f64 = 0.45; + /// Node information for routing. /// /// The `addresses` field stores one or more typed [`MultiAddr`] values that are @@ -1461,6 +1469,19 @@ pub struct DhtCoreEngine { /// Trust score below which a peer is eligible for swap-out. swap_threshold: f64, + /// Trust score below which a peer is avoided for automatic lookup/dial + /// paths, and evicted immediately if it is in the K-closest close group. + quarantine_threshold: f64, + + /// Trust score a quarantined peer must decay back to before normal + /// admission can accept it again. + quarantine_readmit_threshold: f64, + + /// Peers evicted from the close group by quarantine. They remain in this + /// set until discovered naturally and admitted after crossing the readmit + /// threshold. + quarantined_peers: HashSet, + /// Duration of no contact after which a peer is considered stale. /// Defaults to [`LIVE_THRESHOLD`]; overridden in tests to avoid /// `Instant` subtraction overflow on Windows (where `Instant` starts @@ -1506,11 +1527,137 @@ impl DhtCoreEngine { ip_diversity_config: IPDiversityConfig::default(), allow_loopback, swap_threshold, + quarantine_threshold: 0.0, + quarantine_readmit_threshold: DEFAULT_QUARANTINE_READMIT_THRESHOLD, + quarantined_peers: HashSet::new(), live_threshold: LIVE_THRESHOLD, shutdown: CancellationToken::new(), }) } + /// Configure trust quarantine thresholds. + /// + /// A `quarantine_threshold` of `0.0` disables quarantine enforcement. + /// Otherwise, peers below that score are avoided for automatic lookups, + /// and K-closest peers below it are immediately evicted and quarantined. + /// Quarantined peers can only re-enter through normal admission after + /// their decayed trust reaches `quarantine_readmit_threshold`. + pub(crate) fn set_trust_quarantine_thresholds( + &mut self, + quarantine_threshold: f64, + quarantine_readmit_threshold: f64, + ) -> Result<()> { + if !(0.0..1.0).contains(&quarantine_threshold) || quarantine_threshold.is_nan() { + return Err(anyhow!( + "quarantine_threshold must be in [0.0, 1.0), got {quarantine_threshold}" + )); + } + if !(0.0..1.0).contains(&quarantine_readmit_threshold) + || quarantine_readmit_threshold.is_nan() + { + return Err(anyhow!( + "quarantine_readmit_threshold must be in [0.0, 1.0), got {quarantine_readmit_threshold}" + )); + } + if quarantine_threshold > 0.0 && quarantine_readmit_threshold < quarantine_threshold { + return Err(anyhow!( + "quarantine_readmit_threshold ({quarantine_readmit_threshold}) must be >= quarantine_threshold ({quarantine_threshold})" + )); + } + self.quarantine_threshold = quarantine_threshold; + self.quarantine_readmit_threshold = quarantine_readmit_threshold; + Ok(()) + } + + fn quarantine_enabled(&self) -> bool { + self.quarantine_threshold > 0.0 + } + + /// Return whether trust quarantine affects lookup result filtering. + pub(crate) fn trust_quarantine_enabled(&self) -> bool { + self.quarantine_enabled() + } + + fn check_quarantine_admission(&mut self, peer_id: &PeerId, trust_score: f64) -> Result<()> { + if !self.quarantine_enabled() { + return Ok(()); + } + if !trust_score.is_finite() { + return Err(anyhow!( + "peer {} has non-finite trust score", + peer_id.to_hex() + )); + } + if trust_score < self.quarantine_threshold { + self.quarantined_peers.insert(*peer_id); + return Err(anyhow!( + "peer {} below quarantine threshold ({trust_score:.3} < {:.3})", + peer_id.to_hex(), + self.quarantine_threshold + )); + } + if self.quarantined_peers.contains(peer_id) { + if trust_score < self.quarantine_readmit_threshold { + return Err(anyhow!( + "peer {} quarantined until trust >= {:.3} (current {trust_score:.3})", + peer_id.to_hex(), + self.quarantine_readmit_threshold + )); + } + self.quarantined_peers.remove(peer_id); + } + Ok(()) + } + + /// Return whether automatic lookup/dial paths should avoid this peer. + pub(crate) fn should_avoid_for_lookup(&self, peer_id: &PeerId, trust_score: f64) -> bool { + if !self.quarantine_enabled() { + return false; + } + if !trust_score.is_finite() { + return true; + } + trust_score < self.quarantine_threshold + || (self.quarantined_peers.contains(peer_id) + && trust_score < self.quarantine_readmit_threshold) + } + + /// Evict a quarantined peer if it currently occupies the K-closest set. + pub(crate) async fn enforce_close_group_quarantine( + &mut self, + peer_id: &PeerId, + trust_score: f64, + ) -> Vec { + if !self.quarantine_enabled() + || !trust_score.is_finite() + || trust_score >= self.quarantine_threshold + { + return Vec::new(); + } + + let mut routing = self.routing_table.write().await; + let k_before = routing.k_closest_ids(self.k_value); + if !k_before.contains(peer_id) { + return Vec::new(); + } + if routing.find_node_by_id(peer_id).is_none() { + return Vec::new(); + } + + self.quarantined_peers.insert(*peer_id); + routing.remove_node(peer_id); + + let k_after = routing.k_closest_ids(self.k_value); + let mut events = vec![RoutingTableEvent::PeerRemoved(*peer_id)]; + if k_before != k_after { + events.push(RoutingTableEvent::KClosestPeersChanged { + old: k_before, + new: k_after, + }); + } + events + } + /// Override the IP diversity configuration. pub fn set_ip_diversity_config(&mut self, config: IPDiversityConfig) { self.ip_diversity_config = config; @@ -2026,6 +2173,8 @@ impl DhtCoreEngine { )); } + self.check_quarantine_admission(&peer_id, trust_score(&peer_id))?; + // Extract ALL IP addresses from the candidate for diversity checking. // If candidate has no IP-based addresses, it's a non-IP transport — bypass diversity. let candidate_ips: Vec = node @@ -2645,6 +2794,7 @@ impl DhtCoreEngine { candidate_ips: &[IpAddr], trust_score: &impl Fn(&PeerId) -> f64, ) -> Result> { + self.check_quarantine_admission(&candidate.id, trust_score(&candidate.id))?; let mut routing = self.routing_table.write().await; match self.add_with_diversity(&mut routing, candidate, candidate_ips, trust_score, false)? { AdmissionResult::Admitted(events) => Ok(events), @@ -2668,6 +2818,12 @@ impl std::fmt::Debug for DhtCoreEngine { .field("ip_diversity_config", &self.ip_diversity_config) .field("allow_loopback", &self.allow_loopback) .field("swap_threshold", &self.swap_threshold) + .field("quarantine_threshold", &self.quarantine_threshold) + .field( + "quarantine_readmit_threshold", + &self.quarantine_readmit_threshold, + ) + .field("quarantined_peers", &self.quarantined_peers.len()) .finish() } } @@ -4898,6 +5054,105 @@ mod tests { assert!(dht.has_node(&low_peer).await); } + /// A K-closest peer below the quarantine threshold is evicted immediately + /// and cannot be readmitted until its trust has recovered to the readmit + /// threshold. + #[tokio::test] + async fn test_close_group_peer_below_quarantine_is_evicted_until_readmit() { + let mut dht = DhtCoreEngine::new_for_tests(PeerId::from_bytes([0u8; 32])).unwrap(); + dht.set_trust_quarantine_thresholds(0.20, 0.45).unwrap(); + + let mut peer_id_bytes = [0u8; 32]; + peer_id_bytes[31] = 1; + let peer = PeerId::from_bytes(peer_id_bytes); + + dht.add_node_no_trust(make_node_with_addr( + peer_id_bytes, + "/ip4/10.10.0.1/udp/9000/quic", + )) + .await + .unwrap(); + assert!(dht.has_node(&peer).await); + + let events = dht.enforce_close_group_quarantine(&peer, 0.19).await; + assert!( + events + .iter() + .any(|event| matches!(event, RoutingTableEvent::PeerRemoved(id) if *id == peer)), + "close-group peer below quarantine threshold should emit PeerRemoved" + ); + assert!( + !dht.has_node(&peer).await, + "quarantined close-group peer should be removed from RT" + ); + + let early_readmit = dht + .add_node( + make_node_with_addr(peer_id_bytes, "/ip4/10.10.0.1/udp/9000/quic"), + &|id| if *id == peer { 0.30 } else { 0.5 }, + ) + .await; + assert!( + early_readmit.is_err(), + "quarantined peer should not readmit below 0.45" + ); + + let recovered = dht + .add_node( + make_node_with_addr(peer_id_bytes, "/ip4/10.10.0.1/udp/9000/quic"), + &|id| if *id == peer { 0.45 } else { 0.5 }, + ) + .await; + assert!( + recovered.is_ok(), + "quarantined peer should readmit once trust reaches 0.45" + ); + assert!(dht.has_node(&peer).await); + } + + /// A non-close peer below the quarantine threshold is avoided by automatic + /// lookup paths but is not removed merely for being below threshold. + #[tokio::test] + async fn test_non_close_quarantined_peer_is_avoided_but_not_evicted() { + let mut dht = DhtCoreEngine::new( + PeerId::from_bytes([0u8; 32]), + 4, + false, + DEFAULT_SWAP_THRESHOLD, + ) + .unwrap(); + dht.set_trust_quarantine_thresholds(0.20, 0.45).unwrap(); + + for i in 1..=4u8 { + let mut id = [0u8; 32]; + id[31] = i; + dht.add_node_no_trust(make_node_with_addr( + id, + &format!("/ip4/10.{i}.0.1/udp/9000/quic"), + )) + .await + .unwrap(); + } + + let mut far_id = [0u8; 32]; + far_id[0] = 0x80; + let far_peer = PeerId::from_bytes(far_id); + dht.add_node_no_trust(make_node_with_addr(far_id, "/ip4/10.99.0.1/udp/9000/quic")) + .await + .unwrap(); + + let events = dht.enforce_close_group_quarantine(&far_peer, 0.10).await; + assert!( + events.is_empty(), + "non-close peer should not be immediately evicted" + ); + assert!(dht.has_node(&far_peer).await); + assert!( + dht.should_avoid_for_lookup(&far_peer, 0.10), + "automatic lookup should avoid peers below quarantine threshold" + ); + } + // ----------------------------------------------------------------------- // AddressType::Unverified tests // ----------------------------------------------------------------------- diff --git a/src/dht_network_manager.rs b/src/dht_network_manager.rs index 6b8692ba..1b5b6d00 100644 --- a/src/dht_network_manager.rs +++ b/src/dht_network_manager.rs @@ -634,6 +634,14 @@ pub struct DhtNetworkConfig { /// routing table when a better candidate is available. /// Default: 0.0 (disabled). pub swap_threshold: f64, + /// Trust score below which automatic lookup/dial paths avoid a peer, and + /// K-closest peers are immediately evicted into temporary quarantine. + /// Default: 0.0 (disabled). + pub quarantine_threshold: f64, + /// Trust score required before a quarantined peer can be admitted again + /// through the normal discovery/admission path. + /// Default: 0.0 (disabled). + pub quarantine_readmit_threshold: f64, } /// DHT network operation types @@ -1762,6 +1770,12 @@ impl DhtNetworkManager { config.swap_threshold, ) .map_err(|e| P2PError::Dht(DhtError::OperationFailed(e.to_string().into())))?; + dht_instance + .set_trust_quarantine_thresholds( + config.quarantine_threshold, + config.quarantine_readmit_threshold, + ) + .map_err(|e| P2PError::Dht(DhtError::OperationFailed(e.to_string().into())))?; // Propagate IP diversity settings from the node config into the DHT // core engine so diversity overrides take effect on routing table @@ -2077,6 +2091,16 @@ impl DhtNetworkManager { if dht_node.peer_id == this.config.peer_id { continue; } + if this + .should_avoid_automatic_peer(&dht_node.peer_id) + .await + { + trace!( + "Bucket refresh[{bucket_idx}]: skipping quarantined peer {}", + dht_node.peer_id.to_hex() + ); + continue; + } this.dial_addresses( &dht_node.peer_id, &dht_node.typed_addresses(), @@ -2133,6 +2157,13 @@ impl DhtNetworkManager { if dht_node.peer_id == self_id { continue; } + if self.should_avoid_automatic_peer(&dht_node.peer_id).await { + trace!( + "Self-lookup skipping quarantined peer {}", + dht_node.peer_id.to_hex() + ); + continue; + } // Dial if not already connected — try every advertised // address, not just the first, so a stale NAT binding on // one entry doesn't kill the dial. @@ -2259,6 +2290,10 @@ impl DhtNetworkManager { // entirely for clients. Node-mode dials are issued serially below. let mut to_dial: Vec<(PeerId, Vec<(MultiAddr, AddressType)>)> = Vec::new(); for peer_id in peers { + if self.should_avoid_automatic_peer(peer_id).await { + trace!("Bootstrap skipping quarantined peer {}", peer_id.to_hex()); + continue; + } let op = DhtNetworkOperation::FindNode { key }; match self .send_dht_request_with_response_context(peer_id, op, None) @@ -2293,6 +2328,16 @@ impl DhtNetworkManager { // routing table; upgrade-only on existing entries. self.merge_trusted_gossiped_typed_addresses(&trusted_node) .await; + if self + .should_avoid_automatic_peer(&trusted_node.peer_id) + .await + { + trace!( + "DHT bootstrap: skipping quarantined gossiped peer {}", + trusted_node.peer_id.to_hex() + ); + continue; + } if seen.insert(trusted_node.peer_id) && dialable_count > 0 { to_dial.push((trusted_node.peer_id, typed)); } @@ -2627,8 +2672,9 @@ impl DhtNetworkManager { /// Find closest nodes to a key using ONLY the local routing table. /// /// No network requests are made — safe to call from request handlers. - /// Only returns peers that passed the `is_dht_participant` security gate - /// and were added to the Kademlia routing table. + /// Only returns peers that passed the `is_dht_participant` security gate, + /// were added to the Kademlia routing table, and are not below the trust + /// quarantine/readmit thresholds. /// /// Results are sorted by XOR distance to the key. pub async fn find_closest_nodes_local(&self, key: &Key, count: usize) -> Vec { @@ -2639,21 +2685,23 @@ impl DhtNetworkManager { ); let dht_guard = self.dht.read().await; + let candidate_count = if dht_guard.trust_quarantine_enabled() { + dht_guard.routing_table_size().await.max(count) + } else { + count + }; + let trust_score = |peer_id: &PeerId| self.peer_trust_score(peer_id); match dht_guard - .find_nodes_with_publish_seq(&DhtKey::from_bytes(*key), count) + .find_nodes_with_publish_seq(&DhtKey::from_bytes(*key), candidate_count) .await { - Ok(nodes) => nodes - .into_iter() - .filter(|(node, _)| !self.is_local_peer_id(&node.id)) - .map(|(node, publish_seq)| DHTNode { - peer_id: node.id, - address_types: node.address_types, - addresses: node.addresses, - distance: encode_publish_seq_distance(publish_seq), - reliability: SELF_RELIABILITY_SCORE, - }) - .collect(), + Ok(nodes) => Self::lookup_results_from_routing_nodes( + self.config.peer_id, + &dht_guard, + nodes, + &trust_score, + count, + ), Err(e) => { warn!("find_nodes failed for key {}: {e}", hex::encode(key)); Vec::new() @@ -2661,6 +2709,33 @@ impl DhtNetworkManager { } } + fn lookup_results_from_routing_nodes( + local_peer_id: PeerId, + dht: &DhtCoreEngine, + nodes: Vec<(NodeInfo, u64)>, + trust_score: &impl Fn(&PeerId) -> f64, + count: usize, + ) -> Vec { + nodes + .into_iter() + .filter(|(node, _)| node.id != local_peer_id) + .filter_map(|(node, publish_seq)| { + let reliability = trust_score(&node.id); + if dht.should_avoid_for_lookup(&node.id, reliability) { + return None; + } + Some(DHTNode { + peer_id: node.id, + address_types: node.address_types, + addresses: node.addresses, + distance: encode_publish_seq_distance(publish_seq), + reliability, + }) + }) + .take(count) + .collect() + } + /// Find closest nodes to a key using the local routing table, including /// the local node itself in the candidate set. /// @@ -2773,6 +2848,13 @@ impl DhtNetworkManager { // Start with local knowledge let initial = self.find_closest_nodes_local(key, count).await; for node in initial { + if self.should_avoid_automatic_peer(&node.peer_id).await { + trace!( + "[NETWORK] Skipping {}: peer is below trust quarantine/readmit threshold", + node.peer_id.to_hex() + ); + continue; + } if peer_states.is_contactable(&node.peer_id) { if self.lookup_candidate_dial_plan_is_exhausted(&node).await { // Cache exhaustion is a transient, address-view-local @@ -2812,6 +2894,14 @@ impl DhtNetworkManager { if !peer_states.is_contactable(&node.peer_id) { continue; } + if self.should_avoid_automatic_peer(&node.peer_id).await { + peer_states.mark_failed(node.peer_id); + trace!( + "[NETWORK] Skipping {}: peer is below trust quarantine/readmit threshold", + node.peer_id.to_hex() + ); + continue; + } if self.lookup_candidate_dial_plan_is_exhausted(&node).await { // Transient skip, not a terminal failure: keep the peer // contactable so a better address from a later responder @@ -2915,6 +3005,14 @@ impl DhtNetworkManager { if !peer_states.is_contactable(&node.peer_id) { continue; } + if self.should_avoid_automatic_peer(&node.peer_id).await { + peer_states.mark_failed(node.peer_id); + trace!( + "[NETWORK] Skipping gossiped {}: peer is below trust quarantine/readmit threshold", + node.peer_id.to_hex() + ); + continue; + } if self.lookup_candidate_dial_plan_is_exhausted(&node).await { // Transient skip, not a terminal failure: a // single responder's stale/suppressed (e.g. @@ -3586,6 +3684,42 @@ impl DhtNetworkManager { reason, ); } + self.enforce_trust_quarantine(peer_id).await; + } + + pub(crate) async fn enforce_trust_quarantine(&self, peer_id: &PeerId) -> bool { + let Some(ref engine) = self.trust_engine else { + return false; + }; + let trust_score = engine.score(peer_id); + let rt_events = { + let mut dht = self.dht.write().await; + dht.enforce_close_group_quarantine(peer_id, trust_score) + .await + }; + if rt_events.is_empty() { + return false; + } + info!( + "Evicted quarantined close-group peer {} with trust {:.3}", + peer_id.to_hex(), + trust_score + ); + self.broadcast_routing_events(&rt_events); + true + } + + fn peer_trust_score(&self, peer_id: &PeerId) -> f64 { + self.trust_engine + .as_ref() + .map(|engine| engine.score(peer_id)) + .unwrap_or(DEFAULT_NEUTRAL_TRUST) + } + + async fn should_avoid_automatic_peer(&self, peer_id: &PeerId) -> bool { + let trust_score = self.peer_trust_score(peer_id); + let dht = self.dht.read().await; + dht.should_avoid_for_lookup(peer_id, trust_score) } /// Ensure an identity-authenticated channel to `peer_id` exists, @@ -5942,6 +6076,8 @@ impl Default for DhtNetworkConfig { max_concurrent_operations: DEFAULT_MAX_CONCURRENT_OPS, enable_security: true, swap_threshold: 0.0, + quarantine_threshold: 0.0, + quarantine_readmit_threshold: 0.0, } } } @@ -6110,6 +6246,55 @@ mod tests { .expect("lagged receiver should conservatively wake the waiter"); } + #[tokio::test] + async fn lookup_results_do_not_hand_out_quarantined_peers() { + let local_peer = pid(0); + let quarantined_peer = pid(1); + let below_threshold_peer = pid(2); + let healthy_peer = pid(3); + let mut dht = + DhtCoreEngine::new(local_peer, 4, false, DEFAULT_NEUTRAL_TRUST - 0.15).unwrap(); + dht.set_trust_quarantine_thresholds(0.20, 0.45).unwrap(); + + dht.add_node_no_trust(routing_test_node(1)).await.unwrap(); + let events = dht + .enforce_close_group_quarantine(&quarantined_peer, 0.10) + .await; + assert!( + events + .iter() + .any(|event| matches!(event, RoutingTableEvent::PeerRemoved(id) if *id == quarantined_peer)), + "test setup should quarantine the close-group peer" + ); + + let nodes = vec![ + (routing_test_node(0), 0), + (routing_test_node(1), 0), + (routing_test_node(2), 0), + (routing_test_node(3), 0), + ]; + let trust_score = |peer_id: &PeerId| { + if *peer_id == quarantined_peer { + 0.30 + } else if *peer_id == below_threshold_peer { + 0.10 + } else { + DEFAULT_NEUTRAL_TRUST + } + }; + + let results = DhtNetworkManager::lookup_results_from_routing_nodes( + local_peer, + &dht, + nodes, + &trust_score, + 4, + ); + let result_ids: Vec = results.iter().map(|node| node.peer_id).collect(); + + assert_eq!(result_ids, vec![healthy_peer]); + } + #[test] fn is_dialable_rejects_non_quic_transports() { let ble = MultiAddr::new(crate::address::TransportAddr::Ble { @@ -7809,6 +7994,15 @@ mod tests { PeerId::from_bytes([byte; 32]) } + fn routing_test_node(byte: u8) -> NodeInfo { + NodeInfo { + id: pid(byte), + addresses: vec![MultiAddr::quic(sock(&format!("203.0.113.{byte}:9000")))], + address_types: vec![AddressType::Direct], + last_seen: AtomicInstant::now(), + } + } + #[test] fn identity_failure_cache_records_and_checks() { let cache = IdentityFailureCache::new(); diff --git a/src/network.rs b/src/network.rs index 5c0c8959..c66977f4 100644 --- a/src/network.rs +++ b/src/network.rs @@ -578,14 +578,16 @@ impl NodeConfigBuilder { /// For fine-grained control over the threshold, use /// [`adaptive_dht_config`](Self::adaptive_dht_config) instead. pub fn trust_enforcement(mut self, enabled: bool) -> Self { - let threshold = if enabled { - AdaptiveDhtConfig::default().swap_threshold + let adaptive_config = if enabled { + AdaptiveDhtConfig::default() } else { - 0.0 + AdaptiveDhtConfig { + swap_threshold: 0.0, + quarantine_threshold: 0.0, + quarantine_readmit_threshold: 0.0, + } }; - self.adaptive_dht_config = Some(AdaptiveDhtConfig { - swap_threshold: threshold, - }); + self.adaptive_dht_config = Some(adaptive_config); self } @@ -981,6 +983,8 @@ impl P2PNode { max_concurrent_operations: MAX_ACTIVE_REQUESTS, enable_security: true, swap_threshold: 0.0, // Set by AdaptiveDHT::new() from AdaptiveDhtConfig + quarantine_threshold: 0.0, // Set by AdaptiveDHT::new() from AdaptiveDhtConfig + quarantine_readmit_threshold: 0.0, // Set by AdaptiveDHT::new() }; let adaptive_dht = AdaptiveDHT::new( transport.clone(), diff --git a/tests/sybil_protection.rs b/tests/sybil_protection.rs index b3c28963..26d22df5 100644 --- a/tests/sybil_protection.rs +++ b/tests/sybil_protection.rs @@ -12,9 +12,9 @@ //! Integration tests for trust-based peer management (sybil protection). //! -//! These tests verify that low-trust peers are NOT blocked from `send_request` -//! (the lazy swap-out model only replaces them during routing table admission). -//! Trust scores are still tracked and affect routing table swap-out decisions. +//! These tests verify that low-trust peers are NOT blocked from explicit +//! `send_request` calls. Trust scores are still tracked and affect routing +//! table swap-out, close-group quarantine, and automatic lookup decisions. #![allow(clippy::unwrap_used, clippy::expect_used)] @@ -157,6 +157,7 @@ async fn custom_swap_threshold_accepted() { .ipv6(false) .adaptive_dht_config(AdaptiveDhtConfig { swap_threshold: custom_threshold, + ..Default::default() }) .build() .unwrap(); diff --git a/tests/trust_flow.rs b/tests/trust_flow.rs index 904e8fa6..7fa19542 100644 --- a/tests/trust_flow.rs +++ b/tests/trust_flow.rs @@ -309,6 +309,7 @@ async fn custom_swap_threshold_respected() { .ipv6(false) .adaptive_dht_config(AdaptiveDhtConfig { swap_threshold: custom_threshold, + ..Default::default() }) .build() .unwrap(); @@ -400,6 +401,7 @@ async fn invalid_swap_threshold_rejected() { .ipv6(false) .adaptive_dht_config(AdaptiveDhtConfig { swap_threshold: bad_threshold, + ..Default::default() }) .build(); From e58f2472077c6e727110c66e89f6ed3b61eb0011 Mon Sep 17 00:00:00 2001 From: Warm Beer Date: Wed, 20 May 2026 16:25:24 +0200 Subject: [PATCH 02/18] feat(dht)!: enhance close-group admission and quarantine logic Implement stricter trust gating for K-closest set admission and readmission, adding support for filtering newly promoted peers below the readmission threshold. Update routing table logic and wire compatibility to stabilize behavior across nodes. Extend related tests and documentation. BREAKING CHANGE: Adjusts close-group thresholds affecting trust-based peer routing and admission policies. --- docs/ROUTING_TABLE_DESIGN.md | 27 +-- docs/SECURITY_MODEL.md | 5 +- docs/trust-signals-api.md | 6 +- src/adaptive/dht.rs | 10 +- src/dht/core_engine.rs | 329 +++++++++++++++++++++++++++++++++-- src/dht_network_manager.rs | 111 ++++++++++-- 6 files changed, 439 insertions(+), 49 deletions(-) diff --git a/docs/ROUTING_TABLE_DESIGN.md b/docs/ROUTING_TABLE_DESIGN.md index a592da91..332e2d4f 100644 --- a/docs/ROUTING_TABLE_DESIGN.md +++ b/docs/ROUTING_TABLE_DESIGN.md @@ -58,7 +58,7 @@ All parameters are configurable. Values below are a reference profile used for l | `TRUST_PROTECTION_THRESHOLD` | Trust score above which a peer resists swap-closer eviction | `0.7` | | `SWAP_THRESHOLD` | Trust score below which a peer is eligible for replacement when a better candidate needs the slot | `0.35` | | `QUARANTINE_THRESHOLD` | Trust score below which automatic lookup/dial paths avoid the peer, and close-group peers are evicted immediately | `0.20` | -| `QUARANTINE_READMIT_THRESHOLD` | Trust score a quarantined peer must recover to before normal admission accepts it again | `0.45` | +| `QUARANTINE_READMIT_THRESHOLD` | Trust score required for K-closest admission/readmission after quarantine | `0.45` | | `EMA_ALPHA` | EMA smoothing factor — weight of each new observation (higher = faster response) | `0.124` | | `DECAY_LAMBDA` | Per-second exponential decay rate toward neutral (0.5) | `1.394e-5` | | `SELF_LOOKUP_INTERVAL` | Periodic self-lookup cadence (maintenance phase only; bootstrap self-lookups run back-to-back with no interval) | random in `[5 min, 10 min]` | @@ -129,7 +129,7 @@ Note: `K_BUCKET_SIZE` values below 4 produce degenerate behavior (single-peer ro 4. **Address requirement**: A `NodeInfo` with an empty address list MUST NOT be admitted to the routing table. 5. **Authenticated membership**: Only peers that have completed transport-level authentication are eligible for routing table insertion. Unauthenticated peers MUST NOT enter `LocalRT`. 6. **IP diversity**: No enforcement scope (per-bucket or routing-neighborhood) may exceed `IP_EXACT_LIMIT` nodes per exact IP or `IP_SUBNET_LIMIT` nodes per subnet, except via explicit loopback or testnet overrides. -7. **Trust quarantine**: Peers with `TrustScore(self, P) < QUARANTINE_THRESHOLD` MUST be skipped by local lookup result selection, FIND_NODE responses, and automatic lookup/dial candidate selection. If such a peer is in the K-closest-to-self set, it MUST be evicted and quarantined until `TrustScore(self, P) >= QUARANTINE_READMIT_THRESHOLD`. +7. **Trust quarantine and close-group admission**: Peers with `TrustScore(self, P) < QUARANTINE_THRESHOLD` MUST be skipped by local lookup result selection, FIND_NODE responses, and automatic lookup/dial candidate selection. If such a peer is in the K-closest-to-self set, it MUST be evicted and quarantined until `TrustScore(self, P) >= QUARANTINE_READMIT_THRESHOLD`. Any new or promoted K-closest peer MUST have `TrustScore(self, P) >= QUARANTINE_READMIT_THRESHOLD`; peers between the two thresholds may occupy non-close routing-table slots but must not enter the K-closest set. 8. **Trust protection (staleness-gated)**: A peer with `TrustScore(self, P) >= TRUST_PROTECTION_THRESHOLD` **AND** `last_seen` within `LIVE_THRESHOLD` MUST NOT be evicted by swap-closer admission. A peer whose `last_seen` exceeds `LIVE_THRESHOLD` receives no trust protection regardless of score — stale peers MUST NOT hold slots against live candidates. 9. **Deterministic distance**: `Distance(A, B)` is symmetric, deterministic, and consistent across all nodes. Two nodes compute the same distance between the same pair of keys. 10. **Atomic admission**: IP diversity checks, capacity checks, swap-closer evictions, trust score reads, and insertion MUST execute within a single write-locked critical section to prevent TOCTOU races. All `TrustScore` queries during admission (steps 4, 8) MUST occur while the routing table write lock is held. @@ -185,9 +185,9 @@ When a candidate peer `P` with `NodeInfo` and IP address `candidate_ip` is prese 2. **Address check**: If `P.addresses` is empty, reject. 3. **Authentication check**: If `P` has not completed transport-level authentication, reject. 4. **Trust quarantine check**: If `TrustScore(self, P) < QUARANTINE_THRESHOLD`, reject. If `P` was previously quarantined, reject until `TrustScore(self, P) >= QUARANTINE_READMIT_THRESHOLD`. -5. **Update short-circuit**: If `P` already exists in `KBucket(BucketIndex(self, P))`, merge addresses (Section 6.3), refresh `last_seen`, move `P` to tail, and return. The peer already holds its slot — IP diversity and capacity checks are skipped. -6. **Loopback check**: If `candidate_ip` is loopback and loopback is disallowed, reject. If loopback is allowed, skip all IP diversity checks (step 7–8) and proceed directly to step 9. -7. **Non-IP transport bypass**: If `P` has no IP-based address (e.g., Bluetooth, LoRa), skip IP diversity checks and proceed directly to step 9. +5. **Update short-circuit**: If `P` already exists in `KBucket(BucketIndex(self, P))`, merge addresses (Section 6.3), refresh `last_seen`, move `P` to tail, and return. The peer already holds its slot — IP diversity, capacity, and close-group admission checks are skipped. +6. **Loopback check**: If `candidate_ip` is loopback and loopback is disallowed, reject. If loopback is allowed, skip all IP diversity checks (step 7–9) and proceed directly to the close-group admission check (step 10). +7. **Non-IP transport bypass**: If `P` has no IP-based address (e.g., Bluetooth, LoRa), skip IP diversity checks and proceed directly to the close-group admission check (step 10). 8. **IP diversity enforcement** (under write lock — Invariant 10): a. Compute `bucket_idx = BucketIndex(self, P)`. b. Run per-bucket IP diversity check (Section 7.2) against nodes in `KBucket(bucket_idx)`. @@ -213,9 +213,11 @@ When a candidate peer `P` with `NodeInfo` and IP address `candidate_ip` is prese - Per-bucket IP diversity (step 8b): bucket composition may have changed. - Routing-neighborhood IP diversity (step 8c): K-closest set may have changed. - Capacity pre-check (this step): slots may have been filled by concurrent admissions. + - Close-group admission check (step 10): the candidate may now enter the K-closest set after stale evictions. Steps 1–3, 5–7 are not re-evaluated (candidate identity, addresses, authentication, and loopback status are immutable within a single admission attempt). Re-evaluation MUST NOT trigger a second round of stale revalidation — if any check fails during re-evaluation, reject the candidate. This bounds admission latency to a single `STALE_REVALIDATION_TIMEOUT` per admission attempt with a single lock-release window. This prevents TOCTOU races caused by concurrent mutations during the unlocked ping window. If revalidation frees at least one slot and re-evaluation passes, proceed. If no slots freed or re-evaluation fails, reject. -10. **Execute swaps**: Remove all deduplicated swap candidates. Disconnect evicted peers at the transport layer. -11. **Insert**: Add `P` to `KBucket(bucket_idx)`. +10. **Close-group admission check**: Before mutating the routing table, compute whether `P` would enter the K-closest-to-self set after planned swaps. If so, require `TrustScore(self, P) >= QUARANTINE_READMIT_THRESHOLD`; otherwise reject without mutating the routing table. Existing close-group peers above `QUARANTINE_THRESHOLD` are not evicted merely for being below this admission threshold. +11. **Execute swaps**: Remove all deduplicated swap candidates. Disconnect evicted peers at the transport layer. +12. **Insert**: Add `P` to `KBucket(bucket_idx)`. ### 7.2 IP Diversity Enforcement @@ -261,14 +263,15 @@ When any interaction records a trust failure and `TrustScore(self, P)` drops bel 3. Mark `P` as quarantined if it was evicted from the close group. 4. Do not re-admit quarantined `P` until `TrustScore(self, P) >= QUARANTINE_READMIT_THRESHOLD`. 5. If `P` is not in the K-closest-to-self set, it may remain in the routing table, but local lookup result selection, FIND_NODE responses, and automatic lookup/dial paths MUST avoid it while `TrustScore(self, P) < QUARANTINE_THRESHOLD`. +6. If `P` has `QUARANTINE_THRESHOLD <= TrustScore(self, P) < QUARANTINE_READMIT_THRESHOLD`, it may occupy a non-close routing-table slot. If it would newly enter the K-closest set through admission or promotion, remove/reject it until its trust reaches `QUARANTINE_READMIT_THRESHOLD`. Existing close-group peers in this range are retained until they drop below `QUARANTINE_THRESHOLD`. Quarantine is a routing-table and automatic lookup policy. It is not a blanket transport-level block for explicit user-initiated sends. Re-admission path: a quarantined peer can only re-enter when its trust score recovers above `QUARANTINE_READMIT_THRESHOLD` through time-decay toward neutral AND the peer is rediscovered through normal network activity: 1. Peer `P` is returned in a `FIND_NODE` response from another peer during a lookup, or connects through the normal authenticated peer path. -2. Local node checks `TrustScore(self, P)`. If still below `QUARANTINE_READMIT_THRESHOLD` for a quarantined peer, `P` is skipped/rejected. -3. If trust has recovered to `QUARANTINE_READMIT_THRESHOLD`, the standard admission flow (Section 7.1) applies. +2. Local node checks `TrustScore(self, P)`. If still below `QUARANTINE_READMIT_THRESHOLD` for a quarantined peer, or if admitting it would place it in the K-closest set, `P` is skipped/rejected. +3. If trust has recovered to `QUARANTINE_READMIT_THRESHOLD`, the standard admission flow (Section 7.1) applies. Peers below that threshold can still be admitted to non-close routing-table slots as long as they are not quarantined and are at or above `QUARANTINE_THRESHOLD`. No manual probing is required. Natural rediscovery plus trust decay is the temporary-ban mechanism. @@ -585,9 +588,9 @@ All events — internal and consumer-reported — follow the same path through t 1. **Event received**: `report_trust_event(P, event)` is called (by DHT internals or by the consumer). 2. **Category mapping**: Event mapped to positive (successful interaction) or negative (failed interaction). 3. **Weight resolution**: Internal events have implicit weight `1.0`. Consumer events use their caller-specified weight (after validation/clamping). -4. **EMA update**: The trust engine applies time decay, then blends the observation using the EMA model (Section 4). Positive events use observation `1.0`, negative events use `0.0`. The weight scales influence via the continuous formula `score = (1 - EMA_ALPHA)^W * score + (1 - (1 - EMA_ALPHA)^W) * observation`, which generalizes naturally to fractional weights. At reference values (`EMA_ALPHA = 0.3`), a single weight-1.0 failure moves a neutral peer's score from 0.5 to 0.35; a single weight-5.0 failure moves it from 0.5 to ~0.08. +4. **EMA update**: The trust engine applies time decay, then blends the observation using the EMA model (Section 4). Positive events use observation `1.0`, negative events use `0.0`. The weight scales influence via the continuous formula `score = (1 - EMA_ALPHA)^W * score + (1 - (1 - EMA_ALPHA)^W) * observation`, which generalizes naturally to fractional weights. At reference values (`EMA_ALPHA = 0.124`), a single weight-1.0 failure moves a neutral peer's score from 0.5 to ~0.438; a single weight-5.0 failure moves it from 0.5 to ~0.26. 5. **Threshold checks**: - a. **Quarantine check**: If `TrustScore(self, P)` dropped below `QUARANTINE_THRESHOLD`, local lookup results, FIND_NODE responses, and automatic lookup/dial paths avoid the peer. If it is in the K-closest-to-self set, trigger quarantine handling (Section 7.4). + a. **Quarantine check**: If `TrustScore(self, P)` dropped below `QUARANTINE_THRESHOLD`, local lookup results, FIND_NODE responses, and automatic lookup/dial paths avoid the peer. If it is in the K-closest-to-self set, trigger quarantine handling (Section 7.4). This is local routing policy only; it MUST NOT change the FIND_NODE/DHTNode wire shape or reinterpret legacy wire fields, because nodes must remain interoperable with older releases. b. **Swap eligibility**: If `TrustScore(self, P)` dropped below `SWAP_THRESHOLD`, the peer is eligible for lazy replacement when a better candidate needs its slot. c. **Protection evaluation**: If `TrustScore(self, P)` crossed `TRUST_PROTECTION_THRESHOLD` in either direction, the peer's swap-closer protection status changes accordingly (Section 7.3). @@ -698,6 +701,7 @@ Each scenario should assert exact expected outcomes and state transitions. 4. **Quarantined peer rejection**: - Peer with `TrustScore < QUARANTINE_THRESHOLD`, or previously quarantined peer with trust below `QUARANTINE_READMIT_THRESHOLD`. Rejected. Not in routing table. + - Peer with `QUARANTINE_THRESHOLD <= TrustScore < QUARANTINE_READMIT_THRESHOLD` is admitted when it would occupy a non-close routing-table slot. The same peer is rejected when it would enter the K-closest set. 5. **Bucket-full rejection (no stale peers)**: - Bucket at `K_BUCKET_SIZE` capacity, candidate cannot swap-closer, all incumbent peers have `last_seen` within `LIVE_THRESHOLD`. Stale revalidation finds no candidates. Rejected with "bucket at capacity." Routing table unchanged. @@ -800,6 +804,7 @@ Each scenario should assert exact expected outcomes and state transitions. 36. **Close-group quarantine eviction**: - K-closest peer trust drops below 0.20 after failed interaction. Peer is immediately removed from routing table, its trust record is retained, and `PeerRemoved` is emitted. + - Non-close peer with trust 0.30 is retained in the routing table. A close-group removal promotes it into the K-closest set. Local trust gate removes it from the routing table without marking it as below-threshold quarantine. 37. **Quarantined peer inbound admission rejected**: - Quarantined peer initiates inbound connection. Transport authenticates the peer, the normal routing-table admission path checks trust, and admission is rejected until trust reaches 0.45. diff --git a/docs/SECURITY_MODEL.md b/docs/SECURITY_MODEL.md index 3df1a9c2..a48045f5 100644 --- a/docs/SECURITY_MODEL.md +++ b/docs/SECURITY_MODEL.md @@ -88,13 +88,16 @@ Trust affects routing-table membership in two stages: |-----------------|-------------------|---------------| | Lazy swap eligibility | < 0.35 | `swap_threshold` | | Close-group quarantine / lookup avoidance | < 0.20 | `quarantine_threshold` | -| Quarantine readmission | >= 0.45 | `quarantine_readmit_threshold` | +| Close-group admission / quarantine readmission | >= 0.45 | `quarantine_readmit_threshold` | | Staleness | Configurable | `stale_timeout` | Peers outside the K-closest set are not globally evicted solely for low trust. They are omitted from local lookup results, FIND_NODE responses, and automatic lookup paths below the quarantine threshold, and can be lazily replaced when better candidates need the slot. +Peers at or above the quarantine threshold but below the readmission threshold +may occupy non-close routing-table slots, but cannot newly enter the K-closest +set until they recover to the readmission threshold. ### Quarantine Reasons diff --git a/docs/trust-signals-api.md b/docs/trust-signals-api.md index bf842fae..b0da4fa5 100644 --- a/docs/trust-signals-api.md +++ b/docs/trust-signals-api.md @@ -88,7 +88,9 @@ The routing table uses three trust thresholds: and quarantined. - `quarantine_readmit_threshold` (`0.45` by default): a quarantined peer can only re-enter through normal discovery/admission after its decayed trust - reaches this score. + reaches this score. New or promoted K-closest peers must also meet this + threshold; peers between `0.20` and `0.45` may occupy non-close routing-table + slots. ```rust use saorsa_core::AdaptiveDhtConfig; @@ -104,6 +106,8 @@ let config = AdaptiveDhtConfig { Raw DHT routing uses Kademlia XOR distance. Local lookup results, FIND_NODE responses, and automatic network lookups avoid quarantined peers so known-bad contacts do not consume query slots or get handed out as lookup candidates. +This filtering is local policy only; the DHT wire protocol and legacy +`DHTNode` fields remain unchanged for backwards compatibility with older nodes. ## Architecture diff --git a/src/adaptive/dht.rs b/src/adaptive/dht.rs index 70ec6852..bbde1717 100644 --- a/src/adaptive/dht.rs +++ b/src/adaptive/dht.rs @@ -55,8 +55,8 @@ pub struct AdaptiveDhtConfig { /// K-closest peers are evicted immediately into temporary quarantine. /// Default: 0.20 pub quarantine_threshold: f64, - /// Trust score a quarantined peer must decay back to before normal - /// discovery/admission can accept it again. + /// Trust score required before a quarantined peer can re-enter, and before + /// any new or promoted peer can enter the K-closest set. /// Default: 0.45 pub quarantine_readmit_threshold: f64, } @@ -76,9 +76,9 @@ impl AdaptiveDhtConfig { /// /// Returns `Err` if a threshold is outside its safe range or is NaN. /// Values >= 0.5 (neutral trust) would make all unknown peers immediately - /// swap/quarantine eligible since they start at neutral (0.5). The readmit - /// threshold must also stay below neutral because quarantined peers recover - /// by decay toward neutral, not by active probing. + /// swap/quarantine eligible since they start at neutral (0.5). The + /// close-group admission/readmit threshold must also stay below neutral + /// because recovery happens by decay toward neutral, not by active probing. pub fn validate(&self) -> crate::error::P2pResult<()> { if !(0.0..0.5).contains(&self.swap_threshold) || self.swap_threshold.is_nan() { return Err(crate::error::P2PError::Validation( diff --git a/src/dht/core_engine.rs b/src/dht/core_engine.rs index 2acab1e0..70736960 100644 --- a/src/dht/core_engine.rs +++ b/src/dht/core_engine.rs @@ -207,7 +207,7 @@ const DEFAULT_SWAP_THRESHOLD: f64 = 0.35; #[allow(dead_code)] const DEFAULT_QUARANTINE_THRESHOLD: f64 = 0.20; -/// Default trust score required before a quarantined peer can be admitted again. +/// Default trust score required for K-closest admission/readmission. #[allow(dead_code)] const DEFAULT_QUARANTINE_READMIT_THRESHOLD: f64 = 0.45; @@ -1473,8 +1473,8 @@ pub struct DhtCoreEngine { /// paths, and evicted immediately if it is in the K-closest close group. quarantine_threshold: f64, - /// Trust score a quarantined peer must decay back to before normal - /// admission can accept it again. + /// Trust score required before a quarantined peer can re-enter, and before + /// any new or promoted peer can enter the K-closest set. quarantine_readmit_threshold: f64, /// Peers evicted from the close group by quarantine. They remain in this @@ -1541,22 +1541,23 @@ impl DhtCoreEngine { /// Otherwise, peers below that score are avoided for automatic lookups, /// and K-closest peers below it are immediately evicted and quarantined. /// Quarantined peers can only re-enter through normal admission after - /// their decayed trust reaches `quarantine_readmit_threshold`. + /// their decayed trust reaches `quarantine_readmit_threshold`; new or + /// promoted K-closest peers must also meet that threshold. pub(crate) fn set_trust_quarantine_thresholds( &mut self, quarantine_threshold: f64, quarantine_readmit_threshold: f64, ) -> Result<()> { - if !(0.0..1.0).contains(&quarantine_threshold) || quarantine_threshold.is_nan() { + if !(0.0..0.5).contains(&quarantine_threshold) || quarantine_threshold.is_nan() { return Err(anyhow!( - "quarantine_threshold must be in [0.0, 1.0), got {quarantine_threshold}" + "quarantine_threshold must be in [0.0, 0.5), got {quarantine_threshold}" )); } - if !(0.0..1.0).contains(&quarantine_readmit_threshold) + if !(0.0..0.5).contains(&quarantine_readmit_threshold) || quarantine_readmit_threshold.is_nan() { return Err(anyhow!( - "quarantine_readmit_threshold must be in [0.0, 1.0), got {quarantine_readmit_threshold}" + "quarantine_readmit_threshold must be in [0.0, 0.5), got {quarantine_readmit_threshold}" )); } if quarantine_threshold > 0.0 && quarantine_readmit_threshold < quarantine_threshold { @@ -1588,14 +1589,6 @@ impl DhtCoreEngine { peer_id.to_hex() )); } - if trust_score < self.quarantine_threshold { - self.quarantined_peers.insert(*peer_id); - return Err(anyhow!( - "peer {} below quarantine threshold ({trust_score:.3} < {:.3})", - peer_id.to_hex(), - self.quarantine_threshold - )); - } if self.quarantined_peers.contains(peer_id) { if trust_score < self.quarantine_readmit_threshold { return Err(anyhow!( @@ -1606,6 +1599,66 @@ impl DhtCoreEngine { } self.quarantined_peers.remove(peer_id); } + if trust_score < self.quarantine_threshold { + return Err(anyhow!( + "peer {} below quarantine threshold ({trust_score:.3} < {:.3})", + peer_id.to_hex(), + self.quarantine_threshold + )); + } + Ok(()) + } + + fn candidate_enters_close_group_after_removals( + &self, + routing: &KademliaRoutingTable, + candidate_id: &PeerId, + removed_peer_ids: &[PeerId], + ) -> bool { + let mut candidates: Vec<(PeerId, [u8; 32])> = routing + .all_nodes() + .into_iter() + .filter(|node| node.id != *candidate_id && !removed_peer_ids.contains(&node.id)) + .map(|node| { + let distance = xor_distance_bytes(self.node_id.to_bytes(), node.id.to_bytes()); + (node.id, distance) + }) + .collect(); + + candidates.push(( + *candidate_id, + xor_distance_bytes(self.node_id.to_bytes(), candidate_id.to_bytes()), + )); + candidates.sort_by_key(|(_, distance)| *distance); + candidates + .into_iter() + .take(self.k_value) + .any(|(peer_id, _)| peer_id == *candidate_id) + } + + fn check_close_group_admission( + &self, + routing: &KademliaRoutingTable, + peer_id: &PeerId, + trust_score: f64, + removed_peer_ids: &[PeerId], + ) -> Result<()> { + if !self.quarantine_enabled() || trust_score >= self.quarantine_readmit_threshold { + return Ok(()); + } + if !trust_score.is_finite() { + return Err(anyhow!( + "peer {} has non-finite trust score", + peer_id.to_hex() + )); + } + if self.candidate_enters_close_group_after_removals(routing, peer_id, removed_peer_ids) { + return Err(anyhow!( + "peer {} below close-group admission threshold ({trust_score:.3} < {:.3})", + peer_id.to_hex(), + self.quarantine_readmit_threshold + )); + } Ok(()) } @@ -1623,6 +1676,7 @@ impl DhtCoreEngine { } /// Evict a quarantined peer if it currently occupies the K-closest set. + #[cfg(test)] pub(crate) async fn enforce_close_group_quarantine( &mut self, peer_id: &PeerId, @@ -1658,6 +1712,60 @@ impl DhtCoreEngine { events } + /// Enforce trust gates over the current K-closest set. + /// + /// Peers below the quarantine threshold are evicted from the close group + /// regardless of whether they were already close. Peers newly promoted into + /// the close group must meet the higher readmit/admission threshold. + pub(crate) async fn enforce_close_group_trust_gate( + &mut self, + previous_close_group: Option<&[PeerId]>, + trust_score: &impl Fn(&PeerId) -> f64, + ) -> Vec { + if !self.quarantine_enabled() { + return Vec::new(); + } + + let mut routing = self.routing_table.write().await; + let k_before = routing.k_closest_ids(self.k_value); + let mut removed = Vec::new(); + + while let Some(peer_id) = routing + .k_closest_ids(self.k_value) + .into_iter() + .find(|peer_id| { + let score = trust_score(peer_id); + score.is_finite() + && (score < self.quarantine_threshold + || (previous_close_group.is_some_and(|old| !old.contains(peer_id)) + && score < self.quarantine_readmit_threshold)) + }) + { + if trust_score(&peer_id) < self.quarantine_threshold { + self.quarantined_peers.insert(peer_id); + } + routing.remove_node(&peer_id); + removed.push(peer_id); + } + + if removed.is_empty() { + return Vec::new(); + } + + let k_after = routing.k_closest_ids(self.k_value); + let mut events: Vec = removed + .into_iter() + .map(RoutingTableEvent::PeerRemoved) + .collect(); + if k_before != k_after { + events.push(RoutingTableEvent::KClosestPeersChanged { + old: k_before, + new: k_after, + }); + } + events + } + /// Override the IP diversity configuration. pub fn set_ip_diversity_config(&mut self, config: IPDiversityConfig) { self.ip_diversity_config = config; @@ -2173,7 +2281,8 @@ impl DhtCoreEngine { )); } - self.check_quarantine_admission(&peer_id, trust_score(&peer_id))?; + let peer_trust_score = trust_score(&peer_id); + self.check_quarantine_admission(&peer_id, peer_trust_score)?; // Extract ALL IP addresses from the candidate for diversity checking. // If candidate has no IP-based addresses, it's a non-IP transport — bypass diversity. @@ -2197,6 +2306,7 @@ impl DhtCoreEngine { } return Ok(AdmissionResult::Admitted(vec![])); } + self.check_close_group_admission(&routing, &peer_id, peer_trust_score, &[])?; let k_before = routing.k_closest_ids(self.k_value); routing.add_node(node)?; let k_after = routing.k_closest_ids(self.k_value); @@ -2504,6 +2614,7 @@ impl DhtCoreEngine { } return Ok(AdmissionResult::Admitted(vec![])); } + self.check_close_group_admission(routing, &peer_id, trust_score(&peer_id), &[])?; let k_before = routing.k_closest_ids(self.k_value); routing.add_node(node)?; let k_after = routing.k_closest_ids(self.k_value); @@ -2744,6 +2855,21 @@ impl DhtCoreEngine { } } + let mut planned_removals: Vec = + Vec::with_capacity(all_bucket_swaps.len() + all_close_swaps.len()); + planned_removals.extend(all_bucket_swaps.iter().copied()); + for peer_id in &all_close_swaps { + if !planned_removals.contains(peer_id) { + planned_removals.push(*peer_id); + } + } + self.check_close_group_admission( + routing, + &peer_id, + trust_score(&peer_id), + &planned_removals, + )?; + // === Snapshot K-closest BEFORE mutation === let k_before = routing.k_closest_ids(self.k_value); @@ -5110,6 +5236,175 @@ mod tests { assert!(dht.has_node(&peer).await); } + #[test] + fn test_core_quarantine_readmit_threshold_must_be_reachable_by_decay() { + let mut dht = DhtCoreEngine::new_for_tests(PeerId::from_bytes([0u8; 32])).unwrap(); + assert!(dht.set_trust_quarantine_thresholds(0.20, 0.49).is_ok()); + assert!(dht.set_trust_quarantine_thresholds(0.20, 0.50).is_err()); + } + + /// A first-time admission rejected below the quarantine threshold is not a + /// close-group quarantine. Once it recovers above the quarantine threshold, + /// it can enter a non-close routing-table slot without waiting for the + /// stronger close-group admission threshold. + #[tokio::test] + async fn test_below_threshold_admission_can_recover_into_non_close_slot() { + let mut dht = DhtCoreEngine::new( + PeerId::from_bytes([0u8; 32]), + 4, + false, + DEFAULT_SWAP_THRESHOLD, + ) + .unwrap(); + dht.set_trust_quarantine_thresholds(0.20, 0.45).unwrap(); + + for i in 1..=4u8 { + let mut id = [0u8; 32]; + id[31] = i; + dht.add_node_no_trust(make_node_with_addr( + id, + &format!("/ip4/10.{i}.0.1/udp/9000/quic"), + )) + .await + .unwrap(); + } + + let mut peer_id_bytes = [0u8; 32]; + peer_id_bytes[0] = 0x80; + let peer = PeerId::from_bytes(peer_id_bytes); + + let rejected = dht + .add_node( + make_node_with_addr(peer_id_bytes, "/ip4/10.10.0.9/udp/9000/quic"), + &|id| if *id == peer { 0.10 } else { 0.5 }, + ) + .await; + assert!( + rejected.is_err(), + "peer below quarantine threshold should be rejected" + ); + + let recovered_above_quarantine = dht + .add_node( + make_node_with_addr(peer_id_bytes, "/ip4/10.10.0.9/udp/9000/quic"), + &|id| if *id == peer { 0.30 } else { 0.5 }, + ) + .await; + assert!( + recovered_above_quarantine.is_ok(), + "non-close peer should not need the 0.45 close-group threshold" + ); + assert!(dht.has_node(&peer).await); + } + + /// A new peer that would enter the K-closest set must meet the close-group + /// admission threshold, even if it is above the lower quarantine threshold. + #[tokio::test] + async fn test_close_group_admission_requires_readmit_threshold() { + let mut dht = DhtCoreEngine::new_for_tests(PeerId::from_bytes([0u8; 32])).unwrap(); + dht.set_trust_quarantine_thresholds(0.20, 0.45).unwrap(); + + let mut peer_id_bytes = [0u8; 32]; + peer_id_bytes[31] = 9; + let peer = PeerId::from_bytes(peer_id_bytes); + + let below_close_group_threshold = dht + .add_node( + make_node_with_addr(peer_id_bytes, "/ip4/10.10.0.9/udp/9000/quic"), + &|id| if *id == peer { 0.30 } else { 0.5 }, + ) + .await; + assert!( + below_close_group_threshold.is_err(), + "new close-group peer should need trust >= 0.45" + ); + + let recovered = dht + .add_node( + make_node_with_addr(peer_id_bytes, "/ip4/10.10.0.9/udp/9000/quic"), + &|id| if *id == peer { 0.45 } else { 0.5 }, + ) + .await; + assert!( + recovered.is_ok(), + "new close-group peer should enter once trust reaches 0.45" + ); + assert!(dht.has_node(&peer).await); + } + + /// Removing one close peer can promote a non-close peer into the close + /// group. Promoted peers below the close-group admission threshold are + /// removed, while existing close peers above the quarantine threshold stay. + #[tokio::test] + async fn test_close_group_gate_removes_promoted_peers_below_readmit_threshold() { + let mut dht = DhtCoreEngine::new( + PeerId::from_bytes([0u8; 32]), + 4, + false, + DEFAULT_SWAP_THRESHOLD, + ) + .unwrap(); + dht.set_trust_quarantine_thresholds(0.20, 0.45).unwrap(); + + let mut close_peer_ids = Vec::new(); + for i in 1..=4u8 { + let mut id = [0u8; 32]; + id[31] = i; + close_peer_ids.push(PeerId::from_bytes(id)); + dht.add_node_no_trust(make_node_with_addr( + id, + &format!("/ip4/10.{i}.0.1/udp/9000/quic"), + )) + .await + .unwrap(); + } + let previous_close_group = close_peer_ids.clone(); + + let mut promoted_id = [0u8; 32]; + promoted_id[0] = 0x80; + let promoted_peer = PeerId::from_bytes(promoted_id); + dht.add_node_no_trust(make_node_with_addr( + promoted_id, + "/ip4/10.99.0.1/udp/9000/quic", + )) + .await + .unwrap(); + + let no_events = dht + .enforce_close_group_trust_gate(Some(&previous_close_group), &|id| { + if *id == promoted_peer { 0.30 } else { 0.5 } + }) + .await; + assert!( + no_events.is_empty(), + "peer outside the close group should not be evicted by close-group gate" + ); + assert!(dht.has_node(&promoted_peer).await); + + dht.remove_node_by_id(&close_peer_ids[0]).await; + let events = dht + .enforce_close_group_trust_gate(Some(&previous_close_group), &|id| { + if *id == promoted_peer || *id == close_peer_ids[1] { + 0.30 + } else { + 0.5 + } + }) + .await; + + assert!( + events.iter().any( + |event| matches!(event, RoutingTableEvent::PeerRemoved(id) if *id == promoted_peer) + ), + "promoted peer below close-group admission threshold should be removed" + ); + assert!(!dht.has_node(&promoted_peer).await); + assert!( + dht.has_node(&close_peer_ids[1]).await, + "existing close peer above quarantine threshold should stay" + ); + } + /// A non-close peer below the quarantine threshold is avoided by automatic /// lookup paths but is not removed merely for being below threshold. #[tokio::test] diff --git a/src/dht_network_manager.rs b/src/dht_network_manager.rs index 1b5b6d00..22c13dc9 100644 --- a/src/dht_network_manager.rs +++ b/src/dht_network_manager.rs @@ -638,8 +638,8 @@ pub struct DhtNetworkConfig { /// K-closest peers are immediately evicted into temporary quarantine. /// Default: 0.0 (disabled). pub quarantine_threshold: f64, - /// Trust score required before a quarantined peer can be admitted again - /// through the normal discovery/admission path. + /// Trust score required before a quarantined peer can be admitted again, + /// and before any new or promoted peer can enter the K-closest set. /// Default: 0.0 (disabled). pub quarantine_readmit_threshold: f64, } @@ -2729,7 +2729,10 @@ impl DhtNetworkManager { address_types: node.address_types, addresses: node.addresses, distance: encode_publish_seq_distance(publish_seq), - reliability, + // Keep the legacy wire value stable. Trust is local policy + // used for filtering and should not change DHTNode wire + // semantics for older nodes. + reliability: SELF_RELIABILITY_SCORE, }) }) .take(count) @@ -3102,7 +3105,8 @@ impl DhtNetworkManager { let mut dht = self.dht.write().await; let rt_events = dht.remove_node_by_id(&peer_id).await; drop(dht); - self.broadcast_routing_events(&rt_events); + self.broadcast_routing_events_with_quarantine(rt_events) + .await; let _ = self.transport.disconnect_peer(&peer_id).await; } Ok(_) => { @@ -3692,16 +3696,12 @@ impl DhtNetworkManager { return false; }; let trust_score = engine.score(peer_id); - let rt_events = { - let mut dht = self.dht.write().await; - dht.enforce_close_group_quarantine(peer_id, trust_score) - .await - }; + let rt_events = self.enforce_close_group_trust_gate(None).await; if rt_events.is_empty() { return false; } info!( - "Evicted quarantined close-group peer {} with trust {:.3}", + "Evicted quarantined close-group peer(s) after trust update for {} with trust {:.3}", peer_id.to_hex(), trust_score ); @@ -3853,7 +3853,8 @@ impl DhtNetworkManager { let mut dht = self.dht.write().await; dht.remove_node_by_id(peer_id).await }; - self.broadcast_routing_events(&rt_events); + self.broadcast_routing_events_with_quarantine(rt_events) + .await; } // Broadcast and clear. Remove BEFORE sending so any task @@ -5173,7 +5174,8 @@ impl DhtNetworkManager { match add_result { Ok(AdmissionResult::Admitted(rt_events)) => { info!("Added peer {} to DHT routing table", app_peer_id_hex); - self.broadcast_routing_events(&rt_events); + self.broadcast_routing_events_with_quarantine(rt_events) + .await; } Ok(AdmissionResult::StaleRevalidationNeeded { candidate, @@ -5261,7 +5263,8 @@ impl DhtNetworkManager { "Added peer {} to DHT routing table after stale revalidation", app_peer_id_hex ); - this.broadcast_routing_events(&rt_events); + this.broadcast_routing_events_with_quarantine(rt_events) + .await; } Err(e) => { warn!( @@ -5667,10 +5670,84 @@ impl DhtNetworkManager { events }; - self.broadcast_routing_events(&all_events); + self.broadcast_routing_events_with_quarantine(all_events) + .await; info!("Evicted {} offline K-closest peer(s)", non_responders.len()); } + fn routing_events_include_close_group_change(events: &[RoutingTableEvent]) -> bool { + events + .iter() + .any(|event| matches!(event, RoutingTableEvent::KClosestPeersChanged { .. })) + } + + async fn enforce_close_group_trust_gate( + &self, + previous_close_group: Option<&[PeerId]>, + ) -> Vec { + let Some(ref engine) = self.trust_engine else { + return Vec::new(); + }; + let trust_score = |peer_id: &PeerId| engine.score(peer_id); + let rt_events = { + let mut dht = self.dht.write().await; + dht.enforce_close_group_trust_gate(previous_close_group, &trust_score) + .await + }; + if !rt_events.is_empty() { + let removed: Vec = rt_events + .iter() + .filter_map(|event| match event { + RoutingTableEvent::PeerRemoved(peer_id) => Some(peer_id.to_hex()), + _ => None, + }) + .collect(); + info!("Removed close-group trust-gated peer(s): {:?}", removed); + } + rt_events + } + + async fn broadcast_routing_events_with_quarantine(&self, mut events: Vec) { + if !Self::routing_events_include_close_group_change(&events) { + self.broadcast_routing_events(&events); + return; + } + + let original_old_close_group = events.iter().find_map(|event| match event { + RoutingTableEvent::KClosestPeersChanged { old, .. } => Some(old.clone()), + _ => None, + }); + let quarantine_events = self + .enforce_close_group_trust_gate(original_old_close_group.as_deref()) + .await; + if quarantine_events.is_empty() { + self.broadcast_routing_events(&events); + return; + } + + let final_new_close_group = quarantine_events + .iter() + .rev() + .find_map(|event| match event { + RoutingTableEvent::KClosestPeersChanged { new, .. } => Some(new.clone()), + _ => None, + }); + + events.retain(|event| !matches!(event, RoutingTableEvent::KClosestPeersChanged { .. })); + events.extend( + quarantine_events + .into_iter() + .filter(|event| !matches!(event, RoutingTableEvent::KClosestPeersChanged { .. })), + ); + if let (Some(old), Some(new)) = (original_old_close_group, final_new_close_group) + && old != new + { + events.push(RoutingTableEvent::KClosestPeersChanged { old, new }); + } + + self.broadcast_routing_events(&events); + } + /// Translate core engine routing table events into network events and broadcast them. fn broadcast_routing_events(&self, events: &[RoutingTableEvent]) { if self.event_tx.receiver_count() == 0 { @@ -6293,6 +6370,12 @@ mod tests { let result_ids: Vec = results.iter().map(|node| node.peer_id).collect(); assert_eq!(result_ids, vec![healthy_peer]); + assert!( + results + .iter() + .all(|node| (node.reliability - SELF_RELIABILITY_SCORE).abs() < f64::EPSILON), + "trust filtering must not change the legacy DHTNode reliability wire value" + ); } #[test] From 39d696eee8224a58574b37128784cea185cf10e8 Mon Sep 17 00:00:00 2001 From: Warm Beer Date: Wed, 20 May 2026 18:22:19 +0200 Subject: [PATCH 03/18] feat(dht)!: require admission trust for new routing peers Gate all new routing-table admissions at quarantine_readmit_threshold while allowing existing routing-table peers above the quarantine threshold to stay and move into the close group. BREAKING CHANGE: new peers below quarantine_readmit_threshold are no longer admitted to the routing table, even for non-close routing slots. --- docs/ROUTING_TABLE_DESIGN.md | 30 ++--- docs/SECURITY_MODEL.md | 9 +- docs/trust-signals-api.md | 6 +- src/adaptive/dht.rs | 8 +- src/dht/core_engine.rs | 226 ++++++++++++++++++----------------- src/dht_network_manager.rs | 17 +-- 6 files changed, 154 insertions(+), 142 deletions(-) diff --git a/docs/ROUTING_TABLE_DESIGN.md b/docs/ROUTING_TABLE_DESIGN.md index 332e2d4f..af89a54d 100644 --- a/docs/ROUTING_TABLE_DESIGN.md +++ b/docs/ROUTING_TABLE_DESIGN.md @@ -58,7 +58,7 @@ All parameters are configurable. Values below are a reference profile used for l | `TRUST_PROTECTION_THRESHOLD` | Trust score above which a peer resists swap-closer eviction | `0.7` | | `SWAP_THRESHOLD` | Trust score below which a peer is eligible for replacement when a better candidate needs the slot | `0.35` | | `QUARANTINE_THRESHOLD` | Trust score below which automatic lookup/dial paths avoid the peer, and close-group peers are evicted immediately | `0.20` | -| `QUARANTINE_READMIT_THRESHOLD` | Trust score required for K-closest admission/readmission after quarantine | `0.45` | +| `QUARANTINE_READMIT_THRESHOLD` | Trust score required for new routing-table admission/readmission after quarantine | `0.45` | | `EMA_ALPHA` | EMA smoothing factor — weight of each new observation (higher = faster response) | `0.124` | | `DECAY_LAMBDA` | Per-second exponential decay rate toward neutral (0.5) | `1.394e-5` | | `SELF_LOOKUP_INTERVAL` | Periodic self-lookup cadence (maintenance phase only; bootstrap self-lookups run back-to-back with no interval) | random in `[5 min, 10 min]` | @@ -129,10 +129,10 @@ Note: `K_BUCKET_SIZE` values below 4 produce degenerate behavior (single-peer ro 4. **Address requirement**: A `NodeInfo` with an empty address list MUST NOT be admitted to the routing table. 5. **Authenticated membership**: Only peers that have completed transport-level authentication are eligible for routing table insertion. Unauthenticated peers MUST NOT enter `LocalRT`. 6. **IP diversity**: No enforcement scope (per-bucket or routing-neighborhood) may exceed `IP_EXACT_LIMIT` nodes per exact IP or `IP_SUBNET_LIMIT` nodes per subnet, except via explicit loopback or testnet overrides. -7. **Trust quarantine and close-group admission**: Peers with `TrustScore(self, P) < QUARANTINE_THRESHOLD` MUST be skipped by local lookup result selection, FIND_NODE responses, and automatic lookup/dial candidate selection. If such a peer is in the K-closest-to-self set, it MUST be evicted and quarantined until `TrustScore(self, P) >= QUARANTINE_READMIT_THRESHOLD`. Any new or promoted K-closest peer MUST have `TrustScore(self, P) >= QUARANTINE_READMIT_THRESHOLD`; peers between the two thresholds may occupy non-close routing-table slots but must not enter the K-closest set. +7. **Trust quarantine and admission**: Peers with `TrustScore(self, P) < QUARANTINE_THRESHOLD` MUST be skipped by local lookup result selection, FIND_NODE responses, and automatic lookup/dial candidate selection. If such a peer is in the K-closest-to-self set, it MUST be evicted and quarantined until `TrustScore(self, P) >= QUARANTINE_READMIT_THRESHOLD`. Any new routing-table peer MUST have `TrustScore(self, P) >= QUARANTINE_READMIT_THRESHOLD`; existing routing-table peers between the two thresholds may remain in the table, including after moving into the K-closest set. 8. **Trust protection (staleness-gated)**: A peer with `TrustScore(self, P) >= TRUST_PROTECTION_THRESHOLD` **AND** `last_seen` within `LIVE_THRESHOLD` MUST NOT be evicted by swap-closer admission. A peer whose `last_seen` exceeds `LIVE_THRESHOLD` receives no trust protection regardless of score — stale peers MUST NOT hold slots against live candidates. 9. **Deterministic distance**: `Distance(A, B)` is symmetric, deterministic, and consistent across all nodes. Two nodes compute the same distance between the same pair of keys. -10. **Atomic admission**: IP diversity checks, capacity checks, swap-closer evictions, trust score reads, and insertion MUST execute within a single write-locked critical section to prevent TOCTOU races. All `TrustScore` queries during admission (steps 4, 8) MUST occur while the routing table write lock is held. +10. **Atomic admission**: IP diversity checks, capacity checks, swap-closer evictions, trust score reads, and insertion MUST execute within a single exclusive admission critical section to prevent TOCTOU races. Implementations may use a routing-table write lock or an outer DHT-engine write guard that serializes admission. 11. **Monotonic liveness**: `touch_node` updates `last_seen` to the current time and moves the peer to the tail (most recently seen) of its k-bucket. This preserves Kademlia's eviction preference for long-lived peers. 12. **Lookup determinism**: Two nodes with identical `LocalRT` contents compute identical `find_closest_nodes_local(K, count)` results for any key `K` and count. Disagreements between nodes are caused only by routing table divergence, never by algorithm divergence. @@ -184,10 +184,10 @@ When a candidate peer `P` with `NodeInfo` and IP address `candidate_ip` is prese 1. **Self-check**: If `P.id == self.id`, reject. 2. **Address check**: If `P.addresses` is empty, reject. 3. **Authentication check**: If `P` has not completed transport-level authentication, reject. -4. **Trust quarantine check**: If `TrustScore(self, P) < QUARANTINE_THRESHOLD`, reject. If `P` was previously quarantined, reject until `TrustScore(self, P) >= QUARANTINE_READMIT_THRESHOLD`. -5. **Update short-circuit**: If `P` already exists in `KBucket(BucketIndex(self, P))`, merge addresses (Section 6.3), refresh `last_seen`, move `P` to tail, and return. The peer already holds its slot — IP diversity, capacity, and close-group admission checks are skipped. -6. **Loopback check**: If `candidate_ip` is loopback and loopback is disallowed, reject. If loopback is allowed, skip all IP diversity checks (step 7–9) and proceed directly to the close-group admission check (step 10). -7. **Non-IP transport bypass**: If `P` has no IP-based address (e.g., Bluetooth, LoRa), skip IP diversity checks and proceed directly to the close-group admission check (step 10). +4. **Update short-circuit**: If `P` already exists in `KBucket(BucketIndex(self, P))`, merge addresses (Section 6.3), refresh `last_seen`, move `P` to tail, and return. The peer already holds its slot — new-peer trust admission, IP diversity, and capacity checks are skipped. +5. **New-peer trust admission check**: If `TrustScore(self, P) < QUARANTINE_READMIT_THRESHOLD`, reject. If `P` was previously quarantined, reject until `TrustScore(self, P) >= QUARANTINE_READMIT_THRESHOLD`. +6. **Loopback check**: If `candidate_ip` is loopback and loopback is disallowed, reject. If loopback is allowed, skip all IP diversity checks (step 7–9) and proceed directly to insertion/capacity handling. +7. **Non-IP transport bypass**: If `P` has no IP-based address (e.g., Bluetooth, LoRa), skip IP diversity checks and proceed directly to insertion/capacity handling. 8. **IP diversity enforcement** (under write lock — Invariant 10): a. Compute `bucket_idx = BucketIndex(self, P)`. b. Run per-bucket IP diversity check (Section 7.2) against nodes in `KBucket(bucket_idx)`. @@ -263,15 +263,15 @@ When any interaction records a trust failure and `TrustScore(self, P)` drops bel 3. Mark `P` as quarantined if it was evicted from the close group. 4. Do not re-admit quarantined `P` until `TrustScore(self, P) >= QUARANTINE_READMIT_THRESHOLD`. 5. If `P` is not in the K-closest-to-self set, it may remain in the routing table, but local lookup result selection, FIND_NODE responses, and automatic lookup/dial paths MUST avoid it while `TrustScore(self, P) < QUARANTINE_THRESHOLD`. -6. If `P` has `QUARANTINE_THRESHOLD <= TrustScore(self, P) < QUARANTINE_READMIT_THRESHOLD`, it may occupy a non-close routing-table slot. If it would newly enter the K-closest set through admission or promotion, remove/reject it until its trust reaches `QUARANTINE_READMIT_THRESHOLD`. Existing close-group peers in this range are retained until they drop below `QUARANTINE_THRESHOLD`. +6. If `P` has `QUARANTINE_THRESHOLD <= TrustScore(self, P) < QUARANTINE_READMIT_THRESHOLD` and is already in the routing table, it may remain there, including after moving into the K-closest set. New routing-table admissions in this range are rejected until trust reaches `QUARANTINE_READMIT_THRESHOLD`. Quarantine is a routing-table and automatic lookup policy. It is not a blanket transport-level block for explicit user-initiated sends. Re-admission path: a quarantined peer can only re-enter when its trust score recovers above `QUARANTINE_READMIT_THRESHOLD` through time-decay toward neutral AND the peer is rediscovered through normal network activity: 1. Peer `P` is returned in a `FIND_NODE` response from another peer during a lookup, or connects through the normal authenticated peer path. -2. Local node checks `TrustScore(self, P)`. If still below `QUARANTINE_READMIT_THRESHOLD` for a quarantined peer, or if admitting it would place it in the K-closest set, `P` is skipped/rejected. -3. If trust has recovered to `QUARANTINE_READMIT_THRESHOLD`, the standard admission flow (Section 7.1) applies. Peers below that threshold can still be admitted to non-close routing-table slots as long as they are not quarantined and are at or above `QUARANTINE_THRESHOLD`. +2. Local node checks `TrustScore(self, P)`. If still below `QUARANTINE_READMIT_THRESHOLD`, `P` is skipped/rejected. +3. If trust has recovered to `QUARANTINE_READMIT_THRESHOLD`, the standard admission flow (Section 7.1) applies. No manual probing is required. Natural rediscovery plus trust decay is the temporary-ban mechanism. @@ -699,9 +699,9 @@ Each scenario should assert exact expected outcomes and state transitions. 3. **Empty address rejection**: - Candidate with zero addresses. Rejected with error. Routing table unchanged. -4. **Quarantined peer rejection**: - - Peer with `TrustScore < QUARANTINE_THRESHOLD`, or previously quarantined peer with trust below `QUARANTINE_READMIT_THRESHOLD`. Rejected. Not in routing table. - - Peer with `QUARANTINE_THRESHOLD <= TrustScore < QUARANTINE_READMIT_THRESHOLD` is admitted when it would occupy a non-close routing-table slot. The same peer is rejected when it would enter the K-closest set. +4. **New peer admission threshold**: + - New peer with `TrustScore < QUARANTINE_READMIT_THRESHOLD`, including a previously quarantined peer, is rejected. Not in routing table. + - Existing routing-table peer with `QUARANTINE_THRESHOLD <= TrustScore < QUARANTINE_READMIT_THRESHOLD` remains eligible for address/liveness updates and may later move into the K-closest set. 5. **Bucket-full rejection (no stale peers)**: - Bucket at `K_BUCKET_SIZE` capacity, candidate cannot swap-closer, all incumbent peers have `last_seen` within `LIVE_THRESHOLD`. Stale revalidation finds no candidates. Rejected with "bucket at capacity." Routing table unchanged. @@ -804,13 +804,13 @@ Each scenario should assert exact expected outcomes and state transitions. 36. **Close-group quarantine eviction**: - K-closest peer trust drops below 0.20 after failed interaction. Peer is immediately removed from routing table, its trust record is retained, and `PeerRemoved` is emitted. - - Non-close peer with trust 0.30 is retained in the routing table. A close-group removal promotes it into the K-closest set. Local trust gate removes it from the routing table without marking it as below-threshold quarantine. + - Non-close peer with trust 0.30 is retained in the routing table. A close-group removal promotes it into the K-closest set. Local trust gate keeps it because it is already in the routing table and above the quarantine threshold. 37. **Quarantined peer inbound admission rejected**: - Quarantined peer initiates inbound connection. Transport authenticates the peer, the normal routing-table admission path checks trust, and admission is rejected until trust reaches 0.45. 38. **Quarantined peer skipped in lookup results**: - - Quarantined peer appears in `FIND_NODE` response from a node that does not quarantine it. Local node checks trust, finds it below the applicable quarantine/readmit threshold. Peer silently skipped — not dialed. + - Quarantined peer appears in `FIND_NODE` response from a node that does not quarantine it. Local node checks trust, finds it below the applicable quarantine/admission threshold. Peer silently skipped — not dialed. - If the local node is answering a `FIND_NODE` request, quarantined peers are omitted from the response entirely. 39. **Quarantined peer re-admission via lookup discovery after trust recovery**: diff --git a/docs/SECURITY_MODEL.md b/docs/SECURITY_MODEL.md index a48045f5..1f0b3265 100644 --- a/docs/SECURITY_MODEL.md +++ b/docs/SECURITY_MODEL.md @@ -88,16 +88,17 @@ Trust affects routing-table membership in two stages: |-----------------|-------------------|---------------| | Lazy swap eligibility | < 0.35 | `swap_threshold` | | Close-group quarantine / lookup avoidance | < 0.20 | `quarantine_threshold` | -| Close-group admission / quarantine readmission | >= 0.45 | `quarantine_readmit_threshold` | +| New peer admission / quarantine readmission | >= 0.45 | `quarantine_readmit_threshold` | | Staleness | Configurable | `stale_timeout` | Peers outside the K-closest set are not globally evicted solely for low trust. They are omitted from local lookup results, FIND_NODE responses, and automatic lookup paths below the quarantine threshold, and can be lazily replaced when better candidates need the slot. -Peers at or above the quarantine threshold but below the readmission threshold -may occupy non-close routing-table slots, but cannot newly enter the K-closest -set until they recover to the readmission threshold. +Peers already in the routing table at or above the quarantine threshold but +below the readmission threshold may remain in the table, including after moving +into the K-closest set. New routing-table admissions and quarantined +readmissions require the readmission threshold. ### Quarantine Reasons diff --git a/docs/trust-signals-api.md b/docs/trust-signals-api.md index b0da4fa5..2c27a770 100644 --- a/docs/trust-signals-api.md +++ b/docs/trust-signals-api.md @@ -88,9 +88,9 @@ The routing table uses three trust thresholds: and quarantined. - `quarantine_readmit_threshold` (`0.45` by default): a quarantined peer can only re-enter through normal discovery/admission after its decayed trust - reaches this score. New or promoted K-closest peers must also meet this - threshold; peers between `0.20` and `0.45` may occupy non-close routing-table - slots. + reaches this score. New peers must also meet this threshold before entering + the routing table. Existing routing-table peers between `0.20` and `0.45` + may remain in the table, including after moving into the close group. ```rust use saorsa_core::AdaptiveDhtConfig; diff --git a/src/adaptive/dht.rs b/src/adaptive/dht.rs index bbde1717..3d039d88 100644 --- a/src/adaptive/dht.rs +++ b/src/adaptive/dht.rs @@ -35,7 +35,7 @@ const DEFAULT_SWAP_THRESHOLD: f64 = 0.35; /// immediately and all peers are avoided by automatic lookup/dial paths. const DEFAULT_QUARANTINE_THRESHOLD: f64 = 0.20; -/// Default trust score a quarantined peer must recover to before readmission. +/// Default trust score a new or quarantined peer must have for admission. const DEFAULT_QUARANTINE_READMIT_THRESHOLD: f64 = 0.45; /// Maximum weight multiplier per single consumer-reported event. @@ -55,8 +55,8 @@ pub struct AdaptiveDhtConfig { /// K-closest peers are evicted immediately into temporary quarantine. /// Default: 0.20 pub quarantine_threshold: f64, - /// Trust score required before a quarantined peer can re-enter, and before - /// any new or promoted peer can enter the K-closest set. + /// Trust score required before a new peer can enter the routing table, and + /// before a quarantined peer can re-enter. /// Default: 0.45 pub quarantine_readmit_threshold: f64, } @@ -77,7 +77,7 @@ impl AdaptiveDhtConfig { /// Returns `Err` if a threshold is outside its safe range or is NaN. /// Values >= 0.5 (neutral trust) would make all unknown peers immediately /// swap/quarantine eligible since they start at neutral (0.5). The - /// close-group admission/readmit threshold must also stay below neutral + /// new-peer admission/readmit threshold must also stay below neutral /// because recovery happens by decay toward neutral, not by active probing. pub fn validate(&self) -> crate::error::P2pResult<()> { if !(0.0..0.5).contains(&self.swap_threshold) || self.swap_threshold.is_nan() { diff --git a/src/dht/core_engine.rs b/src/dht/core_engine.rs index 70736960..33990191 100644 --- a/src/dht/core_engine.rs +++ b/src/dht/core_engine.rs @@ -207,7 +207,7 @@ const DEFAULT_SWAP_THRESHOLD: f64 = 0.35; #[allow(dead_code)] const DEFAULT_QUARANTINE_THRESHOLD: f64 = 0.20; -/// Default trust score required for K-closest admission/readmission. +/// Default trust score required for new routing-table admission/readmission. #[allow(dead_code)] const DEFAULT_QUARANTINE_READMIT_THRESHOLD: f64 = 0.45; @@ -1473,8 +1473,8 @@ pub struct DhtCoreEngine { /// paths, and evicted immediately if it is in the K-closest close group. quarantine_threshold: f64, - /// Trust score required before a quarantined peer can re-enter, and before - /// any new or promoted peer can enter the K-closest set. + /// Trust score required before a new peer can enter the routing table, and + /// before a quarantined peer can re-enter. quarantine_readmit_threshold: f64, /// Peers evicted from the close group by quarantine. They remain in this @@ -1541,8 +1541,8 @@ impl DhtCoreEngine { /// Otherwise, peers below that score are avoided for automatic lookups, /// and K-closest peers below it are immediately evicted and quarantined. /// Quarantined peers can only re-enter through normal admission after - /// their decayed trust reaches `quarantine_readmit_threshold`; new or - /// promoted K-closest peers must also meet that threshold. + /// their decayed trust reaches `quarantine_readmit_threshold`; new peers + /// must also meet that threshold before entering the routing table. pub(crate) fn set_trust_quarantine_thresholds( &mut self, quarantine_threshold: f64, @@ -1579,7 +1579,7 @@ impl DhtCoreEngine { self.quarantine_enabled() } - fn check_quarantine_admission(&mut self, peer_id: &PeerId, trust_score: f64) -> Result<()> { + fn check_new_peer_admission(&mut self, peer_id: &PeerId, trust_score: f64) -> Result<()> { if !self.quarantine_enabled() { return Ok(()); } @@ -1589,76 +1589,21 @@ impl DhtCoreEngine { peer_id.to_hex() )); } - if self.quarantined_peers.contains(peer_id) { - if trust_score < self.quarantine_readmit_threshold { + if trust_score < self.quarantine_readmit_threshold { + if self.quarantined_peers.contains(peer_id) { return Err(anyhow!( "peer {} quarantined until trust >= {:.3} (current {trust_score:.3})", peer_id.to_hex(), self.quarantine_readmit_threshold )); } - self.quarantined_peers.remove(peer_id); - } - if trust_score < self.quarantine_threshold { - return Err(anyhow!( - "peer {} below quarantine threshold ({trust_score:.3} < {:.3})", - peer_id.to_hex(), - self.quarantine_threshold - )); - } - Ok(()) - } - - fn candidate_enters_close_group_after_removals( - &self, - routing: &KademliaRoutingTable, - candidate_id: &PeerId, - removed_peer_ids: &[PeerId], - ) -> bool { - let mut candidates: Vec<(PeerId, [u8; 32])> = routing - .all_nodes() - .into_iter() - .filter(|node| node.id != *candidate_id && !removed_peer_ids.contains(&node.id)) - .map(|node| { - let distance = xor_distance_bytes(self.node_id.to_bytes(), node.id.to_bytes()); - (node.id, distance) - }) - .collect(); - - candidates.push(( - *candidate_id, - xor_distance_bytes(self.node_id.to_bytes(), candidate_id.to_bytes()), - )); - candidates.sort_by_key(|(_, distance)| *distance); - candidates - .into_iter() - .take(self.k_value) - .any(|(peer_id, _)| peer_id == *candidate_id) - } - - fn check_close_group_admission( - &self, - routing: &KademliaRoutingTable, - peer_id: &PeerId, - trust_score: f64, - removed_peer_ids: &[PeerId], - ) -> Result<()> { - if !self.quarantine_enabled() || trust_score >= self.quarantine_readmit_threshold { - return Ok(()); - } - if !trust_score.is_finite() { return Err(anyhow!( - "peer {} has non-finite trust score", - peer_id.to_hex() - )); - } - if self.candidate_enters_close_group_after_removals(routing, peer_id, removed_peer_ids) { - return Err(anyhow!( - "peer {} below close-group admission threshold ({trust_score:.3} < {:.3})", + "peer {} below new-peer admission threshold ({trust_score:.3} < {:.3})", peer_id.to_hex(), self.quarantine_readmit_threshold )); } + self.quarantined_peers.remove(peer_id); Ok(()) } @@ -1675,6 +1620,26 @@ impl DhtCoreEngine { && trust_score < self.quarantine_readmit_threshold) } + /// Return whether automatic lookup/dial paths should avoid this peer when + /// it might become a new routing-table admission. + pub(crate) async fn should_avoid_automatic_candidate( + &self, + peer_id: &PeerId, + trust_score: f64, + ) -> bool { + if self.should_avoid_for_lookup(peer_id, trust_score) { + return true; + } + if !self.quarantine_enabled() || trust_score >= self.quarantine_readmit_threshold { + return false; + } + self.routing_table + .read() + .await + .find_node_by_id(peer_id) + .is_none() + } + /// Evict a quarantined peer if it currently occupies the K-closest set. #[cfg(test)] pub(crate) async fn enforce_close_group_quarantine( @@ -1715,11 +1680,12 @@ impl DhtCoreEngine { /// Enforce trust gates over the current K-closest set. /// /// Peers below the quarantine threshold are evicted from the close group - /// regardless of whether they were already close. Peers newly promoted into - /// the close group must meet the higher readmit/admission threshold. + /// regardless of whether they were already close. Peers already in the + /// routing table may move into the close group as long as they are not + /// below the quarantine threshold. pub(crate) async fn enforce_close_group_trust_gate( &mut self, - previous_close_group: Option<&[PeerId]>, + _previous_close_group: Option<&[PeerId]>, trust_score: &impl Fn(&PeerId) -> f64, ) -> Vec { if !self.quarantine_enabled() { @@ -1735,10 +1701,7 @@ impl DhtCoreEngine { .into_iter() .find(|peer_id| { let score = trust_score(peer_id); - score.is_finite() - && (score < self.quarantine_threshold - || (previous_close_group.is_some_and(|old| !old.contains(peer_id)) - && score < self.quarantine_readmit_threshold)) + score.is_finite() && score < self.quarantine_threshold }) { if trust_score(&peer_id) < self.quarantine_threshold { @@ -2282,7 +2245,10 @@ impl DhtCoreEngine { } let peer_trust_score = trust_score(&peer_id); - self.check_quarantine_admission(&peer_id, peer_trust_score)?; + let peer_already_known = self.has_node(&peer_id).await; + if !peer_already_known { + self.check_new_peer_admission(&peer_id, peer_trust_score)?; + } // Extract ALL IP addresses from the candidate for diversity checking. // If candidate has no IP-based addresses, it's a non-IP transport — bypass diversity. @@ -2306,7 +2272,6 @@ impl DhtCoreEngine { } return Ok(AdmissionResult::Admitted(vec![])); } - self.check_close_group_admission(&routing, &peer_id, peer_trust_score, &[])?; let k_before = routing.k_closest_ids(self.k_value); routing.add_node(node)?; let k_after = routing.k_closest_ids(self.k_value); @@ -2614,7 +2579,6 @@ impl DhtCoreEngine { } return Ok(AdmissionResult::Admitted(vec![])); } - self.check_close_group_admission(routing, &peer_id, trust_score(&peer_id), &[])?; let k_before = routing.k_closest_ids(self.k_value); routing.add_node(node)?; let k_after = routing.k_closest_ids(self.k_value); @@ -2855,21 +2819,6 @@ impl DhtCoreEngine { } } - let mut planned_removals: Vec = - Vec::with_capacity(all_bucket_swaps.len() + all_close_swaps.len()); - planned_removals.extend(all_bucket_swaps.iter().copied()); - for peer_id in &all_close_swaps { - if !planned_removals.contains(peer_id) { - planned_removals.push(*peer_id); - } - } - self.check_close_group_admission( - routing, - &peer_id, - trust_score(&peer_id), - &planned_removals, - )?; - // === Snapshot K-closest BEFORE mutation === let k_before = routing.k_closest_ids(self.k_value); @@ -2920,7 +2869,7 @@ impl DhtCoreEngine { candidate_ips: &[IpAddr], trust_score: &impl Fn(&PeerId) -> f64, ) -> Result> { - self.check_quarantine_admission(&candidate.id, trust_score(&candidate.id))?; + self.check_new_peer_admission(&candidate.id, trust_score(&candidate.id))?; let mut routing = self.routing_table.write().await; match self.add_with_diversity(&mut routing, candidate, candidate_ips, trust_score, false)? { AdmissionResult::Admitted(events) => Ok(events), @@ -5243,12 +5192,10 @@ mod tests { assert!(dht.set_trust_quarantine_thresholds(0.20, 0.50).is_err()); } - /// A first-time admission rejected below the quarantine threshold is not a - /// close-group quarantine. Once it recovers above the quarantine threshold, - /// it can enter a non-close routing-table slot without waiting for the - /// stronger close-group admission threshold. + /// New peers must meet the readmit/admission threshold even when they would + /// occupy a non-close routing-table slot. #[tokio::test] - async fn test_below_threshold_admission_can_recover_into_non_close_slot() { + async fn test_new_non_close_admission_requires_readmit_threshold() { let mut dht = DhtCoreEngine::new( PeerId::from_bytes([0u8; 32]), 4, @@ -5284,23 +5231,35 @@ mod tests { "peer below quarantine threshold should be rejected" ); - let recovered_above_quarantine = dht + let below_new_peer_admission = dht .add_node( make_node_with_addr(peer_id_bytes, "/ip4/10.10.0.9/udp/9000/quic"), &|id| if *id == peer { 0.30 } else { 0.5 }, ) .await; assert!( - recovered_above_quarantine.is_ok(), - "non-close peer should not need the 0.45 close-group threshold" + below_new_peer_admission.is_err(), + "new non-close peer should need trust >= 0.45" + ); + + let recovered = dht + .add_node( + make_node_with_addr(peer_id_bytes, "/ip4/10.10.0.9/udp/9000/quic"), + &|id| if *id == peer { 0.45 } else { 0.5 }, + ) + .await; + assert!( + recovered.is_ok(), + "new non-close peer should enter once trust reaches 0.45" ); assert!(dht.has_node(&peer).await); } - /// A new peer that would enter the K-closest set must meet the close-group - /// admission threshold, even if it is above the lower quarantine threshold. + /// A new peer that would enter the K-closest set must meet the general + /// new-peer admission threshold, even if it is above the lower quarantine + /// threshold. #[tokio::test] - async fn test_close_group_admission_requires_readmit_threshold() { + async fn test_new_close_group_admission_requires_readmit_threshold() { let mut dht = DhtCoreEngine::new_for_tests(PeerId::from_bytes([0u8; 32])).unwrap(); dht.set_trust_quarantine_thresholds(0.20, 0.45).unwrap(); @@ -5332,11 +5291,49 @@ mod tests { assert!(dht.has_node(&peer).await); } + #[tokio::test] + async fn test_automatic_lookup_skips_unknown_below_admission_threshold() { + let mut dht = DhtCoreEngine::new( + PeerId::from_bytes([0u8; 32]), + 4, + false, + DEFAULT_SWAP_THRESHOLD, + ) + .unwrap(); + dht.set_trust_quarantine_thresholds(0.20, 0.45).unwrap(); + + let mut existing_id = [0u8; 32]; + existing_id[0] = 0x80; + let existing_peer = PeerId::from_bytes(existing_id); + dht.add_node_no_trust(make_node_with_addr( + existing_id, + "/ip4/10.99.0.1/udp/9000/quic", + )) + .await + .unwrap(); + + let mut unknown_id = [0u8; 32]; + unknown_id[0] = 0x81; + let unknown_peer = PeerId::from_bytes(unknown_id); + + assert!( + dht.should_avoid_automatic_candidate(&unknown_peer, 0.30) + .await, + "automatic lookup should skip unknown peers below new-peer admission threshold" + ); + assert!( + !dht.should_avoid_automatic_candidate(&existing_peer, 0.30) + .await, + "existing routing-table peers above quarantine threshold should remain usable" + ); + } + /// Removing one close peer can promote a non-close peer into the close - /// group. Promoted peers below the close-group admission threshold are - /// removed, while existing close peers above the quarantine threshold stay. + /// group. Existing routing-table peers above the quarantine threshold stay + /// even when below the new-peer admission threshold; peers below the + /// quarantine threshold are removed. #[tokio::test] - async fn test_close_group_gate_removes_promoted_peers_below_readmit_threshold() { + async fn test_close_group_gate_allows_existing_promotions_above_quarantine() { let mut dht = DhtCoreEngine::new( PeerId::from_bytes([0u8; 32]), 4, @@ -5393,16 +5390,29 @@ mod tests { .await; assert!( - events.iter().any( + !events.iter().any( |event| matches!(event, RoutingTableEvent::PeerRemoved(id) if *id == promoted_peer) ), - "promoted peer below close-group admission threshold should be removed" + "existing promoted peer above quarantine threshold should stay" ); - assert!(!dht.has_node(&promoted_peer).await); + assert!(dht.has_node(&promoted_peer).await); assert!( dht.has_node(&close_peer_ids[1]).await, "existing close peer above quarantine threshold should stay" ); + + let quarantine_events = dht + .enforce_close_group_trust_gate(Some(&previous_close_group), &|id| { + if *id == promoted_peer { 0.10 } else { 0.5 } + }) + .await; + assert!( + quarantine_events.iter().any( + |event| matches!(event, RoutingTableEvent::PeerRemoved(id) if *id == promoted_peer) + ), + "existing promoted peer below quarantine threshold should be removed" + ); + assert!(!dht.has_node(&promoted_peer).await); } /// A non-close peer below the quarantine threshold is avoided by automatic diff --git a/src/dht_network_manager.rs b/src/dht_network_manager.rs index 22c13dc9..85005a77 100644 --- a/src/dht_network_manager.rs +++ b/src/dht_network_manager.rs @@ -638,8 +638,8 @@ pub struct DhtNetworkConfig { /// K-closest peers are immediately evicted into temporary quarantine. /// Default: 0.0 (disabled). pub quarantine_threshold: f64, - /// Trust score required before a quarantined peer can be admitted again, - /// and before any new or promoted peer can enter the K-closest set. + /// Trust score required before a new peer can enter the routing table, + /// and before a quarantined peer can be admitted again. /// Default: 0.0 (disabled). pub quarantine_readmit_threshold: f64, } @@ -2673,8 +2673,8 @@ impl DhtNetworkManager { /// /// No network requests are made — safe to call from request handlers. /// Only returns peers that passed the `is_dht_participant` security gate, - /// were added to the Kademlia routing table, and are not below the trust - /// quarantine/readmit thresholds. + /// were added to the Kademlia routing table, and are not locally + /// quarantined by trust policy. /// /// Results are sorted by XOR distance to the key. pub async fn find_closest_nodes_local(&self, key: &Key, count: usize) -> Vec { @@ -2853,7 +2853,7 @@ impl DhtNetworkManager { for node in initial { if self.should_avoid_automatic_peer(&node.peer_id).await { trace!( - "[NETWORK] Skipping {}: peer is below trust quarantine/readmit threshold", + "[NETWORK] Skipping {}: peer is below trust quarantine/admission threshold", node.peer_id.to_hex() ); continue; @@ -2900,7 +2900,7 @@ impl DhtNetworkManager { if self.should_avoid_automatic_peer(&node.peer_id).await { peer_states.mark_failed(node.peer_id); trace!( - "[NETWORK] Skipping {}: peer is below trust quarantine/readmit threshold", + "[NETWORK] Skipping {}: peer is below trust quarantine/admission threshold", node.peer_id.to_hex() ); continue; @@ -3011,7 +3011,7 @@ impl DhtNetworkManager { if self.should_avoid_automatic_peer(&node.peer_id).await { peer_states.mark_failed(node.peer_id); trace!( - "[NETWORK] Skipping gossiped {}: peer is below trust quarantine/readmit threshold", + "[NETWORK] Skipping gossiped {}: peer is below trust quarantine/admission threshold", node.peer_id.to_hex() ); continue; @@ -3719,7 +3719,8 @@ impl DhtNetworkManager { async fn should_avoid_automatic_peer(&self, peer_id: &PeerId) -> bool { let trust_score = self.peer_trust_score(peer_id); let dht = self.dht.read().await; - dht.should_avoid_for_lookup(peer_id, trust_score) + dht.should_avoid_automatic_candidate(peer_id, trust_score) + .await } /// Ensure an identity-authenticated channel to `peer_id` exists, From f0e6c58993d93c29fb5a61121114c5d8323d04df Mon Sep 17 00:00:00 2001 From: Warm Beer Date: Wed, 20 May 2026 19:03:53 +0200 Subject: [PATCH 04/18] fix(dht): preserve k-sized routing table during quarantine --- docs/ROUTING_TABLE_DESIGN.md | 19 ++-- docs/trust-signals-api.md | 6 +- src/adaptive/dht.rs | 14 ++- src/dht/core_engine.rs | 191 +++++++++++++++++++++++++++++++---- src/dht_network_manager.rs | 66 +++++++----- src/network.rs | 17 ++-- 6 files changed, 244 insertions(+), 69 deletions(-) diff --git a/docs/ROUTING_TABLE_DESIGN.md b/docs/ROUTING_TABLE_DESIGN.md index af89a54d..afb213d8 100644 --- a/docs/ROUTING_TABLE_DESIGN.md +++ b/docs/ROUTING_TABLE_DESIGN.md @@ -57,7 +57,7 @@ All parameters are configurable. Values below are a reference profile used for l | `IPV6_SUBNET_MASK` | Prefix length for IPv6 subnet grouping | `/48` | | `TRUST_PROTECTION_THRESHOLD` | Trust score above which a peer resists swap-closer eviction | `0.7` | | `SWAP_THRESHOLD` | Trust score below which a peer is eligible for replacement when a better candidate needs the slot | `0.35` | -| `QUARANTINE_THRESHOLD` | Trust score below which automatic lookup/dial paths avoid the peer, and close-group peers are evicted immediately | `0.20` | +| `QUARANTINE_THRESHOLD` | Trust score below which automatic lookup/dial paths avoid the peer, and close-group peers are evicted when the routing table can retain at least K peers | `0.20` | | `QUARANTINE_READMIT_THRESHOLD` | Trust score required for new routing-table admission/readmission after quarantine | `0.45` | | `EMA_ALPHA` | EMA smoothing factor — weight of each new observation (higher = faster response) | `0.124` | | `DECAY_LAMBDA` | Per-second exponential decay rate toward neutral (0.5) | `1.394e-5` | @@ -129,7 +129,7 @@ Note: `K_BUCKET_SIZE` values below 4 produce degenerate behavior (single-peer ro 4. **Address requirement**: A `NodeInfo` with an empty address list MUST NOT be admitted to the routing table. 5. **Authenticated membership**: Only peers that have completed transport-level authentication are eligible for routing table insertion. Unauthenticated peers MUST NOT enter `LocalRT`. 6. **IP diversity**: No enforcement scope (per-bucket or routing-neighborhood) may exceed `IP_EXACT_LIMIT` nodes per exact IP or `IP_SUBNET_LIMIT` nodes per subnet, except via explicit loopback or testnet overrides. -7. **Trust quarantine and admission**: Peers with `TrustScore(self, P) < QUARANTINE_THRESHOLD` MUST be skipped by local lookup result selection, FIND_NODE responses, and automatic lookup/dial candidate selection. If such a peer is in the K-closest-to-self set, it MUST be evicted and quarantined until `TrustScore(self, P) >= QUARANTINE_READMIT_THRESHOLD`. Any new routing-table peer MUST have `TrustScore(self, P) >= QUARANTINE_READMIT_THRESHOLD`; existing routing-table peers between the two thresholds may remain in the table, including after moving into the K-closest set. +7. **Trust quarantine and admission**: Peers with `TrustScore(self, P) < QUARANTINE_THRESHOLD` MUST be skipped by local lookup result selection, FIND_NODE responses, and automatic lookup/dial candidate selection. If such a peer is in the K-closest-to-self set, it MUST be evicted and quarantined until `TrustScore(self, P) >= QUARANTINE_READMIT_THRESHOLD` whenever eviction leaves `LocalRT(self)` with at least K peers. If eviction would shrink `LocalRT(self)` below K peers, the peer remains in the table but is still avoided by automatic lookup policy. Any new routing-table peer MUST have `TrustScore(self, P) >= QUARANTINE_READMIT_THRESHOLD`; existing routing-table peers between the two thresholds may remain in the table, including after moving into the K-closest set. 8. **Trust protection (staleness-gated)**: A peer with `TrustScore(self, P) >= TRUST_PROTECTION_THRESHOLD` **AND** `last_seen` within `LIVE_THRESHOLD` MUST NOT be evicted by swap-closer admission. A peer whose `last_seen` exceeds `LIVE_THRESHOLD` receives no trust protection regardless of score — stale peers MUST NOT hold slots against live candidates. 9. **Deterministic distance**: `Distance(A, B)` is symmetric, deterministic, and consistent across all nodes. Two nodes compute the same distance between the same pair of keys. 10. **Atomic admission**: IP diversity checks, capacity checks, swap-closer evictions, trust score reads, and insertion MUST execute within a single exclusive admission critical section to prevent TOCTOU races. Implementations may use a routing-table write lock or an outer DHT-engine write guard that serializes admission. @@ -258,12 +258,13 @@ Rationale: swap-closer prefers geographically closer peers (lower XOR distance) When any interaction records a trust failure and `TrustScore(self, P)` drops below `QUARANTINE_THRESHOLD`: -1. If `P` is in the K-closest-to-self set, remove `P` from `LocalRT(self)` and emit `PeerRemoved`. +1. If `P` is in the K-closest-to-self set and removal leaves at least K peers in `LocalRT(self)`, remove `P` from `LocalRT(self)` and emit `PeerRemoved`. 2. Keep `P`'s trust record. Do not reset trust on eviction. 3. Mark `P` as quarantined if it was evicted from the close group. 4. Do not re-admit quarantined `P` until `TrustScore(self, P) >= QUARANTINE_READMIT_THRESHOLD`. -5. If `P` is not in the K-closest-to-self set, it may remain in the routing table, but local lookup result selection, FIND_NODE responses, and automatic lookup/dial paths MUST avoid it while `TrustScore(self, P) < QUARANTINE_THRESHOLD`. -6. If `P` has `QUARANTINE_THRESHOLD <= TrustScore(self, P) < QUARANTINE_READMIT_THRESHOLD` and is already in the routing table, it may remain there, including after moving into the K-closest set. New routing-table admissions in this range are rejected until trust reaches `QUARANTINE_READMIT_THRESHOLD`. +5. If removal would shrink `LocalRT(self)` below K peers, keep `P` in the routing table until another peer is admitted and the same eviction can happen without underfilling the table. +6. If `P` is not in the K-closest-to-self set, it may remain in the routing table, but local lookup result selection, FIND_NODE responses, and automatic lookup/dial paths MUST avoid it while `TrustScore(self, P) < QUARANTINE_THRESHOLD`. +7. If `P` has `QUARANTINE_THRESHOLD <= TrustScore(self, P) < QUARANTINE_READMIT_THRESHOLD` and is already in the routing table, it may remain there, including after moving into the K-closest set. New routing-table admissions in this range are rejected until trust reaches `QUARANTINE_READMIT_THRESHOLD`. Quarantine is a routing-table and automatic lookup policy. It is not a blanket transport-level block for explicit user-initiated sends. @@ -412,7 +413,7 @@ Events MUST be emitted reliably for every routing table mutation. Consumers MAY Peers are detected as departed through: -1. **RPC failure**: Failed outbound RPC records trust failure. If trust drops below `QUARANTINE_THRESHOLD` and the peer is in the close group, it is evicted and quarantined (Section 7.4). +1. **RPC failure**: Failed outbound RPC records trust failure. If trust drops below `QUARANTINE_THRESHOLD` and the peer is in the close group, it is evicted and quarantined when the routing table can retain at least K peers (Section 7.4). 2. **Iterative lookup feedback**: Network lookups record success/failure per queried peer. 3. **Self-lookup refresh**: Periodic self-lookups discover that a previously-close peer is no longer returned by the network. 4. **Stale peer revalidation**: When a new candidate contends for a full bucket, all stale peers (not seen within `LIVE_THRESHOLD`) in that bucket are pinged. Non-responders are evicted immediately (Section 7.5). @@ -803,7 +804,7 @@ Each scenario should assert exact expected outcomes and state transitions. - Insert a peer into a bucket that affects the K-closest-to-self set. `KClosestPeersChanged` emitted with correct old and new sets. Insert a peer into a distant bucket that does NOT affect the K-closest set. `KClosestPeersChanged` is NOT emitted. Verify at-most-once semantics: a single admission with multiple swaps emits the event at most once. 36. **Close-group quarantine eviction**: - - K-closest peer trust drops below 0.20 after failed interaction. Peer is immediately removed from routing table, its trust record is retained, and `PeerRemoved` is emitted. + - K-closest peer trust drops below 0.20 after failed interaction. Peer is removed from routing table when the routing table can retain at least K peers, its trust record is retained, and `PeerRemoved` is emitted. - Non-close peer with trust 0.30 is retained in the routing table. A close-group removal promotes it into the K-closest set. Local trust gate keeps it because it is already in the routing table and above the quarantine threshold. 37. **Quarantined peer inbound admission rejected**: @@ -866,10 +867,10 @@ Each scenario should assert exact expected outcomes and state transitions. - Peer starts at neutral trust (0.5). Consumer reports `ApplicationSuccess(1.0)`. Trust score increases above 0.5 (exact value determined by EMA smoothing factor). Peer remains in routing table. 54. **Consumer penalty degrades trust to quarantine**: - - Peer starts at neutral trust (0.5). Consumer reports repeated `ApplicationFailure(3.0)` events. Trust score decreases with each event. After sufficient events, score drops below `QUARANTINE_THRESHOLD` (0.20). If the peer is in the K-closest set, it is evicted and quarantined (Section 7.4). + - Peer starts at neutral trust (0.5). Consumer reports repeated `ApplicationFailure(3.0)` events. Trust score decreases with each event. After sufficient events, score drops below `QUARANTINE_THRESHOLD` (0.20). If the peer is in the K-closest set, it is evicted and quarantined when the routing table can retain at least K peers (Section 7.4). 55. **Consumer penalty triggers close-group quarantine**: - - Peer is in the K-closest set with trust slightly above `QUARANTINE_THRESHOLD`. Consumer reports `ApplicationFailure(weight)` sufficient to push score below `QUARANTINE_THRESHOLD`. Peer is immediately evicted from routing table and quarantined from re-admission until trust reaches `QUARANTINE_READMIT_THRESHOLD`. `PeerRemoved` event emitted. + - Peer is in the K-closest set with trust slightly above `QUARANTINE_THRESHOLD`. Consumer reports `ApplicationFailure(weight)` sufficient to push score below `QUARANTINE_THRESHOLD`. Peer is evicted from routing table, when doing so retains at least K peers, and quarantined from re-admission until trust reaches `QUARANTINE_READMIT_THRESHOLD`. `PeerRemoved` event emitted. 56. **Consumer event for peer not in routing table**: - Peer has no routing table entry. Consumer reports `ApplicationFailure(2.0)`. Trust engine records the event and updates the EMA score (decreases from neutral 0.5). Routing table is unchanged. If the peer later attempts admission, the recorded low trust may cause rejection (Section 7.1 step 4). diff --git a/docs/trust-signals-api.md b/docs/trust-signals-api.md index 2c27a770..765116cc 100644 --- a/docs/trust-signals-api.md +++ b/docs/trust-signals-api.md @@ -11,7 +11,7 @@ responsibility via `TrustEvent::ApplicationSuccess`. The trust system enables: - **Sybil resistance**: Malicious nodes are downscored automatically -- **Close-group quarantine**: K-closest peers below the quarantine threshold are evicted +- **Close-group quarantine**: K-closest peers below the quarantine threshold are evicted when the routing table can retain at least K peers - **Self-healing**: Time decay moves quarantined peers back toward neutral over days - **Lazy swap-out**: Low-trust peers outside the close group are replaced when better candidates arrive @@ -84,8 +84,8 @@ The routing table uses three trust thresholds: for replacement when a better candidate needs the slot. - `quarantine_threshold` (`0.20` by default): peers below this score are skipped by lookup result selection and automatic lookup/dial paths. If such - a peer is currently in the K-closest-to-self set, it is evicted immediately - and quarantined. + a peer is currently in the K-closest-to-self set, it is evicted and + quarantined when the routing table can retain at least K peers. - `quarantine_readmit_threshold` (`0.45` by default): a quarantined peer can only re-enter through normal discovery/admission after its decayed trust reaches this score. New peers must also meet this threshold before entering diff --git a/src/adaptive/dht.rs b/src/adaptive/dht.rs index 3d039d88..f7a8b075 100644 --- a/src/adaptive/dht.rs +++ b/src/adaptive/dht.rs @@ -32,7 +32,8 @@ use std::sync::Arc; const DEFAULT_SWAP_THRESHOLD: f64 = 0.35; /// Default trust score threshold below which close-group peers are evicted -/// immediately and all peers are avoided by automatic lookup/dial paths. +/// when doing so still leaves at least K routing-table peers, and all peers +/// are avoided by automatic lookup/dial paths. const DEFAULT_QUARANTINE_THRESHOLD: f64 = 0.20; /// Default trust score a new or quarantined peer must have for admission. @@ -52,7 +53,8 @@ pub struct AdaptiveDhtConfig { /// Default: 0.35 pub swap_threshold: f64, /// Trust score below which automatic lookup/dial paths avoid a peer, and - /// K-closest peers are evicted immediately into temporary quarantine. + /// K-closest peers are evicted into temporary quarantine when the routing + /// table can keep at least K peers. /// Default: 0.20 pub quarantine_threshold: f64, /// Trust score required before a new peer can enter the routing table, and @@ -194,7 +196,7 @@ impl AdaptiveDHT { /// trust engine injected. Call [`start`](Self::start) to begin DHT /// operations. Trust scores are computed live — low-trust peers are /// swapped out when better candidates arrive, and bad close-group peers - /// are quarantined immediately. + /// are quarantined when the routing table has enough peers. /// /// # Errors /// @@ -239,7 +241,8 @@ impl AdaptiveDHT { /// Trust scores are updated immediately. Peers below the quarantine /// threshold are avoided by lookup result selection and automatic /// lookup/dial paths, and K-closest peers below that threshold are - /// evicted into temporary quarantine. + /// evicted into temporary quarantine when the routing table can keep at + /// least K peers. pub async fn report_trust_event(&self, peer_id: &PeerId, event: TrustEvent) { match event { TrustEvent::ApplicationSuccess(weight) | TrustEvent::ApplicationFailure(weight) => { @@ -299,7 +302,8 @@ impl AdaptiveDHT { /// /// Trust scores are computed live — no background tasks needed. /// Low-trust peers are swapped out when better candidates arrive; close - /// peers below the quarantine threshold are evicted immediately. + /// peers below the quarantine threshold are evicted when the routing table + /// has enough peers. pub async fn start(&self) -> Result<()> { Arc::clone(&self.dht_manager).start().await } diff --git a/src/dht/core_engine.rs b/src/dht/core_engine.rs index 33990191..1e02f567 100644 --- a/src/dht/core_engine.rs +++ b/src/dht/core_engine.rs @@ -1470,7 +1470,8 @@ pub struct DhtCoreEngine { swap_threshold: f64, /// Trust score below which a peer is avoided for automatic lookup/dial - /// paths, and evicted immediately if it is in the K-closest close group. + /// paths, and evicted if it is in the K-closest close group and the + /// routing table can keep at least K peers. quarantine_threshold: f64, /// Trust score required before a new peer can enter the routing table, and @@ -1539,7 +1540,8 @@ impl DhtCoreEngine { /// /// A `quarantine_threshold` of `0.0` disables quarantine enforcement. /// Otherwise, peers below that score are avoided for automatic lookups, - /// and K-closest peers below it are immediately evicted and quarantined. + /// and K-closest peers below it are evicted and quarantined when the + /// routing table can keep at least K peers. /// Quarantined peers can only re-enter through normal admission after /// their decayed trust reaches `quarantine_readmit_threshold`; new peers /// must also meet that threshold before entering the routing table. @@ -1640,7 +1642,8 @@ impl DhtCoreEngine { .is_none() } - /// Evict a quarantined peer if it currently occupies the K-closest set. + /// Evict a quarantined peer if it currently occupies the K-closest set and + /// removal will not shrink the routing table below K peers. #[cfg(test)] pub(crate) async fn enforce_close_group_quarantine( &mut self, @@ -1662,6 +1665,9 @@ impl DhtCoreEngine { if routing.find_node_by_id(peer_id).is_none() { return Vec::new(); } + if routing.node_count() <= self.k_value { + return Vec::new(); + } self.quarantined_peers.insert(*peer_id); routing.remove_node(peer_id); @@ -1680,9 +1686,10 @@ impl DhtCoreEngine { /// Enforce trust gates over the current K-closest set. /// /// Peers below the quarantine threshold are evicted from the close group - /// regardless of whether they were already close. Peers already in the + /// while the routing table has more than K entries. Peers already in the /// routing table may move into the close group as long as they are not - /// below the quarantine threshold. + /// below the quarantine threshold, and low-trust close-group peers are + /// retained rather than shrinking the routing table below K entries. pub(crate) async fn enforce_close_group_trust_gate( &mut self, _previous_close_group: Option<&[PeerId]>, @@ -1696,14 +1703,18 @@ impl DhtCoreEngine { let k_before = routing.k_closest_ids(self.k_value); let mut removed = Vec::new(); - while let Some(peer_id) = routing - .k_closest_ids(self.k_value) - .into_iter() - .find(|peer_id| { - let score = trust_score(peer_id); - score.is_finite() && score < self.quarantine_threshold - }) - { + while routing.node_count() > self.k_value { + let Some(peer_id) = routing + .k_closest_ids(self.k_value) + .into_iter() + .find(|peer_id| { + let score = trust_score(peer_id); + score.is_finite() && score < self.quarantine_threshold + }) + else { + break; + }; + if trust_score(&peer_id) < self.quarantine_threshold { self.quarantined_peers.insert(peer_id); } @@ -2909,6 +2920,8 @@ mod tests { use crate::address::TransportAddr; use std::collections::HashSet; + const SMALL_TEST_K: usize = 4; + #[tokio::test] async fn test_xor_distance() { let key1 = DhtKey::from_bytes([0u8; 32]); @@ -5129,12 +5142,18 @@ mod tests { assert!(dht.has_node(&low_peer).await); } - /// A K-closest peer below the quarantine threshold is evicted immediately - /// and cannot be readmitted until its trust has recovered to the readmit - /// threshold. + /// A K-closest peer below the quarantine threshold is evicted when the + /// routing table has surplus above K and cannot be readmitted until its + /// trust has recovered to the readmit threshold. #[tokio::test] async fn test_close_group_peer_below_quarantine_is_evicted_until_readmit() { - let mut dht = DhtCoreEngine::new_for_tests(PeerId::from_bytes([0u8; 32])).unwrap(); + let mut dht = DhtCoreEngine::new( + PeerId::from_bytes([0u8; 32]), + SMALL_TEST_K, + false, + DEFAULT_SWAP_THRESHOLD, + ) + .unwrap(); dht.set_trust_quarantine_thresholds(0.20, 0.45).unwrap(); let mut peer_id_bytes = [0u8; 32]; @@ -5149,6 +5168,19 @@ mod tests { .unwrap(); assert!(dht.has_node(&peer).await); + const FAR_PEER_FIRST_BYTES: [u8; SMALL_TEST_K] = [0x10, 0x20, 0x40, 0x80]; + for first_byte in FAR_PEER_FIRST_BYTES { + let mut id = [0u8; 32]; + id[0] = first_byte; + dht.add_node_no_trust(make_node_with_addr( + id, + &format!("/ip4/10.{first_byte}.0.1/udp/9000/quic"), + )) + .await + .unwrap(); + } + assert_eq!(dht.routing_table_size().await, SMALL_TEST_K + 1); + let events = dht.enforce_close_group_quarantine(&peer, 0.19).await; assert!( events @@ -5160,6 +5192,7 @@ mod tests { !dht.has_node(&peer).await, "quarantined close-group peer should be removed from RT" ); + assert_eq!(dht.routing_table_size().await, SMALL_TEST_K); let early_readmit = dht .add_node( @@ -5185,6 +5218,105 @@ mod tests { assert!(dht.has_node(&peer).await); } + /// Close-group quarantine should not shrink the routing table below K. + #[tokio::test] + async fn test_close_group_quarantine_keeps_minimum_k_peers() { + let mut dht = DhtCoreEngine::new( + PeerId::from_bytes([0u8; 32]), + SMALL_TEST_K, + false, + DEFAULT_SWAP_THRESHOLD, + ) + .unwrap(); + dht.set_trust_quarantine_thresholds(0.20, 0.45).unwrap(); + + let mut close_peer_ids = Vec::new(); + for i in 1..=SMALL_TEST_K as u8 { + let mut id = [0u8; 32]; + id[31] = i; + let peer_id = PeerId::from_bytes(id); + close_peer_ids.push(peer_id); + dht.add_node_no_trust(make_node_with_addr( + id, + &format!("/ip4/10.{i}.0.1/udp/9000/quic"), + )) + .await + .unwrap(); + } + + let low_peer = close_peer_ids[0]; + let events = dht + .enforce_close_group_trust_gate(None, &|id| if *id == low_peer { 0.10 } else { 0.5 }) + .await; + + assert!( + events.is_empty(), + "trust quarantine should not evict when the routing table is at K" + ); + assert_eq!(dht.routing_table_size().await, SMALL_TEST_K); + assert!(dht.has_node(&low_peer).await); + assert!( + dht.should_avoid_for_lookup(&low_peer, 0.10), + "retained low-trust peer should still be avoided by lookup policy" + ); + } + + /// Once a new routing-table peer creates surplus above K, a previously + /// deferred close-group quarantine can remove the low-trust peer. + #[tokio::test] + async fn test_new_peer_surplus_allows_deferred_close_group_quarantine() { + let mut dht = DhtCoreEngine::new( + PeerId::from_bytes([0u8; 32]), + SMALL_TEST_K, + false, + DEFAULT_SWAP_THRESHOLD, + ) + .unwrap(); + dht.set_trust_quarantine_thresholds(0.20, 0.45).unwrap(); + + let mut close_peer_ids = Vec::new(); + for i in 1..=SMALL_TEST_K as u8 { + let mut id = [0u8; 32]; + id[31] = i; + let peer_id = PeerId::from_bytes(id); + close_peer_ids.push(peer_id); + dht.add_node_no_trust(make_node_with_addr( + id, + &format!("/ip4/10.{i}.0.1/udp/9000/quic"), + )) + .await + .unwrap(); + } + + let low_peer = close_peer_ids[0]; + let deferred = dht + .enforce_close_group_trust_gate(None, &|id| if *id == low_peer { 0.10 } else { 0.5 }) + .await; + assert!(deferred.is_empty()); + + let mut surplus_id = [0u8; 32]; + surplus_id[0] = 0x80; + dht.add_node_no_trust(make_node_with_addr( + surplus_id, + "/ip4/10.99.0.1/udp/9000/quic", + )) + .await + .unwrap(); + assert_eq!(dht.routing_table_size().await, SMALL_TEST_K + 1); + + let events = dht + .enforce_close_group_trust_gate(None, &|id| if *id == low_peer { 0.10 } else { 0.5 }) + .await; + assert!( + events.iter().any( + |event| matches!(event, RoutingTableEvent::PeerRemoved(id) if *id == low_peer) + ), + "surplus peer should allow deferred close-group quarantine" + ); + assert_eq!(dht.routing_table_size().await, SMALL_TEST_K); + assert!(!dht.has_node(&low_peer).await); + } + #[test] fn test_core_quarantine_readmit_threshold_must_be_reachable_by_decay() { let mut dht = DhtCoreEngine::new_for_tests(PeerId::from_bytes([0u8; 32])).unwrap(); @@ -5331,7 +5463,8 @@ mod tests { /// Removing one close peer can promote a non-close peer into the close /// group. Existing routing-table peers above the quarantine threshold stay /// even when below the new-peer admission threshold; peers below the - /// quarantine threshold are removed. + /// quarantine threshold are removed once the routing table has surplus + /// above K. #[tokio::test] async fn test_close_group_gate_allows_existing_promotions_above_quarantine() { let mut dht = DhtCoreEngine::new( @@ -5401,6 +5534,26 @@ mod tests { "existing close peer above quarantine threshold should stay" ); + let retained_events = dht + .enforce_close_group_trust_gate(Some(&previous_close_group), &|id| { + if *id == promoted_peer { 0.10 } else { 0.5 } + }) + .await; + assert!( + retained_events.is_empty(), + "existing promoted peer below quarantine threshold should stay while the routing table is at K" + ); + assert!(dht.has_node(&promoted_peer).await); + + let mut surplus_id = [0u8; 32]; + surplus_id[0] = 0x90; + dht.add_node_no_trust(make_node_with_addr( + surplus_id, + "/ip4/10.100.0.1/udp/9000/quic", + )) + .await + .unwrap(); + let quarantine_events = dht .enforce_close_group_trust_gate(Some(&previous_close_group), &|id| { if *id == promoted_peer { 0.10 } else { 0.5 } @@ -5410,7 +5563,7 @@ mod tests { quarantine_events.iter().any( |event| matches!(event, RoutingTableEvent::PeerRemoved(id) if *id == promoted_peer) ), - "existing promoted peer below quarantine threshold should be removed" + "existing promoted peer below quarantine threshold should be removed once there is surplus" ); assert!(!dht.has_node(&promoted_peer).await); } diff --git a/src/dht_network_manager.rs b/src/dht_network_manager.rs index 85005a77..6298a6b8 100644 --- a/src/dht_network_manager.rs +++ b/src/dht_network_manager.rs @@ -635,7 +635,8 @@ pub struct DhtNetworkConfig { /// Default: 0.0 (disabled). pub swap_threshold: f64, /// Trust score below which automatic lookup/dial paths avoid a peer, and - /// K-closest peers are immediately evicted into temporary quarantine. + /// K-closest peers are evicted into temporary quarantine when the routing + /// table can keep at least K peers. /// Default: 0.0 (disabled). pub quarantine_threshold: f64, /// Trust score required before a new peer can enter the routing table, @@ -5676,10 +5677,13 @@ impl DhtNetworkManager { info!("Evicted {} offline K-closest peer(s)", non_responders.len()); } - fn routing_events_include_close_group_change(events: &[RoutingTableEvent]) -> bool { - events - .iter() - .any(|event| matches!(event, RoutingTableEvent::KClosestPeersChanged { .. })) + fn routing_events_can_enable_quarantine_eviction(events: &[RoutingTableEvent]) -> bool { + events.iter().any(|event| { + matches!( + event, + RoutingTableEvent::PeerAdded(_) | RoutingTableEvent::KClosestPeersChanged { .. } + ) + }) } async fn enforce_close_group_trust_gate( @@ -5709,7 +5713,7 @@ impl DhtNetworkManager { } async fn broadcast_routing_events_with_quarantine(&self, mut events: Vec) { - if !Self::routing_events_include_close_group_change(&events) { + if !Self::routing_events_can_enable_quarantine_eviction(&events) { self.broadcast_routing_events(&events); return; } @@ -5726,24 +5730,32 @@ impl DhtNetworkManager { return; } - let final_new_close_group = quarantine_events - .iter() - .rev() - .find_map(|event| match event { - RoutingTableEvent::KClosestPeersChanged { new, .. } => Some(new.clone()), - _ => None, - }); + if let Some(original_old_close_group) = original_old_close_group { + let final_new_close_group = + quarantine_events + .iter() + .rev() + .find_map(|event| match event { + RoutingTableEvent::KClosestPeersChanged { new, .. } => Some(new.clone()), + _ => None, + }); - events.retain(|event| !matches!(event, RoutingTableEvent::KClosestPeersChanged { .. })); - events.extend( - quarantine_events - .into_iter() - .filter(|event| !matches!(event, RoutingTableEvent::KClosestPeersChanged { .. })), - ); - if let (Some(old), Some(new)) = (original_old_close_group, final_new_close_group) - && old != new - { - events.push(RoutingTableEvent::KClosestPeersChanged { old, new }); + events.retain(|event| !matches!(event, RoutingTableEvent::KClosestPeersChanged { .. })); + events.extend( + quarantine_events.into_iter().filter(|event| { + !matches!(event, RoutingTableEvent::KClosestPeersChanged { .. }) + }), + ); + if let Some(new) = final_new_close_group + && original_old_close_group != new + { + events.push(RoutingTableEvent::KClosestPeersChanged { + old: original_old_close_group, + new, + }); + } + } else { + events.extend(quarantine_events); } self.broadcast_routing_events(&events); @@ -6334,7 +6346,11 @@ mod tests { DhtCoreEngine::new(local_peer, 4, false, DEFAULT_NEUTRAL_TRUST - 0.15).unwrap(); dht.set_trust_quarantine_thresholds(0.20, 0.45).unwrap(); - dht.add_node_no_trust(routing_test_node(1)).await.unwrap(); + for byte in 1..=5u8 { + dht.add_node_no_trust(routing_test_node(byte)) + .await + .unwrap(); + } let events = dht .enforce_close_group_quarantine(&quarantined_peer, 0.10) .await; @@ -8081,7 +8097,7 @@ mod tests { fn routing_test_node(byte: u8) -> NodeInfo { NodeInfo { id: pid(byte), - addresses: vec![MultiAddr::quic(sock(&format!("203.0.113.{byte}:9000")))], + addresses: vec![MultiAddr::quic(sock(&format!("203.0.{byte}.1:9000")))], address_types: vec![AddressType::Direct], last_seen: AtomicInstant::now(), } diff --git a/src/network.rs b/src/network.rs index c66977f4..459dc40a 100644 --- a/src/network.rs +++ b/src/network.rs @@ -565,17 +565,18 @@ impl NodeConfigBuilder { self } - /// Enable or disable trust-based peer swap-out. + /// Enable or disable trust-based routing-table enforcement. /// - /// When `false`, peers are never swapped out of the routing table - /// based on trust scores. Trust scores are still tracked but have - /// no enforcement effect. + /// When `false`, trust scores are still tracked but have no routing-table + /// enforcement effect. /// - /// When `true` (the default), peers whose trust score falls below the - /// swap threshold (0.35) become eligible for replacement when a - /// better candidate arrives. + /// When `true` (the default), the default adaptive DHT policy applies: + /// peers below the swap threshold (0.35) become eligible for replacement, + /// peers below the quarantine threshold (0.20) are avoided by automatic + /// lookup/dial paths, and new routing-table peers must meet the readmission + /// threshold (0.45). /// - /// For fine-grained control over the threshold, use + /// For fine-grained control over these thresholds, use /// [`adaptive_dht_config`](Self::adaptive_dht_config) instead. pub fn trust_enforcement(mut self, enabled: bool) -> Self { let adaptive_config = if enabled { From 8e564efcba4b08746cbfab79557a39f6eadcc6c7 Mon Sep 17 00:00:00 2001 From: Warm Beer Date: Thu, 21 May 2026 09:49:30 +0200 Subject: [PATCH 05/18] fix(docs): document adaptive trust enforcement policy --- src/dht_network_manager.rs | 13 +++++-------- src/network.rs | 10 +++++----- 2 files changed, 10 insertions(+), 13 deletions(-) diff --git a/src/dht_network_manager.rs b/src/dht_network_manager.rs index 6298a6b8..492f5be7 100644 --- a/src/dht_network_manager.rs +++ b/src/dht_network_manager.rs @@ -6206,13 +6206,14 @@ mod tests { } } + const _: () = assert!( + MAX_BUCKET_REFRESH_LOOKUPS_PER_PASS > 0, + "bucket refresh budget must allow at least one lookup" + ); + #[test] fn bucket_refresh_selection_keeps_all_stale_buckets_within_budget() { let refresh_budget = MAX_BUCKET_REFRESH_LOOKUPS_PER_PASS; - assert!( - refresh_budget > 0, - "bucket refresh budget must allow at least one lookup" - ); let candidates: Vec<_> = (0..refresh_budget) .map(|idx| bucket_refresh_candidate(idx, 3_600 + idx as u64)) @@ -6226,10 +6227,6 @@ mod tests { #[test] fn bucket_refresh_selection_caps_large_stale_sets_by_debt() { let refresh_budget = MAX_BUCKET_REFRESH_LOOKUPS_PER_PASS; - assert!( - refresh_budget > 0, - "bucket refresh budget must allow at least one lookup" - ); let candidate_count = refresh_budget * 3; let candidates: Vec<_> = (0..candidate_count) diff --git a/src/network.rs b/src/network.rs index 459dc40a..ff53a2e8 100644 --- a/src/network.rs +++ b/src/network.rs @@ -300,13 +300,13 @@ pub struct NodeConfig { #[serde(default)] pub allow_loopback: bool, - /// Adaptive DHT configuration (trust-based swap-out). + /// Adaptive DHT configuration for trust-based routing enforcement. /// - /// Controls whether peers with low trust scores are eligible for - /// swap-out from the routing table when better candidates arrive. Use - /// `NodeConfigBuilder::trust_enforcement` for a simple on/off toggle. + /// Controls lazy swap-out, close-group quarantine, automatic lookup + /// avoidance, and new-peer/readmission trust thresholds. Use + /// [`NodeConfigBuilder::trust_enforcement`] for a simple on/off toggle. /// - /// Default: enabled with a swap threshold of 0.35. + /// Default: enabled with the default [`AdaptiveDhtConfig`] thresholds. #[serde(default)] pub adaptive_dht_config: AdaptiveDhtConfig, From 2220b72de1432afaedc0e526abe77b75bd88d1c6 Mon Sep 17 00:00:00 2001 From: Warm Beer Date: Thu, 21 May 2026 16:49:21 +0200 Subject: [PATCH 06/18] fix(dht): enable quarantine defaults in core config --- src/dht/core_engine.rs | 21 ++++++++++----------- src/dht_network_manager.rs | 15 ++++++++------- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/src/dht/core_engine.rs b/src/dht/core_engine.rs index 1e02f567..e791f90b 100644 --- a/src/dht/core_engine.rs +++ b/src/dht/core_engine.rs @@ -1528,7 +1528,7 @@ impl DhtCoreEngine { ip_diversity_config: IPDiversityConfig::default(), allow_loopback, swap_threshold, - quarantine_threshold: 0.0, + quarantine_threshold: DEFAULT_QUARANTINE_THRESHOLD, quarantine_readmit_threshold: DEFAULT_QUARANTINE_READMIT_THRESHOLD, quarantined_peers: HashSet::new(), live_threshold: LIVE_THRESHOLD, @@ -3844,25 +3844,24 @@ mod tests { const TEST_STALE_AGE: Duration = Duration::from_secs(2); // ----------------------------------------------------------------------- - // Test 4: low-trust peer admission (lazy swap-out model) + // Test 4: low-trust peer admission is gated by default quarantine policy // ----------------------------------------------------------------------- #[tokio::test] - async fn test_low_trust_candidate_still_admitted() { + async fn test_low_trust_candidate_rejected_by_default_quarantine() { let mut dht = DhtCoreEngine::new_for_tests(PeerId::from_bytes([0u8; 32])).unwrap(); let node = make_node(1, "/ip4/10.0.0.1/udp/9000/quic"); let peer_id = node.id; - // Candidate with trust below swap threshold is still admitted - // (lazy swap-out model: no admission blocking) + // Candidate below the default readmission/admission threshold is rejected. let result = dht .add_node(node, &|id| { if *id == peer_id { 0.1 } else { 0.5 } }) .await; - assert!(result.is_ok(), "low-trust candidate should be admitted"); - assert!(dht.has_node(&peer_id).await); + assert!(result.is_err(), "low-trust candidate should be rejected"); + assert!(!dht.has_node(&peer_id).await); } // ----------------------------------------------------------------------- @@ -4340,7 +4339,7 @@ mod tests { } #[tokio::test] - async fn test_re_evaluate_admits_low_trust_candidate() { + async fn test_re_evaluate_rejects_low_trust_candidate_by_default() { let mut dht = DhtCoreEngine::new( PeerId::from_bytes([0u8; 32]), 20, @@ -4354,14 +4353,14 @@ mod tests { let candidate = make_node_with_addr(id, "/ip4/10.0.0.1/udp/9000/quic"); let candidate_ips = vec!["10.0.0.1".parse().unwrap()]; - // Trust below swap threshold — should still be admitted + // Trust below readmission/admission threshold is rejected by default. let result = dht .re_evaluate_admission(candidate, &candidate_ips, &|_| 0.1) .await; assert!( - result.is_ok(), - "low-trust candidate should be admitted via re-evaluate" + result.is_err(), + "low-trust candidate should be rejected via re-evaluate" ); } diff --git a/src/dht_network_manager.rs b/src/dht_network_manager.rs index 492f5be7..e3f96ebe 100644 --- a/src/dht_network_manager.rs +++ b/src/dht_network_manager.rs @@ -20,7 +20,7 @@ use crate::{ P2PError, PeerId, Result, adaptive::trust::DEFAULT_NEUTRAL_TRUST, - adaptive::{NodeStatisticsUpdate, TrustEngine}, + adaptive::{AdaptiveDhtConfig, NodeStatisticsUpdate, TrustEngine}, address::{MultiAddr, is_lan_ip}, dht::core_engine::{AddressType, AtomicInstant, BucketRefreshCandidate, NodeInfo}, dht::{AdmissionResult, DhtCoreEngine, DhtKey, Key, RoutingTableEvent}, @@ -632,16 +632,16 @@ pub struct DhtNetworkConfig { pub enable_security: bool, /// Trust score below which a peer is eligible for swap-out from the /// routing table when a better candidate is available. - /// Default: 0.0 (disabled). + /// Default: [`AdaptiveDhtConfig::default`]. pub swap_threshold: f64, /// Trust score below which automatic lookup/dial paths avoid a peer, and /// K-closest peers are evicted into temporary quarantine when the routing /// table can keep at least K peers. - /// Default: 0.0 (disabled). + /// Default: [`AdaptiveDhtConfig::default`]. pub quarantine_threshold: f64, /// Trust score required before a new peer can enter the routing table, /// and before a quarantined peer can be admitted again. - /// Default: 0.0 (disabled). + /// Default: [`AdaptiveDhtConfig::default`]. pub quarantine_readmit_threshold: f64, } @@ -6159,15 +6159,16 @@ const DEFAULT_MAX_CONCURRENT_OPS: usize = 100; impl Default for DhtNetworkConfig { fn default() -> Self { + let adaptive_config = AdaptiveDhtConfig::default(); Self { peer_id: PeerId::from_bytes([0u8; 32]), node_config: NodeConfig::default(), request_timeout: Duration::from_secs(DEFAULT_REQUEST_TIMEOUT_SECS), max_concurrent_operations: DEFAULT_MAX_CONCURRENT_OPS, enable_security: true, - swap_threshold: 0.0, - quarantine_threshold: 0.0, - quarantine_readmit_threshold: 0.0, + swap_threshold: adaptive_config.swap_threshold, + quarantine_threshold: adaptive_config.quarantine_threshold, + quarantine_readmit_threshold: adaptive_config.quarantine_readmit_threshold, } } } From c62193b66e0bd4132250b14680e3f4c905f78327 Mon Sep 17 00:00:00 2001 From: Warm Beer Date: Thu, 21 May 2026 17:40:35 +0200 Subject: [PATCH 07/18] fix(dht): address quarantine review feedback --- src/dht/core_engine.rs | 220 ++++++++++++++++++++++++++++--------- src/dht_network_manager.rs | 91 ++++++--------- 2 files changed, 204 insertions(+), 107 deletions(-) diff --git a/src/dht/core_engine.rs b/src/dht/core_engine.rs index e791f90b..211d3232 100644 --- a/src/dht/core_engine.rs +++ b/src/dht/core_engine.rs @@ -9,7 +9,8 @@ use crate::security::{IP_EXACT_LIMIT, IPDiversityConfig, canonicalize_ip, ip_sub use anyhow::{Result, anyhow}; use parking_lot::Mutex as PlMutex; use serde::{Deserialize, Serialize}; -use std::collections::{HashMap, HashSet}; +use std::cmp::Ordering; +use std::collections::{BinaryHeap, HashMap, HashSet, VecDeque}; use std::net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr}; use std::sync::Arc; use std::time::{Duration, Instant}; @@ -211,6 +212,31 @@ const DEFAULT_QUARANTINE_THRESHOLD: f64 = 0.20; #[allow(dead_code)] const DEFAULT_QUARANTINE_READMIT_THRESHOLD: f64 = 0.45; +/// Maximum number of evicted quarantine markers retained by the routing engine. +const MAX_QUARANTINED_PEERS: usize = 8192; + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +struct ClosestNodeCandidate { + distance: [u8; 32], + peer_id: PeerId, + bucket_index: usize, + node_index: usize, +} + +impl Ord for ClosestNodeCandidate { + fn cmp(&self, other: &Self) -> Ordering { + self.distance + .cmp(&other.distance) + .then_with(|| self.peer_id.cmp(&other.peer_id)) + } +} + +impl PartialOrd for ClosestNodeCandidate { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + /// Node information for routing. /// /// The `addresses` field stores one or more typed [`MultiAddr`] values that are @@ -1118,44 +1144,69 @@ impl KademliaRoutingTable { } fn find_closest_nodes(&self, key: &DhtKey, count: usize) -> Vec { - // Collect ALL entries from every bucket. Bucket index correlates with - // distance from *self*, not from key K — peers in distant buckets can - // be closer to K than peers in nearby buckets. The routing table holds - // at most 256 * K_BUCKET_SIZE entries, so a full scan is trivially fast. - let mut candidates: Vec<(NodeInfo, [u8; 32])> = Vec::with_capacity(count * 2); - - for bucket in &self.buckets { - for node in bucket.get_nodes() { - let distance = xor_distance_bytes(node.id.to_bytes(), key.as_bytes()); - candidates.push((node.clone(), distance)); - } - } - - // Sort by distance - candidates.sort_by_key(|a| a.1); - - // Return top `count` nodes - candidates + self.find_closest_node_candidates_filtered(key, count, |_| true) .into_iter() - .take(count) - .map(|(node, _)| node) + .map(|candidate| { + self.buckets[candidate.bucket_index].nodes[candidate.node_index].clone() + }) .collect() } - fn find_closest_nodes_with_publish_seq( + fn find_closest_nodes_with_publish_seq_filtered( &self, key: &DhtKey, count: usize, + include: impl Fn(&NodeInfo) -> bool, ) -> Vec<(NodeInfo, u64)> { - self.find_closest_nodes(key, count) + self.find_closest_node_candidates_filtered(key, count, include) .into_iter() - .map(|node| { + .map(|candidate| { + let node = self.buckets[candidate.bucket_index].nodes[candidate.node_index].clone(); let seq = self.publish_seq_for(&node.id); (node, seq) }) .collect() } + fn find_closest_node_candidates_filtered( + &self, + key: &DhtKey, + count: usize, + include: impl Fn(&NodeInfo) -> bool, + ) -> Vec { + if count == 0 { + return Vec::new(); + } + + let mut closest = BinaryHeap::with_capacity(count); + + for (bucket_index, bucket) in self.buckets.iter().enumerate() { + for (node_index, node) in bucket.get_nodes().iter().enumerate() { + if !include(node) { + continue; + } + + let candidate = ClosestNodeCandidate { + distance: xor_distance_bytes(node.id.to_bytes(), key.as_bytes()), + peer_id: node.id, + bucket_index, + node_index, + }; + + if closest.len() < count { + closest.push(candidate); + } else if closest.peek().is_some_and(|farthest| candidate < *farthest) { + closest.pop(); + closest.push(candidate); + } + } + } + + let mut selected = closest.into_vec(); + selected.sort_unstable(); + selected + } + /// Returns the k-bucket index for a key, or `None` when the key equals /// the local node ID (XOR distance is zero — no valid bucket exists). fn get_bucket_index_for_key(&self, key: &DhtKey) -> Option { @@ -1478,11 +1529,14 @@ pub struct DhtCoreEngine { /// before a quarantined peer can re-enter. quarantine_readmit_threshold: f64, - /// Peers evicted from the close group by quarantine. They remain in this - /// set until discovered naturally and admitted after crossing the readmit + /// Peers evicted from the close group by quarantine. Markers are bounded + /// to cap memory use and are removed when the peer crosses the readmit /// threshold. quarantined_peers: HashSet, + /// FIFO order used to prune the bounded quarantine marker set. + quarantined_peer_order: VecDeque, + /// Duration of no contact after which a peer is considered stale. /// Defaults to [`LIVE_THRESHOLD`]; overridden in tests to avoid /// `Instant` subtraction overflow on Windows (where `Instant` starts @@ -1531,6 +1585,7 @@ impl DhtCoreEngine { quarantine_threshold: DEFAULT_QUARANTINE_THRESHOLD, quarantine_readmit_threshold: DEFAULT_QUARANTINE_READMIT_THRESHOLD, quarantined_peers: HashSet::new(), + quarantined_peer_order: VecDeque::new(), live_threshold: LIVE_THRESHOLD, shutdown: CancellationToken::new(), }) @@ -1576,11 +1631,6 @@ impl DhtCoreEngine { self.quarantine_threshold > 0.0 } - /// Return whether trust quarantine affects lookup result filtering. - pub(crate) fn trust_quarantine_enabled(&self) -> bool { - self.quarantine_enabled() - } - fn check_new_peer_admission(&mut self, peer_id: &PeerId, trust_score: f64) -> Result<()> { if !self.quarantine_enabled() { return Ok(()); @@ -1605,10 +1655,46 @@ impl DhtCoreEngine { self.quarantine_readmit_threshold )); } - self.quarantined_peers.remove(peer_id); + self.forget_quarantined_peer(peer_id); Ok(()) } + #[cfg(test)] + fn remember_quarantined_peer(&mut self, peer_id: PeerId) { + Self::remember_quarantined_peer_in( + &mut self.quarantined_peers, + &mut self.quarantined_peer_order, + peer_id, + ); + } + + fn remember_quarantined_peer_in( + quarantined_peers: &mut HashSet, + quarantined_peer_order: &mut VecDeque, + peer_id: PeerId, + ) { + if !quarantined_peers.insert(peer_id) { + return; + } + + quarantined_peer_order.push_back(peer_id); + while quarantined_peers.len() > MAX_QUARANTINED_PEERS { + let Some(stale_peer) = quarantined_peer_order.pop_front() else { + break; + }; + quarantined_peers.remove(&stale_peer); + } + } + + fn forget_quarantined_peer(&mut self, peer_id: &PeerId) { + if !self.quarantined_peers.remove(peer_id) { + return; + } + + self.quarantined_peer_order + .retain(|quarantined_peer| quarantined_peer != peer_id); + } + /// Return whether automatic lookup/dial paths should avoid this peer. pub(crate) fn should_avoid_for_lookup(&self, peer_id: &PeerId, trust_score: f64) -> bool { if !self.quarantine_enabled() { @@ -1669,7 +1755,11 @@ impl DhtCoreEngine { return Vec::new(); } - self.quarantined_peers.insert(*peer_id); + Self::remember_quarantined_peer_in( + &mut self.quarantined_peers, + &mut self.quarantined_peer_order, + *peer_id, + ); routing.remove_node(peer_id); let k_after = routing.k_closest_ids(self.k_value); @@ -1692,7 +1782,6 @@ impl DhtCoreEngine { /// retained rather than shrinking the routing table below K entries. pub(crate) async fn enforce_close_group_trust_gate( &mut self, - _previous_close_group: Option<&[PeerId]>, trust_score: &impl Fn(&PeerId) -> f64, ) -> Vec { if !self.quarantine_enabled() { @@ -1715,9 +1804,11 @@ impl DhtCoreEngine { break; }; - if trust_score(&peer_id) < self.quarantine_threshold { - self.quarantined_peers.insert(peer_id); - } + Self::remember_quarantined_peer_in( + &mut self.quarantined_peers, + &mut self.quarantined_peer_order, + peer_id, + ); routing.remove_node(&peer_id); removed.push(peer_id); } @@ -1875,15 +1966,16 @@ impl DhtCoreEngine { Ok(routing.find_closest_nodes(key, count)) } - /// Find nodes closest to a key and include the latest authoritative - /// `PublishAddressSet` sequence known for each returned record. - pub async fn find_nodes_with_publish_seq( + /// Find nodes closest to a key after applying a caller-provided routing + /// policy filter. + pub(crate) async fn find_nodes_with_publish_seq_filtered( &self, key: &DhtKey, count: usize, + include: impl Fn(&NodeInfo) -> bool, ) -> Result> { let routing = self.routing_table.read().await; - Ok(routing.find_closest_nodes_with_publish_seq(key, count)) + Ok(routing.find_closest_nodes_with_publish_seq_filtered(key, count, include)) } /// Find nodes closest to a key, including self as a candidate. @@ -2942,6 +3034,14 @@ mod tests { } } + fn peer_id_from_index(index: usize) -> PeerId { + let mut id = [0u8; 32]; + let index_bytes = index.to_be_bytes(); + let offset = id.len() - index_bytes.len(); + id[offset..].copy_from_slice(&index_bytes); + PeerId::from_bytes(id) + } + // ----------------------------------------------------------------------- // KBucket::touch_node tests // ----------------------------------------------------------------------- @@ -5245,7 +5345,7 @@ mod tests { let low_peer = close_peer_ids[0]; let events = dht - .enforce_close_group_trust_gate(None, &|id| if *id == low_peer { 0.10 } else { 0.5 }) + .enforce_close_group_trust_gate(&|id| if *id == low_peer { 0.10 } else { 0.5 }) .await; assert!( @@ -5289,7 +5389,7 @@ mod tests { let low_peer = close_peer_ids[0]; let deferred = dht - .enforce_close_group_trust_gate(None, &|id| if *id == low_peer { 0.10 } else { 0.5 }) + .enforce_close_group_trust_gate(&|id| if *id == low_peer { 0.10 } else { 0.5 }) .await; assert!(deferred.is_empty()); @@ -5304,7 +5404,7 @@ mod tests { assert_eq!(dht.routing_table_size().await, SMALL_TEST_K + 1); let events = dht - .enforce_close_group_trust_gate(None, &|id| if *id == low_peer { 0.10 } else { 0.5 }) + .enforce_close_group_trust_gate(&|id| if *id == low_peer { 0.10 } else { 0.5 }) .await; assert!( events.iter().any( @@ -5323,6 +5423,28 @@ mod tests { assert!(dht.set_trust_quarantine_thresholds(0.20, 0.50).is_err()); } + #[test] + fn test_quarantined_peer_markers_are_bounded() { + let mut dht = DhtCoreEngine::new_for_tests(PeerId::from_bytes([0u8; 32])).unwrap(); + let oldest_peer = peer_id_from_index(0); + let newest_peer = peer_id_from_index(MAX_QUARANTINED_PEERS); + + for index in 0..=MAX_QUARANTINED_PEERS { + dht.remember_quarantined_peer(peer_id_from_index(index)); + } + + assert_eq!(dht.quarantined_peers.len(), MAX_QUARANTINED_PEERS); + assert_eq!(dht.quarantined_peer_order.len(), MAX_QUARANTINED_PEERS); + assert!( + !dht.quarantined_peers.contains(&oldest_peer), + "oldest quarantine marker should be pruned at capacity" + ); + assert!( + dht.quarantined_peers.contains(&newest_peer), + "newest quarantine marker should be retained" + ); + } + /// New peers must meet the readmit/admission threshold even when they would /// occupy a non-close routing-table slot. #[tokio::test] @@ -5487,8 +5609,6 @@ mod tests { .await .unwrap(); } - let previous_close_group = close_peer_ids.clone(); - let mut promoted_id = [0u8; 32]; promoted_id[0] = 0x80; let promoted_peer = PeerId::from_bytes(promoted_id); @@ -5500,7 +5620,7 @@ mod tests { .unwrap(); let no_events = dht - .enforce_close_group_trust_gate(Some(&previous_close_group), &|id| { + .enforce_close_group_trust_gate(&|id| { if *id == promoted_peer { 0.30 } else { 0.5 } }) .await; @@ -5512,7 +5632,7 @@ mod tests { dht.remove_node_by_id(&close_peer_ids[0]).await; let events = dht - .enforce_close_group_trust_gate(Some(&previous_close_group), &|id| { + .enforce_close_group_trust_gate(&|id| { if *id == promoted_peer || *id == close_peer_ids[1] { 0.30 } else { @@ -5534,7 +5654,7 @@ mod tests { ); let retained_events = dht - .enforce_close_group_trust_gate(Some(&previous_close_group), &|id| { + .enforce_close_group_trust_gate(&|id| { if *id == promoted_peer { 0.10 } else { 0.5 } }) .await; @@ -5554,7 +5674,7 @@ mod tests { .unwrap(); let quarantine_events = dht - .enforce_close_group_trust_gate(Some(&previous_close_group), &|id| { + .enforce_close_group_trust_gate(&|id| { if *id == promoted_peer { 0.10 } else { 0.5 } }) .await; diff --git a/src/dht_network_manager.rs b/src/dht_network_manager.rs index e3f96ebe..ffaaf93a 100644 --- a/src/dht_network_manager.rs +++ b/src/dht_network_manager.rs @@ -2686,23 +2686,20 @@ impl DhtNetworkManager { ); let dht_guard = self.dht.read().await; - let candidate_count = if dht_guard.trust_quarantine_enabled() { - dht_guard.routing_table_size().await.max(count) - } else { - count - }; + let dht_key = DhtKey::from_bytes(*key); + let local_peer_id = self.config.peer_id; let trust_score = |peer_id: &PeerId| self.peer_trust_score(peer_id); match dht_guard - .find_nodes_with_publish_seq(&DhtKey::from_bytes(*key), candidate_count) + .find_nodes_with_publish_seq_filtered(&dht_key, count, |node| { + if node.id == local_peer_id { + return false; + } + let score = trust_score(&node.id); + !dht_guard.should_avoid_for_lookup(&node.id, score) + }) .await { - Ok(nodes) => Self::lookup_results_from_routing_nodes( - self.config.peer_id, - &dht_guard, - nodes, - &trust_score, - count, - ), + Ok(nodes) => Self::lookup_results_from_routing_nodes(nodes, count), Err(e) => { warn!("find_nodes failed for key {}: {e}", hex::encode(key)); Vec::new() @@ -2711,32 +2708,22 @@ impl DhtNetworkManager { } fn lookup_results_from_routing_nodes( - local_peer_id: PeerId, - dht: &DhtCoreEngine, nodes: Vec<(NodeInfo, u64)>, - trust_score: &impl Fn(&PeerId) -> f64, count: usize, ) -> Vec { nodes .into_iter() - .filter(|(node, _)| node.id != local_peer_id) - .filter_map(|(node, publish_seq)| { - let reliability = trust_score(&node.id); - if dht.should_avoid_for_lookup(&node.id, reliability) { - return None; - } - Some(DHTNode { - peer_id: node.id, - address_types: node.address_types, - addresses: node.addresses, - distance: encode_publish_seq_distance(publish_seq), - // Keep the legacy wire value stable. Trust is local policy - // used for filtering and should not change DHTNode wire - // semantics for older nodes. - reliability: SELF_RELIABILITY_SCORE, - }) - }) .take(count) + .map(|(node, publish_seq)| DHTNode { + peer_id: node.id, + address_types: node.address_types, + addresses: node.addresses, + distance: encode_publish_seq_distance(publish_seq), + // Keep the legacy wire value stable. Trust is local policy + // used for filtering and should not change DHTNode wire + // semantics for older nodes. + reliability: SELF_RELIABILITY_SCORE, + }) .collect() } @@ -3697,7 +3684,7 @@ impl DhtNetworkManager { return false; }; let trust_score = engine.score(peer_id); - let rt_events = self.enforce_close_group_trust_gate(None).await; + let rt_events = self.enforce_close_group_trust_gate().await; if rt_events.is_empty() { return false; } @@ -5686,18 +5673,14 @@ impl DhtNetworkManager { }) } - async fn enforce_close_group_trust_gate( - &self, - previous_close_group: Option<&[PeerId]>, - ) -> Vec { + async fn enforce_close_group_trust_gate(&self) -> Vec { let Some(ref engine) = self.trust_engine else { return Vec::new(); }; let trust_score = |peer_id: &PeerId| engine.score(peer_id); let rt_events = { let mut dht = self.dht.write().await; - dht.enforce_close_group_trust_gate(previous_close_group, &trust_score) - .await + dht.enforce_close_group_trust_gate(&trust_score).await }; if !rt_events.is_empty() { let removed: Vec = rt_events @@ -5722,9 +5705,7 @@ impl DhtNetworkManager { RoutingTableEvent::KClosestPeersChanged { old, .. } => Some(old.clone()), _ => None, }); - let quarantine_events = self - .enforce_close_group_trust_gate(original_old_close_group.as_deref()) - .await; + let quarantine_events = self.enforce_close_group_trust_gate().await; if quarantine_events.is_empty() { self.broadcast_routing_events(&events); return; @@ -6359,12 +6340,6 @@ mod tests { "test setup should quarantine the close-group peer" ); - let nodes = vec![ - (routing_test_node(0), 0), - (routing_test_node(1), 0), - (routing_test_node(2), 0), - (routing_test_node(3), 0), - ]; let trust_score = |peer_id: &PeerId| { if *peer_id == quarantined_peer { 0.30 @@ -6375,16 +6350,18 @@ mod tests { } }; - let results = DhtNetworkManager::lookup_results_from_routing_nodes( - local_peer, - &dht, - nodes, - &trust_score, - 4, - ); + let lookup_key = DhtKey::from_bytes(*local_peer.as_bytes()); + let nodes = dht + .find_nodes_with_publish_seq_filtered(&lookup_key, 4, |node| { + let score = trust_score(&node.id); + node.id != local_peer && !dht.should_avoid_for_lookup(&node.id, score) + }) + .await + .unwrap(); + let results = DhtNetworkManager::lookup_results_from_routing_nodes(nodes, 4); let result_ids: Vec = results.iter().map(|node| node.peer_id).collect(); - assert_eq!(result_ids, vec![healthy_peer]); + assert_eq!(result_ids, vec![healthy_peer, pid(4), pid(5)]); assert!( results .iter() From 25fd2937ca60df599627f049e208b312bc08fcd1 Mon Sep 17 00:00:00 2001 From: Warm Beer Date: Thu, 21 May 2026 18:12:03 +0200 Subject: [PATCH 08/18] fix(dht): preserve quarantine marker semantics --- docs/ROUTING_TABLE_DESIGN.md | 3 +- src/adaptive/dht.rs | 40 ++++++++- src/dht/core_engine.rs | 163 ++++++++++++++++++++++++++++++----- 3 files changed, 182 insertions(+), 24 deletions(-) diff --git a/docs/ROUTING_TABLE_DESIGN.md b/docs/ROUTING_TABLE_DESIGN.md index afb213d8..f4056fed 100644 --- a/docs/ROUTING_TABLE_DESIGN.md +++ b/docs/ROUTING_TABLE_DESIGN.md @@ -107,7 +107,8 @@ Parameter safety constraints (MUST hold): 1. `IP_EXACT_LIMIT >= 1`. 2. `IP_SUBNET_LIMIT >= 1`. -3. `TRUST_PROTECTION_THRESHOLD > SWAP_THRESHOLD > QUARANTINE_THRESHOLD`. +3. `TRUST_PROTECTION_THRESHOLD > SWAP_THRESHOLD > QUARANTINE_THRESHOLD` when swap enforcement is enabled. `SWAP_THRESHOLD = 0.0` is the explicit + disabled state and may be paired with quarantine enforcement. 4. `ALPHA >= 1`. 5. `LIVE_THRESHOLD > max(SELF_LOOKUP_INTERVAL)` (peers touched by self-lookup must not oscillate between live and stale between consecutive cycles; at reference values: 15 min > 10 min). The 5-minute margin at reference values is sufficient for typical network latencies (sub-second RTTs). Operators in high-latency environments (satellite, Tor overlay) SHOULD increase `LIVE_THRESHOLD` proportionally. 6. `STALE_REVALIDATION_TIMEOUT > 0`. diff --git a/src/adaptive/dht.rs b/src/adaptive/dht.rs index f7a8b075..30b925d9 100644 --- a/src/adaptive/dht.rs +++ b/src/adaptive/dht.rs @@ -81,6 +81,8 @@ impl AdaptiveDhtConfig { /// swap/quarantine eligible since they start at neutral (0.5). The /// new-peer admission/readmit threshold must also stay below neutral /// because recovery happens by decay toward neutral, not by active probing. + /// When swap enforcement is enabled, the swap threshold must remain above + /// the quarantine threshold so quarantine is strictly more severe. pub fn validate(&self) -> crate::error::P2pResult<()> { if !(0.0..0.5).contains(&self.swap_threshold) || self.swap_threshold.is_nan() { return Err(crate::error::P2PError::Validation( @@ -122,6 +124,18 @@ impl AdaptiveDhtConfig { .into(), )); } + if self.swap_threshold > 0.0 + && self.quarantine_threshold > 0.0 + && self.swap_threshold <= self.quarantine_threshold + { + return Err(crate::error::P2PError::Validation( + format!( + "swap_threshold ({}) must be > quarantine_threshold ({}) when both are enabled", + self.swap_threshold, self.quarantine_threshold + ) + .into(), + )); + } Ok(()) } } @@ -418,7 +432,7 @@ mod tests { #[test] fn test_swap_threshold_validation_accepts_valid() { - for &good in &[0.0, 0.15, 0.49] { + for &good in &[0.0, 0.25, 0.49] { let config = AdaptiveDhtConfig { swap_threshold: good, ..Default::default() @@ -428,6 +442,30 @@ mod tests { "swap_threshold {good} should pass validation" ); } + + let quarantine_disabled = AdaptiveDhtConfig { + swap_threshold: 0.15, + quarantine_threshold: 0.0, + quarantine_readmit_threshold: 0.0, + }; + assert!(quarantine_disabled.validate().is_ok()); + } + + #[test] + fn test_swap_threshold_must_exceed_quarantine_threshold_when_enabled() { + let equal = AdaptiveDhtConfig { + swap_threshold: 0.20, + quarantine_threshold: 0.20, + ..Default::default() + }; + assert!(equal.validate().is_err()); + + let below = AdaptiveDhtConfig { + swap_threshold: 0.15, + quarantine_threshold: 0.20, + ..Default::default() + }; + assert!(below.validate().is_err()); } #[test] diff --git a/src/dht/core_engine.rs b/src/dht/core_engine.rs index 211d3232..bcd5f404 100644 --- a/src/dht/core_engine.rs +++ b/src/dht/core_engine.rs @@ -213,6 +213,10 @@ const DEFAULT_QUARANTINE_THRESHOLD: f64 = 0.20; const DEFAULT_QUARANTINE_READMIT_THRESHOLD: f64 = 0.45; /// Maximum number of evicted quarantine markers retained by the routing engine. +/// +/// Markers are only semantically required while a peer has recovered above the +/// quarantine threshold but remains below the readmit threshold; below the +/// quarantine threshold the trust score itself keeps the peer avoided. const MAX_QUARANTINED_PEERS: usize = 8192; #[derive(Clone, Copy, Debug, Eq, PartialEq)] @@ -1530,8 +1534,8 @@ pub struct DhtCoreEngine { quarantine_readmit_threshold: f64, /// Peers evicted from the close group by quarantine. Markers are bounded - /// to cap memory use and are removed when the peer crosses the readmit - /// threshold. + /// to cap memory use; redundant markers are removed when direct trust-score + /// checks are sufficient or when the peer crosses the readmit threshold. quarantined_peers: HashSet, /// FIFO order used to prune the bounded quarantine marker set. @@ -1660,30 +1664,86 @@ impl DhtCoreEngine { } #[cfg(test)] - fn remember_quarantined_peer(&mut self, peer_id: PeerId) { - Self::remember_quarantined_peer_in( + fn remember_quarantined_peer( + &mut self, + peer_id: PeerId, + trust_score: &impl Fn(&PeerId) -> f64, + ) { + let quarantine_threshold = self.quarantine_threshold; + let quarantine_readmit_threshold = self.quarantine_readmit_threshold; + Self::remember_quarantined_peer_with_trust( &mut self.quarantined_peers, &mut self.quarantined_peer_order, + quarantine_threshold, + quarantine_readmit_threshold, peer_id, + trust_score, ); } - fn remember_quarantined_peer_in( + fn quarantine_marker_required_for_score( + score: f64, + quarantine_threshold: f64, + quarantine_readmit_threshold: f64, + ) -> bool { + score.is_finite() && score >= quarantine_threshold && score < quarantine_readmit_threshold + } + + fn prune_redundant_quarantined_peers( + quarantined_peers: &mut HashSet, + quarantined_peer_order: &mut VecDeque, + quarantine_threshold: f64, + quarantine_readmit_threshold: f64, + trust_score: &impl Fn(&PeerId) -> f64, + ) { + let mut retained_order = VecDeque::with_capacity(quarantined_peer_order.len()); + while let Some(peer_id) = quarantined_peer_order.pop_front() { + if !quarantined_peers.contains(&peer_id) { + continue; + } + + let score = trust_score(&peer_id); + if Self::quarantine_marker_required_for_score( + score, + quarantine_threshold, + quarantine_readmit_threshold, + ) { + retained_order.push_back(peer_id); + } else { + quarantined_peers.remove(&peer_id); + } + } + + *quarantined_peer_order = retained_order; + } + + fn remember_quarantined_peer_with_trust( quarantined_peers: &mut HashSet, quarantined_peer_order: &mut VecDeque, + quarantine_threshold: f64, + quarantine_readmit_threshold: f64, peer_id: PeerId, + trust_score: &impl Fn(&PeerId) -> f64, ) { - if !quarantined_peers.insert(peer_id) { + if quarantined_peers.contains(&peer_id) { return; } - quarantined_peer_order.push_back(peer_id); - while quarantined_peers.len() > MAX_QUARANTINED_PEERS { - let Some(stale_peer) = quarantined_peer_order.pop_front() else { - break; - }; - quarantined_peers.remove(&stale_peer); + if quarantined_peers.len() >= MAX_QUARANTINED_PEERS { + Self::prune_redundant_quarantined_peers( + quarantined_peers, + quarantined_peer_order, + quarantine_threshold, + quarantine_readmit_threshold, + trust_score, + ); } + if quarantined_peers.len() >= MAX_QUARANTINED_PEERS { + return; + } + + quarantined_peers.insert(peer_id); + quarantined_peer_order.push_back(peer_id); } fn forget_quarantined_peer(&mut self, peer_id: &PeerId) { @@ -1755,10 +1815,21 @@ impl DhtCoreEngine { return Vec::new(); } - Self::remember_quarantined_peer_in( + let quarantine_threshold = self.quarantine_threshold; + let quarantine_readmit_threshold = self.quarantine_readmit_threshold; + Self::remember_quarantined_peer_with_trust( &mut self.quarantined_peers, &mut self.quarantined_peer_order, + quarantine_threshold, + quarantine_readmit_threshold, *peer_id, + &|id| { + if id == peer_id { + trust_score + } else { + quarantine_threshold + } + }, ); routing.remove_node(peer_id); @@ -1804,10 +1875,15 @@ impl DhtCoreEngine { break; }; - Self::remember_quarantined_peer_in( + let quarantine_threshold = self.quarantine_threshold; + let quarantine_readmit_threshold = self.quarantine_readmit_threshold; + Self::remember_quarantined_peer_with_trust( &mut self.quarantined_peers, &mut self.quarantined_peer_order, + quarantine_threshold, + quarantine_readmit_threshold, peer_id, + trust_score, ); routing.remove_node(&peer_id); removed.push(peer_id); @@ -3013,6 +3089,10 @@ mod tests { use std::collections::HashSet; const SMALL_TEST_K: usize = 4; + const TEST_QUARANTINE_THRESHOLD: f64 = 0.20; + const TEST_QUARANTINE_READMIT_THRESHOLD: f64 = 0.45; + const TEST_QUARANTINE_LOW_SCORE: f64 = 0.10; + const TEST_QUARANTINE_MARKER_REQUIRED_SCORE: f64 = 0.30; #[tokio::test] async fn test_xor_distance() { @@ -5424,24 +5504,63 @@ mod tests { } #[test] - fn test_quarantined_peer_markers_are_bounded() { + fn test_quarantined_peer_marker_cap_keeps_readmit_gap_markers() { let mut dht = DhtCoreEngine::new_for_tests(PeerId::from_bytes([0u8; 32])).unwrap(); + dht.set_trust_quarantine_thresholds( + TEST_QUARANTINE_THRESHOLD, + TEST_QUARANTINE_READMIT_THRESHOLD, + ) + .unwrap(); let oldest_peer = peer_id_from_index(0); - let newest_peer = peer_id_from_index(MAX_QUARANTINED_PEERS); + let low_trust_overflow_peer = peer_id_from_index(MAX_QUARANTINED_PEERS); + let recovered_peer = oldest_peer; + let retained_overflow_peer = peer_id_from_index(MAX_QUARANTINED_PEERS + 1); - for index in 0..=MAX_QUARANTINED_PEERS { - dht.remember_quarantined_peer(peer_id_from_index(index)); + for index in 0..MAX_QUARANTINED_PEERS { + dht.remember_quarantined_peer(peer_id_from_index(index), &|_| { + TEST_QUARANTINE_MARKER_REQUIRED_SCORE + }); } assert_eq!(dht.quarantined_peers.len(), MAX_QUARANTINED_PEERS); assert_eq!(dht.quarantined_peer_order.len(), MAX_QUARANTINED_PEERS); + + dht.remember_quarantined_peer(low_trust_overflow_peer, &|peer_id| { + if *peer_id == low_trust_overflow_peer { + TEST_QUARANTINE_LOW_SCORE + } else { + TEST_QUARANTINE_MARKER_REQUIRED_SCORE + } + }); + + assert_eq!(dht.quarantined_peers.len(), MAX_QUARANTINED_PEERS); + assert!( + dht.quarantined_peers.contains(&oldest_peer), + "markers still in the readmit gap should not be pruned" + ); + assert!( + !dht.quarantined_peers.contains(&low_trust_overflow_peer), + "a below-quarantine peer remains avoided by score and can be dropped when cap is full" + ); + assert!(dht.should_avoid_for_lookup(&oldest_peer, TEST_QUARANTINE_MARKER_REQUIRED_SCORE)); + assert!(dht.should_avoid_for_lookup(&low_trust_overflow_peer, TEST_QUARANTINE_LOW_SCORE)); + + dht.remember_quarantined_peer(retained_overflow_peer, &|peer_id| { + if *peer_id == recovered_peer { + TEST_QUARANTINE_READMIT_THRESHOLD + } else { + TEST_QUARANTINE_MARKER_REQUIRED_SCORE + } + }); + + assert_eq!(dht.quarantined_peers.len(), MAX_QUARANTINED_PEERS); assert!( - !dht.quarantined_peers.contains(&oldest_peer), - "oldest quarantine marker should be pruned at capacity" + !dht.quarantined_peers.contains(&recovered_peer), + "recovered marker should be pruned at capacity" ); assert!( - dht.quarantined_peers.contains(&newest_peer), - "newest quarantine marker should be retained" + dht.quarantined_peers.contains(&retained_overflow_peer), + "new marker should be retained after pruning a recovered peer" ); } From aa75b0d73229bf42f62323d5dbcd7f6aa9115a2c Mon Sep 17 00:00:00 2001 From: Hermes Agent Date: Thu, 25 Jun 2026 16:09:09 +0100 Subject: [PATCH 09/18] fix(dht): preserve stale revalidation admissions --- src/dht/core_engine.rs | 47 +++++++++++++++++++++++++++++++++++++- src/dht_network_manager.rs | 42 ++++++++++++++++++++++++++-------- 2 files changed, 78 insertions(+), 11 deletions(-) diff --git a/src/dht/core_engine.rs b/src/dht/core_engine.rs index bcd5f404..c444958f 100644 --- a/src/dht/core_engine.rs +++ b/src/dht/core_engine.rs @@ -3048,7 +3048,12 @@ impl DhtCoreEngine { candidate_ips: &[IpAddr], trust_score: &impl Fn(&PeerId) -> f64, ) -> Result> { - self.check_new_peer_admission(&candidate.id, trust_score(&candidate.id))?; + let candidate_id = candidate.id; + let candidate_trust_score = trust_score(&candidate_id); + let peer_already_known = self.has_node(&candidate_id).await; + if !peer_already_known { + self.check_new_peer_admission(&candidate_id, candidate_trust_score)?; + } let mut routing = self.routing_table.write().await; match self.add_with_diversity(&mut routing, candidate, candidate_ips, trust_score, false)? { AdmissionResult::Admitted(events) => Ok(events), @@ -4544,6 +4549,46 @@ mod tests { ); } + #[tokio::test] + async fn test_re_evaluate_updates_existing_peer_below_readmit_threshold() { + let mut dht = DhtCoreEngine::new( + PeerId::from_bytes([0u8; 32]), + 20, + false, + DEFAULT_SWAP_THRESHOLD, + ) + .unwrap(); + dht.set_trust_quarantine_thresholds(0.20, 0.45).unwrap(); + + let mut id = [0u8; 32]; + id[0] = 0x80; + let peer = PeerId::from_bytes(id); + dht.add_node_no_trust(make_node_with_addr(id, "/ip4/10.0.0.1/udp/9000/quic")) + .await + .unwrap(); + + let candidate = make_node_with_addr(id, "/ip4/10.0.0.2/udp/9000/quic"); + let candidate_ips = vec!["10.0.0.2".parse().unwrap()]; + + let events = dht + .re_evaluate_admission(candidate, &candidate_ips, &|id| { + if *id == peer { 0.30 } else { 0.5 } + }) + .await + .unwrap(); + + assert!( + events.is_empty(), + "existing peer update should not emit a second admission event" + ); + assert!(dht.has_node(&peer).await); + assert!( + dht.get_node_addresses(&peer) + .await + .contains(&"/ip4/10.0.0.2/udp/9000/quic".parse().unwrap()) + ); + } + #[tokio::test] async fn test_re_evaluate_does_not_trigger_second_revalidation() { // Use k=4 (minimum valid K) so the bucket fills quickly. diff --git a/src/dht_network_manager.rs b/src/dht_network_manager.rs index ffaaf93a..13ece325 100644 --- a/src/dht_network_manager.rs +++ b/src/dht_network_manager.rs @@ -5247,11 +5247,18 @@ impl DhtNetworkManager { }; match result { - Ok(rt_events) => { - info!( - "Added peer {} to DHT routing table after stale revalidation", - app_peer_id_hex - ); + Ok((rt_events, candidate_admitted)) => { + if candidate_admitted { + info!( + "Added peer {} to DHT routing table after stale revalidation", + app_peer_id_hex + ); + } else { + info!( + "Stale revalidation removed stale peers; peer {} was not admitted", + app_peer_id_hex + ); + } this.broadcast_routing_events_with_quarantine(rt_events) .await; } @@ -5473,7 +5480,7 @@ impl DhtNetworkManager { bucket_idx: usize, stale_peers: Vec<(PeerId, usize)>, trust_fn: &impl Fn(&PeerId) -> f64, - ) -> anyhow::Result> { + ) -> anyhow::Result<(Vec, bool)> { if stale_peers.is_empty() { return Err(anyhow::anyhow!("no stale peers to revalidate")); } @@ -5548,12 +5555,27 @@ impl DhtNetworkManager { all_events.extend(removal_events); } - let admission_events = dht + let candidate_id = candidate.id; + let candidate_admitted = match dht .re_evaluate_admission(candidate, &candidate_ips, trust_fn) - .await?; - all_events.extend(admission_events); + .await + { + Ok(admission_events) => { + all_events.extend(admission_events); + true + } + Err(err) if !all_events.is_empty() => { + warn!( + "Candidate {} was not admitted after stale-peer eviction: {}; broadcasting committed removal events", + candidate_id.to_hex(), + err + ); + false + } + Err(err) => return Err(err), + }; - Ok(all_events) + Ok((all_events, candidate_admitted)) } /// Ping a peer to check liveness. From f8fb82ca8ab7149f66ad5b53e154d8322eaf0966 Mon Sep 17 00:00:00 2001 From: Mick van Dijke <12992260+mickvandijke@users.noreply.github.com> Date: Thu, 9 Jul 2026 15:18:17 +0200 Subject: [PATCH 10/18] fix(dht): disable immediate close-group trust eviction --- src/adaptive/dht.rs | 21 ++++---- src/dht/core_engine.rs | 98 ++++++++++++++++++++++---------------- src/dht_network_manager.rs | 16 ++----- src/network.rs | 4 +- 4 files changed, 71 insertions(+), 68 deletions(-) diff --git a/src/adaptive/dht.rs b/src/adaptive/dht.rs index 30b925d9..138fd623 100644 --- a/src/adaptive/dht.rs +++ b/src/adaptive/dht.rs @@ -31,9 +31,8 @@ use std::sync::Arc; /// Default trust score threshold below which a peer is eligible for swap-out const DEFAULT_SWAP_THRESHOLD: f64 = 0.35; -/// Default trust score threshold below which close-group peers are evicted -/// when doing so still leaves at least K routing-table peers, and all peers -/// are avoided by automatic lookup/dial paths. +/// Default trust score threshold below which peers are avoided by automatic +/// lookup/dial paths. const DEFAULT_QUARANTINE_THRESHOLD: f64 = 0.20; /// Default trust score a new or quarantined peer must have for admission. @@ -52,9 +51,7 @@ pub struct AdaptiveDhtConfig { /// Peers are not immediately evicted by this threshold alone. /// Default: 0.35 pub swap_threshold: f64, - /// Trust score below which automatic lookup/dial paths avoid a peer, and - /// K-closest peers are evicted into temporary quarantine when the routing - /// table can keep at least K peers. + /// Trust score below which automatic lookup/dial paths avoid a peer. /// Default: 0.20 pub quarantine_threshold: f64, /// Trust score required before a new peer can enter the routing table, and @@ -254,9 +251,9 @@ impl AdaptiveDHT { /// /// Trust scores are updated immediately. Peers below the quarantine /// threshold are avoided by lookup result selection and automatic - /// lookup/dial paths, and K-closest peers below that threshold are - /// evicted into temporary quarantine when the routing table can keep at - /// least K peers. + /// lookup/dial paths. Immediate close-group eviction is temporarily + /// disabled until trust scoring is stable; low-trust peers remain eligible + /// for lazy swap-out when better candidates arrive. pub async fn report_trust_event(&self, peer_id: &PeerId, event: TrustEvent) { match event { TrustEvent::ApplicationSuccess(weight) | TrustEvent::ApplicationFailure(weight) => { @@ -315,9 +312,9 @@ impl AdaptiveDHT { /// Start the DHT manager. /// /// Trust scores are computed live — no background tasks needed. - /// Low-trust peers are swapped out when better candidates arrive; close - /// peers below the quarantine threshold are evicted when the routing table - /// has enough peers. + /// Low-trust peers are swapped out when better candidates arrive. Immediate + /// close-group eviction is temporarily disabled until trust scoring is + /// stable. pub async fn start(&self) -> Result<()> { Arc::clone(&self.dht_manager).start().await } diff --git a/src/dht/core_engine.rs b/src/dht/core_engine.rs index c444958f..05feb2b2 100644 --- a/src/dht/core_engine.rs +++ b/src/dht/core_engine.rs @@ -208,6 +208,15 @@ const DEFAULT_SWAP_THRESHOLD: f64 = 0.35; #[allow(dead_code)] const DEFAULT_QUARANTINE_THRESHOLD: f64 = 0.20; +/// Immediate close-group eviction is disabled until trust scoring is stable. +/// +/// Low-trust peers remain eligible for lazy swap-out through +/// [`DhtCoreEngine::add_node`], but existing close-group peers are not evicted +/// solely because their score drops below the quarantine threshold. +fn close_group_immediate_eviction_enabled() -> bool { + false +} + /// Default trust score required for new routing-table admission/readmission. #[allow(dead_code)] const DEFAULT_QUARANTINE_READMIT_THRESHOLD: f64 = 0.45; @@ -1525,8 +1534,7 @@ pub struct DhtCoreEngine { swap_threshold: f64, /// Trust score below which a peer is avoided for automatic lookup/dial - /// paths, and evicted if it is in the K-closest close group and the - /// routing table can keep at least K peers. + /// paths. quarantine_threshold: f64, /// Trust score required before a new peer can enter the routing table, and @@ -1598,9 +1606,7 @@ impl DhtCoreEngine { /// Configure trust quarantine thresholds. /// /// A `quarantine_threshold` of `0.0` disables quarantine enforcement. - /// Otherwise, peers below that score are avoided for automatic lookups, - /// and K-closest peers below it are evicted and quarantined when the - /// routing table can keep at least K peers. + /// Otherwise, peers below that score are avoided for automatic lookups. /// Quarantined peers can only re-enter through normal admission after /// their decayed trust reaches `quarantine_readmit_threshold`; new peers /// must also meet that threshold before entering the routing table. @@ -1664,7 +1670,7 @@ impl DhtCoreEngine { } #[cfg(test)] - fn remember_quarantined_peer( + pub(crate) fn remember_quarantined_peer( &mut self, peer_id: PeerId, trust_score: &impl Fn(&PeerId) -> f64, @@ -1790,12 +1796,20 @@ impl DhtCoreEngine { /// Evict a quarantined peer if it currently occupies the K-closest set and /// removal will not shrink the routing table below K peers. + /// + /// Temporarily disabled until trust scoring is considered stable. Low-trust + /// peers continue to leave through lazy swap-out when better candidates are + /// admitted. #[cfg(test)] pub(crate) async fn enforce_close_group_quarantine( &mut self, peer_id: &PeerId, trust_score: f64, ) -> Vec { + if !close_group_immediate_eviction_enabled() { + return Vec::new(); + } + if !self.quarantine_enabled() || !trust_score.is_finite() || trust_score >= self.quarantine_threshold @@ -1846,15 +1860,17 @@ impl DhtCoreEngine { /// Enforce trust gates over the current K-closest set. /// - /// Peers below the quarantine threshold are evicted from the close group - /// while the routing table has more than K entries. Peers already in the - /// routing table may move into the close group as long as they are not - /// below the quarantine threshold, and low-trust close-group peers are - /// retained rather than shrinking the routing table below K entries. + /// Temporarily leaves close-group peers in place even when they are below + /// the quarantine threshold. Lazy swap-out remains responsible for + /// replacing low-trust peers when better candidates arrive. pub(crate) async fn enforce_close_group_trust_gate( &mut self, trust_score: &impl Fn(&PeerId) -> f64, ) -> Vec { + if !close_group_immediate_eviction_enabled() { + return Vec::new(); + } + if !self.quarantine_enabled() { return Vec::new(); } @@ -5366,11 +5382,10 @@ mod tests { assert!(dht.has_node(&low_peer).await); } - /// A K-closest peer below the quarantine threshold is evicted when the - /// routing table has surplus above K and cannot be readmitted until its - /// trust has recovered to the readmit threshold. + /// A K-closest peer below the quarantine threshold is retained while + /// immediate close-group eviction is disabled. #[tokio::test] - async fn test_close_group_peer_below_quarantine_is_evicted_until_readmit() { + async fn test_close_group_peer_below_quarantine_is_not_immediately_evicted() { let mut dht = DhtCoreEngine::new( PeerId::from_bytes([0u8; 32]), SMALL_TEST_K, @@ -5407,26 +5422,29 @@ mod tests { let events = dht.enforce_close_group_quarantine(&peer, 0.19).await; assert!( - events - .iter() - .any(|event| matches!(event, RoutingTableEvent::PeerRemoved(id) if *id == peer)), - "close-group peer below quarantine threshold should emit PeerRemoved" + events.is_empty(), + "close-group peer below quarantine threshold should not be immediately evicted" ); assert!( - !dht.has_node(&peer).await, - "quarantined close-group peer should be removed from RT" + dht.has_node(&peer).await, + "close-group peer should remain in RT while immediate eviction is disabled" + ); + assert_eq!(dht.routing_table_size().await, SMALL_TEST_K + 1); + assert!( + dht.should_avoid_for_lookup(&peer, 0.19), + "retained low-trust peer should still be avoided by lookup policy" ); - assert_eq!(dht.routing_table_size().await, SMALL_TEST_K); - let early_readmit = dht + dht.remove_node_by_id(&peer).await; + let below_admission = dht .add_node( make_node_with_addr(peer_id_bytes, "/ip4/10.10.0.1/udp/9000/quic"), &|id| if *id == peer { 0.30 } else { 0.5 }, ) .await; assert!( - early_readmit.is_err(), - "quarantined peer should not readmit below 0.45" + below_admission.is_err(), + "removed peer should not readmit below 0.45" ); let recovered = dht @@ -5437,7 +5455,7 @@ mod tests { .await; assert!( recovered.is_ok(), - "quarantined peer should readmit once trust reaches 0.45" + "removed peer should readmit once trust reaches 0.45" ); assert!(dht.has_node(&peer).await); } @@ -5485,10 +5503,10 @@ mod tests { ); } - /// Once a new routing-table peer creates surplus above K, a previously - /// deferred close-group quarantine can remove the low-trust peer. + /// Once a new routing-table peer creates surplus above K, a low-trust + /// close-group peer is still retained while immediate eviction is disabled. #[tokio::test] - async fn test_new_peer_surplus_allows_deferred_close_group_quarantine() { + async fn test_new_peer_surplus_does_not_trigger_close_group_quarantine() { let mut dht = DhtCoreEngine::new( PeerId::from_bytes([0u8; 32]), SMALL_TEST_K, @@ -5532,13 +5550,11 @@ mod tests { .enforce_close_group_trust_gate(&|id| if *id == low_peer { 0.10 } else { 0.5 }) .await; assert!( - events.iter().any( - |event| matches!(event, RoutingTableEvent::PeerRemoved(id) if *id == low_peer) - ), - "surplus peer should allow deferred close-group quarantine" + events.is_empty(), + "surplus peer should not trigger close-group quarantine while immediate eviction is disabled" ); - assert_eq!(dht.routing_table_size().await, SMALL_TEST_K); - assert!(!dht.has_node(&low_peer).await); + assert_eq!(dht.routing_table_size().await, SMALL_TEST_K + 1); + assert!(dht.has_node(&low_peer).await); } #[test] @@ -5748,8 +5764,8 @@ mod tests { /// Removing one close peer can promote a non-close peer into the close /// group. Existing routing-table peers above the quarantine threshold stay /// even when below the new-peer admission threshold; peers below the - /// quarantine threshold are removed once the routing table has surplus - /// above K. + /// quarantine threshold are also retained while immediate eviction is + /// disabled. #[tokio::test] async fn test_close_group_gate_allows_existing_promotions_above_quarantine() { let mut dht = DhtCoreEngine::new( @@ -5843,12 +5859,10 @@ mod tests { }) .await; assert!( - quarantine_events.iter().any( - |event| matches!(event, RoutingTableEvent::PeerRemoved(id) if *id == promoted_peer) - ), - "existing promoted peer below quarantine threshold should be removed once there is surplus" + quarantine_events.is_empty(), + "existing promoted peer below quarantine threshold should stay even when there is surplus" ); - assert!(!dht.has_node(&promoted_peer).await); + assert!(dht.has_node(&promoted_peer).await); } /// A non-close peer below the quarantine threshold is avoided by automatic diff --git a/src/dht_network_manager.rs b/src/dht_network_manager.rs index 13ece325..178094f4 100644 --- a/src/dht_network_manager.rs +++ b/src/dht_network_manager.rs @@ -634,9 +634,9 @@ pub struct DhtNetworkConfig { /// routing table when a better candidate is available. /// Default: [`AdaptiveDhtConfig::default`]. pub swap_threshold: f64, - /// Trust score below which automatic lookup/dial paths avoid a peer, and - /// K-closest peers are evicted into temporary quarantine when the routing - /// table can keep at least K peers. + /// Trust score below which automatic lookup/dial paths avoid a peer. + /// Immediate close-group eviction is temporarily disabled until trust + /// scoring is stable. /// Default: [`AdaptiveDhtConfig::default`]. pub quarantine_threshold: f64, /// Trust score required before a new peer can enter the routing table, @@ -6352,15 +6352,7 @@ mod tests { .await .unwrap(); } - let events = dht - .enforce_close_group_quarantine(&quarantined_peer, 0.10) - .await; - assert!( - events - .iter() - .any(|event| matches!(event, RoutingTableEvent::PeerRemoved(id) if *id == quarantined_peer)), - "test setup should quarantine the close-group peer" - ); + dht.remember_quarantined_peer(quarantined_peer, &|_| 0.30); let trust_score = |peer_id: &PeerId| { if *peer_id == quarantined_peer { diff --git a/src/network.rs b/src/network.rs index ff53a2e8..3e4c9f5c 100644 --- a/src/network.rs +++ b/src/network.rs @@ -302,8 +302,8 @@ pub struct NodeConfig { /// Adaptive DHT configuration for trust-based routing enforcement. /// - /// Controls lazy swap-out, close-group quarantine, automatic lookup - /// avoidance, and new-peer/readmission trust thresholds. Use + /// Controls lazy swap-out, automatic lookup avoidance, and + /// new-peer/readmission trust thresholds. Use /// [`NodeConfigBuilder::trust_enforcement`] for a simple on/off toggle. /// /// Default: enabled with the default [`AdaptiveDhtConfig`] thresholds. From dd822a0d0e00f61e6fe085ab905133affec6a213 Mon Sep 17 00:00:00 2001 From: Mick van Dijke <12992260+mickvandijke@users.noreply.github.com> Date: Thu, 9 Jul 2026 15:42:52 +0200 Subject: [PATCH 11/18] fix(dht): weight dial failures for trust avoidance --- src/dht_network_manager.rs | 53 +++++++++++++++++++++++++++++++++++--- 1 file changed, 50 insertions(+), 3 deletions(-) diff --git a/src/dht_network_manager.rs b/src/dht_network_manager.rs index 178094f4..e42ce724 100644 --- a/src/dht_network_manager.rs +++ b/src/dht_network_manager.rs @@ -259,6 +259,12 @@ const DIAL_FAILURE_PUBLISH_EXEMPTION_TTL: Duration = Duration::from_secs(2 * 60) /// Trust-score log reason for a failed pre-request dial. const TRUST_REASON_DHT_DIAL_FAILED: &str = "dht_dial_failed"; +/// Trust weight for failed pre-request dials. +/// +/// At the current trust EMA and decay rate, four evenly-spaced dial failures +/// over six hours take a neutral peer below the 0.20 lookup-avoid threshold. +const DHT_DIAL_FAILURE_TRUST_WEIGHT: f64 = 2.25; + /// Trust-score log reason for a failed post-dial identity exchange. const TRUST_REASON_DHT_IDENTITY_EXCHANGE_FAILED: &str = "dht_identity_exchange_failed"; @@ -3669,10 +3675,21 @@ impl DhtNetworkManager { } async fn record_peer_failure(&self, peer_id: &PeerId, reason: &'static str) { + self.record_peer_failure_weighted(peer_id, reason, 1.0) + .await; + } + + async fn record_peer_failure_weighted( + &self, + peer_id: &PeerId, + reason: &'static str, + weight: f64, + ) { if let Some(ref engine) = self.trust_engine { - engine.update_node_stats_with_reason( + engine.update_node_stats_weighted_with_reason( peer_id, NodeStatisticsUpdate::FailedResponse, + weight, reason, ); } @@ -3885,8 +3902,12 @@ impl DhtNetworkManager { peer_hex, candidates.len() ); - self.record_peer_failure(peer_id, TRUST_REASON_DHT_DIAL_FAILED) - .await; + self.record_peer_failure_weighted( + peer_id, + TRUST_REASON_DHT_DIAL_FAILED, + DHT_DIAL_FAILURE_TRUST_WEIGHT, + ) + .await; return PendingDialOutcome::DialFailed { candidates_count: candidates.len(), }; @@ -6200,6 +6221,32 @@ mod tests { ); } + #[tokio::test] + async fn dial_failure_weight_avoids_neutral_peer_after_four_failures_in_six_hours() { + let engine = TrustEngine::new(); + let peer = pid(42); + let threshold = AdaptiveDhtConfig::default().quarantine_threshold; + let spacing = Duration::from_secs(2 * 60 * 60); + + for failure_index in 0..4 { + if failure_index > 0 { + engine.simulate_elapsed(&peer, spacing).await; + } + engine.update_node_stats_weighted_with_reason( + &peer, + NodeStatisticsUpdate::FailedResponse, + DHT_DIAL_FAILURE_TRUST_WEIGHT, + TRUST_REASON_DHT_DIAL_FAILED, + ); + } + + let score = engine.score(&peer); + assert!( + score < threshold, + "four weighted dial failures over six hours should avoid peer: score={score}, threshold={threshold}" + ); + } + fn bucket_refresh_candidate(index: usize, refresh_debt_secs: u64) -> BucketRefreshCandidate { let refresh_debt = Duration::from_secs(refresh_debt_secs); BucketRefreshCandidate { From aecb0142e3915058b099945d1958d3ff967e1e35 Mon Sep 17 00:00:00 2001 From: Mick van Dijke <12992260+mickvandijke@users.noreply.github.com> Date: Thu, 16 Jul 2026 16:44:22 +0200 Subject: [PATCH 12/18] fix(network): make request transport trust-neutral --- docs/trust-signals-api.md | 14 +++++++++----- src/network.rs | 27 ++++++++++++--------------- tests/trust_flow.rs | 32 ++++++++++++++++++++++++++++++++ 3 files changed, 53 insertions(+), 20 deletions(-) diff --git a/docs/trust-signals-api.md b/docs/trust-signals-api.md index 765116cc..e702a11f 100644 --- a/docs/trust-signals-api.md +++ b/docs/trust-signals-api.md @@ -37,9 +37,10 @@ if trust < 0.3 { ### `report_trust_event(peer_id, event)` -Report a trust event for a peer. Core penalties (connection failures) are -recorded automatically by the DHT layer. Consumers use this API to report -application-level outcomes (rewards and additional penalties). +Report a trust event for a peer. DHT-specific penalties are recorded by the +DHT layer where the failure phase and operation are known. Generic +`P2PNode::send_request` transport errors are trust-neutral. Consumers use this +API to report application-level outcomes (rewards and justified penalties). ```rust pub async fn report_trust_event(&self, peer_id: &PeerId, event: TrustEvent) @@ -69,12 +70,15 @@ are not rewarded. | Event | Severity | Description | Where it fires | |-------|----------|-------------|----------------| -| `ConnectionFailed` | 1x penalty (core) | Could not establish connection | `send_request()` error, `send_dht_request()` RPC failure | -| `ConnectionTimeout` | 1x penalty (core) | Connection attempt timed out | `send_request()` timeout, `send_dht_request()` RPC timeout | +| `ConnectionFailed` | 1x penalty | Could not establish connection | Explicit `report_trust_event` callers | +| `ConnectionTimeout` | 1x penalty | Connection attempt timed out | Explicit `report_trust_event` callers | | `ApplicationSuccess(w)` | Weighted reward (consumer) | Peer completed an application-level task | Consumer code | | `ApplicationFailure(w)` | Weighted penalty (consumer) | Peer failed an application-level task | Consumer code | Note: Peer disconnects are normal connection lifecycle — they do not affect trust. +Generic `P2PNode::send_request` errors are also trust-neutral because that layer +cannot distinguish remote misbehaviour from network failure, application delay, +or local overload. Application-aware callers report trust events explicitly. ## Trust Thresholds diff --git a/src/network.rs b/src/network.rs index 3e4c9f5c..2e2dded7 100644 --- a/src/network.rs +++ b/src/network.rs @@ -1100,20 +1100,27 @@ impl P2PNode { } // ========================================================================= - // Request/Response API — Automatic Trust Feedback + // Request/Response API — Trust-Neutral Transport // ========================================================================= - /// Send a request to a peer and wait for a response with automatic trust penalty reporting. + /// Send a request to a peer and wait for a response. /// /// Unlike fire-and-forget `send_message()`, this method: /// 1. Wraps the payload in a `RequestResponseEnvelope` with a unique message ID /// 2. Sends it on the `/rr/` protocol prefix /// 3. Waits for a matching response (or timeout) - /// 4. Automatically reports failure to the trust engine (success is the expected baseline) /// /// The remote peer's handler should call `send_response()` with the /// incoming message ID to route the response back. /// + /// # Trust neutrality + /// + /// Request transport errors (timeouts, connection failures) are + /// trust-neutral: this method never reports trust events on its own. + /// Application-aware callers that can judge whether a failure reflects + /// peer misbehaviour must explicitly call + /// [`Self::report_trust_event`] when a penalty (or reward) is justified. + /// /// # Arguments /// /// * `peer_id` - Target peer @@ -1138,18 +1145,8 @@ impl P2PNode { data: Vec, timeout: Duration, ) -> Result { - let result = self - .send_request_reconnecting(peer_id, protocol, data, timeout) - .await; - if let Err(ref e) = result { - let event = if matches!(e, P2PError::Timeout(_)) { - TrustEvent::ConnectionTimeout - } else { - TrustEvent::ConnectionFailed - }; - self.report_trust_event(peer_id, event).await; - } - result + self.send_request_reconnecting(peer_id, protocol, data, timeout) + .await } /// Request/response send with reconnect-on-demand. diff --git a/tests/trust_flow.rs b/tests/trust_flow.rs index 7fa19542..9e13618d 100644 --- a/tests/trust_flow.rs +++ b/tests/trust_flow.rs @@ -17,11 +17,16 @@ #![allow(clippy::unwrap_used, clippy::expect_used)] +use std::time::Duration; + use saorsa_core::{AdaptiveDhtConfig, NodeConfig, P2PNode, PeerId, TrustEvent}; /// Default neutral trust score for unknown peers. const NEUTRAL_TRUST: f64 = 0.5; +/// Response wait bound for request/response tests against unreachable peers. +const REQUEST_TIMEOUT: Duration = Duration::from_secs(1); + /// Default trust threshold below which peers become eligible for swap-out. const SWAP_THRESHOLD: f64 = 0.35; @@ -88,6 +93,33 @@ async fn failures_lower_trust_below_neutral() { ); } +/// A `send_request` that fails at the transport layer leaves the target +/// peer's trust unchanged — request transport errors are trust-neutral. +#[tokio::test] +async fn failed_send_request_leaves_trust_unchanged() { + let node = P2PNode::new(test_node_config()).await.unwrap(); + node.start().await.unwrap(); + let unreachable_peer = PeerId::random(); + + let result = node + .send_request( + &unreachable_peer, + "test/echo", + vec![1, 2, 3], + REQUEST_TIMEOUT, + ) + .await; + assert!(result.is_err(), "request to a nonexistent peer must fail"); + + let score = node.peer_trust(&unreachable_peer); + assert!( + (score - NEUTRAL_TRUST).abs() < f64::EPSILON, + "Request transport errors must be trust-neutral; expected {NEUTRAL_TRUST}, got {score}" + ); + + node.stop().await.unwrap(); +} + // --------------------------------------------------------------------------- // Trust event variants // --------------------------------------------------------------------------- From 210fe770c67147b51c6cf2d532e6f28b5432dc10 Mon Sep 17 00:00:00 2001 From: Mick van Dijke <12992260+mickvandijke@users.noreply.github.com> Date: Thu, 16 Jul 2026 17:03:51 +0200 Subject: [PATCH 13/18] docs(adr): record trust quarantine architecture --- ...t-quarantine-and-trust-neutral-requests.md | 192 ++++++++++++++++++ docs/adr/README.md | 1 + 2 files changed, 193 insertions(+) create mode 100644 docs/adr/ADR-016-trust-quarantine-and-trust-neutral-requests.md diff --git a/docs/adr/ADR-016-trust-quarantine-and-trust-neutral-requests.md b/docs/adr/ADR-016-trust-quarantine-and-trust-neutral-requests.md new file mode 100644 index 00000000..7c63340d --- /dev/null +++ b/docs/adr/ADR-016-trust-quarantine-and-trust-neutral-requests.md @@ -0,0 +1,192 @@ +# ADR-016: Trust Quarantine Thresholds and Trust-Neutral Request Transport + +## Status + +Proposed (2026-07-16) — documents the decisions introduced by open PR [#119](https://github.com/WithAutonomi/saorsa-core/pull/119); to be moved to Accepted when the PR merges. + +## Context + +Before PR #119, trust enforcement in the DHT was blunt and partially misattributed: + +- The routing table had a single trust lever — a swap threshold — with no distinction between "this peer is a weak candidate" and "this peer should be actively avoided". A single threshold cannot express three different questions: *when should a peer become replaceable?*, *when should automatic machinery stop selecting a peer?*, and *what does a new or previously-avoided peer have to prove before entering the routing table?* +- Nothing gated new routing-table admissions on trust, so a peer that had been driven to a low score could be forgotten and immediately re-admitted at the same low score via rediscovery. +- `P2PNode::send_request` automatically reported `TrustEvent::ConnectionTimeout` / `TrustEvent::ConnectionFailed` on every transport error ("Request/Response API — Automatic Trust Feedback"). Generic transport failures are **ambiguous**: at that layer a timeout can mean remote misbehaviour, but it can equally mean network congestion, a slow-but-honest application handler, or local overload. The generic layer cannot tell these apart. Worse, an application-aware downstream caller reporting `ApplicationFailure` for the same operation would then **double-penalize** the failed exchange — once automatically at the transport layer and once explicitly at the application layer. +- Trust scores are still stabilizing in real deployments. An immediate-eviction policy tied to a young scoring model risks ejecting honest peers on transient noise and shrinking the routing table below the Kademlia K target. + +The trust model this builds on (see `src/adaptive/trust.rs` and [ADR-007](./ADR-007-adaptive-networking.md)): unknown peers start at neutral trust `0.5`; core records **penalties only** (successful responses are the expected baseline); scores are an EMA (`EMA_WEIGHT = 0.124`) with lazy exponential time decay toward neutral (`DECAY_LAMBDA = 1.394e-5`/s, so a worst-case `0.0` score decays back above `0.35` in ~1 day and above `0.45` in ~46 hours). Rewards are the consumer's responsibility via `TrustEvent::ApplicationSuccess`. + +## Decision + +PR #119 introduces a three-threshold trust quarantine policy for the DHT routing layer, enabled by default, together with a trust-neutral generic request transport. Trust enforcement becomes **routing-table and automatic-selection policy**, never a transport block. + +### 1. Ownership and configuration propagation + +`AdaptiveDHT` remains the **sole creator and trust-policy owner** of the `TrustEngine` (`src/adaptive/dht.rs:198`; module doc at `src/adaptive/dht.rs:14-21`) — all trust signals flow through it. Configuration flows one way: + +```text +NodeConfig.adaptive_dht_config : AdaptiveDhtConfig (src/network.rs:263, #[serde(default)]) + │ validated in AdaptiveDHT::new (src/adaptive/dht.rs:222) + ▼ +DhtNetworkConfig { swap_threshold, quarantine_threshold, quarantine_readmit_threshold } + │ (src/dht_network_manager.rs:626-635) + ▼ +DhtCoreEngine — swap_threshold via constructor; + set_trust_quarantine_thresholds(...) for the quarantine pair + (src/dht_network_manager.rs:1725-1742, src/dht/core_engine.rs:1613-1637) +``` + +`DhtNetworkManager` holds only an injected `Option>`; it never creates one. The policy is **on by default**: `NodeConfig` defaults to `AdaptiveDhtConfig::default()`. + +### 2. Three thresholds, not one + +Defined in `src/adaptive/dht.rs:33-40` (mirrored as documentation constants in `src/dht/core_engine.rs:205-222`): + +| Threshold | Default | Meaning | +|-----------|---------|---------| +| `swap_threshold` | **0.35** | Peer becomes *eligible for lazy swap-out* — replaced only when a better routing-table candidate arrives. Never causes eviction on its own. | +| `quarantine_threshold` | **0.20** | *Automatic avoidance*: lookup/dial machinery stops selecting the peer. The peer is not removed and explicit sends still work. | +| `quarantine_readmit_threshold` | **0.45** | *Admission gate*: a peer **unknown to the routing table** (brand new, or previously quarantined and forgotten) must score at or above this to be admitted/readmitted. | + +**Validation** (`AdaptiveDhtConfig::validate`, `src/adaptive/dht.rs:84-138`): all three must be finite and in `[0.0, 0.5)`; when quarantine is active, `quarantine_readmit_threshold >= quarantine_threshold`; when swap and quarantine are both non-zero, `swap_threshold > quarantine_threshold` (swap is a milder condition than avoidance, quarantine is strictly more severe). The quarantine pair is re-checked at the engine boundary in `set_trust_quarantine_thresholds`; `swap_threshold` is validated only by `AdaptiveDhtConfig::validate`. Values at or above neutral `0.5` are rejected because decay approaches neutral asymptotically: negatively observed peers would remain swap/quarantine-eligible indefinitely, while readmission from below at a neutral cutoff would be unreachable in finite time. Invalid config fails node construction (`AdaptiveDHT::new` returns `Err`). + +**Disabling**: `quarantine_threshold == 0.0` disables quarantine enforcement (`quarantine_enabled()`, `src/dht/core_engine.rs:1640-1642`). `NodeConfigBuilder::trust_enforcement(false)` (`src/network.rs:518-530`) zeroes all three thresholds — scores are still tracked, but nothing is enforced. + +### 3. Only unknown admissions are gated; existing peers in [0.20, 0.45) are preserved + +`check_new_peer_admission` (`src/dht/core_engine.rs:1644-1670`) is called **only when the peer is not already in the routing table** (`add_node`, `src/dht/core_engine.rs:2443-2446`; `re_evaluate_admission`, `src/dht/core_engine.rs:3069-3072`). Non-finite trust scores are rejected defensively. Consequences: + +- An existing routing-table peer whose score sits in `[0.20, 0.45)` **stays in the table** and may move into the close group (close-group membership is pure XOR distance over table contents). It remains eligible for lazy swap-out below 0.35 and is skipped by automatic selection below 0.20 — but it is never ejected merely for the band it occupies. +- Admission at or above 0.45 clears any quarantine marker (`forget_quarantined_peer`, `src/dht/core_engine.rs:1668`) — this is the readmission point. +- `should_avoid_automatic_candidate` (`src/dht/core_engine.rs:1779-1795`) encodes the asymmetry directly: a peer scoring in `[quarantine, readmit)` is avoided as an automatic candidate *only if it is not already in the routing table*. + +### 4. Stale-revalidation concurrency invariant + +Stale-peer revalidation (evict-then-readmit under contention) must not let the new-peer admission gate reject a peer that concurrently became known. The invariant: **a peer already present in the routing table is treated as an update, never as a new admission**. `re_evaluate_admission` skips `check_new_peer_admission` when `has_node(&candidate_id)` is true (`src/dht/core_engine.rs:3069-3072`), so a peer with trust in `[0.20, 0.45)` that entered the table during the revalidation window is updated in place with no second admission event. If re-admission fails *after* stale peers were already evicted, `revalidate_and_retry_admission` still broadcasts the committed removal events rather than failing the flow (`src/dht_network_manager.rs:5197+`). Revalidation itself is bounded (`MAX_CONCURRENT_REVALIDATIONS = 8`, `MAX_CONCURRENT_REVALIDATION_PINGS = 4`, per-bucket guards) and re-evaluation runs with `allow_stale_revalidation: false` to prevent recursion. + +### 5. Automatic filtering everywhere; explicit sends and wire format untouched + +Two engine predicates drive all filtering: `should_avoid_for_lookup` (`src/dht/core_engine.rs:1765-1775` — non-finite, below 0.20, or marked quarantined and below 0.45) and `should_avoid_automatic_candidate` (adds the unknown-peer readmit gate). They are applied on every **automatic** path in `src/dht_network_manager.rs`: + +1. Local lookup results / FIND_NODE response serving — `find_closest_nodes_local` (`:2579-2606`) +2. Iterative lookup local seeding — `find_closest_nodes_network` (`:2740`) +3. Iterative lookup candidate/batch selection (`:2786`) +4. Gossiped nodes from FIND_NODE responses (`:2897`) +5. Bootstrap — both the bootstrap peers themselves and gossiped nodes (`:2192-2195`, `:2230-2239`) +6. Bucket refresh (`:1993-2002`) +7. Self-lookup (`:2059-2065`) + +**Explicit sends stay unblocked**: `send_dht_request` / `send_dht_request_with_response_context` contain no quarantine checks, and `P2PNode::send_request` / `send_message` never consult trust. Quarantine is local selection policy, not a firewall. + +**Wire format is unchanged**: trust is never serialized into DHT lookup results — `lookup_results_from_routing_nodes` keeps the legacy `DHTNode` reliability wire value stable (`src/dht_network_manager.rs:2620-2623`, asserted by test at `:6011`). Older nodes interoperate without change. + +### 6. Immediate close-group trust eviction: implemented but disabled + +Immediate eviction of below-0.20 close-group peers is **currently switched off** while trust scoring stabilizes: `close_group_immediate_eviction_enabled()` is hard-wired to `false` (`src/dht/core_engine.rs:216-218`). `enforce_close_group_trust_gate` (`src/dht/core_engine.rs:1866-1924`) is still wired through `DhtNetworkManager::enforce_trust_quarantine` and `broadcast_routing_events_with_quarantine`, but returns no events; `enforce_close_group_quarantine` is retained under `#[cfg(test)]`. The gated-off bodies preserve the safety property for re-enablement: eviction only proceeds `while routing.node_count() > k_value`, so the routing table **never shrinks below K** for trust reasons. Until the gate flips, lazy swap-out (0.35) is the sole replacement mechanism for low-trust peers, and peers below 0.20 remain in the table but are avoided by the automatic paths above. + +### 7. Bounded quarantine markers + +Quarantine markers (`quarantined_peers: HashSet` plus FIFO `quarantined_peer_order`) are bounded at `MAX_QUARANTINED_PEERS = 8192` (`src/dht/core_engine.rs:229`). The key insight making the bound safe (`quarantine_marker_required_for_score`, `src/dht/core_engine.rs:1690-1696`): **a marker is only semantically required while a peer's score is in `[quarantine_threshold, readmit_threshold)`** — below 0.20 the score itself keeps the peer avoided; at/above 0.45 the peer is readmittable and the marker is cleared. When the set is full, `prune_redundant_quarantined_peers` drops (oldest-first) any marker whose current score no longer requires one; if the set is still full, the new marker is simply not inserted — the score-based avoidance clause covers the peer regardless. Note: with immediate eviction disabled (§6), no production path currently inserts markers; the machinery is preserved intact for re-enablement. + +### 8. Decay and recovery via rediscovery + +There is no active probing of avoided peers. Recovery is `time decay toward neutral` **plus** `natural rediscovery`: a quarantined/avoided peer's score decays back above 0.45 (~46 h from worst case), after which it can re-enter through the normal admission path when it is rediscovered via a FIND_NODE response or an authenticated inbound connection (`docs/ROUTING_TABLE_DESIGN.md:272-278`). Decay-plus-rediscovery *is* the temporary-ban mechanism. + +### 9. Application trust weights capped + +`TrustEvent::ApplicationSuccess(w)` / `ApplicationFailure(w)` weights are clamped to `MAX_CONSUMER_WEIGHT = 5.0` in `AdaptiveDHT::report_trust_event` (`src/adaptive/dht.rs:44`, `:262`) so no single consumer-reported event can dominate the EMA; zero or negative weights are ignored. The clamp deliberately lives in `AdaptiveDHT`, not `TrustEngine` — the engine applies whatever weight it is given (verified by `test_trust_engine_does_not_clamp_weights`), keeping the policy at the ownership boundary. + +### 10. DHT-context scoring: weighted dials, exactly-once requests + +The DHT layer keeps automatic penalties **because it has context** — it knows the operation, the failure phase, and the peer's advertised addresses: + +- **Pre-request dial failure** — `DHT_DIAL_FAILURE_TRUST_WEIGHT = 2.25` (`src/dht_network_manager.rs:250`), reason `dht_dial_failed`, reported in `run_owned_dial` when every candidate address fails. Calibration: four evenly-spaced dial failures over six hours take a neutral peer below the 0.20 avoidance threshold. +- **Failed DHT RPC** (send error or response timeout) — weight 1.0, reason `dht_request_failed`, recorded at the RPC level in `send_dht_request_with_response_context` (`src/dht_network_manager.rs:4076-4079`). +- **Failed identity exchange after dial** — weight 1.0, reason `dht_identity_exchange_failed`. + +**Exactly-once invariant**: each failed DHT request is scored once and only once. (a) Concurrent dials collapse onto a single owner (`run_owned_dial`), so the dial penalty fires once per dial, not once per waiting caller; (b) a dial failure returns early from the request path *before* the RPC-level recording, so it is never also counted as a request failure; (c) RPC failure is recorded at one chokepoint that all callers (including revalidation pings) rely on rather than re-reporting. + +### 11. Generic request transport is trust-neutral (commit `5d5f69f`) + +`P2PNode::send_request` no longer reports any trust event — it is a pure passthrough to `send_request_reconnecting` (`src/network.rs:1044-1053`; section renamed to "Request/Response API — Trust-Neutral Transport"). Rationale: + +- **Ambiguity**: at the generic transport layer, a timeout or connection error cannot be attributed — it may be remote misbehaviour, but equally network congestion, a slow application handler on an honest peer, or local overload. Penalizing on it punishes honest peers for conditions they don't control (`docs/trust-signals-api.md:79-81`). +- **Double-penalty**: application-aware layers (downstream node applications and DHT RPC) already report justified outcomes for the same exchange. Automatic transport penalties would stack a second, unjustified penalty on top of the informed one. + +The division of labour: **layers that can judge, report; layers that can't, stay neutral.** The DHT layer keeps its contextual automatic penalties (§10); applications report `ApplicationSuccess`/`ApplicationFailure` (and, where justified, `ConnectionFailed`/`ConnectionTimeout`) explicitly via `P2PNode::report_trust_event`. Regression test: `failed_send_request_leaves_trust_unchanged` (`tests/trust_flow.rs:100`) asserts a failed `send_request` leaves the peer at neutral 0.5. + +## Invariants + +1. **Threshold ordering**: when quarantine is active, `0 < quarantine_threshold <= quarantine_readmit_threshold < 0.5`; when swap and quarantine are both active, `quarantine_threshold < swap_threshold < 0.5`. Defaults: `0.20 < 0.35` and `0.20 <= 0.45`. +2. **K-sized routing table**: trust enforcement never shrinks the routing table below K (eviction, even when re-enabled, only runs while `node_count > K`). +3. **Known-peer preservation**: a peer already in the routing table is never subjected to the new-peer admission gate — including during stale-revalidation races (§4). +4. **No transport block**: quarantine affects only routing-table membership and automatic selection; explicit sends always go through. +5. **Wire stability**: trust state never leaks into serialized DHT messages; filtering is strictly local policy. +6. **Exactly-once DHT scoring**: one failed DHT request produces exactly one trust penalty (dial 2.25 *or* RPC 1.0 *or* identity-exchange 1.0 — never stacked for the same attempt). +7. **Marker sufficiency**: quarantine markers are required only for scores in `[quarantine, readmit)`; outside that band the score alone determines behaviour, which is what makes the 8192 bound safe. +8. **Penalty-only core**: saorsa-core never auto-rewards; positive signals come exclusively from consumers (capped at weight 5.0). +9. **Trust-neutral generic transport**: `send_request`/`send_message` never report trust events; only application-aware layers do. + +## Consequences + +### Positive + +- **Sybil/misbehaviour pressure without fragility**: low-trust peers stop being handed out by lookups, used for bootstrap/refresh, or dialed by maintenance — while the routing table stays K-sized and honest-but-unlucky peers aren't permanently exiled. +- **Readmission gate closes the forget-and-return loophole**: a peer driven below quarantine cannot be evicted/forgotten and immediately re-admitted at a low score; it must decay back to 0.45 first. +- **No misattributed penalties from generic transport**: honest peers are no longer punished for congestion, slow handlers, or the local node's own overload — and application penalties are no longer doubled by transport-layer penalties for the same exchange. +- **Calibrated avoidance**: the 2.25 dial weight gives persistent unreachability a concrete, documented time-to-avoidance (four spaced failures over six hours from neutral), rather than an emergent accident of unit weights. +- **Fully backward compatible on the wire**: mixed-version networks work; old nodes see identical messages. +- **Bounded memory**: quarantine bookkeeping cannot grow past 8192 entries, and dropping markers under pressure degrades gracefully to score-only avoidance. +- **Operators can turn it off**: `trust_enforcement(false)` gives observe-only mode (scores tracked, nothing enforced) for diagnosis or staged rollout. + +### Negative + +- **Slower reaction to genuinely malicious close-group peers**: with immediate eviction disabled, a below-0.20 peer stays in the close group until lazy swap-out replaces it; it is avoided by automatic paths but still occupies a slot. +- **Applications now own attribution**: any consumer that relied on `send_request`'s automatic penalties gets no trust signal for its request failures unless it explicitly reports a justified outcome. Silent trust erosion of misbehaving peers via generic requests no longer happens. +- **Recovery latency is fixed by decay**: a wrongly-penalized peer needs up to ~46 h (worst case) to become admissible again; there is no active-probe fast path. +- **More configuration surface**: three interdependent thresholds with ordering rules; invalid combinations fail node construction (loudly, by design). + +### Neutral + +- The immediate-eviction machinery (gate function, trust-gate enforcement, marker insertion) ships dark: wired, tested, and preserved, but returning no events until `close_group_immediate_eviction_enabled()` flips. Re-enabling it is a one-line change plus recalibration review — a likely follow-up ADR/amendment once scoring is deemed stable. +- With eviction dark, quarantine *markers* have no production writer; the active enforcement today is score-based avoidance (0.20), the unknown-admission gate (0.45), and lazy swap (0.35). +- Trust scores keep being computed identically in observe-only mode, so enabling enforcement later needs no re-learning period. + +## Compatibility and Breaking Changes + +- **API (breaking)**: `AdaptiveDhtConfig` gains `quarantine_threshold` and `quarantine_readmit_threshold` (with `#[serde(default)]`, so serialized configs deserialize fine); struct-literal construction without `..Default::default()` breaks. `DhtNetworkConfig` gains the same fields. +- **Behavioural (breaking)**: quarantine defaults ON. New routing-table peers must meet 0.45 when enforcement is enabled; peers below 0.20 stop appearing in lookup results and automatic maintenance. `send_request` no longer auto-penalizes failures — consumers relying on that must add explicit `report_trust_event` calls. +- **Wire (non-breaking)**: no message format changes; `DHTNode` reliability keeps its legacy value. + +## Operational Implications + +- **Defaults are live on upgrade** — no config change needed to get the policy; use `trust_enforcement(false)` to opt out. +- Watch trust-score distribution after deployment: since scores are stabilizing, thresholds (especially 0.45 admission on small networks, where rejecting a scarce peer costs more) may need tuning before immediate eviction is re-enabled. +- Reason strings (`dht_dial_failed`, `dht_request_failed`, `dht_identity_exchange_failed`, `application_failure`, …) are logged with score deltas — use them to audit which layer is driving a peer's score. +- Downstream consumers should classify data-availability outcomes and report only justified `ApplicationSuccess`/`ApplicationFailure` events; generic request transport itself contributes no trust signal. +- Small/bootstrap networks: bootstrap peers themselves are trust-filtered — a bootstrap peer driven below 0.20 will be skipped, so keep multiple bootstrap endpoints configured. + +## Alternatives Considered + +**Single trust threshold for everything.** One cutoff for swap, avoidance, and admission. Rejected: the three questions have different costs. Swap-eligibility is cheap and reversible; avoidance affects lookup quality; admission controls table churn. A single value is either too aggressive for avoidance or too lax for admission, and it recreates the forget-and-readmit loophole (evict at X, readmit at X). + +**Hard transport-level block of quarantined peers.** Refuse all sends to below-threshold peers. Rejected: explicit sends are how applications retry, probe, and recover — blocking them turns a local routing preference into a network partition, breaks consumer semantics, and prevents the very interactions whose successes a consumer could report to rehabilitate a peer. + +**Keep immediate close-group eviction active.** Evict on the spot when a K-closest peer drops below 0.20. Rejected *for now*: trust scoring is not yet stable enough; transient noise could eject honest close-group peers and churn the close group. The machinery is retained behind `close_group_immediate_eviction_enabled()` and its K-preservation guard, to be re-enabled once scoring stabilizes. + +**Keep generic automatic request penalties (status quo ante).** Let `send_request` keep reporting `ConnectionFailed`/`ConnectionTimeout`. Rejected: the generic layer cannot distinguish remote misbehaviour from congestion, application delay, or local overload, and it double-counts failures that application-aware layers already report with justified weights. + +**Parse application protocols in core to disambiguate failures.** Teach saorsa-core enough about each application protocol to attribute failures correctly. Rejected: inverts the layering (core is a phonebook and trust substrate, per the DHT-phonebook architecture), couples core releases to every consumer protocol, and still can't see application-level correctness (e.g. "served the wrong chunk"). + +**Per-call trust policy parameter on `send_request`.** Let each call site pass `penalize_on_failure: bool` or a policy enum. Rejected: pushes a breaking signature change onto every downstream caller, and the caller that knows enough to set the flag correctly is exactly the caller that can simply call `report_trust_event` — the explicit-report API already is the per-call policy, without changing the transport signature. + +## References + +- PR: [WithAutonomi/saorsa-core#119](https://github.com/WithAutonomi/saorsa-core/pull/119) — `feat(dht)!: add trust quarantine thresholds`; implementation commits span `898b9fb` through `5d5f69f`, where `5d5f69f` makes generic request transport trust-neutral. +- Key commits: `898b9fb` (thresholds), `f844fc7` (admission gate), `f03fa83` (K-sized table preservation), `6eb61df` (defaults on), `369f1a9` (bounded marker set), `001ca24` (marker semantics), `7159ad4` (stale-revalidation admissions), `f0a0b8b` (disable immediate eviction), `13cf699` (2.25 dial weight), `5d5f69f` (trust-neutral requests) +- Config & ownership: `src/adaptive/dht.rs:33-44` (constants), `:47-72` (`AdaptiveDhtConfig`), `:84-138` (validation), `:217-239` (`AdaptiveDHT::new`); `src/network.rs:255-263` (default enablement), `:505-538` (`trust_enforcement`) +- Engine: `src/dht/core_engine.rs:205-229` (constants, eviction gate, `MAX_QUARANTINED_PEERS`), `:1613-1642` (threshold setter / `quarantine_enabled`), `:1644-1670` (`check_new_peer_admission`), `:1690-1762` (marker lifecycle), `:1765-1795` (avoidance predicates), `:1866-1924` (`enforce_close_group_trust_gate`), `:3055-3081` (`re_evaluate_admission`) +- Manager: `src/dht_network_manager.rs:244-256` (trust reason constants, 2.25 weight), `:3569-3621` (failure recording, avoidance wrapper), `:3771-3806` (owned dial), `:4074-4079` (exactly-once RPC recording), filtering sites §5 +- Trust model: `src/adaptive/trust.rs` (EMA, decay, neutral 0.5) +- Trust-neutral transport: `src/network.rs:1006-1053`, `tests/trust_flow.rs:100` +- Docs updated by this PR: [`docs/trust-signals-api.md`](../trust-signals-api.md), [`docs/SECURITY_MODEL.md`](../SECURITY_MODEL.md), [`docs/ROUTING_TABLE_DESIGN.md`](../ROUTING_TABLE_DESIGN.md) +- Related ADRs: [ADR-006: EigenTrust Reputation System](./ADR-006-eigentrust-reputation.md), [ADR-007: Adaptive Networking with ML](./ADR-007-adaptive-networking.md), [ADR-009: Sybil Protection Mechanisms](./ADR-009-sybil-protection.md) diff --git a/docs/adr/README.md b/docs/adr/README.md index 17d6c4d2..a79e256f 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -35,6 +35,7 @@ An Architecture Decision Record (ADR) is a document that captures an important a | [ADR-006](./ADR-006-eigentrust-reputation.md) | EigenTrust Reputation System | Accepted | Iterative trust computation for Sybil resistance | | [ADR-009](./ADR-009-sybil-protection.md) | Sybil Protection Mechanisms | Accepted | Multi-layered defense against identity attacks | | [ADR-010](./ADR-010-entangled-attestation.md) | Entangled Attestation System | Accepted | Software integrity verification via attestation chains | +| [ADR-016](./ADR-016-trust-quarantine-and-trust-neutral-requests.md) | Trust Quarantine Thresholds and Trust-Neutral Request Transport | Proposed | Three-threshold DHT trust quarantine (0.35 swap / 0.20 avoidance / 0.45 admission) with trust-neutral generic request transport | ### Network Intelligence From c3c1f4f9daa86a58b5a83be452e897cf6ce79ffd Mon Sep 17 00:00:00 2001 From: Mick van Dijke <12992260+mickvandijke@users.noreply.github.com> Date: Tue, 25 Aug 2026 12:02:28 +0200 Subject: [PATCH 14/18] fix(dht): reserve readmit threshold for quarantined peers --- docs/ROUTING_TABLE_DESIGN.md | 11 +- docs/SECURITY_MODEL.md | 8 +- ...t-quarantine-and-trust-neutral-requests.md | 17 +- src/adaptive/dht.rs | 11 +- src/dht/core_engine.rs | 159 ++++++++++-------- src/dht_network_manager.rs | 1 - src/network.rs | 4 +- 7 files changed, 116 insertions(+), 95 deletions(-) diff --git a/docs/ROUTING_TABLE_DESIGN.md b/docs/ROUTING_TABLE_DESIGN.md index f4056fed..4a570a00 100644 --- a/docs/ROUTING_TABLE_DESIGN.md +++ b/docs/ROUTING_TABLE_DESIGN.md @@ -186,7 +186,7 @@ When a candidate peer `P` with `NodeInfo` and IP address `candidate_ip` is prese 2. **Address check**: If `P.addresses` is empty, reject. 3. **Authentication check**: If `P` has not completed transport-level authentication, reject. 4. **Update short-circuit**: If `P` already exists in `KBucket(BucketIndex(self, P))`, merge addresses (Section 6.3), refresh `last_seen`, move `P` to tail, and return. The peer already holds its slot — new-peer trust admission, IP diversity, and capacity checks are skipped. -5. **New-peer trust admission check**: If `TrustScore(self, P) < QUARANTINE_READMIT_THRESHOLD`, reject. If `P` was previously quarantined, reject until `TrustScore(self, P) >= QUARANTINE_READMIT_THRESHOLD`. +5. **New-peer trust admission check**: If `P` carries an explicit quarantine marker, reject until `TrustScore(self, P) >= QUARANTINE_READMIT_THRESHOLD`. Otherwise reject only when `TrustScore(self, P) < QUARANTINE_THRESHOLD`. 6. **Loopback check**: If `candidate_ip` is loopback and loopback is disallowed, reject. If loopback is allowed, skip all IP diversity checks (step 7–9) and proceed directly to insertion/capacity handling. 7. **Non-IP transport bypass**: If `P` has no IP-based address (e.g., Bluetooth, LoRa), skip IP diversity checks and proceed directly to insertion/capacity handling. 8. **IP diversity enforcement** (under write lock — Invariant 10): @@ -265,7 +265,7 @@ When any interaction records a trust failure and `TrustScore(self, P)` drops bel 4. Do not re-admit quarantined `P` until `TrustScore(self, P) >= QUARANTINE_READMIT_THRESHOLD`. 5. If removal would shrink `LocalRT(self)` below K peers, keep `P` in the routing table until another peer is admitted and the same eviction can happen without underfilling the table. 6. If `P` is not in the K-closest-to-self set, it may remain in the routing table, but local lookup result selection, FIND_NODE responses, and automatic lookup/dial paths MUST avoid it while `TrustScore(self, P) < QUARANTINE_THRESHOLD`. -7. If `P` has `QUARANTINE_THRESHOLD <= TrustScore(self, P) < QUARANTINE_READMIT_THRESHOLD` and is already in the routing table, it may remain there, including after moving into the K-closest set. New routing-table admissions in this range are rejected until trust reaches `QUARANTINE_READMIT_THRESHOLD`. +7. If `P` has `QUARANTINE_THRESHOLD <= TrustScore(self, P) < QUARANTINE_READMIT_THRESHOLD`, it may remain in or enter the routing table unless it carries an explicit quarantine marker. A marked peer remains rejected until trust reaches `QUARANTINE_READMIT_THRESHOLD`. Quarantine is a routing-table and automatic lookup policy. It is not a blanket transport-level block for explicit user-initiated sends. @@ -701,9 +701,10 @@ Each scenario should assert exact expected outcomes and state transitions. 3. **Empty address rejection**: - Candidate with zero addresses. Rejected with error. Routing table unchanged. -4. **New peer admission threshold**: - - New peer with `TrustScore < QUARANTINE_READMIT_THRESHOLD`, including a previously quarantined peer, is rejected. Not in routing table. - - Existing routing-table peer with `QUARANTINE_THRESHOLD <= TrustScore < QUARANTINE_READMIT_THRESHOLD` remains eligible for address/liveness updates and may later move into the K-closest set. +4. **New peer admission and quarantine readmission thresholds**: + - A new, unmarked peer with `TrustScore < QUARANTINE_THRESHOLD` is rejected. An unmarked peer in `[QUARANTINE_THRESHOLD, QUARANTINE_READMIT_THRESHOLD)` is admitted normally. + - A peer carrying an explicit quarantine marker is rejected until `TrustScore >= QUARANTINE_READMIT_THRESHOLD`. + - Existing routing-table peers remain eligible for address/liveness updates and may later move into the K-closest set. 5. **Bucket-full rejection (no stale peers)**: - Bucket at `K_BUCKET_SIZE` capacity, candidate cannot swap-closer, all incumbent peers have `last_seen` within `LIVE_THRESHOLD`. Stale revalidation finds no candidates. Rejected with "bucket at capacity." Routing table unchanged. diff --git a/docs/SECURITY_MODEL.md b/docs/SECURITY_MODEL.md index 1f0b3265..61eac213 100644 --- a/docs/SECURITY_MODEL.md +++ b/docs/SECURITY_MODEL.md @@ -88,7 +88,8 @@ Trust affects routing-table membership in two stages: |-----------------|-------------------|---------------| | Lazy swap eligibility | < 0.35 | `swap_threshold` | | Close-group quarantine / lookup avoidance | < 0.20 | `quarantine_threshold` | -| New peer admission / quarantine readmission | >= 0.45 | `quarantine_readmit_threshold` | +| New peer admission | >= 0.20 | `quarantine_threshold` | +| Explicit quarantine readmission | >= 0.45 | `quarantine_readmit_threshold` | | Staleness | Configurable | `stale_timeout` | Peers outside the K-closest set are not globally evicted solely for low trust. @@ -97,8 +98,9 @@ lookup paths below the quarantine threshold, and can be lazily replaced when better candidates need the slot. Peers already in the routing table at or above the quarantine threshold but below the readmission threshold may remain in the table, including after moving -into the K-closest set. New routing-table admissions and quarantined -readmissions require the readmission threshold. +into the K-closest set. New routing-table admissions require the quarantine +threshold; only peers carrying an explicit quarantine marker require the higher +readmission threshold. ### Quarantine Reasons diff --git a/docs/adr/ADR-016-trust-quarantine-and-trust-neutral-requests.md b/docs/adr/ADR-016-trust-quarantine-and-trust-neutral-requests.md index 7c63340d..5ced99d3 100644 --- a/docs/adr/ADR-016-trust-quarantine-and-trust-neutral-requests.md +++ b/docs/adr/ADR-016-trust-quarantine-and-trust-neutral-requests.md @@ -45,19 +45,20 @@ Defined in `src/adaptive/dht.rs:33-40` (mirrored as documentation constants in ` |-----------|---------|---------| | `swap_threshold` | **0.35** | Peer becomes *eligible for lazy swap-out* — replaced only when a better routing-table candidate arrives. Never causes eviction on its own. | | `quarantine_threshold` | **0.20** | *Automatic avoidance*: lookup/dial machinery stops selecting the peer. The peer is not removed and explicit sends still work. | -| `quarantine_readmit_threshold` | **0.45** | *Admission gate*: a peer **unknown to the routing table** (brand new, or previously quarantined and forgotten) must score at or above this to be admitted/readmitted. | +| `quarantine_readmit_threshold` | **0.45** | *Readmission hysteresis*: only a peer carrying an explicit quarantine marker must recover to this score before readmission. | **Validation** (`AdaptiveDhtConfig::validate`, `src/adaptive/dht.rs:84-138`): all three must be finite and in `[0.0, 0.5)`; when quarantine is active, `quarantine_readmit_threshold >= quarantine_threshold`; when swap and quarantine are both non-zero, `swap_threshold > quarantine_threshold` (swap is a milder condition than avoidance, quarantine is strictly more severe). The quarantine pair is re-checked at the engine boundary in `set_trust_quarantine_thresholds`; `swap_threshold` is validated only by `AdaptiveDhtConfig::validate`. Values at or above neutral `0.5` are rejected because decay approaches neutral asymptotically: negatively observed peers would remain swap/quarantine-eligible indefinitely, while readmission from below at a neutral cutoff would be unreachable in finite time. Invalid config fails node construction (`AdaptiveDHT::new` returns `Err`). **Disabling**: `quarantine_threshold == 0.0` disables quarantine enforcement (`quarantine_enabled()`, `src/dht/core_engine.rs:1640-1642`). `NodeConfigBuilder::trust_enforcement(false)` (`src/network.rs:518-530`) zeroes all three thresholds — scores are still tracked, but nothing is enforced. -### 3. Only unknown admissions are gated; existing peers in [0.20, 0.45) are preserved +### 3. New peers use quarantine; explicit readmissions use hysteresis -`check_new_peer_admission` (`src/dht/core_engine.rs:1644-1670`) is called **only when the peer is not already in the routing table** (`add_node`, `src/dht/core_engine.rs:2443-2446`; `re_evaluate_admission`, `src/dht/core_engine.rs:3069-3072`). Non-finite trust scores are rejected defensively. Consequences: +`check_new_peer_admission` (`src/dht/core_engine.rs:1644-1673`) is called **only when the peer is not already in the routing table** (`add_node`, `src/dht/core_engine.rs:2443-2446`; `re_evaluate_admission`, `src/dht/core_engine.rs:3069-3072`). Non-finite trust scores are rejected defensively. Consequences: - An existing routing-table peer whose score sits in `[0.20, 0.45)` **stays in the table** and may move into the close group (close-group membership is pure XOR distance over table contents). It remains eligible for lazy swap-out below 0.35 and is skipped by automatic selection below 0.20 — but it is never ejected merely for the band it occupies. -- Admission at or above 0.45 clears any quarantine marker (`forget_quarantined_peer`, `src/dht/core_engine.rs:1668`) — this is the readmission point. -- `should_avoid_automatic_candidate` (`src/dht/core_engine.rs:1779-1795`) encodes the asymmetry directly: a peer scoring in `[quarantine, readmit)` is avoided as an automatic candidate *only if it is not already in the routing table*. +- An unmarked peer is admitted and remains eligible for automatic lookup at or above 0.20. A transient failure cannot turn its absence from one observer's routing table into an implicit quarantine. +- A marked peer remains blocked below 0.45. Admission at or above 0.45 clears its quarantine marker (`forget_quarantined_peer`) — this is the explicit readmission point. +- `should_avoid_automatic_candidate` applies the same score/marker policy as `should_avoid_for_lookup`; routing-table absence by itself no longer raises the applicable threshold. ### 4. Stale-revalidation concurrency invariant @@ -65,7 +66,7 @@ Stale-peer revalidation (evict-then-readmit under contention) must not let the n ### 5. Automatic filtering everywhere; explicit sends and wire format untouched -Two engine predicates drive all filtering: `should_avoid_for_lookup` (`src/dht/core_engine.rs:1765-1775` — non-finite, below 0.20, or marked quarantined and below 0.45) and `should_avoid_automatic_candidate` (adds the unknown-peer readmit gate). They are applied on every **automatic** path in `src/dht_network_manager.rs`: +Two engine predicates drive all filtering: `should_avoid_for_lookup` (`src/dht/core_engine.rs:1768-1778` — non-finite, below 0.20, or marked quarantined and below 0.45) and `should_avoid_automatic_candidate`, which deliberately applies the same policy to newly discovered candidates. They are applied on every **automatic** path in `src/dht_network_manager.rs`: 1. Local lookup results / FIND_NODE response serving — `find_closest_nodes_local` (`:2579-2606`) 2. Iterative lookup local seeding — `find_closest_nodes_network` (`:2740`) @@ -148,13 +149,13 @@ The division of labour: **layers that can judge, report; layers that can't, stay ### Neutral - The immediate-eviction machinery (gate function, trust-gate enforcement, marker insertion) ships dark: wired, tested, and preserved, but returning no events until `close_group_immediate_eviction_enabled()` flips. Re-enabling it is a one-line change plus recalibration review — a likely follow-up ADR/amendment once scoring is deemed stable. -- With eviction dark, quarantine *markers* have no production writer; the active enforcement today is score-based avoidance (0.20), the unknown-admission gate (0.45), and lazy swap (0.35). +- With eviction dark, quarantine *markers* have no production writer; the active enforcement today is score-based avoidance/admission (0.20) and lazy swap (0.35). The 0.45 hysteresis becomes active when explicit quarantine eviction is enabled. - Trust scores keep being computed identically in observe-only mode, so enabling enforcement later needs no re-learning period. ## Compatibility and Breaking Changes - **API (breaking)**: `AdaptiveDhtConfig` gains `quarantine_threshold` and `quarantine_readmit_threshold` (with `#[serde(default)]`, so serialized configs deserialize fine); struct-literal construction without `..Default::default()` breaks. `DhtNetworkConfig` gains the same fields. -- **Behavioural (breaking)**: quarantine defaults ON. New routing-table peers must meet 0.45 when enforcement is enabled; peers below 0.20 stop appearing in lookup results and automatic maintenance. `send_request` no longer auto-penalizes failures — consumers relying on that must add explicit `report_trust_event` calls. +- **Behavioural (breaking)**: quarantine defaults ON. Unmarked routing-table peers must meet 0.20; explicitly quarantined peers must recover to 0.45. Peers below 0.20 stop appearing in lookup results and automatic maintenance. `send_request` no longer auto-penalizes failures — consumers relying on that must add explicit `report_trust_event` calls. - **Wire (non-breaking)**: no message format changes; `DHTNode` reliability keeps its legacy value. ## Operational Implications diff --git a/src/adaptive/dht.rs b/src/adaptive/dht.rs index 138fd623..ebf9e67d 100644 --- a/src/adaptive/dht.rs +++ b/src/adaptive/dht.rs @@ -35,7 +35,7 @@ const DEFAULT_SWAP_THRESHOLD: f64 = 0.35; /// lookup/dial paths. const DEFAULT_QUARANTINE_THRESHOLD: f64 = 0.20; -/// Default trust score a new or quarantined peer must have for admission. +/// Default trust score an explicitly quarantined peer must regain for readmission. const DEFAULT_QUARANTINE_READMIT_THRESHOLD: f64 = 0.45; /// Maximum weight multiplier per single consumer-reported event. @@ -54,8 +54,9 @@ pub struct AdaptiveDhtConfig { /// Trust score below which automatic lookup/dial paths avoid a peer. /// Default: 0.20 pub quarantine_threshold: f64, - /// Trust score required before a new peer can enter the routing table, and - /// before a quarantined peer can re-enter. + /// Trust score required before an explicitly quarantined peer can re-enter + /// the routing table. New peers that were never quarantined are admitted at + /// or above `quarantine_threshold`. /// Default: 0.45 pub quarantine_readmit_threshold: f64, } @@ -76,8 +77,8 @@ impl AdaptiveDhtConfig { /// Returns `Err` if a threshold is outside its safe range or is NaN. /// Values >= 0.5 (neutral trust) would make all unknown peers immediately /// swap/quarantine eligible since they start at neutral (0.5). The - /// new-peer admission/readmit threshold must also stay below neutral - /// because recovery happens by decay toward neutral, not by active probing. + /// quarantine readmit threshold must also stay below neutral because + /// recovery happens by decay toward neutral, not by active probing. /// When swap enforcement is enabled, the swap threshold must remain above /// the quarantine threshold so quarantine is strictly more severe. pub fn validate(&self) -> crate::error::P2pResult<()> { diff --git a/src/dht/core_engine.rs b/src/dht/core_engine.rs index 05feb2b2..6511af3e 100644 --- a/src/dht/core_engine.rs +++ b/src/dht/core_engine.rs @@ -1608,8 +1608,9 @@ impl DhtCoreEngine { /// A `quarantine_threshold` of `0.0` disables quarantine enforcement. /// Otherwise, peers below that score are avoided for automatic lookups. /// Quarantined peers can only re-enter through normal admission after - /// their decayed trust reaches `quarantine_readmit_threshold`; new peers - /// must also meet that threshold before entering the routing table. + /// their decayed trust reaches `quarantine_readmit_threshold`. Peers that + /// were never explicitly quarantined need only remain at or above + /// `quarantine_threshold`. pub(crate) fn set_trust_quarantine_thresholds( &mut self, quarantine_threshold: f64, @@ -1651,21 +1652,24 @@ impl DhtCoreEngine { peer_id.to_hex() )); } - if trust_score < self.quarantine_readmit_threshold { - if self.quarantined_peers.contains(peer_id) { + if self.quarantined_peers.contains(peer_id) { + if trust_score < self.quarantine_readmit_threshold { return Err(anyhow!( "peer {} quarantined until trust >= {:.3} (current {trust_score:.3})", peer_id.to_hex(), self.quarantine_readmit_threshold )); } + self.forget_quarantined_peer(peer_id); + return Ok(()); + } + if trust_score < self.quarantine_threshold { return Err(anyhow!( - "peer {} below new-peer admission threshold ({trust_score:.3} < {:.3})", + "peer {} below quarantine threshold for new-peer admission ({trust_score:.3} < {:.3})", peer_id.to_hex(), - self.quarantine_readmit_threshold + self.quarantine_threshold )); } - self.forget_quarantined_peer(peer_id); Ok(()) } @@ -1774,24 +1778,17 @@ impl DhtCoreEngine { && trust_score < self.quarantine_readmit_threshold) } - /// Return whether automatic lookup/dial paths should avoid this peer when - /// it might become a new routing-table admission. - pub(crate) async fn should_avoid_automatic_candidate( + /// Return whether automatic lookup/dial paths should avoid this peer. + /// + /// Unknown peers are judged against the quarantine threshold. The higher + /// readmission threshold applies only when an explicit quarantine marker + /// records that this observer previously evicted the peer. + pub(crate) fn should_avoid_automatic_candidate( &self, peer_id: &PeerId, trust_score: f64, ) -> bool { - if self.should_avoid_for_lookup(peer_id, trust_score) { - return true; - } - if !self.quarantine_enabled() || trust_score >= self.quarantine_readmit_threshold { - return false; - } - self.routing_table - .read() - .await - .find_node_by_id(peer_id) - .is_none() + self.should_avoid_for_lookup(peer_id, trust_score) } /// Evict a quarantined peer if it currently occupies the K-closest set and @@ -5436,26 +5433,15 @@ mod tests { ); dht.remove_node_by_id(&peer).await; - let below_admission = dht + let readmitted = dht .add_node( make_node_with_addr(peer_id_bytes, "/ip4/10.10.0.1/udp/9000/quic"), &|id| if *id == peer { 0.30 } else { 0.5 }, ) .await; assert!( - below_admission.is_err(), - "removed peer should not readmit below 0.45" - ); - - let recovered = dht - .add_node( - make_node_with_addr(peer_id_bytes, "/ip4/10.10.0.1/udp/9000/quic"), - &|id| if *id == peer { 0.45 } else { 0.5 }, - ) - .await; - assert!( - recovered.is_ok(), - "removed peer should readmit once trust reaches 0.45" + readmitted.is_ok(), + "a manually removed peer without a quarantine marker should be admitted above 0.20" ); assert!(dht.has_node(&peer).await); } @@ -5625,10 +5611,10 @@ mod tests { ); } - /// New peers must meet the readmit/admission threshold even when they would - /// occupy a non-close routing-table slot. + /// New non-close peers that were never quarantined use the lower quarantine + /// threshold for admission. #[tokio::test] - async fn test_new_non_close_admission_requires_readmit_threshold() { + async fn test_new_non_close_admission_uses_quarantine_threshold() { let mut dht = DhtCoreEngine::new( PeerId::from_bytes([0u8; 32]), 4, @@ -5664,35 +5650,23 @@ mod tests { "peer below quarantine threshold should be rejected" ); - let below_new_peer_admission = dht + let admitted = dht .add_node( make_node_with_addr(peer_id_bytes, "/ip4/10.10.0.9/udp/9000/quic"), &|id| if *id == peer { 0.30 } else { 0.5 }, ) .await; assert!( - below_new_peer_admission.is_err(), - "new non-close peer should need trust >= 0.45" - ); - - let recovered = dht - .add_node( - make_node_with_addr(peer_id_bytes, "/ip4/10.10.0.9/udp/9000/quic"), - &|id| if *id == peer { 0.45 } else { 0.5 }, - ) - .await; - assert!( - recovered.is_ok(), - "new non-close peer should enter once trust reaches 0.45" + admitted.is_ok(), + "new non-close peer above 0.20 should be admitted without a quarantine marker" ); assert!(dht.has_node(&peer).await); } - /// A new peer that would enter the K-closest set must meet the general - /// new-peer admission threshold, even if it is above the lower quarantine - /// threshold. + /// New close-group peers that were never quarantined also use the lower + /// quarantine threshold for admission. #[tokio::test] - async fn test_new_close_group_admission_requires_readmit_threshold() { + async fn test_new_close_group_admission_uses_quarantine_threshold() { let mut dht = DhtCoreEngine::new_for_tests(PeerId::from_bytes([0u8; 32])).unwrap(); dht.set_trust_quarantine_thresholds(0.20, 0.45).unwrap(); @@ -5700,32 +5674,32 @@ mod tests { peer_id_bytes[31] = 9; let peer = PeerId::from_bytes(peer_id_bytes); - let below_close_group_threshold = dht + let below_quarantine = dht .add_node( make_node_with_addr(peer_id_bytes, "/ip4/10.10.0.9/udp/9000/quic"), - &|id| if *id == peer { 0.30 } else { 0.5 }, + &|id| if *id == peer { 0.10 } else { 0.5 }, ) .await; assert!( - below_close_group_threshold.is_err(), - "new close-group peer should need trust >= 0.45" + below_quarantine.is_err(), + "new close-group peer below 0.20 should be rejected" ); - let recovered = dht + let admitted = dht .add_node( make_node_with_addr(peer_id_bytes, "/ip4/10.10.0.9/udp/9000/quic"), - &|id| if *id == peer { 0.45 } else { 0.5 }, + &|id| if *id == peer { 0.30 } else { 0.5 }, ) .await; assert!( - recovered.is_ok(), - "new close-group peer should enter once trust reaches 0.45" + admitted.is_ok(), + "new close-group peer above 0.20 should be admitted without a quarantine marker" ); assert!(dht.has_node(&peer).await); } #[tokio::test] - async fn test_automatic_lookup_skips_unknown_below_admission_threshold() { + async fn test_automatic_lookup_uses_quarantine_marker_for_readmit_hysteresis() { let mut dht = DhtCoreEngine::new( PeerId::from_bytes([0u8; 32]), 4, @@ -5750,20 +5724,63 @@ mod tests { let unknown_peer = PeerId::from_bytes(unknown_id); assert!( - dht.should_avoid_automatic_candidate(&unknown_peer, 0.30) - .await, - "automatic lookup should skip unknown peers below new-peer admission threshold" + !dht.should_avoid_automatic_candidate(&unknown_peer, 0.30), + "automatic lookup should allow an unknown peer above the quarantine threshold" ); assert!( - !dht.should_avoid_automatic_candidate(&existing_peer, 0.30) - .await, + dht.should_avoid_automatic_candidate(&unknown_peer, 0.10), + "automatic lookup should skip an unknown peer below the quarantine threshold" + ); + assert!( + !dht.should_avoid_automatic_candidate(&existing_peer, 0.30), "existing routing-table peers above quarantine threshold should remain usable" ); + + dht.remember_quarantined_peer(unknown_peer, &|_| 0.30); + assert!( + dht.should_avoid_automatic_candidate(&unknown_peer, 0.30), + "an explicitly quarantined peer should remain avoided below the readmit threshold" + ); + } + + #[tokio::test] + async fn test_explicitly_quarantined_peer_requires_readmit_threshold() { + let mut dht = DhtCoreEngine::new_for_tests(PeerId::from_bytes([0u8; 32])).unwrap(); + dht.set_trust_quarantine_thresholds(0.20, 0.45).unwrap(); + + let mut peer_id_bytes = [0u8; 32]; + peer_id_bytes[31] = 9; + let peer = PeerId::from_bytes(peer_id_bytes); + dht.remember_quarantined_peer(peer, &|_| 0.30); + + let rejected = dht + .add_node( + make_node_with_addr(peer_id_bytes, "/ip4/10.10.0.9/udp/9000/quic"), + &|id| if *id == peer { 0.30 } else { 0.5 }, + ) + .await; + assert!( + rejected.is_err(), + "explicitly quarantined peer should remain blocked below 0.45" + ); + + let readmitted = dht + .add_node( + make_node_with_addr(peer_id_bytes, "/ip4/10.10.0.9/udp/9000/quic"), + &|id| if *id == peer { 0.45 } else { 0.5 }, + ) + .await; + assert!( + readmitted.is_ok(), + "explicitly quarantined peer should re-enter at 0.45" + ); + assert!(dht.has_node(&peer).await); + assert!(!dht.quarantined_peers.contains(&peer)); } /// Removing one close peer can promote a non-close peer into the close /// group. Existing routing-table peers above the quarantine threshold stay - /// even when below the new-peer admission threshold; peers below the + /// even when below the explicit readmission threshold; peers below the /// quarantine threshold are also retained while immediate eviction is /// disabled. #[tokio::test] diff --git a/src/dht_network_manager.rs b/src/dht_network_manager.rs index e42ce724..2a86079b 100644 --- a/src/dht_network_manager.rs +++ b/src/dht_network_manager.rs @@ -3725,7 +3725,6 @@ impl DhtNetworkManager { let trust_score = self.peer_trust_score(peer_id); let dht = self.dht.read().await; dht.should_avoid_automatic_candidate(peer_id, trust_score) - .await } /// Ensure an identity-authenticated channel to `peer_id` exists, diff --git a/src/network.rs b/src/network.rs index 2e2dded7..f4341495 100644 --- a/src/network.rs +++ b/src/network.rs @@ -302,8 +302,8 @@ pub struct NodeConfig { /// Adaptive DHT configuration for trust-based routing enforcement. /// - /// Controls lazy swap-out, automatic lookup avoidance, and - /// new-peer/readmission trust thresholds. Use + /// Controls lazy swap-out, automatic lookup avoidance, and quarantined-peer + /// readmission thresholds. Use /// [`NodeConfigBuilder::trust_enforcement`] for a simple on/off toggle. /// /// Default: enabled with the default [`AdaptiveDhtConfig`] thresholds. From fa894a30100870461480c06b35c7d46f6cd0ef24 Mon Sep 17 00:00:00 2001 From: Mick van Dijke <12992260+mickvandijke@users.noreply.github.com> Date: Tue, 25 Aug 2026 15:21:42 +0200 Subject: [PATCH 15/18] feat(dht): enable immediate close-group quarantine --- ...t-quarantine-and-trust-neutral-requests.md | 18 ++-- src/adaptive/dht.rs | 6 +- src/dht/core_engine.rs | 82 +++++++++---------- src/dht_network_manager.rs | 5 +- 4 files changed, 54 insertions(+), 57 deletions(-) diff --git a/docs/adr/ADR-016-trust-quarantine-and-trust-neutral-requests.md b/docs/adr/ADR-016-trust-quarantine-and-trust-neutral-requests.md index 5ced99d3..96ae0cf0 100644 --- a/docs/adr/ADR-016-trust-quarantine-and-trust-neutral-requests.md +++ b/docs/adr/ADR-016-trust-quarantine-and-trust-neutral-requests.md @@ -80,13 +80,13 @@ Two engine predicates drive all filtering: `should_avoid_for_lookup` (`src/dht/c **Wire format is unchanged**: trust is never serialized into DHT lookup results — `lookup_results_from_routing_nodes` keeps the legacy `DHTNode` reliability wire value stable (`src/dht_network_manager.rs:2620-2623`, asserted by test at `:6011`). Older nodes interoperate without change. -### 6. Immediate close-group trust eviction: implemented but disabled +### 6. Immediate close-group trust eviction -Immediate eviction of below-0.20 close-group peers is **currently switched off** while trust scoring stabilizes: `close_group_immediate_eviction_enabled()` is hard-wired to `false` (`src/dht/core_engine.rs:216-218`). `enforce_close_group_trust_gate` (`src/dht/core_engine.rs:1866-1924`) is still wired through `DhtNetworkManager::enforce_trust_quarantine` and `broadcast_routing_events_with_quarantine`, but returns no events; `enforce_close_group_quarantine` is retained under `#[cfg(test)]`. The gated-off bodies preserve the safety property for re-enablement: eviction only proceeds `while routing.node_count() > k_value`, so the routing table **never shrinks below K** for trust reasons. Until the gate flips, lazy swap-out (0.35) is the sole replacement mechanism for low-trust peers, and peers below 0.20 remain in the table but are avoided by the automatic paths above. +Immediate eviction of below-0.20 close-group peers is enabled. `enforce_close_group_trust_gate` is wired through `DhtNetworkManager::enforce_trust_quarantine` and `broadcast_routing_events_with_quarantine`; it records an explicit quarantine marker before removing a peer. Eviction only proceeds `while routing.node_count() > k_value`, so the routing table **never shrinks below K** for trust reasons. When no surplus exists, the peer remains in the table but is avoided by automatic paths until a later enforcement pass can replace it safely. ### 7. Bounded quarantine markers -Quarantine markers (`quarantined_peers: HashSet` plus FIFO `quarantined_peer_order`) are bounded at `MAX_QUARANTINED_PEERS = 8192` (`src/dht/core_engine.rs:229`). The key insight making the bound safe (`quarantine_marker_required_for_score`, `src/dht/core_engine.rs:1690-1696`): **a marker is only semantically required while a peer's score is in `[quarantine_threshold, readmit_threshold)`** — below 0.20 the score itself keeps the peer avoided; at/above 0.45 the peer is readmittable and the marker is cleared. When the set is full, `prune_redundant_quarantined_peers` drops (oldest-first) any marker whose current score no longer requires one; if the set is still full, the new marker is simply not inserted — the score-based avoidance clause covers the peer regardless. Note: with immediate eviction disabled (§6), no production path currently inserts markers; the machinery is preserved intact for re-enablement. +Quarantine markers (`quarantined_peers: HashSet` plus FIFO `quarantined_peer_order`) are bounded at `MAX_QUARANTINED_PEERS = 8192` (`src/dht/core_engine.rs:229`). The key insight making the bound safe (`quarantine_marker_required_for_score`, `src/dht/core_engine.rs:1690-1696`): **a marker is only semantically required while a peer's score is in `[quarantine_threshold, readmit_threshold)`** — below 0.20 the score itself keeps the peer avoided; at/above 0.45 the peer is readmittable and the marker is cleared. When the set is full, `prune_redundant_quarantined_peers` drops (oldest-first) any marker whose current score no longer requires one; if the set is still full, the new marker is simply not inserted — the score-based avoidance clause covers the peer regardless. ### 8. Decay and recovery via rediscovery @@ -118,7 +118,7 @@ The division of labour: **layers that can judge, report; layers that can't, stay ## Invariants 1. **Threshold ordering**: when quarantine is active, `0 < quarantine_threshold <= quarantine_readmit_threshold < 0.5`; when swap and quarantine are both active, `quarantine_threshold < swap_threshold < 0.5`. Defaults: `0.20 < 0.35` and `0.20 <= 0.45`. -2. **K-sized routing table**: trust enforcement never shrinks the routing table below K (eviction, even when re-enabled, only runs while `node_count > K`). +2. **K-sized routing table**: trust enforcement never shrinks the routing table below K; eviction only runs while `node_count > K`. 3. **Known-peer preservation**: a peer already in the routing table is never subjected to the new-peer admission gate — including during stale-revalidation races (§4). 4. **No transport block**: quarantine affects only routing-table membership and automatic selection; explicit sends always go through. 5. **Wire stability**: trust state never leaks into serialized DHT messages; filtering is strictly local policy. @@ -141,16 +141,14 @@ The division of labour: **layers that can judge, report; layers that can't, stay ### Negative -- **Slower reaction to genuinely malicious close-group peers**: with immediate eviction disabled, a below-0.20 peer stays in the close group until lazy swap-out replaces it; it is avoided by automatic paths but still occupies a slot. +- **Close-group churn risk**: transiently driving an honest peer below 0.20 can immediately evict it when surplus capacity exists and hold it out until its score recovers to 0.45. - **Applications now own attribution**: any consumer that relied on `send_request`'s automatic penalties gets no trust signal for its request failures unless it explicitly reports a justified outcome. Silent trust erosion of misbehaving peers via generic requests no longer happens. - **Recovery latency is fixed by decay**: a wrongly-penalized peer needs up to ~46 h (worst case) to become admissible again; there is no active-probe fast path. - **More configuration surface**: three interdependent thresholds with ordering rules; invalid combinations fail node construction (loudly, by design). ### Neutral -- The immediate-eviction machinery (gate function, trust-gate enforcement, marker insertion) ships dark: wired, tested, and preserved, but returning no events until `close_group_immediate_eviction_enabled()` flips. Re-enabling it is a one-line change plus recalibration review — a likely follow-up ADR/amendment once scoring is deemed stable. -- With eviction dark, quarantine *markers* have no production writer; the active enforcement today is score-based avoidance/admission (0.20) and lazy swap (0.35). The 0.45 hysteresis becomes active when explicit quarantine eviction is enabled. -- Trust scores keep being computed identically in observe-only mode, so enabling enforcement later needs no re-learning period. +- Trust scores keep being computed identically in observe-only mode, so disabling and later re-enabling enforcement needs no re-learning period. ## Compatibility and Breaking Changes @@ -161,7 +159,7 @@ The division of labour: **layers that can judge, report; layers that can't, stay ## Operational Implications - **Defaults are live on upgrade** — no config change needed to get the policy; use `trust_enforcement(false)` to opt out. -- Watch trust-score distribution after deployment: since scores are stabilizing, thresholds (especially 0.45 admission on small networks, where rejecting a scarce peer costs more) may need tuning before immediate eviction is re-enabled. +- Watch trust-score distribution and close-group churn after deployment, especially on small networks where rejecting a scarce peer costs more. - Reason strings (`dht_dial_failed`, `dht_request_failed`, `dht_identity_exchange_failed`, `application_failure`, …) are logged with score deltas — use them to audit which layer is driving a peer's score. - Downstream consumers should classify data-availability outcomes and report only justified `ApplicationSuccess`/`ApplicationFailure` events; generic request transport itself contributes no trust signal. - Small/bootstrap networks: bootstrap peers themselves are trust-filtered — a bootstrap peer driven below 0.20 will be skipped, so keep multiple bootstrap endpoints configured. @@ -172,7 +170,7 @@ The division of labour: **layers that can judge, report; layers that can't, stay **Hard transport-level block of quarantined peers.** Refuse all sends to below-threshold peers. Rejected: explicit sends are how applications retry, probe, and recover — blocking them turns a local routing preference into a network partition, breaks consumer semantics, and prevents the very interactions whose successes a consumer could report to rehabilitate a peer. -**Keep immediate close-group eviction active.** Evict on the spot when a K-closest peer drops below 0.20. Rejected *for now*: trust scoring is not yet stable enough; transient noise could eject honest close-group peers and churn the close group. The machinery is retained behind `close_group_immediate_eviction_enabled()` and its K-preservation guard, to be re-enabled once scoring stabilizes. +**Keep immediate close-group eviction disabled.** Rely only on score-based avoidance and lazy swap-out. Rejected after testnet calibration: it leaves a below-0.20 peer occupying a close-group slot despite surplus capacity and leaves the 0.45 readmission hysteresis without a production marker writer. The K-preservation guard limits immediate eviction to routing tables with surplus above K. **Keep generic automatic request penalties (status quo ante).** Let `send_request` keep reporting `ConnectionFailed`/`ConnectionTimeout`. Rejected: the generic layer cannot distinguish remote misbehaviour from congestion, application delay, or local overload, and it double-counts failures that application-aware layers already report with justified weights. diff --git a/src/adaptive/dht.rs b/src/adaptive/dht.rs index ebf9e67d..da864f72 100644 --- a/src/adaptive/dht.rs +++ b/src/adaptive/dht.rs @@ -313,9 +313,9 @@ impl AdaptiveDHT { /// Start the DHT manager. /// /// Trust scores are computed live — no background tasks needed. - /// Low-trust peers are swapped out when better candidates arrive. Immediate - /// close-group eviction is temporarily disabled until trust scoring is - /// stable. + /// Low-trust peers are swapped out when better candidates arrive. Close- + /// group peers below the quarantine threshold are immediately evicted when + /// the routing table has capacity above K. pub async fn start(&self) -> Result<()> { Arc::clone(&self.dht_manager).start().await } diff --git a/src/dht/core_engine.rs b/src/dht/core_engine.rs index 6511af3e..551f7f7b 100644 --- a/src/dht/core_engine.rs +++ b/src/dht/core_engine.rs @@ -208,16 +208,13 @@ const DEFAULT_SWAP_THRESHOLD: f64 = 0.35; #[allow(dead_code)] const DEFAULT_QUARANTINE_THRESHOLD: f64 = 0.20; -/// Immediate close-group eviction is disabled until trust scoring is stable. -/// -/// Low-trust peers remain eligible for lazy swap-out through -/// [`DhtCoreEngine::add_node`], but existing close-group peers are not evicted -/// solely because their score drops below the quarantine threshold. +/// Immediate close-group eviction is enabled for peers below the quarantine +/// threshold. Enforcement retains at least K peers in the routing table. fn close_group_immediate_eviction_enabled() -> bool { - false + true } -/// Default trust score required for new routing-table admission/readmission. +/// Default trust score required for explicitly quarantined-peer readmission. #[allow(dead_code)] const DEFAULT_QUARANTINE_READMIT_THRESHOLD: f64 = 0.45; @@ -1537,8 +1534,7 @@ pub struct DhtCoreEngine { /// paths. quarantine_threshold: f64, - /// Trust score required before a new peer can enter the routing table, and - /// before a quarantined peer can re-enter. + /// Trust score required before an explicitly quarantined peer can re-enter. quarantine_readmit_threshold: f64, /// Peers evicted from the close group by quarantine. Markers are bounded @@ -1794,9 +1790,9 @@ impl DhtCoreEngine { /// Evict a quarantined peer if it currently occupies the K-closest set and /// removal will not shrink the routing table below K peers. /// - /// Temporarily disabled until trust scoring is considered stable. Low-trust - /// peers continue to leave through lazy swap-out when better candidates are - /// admitted. + /// This targeted helper is retained for focused tests; production trust + /// enforcement evaluates the full close group in + /// [`Self::enforce_close_group_trust_gate`]. #[cfg(test)] pub(crate) async fn enforce_close_group_quarantine( &mut self, @@ -1857,9 +1853,8 @@ impl DhtCoreEngine { /// Enforce trust gates over the current K-closest set. /// - /// Temporarily leaves close-group peers in place even when they are below - /// the quarantine threshold. Lazy swap-out remains responsible for - /// replacing low-trust peers when better candidates arrive. + /// Peers below the quarantine threshold are evicted and explicitly marked + /// when the routing table has surplus capacity above K. pub(crate) async fn enforce_close_group_trust_gate( &mut self, trust_score: &impl Fn(&PeerId) -> f64, @@ -5379,10 +5374,10 @@ mod tests { assert!(dht.has_node(&low_peer).await); } - /// A K-closest peer below the quarantine threshold is retained while - /// immediate close-group eviction is disabled. + /// A K-closest peer below the quarantine threshold is evicted and marked + /// when the routing table has surplus capacity. #[tokio::test] - async fn test_close_group_peer_below_quarantine_is_not_immediately_evicted() { + async fn test_close_group_peer_below_quarantine_is_immediately_evicted() { let mut dht = DhtCoreEngine::new( PeerId::from_bytes([0u8; 32]), SMALL_TEST_K, @@ -5419,31 +5414,31 @@ mod tests { let events = dht.enforce_close_group_quarantine(&peer, 0.19).await; assert!( - events.is_empty(), - "close-group peer below quarantine threshold should not be immediately evicted" + events + .iter() + .any(|event| matches!(event, RoutingTableEvent::PeerRemoved(id) if *id == peer)), + "close-group peer below quarantine threshold should be immediately evicted" ); assert!( - dht.has_node(&peer).await, - "close-group peer should remain in RT while immediate eviction is disabled" + !dht.has_node(&peer).await, + "quarantined close-group peer should leave the routing table" ); - assert_eq!(dht.routing_table_size().await, SMALL_TEST_K + 1); + assert_eq!(dht.routing_table_size().await, SMALL_TEST_K); assert!( - dht.should_avoid_for_lookup(&peer, 0.19), - "retained low-trust peer should still be avoided by lookup policy" + dht.quarantined_peers.contains(&peer), + "eviction should create an explicit quarantine marker" ); - dht.remove_node_by_id(&peer).await; - let readmitted = dht + let rejected = dht .add_node( make_node_with_addr(peer_id_bytes, "/ip4/10.10.0.1/udp/9000/quic"), &|id| if *id == peer { 0.30 } else { 0.5 }, ) .await; assert!( - readmitted.is_ok(), - "a manually removed peer without a quarantine marker should be admitted above 0.20" + rejected.is_err(), + "explicitly quarantined peer should remain blocked below 0.45" ); - assert!(dht.has_node(&peer).await); } /// Close-group quarantine should not shrink the routing table below K. @@ -5490,9 +5485,9 @@ mod tests { } /// Once a new routing-table peer creates surplus above K, a low-trust - /// close-group peer is still retained while immediate eviction is disabled. + /// close-group peer is immediately quarantined. #[tokio::test] - async fn test_new_peer_surplus_does_not_trigger_close_group_quarantine() { + async fn test_new_peer_surplus_triggers_close_group_quarantine() { let mut dht = DhtCoreEngine::new( PeerId::from_bytes([0u8; 32]), SMALL_TEST_K, @@ -5536,11 +5531,14 @@ mod tests { .enforce_close_group_trust_gate(&|id| if *id == low_peer { 0.10 } else { 0.5 }) .await; assert!( - events.is_empty(), - "surplus peer should not trigger close-group quarantine while immediate eviction is disabled" + events.iter().any( + |event| matches!(event, RoutingTableEvent::PeerRemoved(id) if *id == low_peer) + ), + "surplus capacity should allow close-group quarantine" ); - assert_eq!(dht.routing_table_size().await, SMALL_TEST_K + 1); - assert!(dht.has_node(&low_peer).await); + assert_eq!(dht.routing_table_size().await, SMALL_TEST_K); + assert!(!dht.has_node(&low_peer).await); + assert!(dht.quarantined_peers.contains(&low_peer)); } #[test] @@ -5781,8 +5779,7 @@ mod tests { /// Removing one close peer can promote a non-close peer into the close /// group. Existing routing-table peers above the quarantine threshold stay /// even when below the explicit readmission threshold; peers below the - /// quarantine threshold are also retained while immediate eviction is - /// disabled. + /// quarantine threshold are evicted once surplus capacity exists. #[tokio::test] async fn test_close_group_gate_allows_existing_promotions_above_quarantine() { let mut dht = DhtCoreEngine::new( @@ -5876,10 +5873,13 @@ mod tests { }) .await; assert!( - quarantine_events.is_empty(), - "existing promoted peer below quarantine threshold should stay even when there is surplus" + quarantine_events.iter().any( + |event| matches!(event, RoutingTableEvent::PeerRemoved(id) if *id == promoted_peer) + ), + "existing promoted peer below quarantine threshold should be evicted when there is surplus" ); - assert!(dht.has_node(&promoted_peer).await); + assert!(!dht.has_node(&promoted_peer).await); + assert!(dht.quarantined_peers.contains(&promoted_peer)); } /// A non-close peer below the quarantine threshold is avoided by automatic diff --git a/src/dht_network_manager.rs b/src/dht_network_manager.rs index 2a86079b..935cb873 100644 --- a/src/dht_network_manager.rs +++ b/src/dht_network_manager.rs @@ -640,9 +640,8 @@ pub struct DhtNetworkConfig { /// routing table when a better candidate is available. /// Default: [`AdaptiveDhtConfig::default`]. pub swap_threshold: f64, - /// Trust score below which automatic lookup/dial paths avoid a peer. - /// Immediate close-group eviction is temporarily disabled until trust - /// scoring is stable. + /// Trust score below which automatic lookup/dial paths avoid a peer and + /// close-group peers are evicted when the routing table has surplus above K. /// Default: [`AdaptiveDhtConfig::default`]. pub quarantine_threshold: f64, /// Trust score required before a new peer can enter the routing table, From f909b079105c803c20dac0eabf11cad4e64a3020 Mon Sep 17 00:00:00 2001 From: Mick van Dijke <12992260+mickvandijke@users.noreply.github.com> Date: Thu, 27 Aug 2026 12:02:24 +0200 Subject: [PATCH 16/18] fix(dht): avoid scoring cached dial skips --- src/dht_network_manager.rs | 157 +++++++++++++++++++++++++++++-------- 1 file changed, 126 insertions(+), 31 deletions(-) diff --git a/src/dht_network_manager.rs b/src/dht_network_manager.rs index 935cb873..b6c36652 100644 --- a/src/dht_network_manager.rs +++ b/src/dht_network_manager.rs @@ -831,6 +831,21 @@ pub struct DhtNetworkManager { lookup_failures: Arc, } +/// Result of walking a peer's bounded dial plan. +/// +/// Keeping [`Self::NoAttempt`] separate from [`Self::AttemptedAndFailed`] +/// prevents callers from treating failure-cache suppression as a fresh +/// transport failure. +#[derive(Debug, PartialEq, Eq)] +enum DialAddressesOutcome { + /// A transport dial succeeded and produced this channel ID. + Connected(String), + /// No transport dial was made (for example, every candidate was cached). + NoAttempt, + /// At least one transport dial was made and every such dial failed. + AttemptedAndFailed, +} + /// Outcome of a shared dial+identity-exchange attempt, broadcast to /// every task that joined the in-flight dial via /// [`DhtNetworkManager::ensure_peer_channel`]. @@ -845,7 +860,8 @@ enum PendingDialOutcome { /// QUIC handshake completed and identity exchange authenticated /// the remote as the expected peer. Connected, - /// Every candidate address failed to dial. + /// No candidate address produced a channel. The owner records a trust + /// failure only when at least one transport dial was actually attempted. DialFailed { candidates_count: usize }, /// The dial succeeded but identity exchange failed or timed out — /// the owning task has already torn down the transport channel. @@ -868,7 +884,7 @@ impl PendingDialOutcome { Self::DialFailed { candidates_count } => { Err(P2PError::Network(NetworkError::PeerNotFound( format!( - "failed to dial {} at any of {} candidate address(es)", + "could not establish a channel to {} using {} candidate address(es)", peer_hex, candidates_count ) .into(), @@ -3494,8 +3510,8 @@ impl DhtNetworkManager { } /// Try dialing the bounded per-family plan chosen by - /// [`Self::select_dial_candidates`]. Returns the transport channel ID on - /// the first success, `None` if every attempted dial failed. + /// [`Self::select_dial_candidates`]. Distinguishes a failed transport dial + /// from a plan where every candidate was skipped without an attempt. /// /// The caller hands in typed pairs from a `DHTNode` (via /// [`DHTNode::typed_addresses`]) or a candidate list returned by @@ -3505,11 +3521,11 @@ impl DhtNetworkManager { /// /// Addresses that failed a dial within the last /// [`DIAL_FAILURE_CACHE_TTL`] are **not re-dialed**, but they still - /// consume one of the plan slots — a fully cached plan therefore - /// returns `None` without trying anything further down the priority - /// list. This stops a peer that republishes the same broken Direct / - /// Unverified / Lan set on every DHT query from causing a dial retry - /// every time we encounter them. + /// consume one of the plan slots — a fully cached plan therefore returns + /// [`DialAddressesOutcome::NoAttempt`] without trying anything further + /// down the priority list. This stops a peer that republishes the same + /// broken Direct / Unverified / Lan set on every DHT query from causing a + /// dial retry every time we encounter them. /// /// Bails out early when the peer is already connected — the caller /// would otherwise be paying N redundant `is_peer_connected` reads @@ -3518,13 +3534,13 @@ impl DhtNetworkManager { &self, peer_id: &PeerId, typed_addresses: &[(MultiAddr, AddressType)], - ) -> Option { + ) -> DialAddressesOutcome { if self.transport.is_peer_connected(peer_id).await { trace!( "dial_addresses: peer {} already connected, skipping dial", peer_id.to_hex() ); - return None; + return DialAddressesOutcome::NoAttempt; } let plan = self.contextual_dial_plan(typed_addresses).await; if plan.is_empty() { @@ -3532,12 +3548,11 @@ impl DhtNetworkManager { "dial_addresses: no dialable addresses for {}", peer_id.to_hex() ); - return None; + return DialAddressesOutcome::NoAttempt; } let mut attempted = 0usize; let mut skipped_cached = 0usize; for (addr, ty) in &plan { - attempted += 1; let Some(socket_addr) = addr.dialable_socket_addr() else { continue; }; @@ -3554,23 +3569,32 @@ impl DhtNetworkManager { ); continue; } + attempted += 1; match self.dial_candidate(peer_id, addr, *ty).await { Some(channel_id) => { self.dial_failure_cache.clear(&socket_addr); - return Some(channel_id); + return DialAddressesOutcome::Connected(channel_id); } None => { self.dial_failure_cache.record_failure(socket_addr, *ty); } } } + if attempted == 0 { + debug!( + "dial_addresses: no address dial attempted for {} ({} skipped from failure cache)", + peer_id.to_hex(), + skipped_cached + ); + return DialAddressesOutcome::NoAttempt; + } debug!( "dial_addresses: all {} attempted address(es) failed for {} ({} skipped from failure cache)", attempted, peer_id.to_hex(), skipped_cached ); - None + DialAddressesOutcome::AttemptedAndFailed } /// Return true when a FIND_NODE candidate has no useful dial attempt left @@ -3893,22 +3917,36 @@ impl DhtNetworkManager { candidates.len() ); - let Some(channel_id) = self.dial_addresses(peer_id, candidates).await else { - warn!( - "[STEP 1b] {} -> {}: dial failed for all {} candidate address(es)", - local_hex, - peer_hex, - candidates.len() - ); - self.record_peer_failure_weighted( - peer_id, - TRUST_REASON_DHT_DIAL_FAILED, - DHT_DIAL_FAILURE_TRUST_WEIGHT, - ) - .await; - return PendingDialOutcome::DialFailed { - candidates_count: candidates.len(), - }; + let channel_id = match self.dial_addresses(peer_id, candidates).await { + DialAddressesOutcome::Connected(channel_id) => channel_id, + DialAddressesOutcome::NoAttempt => { + debug!( + "[STEP 1b] {} -> {}: no dial attempted for {} candidate address(es)", + local_hex, + peer_hex, + candidates.len() + ); + return PendingDialOutcome::DialFailed { + candidates_count: candidates.len(), + }; + } + DialAddressesOutcome::AttemptedAndFailed => { + warn!( + "[STEP 1b] {} -> {}: dial failed for all {} candidate address(es)", + local_hex, + peer_hex, + candidates.len() + ); + self.record_peer_failure_weighted( + peer_id, + TRUST_REASON_DHT_DIAL_FAILED, + DHT_DIAL_FAILURE_TRUST_WEIGHT, + ) + .await; + return PendingDialOutcome::DialFailed { + candidates_count: candidates.len(), + }; + } }; let identity_timeout = self.config.request_timeout.min(IDENTITY_EXCHANGE_TIMEOUT); @@ -6245,6 +6283,63 @@ mod tests { ); } + #[tokio::test] + async fn cached_dial_skips_do_not_reduce_peer_trust() { + let identity = + Arc::new(crate::identity::node_identity::NodeIdentity::from_seed(&[91u8; 32]).unwrap()); + let transport = Arc::new( + crate::transport_handle::TransportHandle::new( + crate::transport_handle::TransportConfig { + listen_addrs: vec![MultiAddr::quic(SocketAddr::from(([127, 0, 0, 1], 0)))], + connection_timeout: Duration::from_millis(100), + max_connections: 16, + event_channel_capacity: 16, + max_message_size: None, + node_identity: identity, + user_agent: "cached-dial-regression".to_string(), + allow_loopback: true, + enable_relay_service: false, + advertise_external_addresses: false, + }, + ) + .await + .unwrap(), + ); + let trust_engine = Arc::new(TrustEngine::new()); + let mut config = DhtNetworkConfig::default(); + config.peer_id = transport.peer_id(); + config.node_config.allow_loopback = true; + let manager = DhtNetworkManager::new( + Arc::clone(&transport), + Some(Arc::clone(&trust_engine)), + config, + ) + .await + .unwrap(); + + let peer = pid(43); + let address = MultiAddr::quic("203.0.113.7:49001".parse().unwrap()); + let socket_addr = address.dialable_socket_addr().unwrap(); + let candidates = vec![(address, AddressType::Direct)]; + manager + .dial_failure_cache + .record_failure(socket_addr, AddressType::Direct); + + assert_eq!( + manager.dial_addresses(&peer, &candidates).await, + DialAddressesOutcome::NoAttempt + ); + for _ in 0..4 { + let outcome = manager + .run_owned_dial(&peer, &candidates, "local", "remote") + .await; + assert!(matches!(outcome, PendingDialOutcome::DialFailed { .. })); + } + assert_eq!(trust_engine.score(&peer), DEFAULT_NEUTRAL_TRUST); + + transport.stop().await.unwrap(); + } + fn bucket_refresh_candidate(index: usize, refresh_debt_secs: u64) -> BucketRefreshCandidate { let refresh_debt = Duration::from_secs(refresh_debt_secs); BucketRefreshCandidate { From 2140acc2c449e00b7a21eaa1de69f7ad0e0e116a Mon Sep 17 00:00:00 2001 From: Mick van Dijke <12992260+mickvandijke@users.noreply.github.com> Date: Thu, 27 Aug 2026 15:09:17 +0200 Subject: [PATCH 17/18] fix(dht): preserve quarantine hysteresis at marker cap --- docs/ROUTING_TABLE_DESIGN.md | 8 +- docs/SECURITY_MODEL.md | 5 +- ...t-quarantine-and-trust-neutral-requests.md | 12 +- src/dht/core_engine.rs | 175 ++++++++++++------ 4 files changed, 138 insertions(+), 62 deletions(-) diff --git a/docs/ROUTING_TABLE_DESIGN.md b/docs/ROUTING_TABLE_DESIGN.md index 4a570a00..39402822 100644 --- a/docs/ROUTING_TABLE_DESIGN.md +++ b/docs/ROUTING_TABLE_DESIGN.md @@ -186,7 +186,7 @@ When a candidate peer `P` with `NodeInfo` and IP address `candidate_ip` is prese 2. **Address check**: If `P.addresses` is empty, reject. 3. **Authentication check**: If `P` has not completed transport-level authentication, reject. 4. **Update short-circuit**: If `P` already exists in `KBucket(BucketIndex(self, P))`, merge addresses (Section 6.3), refresh `last_seen`, move `P` to tail, and return. The peer already holds its slot — new-peer trust admission, IP diversity, and capacity checks are skipped. -5. **New-peer trust admission check**: If `P` carries an explicit quarantine marker, reject until `TrustScore(self, P) >= QUARANTINE_READMIT_THRESHOLD`. Otherwise reject only when `TrustScore(self, P) < QUARANTINE_THRESHOLD`. +5. **New-peer trust admission check**: If `P` carries an explicit quarantine marker, reject until `TrustScore(self, P) >= QUARANTINE_READMIT_THRESHOLD`. Otherwise reject only when `TrustScore(self, P) < QUARANTINE_THRESHOLD`, unless the bounded exact-marker set has overflowed; overflow fails closed by requiring `QUARANTINE_READMIT_THRESHOLD` for unmarked candidates too. 6. **Loopback check**: If `candidate_ip` is loopback and loopback is disallowed, reject. If loopback is allowed, skip all IP diversity checks (step 7–9) and proceed directly to insertion/capacity handling. 7. **Non-IP transport bypass**: If `P` has no IP-based address (e.g., Bluetooth, LoRa), skip IP diversity checks and proceed directly to insertion/capacity handling. 8. **IP diversity enforcement** (under write lock — Invariant 10): @@ -265,7 +265,8 @@ When any interaction records a trust failure and `TrustScore(self, P)` drops bel 4. Do not re-admit quarantined `P` until `TrustScore(self, P) >= QUARANTINE_READMIT_THRESHOLD`. 5. If removal would shrink `LocalRT(self)` below K peers, keep `P` in the routing table until another peer is admitted and the same eviction can happen without underfilling the table. 6. If `P` is not in the K-closest-to-self set, it may remain in the routing table, but local lookup result selection, FIND_NODE responses, and automatic lookup/dial paths MUST avoid it while `TrustScore(self, P) < QUARANTINE_THRESHOLD`. -7. If `P` has `QUARANTINE_THRESHOLD <= TrustScore(self, P) < QUARANTINE_READMIT_THRESHOLD`, it may remain in or enter the routing table unless it carries an explicit quarantine marker. A marked peer remains rejected until trust reaches `QUARANTINE_READMIT_THRESHOLD`. +7. If `P` has `QUARANTINE_THRESHOLD <= TrustScore(self, P) < QUARANTINE_READMIT_THRESHOLD`, it may remain in or enter the routing table unless it carries an explicit quarantine marker or exact-marker overflow has put the engine into fail-closed mode. A marked peer remains rejected until trust reaches `QUARANTINE_READMIT_THRESHOLD`. +8. Exact quarantine markers are capped at `u16::MAX` (65,535). New insertions prune recovered markers in bounded round-robin batches; an insertion at the cap performs a full recovery sweep. Saturation after that sweep sets a sticky fail-closed overflow marker. Quarantine is a routing-table and automatic lookup policy. It is not a blanket transport-level block for explicit user-initiated sends. @@ -702,8 +703,9 @@ Each scenario should assert exact expected outcomes and state transitions. - Candidate with zero addresses. Rejected with error. Routing table unchanged. 4. **New peer admission and quarantine readmission thresholds**: - - A new, unmarked peer with `TrustScore < QUARANTINE_THRESHOLD` is rejected. An unmarked peer in `[QUARANTINE_THRESHOLD, QUARANTINE_READMIT_THRESHOLD)` is admitted normally. + - A new, unmarked peer with `TrustScore < QUARANTINE_THRESHOLD` is rejected. Before marker overflow, an unmarked peer in `[QUARANTINE_THRESHOLD, QUARANTINE_READMIT_THRESHOLD)` is admitted normally. - A peer carrying an explicit quarantine marker is rejected until `TrustScore >= QUARANTINE_READMIT_THRESHOLD`. + - After exact-marker overflow, an unmarked peer below `QUARANTINE_READMIT_THRESHOLD` is also rejected; neutral unknown peers remain admissible. - Existing routing-table peers remain eligible for address/liveness updates and may later move into the K-closest set. 5. **Bucket-full rejection (no stale peers)**: diff --git a/docs/SECURITY_MODEL.md b/docs/SECURITY_MODEL.md index 61eac213..c49616ac 100644 --- a/docs/SECURITY_MODEL.md +++ b/docs/SECURITY_MODEL.md @@ -99,8 +99,9 @@ better candidates need the slot. Peers already in the routing table at or above the quarantine threshold but below the readmission threshold may remain in the table, including after moving into the K-closest set. New routing-table admissions require the quarantine -threshold; only peers carrying an explicit quarantine marker require the higher -readmission threshold. +threshold. Peers carrying an explicit quarantine marker require the higher +readmission threshold; if the bounded exact-marker set has overflowed, unmarked +admissions also fail closed against that higher threshold. ### Quarantine Reasons diff --git a/docs/adr/ADR-016-trust-quarantine-and-trust-neutral-requests.md b/docs/adr/ADR-016-trust-quarantine-and-trust-neutral-requests.md index 96ae0cf0..245a6273 100644 --- a/docs/adr/ADR-016-trust-quarantine-and-trust-neutral-requests.md +++ b/docs/adr/ADR-016-trust-quarantine-and-trust-neutral-requests.md @@ -45,7 +45,7 @@ Defined in `src/adaptive/dht.rs:33-40` (mirrored as documentation constants in ` |-----------|---------|---------| | `swap_threshold` | **0.35** | Peer becomes *eligible for lazy swap-out* — replaced only when a better routing-table candidate arrives. Never causes eviction on its own. | | `quarantine_threshold` | **0.20** | *Automatic avoidance*: lookup/dial machinery stops selecting the peer. The peer is not removed and explicit sends still work. | -| `quarantine_readmit_threshold` | **0.45** | *Readmission hysteresis*: only a peer carrying an explicit quarantine marker must recover to this score before readmission. | +| `quarantine_readmit_threshold` | **0.45** | *Readmission hysteresis*: a peer carrying an explicit quarantine marker must recover to this score; unmarked peers must also do so after exact-marker overflow. | **Validation** (`AdaptiveDhtConfig::validate`, `src/adaptive/dht.rs:84-138`): all three must be finite and in `[0.0, 0.5)`; when quarantine is active, `quarantine_readmit_threshold >= quarantine_threshold`; when swap and quarantine are both non-zero, `swap_threshold > quarantine_threshold` (swap is a milder condition than avoidance, quarantine is strictly more severe). The quarantine pair is re-checked at the engine boundary in `set_trust_quarantine_thresholds`; `swap_threshold` is validated only by `AdaptiveDhtConfig::validate`. Values at or above neutral `0.5` are rejected because decay approaches neutral asymptotically: negatively observed peers would remain swap/quarantine-eligible indefinitely, while readmission from below at a neutral cutoff would be unreachable in finite time. Invalid config fails node construction (`AdaptiveDHT::new` returns `Err`). @@ -56,8 +56,9 @@ Defined in `src/adaptive/dht.rs:33-40` (mirrored as documentation constants in ` `check_new_peer_admission` (`src/dht/core_engine.rs:1644-1673`) is called **only when the peer is not already in the routing table** (`add_node`, `src/dht/core_engine.rs:2443-2446`; `re_evaluate_admission`, `src/dht/core_engine.rs:3069-3072`). Non-finite trust scores are rejected defensively. Consequences: - An existing routing-table peer whose score sits in `[0.20, 0.45)` **stays in the table** and may move into the close group (close-group membership is pure XOR distance over table contents). It remains eligible for lazy swap-out below 0.35 and is skipped by automatic selection below 0.20 — but it is never ejected merely for the band it occupies. -- An unmarked peer is admitted and remains eligible for automatic lookup at or above 0.20. A transient failure cannot turn its absence from one observer's routing table into an implicit quarantine. +- Before exact-marker overflow, an unmarked peer is admitted and remains eligible for automatic lookup at or above 0.20. A transient failure cannot turn its absence from one observer's routing table into an implicit quarantine. - A marked peer remains blocked below 0.45. Admission at or above 0.45 clears its quarantine marker (`forget_quarantined_peer`) — this is the explicit readmission point. +- After exact-marker overflow, unmarked candidates also remain blocked below 0.45 so discarded marker history cannot become a readmission bypass. - `should_avoid_automatic_candidate` applies the same score/marker policy as `should_avoid_for_lookup`; routing-table absence by itself no longer raises the applicable threshold. ### 4. Stale-revalidation concurrency invariant @@ -86,7 +87,7 @@ Immediate eviction of below-0.20 close-group peers is enabled. `enforce_close_gr ### 7. Bounded quarantine markers -Quarantine markers (`quarantined_peers: HashSet` plus FIFO `quarantined_peer_order`) are bounded at `MAX_QUARANTINED_PEERS = 8192` (`src/dht/core_engine.rs:229`). The key insight making the bound safe (`quarantine_marker_required_for_score`, `src/dht/core_engine.rs:1690-1696`): **a marker is only semantically required while a peer's score is in `[quarantine_threshold, readmit_threshold)`** — below 0.20 the score itself keeps the peer avoided; at/above 0.45 the peer is readmittable and the marker is cleared. When the set is full, `prune_redundant_quarantined_peers` drops (oldest-first) any marker whose current score no longer requires one; if the set is still full, the new marker is simply not inserted — the score-based avoidance clause covers the peer regardless. +Quarantine markers (`quarantined_peers: HashSet` plus round-robin `quarantined_peer_order`) are bounded at `MAX_QUARANTINED_PEERS = u16::MAX` (65,535). A marker remains semantically required at every score below the readmit threshold: dropping a marker while its peer is below 0.20 would let lazy time decay later carry the unmarked peer into the 0.20–0.45 admission band. Each new marker insertion checks a bounded batch of existing markers and removes peers that reached 0.45; an insertion at the hard cap scans the full set before deciding that no exact slot is available. If all 65,535 markers are still required, a sticky overflow flag makes unmarked candidates fail closed against the 0.45 readmission threshold. The flag cannot safely clear merely because exact slots later become available, because the engine can no longer identify the peer whose marker overflowed. ### 8. Decay and recovery via rediscovery @@ -123,7 +124,7 @@ The division of labour: **layers that can judge, report; layers that can't, stay 4. **No transport block**: quarantine affects only routing-table membership and automatic selection; explicit sends always go through. 5. **Wire stability**: trust state never leaks into serialized DHT messages; filtering is strictly local policy. 6. **Exactly-once DHT scoring**: one failed DHT request produces exactly one trust penalty (dial 2.25 *or* RPC 1.0 *or* identity-exchange 1.0 — never stacked for the same attempt). -7. **Marker sufficiency**: quarantine markers are required only for scores in `[quarantine, readmit)`; outside that band the score alone determines behaviour, which is what makes the 8192 bound safe. +7. **Marker sufficiency**: an exact quarantine marker remains required until its peer reaches the readmit threshold; if the 65,535-entry exact set overflows, a sticky fail-closed state applies the readmit threshold to unmarked candidates. 8. **Penalty-only core**: saorsa-core never auto-rewards; positive signals come exclusively from consumers (capped at weight 5.0). 9. **Trust-neutral generic transport**: `send_request`/`send_message` never report trust events; only application-aware layers do. @@ -136,12 +137,13 @@ The division of labour: **layers that can judge, report; layers that can't, stay - **No misattributed penalties from generic transport**: honest peers are no longer punished for congestion, slow handlers, or the local node's own overload — and application penalties are no longer doubled by transport-layer penalties for the same exchange. - **Calibrated avoidance**: the 2.25 dial weight gives persistent unreachability a concrete, documented time-to-avoidance (four spaced failures over six hours from neutral), rather than an emergent accident of unit weights. - **Fully backward compatible on the wire**: mixed-version networks work; old nodes see identical messages. -- **Bounded memory**: quarantine bookkeeping cannot grow past 8192 entries, and dropping markers under pressure degrades gracefully to score-only avoidance. +- **Bounded memory with fail-closed overflow**: exact quarantine bookkeeping cannot grow past 65,535 entries; saturation raises the admission requirement for unmarked candidates instead of forgetting readmission history. - **Operators can turn it off**: `trust_enforcement(false)` gives observe-only mode (scores tracked, nothing enforced) for diagnosis or staged rollout. ### Negative - **Close-group churn risk**: transiently driving an honest peer below 0.20 can immediately evict it when surplus capacity exists and hold it out until its score recovers to 0.45. +- **Conservative saturation mode**: after exact-marker overflow, unmarked candidates below 0.45 are treated as potentially quarantined for the rest of the engine lifetime. - **Applications now own attribution**: any consumer that relied on `send_request`'s automatic penalties gets no trust signal for its request failures unless it explicitly reports a justified outcome. Silent trust erosion of misbehaving peers via generic requests no longer happens. - **Recovery latency is fixed by decay**: a wrongly-penalized peer needs up to ~46 h (worst case) to become admissible again; there is no active-probe fast path. - **More configuration surface**: three interdependent thresholds with ordering rules; invalid combinations fail node construction (loudly, by design). diff --git a/src/dht/core_engine.rs b/src/dht/core_engine.rs index 551f7f7b..70ac3162 100644 --- a/src/dht/core_engine.rs +++ b/src/dht/core_engine.rs @@ -220,10 +220,16 @@ const DEFAULT_QUARANTINE_READMIT_THRESHOLD: f64 = 0.45; /// Maximum number of evicted quarantine markers retained by the routing engine. /// -/// Markers are only semantically required while a peer has recovered above the -/// quarantine threshold but remains below the readmit threshold; below the -/// quarantine threshold the trust score itself keeps the peer avoided. -const MAX_QUARANTINED_PEERS: usize = 8192; +/// A marker remains semantically required until the peer reaches the readmit +/// threshold: dropping it while the score is still below the quarantine +/// threshold would let time decay later bypass readmission hysteresis. +const MAX_QUARANTINED_PEERS: usize = u16::MAX as usize; + +/// Maximum existing quarantine markers checked for recovery on an ordinary +/// insertion. A bounded round-robin sweep avoids making a run of new marker +/// insertions quadratic; an insertion at the hard cap scans the full set so it +/// can reclaim every recovered slot before failing closed. +const QUARANTINE_MARKER_PRUNE_BATCH_SIZE: usize = 64; #[derive(Clone, Copy, Debug, Eq, PartialEq)] struct ClosestNodeCandidate { @@ -1538,13 +1544,21 @@ pub struct DhtCoreEngine { quarantine_readmit_threshold: f64, /// Peers evicted from the close group by quarantine. Markers are bounded - /// to cap memory use; redundant markers are removed when direct trust-score - /// checks are sufficient or when the peer crosses the readmit threshold. + /// to cap memory use and removed once the peer crosses the readmit + /// threshold. quarantined_peers: HashSet, - /// FIFO order used to prune the bounded quarantine marker set. + /// Round-robin order used to prune the bounded quarantine marker set. quarantined_peer_order: VecDeque, + /// Whether an exact quarantine marker could not be retained at capacity. + /// + /// This is sticky because, after an ID is discarded, a later reduction in + /// the exact-marker count cannot prove that the discarded peer reached the + /// readmit threshold. While set, unmarked peers fail closed against the + /// readmit threshold rather than bypassing hysteresis through time decay. + quarantine_marker_overflowed: bool, + /// Duration of no contact after which a peer is considered stale. /// Defaults to [`LIVE_THRESHOLD`]; overridden in tests to avoid /// `Instant` subtraction overflow on Windows (where `Instant` starts @@ -1594,6 +1608,7 @@ impl DhtCoreEngine { quarantine_readmit_threshold: DEFAULT_QUARANTINE_READMIT_THRESHOLD, quarantined_peers: HashSet::new(), quarantined_peer_order: VecDeque::new(), + quarantine_marker_overflowed: false, live_threshold: LIVE_THRESHOLD, shutdown: CancellationToken::new(), }) @@ -1648,7 +1663,8 @@ impl DhtCoreEngine { peer_id.to_hex() )); } - if self.quarantined_peers.contains(peer_id) { + let has_exact_marker = self.quarantined_peers.contains(peer_id); + if has_exact_marker || self.quarantine_marker_overflowed { if trust_score < self.quarantine_readmit_threshold { return Err(anyhow!( "peer {} quarantined until trust >= {:.3} (current {trust_score:.3})", @@ -1656,7 +1672,9 @@ impl DhtCoreEngine { self.quarantine_readmit_threshold )); } - self.forget_quarantined_peer(peer_id); + if has_exact_marker { + self.forget_quarantined_peer(peer_id); + } return Ok(()); } if trust_score < self.quarantine_threshold { @@ -1675,81 +1693,76 @@ impl DhtCoreEngine { peer_id: PeerId, trust_score: &impl Fn(&PeerId) -> f64, ) { - let quarantine_threshold = self.quarantine_threshold; let quarantine_readmit_threshold = self.quarantine_readmit_threshold; - Self::remember_quarantined_peer_with_trust( + let overflowed = Self::remember_quarantined_peer_with_trust( &mut self.quarantined_peers, &mut self.quarantined_peer_order, - quarantine_threshold, quarantine_readmit_threshold, peer_id, trust_score, ); + self.quarantine_marker_overflowed |= overflowed; } - fn quarantine_marker_required_for_score( - score: f64, - quarantine_threshold: f64, - quarantine_readmit_threshold: f64, - ) -> bool { - score.is_finite() && score >= quarantine_threshold && score < quarantine_readmit_threshold + fn quarantine_marker_recovered(score: f64, quarantine_readmit_threshold: f64) -> bool { + score.is_finite() && score >= quarantine_readmit_threshold } - fn prune_redundant_quarantined_peers( + fn prune_recovered_quarantined_peers( quarantined_peers: &mut HashSet, quarantined_peer_order: &mut VecDeque, - quarantine_threshold: f64, quarantine_readmit_threshold: f64, trust_score: &impl Fn(&PeerId) -> f64, + scan_limit: usize, ) { - let mut retained_order = VecDeque::with_capacity(quarantined_peer_order.len()); - while let Some(peer_id) = quarantined_peer_order.pop_front() { + let scan_count = quarantined_peer_order.len().min(scan_limit); + for _ in 0..scan_count { + let Some(peer_id) = quarantined_peer_order.pop_front() else { + break; + }; if !quarantined_peers.contains(&peer_id) { continue; } let score = trust_score(&peer_id); - if Self::quarantine_marker_required_for_score( - score, - quarantine_threshold, - quarantine_readmit_threshold, - ) { - retained_order.push_back(peer_id); - } else { + if Self::quarantine_marker_recovered(score, quarantine_readmit_threshold) { quarantined_peers.remove(&peer_id); + } else { + quarantined_peer_order.push_back(peer_id); } } - - *quarantined_peer_order = retained_order; } fn remember_quarantined_peer_with_trust( quarantined_peers: &mut HashSet, quarantined_peer_order: &mut VecDeque, - quarantine_threshold: f64, quarantine_readmit_threshold: f64, peer_id: PeerId, trust_score: &impl Fn(&PeerId) -> f64, - ) { + ) -> bool { if quarantined_peers.contains(&peer_id) { - return; + return false; } + let prune_scan_limit = if quarantined_peers.len() >= MAX_QUARANTINED_PEERS { + quarantined_peer_order.len() + } else { + QUARANTINE_MARKER_PRUNE_BATCH_SIZE + }; + Self::prune_recovered_quarantined_peers( + quarantined_peers, + quarantined_peer_order, + quarantine_readmit_threshold, + trust_score, + prune_scan_limit, + ); if quarantined_peers.len() >= MAX_QUARANTINED_PEERS { - Self::prune_redundant_quarantined_peers( - quarantined_peers, - quarantined_peer_order, - quarantine_threshold, - quarantine_readmit_threshold, - trust_score, - ); - } - if quarantined_peers.len() >= MAX_QUARANTINED_PEERS { - return; + return true; } quarantined_peers.insert(peer_id); quarantined_peer_order.push_back(peer_id); + false } fn forget_quarantined_peer(&mut self, peer_id: &PeerId) { @@ -1770,7 +1783,7 @@ impl DhtCoreEngine { return true; } trust_score < self.quarantine_threshold - || (self.quarantined_peers.contains(peer_id) + || ((self.quarantine_marker_overflowed || self.quarantined_peers.contains(peer_id)) && trust_score < self.quarantine_readmit_threshold) } @@ -1824,10 +1837,9 @@ impl DhtCoreEngine { let quarantine_threshold = self.quarantine_threshold; let quarantine_readmit_threshold = self.quarantine_readmit_threshold; - Self::remember_quarantined_peer_with_trust( + let overflowed = Self::remember_quarantined_peer_with_trust( &mut self.quarantined_peers, &mut self.quarantined_peer_order, - quarantine_threshold, quarantine_readmit_threshold, *peer_id, &|id| { @@ -1838,6 +1850,7 @@ impl DhtCoreEngine { } }, ); + self.quarantine_marker_overflowed |= overflowed; routing.remove_node(peer_id); let k_after = routing.k_closest_ids(self.k_value); @@ -1883,16 +1896,15 @@ impl DhtCoreEngine { break; }; - let quarantine_threshold = self.quarantine_threshold; let quarantine_readmit_threshold = self.quarantine_readmit_threshold; - Self::remember_quarantined_peer_with_trust( + let overflowed = Self::remember_quarantined_peer_with_trust( &mut self.quarantined_peers, &mut self.quarantined_peer_order, - quarantine_threshold, quarantine_readmit_threshold, peer_id, trust_score, ); + self.quarantine_marker_overflowed |= overflowed; routing.remove_node(&peer_id); removed.push(peer_id); } @@ -3091,6 +3103,10 @@ impl std::fmt::Debug for DhtCoreEngine { &self.quarantine_readmit_threshold, ) .field("quarantined_peers", &self.quarantined_peers.len()) + .field( + "quarantine_marker_overflowed", + &self.quarantine_marker_overflowed, + ) .finish() } } @@ -5549,7 +5565,37 @@ mod tests { } #[test] - fn test_quarantined_peer_marker_cap_keeps_readmit_gap_markers() { + fn test_new_quarantine_marker_prunes_only_recovered_markers() { + let mut dht = DhtCoreEngine::new_for_tests(PeerId::from_bytes([0u8; 32])).unwrap(); + dht.set_trust_quarantine_thresholds( + TEST_QUARANTINE_THRESHOLD, + TEST_QUARANTINE_READMIT_THRESHOLD, + ) + .unwrap(); + let below_quarantine_peer = peer_id_from_index(1); + let recovered_peer = peer_id_from_index(2); + let new_peer = peer_id_from_index(3); + + dht.remember_quarantined_peer(below_quarantine_peer, &|_| TEST_QUARANTINE_LOW_SCORE); + dht.remember_quarantined_peer(recovered_peer, &|_| TEST_QUARANTINE_LOW_SCORE); + dht.remember_quarantined_peer(new_peer, &|peer_id| { + if *peer_id == recovered_peer { + TEST_QUARANTINE_READMIT_THRESHOLD + } else { + TEST_QUARANTINE_LOW_SCORE + } + }); + + assert!(dht.quarantined_peers.contains(&below_quarantine_peer)); + assert!( + !dht.quarantined_peers.contains(&recovered_peer), + "new marker insertion should prune peers that reached the readmit threshold" + ); + assert!(dht.quarantined_peers.contains(&new_peer)); + } + + #[test] + fn test_quarantined_peer_marker_cap_prunes_recovered_and_fails_closed() { let mut dht = DhtCoreEngine::new_for_tests(PeerId::from_bytes([0u8; 32])).unwrap(); dht.set_trust_quarantine_thresholds( TEST_QUARANTINE_THRESHOLD, @@ -5561,6 +5607,7 @@ mod tests { let recovered_peer = oldest_peer; let retained_overflow_peer = peer_id_from_index(MAX_QUARANTINED_PEERS + 1); + assert_eq!(MAX_QUARANTINED_PEERS, usize::from(u16::MAX)); for index in 0..MAX_QUARANTINED_PEERS { dht.remember_quarantined_peer(peer_id_from_index(index), &|_| { TEST_QUARANTINE_MARKER_REQUIRED_SCORE @@ -5585,10 +5632,34 @@ mod tests { ); assert!( !dht.quarantined_peers.contains(&low_trust_overflow_peer), - "a below-quarantine peer remains avoided by score and can be dropped when cap is full" + "the exact-marker set should remain bounded when every retained peer still requires hysteresis" ); + assert!(dht.quarantine_marker_overflowed); assert!(dht.should_avoid_for_lookup(&oldest_peer, TEST_QUARANTINE_MARKER_REQUIRED_SCORE)); assert!(dht.should_avoid_for_lookup(&low_trust_overflow_peer, TEST_QUARANTINE_LOW_SCORE)); + assert!( + dht.should_avoid_for_lookup( + &low_trust_overflow_peer, + TEST_QUARANTINE_MARKER_REQUIRED_SCORE + ), + "overflow must remain fail-closed after an untracked peer decays above the quarantine threshold" + ); + assert!( + dht.check_new_peer_admission( + &low_trust_overflow_peer, + TEST_QUARANTINE_MARKER_REQUIRED_SCORE + ) + .is_err(), + "an untracked overflow peer must not bypass the readmit threshold" + ); + assert!( + dht.check_new_peer_admission( + &low_trust_overflow_peer, + TEST_QUARANTINE_READMIT_THRESHOLD + ) + .is_ok(), + "fail-closed overflow should still admit peers at the readmit threshold" + ); dht.remember_quarantined_peer(retained_overflow_peer, &|peer_id| { if *peer_id == recovered_peer { From 764854607c1438bcd7d2c59c23b3f2aa8127b9be Mon Sep 17 00:00:00 2001 From: Mick van Dijke <12992260+mickvandijke@users.noreply.github.com> Date: Thu, 27 Aug 2026 15:42:49 +0200 Subject: [PATCH 18/18] docs(dht): align trust quarantine policy --- docs/ROUTING_TABLE_DESIGN.md | 4 ++-- docs/adr/README.md | 2 +- docs/trust-signals-api.md | 7 ++++--- src/adaptive/dht.rs | 7 ++++--- src/dht_network_manager.rs | 6 ++++-- src/network.rs | 6 ++++-- 6 files changed, 19 insertions(+), 13 deletions(-) diff --git a/docs/ROUTING_TABLE_DESIGN.md b/docs/ROUTING_TABLE_DESIGN.md index 39402822..a875f9d3 100644 --- a/docs/ROUTING_TABLE_DESIGN.md +++ b/docs/ROUTING_TABLE_DESIGN.md @@ -58,7 +58,7 @@ All parameters are configurable. Values below are a reference profile used for l | `TRUST_PROTECTION_THRESHOLD` | Trust score above which a peer resists swap-closer eviction | `0.7` | | `SWAP_THRESHOLD` | Trust score below which a peer is eligible for replacement when a better candidate needs the slot | `0.35` | | `QUARANTINE_THRESHOLD` | Trust score below which automatic lookup/dial paths avoid the peer, and close-group peers are evicted when the routing table can retain at least K peers | `0.20` | -| `QUARANTINE_READMIT_THRESHOLD` | Trust score required for new routing-table admission/readmission after quarantine | `0.45` | +| `QUARANTINE_READMIT_THRESHOLD` | Trust score required for explicitly quarantined-peer readmission, and fail-closed unmarked admission after marker overflow | `0.45` | | `EMA_ALPHA` | EMA smoothing factor — weight of each new observation (higher = faster response) | `0.124` | | `DECAY_LAMBDA` | Per-second exponential decay rate toward neutral (0.5) | `1.394e-5` | | `SELF_LOOKUP_INTERVAL` | Periodic self-lookup cadence (maintenance phase only; bootstrap self-lookups run back-to-back with no interval) | random in `[5 min, 10 min]` | @@ -130,7 +130,7 @@ Note: `K_BUCKET_SIZE` values below 4 produce degenerate behavior (single-peer ro 4. **Address requirement**: A `NodeInfo` with an empty address list MUST NOT be admitted to the routing table. 5. **Authenticated membership**: Only peers that have completed transport-level authentication are eligible for routing table insertion. Unauthenticated peers MUST NOT enter `LocalRT`. 6. **IP diversity**: No enforcement scope (per-bucket or routing-neighborhood) may exceed `IP_EXACT_LIMIT` nodes per exact IP or `IP_SUBNET_LIMIT` nodes per subnet, except via explicit loopback or testnet overrides. -7. **Trust quarantine and admission**: Peers with `TrustScore(self, P) < QUARANTINE_THRESHOLD` MUST be skipped by local lookup result selection, FIND_NODE responses, and automatic lookup/dial candidate selection. If such a peer is in the K-closest-to-self set, it MUST be evicted and quarantined until `TrustScore(self, P) >= QUARANTINE_READMIT_THRESHOLD` whenever eviction leaves `LocalRT(self)` with at least K peers. If eviction would shrink `LocalRT(self)` below K peers, the peer remains in the table but is still avoided by automatic lookup policy. Any new routing-table peer MUST have `TrustScore(self, P) >= QUARANTINE_READMIT_THRESHOLD`; existing routing-table peers between the two thresholds may remain in the table, including after moving into the K-closest set. +7. **Trust quarantine and admission**: Peers with `TrustScore(self, P) < QUARANTINE_THRESHOLD` MUST be skipped by local lookup result selection, FIND_NODE responses, and automatic lookup/dial candidate selection. If such a peer is in the K-closest-to-self set, it MUST be evicted and quarantined until `TrustScore(self, P) >= QUARANTINE_READMIT_THRESHOLD` whenever eviction leaves `LocalRT(self)` with at least K peers. If eviction would shrink `LocalRT(self)` below K peers, the peer remains in the table but is still avoided by automatic lookup policy. New unmarked routing-table peers MUST have `TrustScore(self, P) >= QUARANTINE_THRESHOLD`; peers carrying an explicit quarantine marker MUST reach `QUARANTINE_READMIT_THRESHOLD`. If exact quarantine-marker storage has overflowed, unmarked admissions also MUST reach `QUARANTINE_READMIT_THRESHOLD`. Existing routing-table peers between the two thresholds may remain in the table, including after moving into the K-closest set. 8. **Trust protection (staleness-gated)**: A peer with `TrustScore(self, P) >= TRUST_PROTECTION_THRESHOLD` **AND** `last_seen` within `LIVE_THRESHOLD` MUST NOT be evicted by swap-closer admission. A peer whose `last_seen` exceeds `LIVE_THRESHOLD` receives no trust protection regardless of score — stale peers MUST NOT hold slots against live candidates. 9. **Deterministic distance**: `Distance(A, B)` is symmetric, deterministic, and consistent across all nodes. Two nodes compute the same distance between the same pair of keys. 10. **Atomic admission**: IP diversity checks, capacity checks, swap-closer evictions, trust score reads, and insertion MUST execute within a single exclusive admission critical section to prevent TOCTOU races. Implementations may use a routing-table write lock or an outer DHT-engine write guard that serializes admission. diff --git a/docs/adr/README.md b/docs/adr/README.md index a79e256f..76abbf18 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -35,7 +35,7 @@ An Architecture Decision Record (ADR) is a document that captures an important a | [ADR-006](./ADR-006-eigentrust-reputation.md) | EigenTrust Reputation System | Accepted | Iterative trust computation for Sybil resistance | | [ADR-009](./ADR-009-sybil-protection.md) | Sybil Protection Mechanisms | Accepted | Multi-layered defense against identity attacks | | [ADR-010](./ADR-010-entangled-attestation.md) | Entangled Attestation System | Accepted | Software integrity verification via attestation chains | -| [ADR-016](./ADR-016-trust-quarantine-and-trust-neutral-requests.md) | Trust Quarantine Thresholds and Trust-Neutral Request Transport | Proposed | Three-threshold DHT trust quarantine (0.35 swap / 0.20 avoidance / 0.45 admission) with trust-neutral generic request transport | +| [ADR-016](./ADR-016-trust-quarantine-and-trust-neutral-requests.md) | Trust Quarantine Thresholds and Trust-Neutral Request Transport | Proposed | Three-threshold DHT trust quarantine (0.35 swap / 0.20 avoidance and unmarked admission / 0.45 quarantined readmission) with trust-neutral generic request transport | ### Network Intelligence diff --git a/docs/trust-signals-api.md b/docs/trust-signals-api.md index e702a11f..3761ff6e 100644 --- a/docs/trust-signals-api.md +++ b/docs/trust-signals-api.md @@ -92,9 +92,10 @@ The routing table uses three trust thresholds: quarantined when the routing table can retain at least K peers. - `quarantine_readmit_threshold` (`0.45` by default): a quarantined peer can only re-enter through normal discovery/admission after its decayed trust - reaches this score. New peers must also meet this threshold before entering - the routing table. Existing routing-table peers between `0.20` and `0.45` - may remain in the table, including after moving into the close group. + reaches this score. Unmarked new peers normally need only meet + `quarantine_threshold`; after exact quarantine-marker overflow they also fail + closed at this threshold. Existing routing-table peers between `0.20` and + `0.45` may remain in the table, including after moving into the close group. ```rust use saorsa_core::AdaptiveDhtConfig; diff --git a/src/adaptive/dht.rs b/src/adaptive/dht.rs index da864f72..0f796682 100644 --- a/src/adaptive/dht.rs +++ b/src/adaptive/dht.rs @@ -252,9 +252,10 @@ impl AdaptiveDHT { /// /// Trust scores are updated immediately. Peers below the quarantine /// threshold are avoided by lookup result selection and automatic - /// lookup/dial paths. Immediate close-group eviction is temporarily - /// disabled until trust scoring is stable; low-trust peers remain eligible - /// for lazy swap-out when better candidates arrive. + /// lookup/dial paths. Close-group peers are immediately evicted when the + /// routing table has enough surplus to retain at least K peers; otherwise + /// they remain in the table but are still avoided. Low-trust peers also + /// remain eligible for lazy swap-out when better candidates arrive. pub async fn report_trust_event(&self, peer_id: &PeerId, event: TrustEvent) { match event { TrustEvent::ApplicationSuccess(weight) | TrustEvent::ApplicationFailure(weight) => { diff --git a/src/dht_network_manager.rs b/src/dht_network_manager.rs index b6c36652..4aeeddce 100644 --- a/src/dht_network_manager.rs +++ b/src/dht_network_manager.rs @@ -644,8 +644,10 @@ pub struct DhtNetworkConfig { /// close-group peers are evicted when the routing table has surplus above K. /// Default: [`AdaptiveDhtConfig::default`]. pub quarantine_threshold: f64, - /// Trust score required before a new peer can enter the routing table, - /// and before a quarantined peer can be admitted again. + /// Trust score required before an explicitly quarantined peer can re-enter + /// the routing table. New unmarked peers normally use + /// `quarantine_threshold`; after exact quarantine-marker overflow they + /// fail closed against this readmission threshold too. /// Default: [`AdaptiveDhtConfig::default`]. pub quarantine_readmit_threshold: f64, } diff --git a/src/network.rs b/src/network.rs index f4341495..c60daa0f 100644 --- a/src/network.rs +++ b/src/network.rs @@ -573,8 +573,10 @@ impl NodeConfigBuilder { /// When `true` (the default), the default adaptive DHT policy applies: /// peers below the swap threshold (0.35) become eligible for replacement, /// peers below the quarantine threshold (0.20) are avoided by automatic - /// lookup/dial paths, and new routing-table peers must meet the readmission - /// threshold (0.45). + /// lookup/dial paths, unmarked new routing-table peers must meet the + /// quarantine threshold (0.20), and explicitly quarantined peers must meet + /// the readmission threshold (0.45). If exact quarantine-marker storage + /// overflows, unmarked admissions also fail closed at 0.45. /// /// For fine-grained control over these thresholds, use /// [`adaptive_dht_config`](Self::adaptive_dht_config) instead.