From 83f23951ece892f0cfb5bd6d0c01c2da4d7c8995 Mon Sep 17 00:00:00 2001 From: fnyaker Date: Wed, 14 Oct 2026 17:33:29 +0200 Subject: [PATCH 1/2] Routing: every node insists on a link with its eight nearest neighbours Co-Authored-By: Claude Sonnet 5.5 --- BUGSVULNS.MD | 40 ++++++++++++++++ Docs/Architecture/routing.md | 18 +++++++- Docs/Architecture/scale.md | 2 +- TEST.md | 18 ++++++++ pyproject.toml | 2 +- src/mesh/constants.py | 4 ++ src/node.py | 23 +++++++++- src/version.py | 2 +- tests/test_kad_bootstrap.py | 2 + tests/test_neighborhood.py | 79 ++++++++++++++++++++++++++++++++ tests/tools/fake_oqs.py | 89 ++++++++++++++++++++++++++++++++++++ tests/tools/mesh_sim.py | 42 +++++++++++++++-- 12 files changed, 310 insertions(+), 11 deletions(-) create mode 100644 tests/tools/fake_oqs.py diff --git a/BUGSVULNS.MD b/BUGSVULNS.MD index 1bfb812..319f4e7 100644 --- a/BUGSVULNS.MD +++ b/BUGSVULNS.MD @@ -4143,3 +4143,43 @@ where a node holds no link (93 of 100 on average over four seeds, the same as without, for 4 to 6 packets per second per node at rest instead of 2.4 to 4.4). And there is nothing larger than 240 nodes to measure on this machine: the simulator needs more cores than the 8 it has once the mesh is chatty, and at rest it is the first number to check again. + +## 109 — high — A node was linked to its true nearest neighbour about one time in two, and the others could not be found + +> **Status — fixed** (leaf set). Tests: +> `tests/test_neighborhood.py::TestTheLeafSet`. + +**Where** — `src/node.py`: `_maintain_neighbors`. + +**What happens** — maintenance only insisted on `_NEIGHBOR_FLOOR` = 3 links, and +stopped searching once it held them, whichever nodes they were. A node that +joined through two random members held a handful of links in no particular part +of the id space, was known to a handful of nodes, and had nobody near it in id +space that knew it. Both ways of reaching it failed: greedy forwarding ends at +the node nearest the target and that node had no link to it, and a lookup +cannot find a node its nearest neighbours have never heard of. The `--explain` +output of `tests/tools/mesh_sim.py` on 1000 nodes showed it: the undelivered +first contacts ended at nodes holding 3 to 6 links, with a routing table of 3 +to 9 entries, known to 3 to 7 nodes in the mesh. `leafcheck` measured it +directly: of each node's true 1, 3 and 8 nearest (computed over all ids), 54 %, +53 % and 47 % were linked. + +Two changes that looked like the fix were measured and changed nothing: looking +the target up before the first forward (89 and 89 of 100 against 90 and 88, and +slower), and a rule that looked the target up at a local minimum of the +distance. The cause was not the way packets are carried. + +**Fixed** — the nodes we hold nearest in id space, `_LEAF_SIZE` = 8, are a set +the node insists on. While they are not all linked, or are not what they were at +the last cycle, a cycle searches: it looks up its own id (which is also how +those nodes come to know it) and dials what is missing. One that refused +`_LEAF_GIVE_UP` times is not waited for, and the usual back-off stretches the +cycles while nothing changes. On 1000 nodes over twelve processes with the +crypto stubbed (`--fast`): the true 8 nearest linked 98 % of the time (from +47 %), first contacts 99, 100 and 100 of 100 over three seeds (from 88 to 92), one retry in a hundred +(from eleven), the same 4 packets per second per node at rest. + +**Still open** — nothing larger than 2000 nodes was run (memory: about 3 MB per +simulated node). Hostile nodes inserting near ids to hold a victim in the +searching regime are bounded by `_LEAF_GIVE_UP` and the back-off, but were not +tried. diff --git a/Docs/Architecture/routing.md b/Docs/Architecture/routing.md index 8f2d317..e9b9c21 100644 --- a/Docs/Architecture/routing.md +++ b/Docs/Architecture/routing.md @@ -400,13 +400,29 @@ then in a loop), but what it does depends on what we already hold: neighbourhood, then a directed dial of the `_NEIGHBOR_TARGET = 5` XOR-nearest entries it has no session with (`_connect_routing`, IPv6 then IPv4). It is a directed dial, not a broadcast. -- **Quiet** — as soon as `_NEIGHBOR_FLOOR` links are held: no lookup, no dial. A +- **Quiet** — as soon as `_NEIGHBOR_FLOOR` links are held *and the leaf set is + linked and unchanged* (below): no lookup, no dial, bar promotions and fingers. A node searching for its own id forever is nothing but traffic, and a mesh that never settles is a mesh an adversary can keep busy. Losing one of the three links (`_drop_failed_peer`, keepalive) restarts the search immediately. - `force=True` (join/`bootstrap`) imposes a full searching cycle whatever we already hold: a fresh table has to fill up. +### The leaf set: a node is linked to its nearest neighbours (`_LEAF_SIZE`) + +Greedy forwarding ends at the node nearest the target, and a lookup finds a node +through the nodes near it. Both rely on every node being linked to the nodes +nearest it in id space and known to them. The maintained set used to be three +links of whatever kind (`_NEIGHBOR_FLOOR`), which the join gave by luck, and +measured on 1000 nodes the true nearest neighbour was linked 54 % of the time +(BUGSVULNS 109). + +Each cycle takes the `_LEAF_SIZE` nearest nodes in the routing table. If they +are not all linked, or are not the ones of the last cycle, the cycle is a +searching one (a lookup of our own id, then a dial of whatever is missing); if +they are, the node is quiet. An entry that refused `_LEAF_GIVE_UP` dials is not +waited for. The cycle interval stretches while nothing changes, as before. + ### Links are spread over distance, and a lookup does not buy one (`_finger_candidates`, `_UPGRADE_LINK_SOFT`) Greedy forwarding picks the neighbour nearest the target, and only makes diff --git a/Docs/Architecture/scale.md b/Docs/Architecture/scale.md index 6858b03..fa8dfc5 100644 --- a/Docs/Architecture/scale.md +++ b/Docs/Architecture/scale.md @@ -17,7 +17,7 @@ test for anything that grows. | # | Point | State | |---|---|---| | 1 | Membership chains and the routing answers that carry them | **fixed** (compaction, lookup end); smaller chains on the wire in progress | -| 2 | Links per node, and what probing them costs a relay | **fixed** (probe budget; links spread over distance and no link bought by a lookup, BUGSVULNS 108) | +| 2 | Links per node, and what probing them costs a relay | **fixed** (probe budget; links spread over distance and no link bought by a lookup, BUGSVULNS 108; every node linked to its nearest neighbours, BUGSVULNS 109) | | 3 | Gossip planes (addresses, revocations, abuse, releases) | open | | 4 | Distributed storage (DHT, names, package directory) | open | | 5 | Per-remote-node tables | partly reviewed (below) | diff --git a/TEST.md b/TEST.md index 23fd506..365e762 100644 --- a/TEST.md +++ b/TEST.md @@ -117,6 +117,21 @@ packets sent and relayed, and the **lateness of the slowest worker's event loop* Past a loop lag p99 of a few hundred milliseconds the figures describe the machine. It is a measurement tool, not a test: nothing in `pytest` runs it. +Options worth knowing: + +* `--fast` replaces `oqs` with `tests/tools/fake_oqs.py`: the same key, signature + and ciphertext sizes, hashes instead of lattices. It gave the same figures as + the real thing at 240 nodes and reaches 2000 on twelve cores (about 3 MB per + simulated node, and a join phase that takes minutes: it is sequential). +* After the settling time it prints, for every node, how many of its **true** + 1, 3 and 8 nearest nodes (computed over all ids) it is linked to and has in its + routing table. A routing algorithm that depends on that is checked by it + directly, not by its side effects. +* `--explain` says what both ends of each undelivered first contact looked like: + links, routing table size, and how many nodes know the receiver. +* The same seed moves by five first contacts or more from run to run. Compare + ranges over several seeds, never one run against one run. + ### Hostile requests at the console's API `tests/tools/api_fuzz.py` signs in to a **throwaway** node and sends every route @@ -272,6 +287,9 @@ tests/ ├── test_loop_backtrack.py — a routed packet that │ comes back round a cycle leaves by another neighbour, and what that may │ cost is bounded (replays, fanout, the book) +├── test_neighborhood.py — the maintained neighbourhood, including the leaf set +│ (searching until the nearest known nodes are all linked, quiet after, bounded, +│ not kept searching by a node nobody can reach) ├── test_routing_stability.py — routing regressions: │ the size of a FOUND_NODE, acquiring a route outside the receive loop, │ the return path learned from traffic, bounded teardown, a first diff --git a/pyproject.toml b/pyproject.toml index 2a316fc..c46c621 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.backends.legacy:build" [project] name = "nmesh" -version = "0.4.111" +version = "0.4.112" [tool.pytest.ini_options] testpaths = ["tests"] diff --git a/src/mesh/constants.py b/src/mesh/constants.py index e97c6d2..775f4f1 100644 --- a/src/mesh/constants.py +++ b/src/mesh/constants.py @@ -475,6 +475,8 @@ def _is_ip_address(s: str) -> bool: # the node runs. Failed identities back off independently so dead addresses do # not turn maintenance into a dial storm. _NEIGHBOR_TARGET = 5 +_LEAF_SIZE = 8 # the nearest nodes by XOR this node insists on a link with +_LEAF_GIVE_UP = 3 # failed dials after which a leaf entry stops keeping the node searching _FINGER_LINKS = 2 # links wanted in each distance band that has nodes in it _FINGER_DIALS = 4 # bands filled per maintenance cycle # Floor of live maintained links. Below it the node is *searching*: it runs a @@ -1051,6 +1053,8 @@ def _is_ip_address(s: str) -> bool: "_NEIGHBOR_RETRY_MIN", "_NEIGHBOR_RETRY_TRACKED", "_NEIGHBOR_TARGET", + "_LEAF_SIZE", + "_LEAF_GIVE_UP", "_FINGER_LINKS", "_FINGER_DIALS", "_NEIGHBOR_WATCH_TRACKED", diff --git a/src/node.py b/src/node.py index 3f8b57c..70ee8cf 100644 --- a/src/node.py +++ b/src/node.py @@ -308,6 +308,7 @@ def __init__(self, self._e2e_pending_kem: dict[NodeID, bytes] = {} self._e2e_pending_nonce: dict[NodeID, bytes] = {} self._e2e_pending_data: dict[NodeID, list[bytes]] = {} + self._leaf_last: list[NodeID] = [] # the nearest nodes known at the last cycle self._e2e_attempt: dict[NodeID, float] = {} # target -> last handshake attempt (monotonic) # Responder-side re-key candidates: peer -> (candidate session, expiry). # Promoted only by a DATA packet that decrypts under the candidate. @@ -3474,8 +3475,26 @@ async def _maintain_neighbors(self, *, force: bool = False) -> None: searching = force or len(live) < _NEIGHBOR_FLOOR promotions = self._neighbor_promotions() fingers = self._finger_candidates() - if not searching and not promotions and not fingers: + # The leaf set: the `_LEAF_SIZE` nearest nodes we know. Greedy routing + # ends at the node nearest the target, and that node can only hand a + # packet on if it holds a link with the target; and a lookup can only + # find a node that its nearest neighbours know. Both need every node + # linked to its true nearest, and a node that held three links by luck + # of its join was known to three nodes and could be reached by nobody + # else. While the set is not all linked, or is not what it was last + # cycle, the cycle searches: a lookup of our own id (which is also how + # those nodes learn of us) and a dial of what it turned up. + leaf = [e.node_id for e in self._routing.get_closest(self._id, _LEAF_SIZE)] + linked = {p.authenticated_id for p in self._authenticated_peers()} + # One that has refused three times in a row is not waited for: an entry + # nobody can reach must not keep the node searching for ever. + unsettled = leaf != self._leaf_last or any( + i not in linked and self._neighbor_retry.get(i, (0, 0.0))[0] < _LEAF_GIVE_UP + for i in leaf) + self._leaf_last = leaf + if not searching and not promotions and not fingers and not unsettled: return + searching = searching or unsettled if searching and self._authenticated_peers(): try: @@ -3488,7 +3507,7 @@ async def _maintain_neighbors(self, *, force: bool = False) -> None: now = time.monotonic() live_ids = {p.authenticated_id for p in self._authenticated_peers()} - desired = list(promotions) + fingers + desired = list(promotions) + fingers + [i for i in leaf if i not in live_ids] if searching: desired += [entry.node_id for entry in self._routing.get_closest(self._id, _NEIGHBOR_TARGET)] diff --git a/src/version.py b/src/version.py index 67b22ed..a2e1105 100644 --- a/src/version.py +++ b/src/version.py @@ -10,7 +10,7 @@ import re -__version__ = "0.4.111" +__version__ = "0.4.112" # Releases are tagged ``vX.Y.Z`` (see .github/workflows/release.yml). Anything # after the numbers (``-rc1``, ``+build``) is kept but only used to break ties diff --git a/tests/test_kad_bootstrap.py b/tests/test_kad_bootstrap.py index b916b0f..7d110ce 100644 --- a/tests/test_kad_bootstrap.py +++ b/tests/test_kad_bootstrap.py @@ -173,6 +173,8 @@ async def fake_ensure(node_id, timeout=5.0): monkeypatch.setattr(node, "_ensure_route_to", fake_ensure) monkeypatch.setattr(node, "_finger_candidates", lambda: []) + import src.node as nodemod + monkeypatch.setattr(nodemod, "_LEAF_SIZE", 0) await node._maintain_neighbors() assert attempted == expected diff --git a/tests/test_neighborhood.py b/tests/test_neighborhood.py index 2d7525a..b2f601c 100644 --- a/tests/test_neighborhood.py +++ b/tests/test_neighborhood.py @@ -106,6 +106,7 @@ async def fake_lookup(*args, **kwargs): monkeypatch.setattr(node, "_ensure_route_to", fake_ensure) monkeypatch.setattr(node, "kad_lookup", fake_lookup) monkeypatch.setattr(node, "_finger_candidates", lambda: []) # its own rule, tested apart + monkeypatch.setattr(src.node, "_LEAF_SIZE", 0) # so is the leaf set await node._maintain_neighbors() assert attempted == [] assert looked_up == [] @@ -150,6 +151,7 @@ async def fake_ensure(node_id, timeout=5.0): monkeypatch.setattr(node, "_ensure_route_to", fake_ensure) monkeypatch.setattr(node, "kad_lookup", lambda *a, **k: asyncio.sleep(0)) monkeypatch.setattr(node, "_finger_candidates", lambda: []) + monkeypatch.setattr(src.node, "_LEAF_SIZE", 0) await node._maintain_neighbors() assert attempted == [] @@ -404,7 +406,84 @@ async def fake_lookup(*args, **kwargs): monkeypatch.setattr(node, "_ensure_route_to", fake_ensure) monkeypatch.setattr(node, "kad_lookup", fake_lookup) + monkeypatch.setattr(src.node, "_LEAF_SIZE", 0) await node._maintain_neighbors() assert attempted == [far] assert looked_up == [], "filling a band is not searching" await node.stop() + + +class TestTheLeafSet: + """The nearest nodes we know are linked, and a node keeps searching until they are.""" + + async def _quiet_node(self, monkeypatch): + node = MeshNode(transport_manager=make_manager()) + for _ in range(_NEIGHBOR_FLOOR): + await _attach_peer(node, _id_at_distance(node, 8)) + monkeypatch.setattr(node, "_finger_candidates", lambda: []) + self.attempted, self.looked_up = [], [] + + async def fake_ensure(node_id, timeout=5.0): + self.attempted.append(node_id) + return None + + async def fake_lookup(*args, **kwargs): + self.looked_up.append(args) + return [] + + monkeypatch.setattr(node, "_ensure_route_to", fake_ensure) + monkeypatch.setattr(node, "kad_lookup", fake_lookup) + return node + + @pytest.mark.asyncio + async def test_a_near_node_we_have_no_link_with_is_searched_for_and_dialled(self, monkeypatch): + node = await self._quiet_node(monkeypatch) + near = _id_at_distance(node, 3) + node._routing.add(near, ["fake://near"], b"k") + await node._maintain_neighbors() + assert self.attempted == [near] + assert self.looked_up, "our own id is looked up, which is how they learn of us" + await node.stop() + + @pytest.mark.asyncio + async def test_once_they_are_all_linked_and_unchanged_it_is_quiet(self, monkeypatch): + node = await self._quiet_node(monkeypatch) + near = _id_at_distance(node, 3) + node._routing.add(near, ["fake://near"], b"k") + await _attach_peer(node, near) + await node._maintain_neighbors() # the set is new to us: one look + self.attempted.clear() + self.looked_up.clear() + await node._maintain_neighbors() + assert self.attempted == [] and self.looked_up == [] + await node.stop() + + @pytest.mark.asyncio + async def test_a_new_nearer_node_starts_it_again(self, monkeypatch): + node = await self._quiet_node(monkeypatch) + await node._maintain_neighbors() + self.looked_up.clear() + node._routing.add(_id_at_distance(node, 2), ["fake://nearer"], b"k") + await node._maintain_neighbors() + assert self.looked_up + + @pytest.mark.asyncio + async def test_a_node_nobody_can_reach_does_not_keep_it_searching(self, monkeypatch): + node = await self._quiet_node(monkeypatch) + ghost = _id_at_distance(node, 3) + node._routing.add(ghost, ["fake://ghost"], b"k") + await node._maintain_neighbors() # the set is new to us + node._neighbor_retry[ghost] = (src.node._LEAF_GIVE_UP, time.monotonic() + 3600) + self.looked_up.clear() + await node._maintain_neighbors() + assert self.looked_up == [] + await node.stop() + + @pytest.mark.asyncio + async def test_the_set_is_bounded(self, monkeypatch): + node = await self._quiet_node(monkeypatch) + for index in range(60): + node._routing.add(_id_at_distance(node, 3 + index % 20), [f"fake://n{index}"], b"k") + await node._maintain_neighbors() + assert len(self.attempted) <= src.node._LEAF_SIZE + src.node._NEIGHBOR_TARGET + await node.stop() diff --git a/tests/tools/fake_oqs.py b/tests/tools/fake_oqs.py new file mode 100644 index 0000000..42ea620 --- /dev/null +++ b/tests/tools/fake_oqs.py @@ -0,0 +1,89 @@ +"""A stand-in for `oqs` with the sizes of ML-DSA-65 and ML-KEM-768 and none of +their cost, so a simulation measures routing and not lattice arithmetic. + + from tests.tools import fake_oqs; fake_oqs.install() # before `import src` + +Nothing here is secure: a signature is a hash anyone can compute from the public +key. It exists to carry the right number of bytes through the right code paths. +""" +import hashlib +import os +import sys +import types + +_DSA_PK, _DSA_SK, _DSA_SIG = 1952, 4032, 3309 +_KEM_PK, _KEM_SK, _KEM_CT, _KEM_SS = 1184, 2400, 1088, 32 + + +def _h(data: bytes, size: int = 32) -> bytes: + return hashlib.blake2b(data, digest_size=size).digest() + + +def _pad(data: bytes, size: int) -> bytes: + return data + b"\0" * (size - len(data)) + + +class Signature: + def __init__(self, alg, secret_key=None): + self._sk = bytes(secret_key) if secret_key is not None else None + + @staticmethod + def _pk(sk: bytes) -> bytes: + return _pad(_h(sk[:32], 64), _DSA_PK) + + def generate_keypair(self) -> bytes: + self._sk = _pad(os.urandom(32), _DSA_SK) + return self._pk(self._sk) + + def export_secret_key(self) -> bytes: + return self._sk + + def sign(self, message: bytes) -> bytes: + return _pad(_h(self._pk(self._sk) + bytes(message), 64), _DSA_SIG) + + def verify(self, message: bytes, signature: bytes, public_key: bytes) -> bool: + return len(signature) == _DSA_SIG and bytes(signature[:64]) == _h(bytes(public_key) + bytes(message), 64) + + def free(self): + pass + + +class KeyEncapsulation: + def __init__(self, alg, secret_key=None): + self._sk = bytes(secret_key) if secret_key is not None else None + + def __enter__(self): + return self + + def __exit__(self, *exc): + return False + + @staticmethod + def _pk(sk: bytes) -> bytes: + return _pad(_h(sk[:32], 64), _KEM_PK) + + def generate_keypair(self) -> bytes: + self._sk = _pad(os.urandom(32), _KEM_SK) + return self._pk(self._sk) + + def export_secret_key(self) -> bytes: + return self._sk + + def encap_secret(self, public_key: bytes): + secret = os.urandom(_KEM_SS) + mask = _h(bytes(public_key), _KEM_SS) + return _pad(bytes(a ^ b for a, b in zip(secret, mask)), _KEM_CT), secret + + def decap_secret(self, ciphertext: bytes) -> bytes: + mask = _h(self._pk(self._sk), _KEM_SS) + return bytes(a ^ b for a, b in zip(bytes(ciphertext[:_KEM_SS]), mask)) + + def free(self): + pass + + +def install() -> None: + module = types.ModuleType("oqs") + module.Signature = Signature + module.KeyEncapsulation = KeyEncapsulation + sys.modules["oqs"] = module diff --git a/tests/tools/mesh_sim.py b/tests/tools/mesh_sim.py index 839b53e..9cd3f76 100644 --- a/tests/tools/mesh_sim.py +++ b/tests/tools/mesh_sim.py @@ -25,8 +25,12 @@ # ---------------------------------------------------------------- worker ---- -async def worker(src: str) -> None: +async def worker(src: str, fast: bool = False) -> None: sys.path.insert(0, src) + if fast: + sys.path.insert(0, _HERE) + import fake_oqs + fake_oqs.install() out = os.fdopen(os.dup(1), "w", buffering=1) sys.stdout = sys.stderr # the library prints; the protocol must not mix from src import MeshNode @@ -94,10 +98,28 @@ async def drop(*_args, **_kwargs): if op == "kill": await node.stop() return True + if op == "leafcheck": + ids = [NodeID(bytes.fromhex(h)) for h in req["ids"]] + out = [] + for n in nodes: + if not n._running: + continue + linked = {p.authenticated_id for p in n._peers if p.session is not None and p.authenticated_id} + order = sorted((i for i in ids if i != n.id), key=n.id.distance) + row = [] + for k in req["ks"]: + near = order[:k] + row.append((sum(1 for i in near if i in linked), sum(1 for i in near if n._routing.contains(i)), k)) + out.append(row) + return out + if op == "known_by": + target = NodeID(bytes.fromhex(req["to"])) + return sum(1 for n in nodes if n._running and n._routing.contains(target)) if op == "explain": target = NodeID(bytes.fromhex(req["to"])) linked = {p.authenticated_id for p in node._peers if p.session is not None and p.authenticated_id} - return {"links": len(linked), "knows": node._routing.contains(target), + return {"links": len(linked), "table": len(node._routing.all_entries()), + "knows": node._routing.contains(target), "direct": target in linked, "nearest_gap": min((target.distance(i) for i in linked), default=None) is not None and min(target.distance(i) for i in linked) >= target.distance(node.id)} @@ -174,7 +196,7 @@ async def parent(args) -> None: workers = [] for _ in range(args.procs): proc = await asyncio.create_subprocess_exec( - sys.executable, os.path.abspath(__file__), "--worker", "--src", args.src, + sys.executable, os.path.abspath(__file__), "--worker", "--src", args.src, *(["--fast"] if args.fast else []), stdin=asyncio.subprocess.PIPE, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.DEVNULL, env=env) workers.append(Worker(proc)) @@ -223,6 +245,13 @@ async def counters(): cpu0 = sum(_cpu_seconds(w.proc.pid) for w in workers) wall0 = time.monotonic() + ks = [1, 3, 8] + rows = [r for rs in await asyncio.gather(*[w.call(op="leafcheck", ids=[n[2] for n in nodes], ks=ks) + for w in workers]) for r in rs] + for column, k in enumerate(ks): + linked = sum(r[column][0] for r in rows) / (len(rows) * k) + known = sum(r[column][1] for r in rows) / (len(rows) * k) + print(f"true {k} nearest: {linked:.0%} are linked, {known:.0%} are in the routing table", flush=True) alive = list(range(len(nodes))) if args.blackhole: bad = rnd.sample(alive, int(len(alive) * args.blackhole)) @@ -262,7 +291,8 @@ async def counters(): if got is None and args.explain: sender = await nodes[a][0].call(op="explain", i=nodes[a][1], to=nodes[b][2]) receiver = await nodes[b][0].call(op="explain", i=nodes[b][1], to=nodes[a][2]) - print(f" undelivered: sender {sender}, receiver {receiver}", flush=True) + known = sum(await asyncio.gather(*[w.call(op="known_by", to=nodes[b][2]) for w in workers])) + print(f" undelivered: sender {sender}, receiver {receiver}, receiver known to {known} nodes", flush=True) lags = await asyncio.gather(*[w.call(op="lag") for w in workers]) if args.profile: print((await workers[0].call(op="prof_stop", top=args.profile)).strip(), flush=True) @@ -287,6 +317,8 @@ async def counters(): if __name__ == "__main__": ap = argparse.ArgumentParser() ap.add_argument("--worker", action="store_true") + ap.add_argument("--fast", action="store_true", + help="stub the post-quantum crypto (same sizes, no cost): routing at sizes the real thing cannot reach") ap.add_argument("--src", default=_TREE, help="the tree whose nodes are run") ap.add_argument("--procs", type=int, default=8) ap.add_argument("--nodes", type=int, default=240) @@ -304,4 +336,4 @@ async def counters(): ap.add_argument("--profile", type=int, default=0, metavar="N", help="print the top N functions of the first worker while the pairs run") args = ap.parse_args() - asyncio.run(worker(args.src) if args.worker else parent(args)) + asyncio.run(worker(args.src, args.fast) if args.worker else parent(args)) From 6ca799a348970271e026c59d630842dd4197552b Mon Sep 17 00:00:00 2001 From: fnyaker Date: Wed, 14 Oct 2026 21:17:07 +0200 Subject: [PATCH 2/2] Links: structure held for good, the rest closed when idle, LINK_RELEASE, and the three numbers on the energy bar Co-Authored-By: Claude Sonnet 5.5 --- BUGSVULNS.MD | 25 +++ Docs/Architecture/protocol.md | 1 + Docs/Architecture/routing.md | 27 ++++ Docs/WebConsole/guide | 2 +- TEST.md | 3 + pyproject.toml | 2 +- src/config.py | 36 +++++ src/control/modules/settings.py | 6 +- src/features.py | 9 +- src/mesh/constants.py | 12 +- src/mesh/messages.py | 2 + src/mesh/peers.py | 10 ++ src/node.py | 142 ++++++++++++++++- src/power.py | 66 ++++---- src/version.py | 2 +- tests/test_kad_bootstrap.py | 2 +- tests/test_link_prune.py | 272 ++++++++++++++++++++++++++++++++ tests/test_neighborhood.py | 6 +- tests/tools/mesh_sim.py | 42 +++++ 19 files changed, 626 insertions(+), 41 deletions(-) create mode 100644 tests/test_link_prune.py diff --git a/BUGSVULNS.MD b/BUGSVULNS.MD index 319f4e7..40b93aa 100644 --- a/BUGSVULNS.MD +++ b/BUGSVULNS.MD @@ -4183,3 +4183,28 @@ crypto stubbed (`--fast`): the true 8 nearest linked 98 % of the time (from simulated node). Hostile nodes inserting near ids to hold a victim in the searching regime are bounded by `_LEAF_GIVE_UP` and the back-off, but were not tried. + +## 110 — medium — Every link stayed open for ever, and keeping them open was the whole traffic of an idle node + +> **Status — fixed.** Tests: `tests/test_link_prune.py`. + +**Where** — `src/node.py`: nothing closed a link that nobody used. + +**What happens** — a node on 1000 nodes held 28 to 36 links (the leaf set and +two links per band of distance, plus every link a conversation or a lookup had +ever opened) and, with nothing to say, sent 3.6 packets per second and 0.38 KiB/s +to keep them alive: `PING` and `PONG`, nothing else. The energy and performance +bar moved the cadence of those probes and not the number of links. + +**Fixed** — `_prune_idle_links` (see `routing.md`), the three numbers on the +bar, and `LINK_RELEASE`. A first version without the message closed links the +far end then redialled, which cycled: 0.05 handshakes per second per node and +5 KiB/s, ten times worse than doing nothing. At rest after the settling time on +1000 nodes: 15 links per node, 1.7 packets per second, 0.22 KiB/s, 99 % of the +true 8 nearest linked, 100 of 100 first contacts. + +**Still open** — the first minutes after a start (the settling time) still cost +more than the rest: handshakes and the lookups of every new node (29 KiB/s +per node at 30 nodes, measured earlier). Both ends of a link still +probe it; one probe each way would halve the rest (the quality estimates of each +side come from its own probes, so that is not free). diff --git a/Docs/Architecture/protocol.md b/Docs/Architecture/protocol.md index d17885d..32f09fc 100644 --- a/Docs/Architecture/protocol.md +++ b/Docs/Architecture/protocol.md @@ -140,6 +140,7 @@ and gone once they update. | SPEED_PROBE | 0x31 | padding sent to measure a link by **loading** it. Answered only on a direct authenticated link, only when addressed to us, and only up to `_SPEED_CHUNK` — a stranger must never be able to make this node generate traffic, and a peer must never be able to make it answer *to somebody else* | | SPEED_ECHO | 0x32 | the same padding back, **one for one and byte for byte**. The handler copies the payload rather than producing one, because a reflector that answers more than it was sent is an amplifier — which is the single worst thing this pair could be. The answering side keeps its own ceiling per identity; the asking side cannot raise it | | STOP_RELAY | 0x33 | a destination's **signed** "stop relaying that node to me", for at most `stop_relay.MAX_TTL` — **direct**, re-stamped at each hop: a relay passes the same record back to the link that fed it the refused traffic. Honoured only for the destination that signed it (see [`security.md`](security.md#refusing-a-node-all-the-way-back-stop_relaypy)), negotiated as `stoprelay` | +| LINK_RELEASE | 0x34 | **direct**, empty: "I am closing this link on purpose". The receiver only stops treating its end as a loss to chase (`peer.released`, read by `_note_node_lost`); a payload is charged as abuse and ignored. Sent by `_prune_idle_links` to a peer that announced `linkrelease`, never to one that did not | | INVITE_OFFER | 0x30 | a **rendezvous** an inviter leaves with a relay: "expect a seek for this code, and here is the proof I authorised it". Authenticated link only, and only ever about its own sender. It is what lets an invitation reach a node with no address of its own out of a string short enough for a QR code — the ML-DSA key and signature a relayed invitation needs stay with the relay, and the joiner sends 40 bytes | Groupings (constants): diff --git a/Docs/Architecture/routing.md b/Docs/Architecture/routing.md index e9b9c21..b0be72d 100644 --- a/Docs/Architecture/routing.md +++ b/Docs/Architecture/routing.md @@ -445,6 +445,33 @@ band is the bit length of the XOR distance to us. At rest on 240 nodes: about 3 packets per second per node and 25 to 30 links, against 20 packets per second and a full mesh before. +### Links nobody needs are closed (`_prune_idle_links`, `LINK_RELEASE`) + +An open link costs a probe each way every twenty seconds, and nothing used to +close one. On 1000 nodes the idle traffic of a node was 36 links and 3.6 packets +per second, all `PING` and `PONG` (BUGSVULNS 110). What a node holds for good is +its structure: the leaf set (`leaf_size`, the nearest nodes) and `finger_links` +links in each band of distance, the oldest. The others are closed after +`link_idle_minutes` without a conversation on them (`peer.last_used`, touched by +`DATA` and the E2E handshake, sent, received or relayed; keepalives and lookups +do not count), two per keepalive sweep. + +Left alone: a node we hold several links to (a bundle), relay-only and probe +links, the last `max(_NEIGHBOR_FLOOR, leaf_size)` links, a node whose leaf set +we probably belong to (fewer than `leaf_size` nodes in our table nearer to it +than we are), and a node that redialled after we closed (`peer.peer_wants`, +remembered `_PEER_WANTS_TTL` in `_idle_closed`, bounded). A closed link is not +chased by the side that closed it, and a peer that announced `linkrelease` is +sent `LINK_RELEASE` first so that it does not chase it either. Without that +message the far end saw a loss, redialled, and the pair cycled: measured as +0.05 handshakes per second per node, 5 KiB/s against 0.4 before pruning. + +The three numbers are on the energy and performance bar (`power.py`): from four +near neighbours, one link per band and two idle minutes at the frugal end, to +twelve, three and never closing at the other. Measured at rest on 1000 nodes, +after the settling time: 15 links per node and 1.7 packets per second (0.22 +KiB/s), against 28 to 36 and 3.6 (0.38 KiB/s). + ### Promoting a node seen in transit The maintained set (`_neighbor_slots`) = the `_NEIGHBOR_FLOOR` nearest live diff --git a/Docs/WebConsole/guide b/Docs/WebConsole/guide index 61290d8..264f7e6 100644 --- a/Docs/WebConsole/guide +++ b/Docs/WebConsole/guide @@ -1192,7 +1192,7 @@ console cannot produce a file the node would refuse to start on. Above the form, one bar with ten steps from *Minimal* to *Maximum*: how much this node may spend to stay quick. A step is a named set of **live** settings (`src/power.py`: the four keepalive bounds, `mlo_always`, `dynamic_address`, -`update_check_minutes`, `update_when_active`, `route_hold_minutes`) applied +`update_check_minutes`, `update_when_active`, `route_hold_minutes`, and the link budget: `leaf_size`, `finger_links`, `link_idle_minutes`) applied together through `config.profile`, and remembered like any other edit. Dragging previews the step's name and a one-sentence description; letting go applies it. Step 6, diff --git a/TEST.md b/TEST.md index 365e762..fdab5c1 100644 --- a/TEST.md +++ b/TEST.md @@ -287,6 +287,9 @@ tests/ ├── test_loop_backtrack.py — a routed packet that │ comes back round a cycle leaves by another neighbour, and what that may │ cost is bounded (replays, fanout, the book) +├── test_link_prune.py — idle links closed (structure, links in use, bundles, relay and +│ probe links, nodes whose leaf we are, redialled links kept), told with +│ LINK_RELEASE and not chased, bounded; the link budget's bounds ├── test_neighborhood.py — the maintained neighbourhood, including the leaf set │ (searching until the nearest known nodes are all linked, quiet after, bounded, │ not kept searching by a node nobody can reach) diff --git a/pyproject.toml b/pyproject.toml index c46c621..5d6e4d9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.backends.legacy:build" [project] name = "nmesh" -version = "0.4.112" +version = "0.4.113" [tool.pytest.ini_options] testpaths = ["tests"] diff --git a/src/config.py b/src/config.py index 975f0bc..13f53b3 100644 --- a/src/config.py +++ b/src/config.py @@ -249,6 +249,30 @@ def _as_route_hold(raw: str) -> int: return value +def _as_leaf_size(raw: str) -> int: + """How many of the nearest nodes this one keeps a link with.""" + value = int(str(raw).strip()) + if not 2 <= value <= 32: + raise ValueError("between 2 and 32 nodes") + return value + + +def _as_finger_links(raw: str) -> int: + """Links kept in each band of distance that has nodes in it.""" + value = int(str(raw).strip()) + if not 1 <= value <= 4: + raise ValueError("between 1 and 4 links") + return value + + +def _as_link_idle(raw: str) -> int: + """Minutes a link nobody needs is kept. 0 keeps every link.""" + value = int(str(raw).strip()) + if not 0 <= value <= 1440: + raise ValueError("between 0 and 1440 minutes (0 = never close one)") + return value + + def _as_branch(raw: str) -> str: """A branch to follow instead of the published releases. Empty = releases. @@ -316,6 +340,17 @@ def _as_path(raw: str): "Minutes a node reached through a relay keeps that " "relay before moving to another healthy one (0 = " "spread its traffic over every relay at once)"), + "leaf_size": (_as_leaf_size, 8, True, + "How many of the nearest nodes (by id) this one keeps " + "a permanent link with; fewer saves keepalives, more " + "makes it surer to be found"), + "finger_links": (_as_finger_links, 2, True, + "Permanent links kept in each distance band that has " + "nodes in it; they are what a packet crosses the mesh on"), + "link_idle_minutes": (_as_link_idle, 10, True, + "Minutes a link that is not one of the permanent ones " + "is kept without a conversation on it (0 = never " + "close one)"), "mlo_always": (_as_bool, False, True, "Keep multi-link operation running even when nothing " "is using this node (otherwise it wakes with the " @@ -457,6 +492,7 @@ def _as_path(raw: str): LIVE = frozenset({ "punch_keepalive", "lan_discovery", "transport_balance", "dynamic_address", "mlo_always", "mlo_skew_ms", "mlo_drop_percent", "route_hold_minutes", + "leaf_size", "finger_links", "link_idle_minutes", "keepalive_fast_min_ms", "keepalive_fast_max_ms", "keepalive_slow_min_ms", "keepalive_slow_max_ms", "pseudo", "no_abuse_gossip", "release_quorum", "release_auto_publish", diff --git a/src/control/modules/settings.py b/src/control/modules/settings.py index 1dbafca..3269688 100644 --- a/src/control/modules/settings.py +++ b/src/control/modules/settings.py @@ -59,7 +59,7 @@ ("punch_keepalive",), ("lan_discovery",), ("transport_balance",), ("dynamic_address",), ("mlo_always",), ("pseudo",), ("no_abuse_gossip",), - ("route_hold_minutes",)) + ("route_hold_minutes",), ("leaf_size", "finger_links", "link_idle_minutes")) async def apply_live(node, values: dict, changed) -> tuple[dict, list]: @@ -104,6 +104,10 @@ async def attempt(label, call): if "route_hold_minutes" in changed: await attempt("route_hold_minutes", lambda: node.set_route_hold(values["route_hold_minutes"] * 60)) + if changed & {"leaf_size", "finger_links", "link_idle_minutes"}: + await attempt("link budget", lambda: node.set_link_budget( + leaf_size=values["leaf_size"], finger_links=values["finger_links"], + idle_minutes=values["link_idle_minutes"])) if "dynamic_address" in changed: await attempt("dynamic_address", lambda: node.set_dynamic_address(values["dynamic_address"])) diff --git a/src/features.py b/src/features.py index c5592da..7737127 100644 --- a/src/features.py +++ b/src/features.py @@ -118,17 +118,22 @@ # never sent one — and never judged for not honouring one. STOP_RELAY = "stoprelay" +# Telling the far end that a link is being closed on purpose, so that it does +# not take it for a loss and chase it (`MeshNode._prune_idle_links`). A node that +# never announced it is simply closed on, as before. +LINK_RELEASE = "linkrelease" + SPOKEN = frozenset({CORE, KADEMLIA, E2E, DIRECTORY, PSEUDO, CATALOG, RELEASE, PUNCH, REACH, RELAY, RENDEZVOUS, RENEW, REVOKE, ABUSE, KEEPALIVE, MLO, PACKAGES, HANDOVER, SPEEDTEST, - BLAKE2B_IDS, STOP_RELAY}) + BLAKE2B_IDS, STOP_RELAY, LINK_RELEASE}) # Planes added *after* this negotiation existed. Silence about one of these is # not a node from before the name — it is a node that has never heard of it, # and sending it the new thing is exactly what rule 2 forbids. See # ``MeshNode.peer_announces``, which is the predicate these are asked through. SINCE_NEGOTIATION = frozenset({KEEPALIVE, MLO, PACKAGES, HANDOVER, RENDEZVOUS, - SPEEDTEST, BLAKE2B_IDS, STOP_RELAY}) + SPEEDTEST, BLAKE2B_IDS, STOP_RELAY, LINK_RELEASE}) class FeatureError(Exception): diff --git a/src/mesh/constants.py b/src/mesh/constants.py index 775f4f1..a1b3e48 100644 --- a/src/mesh/constants.py +++ b/src/mesh/constants.py @@ -124,7 +124,9 @@ def _is_ip_address(s: str) -> bool: KA_PROPOSE, KA_REQUEST, # Re-stamped at each hop like a revocation: the record # inside is signed by the destination it speaks for. - STOP_RELAY} + STOP_RELAY, + # About this link only, like the keepalive accord. + LINK_RELEASE} # Stop-relay (`stop_relay.py`). How long a request asks to be honoured, how long # a relay is given to act on one before still being fed counts against it, the # rules a relay holds (in all, and for one destination), how many requests one @@ -476,6 +478,10 @@ def _is_ip_address(s: str) -> bool: # not turn maintenance into a dial storm. _NEIGHBOR_TARGET = 5 _LEAF_SIZE = 8 # the nearest nodes by XOR this node insists on a link with +_LINK_IDLE_SECONDS = 600.0 # a link nobody needs is closed after this long without conversation (0 = never) +_PEER_WANTS_TTL = 3600.0 # a node that redialled a link we closed is left holding it for this long +_IDLE_CLOSED_MAX = 512 # links we closed for being idle, remembered +_PRUNE_PER_SWEEP = 2 # idle links closed at one keepalive sweep _LEAF_GIVE_UP = 3 # failed dials after which a leaf entry stops keeping the node searching _FINGER_LINKS = 2 # links wanted in each distance band that has nodes in it _FINGER_DIALS = 4 # bands filled per maintenance cycle @@ -1055,6 +1061,10 @@ def _is_ip_address(s: str) -> bool: "_NEIGHBOR_TARGET", "_LEAF_SIZE", "_LEAF_GIVE_UP", + "_LINK_IDLE_SECONDS", + "_PEER_WANTS_TTL", + "_IDLE_CLOSED_MAX", + "_PRUNE_PER_SWEEP", "_FINGER_LINKS", "_FINGER_DIALS", "_NEIGHBOR_WATCH_TRACKED", diff --git a/src/mesh/messages.py b/src/mesh/messages.py index 8e5e3d5..32972ea 100644 --- a/src/mesh/messages.py +++ b/src/mesh/messages.py @@ -60,6 +60,7 @@ SPEED_PROBE = 0x31 # padding, to measure a link by loading it SPEED_ECHO = 0x32 # the same padding back — one for one, never more STOP_RELAY = 0x33 # a destination's signed "stop relaying that node to me" +LINK_RELEASE = 0x34 # "I am closing this link on purpose — do not chase me" # Built from this module's own constants so a message type added above can never # be missing here — a trace showing "0x1e" for a type the code knows the name of @@ -99,6 +100,7 @@ "INVITE_SEEK", "KA_PROPOSE", "KA_REQUEST", + "LINK_RELEASE", "KEY_ACCEPT", "KEY_GRANT", "KEY_OFFER", diff --git a/src/mesh/peers.py b/src/mesh/peers.py index a28061a..82694c1 100644 --- a/src/mesh/peers.py +++ b/src/mesh/peers.py @@ -115,6 +115,16 @@ def __init__(self, transport: BaseTransport, is_client_side: bool = False) -> No # A link used only to relay for others (SEEK / RELAY_CARRY) — we do not # try to authenticate to it, so its unsolicited CHALLENGE is ignored. self.relay_only: bool = False + # Last time a conversation (E2E handshake, data) crossed this link, in + # either direction or on its way through. Keepalives and lookups do not + # count: a link carrying only those is held for nothing. + self.last_used: float = 0.0 + # The far end redialled a link we had closed for being idle: it wants + # this one, so it is not ours to close again. + self.peer_wants: bool = False + # The far end said it was closing this link on purpose: its ending is + # not a loss to chase. + self.released: bool = False # A second link to a node we already reach, opened to measure it # (address steering). It is a duplicate on purpose and the pass that # opened it closes the loser, so the duplicate reaper leaves it alone. diff --git a/src/node.py b/src/node.py index 70ee8cf..cc5cf6e 100644 --- a/src/node.py +++ b/src/node.py @@ -309,6 +309,10 @@ def __init__(self, self._e2e_pending_nonce: dict[NodeID, bytes] = {} self._e2e_pending_data: dict[NodeID, list[bytes]] = {} self._leaf_last: list[NodeID] = [] # the nearest nodes known at the last cycle + self._leaf_size = _LEAF_SIZE # nearest nodes we insist on a link with + self._finger_links = _FINGER_LINKS # links per distance band + self._link_idle = _LINK_IDLE_SECONDS # seconds a link nobody needs is kept; 0 = never closed + self._idle_closed: OrderedDict[NodeID, float] = OrderedDict() # links we closed for being idle self._e2e_attempt: dict[NodeID, float] = {} # target -> last handshake attempt (monotonic) # Responder-side re-key candidates: peer -> (candidate session, expiry). # Promoted only by a DATA packet that decrypts under the candidate. @@ -1982,6 +1986,7 @@ async def _link_keepalive_loop(self) -> None: if now >= next_sweep: next_sweep = now + _LINK_KEEPALIVE_INTERVAL self._reap_silent_links() + self._prune_idle_links() self._reap_mute_accepted() self._reap_expired_tarpits() self._update_bundles() @@ -2081,6 +2086,98 @@ def _wake_keepalive(self) -> None: """A link arrived, or its cadence changed: do not wait out the tick.""" self._keepalive_wakeup.set() + def _links_held_for_good(self) -> set[NodeID]: + """The identities whose links are the structure of the mesh: the nearest + nodes (`_leaf_last`) and, in each band of distance, the + `_finger_links` oldest links. Everything else is there because a + conversation or a lookup put it there.""" + keep = set(self._leaf_last) + bands: dict[int, list] = {} + for peer in self._authenticated_peers(): + band = self._id.distance(peer.authenticated_id).bit_length() + bands.setdefault(band, []).append(peer) + for peers in bands.values(): + peers.sort(key=lambda p: p.serial) + keep.update(p.authenticated_id for p in peers[:self._finger_links]) + return keep + + def _likely_in_their_leaf(self, node_id: NodeID) -> bool: + """Whether we are probably one of the nearest nodes of ``node_id``, + from our own table: closing a link its other end insists on only makes + it dial back.""" + mine = node_id.distance(self._id) + nearer = 0 + for entry in self._routing.all_entries(): + if node_id.distance(entry.node_id) < mine: + nearer += 1 + if nearer >= self._leaf_size: + return False + return True + + def _prune_idle_links(self) -> None: + """Close the links nobody needs. Never raises, never awaits. + + A link stays open for ever unless something closes it, and every open + link costs a probe each way every twenty seconds: at rest that is the + whole traffic of a node (measured on 1000 nodes, 36 links and 3.6 + packets per second, all of it PING and PONG). What a node needs open is + its structure — the nearest nodes and a link or two per band of + distance — and the links it is using. The rest is closed after + `_link_idle` seconds without a conversation, a couple per sweep. + + Left alone: a node we hold several links to (a bundle), a link we are + probing or relaying on, one the far end redialled after we closed it, + and the last few links we have. A link closed here is **not chased**: + it was closed on purpose.""" + idle = self._link_idle + if not idle or not self._leaf_last: + return + links = self._authenticated_peers() + if len(links) <= max(_NEIGHBOR_FLOOR, self._leaf_size): + return + now = time.monotonic() + keep = self._links_held_for_good() + per_node: dict[NodeID, int] = {} + for peer in self._peers: + if peer.authenticated_id is not None: + per_node[peer.authenticated_id] = per_node.get(peer.authenticated_id, 0) + 1 + closing = [] + for peer in sorted(links, key=lambda p: p.last_used): + node_id = peer.authenticated_id + if (now - peer.last_used < idle or node_id in keep or peer.peer_wants + or peer.relay_only or peer.probation or per_node.get(node_id, 0) > 1 + or self._likely_in_their_leaf(node_id)): + continue + closing.append(peer) + if len(closing) >= _PRUNE_PER_SWEEP: + break + for peer in closing: + if peer not in self._peers: + continue + self._peers.remove(peer) + self._forget_bundle(peer) + self._idle_closed[peer.authenticated_id] = now + while len(self._idle_closed) > _IDLE_CLOSED_MAX: + self._idle_closed.popitem(last=False) + reason = f"closed: nobody needed it for {now - peer.last_used:.0f} s" + self._log_link_end(peer, reason) + self._spawn_bounded(self._release_and_stop(peer, reason)) + if closing: + self._note_change("links") + self._note_change("nodes") + + async def _release_and_stop(self, peer: '_Peer', reason: str) -> None: + """Say the link is being closed on purpose, to a peer that understands + it, and close it.""" + if self.peer_announces(peer, features.LINK_RELEASE): + try: + async with asyncio.timeout(1.0): + await peer.send(Packet.create(LINK_RELEASE, self._id.raw, + peer.authenticated_id.raw, b"")) + except Exception: # noqa: BLE001 — closing anyway + pass + await self._safe_stop_peer(peer, reason) + def _reap_silent_links(self) -> None: """Cut the links that answer nothing. Never raises, never awaits. @@ -2491,6 +2588,18 @@ async def _handle_ka_propose(self, peer: '_Peer', packet: Packet) -> None: peer.ka_accord_at + self._keepalive_interval(peer)) self._wake_keepalive() + async def _handle_link_release(self, peer: '_Peer', packet: Packet) -> None: + """The far end is closing this link on purpose and says so, so that its + ending is not taken for a loss and chased. + + All it can do is stop us redialling: the flag is read when the link + ends, and a peer that says it and does not close costs us nothing.""" + if peer.authenticated_id is None or peer.session is None: + return + if packet.payload: + return self._charge_abuse(peer) # it carries nothing + peer.released = True + async def _handle_ka_request(self, peer: '_Peer', packet: Packet) -> None: """A peer asks this node to slow its probes down on this link. @@ -3322,7 +3431,7 @@ def _finger_candidates(self) -> list[NodeID]: if now < self._neighbor_retry.get(entry.node_id, (0, 0.0))[1]: continue band = self._id.distance(entry.node_id).bit_length() - if held.get(band, 0) >= _FINGER_LINKS: + if held.get(band, 0) >= self._finger_links: continue if band not in best or entry.last_seen > best[band].last_seen: best[band] = entry @@ -3484,7 +3593,7 @@ async def _maintain_neighbors(self, *, force: bool = False) -> None: # else. While the set is not all linked, or is not what it was last # cycle, the cycle searches: a lookup of our own id (which is also how # those nodes learn of us) and a dial of what it turned up. - leaf = [e.node_id for e in self._routing.get_closest(self._id, _LEAF_SIZE)] + leaf = [e.node_id for e in self._routing.get_closest(self._id, self._leaf_size)] linked = {p.authenticated_id for p in self._authenticated_peers()} # One that has refused three times in a row is not waited for: an entry # nobody can reach must not keep the node searching for ever. @@ -3584,6 +3693,8 @@ def _note_node_lost(self, peer: '_Peer') -> None: return # never was a link to a node we can dial if peer.relay_only or peer.probation: return # not this node's link to that node + if peer.released: + return # it told us it was closing this on purpose if peer.tarpit_until or peer._malformed > _MAX_MALFORMED: return # we cut this one; getting it back is not a goal if self._cert_store.revocation_for(node_id) is not None: @@ -4234,6 +4345,21 @@ def _measured_first_hops(self, target: NodeID, out.append(peer) return out + def set_link_budget(self, *, leaf_size: int, finger_links: int, idle_minutes: int) -> dict: + """How many links this node holds for good, and how long it keeps the + others. See `_prune_idle_links`. Bounded here, so a setting that is + wrong costs a refusal and not a mesh.""" + leaf, fingers, idle = int(leaf_size), int(finger_links), int(idle_minutes) + if not 2 <= leaf <= 32: + raise ValueError("between 2 and 32 nearest nodes") + if not 1 <= fingers <= 4: + raise ValueError("between 1 and 4 links per band") + if not 0 <= idle <= 1440: + raise ValueError("between 0 and 1440 minutes") + self._leaf_size, self._finger_links, self._link_idle = leaf, fingers, idle * 60.0 + self._wake_neighbor_maintenance() + return {"leaf_size": leaf, "finger_links": fingers, "idle_minutes": idle} + def set_route_hold(self, seconds: float) -> float: """How long a routed identity keeps its first hop; 0 spreads it over every measured path. See `_route_lead`.""" @@ -4551,6 +4677,8 @@ async def _send_to_candidates(self, packet: Packet, candidates: list[_Peer], for peer in candidates: try: await peer.send(outgoing) + if packet.type in _CONVERSATION_TYPES: + peer.last_used = time.monotonic() return peer except LinkBusy: # Full, not gone: the next candidate gets the packet and this @@ -5304,6 +5432,12 @@ def _log_link_end(self, peer: '_Peer', why: str) -> None: def _log_link_up(self, peer: '_Peer', how: str) -> None: """A link authenticated: who, over what, which way round.""" + now = time.monotonic() + peer.last_used = now + closed = self._idle_closed.pop(peer.authenticated_id, None) + # We closed this node's link for being idle and it came back: it wants + # it, and closing it again would only start the same cycle over. + peer.peer_wants = closed is not None and now - closed < _PEER_WANTS_TTL if not self.logs.enabled: return self.log("link up", source="peers", topic="link", @@ -7739,6 +7873,8 @@ async def _handle_packet(self, peer: _Peer, packet: Packet) -> None: return if packet.src_id != peer.authenticated_id.raw: return + if packet.type in _CONVERSATION_TYPES: + peer.last_used = time.monotonic() if packet.type in _ROUTABLE_TYPES: if peer.authenticated_id is None: return @@ -14267,6 +14403,7 @@ async def _kick_punched_link(self, peer: '_Peer', SPEED_PROBE: MeshNode._handle_speed_probe, SPEED_ECHO: MeshNode._handle_speed_echo, STOP_RELAY: MeshNode._handle_stop_relay, + LINK_RELEASE: MeshNode._handle_link_release, ECHO_REPLY: MeshNode._handle_echo_reply, CERT_RENEW: MeshNode._handle_cert_renew, CERT_RENEWED: MeshNode._handle_cert_renewed, @@ -14306,4 +14443,5 @@ async def _kick_punched_link(self, peer: '_Peer', KA_PROPOSE: features.KEEPALIVE, KA_REQUEST: features.KEEPALIVE, SPEED_PROBE: features.SPEEDTEST, SPEED_ECHO: features.SPEEDTEST, STOP_RELAY: features.STOP_RELAY, + LINK_RELEASE: features.LINK_RELEASE, } diff --git a/src/power.py b/src/power.py index 64be408..f7c9190 100644 --- a/src/power.py +++ b/src/power.py @@ -33,6 +33,9 @@ "update_check_minutes", "update_when_active", "route_hold_minutes", + "leaf_size", + "finger_links", + "link_idle_minutes", ) LOWEST = 1 @@ -49,47 +52,54 @@ # Step 6 is exactly the defaults in `config.SETTINGS`, so a node nobody touched # reads as "Balanced" rather than "custom" (`test_the_default_is_a_step`). PROFILES = { - 1: ("Minimal", "Spends as little as it can. Connections are checked about " - "once a minute, two connections to one node are never combined, and " - "updates are looked for once a day, only while the node is in use. " - "A node reached through a relay keeps it for an hour.", - (2000, 5000, 60000, 120000, False, False, 1440, True, 60)), + 1: ("Minimal", "Spends as little as it can. Connections checked about once a " + "minute, never combined. Updates daily, only in use. Four near " + "neighbours; other links close after two idle minutes.", + (2000, 5000, 60000, 120000, False, False, 1440, True, 60, 4, 1, 2)), 2: ("Frugal", "Connections checked every 45 seconds or so, never " "combined. Updates twice a day, only while the node is in use. A " - "relay is kept for an hour.", - (1500, 5000, 45000, 90000, False, False, 720, True, 60)), + "relay is kept for an hour." + " Keeps few links, and closes the others after three idle minutes.", + (1500, 5000, 45000, 90000, False, False, 720, True, 60, 4, 1, 3)), 3: ("Saving", "Connections checked every half minute at rest. Two " "connections to one node may be combined, tested twice a second at " "most. Updates every four hours while in use. A relay is kept " - "for half an hour.", - (500, 3000, 30000, 60000, False, False, 240, True, 30)), + "for half an hour." + " Closes links nobody is using after five minutes.", + (500, 3000, 30000, 60000, False, False, 240, True, 30, 6, 1, 5)), 4: ("Light", "Connections checked every 25 seconds at rest. Hourly " - "updates while in use. A relay is kept for half an hour.", - (250, 2000, 25000, 40000, False, False, 60, True, 30)), + "updates while in use. A relay is kept for half an hour." + " Closes links nobody is using after eight minutes.", + (250, 2000, 25000, 40000, False, False, 60, True, 30, 6, 1, 8)), 5: ("Moderate", "A little quieter than the defaults at rest. Updates " - "every half hour. A relay is kept for a quarter of an hour.", - (200, 1000, 20000, 30000, False, False, 30, False, 15)), - 6: ("Balanced", "The defaults. Connections checked every 15 to 20 " - "seconds at rest, extra work only while somebody is using the node, " - "updates every five minutes. A node reached through a relay " - "moves to another one every ten minutes.", - (100, 1000, 15000, 20000, False, False, 5, False, 10)), + "every half hour. A relay is kept for a quarter of an hour." + " Closes links nobody is using after ten minutes, and keeps one per distance band.", + (200, 1000, 20000, 30000, False, False, 30, False, 15, 8, 1, 10)), + 6: ("Balanced", "The defaults. Connections checked every 15 to 20 seconds, " + "updates every five minutes, relays change every ten. Eight " + "near neighbours, two links per distance band; other links " + "close after ten idle minutes.", + (100, 1000, 15000, 20000, False, False, 5, False, 10, 8, 2, 10)), 7: ("Responsive", "Connections checked every ten seconds, and a live " "connection moves to a faster address of the same node when one " - "measures better.", - (100, 1000, 10000, 15000, False, True, 5, False, 10)), + "measures better." + " Idle links are kept twenty minutes.", + (100, 1000, 10000, 15000, False, True, 5, False, 10, 8, 2, 20)), 8: ("Fast", "Two connections to one node stay combined even when nobody " "is using this one, so the second is ready the moment traffic starts. " - "Relays change every five minutes.", - (100, 500, 8000, 12000, True, True, 5, False, 5)), + "Relays change every five minutes." + " Idle links are kept half an hour.", + (100, 500, 8000, 12000, True, True, 5, False, 5, 8, 2, 30)), 9: ("Faster", "Connections checked every five seconds at rest, up to " "twenty times a second while combined. Updates every two minutes. " - "Relays change every two minutes.", - (50, 500, 5000, 10000, True, True, 2, False, 2)), - 10: ("Maximum", "Notices trouble and recovers as fast as it can, " - "whatever it costs, and spreads what goes through relays over all " - "of them at once. For a machine on mains with bandwidth to spare.", - (50, 300, 3000, 6000, True, True, 1, False, 0)), + "Relays change every two minutes." + " Idle links are kept an hour, with ten near neighbours.", + (50, 500, 5000, 10000, True, True, 2, False, 2, 10, 2, 60)), + 10: ("Maximum", "Notices trouble and recovers as fast as it can, and " + "spreads what goes through relays over all of them. Never " + "closes a link; twelve near neighbours, three links per " + "band. For a machine on mains.", + (50, 300, 3000, 6000, True, True, 1, False, 0, 12, 3, 0)), } diff --git a/src/version.py b/src/version.py index a2e1105..bd13785 100644 --- a/src/version.py +++ b/src/version.py @@ -10,7 +10,7 @@ import re -__version__ = "0.4.112" +__version__ = "0.4.113" # Releases are tagged ``vX.Y.Z`` (see .github/workflows/release.yml). Anything # after the numbers (``-rc1``, ``+build``) is kept but only used to break ties diff --git a/tests/test_kad_bootstrap.py b/tests/test_kad_bootstrap.py index 7d110ce..625d55a 100644 --- a/tests/test_kad_bootstrap.py +++ b/tests/test_kad_bootstrap.py @@ -174,7 +174,7 @@ async def fake_ensure(node_id, timeout=5.0): monkeypatch.setattr(node, "_ensure_route_to", fake_ensure) monkeypatch.setattr(node, "_finger_candidates", lambda: []) import src.node as nodemod - monkeypatch.setattr(nodemod, "_LEAF_SIZE", 0) + node._leaf_size = 0 await node._maintain_neighbors() assert attempted == expected diff --git a/tests/test_link_prune.py b/tests/test_link_prune.py new file mode 100644 index 0000000..9f24c6c --- /dev/null +++ b/tests/test_link_prune.py @@ -0,0 +1,272 @@ +""" +Links nobody needs are closed, and the far end is told it was on purpose. + +A link stays open for ever unless something closes it, and each open link costs +a probe each way every twenty seconds: that was the whole traffic of an idle +node. What a node holds for good is its structure (the nearest nodes, a link or +two per band of distance); the rest goes after a while without a conversation. +Closing one must not start a chase on the other side, hence LINK_RELEASE. +""" +import asyncio +import os +import time + +import pytest + +import src.node as nodemod +from src import features +from src.node import MeshNode, LINK_RELEASE +from src.node_id import NodeID +from src.packet import Packet +from tests.conftest import FakeTransport, make_manager + + +def _at(node: MeshNode, band: int, low: int = 0) -> NodeID: + value = int.from_bytes(node.id.raw, "big") ^ ((1 << (band - 1)) | low) + return NodeID(value.to_bytes(20, "big")) + + +async def _link(node, node_id, *, idle=True, announces=False): + fake = FakeTransport() + peer = await node._inject_peer(fake) + peer.authenticated_id = node_id + peer.session = object() + peer.last_used = time.monotonic() - (10_000 if idle else 0) + if announces: + peer.agreed = frozenset({features.LINK_RELEASE}) + return peer, fake + + +async def _node(links=12, leaf=2): + """A node holding ``links`` links in distinct bands, all idle, with a leaf + set of its two nearest and nothing known that would make it somebody's + leaf.""" + node = MeshNode(transport_manager=make_manager()) + node._link_idle = 60.0 + node._leaf_size = leaf + node._finger_links = 1 + made = [] + for band in range(20, 20 + links): + made.append(await _link(node, _at(node, band))) + node._leaf_last = [made[-1][0].authenticated_id, made[-2][0].authenticated_id] + # With an empty table every node looks as if it had us among its nearest, + # which is the one rule tested on its own below. + node._likely_in_their_leaf = lambda _id: False + return node, made + + +def _open(node): + return {p.authenticated_id for p in node._peers if p.session is not None} + + +class TestWhatIsClosed: + async def test_an_idle_link_nobody_needs_is_closed(self): + node, made = await _node() + # two links in one band: the second is not structure + extra, _ = await _link(node, _at(node, 20, 5)) + node._prune_idle_links() + assert extra not in node._peers + await node.stop() + + async def test_the_structure_is_kept(self): + node, made = await _node() + before = _open(node) + node._prune_idle_links() + assert _open(node) == before, "one per band and the leaf set: nothing is extra" + await node.stop() + + async def test_a_link_in_use_is_kept(self): + node, made = await _node() + busy, _ = await _link(node, _at(node, 20, 5), idle=False) + node._prune_idle_links() + assert busy in node._peers + await node.stop() + + async def test_a_node_that_redialled_after_we_closed_is_not_closed_again(self): + node, made = await _node() + wanted, _ = await _link(node, _at(node, 20, 5)) + wanted.peer_wants = True + node._prune_idle_links() + assert wanted in node._peers + await node.stop() + + @pytest.mark.parametrize("flag", ["relay_only", "probation"]) + async def test_a_relay_link_or_a_probe_is_left_alone(self, flag): + node, made = await _node() + odd, _ = await _link(node, _at(node, 20, 5)) + setattr(odd, flag, True) + node._prune_idle_links() + assert odd in node._peers + await node.stop() + + async def test_a_node_we_hold_two_links_to_is_left_alone(self): + node, made = await _node() + twin_id = _at(node, 20, 5) + first, _ = await _link(node, twin_id) + second, _ = await _link(node, twin_id) + node._prune_idle_links() + assert first in node._peers and second in node._peers + await node.stop() + + async def test_a_node_whose_leaf_we_probably_are_is_left_alone(self): + node, made = await _node() + other = _at(node, 20, 5) + extra, _ = await _link(node, other) + del node._likely_in_their_leaf # the real rule + node._leaf_size = 3 + # fewer than three known nodes nearer to it than we are + node._routing.add(NodeID((int.from_bytes(other.raw, "big") ^ 1).to_bytes(20, "big")), + ["fake://a"], b"k") + node._prune_idle_links() + assert extra in node._peers + await node.stop() + + async def test_a_node_that_is_not_near_us_is_closed(self): + node, made = await _node() + other = _at(node, 20, 5) + extra, _ = await _link(node, other) + del node._likely_in_their_leaf # the real rule + for low in (1, 2, 3): + node._routing.add(NodeID((int.from_bytes(other.raw, "big") ^ low).to_bytes(20, "big")), + [f"fake://n{low}"], b"k") + node._prune_idle_links() + assert extra not in node._peers + await node.stop() + + async def test_few_links_are_never_pruned(self): + node, made = await _node(links=3) + node._leaf_size = 8 + extra, _ = await _link(node, _at(node, 5)) + node._prune_idle_links() + assert extra in node._peers + await node.stop() + + async def test_zero_minutes_means_never(self): + node, made = await _node() + node._link_idle = 0.0 + extra, _ = await _link(node, _at(node, 20, 5)) + node._prune_idle_links() + assert extra in node._peers + await node.stop() + + async def test_a_sweep_closes_a_bounded_number(self): + node, made = await _node(links=4) + node._leaf_size = 2 + for low in range(1, 9): + await _link(node, _at(node, 22, low)) + before = len(node._peers) + for low in (1, 2, 3): # make them all "not near" for the other end + pass + node._leaf_size = 2 + node._prune_idle_links() + assert before - len(node._peers) <= nodemod._PRUNE_PER_SWEEP + await node.stop() + + +class TestClosingOnPurpose: + async def test_a_peer_that_understands_is_told_and_one_that_does_not_is_not(self): + node, made = await _node() + told, told_fake = await _link(node, _at(node, 20, 5), announces=True) + silent, silent_fake = await _link(node, _at(node, 20, 6), announces=False) + node._prune_idle_links() + await asyncio.sleep(0.2) + assert [p.type for p in told_fake.sent].count(LINK_RELEASE) == 1 + assert LINK_RELEASE not in [p.type for p in silent_fake.sent] + await node.stop() + + async def test_it_is_not_chased_by_the_node_that_closed_it(self): + node, made = await _node() + extra, _ = await _link(node, _at(node, 20, 5)) + node._prune_idle_links() + assert extra.authenticated_id not in node._reconnect + assert extra.authenticated_id in node._idle_closed + await node.stop() + + async def test_the_book_of_closed_links_is_bounded(self): + node, made = await _node() + for _ in range(nodemod._IDLE_CLOSED_MAX + 50): + node._idle_closed[NodeID(os.urandom(20))] = time.monotonic() + while len(node._idle_closed) > nodemod._IDLE_CLOSED_MAX: + node._idle_closed.popitem(last=False) + extra, _ = await _link(node, _at(node, 20, 5)) + node._prune_idle_links() + assert len(node._idle_closed) <= nodemod._IDLE_CLOSED_MAX + await node.stop() + + async def test_a_link_that_comes_back_is_not_ours_to_close(self): + node = MeshNode(transport_manager=make_manager()) + other = NodeID(os.urandom(20)) + node._idle_closed[other] = time.monotonic() + peer, _ = await _link(node, other) + node._log_link_up(peer, "x") + assert peer.peer_wants is True and other not in node._idle_closed + stranger, _ = await _link(node, NodeID(os.urandom(20))) + node._log_link_up(stranger, "x") + assert stranger.peer_wants is False + await node.stop() + + async def test_a_long_ago_closed_link_is_forgotten(self): + node = MeshNode(transport_manager=make_manager()) + other = NodeID(os.urandom(20)) + node._idle_closed[other] = time.monotonic() - nodemod._PEER_WANTS_TTL - 1 + peer, _ = await _link(node, other) + node._log_link_up(peer, "x") + assert peer.peer_wants is False + await node.stop() + + +class TestBeingToldItWasOnPurpose: + async def test_a_release_marks_the_link(self): + node = MeshNode(transport_manager=make_manager()) + peer, _ = await _link(node, NodeID(os.urandom(20))) + await node._handle_link_release(peer, Packet.create( + LINK_RELEASE, peer.authenticated_id.raw, node.id.raw, b"")) + assert peer.released is True + await node.stop() + + async def test_a_release_that_carries_something_is_charged_and_ignored(self): + node = MeshNode(transport_manager=make_manager()) + peer, _ = await _link(node, NodeID(os.urandom(20))) + await node._handle_link_release(peer, Packet.create( + LINK_RELEASE, peer.authenticated_id.raw, node.id.raw, b"x" * 10)) + assert peer.released is False + await node.stop() + + async def test_a_release_before_authentication_is_ignored(self): + node = MeshNode(transport_manager=make_manager()) + fake = FakeTransport() + peer = await node._inject_peer(fake) + await node._handle_link_release(peer, Packet.create( + LINK_RELEASE, os.urandom(20), node.id.raw, b"")) + assert peer.released is False + await node.stop() + + async def test_a_released_link_that_ends_is_not_chased(self): + node = MeshNode(transport_manager=make_manager()) + other = NodeID(os.urandom(20)) + node._running = True + gone, _ = await _link(node, other) + gone.released = True + node._peers.remove(gone) + node._note_node_lost(gone) + assert other not in node._reconnect + lost, _ = await _link(node, NodeID(os.urandom(20))) + node._peers.remove(lost) + node._note_node_lost(lost) + assert lost.authenticated_id in node._reconnect, "a loss is still chased" + node._running = False + await node.stop() + + +class TestTheBudget: + async def test_it_is_bounded(self): + node = MeshNode(transport_manager=make_manager()) + for bad in ({"leaf_size": 1}, {"leaf_size": 33}, {"finger_links": 0}, + {"finger_links": 5}, {"idle_minutes": -1}, {"idle_minutes": 1441}): + args = {"leaf_size": 8, "finger_links": 2, "idle_minutes": 10, **bad} + with pytest.raises(ValueError): + node.set_link_budget(**args) + assert node.set_link_budget(leaf_size=4, finger_links=1, idle_minutes=0) == { + "leaf_size": 4, "finger_links": 1, "idle_minutes": 0} + assert node._link_idle == 0.0 + await node.stop() diff --git a/tests/test_neighborhood.py b/tests/test_neighborhood.py index b2f601c..3b94648 100644 --- a/tests/test_neighborhood.py +++ b/tests/test_neighborhood.py @@ -106,7 +106,7 @@ async def fake_lookup(*args, **kwargs): monkeypatch.setattr(node, "_ensure_route_to", fake_ensure) monkeypatch.setattr(node, "kad_lookup", fake_lookup) monkeypatch.setattr(node, "_finger_candidates", lambda: []) # its own rule, tested apart - monkeypatch.setattr(src.node, "_LEAF_SIZE", 0) # so is the leaf set + node._leaf_size = 0 # so is the leaf set await node._maintain_neighbors() assert attempted == [] assert looked_up == [] @@ -151,7 +151,7 @@ async def fake_ensure(node_id, timeout=5.0): monkeypatch.setattr(node, "_ensure_route_to", fake_ensure) monkeypatch.setattr(node, "kad_lookup", lambda *a, **k: asyncio.sleep(0)) monkeypatch.setattr(node, "_finger_candidates", lambda: []) - monkeypatch.setattr(src.node, "_LEAF_SIZE", 0) + node._leaf_size = 0 await node._maintain_neighbors() assert attempted == [] @@ -406,7 +406,7 @@ async def fake_lookup(*args, **kwargs): monkeypatch.setattr(node, "_ensure_route_to", fake_ensure) monkeypatch.setattr(node, "kad_lookup", fake_lookup) - monkeypatch.setattr(src.node, "_LEAF_SIZE", 0) + node._leaf_size = 0 await node._maintain_neighbors() assert attempted == [far] assert looked_up == [], "filling a band is not searching" diff --git a/tests/tools/mesh_sim.py b/tests/tools/mesh_sim.py index 9cd3f76..5d49042 100644 --- a/tests/tools/mesh_sim.py +++ b/tests/tools/mesh_sim.py @@ -135,6 +135,25 @@ async def drop(*_args, **_kwargs): buf = io.StringIO() pstats.Stats(prof[0], stream=buf).sort_stats("tottime").print_stats(req.get("top", 14)) return buf.getvalue() + if op == "budget": + for n in nodes: + n.set_link_budget(leaf_size=req["leaf"], finger_links=req["finger"], idle_minutes=req["idle"]) + return True + if op == "trace_start": + from src.node import MESSAGE_NAMES + for n in nodes: + if n._running: + n.trace.start(seconds=req["seconds"], events=1000, names=MESSAGE_NAMES) + return True + if op == "trace_totals": + out = {} + for n in nodes: + for (direction, kind), (cnt, nbytes) in n.trace._totals.items(): + if direction == "out": + row = out.setdefault(n.trace.name(kind), [0, 0]) + row[0] += cnt + row[1] += nbytes + return out if op == "counters": live = [n for n in nodes if n._running] return {"out": sum(n._metrics.total.pkts_out for n in live), @@ -228,6 +247,9 @@ async def counters(): rows = await asyncio.gather(*[w.call(op="counters") for w in workers]) return sum(r["out"] for r in rows), sum(r["relayed"] for r in rows) + if args.link_budget: + leaf, finger, idle = (int(x) for x in args.link_budget.split(",")) + await asyncio.gather(*[w.call(op="budget", leaf=leaf, finger=finger, idle=idle) for w in workers]) c0 = await counters() idle_cpu0 = sum(_cpu_seconds(w.proc.pid) for w in workers) await asyncio.sleep(args.settle) @@ -240,6 +262,22 @@ async def counters(): for g in range(len(nodes))])) print(f"links per node (distinct nodes): min {held[0]}, median {held[len(held) // 2]}, " f"mean {sum(held) / len(held):.1f}, max {held[-1]}", flush=True) + if args.rest_window: + await asyncio.gather(*[w.call(op="trace_start", seconds=args.rest_window + 10) for w in workers]) + await asyncio.sleep(args.rest_window) + rows: dict = {} + for part in await asyncio.gather(*[w.call(op="trace_totals") for w in workers]): + for name, (cnt, size) in part.items(): + row = rows.setdefault(name, [0, 0]) + row[0] += cnt + row[1] += size + per = len(nodes) * args.rest_window + print(f"at rest, per node per second ({args.rest_window:.0f} s window): " + f"{sum(r[0] for r in rows.values()) / per:.2f} packets, " + f"{sum(r[1] for r in rows.values()) / per / 1024:.2f} KiB", flush=True) + for name, (cnt, size) in sorted(rows.items(), key=lambda kv: -kv[1][1])[:10]: + print(f" {name:20s} {cnt / per:6.2f}/s {size / max(cnt, 1):7.0f} B each " + f"{size / per / 1024:6.2f} KiB/s", flush=True) await asyncio.gather(*[w.call(op="lag") for w in workers]) # reset c2 = await counters() cpu0 = sum(_cpu_seconds(w.proc.pid) for w in workers) @@ -316,6 +354,10 @@ async def counters(): if __name__ == "__main__": ap = argparse.ArgumentParser() + ap.add_argument("--rest-window", type=float, default=0.0, + help="after settling, count what each node sends, by message type, for this many seconds") + ap.add_argument("--link-budget", default="", metavar="LEAF,FINGER,IDLE_MIN", + help="set the nodes' permanent-link budget before the settling time (e.g. 8,1,1)") ap.add_argument("--worker", action="store_true") ap.add_argument("--fast", action="store_true", help="stub the post-quantum crypto (same sizes, no cost): routing at sizes the real thing cannot reach")