From 2e8275f6ce52ef514a8de7ec6608e3f48b22ecb0 Mon Sep 17 00:00:00 2001 From: Afi Date: Fri, 16 Oct 2026 11:04:04 +1030 Subject: [PATCH 1/9] optimizations & debug logging --- CHANGELOG.md | 49 ++++++++ README.md | 3 +- design.md | 53 +++++--- rnfs/__init__.py | 1 + rnfs/cli.py | 40 +++++- rnfs/config.py | 3 +- rnfs/content/archive.py | 24 ++-- rnfs/content/cid.py | 13 +- rnfs/content/dag.py | 47 ++++--- rnfs/content/shared_folder.py | 3 +- rnfs/content/storage.py | 130 ++++++++++++++++--- rnfs/debuglog.py | 229 ++++++++++++++++++++++++++++++++++ rnfs/dht/lookup.py | 37 +++++- rnfs/dht/providers.py | 26 ++-- rnfs/dht/routing.py | 21 +++- rnfs/features/jobs.py | 15 +-- rnfs/fetcher.py | 25 ++-- rnfs/gateway.py | 66 +++++++++- rnfs/log.py | 37 +++++- rnfs/names.py | 85 ++++++++----- rnfs/network/client.py | 12 +- rnfs/network/links.py | 128 +++++++++++++------ rnfs/network/server.py | 38 ++++-- rnfs/node.py | 74 ++++++++--- rnfs/pin_names.py | 2 +- rnfs/publisher.py | 75 +++++++---- specification.md | 1 + tests/test_archive.py | 17 +++ tests/test_cid.py | 23 ++++ tests/test_cli.py | 2 + tests/test_client.py | 24 ++++ tests/test_config.py | 2 +- tests/test_dag.py | 8 ++ tests/test_debuglog.py | 209 +++++++++++++++++++++++++++++++ tests/test_fetcher.py | 116 ++++++++++++++++- tests/test_jobs.py | 18 +++ tests/test_links.py | 14 +++ tests/test_lookup.py | 20 +++ tests/test_names.py | 78 ++++++++++++ tests/test_node.py | 66 +++++++++- tests/test_providers.py | 38 ++++++ tests/test_publisher.py | 98 ++++++++++++++- tests/test_routing.py | 32 +++++ tests/test_server.py | 94 +++++++++++++- tests/test_storage.py | 204 ++++++++++++++++++++++++++++++ 45 files changed, 2072 insertions(+), 228 deletions(-) create mode 100644 rnfs/debuglog.py create mode 100644 tests/test_debuglog.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 96a85ba..dabf2de 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,55 @@ [specification.md](specification.md) is the authority on behaviour. +## Unreleased + +Performance work, and a debug log. No wire-format, CID or state-file changes: these nodes and 0.1.4 nodes work +together, and each reads the other's ``. + +Added: +- A debug log file, for diagnosing problems: `debug_log = true` in `rnfs.toml`, or `rnfs daemon --debug-log`, + writes `/debug.log`. It begins with the daemon's version (then the Python, RNS and LXMF versions, + the platform and the settings changed from the defaults), and has every rnfs message at every level whatever + `verbosity` is; Reticulum's own messages, such as an interface that can't connect; each gateway request, + numbered, with the headers that decide it, what was decided (stored or fetched, type, range) and its status, + size and time; errors with tracebacks, and threads that die; and a status line every 5 minutes (counts, + open links, each interface up or down). New detail, at debug level: each DHT lookup's queries (which node + answered or failed, and how the lookup ended), the path to each node (hops, interface), why a link closed or + couldn't be established, each request's outcome or why it went unanswered, and incoming links. The file is + readable by its owner only and never sent; each start keeps the previous run's as `debug.log.1`, and past + 16 MiB the file moves there too, so the two stay under 32 MiB. + +Fixes: +- `gc` (and eviction) could delete blocks of a complete pin. `stat` of a DAG only partly stored remembered the + nodes it didn't find as listing nothing, so once the rest was fetched and pinned, collection didn't see what + those nodes reach. Until a restart, which cleared it. +- `import` pins the roots an archive supplies with one save of `pins.json`; if that save fails, none of them is + pinned (before, those ahead of the failure were). + +Faster: +- `add`, the shared folder and `import` write blocks on up to 16 threads, so the sync to disk each block needs + before its rename overlaps (about 11x on an SSD: 64 MB added at 165–190 MB/s, from 13–17). Every block is + still synced before it is named, and before anything pins it. +- A CID's written form and DHT key are computed once. They showed up in every save of `pins.json`, + `pin_names.json`, `published.json` and in `pin ls`. +- `fetched.json` is saved with the cache records (every 30 s, at `gc` and at stop), not rewritten and synced + under the node's lock on each fetch. Jobs save `jobs.json` once per change, not twice. +- State files written in full (`published.json`, `providers.json`, `name-floor.json`) are written outside the + publisher, provider-store and names locks, from a copy taken under them, so `/find`, `/provide`, resolves and + name announces don't wait for the write. `verify` and the size queries (`stat`, quotas, pinned bytes) read DAG + nodes without the storage lock, as eviction and `gc` already did. +- Recomputing what is provided reuses the last result when no provided DAG changed (about 30 ms to 5 ms at + 5000 pins). +- The DHT routing table finds the closest nodes by walking buckets, not by sorting every node (about 350 µs to + 4 µs per `/find` on a full table). Lookups start from nodes that haven't failed or gone silent, with the + others only to make up K, as `/find` replies do. +- Fetches: a slow provider is paced by how long its response took from when the request was sent, not counting + opening the link or waiting for the link's `/blocks` slot. A request reserves only what one response can carry, + sized to the open link's packet size, so idle providers get the rest; and the batch is sized in one pass. +- Serving `/blocks` no longer reads and hashes the block it then leaves out for the budget; finished responses + are noticed at once rather than within 10 ms; the response monitor sleeps while nothing is being sent. +- DAG nodes decode about 15% faster (struct formats compiled once), accepting and rejecting exactly as before. + ## 0.1.4 Security release from the fifth audit: fixes for denial of service against `/blocks` serving, unbounded downloads, diff --git a/README.md b/README.md index a34b48c..10d0ad5 100644 --- a/README.md +++ b/README.md @@ -600,6 +600,7 @@ lxmf_auto_fetch_from = [""] | `max_records` | `100000` | Provider records stored for others | | `ban_time` | `3600` | How long a node that sent a bad block is ignored | | `verbosity` | `0` | Extra logging: 1 per-query detail, 2 RNS debug output (`daemon -v`, `-vv`) | +| `debug_log` | `false` | Also write `/debug.log` for diagnosing problems (`daemon --debug-log`): it begins with the daemon's version, then has every rnfs message at every level whatever `verbosity` is (which nodes are asked, which fail and why, links that fail or close), Reticulum's own messages (interfaces that can't connect), each gateway request with its outcome, errors with tracebacks, and a status line every 5 minutes. It names what you fetched and browsed, so it is readable by you only and never sent anywhere; each start keeps the previous run's as `debug.log.1`, and the two stay under 32 MiB | **Storage** @@ -705,7 +706,7 @@ node ID is expected, a node contact's nick works too. | Command | Description | |---|---| -| `rnfs daemon [--bootstrap NODE_ID…] [--rnsconfig DIR] [--gateway-port N] [--dht-mode server\|client] [--slow yes\|no\|auto] [--private] [--provide-strategy pinned\|roots] [--no-gateway-scripts] [-v\|-vv]` | Run a node with its API and gateway | +| `rnfs daemon [--bootstrap NODE_ID…] [--rnsconfig DIR] [--gateway-port N] [--dht-mode server\|client] [--slow yes\|no\|auto] [--private] [--provide-strategy pinned\|roots] [--no-gateway-scripts] [--debug-log] [-v\|-vv]` | Run a node with its API and gateway | | `rnfs status` | Node ID, identity hash (what others list in `members` and `remote_pin_identities`), known nodes, records, storage, traffic, jobs, LXMF address. Without a readable API token (another user's health check) only the daemon's version | | `rnfs peers [--json]` | Known nodes: DHT mode, seniority, hops, interface, link rate, last heard, traffic, bans | | `rnfs stats [--json]` | Traffic since start: totals, uploads refused by your limits, per interface | diff --git a/design.md b/design.md index 35f55c3..bbb0994 100644 --- a/design.md +++ b/design.md @@ -18,7 +18,15 @@ authority on behaviour; this file fixes module boundaries and the interfaces bet strings only at the user edge (CLI, API, gateway, logs) and binary only on the wire. - `rnfs/log.py` — logging through RNS's logger. `log()` escapes non-printable characters in every message (remote names can't inject terminal escapes or fake lines); `cap_warnings` keeps a result's first - `MAX_WARNINGS` (100) warnings plus a count line and `warnings_total`. + `MAX_WARNINGS` (100) warnings plus a count line and `warnings_total`. With a debug sink set + (`set_debug_sink`), every message also goes to it, whatever RNS's loglevel. +- `rnfs/debuglog.py` — the debug log file (`debug_log`, `daemon --debug-log`), `/debug.log` (0600), + started by the daemon before the node: version first, then every rnfs message (warnings and errors logged in + an `except` block with their traceback), RNS's own log lines (copied from `RNS.logdest = LOG_CALLBACK`, which + still prints them), threads that die (`threading.excepthook`), a status line every 5 minutes (`status_line`: + the node's status, open links, each interface up or down), and gateway requests, numbered (`tag`). Each start + and every 16 MiB moves the file to `debug.log.1`. Lookups report their queries through `Lookup(report=)` (dht + stays free of RNS), links their path, close reason and each request's outcome, all at `LOG_DEBUG`. - `rnfs/fsutil.py` — the one platform switch (`IS_WINDOWS`), `O_BINARY`, and `replace`/`remove` that retry for about a second on Windows while another handle holds the file; `rename_new` (rename onto a path that must not exist; atomic through `renameat2(RENAME_NOREPLACE)` on Linux or a hard link, else check then rename); @@ -45,6 +53,7 @@ internally. rnfs.py, rnfs/__main__.py entry points (python3 rnfs.py, python3 -m rnfs); the installed command is rnfs.cli:main rnfs/ constants.py config.py errors.py log.py shared foundation + debuglog.py the debug log file, /debug.log (debug_log) fsutil.py cross-platform file operations: replace/remove retried on Windows, O_BINARY, IS_WINDOWS content/ no networking; uses RNS's bundled msgpack and RNS logging, so importing it loads RNS cid.py CIDs (section 4) @@ -128,10 +137,16 @@ packaging/ requirements.lock (hash-pinned runtime dependencies), upda `storage.py` — `Storage(blocks_dir, state_dir, max_storage)`, thread-safe: - blocks: `has(cid)`, `get(cid, verify=False) -> bytes | None` (with `verify`, a block no longer matching its CID is deleted), `put(cid, block)` (verifies, even a block already stored; atomic, the file fsynced), - `used_bytes()`. `BlockStore.sync()` syncs the blocks directory; `Storage` calls it before saving pins or + `used_bytes()`. Adds, the shared folder and archive import store through `with session.writer() as put:`, + which writes blocks on up to 16 threads (`WRITE_THREADS`, at most 32 queued) so their fsyncs overlap, and + has stored every one, or raised the first error, before the block exits: nothing is pinned before its blocks + are. It is handed only blocks built from, or checked against, their CIDs. `BlockStore.sync()` syncs the blocks directory; `Storage` calls it before saving pins or cache records, so they never refer to block names a power loss could undo. -- pins: `pin(cid, source)`, `unpin(cid, source) -> bool`, `pins() -> dict[Cid, set[str]]`, `is_complete(cid)`. +- pins: `pin(cid, source)`, `pin_many(cids, source)` (one save for them all; archive import), + `unpin(cid, source) -> bool`, `pins() -> dict[Cid, set[str]]`, `pin_count()`, `is_complete(cid)`. - cache: `record_fetch(root, names, target)` (keeps the path's nodes and the target's DAG), LRU touched on use. + `cache_generation()` grows whenever records are removed or retargeted, so the node prunes its fetched + targets only then. - protection: `with storage.protect() as session:` — blocks stored or listed in `session.add(cid)` are kept as if pinned until the block exits (fetches, adds, imports in progress). - `check_disk(nbytes)` raises `StorageFull` when storing `nbytes` would leave less than the disk reserve @@ -142,7 +157,9 @@ packaging/ requirements.lock (hash-pinned runtime dependencies), upda first read, without the lock, the DAG nodes they will walk (`_warm`, filling the `_links` and `_steps` memos; completeness via `dag_blocks`), then decide and delete under the lock from the pins, sessions and records current then, so `get` (and `/blocks` serving) waits only for that in-memory walk and the deletions, - and a block that became reachable meanwhile is kept. `pins.json` is still written under the lock on every + and a block that became reachable meanwhile is kept. The size queries (`pinned_bytes`, `dag_bytes`, + `source_usage`, `stat`) read their DAG nodes the same way first. `_links` memoises only stored DAG nodes: a + node not stored yet may be stored later, and a raw block lists nothing. `pins.json` is still written under the lock on every pin and unpin: a pin must be on disk when `pin()` returns (names' crash consistency and `rnfs pin add` depend on it), and before a collection can delete what an `unpin()` released. - `verify() -> stats` re-checks every block, deleting bad ones. @@ -194,6 +211,7 @@ can tell who vouches for its candidates. `server.py` — `RequestServer(destination, config, backend)`. `backend` provides: `is_dht_server() -> bool`, `find(key) -> (providers, nodes)`, `provide(publisher_id, keys, withdrawn) -> bool`, `served_block(cid) -> bytes | None` (only blocks in the DAG of a currently provided CID, re-verified), +optionally `served_block_size(cid) -> int | None` (so `/blocks` stops at the block over its budget unread), `remote_pin(identity_hash, cid, pin) -> bool`, `is_member(identity_hash) -> bool` (private mode). `NodeBackend.find` leaves out providers `RnfsNode.excluded_provider` rejects: banned, and in private mode non-members (`_is_member_publisher`: this node, a publisher whose `/provide` the server let through as a member @@ -205,7 +223,8 @@ node also drops non-members' records at start (`ProviderStore.retain`). link, batched; the keys the node may have recorded: every key, or those of the batches up to and including the first refused or unanswered one), `blocks(node_id, cids, on_progress=None) -> BlocksResult` (status OK / BUSY / FAILED / STALLED; blocks verified against their CIDs; `.blocks: dict[Cid, bytes]`, `.missing` answered None, -`.unsent` cut by the budget, `.bad` hash mismatches), `pin(node_id, cid, pin) -> bool`, `ping(node_id) -> float | +`.unsent` cut by the budget, `.bad` hash mismatches, `.elapsed` from sending to the answer, which the fetcher +paces by), `link_mdu(node_id)` (an open link's, which the fetcher sizes batches by), `pin(node_id, cid, pin) -> bool`, `ping(node_id) -> float | None`, `has_link(node_id)`, `close(node_id)`. `links.py` links only to the `rnfs.dht` destination of the identity RNS recalls, and only if its hash is the node ID asked for (in private mode, also only if that identity is on `members`); the idle sweep spares a link `LinkPool.get()` handed out within `link_idle_timeout`. @@ -475,6 +494,7 @@ name in the output's directory; the CLI sets it for `rnfs get ` withou | `rnfs-join`, `rnfs-publisher`, `rnfs-names`, `rnfs-maintenance` | node | joining (and joining again while under `MIN_SERVERS`), publishing, name announces, periodic saves and eviction | | `rnfs-announcer`, `rnfs-response-monitor`, `rnfs-link-sweeper` | network | rnfs.dht announces, `/blocks` deadlines and displacement, idle links | | `rnfs-lookup`, `rnfs-fetch`, `rnfs-find-providers`, `rnfs-remote-pin` | per operation | lookups, fetch workers, provider search, remote-pin fetches (one thread per source with pins waiting) | +| `rnfs-write` | storage | an add's, shared-folder import's or archive import's block writes (`Session.writer()`), up to 16 at once, ended with it | | `rnfs-job-` | jobs | one per running job, at most `max_jobs`; `stop()` waits ≤ 2 s | | `rnfs-keepers`, `rnfs-keeper-query` | keepers | pending refreshes (≤ 64 queued), own-name sync, saves, re-asking every `announce_interval`; parallel `/name` queries | | `rnfs-follow` | follow | every check and fetch, one at a time | @@ -487,14 +507,17 @@ Locks are taken in this order, never the reverse: `names._lock` → `publisher._ `publisher.refresh()`, while holding the names lock; `Publisher.refresh` recomputes what is provided under its own lock, reading the node's fetched targets (`node._lock`) and walking the pins' DAGs (`storage._lock`); the node reads storage under its lock (`_prune_fetched`). This is how the code behaves today, not a design goal. -Slow work that runs while a lock is held: - -- `Publisher.refresh` walks every pin's DAG (memoised) under the publisher lock, behind the names lock when a - name changes, so an RNS thread delivering a name announce can wait behind a full recompute. -- `Storage.evict`, `gc` and `verify` check every pin and cached target for completeness (`dag_blocks`, - memoised, reading blocks from disk) under the storage lock; `evict` and `gc` also delete blocks under it. - `verify` re-reads the blocks themselves outside it. -- `RnfsNode._note_fetched` and `_prune_fetched` write `fetched.json`, synced to disk, under the node lock. +State files written in full are written outside these locks, from a copy taken under them, one save at a time +behind a save lock taken first (`Storage.save`, `Publisher.save`, `ProviderStore.save`, the name floor, +`fetched.json`, `remote_pins.json`), so an older copy never lands last. Slow work that runs while a lock is +held: + +- `Publisher.refresh` checks every pin's DAG (`dag_blocks`, memoised) under the publisher lock, behind the + names lock when a name changes. It builds the set of blocks served again only when a provided CID went or its + DAG's blocks changed (CIDs only added are unioned in), so a refresh that changed nothing doesn't recompute it. +- `Storage.evict` and `gc` decide what to delete, and delete it, under the storage lock (after reading the DAG + nodes outside it); `verify` checks completeness and re-reads the blocks outside it. +- `pins.json` is written, synced to disk, under the storage lock on every pin and unpin (see `storage.py`). ## State files @@ -697,8 +720,8 @@ Everything in the specification is implemented, with these simplifications and a (`release_gone()`), pins a recorded target again if its pin is missing, and releases `name:` pins nothing records; a key without a `names.json` entry keeps its pins until its next publish or removal (`_stale`, where a release that failed with `OSError` also waits). A key file that is there but can't be loaded is - `_unreadable`: its pins are kept and its `names.json` entry is written back as loaded. `Storage.pin`/`unpin` - undo their change in memory if saving `pins.json` fails. Follow and LXMF do the same for their sources at + `_unreadable`: its pins are kept and its `names.json` entry is written back as loaded. `Storage.pin`/`pin_many`/ + `unpin` undo their change in memory if saving `pins.json` fails. Follow and LXMF do the same for their sources at start (`FollowFeature._reconcile`, `_release_unrecorded`), but release only if their file loaded whole (`MessageLog.clean`); otherwise unrecorded pins are recorded again (the name followed again, or an inbox message per sender listing them), so a later start keeps them too. The name table marks diff --git a/rnfs/__init__.py b/rnfs/__init__.py index 314b8b4..37a30f7 100644 --- a/rnfs/__init__.py +++ b/rnfs/__init__.py @@ -8,6 +8,7 @@ publisher.py provide strategy, publishing and withdrawal in the DHT node.py RnfsNode, which ties the above together into a running node and offers the facade api.py local HTTP API; gateway.py the HTTP gateway +debuglog.py the debug log file (debug_log) cli.py command line: run a daemon, or talk to one through its API """ diff --git a/rnfs/cli.py b/rnfs/cli.py index bcd4867..e38b1d5 100644 --- a/rnfs/cli.py +++ b/rnfs/cli.py @@ -268,13 +268,17 @@ def run_daemon(args): import faulthandler import signal import threading + import time + import traceback import RNS + from rnfs import debuglog from rnfs.api import ApiServer from rnfs.errors import RnfsError from rnfs.gateway import GatewayServer - from rnfs.node import RnfsNode + from rnfs.log import LOG_ERROR, LOG_NOTICE + from rnfs.node import RnfsNode, _make_private_dirs try: config = Config.load(args.dir, reticulum_dir=args.rnsconfig, @@ -282,13 +286,26 @@ def run_daemon(args): bootstrap_nodes=args.bootstrap, verbosity=args.verbose or None, dht_mode=args.dht_mode, slow=args.slow, private=args.private, provide_strategy=args.provide_strategy, - gateway_scripts=False if args.no_gateway_scripts else None) + gateway_scripts=False if args.no_gateway_scripts else None, + debug_log=args.debug_log) except (ValueError, TypeError, OSError) as e: fail(f"invalid settings: {e}") + if config.debug_log: + # Before the node, so it has Reticulum starting (interfaces connecting or not); in the rnfs dir made + # private first, as the node would make it + try: + _make_private_dirs(config) + debug = debuglog.start(config) + except (RnfsError, OSError) as e: + fail(f"can't write the debug log: {e}") + print(f"rnfs: writing the debug log {debug.path}", flush=True) + try: node = RnfsNode(config) except (RnfsError, OSError) as e: # e.g. an rnfs dir that isn't private, or an unreadable state file + if debuglog.current() is not None: # inside the except: with its traceback + debuglog.current().write(LOG_ERROR, "rnfs", f"couldn't start the node: {e!r}", traceback.format_exc()) if RNS.Reticulum.get_instance() is None: fail(str(e)) print(f"rnfs: {printable(str(e))}", file=sys.stderr, flush=True) @@ -301,6 +318,8 @@ def abort(message: str): server.stop() node.stop() print(f"rnfs: {printable(message)}", file=sys.stderr, flush=True) + if debuglog.current() is not None: + debuglog.current().note(f"stopped: {message}", LOG_ERROR) RNS.exit(1) # ends with os._exit, so the message is printed first try: @@ -319,6 +338,10 @@ def abort(message: str): server.start() print(f"rnfs node {node.node_id.hex()}, API on http://127.0.0.1:{api.port}, " f"gateway on http://127.0.0.1:{gateway.port}", flush=True) + debug = debuglog.current() + if debug is not None: + debug.note(f"started: API on port {api.port}, gateway on port {gateway.port}", LOG_NOTICE) + debug.note(debuglog.status_line(node), LOG_NOTICE) # `kill -USR1 ` prints every thread's stack, for debugging hangs. Windows has no SIGUSR1 if hasattr(signal, "SIGUSR1"): faulthandler.register(signal.SIGUSR1, all_threads=True) @@ -331,13 +354,22 @@ def abort(message: str): if hasattr(signal, name): signal.signal(getattr(signal, name), lambda *_: stopping.set()) # Python runs handlers in the main thread, and on Windows a wait isn't interrupted for them: wake each second + status_due = time.monotonic() + debuglog.STATUS_INTERVAL while not stopping.wait(1): - pass + if debug is not None and time.monotonic() >= status_due: + status_due = time.monotonic() + debuglog.STATUS_INTERVAL + try: + debug.note(debuglog.status_line(node), LOG_NOTICE) + except Exception as e: # noqa: BLE001 - a status line mustn't stop the daemon + debug.note(f"status line failed: {e!r}", LOG_ERROR) for server in servers: server.stop() remove_token(token_file, api.token) node.stop() + if debug is not None: + debug.note("stopped", LOG_NOTICE) + debuglog.stop() # RNS.exit() ends with os._exit, which would lose log lines still buffered when stdout is a file sys.stdout.flush() sys.stderr.flush() @@ -725,6 +757,8 @@ def command(subparsers, name: str, run, help: str) -> argparse.ArgumentParser: daemon.add_argument("--private", action="store_true", default=None, help="private mode: members only") daemon.add_argument("--provide-strategy", choices=["pinned", "roots"]) daemon.add_argument("--no-gateway-scripts", action="store_true", help="don't let gateway pages run scripts") + daemon.add_argument("--debug-log", action="store_true", default=None, + help="also write every message, gateway request and RNS message to /debug.log") daemon.add_argument("-v", "--verbose", action="count", default=0, help="log more detail: -v for every DHT query, -vv for RNS debug output") diff --git a/rnfs/config.py b/rnfs/config.py index 59b6b49..200043b 100644 --- a/rnfs/config.py +++ b/rnfs/config.py @@ -36,7 +36,7 @@ OPTIONAL = ("republish_interval", "max_upload_rate") # these may also be unset (None) # Some durations are waited on, and Windows can wait at most ~49 days at once (threading.TIMEOUT_MAX) MAX_DURATION = 30 * DAY -FLAGS = ("private", "gateway_scripts", "default_bootstrap") +FLAGS = ("private", "gateway_scripts", "default_bootstrap", "debug_log") IDENTITY_LISTS = ("members", "remote_pin_identities") # RNS identity hashes: 32 hex digits, optionally _HEX_ID = re.compile(r"[0-9a-fA-F]{32}") @@ -59,6 +59,7 @@ class Config: bootstrap_nodes: list[str] = field(default_factory=list) # node IDs (hex) contacted first when joining default_bootstrap: bool = True # also contact DEFAULT_BOOTSTRAP_NODES when joining (never in private mode) verbosity: int = 0 # added to Reticulum's loglevel: 1 shows per-query detail, 2 shows RNS debug output + debug_log: bool = False # the daemon also writes /debug.log, every message at every level (debuglog.py) slow: str = "auto" # "yes", "no" or "auto" (section 2) slow_bitrate: float = 100_000 diff --git a/rnfs/content/archive.py b/rnfs/content/archive.py index 2c8f4c4..1848556 100644 --- a/rnfs/content/archive.py +++ b/rnfs/content/archive.py @@ -188,16 +188,16 @@ def import_archive(stream: BinaryIO, storage: Storage, pin_source: str | None = with storage.protect() as session: for root in roots: session.add(root) # keeps blocks already stored below the roots until they're pinned - for cid, block in records: - if not storage.has(cid): - storage.check_disk(len(block)) # the same reserve fetches leave free (StorageFull) - try: - session.put(cid, block) # verifies, even a block already stored - except ValueError: - bad += 1 - continue - supplied.add(cid) - stored += 1 + with session.writer() as put: + for cid, block in records: + if not storage.has(cid): + storage.check_disk(len(block)) # the same reserve fetches leave free (StorageFull) + if not cid.verify(block): # even a block already stored; the writer stores it later + bad += 1 + continue + put(cid, block) + supplied.add(cid) + stored += 1 complete, incomplete = [], [] held: set[Cid] = set() # complete roots the archive didn't wholly supply for root in roots: @@ -208,11 +208,11 @@ def import_archive(stream: BinaryIO, storage: Storage, pin_source: str | None = continue if blocks <= supplied: complete.append(root) - if pin_source is not None: - storage.pin(root, pin_source) else: incomplete.append(root) held.add(root) + if pin_source is not None: + storage.pin_many(complete, pin_source) # still protected; pins.json is saved once for them all if bad: log(f"import: skipped {bad} blocks that didn't match their CIDs", LOG_WARNING) for root in incomplete: diff --git a/rnfs/content/cid.py b/rnfs/content/cid.py index 1d4e6b8..e8ace44 100644 --- a/rnfs/content/cid.py +++ b/rnfs/content/cid.py @@ -30,12 +30,15 @@ class Cid: Every constructor raises ValueError for anything that isn't a valid CID. """ - __slots__ = ("binary", "codec") + # _text and _key memoise str() and .key, computed on first use (a CID never changes) + __slots__ = ("binary", "codec", "_text", "_key") def __init__(self, binary: bytes): """Use from_bytes; this assumes binary is already known to be valid.""" self.binary = binary self.codec = RAW if len(binary) == len(_PREFIXES[RAW]) + _DIGEST_LENGTH else DAG + self._text = None + self._key = None @classmethod def from_bytes(cls, binary) -> "Cid": @@ -76,7 +79,9 @@ def digest(self) -> bytes: @property def key(self) -> bytes: """DHT key: SHA-256 of the whole binary CID, truncated to the 128-bit node ID space.""" - return hashlib.sha256(self.binary).digest()[:ID_LENGTH] + if self._key is None: + self._key = hashlib.sha256(self.binary).digest()[:ID_LENGTH] + return self._key @property def is_raw(self) -> bool: @@ -87,7 +92,9 @@ def verify(self, block: bytes) -> bool: return hashlib.sha256(block).digest() == self.digest def __str__(self) -> str: - return "b" + base64.b32encode(self.binary).decode("ascii").lower().rstrip("=") + if self._text is None: + self._text = "b" + base64.b32encode(self.binary).decode("ascii").lower().rstrip("=") + return self._text def __repr__(self) -> str: return f"Cid({self})" diff --git a/rnfs/content/dag.py b/rnfs/content/dag.py index 7c6a87a..d4902e1 100644 --- a/rnfs/content/dag.py +++ b/rnfs/content/dag.py @@ -142,6 +142,13 @@ def encode(node: Node) -> bytes: raise TypeError(f"not a DAG node: {node!r}") +# The big-endian lengths and integers after a msgpack type byte, by type byte (struct formats compiled once) +_ARRAY_LENGTHS = {0xDC: struct.Struct(">H"), 0xDD: struct.Struct(">I")} +_UINTS = {0xCC: struct.Struct(">B"), 0xCD: struct.Struct(">H"), 0xCE: struct.Struct(">I"), 0xCF: struct.Struct(">Q")} +_BIN_LENGTHS = {0xC4: struct.Struct(">B"), 0xC5: struct.Struct(">H"), 0xC6: struct.Struct(">I")} +_STR_LENGTHS = {0xD9: struct.Struct(">B"), 0xDA: struct.Struct(">H"), 0xDB: struct.Struct(">I")} + + class _Reader: """Reads the handful of msgpack forms a DAG node may contain, rejecting every other form.""" @@ -161,21 +168,29 @@ def _take(self, count: int) -> bytes: def _byte(self) -> int: """The next byte.""" - return self._take(1)[0] + pos = self.pos + if pos >= len(self.data): + raise InvalidDag("DAG node is truncated") + self.pos = pos + 1 + return self.data[pos] - def _length(self, size_format: str) -> int: - """A big-endian length or integer of the given struct format.""" - return struct.unpack(size_format, self._take(struct.calcsize(size_format)))[0] + def _length(self, size_format: struct.Struct) -> int: + """A big-endian length or integer of the given compiled struct format.""" + end = self.pos + size_format.size + if end > len(self.data): + raise InvalidDag("DAG node is truncated") + value = size_format.unpack_from(self.data, self.pos)[0] + self.pos = end + return value def array(self) -> int: """An array header; returns its length.""" tag = self._byte() if 0x90 <= tag <= 0x9F: return tag & 0x0F - if tag == 0xDC: - return self._length(">H") - if tag == 0xDD: - return self._length(">I") + size_format = _ARRAY_LENGTHS.get(tag) + if size_format is not None: + return self._length(size_format) raise InvalidDag(f"expected a msgpack array, found type byte 0x{tag:02x}") def uint(self) -> int: @@ -183,26 +198,26 @@ def uint(self) -> int: tag = self._byte() if tag <= 0x7F: return tag - formats = {0xCC: ">B", 0xCD: ">H", 0xCE: ">I", 0xCF: ">Q"} - if tag in formats: - return self._length(formats[tag]) + size_format = _UINTS.get(tag) + if size_format is not None: + return self._length(size_format) raise InvalidDag(f"expected a non-negative msgpack integer, found type byte 0x{tag:02x}") def bin(self) -> bytes: """A bin.""" tag = self._byte() - formats = {0xC4: ">B", 0xC5: ">H", 0xC6: ">I"} - if tag not in formats: + size_format = _BIN_LENGTHS.get(tag) + if size_format is None: raise InvalidDag(f"expected msgpack bin, found type byte 0x{tag:02x}") - return bytes(self._take(self._length(formats[tag]))) + return bytes(self._take(self._length(size_format))) def text(self) -> str: """A str holding valid UTF-8.""" tag = self._byte() if 0xA0 <= tag <= 0xBF: length = tag & 0x1F - elif tag in (0xD9, 0xDA, 0xDB): - length = self._length({0xD9: ">B", 0xDA: ">H", 0xDB: ">I"}[tag]) + elif tag in _STR_LENGTHS: + length = self._length(_STR_LENGTHS[tag]) else: raise InvalidDag(f"expected msgpack str, found type byte 0x{tag:02x}") try: diff --git a/rnfs/content/shared_folder.py b/rnfs/content/shared_folder.py index 48e6e84..23b0301 100644 --- a/rnfs/content/shared_folder.py +++ b/rnfs/content/shared_folder.py @@ -184,7 +184,8 @@ def _import(self, name: str, handle: builder.Handle, st: os.stat_result, path: s warnings: list[str] = [] try: with self.storage.protect() as session: - built = builder.build_open(handle, st, path, session.put, warnings, skip_hard_links=True) + with session.writer() as put: + built = builder.build_open(handle, st, path, put, warnings, skip_hard_links=True) self.storage.pin(built.cid, FOLDER) except (OSError, ValueError, RnfsError) as e: self._failed[name] = sig diff --git a/rnfs/content/storage.py b/rnfs/content/storage.py index 50785ab..f21a32d 100644 --- a/rnfs/content/storage.py +++ b/rnfs/content/storage.py @@ -14,7 +14,8 @@ Pins and cache records are persisted as JSON under state_dir: pins at once, cache records (which change on every fetch) by save(), which the node calls periodically and at shutdown. Thread-safe: one lock guards the indexes; long walks run outside it: completeness walks are discarded if blocks were deleted meanwhile, and -eviction and gc read the DAG nodes they will walk beforehand (_warm), deciding what to delete under the lock. +eviction, gc and the size queries read the DAG nodes they will walk beforehand (_warm), deciding what to delete +(or counting) under the lock. """ import json import os @@ -22,6 +23,7 @@ import shutil import threading from collections import OrderedDict +from concurrent.futures import ThreadPoolExecutor from contextlib import contextmanager from typing import Callable, Iterator @@ -43,6 +45,10 @@ EVICT_MARGIN = 32 # eviction frees max_storage / EVICT_MARGIN more than it must, so it runs rarely DISK_RESERVE_MIN = 256 * MIB # free disk space fetches always leave on the blocks' filesystem... DISK_RESERVE_FRACTION = 0.02 # ...or this fraction of the filesystem, if larger +# Blocks a Session.writer() writes at once: each is synced to disk before its rename, and syncs that overlap +# share the filesystem's journal commits (about 9x the speed of one at a time on an SSD) +WRITE_THREADS = 16 +WRITE_QUEUE = 32 # blocks a writer holds waiting to be written, at most (2 MiB of 64 KiB chunks) DAMAGED = object() # Storage._node's answer for a stored DAG node that is unreadable or no longer matches its CID MISSING = object() # Storage._read_node's answer for a block that isn't stored (or was deleted meanwhile) @@ -160,6 +166,64 @@ def put(self, cid: Cid, block: bytes): self.add(cid) # first, so an eviction between storing and protecting can't take it self._storage._put(cid, block, protected=True) + @contextmanager + def writer(self) -> Iterator[Callable[[Cid, bytes], None]]: + """A put for adds and imports: `with session.writer() as put:`. Each block is protected at once, as by + put(), and stored on one of WRITE_THREADS threads; every one is stored (or the first error raised) + before the with block ends, so nothing is pinned before its blocks are. Callers hand it only blocks + that match their CIDs (a mismatch is raised, but later): a block just built from, or checked.""" + writer = _Writer(self) + try: + yield writer.put + except BaseException: + writer.close(raise_error=False) # the body's error is the one raised; what it queued is still stored + raise + writer.close() + + +class _Writer: + """Session.writer()'s threads: at most WRITE_QUEUE blocks waiting, the first error kept for the caller.""" + + def __init__(self, session: Session): + """A writer storing blocks for session.""" + self._session = session + self._pool = ThreadPoolExecutor(WRITE_THREADS, thread_name_prefix="rnfs-write") + self._room = threading.Semaphore(WRITE_QUEUE) + self._error: BaseException | None = None + + def put(self, cid: Cid, block: bytes): + """Protect cid and queue its block (waiting while WRITE_QUEUE are queued); raises an earlier block's + error, so an add stops at the first one.""" + self._raise() + self._session.add(cid) # first, as in Session.put + self._room.acquire() + try: + self._pool.submit(self._write, cid, block) + except BaseException: + self._room.release() + raise + + def _write(self, cid: Cid, block: bytes): + """Store one block on a writer thread, keeping the first error.""" + try: + self._session._storage._put(cid, block, protected=True) + except BaseException as e: + if self._error is None: + self._error = e + finally: + self._room.release() + + def _raise(self): + """Raise the first error a block met, if any.""" + if self._error is not None: + raise self._error + + def close(self, raise_error: bool = True): + """Wait for every block queued to be stored, then raise the first error (if raise_error).""" + self._pool.shutdown(wait=True) + if raise_error: + self._raise() + class Storage: """Blocks plus pins, cached fetches, protection and eviction.""" @@ -183,6 +247,7 @@ def __init__(self, blocks_dir: str, state_dir: str, max_storage: int): self._steps: dict[Cid, dict[str, tuple[Cid, bool] | None]] = {} self._complete: dict[Cid, frozenset[Cid]] = {} # dag_blocks memo, complete DAGs only self._generation = 0 # incremented whenever blocks are deleted, invalidating walks in progress + self._cache_generation = 0 # incremented whenever cache records are removed or retargeted self._cache_dirty = False # cache records changed since the last save() # False only when nothing has happened since the last collection that could leave a block unreached, # so eviction over max_storage with nothing to evict needn't walk everything again @@ -319,19 +384,31 @@ def get(expect: walker.Expect) -> bytes: def pin(self, cid: Cid, source: str): """Pin cid with source (it lasts while any source holds it). Fetching it is the caller's job.""" + self.pin_many([cid], source) + + def pin_many(self, cids, source: str): + """Pin each of cids with source, as pin() does, saving pins once for them all. If the save fails none of + them is newly pinned, and the error is raised.""" check_source(source) with self._lock: - sources = self._pins.setdefault(cid, set()) - if source not in sources: - sources.add(source) - try: - self._save_pins() - except BaseException: - # Not saved (a full or failing disk): not pinned, so memory says what pins.json does + added = [] + for cid in cids: + sources = self._pins.setdefault(cid, set()) + if source not in sources: + sources.add(source) + added.append(cid) + if not added: + return + try: + self._save_pins() + except BaseException: + # Not saved (a full or failing disk): not pinned, so memory says what pins.json does + for cid in added: + sources = self._pins[cid] sources.discard(source) if not sources: del self._pins[cid] - raise + raise def unpin(self, cid: Cid, source: str) -> bool: """Release source's hold on cid; True if it held it. Blocks go at the next evict() or gc().""" @@ -365,6 +442,11 @@ def is_pinned(self, cid: Cid) -> bool: with self._lock: return cid in self._pins + def pin_count(self) -> int: + """Number of pinned CIDs.""" + with self._lock: + return len(self._pins) + # Cache def record_fetch(self, root: Cid, names: list[str] | tuple[str, ...], target: Cid) -> set[Cid]: @@ -377,6 +459,7 @@ def record_fetch(self, root: Cid, names: list[str] | tuple[str, ...], target: Ci previous = self._cache.get(key) if previous is not None and previous != target: self._maybe_garbage = True + self._cache_generation += 1 self._cache[key] = target self._cache.move_to_end(key) self._cache_dirty = True @@ -391,6 +474,7 @@ def _cap_cache(self) -> set[Cid]: if dropped: self._maybe_garbage = True self._cache_dirty = True + self._cache_generation += 1 return dropped def touch(self, root: Cid, names: list[str] | tuple[str, ...]) -> bool: @@ -413,6 +497,13 @@ def cached_targets(self) -> set[Cid]: with self._lock: return set(self._cache.values()) + def cache_generation(self) -> int: + """A count that grows whenever cache records are removed or lead somewhere new (eviction, gc, the + MAX_CACHE_RECORDS cap, a fetch recorded again with another target); while it is unchanged, + cached_targets() has only gained members.""" + with self._lock: + return self._cache_generation + # Protection @contextmanager @@ -482,12 +573,15 @@ def _step_in(node, name: str) -> tuple[Cid, bool] | None: def _children(self, cid: Cid) -> tuple[Cid, ...] | None: """CIDs listed by a stored DAG node; none for raw, missing or invalid blocks; None if the node is damaged, as its children are unknown (lock held).""" - if cid not in self._links: + listed = self._links.get(cid) + if listed is None: + if cid.is_raw or cid not in self._sizes: + return () # not memoised: a raw block never lists anything, and a missing node may be stored later node = self._node(cid) if node is DAMAGED: return None # not memoised: verify() removes it, or a fetch replaces it - self._links[cid] = self._listed(node) - return self._links[cid] + listed = self._links[cid] = self._listed(node) + return listed def _reach(self, roots, known: set[Cid] | frozenset[Cid] = frozenset(), damaged: set[Cid] | None = None) -> set[Cid]: @@ -546,7 +640,7 @@ def _warm(self, roots: set[Cid], paths: list[tuple[Cid, tuple[str, ...]]]): """ with self._lock: links = dict(self._links) - steps = {cid: dict(names) for cid, names in self._steps.items()} + steps = {cid: dict(names) for cid, names in self._steps.items()} if paths else {} found_links: dict[Cid, tuple[Cid, ...]] = {} found_steps: dict[tuple[Cid, str], tuple[Cid, bool] | None] = {} seen: set[Cid] = set() @@ -687,6 +781,7 @@ def _evict_to(self, limit: int) -> tuple[set[Cid], int, int]: self._cache.popitem(last=False) if evicted: self._cache_dirty = True + self._cache_generation += 1 dead = {cid for cid in self._sizes if cid not in live} deleted, freed = self._delete(dead) self._maybe_garbage = deleted != dead # anything deferred is collected on a later pass @@ -757,6 +852,9 @@ def _targets(self) -> set[Cid]: def pinned_bytes(self) -> int: """Bytes of stored blocks reachable from pins.""" + with self._lock: + pins = set(self._pins) + self._warm(pins, []) # the DAG nodes are read without the lock, as in gc() with self._lock: return sum(self._sizes[cid] for cid in self._reach(self._pins)) @@ -843,6 +941,7 @@ def gc(self) -> dict: if self._cache: self._cache.clear() self._cache_dirty = True + self._cache_generation += 1 deleted, freed = self._collect() incomplete = {cid for cid, blocks in before.items() if blocks & deleted} return {"removed_blocks": len(deleted), "freed_bytes": freed, "incomplete": incomplete, @@ -855,7 +954,8 @@ def verify(self) -> dict: """ with self._lock: cids = list(self._sizes) - before = self._complete_sets(self._targets()) + targets = self._targets() + before = self._complete_sets(targets) # without the lock, as in evict(); dag_blocks checks for deletions bad = [] for cid in cids: if self.blocks.verify(cid): @@ -888,6 +988,7 @@ def content_size(self, cid: Cid) -> int | None: def dag_bytes(self, cid: Cid) -> int: """Bytes of the stored blocks of cid's DAG (each block once), as far as its stored nodes reach.""" + self._warm({cid}, []) # the DAG nodes are read without the lock, as in gc() with self._lock: return sum(self._sizes[block] for block in self._reach({cid})) @@ -902,6 +1003,7 @@ def stat(self, cid: Cid) -> dict: "blocks" counts distinct blocks known from stored nodes, so it is exact only once all DAG nodes are stored; "size" is None until the root is. """ + self._warm({cid}, []) # the DAG nodes are read without the lock, as in gc() with self._lock: known = {cid} stack = [cid] diff --git a/rnfs/debuglog.py b/rnfs/debuglog.py new file mode 100644 index 0000000..276e84c --- /dev/null +++ b/rnfs/debuglog.py @@ -0,0 +1,229 @@ +"""The debug log file (setting `debug_log`, or `daemon --debug-log`): /debug.log, for diagnosing a daemon. + +- It begins with the daemon's version, then the Python, RNS and LXMF versions, the platform, and the settings + that differ from the defaults. Each start moves the previous run's file to debug.log.1; past MAX_SIZE the file + is moved there too and a new one begun, also with the version first, so the two hold at most 2 × MAX_SIZE. +- Every rnfs message at every level, whatever verbosity the console shows (rnfs.log hands each one here), with + the thread that logged it, and, for a warning or error logged while handling an exception, its traceback. +- Reticulum's own messages at its loglevel (interfaces failing to connect, paths, links), copied from its log + output, which still goes to the console as before; threads that die of an exception; and a status line every + STATUS_INTERVAL: the node's counts, its open links and each interface's state. +- Gateway requests (rnfs/gateway.py) are numbered, and the lines logged on a request's thread carry its number. + +The file names what the node looked up, fetched and served, so it is created readable by its owner only (on +Windows, the rnfs dir's access list) and nothing in it is sent anywhere. Every line has non-printable characters +escaped, as the console's do. A file that can't be written stops the debug log, with one message on stderr. +""" +import dataclasses +import os +import platform +import re +import sys +import threading +import time +import traceback +from contextlib import contextmanager + +from rnfs import __version__, fsutil, winacl +from rnfs import log as rnfs_log +from rnfs.fsutil import printable + +DEBUG_LOG_FILE = "debug.log" +MAX_SIZE = 16 * 1024 * 1024 # bytes per file; the one before is kept as debug.log.1 +STATUS_INTERVAL = 300.0 # seconds between status lines +# RNS's loglevels by number, named as its log lines name them +LEVEL_NAMES = {0: "Critical", 1: "Error", 2: "Warning", 3: "Notice", 4: "Info", 5: "Verbose", 6: "Debug", + 7: "Pathing", 8: "Extra"} +# RNS's own log line: "[time] [Level] message" (its compact format has neither, and is copied as is) +_RNS_LINE = re.compile(r"\[[^\]]*\] \[(\w+)\]\s*(.*)", re.DOTALL) +_LEVEL_NUMBERS = {name: number for number, name in LEVEL_NAMES.items()} + +_active: "DebugLog | None" = None +_previous_hook = None # threading.excepthook before start() + + +def current() -> "DebugLog | None": + """The debug log, if the daemon runs with one.""" + return _active + + +def path_in(rnfs_root: str) -> str: + """Where the debug log of the rnfs dir rnfs_root is.""" + return os.path.join(rnfs_root, DEBUG_LOG_FILE) + + +class DebugLog: + """The open file: whole lines, written as they come from any thread, each flushed at once.""" + + def __init__(self, path: str, settings: dict, max_size: int = MAX_SIZE): + """Move any earlier file at path aside and start one with the header. Raises OSError.""" + self.path = path + self.settings = settings + self.max_size = max_size + self._lock = threading.Lock() + self._local = threading.local() # .tag: what this thread's lines are about (a gateway request) + self._fd: int | None = None + self._size = 0 + self._broken = False + with self._lock: + self._begin(continued=False) + + def _begin(self, continued: bool): + """Move the current file to path.1 and start a new one, version first (lock held). Raises OSError.""" + if self._fd is not None: + os.close(self._fd) + self._fd = None + if os.path.lexists(self.path): + fsutil.replace(self.path, self.path + ".1") + # O_EXCL and O_NOFOLLOW: never append to a file or symlink put in its place + self._fd = os.open(self.path, os.O_WRONLY | os.O_CREAT | os.O_EXCL | fsutil.O_NOFOLLOW | fsutil.O_BINARY, + 0o600) + winacl.make_private(self.path) # Windows ignores the mode; elsewhere this does nothing + self._size = 0 + now = time.strftime("%Y-%m-%d %H:%M:%S %z") + lxmf = _package_version("lxmf") + for line in (f"rnfs {__version__} debug log, " + (f"continued {now} (earlier lines: {DEBUG_LOG_FILE}.1)" + if continued else f"started {now} (pid {os.getpid()})"), + f"Python {platform.python_version()} on {platform.platform()}; RNS {_package_version('rns')}; " + f"LXMF {lxmf or 'not installed'}", + f"settings other than the defaults: {self.settings}"): + self._put(printable(line) + "\n") + + def _put(self, text: str): + """Write text, already escaped (lock held). Raises OSError.""" + data = text.encode("utf-8", "backslashreplace") + os.write(self._fd, data) + self._size += len(data) + + def write(self, level: int, source: str, text: str, trace: str | None = None): + """One line, from source ("rnfs", "RNS", "status"…) at level, with a traceback below it if given.""" + stamp = time.time() + when = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(stamp)) + f".{int(stamp % 1 * 1000):03d}" + where = getattr(self._local, "tag", None) or threading.current_thread().name + line = f"[{when}] [{LEVEL_NAMES.get(level, level)}] [{printable(where)}] {source}: {printable(text)}\n" + if trace: + line += "".join(f" | {printable(part)}\n" for part in trace.rstrip("\n").split("\n")) + with self._lock: + if self._broken: + return + try: + if self._size + len(line) > self.max_size: + self._begin(continued=True) + self._put(line) + except OSError as e: + self._broken = True # not logged: that would come back here + print(f"rnfs: the debug log {self.path} stopped: {e}", file=sys.stderr, flush=True) + + def note(self, text: str, level: int = rnfs_log.LOG_DEBUG): + """A line for the debug log only (not the console), from rnfs.""" + self.write(level, "rnfs", text) + + @contextmanager + def tag(self, label: str): + """Mark the lines this thread writes in the with block as about label (e.g. "gateway #12").""" + previous = getattr(self._local, "tag", None) + self._local.tag = label + try: + yield + finally: + self._local.tag = previous + + def close(self): + """Close the file; later lines are dropped.""" + with self._lock: + self._broken = True + if self._fd is not None: + os.close(self._fd) + self._fd = None + + +def _package_version(name: str) -> str | None: + """The installed version of a distribution, or None.""" + try: + from importlib.metadata import version + return version(name) + except Exception: # noqa: BLE001 - not installed, or a bundle without metadata + return None + + +def _rnfs_message(level: int, text: str): + """rnfs.log's sink: every rnfs message; a warning or error with the traceback of the exception being handled, + if any (others logged in an except block are expected outcomes, such as a gateway refusal).""" + debug = _active + if debug is None: + return + trace = traceback.format_exc() if level <= rnfs_log.LOG_WARNING and sys.exc_info()[0] is not None else None + debug.write(level, "rnfs", text, trace) + + +def _rns_output(logstring: str): + """RNS's log output (LOG_CALLBACK): printed as RNS prints it, and copied here unless it's an rnfs message.""" + if threading.main_thread().is_alive(): # as RNS's own console output does + try: + print(logstring) + except Exception: # noqa: BLE001 - as RNS: a closed stdout mustn't break logging + pass + debug = _active + if debug is None or rnfs_log.passing_through(): + return + match = _RNS_LINE.fullmatch(logstring) + if match and match.group(1) in _LEVEL_NUMBERS: + debug.write(_LEVEL_NUMBERS[match.group(1)], "RNS", match.group(2)) + else: + debug.write(rnfs_log.LOG_INFO, "RNS", logstring) + + +def start(config) -> DebugLog: + """Open the debug log of config's rnfs dir and send it everything described above. Raises OSError.""" + global _active, _previous_hook + import RNS + defaults = type(config)() + settings = {field.name: getattr(config, field.name) for field in dataclasses.fields(config) + if getattr(config, field.name) != getattr(defaults, field.name)} + debug = DebugLog(path_in(config.root), settings) + _active = debug + rnfs_log.set_debug_sink(_rnfs_message) + if RNS.logdest == RNS.LOG_STDOUT: # the daemon's own; a file or callback someone set up is left alone + RNS.logcall = _rns_output + RNS.logdest = RNS.LOG_CALLBACK + previous_hook = _previous_hook = threading.excepthook + + def thread_died(args): + """Log a thread's uncaught exception, then report it as before.""" + if args.exc_type is not SystemExit: + name = args.thread.name if args.thread is not None else "a thread" + trace = "".join(traceback.format_exception(args.exc_type, args.exc_value, args.exc_traceback)) + debug.write(rnfs_log.LOG_ERROR, "rnfs", f"{name} died of {args.exc_type.__name__}", trace) + previous_hook(args) + + threading.excepthook = thread_died + return debug + + +def stop(): + """Stop sending anything to the debug log and close it (RNS's output goes back to the console alone).""" + global _active, _previous_hook + import RNS + debug = _active + if debug is None: + return + _active = None + rnfs_log.set_debug_sink(None) + if _previous_hook is not None: + threading.excepthook, _previous_hook = _previous_hook, None + if RNS.logdest == RNS.LOG_CALLBACK and RNS.logcall is _rns_output: + RNS.logdest = RNS.LOG_STDOUT + debug.close() + + +def status_line(node) -> str: + """The node's status (as `rnfs status` gives it), its open links, and each RNS interface's state.""" + import RNS + interfaces = [] + for interface in list(getattr(RNS.Transport, "interfaces", [])): + online = getattr(interface, "online", None) + interfaces.append(f"{interface} {'up' if online else 'down' if online is not None else '?'}" + f" rx {getattr(interface, 'rxb', 0)} B tx {getattr(interface, 'txb', 0)} B") + links = getattr(node, "links", None) + open_links = len(links.open_nodes()) if links is not None else 0 + return f"status {node.status()}; {open_links} open links; interfaces: " + ("; ".join(interfaces) or "none") diff --git a/rnfs/dht/lookup.py b/rnfs/dht/lookup.py index 645561e..8acf7ec 100644 --- a/rnfs/dht/lookup.py +++ b/rnfs/dht/lookup.py @@ -38,6 +38,15 @@ def _never(node_id: bytes) -> bool: return False +def _quiet(text: str) -> None: + """Default report: says nothing.""" + + +def _name(node_id: bytes) -> str: + """A node ID as the log writes it (RNS.prettyhexrep's form, without loading RNS).""" + return f"<{node_id.hex()}>" + + def _valid_id(node_id) -> bool: """True if node_id has the shape of a node ID.""" return isinstance(node_id, bytes) and len(node_id) == ID_LENGTH @@ -54,6 +63,8 @@ class Lookup: even after the run that sent it has returned, so the node can update its routing table. - `exclude(id)`: IDs to ignore in replies, as nodes and as providers (banned nodes). - `timeout`: the time budget, starting at the first run; later runs share it. + - `report(text)`: told each run's start and end, and each query's outcome (the node logs them at + LOG_DEBUG, for the debug log); called from the running thread and the query threads. A Lookup may be run again (find_providers after every provider failed): each run re-queries the nodes, so /find replies give a fresh random half of providers, within the same deadline and unknown-ID budget. @@ -64,7 +75,8 @@ def __init__(self, key: bytes, start_nodes: Iterable[bytes], query: FindQuery, o alpha: int = 3, beta: int = 3, k: int = K, timeout: float = 600, is_known: Callable[[bytes], bool] = _always, on_answer: NodeCallback = _ignore, on_failure: NodeCallback = _ignore, exclude: Callable[[bytes], bool] = _never, - clock: Callable[[], float] = time.monotonic, poll_interval: float = _POLL_INTERVAL): + clock: Callable[[], float] = time.monotonic, poll_interval: float = _POLL_INTERVAL, + report: Callable[[str], None] = _quiet): """A lookup for key; nothing is sent until find_closest() or find_providers() runs it.""" self.key = key self.query = query @@ -79,6 +91,7 @@ def __init__(self, key: bytes, start_nodes: Iterable[bytes], query: FindQuery, o self.exclude = exclude self.clock = clock self.poll_interval = poll_interval + self.report = report self.deadline: float | None = None self._failed: set[bytes] = set() # didn't answer: dropped for the rest of the lookup self._answered: set[bytes] = set() # answered in any run @@ -186,6 +199,8 @@ def _run(self, on_provider: NodeCallback | None, should_stop: Callable[[], bool] self.deadline = self.clock() + self.timeout results: queue.Queue = queue.Queue() self._wake = results + self.report(f"lookup for {self.key.hex()}" + (" (providers)" if on_provider is not None else "") + + f": {len(self._start_nodes)} start nodes, {max(0.0, self.deadline - self.clock()):.0f}s left") try: return self._iterate(results, on_provider, should_stop) finally: @@ -207,14 +222,19 @@ def _iterate(self, results: queue.Queue, on_provider: NodeCallback | None, window = max(self.alpha, self.beta) self._last_known = known # live: find_closest reads them once the run ends self._last_queried = queried + started = self.clock() + why = "stopped" while True: if on_provider is not None and should_stop(): + why = "enough providers" break if self.clock() >= self.deadline: + why = "out of time" break ranked = sorted(known, key=known.__getitem__) if ranked and all(node_id in answered for node_id in ranked[:self.beta]): + why = "complete" break # the beta closest known nodes have all answered: complete # Keep up to alpha queries in flight, to the closest nodes not yet queried. Only nodes ranked @@ -232,10 +252,12 @@ def _iterate(self, results: queue.Queue, on_provider: NodeCallback | None, self._send(node_id, results) in_flight += 1 if in_flight == 0: + why = "no node left to ask" break # every known node was queried and nothing new turned up wait = min(self.deadline - self.clock(), self.poll_interval) if wait <= 0: + why = "out of time" break try: item = results.get(timeout=wait) @@ -270,16 +292,27 @@ def _iterate(self, results: queue.Queue, on_provider: NodeCallback | None, self._reported.add(provider) found.append(provider) on_provider(provider) + failed = len(queried & self._failed) + self.report(f"lookup for {self.key.hex()}: {why} after {self.clock() - started:.1f}s; asked {len(queried)}, " + f"{len(answered)} answered, {failed} failed, {in_flight} still out" + + (f"; {len(found)} new providers" if on_provider is not None else "")) return found def _send(self, node_id: bytes, results: queue.Queue): """Query node_id on a worker thread, which reports the outcome and queues it for the run.""" def work(): """Run the blocking query, report it, and hand the cleaned reply to the run.""" + started = self.clock() try: reply = self._clean(self.query(node_id, self.key)) - except Exception: + except Exception as e: reply = None # one broken node must not abort the lookup + self.report(f"lookup for {self.key.hex()}: asking {_name(node_id)} raised {e!r}") + else: + took = self.clock() - started + self.report(f"lookup for {self.key.hex()}: {_name(node_id)} " + ( + f"failed after {took:.1f}s" if reply is None + else f"answered in {took:.1f}s: {len(reply[0])} providers, {len(reply[1])} nodes")) try: (self.on_failure if reply is None else self.on_answer)(node_id) except Exception: diff --git a/rnfs/dht/providers.py b/rnfs/dht/providers.py index bde89d1..7753f32 100644 --- a/rnfs/dht/providers.py +++ b/rnfs/dht/providers.py @@ -618,22 +618,26 @@ def save(self): with self._lock: if not self._dirty: return + # Only copied under the lock (records change in place); converted and written without it + publishers = [(publisher, entry.first_seen) for publisher, entry in self._publishers.items()] + records = [(key, publisher, record.first_stored, record.last_stored, record.pooled) + for key, held in self._records.items() for publisher, record in held.items()] + departures = list(self._departed.items()) + evictions = list(self._evicted.items()) + self._dirty = False + try: state = { "version": _STATE_VERSION, - "publishers": {publisher.hex(): entry.first_seen - for publisher, entry in self._publishers.items()}, - "records": [[key.hex(), publisher.hex(), record.first_stored, record.last_stored, - int(record.pooled)] - for key, records in self._records.items() for publisher, record in records.items()], + "publishers": {publisher.hex(): first_seen for publisher, first_seen in publishers}, + "records": [[key.hex(), publisher.hex(), first_stored, last_stored, int(pooled)] + for key, publisher, first_stored, last_stored, pooled in records], "departed": [[publisher.hex(), first_seen, departed] - for publisher, (first_seen, departed) in self._departed.items()], + for publisher, (first_seen, departed) in departures], "evicted": [[publisher.hex(), first_seen, evicted] - for publisher, (first_seen, evicted) in self._evicted.items()], + for publisher, (first_seen, evicted) in evictions], } - self._dirty = False - directory = os.path.dirname(os.path.abspath(self.state_path)) - os.makedirs(directory, exist_ok=True) - try: + directory = os.path.dirname(os.path.abspath(self.state_path)) + os.makedirs(directory, exist_ok=True) fsutil.write_atomic(self.state_path, json.dumps(state, separators=(",", ":")).encode(), mode=0o600) except BaseException: with self._lock: diff --git a/rnfs/dht/routing.py b/rnfs/dht/routing.py index bb2081b..f8ab01c 100644 --- a/rnfs/dht/routing.py +++ b/rnfs/dht/routing.py @@ -171,13 +171,24 @@ def closest(self, key: bytes, count: int = K, for_reply: bool = False) -> list[b silent over 2 × announce_interval, so departed nodes stay out of replies without a table refresh. """ check_id(key) + # Distances to key rank whole buckets: those in key's own bucket b are under 2^b, those in every bucket + # below it in [2^b, 2^(b+1)), and those in a bucket i above it in [2^i, 2^(i+1)). So the buckets are + # walked in that order and only those needed for count are sorted, not the whole table (this answers + # /find requests). key is own_id: b is -1, and the buckets rank in order from 0 + b = distance(self.own_id, key).bit_length() - 1 + walk = ([[b]] if b >= 0 else []) + [range(max(b, 0))] + [[i] for i in range(b + 1, BUCKET_COUNT)] + groups: list[list[bytes]] = [] + found = 0 with self._lock: now = self.clock() - # The key's bucket may be sparse or empty, so rank every node rather than one bucket - entries = [e for bucket in self._buckets for e in bucket.values() - if not (for_reply and self._stale(e, now))] - entries.sort(key=lambda e: distance(e.node_id, key)) - return [e.node_id for e in entries[:count]] + for indices in walk: + group = [e.node_id for i in indices for e in self._buckets[i].values() + if not (for_reply and self._stale(e, now))] + groups.append(group) + found += len(group) + if 0 < count <= found: + break + return [n for group in groups for n in sorted(group, key=lambda node_id: distance(node_id, key))][:count] def seniority(self, node_id: bytes) -> float | None: """Seconds since node_id was first seen (kept across a short absence), or None if not in the table.""" diff --git a/rnfs/features/jobs.py b/rnfs/features/jobs.py index 45fc94c..dc109d8 100644 --- a/rnfs/features/jobs.py +++ b/rnfs/features/jobs.py @@ -395,12 +395,12 @@ def status(self) -> dict: states = [job.state for job in self._jobs.values()] return {"jobs_running": states.count(RUNNING), "jobs_queued": states.count(QUEUED)} - def _schedule(self): - """Start queued jobs, oldest first, while fewer than max_jobs run.""" + def _schedule(self) -> bool: + """Start queued jobs, oldest first, while fewer than max_jobs run; True if any started (and were saved).""" starting = [] with self._lock: if not self._started or self._stop_event.is_set(): - return + return False running = sum(1 for job in self._jobs.values() if job.state == RUNNING) for job in self._jobs.values(): if running >= self.max_jobs: @@ -416,6 +416,7 @@ def _schedule(self): thread.start() if starting: self._save_quietly() + return bool(starting) def _run(self, job: Job): """Run a job to its end on this thread, then start the next.""" @@ -516,8 +517,8 @@ def _finish(self, job: Job, state: str, error: str | None, result: dict | None): if state in FINISHED: job.ended.set() _log(f"job {job.id} ({job.kind} {job.target}) {state}" + (f": {error}" if error else "")) - self._save_quietly() - self._schedule() + if not self._schedule(): # starting the next saves this outcome with it + self._save_quietly() # Operations (API) @@ -551,8 +552,8 @@ def submit(self, kind: str, target: str, output: str | None = None, from_node: s self._protect(job) self._jobs[job_id] = job _log(f"job {job_id} queued: {kind} {target}" + (f" to {output}" if output else "")) - self._save_quietly() - self._schedule() + if not self._schedule(): # else saved as it started + self._save_quietly() return job.to_dict() def _job(self, job_id: str) -> Job: diff --git a/rnfs/fetcher.py b/rnfs/fetcher.py index 45d87b7..57b7123 100644 --- a/rnfs/fetcher.py +++ b/rnfs/fetcher.py @@ -64,10 +64,12 @@ import time from collections import OrderedDict, deque from contextlib import contextmanager +from itertools import islice from typing import Callable, Iterator from rnfs.config import Config -from rnfs.constants import CHUNK_SIZE, KIB, MAX_BLOCK_SIZE, MAX_CHILDREN, MAX_DEPTH, MAX_REQUEST_SIZE +from rnfs.constants import (CHUNK_SIZE, KIB, MAX_BLOCK_SIZE, MAX_CHILDREN, MAX_DEPTH, MAX_REQUEST_SIZE, + MAX_RESPONSE_SIZE, RESPONSE_OVERHEAD) from rnfs.content import walker from rnfs.content.cid import Cid from rnfs.content.storage import Storage, read_json, write_json @@ -863,23 +865,29 @@ def _assign(self, pending: deque, in_flight: dict, outstanding: set[Cid], wanted busy_providers += 1 def _take_batch(self, pending: deque, node_id: bytes, wanted: dict[Cid, list[Expect]]) -> list[Cid]: - """The next CIDs to ask node_id for: what its budget and one request packet can hold, at least one. + """The next CIDs to ask node_id for: what one response can carry and one request packet on its link can + hold, at least one. Under a DAG limit, only what the room it leaves can take for sure (_room), but at least one CID when no request is in flight: so a fetch downloads at most about one block beyond the limit. """ budget = rates.requester_max_bytes(self.client.rate_estimate(node_id), self.config) + # However large max_bytes is, the responder's budget is at most this (server.blocks_budget, section 7) + carried = min(budget, MAX_RESPONSE_SIZE - RESPONSE_OVERHEAD) + # The client sends what fits its link's packet and returns the rest: the open link's MDU, else the + # largest request any link allows (test fakes may not know links). Whether a prefix fits only gets + # worse as it grows, so one count covers every batch taken from the front of pending + link_mdu = getattr(self.client, "link_mdu", None) + mdu = (link_mdu(node_id) if link_mdu is not None else None) or MAX_REQUEST_SIZE + fit = max(1, protocol.blocks_batch_size(budget, list(islice(pending, MAX_BATCH)), mdu)) room = self._room(wanted) batch: list[Cid] = [] total = 0 - while pending and len(batch) < MAX_BATCH: + while pending and len(batch) < fit: cid = pending[0] known = wanted[cid][0].size if cid.is_raw else None guess = known if known is not None else (CHUNK_SIZE if cid.is_raw else DAG_NODE_GUESS) - if batch and total + guess > budget: - break - # The largest request any link allows; the client sends what fits its link and returns the rest - if batch and protocol.blocks_batch_size(budget, batch + [cid], MAX_REQUEST_SIZE) <= len(batch): + if batch and total + guess > carried: break if room is not None: room = self._reserve(cid, wanted, room) @@ -905,6 +913,9 @@ def work(): result = self.client.blocks(node_id, cids, on_progress=progress) except Exception as e: result = peer_client.BlocksResult(peer_client.FAILED, unsent=list(cids), reason=str(e)) + if result.elapsed is not None: + # When it actually went out: opening the link or waiting for its /blocks slot isn't its time + sent_at = max(sent_at, self.fetcher.clock() - result.elapsed) self._events.put(("result", node_id, cids, result, sent_at, rate)) threading.Thread(target=work, name="rnfs-fetch", daemon=True).start() diff --git a/rnfs/gateway.py b/rnfs/gateway.py index 76de215..72519a8 100644 --- a/rnfs/gateway.py +++ b/rnfs/gateway.py @@ -38,9 +38,13 @@ - Connections are limited as for the API (rnfs/api.py): the request must arrive within request_timeout, at most max_connections are served at once (more get a 503 carrying the headers above), and a response write may block response_timeout, so clients that stop reading can't hold every slot for ever. +- With the debug log (rnfs/debuglog.py), each request is numbered and logged there with the headers that decide + how it is served (DEBUG_HEADERS), then its status, bytes written and time; what the gateway decided on the way + (name resolutions, stored or fetched, index or listing, type and range) is logged at LOG_DEBUG. """ import codecs import html +import itertools import mimetypes import re import threading @@ -49,11 +53,12 @@ from urllib.parse import parse_qs, quote, unquote, urlsplit from rnfs.api import LocalServer, TimedHandler, error_status, host_allowed, loggable, parse_hex_id, raw_response +from rnfs import debuglog from rnfs import gateway_formats # --- content --- from rnfs.content.cid import Cid from rnfs.content.seal import is_sealed # --- content --- from rnfs.errors import NotFound -from rnfs.log import LOG_ERROR, LOG_VERBOSE, log +from rnfs.log import LOG_DEBUG, LOG_ERROR, LOG_VERBOSE, log CSP_SCRIPTS = ("sandbox allow-scripts allow-forms allow-popups; " "default-src 'self' 'unsafe-inline' 'unsafe-eval' data: blob:; form-action 'self'") @@ -70,6 +75,10 @@ 422: "invalid content: it breaks the rnfs format's rules", 503: "the node is busy; try again shortly", 507: "not enough free disk space on this node", 500: "internal error; see the rnfs daemon's log"} RANGE_NUMBER = re.compile(r"[0-9]{1,18}") # ASCII digits only (str.isdigit() takes "¹²³"), and below 2**63 +# Request headers the debug log shows: those that decide how a request is answered +DEBUG_HEADERS = ("Host", "Range", "If-Range", "If-None-Match", "Sec-Fetch-Site", "Sec-Fetch-Mode", "Sec-Fetch-Dest", + "User-Agent") +_request_numbers = itertools.count(1) # the debug log's numbers for gateway requests # Types some platforms' mimetypes tables lack or get wrong (on Windows, mimetypes also reads the registry, # which installed software edits), so the commonest web types are fixed here too @@ -256,6 +265,24 @@ def put(self, name: str, cid: str): self.entries.popitem(last=False) +class _CountingWriter: + """A response stream counting the bytes written to it, for the debug log; otherwise the stream itself.""" + + def __init__(self, stream): + """Count what is written to stream.""" + self._stream = stream + self.count = 0 + + def write(self, data) -> int: + """Write data and count it.""" + written = self._stream.write(data) + self.count += len(data) + return written + + def __getattr__(self, name): + return getattr(self._stream, name) + + class GatewayServer(LocalServer): """The gateway for node (an RnfsNode, or anything with its facade methods), using config's gateway settings.""" @@ -278,6 +305,7 @@ class GatewayHandler(TimedHandler): names: NameCache = None server_version = "rnfs" sys_version = "" + status_sent: int | None = None # the response's status, once sent (for the debug log) def log_message(self, format, *args): """Log nothing: RNS does the node's logging, and per-request lines would drown it out.""" @@ -293,6 +321,11 @@ def busy_response(cls) -> bytes: # Responses + def send_response(self, code: int, message: str | None = None): + """Send the status line, noting the status.""" + self.status_sent = code + super().send_response(code, message) + def end_headers(self): """Add the headers every response carries, then end the headers.""" self.send_header("X-Content-Type-Options", "nosniff") @@ -351,6 +384,24 @@ def do_OPTIONS(self): self.dispatch() def dispatch(self): + """Serve the request (handle_request); with the debug log, logged there with its outcome.""" + debug = debuglog.current() + if debug is None: + return self.handle_request() + number = next(_request_numbers) + started = time.monotonic() + self.wfile = _CountingWriter(self.wfile) + shown = ", ".join(f"{name}: {self.headers[name]}" for name in DEBUG_HEADERS if self.headers.get(name) is not None) + with debug.tag(f"gateway #{number}"): + debug.note(f"gateway {self.command} {self.path}" + (f" ({shown})" if shown else "")) + try: + self.handle_request() + finally: + debug.note(f"gateway answered {self.status_sent or 'nothing'}: {self.wfile.count} bytes written " + f"(headers included) in {(time.monotonic() - started) * 1000:.0f} ms" + + (", holding a gateway_fetches slot" if self.slot else "")) + + def handle_request(self): """Check the request, serve it, and turn every failure into a response carrying the common headers.""" self.close_connection = True self.headers_sent = False @@ -363,8 +414,10 @@ def dispatch(self): self.request_read() # GET and HEAD have no body self.serve() except GatewayError as e: + log(f"gateway refused {loggable(self.path)} with {e.status}: {e}", LOG_DEBUG) self.respond_error(e.status, str(e), e.headers) except NotFound as e: + log(f"gateway: {loggable(self.path)} not found: {e}", LOG_DEBUG) self.respond_error(404, str(e) or "not found", {}) except Exception as e: # noqa: BLE001 - every error becomes a response, never a dropped connection status = error_status(e) @@ -432,6 +485,10 @@ def serve(self): target = "/".join([root, *names]) if self.limited() and not self.node.is_local(target): self.need_network() + if debuglog.current() is not None: + log(f"gateway: {target} is " + ("stored here" if self.node.is_local(target) else "not all stored here; " + "fetching what it needs") + (", for a web page" if self.limited() else ""), + LOG_DEBUG) info = self.node.info(target, for_gateway=True) etag = f'"{info["cid"]}"' if immutable and etag_matches(self.headers.get("If-None-Match"), etag): @@ -471,10 +528,14 @@ def resolve_name(self, name: str) -> str: self.need_network() cid = self.node.name_resolve(name, for_gateway=True)["cid"] self.names.put(name, cid) + log(f"gateway resolved name {name} to {cid}", LOG_DEBUG) + else: + log(f"gateway: name {name} is {cid} (resolved within {NAME_CACHE_TIME:g} s)", LOG_DEBUG) return cid def send_not_modified(self, cid: str): """304 for a cached immutable response.""" + log(f"gateway: {cid} not modified", LOG_DEBUG) self.send_response(304) self.send_header("ETag", f'"{cid}"') self.send_header("Cache-Control", IMMUTABLE) @@ -494,7 +555,9 @@ def serve_directory(self, target: str, segments: list[str], names: list[str], ca index_target = f"{target}/{INDEX}" if self.limited() and not self.node.is_local(index_target): self.need_network() # the directory may be stored without its index.html + log(f"gateway: serving {index_target}", LOG_DEBUG) return self.serve_file(index_target, entry["size"], INDEX, cache) + log(f"gateway: listing {target} ({len(entries)} entries)", LOG_DEBUG) title = "/" + "/".join([segments[1], segments[2], *names]) + "/" self.send_body(200, "text/html; charset=utf-8", listing_html(title, entries, bool(names)), cache) @@ -518,6 +581,7 @@ def serve_file(self, target: str, size: int, filename: str | None, cache: dict): else: status, start, length = 206, byte_range[0], byte_range[1] - byte_range[0] + 1 headers["Content-Range"] = f"bytes {byte_range[0]}-{byte_range[1]}/{size}" + log(f"gateway: {target} as {content_type}, {length} bytes from byte {start} of {size}", LOG_DEBUG) if self.command == "HEAD": return self.send_stream(status, content_type, length, headers, iter(())) diff --git a/rnfs/log.py b/rnfs/log.py index 84f3e6c..d2ae875 100644 --- a/rnfs/log.py +++ b/rnfs/log.py @@ -1,4 +1,9 @@ -"""Logging through RNS's logger, so rnfs messages share its output, timestamps and loglevel.""" +"""Logging through RNS's logger, so rnfs messages share its output, timestamps and loglevel. + +With `debug_log` on, every message also goes to the debug log file (rnfs/debuglog.py), whatever RNS's loglevel. +""" +import threading + import RNS from rnfs.fsutil import printable @@ -7,14 +12,40 @@ MAX_WARNINGS = 100 # Re-exported so callers only need this module for logging -from RNS import LOG_ERROR, LOG_INFO, LOG_NOTICE, LOG_VERBOSE, LOG_WARNING # noqa: F401 +from RNS import LOG_DEBUG, LOG_ERROR, LOG_INFO, LOG_NOTICE, LOG_VERBOSE, LOG_WARNING # noqa: F401 + +# sink(level, text) given every message as well, at any level: the debug log's (set_debug_sink), or None +_debug_sink = None +# Set while an rnfs message is inside RNS.log, so the debug log's copy of RNS's own output can skip it +_passing = threading.local() def log(message: str, level: int = LOG_INFO): """Log message with an "rnfs" prefix at the given RNS loglevel, non-printable characters escaped.""" # Messages carry remote-chosen names (DAG entries, LXMF links, request paths): escaping here, once, # keeps escape sequences and fake newlines out of every log line without each caller remembering - RNS.log(f"rnfs {printable(message)}", level) + text = printable(message) + sink = _debug_sink + if sink is None: + RNS.log(f"rnfs {text}", level) + return + _passing.active = True + try: + RNS.log(f"rnfs {text}", level) + finally: + _passing.active = False + sink(level, text) + + +def set_debug_sink(sink): + """Give every message from now on to sink(level, text) too (None: stop).""" + global _debug_sink + _debug_sink = sink + + +def passing_through() -> bool: + """True while this thread's rnfs message is inside RNS.log (it reaches the debug log by its sink).""" + return getattr(_passing, "active", False) def node_name(node_id: bytes | None) -> str: diff --git a/rnfs/names.py b/rnfs/names.py index b070885..2ed3788 100644 --- a/rnfs/names.py +++ b/rnfs/names.py @@ -253,6 +253,9 @@ def __init__(self, config: Config, pins, network=None, clock=time.time, monotoni self._floor_dirty = False self._floor_saved_at = float("-inf") # monotonic time of the last floor save self._floor_timer: threading.Timer | None = None # a save waiting for FLOOR_SAVE_INTERVAL to pass + # Floor saves write without self._lock, one at a time, so an older snapshot never lands last. Taken before + # self._lock, never while holding it + self._floor_save_lock = threading.Lock() self._held_dirty = False # heard times changed without a save self._destinations: dict[str, object] = {} @@ -517,7 +520,7 @@ def stop(self): self._floor_timer.cancel() self._floor_timer = None self._save() - self._save_floor() + self._save_floor() def announce_due(self) -> float: """Re-announce every name whose announce_interval has passed; seconds until the next is due.""" @@ -528,11 +531,12 @@ def announce_due(self) -> float: self._announce(alias) if self._held_dirty: self._save() - if self._floor_dirty: - self._save_floor() if not self._next_announce: - return self.config.announce_interval - return max(0.0, min(self._next_announce.values()) - now) + wait = self.config.announce_interval + else: + wait = max(0.0, min(self._next_announce.values()) - now) + self._save_floor() + return wait def run(self, stop_event: threading.Event): """The re-announce loop, for a thread of its own; returns once stop_event is set.""" @@ -577,6 +581,7 @@ def resolve(self, name: bytes, persist: bool, max_age: float = 0.0) -> dict | No # Again: the bounded table may have evicted it meanwhile, letting a replayed announce in self._offer_kept(name) + save_floor = False with self._lock: self._absorb_recalled(name) record = self._record(name, touch=True) @@ -587,9 +592,12 @@ def resolve(self, name: bytes, persist: bool, max_age: float = 0.0) -> dict | No self._held[name] = record self._save() elif name not in self._held: - self._resolved(name, record.sequence) - return {"name": name.hex(), "cid": str(record.cid), "sequence": record.sequence, - "age": max(0.0, self._clock() - record.heard)} + save_floor = self._resolved(name, record.sequence) + result = {"name": name.hex(), "cid": str(record.cid), "sequence": record.sequence, + "age": max(0.0, self._clock() - record.heard)} + if save_floor: + self._save_floor() + return result def _offer_kept(self, name: bytes): """Offer the signed record keepers hold for name, so it is a floor no replayed announce gets under. @@ -769,10 +777,11 @@ def _drop_from_table(self, name: bytes): self._table.pop(name, None) self._heard_only.pop(name, None) - def _resolved(self, name: bytes, sequence: int): + def _resolved(self, name: bytes, sequence: int) -> bool: """A resolve without persist returned name's record: it is no longer only heard, and its sequence is the floor for later resolves (_trim_floor says which are forgotten). A floor raised (or a new one) is saved at - once, or FLOOR_SAVE_INTERVAL after the last save, so a crash soon after loses little.""" + once, or FLOOR_SAVE_INTERVAL after the last save, so a crash soon after loses little. True if the caller + is to save it now, once it has released the lock.""" self._heard_only.pop(name, None) now = self._clock() previous = self._floor.pop(name, None) @@ -780,9 +789,10 @@ def _resolved(self, name: bytes, sequence: int): trimmed = self._trim_floor() if previous is None or self._floor[name][0] != previous[0] or trimmed: self._floor_dirty = True - self._save_floor_soon() - elif not 0 <= now - previous[1] < FLOOR_TOUCH_SAVE: + return self._save_floor_soon() + if not 0 <= now - previous[1] < FLOOR_TOUCH_SAVE: self._floor_dirty = True # only its time renewed: saved with the next change or by announce_due + return False def _trim_floor(self) -> bool: """Forget the least recently resolved names over FLOOR_FACTOR × name_table_size, but not those resolved @@ -798,22 +808,24 @@ def _trim_floor(self) -> bool: trimmed = True return trimmed - def _save_floor_soon(self): - """Save the floor now if the last save was FLOOR_SAVE_INTERVAL ago, else once it is (a timer).""" + def _save_floor_soon(self) -> bool: + """True if the floor is to be saved now (the last save was FLOOR_SAVE_INTERVAL ago), by the caller once it + has released the lock; else it is saved once it is (a timer).""" wait = self._floor_saved_at + FLOOR_SAVE_INTERVAL - self._monotonic() if wait <= 0: - self._save_floor() - elif self._floor_timer is None: + return True + if self._floor_timer is None: timer = threading.Timer(wait, self._save_floor_later) timer.daemon = True self._floor_timer = timer timer.start() + return False def _save_floor_later(self): """The timer's save.""" with self._lock: self._floor_timer = None - self._save_floor() + self._save_floor() def _floor_size(self) -> int: """How many names the floor holds, past which only names resolved within FLOOR_KEEP_AGE are kept.""" @@ -981,19 +993,32 @@ def _load_floor(self): def _save_floor(self): """Write the floor atomically, least recently resolved first, as [name, sequence, resolved], if it changed. Saved by a resolve that raised it (at most every FLOOR_SAVE_INTERVAL, since it holds thousands - of names), announce_due (at least every minute while running) and stop().""" - if not self._floor_dirty: - return - state = {"version": FLOOR_VERSION, - "floor": [[name.hex(), sequence, int(at)] for name, (sequence, at) in self._floor.items()]} - os.makedirs(self.config.state_dir, exist_ok=True) - try: - fsutil.write_atomic(self._floor_path, json.dumps(state, separators=(",", ":")).encode()) - except OSError as e: - log(f"could not save {self._floor_path}: {e}", LOG_WARNING) - return - self._floor_dirty = False - self._floor_saved_at = self._monotonic() + of names), announce_due (at least every minute while running) and stop(). + + Called without the lock: only the copy is taken under it, so resolves and announces needn't wait for + the write. The save counts from when the copy was taken; a failed one is retried by the next.""" + with self._floor_save_lock: + with self._lock: + if not self._floor_dirty: + return + floor = list(self._floor.items()) + saved_at, self._floor_saved_at = self._floor_saved_at, self._monotonic() + self._floor_dirty = False + saved = False + try: + state = {"version": FLOOR_VERSION, + "floor": [[name.hex(), sequence, int(at)] for name, (sequence, at) in floor]} + os.makedirs(self.config.state_dir, exist_ok=True) + try: + fsutil.write_atomic(self._floor_path, json.dumps(state, separators=(",", ":")).encode()) + saved = True + except OSError as e: + log(f"could not save {self._floor_path}: {e}", LOG_WARNING) + finally: + if not saved: + with self._lock: + self._floor_dirty = True + self._floor_saved_at = saved_at def _save(self): """Write published and held records atomically.""" diff --git a/rnfs/network/client.py b/rnfs/network/client.py index f84cfc5..0896f2b 100644 --- a/rnfs/network/client.py +++ b/rnfs/network/client.py @@ -33,6 +33,7 @@ class BlocksResult: unsent: list[Cid] = field(default_factory=list) # not answered: cut by the batch size or budget, or not OK bad: list[Cid] = field(default_factory=list) # blocks that didn't match their CID: ban the provider reason: str = "" # why the request stalled or failed, for logs + elapsed: float | None = None # seconds from sending the request to its answer, if OK (local, not on the wire) class PeerClient: @@ -60,6 +61,12 @@ def rate_estimate(self, node_id: bytes) -> float | None: peer = self.links.shared.get(node_id) return peer.rate if peer is not None else None + def link_mdu(self, node_id: bytes) -> int | None: + """The packet MDU of the open link to node_id, which bounds the requests sent on it, or None.""" + with self.links.lock: + peer = self.links.shared.get(node_id) + return peer.mdu() if peer is not None and peer.is_active() else None + def close(self, node_id: bytes): """Close the shared link to node_id, cancelling any transfer on it.""" self.links.close(node_id) @@ -221,7 +228,10 @@ def _blocks(self, node_id: bytes, cids: list[Cid], on_progress: Callable[[float] if peer.record_rate(exchange, self.config.min_rate_sample): log(f"link to {name} measured at {size(peer.rate / 8)}/s", LOG_VERBOSE) - return self._check_blocks(name, batch, entries, rest) + result = self._check_blocks(name, batch, entries, rest) + # From when it was sent: not opening the link or waiting for its /blocks slot + result.elapsed = max(0.0, self.links.clock() - exchange.sent_at) + return result def _stall_watchdog(self, rate: float | None) -> Callable[[Exchange, float], str | None]: """Stall test for a /blocks response on a link with this rate estimate (section 7, step 5).""" diff --git a/rnfs/network/links.py b/rnfs/network/links.py index a66e4ef..b3714b7 100644 --- a/rnfs/network/links.py +++ b/rnfs/network/links.py @@ -25,7 +25,7 @@ from rnfs.config import Config from rnfs.constants import APP_NAME, DHT_ASPECT, MAX_REQUESTS_PER_LINK -from rnfs.log import LOG_VERBOSE, LOG_WARNING, log, node_name +from rnfs.log import LOG_DEBUG, LOG_VERBOSE, LOG_WARNING, log, node_name, size from rnfs.network import guards, protocol, rates POLL_INTERVAL = 0.25 # how often a waiting request checks its link, progress and stall conditions @@ -34,6 +34,27 @@ DEADLINE_FACTOR = 2 # a request other than /blocks may take this many times RNS's own request timeout # What RNS adds to rtt × traffic_timeout_factor for its request timeout (Link.request with timeout=None) RESPONSE_GRACE = protocol.REQUEST_TIMEOUT_WITHOUT_RTT +# Why RNS closed a link (Link.teardown_reason), for the log +CLOSE_REASONS = {RNS.Link.TIMEOUT: "timed out", RNS.Link.INITIATOR_CLOSED: "closed by this node", + RNS.Link.DESTINATION_CLOSED: "closed by the other node"} +RECEIPT_FAILED = RNS.RequestReceipt.FAILED # a request RNS gave up on + + +def close_reason(link) -> str: + """Why link closed, as the log says it.""" + return CLOSE_REASONS.get(getattr(link, "teardown_reason", None), "closed") + + +def path_description(node_id: bytes) -> str: + """How far RNS's path to node_id goes and through which interface, for the log ("" if it can't tell).""" + try: + hops = RNS.Transport.hops_to(node_id) + interface = RNS.Transport.next_hop_interface(node_id) + except Exception: + return "" + if hops >= RNS.Transport.PATHFINDER_M: + return "" + return f"{hops} hop{'s' if hops != 1 else ''}" + (f" via {interface}" if interface is not None else "") class Exchange: @@ -253,10 +274,11 @@ def _open(self, peer: PeerLink) -> bool: if not RNS.Transport.await_path(node_id, timeout=self.config.path_timeout): log(f"no path to {name} after {self.config.path_timeout:g}s") return False + log(f"path to {name}: {path_description(node_id) or 'known'}", LOG_DEBUG) identity = self._recall(node_id) if identity is None: - log(f"have a path to {name} but not its identity") + log(f"have a path to {name} but not its identity (none after {self.config.path_timeout:g}s)") return False destination = RNS.Destination(identity, RNS.Destination.OUT, RNS.Destination.SINGLE, APP_NAME, DHT_ASPECT) @@ -276,15 +298,19 @@ def on_closed(closed_link): """Forget the link; RNS runs this synchronously inside teardown, so it only takes our lock.""" self._forget(peer) settled.set() - log(f"link to {name} closed", LOG_VERBOSE) + log(f"link to {name} {close_reason(closed_link)}", LOG_VERBOSE) log(f"connecting to {name}", LOG_VERBOSE) + started = time.monotonic() link = RNS.Link(destination, established_callback=lambda _: settled.set(), closed_callback=on_closed) peer.link = link timeout = getattr(link, "establishment_timeout", None) or self.config.path_timeout settled.wait(timeout + ESTABLISH_GRACE) if link.status != RNS.Link.ACTIVE: - log(f"could not connect to {name}") + route = path_description(node_id) + why = (close_reason(link) if link.status == RNS.Link.CLOSED + else f"not established within {time.monotonic() - started:.0f}s") + log(f"could not connect to {name}: {why}" + (f" ({route})" if route else "")) link.teardown() peer.mark_closed() return False @@ -448,43 +474,71 @@ def request(self, peer: PeerLink, path: str, data, blocks: bool = False, max_res return exchange exchange.receipt = receipt exchange.sent_at = exchange.last_progress = self.clock() - - reported = 0.0 - missing_since = None - while not exchange.done.wait(POLL_INTERVAL): - if not peer.is_active(): - break - # A response Resource cancelled by the responder never fails the receipt (RNS ignores the - # timeout once a response is being received), so notice that it's gone from the link - if exchange.response_started is not None and not exchange.answered: - if self._receiving(peer.link, getattr(receipt, "request_id", None)): - missing_since = None - elif missing_since is None: - missing_since = self.clock() - elif self.clock() - missing_since > CANCEL_GRACE: - exchange.cancelled = True - break - if on_progress is not None and exchange.progress > reported: - reported = exchange.progress - on_progress(reported) - if watchdog is not None: - reason = watchdog(exchange, self.clock()) - if reason: - exchange.stalled = reason - break - if deadline is not None and self.clock() - exchange.sent_at > deadline: - # Only reachable while RNS is receiving a response Resource: otherwise its own timeout - # fails the request first. Honest answers to these requests fit one packet. - exchange.stalled = f"no answer within {deadline:.0f}s" - log(f"{node_name(peer.node_id)} took over {deadline:.0f}s to answer {path}; giving up", - LOG_VERBOSE) - break - if exchange.answered and on_progress is not None and reported < 1.0: - on_progress(1.0) + try: + self._wait(peer, path, exchange, receipt, deadline, watchdog, on_progress) + finally: + self._log_outcome(peer, path, exchange) return exchange finally: peer.release(blocks) + def _wait(self, peer: PeerLink, path: str, exchange: Exchange, receipt, deadline: float | None, + watchdog: Callable[[Exchange, float], str | None] | None, on_progress: Callable[[float], None] | None): + """request()'s wait for the outcome of a request sent.""" + reported = 0.0 + missing_since = None + while not exchange.done.wait(POLL_INTERVAL): + if not peer.is_active(): + break + # A response Resource cancelled by the responder never fails the receipt (RNS ignores the + # timeout once a response is being received), so notice that it's gone from the link + if exchange.response_started is not None and not exchange.answered: + if self._receiving(peer.link, getattr(receipt, "request_id", None)): + missing_since = None + elif missing_since is None: + missing_since = self.clock() + elif self.clock() - missing_since > CANCEL_GRACE: + exchange.cancelled = True + break + if on_progress is not None and exchange.progress > reported: + reported = exchange.progress + on_progress(reported) + if watchdog is not None: + reason = watchdog(exchange, self.clock()) + if reason: + exchange.stalled = reason + break + if deadline is not None and self.clock() - exchange.sent_at > deadline: + # Only reachable while RNS is receiving a response Resource: otherwise its own timeout + # fails the request first. Honest answers to these requests fit one packet. + exchange.stalled = f"no answer within {deadline:.0f}s" + log(f"{node_name(peer.node_id)} took over {deadline:.0f}s to answer {path}; giving up", + LOG_VERBOSE) + break + if exchange.answered and on_progress is not None and reported < 1.0: + on_progress(1.0) + + def _log_outcome(self, peer: PeerLink, path: str, exchange: Exchange): + """Log, at LOG_DEBUG, how a request sent ended: answered (in what time, how large) or why not.""" + name = node_name(peer.node_id) + took = self.clock() - exchange.sent_at + if exchange.answered: + answer = "refused" if exchange.response is False else "answered" + log(f"{name} {answer} {path} in {took:.2f}s" + + (f" ({size(exchange.response_size)})" if exchange.response_size else ""), LOG_DEBUG) + return + if exchange.stalled: + why = exchange.stalled + elif exchange.cancelled: + why = "it cancelled its response" + elif not peer.is_active(): + why = "the link " + (close_reason(peer.link) if peer.link is not None else "closed") + elif getattr(exchange.receipt, "status", None) == RECEIPT_FAILED: + why = "RNS gave up on it (timed out, or the response failed or was too large)" + else: + why = "no outcome" + log(f"{path} to {name} not answered after {took:.1f}s: {why}", LOG_DEBUG) + @staticmethod def request_deadline(peer: PeerLink) -> float: """Seconds a request other than /blocks may take from being sent: DEADLINE_FACTOR × RNS's own diff --git a/rnfs/network/server.py b/rnfs/network/server.py index cd91c3d..aaa8712 100644 --- a/rnfs/network/server.py +++ b/rnfs/network/server.py @@ -28,9 +28,10 @@ from rnfs.constants import (BLOCKS_PATH, CHUNK_SIZE, FIND_PATH, MAX_REQUEST_SIZE, MAX_REQUESTS_PER_LINK, MAX_RESPONSE_SIZE, PIN_PATH, PROVIDE_PATH, RESPONSE_OVERHEAD) from rnfs.content.cid import Cid -from rnfs.log import LOG_ERROR, LOG_VERBOSE, LOG_WARNING, log, node_name, plural, size +from rnfs.log import LOG_DEBUG, LOG_ERROR, LOG_VERBOSE, LOG_WARNING, log, node_name, plural, size from rnfs.network import guards, protocol, rates from rnfs.network.identity import node_id_for_identity +from rnfs.network.links import close_reason from rnfs import mesh SLOW_POOL = "slow" @@ -96,6 +97,10 @@ def provide(self, publisher_id: bytes, keys: list[bytes], withdrawn: list[bytes] def served_block(self, cid: Cid) -> bytes | None: """cid's block if it's in the DAG of a CID this node provides, re-verified; else None.""" + def served_block_size(self, cid: Cid) -> int | None: + """Optional (looked up with getattr): the stored length of the block served_block(cid) would return, or + None if it isn't served (or isn't known). Lets /blocks stop at the block over its budget unread.""" + def remote_pin(self, identity_hash: bytes, cid: Cid, pin: bool) -> bool: """Accept (True) or refuse a /pin from the identity with this 16-byte hash.""" @@ -117,15 +122,22 @@ def blocks_budget(max_bytes: int, rate_estimate: float | None, upload_cap: float return min(limit, rates.transfer_bytes(response_target, rate)) -def select_blocks(cids: list[Cid], served_block: Callable[[Cid], bytes | None], limit: int) -> list[bytes | None]: +def select_blocks(cids: list[Cid], served_block: Callable[[Cid], bytes | None], limit: int, + served_size: Callable[[Cid], int | None] | None = None) -> list[bytes | None]: """Response entries for cids: each block or None, stopping before the first block over the budget. - The first served block is always included, even if it alone exceeds limit. + The first served block is always included, even if it alone exceeds limit. served_size(cid), if given, is + the length of the block served_block(cid) would return (None if unknown): the block over the budget is then + left out without being read and hashed. """ entries: list[bytes | None] = [] total = 0 served_any = False for cid in cids: + if served_any and served_size is not None: + known = served_size(cid) + if known is not None and total + known > limit: + break block = served_block(cid) if block is None: entries.append(None) @@ -710,6 +722,9 @@ def _link_established(self, link): RNS calls this on its packet-handling thread, so it only does quick work. """ self._track(link) + log(f"incoming link {RNS.prettyhexrep(link.link_id)} established" + + (f" on {link.attached_interface}" if getattr(link, "attached_interface", None) is not None else ""), + LOG_DEBUG) # Private mode: outsiders' links are closed first, so they can't push members' idle links out keep = self._member_links() if self.config.private else frozenset() admitted, victim = self.incoming.admit(link.link_id, keep) @@ -748,6 +763,9 @@ def _track(self, link): def _link_closed(self, link): """Forget a closed link and free everything its responses held.""" link_id = link.link_id + opened = getattr(link, "activated_at", None) + log(f"incoming link {RNS.prettyhexrep(link_id)} {close_reason(link)}" + + (f" after {time.time() - opened:.0f}s" if isinstance(opened, (int, float)) else ""), LOG_DEBUG) with self.lock: self.links.pop(link_id, None) self.watched.discard(link_id) @@ -930,7 +948,8 @@ def _handle_blocks(self, path, data, request_id, link_id, remote_identity): log("refused /blocks: that requester's responses in flight hold half of upload_budget", LOG_VERBOSE) return False limit = min(limit, max(0, share - RESPONSE_OVERHEAD)) - entries = select_blocks(cids, self.backend.served_block, limit) + entries = select_blocks(cids, self.backend.served_block, limit, + getattr(self.backend, "served_block_size", None)) body = protocol.encode_blocks_response(entries) packed = protocol.packed_response_size(request_id, body) mdu = link.get_mdu() or protocol.DEFAULT_MDU @@ -1030,7 +1049,8 @@ def _previous_done(self, link, link_id: bytes) -> bool: if getattr(resource, "status", 0) >= RNS.Resource.COMPLETE: self._concluded(response, link, resource) break - time.sleep(0.01) + # Its Resource's callback finishes it as the proof is processed (_hook); the status is a backup + response.done.wait(0.01) if not response.finished: return False return True @@ -1320,12 +1340,14 @@ def _check_progress(self, response: Response, now: float) -> bool: return True def _monitor_loop(self): - """Run check_responses until stopped.""" + """Run check_responses until stopped; with no response handed to RNS, wait for one (or stop).""" while not self.stop_event.is_set(): try: searching = self.check_responses() + # check_responses only looks at responses handed to RNS, and each handover sets wake + idle = not any(r.size is not None and not r.finished for r in self.pools.responses()) except Exception as e: log(f"error tracking /blocks responses: {e}", LOG_WARNING) - searching = False - self.wake.wait(MONITOR_FAST_TICK if searching else MONITOR_TICK) + searching = idle = False + self.wake.wait(None if idle else MONITOR_FAST_TICK if searching else MONITOR_TICK) self.wake.clear() diff --git a/rnfs/node.py b/rnfs/node.py index a760bd0..bbd8942 100644 --- a/rnfs/node.py +++ b/rnfs/node.py @@ -25,7 +25,7 @@ from rnfs import __version__, features, fsutil, winacl from rnfs.config import Config -from rnfs.constants import APP_NAME, DEFAULT_BOOTSTRAP_NODES, DHT_ASPECT, MODE_SERVER, PARTIAL_SUFFIX +from rnfs.constants import APP_NAME, DEFAULT_BOOTSTRAP_NODES, DHT_ASPECT, K, MODE_SERVER, PARTIAL_SUFFIX from rnfs.content import archive, builder, dag, edit, seal, walker from rnfs.content.cid import Cid from rnfs import node_content # --- content: add --no-pin/--to/--chunker, sealed files, gateway formats --- @@ -38,7 +38,7 @@ from rnfs.errors import InvalidDag, NotFound, RnfsError, TooLarge from rnfs.fetcher import Fetch, Fetcher, describe from rnfs.mesh import RnsPaths -from rnfs.log import LOG_ERROR, LOG_NOTICE, LOG_VERBOSE, LOG_WARNING, cap_warnings, log, node_name, plural, size +from rnfs.log import LOG_DEBUG, LOG_ERROR, LOG_NOTICE, LOG_VERBOSE, LOG_WARNING, cap_warnings, log, node_name, plural, size from rnfs.names import NameManager, parse_name from rnfs.pin_names import PinNames from rnfs.network import guards, rates @@ -53,7 +53,7 @@ REMOTE_PINS_FILE = "remote_pins.json" # remote pins accepted and not yet fetched: [[source, cid], ...] in order FETCHED_FILE = "fetched.json" # cached targets fetched for the user (not the gateway): the roots strategy's SAVE_INTERVAL = 300.0 # seconds between saves of provider records -STATE_SAVE_INTERVAL = 30.0 # seconds between saves of cache records and invalid roots, when they changed +STATE_SAVE_INTERVAL = 30.0 # seconds between saves of cache records, fetched targets and invalid roots, when changed ARCHIVE_BLOCK_OVERHEAD = 64 # bytes an archive spends on each block beyond the block (msgpack framing, CID) EVICT_INTERVAL = 60.0 # seconds between max_storage checks when nothing else triggers one GATEWAY_NAME_AGE = 60.0 # a gateway resolution reuses a record heard this recently @@ -210,6 +210,12 @@ def served_block(self, cid: Cid) -> bytes | None: self.node.refresh_soon() return block + def served_block_size(self, cid: Cid) -> int | None: + """The stored length of the block served_block(cid) would return, or None if it isn't served.""" + if not self.node.publisher.serves(cid): + return None + return self.node.storage.block_size(cid) + def remote_pin(self, identity_hash: bytes, cid: Cid, pin: bool) -> bool: """Accept a /pin from an allowed identity (the server checked remote_pin_identities).""" return self.node.remote_pin(identity_hash, cid, pin) @@ -320,6 +326,8 @@ def _setup(self, config: Config, node_id: bytes, client, slow: bool, names_netwo self._fetched.add(Cid.parse(text)) except (ValueError, TypeError): continue + self._fetched_dirty = False # changed since last saved (by _save_state: maintenance, gc and stop) + self._fetched_save_lock = threading.Lock() # one save at a time, so an older snapshot never lands last # Remote source -> CIDs waiting or being fetched for it, in the order accepted (fetched one at a time) self._remote_reserved: dict[str, dict[Cid, None]] = {} self._remote_released: set[tuple[str, Cid]] = set() # remote pins released while still being fetched @@ -392,8 +400,10 @@ def stop(self, timeout: float = STOP_TIMEOUT): self.links.close_all() def _save_state(self): - """Save cache records and invalid roots if they changed (each change isn't written at once).""" - for save in (self.storage.save, self.fetcher.save, *(f.save for f in self.features.values())): + """Save cache records, fetched targets and invalid roots if they changed (each change isn't written at + once).""" + for save in (self.storage.save, self._save_fetched, self.fetcher.save, + *(f.save for f in self.features.values())): try: save() except OSError as e: @@ -402,11 +412,19 @@ def _save_state(self): # DHT wiring def _lookup(self, key: bytes) -> Lookup: - """A lookup for key starting from the routing table, counting answers and failures in it.""" - return Lookup(key, self.routing.closest(key), self.client.find, self.node_id, + """A lookup for key starting from the routing table, counting answers and failures in it. + + It starts from the closest nodes that haven't failed since last heard or gone silent (those a /find + reply would give), with others only to make up K, so departed nodes don't take its first queries. + """ + start = self.routing.closest(key, for_reply=True) + if len(start) < K: + start += [n for n in self.routing.closest(key) if n not in start][:K - len(start)] + return Lookup(key, start, self.client.find, self.node_id, alpha=self.config.effective_alpha(self.slow), beta=self.config.beta, timeout=self.config.lookup_timeout, is_known=self._is_known, - on_answer=self._answered, on_failure=self.routing.failed, exclude=self.routing.is_banned) + on_answer=self._answered, on_failure=self.routing.failed, exclude=self.routing.is_banned, + report=lambda text: log(text, LOG_DEBUG)) def _answered(self, node_id: bytes): """A node answered /find, proving it is a reachable server.""" @@ -561,9 +579,11 @@ def _enforce_storage(self): def _make_room(self, nbytes: int): """Evict cached fetches so nbytes more fit in max_storage, withdrawing targets no longer complete.""" + before = self.storage.cache_generation() if self.storage.evict(nbytes): self.publisher.refresh() - self._prune_fetched() + if self.storage.cache_generation() != before: # else no cached target went, and there's nothing to prune + self._prune_fetched() def maintain(self, now: float, last: dict) -> None: """One maintenance pass: shared folder, provider records, storage. last holds when each ran.""" @@ -616,7 +636,7 @@ def _prune_fetched(self): if self._fetched <= cached: return self._fetched &= cached - write_json(self._fetched_path, sorted(str(cid) for cid in self._fetched)) + self._fetched_dirty = True def _note_fetched(self, target: Cid): """Remember a cached target fetched for the user.""" @@ -624,7 +644,23 @@ def _note_fetched(self, target: Cid): if target in self._fetched: return self._fetched.add(target) - write_json(self._fetched_path, sorted(str(cid) for cid in self._fetched)) + self._fetched_dirty = True + + def _save_fetched(self): + """Save the fetched targets if they changed. With the cache records they're intersected with, which are + saved in the same pass: a crash loses at most what changed since the last pass, as for those.""" + with self._fetched_save_lock: + with self._lock: + if not self._fetched_dirty: + return + value = sorted(str(cid) for cid in self._fetched) + self._fetched_dirty = False + try: + write_json(self._fetched_path, value) + except BaseException: + with self._lock: + self._fetched_dirty = True + raise # Fetching @@ -686,11 +722,12 @@ def add_path(self, path: str, wrap: bool = False, only_hash: bool = False, pin: built, warnings = builder.add_path(path, builder.NO_PUT, wrap) else: with self.storage.protect() as session: - if recipients: - built = node_content.add_sealed_path(path, session.put, wrap, recipients, seal_name) - warnings = [] - else: - built, warnings = builder.add_path(path, session.put, wrap) + with session.writer() as put: + if recipients: + built = node_content.add_sealed_path(path, put, wrap, recipients, seal_name) + warnings = [] + else: + built, warnings = builder.add_path(path, put, wrap) node_content.keep_added(self, built.cid, pin) if pin: self.pin_names.record(built.cid, os.path.abspath(path)) @@ -709,7 +746,8 @@ def add_stream(self, stream: BinaryIO, size: int | None = None, name: str | None built = builder.add_stream(stream, builder.NO_PUT, size, name) else: with self.storage.protect() as session: - built = builder.add_stream(stream, session.put, size, name) + with session.writer() as put: + built = builder.add_stream(stream, put, size, name) node_content.keep_added(self, built.cid, pin) if pin: self.pin_names.record(built.cid, name or seal_name or "(stdin)") @@ -1268,7 +1306,7 @@ def status(self) -> dict: "identity_hash": identity.hash.hex() if (identity := getattr(self, "identity", None)) else None, "dht_mode": self.config.dht_mode, "slow": self.slow, "private": self.config.private, "known_nodes": self.routing.node_count(), - "records": self.records.record_count(), "pins": len(self.storage.pins()), + "records": self.records.record_count(), "pins": self.storage.pin_count(), "provided": len(self.publisher.provided()), "blocks": self.storage.block_count(), "used_bytes": self.storage.used_bytes(), "max_storage": self.config.max_storage, "names": len(self.names.list()), diff --git a/rnfs/pin_names.py b/rnfs/pin_names.py index 80747d1..d832d7e 100644 --- a/rnfs/pin_names.py +++ b/rnfs/pin_names.py @@ -64,4 +64,4 @@ def prune(self, pinned: set[Cid]): def _save(self): """Write the names atomically (called with the lock held).""" - write_json(self.path, {str(cid): name for cid, name in sorted(self._names.items(), key=lambda i: str(i[0]))}) + write_json(self.path, dict(sorted((str(cid), name) for cid, name in self._names.items()))) diff --git a/rnfs/publisher.py b/rnfs/publisher.py index f0cc6d9..f369b85 100644 --- a/rnfs/publisher.py +++ b/rnfs/publisher.py @@ -129,9 +129,15 @@ def __init__(self, own_id: bytes, config: Config, storage: Storage, providers, c self.retry_interval = retry_interval self._state_path = None if state_dir is None else os.path.join(state_dir, STATE_FILE) self._lock = threading.RLock() + self._save_lock = threading.Lock() # one save at a time, so an older snapshot never lands last self._wake = threading.Event() self._provided: frozenset[Cid] = frozenset() self._served: frozenset[Cid] = frozenset() + # What _compute last built, and from what, so a refresh that changed nothing needn't build it again: the + # served set from each provided CID's dag_blocks (memoised by storage, so the same object while its DAG + # stays complete), and the DhtKeys from the keys the key sources returned + self._served_from: tuple[dict[Cid, frozenset[Cid]], frozenset[Cid]] = ({}, frozenset()) + self._keys_from: tuple[set[bytes], frozenset[DhtKey]] = (set(), frozenset()) self._published: dict[Cid, tuple[float, list[bytes]]] = {} # CID -> (when, nodes that recorded it) # Other nodes that may still hold our record of a CID (earlier rounds' closest nodes, or a round cut # short), with when it was stored there; withdrawn from too, and forgotten once PROVIDER_TTL has passed @@ -204,16 +210,19 @@ def when(number) -> float: continue def save(self): - """Persist the publish state (atomic).""" + """Persist the publish state (atomic). Called without the lock: it is written outside it.""" if self._state_path is None: return - with self._lock: - value = {"published": {str(cid): [when, [n.hex() for n in nodes]] - for cid, (when, nodes) in self._published.items()}, - "earlier": {str(cid): {n.hex(): when for n, when in nodes.items()} - for cid, nodes in self._earlier.items() if nodes}, - "withdraw": {node.hex(): sorted(k.hex() for k in keys) - for node, keys in self._withdrawals.items() if keys}} + # Saves come from refresh(), the publish loop and stop(): one at a time, or a snapshot taken before the + # other's could be written last + with self._save_lock: + with self._lock: + value = {"published": {str(cid): [when, [n.hex() for n in nodes]] + for cid, (when, nodes) in self._published.items()}, + "earlier": {str(cid): {n.hex(): when for n, when in nodes.items()} + for cid, nodes in self._earlier.items() if nodes}, + "withdraw": {node.hex(): sorted(k.hex() for k in keys) + for node, keys in self._withdrawals.items() if keys}} write_json(self._state_path, value) # The provide set @@ -233,25 +242,40 @@ def serves(self, cid: Cid) -> bool: return cid in self._served # a frozenset swapped whole, so reading it needs no lock def _compute(self) -> tuple[frozenset[Cid], frozenset[Cid]]: - """(provided CIDs, blocks they reach) from storage and the strategy.""" + """(provided CIDs, blocks they reach) from storage and the strategy. Lock held. + + The served set and the DhtKeys are those of the last call when what they are built from hasn't changed + (most refreshes): the same complete CIDs with the same dag_blocks objects, and the same keys. CIDs only + added since add their blocks to it; only one gone or changed builds it again. + """ candidates = set(self.storage.pins()) if self.config.provide_strategy == "roots": candidates |= self.storage.cached_targets() & set(self.cached_eligible()) - provided = set() - served: set[Cid] = set() + dags: dict[Cid, frozenset[Cid]] = {} for cid in candidates: try: - blocks = self.storage.dag_blocks(cid) # memoised; raises unless complete + dags[cid] = self.storage.dag_blocks(cid) # memoised; raises unless complete except (NotFound, InvalidDag): continue - provided.add(cid) - served |= blocks + keys: set[bytes] = set() for source in list(self._key_sources): try: - provided |= {DhtKey(key) for key in source()} + keys |= {bytes(key) for key in source()} # as DhtKey takes them except Exception as e: # noqa: BLE001 - a broken feature mustn't stop CIDs being provided log(f"a provided-key source failed: {e!r}", LOG_ERROR) - return frozenset(provided), frozenset(served) + last_dags, served = self._served_from + new = [blocks for cid, blocks in dags.items() if last_dags.get(cid) is not blocks] + if len(dags) - len(new) < len(last_dags): # one went, or its blocks changed: built again + blocks_reached: set[Cid] = set() + for blocks in dags.values(): + blocks_reached |= blocks + served = frozenset(blocks_reached) + elif new: # only some added (a pin or a fetch) + served = served.union(*new) + self._served_from = (dags, served) + if keys != self._keys_from[0]: + self._keys_from = (keys, frozenset(DhtKey(key) for key in keys)) + return frozenset(dags) | self._keys_from[1], served def refresh(self): """Recompute what is provided (after pins, fetches, evictions); withdraw what no longer is, at once.""" @@ -282,17 +306,18 @@ def _provide_again(self, cid: "Cid | DhtKey") -> bool: there again, deleting the fresh record (section 6). Those nodes may still hold a record of it, so they join its earlier nodes, to be withdrawn from should it stop being provided again. """ - self.providers.store(cid.key, self.own_id) # refresh() withdrew it when it stopped being provided + key = cid.key + self.providers.store(key, self.own_id) # refresh() withdrew it when it stopped being provided now = self.clock() cancelled = False - for node_id, keys in list(self._withdrawals.items()): - if cid.key in keys: - keys.discard(cid.key) - if not keys: - del self._withdrawals[node_id] - earlier = self._earlier.setdefault(cid, {}) - earlier[node_id] = max(now, earlier.get(node_id, now)) - cancelled = True + for node_id in [node_id for node_id, keys in self._withdrawals.items() if key in keys]: + keys = self._withdrawals[node_id] + keys.discard(key) + if not keys: + del self._withdrawals[node_id] + earlier = self._earlier.setdefault(cid, {}) + earlier[node_id] = max(now, earlier.get(node_id, now)) + cancelled = True return cancelled # Publishing diff --git a/specification.md b/specification.md index ddaa37c..c29b0c8 100644 --- a/specification.md +++ b/specification.md @@ -1275,6 +1275,7 @@ setting. | `default_bootstrap` | on | Also contact the default bootstrap nodes when joining; never in private mode (section 6) | | `reticulum_dir` | RNS's own | RNS config directory, the one `rnsd` uses by default | | `verbosity` | 0 | Extra logging: 1 per-query detail, 2 RNS debug output | +| `debug_log` | off | Also write every log message (at every level), RNS's own, and each gateway request to a debug log file in the rnfs dir, readable by its owner only. Local only: nothing in it is sent | | `upload_budget` | none (unlimited) | Bytes per hour of `/blocks` responses, node-wide; a token bucket charged for bytes sent, at most half of it per requester in flight (section 3) | | `serve_interfaces` | empty (all) | Serve `/blocks` only on these RNS interfaces (names or prefixes, case-insensitive; a spawned interface also matches its parent's name) | | `no_serve_interfaces` | empty | Never serve `/blocks` on these interfaces; takes precedence over `serve_interfaces` | diff --git a/tests/test_archive.py b/tests/test_archive.py index edecd81..7acad48 100644 --- a/tests/test_archive.py +++ b/tests/test_archive.py @@ -74,6 +74,23 @@ def test_round_trip_pins_roots(self): self.assertEqual(result["blocks"], len(self.records(data)) - 1) self.assertEqual(len(self.records(self.export([self.root.cid]))), len(self.records(data))) + def test_roots_are_pinned_with_one_save(self): + data = self.export([self.root.cid, self.other.cid, self.root.cid]) + with mock.patch.object(self.destination, "_save_pins", wraps=self.destination._save_pins) as save: + result = archive.import_archive(io.BytesIO(data), self.destination) + self.assertEqual(save.call_count, 1) + self.assertEqual(result["complete"], [self.root.cid, self.other.cid, self.root.cid]) + self.assertEqual(self.storage("destination").pins(), {self.root.cid: {USER}, self.other.cid: {USER}}) + + def test_failed_pin_save_pins_no_root(self): + data = self.export([self.root.cid, self.other.cid]) + with mock.patch("rnfs.content.storage.write_json", side_effect=OSError(28, "No space left on device")): + with self.assertRaises(OSError): + archive.import_archive(io.BytesIO(data), self.destination) + self.assertEqual(self.destination.pins(), {}) + self.assertEqual(self.storage("destination").pins(), {}) + self.assertTrue(self.destination.is_complete(self.root.cid)) # the blocks are kept, as cache + def test_inner_target(self): target = walker.resolve(self.root.cid, ["b"], self.source.getter()) data = self.export([target]) diff --git a/tests/test_cid.py b/tests/test_cid.py index 5b1440b..d9760c9 100644 --- a/tests/test_cid.py +++ b/tests/test_cid.py @@ -1,6 +1,8 @@ """Tests for CIDs (specification section 4): binary and written forms, validation, DHT keys.""" import base64 +import copy import hashlib +import pickle import unittest from RNS.vendor import umsgpack @@ -51,6 +53,27 @@ def test_key(self): self.assertEqual(raw.key, hashlib.sha256(raw.binary).digest()[:16]) self.assertNotEqual(raw.key, node.key) # identical bytes, different codecs, different keys + def test_written_form_and_key_are_memoised(self): + value = Cid.for_block(RAW, b"hello world") + self.assertEqual(str(value), HELLO_WORLD) + self.assertIs(str(value), str(value)) # computed once + self.assertEqual(value.key, hashlib.sha256(value.binary).digest()[:ID_LENGTH]) + self.assertIs(value.key, value.key) + # Memoising changes nothing a CID is compared, hashed or copied by + fresh = Cid.from_bytes(value.binary) + self.assertEqual(fresh, value) + self.assertEqual(hash(fresh), hash(value)) + self.assertEqual({fresh: 1}[value], 1) + self.assertEqual(repr(fresh), repr(value)) + self.assertEqual(repr(value), f"Cid({HELLO_WORLD})") + self.assertEqual(Cid.parse(HELLO_WORLD), value) + for copied in (copy.copy(value), copy.deepcopy(fresh), pickle.loads(pickle.dumps(value))): + self.assertEqual(copied, value) + self.assertEqual(hash(copied), hash(value)) + self.assertEqual(str(copied), HELLO_WORLD) + self.assertEqual(copied.key, value.key) + self.assertEqual(copied.codec, RAW) + def test_verify(self): value = Cid.for_block(RAW, b"real") self.assertTrue(value.verify(b"real")) diff --git a/tests/test_cli.py b/tests/test_cli.py index c46fed9..edae1bb 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -87,6 +87,8 @@ def test_daemon_options(self): ("client", "yes", True, 2, 0)) args = cli.build_parser().parse_args(["daemon"]) self.assertEqual((args.bootstrap, args.private, args.dht_mode), (None, None, None)) + self.assertIsNone(args.debug_log) # rnfs.toml's debug_log then decides + self.assertTrue(cli.build_parser().parse_args(["daemon", "--debug-log"]).debug_log) def test_every_spec_command_parses(self): cid = str(Cid.for_block(RAW, b"")) diff --git a/tests/test_client.py b/tests/test_client.py index 07cff8d..4a385c4 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -200,6 +200,30 @@ def test_small_response_gives_no_rate(self): self.call(lambda: self.client.blocks(NODE, [cid])) self.assertIsNone(self.client.rate_estimate(NODE)) + def test_blocks_elapsed_counts_from_sending(self): + cid, block = block_of(100, 1) + + def answer(link, path, data): + """Answer after 0.1 s.""" + time.sleep(0.1) + return [block] + self.respond(answer) + fakes.FakeLink.establish_delay = 0.5 + start = time.monotonic() + result = self.call(lambda: self.client.blocks(NODE, [cid])) + self.assertGreaterEqual(time.monotonic() - start, 0.6) + self.assertGreaterEqual(result.elapsed, 0.1) + self.assertLess(result.elapsed, 0.5) # opening the link isn't counted + self.respond(lambda link, path, data: False) + self.assertIsNone(self.call(lambda: self.client.blocks(NODE, [cid])).elapsed) + + def test_link_mdu(self): + self.assertIsNone(self.client.link_mdu(NODE)) + peer = self.call(lambda: self.pool.get(NODE)) + self.assertEqual(self.client.link_mdu(NODE), 431) + self.pool.close_link(peer) + self.assertIsNone(self.client.link_mdu(NODE)) + def test_blocks_batch_cut(self): cids = [block_of(10, i)[0] for i in range(15)] self.respond(lambda link, path, data: [None] * len(data[1])) diff --git a/tests/test_config.py b/tests/test_config.py index 6c547e0..0b6c49b 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -19,7 +19,7 @@ def test_defaults_are_valid(self): def test_flags_must_be_booleans(self): # "false" and "no" are non-empty strings, which would read as true: scripts on, private mode on - for name in ("gateway_scripts", "private", "default_bootstrap"): + for name in ("gateway_scripts", "private", "default_bootstrap", "debug_log"): for value in ("false", "no", 0, 1, None): self.assert_invalid(**{name: value}) Config(**{name: False}).validate() diff --git a/tests/test_dag.py b/tests/test_dag.py index c51f34e..11a5914 100644 --- a/tests/test_dag.py +++ b/tests/test_dag.py @@ -122,6 +122,14 @@ def test_rejects_non_shortest_headers(self): with self.subTest(reason): self.assertInvalid(block) + def test_rejects_every_truncation(self): + # str8, bin8 and uint64 headers, and fixarray, fixstr and fixint ones + block = pack([2, [["n" * 40, CHUNK.binary, 2**40, 0], ["o", NODE.binary, 7, 2]]]) + dag.decode(block) + for end in range(len(block)): + with self.subTest(end=end): + self.assertInvalid(block[:end]) + def test_rejects_invalid_utf8(self): block = pack([2, [["ab", CHUNK.binary, 1, 0]]]).replace(b"\xa2ab", b"\xa2\xff\xfe") self.assertInvalid(block) diff --git a/tests/test_debuglog.py b/tests/test_debuglog.py new file mode 100644 index 0000000..f4bfa73 --- /dev/null +++ b/tests/test_debuglog.py @@ -0,0 +1,209 @@ +"""Tests for the debug log file (rnfs/debuglog.py), the hooks that feed it, and what gateway requests write there.""" +import contextlib +import io +import os +import stat +import tempfile +import threading +import unittest +from unittest import mock + +import RNS + +from rnfs import __version__, debuglog, fsutil, log +from rnfs.config import Config +from rnfs.gateway import GatewayServer +from tests.test_api import FakeNode, raw_request + + +class DebugLogTestCase(unittest.TestCase): + """Base: a debug log started over a temporary rnfs dir, stopped (hooks undone) after each test.""" + + def setUp(self): + """Start the debug log with RNS printing to the console, as the daemon does.""" + self.tmp = tempfile.TemporaryDirectory() + self.addCleanup(self.tmp.cleanup) + self.config = Config(rnfs_dir=self.tmp.name, debug_log=True, gateway_fetches=3) + self.addCleanup(self.restore_rns, RNS.logdest, RNS.loglevel) + RNS.logdest = RNS.LOG_STDOUT + self.addCleanup(debuglog.stop) + self.debug = debuglog.start(self.config) + + @staticmethod + def restore_rns(logdest, loglevel): + """Put RNS's logging back as it was.""" + RNS.logdest, RNS.loglevel = logdest, loglevel + + def lines(self, path: str | None = None) -> list[str]: + """The file's lines.""" + with open(path or self.debug.path, encoding="utf-8") as f: + return f.read().splitlines() + + def logged(self, needle: str) -> list[str]: + """The lines containing needle.""" + return [line for line in self.lines() if needle in line] + + +class FileTest(DebugLogTestCase): + + def test_begins_with_the_version_then_versions_and_settings(self): + first, second, third = self.lines()[:3] + self.assertTrue(first.startswith(f"rnfs {__version__} debug log, started "), first) + self.assertIn(f"(pid {os.getpid()})", first) + self.assertTrue(second.startswith("Python "), second) + self.assertIn("RNS ", second) + self.assertIn("'debug_log': True", third) + self.assertIn("'gateway_fetches': 3", third) + self.assertNotIn("'max_storage'", third) # only settings that differ from the defaults + self.assertEqual(self.debug.path, os.path.join(self.tmp.name, "debug.log")) + + @unittest.skipIf(fsutil.IS_WINDOWS, "no POSIX modes on Windows") + def test_readable_by_its_owner_only(self): + self.assertEqual(stat.S_IMODE(os.stat(self.debug.path).st_mode) & 0o077, 0) + + def test_each_start_keeps_the_last_run_as_dot_1(self): + log.log("from the first run", log.LOG_NOTICE) + debuglog.stop() + self.debug = debuglog.start(self.config) + self.assertTrue(self.lines(self.debug.path + ".1")[0].startswith(f"rnfs {__version__} debug log")) + self.assertTrue(any("from the first run" in line for line in self.lines(self.debug.path + ".1"))) + self.assertFalse(self.logged("from the first run")) + + def test_a_full_file_moves_aside_and_the_next_begins_with_the_version_too(self): + small = debuglog.DebugLog(os.path.join(self.tmp.name, "small.log"), {}, max_size=2000) + self.addCleanup(small.close) + for i in range(60): + small.note(f"line {i} " + "x" * 40) + current = self.lines(small.path) + self.assertTrue(current[0].startswith(f"rnfs {__version__} debug log, continued "), current[0]) + self.assertLessEqual(os.path.getsize(small.path), 2000) + self.assertTrue(os.path.exists(small.path + ".1")) + self.assertIn("line 59", current[-1]) + + def test_a_failing_disk_stops_it_quietly(self): + stderr = io.StringIO() + with mock.patch("os.write", side_effect=OSError(28, "No space left on device")), \ + contextlib.redirect_stderr(stderr): + log.log("lost", log.LOG_NOTICE) + log.log("lost too", log.LOG_NOTICE) + self.assertEqual(stderr.getvalue().count("debug log"), 1) + log.log("after", log.LOG_NOTICE) + self.assertFalse(self.logged("after")) + + +class MessagesTest(DebugLogTestCase): + + def test_every_rnfs_message_whatever_the_console_shows(self): + RNS.loglevel = RNS.LOG_ERROR # the console shows errors only + log.log("per-query detail", log.LOG_VERBOSE) + log.log("debug detail", log.LOG_DEBUG) + verbose, = self.logged("per-query detail") + self.assertIn("[Verbose]", verbose) + self.assertIn(f"[{threading.current_thread().name}] rnfs: per-query detail", verbose) + self.assertIn("[Debug]", self.logged("debug detail")[0]) + + def test_lines_are_escaped(self): + log.log("name '\x1b]0;owned\x07'\nrnfs fake line", log.LOG_WARNING) + line, = self.logged("owned") + self.assertTrue(line.isprintable()) + self.assertEqual(self.logged("rnfs fake line"), [line]) # one line: the newline is escaped + + def test_an_expected_outcome_logged_in_an_except_block_has_no_traceback(self): + try: + raise ValueError("a refusal") + except ValueError: + log.log("refused, as expected", log.LOG_DEBUG) + self.assertTrue(self.lines()[-1].endswith("rnfs: refused, as expected")) + + def test_a_message_logged_while_handling_an_exception_has_its_traceback(self): + try: + raise ValueError("the cause") + except ValueError as e: + log.log(f"it failed: {e!r}", log.LOG_ERROR) + lines = self.lines() + at = next(i for i, line in enumerate(lines) if "it failed" in line) + trace = lines[at + 1:at + 6] + self.assertTrue(all(line.startswith(" | ") for line in trace[:2]), trace) + self.assertTrue(any("ValueError: the cause" in line for line in lines[at + 1:])) + log.log("no exception here", log.LOG_ERROR) + self.assertTrue(self.lines()[-1].endswith("rnfs: no exception here")) # nothing below it + + def test_rns_messages_still_print_and_are_copied_once(self): + RNS.loglevel = RNS.LOG_INFO + out = io.StringIO() + with contextlib.redirect_stdout(out): + RNS.log("TCPClientInterface[backbone] could not connect", RNS.LOG_ERROR) + log.log("an rnfs line", log.LOG_NOTICE) + printed = out.getvalue() + self.assertIn("could not connect", printed) + self.assertIn("rnfs an rnfs line", printed) + rns, = self.logged("could not connect") + self.assertIn("[Error]", rns) + self.assertIn("RNS: TCPClientInterface[backbone] could not connect", rns) + self.assertEqual(len(self.logged("an rnfs line")), 1) # by rnfs.log's sink, not again as RNS output + self.assertIn("rnfs: an rnfs line", self.logged("an rnfs line")[0]) + + def test_a_thread_that_dies_is_logged(self): + def die(): + """Raise in a thread.""" + raise RuntimeError("boom") + + with contextlib.redirect_stderr(io.StringIO()): # the usual report still goes to stderr + thread = threading.Thread(target=die, name="rnfs-test-thread") + thread.start() + thread.join() + died, = self.logged("rnfs-test-thread died of RuntimeError") + self.assertIn("[Error]", died) + self.assertTrue(self.logged("RuntimeError: boom")) + + def test_stop_undoes_every_hook(self): + hook = threading.excepthook + debuglog.stop() + self.assertIsNone(debuglog.current()) + self.assertEqual(RNS.logdest, RNS.LOG_STDOUT) + self.assertIsNot(threading.excepthook, hook) + log.log("after stop", log.LOG_NOTICE) + self.assertFalse(self.logged("after stop")) + + def test_status_line(self): + node = mock.Mock(links=None) + node.status.return_value = {"known_nodes": 4, "pins": 2} + line = debuglog.status_line(node) + self.assertIn("'known_nodes': 4", line) + self.assertIn("interfaces:", line) + + +class GatewayRequestsTest(DebugLogTestCase): + """What the gateway writes for each request.""" + + def setUp(self): + """A gateway over a FakeNode, with the debug log on.""" + super().setUp() + self.node = FakeNode() + self.server = GatewayServer(self.node, self.config, 0) + self.server.start() + self.addCleanup(self.server.stop) + self.text = self.node.put_file(b"hello gateway") + + def test_a_request_and_its_outcome(self): + status, _, body = raw_request(self.server.port, "GET", f"/rnfs/{self.text}", None, + {"Range": "bytes=0-4"}, "default") + self.assertEqual((status, body), (206, b"hello")) + start, = self.logged(f"gateway GET /rnfs/{self.text} (") + self.assertIn("Range: bytes=0-4", start) + number = start.split("gateway #")[1].split("]")[0] + tagged = self.logged(f"gateway #{number}]") + self.assertIn("is stored here", "\n".join(tagged)) + self.assertIn("5 bytes from byte 0 of 13", "\n".join(tagged)) + self.assertIn("gateway answered 206", tagged[-1]) + self.assertIn("bytes written (headers included) in", tagged[-1]) + + def test_refusals_are_logged_with_their_reason(self): + status, _, _ = raw_request(self.server.port, "GET", "/nowhere", None, None, "default") + self.assertEqual(status, 404) + self.assertTrue(self.logged("gateway refused /nowhere with 404")) + self.assertIn("gateway answered 404", self.logged("gateway answered")[-1]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_fetcher.py b/tests/test_fetcher.py index e10dd45..5e1c005 100644 --- a/tests/test_fetcher.py +++ b/tests/test_fetcher.py @@ -1,16 +1,18 @@ """Tests for fetching targets (rnfs/fetcher.py; specification sections 6 and 7) over a fake client and lookup.""" import os import shutil +import random import tempfile import threading import time import unittest +from collections import deque from unittest import mock import RNS from rnfs.config import Config -from rnfs.constants import CHUNK_SIZE +from rnfs.constants import CHUNK_SIZE, MAX_BLOCK_SIZE, MAX_REQUEST_SIZE, MAX_RESPONSE_SIZE, RESPONSE_OVERHEAD from rnfs.content import builder, dag, walker from rnfs.content.cid import DAG, RAW, Cid from rnfs.content.dag import Child, Entry, FileNode @@ -20,6 +22,7 @@ from rnfs.errors import InvalidDag, NotFound, StorageFull, TooLarge from rnfs import fetcher as fetcher_module from rnfs.fetcher import Fetcher +from rnfs.network import protocol, server from rnfs.network.client import BUSY, FAILED, OK, STALLED, BlocksResult OWN = bytes(16) @@ -375,6 +378,117 @@ def test_paced_provider_isnt_dropped_meanwhile(self): self.result(OK, sent_at=self.now - 40, rate=50_000) self.assertTrue(self.fetch._waiting_for_busy()) + def sent(self, elapsed: float | None): + """Send P1 a request that spends 30 s opening the link, then 40 s from sending it to its answer (reported + as elapsed), and handle the result it queues.""" + cid = self.built.cid + self.client.rate = 50_000 + + def answer(cids): + """Open the link, then send and get the answer.""" + self.now += 30 + 40 + return BlocksResult(OK, blocks={cid: self.network[cid]}, elapsed=elapsed) + self.client.behaviour[P1] = answer + self.fetch._send(P1, [cid]) + _, node_id, cids, result, sent_at, rate = self.fetch._events.get(timeout=5) + self.fetch._handle(node_id, cids, result, False, {}, {}, set(), sent_at, rate) + + def test_pacing_counts_from_when_the_request_was_sent(self): + self.sent(40) + # As long again as the request took once sent: not the link opening before it + self.assertEqual(self.provider.busy_until, self.now + 40) + self.assertEqual(self.provider.pacing_until, self.now - 40 + 4 * 40) + self.assertEqual(self.provider.pacing_retry, 5) + + def test_pacing_without_a_send_time_counts_the_whole_call(self): + self.sent(None) + self.assertEqual(self.provider.busy_until, self.now + 70) + self.assertEqual(self.provider.pacing_until, self.now - 70 + 4 * 70) + + +def old_take_batch(fetch, pending, node_id, wanted, mdu): + """Fetch._take_batch as it was, packing each prefix in turn to see if it fits, with the cap on the bytes + reserved that it has now: the reference the single count over pending must match.""" + budget = fetcher_module.rates.requester_max_bytes(fetch.client.rate_estimate(node_id), fetch.config) + carried = min(budget, MAX_RESPONSE_SIZE - RESPONSE_OVERHEAD) + room = fetch._room(wanted) + batch, total = [], 0 + while pending and len(batch) < fetcher_module.MAX_BATCH: + cid = pending[0] + known = wanted[cid][0].size if cid.is_raw else None + guess = known if known is not None else (CHUNK_SIZE if cid.is_raw else fetcher_module.DAG_NODE_GUESS) + if batch and total + guess > carried: + break + if batch and protocol.blocks_batch_size(budget, batch + [cid], mdu) <= len(batch): + break + if room is not None: + room = fetch._reserve(cid, wanted, room) + if not fetch._fits(room) and (batch or fetch._in_flight): + break + batch.append(pending.popleft()) + total += guess + return batch + + +class BatchTest(FetcherTest): + """What one request asks for: no more than a response can carry, or the provider's link can send.""" + + def setUp(self): + """An open fetch, and the expectations of 40 raw chunks of 64 KiB.""" + super().setUp() + self.fetch = self.fetcher.open(self.file(b"root").cid).__enter__() + self.addCleanup(self.fetch.__exit__, None, None, None) + self.chunks = [walker.Expect(Cid.for_block(RAW, bytes([i]) * 8), CHUNK_SIZE, walker.FILE_TYPES, 2) + for i in range(40)] + self.wanted = {expect.cid: [expect] for expect in self.chunks} + + def take(self) -> list[Cid]: + """The batch for P1 from every chunk.""" + return self.fetch._take_batch(deque(self.wanted), P1, self.wanted) + + def test_capped_at_what_a_response_carries(self): + # At 1 Gbit/s max_bytes asks for ~7.5 GB, but a response carries at most MAX_RESPONSE_SIZE less overhead + self.client.rate = 1e9 + batch = self.take() + max_bytes = fetcher_module.rates.requester_max_bytes(self.client.rate, self.config) + served = server.select_blocks(batch, lambda cid: bytes(CHUNK_SIZE), + server.blocks_budget(max_bytes, self.client.rate, None, 60)) + self.assertEqual(len(batch), (MAX_RESPONSE_SIZE - RESPONSE_OVERHEAD) // CHUNK_SIZE) + self.assertEqual(len(served), len(batch)) # every CID reserved comes back + + def test_sized_to_the_open_links_packet(self): + self.client.rate = 1e9 + self.wanted = {expect.cid: [walker.Expect(expect.cid, 10, walker.FILE_TYPES, 2)] for expect in self.chunks} + max_bytes = fetcher_module.rates.requester_max_bytes(self.client.rate, self.config) + # No link known: the largest request any link allows + self.assertEqual(len(self.take()), protocol.blocks_batch_size(max_bytes, list(self.wanted), MAX_REQUEST_SIZE)) + self.client.link_mdu = lambda node_id: None + self.assertEqual(len(self.take()), protocol.blocks_batch_size(max_bytes, list(self.wanted), MAX_REQUEST_SIZE)) + self.client.link_mdu = lambda node_id: 431 + self.assertEqual(len(self.take()), protocol.blocks_batch_size(max_bytes, list(self.wanted), 431)) + self.assertLess(len(self.take()), protocol.blocks_batch_size(max_bytes, list(self.wanted), MAX_REQUEST_SIZE)) + + def test_one_count_matches_packing_each_prefix(self): + rng = random.Random(7) + for trial in range(300): + expects = [] + for i in range(rng.randrange(0, 45)): + if rng.random() < 0.7: + size = rng.choice([None, 0, 10, 1000, CHUNK_SIZE, rng.randrange(1, CHUNK_SIZE)]) + expects.append(walker.Expect(Cid.for_block(RAW, rng.randbytes(8)), size, walker.FILE_TYPES, 2)) + else: + expects.append(walker.Expect(Cid.for_block(DAG, rng.randbytes(8)), None, walker.ANY_TYPES, 2)) + wanted = {expect.cid: [expect] for expect in expects} + self.client.rate = rng.choice([None, 1200, 50_000, 1e6, 1e9, rng.uniform(1, 1e10)]) + mdu = rng.choice([None, 0, 120, 300, 431, 1000, 1500, 9000]) + self.client.link_mdu = lambda node_id: mdu + self.fetch.max_dag_bytes = rng.choice([None, None, 0, CHUNK_SIZE, 5 * MAX_BLOCK_SIZE]) + with self.subTest(trial=trial): + old, new = deque(wanted), deque(wanted) + expected = old_take_batch(self.fetch, old, P1, wanted, mdu or MAX_REQUEST_SIZE) + self.assertEqual(self.fetch._take_batch(new, P1, wanted), expected) + self.assertEqual(list(new), list(old)) + class ValidityTest(FetcherTest): """Reader checks, max_download_size, space.""" diff --git a/tests/test_jobs.py b/tests/test_jobs.py index f796bf1..5580aba 100644 --- a/tests/test_jobs.py +++ b/tests/test_jobs.py @@ -191,6 +191,24 @@ def test_cancel_running_job_stops_its_fetch(self): self.assertFalse(os.path.exists(self.out("cancelled"))) self.assertEqual(self.b.storage._sessions, []) # its protection is released + def test_submit_saves_once_whether_the_job_starts_or_waits(self): + self.b.config.max_jobs = 1 + first, second = self.share(b"first"), self.share(b"second") + self.net.gate.clear() + path = os.path.join(self.b.config.state_dir, jobs.JOBS_FILE) + from rnfs.content import storage + with mock.patch.object(storage, "write_json", wraps=storage.write_json) as write: + started = self.jobs.submit("get", first, self.out("first"))["id"] + self.assertEqual(write.call_count, 1) # saved as it started, not also before + queued = self.jobs.submit("get", second, self.out("second"))["id"] + self.assertEqual(write.call_count, 2) # nothing started, so saved as queued + with open(path, encoding="utf-8") as f: + saved = {job["id"]: job["state"] for job in json.load(f)["jobs"]} + self.assertEqual(saved, {started: "running", queued: "queued"}) + self.net.gate.set() + self.assertEqual(self.finished(started)["state"], "done") + self.assertEqual(self.finished(queued)["state"], "done") + def test_cancel_queued_job_and_concurrency_limit(self): self.b.config.max_jobs = 1 cids = [self.share(f"file {i}".encode()) for i in range(3)] diff --git a/tests/test_links.py b/tests/test_links.py index fa7aea5..c455cad 100644 --- a/tests/test_links.py +++ b/tests/test_links.py @@ -301,6 +301,20 @@ def test_request_failure(self): peer = run(self, lambda: self.pool.get(NODE)) self.assertFalse(run(self, lambda: self.pool.request(peer, "/find", b"k")).answered) + def test_request_outcomes_are_logged_with_the_reason(self): + FakeLink.responder = staticmethod(lambda link, path, data: [path, data]) + peer = run(self, lambda: self.pool.get(NODE)) + with mock.patch.object(links, "log") as logged: + run(self, lambda: self.pool.request(peer, "/find", b"k")) + self.assertTrue(any("answered /find in" in call.args[0] and call.args[1] == links.LOG_DEBUG + for call in logged.call_args_list), logged.call_args_list) + FakeLink.responder = staticmethod(lambda link, path, data: FAIL) + with mock.patch.object(links, "log") as logged: + run(self, lambda: self.pool.request(peer, "/find", b"k")) + message = logged.call_args_list[-1].args[0] + self.assertIn("/find to", message) + self.assertIn("not answered after", message) + def test_request_gives_up_when_link_closes(self): FakeLink.responder = staticmethod(lambda link, path, data: SILENT) peer = run(self, lambda: self.pool.get(NODE)) diff --git a/tests/test_lookup.py b/tests/test_lookup.py index 26c7641..b02b9c4 100644 --- a/tests/test_lookup.py +++ b/tests/test_lookup.py @@ -204,6 +204,26 @@ def test_ends_when_every_known_node_queried(self): self.assertEqual(sorted(result), [a, b]) self.assertEqual(sorted(net.queried), [a, b]) # each queried once + def test_reports_each_query_and_how_the_run_ended(self): + net = self.network() + a, b, gone = ident(1), ident(2), ident(3) + net.add(a, knows=[b, gone]) + net.add(b, knows=[a]) + reports: list[str] = [] + lock = threading.Lock() + + def report(text): + """Collect reports (query threads report too).""" + with lock: + reports.append(text) + + self.lookup(net, ident(100), [a], report=report).find_closest() + key = ident(100).hex() + self.assertTrue(reports[0].startswith(f"lookup for {key}: 1 start nodes"), reports[0]) + self.assertTrue(any(f"<{a.hex()}> answered in" in text for text in reports), reports) + self.assertTrue(any(f"<{gone.hex()}> failed after" in text for text in reports), reports) + self.assertIn("asked 3, 2 answered, 1 failed", reports[-1]) + def test_no_start_nodes(self): net = self.network() self.assertEqual(self.lookup(net, ident(5), []).find_closest(), []) diff --git a/tests/test_names.py b/tests/test_names.py index 198ca28..209cc59 100644 --- a/tests/test_names.py +++ b/tests/test_names.py @@ -1,4 +1,5 @@ """Tests for mutable names (specification section 9), with RNS's network side replaced by fakes.""" +import json import os import stat import tempfile @@ -623,5 +624,82 @@ def test_resolve_rejects_bad_names(self): self.manager.resolve(b"short", persist=False) +class FloorSaveTest(NamesTestCase): + """name-floor.json is written outside the manager's lock, in order, and a failed write is tried again.""" + + def setUp(self): + """A started manager.""" + super().setUp() + self.manager.start() + self.addCleanup(self.manager.stop) + self.path = os.path.join(self.config.state_dir, names.FLOOR_FILE) + + def resolve_for_gateway(self, i: int, sequence: int = 1): + """Resolve other_name(i), just announced at sequence, without persist (so it goes in the floor).""" + self.network.announce(other_name(i), encode_record(sequence, CID_A), self.clock.now) + self.assertEqual(self.manager.resolve(other_name(i), persist=False, max_age=60)["sequence"], sequence) + + def saved(self) -> list: + """The floor entries in the file.""" + with open(self.path, encoding="utf-8") as f: + return json.load(f)["floor"] + + def test_file_content(self): + self.resolve_for_gateway(1, sequence=7) + self.clock.now += 0.5 + self.resolve_for_gateway(2) + self.manager.stop() + with open(self.path, "rb") as f: + self.assertEqual(f.read(), json.dumps({"version": 1, "floor": [ + [other_name(1).hex(), 7, 1_000_000], [other_name(2).hex(), 1, 1_000_000]]}, + separators=(",", ":")).encode()) + + def test_written_without_the_lock(self): + real_write = fsutil.write_atomic + free = [] + + def write(path, data, *args, **kwargs): + """The floor's write: another thread can take the manager's lock meanwhile.""" + if path == self.path: + lock = self.manager._lock + thread = threading.Thread(target=lambda: free.append(lock.acquire(timeout=5)) or lock.release()) + thread.start() + thread.join() + return real_write(path, data, *args, **kwargs) + + with mock.patch.object(names.fsutil, "write_atomic", write): + self.resolve_for_gateway(1) + self.assertEqual(free, [True]) + self.assertEqual([entry[0] for entry in self.saved()], [other_name(1).hex()]) + + def test_saves_never_land_out_of_order(self): + """A save that copies the floor while another is still writing it isn't overwritten by the older copy.""" + real_write = fsutil.write_atomic + second = [] + + def write(path, data, *args, **kwargs): + """The first save's write: meanwhile another resolve raises the floor and saves it.""" + if path == self.path and not second: + self.clock.now += names.FLOOR_SAVE_INTERVAL + second.append(threading.Thread(target=self.resolve_for_gateway, args=(2,))) + second[0].start() + second[0].join(0.2) + return real_write(path, data, *args, **kwargs) + + with mock.patch.object(names.fsutil, "write_atomic", write): + self.resolve_for_gateway(1) + second[0].join(5) + self.assertEqual([entry[0] for entry in self.saved()], [other_name(1).hex(), other_name(2).hex()]) + + def test_failed_save_is_tried_again_at_once(self): + with mock.patch.object(names.fsutil, "write_atomic", side_effect=OSError("disk full")): + self.resolve_for_gateway(1) + self.assertFalse(os.path.exists(self.path)) + self.resolve_for_gateway(2) # within FLOOR_SAVE_INTERVAL of the failed save, which doesn't count + self.assertEqual([entry[0] for entry in self.saved()], [other_name(1).hex(), other_name(2).hex()]) + self.resolve_for_gateway(3) # but the save that worked does + self.assertEqual(len(self.saved()), 2) + + if __name__ == "__main__": unittest.main() diff --git a/tests/test_node.py b/tests/test_node.py index 209118e..25d6513 100644 --- a/tests/test_node.py +++ b/tests/test_node.py @@ -1,6 +1,8 @@ """Tests for RnfsNode's facade and backend (rnfs/node.py), on offline nodes with a fake network client.""" import io +import json import os +import random import shutil import stat import tempfile @@ -12,7 +14,7 @@ import RNS from rnfs.config import Config -from rnfs.constants import CHUNK_SIZE, DEFAULT_BOOTSTRAP_NODES +from rnfs.constants import CHUNK_SIZE, DEFAULT_BOOTSTRAP_NODES, K from rnfs.content.cid import RAW, Cid from rnfs.content.storage import remote_source from rnfs import fetcher @@ -323,8 +325,11 @@ def test_served_block_only_from_provided_dags(self): other = Cid.for_block(RAW, b"cached only") cached.put(other, b"cached only") self.assertIsNone(self.a.backend.served_block(other)) + self.assertEqual(self.a.backend.served_block_size(cid), len(self.a.backend.served_block(cid))) + self.assertIsNone(self.a.backend.served_block_size(other)) # sizes only of what is served self.a.pin_rm(str(cid)) self.assertIsNone(self.a.backend.served_block(cid)) + self.assertIsNone(self.a.backend.served_block_size(cid)) def test_damaged_block_not_served(self): cid = Cid.parse(self.a.add_stream(io.BytesIO(b"will be damaged"))["cid"]) @@ -641,6 +646,65 @@ def cached_targets(): self.a._prune_fetched() self.assertEqual(held, [True]) + def test_fetched_targets_are_saved_with_the_state_and_kept_at_a_restart(self): + cid = self.a.add_stream(io.BytesIO(b"fetched for the user"))["cid"] + self.advertise(cid) + self.b.get(cid, os.path.join(self.tmp.name, "out")) + path = os.path.join(self.b.config.state_dir, node_module.FETCHED_FILE) + self.assertFalse(os.path.exists(path)) # not written on each fetch, under the node's lock... + self.b._save_state() # ...but by the next state save (maintenance, gc and stop) + with open(path, encoding="utf-8") as f: + self.assertEqual(json.load(f), [cid]) + self.b = self.restart(self.b, "b", B_ID) + self.assertEqual(self.b._eligible_targets(), {Cid.parse(cid)}) + self.assertIn(Cid.parse(cid), self.b.publisher.provided()) + + def test_failed_fetched_save_is_tried_again(self): + self.b._note_fetched(Cid.for_block(RAW, b"target")) + with mock.patch.object(node_module, "write_json", side_effect=OSError("disk full")): + with self.assertRaises(OSError): + self.b._save_fetched() + self.b._save_fetched() + with open(os.path.join(self.b.config.state_dir, node_module.FETCHED_FILE), encoding="utf-8") as f: + self.assertEqual(json.load(f), [str(Cid.for_block(RAW, b"target"))]) + + +class MakeRoomTest(NodeTest): + """Eviction for room, as each stored response batch asks for.""" + + def test_fetched_targets_are_pruned_only_when_cache_records_went(self): + cid = Cid.parse(self.a.add_stream(io.BytesIO(os.urandom(5000)), pin=False)["cid"]) # a cached fetch + with mock.patch.object(self.a, "_prune_fetched", wraps=self.a._prune_fetched) as prune: + self.a._make_room(0) # within max_storage: nothing evicted + prune.assert_not_called() + self.a.storage.max_storage = 1 + self.a._make_room(0) + prune.assert_called_once() + self.a._make_room(0) # over max_storage still, but with nothing left to evict + prune.assert_called_once() + self.assertEqual(self.a._eligible_targets(), set()) + self.assertFalse(self.a.storage.has(cid)) + + +class LookupStartTest(NodeTest): + """Where the node's lookups start.""" + + def test_nodes_that_failed_start_lookups_only_to_make_up_k(self): + rng = random.Random(7) + others = [bytes(rng.getrandbits(8) for _ in range(16)) for _ in range(K)] + for node_id in others: + self.assertTrue(self.b.routing.heard(node_id)) + failed = others[0] + self.b.routing.failed(failed) # the closest node to its own ID, but no longer answering + start = self.b._lookup(failed)._start_nodes + self.assertNotIn(failed, start) + self.assertEqual(len(start), K) # A and the others that haven't failed + self.b.routing.remove(others[1]) + self.b.routing.remove(others[2]) + start = self.b._lookup(failed)._start_nodes + self.assertIn(failed, start) # fewer than K healthy: made up from the rest + self.assertEqual(len(start), K - 1) + class JoinTest(NodeTest): """Joining: bootstrap nodes first (the user's, then the defaults), then a lookup; again while too few.""" diff --git a/tests/test_providers.py b/tests/test_providers.py index 32cc97a..22fd46e 100644 --- a/tests/test_providers.py +++ b/tests/test_providers.py @@ -423,6 +423,44 @@ def slow_replace(source, target): with open(self.path, encoding="utf-8") as f: self.assertEqual(len(json.load(f)["records"]), 2) + def test_save_writes_the_whole_state(self): + """The file has every publisher, record (pooled or not), departure and eviction, in the store's order.""" + store = ProviderStore(20, self.path, clock=self.clock, seniority=DAY) + for publisher in (1, 2): + for i in range(9): + store.store(ident(100 * publisher + i), ident(publisher)) + self.clock.now += 1 + for publisher in (3, 4, 5): # the main part is full: newcomers share the pool, and 5 evicts 3 or 4 + store.store(ident(1000 + publisher), ident(publisher)) + self.clock.now += 1 + store.store(ident(101), ident(1)) # refreshed in place + store.withdraw(ident(100), ident(1)) + self.clock.now += DAY + store.withdraw(ident(1005), ident(5)) # 5 departs, seen for long enough to be remembered + with store._lock: + expected = { + "version": 1, + "publishers": {p.hex(): e.first_seen for p, e in store._publishers.items()}, + "records": [[k.hex(), p.hex(), r.first_stored, r.last_stored, int(r.pooled)] + for k, held in store._records.items() for p, r in held.items()], + "departed": [[p.hex(), seen, left] for p, (seen, left) in store._departed.items()], + "evicted": [[p.hex(), seen, left] for p, (seen, left) in store._evicted.items()], + } + self.assertTrue(all(expected[section] for section in ("departed", "evicted"))) + self.assertIn(1, [record[4] for record in expected["records"]]) + store.save() + with open(self.path, "rb") as f: + self.assertEqual(f.read(), json.dumps(expected, separators=(",", ":")).encode()) + + def test_failed_save_is_tried_again(self): + store = self.open() + store.store(KEY, ident(1)) + with mock.patch.object(fsutil, "write_atomic", side_effect=OSError("disk full")): + with self.assertRaises(OSError): + store.save() + store.save() + self.assertEqual(self.open().providers(KEY), [ident(1)]) + def test_corrupt_file_starts_empty(self): os.makedirs(os.path.dirname(self.path)) with open(self.path, "w", encoding="utf-8") as f: diff --git a/tests/test_publisher.py b/tests/test_publisher.py index f8306b1..c084299 100644 --- a/tests/test_publisher.py +++ b/tests/test_publisher.py @@ -2,7 +2,9 @@ import json import os import tempfile +import threading import unittest +from unittest import mock import RNS @@ -12,7 +14,8 @@ from rnfs.content.cid import RAW, Cid from rnfs.content.storage import USER, Storage from rnfs.dht.providers import ProviderStore -from rnfs.publisher import SLOW_LOOKUP_PACE, SLOW_ROUND_LOOKUPS, STATE_FILE, Publisher +from rnfs import publisher as publisher_module +from rnfs.publisher import SLOW_LOOKUP_PACE, SLOW_ROUND_LOOKUPS, STATE_FILE, DhtKey, Publisher from tests.support import FakeClock OWN = bytes(16) @@ -356,6 +359,99 @@ def test_served_blocks_are_the_provided_dags(self): self.assertTrue(self.publisher.serves(block)) self.assertFalse(self.publisher.serves(Cid.for_block(RAW, b"other"))) + def test_refresh_builds_the_sets_again_only_when_they_change(self): + """Each refresh provides and serves exactly the complete candidates' DAGs and the sources' keys; when + none of them changed, the same served set is kept rather than built again.""" + keys: set[bytes] = set() + self.publisher.add_key_source(lambda: keys) + self.eligible = set() + + def check(): + """The publisher's sets are those of storage and the key source now.""" + self.publisher.refresh() + candidates = set(self.storage.pins()) | (self.storage.cached_targets() & self.eligible) + dags = {cid: self.storage.dag_blocks(cid) for cid in candidates if self.storage.is_complete(cid)} + self.assertEqual(self.publisher.provided(), set(dags)) + self.assertEqual(self.publisher.provided_keys(), keys) + self.assertEqual(self.publisher._served, frozenset().union(*dags.values())) + return self.publisher._served + + a = self.pinned(os.urandom(100_000)) + served = check() + self.assertIs(check(), served) # nothing changed + b = self.pinned(os.urandom(100_000)) # added + self.assertIsNot(check(), served) + keys.add(bytes([9]) * 16) + served = check() + self.assertIn(DhtKey(bytes([9]) * 16), self.publisher._provided) + self.assertIs(check(), served) + # A pin whose DAG is missing a block isn't provided until the block is stored + blocks: dict[Cid, bytes] = {} + partial = builder.build_bytes(os.urandom(100_000), lambda cid, block: blocks.setdefault(cid, block)).cid + missing = next(cid for cid in blocks if cid.is_raw) + for cid, block in blocks.items(): + if cid != missing: + self.storage.put(cid, block) + self.storage.pin(partial, USER) + self.assertIs(check(), served) + self.storage.put(missing, blocks[missing]) + self.assertIsNot(check(), served) + self.storage.unpin(a, USER) # gone + served = check() + self.assertFalse(self.publisher.serves(a)) + # A cached target the roots strategy provides, until it is evicted + self.config.provide_strategy = "roots" + fetched = builder.build_bytes(b"fetched", self.storage.put) + self.storage.record_fetch(fetched.cid, [], fetched.cid) + self.eligible.add(fetched.cid) + self.assertIsNot(check(), served) + self.storage.gc() + check() + keys.clear() + check() + self.assertEqual(self.publisher.provided(), {b, partial}) + + def test_refresh_withdraws_what_isnt_provided_when_the_sets_are_unchanged(self): + """State loaded at startup is withdrawn even when the provided set is the same as before (empty).""" + cid = self.pinned(b"once provided") + self.publisher.refresh() + self.publisher.publish_due() + self.storage.unpin(cid, USER) + restarted = self.make_publisher() + restarted.refresh() + self.assertEqual(restarted.provided(), set()) + restarted.publish_due() + self.assertEqual(self.withdrawn(), {N1: {cid.key}, N2: {cid.key}}) + + def test_saved_without_the_lock_and_in_order(self): + """published.json is written outside the lock, and a save that snapshots meanwhile lands last.""" + a, b = self.pinned(b"a"), self.pinned(b"b") + self.publisher.refresh() + real_write = publisher_module.write_json + free, second = [], [] + + def write(path, value): + """The first save's write: another thread takes the lock, then unpins b and saves.""" + if not second: + lock = self.publisher._lock + thread = threading.Thread(target=lambda: free.append(lock.acquire(timeout=5)) or lock.release()) + thread.start() + thread.join() + self.storage.unpin(b, USER) + second.append(threading.Thread(target=self.publisher.refresh)) + second[0].start() + second[0].join(0.2) + real_write(path, value) + + with mock.patch.object(publisher_module, "write_json", write): + self.publisher.publish_due() + second[0].join(5) + self.assertEqual(free, [True]) + with open(os.path.join(self.state, STATE_FILE), encoding="utf-8") as f: + saved = json.load(f) + self.assertEqual(set(saved["published"]), {str(a)}) + self.assertEqual(saved["withdraw"], {N1.hex(): [b.key.hex()], N2.hex(): [b.key.hex()]}) + if __name__ == "__main__": unittest.main() diff --git a/tests/test_routing.py b/tests/test_routing.py index fe2d1a7..a2e2218 100644 --- a/tests/test_routing.py +++ b/tests/test_routing.py @@ -80,6 +80,38 @@ def test_closest_ranks_every_bucket(self): self.assertEqual(self.table.closest(key), expected) self.assertEqual(len(self.table.closest(key, count=3)), 3) + def test_closest_matches_ranking_every_node(self): + """Walking buckets from the key's gives exactly what sorting the whole table by distance gives.""" + rng = random.Random(3) + for _ in range(60): + own = rng.randbytes(ID_LENGTH) + own_value = int.from_bytes(own, "big") + table = RoutingTable(own, self.config, clock=self.clock) + # Random IDs fill the far buckets; IDs sharing a prefix with own reach the near ones, some left empty + ids = [rng.randbytes(ID_LENGTH) for _ in range(rng.randrange(0, 300))] + for bucket in rng.sample(range(BUCKET_COUNT), rng.randrange(0, BUCKET_COUNT)): + for _ in range(rng.randrange(1, K + 3)): + ids.append((own_value ^ (1 << bucket) ^ rng.getrandbits(bucket)).to_bytes(ID_LENGTH, "big")) + for n in ids: + table.heard(n) + stored = table.nodes() + # Some go silent or fail a query, so replies leave them out + self.clock.now += 2 * ANNOUNCE + 1 + for n in stored: + if rng.random() < 0.5: + table.heard(n) + if rng.random() < 0.3: + table.failed(n) + live = [n for n in stored if table.standing(n, 0) < 2] + keys = [own, rng.randbytes(ID_LENGTH)] + rng.sample(stored, min(3, len(stored))) + keys += [(own_value ^ rng.getrandbits(rng.randrange(1, 129))).to_bytes(ID_LENGTH, "big") for _ in range(3)] + for key in keys: + for count in (0, 1, 3, K, len(stored) + 5): + expected = sorted(stored, key=lambda n: distance(n, key))[:count] + self.assertEqual(table.closest(key, count), expected) + expected = sorted(live, key=lambda n: distance(n, key))[:count] + self.assertEqual(table.closest(key, count, for_reply=True), expected) + class ReplacementTest(RoutingTestCase): """A full bucket keeps its nodes unless the least recently heard has stopped answering or gone silent.""" diff --git a/tests/test_server.py b/tests/test_server.py index 7a50335..2720366 100644 --- a/tests/test_server.py +++ b/tests/test_server.py @@ -11,8 +11,8 @@ from RNS.vendor import umsgpack from rnfs.config import Config -from rnfs.constants import (BLOCKS_PATH, FIND_PATH, ID_LENGTH, K, KIB, MAX_REQUEST_SIZE, MAX_RESPONSE_SIZE, PIN_PATH, - PROVIDE_PATH, RESPONSE_OVERHEAD) +from rnfs.constants import (BLOCKS_PATH, CHUNK_SIZE, FIND_PATH, ID_LENGTH, K, KIB, MAX_REQUEST_SIZE, MAX_RESPONSE_SIZE, + PIN_PATH, PROVIDE_PATH, RESPONSE_OVERHEAD) from rnfs.content.cid import RAW, Cid from rnfs.network import guards, protocol, server from rnfs.network.identity import node_id_for_identity @@ -60,6 +60,36 @@ def test_upload_cap_lowers_rate(self): def test_max_bytes(self): self.assertEqual(server.blocks_budget(5000, 1e9, None, 60), 5000) + def test_block_over_the_budget_isnt_read(self): + (c1, b1), (c2, _), (c3, b3) = block_of(40_000, 1), block_of(10, 2), block_of(30_000, 3) + held, reads = {c1: b1, c3: b3}, [] + + def read(cid): + """A served block, noting the read.""" + reads.append(cid) + return held.get(cid) + sizes = {cid: len(block) for cid, block in held.items()} + self.assertEqual(server.select_blocks([c1, c2, c3], read, 60_000, sizes.get), [b1, None]) + self.assertEqual(reads, [c1, c2]) + # The first served block is read whatever its size + reads.clear() + self.assertEqual(server.select_blocks([c3, c1], read, 0, sizes.get), [b3]) + self.assertEqual(reads, [c3]) + + def test_sizes_leave_the_entries_unchanged(self): + rng = random.Random(3) + for trial in range(300): + blocks = [block_of(rng.choice([0, 10, 1000, 40_000, 100 * KIB, rng.randrange(1, 128 * KIB)]), i) + for i in range(rng.randrange(0, 12))] + held = {cid: block for cid, block in blocks if rng.random() < 0.8} + # A size may be unknown (None), and is never given for a block that isn't served + sizes = {cid: len(block) for cid, block in held.items() if rng.random() < 0.8} + cids = [cid for cid, _ in blocks] + limit = rng.choice([0, 1000, 60_000, CHUNK_SIZE, MAX_RESPONSE_SIZE - RESPONSE_OVERHEAD]) + with self.subTest(trial=trial): + self.assertEqual(server.select_blocks(cids, held.get, limit, sizes.get), + server.select_blocks(cids, held.get, limit)) + class ResponsePoolsTest(unittest.TestCase): """Slow and fast pools, displacement and refusal.""" @@ -460,6 +490,66 @@ def test_response_is_tracked_until_complete_and_gives_rate(self): self.assertEqual(self.server.incoming.in_progress(link.link_id), (0, 0)) self.assertEqual(self.server.rates.get(link.link_id), 50_000) + def test_backend_sizes_spare_reading_the_block_over_the_budget(self): + link = self.connect(b"a") + self.server.rates.record(link.link_id, 64 * KIB, 8000 * 8) # slow: room for two 100 KiB blocks + reads = [] + served = self.backend.served_block + self.backend.served_block = lambda cid: reads.append(cid) or served(cid) + self.backend.served_block_size = lambda cid: len(self.backend.blocks[cid]) + cids = [cid for cid, _ in self.big] + self.assertEqual(self.blocks(link, cids), [block for _, block in self.big[:2]]) + self.assertEqual(reads, cids[:2]) + + def test_idle_monitor_waits_for_a_handover(self): + waits = [] + + class Wake: + """Records the monitor's waits; the second one stops it.""" + + def wait(_, timeout=None): + """Note the timeout.""" + waits.append(timeout) + if len(waits) == 2: + self.server.stop_event.set() + + def clear(_): + """Nothing to clear.""" + + def set(_): + """Nothing to wake.""" + self.server.wake = Wake() + self.server._monitor_loop() + self.assertEqual(waits, [None, None]) # no response handed to RNS: nothing to look at + # A response handed over, its Resource not found yet: soon + self.server.stop_event.clear() + waits.clear() + self.blocks(self.connect(b"a")) + self.server._monitor_loop() + self.assertEqual(waits, [server.MONITOR_FAST_TICK, server.MONITOR_FAST_TICK]) + + def test_idle_monitor_wakes_for_a_handover_and_stops_promptly(self): + passes = [] + check = self.server.check_responses + self.server.check_responses = lambda: passes.append(1) or check() + self.server.monitor.start() + self.addCleanup(self.server.monitor.join, 5) + self.addCleanup(self.server.stop) + deadline = time.monotonic() + 5 + while not passes and time.monotonic() < deadline: + time.sleep(0.01) + time.sleep(2 * server.MONITOR_TICK) + self.assertEqual(len(passes), 1) # idle: no ticks + self.blocks(self.connect(b"a")) + while len(passes) < 2 and time.monotonic() < deadline: + time.sleep(0.01) + self.assertGreaterEqual(len(passes), 2) + self.server._finish(self.server.pools.responses()[0]) + time.sleep(2 * server.MONITOR_FAST_TICK) # idle again + self.server.stop() + self.server.monitor.join(1) + self.assertFalse(self.server.monitor.is_alive()) + def test_fast_transfer_concluded_before_the_monitor_looks(self): link = self.connect(b"a") self.blocks(link) diff --git a/tests/test_storage.py b/tests/test_storage.py index 402ca87..3d67416 100644 --- a/tests/test_storage.py +++ b/tests/test_storage.py @@ -104,6 +104,80 @@ def worker(chunk): self.assertEqual(self.storage.block_count(), 200) +class WriterTest(StorageTest): + """Session.writer(): blocks stored on several threads, all of them before the with block ends.""" + + def test_every_block_is_stored_and_protected_when_it_ends(self): + data = os.urandom(40 * 65536 + 5) + with self.storage.protect() as session: + with session.writer() as put: + built = builder.build_bytes(data, put) + self.assertTrue(self.storage.is_complete(built.cid)) + self.assertEqual(self.storage.gc()["removed_blocks"], 0) # protected while the session lasts + self.storage.pin(built.cid, USER) + self.assertEqual(built.cid, builder.build_bytes(data, builder.NO_PUT).cid) + self.assertEqual(self.storage.used_bytes(), sum(self.storage.blocks.size(c) for c in self.blocks_of(built.cid))) + reopened = self.open() + self.assertTrue(reopened.is_complete(built.cid)) + + def test_writes_overlap(self): + # Each put waits for its sync to disk; one after another they take as long as all of those together + writing, most, started = 0, 0, 0 + lock = threading.Lock() + real = self.storage.blocks.put + both = threading.Barrier(2, timeout=5) + + def put(cid, block): + """Count the puts in progress, and hold the first two until both are.""" + nonlocal writing, most, started + with lock: + writing += 1 + started += 1 + most = max(most, writing) + first_two = started <= 2 + try: + if first_two: + both.wait() + real(cid, block) + finally: + with lock: + writing -= 1 + + with mock.patch.object(self.storage.blocks, "put", put), self.storage.protect() as session: + with session.writer() as write: + builder.build_bytes(os.urandom(8 * 65536), write) + self.assertGreaterEqual(most, 2) + + def test_a_failed_write_is_raised_and_the_rest_still_finish(self): + real = self.storage.blocks.put + calls = [] + + def put(cid, block): + """Fail the third block, as a full disk would.""" + calls.append(cid) + if len(calls) == 3: + raise OSError(28, "No space left on device") + real(cid, block) + + with mock.patch.object(self.storage.blocks, "put", put), self.storage.protect() as session: + with self.assertRaises(OSError): + with session.writer() as write: + builder.build_bytes(os.urandom(20 * 65536), write) + self.assertEqual(self.storage.pins(), {}) + # Every block written is indexed, and nothing else: a write in progress at the error isn't lost track of + self.assertEqual(set(self.storage._sizes), {c for c in calls if self.storage.blocks.has(c)}) + + def test_an_error_in_the_body_waits_for_what_it_queued(self): + blocks = [os.urandom(1000) for _ in range(10)] + with self.storage.protect() as session: + with self.assertRaises(RuntimeError): + with session.writer() as put: + for block in blocks: + put(Cid.for_block(RAW, block), block) + raise RuntimeError("the add failed") + self.assertEqual(self.storage.block_count(), 10) # kept as cache, as before + + class PinTest(StorageTest): """Pins and their sources.""" @@ -135,6 +209,46 @@ def test_persisted(self): self.storage.pin(cid, USER) self.assertEqual(self.open().pins(), {cid: {"folder", "user"}}) + def test_pin_many_saves_once(self): + a, b, c = (self.add(b"%d" % i).cid for i in range(3)) + self.storage.pin(a, USER) + with mock.patch.object(self.storage, "_save_pins", wraps=self.storage._save_pins) as save: + self.storage.pin_many([a, b, c, b], USER) + self.assertEqual(save.call_count, 1) + self.storage.pin_many([a, c], USER) # nothing new: nothing to save + self.storage.pin_many([], USER) + self.assertEqual(save.call_count, 1) + self.assertEqual(self.storage.pins(), {a: {"user"}, b: {"user"}, c: {"user"}}) + self.assertEqual(self.open().pins(), self.storage.pins()) + with self.assertRaises(ValueError): + self.storage.pin_many([a], "users") + + def test_failed_save_pins_nothing(self): + a, b, c = (self.add(b"%d" % i).cid for i in range(3)) + self.storage.pin(a, USER) + self.storage.pin(b, FOLDER) + failure = OSError(28, "No space left on device") + for pin in (lambda: self.storage.pin_many([a, b, c], USER), lambda: self.storage.pin(c, USER)): + with mock.patch.object(storage_module, "write_json", side_effect=failure): + with self.assertRaises(OSError): + pin() + # Memory says what pins.json does + self.assertEqual(self.storage.pins(), {a: {"user"}, b: {"folder"}}) + self.assertEqual(self.open().pins(), self.storage.pins()) + self.assertEqual(self.storage.pin_count(), 2) + + def test_pin_count(self): + self.assertEqual(self.storage.pin_count(), 0) + a, b = self.add(b"a").cid, self.add(b"b").cid + self.storage.pin(a, USER) + self.storage.pin(a, FOLDER) + self.storage.pin(b, USER) + self.assertEqual(self.storage.pin_count(), 2) # CIDs, not sources + self.storage.unpin(a, USER) + self.assertEqual(self.storage.pin_count(), 2) + self.storage.unpin(a, FOLDER) + self.assertEqual(self.storage.pin_count(), 1) + def test_damaged_state_is_moved_aside(self): with open(os.path.join(self.tmp.name, "state", "pins.json"), "w", encoding="utf-8") as f: f.write("{not json") @@ -440,6 +554,64 @@ def test_verify(self): self.assertEqual(stats["incomplete"], {root.cid}) self.assertFalse(self.storage.is_complete(root.cid)) # the memo was invalidated + def test_verify_checks_completeness_without_the_lock(self): + pinned, cached, partial = (self.add(byte * 70000) for byte in (b"A", b"B", b"C")) + self.storage.pin(pinned.cid, USER) + self.storage.record_fetch(cached.cid, [], cached.cid) + self.storage.record_fetch(partial.cid, [], partial.cid) + chunks = {built.cid: [child.cid for child in dag.decode(self.storage.get(built.cid)).children] + for built in (pinned, partial)} + self.storage._delete({chunks[partial.cid][0]}) + self.storage = self.open() # nothing memoised as complete + damaged = {chunks[pinned.cid][1], chunks[partial.cid][1]} # chunks: the completeness walk doesn't read them + for cid in damaged: + with open(self.storage.blocks.path(cid), "wb") as f: + f.write(b"damaged") + held = [] + real = Storage._complete_sets + with mock.patch.object(Storage, "_complete_sets", + lambda s, cids: (held.append(s._lock._is_owned()), real(s, cids))[1]): + stats = self.storage.verify() + self.assertEqual(held, [False]) + self.assertEqual(set(stats["bad"]), damaged) + self.assertEqual(stats["incomplete"], {pinned.cid}) # not partial: it was incomplete before + + def test_cache_generation(self): + files = [self.add(b"%d" % i * 1000) for i in range(4)] + generation = self.storage.cache_generation() + + def changed() -> bool: + """True if the cache generation moved since the last call.""" + nonlocal generation + moved, generation = self.storage.cache_generation() != generation, self.storage.cache_generation() + return moved + + self.storage.record_fetch(files[0].cid, [], files[0].cid) + self.storage.record_fetch(files[1].cid, [], files[1].cid) + self.storage.record_fetch(files[1].cid, [], files[1].cid) # touched again, same target + self.storage.touch(files[0].cid, []) + self.assertFalse(changed()) # records added or touched only + self.storage.record_fetch(files[1].cid, [], files[2].cid) + self.assertTrue(changed()) # files[1] is no longer a cached target + self.storage.evict() + self.assertFalse(changed()) # under max_storage + self.storage.max_storage = 3000 + self.storage.evict() + self.assertFalse(changed()) # deleting unreached blocks was enough + self.storage.max_storage = 1500 + self.storage.evict() + self.assertTrue(changed()) + self.assertEqual(len(self.storage.cached_fetches()), 1) + self.storage.gc() + self.assertTrue(changed()) + self.storage.gc() + self.assertFalse(changed()) # no records left to remove + with mock.patch.object(storage_module, "MAX_CACHE_RECORDS", 1): + self.storage.record_fetch(files[0].cid, [], files[0].cid) + self.assertFalse(changed()) + self.storage.record_fetch(files[3].cid, [], files[3].cid) + self.assertTrue(changed()) # the cap dropped a record + def big_pinned_file(self) -> tuple[Cid, Cid]: """A pinned file of 300 chunks (a root over two index nodes); returns (root, its last child node).""" built = self.add(os.urandom(300 * 64 * 1024)) @@ -554,6 +726,38 @@ def test_stat(self): self.assertEqual(stat["blocks"], 5) # root, a's node and 2 chunks, b's node (its chunks are unknown) self.assertEqual(stat["local_blocks"], 4) + def test_stat_of_a_partial_dag_doesnt_hide_nodes_stored_later(self): + root, a, b = self.tree() + blocks = {cid: self.storage.get(cid) for cid in self.blocks_of(root.cid)} + self.storage = Storage(os.path.join(self.tmp.name, "blocks2"), os.path.join(self.tmp.name, "state2"), 10**9) + self.storage.put(root.cid, blocks[root.cid]) + self.storage.pin(root.cid, USER) + self.assertEqual(self.storage.stat(root.cid)["local_blocks"], 1) # a's and b's nodes aren't stored yet + for cid, block in blocks.items(): + self.storage.put(cid, block) + self.assertEqual(self.storage.stat(root.cid)["local_blocks"], 7) + self.assertEqual(self.storage.gc()["removed_blocks"], 0) # a's and b's chunks are reached through them + self.assertTrue(self.storage.is_complete(root.cid)) + + def test_size_queries_read_nodes_without_the_lock(self): + root, a, b = self.tree() + self.storage.pin(root.cid, remote_source(IDENTITY)) + expected = {"pinned": self.storage.pinned_bytes(), "dag": self.storage.dag_bytes(root.cid), + "usage": self.storage.source_usage(remote_source(IDENTITY)), "stat": self.storage.stat(root.cid)} + queries = {"pinned": lambda s: s.pinned_bytes(), "dag": lambda s: s.dag_bytes(root.cid), + "usage": lambda s: s.source_usage(remote_source(IDENTITY)), "stat": lambda s: s.stat(root.cid)} + for name, query in queries.items(): + with self.subTest(name): + self.storage = self.open() # nothing memoised + held = [] + real = Storage._read_node + with mock.patch.object(Storage, "_read_node", + lambda s, cid: (held.append(s._lock._is_owned()), real(s, cid))[1]): + self.assertEqual(query(self.storage), expected[name]) + self.assertEqual(len(held), 3) # root, a's node and b's node, once each + self.assertNotIn(True, held) + self.assertFalse(any(cid.is_raw for cid in self.storage._links)) # raw blocks list nothing + if __name__ == "__main__": unittest.main() From 407c7e10ae1bda617efac4ef5e518881e4eeaef4 Mon Sep 17 00:00:00 2001 From: Afi Date: Fri, 16 Oct 2026 11:52:40 +1030 Subject: [PATCH 2/9] Add logging for DHT lookup failures in RnfsNode --- rnfs/node.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/rnfs/node.py b/rnfs/node.py index bbd8942..40df3ca 100644 --- a/rnfs/node.py +++ b/rnfs/node.py @@ -520,7 +520,8 @@ def join(self): alpha=self.config.effective_alpha(self.slow), beta=self.config.beta, timeout=self.config.lookup_timeout, is_known=self._is_known, on_answer=self._answered, on_failure=self.routing.failed, - exclude=lambda node_id: node_id in contacted or self.routing.is_banned(node_id)) + exclude=lambda node_id: node_id in contacted or self.routing.is_banned(node_id), + report=lambda text: log(text, LOG_DEBUG)) lookup.find_closest() log(f"joined the DHT: {plural(self.routing.node_count(), 'server')} known", LOG_NOTICE) From de3a1715d9f79ac6ec0df1657cc9eec5c5205769 Mon Sep 17 00:00:00 2001 From: Afi Date: Fri, 16 Oct 2026 14:28:08 +1030 Subject: [PATCH 3/9] Enhance debug logging: include more detailed RNS messages and clarify link closure reasons --- CHANGELOG.md | 3 ++- README.md | 2 +- design.md | 4 ++-- rnfs/debuglog.py | 13 +++++++++---- rnfs/network/links.py | 19 ++++++++++++++----- tests/test_debuglog.py | 12 ++++++++++++ tests/test_links.py | 13 +++++++++++++ 7 files changed, 53 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index dabf2de..37e852a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,7 +11,8 @@ Added: - A debug log file, for diagnosing problems: `debug_log = true` in `rnfs.toml`, or `rnfs daemon --debug-log`, writes `/debug.log`. It begins with the daemon's version (then the Python, RNS and LXMF versions, the platform and the settings changed from the defaults), and has every rnfs message at every level whatever - `verbosity` is; Reticulum's own messages, such as an interface that can't connect; each gateway request, + `verbosity` is; Reticulum's own messages at Info and more severe, such as an interface that can't connect or a + socket that drops (its Verbose and Debug detail stays on the console only); each gateway request, numbered, with the headers that decide it, what was decided (stored or fetched, type, range) and its status, size and time; errors with tracebacks, and threads that die; and a status line every 5 minutes (counts, open links, each interface up or down). New detail, at debug level: each DHT lookup's queries (which node diff --git a/README.md b/README.md index 10d0ad5..db4ac2d 100644 --- a/README.md +++ b/README.md @@ -600,7 +600,7 @@ lxmf_auto_fetch_from = [""] | `max_records` | `100000` | Provider records stored for others | | `ban_time` | `3600` | How long a node that sent a bad block is ignored | | `verbosity` | `0` | Extra logging: 1 per-query detail, 2 RNS debug output (`daemon -v`, `-vv`) | -| `debug_log` | `false` | Also write `/debug.log` for diagnosing problems (`daemon --debug-log`): it begins with the daemon's version, then has every rnfs message at every level whatever `verbosity` is (which nodes are asked, which fail and why, links that fail or close), Reticulum's own messages (interfaces that can't connect), each gateway request with its outcome, errors with tracebacks, and a status line every 5 minutes. It names what you fetched and browsed, so it is readable by you only and never sent anywhere; each start keeps the previous run's as `debug.log.1`, and the two stay under 32 MiB | +| `debug_log` | `false` | Also write `/debug.log` for diagnosing problems (`daemon --debug-log`): it begins with the daemon's version, then has every rnfs message at every level whatever `verbosity` is (which nodes are asked, which fail and why, links that fail or close), Reticulum's own messages at Info and above (interfaces that can't connect or drop; not its debug detail), each gateway request with its outcome, errors with tracebacks, and a status line every 5 minutes. It names what you fetched and browsed, so it is readable by you only and never sent anywhere; each start keeps the previous run's as `debug.log.1`, and the two stay under 32 MiB | **Storage** diff --git a/design.md b/design.md index bbb0994..e4b5914 100644 --- a/design.md +++ b/design.md @@ -22,8 +22,8 @@ authority on behaviour; this file fixes module boundaries and the interfaces bet (`set_debug_sink`), every message also goes to it, whatever RNS's loglevel. - `rnfs/debuglog.py` — the debug log file (`debug_log`, `daemon --debug-log`), `/debug.log` (0600), started by the daemon before the node: version first, then every rnfs message (warnings and errors logged in - an `except` block with their traceback), RNS's own log lines (copied from `RNS.logdest = LOG_CALLBACK`, which - still prints them), threads that die (`threading.excepthook`), a status line every 5 minutes (`status_line`: + an `except` block with their traceback), RNS's own log lines at Info or more severe (`RNS_LEVEL`; copied from + `RNS.logdest = LOG_CALLBACK`, which still prints all of them), threads that die (`threading.excepthook`), a status line every 5 minutes (`status_line`: the node's status, open links, each interface up or down), and gateway requests, numbered (`tag`). Each start and every 16 MiB moves the file to `debug.log.1`. Lookups report their queries through `Lookup(report=)` (dht stays free of RNS), links their path, close reason and each request's outcome, all at `LOG_DEBUG`. diff --git a/rnfs/debuglog.py b/rnfs/debuglog.py index 276e84c..005f779 100644 --- a/rnfs/debuglog.py +++ b/rnfs/debuglog.py @@ -5,8 +5,10 @@ is moved there too and a new one begun, also with the version first, so the two hold at most 2 × MAX_SIZE. - Every rnfs message at every level, whatever verbosity the console shows (rnfs.log hands each one here), with the thread that logged it, and, for a warning or error logged while handling an exception, its traceback. -- Reticulum's own messages at its loglevel (interfaces failing to connect, paths, links), copied from its log - output, which still goes to the console as before; threads that die of an exception; and a status line every +- Reticulum's own messages at Info and more severe (RNS_LEVEL) that its loglevel lets through: interfaces + failing to connect, sockets dropping and reconnecting. Its Verbose and Debug detail (handshakes, ratchets, + table cleaning, other nodes' announces) isn't copied, and rnfs logs what matters of links itself. The console + still gets all of RNS's output as before. Also threads that die of an exception, and a status line every STATUS_INTERVAL: the node's counts, its open links and each interface's state. - Gateway requests (rnfs/gateway.py) are numbered, and the lines logged on a request's thread carry its number. @@ -31,6 +33,7 @@ DEBUG_LOG_FILE = "debug.log" MAX_SIZE = 16 * 1024 * 1024 # bytes per file; the one before is kept as debug.log.1 STATUS_INTERVAL = 300.0 # seconds between status lines +RNS_LEVEL = 4 # RNS.LOG_INFO: Reticulum's own messages this severe or more are copied # RNS's loglevels by number, named as its log lines name them LEVEL_NAMES = {0: "Critical", 1: "Error", 2: "Warning", 3: "Notice", 4: "Info", 5: "Verbose", 6: "Debug", 7: "Pathing", 8: "Extra"} @@ -168,9 +171,11 @@ def _rns_output(logstring: str): return match = _RNS_LINE.fullmatch(logstring) if match and match.group(1) in _LEVEL_NUMBERS: - debug.write(_LEVEL_NUMBERS[match.group(1)], "RNS", match.group(2)) + level = _LEVEL_NUMBERS[match.group(1)] + if level <= RNS_LEVEL: + debug.write(level, "RNS", match.group(2)) else: - debug.write(rnfs_log.LOG_INFO, "RNS", logstring) + debug.write(rnfs_log.LOG_INFO, "RNS", logstring) # a format without levels: kept, as it can't be told def start(config) -> DebugLog: diff --git a/rnfs/network/links.py b/rnfs/network/links.py index b3714b7..24fbdc1 100644 --- a/rnfs/network/links.py +++ b/rnfs/network/links.py @@ -34,15 +34,24 @@ DEADLINE_FACTOR = 2 # a request other than /blocks may take this many times RNS's own request timeout # What RNS adds to rtt × traffic_timeout_factor for its request timeout (Link.request with timeout=None) RESPONSE_GRACE = protocol.REQUEST_TIMEOUT_WITHOUT_RTT -# Why RNS closed a link (Link.teardown_reason), for the log -CLOSE_REASONS = {RNS.Link.TIMEOUT: "timed out", RNS.Link.INITIATOR_CLOSED: "closed by this node", - RNS.Link.DESTINATION_CLOSED: "closed by the other node"} +# Link.teardown_reason: a timeout, or which end closed it (the one that opened it, or the other) +LINK_TIMEOUT, INITIATOR_CLOSED, DESTINATION_CLOSED = (RNS.Link.TIMEOUT, RNS.Link.INITIATOR_CLOSED, + RNS.Link.DESTINATION_CLOSED) RECEIPT_FAILED = RNS.RequestReceipt.FAILED # a request RNS gave up on def close_reason(link) -> str: - """Why link closed, as the log says it.""" - return CLOSE_REASONS.get(getattr(link, "teardown_reason", None), "closed") + """Why link closed, as the log says it: timed out, or closed by this node or the other. RNS names the end + that closed it by its role, so for a link the other node opened (link.initiator false) they swap.""" + reason = getattr(link, "teardown_reason", None) + if reason == LINK_TIMEOUT: + return "timed out" + ours = INITIATOR_CLOSED if getattr(link, "initiator", True) else DESTINATION_CLOSED + if reason == ours: + return "closed by this node" + if reason in (INITIATOR_CLOSED, DESTINATION_CLOSED): + return "closed by the other node" + return "closed" def path_description(node_id: bytes) -> str: diff --git a/tests/test_debuglog.py b/tests/test_debuglog.py index f4bfa73..1f19e57 100644 --- a/tests/test_debuglog.py +++ b/tests/test_debuglog.py @@ -143,6 +143,18 @@ def test_rns_messages_still_print_and_are_copied_once(self): self.assertEqual(len(self.logged("an rnfs line")), 1) # by rnfs.log's sink, not again as RNS output self.assertIn("rnfs: an rnfs line", self.logged("an rnfs line")[0]) + def test_rns_detail_below_info_isnt_copied(self): + RNS.loglevel = RNS.LOG_DEBUG + out = io.StringIO() + with contextlib.redirect_stdout(out): + RNS.log("Cleaning known destinations at background priority...", RNS.LOG_DEBUG) + RNS.log("Link establishment timed out", RNS.LOG_VERBOSE) + RNS.log("Reconnected socket for TCPInterface[backbone]", RNS.LOG_INFO) + self.assertIn("Cleaning known destinations", out.getvalue()) # the console is as before + self.assertFalse(self.logged("Cleaning known destinations")) + self.assertFalse(self.logged("Link establishment timed out")) + self.assertTrue(self.logged("RNS: Reconnected socket")) + def test_a_thread_that_dies_is_logged(self): def die(): """Raise in a thread.""" diff --git a/tests/test_links.py b/tests/test_links.py index c455cad..fcb5d10 100644 --- a/tests/test_links.py +++ b/tests/test_links.py @@ -315,6 +315,19 @@ def test_request_outcomes_are_logged_with_the_reason(self): self.assertIn("/find to", message) self.assertIn("not answered after", message) + def test_close_reason_names_the_end_that_closed_the_link(self): + def link(initiator, reason): + """A closed link as RNS leaves it.""" + return types.SimpleNamespace(initiator=initiator, teardown_reason=reason) + + # RNS records the role of the end that closed it: for a link opened by the other node, the roles swap + self.assertEqual(links.close_reason(link(True, links.INITIATOR_CLOSED)), "closed by this node") + self.assertEqual(links.close_reason(link(True, links.DESTINATION_CLOSED)), "closed by the other node") + self.assertEqual(links.close_reason(link(False, links.INITIATOR_CLOSED)), "closed by the other node") + self.assertEqual(links.close_reason(link(False, links.DESTINATION_CLOSED)), "closed by this node") + self.assertEqual(links.close_reason(link(False, links.LINK_TIMEOUT)), "timed out") + self.assertEqual(links.close_reason(types.SimpleNamespace()), "closed") + def test_request_gives_up_when_link_closes(self): FakeLink.responder = staticmethod(lambda link, path, data: SILENT) peer = run(self, lambda: self.pool.get(NODE)) From 0d13e92d6ac99c29a25b4796ecdfc7cb077f51e1 Mon Sep 17 00:00:00 2001 From: Afi Date: Fri, 16 Oct 2026 18:48:41 +1030 Subject: [PATCH 4/9] Keep state in one SQLite database, migrating the JSON state files at startup The 17 JSON files under /state/ are replaced by state/rnfs.db (the standard sqlite3 module; no new dependency). Nothing changes on the wire: request paths, messages, CIDs, DAG bytes and provider and name records are as before, and 0.1.4 and new nodes find, fetch and resolve each other's content and names. rnfs/statedb.py is the database layer: one shared connection per state directory (each store holds a counted reference), WAL mode with busy_timeout, and files created 0600. The schema is a list of steps counted by PRAGMA user_version. Startup runs PRAGMA quick_check and refuses, with what to do, a damaged database, one from a newer version, or a Python without _sqlite3. transaction(durable) is the only way to write and its lock is the innermost lock of all: FULL for state saved on each change, NORMAL for batched saves. SQLite failures raise StateDbError, an OSError. Migration: at startup each legacy file still present is read with read_json and the old shape check. Its importer checks every record as the old loader did, and inserts them in one transaction that also records the file in the migrated table. The file is then renamed .migrated and the directory synced. A crash between the commit and the rename only renames at the next start, so no file is imported twice or by half. Every format written by 0.1.3, 0.1.4 and dev is accepted. Damaged files are handled as before: moved to .damaged, except an unreadable pins.json or a damaged names.json still stops startup. The upgrade is one-way. Stores keep their state in memory as the read path and write only what changed. A pin or release is one row, committed durably after blocks.sync(), and memory is rolled back if the commit fails. Cache records, provider records, publish state, the name floor, keepers and the LXMF inbox and outbox track changed keys. Every validation and cap of the JSON loaders still applies at load; malformed rows are dropped with a warning and deleted. The reconcile logic of names, follow and LXMF is unchanged. Measured: Storage.pin() takes 2.5 ms at any pin count (was 5 to 14 ms at 100 to 20,000 pins). A provider save with one change in 100k records takes 0.02 ms (was 155 ms), a cache save with one change in 10k 0.02 ms (was 13 ms). Startup with 20k pins and 100k records takes 354 ms (was 463 ms). Behaviour changes: batched saves are no longer synced at each save, so a power loss may lose the last one (never part of one). A state directory that can't be written stops startup with a clear message. An unparsable providers.json is kept as .damaged. published.json IDs must be 16 bytes. A malformed names row stops startup, as a malformed names.json did. Tests: existing tests read the database through new helpers in tests/support.py and close their stores so Windows can delete temp dirs. The damaged-state tests now cover damaged legacy files and damaged rows. New: test_statedb.py; test_migrate_*.py per store; test_upgrade_014.py, which loads tests/fixtures/rnfs-0.1.4/, a node directory written by 0.1.4's own code; its blocks are rebuilt by the test, not checked in. The smoke test now checks the database holds a pin and imported a 0.1.4 file; sqlite3 is named in the PyInstaller spec. Docs: design.md (state database, migration, lock order, State files), specification.md (storage and jobs wording), CHANGELOG.md, README.md. Test and lint caches added to .gitignore. --- .gitignore | 6 + CHANGELOG.md | 47 +- README.md | 2 +- design.md | 134 +++-- packaging/pyinstaller/rnfs.spec | 4 +- packaging/smoke_test.py | 36 +- rnfs/content/archive.py | 2 +- rnfs/content/shared_folder.py | 75 ++- rnfs/content/storage.py | 214 ++++--- rnfs/dht/providers.py | 364 +++++++++--- rnfs/features/__init__.py | 3 + rnfs/features/contacts.py | 93 ++- rnfs/features/follow.py | 133 +++-- rnfs/features/jobs.py | 100 +++- rnfs/features/keepers.py | 136 +++-- rnfs/features/lxmf.py | 214 +++++-- rnfs/fetcher.py | 61 +- rnfs/log.py | 2 +- rnfs/names.py | 378 +++++++++--- rnfs/node.py | 137 +++-- rnfs/pin_names.py | 81 ++- rnfs/publisher.py | 261 ++++++-- rnfs/statedb.py | 560 ++++++++++++++++++ roadmap.md | 14 - specification.md | 26 +- tests/fixtures/rnfs-0.1.4/README.md | 54 ++ tests/fixtures/rnfs-0.1.4/files/notes.txt | 1 + tests/fixtures/rnfs-0.1.4/fixture_content.py | 25 + tests/fixtures/rnfs-0.1.4/generate.py | 407 +++++++++++++ tests/fixtures/rnfs-0.1.4/names/imported | 1 + tests/fixtures/rnfs-0.1.4/names/photos | 2 + tests/fixtures/rnfs-0.1.4/state/cache.json | 1 + tests/fixtures/rnfs-0.1.4/state/contacts.json | 1 + tests/fixtures/rnfs-0.1.4/state/fetched.json | 1 + tests/fixtures/rnfs-0.1.4/state/follow.json | 1 + tests/fixtures/rnfs-0.1.4/state/invalid.json | 1 + tests/fixtures/rnfs-0.1.4/state/jobs.json | 1 + tests/fixtures/rnfs-0.1.4/state/keepers.json | 1 + .../fixtures/rnfs-0.1.4/state/lxmf_inbox.json | 1 + .../rnfs-0.1.4/state/lxmf_outbox.json | 1 + .../fixtures/rnfs-0.1.4/state/name-floor.json | 1 + tests/fixtures/rnfs-0.1.4/state/names.json | 32 + .../fixtures/rnfs-0.1.4/state/pin_names.json | 1 + tests/fixtures/rnfs-0.1.4/state/pins.json | 1 + .../fixtures/rnfs-0.1.4/state/providers.json | 1 + .../fixtures/rnfs-0.1.4/state/published.json | 1 + .../rnfs-0.1.4/state/remote_pins.json | 1 + .../rnfs-0.1.4/state/shared_folder.json | 1 + tests/security/_pages_helpers.py | 7 +- tests/security/test_fuzz_parsers.py | 2 + tests/security/test_h2_display_page_pins.py | 2 +- .../security/test_l10_index_page_injection.py | 2 +- .../security/test_l11_display_page_micron.py | 2 +- tests/security/test_l12_lxmf_inbox.py | 4 +- tests/security/test_l4_name_rollback.py | 19 +- tests/security/test_l4r_name_floor.py | 19 +- tests/security/test_l8_folder_swap.py | 2 + tests/security/test_l8r2_hard_links.py | 2 + tests/security/test_m4_provider_admission.py | 42 +- tests/security/test_m4r_newcomer_pool.py | 49 +- tests/security/test_m6_index_page_full_run.py | 2 +- tests/security/test_m6_index_page_names.py | 2 +- tests/security/test_m6_pin_ls_sealed.py | 2 +- .../test_n1_display_page_unbounded.py | 2 +- .../test_new1_publisher_withdraw_race.py | 2 + tests/security/test_new2_follow_lxmf_pins.py | 47 +- .../security/test_new2_name_publish_crash.py | 60 +- tests/security/test_new3_remote_pin_wakeup.py | 2 +- .../test_new4_private_mode_records.py | 4 +- .../security/test_new5_storage_lock_walks.py | 2 + .../test_new6_publisher_crash_cut_round.py | 19 +- .../test_new6_remote_pins_bad_entry.py | 39 +- .../security/test_new6_remote_pins_restart.py | 4 +- .../test_r1_reconcile_damaged_state.py | 142 +++-- tests/security/test_r2_name_key_unreadable.py | 25 +- tests/support.py | 46 ++ tests/test_archive.py | 10 +- tests/test_contacts.py | 57 +- tests/test_fetcher.py | 45 +- tests/test_gateway_formats.py | 2 + tests/test_get_partial.py | 1 + tests/test_jobs.py | 91 ++- tests/test_lxmf.py | 252 ++++++-- tests/test_migrate_dht.py | 339 +++++++++++ tests/test_migrate_features.py | 425 +++++++++++++ tests/test_migrate_names.py | 350 +++++++++++ tests/test_migrate_small.py | 288 +++++++++ tests/test_migrate_storage.py | 332 +++++++++++ tests/test_names.py | 270 +++++++-- tests/test_names_airtime.py | 1 + tests/test_names_keep.py | 91 ++- tests/test_node.py | 36 +- tests/test_pacing.py | 2 + tests/test_pin_names.py | 57 +- tests/test_providers.py | 316 ++++++++-- tests/test_publisher.py | 155 ++++- tests/test_shared_folder.py | 84 ++- tests/test_statedb.py | 433 ++++++++++++++ tests/test_storage.py | 124 ++-- tests/test_upgrade_014.py | 421 +++++++++++++ 100 files changed, 7153 insertions(+), 1202 deletions(-) create mode 100644 rnfs/statedb.py create mode 100644 tests/fixtures/rnfs-0.1.4/README.md create mode 100644 tests/fixtures/rnfs-0.1.4/files/notes.txt create mode 100644 tests/fixtures/rnfs-0.1.4/fixture_content.py create mode 100644 tests/fixtures/rnfs-0.1.4/generate.py create mode 100644 tests/fixtures/rnfs-0.1.4/names/imported create mode 100644 tests/fixtures/rnfs-0.1.4/names/photos create mode 100644 tests/fixtures/rnfs-0.1.4/state/cache.json create mode 100644 tests/fixtures/rnfs-0.1.4/state/contacts.json create mode 100644 tests/fixtures/rnfs-0.1.4/state/fetched.json create mode 100644 tests/fixtures/rnfs-0.1.4/state/follow.json create mode 100644 tests/fixtures/rnfs-0.1.4/state/invalid.json create mode 100644 tests/fixtures/rnfs-0.1.4/state/jobs.json create mode 100644 tests/fixtures/rnfs-0.1.4/state/keepers.json create mode 100644 tests/fixtures/rnfs-0.1.4/state/lxmf_inbox.json create mode 100644 tests/fixtures/rnfs-0.1.4/state/lxmf_outbox.json create mode 100644 tests/fixtures/rnfs-0.1.4/state/name-floor.json create mode 100644 tests/fixtures/rnfs-0.1.4/state/names.json create mode 100644 tests/fixtures/rnfs-0.1.4/state/pin_names.json create mode 100644 tests/fixtures/rnfs-0.1.4/state/pins.json create mode 100644 tests/fixtures/rnfs-0.1.4/state/providers.json create mode 100644 tests/fixtures/rnfs-0.1.4/state/published.json create mode 100644 tests/fixtures/rnfs-0.1.4/state/remote_pins.json create mode 100644 tests/fixtures/rnfs-0.1.4/state/shared_folder.json create mode 100644 tests/test_migrate_dht.py create mode 100644 tests/test_migrate_features.py create mode 100644 tests/test_migrate_names.py create mode 100644 tests/test_migrate_small.py create mode 100644 tests/test_migrate_storage.py create mode 100644 tests/test_statedb.py create mode 100644 tests/test_upgrade_014.py diff --git a/.gitignore b/.gitignore index e19c2d4..d44c34c 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,9 @@ dist/ venv/ .flatpak-builder/ build-dir/ +# Left by test and lint runs +.ruff_cache/ +.hypothesis/ +.coverage +.coverage.* +htmlcov/ diff --git a/CHANGELOG.md b/CHANGELOG.md index 37e852a..08071cd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,8 +4,35 @@ ## Unreleased -Performance work, and a debug log. No wire-format, CID or state-file changes: these nodes and 0.1.4 nodes work -together, and each reads the other's ``. +Performance work, a debug log, and state kept in one SQLite database. No wire-format or CID changes: these +nodes and 0.1.4 nodes work together. The upgrade of an `` is automatic and one-way (an older version +can't read it afterwards). + +Changed: +- State is kept in one SQLite database, `/state/rnfs.db` (Python's standard `sqlite3`; no new + dependency), instead of 17 JSON files. At the first start the JSON files of 0.1.4 or 0.1.3 (pins, cache + records, pin names, fetched targets, remote pins waiting, invalid roots, publish state, provider records, + names and the name floor, the shared folder, jobs, keepers, follows, contacts, the LXMF inbox and outbox) are + imported, each in one transaction, with every check the old loaders applied, and kept as `.migrated` + (a node that skips versions upgrades the same way). A damaged file is moved aside as `.damaged` as before; an + unreadable `pins.json` or a damaged `names.json` still stops startup. Nothing changes on the network: request + paths, messages, CIDs, DAG bytes, provider and name records are as before. +- Saves write only what changed: a pin or release is one row, about 2.5 ms whatever the number of pins + (before: `pins.json` rewritten whole, 5 ms at 100 pins and 14 ms at 20 000); a provider-records save writes + the records changed since the last one (one change at 100 000 records: 0.02 ms, from 155 ms); a cache save + the records touched (0.02 ms, from 13 ms at 10 000 records). Startup with 20 000 pins and 100 000 provider + records loads in 350 ms, from 460. +- Pins, names, jobs and the other state saved on each change are on disk when the change returns, as before. + State saved in batches (cache records, fetched targets, invalid roots, remote pins waiting, provider records, + publish state, the name floor, keepers, the LXMF inbox and outbox) is no longer synced to disk at each save: + a power loss (not a crash) may lose the last of those saves as well as what changed since, never part of + one, and never one without every earlier save. +- A `providers.json` that can't be parsed is now kept as `.damaged` (before, the node started without its records + and overwrote it). Node IDs and keys in `published.json` must be 16 bytes (rnfs never wrote others). +- The database and its `-wal` and `-shm` files are readable by their owner only. rnfs refuses to start on a + damaged database (it runs SQLite's `quick_check` at startup) or one written by a newer version, and says how + to recover: the `.migrated` files are kept for that. A state directory it can't write to now stops startup + too, saying so (before, the node started and every save failed). Added: - A debug log file, for diagnosing problems: `debug_log = true` in `rnfs.toml`, or `rnfs daemon --debug-log`, @@ -25,20 +52,20 @@ Fixes: - `gc` (and eviction) could delete blocks of a complete pin. `stat` of a DAG only partly stored remembered the nodes it didn't find as listing nothing, so once the rest was fetched and pinned, collection didn't see what those nodes reach. Until a restart, which cleared it. -- `import` pins the roots an archive supplies with one save of `pins.json`; if that save fails, none of them is +- `import` pins the roots an archive supplies with one save of the pins; if that save fails, none of them is pinned (before, those ahead of the failure were). Faster: - `add`, the shared folder and `import` write blocks on up to 16 threads, so the sync to disk each block needs before its rename overlaps (about 11x on an SSD: 64 MB added at 165–190 MB/s, from 13–17). Every block is still synced before it is named, and before anything pins it. -- A CID's written form and DHT key are computed once. They showed up in every save of `pins.json`, - `pin_names.json`, `published.json` and in `pin ls`. -- `fetched.json` is saved with the cache records (every 30 s, at `gc` and at stop), not rewritten and synced - under the node's lock on each fetch. Jobs save `jobs.json` once per change, not twice. -- State files written in full (`published.json`, `providers.json`, `name-floor.json`) are written outside the - publisher, provider-store and names locks, from a copy taken under them, so `/find`, `/provide`, resolves and - name announces don't wait for the write. `verify` and the size queries (`stat`, quotas, pinned bytes) read DAG +- A CID's written form and DHT key are computed once. They showed up in every save of the pins, pin names and + publish state, and in `pin ls`. +- The targets fetched for the user (for `roots`) are saved with the cache records (every 30 s, at `gc` and at + stop), not rewritten and synced under the node's lock on each fetch. Jobs are saved once per change, not twice. +- The publish state, provider records and name floor are written outside the publisher, provider-store and + names locks, from a copy taken under them, so `/find`, `/provide`, resolves and name announces don't wait for + the write. `verify` and the size queries (`stat`, quotas, pinned bytes) read DAG nodes without the storage lock, as eviction and `gc` already did. - Recomputing what is provided reuses the last result when no provided DAG changed (about 30 ms to 5 ms at 5000 pins). diff --git a/README.md b/README.md index db4ac2d..a08a080 100644 --- a/README.md +++ b/README.md @@ -367,7 +367,7 @@ Uninstalling never deletes it. | `storage/identity` | the node's RNS identity; your node ID comes from it | | `storage/lxmf_identity` | a separate identity for your LXMF address | | `blocks/` | stored blocks | -| `state/` | pins, cache records, provider records, names' sequences, jobs, kept and followed names, contacts, LXMF inbox and outbox | +| `state/rnfs.db` | pins, cache records, provider records, names' sequences, jobs, kept and followed names, contacts, LXMF inbox and outbox (one SQLite database; an older version's JSON files there are imported at the first start and kept as `*.migrated`) | | `names/` | name keys (back them up: `rnfs name export`) | | `files/` | the shared folder: anything put here is added, pinned and shared automatically | | `api_token` | the running daemon's API token | diff --git a/design.md b/design.md index e4b5914..3c2ebed 100644 --- a/design.md +++ b/design.md @@ -40,12 +40,31 @@ Node IDs, DHT keys and names are 16-byte `bytes`. Identity hashes (for `members` pin source `remote:`) are RNS identity hashes, and LXMF addresses (`lxmf_*` settings, pin source `lxmf:
`) are `lxmf.delivery` destination hashes, all written as 32 hex characters. -Persistent state is JSON (or msgpack) under `config.state_dir`, written atomically and durably -(`storage.write_json`: temp file fsynced, `os.replace`, then the directory synced). `storage.read_json(path, -shape)` treats only a file that isn't JSON, fails `shape` or is a directory as damaged: it is moved to -`.damaged` (`.damaged.1`, … keeping earlier copies) and the node starts without it; any other `OSError` stops -startup, since starting without `pins.json` would lose every pin. Everything shared between threads locks -internally. +- `rnfs/statedb.py` — the state database, `/state/rnfs.db` (standard `sqlite3`, WAL mode, + `busy_timeout`; created 0600, which SQLite copies to its `-wal` and `-shm` files; on Windows the state + directory's access list covers them). `connect(state_dir)` hands each store a reference (`StateDb`) to one + shared connection per directory, the last `close()` closing it; opening runs `quick_check` and refuses a + damaged database, or one whose `user_version` is past the schema steps this code knows (`SCHEMA`, applied + one transaction each), saying what to do. `transaction(durable)` is the only way to write: `durable` commits + with `synchronous=FULL` (on disk when it returns), otherwise `NORMAL` (a power loss may lose the last commits, + never part of one or one without those before it); a sqlite failure raises `StateDbError`, an `OSError`. + Text that may hold lone surrogates (file names, remote strings) goes through `dump`/`load` (JSON text); + sequences are decimal text, since they pass SQLite's 64-bit integers. Without `_sqlite3` the daemon stops + with a message saying so. + +Persistent state lives in the state database. Each store keeps its state in memory, which is all reads use, +loads it once at startup (checking every row as the JSON loaders checked every record: a malformed row is +logged, skipped and deleted) and then writes only what changed: a pin is one row inserted or deleted, batched +stores write the records changed since their last save. At every `connect`, `statedb.migrate` first imports +the JSON state files of 0.1.4 and earlier (`LEGACY`): each through its owner's importer, which checks every +record as the old loader did, in one transaction that also records the file in the `migrated` table; then the +file is renamed `.migrated` and the directory synced. A file the table already lists (a crash between +the commit and the rename) is only renamed, so none is imported twice or by half. The files are read with +`statedb.read_json(path, shape)`, which treats only a file that isn't JSON, fails `shape` or is a directory as +damaged: it is moved to `.damaged` (`.damaged.1`, … keeping earlier copies) and the node starts without it; any +other `OSError` stops startup, since starting without `pins.json` would lose every pin (`names.json` stops it +when damaged too, as it always did: starting afresh would allow rollback). Everything shared between threads +locks internally. ## Layout and ownership @@ -55,6 +74,7 @@ rnfs/ constants.py config.py errors.py log.py shared foundation debuglog.py the debug log file, /debug.log (debug_log) fsutil.py cross-platform file operations: replace/remove retried on Windows, O_BINARY, IS_WINDOWS + statedb.py the state database, /state/rnfs.db: schema, transactions, JSON state migration content/ no networking; uses RNS's bundled msgpack and RNS logging, so importing it loads RNS cid.py CIDs (section 4) dag.py DAG node encoding, strict decoding, names (section 5) @@ -159,9 +179,12 @@ packaging/ requirements.lock (hash-pinned runtime dependencies), upda records current then, so `get` (and `/blocks` serving) waits only for that in-memory walk and the deletions, and a block that became reachable meanwhile is kept. The size queries (`pinned_bytes`, `dag_bytes`, `source_usage`, `stat`) read their DAG nodes the same way first. `_links` memoises only stored DAG nodes: a - node not stored yet may be stored later, and a raw block lists nothing. `pins.json` is still written under the lock on every - pin and unpin: a pin must be on disk when `pin()` returns (names' crash consistency and `rnfs pin add` - depend on it), and before a collection can delete what an `unpin()` released. + node not stored yet may be stored later, and a raw block lists nothing. A pin's or unpin's row is still + committed durably under the lock: a pin must be on disk when `pin()` returns (names' crash consistency and + `rnfs pin add` depend on it), and before a collection can delete what an `unpin()` released. If the commit + fails, `pin`/`pin_many`/`unpin` undo their change in memory and raise. Cache records track the keys changed + since the last `save()`; those still cached are always the most recently used, so a save writes them with + new `used` values (the table's LRU order) and deletes the others. - `verify() -> stats` re-checks every block, deleting bad ones. - A stored DAG node that no longer matches its CID (`DAMAGED`) has unknown children: while anything live reaches one, `evict()` and `gc()` delete nothing and log it once (`_held_back`) until `verify()` removes it or @@ -335,7 +358,7 @@ possible future way to let pages read their own DAG. decrypting only the covering segments), `parse_recipient`. AES-256-CBC and HKDF come from `RNS.Cryptography`, HMAC/SHA-256 from the standard library: no new dependency. - `node_content.py` — `keep_added` (a `user` pin, or for `--no-pin` a cached fetch recorded in - `fetched.json` for the roots strategy), `seal_recipients` (`RNS.Identity.recall` by destination hash, then + the fetched targets for the roots strategy), `seal_recipients` (`RNS.Identity.recall` by destination hash, then identity hash; the node's own identity; contacts), `open_sealed` over a Fetch's `read_range`, `sealed_info`, `write_sealed`, `block`, `export_stream`, and the API option parser `add_options`. - `gateway_formats.py` — `?format=raw|archive`; `archive.archive_size` gives the exact length, so the archive @@ -503,13 +526,14 @@ name in the output's directory; the CLI sets it for `rnfs get ` withou The API and gateway run one thread per connection, capped (32 and 64). Locks are taken in this order, never the reverse: `names._lock` → `publisher._lock` → `node._lock` → -`storage._lock`. `NameManager.publish`, `remove` and `release_gone` pin and unpin, and so call +`storage._lock` → the state database's lock (`statedb`, innermost: a store opens a transaction only once it +holds every lock the change needs, and inside it takes no other lock and does no network I/O). `NameManager.publish`, `remove` and `release_gone` pin and unpin, and so call `publisher.refresh()`, while holding the names lock; `Publisher.refresh` recomputes what is provided under its own lock, reading the node's fetched targets (`node._lock`) and walking the pins' DAGs (`storage._lock`); the node reads storage under its lock (`_prune_fetched`). This is how the code behaves today, not a design goal. -State files written in full are written outside these locks, from a copy taken under them, one save at a time -behind a save lock taken first (`Storage.save`, `Publisher.save`, `ProviderStore.save`, the name floor, -`fetched.json`, `remote_pins.json`), so an older copy never lands last. Slow work that runs while a lock is +Batched saves write outside these locks, from the changes collected under them, one save at a time behind a +save lock taken first (`Storage.save`, `Publisher.save`, `ProviderStore.save`, the name floor, the fetched +targets, the remote pins waiting), so an older snapshot never lands last. Slow work that runs while a lock is held: - `Publisher.refresh` checks every pin's DAG (`dag_blocks`, memoised) under the publisher lock, behind the @@ -517,26 +541,41 @@ held: DAG's blocks changed (CIDs only added are unioned in), so a refresh that changed nothing doesn't recompute it. - `Storage.evict` and `gc` decide what to delete, and delete it, under the storage lock (after reading the DAG nodes outside it); `verify` checks completeness and re-reads the blocks outside it. -- `pins.json` is written, synced to disk, under the storage lock on every pin and unpin (see `storage.py`). +- Every pin and unpin commits its row durably (`synchronous=FULL`, after the blocks are synced) under the + storage lock (see `storage.py`). ## State files -Under `/state/`, JSON written atomically from a temp file synced to disk: `pins.json`, `cache.json` -(storage); `pin_names.json` (rnfs/pin_names.py: where each pin was added from, local only, pruned to the pin -list); `providers.json` (provider records); `published.json` (publisher: last rounds, earlier rounds' nodes, -pending withdrawals); `fetched.json` (targets fetched for the user, for `roots`); `remote_pins.json` (remote pins accepted and -not yet fetched, `[[source, cid], ...]` in order); `names.json` (sequences, name -table); `name-floor.json` (`[[name, sequence, resolved], ...]`: highest sequence of names resolved for the -gateway or auto-fetch and when, at most 4 × `name_table_size`, or 8 × for names resolved within 7 days; saved by -a resolve raising it, at most every 5 s, and by the re-announce loop when changed; entries without the time load -as resolved long ago); `invalid.json` (invalid roots); `shared_folder.json`; `jobs.json` (results keep at most 100 warnings -plus `warnings_total`; older results are capped at load); `keepers.json` (records re-verified at load); -`follow.json`; `contacts.json`; `lxmf_inbox.json` (500 messages, 4 MiB, 50 per non-auto-fetch sender, 16 KiB per -record) and `lxmf_outbox.json` (200; unfinished messages are marked failed at a restart), both UTF-8 JSON with -non-ASCII characters written as is. Elsewhere: `storage/identity`, `storage/lxmf_identity` (0600), -`names/` (name keys), `lxmf/` (LXMF's ratchets, delivered-message IDs, stamp costs), `api_token` (while -the daemon runs). Malformed records in feature state files (wrong types, duplicate IDs) are dropped at load -with a warning. Outputs being written live beside the output as `.rnfs-get-.partial` (`RnfsNode.get`) +`/state/rnfs.db` (rnfs/statedb.py), one table set per store; "FULL" commits are on disk when the +change returns, "NORMAL" ones are batched saves a power loss may lose the last of: +- storage: `pins` (one row per source; each pin and release FULL, after the blocks are synced) and `cache` + (cached fetches, LRU order in `used`, at most 10 000; NORMAL every 30 s and at stop, changed records only); +- `pin_names` (rnfs/pin_names.py: where each pin was added from, local only, pruned to the pin list; FULL); +- node: `fetched` (targets fetched for the user, for `roots`; with the cache records) and `remote_pins` (remote + pins accepted and not yet fetched, in order; about once a second and at stop, rewritten whole); fetcher: + `invalid` (invalid roots, oldest first; with the cache records); +- `published`, `published_earlier`, `withdrawals` (publisher: last rounds, earlier rounds' nodes, pending + withdrawals; NORMAL every 10 nodes or 30 s of a round and at stop, changed items only); +- `provider_publishers`, `provider_records`, `provider_departed`, `provider_evicted` (NORMAL every 300 s and at + stop, changed rows only); +- `names_published`, `names_held` (sequences as decimal text, up to 2^64 − 1; FULL, changed rows only; a + malformed row stops startup, as a malformed `names.json` did, since skipping it could allow rollback) and + `name_floor` (highest sequence of names resolved for the gateway or auto-fetch and when, in `used` order, at + most 4 × `name_table_size`, or 8 × for names resolved within 7 days; NORMAL, saved by a resolve raising it, at + most every 5 s, and by the re-announce loop when changed); +- `shared_folder` (FULL); `jobs` (record per job in creation order; FULL, including progress saves; results keep + at most 100 warnings plus `warnings_total`, older results capped at load); `keepers` (records re-verified at + load; NORMAL); `follow` (FULL); `contacts` (FULL); `lxmf_messages` (inbox: 500 messages, 4 MiB, 50 per + non-auto-fetch sender, 16 KiB per record; outbox 200, unfinished messages marked failed at a restart; NORMAL, + changed messages only; ids and `next_id` stay below 2^63); +- `meta` (per-store values: follow's and each box's `clean`, set by every save, which the reconcile at start + requires; each box's `next_id`) and `migrated` (JSON files imported). + +Free text that may hold lone surrogates (file names, LXMF records, cache paths) is stored as JSON text +(`statedb.dump`). Rows that fail the checks the JSON loaders applied are dropped at load with a warning and +deleted. The JSON files of 0.1.4 and earlier, once imported, stay as `.migrated`. Elsewhere: +`storage/identity`, `storage/lxmf_identity` (0600), `names/` (name keys), `lxmf/` (LXMF's ratchets, +delivered-message IDs, stamp costs), `api_token` (while the daemon runs). Outputs being written live beside the output as `.rnfs-get-.partial` (`RnfsNode.get`) or `.rnfs-job-.partial` (get jobs) until renamed; a failed, cancelled or shut-down job removes its partial, and a resumed job removes one left by a crash. @@ -545,8 +584,11 @@ and a resumed job removes one left by a crash. The commands CI runs the tests with are in `.github/workflows/tests.yml`. Feature tests live in their own files (`test_jobs.py`, `test_mesh.py`, `test_names_keep.py`, `test_contacts.py`, `test_seal.py`, `test_chunker.py`, `test_pin_false.py`, `test_gateway_formats.py`, `test_lxmf.py`), and `test_e2e.py` has a marked block per -feature. CI (`.github/workflows/tests.yml`) runs them on Linux, macOS and Windows; `packaging/smoke_test.py` -checks a built package (`--version`, `--help`, daemon and status). +feature. The state database has `test_statedb.py`, each store's migration a `test_migrate_*.py`, and +`test_upgrade_014.py` loads `tests/fixtures/rnfs-0.1.4/`, a node directory 0.1.4 wrote (its `generate.py` +writes it again). CI runs them on Linux, macOS and Windows; `packaging/smoke_test.py` checks a built package +(`--version`, `--help`, daemon and status, an add, and that the state database holds the pin and imported a +0.1.4 state file). ## Status @@ -565,8 +607,8 @@ Everything in the specification is implemented, with these simplifications and a - Withdrawals are sent once, with any provides due for the same node. One that fails isn't retried, since the record expires after `PROVIDER_TTL`. A CID is withdrawn from the nodes of its last round and from nodes of earlier rounds, or of a round cut short, that may still hold its record (`Publisher._earlier`, persisted in - `published.json`, forgotten after `PROVIDER_TTL`). Each node that accepts a key is added there as its - `/provide` returns, and `published.json` is saved every 10 nodes or 30 s of a round + `published_earlier`, forgotten after `PROVIDER_TTL`). Each node that accepts a key is added there as its + `/provide` returns, and the publish state is saved every 10 nodes or 30 s of a round (`SAVE_EVERY_NODES`, `SAVE_EVERY_SECONDS`), so a crash loses at most that much of where a round stored. A round stopped before every node was sent its keys leaves its CIDs due, and persisted times later than now are clamped to now. @@ -641,7 +683,7 @@ Everything in the specification is implemented, with these simplifications and a (and a sealed file's write-out) counts what it still has to fetch too (the verified size less the bytes of the target's DAG already stored). If the destination's free space can't be read, the check raises the `OSError` (same errno, saying so) instead of letting the write go ahead unchecked. -- Roots found invalid are remembered in `state/invalid.json` (at most 10 000, oldest dropped), and there is +- Roots found invalid are remembered in the state database's `invalid` table (at most 10 000, oldest dropped), and there is no command to clear them. Cache records are capped at 10 000 (LRU). Both are saved every 30 s and at stop. - A remote pin's usage is the larger of its content size and its DAG's block bytes. It is dropped once the root block shows the quota would be exceeded, or once the blocks fetched exceed it. A source's remote pins @@ -649,7 +691,7 @@ Everything in the specification is implemented, with these simplifications and a at most 64 (`REMOTE_PIN_QUEUE`) may be waiting or fetching, and further `/pin`s are refused. A worker leaves `_remote_workers` in the same critical section in which it finds its queue empty, so a `/pin` arriving as it exits starts a new one; if `Thread.start()` fails the pin is given back and refused. The pins waiting are - saved to `state/remote_pins.json` by maintenance (when changed) and at stop, and `_resume_remote_pins` queues + saved to the `remote_pins` table by maintenance (when changed; rewritten whole, a few rows) and at stop, and `_resume_remote_pins` queues them again at start (skipping those already pinned or whose identity left `remote_pin_identities`); workers wait for the first join before fetching, and a fetch cut short by stopping stays queued. - Byte-range reads (`cat`, gateway) walk the file lazily and fetch 32 chunks at a time ahead of what has been @@ -712,23 +754,23 @@ Everything in the specification is implemented, with these simplifications and a where one current answer is enough once a sequence is known). On a slow node that is still at most two `/name`; a fast node with no known sequence and no established keepers asks all 3 batches (up to 9), and the lookup runs to completion unless two established nodes list keepers. -- **Names.** A key imported, or found without an entry in `names.json`, is marked `imported`: its first publish +- **Names.** A key imported, or found without a `names_published` row, is marked `imported`: its first publish offers kept records, makes a path request and queries keepers before choosing a sequence. `publish()` pins - the new target, saves `names.json` (the commit point; a failed save rolls the record back and releases the - new pin) and only then releases the old target, so `names.json`'s target is pinned at every step. Once the + the new target, saves its row (the commit point; a failed commit rolls the record back and releases the + new pin) and only then releases the old target, so the saved target is pinned at every step. Once the features are set up, `reconcile_pins(storage.pins())` releases the pins of name keys deleted by hand (`release_gone()`), pins a recorded target again if its pin is missing, and releases `name:` pins nothing - records; a key without a `names.json` entry keeps its pins until its next publish or removal (`_stale`, + records; a key without a saved record keeps its pins until its next publish or removal (`_stale`, where a release that failed with `OSError` also waits). A key file that is there but can't be loaded is - `_unreadable`: its pins are kept and its `names.json` entry is written back as loaded. `Storage.pin`/`pin_many`/ - `unpin` undo their change in memory if saving `pins.json` fails. Follow and LXMF do the same for their sources at - start (`FollowFeature._reconcile`, `_release_unrecorded`), but release only if their file loaded whole - (`MessageLog.clean`); otherwise unrecorded pins are recorded again (the name followed again, or an inbox + `_unreadable`: its pins are kept and its row is never touched by saves. `Storage.pin`/`pin_many`/ + `unpin` undo their change in memory if the commit fails. Follow and LXMF do the same for their sources at + start (`FollowFeature._reconcile`, `_release_unrecorded`), but release only if their state loaded whole (the + `clean` meta value, set by every save; `MessageLog.clean`); otherwise unrecorded pins are recorded again (the name followed again, or an inbox message per sender listing them), so a later start keeps them too. The name table marks entries that were only heard (`_heard_only`, its own LRU, so finding one to evict is O(1)); an announce with no waiting resolve, of a name without a kept record, may only evict one of those. A resolve with `persist=False` (gateway, LXMF auto-fetch) raises the name's floor (`_floor`: name -> (sequence, wall time - last resolved), in resolve order), which `_offer` checks. A raise (or a new name) saves `name-floor.json` + last resolved), in resolve order), which `_offer` checks. A raise (or a new name) saves the floor at once if the last save was `FLOOR_SAVE_INTERVAL` (5 s) ago, else from a one-shot `threading.Timer`; renewing only the time marks it for the next save after an hour. `_trim_floor` drops the least recently resolved past `FLOOR_FACTOR` × `name_table_size`, but stops at one resolved within `FLOOR_KEEP_AGE` (7 days) diff --git a/packaging/pyinstaller/rnfs.spec b/packaging/pyinstaller/rnfs.spec index 022e937..8e3afc1 100644 --- a/packaging/pyinstaller/rnfs.spec +++ b/packaging/pyinstaller/rnfs.spec @@ -40,7 +40,9 @@ def required(package: str) -> str: VERSION = rnfs_version() hiddenimports = ( - collect_submodules("rnfs") # rnfs.features loads feature modules by name with importlib + # rnfs.features loads feature modules by name with importlib, and rnfs.statedb its migration's importers + collect_submodules("rnfs") + + ["sqlite3"] # the state database; imported inside a function, so named here (PyInstaller adds _sqlite3) + collect_submodules(required("RNS")) + collect_submodules(required("LXMF")) + collect_submodules("serial") # pyserial: Serial, KISS and RNode interfaces diff --git a/packaging/smoke_test.py b/packaging/smoke_test.py index d76f80d..595d907 100755 --- a/packaging/smoke_test.py +++ b/packaging/smoke_test.py @@ -8,12 +8,16 @@ (no shared instance, so a running rnsd isn't touched; one TCP server interface on 127.0.0.1 and nothing that reaches the network), on free ports, polls `status` until the daemon answers, checks the TCP interface is listening (which proves RNS found its dynamically loaded interface modules, the part a frozen build can -miss), stops the daemon the way Ctrl+C would and checks it exits. Exits non-zero on any failure. -Uses only the standard library, so any Python 3 can run it. +miss), adds a file, stops the daemon the way Ctrl+C would and checks it exits. The state database must then +hold the pin, and a JSON state file of rnfs 0.1.4 placed in the directory beforehand must have been imported +into it (which proves the build has sqlite3 and the modules the migration imports by name). Exits non-zero on +any failure. Uses only the standard library, so any Python 3 can run it. """ +import json import os import signal import socket +import sqlite3 import subprocess import sys import tempfile @@ -80,6 +84,14 @@ def main(): interface_port = free_port() f.write(RETICULUM_CONFIG.format(port=interface_port)) ports = ["--dir", rnfs_dir, "--api-port", str(free_port())] + # A state file as rnfs 0.1.4 wrote it, which the daemon imports into its state database at startup + state_dir = os.path.join(rnfs_dir, "state") + os.makedirs(state_dir, mode=0o700) + with open(os.path.join(state_dir, "pin_names.json"), "w", encoding="utf-8") as f: + json.dump({}, f) + added = os.path.join(tmp, "added.txt") + with open(added, "w", encoding="utf-8") as f: + f.write("smoke test") log_path = os.path.join(tmp, "daemon.log") with open(log_path, "w", encoding="utf-8") as log: daemon = subprocess.Popen( @@ -104,6 +116,7 @@ def main(): sys.exit(f"smoke test: RNS's TCP interface isn't listening on {interface_port} ({e}): " "were RNS's interface modules bundled?") print(f"RNS TCP interface listening on 127.0.0.1:{interface_port}") + run(command, "add", *ports, added) finally: if daemon.poll() is None: daemon.send_signal(signal.CTRL_BREAK_EVENT if WINDOWS else signal.SIGINT) @@ -122,8 +135,27 @@ def main(): print("--- daemon log ---\n" + log.read().rstrip() + "\n------------------") if not stopped_cleanly: sys.exit("smoke test: daemon didn't stop cleanly") + check_state(state_dir) print("smoke test passed") +def check_state(state_dir: str): + """Exit unless the state database holds a pin and records pin_names.json as imported.""" + path = os.path.join(state_dir, "rnfs.db") + if not os.path.isfile(path): + sys.exit(f"smoke test: no state database at {path}") + conn = sqlite3.connect(path) + try: + pins = conn.execute("SELECT count(*) FROM pins").fetchone()[0] + migrated = [row[0] for row in conn.execute("SELECT file FROM migrated")] + finally: + conn.close() + print(f"state database: {pins} pin(s), imported {migrated}") + if pins < 1: + sys.exit("smoke test: the file added isn't pinned in the state database") + if migrated != ["pin_names.json"] or not os.path.exists(os.path.join(state_dir, "pin_names.json.migrated")): + sys.exit("smoke test: pin_names.json wasn't imported into the state database") + + if __name__ == "__main__": main() diff --git a/rnfs/content/archive.py b/rnfs/content/archive.py index 1848556..fdf6abb 100644 --- a/rnfs/content/archive.py +++ b/rnfs/content/archive.py @@ -212,7 +212,7 @@ def import_archive(stream: BinaryIO, storage: Storage, pin_source: str | None = incomplete.append(root) held.add(root) if pin_source is not None: - storage.pin_many(complete, pin_source) # still protected; pins.json is saved once for them all + storage.pin_many(complete, pin_source) # still protected; the pins are saved once for them all if bad: log(f"import: skipped {bad} blocks that didn't match their CIDs", LOG_WARNING) for root in incomplete: diff --git a/rnfs/content/shared_folder.py b/rnfs/content/shared_folder.py index 23b0301..e30d257 100644 --- a/rnfs/content/shared_folder.py +++ b/rnfs/content/shared_folder.py @@ -3,7 +3,8 @@ Every top-level file and directory in it is imported as with `add` (copied into the block store) and pinned with source "folder"; a changed or removed entry releases its old pin. Entries are recognised as unchanged by a signature of path, size, modification time and execute bit (recursively for directories), -remembered across restarts, so unchanged entries aren't hashed again. +remembered across restarts (the state database's shared_folder table, rnfs.statedb: after a scan that changed +any, the entries changed are written durably), so unchanged entries aren't hashed again. """ import hashlib import json @@ -11,15 +12,29 @@ import stat import threading -from rnfs import constants +from rnfs import constants, statedb from rnfs.content import builder from rnfs.content.builder import display_path from rnfs.content.cid import Cid -from rnfs.content.storage import FOLDER, Storage, read_json, shaped, write_json +from rnfs.content.storage import FOLDER, Storage from rnfs.errors import RnfsError from rnfs.log import LOG_INFO, LOG_WARNING, log -STATE_FILE = "shared_folder.json" + +def import_entries(value: dict, tx) -> int: + """Import shared_folder.json (statedb.migrate): its entries replace the shared_folder table; one that doesn't + parse is skipped, as rnfs 0.1.4 did. Returns the number imported.""" + tx.execute("DELETE FROM shared_folder") + count = 0 + for name, record in value.get("entries", {}).items(): + try: + # Inside the try: a signature SQLite can't store (a lone surrogate) is an invalid record too + tx.execute("INSERT OR REPLACE INTO shared_folder (name, signature, cid) VALUES (?, ?, ?)", + (statedb.dump(name), str(record[0]), Cid.parse(record[1]).binary)) + except (ValueError, TypeError, LookupError): + continue + count += 1 + return count def signature(path: str) -> str: @@ -74,26 +89,40 @@ class SharedFolder: """Keeps the shared folder's entries imported and pinned with source "folder".""" def __init__(self, directory: str, storage: Storage, state_dir: str | None = None): - """Track directory (created if missing at the first scan); the signature cache lives in state_dir.""" + """Track directory (created if missing at the first scan); the signature cache lives in state_dir's + state database (not kept without one).""" self.directory = os.path.expanduser(directory) self.storage = storage - self._state_path = os.path.join(os.path.expanduser(state_dir), STATE_FILE) if state_dir else None + self._db = statedb.connect(os.path.expanduser(state_dir)) if state_dir else None self._lock = threading.Lock() self._known: dict[str, tuple[str, Cid]] = {} # entry name -> (signature, CID) of its last import + self._saved: dict[str, tuple[str, Cid]] = {} # what the shared_folder table holds self._failed: dict[str, str] = {} # entry name -> signature that failed to import, not retried until it changes self._warned: set[str] = set() # skipped entries already warned about - self._load() + try: + self._load() + except BaseException: + self.close() + raise def _load(self): - """Load the signature cache, ignoring anything that doesn't parse.""" - if self._state_path is None: + """Load the signature cache; a row that doesn't parse is logged and deleted.""" + if self._db is None: return - value = read_json(self._state_path, shaped("entries", dict)) or {} - for name, record in value.get("entries", {}).items(): + bad = [] + for text, sig, binary in self._db.query("SELECT name, signature, cid FROM shared_folder"): try: - self._known[name] = (str(record[0]), Cid.parse(record[1])) - except (ValueError, TypeError, LookupError): - continue + name = statedb.load(text) + if not isinstance(name, str) or not isinstance(sig, str): + raise TypeError("not a string") + self._known[name] = (sig, Cid.from_bytes(binary)) + except (ValueError, TypeError): + log(f"shared folder: ignoring invalid saved entry {text!r}", LOG_WARNING) + bad.append((text,)) + self._saved = dict(self._known) + if bad: + with self._db.transaction() as tx: + tx.executemany("DELETE FROM shared_folder WHERE name = ?", bad) def names(self) -> dict[Cid, str]: """CID → the shared-folder entry name it was imported from, for `pin ls`.""" @@ -101,11 +130,21 @@ def names(self) -> dict[Cid, str]: return {cid: name for name, (_, cid) in self._known.items()} def _save(self): - """Persist the signature cache.""" - if self._state_path is None: + """Persist the entries of the signature cache that differ from what the table holds, durably (lock held).""" + if self._db is None: return - os.makedirs(os.path.dirname(self._state_path), exist_ok=True) - write_json(self._state_path, {"entries": {name: [sig, str(cid)] for name, (sig, cid) in self._known.items()}}) + changed = [(statedb.dump(name), sig, cid.binary) for name, (sig, cid) in self._known.items() + if self._saved.get(name) != (sig, cid)] + gone = [(statedb.dump(name),) for name in self._saved if name not in self._known] + with self._db.transaction(durable=True) as tx: + tx.executemany("DELETE FROM shared_folder WHERE name = ?", gone) + tx.executemany("INSERT OR REPLACE INTO shared_folder (name, signature, cid) VALUES (?, ?, ?)", changed) + self._saved = dict(self._known) + + def close(self): + """Release the state database (the node calls this once it has saved everything at shutdown).""" + if self._db is not None: + self._db.close() def scan(self) -> set[Cid]: """Import new and changed entries, release the pins of changed and removed ones; returns the folder's pins.""" diff --git a/rnfs/content/storage.py b/rnfs/content/storage.py index f21a32d..b08bbf9 100644 --- a/rnfs/content/storage.py +++ b/rnfs/content/storage.py @@ -11,13 +11,15 @@ pins alone fill it. The hard bound is the disk: check_disk() refuses blocks that would leave less than DISK_RESERVE free on the filesystem holding the blocks. -Pins and cache records are persisted as JSON under state_dir: pins at once, cache records (which change on -every fetch) by save(), which the node calls periodically and at shutdown. Thread-safe: one lock guards the +Pins and cache records are persisted in the state database (rnfs.statedb), and only read from it at startup: +each pin and release at once, as one row inserted or deleted and committed durably (after the blocks it keeps +are synced), and cache records (which change on every fetch) by save(), which the node calls periodically and +at shutdown, writing only the records changed since the last save. Thread-safe: one lock guards the indexes; long walks run outside it: completeness walks are discarded if blocks were deleted meanwhile, and eviction, gc and the size queries read the DAG nodes they will walk beforehand (_warm), deciding what to delete (or counting) under the lock. """ -import json +import itertools import os import re import shutil @@ -27,7 +29,7 @@ from contextlib import contextmanager from typing import Callable, Iterator -from rnfs import fsutil +from rnfs import statedb from rnfs.constants import MAX_BLOCK_SIZE, MIB from rnfs.content import dag, walker from rnfs.content.blockstore import BlockStore @@ -39,8 +41,6 @@ USER, FOLDER = "user", "folder" NAME_PREFIX, REMOTE_PREFIX = "name:", "remote:" LXMF_PREFIX = "lxmf:" -PINS_FILE = "pins.json" -CACHE_FILE = "cache.json" MAX_CACHE_RECORDS = 10_000 # cached fetches remembered; past this the least recently used record is dropped EVICT_MARGIN = 32 # eviction frees max_storage / EVICT_MARGIN more than it must, so it runs rarely DISK_RESERVE_MIN = 256 * MIB # free disk space fetches always leave on the blocks' filesystem... @@ -90,59 +90,47 @@ def remote_source(identity_hash: bytes | str) -> str: return REMOTE_PREFIX + text -def write_json(path: str, value, ensure_ascii: bool = True): - """Write value as JSON to path atomically and durably (temporary file synced to disk, then renamed), readable - by its owner only. With ensure_ascii False, non-ASCII characters are written as UTF-8 rather than escaped.""" - # backslashreplace: a lone surrogate, which ensure_ascii False leaves as is, is written as its JSON escape - data = json.dumps(value, ensure_ascii=ensure_ascii).encode("utf-8", "backslashreplace") - fsutil.write_atomic(path, data, mode=0o600) # the rename is retried on Windows while read_json has path open - - -def read_json(path: str, shape: Callable[[object], bool] | None = None): - """The JSON value in path, or None if it doesn't exist. A damaged file (not JSON, failing shape if given, - or a directory) is moved aside, not lost, and gives None. Raises OSError if path can't be read: a transient - or permission error isn't damage, and starting without the file could lose what it records (pins).""" - try: - with open(path, encoding="utf-8") as f: - value = json.load(f) # an empty file, as a crash can leave, is a ValueError too - except FileNotFoundError: - return None - except ValueError as e: - problem = str(e) - except OSError: - if not os.path.isdir(path): # Windows reports opening a directory as PermissionError - raise - problem = "it is a directory" - else: - if shape is None or shape(value): - return value - problem = "unexpected structure" - try: - aside = _move_aside(path) - except OSError as moving: - # A read-only state dir: start without it rather than not at all - log(f"storage: {path} is damaged ({problem}) and couldn't be moved aside ({moving}); starting without it", - LOG_ERROR) - return None - log(f"storage: {path} is damaged ({problem}); moved to {aside} and starting without it", LOG_ERROR) - return None - - -def _move_aside(path: str) -> str: - """Rename path to path.damaged, or path.damaged. so an earlier damaged copy is kept; the new path.""" - for n in range(1000): - aside = path + ".damaged" + (f".{n}" if n else "") +def import_pins(value: dict, tx) -> int: + """Import pins.json (statedb.migrate): its pins replace the pins table, each record checked as rnfs 0.1.4 + loaded it (an invalid one is logged and skipped). Returns the number of pins imported.""" + tx.execute("DELETE FROM pins") + count = 0 + for text, sources in value.get("pins", {}).items(): try: - fsutil.rename_new(path, aside) - return aside - except FileExistsError: + cid = Cid.parse(text) + if not isinstance(sources, list): + raise TypeError("sources aren't a list") + for source in sources: + check_source(source) + # Inside the try: a source SQLite can't store (a lone surrogate) is an invalid record too + tx.executemany("INSERT OR IGNORE INTO pins (cid, source) VALUES (?, ?)", + [(cid.binary, source) for source in sources]) + except (ValueError, TypeError): + log(f"storage: ignoring invalid pin record {text!r}", LOG_ERROR) continue - raise FileExistsError(f"too many damaged copies of {path}") + count += bool(sources) + return count -def shaped(key: str, kind: type) -> Callable[[object], bool]: - """A read_json shape: an object whose key, if present, holds a kind.""" - return lambda value: isinstance(value, dict) and isinstance(value.get(key, kind()), kind) +def import_cache(value: dict, tx) -> int: + """Import cache.json (statedb.migrate): its records, least recently used first, replace the cache table, each + checked as rnfs 0.1.4 loaded it. Returns the number of records imported.""" + tx.execute("DELETE FROM cache") + records: OrderedDict[tuple[Cid, tuple[str, ...]], Cid] = OrderedDict() + for record in value.get("fetches", []): + try: + root, names, target = record + if not isinstance(names, list): + raise TypeError("names aren't a list") + for name in names: + dag.check_name(name) + records[(Cid.parse(root), tuple(names))] = Cid.parse(target) # a repeated key keeps its first place + except (ValueError, TypeError): + log(f"storage: ignoring invalid cache record {record!r}", LOG_ERROR) + tx.executemany("INSERT INTO cache (root, path, target, used) VALUES (?, ?, ?, ?)", + [(root.binary, statedb.dump(list(names)), target.binary, used) + for used, ((root, names), target) in enumerate(records.items())]) + return len(records) class Session: @@ -233,7 +221,7 @@ def __init__(self, blocks_dir: str, state_dir: str, max_storage: int): self._lock = threading.RLock() self.blocks = BlockStore(blocks_dir) self.state_dir = os.path.expanduser(state_dir) - os.makedirs(self.state_dir, mode=0o700, exist_ok=True) + self._db = statedb.connect(self.state_dir) self.max_storage = max_storage self._save_lock = threading.Lock() # keeps cache saves in order without holding _lock while writing @@ -248,65 +236,98 @@ def __init__(self, blocks_dir: str, state_dir: str, max_storage: int): self._complete: dict[Cid, frozenset[Cid]] = {} # dag_blocks memo, complete DAGs only self._generation = 0 # incremented whenever blocks are deleted, invalidating walks in progress self._cache_generation = 0 # incremented whenever cache records are removed or retargeted - self._cache_dirty = False # cache records changed since the last save() + # Cache records changed (or removed) since the last save(); every one is among the most recently used + self._cache_dirty: set[tuple[Cid, tuple[str, ...]]] = set() + self._cache_used = 0 # the cache table's next used value: larger than every one saved # False only when nothing has happened since the last collection that could leave a block unreached, # so eviction over max_storage with nothing to evict needn't walk everything again self._maybe_garbage = True self._reported_damaged: set[Cid] = set() # damaged DAG nodes already logged as stopping collection - self._load() + try: + self._load() + except BaseException: + self._db.close() + raise # Persistence def _load(self): - """Load pins and cache records, skipping entries that don't parse.""" - pins = read_json(os.path.join(self.state_dir, PINS_FILE), shaped("pins", dict)) or {} - for text, sources in pins.get("pins", {}).items(): + """Load pins and cache records; rows that don't parse are logged and deleted, as are cache records past + MAX_CACHE_RECORDS.""" + bad_pins = [] + for binary, source in self._db.query("SELECT cid, source FROM pins"): try: - cid = Cid.parse(text) - if not isinstance(sources, list): - raise TypeError("sources aren't a list") - for source in sources: - check_source(source) - if sources: - self._pins[cid] = set(sources) + cid = Cid.from_bytes(binary) + check_source(source) except (ValueError, TypeError): - log(f"storage: ignoring invalid pin record {text!r}", LOG_ERROR) - cache = read_json(os.path.join(self.state_dir, CACHE_FILE), shaped("fetches", list)) or {} - for record in cache.get("fetches", []): + log(f"storage: ignoring invalid pin record {binary!r} {source!r}", LOG_ERROR) + bad_pins.append((binary, source)) + continue + self._pins.setdefault(cid, set()).add(source) + bad_cache = [] + for root, path, target, used in self._db.query("SELECT root, path, target, used FROM cache ORDER BY used"): try: - root, names, target = record + names = statedb.load(path) if not isinstance(names, list): raise TypeError("names aren't a list") for name in names: dag.check_name(name) - self._cache[(Cid.parse(root), tuple(names))] = Cid.parse(target) + key = (Cid.from_bytes(root), tuple(names)) + self._cache[key] = Cid.from_bytes(target) except (ValueError, TypeError): - log(f"storage: ignoring invalid cache record {record!r}", LOG_ERROR) + log(f"storage: ignoring invalid cache record {root!r} {path!r}", LOG_ERROR) + bad_cache.append((root, path)) + continue + if isinstance(used, int): + self._cache_used = max(self._cache_used, used + 1) + capped = [(root.binary, statedb.dump(list(names))) for root, names in + list(self._cache)[:max(0, len(self._cache) - MAX_CACHE_RECORDS)]] self._cap_cache() - - def _save_pins(self): - """Persist pins (called with the lock held).""" - value = {"pins": {str(cid): sorted(sources) for cid, sources in self._pins.items()}} + self._cache_dirty.clear() + if bad_pins or bad_cache or capped: + with self._db.transaction() as tx: + tx.executemany("DELETE FROM pins WHERE cid = ? AND source = ?", bad_pins) + tx.executemany("DELETE FROM cache WHERE root = ? AND path = ?", bad_cache + capped) + + def _save_pins(self, source: str, added=(), removed=()): + """Persist source's new pins of added and releases of removed, durably (called with the lock held).""" self.blocks.sync() # so pins saved never outlast the blocks they keep - write_json(os.path.join(self.state_dir, PINS_FILE), value) + with self._db.transaction(durable=True) as tx: + tx.executemany("INSERT OR IGNORE INTO pins (cid, source) VALUES (?, ?)", + [(cid.binary, source) for cid in added]) + tx.executemany("DELETE FROM pins WHERE cid = ? AND source = ?", [(cid.binary, source) for cid in removed]) def save(self): - """Persist cache records, least recently used first, if they changed since the last save.""" + """Persist the cache records changed since the last save, least recently used first.""" with self._save_lock: with self._lock: if not self._cache_dirty: return - value = {"fetches": [[str(root), list(names), str(target)] - for (root, names), target in self._cache.items()]} - self._cache_dirty = False + dirty, self._cache_dirty = self._cache_dirty, set() + # The records changed since the last save are the most recently used (each change moved its + # record to the end), so those still cached are the last ones, in order + kept = sum(1 for key in dirty if key in self._cache) + changed = list(itertools.islice(reversed(self._cache.items()), kept))[::-1] + rows = [(root.binary, statedb.dump(list(names)), target.binary, self._cache_used + n) + for n, ((root, names), target) in enumerate(changed)] + self._cache_used += len(rows) + gone = [(root.binary, statedb.dump(list(names))) for root, names in dirty if + (root, names) not in self._cache] try: self.blocks.sync() - write_json(os.path.join(self.state_dir, CACHE_FILE), value) + with self._db.transaction() as tx: + tx.executemany("DELETE FROM cache WHERE root = ? AND path = ?", gone) + tx.executemany("INSERT OR REPLACE INTO cache (root, path, target, used) VALUES (?, ?, ?, ?)", + rows) except BaseException: with self._lock: - self._cache_dirty = True + self._cache_dirty |= dirty raise + def close(self): + """Release the state database (the node calls this once it has saved everything at shutdown).""" + self._db.close() + # Blocks def has(self, cid: Cid) -> bool: @@ -400,9 +421,9 @@ def pin_many(self, cids, source: str): if not added: return try: - self._save_pins() + self._save_pins(source, added=added) except BaseException: - # Not saved (a full or failing disk): not pinned, so memory says what pins.json does + # Not saved (a full or failing disk): not pinned, so memory says what the database does for cid in added: sources = self._pins[cid] sources.discard(source) @@ -421,9 +442,9 @@ def unpin(self, cid: Cid, source: str) -> bool: del self._pins[cid] self._maybe_garbage = True try: - self._save_pins() + self._save_pins(source, removed=[cid]) except BaseException: - self._pins.setdefault(cid, set()).add(source) # still pinned, as pins.json says + self._pins.setdefault(cid, set()).add(source) # still pinned, as the database says raise return True @@ -462,7 +483,7 @@ def record_fetch(self, root: Cid, names: list[str] | tuple[str, ...], target: Ci self._cache_generation += 1 self._cache[key] = target self._cache.move_to_end(key) - self._cache_dirty = True + self._cache_dirty.add(key) dropped = self._cap_cache() return dropped - set(self._cache.values()) @@ -470,10 +491,11 @@ def _cap_cache(self) -> set[Cid]: """Drop the least recently used records past MAX_CACHE_RECORDS; their targets (lock held).""" dropped = set() while len(self._cache) > MAX_CACHE_RECORDS: - dropped.add(self._cache.popitem(last=False)[1]) + key, target = self._cache.popitem(last=False) + dropped.add(target) + self._cache_dirty.add(key) if dropped: self._maybe_garbage = True - self._cache_dirty = True self._cache_generation += 1 return dropped @@ -484,7 +506,7 @@ def touch(self, root: Cid, names: list[str] | tuple[str, ...]) -> bool: if key not in self._cache: return False self._cache.move_to_end(key) - self._cache_dirty = True + self._cache_dirty.add(key) return True def cached_fetches(self) -> list[tuple[Cid, tuple[str, ...], Cid]]: @@ -778,9 +800,8 @@ def _evict_to(self, limit: int) -> tuple[set[Cid], int, int]: return set(), 0, 0 evicted = len(records) - keep for _ in range(evicted): - self._cache.popitem(last=False) + self._cache_dirty.add(self._cache.popitem(last=False)[0]) if evicted: - self._cache_dirty = True self._cache_generation += 1 dead = {cid for cid in self._sizes if cid not in live} deleted, freed = self._delete(dead) @@ -939,8 +960,8 @@ def gc(self) -> dict: before = self._with_memoised(before) uncached = set(self._cache.values()) if self._cache: + self._cache_dirty.update(self._cache) self._cache.clear() - self._cache_dirty = True self._cache_generation += 1 deleted, freed = self._collect() incomplete = {cid for cid, blocks in before.items() if blocks & deleted} @@ -1016,3 +1037,4 @@ def stat(self, cid: Cid) -> dict: local_bytes = sum(self._sizes[c] for c in local) return {"size": self.content_size(cid), "blocks": len(known), "local_blocks": len(local), "local_bytes": local_bytes} + diff --git a/rnfs/dht/providers.py b/rnfs/dht/providers.py index 7753f32..e48bbdf 100644 --- a/rnfs/dht/providers.py +++ b/rnfs/dht/providers.py @@ -20,12 +20,16 @@ newcomer whose last record was evicted from the pool is kept too, in a smaller memory that forgets the most recently first seen first, so a flood can't keep a new publisher new forever. -Times are wall clock and persisted, so a time in the future (the clock ran ahead, then was corrected) is -clamped to now on load and at each expire(); otherwise such records would outlive their TTL. +Records, first-seen times and departures are persisted in the state database (rnfs.statedb: the +provider_records, provider_publishers, provider_departed and provider_evicted tables) by save(), which the node +calls periodically and at shutdown, writing only the rows changed since the last save; they are read from it +only at startup. Times are wall clock and persisted, so a time in the future (the clock ran ahead, then was +corrected) is clamped to now on load and at each expire(); otherwise such records would outlive their TTL. """ import heapq import itertools import json +import math import os import random import threading @@ -33,11 +37,12 @@ from collections import OrderedDict from collections.abc import Callable, Iterable -from rnfs import fsutil +from rnfs import statedb from rnfs.constants import ID_LENGTH, K, PROVIDER_TTL from rnfs.dht.routing import check_id +from rnfs.log import LOG_ERROR, LOG_WARNING, log, plural -_STATE_VERSION = 1 +_STATE_VERSION = 1 # of providers.json, which rnfs 0.1.4 and earlier wrote _EXPIRE_INTERVAL = 60 # when full, scan for expired records at most this often (seconds), not on every store PUBLISHER_SHARE = 100 # a publisher may hold at most max_records // PUBLISHER_SHARE records ... MIN_PER_PUBLISHER = 10 # ... but at least this many, so small stores still work @@ -53,6 +58,101 @@ def _never(node_id: bytes) -> bool: return False +def _is_id(value) -> bool: + """True if value is a node ID or DHT key (ID_LENGTH bytes).""" + return isinstance(value, bytes) and len(value) == ID_LENGTH + + +def _number(value) -> float | None: + """A saved time as a float, or None if it isn't a number or is NaN (infinities are kept, as the JSON file's + float() kept them: the load's clamping and TTL check deal with them).""" + if isinstance(value, bool) or not isinstance(value, (int, float)) or math.isnan(value): + return None + return float(value) + + +# Migration of providers.json (rnfs 0.1.4 and earlier) + + +def _parse_state(state) -> tuple[dict[bytes, float], list, list, list]: + """(first seen by publisher, records, departures, evictions) of a providers.json value, converted as rnfs + 0.1.4 loaded it: records are (key, publisher, first stored, last stored, pooled), files written before the + newcomer pool having four fields per record and no departures or evictions. Raises ValueError (a version + other than _STATE_VERSION), KeyError, TypeError, AttributeError, OverflowError or IndexError if anything in it + is malformed: those versions then started empty.""" + if state.get("version") != _STATE_VERSION: + raise ValueError(f"version {state.get('version')!r}, not {_STATE_VERSION}") + first_seen = {bytes.fromhex(p): float(t) for p, t in state["publishers"].items()} + records = [(bytes.fromhex(r[0]), bytes.fromhex(r[1]), float(r[2]), float(r[3]), len(r) > 4 and bool(r[4])) + for r in state["records"]] + departed = [(bytes.fromhex(p), float(seen), float(left)) for p, seen, left in state.get("departed", [])] + evicted = [(bytes.fromhex(p), float(seen), float(left)) for p, seen, left in state.get("evicted", [])] + return first_seen, records, departed, evicted + + +_MALFORMED = (ValueError, KeyError, TypeError, AttributeError, OverflowError, IndexError) # _parse_state's errors + + +def read_records(path: str, shape=None): + """providers.json's value, for statedb.migrate (shape is unused), or None if there is none to import. + + rnfs 0.1.4 started empty on any problem with it, and the next save overwrote it. Now one that isn't JSON, is + of another version or holds a malformed record is moved aside as damaged (the node starts without its + records, which publishers republish), and one that can't be read is logged and left, to import at the next + start. + """ + try: + with open(path, encoding="utf-8") as f: + state = json.load(f) + except FileNotFoundError: + return None + except ValueError as e: + statedb.set_aside(path, str(e)) + return None + except OSError as e: + if os.path.isdir(path): # Windows reports opening a directory as PermissionError + statedb.set_aside(path, "it is a directory") + else: + log(f"providers: could not read {path} ({e}); starting without its records, and importing it at the " + "next start", LOG_ERROR) + return None + try: + _parse_state(state) + except _MALFORMED as e: + statedb.set_aside(path, f"malformed: {e!r}") + return None + return state + + +def import_records(value: dict, tx) -> int: + """Import providers.json (statedb.migrate): its publishers, records, departures and evictions replace the + four provider tables, converted as rnfs 0.1.4 loaded them. What that load dropped whatever the clock and + limits were (NaN times, IDs of the wrong length, a record repeated: the first stored is kept) is left out; + the rest (expiry, future times, limits, records of unknown publishers) is the load's. Returns the number of + records imported.""" + for table in ("provider_publishers", "provider_records", "provider_departed", "provider_evicted"): + tx.execute(f"DELETE FROM {table}") + try: + first_seen, records, departed, evicted = _parse_state(value) + except _MALFORMED as e: # read_records checked it: not expected + log(f"providers: not importing providers.json ({e!r}); starting without its records", LOG_ERROR) + return 0 + tx.executemany("INSERT INTO provider_publishers (publisher, first_seen) VALUES (?, ?)", + [(p, t) for p, t in first_seen.items() if _is_id(p) and t == t]) + kept: dict[tuple[bytes, bytes], tuple] = {} + for key, publisher, first, last, pooled in sorted(records, key=lambda record: record[2]): + if _is_id(key) and _is_id(publisher) and first == first and last == last: + kept.setdefault((key, publisher), (key, publisher, first, last, int(pooled))) + tx.executemany("INSERT INTO provider_records (key, publisher, first_stored, last_stored, pooled)" + " VALUES (?, ?, ?, ?, ?)", list(kept.values())) + for table, entries in (("provider_departed", departed), ("provider_evicted", evicted)): + # The load applied them oldest first, a later one replacing an earlier one of the same publisher + tx.executemany(f"INSERT OR REPLACE INTO {table} (publisher, first_seen, at) VALUES (?, ?, ?)", + [(p, seen, left) for p, seen, left in sorted(entries, key=lambda item: item[2]) + if _is_id(p) and seen == seen and left == left]) + return len(kept) + + class _Record: """When a record was first stored and last stored (republished), and whether it is in the newcomer pool.""" @@ -85,17 +185,17 @@ def __init__(self, first_seen: float, order: int): class ProviderStore: """Provider records, publishers' first-seen times, and /find reply selection. Thread-safe. - `clock` is wall-clock seconds, since times are persisted. `state_path` (None: memory only) is where - save() writes records and first-seen times as JSON; it is loaded on construction. `rng` picks the + `clock` is wall-clock seconds, since times are persisted. `state_dir` (None: memory only) holds the state + database save() writes records and first-seen times to; they are loaded on construction. `rng` picks the random half of /find replies, the publisher a full key evicts and, among the publishers holding the most pooled records, the one a full pool evicts from. """ - def __init__(self, max_records: int, state_path: str | None, ttl: float = PROVIDER_TTL, + def __init__(self, max_records: int, state_dir: str | None, ttl: float = PROVIDER_TTL, clock: Callable[[], float] = time.time, rng: random.Random = random, max_per_publisher: int | None = None, max_per_key: int = MAX_PUBLISHERS_PER_KEY, own_id: bytes | None = None, seniority: float = 0.0): - """Store of at most max_records records, loaded from state_path if it exists. + """Store of at most max_records records, loaded from the state database in state_dir if given. max_per_publisher defaults to max(MIN_PER_PUBLISHER, max_records // PUBLISHER_SHARE); a publisher seen for less than `seniority` seconds (the node passes provider_seniority; 0: no ramp-up) may hold @@ -112,7 +212,6 @@ def __init__(self, max_records: int, state_path: str | None, ttl: float = PROVID self.max_per_key = max_per_key self.own_id = own_id self.seniority = seniority - self.state_path = state_path self.ttl = ttl self.clock = clock self.rng = rng @@ -139,12 +238,22 @@ def __init__(self, max_records: int, state_path: str | None, ttl: float = PROVID self._evicted_size = max(MIN_PER_PUBLISHER, self.pool_size // EVICTED_SHARE) self._count = 0 self._main_count = 0 - self._dirty = False + # Rows changed since the last save (only tracked when persisted): records by (key, publisher), publishers, + # and publishers whose departed or evicted entry changed. save() writes each as memory has it then, or + # deletes it if memory has none + self._dirty_records: set[tuple[bytes, bytes]] = set() + self._dirty_publishers: set[bytes] = set() + self._dirty_gone: set[bytes] = set() self._last_expire = float("-inf") self._lock = threading.RLock() self._save_lock = threading.Lock() # one save at a time, so an older snapshot never lands last - if state_path is not None: - self._load() + self._db = None if state_dir is None else statedb.connect(state_dir) + if self._db is not None: + try: + self._load() + except BaseException: + self._db.close() + raise # Storing and withdrawing @@ -168,7 +277,7 @@ def store(self, key: bytes, publisher: bytes) -> bool: record.last_stored = now if record.pooled and self._main_count < self.main_size: self._unpool(key, publisher, record) # room outside the pool: it leaves random eviction - self._dirty = True + self._touch(key, publisher) return True # An expired record still counted is replaced, starting a new first-stored time self._remove(key, publisher) @@ -184,10 +293,10 @@ def store(self, key: bytes, publisher: bytes) -> bool: entry = self._publishers.get(publisher) if entry is None: first_seen = self._departed_first_seen(publisher, now) - self._departed.pop(publisher, None) - self._evicted.pop(publisher, None) + self._forget_gone(publisher) entry = self._publishers[publisher] = _Publisher(now if first_seen is None else first_seen, next(self._order)) + self._touch_publisher(publisher) entry.keys.add(key) record = self._records.setdefault(key, {})[publisher] = _Record(now, now, pooled) self._count += 1 @@ -195,15 +304,13 @@ def store(self, key: bytes, publisher: bytes) -> bool: self._pool_add(key, publisher) else: self._add_main(publisher, entry) - self._dirty = True + self._touch(key, publisher) return True def withdraw(self, key: bytes, publisher: bytes) -> bool: """Remove publisher's record for key. True if there was one.""" with self._lock: - removed = self._remove(key, publisher) - self._dirty = self._dirty or removed - return removed + return self._remove(key, publisher) def provide(self, publisher: bytes, keys: Iterable[bytes], withdrawn: Iterable[bytes] = ()) -> int: """Apply one /provide batch from publisher: withdraw `withdrawn`, then store `keys`. @@ -237,9 +344,7 @@ def retain(self, keep: Callable[[bytes], bool]) -> int: for key in list(entry.keys) if entry is not None else (): removed += self._remove(key, publisher) # Not kept as departed either: its standing here came from records it shouldn't have had - self._departed.pop(publisher, None) - self._evicted.pop(publisher, None) - self._dirty = self._dirty or removed > 0 + self._forget_gone(publisher) return removed def allowance(self, publisher: bytes) -> int | None: @@ -282,7 +387,6 @@ def _within_limits(self, key: bytes, publisher: bytes, now: float) -> bool: senior = min(K // 2, self.max_per_key - 1) victim = list(records)[self.rng.randrange(senior, len(records))] self._remove(key, victim) - self._dirty = True return True def _sweep_publisher(self, publisher: bytes, entry: _Publisher, now: float): @@ -292,7 +396,6 @@ def _sweep_publisher(self, publisher: bytes, entry: _Publisher, now: float): record = self._records[key][publisher] if now - record.last_stored >= self.ttl: self._remove(key, publisher) - self._dirty = True else: earliest = min(earliest, record.last_stored) entry.sweep_after = earliest + self.ttl @@ -303,7 +406,6 @@ def _drop_expired(self, pairs: list[tuple[bytes, bytes]], now: float): record = self._records.get(key, {}).get(publisher) if record is not None and now - record.last_stored >= self.ttl: self._remove(key, publisher) - self._dirty = True def _make_room(self, publisher: bytes, now: float) -> bool | None: """Where a new record of publisher goes: False outside the pool, True in it, None if refused. Evicts @@ -438,6 +540,7 @@ def _remove(self, key: bytes, publisher: bytes, evicted: bool = False) -> bool: if not records: del self._records[key] self._count -= 1 + self._touch(key, publisher) entry = self._publishers[publisher] entry.keys.discard(key) if record.pooled: @@ -447,6 +550,7 @@ def _remove(self, key: bytes, publisher: bytes, evicted: bool = False) -> bool: entry.main -= 1 if not entry.keys: del self._publishers[publisher] + self._touch_publisher(publisher) self._depart(publisher, entry.first_seen, self.clock(), evicted) return True @@ -455,23 +559,48 @@ def _depart(self, publisher: bytes, first_seen: float, now: float, evicted: bool `seniority` (a younger one has no standing to lose, and leaving those out means a flood of fresh identities can't push the others out). A younger one whose last record a full pool evicted is kept in a memory of its own, so being pushed out doesn't make it new again. Caller holds the lock.""" - self._departed.pop(publisher, None) - self._evicted.pop(publisher, None) + self._forget_gone(publisher) if now - first_seen < self.seniority: if evicted: self._remember_evicted(publisher, first_seen, now) return self._departed[publisher] = (first_seen, now) + self._touch_gone(publisher) while len(self._departed) > self._departed_size: - self._departed.popitem(last=False) + self._touch_gone(self._departed.popitem(last=False)[0]) def _remember_evicted(self, publisher: bytes, first_seen: float, now: float): """Keep a pool-evicted newcomer's first-seen time; when the memory is full, the one first seen most recently (perhaps this one) is forgotten. Caller holds the lock.""" self._evicted[publisher] = (first_seen, now) + self._touch_gone(publisher) if len(self._evicted) > self._evicted_size: newest = max(self._evicted, key=lambda p: self._evicted[p][0]) del self._evicted[newest] + self._touch_gone(newest) + + def _forget_gone(self, publisher: bytes): + """Drop publisher's departed or evicted entry, if it has one. Caller holds the lock.""" + departed = self._departed.pop(publisher, None) + evicted = self._evicted.pop(publisher, None) + if departed is not None or evicted is not None: + self._touch_gone(publisher) + + def _touch(self, key: bytes, publisher: bytes): + """Note that publisher's record for key changed (or went), for the next save. Caller holds the lock.""" + if self._db is not None: + self._dirty_records.add((key, publisher)) + + def _touch_publisher(self, publisher: bytes): + """Note that publisher's first-seen time changed (or went), for the next save. Caller holds the lock.""" + if self._db is not None: + self._dirty_publishers.add(publisher) + + def _touch_gone(self, publisher: bytes): + """Note that publisher's departed or evicted entry changed (or went), for the next save. Caller holds the + lock.""" + if self._db is not None: + self._dirty_gone.add(publisher) def _departed_first_seen(self, publisher: bytes, now: float) -> float | None: """First-seen time of a publisher whose last record went less than a TTL ago, else None. Caller holds @@ -505,40 +634,38 @@ def _expire(self, now: float) -> int: self._remove(key, publisher) # Departures are in order, oldest first while self._departed and now - next(iter(self._departed.values()))[1] >= self.ttl: - self._departed.popitem(last=False) - self._dirty = True + self._touch_gone(self._departed.popitem(last=False)[0]) while self._evicted and now - next(iter(self._evicted.values()))[1] >= self.ttl: - del self._evicted[next(iter(self._evicted))] - self._dirty = True - self._dirty = self._dirty or bool(expired) + publisher = next(iter(self._evicted)) + del self._evicted[publisher] + self._touch_gone(publisher) return len(expired) def _clamp(self, now: float): """Bring every time after now back to now (the clock was corrected backwards). Caller holds the lock.""" stored_moved = False - for records in self._records.values(): - for record in records.values(): - if record.last_stored > now: - record.last_stored = now - stored_moved = self._dirty = True - if record.first_stored > now: - record.first_stored = now - self._dirty = True + for key, records in self._records.items(): + for publisher, record in records.items(): + if record.last_stored > now or record.first_stored > now: + stored_moved = stored_moved or record.last_stored > now + record.last_stored = min(record.last_stored, now) + record.first_stored = min(record.first_stored, now) + self._touch(key, publisher) moved = False - for entry in self._publishers.values(): + for publisher, entry in self._publishers.items(): if stored_moved: entry.sweep_after = float("-inf") # expiries moved earlier: its next sweep can't be skipped if entry.first_seen > now: entry.first_seen = now + self._touch_publisher(publisher) moved = True for memory in (self._departed, self._evicted): for publisher, (first_seen, departed) in list(memory.items()): if first_seen > now or departed > now: memory[publisher] = (min(first_seen, now), min(departed, now)) - self._dirty = True + self._touch_gone(publisher) if moved: self._rebuild_newest() - self._dirty = True # Queries @@ -606,75 +733,95 @@ def first_seen(self, publisher: bytes) -> float | None: # Persistence def save(self): - """Write records and first-seen times to state_path atomically, if anything changed since the last save. + """Write the records, first-seen times and departures changed since the last save, if any. - Call it periodically and at shutdown; storing doesn't save, since a full store is megabytes. + Call it periodically and at shutdown; storing doesn't save, since records change with every /provide. """ - if self.state_path is None: + if self._db is None: return # Saves come from the maintenance thread and from stop(): one at a time, or a snapshot taken before - # the other's could be written last, with _dirty already cleared + # the other's could be written last, with the changes it holds already cleared with self._save_lock: with self._lock: - if not self._dirty: + records, publishers, gone = self._dirty_records, self._dirty_publishers, self._dirty_gone + if not (records or publishers or gone): return - # Only copied under the lock (records change in place); converted and written without it - publishers = [(publisher, entry.first_seen) for publisher, entry in self._publishers.items()] - records = [(key, publisher, record.first_stored, record.last_stored, record.pooled) - for key, held in self._records.items() for publisher, record in held.items()] - departures = list(self._departed.items()) - evictions = list(self._evicted.items()) - self._dirty = False + self._dirty_records, self._dirty_publishers, self._dirty_gone = set(), set(), set() + # Only copied under the lock (records change in place); written without it + stored, removed = [], [] + for key, publisher in records: + record = self._records.get(key, {}).get(publisher) + if record is None: + removed.append((key, publisher)) + else: + stored.append((key, publisher, record.first_stored, record.last_stored, int(record.pooled))) + seen = [(p, self._publishers[p].first_seen) for p in publishers if p in self._publishers] + unseen = [(p,) for p in publishers if p not in self._publishers] + departed = [(p, *self._departed[p]) for p in gone if p in self._departed] + evicted = [(p, *self._evicted[p]) for p in gone if p in self._evicted] + no_longer = [(p,) for p in gone] + stored.sort() # in key order, as the table is: a full save writes a third faster try: - state = { - "version": _STATE_VERSION, - "publishers": {publisher.hex(): first_seen for publisher, first_seen in publishers}, - "records": [[key.hex(), publisher.hex(), first_stored, last_stored, int(pooled)] - for key, publisher, first_stored, last_stored, pooled in records], - "departed": [[publisher.hex(), first_seen, departed] - for publisher, (first_seen, departed) in departures], - "evicted": [[publisher.hex(), first_seen, evicted] - for publisher, (first_seen, evicted) in evictions], - } - directory = os.path.dirname(os.path.abspath(self.state_path)) - os.makedirs(directory, exist_ok=True) - fsutil.write_atomic(self.state_path, json.dumps(state, separators=(",", ":")).encode(), mode=0o600) + with self._db.transaction() as tx: + tx.executemany("DELETE FROM provider_records WHERE key = ? AND publisher = ?", removed) + tx.executemany("INSERT OR REPLACE INTO provider_records (key, publisher, first_stored, last_stored," + " pooled) VALUES (?, ?, ?, ?, ?)", stored) + tx.executemany("DELETE FROM provider_publishers WHERE publisher = ?", unseen) + tx.executemany("INSERT OR REPLACE INTO provider_publishers (publisher, first_seen) VALUES (?, ?)", + seen) + tx.executemany("DELETE FROM provider_departed WHERE publisher = ?", no_longer) + tx.executemany("DELETE FROM provider_evicted WHERE publisher = ?", no_longer) + tx.executemany("INSERT OR REPLACE INTO provider_departed (publisher, first_seen, at)" + " VALUES (?, ?, ?)", departed) + tx.executemany("INSERT OR REPLACE INTO provider_evicted (publisher, first_seen, at)" + " VALUES (?, ?, ?)", evicted) except BaseException: - with self._lock: - self._dirty = True # try again next time + with self._lock: # try again next time + self._dirty_records |= records + self._dirty_publishers |= publishers + self._dirty_gone |= gone raise + def close(self): + """Release the state database (the node calls this once it has saved everything at shutdown).""" + if self._db is not None: + self._db.close() + def _load(self): - """Load state_path if it exists. A missing or unreadable file starts empty: records get republished. + """Load the saved records, first-seen times and departures, applying the TTL and limits as configured now. - Files written before the newcomer pool (four fields per record, no departures) load with every record - outside the pool, as far as it has room. + Rows that don't load (malformed, expired, past a limit, of a publisher left without records) or that load + changed (times clamped, a record moved in or out of the pool) are marked changed, so the next save deletes + or rewrites them and the tables hold what memory does. Files rnfs 0.1.3 wrote, before the newcomer pool, + were imported with every record outside it: those load outside the pool, as far as it has room. """ - try: - with open(self.state_path, encoding="utf-8") as f: - state = json.load(f) - if state.get("version") != _STATE_VERSION: - return - first_seen = {bytes.fromhex(p): float(t) for p, t in state["publishers"].items()} - records = [(bytes.fromhex(r[0]), bytes.fromhex(r[1]), float(r[2]), float(r[3]), - len(r) > 4 and bool(r[4])) for r in state["records"]] - departed = [(bytes.fromhex(p), float(seen), float(left)) for p, seen, left in state.get("departed", [])] - evicted = [(bytes.fromhex(p), float(seen), float(left)) for p, seen, left in state.get("evicted", [])] - except (OSError, ValueError, KeyError, TypeError, AttributeError, OverflowError, IndexError): - return + publisher_rows = self._db.query("SELECT publisher, first_seen FROM provider_publishers") + record_rows = self._db.query("SELECT key, publisher, first_stored, last_stored, pooled FROM provider_records") + departed_rows = self._db.query("SELECT publisher, first_seen, at FROM provider_departed") + evicted_rows = self._db.query("SELECT publisher, first_seen, at FROM provider_evicted") now = self.clock() + malformed = 0 # Times in the future (a clock that ran ahead, since corrected) count as now; NaN is dropped - first_seen = {p: min(t, now) for p, t in first_seen.items() if t == t} - records = [(k, p, min(first, now), min(last, now), pooled) for k, p, first, last, pooled in records - if first == first and last == last] + first_seen = {} + for publisher, seen in publisher_rows: + seen = _number(seen) + if seen is None or not _is_id(publisher): + malformed += 1 + continue + first_seen[publisher] = min(seen, now) + records = [] + for key, publisher, first, last, pooled in record_rows: + first, last = _number(first), _number(last) + if first is None or last is None or not _is_id(key) or not isinstance(pooled, int): + malformed += 1 + continue + records.append((key, publisher, min(first, now), min(last, now), bool(pooled))) # Rebuild first-seen order, oldest publisher first for publisher, seen in sorted(first_seen.items(), key=lambda item: item[1]): - if len(publisher) == ID_LENGTH: - self._publishers[publisher] = _Publisher(seen, next(self._order)) + self._publishers[publisher] = _Publisher(seen, next(self._order)) # Oldest records first, so each key's records are in first-stored order and the limits keep the oldest for key, publisher, first, last, pooled in sorted(records, key=lambda record: record[2]): - if (len(key) != ID_LENGTH or publisher not in self._publishers or now - last >= self.ttl - or publisher in self._records.get(key, {}) + if (publisher not in self._publishers or now - last >= self.ttl or (publisher != self.own_id and len(self._publishers[publisher].keys) >= self.max_per_publisher) or len(self._records.get(key, ())) >= self.max_per_key): @@ -697,11 +844,34 @@ def _load(self): for publisher in [p for p, entry in self._publishers.items() if not entry.keys]: del self._publishers[publisher] self._rebuild_newest() + departed, evicted = [], [] + for rows, entries in ((departed_rows, departed), (evicted_rows, evicted)): + for publisher, seen, left in rows: + seen, left = _number(seen), _number(left) + if seen is None or left is None or not _is_id(publisher): + malformed += 1 + continue + entries.append((publisher, seen, left)) for publisher, seen, left in sorted(departed, key=lambda item: item[2]): - if (len(publisher) == ID_LENGTH and seen == seen and left == left and publisher not in self._publishers - and now - min(left, now) < self.ttl): + if publisher not in self._publishers and now - min(left, now) < self.ttl: self._depart(publisher, min(seen, now), min(left, now)) for publisher, seen, left in sorted(evicted, key=lambda item: item[2]): - if (len(publisher) == ID_LENGTH and seen == seen and left == left and publisher not in self._publishers - and publisher not in self._departed and now - min(left, now) < self.ttl): + if (publisher not in self._publishers and publisher not in self._departed + and now - min(left, now) < self.ttl): self._remember_evicted(publisher, min(seen, now), min(left, now)) + if malformed: + log(f"providers: ignoring {plural(malformed, 'malformed row')} in the state database", LOG_WARNING) + # The rows memory doesn't hold as they are + self._dirty_publishers = {publisher for publisher, seen in publisher_rows + if getattr(self._publishers.get(publisher), "first_seen", None) != seen} + self._dirty_records = {(key, publisher) for key, publisher, first, last, pooled in record_rows + if self._saved_differs(key, publisher, first, last, pooled)} + self._dirty_gone = ({publisher for publisher, seen, left in departed_rows + if self._departed.get(publisher) != (seen, left)} + | {publisher for publisher, seen, left in evicted_rows + if self._evicted.get(publisher) != (seen, left)}) + + def _saved_differs(self, key, publisher, first, last, pooled) -> bool: + """True unless memory holds a record of publisher for key with exactly these saved values (_load).""" + record = self._records.get(key, {}).get(publisher) + return record is None or (record.first_stored, record.last_stored, int(record.pooled)) != (first, last, pooled) diff --git a/rnfs/features/__init__.py b/rnfs/features/__init__.py index 51b45f2..ab050ee 100644 --- a/rnfs/features/__init__.py +++ b/rnfs/features/__init__.py @@ -41,6 +41,9 @@ def stop(self): def save(self): """Save state that is written in batches (called periodically and at stop).""" + def close(self): + """Release the state database, once the node has saved everything at shutdown (after stop and save).""" + def status(self) -> dict: """Fields added to `rnfs status`.""" return {} diff --git a/rnfs/features/contacts.py b/rnfs/features/contacts.py index 8f09b35..64536ad 100644 --- a/rnfs/features/contacts.py +++ b/rnfs/features/contacts.py @@ -11,7 +11,8 @@ or probe the address book). Wherever the CLI takes a node ID (`get --from`, `ping`, `pin remote`), a node contact's nick works too (node_arg). The LXMF feature looks nicks up with lookup(nick, "lxmf"), and trusts senders that has(address, "lxmf") finds. -- Stored in /contacts.json (atomic, UTF-8), at most MAX_CONTACTS entries. +- Stored in the state database's contacts table (rnfs.statedb), each change written at once, durably; at most + MAX_CONTACTS entries. No RNS at module level (the CLI imports this module). """ @@ -26,10 +27,8 @@ from rnfs.fsutil import printable NAME = "contacts" -CONTACTS_FILE = "contacts.json" -STATE_VERSION = 1 KINDS = ("name", "node", "lxmf") -MAX_CONTACTS = 10_000 # an address book is typed in by hand; this only bounds the state file +MAX_CONTACTS = 10_000 # an address book is typed in by hand; this only bounds the saved state EXPAND_AGE = 60.0 # seconds a nick's resolved CID is reused when a target starts with a contact's nick _NICK = re.compile(r"[a-z0-9][a-z0-9._-]{0,31}") _HEX_ID = re.compile(r"[0-9a-f]{32}") @@ -70,40 +69,70 @@ def check_kind(kind) -> str: return kind +def import_contacts(value, tx) -> int: + """Import contacts.json (statedb.migrate): its contacts replace the contacts table, each checked as rnfs 0.1.4 + loaded it (a malformed one is logged and skipped), at most MAX_CONTACTS. Anything but an object with a + "contacts" object imports nothing. Returns the number imported.""" + tx.execute("DELETE FROM contacts") + entries = value.get("contacts") if isinstance(value, dict) else None + if not isinstance(entries, dict): + return 0 + count = 0 + for nick, entry in entries.items(): + try: + check_nick(nick) + tx.execute("INSERT INTO contacts (nick, kind, id) VALUES (?, ?, ?)", + (nick, check_kind(entry["kind"]), parse_hex(entry["id"]))) + count += 1 + except (KeyError, TypeError, ValueError): + ContactsFeature._log(f"ignoring malformed contact {nick!r} in contacts.json", LOG_WARNING) + if count >= MAX_CONTACTS: + break + return count + + class ContactsFeature(Feature): """The address book. Thread-safe.""" name = NAME def __init__(self, node): - """Load the address book from the state dir.""" + """Load the address book from the state database.""" + from rnfs import statedb super().__init__(node) - import os - self._path = os.path.join(node.config.state_dir, CONTACTS_FILE) self._lock = threading.Lock() self._contacts: dict[str, tuple[str, bytes]] = {} # nick -> (kind, hash) self._nicks: dict[tuple[str, bytes], int] = {} # (kind, hash) -> how many nicks name it, for has() + self._dirty: set[str] = set() # nicks changed (or deleted) whose save failed, written with the next # name -> (CID, monotonic time of the resolve that found it), reused for EXPAND_AGE self._expanded: dict[bytes, tuple[str, float]] = {} - self._load() + self._db = statedb.connect(node.config.state_dir) + try: + self._load() + except BaseException: + self.close() + raise # State def _load(self): - """Read contacts.json, skipping malformed entries (it is our own file, but may be edited by hand).""" - from rnfs.content.storage import read_json - state = read_json(self._path) - entries = state.get("contacts") if isinstance(state, dict) else None - if not isinstance(entries, dict): - return - for nick, entry in entries.items(): + """Read the contacts table, by nick; malformed rows are logged and deleted, as are any past MAX_CONTACTS.""" + bad = [] + for nick, kind, ident in self._db.query("SELECT nick, kind, id FROM contacts ORDER BY nick"): + if len(self._contacts) >= MAX_CONTACTS: + bad.append((nick,)) # never added (the address book is full): dropped, as from contacts.json + continue try: check_nick(nick) - self._set(nick, (check_kind(entry["kind"]), parse_hex(entry["id"]))) - except (KeyError, TypeError, ValueError): - self._log(f"ignoring malformed contact {nick!r} in {self._path}", LOG_WARNING) - if len(self._contacts) >= MAX_CONTACTS: - break + if not isinstance(ident, bytes) or len(ident) != 16: + raise ValueError("not a 16-byte hash") + self._set(nick, (check_kind(kind), ident)) + except (TypeError, ValueError): + self._log(f"ignoring malformed contact {nick!r}", LOG_WARNING) + bad.append((nick,)) + if bad: + with self._db.transaction() as tx: + tx.executemany("DELETE FROM contacts WHERE nick = ?", bad) def _set(self, nick: str, value: tuple[str, bytes] | None): """Set or (None) delete a contact, keeping _nicks in step (lock held, or loading).""" @@ -116,12 +145,20 @@ def _set(self, nick: str, value: tuple[str, bytes] | None): self._contacts[nick] = value self._nicks[value] = self._nicks.get(value, 0) + 1 - def _save(self): - """Write contacts.json atomically (lock held).""" - from rnfs.content.storage import write_json - write_json(self._path, {"version": STATE_VERSION, - "contacts": {nick: {"kind": kind, "id": ident.hex()} - for nick, (kind, ident) in sorted(self._contacts.items())}}) + def _save(self, nick: str): + """Write nick's row, and those of earlier changes whose save failed, durably (lock held). If it fails they + are written with the next change.""" + self._dirty.add(nick) + rows = [(other, *self._contacts[other]) for other in self._dirty if other in self._contacts] + gone = [(other,) for other in self._dirty if other not in self._contacts] + with self._db.transaction(durable=True) as tx: + tx.executemany("DELETE FROM contacts WHERE nick = ?", gone) + tx.executemany("INSERT OR REPLACE INTO contacts (nick, kind, id) VALUES (?, ?, ?)", rows) + self._dirty.clear() + + def close(self): + """Release the state database (the node calls this once it has saved everything at shutdown).""" + self._db.close() @staticmethod def _log(message: str, level: int = LOG_INFO): @@ -144,7 +181,7 @@ def add(self, nick: str, ident_hex: str, kind: str | None = None) -> dict: if nick not in self._contacts and len(self._contacts) >= MAX_CONTACTS: raise ValueError(f"the address book is full ({MAX_CONTACTS} contacts)") self._set(nick, (kind, ident)) - self._save() + self._save(nick) self._log(f"added {nick}: {kind} {ident.hex()}") return {"nick": nick, "kind": kind, "id": ident.hex()} @@ -155,7 +192,7 @@ def remove(self, nick: str): if nick not in self._contacts: raise NotFound(f"no contact {nick}") self._set(nick, None) - self._save() + self._save(nick) def list(self) -> list[dict]: """Every contact, by nick.""" diff --git a/rnfs/features/follow.py b/rnfs/features/follow.py index 14283c5..8c444a3 100644 --- a/rnfs/features/follow.py +++ b/rnfs/features/follow.py @@ -13,11 +13,12 @@ - One background thread does every check and fetch, one at a time, so following many names never runs more than one transfer. Fetches obey the usual limits (max_download_size, the DAG byte bound of the new target's declared size, disk reserve); a followed target is a pin, so it is provided. -- Followed names count against keep_max_names. State: /follow.json (atomic). +- Followed names count against keep_max_names. State: the follow table of the state database (rnfs.statedb), + each change committed durably as it is made (the rows of the names it changes), and a meta value saying it + was saved whole ("clean"), which the reconcile at start needs. No RNS at module level (the CLI imports this module). """ -import os import threading import time @@ -30,8 +31,6 @@ from rnfs.fsutil import printable NAME = "follow" -FOLLOW_FILE = "follow.json" -STATE_VERSION = 1 SOURCE_PREFIX = "follow:" RETRY_INTERVAL = 300.0 # seconds before a failed check or fetch is first tried again; doubles per failure FAR_FUTURE = float("inf") @@ -49,6 +48,31 @@ def follow_source(name: bytes) -> str: return SOURCE_PREFIX + name.hex() +def import_follow(value, tx) -> int: + """Import follow.json (statedb.migrate): its names replace the follow table, each entry checked as rnfs 0.1.4 + loaded it (a malformed one is skipped). The state is clean only if every entry was read, as the reconcile at + start needs (FollowFeature._reconcile). Returns the number of names imported.""" + from rnfs import statedb + tx.execute("DELETE FROM follow") + entries = value.get("names") if isinstance(value, dict) else None + clean, names = isinstance(entries, dict), set() + for name_hex, entry in entries.items() if isinstance(entries, dict) else (): + try: + name = bytes.fromhex(name_hex) + pinned = Cid.parse(entry["pinned"]) if entry.get("pinned") is not None else None + if len(name) != ID_LENGTH: + raise ValueError("not a name") + # A name repeated in another case is the same name: the last entry wins, as it did in memory + tx.execute("INSERT OR REPLACE INTO follow (name, pinned) VALUES (?, ?)", + (name, pinned.binary if pinned is not None else None)) + except (AttributeError, KeyError, TypeError, ValueError): + clean = False + continue + names.add(name) + statedb.set_meta(tx, NAME, "clean", int(clean)) + return len(names) + + def _log(message: str, level: int = LOG_VERBOSE): """Log with this feature's prefix.""" _common.log(f"follow: {message}", level) @@ -61,10 +85,12 @@ class FollowFeature(Feature): def __init__(self, node): """Load followed names; no network.""" + from rnfs import statedb super().__init__(node) - self._path = os.path.join(node.config.state_dir, FOLLOW_FILE) + self._db = statedb.connect(node.config.state_dir) self._lock = threading.Lock() self._pinned: dict[bytes, Cid | None] = {} # followed name -> target pinned for it (complete) + self._unsaved: set[bytes] = set() # names a failed save didn't write: the next save writes them too self._state: dict[bytes, str] = {} # name -> what the last check did, for `following` self._due: dict[bytes, tuple[float, bool]] = {} # name -> (monotonic time, use the network) self._failures: dict[bytes, int] = {} # name -> checks failed in a row (for the retry backoff) @@ -74,37 +100,41 @@ def __init__(self, node): self._thread: threading.Thread | None = None self._keepers = node.feature("keepers") self._keepers.add_listener(self._changed) - self._reconcile(self._load()) + try: + self._reconcile(self._load()) + except BaseException: + self._db.close() + raise # State def _load(self) -> bool: - """Read follow.json, skipping malformed entries; True if it was there and every entry was read.""" - from rnfs.content.storage import read_json - state = read_json(self._path) # a damaged file is moved aside and gives None, as a missing one does - entries = state.get("names") if isinstance(state, dict) else None - if not isinstance(entries, dict): - return False - clean = True - for name_hex, entry in entries.items(): + """Read the followed names, skipping malformed rows (deleted); True if the state was saved whole (its + clean meta value: never saved, or follow.json missing, damaged or with a malformed entry when it was + imported, isn't) and every row was read.""" + from rnfs import statedb + clean = self._db.meta(NAME, "clean") == 1 + bad = [] + for name, pinned in self._db.query("SELECT name, pinned FROM follow"): try: - name = bytes.fromhex(name_hex) - pinned = Cid.parse(entry["pinned"]) if entry.get("pinned") is not None else None - except (AttributeError, KeyError, TypeError, ValueError): - clean = False - continue - if len(name) == ID_LENGTH: - self._pinned[name] = pinned - else: - clean = False + if type(name) is not bytes or len(name) != ID_LENGTH: + raise ValueError("not a name") + self._pinned[name] = Cid.from_bytes(pinned) if pinned is not None else None + except (TypeError, ValueError): + bad.append(name) + if bad: + clean = False + with self._db.transaction(durable=True) as tx: # and not clean, until the next save + tx.executemany("DELETE FROM follow WHERE name = ?", [(name,) for name in bad]) + statedb.set_meta(tx, NAME, "clean", 0) return clean def _reconcile(self, clean: bool): - """Make the follow pins match follow.json. If it was read whole (clean), release follow pins of names it - doesn't list: a crash inside unfollow, after follow.json was saved, leaves one that nothing else would - release (`pin rm` releases only the user's pins). Otherwise (missing, damaged or with a malformed entry) - such a pin is the record of a name still followed: follow it again, so it is kept, checked and can be - unfollowed.""" + """Make the follow pins match the saved names. If they were read whole (clean), release follow pins of + names they don't list: a crash inside unfollow, after the name was deleted, leaves one that nothing else + would release (`pin rm` releases only the user's pins). Otherwise (never saved, follow.json missing or + damaged when imported, or a malformed entry) such a pin is the record of a name still followed: follow it + again, so it is kept, checked and can be unfollowed.""" held: dict[bytes, set[Cid]] = {} for cid, sources in self.node.storage.pins().items(): for source in sources: @@ -122,12 +152,12 @@ def _reconcile(self, clean: bool): for name, cids in sorted(held.items()): # One target is its pin; several (a crash during a check) are sorted out by its next check self._pinned[name] = next(iter(cids)) if len(cids) == 1 else None - _log(f"{FOLLOW_FILE} is missing, damaged or incomplete: following name {name.hex()} again, " + _log(f"the followed names are missing, damaged or incomplete: following name {name.hex()} again, " f"as its pin of {', '.join(sorted(str(cid) for cid in cids))} shows", LOG_WARNING) try: - self._save() - except OSError as e: # the pins are kept; follow.json is rebuilt again at the next start - _log(f"saving {FOLLOW_FILE} failed: {e}", LOG_ERROR) + self._save(*held) + except OSError as e: # the pins are kept; the names are followed again at the next start + _log(f"saving the followed names failed: {e}", LOG_ERROR) return for name, cids in held.items(): source = follow_source(name) @@ -138,12 +168,23 @@ def _reconcile(self, clean: bool): except OSError as e: # e.g. a read-only state directory: tried again at the next start _log(f"could not release {cid}: {e}", LOG_ERROR) - def _save(self): - """Write follow.json atomically (lock held).""" - from rnfs.content.storage import write_json - write_json(self._path, {"version": STATE_VERSION, - "names": {name.hex(): {"pinned": str(cid) if cid is not None else None} - for name, cid in sorted(self._pinned.items())}}) + def _save(self, *names: bytes): + """Persist the entries of names, and of any a failed save left unsaved, durably, and mark the state clean + (lock held). Raises OSError; the names are then written by the next save.""" + from rnfs import statedb + names = self._unsaved.union(names) + try: + with self._db.transaction(durable=True) as tx: + tx.executemany("DELETE FROM follow WHERE name = ?", + [(name,) for name in names if name not in self._pinned]) + tx.executemany("INSERT OR REPLACE INTO follow (name, pinned) VALUES (?, ?)", + [(name, self._pinned[name].binary if self._pinned[name] is not None else None) + for name in sorted(names) if name in self._pinned]) + statedb.set_meta(tx, NAME, "clean", 1) + except BaseException: + self._unsaved = names + raise + self._unsaved = set() # Lifecycle @@ -161,6 +202,10 @@ def stop(self): self._stop.set() self._wake.set() + def close(self): + """Release the state database.""" + self._db.close() + def status(self) -> dict: """Names followed.""" with self._lock: @@ -174,7 +219,7 @@ def follow(self, name: bytes) -> dict: with self._lock: if name not in self._pinned: self._pinned[name] = None - self._save() + self._save(name) _log(f"following name {name.hex()}", LOG_NOTICE) self._failures.pop(name, None) # asked again by the user: start the backoff afresh self._due[name] = (time.monotonic(), True) @@ -191,7 +236,7 @@ def unfollow(self, name: bytes): self._state.pop(name, None) self._failures.pop(name, None) self._resolved.discard(name) - self._save() + self._save(name) # Every target its source holds: a crash during an earlier check may have left more than the one recorded source = follow_source(name) for cid in self.node.storage.pinned_by(source) | ({pinned} if pinned is not None else set()): @@ -269,9 +314,9 @@ def check(self, name: bytes, network: bool = True) -> bool: previous = self._pinned[name] self._pinned[name] = target try: - self._save() - except OSError as e: # the pins are right; follow.json catches up with the next save - _log(f"saving {FOLLOW_FILE} failed: {e}", LOG_ERROR) + self._save(name) + except OSError as e: # the pins are right; the next save writes the name too + _log(f"saving the followed names failed: {e}", LOG_ERROR) except RnfsError as e: self._set_state(name, f"fetch of {target} failed: {e}") _log(f"fetching {target} for name {name.hex()} failed: {e}", LOG_WARNING) @@ -279,7 +324,7 @@ def check(self, name: bytes, network: bool = True) -> bool: if not followed: return True # unfollowed while fetching: nothing was pinned # The old target is released only now that the new one is complete, with any other the source holds (a - # crash between pinning and saving follow.json leaves the target pinned but not recorded) + # crash between pinning and saving the name leaves the target pinned but not recorded) for cid in self.node.storage.pinned_by(source) | ({previous} if previous is not None else set()): if cid != target: self.node.storage.unpin(cid, source) diff --git a/rnfs/features/jobs.py b/rnfs/features/jobs.py index dc109d8..44cccdd 100644 --- a/rnfs/features/jobs.py +++ b/rnfs/features/jobs.py @@ -8,9 +8,10 @@ fetcher.observing(): the fetch reports the verified size of what it fetches and each block it meets, and raises Cancelled at its next step (within fetcher.POLL) once the job is cancelled or the node stops. At most max_jobs run at once (2; 1 on slow nodes); queued jobs start oldest first. -- Jobs are saved to /jobs.json (atomically) at every state change and with the node's periodic - saves. At startup, jobs that were queued or running are queued again and resume where they stopped, - since blocks already stored are never fetched again. +- Jobs are saved in the state database's jobs table (rnfs.statedb), durably, at every state change and with + the node's periodic saves (keeping a running job's progress recent); each save writes only the jobs changed + since the last. At startup, jobs that were queued or running are queued again and resume where they + stopped, since blocks already stored are never fetched again. - Resuming needs those blocks to stay stored: a fetch's own protection session ends with the fetch, after which a get's or export's blocks are evictable cache and a pin's are unreferenced. So while a job is queued or running, this feature holds a protection session on its root, which keeps everything stored below it @@ -45,7 +46,6 @@ from rnfs.features._common import LOG_ERROR, LOG_INFO, LOG_NOTICE, LOG_WARNING NAME = "jobs" -JOBS_FILE = "jobs.json" KINDS = ("get", "pin", "export") QUEUED, RUNNING, DONE, FAILED, CANCELLED = "queued", "running", "done", "failed", "cancelled" STATES = (QUEUED, RUNNING, DONE, FAILED, CANCELLED) @@ -164,7 +164,7 @@ def to_dict(self) -> dict: return {**self.to_record(), "state": self.state, "rate": rate, "eta": eta} def to_record(self) -> dict: - """The job as saved in jobs.json (a running job is saved as such, and queued again when loaded).""" + """The job as saved (a running job is saved as such, and queued again when loaded).""" return {"id": self.id, "kind": self.kind, "target": self.target, "output": self.output, "from": self.from_node, "sealed_name": self.sealed_name, "fallback_dht": self.fallback_dht, "state": self.state, "created": self.created, "started": self.started, @@ -174,7 +174,7 @@ def to_record(self) -> dict: @classmethod def from_record(cls, record) -> "Job": - """A job from a jobs.json record. Raises ValueError if anything in it is malformed.""" + """A job from a saved record. Raises ValueError if anything in it is malformed.""" if not isinstance(record, dict): raise ValueError("not an object") job_id, kind, target = record.get("id"), record.get("kind"), record.get("target") @@ -218,7 +218,7 @@ def _count(value, default): def _capped(result: dict) -> dict: - """result with at most log.MAX_WARNINGS warnings (and a count of the rest), so jobs.json stays small.""" + """result with at most log.MAX_WARNINGS warnings (and a count of the rest), so saved jobs stay small.""" from rnfs import log warnings = result.get("warnings") # node.get caps its own (adding warnings_total); this covers results saved before it did @@ -252,6 +252,27 @@ def check_job_params(kind: str, target, output, from_node, sealed_name=False, fa raise ValueError(f"only get jobs take {name}") +def import_jobs(value: dict, tx) -> int: + """Import jobs.json (statedb.migrate): its jobs, in order, replace the jobs table, each checked as rnfs 0.1.4 + loaded it (a malformed one is logged and skipped; of a repeated id, the first is kept) and saved as + Job.to_record() gives it. Returns the number imported.""" + from rnfs import statedb + tx.execute("DELETE FROM jobs") + ids: set[str] = set() + for record in value.get("jobs", []): + try: + job = Job.from_record(record) + if job.id in ids: + continue + tx.execute("INSERT INTO jobs (id, position, record) VALUES (?, ?, ?)", + (job.id, len(ids), statedb.dump(job.to_record()))) + except (ValueError, TypeError, OverflowError) as e: + _log(f"ignoring a saved job ({e})", LOG_WARNING) + continue + ids.add(job.id) + return len(ids) + + def check_job_id(text) -> str: """text if it is a job ID (8 hex characters), else ValueError.""" if not isinstance(text, str) or not _JOB_ID.fullmatch(text): @@ -266,6 +287,7 @@ class JobsFeature(Feature): def __init__(self, node): """Load saved jobs, queueing again those that hadn't finished and protecting their blocks.""" + from rnfs import statedb super().__init__(node) self._lock = threading.Lock() self._save_lock = threading.Lock() # keeps saves in order without holding _lock while writing @@ -274,8 +296,16 @@ def __init__(self, node): self._started = False self._dirty = False # jobs changed since the last save self._stop_event = threading.Event() - self._path = os.path.join(node.config.state_dir, JOBS_FILE) - self._load() + # What the jobs table holds: id -> (position, dump()ed record); and the next new job's position (after + # every one saved). Both used under _save_lock only + self._saved: dict[str, tuple[int, str]] = {} + self._position = 0 + self._db = statedb.connect(node.config.state_dir) + try: + self._load() + except BaseException: + self.close() + raise @property def max_jobs(self) -> int: @@ -286,17 +316,24 @@ def max_jobs(self) -> int: # Persistence def _load(self): - """Read jobs.json, skipping malformed records.""" - from rnfs.content.storage import read_json, shaped - value = read_json(self._path, shaped("jobs", list)) or {} - for record in value.get("jobs", []): + """Read the jobs table, oldest first; a malformed row is logged and deleted.""" + from rnfs import statedb + bad = [] + for job_id, position, text in self._db.query("SELECT id, position, record FROM jobs ORDER BY position"): try: - job = Job.from_record(record) + if not isinstance(position, int): + raise TypeError("bad position") + self._position = max(self._position, position + 1) + job = Job.from_record(statedb.load(text)) + if job.id != job_id: + raise ValueError("its id isn't its row's") except (ValueError, TypeError, OverflowError) as e: _log(f"ignoring a saved job ({e})", LOG_WARNING) + bad.append((job_id,)) continue if job.id in self._jobs: continue + self._saved[job.id] = (position, text) if job.state in ACTIVE: job.state = QUEUED # interrupted: it resumes once the node runs job.stop_event = self._stop_event @@ -308,23 +345,40 @@ def _load(self): resumed = sum(1 for job in self._jobs.values() if job.state == QUEUED) if resumed: _log(f"{resumed} unfinished job(s) will resume") + if bad: + with self._db.transaction() as tx: + tx.executemany("DELETE FROM jobs WHERE id = ?", bad) def save(self): - """Write jobs.json if jobs changed or one runs (the node calls this periodically, keeping saved - progress recent).""" - from rnfs.content.storage import write_json + """Save the jobs changed since the last save, and forget those gone, durably, if jobs changed or one runs + (the node calls this periodically, keeping saved progress recent).""" + from rnfs import statedb with self._save_lock: with self._lock: if not self._dirty and not any(job.state == RUNNING for job in self._jobs.values()): return - value = {"jobs": [job.to_record() for job in self._jobs.values()]} + records = [(job_id, statedb.dump(job.to_record())) for job_id, job in self._jobs.items()] self._dirty = False + saved, position, rows = {}, self._position, [] + for job_id, text in records: + old = self._saved.get(job_id) + if old is None: + old = (position, None) + position += 1 + if old[1] != text: + rows.append((job_id, old[0], text)) + saved[job_id] = (old[0], text) + gone = [(job_id,) for job_id in self._saved if job_id not in saved] try: - write_json(self._path, value) + if rows or gone: + with self._db.transaction(durable=True) as tx: # a queued job must survive a crash + tx.executemany("DELETE FROM jobs WHERE id = ?", gone) + tx.executemany("INSERT OR REPLACE INTO jobs (id, position, record) VALUES (?, ?, ?)", rows) except BaseException: with self._lock: self._dirty = True raise + self._saved, self._position = saved, position def _save_quietly(self): """save() after a change, logging rather than raising: a job's outcome mustn't be lost to a full disk.""" @@ -333,7 +387,11 @@ def _save_quietly(self): try: self.save() except OSError as e: - _log(f"saving {self._path} failed: {e!r}", LOG_ERROR) + _log(f"saving the jobs failed: {e!r}", LOG_ERROR) + + def close(self): + """Release the state database (the node calls this once it has saved everything at shutdown).""" + self._db.close() def _cap_finished(self): """Forget the oldest finished jobs beyond max_finished_jobs (lock held, or before any thread runs). @@ -387,7 +445,7 @@ def stop(self): try: self.save() # only if something changed: offline test nodes with no jobs write nothing except OSError as e: - _log(f"saving {self._path} failed: {e!r}", LOG_ERROR) + _log(f"saving the jobs failed: {e!r}", LOG_ERROR) def status(self) -> dict: """Running and queued job counts for `rnfs status`.""" diff --git a/rnfs/features/keepers.py b/rnfs/features/keepers.py index ad78a5e..b54e612 100644 --- a/rnfs/features/keepers.py +++ b/rnfs/features/keepers.py @@ -30,7 +30,8 @@ asked again every announce_interval, at most MAX_PENDING at a time, round robin. - **A floor.** Kept records are offered to NameManager at startup and on every resolve, so a replayed older announce is never accepted for a kept name, even after a restart or name table eviction. -- State: /keepers.json (atomic; records re-verified at load). +- State: the keepers table of the state database (rnfs.statedb), the names changed since the last save written + in batches (records re-verified at load). No RNS at module level (the CLI imports this module): it is imported inside functions. """ @@ -50,8 +51,6 @@ NAME = "keepers" NAME_PATH = "/name" KEY_DOMAIN = b"rnfs-name" -KEEPERS_FILE = "keepers.json" -STATE_VERSION = 1 # Field sizes of an RNS announce (RNS.Identity: KEYSIZE, NAME_HASH_LENGTH, random hash, RATCHETSIZE, SIGLENGTH) PUBLIC_KEY_LENGTH = 64 NAME_HASH_LENGTH = 10 @@ -244,61 +243,116 @@ def name_record(self, node_id: bytes, name: bytes): # The store +def _checked(name, reasons, fields) -> tuple[set[str], SignedRecord | None] | None: + """A kept name's reasons and re-verified record (None if it has none, or it doesn't verify) from its saved + form, or None if the entry is dropped: not a name, or no known reason. Raises AttributeError, KeyError, + TypeError or ValueError if it is malformed (dropped too).""" + reasons = {reason for reason in reasons if reason in REASONS} + if len(name) != ID_LENGTH or not reasons: + return None + record = None + if fields is not None: + record = parse_signed(name, [bytes.fromhex(field) for field in fields]) + return reasons, record + + +def _row(name: bytes, reasons: set[str], record: SignedRecord | None) -> tuple: + """A kept name's row in the keepers table.""" + from rnfs import statedb + return (name, statedb.dump(sorted(reasons)), + statedb.dump([field.hex() for field in record.wire()]) if record is not None else None) + + +def import_keepers(value, tx) -> int: + """Import keepers.json (statedb.migrate): its names replace the keepers table, each entry checked as rnfs 0.1.4 + loaded it (a malformed one is skipped; a record that doesn't verify is dropped, its name kept). Returns the + number of names imported.""" + tx.execute("DELETE FROM keepers") + entries = value.get("names") if isinstance(value, dict) else None + names = set() + for name_hex, entry in entries.items() if isinstance(entries, dict) else (): + try: + name = bytes.fromhex(name_hex) + checked = _checked(name, entry["reasons"], entry.get("record")) + if checked is None: + continue + # A name repeated in another case is the same name: the last entry wins, as it did in memory + tx.execute("INSERT OR REPLACE INTO keepers (name, reasons, record) VALUES (?, ?, ?)", + _row(name, *checked)) + except (AttributeError, KeyError, TypeError, ValueError): + continue + names.add(name) + return len(names) + + class KeeperStore: """Kept names, why each is kept, and the newest signed record of each. Thread-safe.""" - def __init__(self, path: str, max_names: int): - """Load path; at most max_names names kept for others (own names not counted).""" - self.path = path + def __init__(self, state_dir: str, max_names: int): + """Load the kept names from the state database in state_dir; at most max_names names kept for others + (own names not counted). Close it when done.""" + from rnfs import statedb self.max_names = max_names self._lock = threading.Lock() self._reasons: dict[bytes, set[str]] = {} self._records: dict[bytes, SignedRecord] = {} - self.dirty = False + self._dirty: set[bytes] = set() # names changed (or released) since the last save self._save_lock = threading.Lock() # one save at a time, so an older snapshot never lands last - self._load() + self._db = statedb.connect(state_dir) + try: + self._load() + except BaseException: + self._db.close() + raise def _load(self): - """Read the state file, re-verifying every record (a damaged entry is dropped, not trusted).""" - from rnfs.content.storage import read_json - state = read_json(self.path) - entries = state.get("names") if isinstance(state, dict) else None - if not isinstance(entries, dict): - return - for name_hex, entry in entries.items(): + """Read the kept names, re-verifying every record: a malformed row is dropped (deleted), not trusted, and + a record that doesn't verify is dropped from its row.""" + from rnfs import statedb + gone, unverified = [], [] + for name, reasons, fields in self._db.query("SELECT name, reasons, record FROM keepers"): try: - name = bytes.fromhex(name_hex) - reasons = {reason for reason in entry["reasons"] if reason in REASONS} - if len(name) != ID_LENGTH or not reasons: - continue - record = None - if entry.get("record") is not None: - record = parse_signed(name, [bytes.fromhex(field) for field in entry["record"]]) + if type(name) is not bytes: + raise TypeError("not a name") + checked = _checked(name, statedb.load(reasons), statedb.load(fields) if fields is not None else None) except (AttributeError, KeyError, TypeError, ValueError): + checked = None + if checked is None: + gone.append(name) continue - self._reasons[name] = reasons + self._reasons[name], record = checked if record is not None: self._records[name] = record + elif fields is not None: + unverified.append(name) + if gone or unverified: + with self._db.transaction() as tx: + tx.executemany("DELETE FROM keepers WHERE name = ?", [(name,) for name in gone]) + tx.executemany("UPDATE keepers SET record = NULL WHERE name = ?", [(name,) for name in unverified]) def save(self): - """Write the state file atomically if anything changed.""" - from rnfs.content.storage import write_json + """Write the names changed since the last save, if any.""" with self._save_lock: with self._lock: - if not self.dirty: + if not self._dirty: return - state = {"version": STATE_VERSION, - "names": {name.hex(): {"reasons": sorted(reasons), - "record": ([field.hex() for field in self._records[name].wire()] - if name in self._records else None)} - for name, reasons in sorted(self._reasons.items())}} - self.dirty = False + dirty, self._dirty = self._dirty, set() + rows = [_row(name, self._reasons[name], self._records.get(name)) + for name in sorted(dirty) if name in self._reasons] + gone = [(name,) for name in dirty if name not in self._reasons] try: - write_json(self.path, state) - except OSError: - self.dirty = True # try again at the next save + with self._db.transaction() as tx: + tx.executemany("DELETE FROM keepers WHERE name = ?", gone) + tx.executemany("INSERT OR REPLACE INTO keepers (name, reasons, record) VALUES (?, ?, ?)", rows) + except BaseException: + with self._lock: + self._dirty |= dirty # tried again at the next save raise + def close(self): + """Release the state database.""" + self._db.close() + def _others(self) -> int: """Names kept for others (lock held).""" return sum(1 for reasons in self._reasons.values() if OWN not in reasons) @@ -312,7 +366,7 @@ def keep(self, name: bytes, reason: str) -> bool: if reasons is None and reason != OWN and self._others() >= self.max_names: raise ValueError(f"already keeping {self.max_names} names (keep_max_names)") self._reasons.setdefault(name, set()).add(reason) - self.dirty = True + self._dirty.add(name) return True def release(self, name: bytes, reason: str) -> bool: @@ -325,7 +379,7 @@ def release(self, name: bytes, reason: str) -> bool: if not reasons: del self._reasons[name] self._records.pop(name, None) - self.dirty = True + self._dirty.add(name) return True def offer(self, record: SignedRecord) -> bool: @@ -334,7 +388,7 @@ def offer(self, record: SignedRecord) -> bool: if record.name not in self._reasons or not record.newer_than(self._records.get(record.name)): return False self._records[record.name] = record - self.dirty = True + self._dirty.add(record.name) return True def get(self, name: bytes) -> SignedRecord | None: @@ -374,7 +428,7 @@ class KeepersFeature(Feature): def __init__(self, node): """Load kept names, hook into names and the publisher; no network.""" super().__init__(node) - self.store = KeeperStore(os.path.join(node.config.state_dir, KEEPERS_FILE), node.config.keep_max_names) + self.store = KeeperStore(node.config.state_dir, node.config.keep_max_names) links = getattr(node, "links", None) # Offline test nodes have no LinkPool: their fake client may answer name_record itself self.client = KeeperClient(links) if links is not None else node.client @@ -415,6 +469,10 @@ def save(self): """Save kept records if they changed.""" self.store.save() + def close(self): + """Release the state database.""" + self.store.close() + def status(self) -> dict: """Names kept.""" return {"kept_names": self.store.count()} diff --git a/rnfs/features/lxmf.py b/rnfs/features/lxmf.py index f69fb42..4d3a176 100644 --- a/rnfs/features/lxmf.py +++ b/rnfs/features/lxmf.py @@ -12,10 +12,13 @@ --propagate, or automatically when direct delivery fails and lxmf_propagation_node is set. A propagation node only announcing itself is used only when the user asks (--nearest): it could be anyone's. The outbox records each message's state. -- Receiving: messages with rnfs links go to a bounded inbox (state/lxmf_inbox.json). Links from senders in - lxmf_auto_fetch_from are fetched and pinned with source `lxmf:`, within lxmf_auto_fetch_quota per - sender; removing the message from the inbox releases those pins, and a message holding pins is never - dropped to make room for others. lxmf_accept_from restricts senders. +- Receiving: messages with rnfs links go to a bounded inbox. Links from senders in lxmf_auto_fetch_from are + fetched and pinned with source `lxmf:`, within lxmf_auto_fetch_quota per sender; removing the + message from the inbox releases those pins, and a message holding pins is never dropped to make room for + others. lxmf_accept_from restricts senders. +- The inbox and outbox are kept in the state database (rnfs.statedb, the lxmf_messages table) and saved in + batches, each save writing the records changed since the last; outbox messages still unfinished at a start + are marked failed (LXMF doesn't persist its outbound queue). - Messages waiting at lxmf_propagation_node are downloaded every lxmf_sync_interval. Everything received is untrusted: fields are parsed strictly and capped, text is capped and escaped when @@ -74,8 +77,10 @@ SENDER_BURST = 4 SENDER_ALLOWANCES = 256 # senders whose allowance is remembered, least recently seen forgotten (back to full) MAX_ENTRY_BYTES = 16 * 1024 # an inbox record as JSON: links beyond it are dropped, then text is cut -MAX_INBOX_BYTES = 4 * 1024 * 1024 # every inbox record as JSON (held in memory, rewritten on change) +MAX_INBOX_BYTES = 4 * 1024 * 1024 # every inbox record as JSON (held in memory; a save writes those changed) OUTBOX_LIMIT = 200 +INBOX, OUTBOX = "inbox", "outbox" # the MessageLog boxes: the lxmf_messages table's box, and their meta store +MAX_ID = 2 ** 63 - 2 # largest record id: the next id must still fit SQLite's integers AUTO_QUEUE_LIMIT = 64 # auto-fetches waiting (taken round-robin by sender); more are skipped DELIVERY_LIMIT_KB = 256 # largest LXMF message accepted (LXMF's delivery_limit, in kB); also caps PN downloads HOLD_LIMIT = 32 # messages from senders not known yet, held while their identity is requested; more are dropped @@ -419,60 +424,127 @@ def message_text(text: str | None, links: list[Link]) -> str: def _json_bytes(value) -> int: - """Bytes value takes as UTF-8 JSON, as MessageLog writes it.""" + """Bytes value takes as UTF-8 JSON, as the state database stores a record (statedb.dump).""" return len(json.dumps(value, ensure_ascii=False).encode("utf-8", "backslashreplace")) -def _write_json(path: str, value): - """Write value as UTF-8 JSON (non-ASCII as is, not as 6-byte escapes) to path atomically and durably: a - power loss leaves the old or the new file whole (an emptied inbox would leave lxmf: pins nothing releases).""" - from rnfs.content.storage import write_json - write_json(path, value, ensure_ascii=False) +def _records(entries: list, limit: int, max_bytes: int | None, valid) -> tuple[list[dict], int]: + """The records a log keeps of entries (loaded from the database, or a JSON file), oldest first, and how many + were malformed: not a dict with an id (an int, 1 to MAX_ID) no earlier record has, that valid() accepts if + given. Of the others, the oldest past limit are dropped, then the oldest past max_bytes as JSON (one is kept + whatever its size).""" + kept, seen, bad = [], set(), 0 + for entry in entries: + if isinstance(entry, dict) and type(entry.get("id")) is int and 0 < entry["id"] <= MAX_ID \ + and entry["id"] not in seen and (valid is None or valid(entry)): + kept.append(entry) + seen.add(entry["id"]) + else: + bad += 1 + kept = kept[-limit:] + sizes = [_json_bytes(e) for e in kept] + total, first = sum(sizes), 0 + while max_bytes is not None and len(kept) - first > 1 and total > max_bytes: + total -= sizes[first] + first += 1 + return kept[first:], bad + + +def _next_id(saved, entries: list[dict]) -> int: + """The id the next record gets: the saved next id (if it is an int that fits), past every record's.""" + return max([saved if type(saved) is int and saved <= MAX_ID else 1] + [e["id"] + 1 for e in entries]) + + +def _import_log(box: str, value, tx, limit: int, max_bytes: int | None, valid, file: str) -> int: + """Import a message log file of rnfs 0.1.4 (statedb.migrate): its records replace box's, checked and bounded + as it loaded them, and it is clean only if every record was read. Returns the number of records imported.""" + from rnfs import statedb + tx.execute("DELETE FROM lxmf_messages WHERE box = ?", (box,)) + entries, clean, next_id = [], False, 1 + if isinstance(value, dict): + messages = value.get("messages") + entries, bad = _records(messages if isinstance(messages, list) else [], limit, max_bytes, valid) + bad += not isinstance(messages, list) + if bad: + _log(f"dropped {bad} malformed record(s) from {file}", LOG_WARNING) + clean = not bad + next_id = _next_id(value.get("next_id"), entries) + tx.executemany("INSERT INTO lxmf_messages (box, id, record) VALUES (?, ?, ?)", + [(box, entry["id"], statedb.dump(entry)) for entry in entries]) + statedb.set_meta(tx, box, "next_id", next_id) + statedb.set_meta(tx, box, "clean", int(clean)) + return len(entries) + + +def import_inbox(value, tx) -> int: + """Import lxmf_inbox.json (statedb.migrate); the number of records imported.""" + return _import_log(INBOX, value, tx, INBOX_LIMIT, MAX_INBOX_BYTES, _valid_inbox_entry, "lxmf_inbox.json") + + +def import_outbox(value, tx) -> int: + """Import lxmf_outbox.json (statedb.migrate); the number of records imported.""" + return _import_log(OUTBOX, value, tx, OUTBOX_LIMIT, None, _valid_outbox_entry, "lxmf_outbox.json") class MessageLog: - """A bounded list of message records with stable ids, saved atomically as UTF-8 JSON. Thread-safe. + """A bounded list of message records with stable ids, kept in the state database (rnfs.statedb) as one row + per record in its box: a save writes the records added, changed or removed since the last. Thread-safe. At most limit records and max_bytes of JSON are kept. To make room, the oldest record protect() is false for is dropped; a protected one only when every other is protected too. add() may also name records that - are dropped only for each other's (reserved). valid() checks records loaded from the file (malformed ones - are dropped; clean says whether the file was there and every record in it was read). + are dropped only for each other's (reserved). valid() checks records loaded (malformed ones are dropped; + clean says whether the box was saved whole, or imported from a whole file, and every record was read). """ - def __init__(self, path: str, limit: int, max_bytes: int | None = None, protect=None, valid=None): - """Load path if it exists (malformed records are dropped).""" - self.path = path + def __init__(self, state_dir: str, box: str, limit: int, max_bytes: int | None = None, protect=None, + valid=None): + """Load box's records from the state database in state_dir (malformed ones are dropped). Close it when + done.""" + from rnfs import statedb + self.box = box self.limit = limit self.max_bytes = max_bytes self.protect = protect or (lambda entry: False) self._lock = threading.Lock() self._save_lock = threading.Lock() # one save at a time, so an older snapshot never lands last - self._dirty = False + self._dirty: set[int] = set() # ids of records added, changed or removed since the last save self._next_id = 1 self._entries: list[dict] = [] self._sizes: dict[int, int] = {} # id -> bytes as JSON - from rnfs.content.storage import read_json - stored = read_json(path) # a damaged file is moved aside and gives None, as a missing one does - self.clean = False # the file was there and every record in it was read - if isinstance(stored, dict): - entries = stored.get("messages") - bad, seen = 0 if isinstance(entries, list) else 1, set() - for entry in entries if isinstance(entries, list) else []: - if isinstance(entry, dict) and type(entry.get("id")) is int and entry["id"] > 0 \ - and entry["id"] not in seen and (valid is None or valid(entry)): - self._entries.append(entry) - seen.add(entry["id"]) - else: - bad += 1 - if bad: - _log(f"dropped {bad} malformed record(s) from {path}", LOG_WARNING) - self.clean = not bad - self._entries = self._entries[-limit:] - self._sizes = {e["id"]: _json_bytes(e) for e in self._entries} - while self.max_bytes is not None and len(self._entries) > 1 and sum(self._sizes.values()) > max_bytes: - self._sizes.pop(self._entries.pop(0)["id"], None) - next_id = stored.get("next_id") - self._next_id = max([next_id if type(next_id) is int else 1] + [e["id"] + 1 for e in self._entries]) + self.clean = False # the box was saved whole and every record in it was read + self._db = statedb.connect(state_dir) + try: + self._load(valid) + except BaseException: + self._db.close() + raise + + def _load(self, valid): + """Read the box's records, oldest first, bounded as rnfs 0.1.4 read its file; rows dropped are deleted (and + the box isn't clean if one was malformed, until the next save).""" + from rnfs import statedb + rows = self._db.query("SELECT id, record FROM lxmf_messages WHERE box = ? ORDER BY id", (self.box,)) + loaded = [] + for row_id, text in rows: + try: + entry = statedb.load(text) + except (TypeError, ValueError): + entry = None + # A record whose id isn't its row's is malformed too + loaded.append(entry if isinstance(entry, dict) and entry.get("id") == row_id else None) + self._entries, bad = _records(loaded, self.limit, self.max_bytes, valid) + if bad: + _log(f"dropped {bad} malformed record(s) from the {self.box}", LOG_WARNING) + self.clean = self._db.meta(self.box, "clean") == 1 and not bad + self._sizes = {e["id"]: _json_bytes(e) for e in self._entries} + self._next_id = _next_id(self._db.meta(self.box, "next_id"), self._entries) + kept = {e["id"] for e in self._entries} + gone = [(self.box, row_id) for row_id, _ in rows if row_id not in kept] + if gone: + with self._db.transaction() as tx: + tx.executemany("DELETE FROM lxmf_messages WHERE box = ? AND id = ?", gone) + if bad: + statedb.set_meta(tx, self.box, "clean", 0) def add(self, entry: dict, per_from: int | None = None, reserved=None) -> tuple[int | None, list[dict]]: """Append entry with a new id; returns the id (None if entry itself was dropped: every other record is @@ -482,6 +554,9 @@ def add(self, entry: dict, per_from: int | None = None, reserved=None) -> tuple[ (trusted senders' records, so new identities can't push them out).""" reserved = reserved or (lambda record: False) with self._lock: + if self._next_id > MAX_ID: # only a hand-edited next id gets here: refused, so saves keep working + _log(f"the {self.box} has used every record id; refusing a new record", LOG_ERROR) + return None, [] entry["id"] = self._next_id self._next_id += 1 self._entries.append(entry) @@ -509,7 +584,7 @@ def add(self, entry: dict, per_from: int | None = None, reserved=None) -> tuple[ self._entries = [e for e in self._entries if id(e) not in gone] for e in dropped: self._sizes.pop(e["id"], None) - self._dirty = True + self._dirty.update(e["id"] for e in [entry, *dropped]) # a refused entry still took an id return (None if id(entry) in gone else entry["id"]), [e for e in dropped if e is not entry] def _over(self, dropped: list[dict]) -> bool: @@ -548,7 +623,7 @@ def update(self, entry_id: int, **values) -> bool: if entry["id"] == entry_id: entry.update(values) self._sizes[entry_id] = _json_bytes(entry) - self._dirty = True + self._dirty.add(entry_id) return True return False @@ -561,7 +636,7 @@ def remove(self, entry_id: int | None) -> list[dict]: self._entries = [e for e in self._entries if id(e) not in gone] for e in removed: self._sizes.pop(e["id"], None) - self._dirty = True + self._dirty.update(e["id"] for e in removed) return removed def entries(self) -> list[dict]: @@ -575,20 +650,32 @@ def __len__(self) -> int: return len(self._entries) def save(self): - """Write the log if it changed (atomic; raises OSError).""" + """Write the records added, changed or removed since the last save, if any, and the next id, in one + transaction; the box is then clean. Raises OSError (they are written by the next save).""" + from rnfs import statedb with self._save_lock: with self._lock: if not self._dirty: return - snapshot = {"next_id": self._next_id, "messages": [dict(e) for e in self._entries]} - self._dirty = False + dirty, self._dirty = self._dirty, set() + rows = [(self.box, e["id"], statedb.dump(e)) for e in self._entries if e["id"] in dirty] + gone = [(self.box, entry_id) for entry_id in dirty.difference(e["id"] for e in self._entries)] + next_id = self._next_id try: - _write_json(self.path, snapshot) - except OSError: + with self._db.transaction() as tx: + tx.executemany("DELETE FROM lxmf_messages WHERE box = ? AND id = ?", gone) + tx.executemany("INSERT OR REPLACE INTO lxmf_messages (box, id, record) VALUES (?, ?, ?)", rows) + statedb.set_meta(tx, self.box, "next_id", next_id) + statedb.set_meta(tx, self.box, "clean", 1) + except BaseException: with self._lock: - self._dirty = True + self._dirty |= dirty raise + def close(self): + """Release the state database.""" + self._db.close() + def _str_or_none(value) -> bool: """Whether value is a string or None.""" @@ -739,10 +826,13 @@ def __init__(self, node): self.off_reason = "LXMF has not been started" self.accept_from = {parse_address(a).hex() for a in config.lxmf_accept_from} self.auto_fetch_from = {parse_address(a).hex() for a in config.lxmf_auto_fetch_from} - self.inbox = MessageLog(os.path.join(config.state_dir, "lxmf_inbox.json"), INBOX_LIMIT, MAX_INBOX_BYTES, - protect=_holds_pins, valid=_valid_inbox_entry) - self.outbox = MessageLog(os.path.join(config.state_dir, "lxmf_outbox.json"), OUTBOX_LIMIT, - valid=_valid_outbox_entry) + self.inbox = MessageLog(config.state_dir, INBOX, INBOX_LIMIT, MAX_INBOX_BYTES, protect=_holds_pins, + valid=_valid_inbox_entry) + try: + self.outbox = MessageLog(config.state_dir, OUTBOX, OUTBOX_LIMIT, valid=_valid_outbox_entry) + except BaseException: + self.inbox.close() + raise for entry in self.outbox.entries(): if entry.get("state") not in FINAL_STATES and not self._propagated_sent(entry): # LXMF doesn't persist its outbound queue @@ -767,9 +857,10 @@ def _release_unrecorded(self): few seconds, so a crash can leave a pin whose message is gone or still queued; nothing else would release it (`pin rm` releases only the user's pins). A queued link is fetched and pinned again at start(). - Only if the inbox file was read whole: when it is missing, damaged (moved aside) or had a malformed record, - such a pin may be one a lost message held. Then each sender's unrecorded pins are kept and recorded in a - new inbox message, so they are kept at later starts too and removing that message releases them.""" + Only if the saved inbox was read whole (clean): when it was never saved, its file was missing or damaged + (moved aside) when imported, or it had a malformed record, such a pin may be one a lost message held. + Then each sender's unrecorded pins are kept and recorded in a new inbox message, so they are kept at + later starts too and removing that message releases them.""" from rnfs.content.storage import LXMF_PREFIX recorded = {(entry.get("from"), result.get("cid")) for entry in self.inbox.entries() for result in entry.get("auto") or [] if isinstance(result, dict) and result.get("state") == "pinned"} @@ -798,12 +889,12 @@ def _record_kept(self, sender: str, cids: list[Cid]): if _HEX32.fullmatch(sender) is None: # not a source this feature makes: kept, never recorded _log(f"keeping {listed}, pinned for {sender!r}, which isn't an LXMF address", LOG_WARNING) return - _log(f"the inbox file is missing, damaged or incomplete: keeping {listed}, pinned for a message from " + _log(f"the saved inbox is missing, damaged or incomplete: keeping {listed}, pinned for a message from " f"{sender} it no longer has, in a new inbox message (remove it to release them)", LOG_WARNING) links = [Link(cid) for cid in cids] entry = {"hash": "", "from": sender, "display_name": None, "sent": None, "received": time.time(), "title": "", "method": "unknown", "links": [link.to_json() for link in links], - "text": "Restored by rnfs at start: the inbox file was missing or damaged, and these links were " + "text": "Restored by rnfs at start: the saved inbox was missing or damaged, and these links were " "pinned for a message from this sender that it no longer has. Remove this message to " "release them.", "auto": [{"state": "pinned", "cid": str(cid)} for cid in cids]} @@ -871,6 +962,11 @@ def save(self): self.inbox.save() self.outbox.save() + def close(self): + """Release the state database.""" + self.inbox.close() + self.outbox.close() + def status(self) -> dict: """The LXMF address (None when off) and the number of inbox messages.""" return {"lxmf_address": self.network.address.hex() if self.network is not None else None, @@ -1192,7 +1288,7 @@ def _resolve(self, link: Link, persist: bool = True) -> tuple[Cid, list[str]]: """The root CID and path a link names, resolving a name (falling back to the CID it was sent with). persist False (auto-fetch): the name's record goes only into the bounded table, so names arriving - unasked don't grow names.json; its sequence still goes into the bounded, saved floor, so a later + unasked don't grow the held records; its sequence still goes into the bounded, saved floor, so a later auto-fetch of the name never resolves to an older record.""" if link.name is None: return link.cid, list(link.path) diff --git a/rnfs/fetcher.py b/rnfs/fetcher.py index 57b7123..3ad8805 100644 --- a/rnfs/fetcher.py +++ b/rnfs/fetcher.py @@ -58,7 +58,6 @@ Within `with observing(observer):`, every fetch the thread opens reports its total size and each block it meets to the Observer, and raises Cancelled once observer.cancelled() is true (background jobs use this). """ -import os import queue import threading import time @@ -67,12 +66,13 @@ from itertools import islice from typing import Callable, Iterator +from rnfs import statedb from rnfs.config import Config from rnfs.constants import (CHUNK_SIZE, KIB, MAX_BLOCK_SIZE, MAX_CHILDREN, MAX_DEPTH, MAX_REQUEST_SIZE, MAX_RESPONSE_SIZE, RESPONSE_OVERHEAD) from rnfs.content import walker from rnfs.content.cid import Cid -from rnfs.content.storage import Storage, read_json, write_json +from rnfs.content.storage import Storage from rnfs.content.walker import Expect from rnfs.errors import InvalidDag, NotFound, RnfsError, TooLarge from rnfs.log import LOG_VERBOSE, LOG_WARNING, log, node_name, plural @@ -86,8 +86,23 @@ POLL = 0.5 # seconds between checks while waiting for responses DAG_NODE_GUESS = 8 * KIB # assumed size of a DAG node block when reserving a request's batch MAX_BATCH = 32 # CIDs reserved for one request at most; the client sends as many as fit one packet -INVALID_FILE = "invalid.json" MAX_INVALID = 10_000 # roots remembered as invalid; past this the oldest is forgotten + + +def import_invalid(value: list, tx) -> int: + """Import invalid.json (statedb.migrate): its roots, oldest first, replace the invalid table; one that doesn't + parse is skipped, as rnfs 0.1.4 did, and only the newest MAX_INVALID are kept. Returns the number imported.""" + tx.execute("DELETE FROM invalid") + roots: OrderedDict[Cid, None] = OrderedDict() + for text in value: + try: + roots[Cid.parse(text)] = None + except (ValueError, TypeError): + continue + kept = list(roots)[-MAX_INVALID:] + tx.executemany("INSERT INTO invalid (cid, position) VALUES (?, ?)", + [(cid.binary, position) for position, cid in enumerate(kept)]) + return len(kept) # The DAG byte bound (dag_bound and fetch_dag; design.md, "DAG byte bound"). Writers cut chunks of CHUNK_SIZE, # or with FastCDC of over 16 KiB, so a file has at most one chunk per 16 KiB of content beyond its last; each # costs at most 51 bytes of file node (a [cid, size] child), plus about 1/255 of that for the levels above @@ -256,17 +271,23 @@ def __init__(self, storage: Storage, client, config: Config, own_id: bytes, rout # RNS's (mesh.RnsPaths.info), and without it nothing is known and ranking falls back to reply order self.path_info: Callable[[bytes], mesh.PathInfo] = mesh.no_path_info self._lock = threading.Lock() - self._invalid_path = None if state_dir is None else os.path.join(state_dir, INVALID_FILE) + self._db = None if state_dir is None else statedb.connect(state_dir) self._invalid: OrderedDict[Cid, None] = OrderedDict() # oldest first - self._invalid_dirty = False - if self._invalid_path is not None: - for text in read_json(self._invalid_path) or []: + self._invalid_saved: set = set() # the binary CIDs the invalid table holds + self._invalid_position = 0 # the next root's position in the table: after every one saved + if self._db is not None: + for binary, position in self._db.query("SELECT cid, position FROM invalid ORDER BY position"): + self._invalid_saved.add(binary) + if isinstance(position, int): + self._invalid_position = max(self._invalid_position, position + 1) try: - self._invalid[Cid.parse(text)] = None - except (ValueError, TypeError): + self._invalid[Cid.from_bytes(binary)] = None + except ValueError: continue while len(self._invalid) > MAX_INVALID: self._invalid.popitem(last=False) + # Changed since the last save; a row that didn't parse, or past MAX_INVALID, is deleted by the next + self._invalid_dirty = len(self._invalid) != len(self._invalid_saved) @property def max_parallel(self) -> int: @@ -303,16 +324,30 @@ def mark_invalid(self, root: Cid): log(f"marked {root} invalid locally", LOG_WARNING) def save(self): - """Persist the invalid roots if they changed (the node calls this periodically and at shutdown).""" - if self._invalid_path is None: + """Persist the invalid roots added or forgotten since the last save, if any (the node calls this + periodically and at shutdown).""" + if self._db is None: return with self._lock: if not self._invalid_dirty: return - # Written under the lock, so saves can't overtake each other; it is small and rarely changes - write_json(self._invalid_path, [str(cid) for cid in self._invalid]) + # Written under the lock, so saves can't overtake each other; it rarely changes + current = [cid.binary for cid in self._invalid] + added = [binary for binary in current if binary not in self._invalid_saved] + gone = self._invalid_saved.difference(current) + with self._db.transaction() as tx: + tx.executemany("DELETE FROM invalid WHERE cid = ?", [(binary,) for binary in gone]) + tx.executemany("INSERT OR REPLACE INTO invalid (cid, position) VALUES (?, ?)", + [(binary, self._invalid_position + n) for n, binary in enumerate(added)]) + self._invalid_position += len(added) + self._invalid_saved = set(current) self._invalid_dirty = False + def close(self): + """Release the state database (the node calls this once it has saved everything at shutdown).""" + if self._db is not None: + self._db.close() + class Fetch: """One fetch of a target: its protection session, providers, lookup and the blocks it has stored.""" diff --git a/rnfs/log.py b/rnfs/log.py index d2ae875..8aa9fe3 100644 --- a/rnfs/log.py +++ b/rnfs/log.py @@ -8,7 +8,7 @@ from rnfs.fsutil import printable -# Warnings a result keeps (a write-out skipping 50,000 entries mustn't make a 16 MB response or jobs.json) +# Warnings a result keeps (a write-out skipping 50,000 entries mustn't make a 16 MB response or saved job) MAX_WARNINGS = 100 # Re-exported so callers only need this module for logging diff --git a/rnfs/names.py b/rnfs/names.py index 2ed3788..f35e8df 100644 --- a/rnfs/names.py +++ b/rnfs/names.py @@ -9,6 +9,13 @@ within seconds of a resolve raising it, where names resolved in the last FLOOR_KEEP_AGE are forgotten only past a hard bound. +State lives in the state database (rnfs.statedb), read only at startup. The records of names published here and +of names resolved for the user (names_published, names_held) are saved at once and durably, each save writing only +the rows that changed; that save is a publish's commit point. A malformed one stops startup: starting without it +would allow rollback. Floor saves (name_floor) write only the entries changed too, with synchronous=NORMAL, so a +power loss may lose the last of them; a malformed floor entry is dropped. The names.json and name-floor.json of +rnfs 0.1.4 and earlier are imported once (import_names, import_floor). + Every RNS call goes through `RnsNetwork`, so tests swap in a fake and run without Reticulum. """ import json @@ -23,11 +30,10 @@ import RNS from RNS.vendor import umsgpack -from rnfs import fsutil +from rnfs import fsutil, statedb from rnfs.config import _HEX_ID, Config # _HEX_ID: exactly 32 hex digits, as identity lists are checked from rnfs.constants import APP_NAME, ID_LENGTH, NAME_ASPECT from rnfs.content.cid import Cid -from rnfs.content.storage import read_json from rnfs.errors import NotFound, RnfsError from rnfs.log import LOG_ERROR, LOG_NOTICE, LOG_VERBOSE, LOG_WARNING, log @@ -37,9 +43,8 @@ # A record is at most ~52 bytes (array header, uint64, bin header, 39-byte CID); anything longer is # rejected before decoding, which keeps the announce handler cheap under a flood MAX_RECORD_SIZE = 64 -STATE_FILE = "names.json" -STATE_VERSION = 1 -FLOOR_FILE = "name-floor.json" +STATE_FILE = "names.json" # the published and held records of rnfs 0.1.4 and earlier (import_names) +FLOOR_FILE = "name-floor.json" # the floor of rnfs 0.1.4 and earlier (import_floor) FLOOR_VERSION = 1 FLOOR_FACTOR = 4 # the floor holds up to this many times name_table_size names, so it outlasts table churn ... FLOOR_KEEP_AGE = 7 * 24 * 3600.0 # ... except that names resolved this recently (seconds) are kept past that ... @@ -48,6 +53,8 @@ FLOOR_TOUCH_SAVE = 3600.0 # a resolve that only renews a floor entry's time marks it for the next save after this PIN_SOURCE_PREFIX = "name:" _ALIAS = re.compile(r"[A-Za-z0-9][A-Za-z0-9._-]{0,63}") +_DECIMAL = re.compile(r"-?[0-9]+") # a sequence column (decimal text: sequences don't fit SQLite's integers) +_MAX_USED = 2**62 # floor rows' used values stay below this, so the ones after them still fit SQLite's integers _POLL_INTERVAL = 0.25 # how often a refresh checks whether RNS has heard a path response _MAX_LOOP_WAIT = 60 # the announce loop wakes at least this often, so new schedules are picked up @@ -138,11 +145,137 @@ class _Published: sequence: int cid: Cid | None - # A key imported from elsewhere (or found with no state: names.json was lost) may already have records on - # the network and at keepers, so its first publish asks both before choosing a sequence + # A key imported from elsewhere (or found with no saved record: the state was lost) may already have records + # on the network and at keepers, so its first publish asks both before choosing a sequence imported: bool = False +# State + + +def _sequence(text) -> int: + """A sequence from its decimal text column. Raises ValueError (TypeError if it isn't text).""" + if not isinstance(text, str): + raise TypeError(f"sequence {text!r} isn't text") + if not _DECIMAL.fullmatch(text): + raise ValueError(f"sequence {text!r} isn't decimal") + return int(text) + + +def _published_row(alias, sequence, cid, imported) -> _Published: + """A names_published row as what it records. Raises ValueError or TypeError if it is malformed.""" + if not isinstance(alias, str) or not _ALIAS.fullmatch(alias): + raise ValueError(f"invalid alias {alias!r}") + if type(imported) is not int or imported not in (0, 1): + raise ValueError(f"invalid imported flag {imported!r} of {alias}") + return _Published(_sequence(sequence), None if cid is None else Cid.from_bytes(cid), bool(imported)) + + +def _held_row(name, sequence, cid, heard) -> tuple[bytes, Record]: + """A names_held row as (name, record). Raises ValueError or TypeError if it is malformed.""" + name = _check_name(name) + if type(heard) not in (int, float): + raise TypeError(f"heard time {heard!r} of {name.hex()} isn't a number") + return name, Record(_sequence(sequence), Cid.from_bytes(cid), float(heard)) + + +def read_names(path: str, shape=None): + """names.json's value, for statedb.migrate (shape is unused), or None if there is none. Raises RnfsError if it + can't be read or isn't JSON, leaving it in place: starting without it would allow rollback.""" + try: + with open(path, "rb") as f: + value = json.load(f) + except FileNotFoundError: + return None + except (OSError, ValueError) as e: + raise RnfsError(f"could not read {path}: {e}") from None + if not isinstance(value, dict): # null too, which rnfs 0.1.4 refused as malformed + raise RnfsError(f"malformed {path}: not an object") + return value + + +def import_names(value, tx) -> int: + """Import names.json (statedb.migrate): its published and held records replace the names_published and + names_held rows, every one checked as rnfs 0.1.4 loaded it, whether or not its key file is there. A malformed + record raises RnfsError, which rolls the import back and stops startup, as it did then; an alias no key file + can have is skipped with a warning. Returns the number of records imported.""" + published, held = [], {} + try: + for alias, entry in value.get("published", {}).items(): + cid = Cid.parse(entry["cid"]) if entry["cid"] is not None else None + row = (alias, str(int(entry["sequence"])), None if cid is None else cid.binary, + int(bool(entry.get("imported", False)))) + if not _ALIAS.fullmatch(alias): + # Never a key's (_load_keys reads only files so named); reconcile_pins releases any pin it has + log(f"skipping the record of {alias!r} in {STATE_FILE}: not a valid alias", LOG_WARNING) + continue + published.append(row) + for name_hex, entry in value.get("held", {}).items(): + heard = float(entry["heard"]) + held[parse_name(name_hex)] = (str(int(entry["sequence"])), Cid.parse(entry["cid"]).binary, + 0.0 if math.isnan(heard) else heard) # SQLite has no NaN: heard long ago + except (AttributeError, KeyError, OverflowError, TypeError, ValueError) as e: + raise RnfsError(f"malformed {STATE_FILE}: {e}") from None + tx.execute("DELETE FROM names_published") + tx.execute("DELETE FROM names_held") + tx.executemany("INSERT INTO names_published (alias, sequence, cid, imported) VALUES (?, ?, ?, ?)", published) + tx.executemany("INSERT INTO names_held (name, sequence, cid, heard) VALUES (?, ?, ?, ?)", + [(name, *row) for name, row in held.items()]) + return len(published) + len(held) + + +def _floor_shape(value) -> bool: + """True if value is a name-floor.json of FLOOR_VERSION: {"version", "floor": [entry, ...]}.""" + return isinstance(value, dict) and value.get("version") == FLOOR_VERSION and isinstance(value.get("floor"), list) + + +def read_floor(path: str, shape=None): + """name-floor.json's value, for statedb.migrate (shape is unused), or None if there is none. A damaged file is + moved aside and gives None: the node starts without it, since it only bounds rollback of gateway and + auto-fetch resolves. Raises OSError if it can't be read.""" + return statedb.read_json(path, _floor_shape) + + +def _resolve_time(rest: list) -> float: + """The resolve time a name-floor.json entry ends with: 0.0 (long ago) unless it is one finite number, as + entries written before resolve times were kept lack it.""" + if len(rest) != 1 or type(rest[0]) not in (int, float): + return 0.0 + try: + at = float(rest[0]) + except OverflowError: # an integer past float's range + return 0.0 + return at if math.isfinite(at) else 0.0 + + +def import_floor(value: dict, tx) -> int: + """Import name-floor.json (statedb.migrate): its [name, sequence, resolved] entries, least recently resolved + first, replace the name_floor rows, each checked as rnfs 0.1.4 loaded it: one that doesn't parse is skipped + (and counted in a warning), one without a valid resolve time counts as resolved long ago, and a name listed + twice keeps the higher sequence, at its later place. Held names, future times and the floor's bound are left + to the load (_load_floor). Returns the number of entries imported.""" + floor: OrderedDict[bytes, tuple[int, float]] = OrderedDict() + skipped = 0 + for entry in value["floor"]: + try: + name_hex, sequence, *rest = entry + name = parse_name(name_hex) + except (TypeError, ValueError): + skipped += 1 + continue + if type(sequence) is not int or not 0 <= sequence <= MAX_SEQUENCE: + skipped += 1 + continue + earlier = floor.pop(name, None) + floor[name] = (max(sequence, earlier[0] if earlier else 0), _resolve_time(rest)) + if skipped: + log(f"skipped {skipped} malformed entr{'y' if skipped == 1 else 'ies'} in {FLOOR_FILE}", LOG_WARNING) + tx.execute("DELETE FROM name_floor") + tx.executemany("INSERT INTO name_floor (name, sequence, resolved, used) VALUES (?, ?, ?, ?)", + [(name, str(sequence), at, used) for used, (name, (sequence, at)) in enumerate(floor.items())]) + return len(floor) + + # RNS access @@ -218,7 +351,8 @@ class NameManager: """This node's name keys and records, and the records it has heard. Thread-safe. Lifecycle: construct (loads keys and state, no RNS needed), reconcile_pins() once the node and its features - are set up, start() once Reticulum runs, run(stop_event) on a thread for re-announces, stop() at shutdown. + are set up, start() once Reticulum runs, run(stop_event) on a thread for re-announces, stop() and close() at + shutdown. """ def __init__(self, config: Config, pins, network=None, clock=time.time, monotonic=time.monotonic): @@ -229,7 +363,6 @@ def __init__(self, config: Config, pins, network=None, clock=time.time, monotoni self._clock = clock # wall clock: heard times are persisted across restarts self._monotonic = monotonic # announce schedule self._lock = threading.RLock() - self._state_path = os.path.join(config.state_dir, STATE_FILE) self._keys: dict[str, RNS.Identity] = {} self._names: dict[str, bytes] = {} # alias -> name @@ -238,6 +371,10 @@ def __init__(self, config: Config, pins, network=None, clock=time.time, monotoni # Highest records of names published here or resolved through the CLI or API: persisted, unbounded # (they only grow by the user's own actions) self._held: dict[bytes, Record] = {} + # What the names_published rows (but those of _unreadable aliases) and the names_held rows record, so a + # save writes only the rows that changed + self._saved_published: dict[str, _Published] = {} + self._saved_held: dict[bytes, Record] = {} # Every other name's record: in memory, least recently used first, at most name_table_size self._table: OrderedDict[bytes, Record] = OrderedDict() # The table's names that were only heard, never resolved here, least recently used first. An announce of a @@ -249,8 +386,10 @@ def __init__(self, config: Config, pins, network=None, clock=time.time, monotoni # FLOOR_HARD_FACTOR times that) while those resolved within FLOOR_KEEP_AGE need the room. Saved, so such a # resolve never goes back after table churn or a restart self._floor: OrderedDict[bytes, tuple[int, float]] = OrderedDict() - self._floor_path = os.path.join(config.state_dir, FLOOR_FILE) - self._floor_dirty = False + self._floor_used: dict[bytes, int] = {} # each floor name's used value, its place in the name_floor table + self._floor_next_used = 0 # above every used value given out + self._floor_changed: set[bytes] = set() # floor names whose row the next save writes (or deletes) + self._floor_dirty = False # a floor save is due self._floor_saved_at = float("-inf") # monotonic time of the last floor save self._floor_timer: threading.Timer | None = None # a save waiting for FLOOR_SAVE_INTERVAL to pass # Floor saves write without self._lock, one at a time, so an older snapshot never lands last. Taken before @@ -269,20 +408,25 @@ def __init__(self, config: Config, pins, network=None, clock=time.time, monotoni # Pins of names whose key file was deleted by hand, released by release_gone() once the node is set up self._gone: list[tuple[str, Cid]] = [] # Aliases whose key file is there but couldn't be read (permissions, damage): not names this node can - # use, but not gone either, so their pins stay and their names.json entry (or None) is written back as - # it was, for when the file can be read again - self._unreadable: dict[str, dict | None] = {} - # Aliases with an entry in names.json: for these the recorded target (or none) is the only CID their pin - # source may hold (reconcile_pins). A key without one (names.json lost) may have pins nothing records + # use, but not gone either, so their pins stay and saves leave their names_published row (if any) as it + # is, for when the file can be read again + self._unreadable: set[str] = set() + # Aliases with a saved record: for these the recorded target (or none) is the only CID their pin source + # may hold (reconcile_pins). A key without one (the state was lost) may have pins nothing records self._recorded: set[str] = set() # alias -> CIDs its pin source holds that its record doesn't (left by a crash, or an unpin that failed): # released by its next publish or removal self._stale: dict[str, set[Cid]] = {} self._remove_old_temps() - self._load_keys() - self._load_state() - self._load_floor() + self._db = statedb.connect(config.state_dir) + try: + self._load_keys() + self._load_state() + self._load_floor() + except BaseException: + self._db.close() + raise # Keys @@ -360,14 +504,14 @@ def release_gone(self): self._save() # its entry is dropped only once its pin is released def reconcile_pins(self, pins: dict[Cid, set[str]]): - """Make the name pins (pins: every pin and its sources, as storage.pins() gives them) match names.json, - once the node is set up: release_gone(), then pin each recorded target that lacks its name's pin and - release name pins nothing records (a crash inside publish() can leave one, and earlier rnfs versions + """Make the name pins (pins: every pin and its sources, as storage.pins() gives them) match the saved + records, once the node is set up: release_gone(), then pin each recorded target that lacks its name's pin + and release name pins nothing records (a crash inside publish() can leave one, and earlier rnfs versions could leave one that `pin rm` and `name rm` couldn't remove). - A key with no names.json entry (the file was lost) keeps its pins until its next publish or removal, - which release them; a pin of an alias with no key file at all is released, but one whose key file is - there and can't be read is kept. + A key with no saved record (the state was lost) keeps its pins until its next publish or removal, which + release them; a pin of an alias with no key file at all is released, but one whose key file is there and + can't be read is kept. """ self.release_gone() with self._lock: @@ -389,19 +533,20 @@ def reconcile_pins(self, pins: dict[Cid, set[str]]): recorded = self._published[alias].cid if alias not in self._recorded: if cids: - log(f"name {alias} has no record in {STATE_FILE}; its pins are released at its next " - "publish", LOG_WARNING) + log(f"name {alias} has no saved record; its pins are released at its next publish", + LOG_WARNING) self._stale.setdefault(alias, set()).update(cids) continue if recorded is not None and recorded not in cids: - log(f"pinning {recorded} again for name {alias}, as {STATE_FILE} records", LOG_WARNING) + log(f"pinning {recorded} again for name {alias}, as its saved record says", LOG_WARNING) try: self._pins.pin(recorded, PIN_SOURCE_PREFIX + alias) except OSError as e: # e.g. a read-only state directory: start all the same log(f"could not pin {recorded} for name {alias}: {e}", LOG_ERROR) extra = cids - {recorded} if extra: - log(f"releasing {len(extra)} pin(s) of name {alias} that {STATE_FILE} doesn't record", LOG_WARNING) + log(f"releasing {len(extra)} pin(s) of name {alias} that its saved record doesn't hold", + LOG_WARNING) self._release(alias, extra) def _release(self, alias: str, cids): @@ -459,7 +604,7 @@ def publish(self, alias: str, cid: Cid) -> dict: raise RnfsError(f"name {alias} has reached the highest sequence") source = PIN_SOURCE_PREFIX + alias - # Crash-consistent: pin the new target, then save names.json (the commit point), then release the old + # Crash-consistent: pin the new target, then save the records (the commit point), then release the old # target. A crash leaves at most an extra pin of this source, which reconcile_pins() releases at the # next start, and the recorded target is pinned at every step (blocks they share stay protected) self._pins.pin(cid, source) @@ -469,7 +614,7 @@ def publish(self, alias: str, cid: Cid) -> dict: try: self._save() except Exception: - # Not published: back to what names.json still says (ENOSPC, EIO) + # Not published: back to what the database still says (ENOSPC, EIO) self._published[alias] = previous if held_before is None: self._held.pop(name, None) @@ -522,6 +667,14 @@ def stop(self): self._save() self._save_floor() + def close(self): + """Release the state database, once stop() has saved everything (idempotent).""" + with self._lock: + if self._floor_timer is not None: + self._floor_timer.cancel() + self._floor_timer = None + self._db.close() + def announce_due(self) -> float: """Re-announce every name whose announce_interval has passed; seconds until the next is due.""" with self._lock: @@ -761,6 +914,7 @@ def _store(self, name: bytes, record: Record, held_here: bool, unasked: bool = F floor = self._floor.get(name) if floor is not None and record.sequence > floor[0]: self._floor[name] = (record.sequence, floor[1]) + self._floor_changed.add(name) self._floor_dirty = True if len(self._table) > limit: # name_table_size 0: nothing is kept self._drop_from_table(name) @@ -786,6 +940,9 @@ def _resolved(self, name: bytes, sequence: int) -> bool: now = self._clock() previous = self._floor.pop(name, None) self._floor[name] = (sequence if previous is None else max(previous[0], sequence), now) + self._floor_used[name] = self._floor_next_used # last: the most recently resolved + self._floor_next_used += 1 + self._floor_changed.add(name) # written by the next save, even if this change doesn't call for one trimmed = self._trim_floor() if previous is None or self._floor[name][0] != previous[0] or trimmed: self._floor_dirty = True @@ -804,7 +961,9 @@ def _trim_floor(self) -> bool: _, at = next(iter(self._floor.values())) if len(self._floor) <= FLOOR_HARD_FACTOR * size and now - at < FLOOR_KEEP_AGE: break - self._floor.popitem(last=False) + name, _ = self._floor.popitem(last=False) + del self._floor_used[name] + self._floor_changed.add(name) trimmed = True return trimmed @@ -906,7 +1065,7 @@ def _remove_old_temps(self): entries = [] old = [os.path.join(self.config.names_dir, entry) for entry in entries if entry.startswith(".") and entry.endswith(".tmp") and _ALIAS.fullmatch(entry[1:-4])] - for path in (self._state_path + ".tmp", *old): + for path in (os.path.join(self.config.state_dir, STATE_FILE + ".tmp"), *old): try: fsutil.remove(path) except OSError: @@ -925,7 +1084,7 @@ def _load_keys(self): identity = RNS.Identity.from_file(path) if os.path.isfile(path) else None # None: a dangling symlink if identity is None: log(f"could not read name key {path}; skipping it, and keeping its pins and records", LOG_ERROR) - self._unreadable[alias] = None + self._unreadable.add(alias) continue name = name_for_identity(identity) if name in self._aliases: @@ -938,102 +1097,125 @@ def _load_keys(self): # State; callers hold the lock def _load_state(self): - """Read persisted records. A corrupt file is an error: starting afresh would allow rollback.""" - try: - with open(self._state_path, "rb") as f: - state = json.load(f) - except FileNotFoundError: - state = {} - except (OSError, ValueError) as e: - raise RnfsError(f"could not read {self._state_path}: {e}") from None + """Read the published and held records. A malformed row is an error, as a malformed names.json was: + starting without it would allow rollback.""" try: - for alias, entry in state.get("published", {}).items(): - cid = Cid.parse(entry["cid"]) if entry["cid"] is not None else None - published = _Published(int(entry["sequence"]), cid, bool(entry.get("imported", False))) + for row in self._db.query("SELECT alias, sequence, cid, imported FROM names_published ORDER BY alias"): + alias, published = row[0], _published_row(*row) + if alias in self._unreadable: + continue # left as it is by every save, for when the key file can be read again + self._saved_published[alias] = published if alias in self._keys: self._published[alias] = published self._recorded.add(alias) - elif alias in self._unreadable: - self._unreadable[alias] = entry # written back unchanged by _save - elif cid is not None: + elif published.cid is not None: # Its key file was deleted by hand: nothing will republish it, so release its pin - self._gone.append((alias, cid)) - for name_hex, entry in state.get("held", {}).items(): - self._held[parse_name(name_hex)] = Record(int(entry["sequence"]), Cid.parse(entry["cid"]), - float(entry["heard"])) - except (AttributeError, KeyError, TypeError, ValueError) as e: - raise RnfsError(f"malformed {self._state_path}: {e}") from None + self._gone.append((alias, published.cid)) + for row in self._db.query("SELECT name, sequence, cid, heard FROM names_held ORDER BY name"): + name, record = _held_row(*row) + self._held[name] = record + except (TypeError, ValueError) as e: + raise RnfsError(f"malformed name record in {self._db.path} ({e}); rnfs won't start on it, since starting " + "without it would let a name go back to an older record") from None + self._saved_held = dict(self._held) for alias in self._keys: - # A key with no state entry (names.json lost, or the key copied in by hand) may have been published - # elsewhere: treated as imported, so its first publish looks for its records first + # A key with no saved record (the state was lost, or the key copied in by hand) may have been + # published elsewhere: treated as imported, so its first publish looks for its records first self._published.setdefault(alias, _Published(0, None, imported=True)) def _load_floor(self): - """Read the floor of names resolved without persist. A damaged file is moved aside and the node starts - without it (it only bounds rollback of gateway and auto-fetch resolves); entries that don't parse are - skipped. An entry without a valid resolve time (written before they were kept) counts as resolved long - ago, and one in the future as now.""" - state = read_json(self._floor_path, shape=lambda value: isinstance(value, dict) - and value.get("version") == FLOOR_VERSION and isinstance(value.get("floor"), list)) - if state is None: - return + """Read the floor of names resolved without persist, least recently resolved first. A row that doesn't + parse is logged and deleted (the floor only bounds rollback of gateway and auto-fetch resolves), as are + those of names held for the user and those _trim_floor drops. A resolve time in the future (the clock was + corrected since) counts as now, and the next save writes it so.""" now = self._clock() - for entry in state["floor"]: + dropped = [] + for name, sequence, resolved, used in self._db.query( + "SELECT name, sequence, resolved, used FROM name_floor ORDER BY used"): try: - name_hex, sequence, *at = entry - name = parse_name(name_hex) - except (TypeError, ValueError): + _check_name(name) + sequence = _sequence(sequence) + if not 0 <= sequence <= MAX_SEQUENCE: + raise ValueError(f"sequence {sequence} out of range") + if type(resolved) not in (int, float) or not math.isfinite(resolved): + raise ValueError(f"invalid resolve time {resolved!r}") + if type(used) is not int or not 0 <= used < _MAX_USED: + raise ValueError(f"invalid used value {used!r}") + except (TypeError, ValueError) as e: + log(f"dropping an invalid name floor entry {name!r}: {e}", LOG_WARNING) + dropped.append(name) + continue + if name in self._held: + dropped.append(name) # resolved for the user since: its held record is its floor continue - at = at[0] if len(at) == 1 and type(at[0]) in (int, float) and math.isfinite(at[0]) else 0.0 - if type(sequence) is int and 0 <= sequence <= MAX_SEQUENCE and name not in self._held: - earlier = self._floor.pop(name, None) # listed twice (a damaged file): the higher sequence holds - self._floor[name] = (max(sequence, earlier[0] if earlier else 0), min(float(at), now)) + self._floor[name] = (sequence, min(float(resolved), now)) + self._floor_used[name] = used + self._floor_next_used = max(self._floor_next_used, used + 1) + if resolved > now: + self._floor_changed.add(name) self._trim_floor() + trimmed = [name for name in self._floor_changed if name not in self._floor] + self._floor_changed.difference_update(trimmed) + if dropped or trimmed: + with self._db.transaction() as tx: + tx.executemany("DELETE FROM name_floor WHERE name = ?", [(name,) for name in dropped + trimmed]) def _save_floor(self): - """Write the floor atomically, least recently resolved first, as [name, sequence, resolved], if it - changed. Saved by a resolve that raised it (at most every FLOOR_SAVE_INTERVAL, since it holds thousands - of names), announce_due (at least every minute while running) and stop(). + """Write the floor entries changed (or forgotten) since the last save, if a save is due, with + synchronous=NORMAL. Saved by a resolve that raised it (at most every FLOOR_SAVE_INTERVAL, since it holds + thousands of names), announce_due (at least every minute while running) and stop(). - Called without the lock: only the copy is taken under it, so resolves and announces needn't wait for - the write. The save counts from when the copy was taken; a failed one is retried by the next.""" + Called without the lock: only the rows are made under it, so resolves and announces needn't wait for + the write. The save counts from when they were made; a failed one is retried by the next.""" with self._floor_save_lock: with self._lock: if not self._floor_dirty: return - floor = list(self._floor.items()) + changed, self._floor_changed = self._floor_changed, set() + rows = [(name, str(self._floor[name][0]), self._floor[name][1], self._floor_used[name]) + for name in changed if name in self._floor] + gone = [(name,) for name in changed if name not in self._floor] saved_at, self._floor_saved_at = self._floor_saved_at, self._monotonic() self._floor_dirty = False saved = False try: - state = {"version": FLOOR_VERSION, - "floor": [[name.hex(), sequence, int(at)] for name, (sequence, at) in floor]} - os.makedirs(self.config.state_dir, exist_ok=True) try: - fsutil.write_atomic(self._floor_path, json.dumps(state, separators=(",", ":")).encode()) + with self._db.transaction() as tx: + tx.executemany("DELETE FROM name_floor WHERE name = ?", gone) + tx.executemany("INSERT OR REPLACE INTO name_floor (name, sequence, resolved, used) " + "VALUES (?, ?, ?, ?)", rows) saved = True except OSError as e: - log(f"could not save {self._floor_path}: {e}", LOG_WARNING) + log(f"could not save the name floor: {e}", LOG_WARNING) finally: if not saved: with self._lock: + self._floor_changed |= changed self._floor_dirty = True self._floor_saved_at = saved_at def _save(self): - """Write published and held records atomically.""" - published = {alias: {"sequence": p.sequence, "cid": str(p.cid) if p.cid is not None else None, - "imported": p.imported} - for alias, p in self._published.items()} - for alias, entry in self._unreadable.items(): - if entry is not None and alias not in published: - published[alias] = entry - state = { - "version": STATE_VERSION, - "published": dict(sorted(published.items())), - "held": {name.hex(): {"sequence": r.sequence, "cid": str(r.cid), "heard": r.heard} - for name, r in sorted(self._held.items())}, - } - os.makedirs(self.config.state_dir, exist_ok=True) - fsutil.write_atomic(self._state_path, json.dumps(state, indent=1).encode()) + """Write the published and held records changed since the last save, durably (on disk when it returns: + this is a publish's commit point); the rows of _unreadable aliases are left as they are. Raises OSError if + the database fails, having written nothing.""" + published = [(alias, p) for alias, p in self._published.items() if self._saved_published.get(alias) != p] + unpublished = [alias for alias in self._saved_published if alias not in self._published] + held = [(name, r) for name, r in self._held.items() if self._saved_held.get(name) != r] + unheld = [name for name in self._saved_held if name not in self._held] + if published or unpublished or held or unheld: + with self._db.transaction(durable=True) as tx: + tx.executemany("DELETE FROM names_published WHERE alias = ?", [(alias,) for alias in unpublished]) + tx.executemany("INSERT OR REPLACE INTO names_published (alias, sequence, cid, imported) " + "VALUES (?, ?, ?, ?)", + [(alias, str(p.sequence), None if p.cid is None else p.cid.binary, int(p.imported)) + for alias, p in published]) + tx.executemany("DELETE FROM names_held WHERE name = ?", [(name,) for name in unheld]) + tx.executemany("INSERT OR REPLACE INTO names_held (name, sequence, cid, heard) VALUES (?, ?, ?, ?)", + [(name, str(r.sequence), r.cid.binary, r.heard) for name, r in held]) + for alias in unpublished: + del self._saved_published[alias] + self._saved_published.update(published) + for name in unheld: + del self._saved_held[name] + self._saved_held.update(held) self._held_dirty = False diff --git a/rnfs/node.py b/rnfs/node.py index 40df3ca..e1943a0 100644 --- a/rnfs/node.py +++ b/rnfs/node.py @@ -2,8 +2,9 @@ facade that the API, gateway and CLI call (design.md, "Node facade"). Startup: Reticulum from (attaching to a shared instance if its config shares one), the node's -identity and its rnfs.dht destination, then the routing table, provider records, storage, shared folder, -fetcher, publisher and names; the request server answers through NodeBackend. Background threads: the +identity and its rnfs.dht destination, then the state database (rnfs.statedb, which imports the JSON state files +of earlier versions first), the routing table, provider records, storage, shared folder, fetcher, publisher and +names; the request server answers through NodeBackend. Background threads: the announcer, the join (bootstrap nodes, then a lookup for our own ID, repeated while too few servers are known), the publish loop, the name re-announce loop and maintenance (shared-folder scans, provider record expiry and saves, max_storage enforcement). @@ -23,7 +24,7 @@ import RNS -from rnfs import __version__, features, fsutil, winacl +from rnfs import __version__, features, fsutil, statedb, winacl from rnfs.config import Config from rnfs.constants import APP_NAME, DEFAULT_BOOTSTRAP_NODES, DHT_ASPECT, K, MODE_SERVER, PARTIAL_SUFFIX from rnfs.content import archive, builder, dag, edit, seal, walker @@ -31,7 +32,7 @@ from rnfs import node_content # --- content: add --no-pin/--to/--chunker, sealed files, gateway formats --- from rnfs.content import chunker as content_chunker from rnfs.content.shared_folder import SharedFolder -from rnfs.content.storage import FOLDER, USER, Storage, disk_reserve, read_json, remote_source, write_json +from rnfs.content.storage import FOLDER, USER, Storage, disk_reserve, remote_source from rnfs.dht.lookup import Lookup from rnfs.dht.providers import ProviderStore from rnfs.dht.routing import RoutingTable @@ -49,9 +50,6 @@ from rnfs.network.server import RequestServer from rnfs.publisher import Publisher -PROVIDERS_FILE = "providers.json" -REMOTE_PINS_FILE = "remote_pins.json" # remote pins accepted and not yet fetched: [[source, cid], ...] in order -FETCHED_FILE = "fetched.json" # cached targets fetched for the user (not the gateway): the roots strategy's SAVE_INTERVAL = 300.0 # seconds between saves of provider records STATE_SAVE_INTERVAL = 30.0 # seconds between saves of cache records, fetched targets and invalid roots, when changed ARCHIVE_BLOCK_OVERHEAD = 64 # bytes an archive spends on each block beyond the block (msgpack framing, CID) @@ -65,6 +63,43 @@ KIND_NAMES = {dag.KIND_FILE: "file", dag.KIND_EXECUTABLE: "executable", dag.KIND_DIRECTORY: "directory"} +def import_fetched(value: list, tx) -> int: + """Import fetched.json (statedb.migrate): its targets replace the fetched table; one that doesn't parse is + skipped, as rnfs 0.1.4 did. Returns the number imported.""" + tx.execute("DELETE FROM fetched") + cids = set() + for text in value: + try: + cids.add(Cid.parse(text)) + except (ValueError, TypeError): + continue + tx.executemany("INSERT INTO fetched (cid) VALUES (?)", [(cid.binary,) for cid in cids]) + return len(cids) + + +def import_remote_pins(value: list, tx) -> int: + """Import remote_pins.json (statedb.migrate): its [source, cid] pairs, in order, replace the remote_pins + table; malformed ones are counted in a warning and skipped, as rnfs 0.1.4 did. Which sources are still + allowed, duplicates and REMOTE_PIN_QUEUE are left to the load (_load_remote_pins). Returns the number + imported.""" + tx.execute("DELETE FROM remote_pins") + count = malformed = 0 + for entry in value: + try: + source, text = entry + if not isinstance(source, str) or not isinstance(text, str): + raise TypeError("not a pair of strings") # a list or dict source can't even be looked up + tx.execute("INSERT INTO remote_pins (position, source, cid) VALUES (?, ?, ?)", + (count, source, Cid.parse(text).binary)) + except (ValueError, TypeError): + malformed += 1 + continue + count += 1 + if malformed: + log(f"skipped {malformed} malformed entr{'y' if malformed == 1 else 'ies'} in remote_pins.json", LOG_WARNING) + return count + + def _make_private_dirs(config: Config): """Create the rnfs dir and the node's subdirectories readable by the owner only (0700; on Windows, a protected access list for the user and SYSTEM from the moment each is created, see winacl). @@ -252,8 +287,8 @@ def __init__(self, config: Config): APP_NAME, DHT_ASPECT) slow = rates.is_slow_node(config, self.reticulum) self.links = LinkPool(self.identity, config) - self._setup(config, bytes(self.destination.hash), PeerClient(self.links, config), slow, - names_network=None, is_known=_rns_is_known) + self._setup_or_close(config, bytes(self.destination.hash), PeerClient(self.links, config), slow, + names_network=None, is_known=_rns_is_known) # mesh: hops and next-hop interfaces rank providers; one set of traffic counters for client and server self.paths = RnsPaths(self.reticulum) self.fetcher.path_info = self.paths.info @@ -278,10 +313,20 @@ def offline(cls, config: Config, node_id: bytes, client, names_network, slow: bo """ node = cls.__new__(cls) node.reticulum = node.links = node.server = node.announcer = node.announce_handler = None - node._setup(config, node_id, client, slow, names_network=names_network, is_known=lambda node_id: True) + node._setup_or_close(config, node_id, client, slow, names_network=names_network, + is_known=lambda node_id: True) node.joined.set() return node + def _setup_or_close(self, *args, **kwargs): + """_setup, releasing the state database references taken so far if it raises (a store refusing to + start), so a caller in the same process can try again or remove the directory.""" + try: + self._setup(*args, **kwargs) + except BaseException: + self._close_state() + raise + def _setup(self, config: Config, node_id: bytes, client, slow: bool, names_network, is_known): """Everything that doesn't need Reticulum itself.""" self.config = config @@ -299,10 +344,12 @@ def _setup(self, config: Config, node_id: bytes, client, slow: bool, names_netwo except ValueError as e: log(f"ignoring bootstrap node: {e}", LOG_WARNING) _make_private_dirs(config) + # First, so the JSON state files of earlier versions are imported before any store loads its state + self._db = statedb.connect(config.state_dir) self.routing = RoutingTable(node_id, config, admit=self._is_member_node if config.private else None) - self.records = ProviderStore(config.max_records, os.path.join(config.state_dir, PROVIDERS_FILE), - own_id=node_id, seniority=config.provider_seniority) + self.records = ProviderStore(config.max_records, config.state_dir, own_id=node_id, + seniority=config.provider_seniority) if config.private: # Records stored while the node was public, or from a member removed since, are never handed out removed = self.records.retain(self._is_member_publisher) @@ -319,20 +366,21 @@ def _setup(self, config: Config, node_id: bytes, client, slow: bool, names_netwo self.backend = NodeBackend(self) self._lock = threading.Lock() - self._fetched_path = os.path.join(config.state_dir, FETCHED_FILE) self._fetched: set[Cid] = set() - for text in read_json(self._fetched_path, shape=lambda value: isinstance(value, list)) or []: + self._fetched_saved: set = set() # what the fetched table holds + for (binary,) in self._db.query("SELECT cid FROM fetched"): + self._fetched_saved.add(binary) try: - self._fetched.add(Cid.parse(text)) - except (ValueError, TypeError): + self._fetched.add(Cid.from_bytes(binary)) + except ValueError: continue - self._fetched_dirty = False # changed since last saved (by _save_state: maintenance, gc and stop) + # Changed since last saved (by _save_state: maintenance, gc and stop); a row that didn't parse is deleted + self._fetched_dirty = len(self._fetched) != len(self._fetched_saved) self._fetched_save_lock = threading.Lock() # one save at a time, so an older snapshot never lands last # Remote source -> CIDs waiting or being fetched for it, in the order accepted (fetched one at a time) self._remote_reserved: dict[str, dict[Cid, None]] = {} self._remote_released: set[tuple[str, Cid]] = set() # remote pins released while still being fetched self._remote_workers: set[str] = set() # remote sources with a thread fetching their pins - self._remote_pins_path = os.path.join(config.state_dir, REMOTE_PINS_FILE) # those waiting, for a restart self._remote_pins_dirty = False # changed since last saved (by maintenance, and at stop) self._remote_pins_save_lock = threading.Lock() # one save at a time, so an older snapshot never lands last self._refresh_wanted = threading.Event() @@ -344,7 +392,7 @@ def _setup(self, config: Config, node_id: bytes, client, slow: bool, names_netwo for cls in features.feature_classes(): self.features[cls.name] = cls(self) # Releasing a deleted name key's pin recomputes what is provided, so it waits for the features too; and - # name pins are made to match names.json, which a crash inside a publish can leave them disagreeing with + # name pins are made to match the names state, which a crash inside a publish can leave them disagreeing with self.names.reconcile_pins(self.storage.pins()) # What is provided, before /blocks can be asked; after the features, whose keys (kept names) would # otherwise all be withdrawn at every start @@ -396,9 +444,23 @@ def stop(self, timeout: float = STOP_TIMEOUT): self._save_remote_pins() self.publisher.save() self._save_state() + self._close_state() if self.links is not None: self.links.close_all() + def _close_state(self): + """Release every store's reference to the state database, once everything is saved; the last closes it. + After a _setup that failed, only the stores it made.""" + names = ("records", "storage", "shared_folder", "pin_names", "fetcher", "publisher", "names") + stores = [getattr(self, name) for name in names if hasattr(self, name)] + for store in (*stores, *getattr(self, "features", {}).values(), getattr(self, "_db", None)): + if store is None: + continue + try: + store.close() + except Exception as e: # noqa: BLE001 - release the rest + log(f"closing state failed: {e!r}", LOG_ERROR) + def _save_state(self): """Save cache records, fetched targets and invalid roots if they changed (each change isn't written at once).""" @@ -648,20 +710,25 @@ def _note_fetched(self, target: Cid): self._fetched_dirty = True def _save_fetched(self): - """Save the fetched targets if they changed. With the cache records they're intersected with, which are - saved in the same pass: a crash loses at most what changed since the last pass, as for those.""" + """Save the fetched targets added or forgotten since the last save, if any. With the cache records they're + intersected with, which are saved in the same pass: a crash loses at most what changed since the last + pass, as for those.""" with self._fetched_save_lock: with self._lock: if not self._fetched_dirty: return - value = sorted(str(cid) for cid in self._fetched) + current = {cid.binary for cid in self._fetched} self._fetched_dirty = False + added, gone = current - self._fetched_saved, self._fetched_saved - current try: - write_json(self._fetched_path, value) + with self._db.transaction() as tx: + tx.executemany("DELETE FROM fetched WHERE cid = ?", [(binary,) for binary in gone]) + tx.executemany("INSERT OR IGNORE INTO fetched (cid) VALUES (?)", [(binary,) for binary in added]) except BaseException: with self._lock: self._fetched_dirty = True raise + self._fetched_saved = current # Fetching @@ -991,7 +1058,7 @@ def remote_pin(self, identity_hash: bytes, cid: Cid, pin: bool) -> bool: A source's pins are fetched one at a time, so each is checked against its quota with the earlier ones counted; one with REMOTE_PIN_QUEUE pins waiting is refused (False), and so is one whose fetch thread - can't be started. Accepted pins are saved (remote_pins.json) until fetched, and resumed at the next + can't be started. Accepted pins are saved (the remote_pins table) until fetched, and resumed at the next start if the node stops first. """ source = remote_source(identity_hash) @@ -1080,11 +1147,14 @@ def _save_remote_pins(self): with self._lock: if not self._remote_pins_dirty: return - waiting = [[source, str(cid)] for source, cids in self._remote_reserved.items() for cid in cids + waiting = [(source, cid) for source, cids in self._remote_reserved.items() for cid in cids if (source, cid) not in self._remote_released] self._remote_pins_dirty = False try: - write_json(self._remote_pins_path, waiting) + with self._db.transaction() as tx: # a few rows (at most REMOTE_PIN_QUEUE per source): rewritten + tx.execute("DELETE FROM remote_pins") + tx.executemany("INSERT INTO remote_pins (position, source, cid) VALUES (?, ?, ?)", + [(position, source, cid.binary) for position, (source, cid) in enumerate(waiting)]) except OSError as e: with self._lock: self._remote_pins_dirty = True # try again next time @@ -1092,22 +1162,21 @@ def _save_remote_pins(self): def _load_remote_pins(self) -> list[tuple[str, Cid]]: """Remote pins saved as waiting, oldest first, from identities still on remote_pin_identities (at most - REMOTE_PIN_QUEUE each); others are dropped. Caller holds no lock (it reads a file).""" + REMOTE_PIN_QUEUE each); others are dropped. Caller holds no lock (it reads the database).""" allowed = {remote_source(h.strip().strip("<>")) for h in self.config.remote_pin_identities} try: - saved = read_json(self._remote_pins_path, shape=lambda value: isinstance(value, list)) or [] + saved = self._db.query("SELECT source, cid FROM remote_pins ORDER BY position") except OSError as e: log(f"reading the remote pins waiting failed: {e!r}", LOG_ERROR) return [] pins: dict[tuple[str, Cid], None] = {} counts: dict[str, int] = {} malformed = 0 - for entry in saved: + for source, binary in saved: try: - source, text = entry - if not isinstance(source, str) or not isinstance(text, str): - raise TypeError("not a pair of strings") # a list or dict source can't even be looked up - cid = Cid.parse(text) + if not isinstance(source, str): + raise TypeError("not a string") + cid = Cid.from_bytes(binary) except (ValueError, TypeError): malformed += 1 continue @@ -1116,7 +1185,7 @@ def _load_remote_pins(self) -> list[tuple[str, Cid]]: counts[source] = counts.get(source, 0) + 1 pins[(source, cid)] = None if malformed: - log(f"skipped {malformed} malformed entr{'y' if malformed == 1 else 'ies'} in {self._remote_pins_path}", + log(f"skipped {malformed} malformed remote pin{'' if malformed == 1 else 's'} saved as waiting", LOG_WARNING) return list(pins) diff --git a/rnfs/pin_names.py b/rnfs/pin_names.py index d832d7e..86397f3 100644 --- a/rnfs/pin_names.py +++ b/rnfs/pin_names.py @@ -2,33 +2,67 @@ A CID names only content, so `add photo.jpg` keeps no file name. This records, for this node's own user, what each pin was added from (an absolute path, "(stdin)", the target given to `pin add`, an archive) so -listings can show it. It is local only: never sent anywhere, and it doesn't change any CID. +listings can show it. It is local only: never sent anywhere, and it doesn't change any CID. Kept in the state +database's pin_names table (rnfs.statedb): each change is one row written at once, durably. """ -import os import threading +from rnfs import statedb from rnfs.content.cid import Cid -from rnfs.content.storage import read_json, write_json +from rnfs.log import LOG_WARNING, log -NAMES_FILE = "pin_names.json" MAX_NAME_LENGTH = 4096 # an origin longer than this (a very deep path) is cut, not refused +def import_names(value: dict, tx) -> int: + """Import pin_names.json (statedb.migrate): its names replace the pin_names table, each checked as rnfs 0.1.4 + loaded it (one that doesn't parse is skipped, a long one cut to MAX_NAME_LENGTH). Returns the number + imported.""" + tx.execute("DELETE FROM pin_names") + imported = set() + for text, name in value.items(): + try: + if not isinstance(name, str): + continue + cid = Cid.parse(text) + tx.execute("INSERT OR REPLACE INTO pin_names (cid, name) VALUES (?, ?)", + (cid.binary, statedb.dump(name[:MAX_NAME_LENGTH]))) + except (ValueError, TypeError): + continue + imported.add(cid) + return len(imported) + + class PinNames: """Pin → where it was added from. Thread-safe; entries for CIDs no longer pinned are pruned.""" def __init__(self, state_dir: str): - """Load the names saved in state_dir, skipping anything that doesn't parse.""" - self.path = os.path.join(state_dir, NAMES_FILE) + """Load the names saved in state_dir; a row that doesn't parse is logged and deleted.""" + self._db = statedb.connect(state_dir) self._lock = threading.Lock() self._names: dict[Cid, str] = {} - value = read_json(self.path, shape=lambda value: isinstance(value, dict)) or {} - for text, name in value.items(): + self._dirty: set[Cid] = set() # CIDs whose name changed (or went) and whose save failed + try: + self._load() + except BaseException: + self._db.close() + raise + + def _load(self): + """Read the pin_names table.""" + bad = [] + for binary, text in self._db.query("SELECT cid, name FROM pin_names"): try: - if isinstance(name, str): - self._names[Cid.parse(text)] = name[:MAX_NAME_LENGTH] + name = statedb.load(text) + if not isinstance(name, str): + raise TypeError("not a string") + self._names[Cid.from_bytes(binary)] = name[:MAX_NAME_LENGTH] except (ValueError, TypeError): - continue + log(f"pin names: ignoring invalid record {binary!r} {text!r}", LOG_WARNING) + bad.append((binary,)) + if bad: + with self._db.transaction() as tx: + tx.executemany("DELETE FROM pin_names WHERE cid = ?", bad) def record(self, cid: Cid, name: str): """Record that cid was added from name; a later add of the same content replaces it.""" @@ -37,7 +71,7 @@ def record(self, cid: Cid, name: str): if self._names.get(cid) == name: return self._names[cid] = name - self._save() + self._save([cid]) def record_if_unknown(self, cid: Cid, name: str): """Record name for cid unless it already has one (a pin add of something added here keeps its path).""" @@ -45,7 +79,7 @@ def record_if_unknown(self, cid: Cid, name: str): if cid in self._names: return self._names[cid] = name[:MAX_NAME_LENGTH] - self._save() + self._save([cid]) def get(self, cid: Cid) -> str | None: """Where cid was added from, or None if unknown.""" @@ -53,15 +87,26 @@ def get(self, cid: Cid) -> str | None: return self._names.get(cid) def prune(self, pinned: set[Cid]): - """Forget the names of CIDs that are no longer pinned, so the file stays as small as the pin list.""" + """Forget the names of CIDs that are no longer pinned, so the table stays as small as the pin list.""" with self._lock: gone = [cid for cid in self._names if cid not in pinned] if not gone: return for cid in gone: del self._names[cid] - self._save() + self._save(gone) + + def _save(self, changed: list[Cid]): + """Write the rows of changed, and of earlier changes whose save failed, durably (lock held). If it fails + they are written with the next change.""" + self._dirty.update(changed) + names = [(cid.binary, statedb.dump(self._names[cid])) for cid in self._dirty if cid in self._names] + gone = [(cid.binary,) for cid in self._dirty if cid not in self._names] + with self._db.transaction(durable=True) as tx: + tx.executemany("DELETE FROM pin_names WHERE cid = ?", gone) + tx.executemany("INSERT OR REPLACE INTO pin_names (cid, name) VALUES (?, ?)", names) + self._dirty.clear() - def _save(self): - """Write the names atomically (called with the lock held).""" - write_json(self.path, dict(sorted((str(cid), name) for cid, name in self._names.items()))) + def close(self): + """Release the state database (the node calls this once it has saved everything at shutdown).""" + self._db.close() diff --git a/rnfs/publisher.py b/rnfs/publisher.py index f369b85..35d8e21 100644 --- a/rnfs/publisher.py +++ b/rnfs/publisher.py @@ -11,10 +11,12 @@ from the nodes it was last published to, at once, and from nodes of earlier rounds (or of a round cut short) whose record may not have expired yet: the closest set changes as nodes come and go. -Last-published nodes and times are persisted in /published.json, so a restart doesn't -republish what isn't due. A CID whose lookup found no node at all (nothing was sent) is retried after -RETRY_INTERVAL rather than a whole republish interval; one whose round was stopped before every node was -sent its keys stays due. Times are wall clock: any in the future (the clock was corrected) count as now. +Last-published nodes and times, earlier rounds' nodes and pending withdrawals are persisted in the state +database (rnfs.statedb: the published, published_earlier and withdrawals tables), each save writing only the +items and nodes changed since the last, so a restart doesn't republish what isn't due. A CID whose lookup found +no node at all (nothing was sent) is retried after RETRY_INTERVAL rather than a whole republish interval; one +whose round was stopped before every node was sent its keys stays due. Times are wall clock: any in the future +(the clock was corrected) count as now. - **Feature keys** (DhtKey, e.g. kept names) are republished at max(republish interval, KEY_MIN_INTERVAL), each with its own fixed jitter of up to +-KEY_JITTER, so they don't all come due in one burst; always @@ -26,23 +28,21 @@ """ import hashlib import math -import os import threading import time from typing import Callable +from rnfs import statedb from rnfs.config import Config -from rnfs.constants import DAY, HOUR, PROVIDER_TTL +from rnfs.constants import DAY, HOUR, ID_LENGTH, PROVIDER_TTL from rnfs.content.cid import Cid -from rnfs.content.storage import Storage, read_json, shaped, write_json +from rnfs.content.storage import Storage from rnfs.errors import InvalidDag, NotFound -from rnfs.log import LOG_ERROR, LOG_VERBOSE, log, node_name, plural +from rnfs.log import LOG_ERROR, LOG_VERBOSE, LOG_WARNING, log, node_name, plural -STATE_FILE = "published.json" -SECTIONS = ("published", "earlier", "withdraw") # the state file's objects RETRY_INTERVAL = 30.0 # seconds before retrying a CID whose lookup found no nodes to publish to MAX_WAIT = 60.0 # the loop wakes at least this often -KEY_PREFIX = "key:" # how a DhtKey is written in the state file +KEY_PREFIX = "key:" # how a DhtKey is written as a published item (str()) # A kept name's record changes rarely and keepers answer for years-old ones; its provider record lives # PROVIDER_TTL (7 days), so republishing every 3 days still survives a missed round KEY_MIN_INTERVAL = 3 * DAY @@ -101,10 +101,82 @@ def key_interval(key: bytes, base: float) -> float: def _parse_item(text: str) -> "Cid | DhtKey": - """A provided item as the state file writes it. Raises ValueError.""" + """A provided item as str() writes it (the published tables' item). Raises ValueError.""" return DhtKey.parse(text) if isinstance(text, str) and text.startswith(KEY_PREFIX) else Cid.parse(text) +def _item(text) -> "Cid | DhtKey": + """A saved provided item: _parse_item, but only an ID_LENGTH-byte DhtKey. Raises ValueError.""" + item = _parse_item(text) + if isinstance(item, DhtKey) and len(item.key) != ID_LENGTH: + raise ValueError(f"not a provided key: {text!r}") + return item + + +def _time(number) -> float: + """A saved time; ValueError if it isn't a finite number.""" + if isinstance(number, bool) or not isinstance(number, (int, float)) or not math.isfinite(number): + raise ValueError(f"not a time: {number!r}") + return float(number) + + +def _id(text) -> bytes: + """A node ID or key as published.json wrote it (hex). Raises ValueError or TypeError.""" + value = bytes.fromhex(text) + if len(value) != ID_LENGTH: + raise ValueError(f"not {ID_LENGTH} bytes: {text!r}") + return value + + +def _ids(blob) -> list[bytes]: + """The node IDs a published row's nodes concatenates. Raises ValueError.""" + if not isinstance(blob, bytes) or len(blob) % ID_LENGTH: + raise ValueError(f"not a list of node IDs: {blob!r}") + return [blob[i:i + ID_LENGTH] for i in range(0, len(blob), ID_LENGTH)] + + +def import_published(value: dict, tx) -> int: + """Import published.json (statedb.migrate): its last rounds, earlier rounds' nodes and pending withdrawals + replace the published, published_earlier and withdrawals tables. Each entry is checked as rnfs 0.1.4 loaded + it, one of the wrong shape being skipped, and node IDs, keys and DhtKeys must be ID_LENGTH bytes; times are + kept as written (the load clamps them and drops earlier nodes past PROVIDER_TTL). Returns the number of rows + imported.""" + for table in ("published", "published_earlier", "withdrawals"): + tx.execute(f"DELETE FROM {table}") + published: dict[str, tuple[float, bytes]] = {} + for text, record in value.get("published", {}).items(): + try: + at, nodes = record + if not isinstance(nodes, list): + raise TypeError("nodes must be a list") + published[str(_item(text))] = (_time(at), b"".join(_id(n) for n in nodes)) + except (ValueError, TypeError, OverflowError): + continue + earlier: dict[str, dict[bytes, float]] = {} + for text, nodes in value.get("earlier", {}).items(): + try: + if not isinstance(nodes, dict): + raise TypeError("nodes must be an object") + earlier[str(_item(text))] = {_id(n): _time(stored) for n, stored in nodes.items()} + except (ValueError, TypeError, OverflowError): + continue + withdrawals: dict[bytes, set[bytes]] = {} + for node_hex, keys in value.get("withdraw", {}).items(): + try: + if not isinstance(keys, list): + raise TypeError("keys must be a list") + withdrawals[_id(node_hex)] = {_id(k) for k in keys} + except (ValueError, TypeError): + continue + rows = [[(text, at, nodes) for text, (at, nodes) in published.items()], + [(text, node, at) for text, nodes in earlier.items() for node, at in nodes.items()], + [(node, key) for node, keys in withdrawals.items() for key in keys]] + tx.executemany("INSERT INTO published (item, at, nodes) VALUES (?, ?, ?)", rows[0]) + tx.executemany("INSERT INTO published_earlier (item, node, at) VALUES (?, ?, ?)", rows[1]) + tx.executemany("INSERT INTO withdrawals (node, key) VALUES (?, ?)", rows[2]) + return sum(len(table) for table in rows) + + class Publisher: """The provide set, the served set, and the publish/withdraw schedule. Thread-safe.""" @@ -127,7 +199,6 @@ def __init__(self, own_id: bytes, config: Config, storage: Storage, providers, c self.cached_eligible = cached_eligible self.clock = clock self.retry_interval = retry_interval - self._state_path = None if state_dir is None else os.path.join(state_dir, STATE_FILE) self._lock = threading.RLock() self._save_lock = threading.Lock() # one save at a time, so an older snapshot never lands last self._wake = threading.Event() @@ -147,7 +218,17 @@ def __init__(self, own_id: bytes, config: Config, storage: Storage, providers, c self._next_lookup = 0.0 # slow nodes: clock time before which no lookup round starts # Functions returning the non-CID keys features provide (add_key_source); polled by refresh() self._key_sources: list[Callable[[], set[bytes]]] = [] - self._load() + # Changed since the last save (only tracked when persisted): items whose published or earlier rows, and + # nodes whose withdrawals, save() writes again + self._dirty_items: set = set() + self._dirty_nodes: set = set() + self._db = None if state_dir is None else statedb.connect(state_dir) + if self._db is not None: + try: + self._load() + except BaseException: + self._db.close() + raise def add_key_source(self, source: Callable[[], set[bytes]]): """Also provide the keys source() returns (DhtKeys), recomputed at each refresh(). Must not block.""" @@ -168,62 +249,122 @@ def interval_for(self, item: "Cid | DhtKey") -> float: # Persistence def _load(self): - """Load last-published times and nodes, and pending withdrawals. A file of the wrong shape is set aside - as damaged; entries of the wrong shape are skipped.""" - if self._state_path is None: - return - value = read_json(self._state_path, shape=lambda v: all(shaped(key, dict)(v) for key in SECTIONS)) or {} + """Load last-published times and nodes, earlier rounds' nodes and pending withdrawals. Rows of the wrong + shape are skipped (logged) and deleted; those the load changes (times clamped, earlier nodes past + PROVIDER_TTL dropped) are marked changed, so the next save writes them as memory has them.""" now = self.clock() - def entries(key: str) -> list: - """The items of value[key].""" - return list(value.get(key, {}).items()) - def when(number) -> float: - """A persisted time, clamped to now; ValueError if it isn't a finite number.""" - if isinstance(number, bool) or not isinstance(number, (int, float)) or not math.isfinite(number): - raise ValueError(f"not a time: {number!r}") - return min(float(number), now) + """A saved time, clamped to now; ValueError if it isn't a finite number.""" + return min(_time(number), now) + + doomed: set = set() # item texts whose rows are deleted here: they don't parse, or aren't as str() writes + malformed = 0 - for text, record in entries("published"): + def item_of(text) -> "Cid | DhtKey | None": + """The item a row's text names, or None if it doesn't parse (counted and doomed).""" + nonlocal malformed try: - published, nodes = record - if not isinstance(nodes, list): - raise TypeError("nodes must be a list") - self._published[_parse_item(text)] = (when(published), [bytes.fromhex(n) for n in nodes]) - except (ValueError, TypeError, OverflowError): + item = _item(text) + except (ValueError, TypeError): + malformed += 1 + doomed.add(text) + return None + if str(item) != text: # deleted, and written again as str() writes it + doomed.add(text) + self._touch_item(item) + return item + + for text, at, nodes in self._db.query("SELECT item, at, nodes FROM published"): + item = item_of(text) + if item is None: continue - for text, nodes in entries("earlier"): try: - if not isinstance(nodes, dict): - raise TypeError("nodes must be an object") - earlier = {bytes.fromhex(n): when(stored) for n, stored in nodes.items()} - self._earlier[_parse_item(text)] = {n: t for n, t in earlier.items() if now - t < PROVIDER_TTL} + self._published[item] = (when(at), _ids(nodes)) except (ValueError, TypeError, OverflowError): + malformed += 1 + self._touch_item(item) + continue + if self._published[item][0] != at: + self._touch_item(item) + for text, node, stored in self._db.query("SELECT item, node, at FROM published_earlier"): + item = item_of(text) + if item is None: continue - for node_hex, keys in entries("withdraw"): try: - if not isinstance(keys, list): - raise TypeError("keys must be a list") - self._withdrawals[bytes.fromhex(node_hex)] = {bytes.fromhex(k) for k in keys} - except (ValueError, TypeError): + if not isinstance(node, bytes) or len(node) != ID_LENGTH: + raise ValueError(f"not a node ID: {node!r}") + stored_at = when(stored) + except (ValueError, TypeError, OverflowError): + malformed += 1 + self._touch_item(item) continue + if now - stored_at < PROVIDER_TTL: + self._earlier.setdefault(item, {})[node] = stored_at + if stored_at != stored or now - stored_at >= PROVIDER_TTL: + self._touch_item(item) + for node, key in self._db.query("SELECT node, key FROM withdrawals"): + if not all(isinstance(value, bytes) and len(value) == ID_LENGTH for value in (node, key)): + malformed += 1 + self._touch_node(node) + continue + self._withdrawals.setdefault(node, set()).add(key) + if malformed: + log(f"publisher: ignoring {plural(malformed, 'malformed row')} in the state database", LOG_WARNING) + if doomed: + with self._db.transaction() as tx: + tx.executemany("DELETE FROM published WHERE item = ?", [(text,) for text in doomed]) + tx.executemany("DELETE FROM published_earlier WHERE item = ?", [(text,) for text in doomed]) def save(self): - """Persist the publish state (atomic). Called without the lock: it is written outside it.""" - if self._state_path is None: + """Persist the publish state changed since the last save, if any. Called without the lock: it is written + outside it. Raises OSError (StateDbError) if the database fails; what it held is then written next time.""" + if self._db is None: return # Saves come from refresh(), the publish loop and stop(): one at a time, or a snapshot taken before the # other's could be written last with self._save_lock: with self._lock: - value = {"published": {str(cid): [when, [n.hex() for n in nodes]] - for cid, (when, nodes) in self._published.items()}, - "earlier": {str(cid): {n.hex(): when for n, when in nodes.items()} - for cid, nodes in self._earlier.items() if nodes}, - "withdraw": {node.hex(): sorted(k.hex() for k in keys) - for node, keys in self._withdrawals.items() if keys}} - write_json(self._state_path, value) + items, nodes = self._dirty_items, self._dirty_nodes + if not items and not nodes: + return + self._dirty_items, self._dirty_nodes = set(), set() + texts = {item: str(item) for item in items} + published = [(texts[item], self._published[item][0], b"".join(self._published[item][1])) + for item in items if item in self._published] + earlier = [(texts[item], node, at) + for item in items for node, at in self._earlier.get(item, {}).items()] + withdrawals = [(node, key) for node in nodes for key in self._withdrawals.get(node, ())] + try: + with self._db.transaction() as tx: + tx.executemany("DELETE FROM published WHERE item = ?", [(text,) for text in texts.values()]) + tx.executemany("DELETE FROM published_earlier WHERE item = ?", + [(text,) for text in texts.values()]) + tx.executemany("INSERT OR REPLACE INTO published (item, at, nodes) VALUES (?, ?, ?)", published) + tx.executemany("INSERT OR REPLACE INTO published_earlier (item, node, at) VALUES (?, ?, ?)", + earlier) + tx.executemany("DELETE FROM withdrawals WHERE node = ?", [(node,) for node in nodes]) + tx.executemany("INSERT OR REPLACE INTO withdrawals (node, key) VALUES (?, ?)", withdrawals) + except BaseException: + with self._lock: # written by the next save + self._dirty_items |= items + self._dirty_nodes |= nodes + raise + + def close(self): + """Release the state database (the node calls this once it has saved everything at shutdown).""" + if self._db is not None: + self._db.close() + + def _touch_item(self, item: "Cid | DhtKey"): + """Note that item's published or earlier entry changed, for the next save. Caller holds the lock.""" + if self._db is not None: + self._dirty_items.add(item) + + def _touch_node(self, node_id: bytes): + """Note that the withdrawals queued for node_id changed, for the next save. Caller holds the lock.""" + if self._db is not None: + self._dirty_nodes.add(node_id) # The provide set @@ -288,8 +429,12 @@ def refresh(self): self._retry_at.pop(cid, None) self.providers.withdraw(cid.key, self.own_id) record = self._published.pop(cid, None) - for node_id in set(record[1] if record else []) | set(self._earlier.pop(cid, {})): + earlier = self._earlier.pop(cid, {}) + if record is not None or earlier: + self._touch_item(cid) + for node_id in set(record[1] if record else []) | set(earlier): self._withdrawals.setdefault(node_id, set()).add(cid.key) + self._touch_node(node_id) cancelled = [cid for cid in added if self._provide_again(cid)] if removed: log(f"no longer providing {plural(len(removed), 'CID')}", LOG_VERBOSE) @@ -317,6 +462,8 @@ def _provide_again(self, cid: "Cid | DhtKey") -> bool: del self._withdrawals[node_id] earlier = self._earlier.setdefault(cid, {}) earlier[node_id] = max(now, earlier.get(node_id, now)) + self._touch_node(node_id) + self._touch_item(cid) cancelled = True return cancelled @@ -340,6 +487,7 @@ def _clamp(self, now: float): for cid, (when, nodes) in list(self._published.items()): if when > now: self._published[cid] = (now, nodes) + self._touch_item(cid) for cid, retry in list(self._retry_at.items()): self._retry_at[cid] = min(retry, now + self.retry_interval) self._next_lookup = min(self._next_lookup, now + SLOW_LOOKUP_PACE * SLOW_ROUND_LOOKUPS) @@ -410,6 +558,7 @@ def publish_due(self, should_stop: Callable[[], bool] = lambda: False) -> dict: self._withdrawals[node_id] -= superseded if not self._withdrawals[node_id]: del self._withdrawals[node_id] + self._touch_node(node_id) accepted: dict[Cid, list[bytes]] = {cid: [] for cid in closest} sent_withdrawals: dict[bytes, set[bytes]] = {} unsaved, saved_at = 0, self.clock() @@ -437,8 +586,10 @@ def publish_due(self, should_stop: Callable[[], bool] = lambda: False) -> dict: if cid in self._provided: earlier = self._earlier.setdefault(cid, {}) earlier[node_id] = max(answered, earlier.get(node_id, answered)) + self._touch_item(cid) else: self._withdrawals.setdefault(node_id, set()).add(cid.key) + self._touch_node(node_id) missed = sum(1 for cid in cids if cid.key not in recorded) if missed: log(f"{node_name(node_id)} didn't record {plural(missed, 'key')}", LOG_VERBOSE) @@ -455,16 +606,20 @@ def publish_due(self, should_stop: Callable[[], bool] = lambda: False) -> dict: self._withdrawals[node_id] = remaining else: self._withdrawals.pop(node_id, None) + if keys: + self._touch_node(node_id) for cid, nodes in closest.items(): if cid not in self._provided: # Stopped being provided while this round ran: withdraw what was just stored for node_id in accepted[cid]: self._withdrawals.setdefault(node_id, set()).add(cid.key) + self._touch_node(node_id) continue if not nodes: self._retry_at[cid] = self.clock() + self.retry_interval continue self._retry_at.pop(cid, None) + self._touch_item(cid) # Nodes this round didn't store it on may still hold an earlier record: withdrawn from too earlier = {n: t for n, t in self._earlier.pop(cid, {}).items() if now - t < PROVIDER_TTL} previous = self._published.pop(cid, None) diff --git a/rnfs/statedb.py b/rnfs/statedb.py new file mode 100644 index 0000000..58c080a --- /dev/null +++ b/rnfs/statedb.py @@ -0,0 +1,560 @@ +"""The state database: everything rnfs persists under /state/, in one SQLite file, rnfs.db. + +- Opened through the standard sqlite3 module, in WAL mode, readable by its owner only (rnfs.db is created 0600, and + SQLite gives its -wal and -shm files the database's mode; on Windows the state directory's access list, + winacl, covers all three). +- One connection per directory per process, shared by every store through connect(), which hands each caller its + own StateDb reference; the last close() closes the connection. One writer at a time is all SQLite allows + anyway, and every load happens at startup, so more connections would buy nothing but SQLITE_BUSY waits. +- transaction() is the only way to write. Its lock is the innermost lock of all (design.md, Threads): a store + takes every lock of its own first, prepares its rows, then writes them, and takes no other lock and does no + network I/O inside. durable=True commits with synchronous=FULL (on disk when the call returns: pins, names, + jobs and the other stores saved on each change), False with NORMAL (stores saved periodically: a power loss + may lose the last commits, never part of one, and never a commit without every earlier one). +- The schema is a list of steps, SCHEMA; PRAGMA user_version counts those applied. A database from a newer rnfs + (more steps than this code knows) or one failing PRAGMA quick_check stops startup, and is left as it is. +- At every connect(), migrate() imports the JSON state files earlier versions wrote (LEGACY), each in one + transaction recorded in the migrated table, then renames it to .migrated. A crash between the commit and + the rename leaves the record, so the next start only renames the file: nothing is imported twice or by half. + +Stores keep their state in memory and read the database only when they load. Text that may hold lone +surrogates (file names, remote strings) is stored through dump() and load(); sequences, which may not fit +SQLite's signed 64-bit integers, as decimal text. +""" +import importlib +import json +import os +import stat +import threading +import time +from collections.abc import Callable, Iterator +from contextlib import contextmanager +from dataclasses import dataclass + +from rnfs import fsutil +from rnfs.errors import RnfsError +from rnfs.log import LOG_ERROR, LOG_NOTICE, log, plural + +DB_FILE = "rnfs.db" +BUSY_TIMEOUT = 10.0 # seconds a commit waits for another process's (two daemons on one directory) +JOURNAL_SIZE_LIMIT = 4 * 1024 * 1024 # bytes the WAL file is truncated to after a checkpoint +MIGRATED_SUFFIX = ".migrated" + +# Each step brings PRAGMA user_version from its index to its index + 1, in one transaction. Steps are only ever +# appended: a store whose layout changes gets a new step (ALTER TABLE, or a new table and an INSERT ... SELECT). +SCHEMA: tuple[tuple[str, tuple[str, ...]], ...] = ( + ("statedb", ( + # JSON state files imported, so each is imported once (migrate) + "CREATE TABLE migrated (file TEXT PRIMARY KEY, records INTEGER NOT NULL, at REAL NOT NULL)", + # Small per-store values: (store, key) -> value + "CREATE TABLE meta (store TEXT NOT NULL, key TEXT NOT NULL, value, PRIMARY KEY (store, key)) WITHOUT ROWID", + )), + ("storage", ( + # One row per pin source: pinning is an INSERT, releasing a DELETE + "CREATE TABLE pins (cid BLOB NOT NULL, source TEXT NOT NULL, PRIMARY KEY (cid, source)) WITHOUT ROWID", + # Cached fetches; path is the dump()ed list of names, used orders them (least recently used lowest) + "CREATE TABLE cache (root BLOB NOT NULL, path TEXT NOT NULL, target BLOB NOT NULL, used INTEGER NOT NULL," + " PRIMARY KEY (root, path)) WITHOUT ROWID", + )), + ("pin_names", ( + "CREATE TABLE pin_names (cid BLOB PRIMARY KEY, name TEXT NOT NULL) WITHOUT ROWID", # name: dump()ed + )), + ("node", ( + # Targets fetched for the user (the roots strategy) + "CREATE TABLE fetched (cid BLOB PRIMARY KEY) WITHOUT ROWID", + # Remote pins accepted and not yet fetched, in the order accepted (position) + "CREATE TABLE remote_pins (position INTEGER PRIMARY KEY, source TEXT NOT NULL, cid BLOB NOT NULL)", + )), + ("fetcher", ( + # Roots found invalid, oldest (lowest position) first + "CREATE TABLE invalid (cid BLOB PRIMARY KEY, position INTEGER NOT NULL) WITHOUT ROWID", + )), + ("publisher", ( + # item: str() of a provided Cid or DhtKey, as published.json keyed it; nodes: their IDs concatenated + "CREATE TABLE published (item TEXT PRIMARY KEY, at REAL NOT NULL, nodes BLOB NOT NULL) WITHOUT ROWID", + "CREATE TABLE published_earlier (item TEXT NOT NULL, node BLOB NOT NULL, at REAL NOT NULL," + " PRIMARY KEY (item, node)) WITHOUT ROWID", + "CREATE TABLE withdrawals (node BLOB NOT NULL, key BLOB NOT NULL, PRIMARY KEY (node, key)) WITHOUT ROWID", + )), + ("providers", ( + "CREATE TABLE provider_publishers (publisher BLOB PRIMARY KEY, first_seen REAL NOT NULL) WITHOUT ROWID", + "CREATE TABLE provider_records (key BLOB NOT NULL, publisher BLOB NOT NULL, first_stored REAL NOT NULL," + " last_stored REAL NOT NULL, pooled INTEGER NOT NULL, PRIMARY KEY (key, publisher)) WITHOUT ROWID", + "CREATE TABLE provider_departed (publisher BLOB PRIMARY KEY, first_seen REAL NOT NULL, at REAL NOT NULL)" + " WITHOUT ROWID", + "CREATE TABLE provider_evicted (publisher BLOB PRIMARY KEY, first_seen REAL NOT NULL, at REAL NOT NULL)" + " WITHOUT ROWID", + )), + ("names", ( + # Names published here, by alias; sequences are decimal text (up to 2^64 - 1, past SQLite's integers) + "CREATE TABLE names_published (alias TEXT PRIMARY KEY, sequence TEXT NOT NULL, cid BLOB," + " imported INTEGER NOT NULL) WITHOUT ROWID", + "CREATE TABLE names_held (name BLOB PRIMARY KEY, sequence TEXT NOT NULL, cid BLOB NOT NULL," + " heard REAL NOT NULL) WITHOUT ROWID", + # Highest sequence of names resolved without persist; used orders them (least recently resolved lowest) + "CREATE TABLE name_floor (name BLOB PRIMARY KEY, sequence TEXT NOT NULL, resolved REAL NOT NULL," + " used INTEGER NOT NULL) WITHOUT ROWID", + )), + ("shared_folder", ( + # name: the dump()ed entry name (a file name, which may not be valid Unicode) + "CREATE TABLE shared_folder (name TEXT PRIMARY KEY, signature TEXT NOT NULL, cid BLOB NOT NULL)" + " WITHOUT ROWID", + )), + ("jobs", ( + # record: the dump()ed Job.to_record(); position keeps creation order + "CREATE TABLE jobs (id TEXT PRIMARY KEY, position INTEGER NOT NULL, record TEXT NOT NULL) WITHOUT ROWID", + )), + ("keepers", ( + # reasons: the dump()ed sorted list; record: the dump()ed list of the record's fields in hex, or NULL + "CREATE TABLE keepers (name BLOB PRIMARY KEY, reasons TEXT NOT NULL, record TEXT) WITHOUT ROWID", + )), + ("follow", ( + "CREATE TABLE follow (name BLOB PRIMARY KEY, pinned BLOB) WITHOUT ROWID", + )), + ("contacts", ( + "CREATE TABLE contacts (nick TEXT PRIMARY KEY, kind TEXT NOT NULL, id BLOB NOT NULL) WITHOUT ROWID", + )), + ("lxmf", ( + # box: "inbox" or "outbox"; record: the dump()ed message record (its id is the id column) + "CREATE TABLE lxmf_messages (box TEXT NOT NULL, id INTEGER NOT NULL, record TEXT NOT NULL," + " PRIMARY KEY (box, id)) WITHOUT ROWID", + )), +) + + +class StateDbError(OSError): + """A state database read or write failed (a full or failing disk, a closed database). An OSError, so callers + that handle a failed save of a state file handle this the same way.""" + + +# Text columns + + +def dump(value) -> str: + """value as JSON text for a TEXT column: non-ASCII characters as is, a lone surrogate (from a file name that + isn't UTF-8, or a remote string) as its JSON escape, which SQLite can store and load() turns back into it.""" + return json.dumps(value, ensure_ascii=False).encode("utf-8", "backslashreplace").decode("utf-8") + + +def load(text: str): + """The value dump() wrote. Raises ValueError (or TypeError) if text isn't JSON.""" + return json.loads(text) + + +def set_meta(tx, store: str, key: str, value): + """Set store's meta value key inside transaction tx (an integer, real, text or bytes).""" + tx.execute("INSERT OR REPLACE INTO meta (store, key, value) VALUES (?, ?, ?)", (store, key, value)) + + +# The database + + +def _sqlite3(): + """The sqlite3 module. Raises RnfsError if this Python was built without it.""" + try: + import sqlite3 + except ImportError as e: + raise RnfsError(f"this Python has no SQLite support ({e}); rnfs keeps its state in an SQLite database, " + "so it needs a Python built with the sqlite3 module (_sqlite3)") from None + return sqlite3 + + +class _Database: + """One open state database, shared by every StateDb reference to it in this process.""" + + def __init__(self, state_dir: str): + """Open (creating it if missing) and check the database in state_dir, and bring its schema up to date. + Raises RnfsError if it can't be opened, is damaged or is newer than this code.""" + self.sqlite3 = _sqlite3() + self.state_dir = state_dir + self.path = os.path.join(state_dir, DB_FILE) + self.lock = threading.Lock() # innermost: held for each transaction and read, never while taking another + self._owner: int | None = None # the thread inside a transaction + self.migrate_lock = threading.Lock() # one migrate() at a time, so no file is imported twice + self.references = 0 + self.synchronous: str | None = None + _create_private(self.path) + try: + self.conn = self.sqlite3.connect(self.path, timeout=BUSY_TIMEOUT, isolation_level=None, + check_same_thread=False) + except self.sqlite3.Error as e: + raise RnfsError(f"could not open the state database {self.path}: {e}") from None + try: + self._check() + self._upgrade() + except BaseException: + self.conn.close() + raise + + def _check(self): + """Switch to WAL mode and run quick_check. Raises RnfsError if the file isn't a sound database.""" + try: + self.conn.execute(f"PRAGMA busy_timeout = {int(BUSY_TIMEOUT * 1000)}") + mode = self.conn.execute("PRAGMA journal_mode = WAL").fetchone()[0] + if str(mode).lower() != "wal": + raise RnfsError(f"could not switch the state database {self.path} to WAL mode (it is in {mode})") + self.conn.execute(f"PRAGMA journal_size_limit = {JOURNAL_SIZE_LIMIT}") + problems = [row[0] for row in self.conn.execute("PRAGMA quick_check").fetchall()] + except self.sqlite3.DatabaseError as e: + self._raise_open_error(e) + if problems != ["ok"]: + raise RnfsError(damaged_message(self.path, "; ".join(str(p) for p in problems[:5]))) + + def _raise_open_error(self, e): + """Raise RnfsError for a failure opening the database: damaged if SQLite found it corrupt or not a + database; otherwise what failed (a read-only directory, a disk error, a lock), which isn't damage.""" + code = getattr(e, "sqlite_errorcode", None) + if code is not None and code & 0xFF in (self.sqlite3.SQLITE_CORRUPT, self.sqlite3.SQLITE_NOTADB): + raise RnfsError(damaged_message(self.path, str(e))) from None + raise RnfsError(f"could not open the state database {self.path}: {e}. It was left as it is: check that the " + "state directory is writable by this user and the disk works, then start again") from None + + def _upgrade(self): + """Apply the schema steps this database lacks, each in its own transaction. Raises RnfsError if it was + written by a newer rnfs.""" + try: + version = self.conn.execute("PRAGMA user_version").fetchone()[0] + except self.sqlite3.DatabaseError as e: + self._raise_open_error(e) + if version > len(SCHEMA): + raise RnfsError(f"the state database {self.path} was written by a newer version of rnfs (schema " + f"{version}; this version knows up to {len(SCHEMA)}); run that version or a later " + "one (rnfs never moves state back to an older format)") + for number in range(version, len(SCHEMA)): + with self.transaction(durable=True) as tx: + for statement in SCHEMA[number][1]: + tx.execute(statement) + tx.execute(f"PRAGMA user_version = {number + 1}") + + @contextmanager + def transaction(self, durable: bool) -> Iterator: + """A write transaction (BEGIN IMMEDIATE ... COMMIT), yielding the connection to execute statements on. + Rolled back if the body raises. sqlite3 errors are raised as StateDbError.""" + self._not_inside() + with self.lock: + self._owner = threading.get_ident() + try: + conn = self._conn() + try: + level = "FULL" if durable else "NORMAL" + if self.synchronous != level: + conn.execute(f"PRAGMA synchronous = {level}") # takes effect from the next commit + self.synchronous = level + conn.execute("BEGIN IMMEDIATE") + except self.sqlite3.Error as e: + raise StateDbError(f"state database {self.path}: {e}") from e + try: + yield conn + self._commit(conn) + except BaseException as e: + if conn.in_transaction: + try: + conn.execute("ROLLBACK") + except self.sqlite3.Error: + pass # a failed COMMIT may have rolled back already + # A value SQLite can't take (an integer past 64 bits, text with a lone surrogate) fails as the + # disk would, so callers that survive a failed save survive it too + if isinstance(e, (self.sqlite3.Error, OverflowError, UnicodeEncodeError)): + raise StateDbError(f"state database {self.path}: {e}") from e + raise + finally: + self._owner = None + + def _not_inside(self): + """Raise RuntimeError if this thread is inside a transaction already (the lock isn't reentrant).""" + if self._owner == threading.get_ident(): + raise RuntimeError("the state database was used inside one of its own transactions") + + def _commit(self, conn): + """Commit the open transaction (tests replace this to make a commit fail).""" + conn.execute("COMMIT") + + def query(self, sql: str, params=()) -> list[tuple]: + """Every row sql gives. sqlite3 errors are raised as StateDbError.""" + self._not_inside() + with self.lock: + try: + return self._conn().execute(sql, params).fetchall() + except self.sqlite3.Error as e: + raise StateDbError(f"state database {self.path}: {e}") from e + + def _conn(self): + """The connection; StateDbError once closed.""" + if self.conn is None: + raise StateDbError(f"the state database {self.path} is closed") + return self.conn + + def close(self): + """Close the connection (SQLite checkpoints the WAL into the database file).""" + with self.lock: + if self.conn is not None: + try: + self.conn.close() + finally: + self.conn = None + + +def damaged_message(path: str, problem: str) -> str: + """The startup error for a damaged state database: what is wrong and how to recover.""" + directory = os.path.dirname(path) + return (f"the state database {path} is damaged ({problem}). rnfs has left it as it is and won't start on it. " + f"To recover, stop rnfs and move {DB_FILE} (with {DB_FILE}-wal and {DB_FILE}-shm, if there) out of " + f"{directory}: rnfs then starts with fresh state. The JSON files of the version you upgraded from are " + f"kept there as {MIGRATED_SUFFIX}; renamed back to , they are imported again at the next " + "start (everything changed since the upgrade is lost). Blocks and keys are kept in any case.") + + +def _create_private(path: str): + """Create the database file readable by its owner only, before SQLite opens it, and tighten an existing + one (and its -wal and -shm files, whose mode SQLite copies from it) that others could read.""" + try: + fd = os.open(path, os.O_RDWR | os.O_CREAT | fsutil.O_BINARY, 0o600) + except OSError as e: + raise RnfsError(f"could not create the state database {path}: {e}") from None + os.close(fd) + if fsutil.IS_WINDOWS: + return # no mode bits there: the state directory's access list (winacl) covers what is created in it + for name in (path, path + "-wal", path + "-shm"): + try: + info = os.stat(name) + if stat.S_IMODE(info.st_mode) & 0o077: + os.chmod(name, 0o600) + except FileNotFoundError: + continue + except OSError as e: + raise RnfsError(f"could not make {name} private to its owner: {e}") from None + + +_open: dict[str, _Database] = {} # by normalised state directory +_open_lock = threading.Lock() + + +class StateDb: + """One store's reference to the state database of a state directory (see connect).""" + + def __init__(self, database: _Database): + """A reference to database; use connect() rather than making one directly.""" + self._database: _Database | None = database + self.path = database.path + + def _db(self) -> _Database: + """The shared database; StateDbError once this reference is closed.""" + database = self._database + if database is None: + raise StateDbError(f"the state database {self.path} is closed") + return database + + def transaction(self, durable: bool = False): + """A write transaction: `with db.transaction(durable) as tx: tx.execute(...)`, committed when the block + ends, rolled back if it raises. durable: synchronous=FULL, on disk when the block ends; else NORMAL. + + Take it only with every lock the change needs already held, and inside it take no other lock and do no + network I/O. Raises StateDbError (an OSError) if the database fails; the change is then not made.""" + return self._db().transaction(durable) + + def query(self, sql: str, params=()) -> list[tuple]: + """Every row of a read (stores read only when they load). Raises StateDbError.""" + return self._db().query(sql, params) + + def meta(self, store: str, key: str, default=None): + """store's meta value key, or default if it was never set.""" + rows = self.query("SELECT value FROM meta WHERE store = ? AND key = ?", (store, key)) + return rows[0][0] if rows else default + + def close(self): + """Release this reference (idempotent); the last one closes the database.""" + with _open_lock: + database, self._database = self._database, None + if database is None: + return + database.references -= 1 + if database.references == 0: + _open.pop(_key(database.state_dir), None) + database.close() + + +def _key(state_dir: str) -> str: + """The registry key for a state directory.""" + return os.path.normcase(os.path.realpath(state_dir)) + + +def connect(state_dir: str) -> StateDb: + """A reference to the state database in state_dir (created 0700 if missing), opening it if this process + hasn't, and importing any JSON state files found there first (migrate). Close it when done. + + Raises RnfsError if the database can't be opened, is damaged or is newer than this code, or if migrating a + file must stop startup (an unreadable pins.json, a damaged names.json), and OSError if a file can't be read.""" + state_dir = os.path.expanduser(state_dir) + os.makedirs(state_dir, mode=0o700, exist_ok=True) + with _open_lock: + key = _key(state_dir) + database = _open.get(key) + if database is None: + database = _Database(state_dir) + _open[key] = database + database.references += 1 + db = StateDb(database) + try: + with database.migrate_lock: + migrate(db, state_dir) + except BaseException: + db.close() + raise + return db + + +def open_databases() -> list[str]: + """Paths of the state databases this process has open (tests check none is left open).""" + with _open_lock: + return [database.path for database in _open.values()] + + +# Migration of the JSON state files of rnfs 0.1.4 and earlier + + +@dataclass(frozen=True) +class Legacy: + """A JSON state file and how it is imported. + + read(path, shape) gives its value (None: nothing to import; damaged files are handled as read_json does), + by default read_json. importer(value, tx) checks each record as the store's loader always has (dropping + malformed ones with a warning), replaces the store's rows with them inside transaction tx and returns how + many it imported. Both are "module:function", imported when needed.""" + + file: str + importer: str + shape: Callable[[object], bool] | None = None + read: str | None = None + unreadable_ok: bool = False # an unreadable file (OSError) is logged and left for the next start + + +def shaped(key: str, kind: type) -> Callable[[object], bool]: + """A read_json shape: an object whose key, if present, holds a kind.""" + return lambda value: isinstance(value, dict) and isinstance(value.get(key, kind()), kind) + + +def _is(kind: type) -> Callable[[object], bool]: + """A read_json shape: a value of kind.""" + return lambda value: isinstance(value, kind) + + +LEGACY: tuple[Legacy, ...] = ( + Legacy("pins.json", "rnfs.content.storage:import_pins", shaped("pins", dict)), + Legacy("cache.json", "rnfs.content.storage:import_cache", shaped("fetches", list)), + Legacy("pin_names.json", "rnfs.pin_names:import_names", _is(dict)), + Legacy("fetched.json", "rnfs.node:import_fetched", _is(list)), + Legacy("remote_pins.json", "rnfs.node:import_remote_pins", _is(list), unreadable_ok=True), + Legacy("invalid.json", "rnfs.fetcher:import_invalid", _is(list)), + Legacy("published.json", "rnfs.publisher:import_published", + lambda value: all(shaped(key, dict)(value) for key in ("published", "earlier", "withdraw"))), + Legacy("providers.json", "rnfs.dht.providers:import_records", read="rnfs.dht.providers:read_records"), + Legacy("names.json", "rnfs.names:import_names", read="rnfs.names:read_names"), + Legacy("name-floor.json", "rnfs.names:import_floor", read="rnfs.names:read_floor"), + Legacy("shared_folder.json", "rnfs.content.shared_folder:import_entries", shaped("entries", dict)), + Legacy("jobs.json", "rnfs.features.jobs:import_jobs", shaped("jobs", list)), + Legacy("keepers.json", "rnfs.features.keepers:import_keepers"), + Legacy("follow.json", "rnfs.features.follow:import_follow"), + Legacy("contacts.json", "rnfs.features.contacts:import_contacts"), + Legacy("lxmf_inbox.json", "rnfs.features.lxmf:import_inbox"), + Legacy("lxmf_outbox.json", "rnfs.features.lxmf:import_outbox"), +) + + +def _resolve(target: str) -> Callable: + """The function a "module:function" string names.""" + module, _, name = target.partition(":") + return getattr(importlib.import_module(module), name) + + +def migrate(db: StateDb, state_dir: str): + """Import every legacy JSON state file in state_dir not imported yet, one transaction each, recording it in + the migrated table in the same transaction; then rename it to .migrated and sync the directory. A file + the table already lists (a crash came between the commit and the rename) is only renamed.""" + present = [item for item in LEGACY if os.path.lexists(os.path.join(state_dir, item.file))] + if not present: + return + done = {row[0] for row in db.query("SELECT file FROM migrated")} + for item in present: + path = os.path.join(state_dir, item.file) + if item.file in done: + log(f"state: {item.file} was imported into {DB_FILE} before; finishing its migration", LOG_NOTICE) + _keep_original(path, state_dir) + continue + try: + value = (_resolve(item.read) if item.read else read_json)(path, item.shape) + except OSError as e: + if not item.unreadable_ok: + raise + log(f"state: could not read {path} ({e}); leaving it to import at the next start", LOG_ERROR) + continue + if value is None: + continue # damaged and moved aside (or gone meanwhile): the store starts without it, as before + importer = _resolve(item.importer) + with db.transaction(durable=True) as tx: + count = importer(value, tx) + tx.execute("INSERT INTO migrated (file, records, at) VALUES (?, ?, ?)", (item.file, count, time.time())) + kept = _keep_original(path, state_dir) + log(f"state: imported {item.file} into {DB_FILE} ({plural(count, 'record')})" + + (f"; the original is kept as {os.path.basename(kept)}" if kept else ""), LOG_NOTICE) + + +def _keep_original(path: str, state_dir: str) -> str | None: + """Rename an imported file to .migrated (.migrated.1, ... if taken) and sync the directory; the new path, + or None if it couldn't be renamed (logged: the next start tries again, and won't import it twice).""" + try: + kept = _move_aside(path, MIGRATED_SUFFIX) + fsutil.sync_directory(state_dir) + except OSError as e: + log(f"state: could not rename {path} after importing it ({e}); it is renamed at the next start", LOG_ERROR) + return None + return kept + + +# Reading the JSON state files + + +def read_json(path: str, shape: Callable[[object], bool] | None = None): + """The JSON value in path, or None if it doesn't exist. A damaged file (not JSON, failing shape if given, + or a directory) is moved aside, not lost, and gives None. Raises OSError if path can't be read: a transient + or permission error isn't damage, and starting without the file could lose what it records (pins).""" + try: + with open(path, encoding="utf-8") as f: + value = json.load(f) # an empty file, as a crash can leave, is a ValueError too + except FileNotFoundError: + return None + except ValueError as e: + problem = str(e) + except OSError: + if not os.path.isdir(path): # Windows reports opening a directory as PermissionError + raise + problem = "it is a directory" + else: + if shape is None or shape(value): + return value + problem = "unexpected structure" + set_aside(path, problem) + return None + + +def set_aside(path: str, problem: str): + """Move a damaged state file aside to .damaged (logged with problem), so it isn't lost; if that fails + (a read-only state directory), log that the node starts without it.""" + try: + aside = _move_aside(path, ".damaged") + except OSError as moving: + log(f"storage: {path} is damaged ({problem}) and couldn't be moved aside ({moving}); starting without it", + LOG_ERROR) + return + log(f"storage: {path} is damaged ({problem}); moved to {aside} and starting without it", LOG_ERROR) + + +def _move_aside(path: str, suffix: str) -> str: + """Rename path to path + suffix, or path + suffix + . so an earlier copy is kept; the new path.""" + for n in range(1000): + aside = path + suffix + (f".{n}" if n else "") + try: + fsutil.rename_new(path, aside) + return aside + except FileExistsError: + continue + raise FileExistsError(f"too many {suffix} copies of {path}") diff --git a/roadmap.md b/roadmap.md index 19d90fb..06715c3 100644 --- a/roadmap.md +++ b/roadmap.md @@ -1,17 +1,3 @@ -# 0.1.3 - -## Default Bootstrap Node List -DEFAULT_BOOTSTRAP_NODES global variable: node IDs of official, always-on DHT servers, giving new nodes a first contact when no announces reach them (e.g. behind a LoRa access point) -* Used in the startup join alongside `bootstrap_nodes` (the user's own are tried first), as IPFS does -* Used again, with backoff, whenever fewer than about 3 servers are known, so isolated nodes and small islands rejoin -* Tried in random order, with their own path-request budget, so one offline node doesn't block the rest -* Skipped in private mode, or when a setting turns them off -* After the join they are ordinary DHT servers, with no extra trust - -# 0.1.4 - -Big fabel 5.5 audit and valnerability patch - # 0.1.5 ## CID denylist diff --git a/specification.md b/specification.md index c29b0c8..f2a0940 100644 --- a/specification.md +++ b/specification.md @@ -686,6 +686,11 @@ and repeat. - Blocks are stored in `/blocks/`, one file per CID, written atomically after verification, and synced to disk before any pin or cache record that refers to them is saved, so a power loss can't leave a pin without its blocks or an empty file under a CID name. +- rnfs keeps the rest of its state (pins, cache records, names, provider records, jobs, ...) in one SQLite + database, `/state/rnfs.db`, readable by its owner only. The JSON state files earlier versions wrote + under `/state/` are imported into it at the first start, each once and whole, and kept as + `.migrated`; the upgrade is one-way. A damaged database, or one written by a newer version, stops the + node rather than being started over. **Pins.** - Pins are recursive: a pinned CID keeps every block of its DAG. @@ -696,16 +701,17 @@ and repeat. - `pin rm` removes only the `user` source. Other sources are released by whatever created them: the folder scan; `name publish` or `name rm`; `/pin` with `false` or a remote quota being exceeded; `name unfollow`, or a followed name's newer target once that is complete; removing the LXMF messages that hold it. -- Pins are saved at once; the state that records why a source holds a pin (`names.json`, `follow.json`, the - LXMF inbox) is saved separately, so a crash can leave them disagreeing. At start the node makes `name:` pins - match `names.json` (pinning a recorded target again, releasing the rest), and releases `follow:` pins of +- Pins are saved at once; the state that records why a source holds a pin (the names' records, the followed + names, the LXMF inbox) is saved separately, so a crash can leave them disagreeing. At start the node makes + `name:` pins match the names' records (pinning a recorded target again, releasing the rest), and releases + `follow:` pins of names no longer followed and `lxmf:` pins no saved inbox message records (a queued auto-fetch is fetched again). A followed name's check releases every other target its source holds. -- That release needs the state read whole. If `follow.json` or the inbox file is missing, damaged (moved - aside as `.damaged`) or holds a malformed entry, such pins are kept and recorded again: a `follow:` pin's - name is followed again, and a sender's `lxmf:` pins get a new inbox message listing them, whose removal - releases them. A name key file that is there but can't be read keeps its name's pins and `names.json` - entry; only a missing key file counts as a removed name. +- That release needs the state read whole. If the followed names or the inbox were never saved (or were + imported from a file that was damaged and moved aside as `.damaged`), or hold a malformed entry, such pins + are kept and recorded again: a `follow:` pin's name is followed again, and a sender's `lxmf:` pins get a new + inbox message listing them, whose removal releases them. A name key file that is there but can't be read + keeps its name's pins and records; only a missing key file counts as a removed name. - A pin is provided only once it is complete. Blocks of fetches, adds and imports in progress, and of queued or running jobs (section 11), are protected as if pinned. @@ -759,7 +765,7 @@ could have linked in. It is imported at the next scan after its other links are refuse `/pin` from a source with many pins waiting (rnfs: 64). - The requester learns that a remote pin is complete when the node shows up as a provider (`providers`). - `/pin` is best effort: an accepted pin may still be dropped (quota, or the CID can't be found). rnfs saves the - pins waiting (`state/remote_pins.json`, about once a second and at stop) and resumes them after a restart, + pins waiting (in its state database, about once a second and at stop) and resumes them after a restart, unless their identity has left `remote_pin_identities`; a crash may lose those accepted in the last second. - `rnfs pin remote add|rm ` sends these requests over an identified link. @@ -1166,7 +1172,7 @@ optionally from one node), `pin` (fetch a target completely and pin it with sour `cancelled`. Its result, when done, MUST be what the equivalent API call returns (`POST /api/get`, `/api/pin/add`, `/api/export`), and its error, when failed, the message that call would have reported. - At most `max_jobs` jobs run at once; queued jobs MUST be started oldest first. -- Jobs MUST be persisted in the state directory, written atomically. When the node starts, jobs that were +- Jobs MUST be persisted in the state directory, each change saved atomically and durably. When the node starts, jobs that were queued or running MUST be queued again. A resumed job MUST NOT fetch again blocks that are still stored. - While a job is queued or running, the blocks stored below its root MUST be kept as if pinned, including across a restart, so a resumed job continues where it stopped. A failed or cancelled job's blocks are diff --git a/tests/fixtures/rnfs-0.1.4/README.md b/tests/fixtures/rnfs-0.1.4/README.md new file mode 100644 index 0000000..edb73ed --- /dev/null +++ b/tests/fixtures/rnfs-0.1.4/README.md @@ -0,0 +1,54 @@ +# rnfs 0.1.4 directory fixture + +An rnfs directory exactly as rnfs **0.1.4** (commit `ffd23976e07ce41abc3e603d066472531a98b0cb`) writes it, with +every one of its 17 JSON state files populated. `tests/test_upgrade_014.py` copies it, starts the current code on +the copy and checks that the state database holds everything the files held. + +It was written by `generate.py` (in this directory), running 0.1.4's own code: an offline `RnfsNode` with the +fakes of 0.1.4's own tests, its stores' public methods where they exist, and otherwise their in-memory state set as +their own code sets it, then saved by the store's own save. The generator checks that 0.1.4 loads the result +again without dropping or reconciling away any record. + +## What is in it + +- `state/`: the 17 state files. Notable cases: pins of every source kind (`user`, `folder`, `name:photos`, + `remote:`, `follow:`, `lxmf:`), each matching the file that accounts for it (names.json, + follow.json, the inbox message that auto-fetched it); cache records with Japanese and Greek path names, in + least-recently-used order; a pin name holding a lone surrogate (the path of a file whose name isn't UTF-8); + published, earlier and withdraw sections, with kept names' DHT keys (`key:`); provider records in the main + store and in the newcomer pool, with departed and evicted publishers; a published name, an imported one never + published, held records and a floor (one at the highest sequence, 2^64 - 1); kept names with signed records + (own, followed, user) and one without; jobs done (with a result and warnings), failed and queued; inbox and + outbox messages with non-ASCII and non-BMP text, outbox messages delivered, sent through a propagation node, + and unfinished (outbound, sending, and a direct one that fell back to propagation). +- `names/photos`, `names/imported`: name keys. **Throwaway test keys, generated for this fixture: never use them + for anything else.** +- `files/notes.txt`: the shared folder's one entry (shared_folder.json records its scan). +- `fixture_content.py`: the files 0.1.4 added (data only). There is no `blocks/`: the test adds the same files + with the current code to make the blocks, so every pin is complete and provided, and checks the CIDs it gets + are those 0.1.4 recorded. + +Left out: the node identity (`storage/`, which a node creates), `lxmf/` router state, `api_token` and logs. No +`rnfs.toml`: the settings the test starts with are `SETTINGS` in generate.py (a remote pin identity, an LXMF +auto-fetch sender and `max_records = 40`, so the newcomer pool is reachable with a few records). + +## Times + +The wall clock was faked: records were written between 3 days before and an hour after T0 = 1792054800 +(2026-10-15 09:00 UTC). The test runs the new code at T0 + 1 day, so no provider record or earlier publish +round has expired. + +## Regenerating + +From the repository root (the generator inserts the 0.1.4 tree first on `sys.path` and checks that is where +`rnfs` comes from): + + mkdir /tmp/rnfs-0.1.4 && git archive v0.1.4 | tar -x -C /tmp/rnfs-0.1.4 + python3 tests/fixtures/rnfs-0.1.4/generate.py /tmp/rnfs-0.1.4 /tmp/rnfs-fixture-out + +then replace `state/`, `names/` and `files/` here with those in `/tmp/rnfs-fixture-out/rnfs-0.1.4` (the generator +checks 0.1.4 loads everything, with the blocks, before it drops them). +Keys, signatures, job IDs and message hashes are random, so every run writes different bytes; the test reads +its expectations from the files, so it needs no change. + +The fixture is test data only: the sdist prunes `tests/` (MANIFEST.in) and the wheel packages only `rnfs*`. diff --git a/tests/fixtures/rnfs-0.1.4/files/notes.txt b/tests/fixtures/rnfs-0.1.4/files/notes.txt new file mode 100644 index 0000000..713febb --- /dev/null +++ b/tests/fixtures/rnfs-0.1.4/files/notes.txt @@ -0,0 +1 @@ +shared folder notes diff --git a/tests/fixtures/rnfs-0.1.4/fixture_content.py b/tests/fixtures/rnfs-0.1.4/fixture_content.py new file mode 100644 index 0000000..db738d0 --- /dev/null +++ b/tests/fixtures/rnfs-0.1.4/fixture_content.py @@ -0,0 +1,25 @@ +"""The files the rnfs-0.1.4 fixture adds, shared by generate.py (which adds them with rnfs 0.1.4) and +tests/test_upgrade_014.py (which adds them again with the current code, to make the blocks: the fixture keeps no +blocks/ of its own). Data only, so either version of rnfs can import it. +""" + +# Added from a stream (rnfs add -): (data, the name it is wrapped as, or None) +USER = (b"Bonjour \xc3\xa0 tous\n", "Résumé – 2026.txt") +PLAIN = (b"plain bytes from stdin\n", None) +PICTURE = (b"\xff\xd8\xff\xe0 not really a jpeg", "photo.jpg") +ODD = (b"a file named in Latin-1\n", None) # pin_names.json records it under a path that isn't UTF-8 +REMOTE_PIN = (b"pinned for a remote identity\n", None) +KEPT_TARGET = (b"a kept name's target\n", None) +FOLLOWED_TARGET = (b"the latest issue of a followed name\n", None) +RESOLVED_TARGET = (b"resolved through the CLI\n", None) +FEED = (b"auto-fetched from a trusted sender\n", None) +DROPPED = (b"pinned, published, then released\n", None) +STREAMS = (USER, PLAIN, PICTURE, ODD, REMOTE_PIN, KEPT_TARGET, FOLLOWED_TARGET, RESOLVED_TARGET, FEED, DROPPED) + +# Added from a directory (rnfs add ): {name: bytes, or a subdirectory's dict} +ALBUM = {"2026": {"sommer.jpg": b"jpeg bytes", "été.jpg": b"more jpeg"}} +DOCS = {"日本語": {"メモ.md": b"# memo\n"}, "café.txt": b"coffee\n", "Ελληνικά": {"αρχείο.txt": b"greek\n"}} +TREES = (ALBUM, DOCS) + +# The shared folder's one entry, files/notes.txt (checked in, as the folder scan imports it) +NOTES = ("notes.txt", b"shared folder notes\n") diff --git a/tests/fixtures/rnfs-0.1.4/generate.py b/tests/fixtures/rnfs-0.1.4/generate.py new file mode 100644 index 0000000..887dfce --- /dev/null +++ b/tests/fixtures/rnfs-0.1.4/generate.py @@ -0,0 +1,407 @@ +"""Write the tests/fixtures/rnfs-0.1.4 fixture: an rnfs directory exactly as rnfs 0.1.4 writes it, every state +file populated (see README.md there). + +Run it with the rnfs 0.1.4 source tree (git archive v0.1.4, which includes its tests/ package), never the current +one: it puts that tree first on sys.path and checks rnfs is imported from there. From the repository root: + + mkdir /tmp/rnfs-0.1.4 && git archive v0.1.4 | tar -x -C /tmp/rnfs-0.1.4 + python3 tests/fixtures/rnfs-0.1.4/generate.py /tmp/rnfs-0.1.4 /tmp/rnfs-fixture-out + # then replace tests/fixtures/rnfs-0.1.4/{state,names,files} with those in + # /tmp/rnfs-fixture-out/rnfs-0.1.4 + +Everything is done through 0.1.4's own objects: an offline RnfsNode (RnfsNode.offline, with the fakes of 0.1.4's +own tests: tests.test_node.NetClient, tests.test_names.FakeNetwork, tests.test_lxmf.FakeLxmf), its stores' public +methods where they exist, and otherwise their in-memory state set as their own code sets it, then saved by the +store's own save. The wall clock is a fake (time.time patched): records span three days before T0 to about an +hour after it, so the test knows how old every record is. Keys, signatures, job IDs and message hashes are +random, so each run writes different bytes. +""" +import io +import json +import os +import random +import shutil +import sys +from unittest import mock + +V014, OUT = (os.path.abspath(arg) for arg in sys.argv[1:3]) +SCRATCH = os.path.join(OUT, "scratch") # directories add_path imports +sys.path.insert(0, V014) +sys.path.insert(1, os.path.dirname(os.path.abspath(__file__))) # fixture_content, next to this file + +import RNS # noqa: E402 + +import rnfs # noqa: E402 + +assert rnfs.__file__.startswith(V014 + os.sep), rnfs.__file__ +assert rnfs.__version__ == "0.1.4", rnfs.__version__ + +from rnfs.config import Config # noqa: E402 +from rnfs.content.cid import RAW, Cid # noqa: E402 +from rnfs.content.storage import remote_source # noqa: E402 +from rnfs.features import keepers as keepers_module # noqa: E402 +from rnfs.features.lxmf import Incoming, Link, encode_field # noqa: E402 +from rnfs.names import encode_record, name_for_identity # noqa: E402 +from rnfs.node import RnfsNode # noqa: E402 +from tests.test_lxmf import FakeLxmf # noqa: E402 +from tests.test_names import FakeNetwork # noqa: E402 +from tests.test_node import NetClient # noqa: E402 + +import fixture_content as content # noqa: E402 + +DAY, HOUR = 86400.0, 3600.0 +T0 = 1792054800.0 # 2026-10-15 09:00 UTC, just after 0.1.4 was tagged: the fake wall clock's start +NODE_ID = bytes.fromhex("4e0de1d04e0de1d04e0de1d04e0de1d0") +REMOTE = "0123456789abcdef0123456789abcdef" # an identity on remote_pin_identities +AUTO_SENDER = bytes.fromhex("a0a1a2a3a4a5a6a7a8a9aaabacadaeaf") # LXMF sender on lxmf_auto_fetch_from +OTHER_SENDER = bytes.fromhex("b0b1b2b3b4b5b6b7b8b9babbbcbdbebf") # an LXMF sender not trusted +LXMF_ADDRESS = bytes.fromhex("c0c1c2c3c4c5c6c7c8c9cacbcccdcecf") # this node's LXMF address +PROPAGATION_NODE = bytes.fromhex("d0d1d2d3d4d5d6d7d8d9dadbdcdddedf") +PEERS = [bytes([0x10 + i]) * 16 for i in range(4)] # DHT nodes published to +# Settings the test starts the node with too (none of them is written to rnfs.toml) +SETTINGS = {"remote_pin_identities": [REMOTE], "lxmf_auto_fetch_from": [AUTO_SENDER.hex()], "max_records": 40, + "slow": "no", "path_timeout": 2, "lookup_timeout": 2} + + +class Clock: + """The fake wall clock (time.time), moved by hand.""" + + def __init__(self, now: float): + """Start at now.""" + self.now = now + + def __call__(self) -> float: + """The current time.""" + return self.now + + +def build(root: str): + """Create and populate the rnfs directory root with an offline 0.1.4 node.""" + clock = Clock(T0) + with mock.patch("time.time", clock): + config = Config(rnfs_dir=root, **SETTINGS) + config.validate() + network = FakeNetwork() + node = RnfsNode.offline(config, NODE_ID, NetClient(), network) + # These took time.time as a default argument, before it was patched (they loaded nothing: a new directory) + node.records.clock = node.publisher.clock = node.names._clock = clock + try: + populate(node, network, clock) + finally: + node.stop() + + +def providers_phase(node: RnfsNode, clock: Clock): + """providers.json: other publishers' records, stored over the three days before T0 (the clock moves only + forward, as the store clamps times in the future back to now): seniors in the main store, one departed, + newcomers in the newcomer pool and one evicted from it. The node's own records are stored later, at T0, by + the publisher (evicting records of the newest publisher outside the pool, as own records always get in).""" + records = node.records + records.rng = random.Random(14) + clock.now = T0 - 3 * DAY + publishers = [bytes([0x30 + i]) + bytes(15) for i in range(12)] + + def key(i: int) -> bytes: + """A DHT key.""" + return bytes([0x80, i]) + bytes(14) + + for i in range(8): + records.store(key(i), publishers[0]) # a senior + for i in range(4, 10): + records.store(key(i), publishers[1]) + records.store(key(20), publishers[2]) # departs + clock.now = T0 - 1.5 * DAY + records.withdraw(key(20), publishers[2]) + assert publishers[2] in records._departed + filler, i = 3, 30 + while records._main_count < records.main_size: + clock.now += 60 + if not records.store(key(i), publishers[filler]): + filler += 1 + i += 1 + clock.now = T0 - HOUR + newcomers = publishers[filler + 1:filler + 1 + records.pool_size + 1] + for n, newcomer in enumerate(newcomers): + clock.now += 60 + assert records.store(key(100 + n), newcomer) + assert records._evicted and records._pool_count == records.pool_size, (records._evicted, records._pool_count) + clock.now = T0 + return lambda: records.store(key(0), publishers[0]) # refreshed at the end: first stored 3 days before + + +def populate(node: RnfsNode, network: FakeNetwork, clock: Clock): + """Every state file, through the node's stores.""" + refresh_senior = providers_phase(node, clock) + storage, names = node.storage, node.names + jobs, keepers, follow = node.feature("jobs"), node.feature("keepers"), node.feature("follow") + contacts, lxmf = node.feature("contacts"), node.feature("lxmf") + + def tick(seconds: float = 60.0): + """Let time pass.""" + clock.now += seconds + + def add(data: bytes, name: str | None = None, pin: bool = True) -> Cid: + """add_stream (add of stdin, wrapped as name if given).""" + tick() + return Cid.parse(node.add_stream(io.BytesIO(data), name=name, pin=pin)["cid"]) + + def add_tree(tree: dict, pin: bool) -> Cid: + """add_path of a directory holding tree ({name: bytes or subtree}).""" + tick() + directory = os.path.join(SCRATCH, f"tree-{clock.now:.0f}") + + def write(path: str, entries: dict): + """Create path holding entries.""" + os.makedirs(path) + for entry, value in entries.items(): + if isinstance(value, dict): + write(os.path.join(path, entry), value) + else: + with open(os.path.join(path, entry), "wb") as f: + f.write(value) + + write(directory, tree) + return Cid.parse(node.add_path(directory, pin=pin)["cid"]) + + def child(root: Cid, *path: str) -> Cid: + """The CID at root/path.""" + return Cid.parse(node.stat("/".join([str(root), *path]))["cid"]) + + # pins.json: user, with pin_names.json + user = add(*content.USER) # pin name: the name it was wrapped as + plain = add(*content.PLAIN) + picture = add(*content.PICTURE) + storage.pin(picture, "user") + node.pin_names.record(picture, "/home/alice/Pictures/Fête à Montréal/photo \U0001f389.jpg") + # add of a file whose name isn't UTF-8: its path holds a lone surrogate (os.fsdecode's surrogateescape) + odd = add(*content.ODD) + node.pin_names.record(odd, "/home/alice/Downloads/b\udcffd name.txt") + + # folder: a real file in files/, scanned + os.makedirs(node.config.files_dir, exist_ok=True) + with open(os.path.join(node.config.files_dir, content.NOTES[0]), "wb") as f: + f.write(content.NOTES[1]) + tick() + node.shared_folder.scan() + node.publisher.refresh() + + # name:, names.json, a key in names/photos, keepers' own record + album = add_tree(content.ALBUM, pin=False) + names.create("photos") + tick() + node.name_publish("photos", str(album)) + imported = RNS.Identity() + names.import_key("imported", imported.get_private_key()) # imported, never published here + + # remote:, and remote pins waiting (remote_pins.json) + remote = add(*content.REMOTE_PIN, pin=False) + storage.pin(remote, remote_source(REMOTE)) + waiting = [Cid.for_block(RAW, b"waiting %d" % i) for i in range(2)] + with node._lock: + node._remote_reserved[remote_source(REMOTE)] = {cid: None for cid in waiting} + node._remote_pins_dirty = True + node._save_remote_pins() + + # cache.json and fetched.json: add --no-pin records, a path into a directory with non-ASCII names + docs = add_tree(content.DOCS, pin=False) + storage.record_fetch(docs, ["日本語", "メモ.md"], child(docs, "日本語", "メモ.md")) + storage.record_fetch(docs, ["Ελληνικά", "αρχείο.txt"], child(docs, "Ελληνικά", "αρχείο.txt")) + node._note_fetched(child(docs, "Ελληνικά", "αρχείο.txt")) + # a gateway fetch: cached, never noted as fetched for the user (so never provided) + storage.record_fetch(docs, ["caf\u00e9.txt"], child(docs, "caf\u00e9.txt")) + storage.touch(album, []) # used again: most recently used + tick() + + # invalid.json + for i in range(3): + node.fetcher.mark_invalid(Cid.for_block(RAW, b"invalid %d" % i)) + + # keepers.json: a user-kept name with a record signed by its owner, one with none yet + kept_owner = RNS.Identity() + kept_name = name_for_identity(kept_owner) + kept_target = add(*content.KEPT_TARGET, pin=False) + keepers.keep(kept_name, "user") + tick() + keepers._store(keepers_module.sign_record(kept_owner, kept_name, encode_record(7, kept_target))) + unanswered = name_for_identity(RNS.Identity()) + keepers.keep(unanswered, "user") + + # follow:, follow.json; the followed name's record is kept and held (names.json "held") + followed_owner = RNS.Identity() + followed = name_for_identity(followed_owner) + followed_target = add(*content.FOLLOWED_TARGET, pin=False) + tick() + follow.follow(followed) + keepers._store(keepers_module.sign_record(followed_owner, followed, encode_record(12, followed_target))) + tick() + assert follow.check(followed, network=False) + + # names.json "held": a name resolved through the CLI; name-floor.json: one resolved for the gateway + resolved_owner_target = add(*content.RESOLVED_TARGET, pin=False) + held = name_for_identity(RNS.Identity()) + tick() + network.store(held, encode_record(3, resolved_owner_target), clock.now) + assert names.resolve(held, persist=True) is not None + for i, sequence in enumerate((5, 2**64 - 1)): + floor_name = name_for_identity(RNS.Identity()) + tick() + network.store(floor_name, encode_record(sequence, plain), clock.now) + assert names.resolve(floor_name, persist=False) is not None + + # contacts.json: one of each kind + contacts.add("alice", followed.hex(), kind="name") + contacts.add("relay.1", PEERS[0].hex(), kind="node") + contacts.add("bob", OTHER_SENDER.hex(), kind="lxmf") + + # lxmf_inbox.json: a message from an auto-fetch sender, both links fetched and pinned (lxmf:) + feed = add(*content.FEED, pin=False) + lxmf.attach(FakeLxmf(address=LXMF_ADDRESS, propagation_node=PROPAGATION_NODE)) + links = [Link(feed, size=36, from_node=PEERS[1]), Link(docs, path=("café.txt",))] + text = "Fotos vom Wochenende \U0001f4f7\n\n" + "\n".join(link.uri() for link in links) + tick() + first = lxmf.received(Incoming(AUTO_SENDER, os.urandom(32), clock.now - 30, "Grüße 🌻".encode(), + text.encode(), encode_field(links), "direct", "Zoë")) + lxmf.process_auto_fetches() + assert all(a["state"] == "pinned" for a in lxmf.inbox.get(first)["auto"]), lxmf.inbox.get(first) + # ... and two from a sender not trusted: a name link in the text only, and non-BMP characters + name_link = Link(name=names.name("photos"), path=("2026", "été.jpg")) + tick() + lxmf.received(Incoming(OTHER_SENDER, os.urandom(32), clock.now - 5, "Привет".encode(), + f"Посмотри это: {name_link.uri()}".encode(), {}, "propagated", "Bøb \U0001f98a")) + tick() + lxmf.received(Incoming(OTHER_SENDER, os.urandom(32), None, b"", + f"\U0001d11e music \U0001f3b5 rnfs:{user}".encode(), {}, "opportunistic", None)) + + # lxmf_outbox.json: delivered, propagated and sent, and three unfinished: a direct one that fell back to + # propagation, one outbound, one sending (0.1.4 marks them failed when it next starts) + tick() + delivered = lxmf.send(OTHER_SENDER.hex(), str(user), text="Voilà le fichier ✅") + lxmf.outbound_changed(bytes.fromhex(delivered["hash"]), "delivered", 1.0) + tick() + propagated = lxmf.send("bob", f"name/{names.name('photos').hex()}/2026", + text="via the propagation node \U0001f4e1", propagate=True) + lxmf.outbound_changed(bytes.fromhex(propagated["hash"]), "sent", None) + tick() + fallback = lxmf.send(OTHER_SENDER.hex(), str(picture), text="") + lxmf.outbound_changed(bytes.fromhex(fallback["hash"]), "failed", None) # resent through the propagation node + tick() + lxmf.send(AUTO_SENDER.hex(), f"{docs}/日本語", text="not delivered yet \u23f3") + tick() + sending = lxmf.send(AUTO_SENDER.hex(), str(plain), text="half way \U0001f69a") + lxmf.outbound_changed(bytes.fromhex(sending["hash"]), "sending", 0.4567) + lxmf.save() + + # jobs.json: a finished get with a result and warnings, a failed pin, and a queued pin (to resume) + tick() + got = jobs.submit("get", f"{docs}/café.txt", output="/home/alice/Downloads/café \U0001f600.txt") + job = jobs._job(got["id"]) + job.begin_run() + job.total(7) + job.block(child(docs, "café.txt"), 7, True) + tick(5) + jobs._finish(job, "done", None, {"cid": str(child(docs, "café.txt")), "size": 7, "warnings": [ + "skipped symlink café/link → elsewhere", "kept \U0001f600 as is"], + "output": job.output}) + tick() + failed = jobs._job(jobs.submit("pin", str(Cid.for_block(RAW, b"nobody has this")))["id"]) + failed.begin_run() + jobs._finish(failed, "failed", "no providers found for this CID", None) + tick() + jobs.submit("pin", f"{docs}/Ελληνικά") + + # published.json: everything provided was published to two peers; earlier rounds' nodes for some; a pin + # released since leaves withdrawals queued for the nodes it was published to + dropped = add(*content.DROPPED) + publisher = node.publisher + publisher.refresh() + tick() + with publisher._lock: + for i, item in enumerate(sorted(publisher._provided, key=str)): + publisher._published[item] = (clock.now - i * HOUR, [PEERS[i % 4], PEERS[(i + 1) % 4]]) + if i % 2 == 0: + publisher._earlier[item] = {PEERS[(i + 2) % 4]: clock.now - 3 * DAY} + storage.unpin(dropped, "user") + publisher.refresh() # saves: withdrawals of dropped's key queued for PEERS + assert publisher._withdrawals, "no withdrawals queued" + tick() + refresh_senior() + + +def check(root: str): + """Start 0.1.4 again on a copy of root, a day later: it must load every record (the fixture is consistent: + nothing is reconciled away). Its clocks are faked as the test fakes the new code's.""" + import functools + + from rnfs import node as node_module + + copy = root + "-check" + shutil.copytree(root, copy) + clock = Clock(T0 + DAY) + + def state(name: str): + """A state file of the fixture.""" + with open(os.path.join(root, "state", name), encoding="utf-8") as f: + return json.load(f) + + with mock.patch("time.time", clock), \ + mock.patch.object(node_module, "ProviderStore", functools.partial(node_module.ProviderStore, + clock=clock)), \ + mock.patch.object(node_module, "Publisher", functools.partial(node_module.Publisher, clock=clock)), \ + mock.patch.object(node_module, "NameManager", functools.partial(node_module.NameManager, clock=clock)): + config = Config(rnfs_dir=copy, **SETTINGS) + node = RnfsNode.offline(config, NODE_ID, NetClient(), FakeNetwork()) + try: + pins = {Cid.parse(c): set(s) for c, s in state("pins.json")["pins"].items()} + assert node.storage.pins() == pins, (node.storage.pins(), pins) + assert all(node.storage.is_complete(cid) for cid in pins) + assert len(node.storage.cached_fetches()) == len(state("cache.json")["fetches"]) + assert {str(c) for c in node._eligible_targets()} == set(state("fetched.json")) + assert len(node._load_remote_pins()) == len(state("remote_pins.json")) + assert {str(c) for c in node.publisher._published} == set(state("published.json")["published"]) + assert {n.hex() for n in node.publisher._withdrawals} == set(state("published.json")["withdraw"]) + stored = {(k, p) for k, p, *_ in state("providers.json")["records"]} + loaded = {(k.hex(), p.hex()) for k, ps in node.records._records.items() for p in ps} + assert loaded == stored, (loaded ^ stored) + assert len(node.records._departed) == len(state("providers.json")["departed"]) + assert len(node.records._evicted) == len(state("providers.json")["evicted"]) + assert {n.hex() for n in node.names._held} == set(state("names.json")["held"]) + assert len(node.names._floor) == len(state("name-floor.json")["floor"]) + assert len(node.shared_folder._known) == len(state("shared_folder.json")["entries"]) + jobs = node.feature("jobs")._jobs + assert [j for j in jobs] == [j["id"] for j in state("jobs.json")["jobs"]] + keepers = node.feature("keepers").store + assert {n.hex() for n in keepers.names()} == set(state("keepers.json")["names"]) + assert len(keepers._records) == sum(1 for e in state("keepers.json")["names"].values() if e["record"]) + assert {n.hex() for n in node.feature("follow")._pinned} == set(state("follow.json")["names"]) + assert set(node.feature("contacts")._contacts) == set(state("contacts.json")["contacts"]) + lxmf = node.feature("lxmf") + assert lxmf.inbox.clean and len(lxmf.inbox) == len(state("lxmf_inbox.json")["messages"]) + assert len(lxmf.outbox) == len(state("lxmf_outbox.json")["messages"]) + finally: + node.stop() + shutil.rmtree(copy) + + +def prune(root: str): + """Drop what the fixture doesn't need: LXMF router state, the node identity (rnfs makes one), the API token, + logs, and the blocks (the test makes them again from fixture_content).""" + for name in ("lxmf", "api_token", "debug.log", "storage", "blocks"): + path = os.path.join(root, name) + if os.path.isdir(path): + shutil.rmtree(path) + elif os.path.exists(path): + os.remove(path) + + +if __name__ == "__main__": + RNS.loglevel = RNS.LOG_CRITICAL + root = os.path.join(OUT, "rnfs-0.1.4") + for path in (root, SCRATCH): + if os.path.exists(path): + shutil.rmtree(path) + os.makedirs(SCRATCH) + build(root) + check(root) # with the blocks, before they are pruned + prune(root) + shutil.rmtree(SCRATCH) + print(root) diff --git a/tests/fixtures/rnfs-0.1.4/names/imported b/tests/fixtures/rnfs-0.1.4/names/imported new file mode 100644 index 0000000..decc608 --- /dev/null +++ b/tests/fixtures/rnfs-0.1.4/names/imported @@ -0,0 +1 @@ +b?ub!PhV+Ec#nTfk٥AQ} "Ͱk5 \ No newline at end of file diff --git a/tests/fixtures/rnfs-0.1.4/names/photos b/tests/fixtures/rnfs-0.1.4/names/photos new file mode 100644 index 0000000..d25014a --- /dev/null +++ b/tests/fixtures/rnfs-0.1.4/names/photos @@ -0,0 +1,2 @@ +(񹐤Ex>q? +T6ײQ`2ݍ2@|g?W!DӂUimp[ƃnW \ No newline at end of file diff --git a/tests/fixtures/rnfs-0.1.4/state/cache.json b/tests/fixtures/rnfs-0.1.4/state/cache.json new file mode 100644 index 0000000..46e58ed --- /dev/null +++ b/tests/fixtures/rnfs-0.1.4/state/cache.json @@ -0,0 +1 @@ +{"fetches": [["bafkreihvcmhog4opp7chojb6caafzh7ragthgixbrzcqtg42ei3xfav2nq", [], "bafkreihvcmhog4opp7chojb6caafzh7ragthgixbrzcqtg42ei3xfav2nq"], ["bagaibqabciqjpffm54fysdaxqxjggmfjifzfvoyfsy256jovqmnjy6dpbbpg7lq", [], "bagaibqabciqjpffm54fysdaxqxjggmfjifzfvoyfsy256jovqmnjy6dpbbpg7lq"], ["bagaibqabciqjpffm54fysdaxqxjggmfjifzfvoyfsy256jovqmnjy6dpbbpg7lq", ["\u65e5\u672c\u8a9e", "\u30e1\u30e2.md"], "bafkreibvbnvd5lbk76cb3cq35w77w252b4koramjhn426wy3xjy2vnp6yq"], ["bagaibqabciqjpffm54fysdaxqxjggmfjifzfvoyfsy256jovqmnjy6dpbbpg7lq", ["\u0395\u03bb\u03bb\u03b7\u03bd\u03b9\u03ba\u03ac", "\u03b1\u03c1\u03c7\u03b5\u03af\u03bf.txt"], "bafkreibevdhpo2ydbraaurnjq3wtziaclbsmqpquyh6ntesaf7lmove4vq"], ["bagaibqabciqfbyihpeqy3nfafzl4qcbevvbpvtgaak2wxadkneio5dfclm3m6hi", [], "bagaibqabciqfbyihpeqy3nfafzl4qcbevvbpvtgaak2wxadkneio5dfclm3m6hi"], ["bafkreibkk5apo3fy4r2vriim3bnah5ldx2g2e4zqnjkfstweknpscdmutm", [], "bafkreibkk5apo3fy4r2vriim3bnah5ldx2g2e4zqnjkfstweknpscdmutm"], ["bafkreigyxunorje5oovwdvvvtsjcb5tjmvi4lcsyabxv7t5ngcuabk3xh4", [], "bafkreigyxunorje5oovwdvvvtsjcb5tjmvi4lcsyabxv7t5ngcuabk3xh4"], ["bafkreiherkniycepddysts7hctkszprpeyqcb35geojps2mcapym6x46zi", [], "bafkreiherkniycepddysts7hctkszprpeyqcb35geojps2mcapym6x46zi"], ["bafkreif3aytvfpqujesmyqwwmlhzy3k2w7gfyw7ezitoxbmwzguktd34dm", [], "bafkreif3aytvfpqujesmyqwwmlhzy3k2w7gfyw7ezitoxbmwzguktd34dm"], ["bagaibqabciqjpffm54fysdaxqxjggmfjifzfvoyfsy256jovqmnjy6dpbbpg7lq", ["caf\u00e9.txt"], "bafkreicllqx56gk2tdcidha3bgi7bcwco4p6v24ij5kvymvq5dxwdf27ae"]]} \ No newline at end of file diff --git a/tests/fixtures/rnfs-0.1.4/state/contacts.json b/tests/fixtures/rnfs-0.1.4/state/contacts.json new file mode 100644 index 0000000..924b480 --- /dev/null +++ b/tests/fixtures/rnfs-0.1.4/state/contacts.json @@ -0,0 +1 @@ +{"version": 1, "contacts": {"alice": {"kind": "name", "id": "bcc7900726de195e4cc4b09e67f02966"}, "bob": {"kind": "lxmf", "id": "b0b1b2b3b4b5b6b7b8b9babbbcbdbebf"}, "relay.1": {"kind": "node", "id": "10101010101010101010101010101010"}}} \ No newline at end of file diff --git a/tests/fixtures/rnfs-0.1.4/state/fetched.json b/tests/fixtures/rnfs-0.1.4/state/fetched.json new file mode 100644 index 0000000..4b6b62a --- /dev/null +++ b/tests/fixtures/rnfs-0.1.4/state/fetched.json @@ -0,0 +1 @@ +["bafkreibevdhpo2ydbraaurnjq3wtziaclbsmqpquyh6ntesaf7lmove4vq", "bafkreibkk5apo3fy4r2vriim3bnah5ldx2g2e4zqnjkfstweknpscdmutm", "bafkreif3aytvfpqujesmyqwwmlhzy3k2w7gfyw7ezitoxbmwzguktd34dm", "bafkreigyxunorje5oovwdvvvtsjcb5tjmvi4lcsyabxv7t5ngcuabk3xh4", "bafkreiherkniycepddysts7hctkszprpeyqcb35geojps2mcapym6x46zi", "bafkreihvcmhog4opp7chojb6caafzh7ragthgixbrzcqtg42ei3xfav2nq", "bagaibqabciqfbyihpeqy3nfafzl4qcbevvbpvtgaak2wxadkneio5dfclm3m6hi", "bagaibqabciqjpffm54fysdaxqxjggmfjifzfvoyfsy256jovqmnjy6dpbbpg7lq"] \ No newline at end of file diff --git a/tests/fixtures/rnfs-0.1.4/state/follow.json b/tests/fixtures/rnfs-0.1.4/state/follow.json new file mode 100644 index 0000000..9532fd3 --- /dev/null +++ b/tests/fixtures/rnfs-0.1.4/state/follow.json @@ -0,0 +1 @@ +{"version": 1, "names": {"bcc7900726de195e4cc4b09e67f02966": {"pinned": "bafkreigyxunorje5oovwdvvvtsjcb5tjmvi4lcsyabxv7t5ngcuabk3xh4"}}} \ No newline at end of file diff --git a/tests/fixtures/rnfs-0.1.4/state/invalid.json b/tests/fixtures/rnfs-0.1.4/state/invalid.json new file mode 100644 index 0000000..9bafe2f --- /dev/null +++ b/tests/fixtures/rnfs-0.1.4/state/invalid.json @@ -0,0 +1 @@ +["bafkreicuzem5bjal4mwj4kmzavvzoqkhtyt47mz7t3tsbqswqdohetk4w4", "bafkreieqpx5nxvsmjzt5vv7kk7d2cu7n2ky3y3oupplaffenbqx4qodamu", "bafkreigcfjk46gpgsldrp3krvcynvcga2q5a6zt6a7l7tazzh6bcdgbcju"] \ No newline at end of file diff --git a/tests/fixtures/rnfs-0.1.4/state/jobs.json b/tests/fixtures/rnfs-0.1.4/state/jobs.json new file mode 100644 index 0000000..ea3ccfb --- /dev/null +++ b/tests/fixtures/rnfs-0.1.4/state/jobs.json @@ -0,0 +1 @@ +{"jobs": [{"id": "3bd70342", "kind": "get", "target": "bagaibqabciqjpffm54fysdaxqxjggmfjifzfvoyfsy256jovqmnjy6dpbbpg7lq/caf\u00e9.txt", "output": "/home/alice/Downloads/caf\u00e9 \ud83d\ude00.txt", "from": null, "sealed_name": false, "fallback_dht": false, "state": "done", "created": 1792056540.0, "started": 1792056540.0, "finished": 1792056545.0, "error": null, "result": {"cid": "bafkreicllqx56gk2tdcidha3bgi7bcwco4p6v24ij5kvymvq5dxwdf27ae", "size": 7, "warnings": ["skipped symlink caf\u00e9/link \u2192 elsewhere", "kept \ud83d\ude00 as is"], "output": "/home/alice/Downloads/caf\u00e9 \ud83d\ude00.txt"}, "runs": 1, "total_bytes": 7, "done_bytes": 7, "blocks": 1, "blocks_total": null, "downloaded_bytes": 7}, {"id": "6a322a52", "kind": "pin", "target": "bafkreiemqpndzewx4bioklzfvhtiegsfny5fxuc3nkjk2vmcpjxiammsti", "output": null, "from": null, "sealed_name": false, "fallback_dht": false, "state": "failed", "created": 1792056605.0, "started": 1792056605.0, "finished": 1792056605.0, "error": "no providers found for this CID", "result": null, "runs": 1, "total_bytes": null, "done_bytes": 0, "blocks": 0, "blocks_total": null, "downloaded_bytes": 0}, {"id": "da44b5c5", "kind": "pin", "target": "bagaibqabciqjpffm54fysdaxqxjggmfjifzfvoyfsy256jovqmnjy6dpbbpg7lq/\u0395\u03bb\u03bb\u03b7\u03bd\u03b9\u03ba\u03ac", "output": null, "from": null, "sealed_name": false, "fallback_dht": false, "state": "queued", "created": 1792056665.0, "started": null, "finished": null, "error": null, "result": null, "runs": 0, "total_bytes": null, "done_bytes": 0, "blocks": 0, "blocks_total": null, "downloaded_bytes": 0}]} \ No newline at end of file diff --git a/tests/fixtures/rnfs-0.1.4/state/keepers.json b/tests/fixtures/rnfs-0.1.4/state/keepers.json new file mode 100644 index 0000000..08ffc5d --- /dev/null +++ b/tests/fixtures/rnfs-0.1.4/state/keepers.json @@ -0,0 +1 @@ +{"version": 1, "names": {"05c7214ca11e7646c79340c6b6d9866c": {"reasons": ["own"], "record": ["e5ee52baf6bf767a65fc00d5acf5231c3ab7509438494eec22e4cdb9eae33d3cb4d1e823db4955856e0f57c46987475752a2aa95721399815e3f955d98e23cb2", "783c4f3005a78c29de83", "2e62f40c0a006ad097b4", "", "eb5f4d8539b58b495781c8ba83765ed04318d1718032b8d2af0ad694b340480aa6a9863e4fa057aedb7681f7e41145be6a61a8091e4dd02d10c71920ad932701", "9201c427018080c001122050e10779218db4a02e57c80824ad42faccc002b56b806a6910ee8ca25b36cf1d"]}, "07defa27cdd333a37d1c1f425770a541": {"reasons": ["user"], "record": null}, "bcc7900726de195e4cc4b09e67f02966": {"reasons": ["follow"], "record": ["8e6e4ec6192178ce7ea841c1603ee6756fe9bed343f7780234889a6fb4da5414274d5caf3ca30bb9017e5ac96eb3a75622171754072025b11210c98a0e8e546a", "783c4f3005a78c29de83", "ff7788273d006ad09958", "", "9c523b5797fa9e96ebcf18a1fef4e33aa25c682c158b34592f3942a63ab98dff38b8e8f890649ad259d3c1ce6ea25490077b1c26ed7ec0a891b4f0ae0d31db00", "920cc42401551220d8bd1ae8a49d73ab61d6b59c9220f6696551c58a58006f5fcfad30a800ab773f"]}, "f35e61151054781eb9f985f470691a77": {"reasons": ["user"], "record": ["8db0daa5ca4c0979d432ce2b7ffac8f5057a4a77245494128a636f961309e54f4b2b7e004d1888419fa0de6e61ee896a40996edf79ecd1da36a56ac982962f3b", "783c4f3005a78c29de83", "d2a612e452006ad098e0", "", "f1cf556174e0341ee785bf34f236f06aae1cbda68547a6c5716ace79d29711641ac8b7afde40eb3ef226fad4ce416e4e6a343546be4c1de920fda5004b3d250d", "9207c424015512202a5740f76cb8e47558a10cd85a03f563be8da273306a54594ec4535f210d949b"]}}} \ No newline at end of file diff --git a/tests/fixtures/rnfs-0.1.4/state/lxmf_inbox.json b/tests/fixtures/rnfs-0.1.4/state/lxmf_inbox.json new file mode 100644 index 0000000..7eb06bd --- /dev/null +++ b/tests/fixtures/rnfs-0.1.4/state/lxmf_inbox.json @@ -0,0 +1 @@ +{"next_id": 4, "messages": [{"hash": "7b4121eec4d068e3eeee0a668158e14ad322a1ef925c0be28aa6a26a9dde4ebf", "from": "a0a1a2a3a4a5a6a7a8a9aaabacadaeaf", "display_name": "Zoë", "sent": 1792056030.0, "received": 1792056060.0, "title": "Grüße 🌻", "text": "Fotos vom Wochenende 📷\n\nrnfs:bafkreif3aytvfpqujesmyqwwmlhzy3k2w7gfyw7ezitoxbmwzguktd34dm?from=11111111111111111111111111111111\nrnfs:bagaibqabciqjpffm54fysdaxqxjggmfjifzfvoyfsy256jovqmnjy6dpbbpg7lq/caf%C3%A9.txt", "method": "direct", "links": [{"uri": "rnfs:bafkreif3aytvfpqujesmyqwwmlhzy3k2w7gfyw7ezitoxbmwzguktd34dm?from=11111111111111111111111111111111", "cid": "bafkreif3aytvfpqujesmyqwwmlhzy3k2w7gfyw7ezitoxbmwzguktd34dm", "name": null, "path": [], "size": 36, "from": "11111111111111111111111111111111"}, {"uri": "rnfs:bagaibqabciqjpffm54fysdaxqxjggmfjifzfvoyfsy256jovqmnjy6dpbbpg7lq/caf%C3%A9.txt", "cid": "bagaibqabciqjpffm54fysdaxqxjggmfjifzfvoyfsy256jovqmnjy6dpbbpg7lq", "name": null, "path": ["café.txt"], "size": null, "from": null}], "auto": [{"state": "pinned", "cid": "bafkreif3aytvfpqujesmyqwwmlhzy3k2w7gfyw7ezitoxbmwzguktd34dm"}, {"state": "pinned", "cid": "bafkreicllqx56gk2tdcidha3bgi7bcwco4p6v24ij5kvymvq5dxwdf27ae"}], "id": 1}, {"hash": "b20fef4fb2d0e2869b3570019d9c05a64870b737d86a2e8d13660e452ef87c13", "from": "b0b1b2b3b4b5b6b7b8b9babbbcbdbebf", "display_name": "Bøb 🦊", "sent": 1792056115.0, "received": 1792056120.0, "title": "Привет", "text": "Посмотри это: rnfs:name/05c7214ca11e7646c79340c6b6d9866c/2026/%C3%A9t%C3%A9.jpg", "method": "propagated", "links": [{"uri": "rnfs:name/05c7214ca11e7646c79340c6b6d9866c/2026/%C3%A9t%C3%A9.jpg", "cid": null, "name": "05c7214ca11e7646c79340c6b6d9866c", "path": ["2026", "été.jpg"], "size": null, "from": null}], "auto": null, "id": 2}, {"hash": "584b41e5ba4074842544bf3368769200656aaddbca1d2863f897e6f2817952b6", "from": "b0b1b2b3b4b5b6b7b8b9babbbcbdbebf", "display_name": null, "sent": null, "received": 1792056180.0, "title": "", "text": "𝄞 music 🎵 rnfs:bagaibqabciqliwzhsidg2viwb4ux33yfxkfct2w5ehjbxiihvbbpjeq7okvr6da", "method": "opportunistic", "links": [{"uri": "rnfs:bagaibqabciqliwzhsidg2viwb4ux33yfxkfct2w5ehjbxiihvbbpjeq7okvr6da", "cid": "bagaibqabciqliwzhsidg2viwb4ux33yfxkfct2w5ehjbxiihvbbpjeq7okvr6da", "name": null, "path": [], "size": null, "from": null}], "auto": null, "id": 3}]} \ No newline at end of file diff --git a/tests/fixtures/rnfs-0.1.4/state/lxmf_outbox.json b/tests/fixtures/rnfs-0.1.4/state/lxmf_outbox.json new file mode 100644 index 0000000..1e5c733 --- /dev/null +++ b/tests/fixtures/rnfs-0.1.4/state/lxmf_outbox.json @@ -0,0 +1 @@ +{"next_id": 6, "messages": [{"hash": "93d3f14fa1ad9e4c7b89988e68d6449f0b47b1bce1299bdb6fd33f26743991bd", "to": "b0b1b2b3b4b5b6b7b8b9babbbcbdbebf", "time": 1792056240.0, "links": [{"uri": "rnfs:bagaibqabciqliwzhsidg2viwb4ux33yfxkfct2w5ehjbxiihvbbpjeq7okvr6da?from=4e0de1d04e0de1d04e0de1d04e0de1d0", "cid": "bagaibqabciqliwzhsidg2viwb4ux33yfxkfct2w5ehjbxiihvbbpjeq7okvr6da", "name": null, "path": [], "size": 16, "from": "4e0de1d04e0de1d04e0de1d04e0de1d0"}], "text": "Voilà le fichier ✅", "method": "direct", "state": "delivered", "progress": 1.0, "id": 1}, {"hash": "59ae57cd8477123689cd483035d3386230f1a708b1d45dbb71338b2ac5a2197c", "to": "b0b1b2b3b4b5b6b7b8b9babbbcbdbebf", "time": 1792056300.0, "links": [{"uri": "rnfs:name/05c7214ca11e7646c79340c6b6d9866c/2026?from=4e0de1d04e0de1d04e0de1d04e0de1d0", "cid": "bagaibqabciqfbyihpeqy3nfafzl4qcbevvbpvtgaak2wxadkneio5dfclm3m6hi", "name": "05c7214ca11e7646c79340c6b6d9866c", "path": ["2026"], "size": 19, "from": "4e0de1d04e0de1d04e0de1d04e0de1d0"}], "text": "via the propagation node 📡", "method": "propagated", "state": "sent", "progress": 0.0, "id": 2}, {"hash": "64bfcf1cb5c0b71e992de816e697e5ce0cb0f744b8a707bf8eb8fd602399cab0", "to": "b0b1b2b3b4b5b6b7b8b9babbbcbdbebf", "time": 1792056360.0, "links": [{"uri": "rnfs:bagaibqabciqen3e7u4m7fn7ak6q4vwpeihbka3hewbvwwkt7njnzaex4ifrk3pi?from=4e0de1d04e0de1d04e0de1d04e0de1d0", "cid": "bagaibqabciqen3e7u4m7fn7ak6q4vwpeihbka3hewbvwwkt7njnzaex4ifrk3pi", "name": null, "path": [], "size": 22, "from": "4e0de1d04e0de1d04e0de1d04e0de1d0"}], "text": "", "method": "propagated", "state": "outbound", "progress": 0.0, "id": 3, "direct_hash": "e57ea396179755a3aa63ca5fdafa8ac20a47ab4cd2134df99da0535ef6d548ca", "fallback": true}, {"hash": "75146bd05a2bb4c374a1490dc343371d424723b23f3073177d8cf35e5ceb431a", "to": "a0a1a2a3a4a5a6a7a8a9aaabacadaeaf", "time": 1792056420.0, "links": [{"uri": "rnfs:bagaibqabciqjpffm54fysdaxqxjggmfjifzfvoyfsy256jovqmnjy6dpbbpg7lq/%E6%97%A5%E6%9C%AC%E8%AA%9E?from=4e0de1d04e0de1d04e0de1d04e0de1d0", "cid": "bagaibqabciqjpffm54fysdaxqxjggmfjifzfvoyfsy256jovqmnjy6dpbbpg7lq", "name": null, "path": ["日本語"], "size": 7, "from": "4e0de1d04e0de1d04e0de1d04e0de1d0"}], "text": "not delivered yet ⏳", "method": "direct", "state": "outbound", "progress": 0.0, "id": 4}, {"hash": "ae6c0f21ba27ea8a47bfb3607738da601e19cc3ff3eb58744c4c26ff89e27358", "to": "a0a1a2a3a4a5a6a7a8a9aaabacadaeaf", "time": 1792056480.0, "links": [{"uri": "rnfs:bafkreif2bqifzcfpnduaemjb275npgbk37jmomt3mugspfq2ewxz7ogjcm?from=4e0de1d04e0de1d04e0de1d04e0de1d0", "cid": "bafkreif2bqifzcfpnduaemjb275npgbk37jmomt3mugspfq2ewxz7ogjcm", "name": null, "path": [], "size": 23, "from": "4e0de1d04e0de1d04e0de1d04e0de1d0"}], "text": "half way 🚚", "method": "direct", "state": "sending", "progress": 0.457, "id": 5}]} \ No newline at end of file diff --git a/tests/fixtures/rnfs-0.1.4/state/name-floor.json b/tests/fixtures/rnfs-0.1.4/state/name-floor.json new file mode 100644 index 0000000..6631290 --- /dev/null +++ b/tests/fixtures/rnfs-0.1.4/state/name-floor.json @@ -0,0 +1 @@ +{"version":1,"floor":[["2a7485ee7ccbd5e2196e208cfaef1b9b",5,1792055880],["49154ef17f86f5401d7d07b4bf8f96f4",18446744073709551615,1792055940]]} \ No newline at end of file diff --git a/tests/fixtures/rnfs-0.1.4/state/names.json b/tests/fixtures/rnfs-0.1.4/state/names.json new file mode 100644 index 0000000..89a9228 --- /dev/null +++ b/tests/fixtures/rnfs-0.1.4/state/names.json @@ -0,0 +1,32 @@ +{ + "version": 1, + "published": { + "imported": { + "sequence": 0, + "cid": null, + "imported": true + }, + "photos": { + "sequence": 1, + "cid": "bagaibqabciqfbyihpeqy3nfafzl4qcbevvbpvtgaak2wxadkneio5dfclm3m6hi", + "imported": false + } + }, + "held": { + "05c7214ca11e7646c79340c6b6d9866c": { + "sequence": 1, + "cid": "bagaibqabciqfbyihpeqy3nfafzl4qcbevvbpvtgaak2wxadkneio5dfclm3m6hi", + "heard": 1792055220.0 + }, + "a9f96ee8afd8b8bee788a442a84e555e": { + "sequence": 3, + "cid": "bafkreiherkniycepddysts7hctkszprpeyqcb35geojps2mcapym6x46zi", + "heard": 1792055820.0 + }, + "bcc7900726de195e4cc4b09e67f02966": { + "sequence": 12, + "cid": "bafkreigyxunorje5oovwdvvvtsjcb5tjmvi4lcsyabxv7t5ngcuabk3xh4", + "heard": 1792055640.0 + } + } +} \ No newline at end of file diff --git a/tests/fixtures/rnfs-0.1.4/state/pin_names.json b/tests/fixtures/rnfs-0.1.4/state/pin_names.json new file mode 100644 index 0000000..d098adb --- /dev/null +++ b/tests/fixtures/rnfs-0.1.4/state/pin_names.json @@ -0,0 +1 @@ +{"bafkreicvrk6gu53w2h23u5lgutxbgvfs75j63zt36gh2m24at7kbhnjsxe": "(stdin)", "bafkreie4x56ixo4ws7p6zb6cx7q7ftabyvkjhtkmhrw2pfqjhh6oomrnmy": "/home/alice/Downloads/b\udcffd name.txt", "bafkreif2bqifzcfpnduaemjb275npgbk37jmomt3mugspfq2ewxz7ogjcm": "(stdin)", "bagaibqabciqen3e7u4m7fn7ak6q4vwpeihbka3hewbvwwkt7njnzaex4ifrk3pi": "/home/alice/Pictures/F\u00eate \u00e0 Montr\u00e9al/photo \ud83c\udf89.jpg", "bagaibqabciqliwzhsidg2viwb4ux33yfxkfct2w5ehjbxiihvbbpjeq7okvr6da": "R\u00e9sum\u00e9 \u2013 2026.txt"} \ No newline at end of file diff --git a/tests/fixtures/rnfs-0.1.4/state/pins.json b/tests/fixtures/rnfs-0.1.4/state/pins.json new file mode 100644 index 0000000..8d16147 --- /dev/null +++ b/tests/fixtures/rnfs-0.1.4/state/pins.json @@ -0,0 +1 @@ +{"pins": {"bagaibqabciqliwzhsidg2viwb4ux33yfxkfct2w5ehjbxiihvbbpjeq7okvr6da": ["user"], "bafkreif2bqifzcfpnduaemjb275npgbk37jmomt3mugspfq2ewxz7ogjcm": ["user"], "bagaibqabciqen3e7u4m7fn7ak6q4vwpeihbka3hewbvwwkt7njnzaex4ifrk3pi": ["user"], "bafkreie4x56ixo4ws7p6zb6cx7q7ftabyvkjhtkmhrw2pfqjhh6oomrnmy": ["user"], "bafkreieeka3tp4k5fln7denhbcyl37wnnwy2b3eteebcer7ebbo5x44osi": ["folder"], "bagaibqabciqfbyihpeqy3nfafzl4qcbevvbpvtgaak2wxadkneio5dfclm3m6hi": ["name:photos"], "bafkreihvcmhog4opp7chojb6caafzh7ragthgixbrzcqtg42ei3xfav2nq": ["remote:0123456789abcdef0123456789abcdef"], "bafkreigyxunorje5oovwdvvvtsjcb5tjmvi4lcsyabxv7t5ngcuabk3xh4": ["follow:bcc7900726de195e4cc4b09e67f02966"], "bafkreif3aytvfpqujesmyqwwmlhzy3k2w7gfyw7ezitoxbmwzguktd34dm": ["lxmf:a0a1a2a3a4a5a6a7a8a9aaabacadaeaf"], "bafkreicllqx56gk2tdcidha3bgi7bcwco4p6v24ij5kvymvq5dxwdf27ae": ["lxmf:a0a1a2a3a4a5a6a7a8a9aaabacadaeaf"]}} \ No newline at end of file diff --git a/tests/fixtures/rnfs-0.1.4/state/providers.json b/tests/fixtures/rnfs-0.1.4/state/providers.json new file mode 100644 index 0000000..12eb9d1 --- /dev/null +++ b/tests/fixtures/rnfs-0.1.4/state/providers.json @@ -0,0 +1 @@ +{"version":1,"publishers":{"30000000000000000000000000000000":1791795600.0,"31000000000000000000000000000000":1791795600.0,"33000000000000000000000000000000":1791925260.0,"37000000000000000000000000000000":1792051320.0,"38000000000000000000000000000000":1792051380.0,"39000000000000000000000000000000":1792051440.0,"3a000000000000000000000000000000":1792051500.0,"4e0de1d04e0de1d04e0de1d04e0de1d0":1792054860.0},"records":[["80000000000000000000000000000000","30000000000000000000000000000000",1791795600.0,1792056845.0,0],["80010000000000000000000000000000","30000000000000000000000000000000",1791795600.0,1791795600.0,0],["80020000000000000000000000000000","30000000000000000000000000000000",1791795600.0,1791795600.0,0],["80030000000000000000000000000000","30000000000000000000000000000000",1791795600.0,1791795600.0,0],["80040000000000000000000000000000","30000000000000000000000000000000",1791795600.0,1791795600.0,0],["80040000000000000000000000000000","31000000000000000000000000000000",1791795600.0,1791795600.0,0],["80050000000000000000000000000000","30000000000000000000000000000000",1791795600.0,1791795600.0,0],["80050000000000000000000000000000","31000000000000000000000000000000",1791795600.0,1791795600.0,0],["80060000000000000000000000000000","30000000000000000000000000000000",1791795600.0,1791795600.0,0],["80060000000000000000000000000000","31000000000000000000000000000000",1791795600.0,1791795600.0,0],["80070000000000000000000000000000","30000000000000000000000000000000",1791795600.0,1791795600.0,0],["80070000000000000000000000000000","31000000000000000000000000000000",1791795600.0,1791795600.0,0],["80080000000000000000000000000000","31000000000000000000000000000000",1791795600.0,1791795600.0,0],["80090000000000000000000000000000","31000000000000000000000000000000",1791795600.0,1791795600.0,0],["80240000000000000000000000000000","33000000000000000000000000000000",1791925620.0,1791925620.0,0],["80250000000000000000000000000000","33000000000000000000000000000000",1791925680.0,1791925680.0,0],["80260000000000000000000000000000","33000000000000000000000000000000",1791925740.0,1791925740.0,0],["80270000000000000000000000000000","33000000000000000000000000000000",1791925800.0,1791925800.0,0],["80650000000000000000000000000000","37000000000000000000000000000000",1792051320.0,1792051320.0,1],["80660000000000000000000000000000","38000000000000000000000000000000",1792051380.0,1792051380.0,1],["80670000000000000000000000000000","39000000000000000000000000000000",1792051440.0,1792051440.0,1],["80680000000000000000000000000000","3a000000000000000000000000000000",1792051500.0,1792051500.0,1],["0918207803da9867093b2cffe7b7bb2f","4e0de1d04e0de1d04e0de1d04e0de1d0",1792054860.0,1792054860.0,0],["bf641d47a118e7693be211176667d353","4e0de1d04e0de1d04e0de1d04e0de1d0",1792054920.0,1792054920.0,0],["cc92f5dd8433c61479d0463e6adf4b50","4e0de1d04e0de1d04e0de1d04e0de1d0",1792054980.0,1792054980.0,0],["64f4edd9e5e844b50f7a0ae8b85a898f","4e0de1d04e0de1d04e0de1d04e0de1d0",1792055040.0,1792055040.0,0],["5e3615897e5aabcf904002aa1c8c239c","4e0de1d04e0de1d04e0de1d04e0de1d0",1792055100.0,1792055100.0,0],["005bed22d2fd8d322fb80b140598a0c3","4e0de1d04e0de1d04e0de1d04e0de1d0",1792055160.0,1792055160.0,0],["ff3e2155432da6e46f967c4c688053b2","4e0de1d04e0de1d04e0de1d04e0de1d0",1792055220.0,1792055220.0,0],["368f11e51cc31ac9cb717a07d8d12139","4e0de1d04e0de1d04e0de1d04e0de1d0",1792055280.0,1792055280.0,0],["075d5889d921fccee0f78cb9a3257b24","4e0de1d04e0de1d04e0de1d04e0de1d0",1792055340.0,1792055340.0,0],["7973a606727dce5446b2f0242d1c401e","4e0de1d04e0de1d04e0de1d04e0de1d0",1792055460.0,1792055460.0,0],["b60b450b4d0bfdb631405273f3800390","4e0de1d04e0de1d04e0de1d04e0de1d0",1792055460.0,1792055460.0,0],["b362bf1c891e9f8bf935854c9a4a89da","4e0de1d04e0de1d04e0de1d04e0de1d0",1792055520.0,1792055520.0,0],["24b2f721abc9dd6f9f7b8be02b20e1f8","4e0de1d04e0de1d04e0de1d04e0de1d0",1792055580.0,1792055580.0,0],["b14a6a65ed6d7fd082f130f7d0c00ba9","4e0de1d04e0de1d04e0de1d04e0de1d0",1792055640.0,1792055640.0,0],["9fb861bafe2a0229fa850594ffa8d4e9","4e0de1d04e0de1d04e0de1d04e0de1d0",1792055760.0,1792055760.0,0],["b6a2e77295bec80e226b3d3c734bdecb","4e0de1d04e0de1d04e0de1d04e0de1d0",1792056000.0,1792056000.0,0],["8d616737498993836c17fa8fbc37cf05","4e0de1d04e0de1d04e0de1d04e0de1d0",1792056060.0,1792056060.0,0]],"departed":[["32000000000000000000000000000000",1791795600.0,1791925200.0],["35000000000000000000000000000000",1791926580.0,1792054920.0],["34000000000000000000000000000000",1791925920.0,1792055520.0]],"evicted":[["36000000000000000000000000000000",1792051260.0,1792051500.0]]} \ No newline at end of file diff --git a/tests/fixtures/rnfs-0.1.4/state/published.json b/tests/fixtures/rnfs-0.1.4/state/published.json new file mode 100644 index 0000000..5fbe127 --- /dev/null +++ b/tests/fixtures/rnfs-0.1.4/state/published.json @@ -0,0 +1 @@ +{"published": {"bafkreibevdhpo2ydbraaurnjq3wtziaclbsmqpquyh6ntesaf7lmove4vq": [1792056785.0, ["10101010101010101010101010101010", "11111111111111111111111111111111"]], "bafkreibkk5apo3fy4r2vriim3bnah5ldx2g2e4zqnjkfstweknpscdmutm": [1792053185.0, ["11111111111111111111111111111111", "12121212121212121212121212121212"]], "bafkreicllqx56gk2tdcidha3bgi7bcwco4p6v24ij5kvymvq5dxwdf27ae": [1792049585.0, ["12121212121212121212121212121212", "13131313131313131313131313131313"]], "bafkreie4x56ixo4ws7p6zb6cx7q7ftabyvkjhtkmhrw2pfqjhh6oomrnmy": [1792042385.0, ["10101010101010101010101010101010", "11111111111111111111111111111111"]], "bafkreieeka3tp4k5fln7denhbcyl37wnnwy2b3eteebcer7ebbo5x44osi": [1792038785.0, ["11111111111111111111111111111111", "12121212121212121212121212121212"]], "bafkreif2bqifzcfpnduaemjb275npgbk37jmomt3mugspfq2ewxz7ogjcm": [1792035185.0, ["12121212121212121212121212121212", "13131313131313131313131313131313"]], "bafkreif3aytvfpqujesmyqwwmlhzy3k2w7gfyw7ezitoxbmwzguktd34dm": [1792031585.0, ["13131313131313131313131313131313", "10101010101010101010101010101010"]], "bafkreigyxunorje5oovwdvvvtsjcb5tjmvi4lcsyabxv7t5ngcuabk3xh4": [1792027985.0, ["10101010101010101010101010101010", "11111111111111111111111111111111"]], "bafkreiherkniycepddysts7hctkszprpeyqcb35geojps2mcapym6x46zi": [1792024385.0, ["11111111111111111111111111111111", "12121212121212121212121212121212"]], "bafkreihvcmhog4opp7chojb6caafzh7ragthgixbrzcqtg42ei3xfav2nq": [1792020785.0, ["12121212121212121212121212121212", "13131313131313131313131313131313"]], "bagaibqabciqen3e7u4m7fn7ak6q4vwpeihbka3hewbvwwkt7njnzaex4ifrk3pi": [1792017185.0, ["13131313131313131313131313131313", "10101010101010101010101010101010"]], "bagaibqabciqfbyihpeqy3nfafzl4qcbevvbpvtgaak2wxadkneio5dfclm3m6hi": [1792013585.0, ["10101010101010101010101010101010", "11111111111111111111111111111111"]], "bagaibqabciqjpffm54fysdaxqxjggmfjifzfvoyfsy256jovqmnjy6dpbbpg7lq": [1792009985.0, ["11111111111111111111111111111111", "12121212121212121212121212121212"]], "bagaibqabciqliwzhsidg2viwb4ux33yfxkfct2w5ehjbxiihvbbpjeq7okvr6da": [1792006385.0, ["12121212121212121212121212121212", "13131313131313131313131313131313"]], "key:b14a6a65ed6d7fd082f130f7d0c00ba9": [1792002785.0, ["13131313131313131313131313131313", "10101010101010101010101010101010"]], "key:b362bf1c891e9f8bf935854c9a4a89da": [1791999185.0, ["10101010101010101010101010101010", "11111111111111111111111111111111"]], "key:ff3e2155432da6e46f967c4c688053b2": [1791995585.0, ["11111111111111111111111111111111", "12121212121212121212121212121212"]]}, "earlier": {"bafkreibevdhpo2ydbraaurnjq3wtziaclbsmqpquyh6ntesaf7lmove4vq": {"12121212121212121212121212121212": 1791797585.0}, "bafkreicllqx56gk2tdcidha3bgi7bcwco4p6v24ij5kvymvq5dxwdf27ae": {"10101010101010101010101010101010": 1791797585.0}, "bafkreie4x56ixo4ws7p6zb6cx7q7ftabyvkjhtkmhrw2pfqjhh6oomrnmy": {"12121212121212121212121212121212": 1791797585.0}, "bafkreif2bqifzcfpnduaemjb275npgbk37jmomt3mugspfq2ewxz7ogjcm": {"10101010101010101010101010101010": 1791797585.0}, "bafkreigyxunorje5oovwdvvvtsjcb5tjmvi4lcsyabxv7t5ngcuabk3xh4": {"12121212121212121212121212121212": 1791797585.0}, "bafkreihvcmhog4opp7chojb6caafzh7ragthgixbrzcqtg42ei3xfav2nq": {"10101010101010101010101010101010": 1791797585.0}, "bagaibqabciqfbyihpeqy3nfafzl4qcbevvbpvtgaak2wxadkneio5dfclm3m6hi": {"12121212121212121212121212121212": 1791797585.0}, "bagaibqabciqliwzhsidg2viwb4ux33yfxkfct2w5ehjbxiihvbbpjeq7okvr6da": {"10101010101010101010101010101010": 1791797585.0}, "key:b362bf1c891e9f8bf935854c9a4a89da": {"12121212121212121212121212121212": 1791797585.0}}, "withdraw": {"13131313131313131313131313131313": ["ffe8f9db57e41110e0abce7a6ce6bba0"], "10101010101010101010101010101010": ["ffe8f9db57e41110e0abce7a6ce6bba0"]}} \ No newline at end of file diff --git a/tests/fixtures/rnfs-0.1.4/state/remote_pins.json b/tests/fixtures/rnfs-0.1.4/state/remote_pins.json new file mode 100644 index 0000000..de15fb5 --- /dev/null +++ b/tests/fixtures/rnfs-0.1.4/state/remote_pins.json @@ -0,0 +1 @@ +[["remote:0123456789abcdef0123456789abcdef", "bafkreie47tg2yw2hrduj27xkjexodwj7cyk3vjhqbixhcbjas4b6vggvuu"], ["remote:0123456789abcdef0123456789abcdef", "bafkreifwqdmdus2doogpg3dlr2lf3czs2oyp2lq3mq7olhhu4c3yp44u5y"]] \ No newline at end of file diff --git a/tests/fixtures/rnfs-0.1.4/state/shared_folder.json b/tests/fixtures/rnfs-0.1.4/state/shared_folder.json new file mode 100644 index 0000000..2e6ca82 --- /dev/null +++ b/tests/fixtures/rnfs-0.1.4/state/shared_folder.json @@ -0,0 +1 @@ +{"entries": {"notes.txt": ["6baeb78e6b0bb791f691f2860e2fc0be964fc2d89cb04c82f1378c1f257c398e", "bafkreieeka3tp4k5fln7denhbcyl37wnnwy2b3eteebcer7ebbo5x44osi"]}} \ No newline at end of file diff --git a/tests/security/_pages_helpers.py b/tests/security/_pages_helpers.py index 91bbabf..073ca79 100644 --- a/tests/security/_pages_helpers.py +++ b/tests/security/_pages_helpers.py @@ -33,8 +33,9 @@ INDEX_PAGE = os.path.join(PAGES_DIR, "file-index-node", "index.mu") -def make_nodes(tmp, settings=None): - """A provider node A and a victim node B joined by a NetClient, as tests.test_node does.""" +def make_nodes(tmp, settings=None, test=None): + """A provider node A and a victim node B joined by a NetClient, as tests.test_node does; stopped (which + closes their state databases) when test, a TestCase, cleans up.""" net = NetClient() net.sender = B_ID nodes = {} @@ -44,6 +45,8 @@ def make_nodes(tmp, settings=None): node = RnfsNode.offline(config, node_id, net, FakeNetwork()) net.nodes[node_id] = node nodes[name] = node + if test is not None: + test.addCleanup(node.stop) nodes["b"].routing.heard(A_ID) return net, nodes["a"], nodes["b"] diff --git a/tests/security/test_fuzz_parsers.py b/tests/security/test_fuzz_parsers.py index 8762026..ca0c26b 100644 --- a/tests/security/test_fuzz_parsers.py +++ b/tests/security/test_fuzz_parsers.py @@ -401,6 +401,7 @@ def setUp(self): self.tmp = tempfile.mkdtemp() self.addCleanup(shutil.rmtree, self.tmp, ignore_errors=True) self.storage = Storage(os.path.join(self.tmp, "blocks"), os.path.join(self.tmp, "state"), 10**9) + self.addCleanup(self.storage.close) def read(self, data: bytes) -> list | None: """Every record of the archive in data, or None if it is rejected (ValueError).""" @@ -437,6 +438,7 @@ def test_import_any_stream(self, data): unique_by=lambda entry: entry[0]), st.data()) def test_round_trip_and_mutations(self, files, data): source = Storage(os.path.join(self.tmp, "source"), os.path.join(self.tmp, "source-state"), 10**9) + self.addCleanup(source.close) entries = sorted((Entry(name, builder.build_bytes(content, source.put).cid, len(content), 0) for name, content in files), key=lambda entry: entry.name) root = builder.build_directory(entries, source.put).cid diff --git a/tests/security/test_h2_display_page_pins.py b/tests/security/test_h2_display_page_pins.py index 165475b..e3f6020 100644 --- a/tests/security/test_h2_display_page_pins.py +++ b/tests/security/test_h2_display_page_pins.py @@ -50,7 +50,7 @@ def setUp(self): """Two offline nodes joined by a NetClient, and the page's data dir.""" self.tmp = tempfile.TemporaryDirectory() self.addCleanup(self.tmp.cleanup) - _, self.provider, self.victim = helpers.make_nodes(self.tmp.name) + _, self.provider, self.victim = helpers.make_nodes(self.tmp.name, test=self) self.addCleanup(self.provider.stop) self.addCleanup(self.victim.stop) self.data_dir = os.path.join(self.tmp.name, "page-data") diff --git a/tests/security/test_l10_index_page_injection.py b/tests/security/test_l10_index_page_injection.py index 7568f90..f8f8099 100644 --- a/tests/security/test_l10_index_page_injection.py +++ b/tests/security/test_l10_index_page_injection.py @@ -47,7 +47,7 @@ def setUp(self): """One offline node holding the crafted directory.""" self.tmp = tempfile.TemporaryDirectory() self.addCleanup(self.tmp.cleanup) - _, self.node, _ = helpers.make_nodes(self.tmp.name) + _, self.node, _ = helpers.make_nodes(self.tmp.name, test=self) self.addCleanup(self.node.stop) def pin_crafted_directory(self, entry_name: str) -> str: diff --git a/tests/security/test_l11_display_page_micron.py b/tests/security/test_l11_display_page_micron.py index d7a36fa..30c28ff 100644 --- a/tests/security/test_l11_display_page_micron.py +++ b/tests/security/test_l11_display_page_micron.py @@ -53,7 +53,7 @@ def setUp(self): """An offline node that has the hostile file pinned (source user), and the page's data dir.""" self.tmp = tempfile.TemporaryDirectory() self.addCleanup(self.tmp.cleanup) - _, self.node, _ = helpers.make_nodes(self.tmp.name) + _, self.node, _ = helpers.make_nodes(self.tmp.name, test=self) self.addCleanup(self.node.stop) self.cid = self.node.add_stream(io.BytesIO(HOSTILE.encode("utf-8")))["cid"] self.data_dir = os.path.join(self.tmp.name, "page-data") diff --git a/tests/security/test_l12_lxmf_inbox.py b/tests/security/test_l12_lxmf_inbox.py index dc29163..13c5ed6 100644 --- a/tests/security/test_l12_lxmf_inbox.py +++ b/tests/security/test_l12_lxmf_inbox.py @@ -126,7 +126,9 @@ def test_contacts_lookup_by_address(self): self.assertTrue(book.has(CONTACT, "lxmf")) # carol2 still names it book.add("carol2", EARLY.hex(), kind="lxmf") # replaced self.assertFalse(book.has(CONTACT, "lxmf")) - self.assertTrue(ContactsFeature(self.b).has(EARLY, "lxmf")) # loaded from contacts.json + again = ContactsFeature(self.b) + self.addCleanup(again.close) + self.assertTrue(again.has(EARLY, "lxmf")) # loaded from the saved contacts if __name__ == "__main__": diff --git a/tests/security/test_l4_name_rollback.py b/tests/security/test_l4_name_rollback.py index b2d9ab0..4c039dd 100644 --- a/tests/security/test_l4_name_rollback.py +++ b/tests/security/test_l4_name_rollback.py @@ -17,7 +17,7 @@ from rnfs.config import Config from rnfs.content.cid import RAW, Cid from rnfs.names import NameManager, encode_record -from tests.support import FakeClock +from tests.support import FakeClock, write_legacy from tests.test_names import FakeNetwork, FakePins N = bytes([0x4E] * 16) @@ -46,6 +46,7 @@ def setUp(self): self.clock = FakeClock(1_000_000.0) self.manager = NameManager(self.config, FakePins(), network=self.network, clock=self.clock, monotonic=self.clock) + self.addCleanup(self.manager.close) self.manager.start() self.addCleanup(self.manager.stop) self.network.announce(N, encode_record(5, NEW), self.clock.now) @@ -81,6 +82,7 @@ def test_no_rollback_after_restart(self): self.manager.stop() self.network.app_data.clear() restarted = NameManager(self.config, FakePins(), network=self.network, clock=self.clock, monotonic=self.clock) + self.addCleanup(restarted.close) restarted.start() self.addCleanup(restarted.stop) self.replay_old() @@ -99,11 +101,13 @@ def setUp(self): self.addCleanup(setattr, RNS, "loglevel", saved) tmp = tempfile.TemporaryDirectory() self.addCleanup(tmp.cleanup) + self.tmp = tmp.name self.config = Config(rnfs_dir=tmp.name, path_timeout=0.05, name_table_size=3) self.network = FakeNetwork() self.clock = FakeClock(1_000_000.0) self.manager = NameManager(self.config, FakePins(), network=self.network, clock=self.clock, monotonic=self.clock) + self.addCleanup(self.manager.close) self.manager.start() self.addCleanup(self.manager.stop) self.network.announce(N, encode_record(5, NEW), self.clock.now) @@ -128,13 +132,14 @@ def test_floor_bounded(self): self.assertEqual(len(self.manager._floor), names.FLOOR_FACTOR * 3) def test_damaged_floor_file_is_set_aside(self): - self.manager.stop() - path = os.path.join(self.config.state_dir, names.FLOOR_FILE) - with open(path, "w", encoding="utf-8") as f: - f.write("{not json") - restarted = NameManager(self.config, FakePins(), network=self.network, clock=self.clock, monotonic=self.clock) + """A damaged name-floor.json (of rnfs 0.1.4 or earlier) found at an upgrade is moved aside, and the node + starts without it.""" + config = Config(rnfs_dir=os.path.join(self.tmp, "upgraded"), path_timeout=0.05, name_table_size=3) + write_legacy(config.state_dir, names.FLOOR_FILE, text="{not json") + restarted = NameManager(config, FakePins(), network=self.network, clock=self.clock, monotonic=self.clock) + self.addCleanup(restarted.close) self.assertEqual(len(restarted._floor), 0) - self.assertTrue(any(entry.startswith(names.FLOOR_FILE + ".damaged") for entry in os.listdir(self.config.state_dir))) + self.assertTrue(any(entry.startswith(names.FLOOR_FILE + ".damaged") for entry in os.listdir(config.state_dir))) if __name__ == "__main__": diff --git a/tests/security/test_l4r_name_floor.py b/tests/security/test_l4r_name_floor.py index 10d5f84..c1fe78f 100644 --- a/tests/security/test_l4r_name_floor.py +++ b/tests/security/test_l4r_name_floor.py @@ -4,12 +4,10 @@ replayed old announce of a name back in. A NameManager over FakeNetwork (as tests/security/test_l4_name_rollback.py) resolves name N at sequence 5 for the -gateway. A "crash" is a second manager on the same state directory without stop(). The floor must be on disk at once -(within FLOOR_SAVE_INTERVAL of the last save), names resolved within FLOOR_KEEP_AGE must outlast churn up to a hard -bound, and the file must stay bounded. +gateway. A "crash" is a second manager on the same state directory without stop(). The floor must be committed at +once (within FLOOR_SAVE_INTERVAL of the last save), names resolved within FLOOR_KEEP_AGE must outlast churn up to a +hard bound, and the saved floor must stay bounded. """ -import json -import os import tempfile import unittest @@ -19,7 +17,7 @@ from rnfs.config import Config from rnfs.content.cid import RAW, Cid from rnfs.names import NameManager, encode_record -from tests.support import FakeClock +from tests.support import FakeClock, state_rows from tests.test_names import FakeNetwork, FakePins N = bytes([0x4E] * 16) @@ -53,6 +51,7 @@ def setUp(self): def start_manager(self) -> NameManager: """A manager on the state directory, started.""" manager = NameManager(self.config, FakePins(), network=self.network, clock=self.clock, monotonic=self.clock) + self.addCleanup(manager.close) manager.start() self.addCleanup(manager.stop) return manager @@ -70,12 +69,8 @@ def replayed_resolve(self, manager: NameManager): return manager.resolve(N, persist=False) def saved_names(self) -> set[bytes]: - """The names in name-floor.json (none if there is no such file).""" - try: - with open(os.path.join(self.config.state_dir, names.FLOOR_FILE), encoding="utf-8") as f: - return {bytes.fromhex(entry[0]) for entry in json.load(f)["floor"]} - except FileNotFoundError: - return set() + """The names the saved floor (the name_floor table) holds.""" + return {name for (name,) in state_rows(self.config.state_dir, "SELECT name FROM name_floor")} def test_floor_survives_a_crash_right_after_the_resolve(self): crashed = self.start_manager() # the first one never stopped, nor reached announce_due diff --git a/tests/security/test_l8_folder_swap.py b/tests/security/test_l8_folder_swap.py index 22370f2..225c3b7 100644 --- a/tests/security/test_l8_folder_swap.py +++ b/tests/security/test_l8_folder_swap.py @@ -41,7 +41,9 @@ def make_folder(self): with open(os.path.join(self.drop, "sub", "harmless.txt"), "wb") as f: f.write(b"hi") self.storage = Storage(os.path.join(self.tmp.name, "blocks"), os.path.join(self.tmp.name, "state"), 10**9) + self.addCleanup(self.storage.close) self.folder = SharedFolder(self.files, self.storage, os.path.join(self.tmp.name, "state")) + self.addCleanup(self.folder.close) def swap(self, path: str): """What the attacker does: move the directory at path aside and put a link to the private one there.""" diff --git a/tests/security/test_l8r2_hard_links.py b/tests/security/test_l8r2_hard_links.py index e6da5f6..e3e04f9 100644 --- a/tests/security/test_l8r2_hard_links.py +++ b/tests/security/test_l8r2_hard_links.py @@ -49,7 +49,9 @@ def setUp(self): with open(self.secret, "wb") as f: f.write(SECRET) self.storage = Storage(os.path.join(self.tmp.name, "blocks"), os.path.join(self.tmp.name, "state"), 10**9) + self.addCleanup(self.storage.close) self.folder = SharedFolder(self.files, self.storage, os.path.join(self.tmp.name, "state")) + self.addCleanup(self.folder.close) self.secret_cid = builder.add_path(self.secret, builder.NO_PUT)[0].cid log = mock.patch.object(shared_folder, "log") self.log = log.start() diff --git a/tests/security/test_m4_provider_admission.py b/tests/security/test_m4_provider_admission.py index d48ef2c..d40c757 100644 --- a/tests/security/test_m4_provider_admission.py +++ b/tests/security/test_m4_provider_admission.py @@ -165,25 +165,29 @@ class PersistenceTest(unittest.TestCase): """The pool and remembered first-seen times survive a restart.""" def test_round_trip(self): - with tempfile.TemporaryDirectory() as directory: - path = os.path.join(directory, "providers.json") - clock = FakeClock(1_700_000_000.0) - store = ProviderStore(100, path, clock=clock, rng=random.Random(1), seniority=DAY) - gone = ident(9) - store.store(ident(1), gone) - clock.now += 2 * DAY - store.withdraw(ident(1), gone) - for n in range(10): # 90 outside the pool, then the last 10 pooled - for i in range(10): - store.store(ident(100 * (n + 1) + i), ident(10 + n)) - store.save() - loaded = ProviderStore(100, path, clock=clock, rng=random.Random(1), seniority=DAY) - self.assertEqual(loaded.record_count(), 100) - self.assertEqual(loaded.allowance(gone), loaded.max_per_publisher) - # Pooled records stay pooled: a newcomer evicts one of them, not a record outside the pool - self.assertTrue(loaded.store(ident(5), ident(99))) - self.assertEqual(sum(len(loaded.providers(ident(100 * (n + 1) + i))) for n in range(9) for i in range(10)), - 90) + """A restarted store keeps which records are pooled and a departed publisher's first-seen time.""" + directory = tempfile.TemporaryDirectory() + self.addCleanup(directory.cleanup) + state = os.path.join(directory.name, "state") + clock = FakeClock(1_700_000_000.0) + store = ProviderStore(100, state, clock=clock, rng=random.Random(1), seniority=DAY) + self.addCleanup(store.close) + gone = ident(9) + store.store(ident(1), gone) + clock.now += 2 * DAY + store.withdraw(ident(1), gone) + for n in range(10): # 90 outside the pool, then the last 10 pooled + for i in range(10): + store.store(ident(100 * (n + 1) + i), ident(10 + n)) + store.save() + loaded = ProviderStore(100, state, clock=clock, rng=random.Random(1), seniority=DAY) + self.addCleanup(loaded.close) + self.assertEqual(loaded.record_count(), 100) + self.assertEqual(loaded.allowance(gone), loaded.max_per_publisher) + # Pooled records stay pooled: a newcomer evicts one of them, not a record outside the pool + self.assertTrue(loaded.store(ident(5), ident(99))) + self.assertEqual(sum(len(loaded.providers(ident(100 * (n + 1) + i))) for n in range(9) for i in range(10)), + 90) class PerPublisherCapTest(unittest.TestCase): diff --git a/tests/security/test_m4r_newcomer_pool.py b/tests/security/test_m4r_newcomer_pool.py index 2d23fcd..eb2e9eb 100644 --- a/tests/security/test_m4r_newcomer_pool.py +++ b/tests/security/test_m4r_newcomer_pool.py @@ -39,9 +39,12 @@ def fill_main(store: ProviderStore, clock: FakeClock): class NewcomerPoolTest(unittest.TestCase): """Stores with provider_seniority one day and seeded eviction picks.""" - def make_store(self, max_records: int, path: str | None = None) -> ProviderStore: - """A store on self.clock, its newcomer allowance a tenth of the full one when large enough.""" - return ProviderStore(max_records, path, clock=self.clock, rng=random.Random(1), seniority=DAY) + def make_store(self, max_records: int, state: str | None = None) -> ProviderStore: + """A store on self.clock, its newcomer allowance a tenth of the full one when large enough (persisted in + state, if given, and closed when the test ends).""" + store = ProviderStore(max_records, state, clock=self.clock, rng=random.Random(1), seniority=DAY) + self.addCleanup(store.close) + return store def setUp(self): """The clock.""" @@ -62,25 +65,27 @@ def test_newcomer_keeps_a_fair_share_of_the_pool(self): self.assertGreaterEqual(kept, fair, f"H kept {kept} of {len(keys)} records; a fair share of the pool is {fair}") def test_first_seen_outlasts_eviction_from_the_pool(self): - with tempfile.TemporaryDirectory() as directory: - path = os.path.join(directory, "providers.json") - store = self.make_store(10_000, path) - fill_main(store, self.clock) - self.assertTrue(store.store(ident(2 << 100), H)) - first_seen = store.first_seen(H) - self.clock.now += 60 - for i in range(3 * store.pool_size): # fresh identities, one record each - self.clock.now += 1 - store.store(ident((3 << 100) + i), ident((0x5 << 120) + i)) - self.assertIsNone(store.first_seen(H), "H still holds a record: the flood didn't evict it") - store.save() - store = self.make_store(10_000, path) # and a restart - self.clock.now += DAY - # M4-R: fails until first-seen times of newcomers evicted from the pool are kept - self.assertEqual(store.allowance(H), store.max_per_publisher, - "H, first seen over a day ago, is new again after the flood") - self.assertTrue(store.store(ident(4 << 100), H)) - self.assertEqual(store.first_seen(H), first_seen) + """A newcomer whose last record a flood evicted from the pool keeps its first-seen time, across a restart.""" + directory = tempfile.TemporaryDirectory() + self.addCleanup(directory.cleanup) # after the stores are closed (cleanups run last first) + state = os.path.join(directory.name, "state") + store = self.make_store(10_000, state) + fill_main(store, self.clock) + self.assertTrue(store.store(ident(2 << 100), H)) + first_seen = store.first_seen(H) + self.clock.now += 60 + for i in range(3 * store.pool_size): # fresh identities, one record each + self.clock.now += 1 + store.store(ident((3 << 100) + i), ident((0x5 << 120) + i)) + self.assertIsNone(store.first_seen(H), "H still holds a record: the flood didn't evict it") + store.save() + store = self.make_store(10_000, state) # and a restart + self.clock.now += DAY + # M4-R: fails until first-seen times of newcomers evicted from the pool are kept + self.assertEqual(store.allowance(H), store.max_per_publisher, + "H, first seen over a day ago, is new again after the flood") + self.assertTrue(store.store(ident(4 << 100), H)) + self.assertEqual(store.first_seen(H), first_seen) def test_memory_of_evicted_newcomers_is_bounded(self): """Bounded, and the newcomers first seen most recently are forgotten first, so a flood can't push out H.""" diff --git a/tests/security/test_m6_index_page_full_run.py b/tests/security/test_m6_index_page_full_run.py index 529b700..1224147 100644 --- a/tests/security/test_m6_index_page_full_run.py +++ b/tests/security/test_m6_index_page_full_run.py @@ -36,7 +36,7 @@ def setUp(self): """One offline node and a page data dir.""" self.tmp = tempfile.TemporaryDirectory() self.addCleanup(self.tmp.cleanup) - _, self.node, _ = helpers.make_nodes(self.tmp.name) + _, self.node, _ = helpers.make_nodes(self.tmp.name, test=self) self.addCleanup(self.node.stop) self.data_dir = os.path.join(self.tmp.name, "page-data") diff --git a/tests/security/test_m6_index_page_names.py b/tests/security/test_m6_index_page_names.py index d9962d7..0cd6333 100644 --- a/tests/security/test_m6_index_page_names.py +++ b/tests/security/test_m6_index_page_names.py @@ -43,7 +43,7 @@ def setUp(self): """One offline node with an identity (so it can seal to itself).""" self.tmp = tempfile.TemporaryDirectory() self.addCleanup(self.tmp.cleanup) - _, self.node, _ = helpers.make_nodes(self.tmp.name) + _, self.node, _ = helpers.make_nodes(self.tmp.name, test=self) self.addCleanup(self.node.stop) helpers.give_identity(self.node) diff --git a/tests/security/test_m6_pin_ls_sealed.py b/tests/security/test_m6_pin_ls_sealed.py index aef1086..88f5f0b 100644 --- a/tests/security/test_m6_pin_ls_sealed.py +++ b/tests/security/test_m6_pin_ls_sealed.py @@ -41,7 +41,7 @@ def setUp(self): """One offline node with an identity, and a provider it can fetch from.""" self.tmp = tempfile.TemporaryDirectory() self.addCleanup(self.tmp.cleanup) - _, self.provider, self.node = helpers.make_nodes(self.tmp.name) + _, self.provider, self.node = helpers.make_nodes(self.tmp.name, test=self) self.addCleanup(self.provider.stop) self.addCleanup(self.node.stop) helpers.give_identity(self.node) diff --git a/tests/security/test_n1_display_page_unbounded.py b/tests/security/test_n1_display_page_unbounded.py index f893564..2dd6bd5 100644 --- a/tests/security/test_n1_display_page_unbounded.py +++ b/tests/security/test_n1_display_page_unbounded.py @@ -42,7 +42,7 @@ def setUp(self): """A provider serving the bomb DAG and a victim that serves the page.""" self.tmp = tempfile.TemporaryDirectory() self.addCleanup(self.tmp.cleanup) - _, self.provider, self.victim = helpers.make_nodes(self.tmp.name) + _, self.provider, self.victim = helpers.make_nodes(self.tmp.name, test=self) self.addCleanup(self.provider.stop) self.addCleanup(self.victim.stop) self.root, blocks = helpers.bomb_dag(fanout=5, children=3000) diff --git a/tests/security/test_new1_publisher_withdraw_race.py b/tests/security/test_new1_publisher_withdraw_race.py index f7cb6ce..95951d7 100644 --- a/tests/security/test_new1_publisher_withdraw_race.py +++ b/tests/security/test_new1_publisher_withdraw_race.py @@ -66,11 +66,13 @@ def setUp(self): state = os.path.join(self.tmp.name, "state") config = Config(rnfs_dir=self.tmp.name, republish_interval=1000) self.storage = Storage(os.path.join(self.tmp.name, "blocks"), state, 10**9) + self.addCleanup(self.storage.close) self.records = ProviderStore(1000, None) self.client = HoldingClient() self.clock = FakeClock(1_000_000.0) self.publisher = Publisher(OWN, config, self.storage, self.records, self.client, lambda key: FakeLookup(NODES), False, state, clock=self.clock) + self.addCleanup(self.publisher.close) self.cid = builder.build_bytes(b"some content", self.storage.put).cid self.storage.pin(self.cid, USER) self.publisher.refresh() diff --git a/tests/security/test_new2_follow_lxmf_pins.py b/tests/security/test_new2_follow_lxmf_pins.py index 0cced6d..0311f6c 100644 --- a/tests/security/test_new2_follow_lxmf_pins.py +++ b/tests/security/test_new2_follow_lxmf_pins.py @@ -1,13 +1,13 @@ """NEW-2 (the same pattern in other pin sources): follow: and lxmf: pins that a crash left unrecorded were never released. -- follow: unfollow saves follow.json before releasing the pin, and check() pins the new target before saving - follow.json and only then releases the target recorded there. A crash in between left a `follow:` pin - that no later check, unfollow or `pin rm` released. +- follow: unfollow saves the followed names before releasing the pin, and check() pins the new target before + saving the name's and only then releases the target recorded there. A crash in between left a + `follow:` pin that no later check, unfollow or `pin rm` released. - lxmf: an auto-fetched pin is saved at once, the inbox only every few seconds, so a crash could leave an `lxmf:` pin whose message the saved inbox doesn't have; inbox removal could never release it. The crash is a BaseException from a wrapped call, and the node or feature is then built again on the same -directory without stop(), as in a real crash. +directory without stop(), as in a real crash (each is stopped and closed only when the test is done). """ import io import os @@ -17,12 +17,14 @@ import RNS +from rnfs import statedb from rnfs.config import Config from rnfs.content.cid import Cid from rnfs.content.storage import USER from rnfs.features import lxmf from rnfs.features.follow import follow_source from rnfs.node import RnfsNode +from tests.support import state_rows from tests.test_lxmf import SENDER, FeatureTestCase from tests.test_names import FakeNetwork from tests.test_names_keep import KeeperNet @@ -56,7 +58,9 @@ def setUp(self): self.net = KeeperNet() self.net.sender = B_ID self.a = self.make_node("a", A_ID) + self.addCleanup(self.a.stop) self.b = self.make_node("b", B_ID) + self.addCleanup(self.b.stop) self.a.routing.heard(B_ID) self.b.routing.heard(A_ID) self.name = bytes.fromhex(self.a.name_create("site")["name"]) @@ -78,7 +82,7 @@ def publish(self, data: bytes, sequence: int, follower: RnfsNode) -> Cid: return cid def test_crash_inside_unfollow_leaves_no_pin_after_restart(self): - """Crash after follow.json no longer lists the name, before its pin is released.""" + """Crash after the name's row is deleted, before its pin is released.""" follow = self.b.feature("follow") cid1 = self.publish(b"version one", 1, self.b) follow.follow(self.name) @@ -92,7 +96,7 @@ def test_crash_inside_unfollow_leaves_no_pin_after_restart(self): self.assertEqual(b.storage.pinned_by(self.source), set(), f"{cid1} stays pinned for a name not followed") def test_crash_between_pinning_and_saving_is_cleaned_up_by_the_next_move(self): - """Crash after pinning the new target, before follow.json records it; the name then moves again.""" + """Crash after pinning the new target, before the name's row records it; the name then moves again.""" follow = self.b.feature("follow") self.publish(b"version one", 1, self.b) follow.follow(self.name) @@ -111,6 +115,35 @@ def test_crash_between_pinning_and_saving_is_cleaned_up_by_the_next_move(self): b.feature("follow").unfollow(self.name) self.assertEqual(b.storage.pinned_by(self.source), set()) + def test_crash_inside_the_commit_recording_the_new_target(self): + """The process dies inside the commit of the name's new target: nothing of it is saved, and the next + start and move clean up as above.""" + follow = self.b.feature("follow") + cid1 = self.publish(b"version one", 1, self.b) + follow.follow(self.name) + self.assertTrue(follow.check(self.name, network=False)) + cid2 = self.publish(b"version two", 2, self.b) + commit = statedb._Database._commit + + def crashing(database, conn): + """Die in the commit that records cid2 as the name's target.""" + if conn.execute("SELECT count(*) FROM follow WHERE pinned = ?", (cid2.binary,)).fetchone()[0]: + raise Crash() + commit(database, conn) + + with mock.patch.object(statedb._Database, "_commit", crashing): + with self.assertRaises(Crash): + follow.check(self.name, network=False) + state = self.b.config.state_dir + self.assertEqual(state_rows(state, "SELECT pinned FROM follow"), [(cid1.binary,)]) + b = self.make_node("b", B_ID) + self.addCleanup(b.stop) + b.routing.heard(A_ID) + self.assertEqual(b.storage.pinned_by(self.source), {cid1, cid2}) # a followed name's: kept + cid3 = self.publish(b"version three", 3, b) + self.assertTrue(b.feature("follow").check(self.name, network=False)) + self.assertEqual(b.storage.pinned_by(self.source), {cid3}) + class LxmfCrashTest(FeatureTestCase): """B auto-fetches a link from a trusted sender, then crashes before its inbox is saved.""" @@ -127,7 +160,7 @@ def test_pin_of_a_message_the_saved_inbox_lacks_is_released_at_start(self): self.assertEqual(feature.inbox.get(lost)["auto"][0]["state"], "pinned") self.b.storage.pin(other, USER) # another source's pin is never touched # Crash before the inbox is saved: the feature is built again from what is on disk - again = lxmf.LxmfFeature(self.b) + again = self.reloaded() source = f"lxmf:{SENDER.hex()}" # NEW-2: fails until lxmf: pins the saved inbox doesn't record are released at start self.assertEqual(self.b.storage.pinned_by(source), {target}) diff --git a/tests/security/test_new2_name_publish_crash.py b/tests/security/test_new2_name_publish_crash.py index c72cc4c..030c03e 100644 --- a/tests/security/test_new2_name_publish_crash.py +++ b/tests/security/test_new2_name_publish_crash.py @@ -1,24 +1,29 @@ """NEW-2: a crash or a failed save inside `name publish` left a `name:` pin nothing could remove. -publish() pinned the new target, released the old one (each saved to pins.json at once) and only then saved -names.json. A crash in between left names.json pointing at the old target, now unpinned (gc deleted it while the -name was announced at it), and the new target pinned by `name:`, which `pin rm` refuses and `name rm` and -later publishes never released. A full disk at names.json had the same effect without a crash. These tests crash -an offline node at each step of a publish (a BaseException from a wrapped pin store, as the reviewer's script -does), restart it on the same directory and check that the name pins match names.json, that gc keeps the -recorded target and that `name rm` leaves no pin; and they make the saves fail with OSError. +publish() pinned the new target, released the old one (each saved at once) and only then saved the name's record +(then names.json, now the state database). A crash in between left the record pointing at the old target, now +unpinned (gc deleted it while the name was announced at it), and the new target pinned by `name:`, which +`pin rm` refuses and `name rm` and later publishes never released. A full disk at the record's save had the same +effect without a crash. These tests crash an offline node at each step of a publish (a BaseException from a +wrapped pin store, as the reviewer's script does), restart it on the same directory and check that the name pins +match the saved record, that gc keeps the recorded target and that `name rm` leaves no pin; and they make the +saves fail with OSError. """ import errno import io import os import tempfile import unittest +from contextlib import contextmanager +from unittest import mock import RNS from rnfs.config import Config from rnfs.content.cid import Cid from rnfs.node import RnfsNode +from rnfs.statedb import StateDbError +from tests.support import state_rows from tests.test_names import FakeNetwork from tests.test_node import NetClient @@ -97,7 +102,7 @@ def name_pins(node: RnfsNode) -> set[Cid]: """The CIDs the name's pin source holds.""" return node.storage.pinned_by(SOURCE) - def test_crash_at_each_step_leaves_pins_matching_names_json(self): + def test_crash_at_each_step_leaves_pins_matching_the_record(self): """After the restart, the name's pins are exactly its recorded target, which gc keeps and `name rm` releases, wherever the publish crashed.""" problems = [] @@ -112,13 +117,15 @@ def crash(): a.names._save = crash with self.assertRaises(Crash): a.name_publish("site", str(cid2)) - # a is abandoned without stop(), as in a real crash: only what it wrote survives + # a is abandoned without stop(), as in a real crash: only what it wrote survives. Its references + # to the state database are released once done with, so the directory can be removed (Windows) + self.addCleanup(a._close_state) b = self.open_node(root) recorded = Cid.parse(b.name_list()[0]["cid"]) if self.name_pins(b) != {recorded}: - problems.append(f"{step}: name pins {sorted(map(str, self.name_pins(b)))} but names.json " - f"records {recorded}") + problems.append(f"{step}: name pins {sorted(map(str, self.name_pins(b)))} but the saved " + f"record is {recorded}") b.gc() if not b.storage.is_complete(recorded): problems.append(f"{step}: gc deleted the recorded target {recorded}") @@ -126,38 +133,43 @@ def crash(): if self.name_pins(b): problems.append(f"{step}: `name rm` left {sorted(map(str, self.name_pins(b)))} pinned") b.stop() - # NEW-2: fails until publish saves names.json between pinning the new target and releasing the old one, - # and the node reconciles name pins with names.json at start + # NEW-2: fails until publish saves the record between pinning the new target and releasing the old one, + # and the node reconciles name pins with the saved records at start self.assertEqual(problems, []) def test_failed_save_rolls_the_publish_back(self): - """names.json can't be written (ENOSPC): the publish fails and nothing changes, in memory or on disk.""" + """The record can't be committed (ENOSPC): the publish fails and nothing changes, in memory or on disk.""" a, cid1, cid2 = self.first_publish(os.path.join(self.tmp.name, "a")) - save = a.names._save + transaction = a.names._db.transaction + saved = state_rows(a.config.state_dir, "SELECT * FROM names_published") - def full(): - a.names._save = save - raise OSError(errno.ENOSPC, "No space left on device") - a.names._save = full - with self.assertRaises(OSError): + @contextmanager + def full(durable=False): + """The names' transaction, whose commit fails as on a full disk (pins still save).""" + with transaction(durable) as tx: + yield tx + raise StateDbError(errno.ENOSPC, "database or disk is full") + + with mock.patch.object(a.names._db, "transaction", full), self.assertRaises(OSError): a.name_publish("site", str(cid2)) listed = a.name_list()[0] - # NEW-2: fails until a failed names.json save rolls back the publish's pin changes + # NEW-2: fails until a failed save of the record rolls back the publish's pin changes self.assertEqual(self.name_pins(a), {cid1}) self.assertEqual((listed["cid"], listed["sequence"]), (str(cid1), 1)) + self.assertEqual(state_rows(a.config.state_dir, "SELECT * FROM names_published"), saved) self.assertEqual(a.name_publish("site", str(cid2))["sequence"], 2) # trying again works self.assertEqual(self.name_pins(a), {cid2}) a.stop() def test_failed_release_of_the_old_target_is_retried(self): - """Releasing the old target fails (pins.json can't be written): the publish stands, and the next one + """Releasing the old target fails (the pin's release can't be saved): the publish stands, and the next one releases it.""" a, cid1, cid2 = self.first_publish(os.path.join(self.tmp.name, "a")) a.names._pins = FailingPins(a.names._pins, "before unpin", OSError(errno.EIO, "Input/output error")) try: result = a.name_publish("site", str(cid2)) except OSError as e: - # NEW-2: fails until a release that fails after names.json was saved is kept for later, not raised + # NEW-2: fails until a release that fails after the record was saved is kept for later, not raised self.fail(f"the publish failed after pinning its new target: {e}") self.assertEqual(result["sequence"], 2) self.assertEqual(self.name_pins(a), {cid1, cid2}) # cid1 still held: nothing was lost @@ -176,7 +188,7 @@ def test_name_pins_nothing_records_are_released_at_start(self): a.storage.pin(cid2, "name:gone") # an alias whose key and record are both gone a.stop() b = self.open_node(root) - # NEW-2: fails until name pins are reconciled with names.json at start + # NEW-2: fails until name pins are reconciled with the saved records at start self.assertEqual(self.name_pins(b), {cid1}) self.assertEqual(b.storage.pinned_by("name:gone"), set()) b.stop() diff --git a/tests/security/test_new3_remote_pin_wakeup.py b/tests/security/test_new3_remote_pin_wakeup.py index e558333..e0c0b3a 100644 --- a/tests/security/test_new3_remote_pin_wakeup.py +++ b/tests/security/test_new3_remote_pin_wakeup.py @@ -89,7 +89,7 @@ def setUp(self): self.tmp = tempfile.TemporaryDirectory() self.addCleanup(self.tmp.cleanup) # make_nodes' second node (B) already knows the first (A): here A provides and B is asked to pin - _, self.provider, self.a = helpers.make_nodes(self.tmp.name) + _, self.provider, self.a = helpers.make_nodes(self.tmp.name, test=self) self.addCleanup(self.provider.stop) self.addCleanup(self.a.stop) self.cids = [] diff --git a/tests/security/test_new4_private_mode_records.py b/tests/security/test_new4_private_mode_records.py index 46b6836..12319f9 100644 --- a/tests/security/test_new4_private_mode_records.py +++ b/tests/security/test_new4_private_mode_records.py @@ -1,6 +1,6 @@ """NEW-4: in private mode, /find replies listed provider records of non-members stored before the switch. -NodeBackend.find filtered providers only by bans, and ProviderStore._load reloaded providers.json with no +NodeBackend.find filtered providers only by bans, and ProviderStore._load reloaded saved records with no membership check. The request server refuses non-members' /provide once private mode is on, but records already held (stored while the node was public, or from a member removed since) kept being handed to members for up to PROVIDER_TTL, and a member's client then opened identified links to those outsiders. @@ -83,7 +83,7 @@ def test_records_from_before_private_mode_are_not_handed_out(self): for publisher in (OUTSIDER, MEMBER): self.assertTrue(public.backend.provide(publisher, [KEY], [])) public.records.store(KEY, OWN) - public.stop() # saves providers.json + public.stop() # saves the provider records private = self.open(**self.members(MEMBER)) providers, _ = private.backend.find(KEY) diff --git a/tests/security/test_new5_storage_lock_walks.py b/tests/security/test_new5_storage_lock_walks.py index 15f89fa..c15d886 100644 --- a/tests/security/test_new5_storage_lock_walks.py +++ b/tests/security/test_new5_storage_lock_walks.py @@ -40,6 +40,7 @@ def setUp(self): self.blocks = os.path.join(self.tmp.name, "blocks") self.state = os.path.join(self.tmp.name, "state") storage = Storage(self.blocks, self.state, 10**12) + self.addCleanup(storage.close) self.roots = [] for d in range(12): entries = [] @@ -59,6 +60,7 @@ def setUp(self): storage.save() self.used = storage.used_bytes() self.storage = Storage(self.blocks, self.state, self.used // 2) + self.addCleanup(self.storage.close) @staticmethod def entry(storage: Storage, root, name: str): diff --git a/tests/security/test_new6_publisher_crash_cut_round.py b/tests/security/test_new6_publisher_crash_cut_round.py index 75938ef..b8b41db 100644 --- a/tests/security/test_new6_publisher_crash_cut_round.py +++ b/tests/security/test_new6_publisher_crash_cut_round.py @@ -1,10 +1,10 @@ """NEW-6 (publisher half): records stored during a publish round cut short by a crash were never withdrawn. -Where a round stored a CID was only written to published.json at the round's end, so after a crash mid-round -nothing listed those nodes, and when the CID later stopped being provided they kept the record for up to -PROVIDER_TTL (section 6 SHOULD: withdraw from nodes of a round cut short). These tests crash a round from the fake -client after some nodes accepted, restart the publisher on the same state directory, stop providing the CID, and -check that every node saved as accepted (every SAVE_EVERY_NODES nodes or SAVE_EVERY_SECONDS) gets a withdrawal. +Where a round stored a CID was only saved at the round's end, so after a crash mid-round nothing listed those +nodes, and when the CID later stopped being provided they kept the record for up to PROVIDER_TTL (section 6 +SHOULD: withdraw from nodes of a round cut short). These tests crash a round from the fake client after some +nodes accepted, restart the publisher on the same state directory, stop providing the CID, and check that every +node saved as accepted (every SAVE_EVERY_NODES nodes or SAVE_EVERY_SECONDS) gets a withdrawal. """ import os import tempfile @@ -71,14 +71,17 @@ def setUp(self): self.state = os.path.join(self.tmp.name, "state") self.config = Config(rnfs_dir=self.tmp.name, republish_interval=1000) self.storage = Storage(os.path.join(self.tmp.name, "blocks"), self.state, 10**9) + self.addCleanup(self.storage.close) self.clock = FakeClock(1_000_000.0) self.cid = builder.build_bytes(b"some content", self.storage.put).cid self.storage.pin(self.cid, USER) def publisher(self, client, nodes) -> Publisher: - """A publisher on the shared state directory whose lookups find nodes.""" - return Publisher(OWN, self.config, self.storage, ProviderStore(1000, None), client, - lambda key: FakeLookup(nodes), False, self.state, clock=self.clock) + """A publisher on the shared state directory whose lookups find nodes (closed when the test ends).""" + publisher = Publisher(OWN, self.config, self.storage, ProviderStore(1000, None), client, + lambda key: FakeLookup(nodes), False, self.state, clock=self.clock) + self.addCleanup(publisher.close) + return publisher def crash_then_unpin(self, nodes, crash_at: int, step: float = 0.0) -> set[bytes]: """Crash the first round before request crash_at; restart, unpin, publish; the nodes withdrawn from.""" diff --git a/tests/security/test_new6_remote_pins_bad_entry.py b/tests/security/test_new6_remote_pins_bad_entry.py index 21e6c59..4904df2 100644 --- a/tests/security/test_new6_remote_pins_bad_entry.py +++ b/tests/security/test_new6_remote_pins_bad_entry.py @@ -5,9 +5,12 @@ and _start_threads, and the node failed to start. The fixed node skips such entries (with a log line) and resumes the rest. +The pins waiting now live in the state database: a remote_pins.json from an earlier version is imported at the +first start, skipping malformed entries the same way, and malformed rows in the database are skipped at load. + The pinning node is an offline node; a provider joined to it by tests.test_node's NetClient holds the content. -The file is written as a damaged or hand-edited one might be, and the node is started the way RnfsNode.__init__ -starts it (_start_threads). +The file (or row) is written as a damaged or hand-edited one might be, and the node is started the way +RnfsNode.__init__ starts it (_start_threads). """ import io import json @@ -22,8 +25,8 @@ from rnfs.config import Config from rnfs.content.cid import Cid from rnfs.content.storage import remote_source -from rnfs.node import REMOTE_PINS_FILE, RnfsNode -from tests.support import wait_for +from rnfs.node import RnfsNode +from tests.support import state_execute, wait_for from tests.test_names import FakeNetwork from tests.test_node import A_ID, B_ID, NetClient @@ -65,12 +68,19 @@ def open(self, name, node_id) -> RnfsNode: self.addCleanup(node.stop) return node - def start_with(self, entries) -> RnfsNode: - """Write entries as B's remote_pins.json, then open B and start it as RnfsNode.__init__ does.""" + def start_with(self, entries=None, rows=()) -> RnfsNode: + """Write entries as B's remote_pins.json (from an earlier version) or rows into its remote_pins table, then + open B and start it as RnfsNode.__init__ does.""" state_dir = Config(rnfs_dir=os.path.join(self.tmp.name, "b")).state_dir os.makedirs(state_dir, exist_ok=True) - with open(os.path.join(state_dir, REMOTE_PINS_FILE), "w", encoding="utf-8") as f: - json.dump(entries, f) + if entries is not None: + with open(os.path.join(state_dir, "remote_pins.json"), "w", encoding="utf-8") as f: + json.dump(entries, f) + if rows: + self.open("b", B_ID).stop() # creates the database + for position, (source, cid) in enumerate(rows): + state_execute(state_dir, "INSERT INTO remote_pins (position, source, cid) VALUES (?, ?, ?)", + (position, source, cid)) node = self.open("b", B_ID) node.routing.heard(A_ID) try: @@ -89,9 +99,20 @@ def test_unhashable_sources_are_skipped(self): [SOURCE, str(self.cid)]]) self.assertTrue(wait_for(lambda: SOURCE in node.storage.pins().get(self.cid, set()), 15), "the good remote pin wasn't resumed") - self.assertTrue(any("4 malformed entries" in message and REMOTE_PINS_FILE in message + self.assertTrue(any("4 malformed entries" in message and "remote_pins.json" in message for message in logged), logged) + def test_malformed_rows_are_skipped(self): + """Rows whose source isn't text or whose CID isn't a binary CID are skipped and logged; the good one is + still fetched.""" + logged = [] + with mock.patch.object(node_module, "log", side_effect=lambda message, *args: logged.append(message)): + node = self.start_with(rows=[(SOURCE, str(self.cid)), (SOURCE, self.cid.binary[:-1]), + (SOURCE.encode(), self.cid.binary), (SOURCE, self.cid.binary)]) + self.assertTrue(wait_for(lambda: SOURCE in node.storage.pins().get(self.cid, set()), 15), + "the good remote pin wasn't resumed") + self.assertTrue(any("3 malformed remote pins" in message for message in logged), logged) + if __name__ == "__main__": unittest.main() diff --git a/tests/security/test_new6_remote_pins_restart.py b/tests/security/test_new6_remote_pins_restart.py index 6b2b4ee..9b6163b 100644 --- a/tests/security/test_new6_remote_pins_restart.py +++ b/tests/security/test_new6_remote_pins_restart.py @@ -2,8 +2,8 @@ RnfsNode kept accepted /pin requests only in memory (_remote_reserved). The requester got True, but if the node stopped before fetching them, nothing fetched them after the restart, and nothing said so. The fixed -node saves the pins waiting (state/remote_pins.json, in the order accepted) and resumes them at start; pins -released before the stop, or from identities no longer on remote_pin_identities, are not resumed. +node saves the pins waiting (the state database's remote_pins table, in the order accepted) and resumes them at +start; pins released before the stop, or from identities no longer on remote_pin_identities, are not resumed. The tests use two offline nodes joined by tests.test_node's NetClient. The provider's /blocks answers are held while the pinning node stops, so its pins are still waiting then; the node is then reopened over the same diff --git a/tests/security/test_r1_reconcile_damaged_state.py b/tests/security/test_r1_reconcile_damaged_state.py index 280b6e7..e8c7469 100644 --- a/tests/security/test_r1_reconcile_damaged_state.py +++ b/tests/security/test_r1_reconcile_damaged_state.py @@ -1,15 +1,20 @@ -"""R-1: the start-up reconcile released follow: and lxmf: pins when follow.json or the LXMF inbox was missing, -damaged or had a malformed entry. +"""R-1: the start-up reconcile released follow: and lxmf: pins when the followed names or the LXMF inbox were +missing, damaged or had a malformed entry. -NEW-2 made the node release, at start, follow pins of names follow.json doesn't list and lxmf pins no inbox -message records. But a follow.json or lxmf_inbox.json that is missing, damaged (read_json moves it aside) or -holds a malformed entry lists nothing for the names and messages it lost, so a restart released pins of names -still followed and of messages still wanted. The fixed reconcile releases only when the file was read whole; -otherwise it keeps the pins and records them again (the name is followed again; the sender's pins get a new -inbox message), so they also survive later starts and can still be released by unfollowing or removing it. +NEW-2 made the node release, at start, follow pins of names the follow state doesn't list and lxmf pins no inbox +message records. But a follow state or inbox that is missing, damaged or holds a malformed entry lists nothing +for the names and messages it lost, so a restart released pins of names still followed and of messages still +wanted. The fixed reconcile releases only when the state was read whole (clean); otherwise it keeps the pins and +records them again (the name is followed again; the sender's pins get a new inbox message), so they also survive +later starts and can still be released by unfollowing or removing it. + +The state lives in the state database; follow.json and lxmf_inbox.json are what rnfs 0.1.4 wrote, imported at +the first start. Each test takes B's saved follow or inbox state away between two starts and puts back a version +of it: none (missing), the 0.1.4 file truncated (moved aside as damaged when imported) or with the entry +malformed (imported, but not clean), or the database's own rows with the entry malformed or the clean value gone. The follow tests use two offline nodes (A owns a name B follows) on KeeperNet, as test_new2_follow_lxmf_pins -does; the lxmf tests use tests.test_lxmf's FeatureTestCase. Each damages the state file between two starts. +does; the lxmf tests use tests.test_lxmf's FeatureTestCase. """ import io import json @@ -22,8 +27,9 @@ from rnfs.config import Config from rnfs.content.cid import Cid from rnfs.features import lxmf -from rnfs.features.follow import FOLLOW_FILE, follow_source +from rnfs.features.follow import follow_source from rnfs.node import RnfsNode +from tests.support import state_execute, write_legacy from tests.test_lxmf import SENDER, FeatureTestCase from tests.test_names import FakeNetwork from tests.test_names_keep import KeeperNet @@ -43,27 +49,27 @@ def tearDownModule(): RNS.loglevel = _saved_loglevel -def damage(path: str, how: str, malformed): - """Make the state file at path missing, damaged (truncated JSON) or hold one malformed entry: malformed(state) - changes the loaded state in place.""" - if how == "missing": - os.remove(path) - elif how == "damaged": - with open(path, "r+", encoding="utf-8") as f: - text = f.read() - f.seek(0) - f.truncate() - f.write(text[:len(text) // 2]) - else: - with open(path, encoding="utf-8") as f: - state = json.load(f) - malformed(state) - with open(path, "w", encoding="utf-8") as f: - json.dump(state, f) +def damage(state_dir: str, how: str, store: str, name: str, legacy, malformed_row: tuple[str, tuple]): + """Take store's saved state (its rows and meta values) away and put back a version that isn't whole: none + ("missing"), the legacy file `name` holding legacy truncated ("damaged") or with an entry malformed + ("malformed": legacy is that), the database's rows with an entry malformed ("malformed row": malformed_row's + statement and parameters are run instead), or its meta values without the clean one ("unclean").""" + if how == "malformed row": + state_execute(state_dir, *malformed_row) + return + table, where = ("lxmf_messages", " WHERE box = 'inbox'") if store == lxmf.INBOX else (store, "") + state_execute(state_dir, f"DELETE FROM {table}{where}") + state_execute(state_dir, "DELETE FROM meta WHERE store = ?" + (" AND key = 'clean'" if how == "unclean" else ""), + (store,)) + if how == "damaged": + text = json.dumps(legacy) + write_legacy(state_dir, name, text=text[:len(text) // 2]) + elif how == "malformed": + write_legacy(state_dir, name, legacy) class FollowDamagedStateTest(unittest.TestCase): - """B follows A's name; follow.json is then lost or damaged and B restarts.""" + """B follows A's name; its saved follow state is then lost or damaged and B restarts.""" def setUp(self): """Two nodes that know each other, A's name, and B following it at its first target.""" @@ -87,7 +93,7 @@ def setUp(self): follow.follow(self.name) self.assertTrue(follow.check(self.name, network=False)) self.assertEqual(self.b.storage.pinned_by(self.source), {self.cid}) - self.path = os.path.join(self.b.config.state_dir, FOLLOW_FILE) + self.state = self.b.config.state_dir def make_node(self, label: str, node_id: bytes) -> RnfsNode: """An offline node on its own directory (a restart if it was used before).""" @@ -104,71 +110,109 @@ def restart(self) -> RnfsNode: return self.b def check_kept(self, how: str): - """After follow.json is made `how`, restarts keep the pin and follow the name again; unfollow releases.""" + """After the follow state is made `how`, restarts keep the pin and follow the name again; unfollow + releases.""" self.b.stop() - damage(self.path, how, lambda state: state["names"][self.name.hex()].update(pinned="not-a-cid")) + # follow.json as rnfs 0.1.4 wrote it, with the name's entry malformed + legacy = {"version": 1, "names": {self.name.hex(): {"pinned": "not-a-cid" if how == "malformed" else + str(self.cid)}}} + damage(self.state, how, "follow", "follow.json", legacy, + ("UPDATE follow SET pinned = ? WHERE name = ?", (b"\x01\x02", self.name))) b = self.restart() - # R-1: fails until follow pins are released at start only when follow.json was read whole + # R-1: fails until follow pins are released at start only when the follow state was read whole self.assertEqual(b.storage.pinned_by(self.source), {self.cid}, - f"a {how} {FOLLOW_FILE} released the pin of a name still followed") + f"a {how} follow state released the pin of a name still followed") self.assertEqual([entry["name"] for entry in b.feature("follow").following()], [self.name.hex()]) - b = self.restart() # follow.json was written again: the next start keeps the pin too + if how == "damaged": + self.assertTrue(os.path.exists(os.path.join(self.state, "follow.json.damaged"))) + elif how == "malformed": + self.assertTrue(os.path.exists(os.path.join(self.state, "follow.json.migrated"))) + b = self.restart() # the name was saved again: the next start keeps the pin too self.assertEqual(b.storage.pinned_by(self.source), {self.cid}) self.assertEqual(b.feature("follow").following()[0]["pinned"], str(self.cid)) b.feature("follow").unfollow(self.name) self.assertEqual(b.storage.pinned_by(self.source), set()) - def test_missing_follow_file_keeps_pins(self): - """follow.json deleted.""" + def test_missing_follow_state_keeps_pins(self): + """No follow state saved.""" self.check_kept("missing") def test_damaged_follow_file_keeps_pins(self): - """follow.json truncated (moved aside as damaged at start).""" + """follow.json truncated (moved aside as damaged when imported).""" self.check_kept("damaged") def test_malformed_follow_entry_keeps_pins(self): - """follow.json with the name's entry malformed (skipped at load).""" + """follow.json with the name's entry malformed (skipped when imported).""" self.check_kept("malformed") + def test_malformed_follow_row_keeps_pins(self): + """The name's row with a pinned value that isn't a CID (dropped at load).""" + self.check_kept("malformed row") + + def test_follow_state_without_its_clean_value_keeps_pins(self): + """The name's row lost with the value saying the state was saved whole (with it, the pin is released: + test_new2_follow_lxmf_pins).""" + self.check_kept("unclean") + class LxmfDamagedStateTest(FeatureTestCase): - """B auto-fetches and pins a link from a trusted sender; its inbox file is then lost or damaged.""" + """B auto-fetches and pins a link from a trusted sender; its saved inbox is then lost or damaged.""" def check_kept(self, how: str): - """After lxmf_inbox.json is made `how`, the pin is kept and recorded; removing that record releases it.""" + """After the inbox is made `how`, the pin is kept and recorded; removing that record releases it.""" feature = self.feature_node(lxmf_auto_fetch_from=[SENDER.hex()]) kept = feature.received(self.link_message(number=1)) feature.process_auto_fetches() feature.save() target = Cid.parse(feature.inbox.get(kept)["auto"][0]["cid"]) source = f"lxmf:{SENDER.hex()}" - path = os.path.join(self.b.config.state_dir, "lxmf_inbox.json") - damage(path, how, lambda state: state["messages"][0].update(links="not a list")) - again = lxmf.LxmfFeature(self.b) - # R-1: fails until lxmf pins are released at start only when the inbox file was read whole + # lxmf_inbox.json as rnfs 0.1.4 wrote it, with the message's record malformed + messages = feature.inbox_list() + legacy = {"next_id": 2, "messages": [{**messages[0], "links": "not a list"} if how == "malformed" else + messages[0]]} + self.b.stop() + state = self.b.config.state_dir + damage(state, how, lxmf.INBOX, "lxmf_inbox.json", legacy, + ("UPDATE lxmf_messages SET record = '{not json' WHERE box = 'inbox' AND id = ?", (kept,))) + self.b = self.make_node("b", B_ID, lxmf_auto_fetch_from=[SENDER.hex()]) + again = self.b.feature("lxmf") + # R-1: fails until lxmf pins are released at start only when the inbox was read whole self.assertEqual(self.b.storage.pinned_by(source), {target}, - f"a {how} inbox file released the pin of a message the inbox lost") + f"a {how} inbox released the pin of a message the inbox lost") records = [entry for entry in again.inbox_list() if {"state": "pinned", "cid": str(target)} in (entry.get("auto") or [])] self.assertEqual([entry["from"] for entry in records], [SENDER.hex()]) self.assertEqual([lxmf.Link.from_json(link).cid for link in records[0]["links"]], [target]) - again = lxmf.LxmfFeature(self.b) # the record was saved: the next start keeps the pin too + if how == "damaged": + self.assertTrue(os.path.exists(os.path.join(state, "lxmf_inbox.json.damaged"))) + elif how == "malformed": + self.assertTrue(os.path.exists(os.path.join(state, "lxmf_inbox.json.migrated"))) + again = self.reloaded() # the record was saved: the next start keeps the pin too self.assertEqual(self.b.storage.pinned_by(source), {target}) self.assertEqual(again.inbox_rm(records[0]["id"]), 1) self.assertEqual(self.b.storage.pinned_by(source), set()) def test_missing_inbox_keeps_pins(self): - """lxmf_inbox.json deleted.""" + """No inbox saved.""" self.check_kept("missing") def test_damaged_inbox_keeps_pins(self): - """lxmf_inbox.json truncated (moved aside as damaged at start).""" + """lxmf_inbox.json truncated (moved aside as damaged when imported).""" self.check_kept("damaged") def test_malformed_inbox_record_keeps_pins(self): - """lxmf_inbox.json with the message's record malformed (dropped at load).""" + """lxmf_inbox.json with the message's record malformed (dropped when imported).""" self.check_kept("malformed") + def test_malformed_inbox_row_keeps_pins(self): + """The message's row isn't JSON (dropped at load).""" + self.check_kept("malformed row") + + def test_inbox_without_its_clean_value_keeps_pins(self): + """The message's row lost with the value saying the inbox was saved whole (with it, the pin is released: + test_new2_follow_lxmf_pins).""" + self.check_kept("unclean") + if __name__ == "__main__": unittest.main() diff --git a/tests/security/test_r2_name_key_unreadable.py b/tests/security/test_r2_name_key_unreadable.py index 8703ffa..39f7fe1 100644 --- a/tests/security/test_r2_name_key_unreadable.py +++ b/tests/security/test_r2_name_key_unreadable.py @@ -1,16 +1,15 @@ """R-2: a name key file that exists but can't be read at start was treated as deleted, releasing the name's pin -and dropping its names.json record. +and dropping its saved record (then in names.json). -NameManager skipped a key file RNS couldn't load (no permission, damaged), so its alias had no key; the -names.json entry then counted as a key "deleted by hand" (release_gone() released the pin and the save dropped -the entry) and reconcile_pins released any pin of an alias with no key. Once the file could be read again the -name was back with no target and sequence 0. The fixed manager treats only a missing key file as removed: an -unreadable one keeps its pins, and its names.json entry is written back unchanged by every save. +NameManager skipped a key file RNS couldn't load (no permission, damaged), so its alias had no key; its saved +record then counted as a key "deleted by hand" (release_gone() released the pin and the save dropped the record) +and reconcile_pins released any pin of an alias with no key. Once the file could be read again the name was back +with no target and sequence 0. The fixed manager treats only a missing key file as removed: an unreadable one +keeps its pins, and every save leaves its names_published row as it is. Offline node (RnfsNode.offline over tests.test_names' FakeNetwork), restarted on the same directory. """ import io -import json import os import tempfile import unittest @@ -20,6 +19,7 @@ from rnfs.config import Config from rnfs.content.cid import Cid from rnfs.node import RnfsNode +from tests.support import state_rows from tests.test_names import FakeNetwork from tests.test_node import A_ID, NetClient @@ -49,9 +49,8 @@ def setUp(self): self.node.names.create("site") self.node.names.publish("site", self.cid) self.key = self.node.names._key_path("site") - self.state = os.path.join(self.config.state_dir, "names.json") self.recorded = self.published()["site"] - self.assertEqual(self.recorded["sequence"], 1) + self.assertEqual(self.recorded, ("1", self.cid.binary, 0)) def open(self) -> RnfsNode: """The node, opened (again) on its directory.""" @@ -68,9 +67,9 @@ def restart(self) -> RnfsNode: return self.node def published(self) -> dict: - """names.json's published entries.""" - with open(self.state, encoding="utf-8") as f: - return json.load(f)["published"] + """The names_published rows: alias -> (sequence, cid, imported).""" + rows = state_rows(self.config.state_dir, "SELECT alias, sequence, cid, imported FROM names_published") + return {alias: tuple(rest) for alias, *rest in rows} def check_kept(self, make_unreadable, make_readable): """While the key can't be read the pin and record stay (across a save); once readable, the name is back.""" @@ -79,7 +78,7 @@ def check_kept(self, make_unreadable, make_readable): # R-2: fails until only a missing key file counts as a removed name self.assertEqual(node.storage.pinned_by("name:site"), {self.cid}, "an unreadable key file released the name's pin") - node.names.create("other") # saves names.json + node.names.create("other") # saves the records self.assertEqual(self.published().get("site"), self.recorded, "the name's record was dropped") make_readable() node = self.restart() diff --git a/tests/support.py b/tests/support.py index efa63ab..b427d30 100644 --- a/tests/support.py +++ b/tests/support.py @@ -1,9 +1,15 @@ """Shared test helpers (not a test module): what the platform running the tests can do, for skipping tests that need POSIX features, and small doubles and helpers several test modules use.""" +import json import os import socket +import sqlite3 import tempfile import time +from contextlib import contextmanager +from unittest import mock + +from rnfs import statedb # Unix permission bits (modes, umask, the execute bit): Windows has none, and its st_mode is synthesised POSIX_MODES = os.name != "nt" @@ -69,3 +75,43 @@ def wait_for(condition, timeout: float = 10.0, what: str | None = None, interval if not result and what is not None: raise AssertionError(f"timed out waiting for {what}") return result + + +# The state database (rnfs.statedb) + + +def state_rows(state_dir: str, sql: str, params=()) -> list[tuple]: + """The rows a read of the state database in state_dir gives, through a connection of its own (closed again), + so it sees only what was committed: for tests checking what a store saved.""" + conn = sqlite3.connect(os.path.join(state_dir, statedb.DB_FILE)) + try: + return conn.execute(sql, params).fetchall() + finally: + conn.close() + + +def state_execute(state_dir: str, sql: str, params=()): + """Run one write on the state database in state_dir and commit it, through a connection of its own: for tests + planting rows a store must reject at load.""" + conn = sqlite3.connect(os.path.join(state_dir, statedb.DB_FILE)) + try: + with conn: + conn.execute(sql, params) + finally: + conn.close() + + +def write_legacy(state_dir: str, name: str, value=None, text: str | None = None) -> str: + """Write a JSON state file as rnfs 0.1.4 did (value as JSON, or text as is) into state_dir; its path.""" + os.makedirs(state_dir, exist_ok=True) + path = os.path.join(state_dir, name) + with open(path, "w", encoding="utf-8") as f: + f.write(text if text is not None else json.dumps(value)) + return path + + +@contextmanager +def failing_commits(error: str = "disk I/O error"): + """Within the with block, every state database commit fails as a full or failing disk makes it fail.""" + with mock.patch.object(statedb._Database, "_commit", side_effect=sqlite3.OperationalError(error)): + yield diff --git a/tests/test_archive.py b/tests/test_archive.py index 7acad48..d50ae90 100644 --- a/tests/test_archive.py +++ b/tests/test_archive.py @@ -12,6 +12,7 @@ from rnfs.content.dag import DirectoryNode, Entry from rnfs.content.storage import USER, Storage from rnfs.errors import StorageFull, TooLarge +from tests.support import failing_commits class ArchiveTest(unittest.TestCase): @@ -31,8 +32,11 @@ def setUp(self): Entry("c", self.other.cid, 5, 0)], put) def storage(self, name: str) -> Storage: - """Storage under the temporary directory.""" - return Storage(os.path.join(self.tmp.name, name, "blocks"), os.path.join(self.tmp.name, name, "state"), 10**9) + """Storage under the temporary directory (closed before the directory is removed).""" + storage = Storage(os.path.join(self.tmp.name, name, "blocks"), os.path.join(self.tmp.name, name, "state"), + 10**9) + self.addCleanup(storage.close) + return storage def export(self, roots) -> bytes: """Export roots from the source.""" @@ -84,7 +88,7 @@ def test_roots_are_pinned_with_one_save(self): def test_failed_pin_save_pins_no_root(self): data = self.export([self.root.cid, self.other.cid]) - with mock.patch("rnfs.content.storage.write_json", side_effect=OSError(28, "No space left on device")): + with failing_commits("database or disk is full"): with self.assertRaises(OSError): archive.import_archive(io.BytesIO(data), self.destination) self.assertEqual(self.destination.pins(), {}) diff --git a/tests/test_contacts.py b/tests/test_contacts.py index 7dc5247..024ae9f 100644 --- a/tests/test_contacts.py +++ b/tests/test_contacts.py @@ -20,6 +20,7 @@ from rnfs.features.contacts import check_nick, is_nick, node_arg from rnfs.node import RnfsNode +from tests.support import failing_commits, state_execute, state_rows, write_legacy from tests.test_names import FakeNetwork from tests.test_node import NetClient @@ -94,7 +95,18 @@ def make_node(self, label: str, node_id: bytes) -> RnfsNode: self.addCleanup(node.stop) return node + def reopen(self) -> contacts.ContactsFeature: + """B's address book loaded again from its state, closed when the test ends.""" + book = contacts.ContactsFeature(self.b) + self.addCleanup(book.close) + return book + + def rows(self) -> list[tuple]: + """What B's contacts table holds, by nick.""" + return state_rows(self.b.config.state_dir, "SELECT nick, kind, id FROM contacts ORDER BY nick") + def test_add_list_remove_persist(self): + """Contacts are saved as added and removed, and loaded again.""" self.book.add("alice", self.name.hex(), "name") self.book.add("anode", "<" + A_ID.hex().upper() + ">", "node") self.assertEqual(self.book.list(), [{"nick": "alice", "kind": "name", "id": self.name.hex()}, @@ -103,13 +115,11 @@ def test_add_list_remove_persist(self): self.assertIsNone(self.book.lookup("anode", "lxmf")) self.assertIsNone(self.book.lookup("nobody")) self.assertIsNone(self.book.lookup("Bad Nick")) - again = contacts.ContactsFeature(self.b) - self.assertEqual(again.list(), self.book.list()) + self.assertEqual(self.reopen().list(), self.book.list()) self.book.remove("anode") with self.assertRaises(NotFound): self.book.remove("anode") - with open(os.path.join(self.b.config.state_dir, contacts.CONTACTS_FILE), encoding="utf-8") as f: - self.assertEqual(list(json.load(f)["contacts"]), ["alice"]) + self.assertEqual(self.rows(), [("alice", "name", self.name)]) def test_invalid_input(self): for nick, ident, kind in (("Alice", self.name.hex(), "name"), ("alice", "xyz", "name"), @@ -118,13 +128,38 @@ def test_invalid_input(self): self.book.add(nick, ident, kind) def test_malformed_file_entries_skipped(self): - path = os.path.join(self.b.config.state_dir, contacts.CONTACTS_FILE) - with open(path, "w", encoding="utf-8") as f: - json.dump({"version": 1, "contacts": {"ok": {"kind": "node", "id": A_ID.hex()}, - "BAD": {"kind": "node", "id": A_ID.hex()}, - "worse": {"kind": "spaceship", "id": A_ID.hex()}, - "worst": "nope"}}, f) - self.assertEqual([c["nick"] for c in contacts.ContactsFeature(self.b).list()], ["ok"]) + """contacts.json's malformed entries are skipped when it is imported.""" + write_legacy(self.b.config.state_dir, "contacts.json", + {"version": 1, "contacts": {"ok": {"kind": "node", "id": A_ID.hex()}, + "BAD": {"kind": "node", "id": A_ID.hex()}, + "worse": {"kind": "spaceship", "id": A_ID.hex()}, + "worst": "nope"}}) + self.assertEqual([c["nick"] for c in self.reopen().list()], ["ok"]) + + def test_malformed_rows_deleted(self): + """Rows that aren't a valid contact, and any past MAX_CONTACTS, are skipped at load and deleted.""" + self.book.add("ok", A_ID.hex(), "node") + self.book.add("zed", B_ID.hex(), "node") + for nick, kind, ident in (("BAD", "node", A_ID), ("worse", "spaceship", A_ID), ("short", "node", b"\x01"), + ("text", "node", A_ID.hex()), ("ab" * 16, "node", A_ID)): + state_execute(self.b.config.state_dir, "INSERT INTO contacts (nick, kind, id) VALUES (?, ?, ?)", + (nick, kind, ident)) + with mock.patch.object(contacts, "MAX_CONTACTS", 1): + self.assertEqual([c["nick"] for c in self.reopen().list()], ["ok"]) + self.assertEqual(self.rows(), [("ok", "node", A_ID)]) + + def test_failed_save_is_written_with_the_next_change(self): + """A change whose commit failed raises and stays in memory, as before; the next change saves it too.""" + with failing_commits(), self.assertRaises(OSError): + self.book.add("anode", A_ID.hex(), "node") + self.assertEqual(self.book.lookup("anode", "node"), A_ID) + self.assertEqual(self.rows(), []) + self.book.add("bnode", B_ID.hex(), "node") + self.assertEqual(self.rows(), [("anode", "node", A_ID), ("bnode", "node", B_ID)]) + with failing_commits(), self.assertRaises(OSError): + self.book.remove("anode") + self.book.remove("bnode") + self.assertEqual(self.rows(), []) def test_kind_detection(self): self.assertEqual(self.book.add("alice", self.name.hex())["kind"], "name") # a record B holds diff --git a/tests/test_fetcher.py b/tests/test_fetcher.py index 5e1c005..c7fa933 100644 --- a/tests/test_fetcher.py +++ b/tests/test_fetcher.py @@ -24,6 +24,7 @@ from rnfs.fetcher import Fetcher from rnfs.network import protocol, server from rnfs.network.client import BUSY, FAILED, OK, STALLED, BlocksResult +from tests.support import failing_commits, state_execute, state_rows OWN = bytes(16) P1, P2, P3, P4 = (bytes([i]) * 16 for i in (1, 2, 3, 4)) @@ -130,6 +131,7 @@ def setUp(self): self.addCleanup(self.tmp.cleanup) self.config = Config(rnfs_dir=self.tmp.name, max_parallel_providers=3, slow="no") self.storage = Storage(os.path.join(self.tmp.name, "blocks"), os.path.join(self.tmp.name, "state"), 10**9) + self.addCleanup(self.storage.close) self.routing = RoutingTable(OWN, self.config) self.client = FakeClient() self.network: dict[Cid, bytes] = {} @@ -144,8 +146,10 @@ def make_lookup(key): lookup = FakeLookup(self.runs) self.lookups.append(lookup) return lookup - return Fetcher(self.storage, self.client, self.config, OWN, self.routing, make_lookup, slow, - os.path.join(self.tmp.name, "state"), busy_delay=0.01) + fetcher = Fetcher(self.storage, self.client, self.config, OWN, self.routing, make_lookup, slow, + os.path.join(self.tmp.name, "state"), busy_delay=0.01) + self.addCleanup(fetcher.close) + return fetcher def put(self, cid: Cid, block: bytes): """Builder callback storing into the network.""" @@ -526,6 +530,7 @@ def test_blocks_damaged_on_disk_are_fetched_again(self): f.write(damage) # As after a restart, whose scan records the damaged files' lengths self.storage = Storage(os.path.join(self.tmp.name, "blocks"), os.path.join(self.tmp.name, "state"), 10**9) + self.addCleanup(self.storage.close) self.fetcher = self.make_fetcher() self.runs.append([P1]) self.fetch_all(built.cid) @@ -877,19 +882,45 @@ def test_nested_fetch_marks_only_its_own_root_invalid(self): self.assertFalse(self.fetcher.is_invalid(own.cid)) def test_invalid_roots_are_capped_and_saved_in_batches(self): - writes = [] - real = fetcher_module.write_json + state = os.path.join(self.tmp.name, "state") with mock.patch.object(fetcher_module, "MAX_INVALID", 3), \ - mock.patch.object(fetcher_module, "write_json", lambda *a: (writes.append(a), real(*a))): + mock.patch.object(self.fetcher._db, "transaction", wraps=self.fetcher._db.transaction) as writes: roots = [Cid.for_block(RAW, b"%d" % i) for i in range(5)] for root in roots: self.fetcher.mark_invalid(root) - self.assertEqual(writes, []) + self.assertEqual(writes.call_count, 0) + self.assertEqual(state_rows(state, "SELECT cid FROM invalid"), []) self.fetcher.save() self.fetcher.save() - self.assertEqual(len(writes), 1) + self.assertEqual(writes.call_count, 1) self.assertEqual([self.fetcher.is_invalid(r) for r in roots], [False, False, True, True, True]) self.assertEqual([self.make_fetcher().is_invalid(r) for r in roots], [False, False, True, True, True]) + # The oldest goes when another is marked; only the change is written + self.fetcher.mark_invalid(roots[0]) + self.fetcher.save() + self.assertEqual([self.make_fetcher().is_invalid(r) for r in roots], [True, False, False, True, True]) + self.assertEqual(state_rows(state, "SELECT cid FROM invalid ORDER BY position"), + [(roots[3].binary,), (roots[4].binary,), (roots[0].binary,)]) + + def test_failed_invalid_save_is_tried_again(self): + root = Cid.for_block(RAW, b"bad") + self.fetcher.mark_invalid(root) + with failing_commits(): + with self.assertRaises(OSError): + self.fetcher.save() + self.fetcher.save() + self.assertTrue(self.make_fetcher().is_invalid(root)) + + def test_invalid_rows_are_skipped_and_deleted(self): + root = Cid.for_block(RAW, b"bad") + self.fetcher.mark_invalid(root) + self.fetcher.save() + state = os.path.join(self.tmp.name, "state") + state_execute(state, "INSERT INTO invalid (cid, position) VALUES (?, 7)", (str(root),)) + fetcher = self.make_fetcher() + self.assertTrue(fetcher.is_invalid(root)) + fetcher.save() + self.assertEqual(state_rows(state, "SELECT cid FROM invalid"), [(root.binary,)]) MAX_BATCH_BYTES = BATCH * 80 * 1024 # what one fake response can hold at most diff --git a/tests/test_gateway_formats.py b/tests/test_gateway_formats.py index 4ec6b30..6b3b54b 100644 --- a/tests/test_gateway_formats.py +++ b/tests/test_gateway_formats.py @@ -91,6 +91,7 @@ def test_archive(self): with open(exported, "rb") as f: self.assertEqual(f.read(), body) other = Storage(os.path.join(self.tmp.name, "o", "blocks"), os.path.join(self.tmp.name, "o", "state"), 10**9) + self.addCleanup(other.close) stats = archive.import_archive(io.BytesIO(body), other) self.assertEqual([str(cid) for cid in stats["roots"]], [self.root]) @@ -100,6 +101,7 @@ def test_archive_of_a_path_and_head(self): self.assertEqual(status, 200) self.assertEqual(headers["ETag"], f'"{target}.archive"') other = Storage(os.path.join(self.tmp.name, "o", "blocks"), os.path.join(self.tmp.name, "o", "state"), 10**9) + self.addCleanup(other.close) self.assertEqual([str(c) for c in archive.import_archive(io.BytesIO(body), other)["roots"]], [target]) # A HEAD never gets an archive, stored or not (L6) self.assertEqual(self.get(f"/rnfs/{self.root}/sub?format=archive", method="HEAD")[0], 409) diff --git a/tests/test_get_partial.py b/tests/test_get_partial.py index e0dc4b8..54f882f 100644 --- a/tests/test_get_partial.py +++ b/tests/test_get_partial.py @@ -20,6 +20,7 @@ def setUp(self): self.dir = tmp.name self.node = RnfsNode.offline(Config(rnfs_dir=os.path.join(self.dir, "node")), b"\x01" * 16, client=mock.Mock(), names_network=mock.Mock()) + self.addCleanup(self.node.stop) src = os.path.join(self.dir, "src") os.makedirs(os.path.join(src, "sub")) for i in range(5): diff --git a/tests/test_jobs.py b/tests/test_jobs.py index 5580aba..72b50dd 100644 --- a/tests/test_jobs.py +++ b/tests/test_jobs.py @@ -13,7 +13,7 @@ import RNS -from rnfs import cli +from rnfs import cli, statedb from rnfs.api import ApiServer from rnfs.config import Config from rnfs.constants import CHUNK_SIZE @@ -21,7 +21,7 @@ from rnfs.content.storage import USER from rnfs.features import jobs from rnfs.node import RnfsNode -from tests.support import real, wait_for +from tests.support import failing_commits, real, state_execute, state_rows, wait_for, write_legacy from tests.test_api import FakeNode from tests.test_names import FakeNetwork from tests.test_node import A_ID, B_ID, NetClient @@ -116,6 +116,11 @@ def finished(self, job_id: str, timeout: float = 10.0) -> dict: self.assertTrue(wait_for(lambda: self.jobs.get(job_id)["state"] in jobs.FINISHED, timeout), self.jobs.get(job_id)) return self.jobs.get(job_id) + def saved(self, name: str = "b") -> list[dict]: + """The job records node name's jobs table holds, oldest first.""" + rows = state_rows(os.path.join(self.tmp.name, name, "state"), "SELECT record FROM jobs ORDER BY position") + return [statedb.load(record) for (record,) in rows] + class JobsTest(JobsBase): """B fetching from A through jobs.""" @@ -192,18 +197,16 @@ def test_cancel_running_job_stops_its_fetch(self): self.assertEqual(self.b.storage._sessions, []) # its protection is released def test_submit_saves_once_whether_the_job_starts_or_waits(self): + """A submitted job is saved once: as it starts, or as queued when it can't start yet.""" self.b.config.max_jobs = 1 first, second = self.share(b"first"), self.share(b"second") self.net.gate.clear() - path = os.path.join(self.b.config.state_dir, jobs.JOBS_FILE) - from rnfs.content import storage - with mock.patch.object(storage, "write_json", wraps=storage.write_json) as write: + with mock.patch.object(self.jobs._db, "transaction", wraps=self.jobs._db.transaction) as write: started = self.jobs.submit("get", first, self.out("first"))["id"] self.assertEqual(write.call_count, 1) # saved as it started, not also before queued = self.jobs.submit("get", second, self.out("second"))["id"] self.assertEqual(write.call_count, 2) # nothing started, so saved as queued - with open(path, encoding="utf-8") as f: - saved = {job["id"]: job["state"] for job in json.load(f)["jobs"]} + saved = {job["id"]: job["state"] for job in self.saved()} self.assertEqual(saved, {started: "running", queued: "queued"}) self.net.gate.set() self.assertEqual(self.finished(started)["state"], "done") @@ -238,6 +241,7 @@ def test_finished_jobs_are_capped_and_cleared(self): self.assertEqual(self.jobs.clear(ids[4]), 1) self.assertEqual(self.jobs.clear(), 2) self.assertEqual(self.jobs.list(), []) + self.assertEqual(self.saved(), []) # forgotten jobs are deleted def test_recently_finished_jobs_outlive_the_cap(self): # A CLI following a fast job asks for it once it ends: a small cap mustn't have forgotten it already @@ -278,10 +282,10 @@ def test_resume_removes_what_an_interrupted_run_left(self): self.a.records.store(Cid.parse(cid).key, A_ID) self.b.stop() output = self.out("dir-out") - path = os.path.join(self.tmp.name, "b", "state", jobs.JOBS_FILE) - with open(path, "w", encoding="utf-8") as f: - json.dump({"jobs": [{"id": "0123abcd", "kind": "get", "target": cid, "output": output, - "state": "running", "created": 1.0}]}, f) + # As rnfs 0.1.4 left it (imported at the restart), a record without the fields it can do without + write_legacy(os.path.join(self.tmp.name, "b", "state"), "jobs.json", + {"jobs": [{"id": "0123abcd", "kind": "get", "target": cid, "output": output, + "state": "running", "created": 1.0}]}) partial = os.path.join(self.tmp.name, ".rnfs-job-0123abcd.partial") os.makedirs(os.path.join(partial, "sub")) with open(os.path.join(partial, "sub", "f.txt"), "wb") as f: @@ -367,7 +371,8 @@ def noisy(*args, **kwargs): self.assertEqual(warnings[-1], f"... and {5000 - log.MAX_WARNINGS} more warnings") self.assertEqual(job["result"]["warnings_total"], 5000) self.jobs.save() - self.assertLess(os.path.getsize(os.path.join(self.b.config.state_dir, jobs.JOBS_FILE)), 20_000) + (record,) = state_rows(self.b.config.state_dir, "SELECT record FROM jobs WHERE id = ?", (job["id"],))[0] + self.assertLess(len(record), 20_000) # Persistence and resume @@ -380,9 +385,7 @@ def test_resume_after_restart_keeps_fetched_blocks(self): self.assertTrue(wait_for(lambda: self.net.waiting > 0 and not self.net.gate.is_set())) self.assertTrue(wait_for(lambda: self.jobs.get(job["id"])["done_bytes"] == 2 * CHUNK_SIZE)) self.b.stop() # the running job is interrupted and saved as queued - with open(os.path.join(self.b.config.state_dir, jobs.JOBS_FILE), encoding="utf-8") as f: - saved = json.load(f)["jobs"] - self.assertEqual([(r["id"], r["state"]) for r in saved], [(job["id"], "queued")]) + self.assertEqual([(r["id"], r["state"]) for r in self.saved()], [(job["id"], "queued")]) fetched = {c for c in self.net.requested[:3]} self.net.gate.set() @@ -405,33 +408,73 @@ def test_resume_after_restart_keeps_fetched_blocks(self): self.assertEqual(f.read(), data) def test_finished_jobs_survive_restart(self): + """Finished jobs are loaded again after a restart, oldest first.""" cid = self.b.add_stream(io.BytesIO(b"kept"))["cid"] job_id = self.finished(self.jobs.submit("pin", cid)["id"])["id"] + other = self.finished(self.jobs.submit("pin", cid)["id"])["id"] self.b.stop() b2 = self.make_node("b", B_ID) job = b2.feature("jobs").get(job_id) self.assertEqual((job["state"], job["result"]), ("done", {"cid": cid})) + self.assertEqual([j["id"] for j in b2.feature("jobs").list()], [job_id, other]) def test_damaged_jobs_file_records_skipped(self): - path = os.path.join(self.tmp.name, "c", "state", jobs.JOBS_FILE) - os.makedirs(os.path.dirname(path)) + """jobs.json's malformed records are skipped when it is imported, and a repeated id keeps the first.""" cid = str(Cid.for_block(RAW, b"c")) good = {"id": "0123abcd", "kind": "pin", "target": cid, "state": "running", "created": 1.0} - with open(path, "w", encoding="utf-8") as f: - json.dump({"jobs": [good, {"id": "nothex!!", "kind": "pin", "target": cid, "state": "queued"}, - {**good, "id": "0123abce", "kind": "get", "output": "relative"}, "junk"]}, f) + write_legacy(os.path.join(self.tmp.name, "c", "state"), "jobs.json", + {"jobs": [good, {"id": "nothex!!", "kind": "pin", "target": cid, "state": "queued"}, + {**good, "id": "0123abce", "kind": "get", "output": "relative"}, "junk", + {**good, "state": "done"}]}) c = self.make_node("c", bytes([0xC] * 16)) self.assertEqual([(j["id"], j["state"]) for j in c.feature("jobs").list()], [("0123abcd", "queued")]) def test_jobs_file_of_the_wrong_shape_is_moved_aside(self): - path = os.path.join(self.tmp.name, "c", "state", jobs.JOBS_FILE) - os.makedirs(os.path.dirname(path)) - with open(path, "w", encoding="utf-8") as f: - json.dump({"jobs": {"not": "a list"}}, f) + """A jobs.json of the wrong shape is moved aside when imported, and the node starts without it.""" + path = write_legacy(os.path.join(self.tmp.name, "c", "state"), "jobs.json", {"jobs": {"not": "a list"}}) c = self.make_node("c", bytes([0xC] * 16)) self.assertEqual(c.feature("jobs").list(), []) self.assertTrue(os.path.exists(path + ".damaged")) + def test_damaged_rows_are_deleted(self): + """Rows whose record isn't a valid job (or another job's, or with no position) are skipped at load and + deleted; the others load in order.""" + cid = str(Cid.for_block(RAW, b"c")) + c = self.make_node("c", bytes([0xC] * 16)) + state = c.config.state_dir + c.stop() + good = {"id": "0123abcd", "kind": "pin", "target": cid, "state": "done", "created": 1.0} + for job_id, position, record in (("0123abce", 1, "not json"), ("0123abcf", 2, statedb.dump(good)), + ("0123abd0", 3, statedb.dump({**good, "id": "0123abd0", "kind": "x"})), + ("0123abd1", "x", statedb.dump({**good, "id": "0123abd1"})), + ("0123abcd", 9, statedb.dump(good)), + ("0123abd2", 5, statedb.dump({**good, "id": "0123abd2"}))): + state_execute(state, "INSERT INTO jobs (id, position, record) VALUES (?, ?, ?)", (job_id, position, record)) + c = self.make_node("c", bytes([0xC] * 16)) + self.assertEqual([j["id"] for j in c.feature("jobs").list()], ["0123abd2", "0123abcd"]) + self.assertEqual([r["id"] for r in self.saved("c")], ["0123abd2", "0123abcd"]) + + def test_a_save_writes_only_the_jobs_changed(self): + """A save rewrites only the jobs whose record changed since the last; new ones go after the others.""" + first = self.finished(self.jobs.submit("pin", self.b.add_stream(io.BytesIO(b"1"))["cid"])["id"])["id"] + self.assertTrue(wait_for(lambda: [r["state"] for r in self.saved()] == ["done"])) # its outcome saved + state_execute(self.b.config.state_dir, "UPDATE jobs SET record = ? WHERE id = ?", + (statedb.dump({**self.saved()[0], "error": "planted"}), first)) + second = self.finished(self.jobs.submit("pin", self.b.add_stream(io.BytesIO(b"2"))["cid"])["id"])["id"] + self.assertTrue(wait_for(lambda: [r["state"] for r in self.saved()] == ["done", "done"])) + self.assertEqual([(r["id"], r["error"]) for r in self.saved()], [(first, "planted"), (second, None)]) + + def test_failed_save_leaves_the_job_to_the_next(self): + """A job whose save failed (logged, not raised) is still saved by the next save.""" + self.b.config.max_jobs = 0 # stays queued + cid = self.b.add_stream(io.BytesIO(b"later"))["cid"] + with failing_commits(): + job_id = self.jobs.submit("pin", cid)["id"] + self.assertEqual(self.saved(), []) + self.assertTrue(self.jobs._dirty) + self.jobs.save() + self.assertEqual([(r["id"], r["state"]) for r in self.saved()], [(job_id, "queued")]) + def test_huge_numbers_in_a_record_are_ignored(self): cid = str(Cid.for_block(RAW, b"c")) record = {"id": "0123abcd", "kind": "pin", "target": cid, "state": "done", "created": 10**400, diff --git a/tests/test_lxmf.py b/tests/test_lxmf.py index e762170..f0e8427 100644 --- a/tests/test_lxmf.py +++ b/tests/test_lxmf.py @@ -13,6 +13,7 @@ import RNS +from rnfs import statedb from rnfs.config import Config from rnfs.content import storage as storage_module from rnfs.content.cid import Cid @@ -22,7 +23,7 @@ from rnfs.features.lxmf import FIELD_CUSTOM_DATA, FIELD_CUSTOM_TYPE, Incoming, Link, MessageLog from rnfs.node import RnfsNode -from tests.support import FakeClock +from tests.support import FakeClock, failing_commits, state_execute, state_rows, write_legacy from tests.test_names import FakeNetwork from tests.test_node import NetClient @@ -194,32 +195,60 @@ class MessageLogTest(unittest.TestCase): """The bounded, persistent message log behind the inbox and outbox.""" def setUp(self): - """A temporary state file.""" + """A temporary state directory.""" self.tmp = tempfile.TemporaryDirectory() self.addCleanup(self.tmp.cleanup) - self.path = os.path.join(self.tmp.name, "log.json") + self.state = self.tmp.name + + def open(self, limit: int = 10, box: str = lxmf.INBOX, **settings) -> MessageLog: + """A log of box in the state directory (closed before the directory is removed).""" + log = MessageLog(self.state, box, limit, **settings) + self.addCleanup(log.close) + return log + + def rows(self, box: str = lxmf.INBOX) -> dict: + """The saved records of box: id -> record text.""" + return dict(state_rows(self.state, "SELECT id, record FROM lxmf_messages WHERE box = ?", (box,))) def test_bounded_oldest_dropped(self): - log = MessageLog(self.path, 3) + """Past the limit the oldest record is dropped.""" + log = self.open(3) dropped = [log.add({"text": f"m{i}"})[1] for i in range(5)] self.assertEqual([e["id"] for e in log.entries()], [3, 4, 5]) self.assertEqual([[e["text"] for e in d] for d in dropped], [[], [], [], ["m0"], ["m1"]]) def test_persistence(self): - log = MessageLog(self.path, 10) + """Records are saved as UTF-8 JSON, one row each; ids are never reused; a lower limit keeps the newest.""" + log = self.open() log.add({"text": "héllo ✓"}) second, _ = log.add({"text": "two"}) log.remove(second) log.save() - with open(self.path, encoding="utf-8") as f: - self.assertEqual(json.load(f)["messages"][0]["text"], "héllo ✓") - again = MessageLog(self.path, 10) + self.assertEqual(self.rows(), {1: '{"text": "héllo ✓", "id": 1}'}) + again = self.open() self.assertEqual([e["text"] for e in again.entries()], ["héllo ✓"]) + self.assertTrue(again.clean) self.assertEqual(again.add({})[0], 3) # ids are never reused - self.assertEqual(MessageLog(self.path, 1).entries()[0]["id"], 1) # a lower limit keeps the newest + self.assertEqual(self.open(1).entries()[0]["id"], 1) # a lower limit keeps the newest + + def test_records_round_trip_exactly(self): + """Lone surrogates, non-BMP and other non-ASCII characters, key order and nested values come back as + they were saved.""" + log = self.open() + records = [{"title": "\udc80 lone", "text": "emoji \U0001F600 and \ud83d alone", "display_name": "名前 \udfff", + "links": [{"uri": "rnfs:x", "size": 3}], "auto": [{"state": "pinned", "cid": "c"}], "sent": 1.5}, + {"z": None, "a": [1, [2, {"b": "\x00\n"}]], "text": "\U0010FFFF"}] + for record in records: + log.add(dict(record)) + log.save() + loaded = self.open().entries() + self.assertEqual(loaded, [{**record, "id": n} for n, record in enumerate(records, 1)]) + self.assertEqual([list(e) for e in loaded], [list(record) + ["id"] for record in records]) + self.assertEqual([lxmf._json_bytes(e) for e in loaded], [lxmf._json_bytes(e) for e in log.entries()]) def test_update_find_remove_all(self): - log = MessageLog(self.path, 10) + """Records are updated, found by a field and removed (all of them for None).""" + log = self.open() first, _ = log.add({"hash": "aa"}) log.add({"hash": "bb"}) self.assertTrue(log.update(first, state="x")) @@ -229,49 +258,125 @@ def test_update_find_remove_all(self): self.assertEqual(len(log.remove(None)), 2) self.assertEqual(len(log), 0) + def test_a_save_writes_only_the_records_changed(self): + """A save writes the records added, changed or removed since the last, with the next id.""" + log = self.open() + for n in range(3): + log.add({"n": n}) + log.save() + state_execute(self.state, "UPDATE lxmf_messages SET record = '{\"n\": 0, \"id\": 1, \"mark\": 1}' " + "WHERE box = 'inbox' AND id = 1") + log.update(2, n=20) + log.remove(3) + log.add({"n": 4}) + log.save() + self.assertEqual(self.rows(), {1: '{"n": 0, "id": 1, "mark": 1}', 2: '{"n": 20, "id": 2}', + 4: '{"n": 4, "id": 4}'}) + self.assertEqual(state_rows(self.state, "SELECT key, value FROM meta WHERE store = 'inbox' ORDER BY key"), + [("clean", 1), ("next_id", 5)]) + log.save() # nothing changed: nothing written + self.assertEqual(state_rows(self.state, "SELECT value FROM meta WHERE store = 'inbox' AND key = 'next_id'"), + [(5,)]) + + def test_boxes_are_apart(self): + """The inbox and outbox share the table, each with its rows and next id.""" + inbox, outbox = self.open(), self.open(box=lxmf.OUTBOX) + inbox.add({"to": "in"}) + inbox.add({"to": "in"}) + outbox.add({"to": "out"}) + inbox.save() + outbox.save() + self.assertEqual([e["to"] for e in self.open(box=lxmf.OUTBOX).entries()], ["out"]) + self.assertEqual(self.open(box=lxmf.OUTBOX).add({})[0], 2) + self.assertEqual(len(self.open().entries()), 2) + def test_saves_never_land_out_of_order(self): - # L13: a save that took its snapshot first can't be written after a later one + """L13: a save that took its snapshot first can't be written after a later one; adding doesn't wait for + a slow save.""" import threading - log = MessageLog(self.path, 10) + log = self.open() log.add({"n": 1}) - write = lxmf._write_json + commit = statedb._Database._commit entered, gate = threading.Event(), threading.Event() - def slow(path, value): - """The first save stalls in its write until released.""" + def slow(database, conn): + """The first save stalls in its commit until released.""" if threading.current_thread().name == "first": entered.set() gate.wait(5) - write(path, value) + commit(database, conn) - with mock.patch.object(lxmf, "_write_json", side_effect=slow): + with mock.patch.object(statedb._Database, "_commit", slow): first = threading.Thread(target=log.save, name="first") first.start() entered.wait(5) - log.add({"n": 2}) + log.add({"n": 2}) # doesn't wait for the save threading.Timer(0.2, gate.set).start() log.save() # waits for the first save, then writes the newer records first.join(5) - with open(self.path, encoding="utf-8") as f: - self.assertEqual([e["n"] for e in json.load(f)["messages"]], [1, 2]) + self.assertEqual([e["n"] for e in self.open().entries()], [1, 2]) - def test_writes_are_synced(self): - # L18: the file is fsynced before it replaces the old one (and its directory after, where possible) - log = MessageLog(self.path, 10) + def test_a_failed_save_is_tried_again(self): + """L18: a save is one transaction: if it fails nothing of it is written (the records saved before stay + whole), and the next save writes what it would have.""" + log = self.open() log.add({"n": 1}) - with mock.patch.object(lxmf.os, "fsync", wraps=os.fsync) as fsync: - log.save() - self.assertGreaterEqual(fsync.call_count, 1 if os.name == "nt" else 2) + log.save() + log.update(1, n=10) + log.add({"n": 2}) + with failing_commits(): + with self.assertRaises(OSError): + log.save() + self.assertEqual(self.rows(), {1: '{"n": 1, "id": 1}'}) + self.assertEqual([e["n"] for e in log.entries()], [10, 2]) # memory keeps them + log.save() + self.assertEqual([e["n"] for e in self.open().entries()], [10, 2]) - def test_damaged_and_malformed(self): - with open(self.path, "w", encoding="utf-8") as f: - f.write("{not json") - self.assertEqual(MessageLog(self.path, 10).entries(), []) - with open(self.path, "w", encoding="utf-8") as f: - json.dump({"next_id": "x", "messages": [{"id": 4}, {"id": "5"}, {"id": 0}, "junk", {"id": True}]}, f) - log = MessageLog(self.path, 10) - self.assertEqual([e["id"] for e in log.entries()], [4]) - self.assertEqual(log.add({})[0], 5) + def test_malformed_rows_are_dropped_and_deleted(self): + """Rows that aren't JSON, a dict with a positive integer id (up to MAX_ID) equal to the row's, or that + valid() refuses are dropped and deleted; the box is then not clean until the next save.""" + log = self.open() + log.add({"n": 1}) + log.save() + for row_id, text in ((5, '{"id": "5"}'), (6, '{"id": 7}'), (8, "{not json"), (9, '"junk"'), + (10, '{"id": true}'), (11, '{"id": 11, "bad": 1}'), ("x12", '{"id": 12}')): + state_execute(self.state, "INSERT INTO lxmf_messages (box, id, record) VALUES ('inbox', ?, ?)", + (row_id, text)) + state_execute(self.state, "INSERT INTO lxmf_messages (box, id, record) VALUES ('inbox', ?, ?)", + (lxmf.MAX_ID + 1, '{"id": %d}' % (lxmf.MAX_ID + 1))) + state_execute(self.state, "UPDATE meta SET value = 'x' WHERE store = 'inbox' AND key = 'next_id'") + log = self.open(valid=lambda entry: "bad" not in entry) + self.assertEqual([e["id"] for e in log.entries()], [1]) + self.assertFalse(log.clean) + self.assertEqual(list(self.rows()), [1]) + self.assertEqual(log.add({})[0], 2) # a next id that isn't an integer is ignored + again = self.open() + self.assertFalse(again.clean) # not saved since + log.save() + self.assertTrue(self.open().clean) + + def test_unsaved_state_is_not_clean(self): + """A box never saved, or whose clean value is missing, isn't clean.""" + self.assertFalse(self.open().clean) + log = self.open() + log.add({"n": 1}) + log.save() + self.assertTrue(self.open().clean) + state_execute(self.state, "DELETE FROM meta WHERE store = 'inbox' AND key = 'clean'") + self.assertFalse(self.open().clean) + self.assertEqual(len(self.open().entries()), 1) + + def test_bounds_at_load_delete_rows(self): + """Rows past the limit or the byte bound at load are deleted; that doesn't make the box unclean.""" + log = self.open() + for n in range(5): + log.add({"text": "x" * 100}) + log.save() + bounded = self.open(3, max_bytes=250) + self.assertEqual([e["id"] for e in bounded.entries()], [4, 5]) + self.assertTrue(bounded.clean) + self.assertEqual(sorted(self.rows()), [4, 5]) + self.assertEqual(self.open().add({})[0], 6) class FakeLxmf: @@ -354,12 +459,21 @@ def make_node(self, name: str, node_id: bytes, **settings) -> RnfsNode: def feature_node(self, network=None, **settings): """Node B with the lxmf feature attached to a stand-in network (jobs left out unless a test adds it).""" self.b = self.make_node("b", B_ID, **settings) - self.b.features.pop("jobs", None) + jobs = self.b.features.pop("jobs", None) + if jobs is not None: + jobs.close() # the node no longer closes it at stop self.feature = self.b.feature("lxmf") self.network = network or FakeLxmf() self.feature.attach(self.network) return self.feature + def reloaded(self) -> lxmf.LxmfFeature: + """B's lxmf feature built again from the saved state, as at a restart (closed before the directory is + removed).""" + again = lxmf.LxmfFeature(self.b) + self.addCleanup(again.close) + return again + def link_message(self, link=None, number=1, source=SENDER, text=b"have a look"): """A message carrying link in its field and its text.""" link = link or self.link @@ -429,7 +543,7 @@ def test_inbox_is_bounded_and_persists(self): feature.received(self.link_message(number=number)) self.assertEqual([e["id"] for e in feature.inbox_list()], [3, 4, 5]) feature.save() - again = lxmf.LxmfFeature(self.b) + again = self.reloaded() self.assertEqual([e["id"] for e in again.inbox_list()], [3, 4, 5]) def test_inbox_rm(self): @@ -720,6 +834,7 @@ def test_contact_nick(self): contacts = mock.Mock() contacts.lookup.side_effect = lambda nick, kind=None: SENDER if nick == "alice" and kind in ("lxmf", None) \ else (OTHER if nick == "bob" else None) + self.addCleanup(self.b.features["contacts"].close) # the node closes the stand-in instead self.b.features["contacts"] = contacts feature.send("alice", str(self.cid)) self.assertEqual(self.network.sent[0][0], SENDER) @@ -737,8 +852,10 @@ def test_unfinished_messages_fail_after_restart(self): feature = self.feature_node() feature.send(SENDER.hex(), str(self.cid)) feature.save() - again = lxmf.LxmfFeature(self.b) + again = self.reloaded() self.assertEqual(again.outbox_list()[0]["state"], "failed") + again.save() + self.assertEqual(self.reloaded().outbox_list()[0]["state"], "failed") class DisabledTest(FeatureTestCase): @@ -945,8 +1062,7 @@ def test_inbox_bytes_are_bounded_and_saved_as_utf8(self): self.assertLessEqual(sum(sizes), 20_000) self.assertEqual(feature.inbox_list()[-1]["id"], 20) feature.save() - with open(feature.inbox.path, encoding="utf-8") as f: - raw = f.read() + raw = "".join(text for text, in state_rows(self.b.config.state_dir, "SELECT record FROM lxmf_messages")) self.assertIn("é", raw) self.assertNotIn("\\u00e9", raw) @@ -974,23 +1090,40 @@ def test_inbox_get_prints_escaped(self): self.assertNotIn("\x1b", out.getvalue()) self.assertNotIn("\x07", out.getvalue()) + GOOD_IN = {"id": 2, "hash": "aa", "from": SENDER.hex(), "links": [], "auto": None, "text": "t"} + GOOD_OUT = {"id": 3, "hash": "bb", "to": SENDER.hex(), "state": "delivered", "method": "direct", "links": [], + "text": ""} + BAD = {"id": 1, "links": "x", "auto": 5, "from": 3, "state": []} + def test_malformed_records_are_dropped_at_load(self): - # BG-L8 (PoC poc_state_shapes.py) + """BG-L8 (PoC poc_state_shapes.py): records of the wrong shape in the files of rnfs 0.1.4 are dropped + when they are imported.""" + state = Config(rnfs_dir=os.path.join(self.tmp.name, "b")).state_dir + for name, good, wrong in (("lxmf_inbox.json", self.GOOD_IN, {"auto": [1]}), + ("lxmf_outbox.json", self.GOOD_OUT, {"state": ["x"]})): + write_legacy(state, name, {"messages": [self.BAD, {**good, **wrong, "id": 7}, good, good], "next_id": "x"}) self.b = self.make_node("b", B_ID) - state = self.b.config.state_dir - good_in = {"id": 2, "hash": "aa", "from": SENDER.hex(), "links": [], "auto": None, "text": "t"} - good_out = {"id": 3, "hash": "bb", "to": SENDER.hex(), "state": "delivered", "method": "direct", - "links": [], "text": ""} - bad = {"id": 1, "links": "x", "auto": 5, "from": 3, "state": []} - for name, good, wrong in (("lxmf_inbox.json", good_in, {"auto": [1]}), - ("lxmf_outbox.json", good_out, {"state": ["x"]})): - with open(os.path.join(state, name), "w", encoding="utf-8") as f: - json.dump({"messages": [bad, {**good, **wrong, "id": 7}, good, good], "next_id": "x"}, f) - feature = lxmf.LxmfFeature(self.b) + feature = self.b.feature("lxmf") self.assertEqual([e["id"] for e in feature.inbox_list()], [2]) self.assertEqual([e["id"] for e in feature.outbox_list()], [3]) + self.assertFalse(feature.inbox.clean) self.assertEqual(feature.inbox_rm(None), 1) + def test_malformed_rows_are_dropped_at_load(self): + """BG-L8 for the database: rows of the wrong shape are dropped and deleted.""" + self.b = self.make_node("b", B_ID) + state = self.b.config.state_dir + for box, good, wrong in ((lxmf.INBOX, self.GOOD_IN, {"auto": [1]}), + (lxmf.OUTBOX, self.GOOD_OUT, {"state": ["x"]})): + for record in (self.BAD, {**good, **wrong, "id": 7}, good): + state_execute(state, "INSERT INTO lxmf_messages (box, id, record) VALUES (?, ?, ?)", + (box, record["id"], json.dumps(record))) + feature = self.reloaded() + self.assertEqual([e["id"] for e in feature.inbox_list()], [2]) + self.assertEqual([e["id"] for e in feature.outbox_list()], [3]) + self.assertEqual(state_rows(state, "SELECT box, id FROM lxmf_messages ORDER BY box, id"), + [(lxmf.INBOX, 2), (lxmf.OUTBOX, 3)]) + def test_privacy_defaults(self): feature = self.feature_node(lxmf_send_from_hint=False) feature.send(SENDER.hex(), f"{self.cid}/file.txt") @@ -1006,11 +1139,11 @@ class RestartAndCancelTest(FeatureTestCase): def restarted(self) -> lxmf.LxmfFeature: """B's lxmf feature saved, then loaded again and started on a stand-in network (no threads run).""" self.feature.save() - again = lxmf.LxmfFeature(self.b) + again = self.reloaded() with mock.patch.object(lxmf, "LxmfNetwork", return_value=FakeLxmf()), \ mock.patch.object(again, "_run"), mock.patch.object(again, "_run_auto_fetch"): again.start() - self.addCleanup(again.stop) + self.addCleanup(again.stop) # before it is closed return again def test_queued_auto_fetches_survive_a_restart(self): @@ -1130,13 +1263,18 @@ class MessageLogBoundsTest(unittest.TestCase): """MessageLog's protection of pin holders and its byte bound.""" def setUp(self): - """A temporary state file.""" + """A temporary state directory.""" self.tmp = tempfile.TemporaryDirectory() self.addCleanup(self.tmp.cleanup) - self.path = os.path.join(self.tmp.name, "log.json") + + def open(self, limit: int, **settings) -> MessageLog: + """An inbox log in the state directory (closed before the directory is removed).""" + log = MessageLog(self.tmp.name, lxmf.INBOX, limit, **settings) + self.addCleanup(log.close) + return log def test_protected_entries_stay(self): - log = MessageLog(self.path, 3, protect=lambda e: e.get("keep", False)) + log = self.open(3, protect=lambda e: e.get("keep", False)) log.add({"keep": True}) for i in range(5): log.add({"text": f"m{i}"}) @@ -1151,7 +1289,7 @@ def test_protected_entries_stay(self): self.assertEqual((entry_id, [e["id"] for e in dropped]), (10, [1])) def test_per_from_and_bytes(self): - log = MessageLog(self.path, 100, max_bytes=1000) + log = self.open(100, max_bytes=1000) for i in range(5): log.add({"from": "a", "text": str(i)}, per_from=2) self.assertEqual([e["text"] for e in log.entries()], ["3", "4"]) diff --git a/tests/test_migrate_dht.py b/tests/test_migrate_dht.py new file mode 100644 index 0000000..a3a232e --- /dev/null +++ b/tests/test_migrate_dht.py @@ -0,0 +1,339 @@ +"""Migration of the DHT state files of rnfs 0.1.3 and 0.1.4 into the state database: providers.json (provider +records) and published.json (the publisher's last rounds, earlier rounds' nodes and pending withdrawals). + +For each: a valid file imports exactly (0.1.3's providers.json too, written before the newcomer pool), a +damaged one is moved aside, an import interrupted after its commit isn't repeated, a second start changes +nothing, a .migrated file left behind is ignored, and a failed commit imports nothing.""" +import json +import os +import tempfile +import unittest +from unittest import mock + +import RNS + +from rnfs import statedb +from rnfs.config import Config +from rnfs.constants import DAY, ID_LENGTH +from rnfs.content.cid import RAW, Cid +from rnfs.dht.providers import ProviderStore +from rnfs.publisher import DhtKey, Publisher +from tests.support import POSIX_MODES, FakeClock, failing_commits, state_rows, write_legacy +from tests.test_providers import held, saved + +T = 1_700_000_000.0 +P1, P2, P3, P4, P5 = (bytes([n]) * ID_LENGTH for n in range(1, 6)) +K1, K2, K3 = (bytes([0xA0 + n]) * ID_LENGTH for n in range(1, 4)) +N1, N2, N3 = (bytes([0xB0 + n]) * ID_LENGTH for n in range(1, 4)) +A, B = (Cid.for_block(RAW, bytes([n])) for n in range(2)) +NO_ROWS = {"publishers": {}, "records": {}, "departed": {}, "evicted": {}} + +_saved_loglevel = RNS.loglevel + + +def setUpModule(): + """Silence the log while these tests run.""" + RNS.loglevel = RNS.LOG_CRITICAL + + +def tearDownModule(): + """Restore RNS's loglevel.""" + RNS.loglevel = _saved_loglevel + + +class MigrateTest(unittest.TestCase): + """A state directory under a temporary directory, with the state files written into it.""" + + NAME = "" # the state file + + def setUp(self): + """An empty state directory.""" + self.tmp = tempfile.TemporaryDirectory() + self.addCleanup(self.tmp.cleanup) + self.state = os.path.join(self.tmp.name, "state") + self.path = os.path.join(self.state, self.NAME) + + def migrated(self) -> list[tuple]: + """The migrated table's record count for the state file, if it lists it.""" + return state_rows(self.state, "SELECT records FROM migrated WHERE file = ?", (self.NAME,)) + + def interrupted(self, value, changed, load): + """Import the file holding value, as if the process died between the commit and the rename; write changed + into the file left behind and start again: load() must show value's state, and the file be renamed and + kept as it was left.""" + write_legacy(self.state, self.NAME, value) + with mock.patch.object(statedb, "_keep_original", return_value=None): + first = load() + self.assertTrue(os.path.exists(self.path)) + write_legacy(self.state, self.NAME, changed) + self.assertEqual(load(), first) + self.assertFalse(os.path.exists(self.path)) + with open(self.path + ".migrated", encoding="utf-8") as f: + self.assertEqual(json.load(f), changed) + return first + + def migrated_again(self, changed, load, expected): + """A .migrated left behind (and the database's state) survive another start unchanged.""" + write_legacy(self.state, self.NAME + ".migrated", changed) + self.assertEqual(load(), expected) + + +class ProvidersTest(MigrateTest): + """providers.json: {"version": 1, "publishers": {id: first seen}, "records": [[key, publisher, first stored, + last stored, pooled], ...], "departed": [[publisher, first seen, when], ...], "evicted": [...]}.""" + + NAME = "providers.json" + VALUE = {"version": 1, + "publishers": {P1.hex(): T - 3 * DAY, P2.hex(): T - 2 * DAY, P3.hex(): T}, + "records": [[K1.hex(), P1.hex(), T - 3 * DAY, T - 1, 0], [K2.hex(), P2.hex(), T - 2 * DAY, T - 2, 1], + [K1.hex(), P3.hex(), T, T, 0]], + "departed": [[P4.hex(), T - 5 * DAY, T - DAY]], + "evicted": [[P5.hex(), T - 60, T - 30]]} + ROWS = {"publishers": {P1: T - 3 * DAY, P2: T - 2 * DAY, P3: T}, + "records": {(K1, P1): (T - 3 * DAY, T - 1, 0), (K2, P2): (T - 2 * DAY, T - 2, 1), (K1, P3): (T, T, 0)}, + "departed": {P4: (T - 5 * DAY, T - DAY)}, + "evicted": {P5: (T - 60, T - 30)}} + + def store(self, **settings) -> ProviderStore: + """A store on the state directory at T + 10 (closed before the directory is removed).""" + store = ProviderStore(100, self.state, clock=FakeClock(T + 10), seniority=DAY, **settings) + self.addCleanup(store.close) + return store + + def load(self) -> dict: + """What a store on the state directory loads.""" + return held(self.store()) + + def test_valid_file_imports_exactly(self): + """Every publisher, record (pooled or not), departure and eviction is imported as written and loaded; + the file is kept as providers.json.migrated, and a second start changes nothing.""" + write_legacy(self.state, self.NAME, self.VALUE) + self.assertEqual(self.load(), self.ROWS) + self.assertEqual(saved(self.state), self.ROWS) + self.assertEqual(self.migrated(), [(3,)]) + self.assertFalse(os.path.exists(self.path)) + self.assertTrue(os.path.exists(self.path + ".migrated")) + self.assertEqual(self.load(), self.ROWS) + self.assertEqual(saved(self.state), self.ROWS) + self.assertFalse(os.path.exists(self.path + ".migrated.1")) + + def test_file_from_before_the_newcomer_pool_imports(self): + """rnfs 0.1.3's file, four fields per record and no departures or evictions, imports every record outside + the pool.""" + value = {"version": 1, "publishers": self.VALUE["publishers"], + "records": [record[:4] for record in self.VALUE["records"]]} + write_legacy(self.state, self.NAME, value) + expected = {"publishers": self.ROWS["publishers"], + "records": {pair: (first, last, 0) for pair, (first, last, _) in self.ROWS["records"].items()}, + "departed": {}, "evicted": {}} + self.assertEqual(self.load(), expected) + self.assertEqual(saved(self.state), expected) + + def test_what_any_load_drops_is_left_out(self): + """NaN times, IDs of the wrong length and a repeated record (the first stored is kept) aren't imported; + expired records and those of a publisher without a first-seen time are, and the load drops them (the + next save deletes them). Numbers written as text convert as rnfs 0.1.4 converted them.""" + nan = float("nan") + value = {"version": 1, + "publishers": {P1.hex(): "1699990000", P2.hex(): nan, "abcd": T}, + "records": [[K1.hex(), P1.hex(), T, T, 0], [K1.hex(), P1.hex(), T - 1, T - 1, 1], + [K2.hex(), P1.hex(), nan, T, 0], ["abcd", P1.hex(), T, T, 0], + [K3.hex(), P1.hex(), T - 30 * DAY, T - 30 * DAY, 0], # expired + [K3.hex(), P3.hex(), T, T, 0]], # P3 has no first-seen time + "departed": [[P4.hex(), nan, T], [P5.hex(), T, T], [P5.hex(), T - 9 * DAY, T - 1]]} + write_legacy(self.state, self.NAME, value) + rows = {"publishers": {P1: 1699990000.0}, + "records": {(K1, P1): (T - 1, T - 1, 1), (K3, P1): (T - 30 * DAY, T - 30 * DAY, 0), + (K3, P3): (T, T, 0)}, + "departed": {P5: (T, T)}, "evicted": {}} # the one that went last + store = self.store() + self.assertEqual(saved(self.state), rows) + self.assertEqual(held(store), {"publishers": {P1: 1699990000.0}, "records": {(K1, P1): (T - 1, T - 1, 1)}, + "departed": {}, "evicted": {}}) # P5 departed younger than seniority + store.save() + self.assertEqual(saved(self.state), held(store)) + + def test_damaged_file_is_moved_aside(self): + """A file that isn't JSON, is of another version or holds a malformed record (rnfs 0.1.4 started empty + on each) is moved to .damaged, and the store starts empty.""" + for text in ("{not json", "", "[]", json.dumps(dict(self.VALUE, version=2)), + json.dumps(dict(self.VALUE, records=[["zz", P1.hex(), T, T]])), + json.dumps(dict(self.VALUE, records=[[K1.hex(), P1.hex(), T]])), + json.dumps(dict(self.VALUE, departed=[[P4.hex(), "soon", T]])), + json.dumps({"version": 1, "records": []})): + with self.subTest(text=text): + write_legacy(self.state, self.NAME, text=text) + self.assertEqual(self.load(), NO_ROWS) + self.assertFalse(os.path.exists(self.path)) + self.assertTrue(os.path.exists(self.path + ".damaged.7")) + self.assertEqual(saved(self.state), NO_ROWS) + self.assertEqual(self.migrated(), []) + + def test_directory_is_moved_aside(self): + """A directory where the file should be is moved aside too.""" + os.makedirs(self.path) + self.assertEqual(self.load(), NO_ROWS) + self.assertTrue(os.path.isdir(self.path + ".damaged")) + + @unittest.skipUnless(POSIX_MODES and hasattr(os, "getuid") and os.getuid() != 0, "needs permission checks") + def test_unreadable_file_is_left_for_the_next_start(self): + """A file that can't be read is logged and left (the store starts empty: records are republished), and + imported once it can be read.""" + write_legacy(self.state, self.NAME, self.VALUE) + os.chmod(self.path, 0) + try: + self.assertEqual(self.load(), NO_ROWS) + self.assertTrue(os.path.exists(self.path)) + finally: + os.chmod(self.path, 0o600) + self.assertEqual(self.load(), self.ROWS) + + def test_interrupted_migration_isnt_repeated(self): + """A crash between the commit and the rename: the next start only renames the file.""" + changed = dict(self.VALUE, records=[]) + state = self.interrupted(self.VALUE, changed, self.load) + self.assertEqual(state, self.ROWS) + self.assertEqual(saved(self.state), self.ROWS) + self.migrated_again(changed, self.load, self.ROWS) + + def test_failed_commit_imports_nothing(self): + """An import whose commit fails stops the start and leaves the file, imported at the next.""" + write_legacy(self.state, self.NAME, self.VALUE) + with failing_commits(): + with self.assertRaises(OSError): + self.store() + self.assertTrue(os.path.exists(self.path)) + self.assertEqual(self.load(), self.ROWS) + + def test_failed_save_after_the_import_is_written_next_time(self): + """Changes after the import whose save fails stay to be saved: the next save writes them.""" + write_legacy(self.state, self.NAME, self.VALUE) + store = self.store() + self.assertTrue(store.store(K3, P1)) + self.assertTrue(store.withdraw(K1, P3)) + with failing_commits(): + with self.assertRaises(OSError): + store.save() + self.assertEqual(saved(self.state), self.ROWS) + store.save() + self.assertEqual(saved(self.state), held(store)) + self.assertIn((K3, P1), saved(self.state)["records"]) + + +class PublishedTest(MigrateTest): + """published.json: {"published": {item: [when, [node, ...]]}, "earlier": {item: {node: when}}, + "withdraw": {node: [key, ...]}}, items being CIDs or (feature keys) "key:" and the key in hex, as rnfs + 0.1.3 and 0.1.4 both wrote it.""" + + NAME = "published.json" + KEY = DhtKey(K3) + VALUE = {"published": {str(A): [T - 100, [N1.hex(), N2.hex()]], str(KEY): [T - 50, []]}, + "earlier": {str(A): {N3.hex(): T - 200}, str(B): {N1.hex(): T - 300, N2.hex(): T - 400}}, + "withdraw": {N1.hex(): [K1.hex(), K2.hex()], N3.hex(): []}} + STATE = ({A: (T - 100, [N1, N2]), KEY: (T - 50, [])}, + {A: {N3: T - 200}, B: {N1: T - 300, N2: T - 400}}, + {N1: {K1, K2}, N3: set()}) + + def publisher(self) -> Publisher: + """A publisher on the state directory at T (closed before the directory is removed).""" + publisher = Publisher(bytes(ID_LENGTH), Config(), None, None, None, None, state_dir=self.state, + clock=FakeClock(T)) + self.addCleanup(publisher.close) + return publisher + + def load(self) -> tuple: + """(published, earlier, withdrawals) as a publisher on the state directory loads them.""" + publisher = self.publisher() + return publisher._published, publisher._earlier, publisher._withdrawals + + def rows(self) -> tuple: + """The publisher's tables, sorted.""" + return tuple(sorted(state_rows(self.state, f"SELECT * FROM {table}")) + for table in ("published", "published_earlier", "withdrawals")) + + def test_valid_file_imports_exactly(self): + """Every last round (CIDs and feature keys), earlier node and withdrawal is imported as written and + loaded; the file is kept as published.json.migrated, and a second start changes nothing.""" + write_legacy(self.state, self.NAME, self.VALUE) + expected = (sorted([(str(A), T - 100, N1 + N2), (str(self.KEY), T - 50, b"")]), + sorted([(str(A), N3, T - 200), (str(B), N1, T - 300), (str(B), N2, T - 400)]), + sorted([(N1, K1), (N1, K2)])) + loaded = self.load() + self.assertEqual(loaded[:2], self.STATE[:2]) + self.assertEqual(loaded[2], {N1: {K1, K2}}) # an empty list queues nothing + self.assertEqual(self.rows(), expected) + self.assertEqual(self.migrated(), [(7,)]) + self.assertTrue(os.path.exists(self.path + ".migrated")) + self.assertEqual(self.load(), loaded) + self.assertEqual(self.rows(), expected) + + def test_entries_of_the_wrong_shape_are_skipped(self): + """Entries rnfs 0.1.4 skipped are skipped one by one, and so are those with node IDs, keys or feature keys + not ID_LENGTH bytes (which it accepted); a feature key in capitals is written as str() writes it.""" + value = {"published": {"not an item": [T, []], str(A): [T, "0101"], str(B): [float("inf"), []], + "key:abcd": [T, []], "key:" + K1.hex().upper(): [T - 1, [N1.hex()]], + str(DhtKey(K2)): [T - 2, ["01"]]}, + "earlier": {str(A): [], str(B): {N1.hex(): "now"}, str(self.KEY): {"zz": T}, + str(DhtKey(K2)): {N2.hex(): T - 3}}, + "withdraw": {N1.hex(): "ab", N2.hex(): ["ab"], "01": [K1.hex()], N3.hex(): [K3.hex()]}} + write_legacy(self.state, self.NAME, value) + self.assertEqual(self.load(), ({DhtKey(K1): (T - 1, [N1])}, {DhtKey(K2): {N2: T - 3}}, {N3: {K3}})) + self.assertEqual(self.rows()[0], [(str(DhtKey(K1)), T - 1, N1)]) + + def test_damaged_file_is_moved_aside(self): + """A file that isn't JSON or has the wrong shape is moved to .damaged and the publisher starts without + it.""" + for text in ("{not json", "", "[]", '{"published": []}', '{"earlier": {}, "withdraw": []}'): + with self.subTest(text=text): + write_legacy(self.state, self.NAME, text=text) + self.assertEqual(self.load(), ({}, {}, {})) + self.assertFalse(os.path.exists(self.path)) + self.assertTrue(os.path.exists(self.path + ".damaged.4")) + self.assertEqual(self.migrated(), []) + + @unittest.skipUnless(POSIX_MODES and hasattr(os, "getuid") and os.getuid() != 0, "needs permission checks") + def test_unreadable_file_stops_startup(self): + """A file that can't be read stops the start, as it did (read_json), and is imported once it can be.""" + write_legacy(self.state, self.NAME, self.VALUE) + os.chmod(self.path, 0) + try: + with self.assertRaises(PermissionError): + self.publisher() + finally: + os.chmod(self.path, 0o600) + self.assertEqual(self.load()[:2], self.STATE[:2]) + + def test_interrupted_migration_isnt_repeated(self): + """A crash between the commit and the rename: the next start only renames the file.""" + changed = {"published": {}, "earlier": {}, "withdraw": {N2.hex(): [K3.hex()]}} + state = self.interrupted(self.VALUE, changed, self.load) + self.assertEqual(state[:2], self.STATE[:2]) + self.migrated_again(changed, self.load, state) + + def test_failed_commit_imports_nothing(self): + """An import whose commit fails stops the start and leaves the file, imported at the next.""" + write_legacy(self.state, self.NAME, self.VALUE) + with failing_commits(): + with self.assertRaises(OSError): + self.publisher() + self.assertTrue(os.path.exists(self.path)) + self.assertEqual(self.load()[:2], self.STATE[:2]) + + def test_failed_save_after_the_import_is_written_next_time(self): + """Changes after the import whose save fails stay to be saved: the next save writes them.""" + write_legacy(self.state, self.NAME, self.VALUE) + publisher = self.publisher() + with publisher._lock: + publisher._withdrawals[N2] = {K3} + publisher._touch_node(N2) + del publisher._published[A] + publisher._touch_item(A) + with failing_commits(): + with self.assertRaises(OSError): + publisher.save() + publisher.save() + self.assertEqual(self.load(), ({self.KEY: (T - 50, [])}, self.STATE[1], {N1: {K1, K2}, N2: {K3}})) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_migrate_features.py b/tests/test_migrate_features.py new file mode 100644 index 0000000..2f43501 --- /dev/null +++ b/tests/test_migrate_features.py @@ -0,0 +1,425 @@ +"""Migration of the follow, keepers and LXMF state files of rnfs 0.1.3 and 0.1.4 into the state database: +follow.json, keepers.json, lxmf_inbox.json and lxmf_outbox.json (both versions wrote them alike). + +For each: a valid file imports exactly, a damaged one is handled as those versions handled it (follow and the +inbox then aren't clean, so the reconcile at start keeps their pins), an import interrupted after its commit +isn't repeated, a second start changes nothing, a .migrated file left behind is ignored, and an import whose +commit fails leaves the file to import at the next start.""" +import json +import os +import tempfile +import unittest +from unittest import mock + +import RNS + +from rnfs import statedb +from rnfs.config import Config +from rnfs.content.cid import RAW, Cid +from rnfs.features import lxmf +from rnfs.features.keepers import FOLLOW, OWN, USER, KeeperStore, sign_record +from rnfs.names import encode_record, name_for_identity +from rnfs.node import RnfsNode +from tests.support import failing_commits, state_execute, state_rows, write_legacy +from tests.test_names import FakeNetwork +from tests.test_node import NetClient + +A, B, C = (Cid.for_block(RAW, bytes([i])) for i in range(3)) +N1, N2, N3 = bytes(range(16)), bytes([0xEE] * 16), bytes([0x33] * 16) +SENDER = "51" * 16 + +_saved_loglevel = RNS.loglevel + + +def setUpModule(): + """Silence the log while these tests run.""" + RNS.loglevel = RNS.LOG_CRITICAL + + +def tearDownModule(): + """Restore RNS's loglevel.""" + RNS.loglevel = _saved_loglevel + + +def written(value) -> str: + """value as rnfs 0.1.4 wrote the LXMF logs: UTF-8 JSON, non-ASCII as is, a lone surrogate as its escape.""" + return json.dumps(value, ensure_ascii=False).encode("utf-8", "backslashreplace").decode("utf-8") + + +class MigrateTest(unittest.TestCase): + """An rnfs directory under a temporary directory, with the state files written into /state.""" + + FILE = "" + + def setUp(self): + """An empty rnfs directory.""" + self.tmp = tempfile.TemporaryDirectory() + self.addCleanup(self.tmp.cleanup) + self.config = Config(rnfs_dir=os.path.join(self.tmp.name, "node")) + self.state = self.config.state_dir + + def start(self): + """Open and close the state database, as every start does first: the files there are imported.""" + statedb.connect(self.state).close() + + def node(self) -> RnfsNode: + """An offline node over the directory (stopped before the directory is removed).""" + node = RnfsNode.offline(self.config, b"\x07" * 16, NetClient(), FakeNetwork()) + self.addCleanup(node.stop) + return node + + def rows(self) -> list: + """The store's rows and meta values, as the database holds them.""" + raise NotImplementedError + + def records(self) -> int: + """The records the migrated table says the file had.""" + return state_rows(self.state, "SELECT records FROM migrated WHERE file = ?", (self.FILE,))[0][0] + + +class Common: + """Tests every file passes (mixed into a MigrateTest giving FILE, VALUE, CHANGED and COUNT).""" + + def test_interrupted_migration_isnt_repeated(self): + """A crash between the commit and the rename: the next start only renames the file, whatever it holds + by then, and the rows stay as imported.""" + path = write_legacy(self.state, self.FILE, text=written(self.VALUE)) + with mock.patch.object(statedb, "_keep_original", return_value=None): + self.start() + self.assertTrue(os.path.exists(path)) + imported = self.rows() + write_legacy(self.state, self.FILE, self.CHANGED) + self.start() + self.assertEqual(self.rows(), imported) + self.assertFalse(os.path.exists(path)) + with open(path + statedb.MIGRATED_SUFFIX, encoding="utf-8") as f: + self.assertEqual(json.load(f), self.CHANGED) # the file as it was, kept + self.assertEqual(self.records(), self.COUNT) + + def test_second_start_and_a_left_migrated_file_change_nothing(self): + """Once imported, another start, or a .migrated file left behind, changes nothing.""" + write_legacy(self.state, self.FILE, text=written(self.VALUE)) + self.start() + imported = self.rows() + self.assertTrue(imported) + self.start() + write_legacy(self.state, self.FILE + statedb.MIGRATED_SUFFIX, self.CHANGED) + self.start() + self.assertEqual(self.rows(), imported) + self.assertEqual(state_rows(self.state, "SELECT count(*) FROM migrated"), [(1,)]) + + def test_failed_commit_leaves_the_file(self): + """An import whose commit fails stops the start, imports nothing and leaves the file, imported by the + next start.""" + self.start() # the database exists: only the import's commit fails + path = write_legacy(self.state, self.FILE, text=written(self.VALUE)) + with failing_commits(): + with self.assertRaises(OSError): + self.start() + self.assertTrue(os.path.exists(path)) + self.assertEqual(state_rows(self.state, "SELECT count(*) FROM migrated"), [(0,)]) + self.start() + self.assertFalse(os.path.exists(path)) + self.assertEqual(self.records(), self.COUNT) + + +class FollowTest(Common, MigrateTest): + """follow.json: {"version": 1, "names": {name: {"pinned": cid or null}}}.""" + + FILE = "follow.json" + VALUE = {"version": 1, "names": {N1.hex(): {"pinned": str(A)}, N2.hex(): {"pinned": None}}} + CHANGED = {"version": 1, "names": {N3.hex(): {"pinned": str(C)}}} + COUNT = 2 + + def rows(self) -> list: + """The follow table and its meta values.""" + return (state_rows(self.state, "SELECT name, pinned FROM follow ORDER BY name") + + state_rows(self.state, "SELECT key, value FROM meta WHERE store = 'follow'")) + + def clean(self): + """The follow state's clean value (None if missing).""" + rows = state_rows(self.state, "SELECT value FROM meta WHERE store = 'follow' AND key = 'clean'") + return rows[0][0] if rows else None + + def test_valid_file_imports_exactly(self): + """Every name and its pin is imported, and the state is clean; a node follows them.""" + write_legacy(self.state, self.FILE, self.VALUE) + node = self.node() + self.assertEqual({(e["name"], e["pinned"]) for e in node.feature("follow").following()}, + {(N1.hex(), str(A)), (N2.hex(), None)}) + self.assertEqual(self.rows(), [(N1, A.binary), (N2, None), ("clean", 1)]) + self.assertTrue(os.path.exists(os.path.join(self.state, "follow.json.migrated"))) + + def test_malformed_entries_are_skipped_and_not_clean(self): + """Entries that don't parse, or aren't names, are skipped as rnfs 0.1.4 skipped them; the state isn't + clean then (the reconcile at start keeps follow pins it doesn't list).""" + write_legacy(self.state, self.FILE, {"names": {N1.hex(): {"pinned": str(A)}, "zz": {"pinned": None}, + "abcd": {"pinned": None}, N2.hex(): {"pinned": "bad"}, + N3.hex(): "not a dict"}}) + self.start() + self.assertEqual(self.rows(), [(N1, A.binary), ("clean", 0)]) + self.assertEqual(self.records(), 1) + + def test_damaged_file_is_moved_aside_and_not_clean(self): + """A file that isn't JSON is moved to .damaged and nothing is imported: the clean value stays missing. + One holding no names is imported as not clean, as 0.1.4 loaded it.""" + write_legacy(self.state, self.FILE, text="{not json") + self.start() + self.assertTrue(os.path.exists(os.path.join(self.state, "follow.json.damaged"))) + self.assertIsNone(self.clean()) + for value in ([], {"names": []}): + with self.subTest(value=value): + write_legacy(self.state, self.FILE, value) + forget_import(self.state, self.FILE) + self.start() + self.assertEqual(self.rows(), [("clean", 0)]) + + def test_reconcile_after_the_upgrade(self): + """Upgrading with a damaged follow.json, the pin of a name it lost is kept and the name followed again; + with a whole one, a follow pin of a name it doesn't list is released.""" + source = "follow:" + N3.hex() + for follow, kept in (("{not json", True), (json.dumps(self.VALUE), False)): + with self.subTest(follow=follow): + self.config = Config(rnfs_dir=os.path.join(self.tmp.name, "node" + str(kept))) + self.state = self.config.state_dir + write_legacy(self.state, "pins.json", {"pins": {str(C): [source]}}) + write_legacy(self.state, self.FILE, text=follow) + node = self.node() + self.assertEqual(node.storage.pinned_by(source), {C} if kept else set()) + self.assertEqual(N3.hex() in [e["name"] for e in node.feature("follow").following()], kept) + + +class KeepersTest(Common, MigrateTest): + """keepers.json: {"version": 1, "names": {name: {"reasons": [...], "record": [field hex, ...] or null}}}.""" + + FILE = "keepers.json" + COUNT = 2 + + @classmethod + def setUpClass(cls): + """A name key and a record it signed.""" + cls.identity = RNS.Identity() + cls.name = name_for_identity(cls.identity) + cls.record = sign_record(cls.identity, cls.name, encode_record(7, A), 1_700_000_000) + cls.fields = [field.hex() for field in cls.record.wire()] + cls.VALUE = {"version": 1, "names": {cls.name.hex(): {"reasons": [FOLLOW, USER], "record": cls.fields}, + N1.hex(): {"reasons": [OWN], "record": None}}} + cls.CHANGED = {"version": 1, "names": {N2.hex(): {"reasons": [USER], "record": None}}} + + def rows(self) -> list: + """The keepers table.""" + return state_rows(self.state, "SELECT name, reasons, record FROM keepers ORDER BY name") + + def load(self) -> dict: + """Name -> (reasons, the record's sequence or None), as a store loads them.""" + store = KeeperStore(self.state, 10) + try: + return {name: (store.reasons(name), store.get(name).sequence if store.get(name) else None) + for name in store.names()} + finally: + store.close() + + def test_valid_file_imports_exactly(self): + """Every name, its reasons and its record are imported; the record verifies at load.""" + write_legacy(self.state, self.FILE, self.VALUE) + self.assertEqual(self.load(), {self.name: ({FOLLOW, USER}, 7), N1: ({OWN}, None)}) + self.assertEqual(sorted(self.rows()), sorted([(self.name, '["follow", "user"]', json.dumps(self.fields)), + (N1, '["own"]', None)])) + + def test_malformed_entries_are_skipped(self): + """Entries that don't parse, aren't names or have no known reason are skipped; a record that doesn't + verify is dropped and its name kept, as rnfs 0.1.4 loaded them.""" + forged = list(self.fields) + forged[4] = "00" * 64 # the signature + write_legacy(self.state, self.FILE, {"names": { + self.name.hex(): {"reasons": [USER, "nobody"], "record": forged}, "zz": {"reasons": [USER]}, + "abcd": {"reasons": [USER]}, N1.hex(): {"reasons": ["nobody"]}, N2.hex(): {"record": None}, + N3.hex(): {"reasons": [USER], "record": ["zz"]}, bytes([4] * 16).hex(): {"reasons": 5}}}) + self.assertEqual(self.load(), {self.name: ({USER}, None)}) + self.assertEqual(self.rows(), [(self.name, '["user"]', None)]) + self.assertEqual(self.records(), 1) + + def test_damaged_file_is_moved_aside(self): + """A file that isn't JSON is moved to .damaged and nothing is kept; any other value without names + imports nothing, as rnfs 0.1.4 loaded it.""" + write_legacy(self.state, self.FILE, text="{not json") + self.assertEqual(self.load(), {}) + self.assertTrue(os.path.exists(os.path.join(self.state, "keepers.json.damaged"))) + write_legacy(self.state, self.FILE, [1, 2]) + self.assertEqual(self.load(), {}) + self.assertEqual(self.records(), 0) + + +INBOX_RECORDS = [ + {"hash": "aa" * 32, "from": SENDER, "display_name": "Ünïcode \udc80 \U0001F600", "sent": 1.7e9, + "received": 1.7e9 + 1, "title": "tïtle \ud83d", "text": "rnfs:x ✓ \U0010FFFF", "method": "direct", + "links": [{"uri": f"rnfs:{A}", "size": 3}], "auto": [{"state": "pinned", "cid": str(A)}], "id": 3}, + {"hash": "bb" * 32, "from": SENDER, "display_name": None, "sent": None, "received": 1.7e9 + 2, "title": "", + "text": "", "method": "propagated", "links": [{"uri": f"rnfs:{B}"}], "auto": None, "id": 5}, +] +OUTBOX_RECORDS = [ + {"hash": "cc" * 32, "to": SENDER, "time": 1.7e9, "links": [{"uri": f"rnfs:{A}"}], "text": "héllo \udc80", + "method": "direct", "state": "delivered", "progress": 1.0, "id": 1}, + {"hash": "dd" * 32, "to": SENDER, "time": 1.7e9, "links": [{"uri": f"rnfs:{B}"}], "text": "", + "method": "direct", "state": "outbound", "progress": 0.0, "id": 2}, +] + + +class LogTests: + """Tests of a message log file, {"next_id": n, "messages": [record, ...]} oldest first (mixed into a + MigrateTest giving the box and its bounds).""" + + BOX = "" + LIMIT = 0 + MAX_BYTES = None + VALID = None + RECORDS: list = [] + + def rows(self) -> list: + """The box's rows and meta values.""" + return (state_rows(self.state, "SELECT id, record FROM lxmf_messages WHERE box = ? ORDER BY id", (self.BOX,)) + + state_rows(self.state, "SELECT key, value FROM meta WHERE store = ? ORDER BY key", (self.BOX,))) + + def load(self) -> tuple[list[dict], bool, int]: + """The records a log of the box loads, whether it is clean, and the next id.""" + log = lxmf.MessageLog(self.state, self.BOX, self.LIMIT, self.MAX_BYTES, valid=self.VALID) + try: + return log.entries(), log.clean, log._next_id + finally: + log.close() + + def test_valid_file_imports_exactly(self): + """Every record is imported exactly (lone surrogates and other non-ASCII text included), in order, + with the next id; the box is clean.""" + write_legacy(self.state, self.FILE, text=written({"next_id": 9, "messages": self.RECORDS})) + self.assertEqual(self.load(), (self.RECORDS, True, 9)) + self.assertEqual([list(e) for e in self.load()[0]], [list(e) for e in self.RECORDS]) # key order too + self.assertEqual(self.records(), len(self.RECORDS)) + + def test_malformed_records_are_dropped_and_not_clean(self): + """Records that aren't dicts with a new positive integer id, that the box's check refuses, or whose id + SQLite can't store are dropped (with a warning), as rnfs 0.1.4 dropped them; the box isn't clean.""" + good = self.RECORDS[0] + messages = [good, {**good, "id": "4"}, {**good, "id": 0}, {**good, "id": True}, "junk", {**good}, + {**good, "id": 2 ** 63}, {**good, "id": 2 ** 70}, {**good, "id": 8, "links": "x"}] + write_legacy(self.state, self.FILE, text=written({"next_id": "x", "messages": messages})) + with mock.patch.object(lxmf, "_log") as warned: + entries, clean, next_id = self.load() + self.assertEqual((entries, clean, next_id), ([good], False, good["id"] + 1)) + self.assertEqual(warned.call_args.args[0], f"dropped 8 malformed record(s) from {self.FILE}") + self.assertEqual(self.records(), 1) + + def test_exhausted_ids_refuse_new_records(self): + """A next id at SQLite's largest integer (only a hand-edited file holds one) is refused; one just below + it is the last id given out, and saves keep working: no id or next id past 64 bits is ever stored.""" + statedb.connect(self.state).close() + for saved, expected in ((2 ** 63 - 1, 1), (lxmf.MAX_ID, lxmf.MAX_ID)): + with self.subTest(saved=saved): + forget_import(self.state, self.FILE) + write_legacy(self.state, self.FILE, text=written({"next_id": saved, "messages": []})) + log = lxmf.MessageLog(self.state, self.BOX, self.LIMIT, self.MAX_BYTES, valid=self.VALID) + try: + self.assertEqual(log._next_id, expected) + log._next_id = lxmf.MAX_ID + 1 # as if every id up to MAX_ID were used + self.assertEqual(log.add(dict(self.RECORDS[0]))[0], None) + log.save() + finally: + log.close() + + def test_limits_apply(self): + """Past the box's limit (and the inbox's byte bound) the oldest records are dropped, as rnfs 0.1.4 did + at load; that leaves the box clean.""" + messages = [{**self.RECORDS[0], "id": n, "text": "x" * 1000} for n in range(1, 6)] + write_legacy(self.state, self.FILE, text=written({"next_id": 2, "messages": messages})) + with mock.patch.object(lxmf, self.LIMIT_NAME, 3): + self.start() + self.assertEqual([row_id for row_id, _ in self.rows()[:-2]], [3, 4, 5]) + self.assertEqual(self.rows()[-2:], [("clean", 1), ("next_id", 6)]) + + def test_damaged_file_is_moved_aside(self): + """A file that isn't JSON is moved to .damaged and nothing is imported (the box isn't clean: it was + never saved); one that isn't an object, or whose messages aren't a list, imports nothing and isn't + clean.""" + write_legacy(self.state, self.FILE, text="{not json") + self.assertEqual(self.load(), ([], False, 1)) + self.assertTrue(os.path.exists(os.path.join(self.state, self.FILE + ".damaged"))) + for value in ([self.RECORDS[0]], {"messages": {"1": self.RECORDS[0]}, "next_id": 4}): + with self.subTest(value=value): + forget_import(self.state, self.FILE) + write_legacy(self.state, self.FILE, value) + self.assertEqual(self.load()[:2], ([], False)) + + +class InboxTest(Common, LogTests, MigrateTest): + """lxmf_inbox.json.""" + + FILE = "lxmf_inbox.json" + BOX = lxmf.INBOX + LIMIT = lxmf.INBOX_LIMIT + LIMIT_NAME = "INBOX_LIMIT" + MAX_BYTES = lxmf.MAX_INBOX_BYTES + VALID = staticmethod(lxmf._valid_inbox_entry) + RECORDS = INBOX_RECORDS + VALUE = {"next_id": 6, "messages": INBOX_RECORDS} + CHANGED = {"next_id": 2, "messages": [{**INBOX_RECORDS[1], "id": 1}]} + COUNT = 2 + + def test_byte_bound_applies(self): + """Past MAX_INBOX_BYTES the oldest records are dropped, one kept whatever its size.""" + messages = [{**self.RECORDS[1], "id": n, "text": "é" * 1000} for n in range(1, 4)] + write_legacy(self.state, self.FILE, text=written({"messages": messages})) + with mock.patch.object(lxmf, "MAX_INBOX_BYTES", 5000): + self.start() + self.assertEqual([row_id for row_id, _ in self.rows()[:-2]], [2, 3]) + + def test_reconcile_after_the_upgrade(self): + """Upgrading with the inbox, a node's LXMF feature lists its messages and keeps the pins they record; + with a damaged inbox, a pin no message records is kept and recorded in a new message; with a whole one, + it is released.""" + source = "lxmf:" + SENDER + for inbox, kept in (("{not json", True), (written(self.VALUE), False)): + with self.subTest(inbox=inbox): + self.config = Config(rnfs_dir=os.path.join(self.tmp.name, "node" + str(kept))) + self.state = self.config.state_dir + write_legacy(self.state, "pins.json", {"pins": {str(A): [source], str(C): [source]}}) + write_legacy(self.state, self.FILE, text=inbox) + node = self.node() + messages = node.feature("lxmf").inbox_list() + self.assertEqual(node.storage.pinned_by(source), {A, C} if kept else {A}) + if kept: + self.assertEqual([m["auto"] for m in messages], [[{"state": "pinned", "cid": str(A)}, + {"state": "pinned", "cid": str(C)}]]) + else: + self.assertEqual(messages, self.RECORDS) + + +class OutboxTest(Common, LogTests, MigrateTest): + """lxmf_outbox.json.""" + + FILE = "lxmf_outbox.json" + BOX = lxmf.OUTBOX + LIMIT = lxmf.OUTBOX_LIMIT + LIMIT_NAME = "OUTBOX_LIMIT" + VALID = staticmethod(lxmf._valid_outbox_entry) + RECORDS = OUTBOX_RECORDS + VALUE = {"next_id": 3, "messages": OUTBOX_RECORDS} + CHANGED = {"next_id": 2, "messages": [OUTBOX_RECORDS[0]]} + COUNT = 2 + + def test_unfinished_messages_fail_at_the_first_start(self): + """A message still unfinished in the file is marked failed when the node starts, as at any restart.""" + write_legacy(self.state, self.FILE, text=written(self.VALUE)) + node = self.node() + self.assertEqual([e["state"] for e in node.feature("lxmf").outbox_list()], ["delivered", "failed"]) + + +def forget_import(state_dir: str, name: str): + """Forget that name was imported (and its .migrated copy), so another version of it is imported.""" + state_execute(state_dir, "DELETE FROM migrated WHERE file = ?", (name,)) + path = os.path.join(state_dir, name + statedb.MIGRATED_SUFFIX) + if os.path.exists(path): + os.remove(path) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_migrate_names.py b/tests/test_migrate_names.py new file mode 100644 index 0000000..65db84a --- /dev/null +++ b/tests/test_migrate_names.py @@ -0,0 +1,350 @@ +"""Migration of names.json and name-floor.json, the names state of rnfs 0.1.4 and earlier, into the state database +(names_published, names_held and name_floor). + +For each: a valid file imports exactly, a damaged one is handled as those versions handled it (a damaged names.json +stops startup and is left in place; a damaged name-floor.json is moved aside), an import interrupted after its +commit isn't repeated, a second start changes nothing, a .migrated file left behind is ignored, and the formats +older versions wrote load as they did. A failed commit changes nothing.""" +import json +import os +import tempfile +import unittest +from unittest import mock + +import RNS + +from rnfs import names, statedb +from rnfs.config import Config +from rnfs.content.cid import DAG, RAW, Cid +from rnfs.errors import RnfsError +from rnfs.names import NameManager, Record +from tests.support import POSIX_MODES, FakeClock, failing_commits, state_rows, write_legacy +from tests.test_names import FakeNetwork, FakePins + +A, B, C = Cid.for_block(RAW, b"a"), Cid.for_block(RAW, b"b"), Cid.for_block(DAG, b"c") +MAX = names.MAX_SEQUENCE +N = [bytes([i]) * 16 for i in range(16)] # names that aren't local +NOW = 1_000_000.0 + +_saved_loglevel = RNS.loglevel + + +def setUpModule(): + """Silence the log while these tests run.""" + RNS.loglevel = RNS.LOG_CRITICAL + + +def tearDownModule(): + """Restore RNS's loglevel.""" + RNS.loglevel = _saved_loglevel + + +class MigrateNamesTest(unittest.TestCase): + """An rnfs directory under a temporary directory, with the state files written into /state.""" + + def setUp(self): + """An empty rnfs directory, and fakes for the managers.""" + self.tmp = tempfile.TemporaryDirectory() + self.addCleanup(self.tmp.cleanup) + self.config = Config(rnfs_dir=os.path.join(self.tmp.name, "node"), path_timeout=0.05, name_table_size=3) + self.state = self.config.state_dir + self.clock = FakeClock(NOW) + self.pins = FakePins() + + def manager(self) -> NameManager: + """A manager over the directory (closed before the directory is removed).""" + manager = NameManager(self.config, self.pins, network=FakeNetwork(), clock=self.clock, monotonic=self.clock) + self.addCleanup(manager.close) + return manager + + def key(self, alias: str) -> bytes: + """Write a new name key file for alias, as `name create` did; its name.""" + identity = RNS.Identity() + os.makedirs(self.config.names_dir, exist_ok=True) + with open(os.path.join(self.config.names_dir, alias), "wb") as f: + f.write(identity.get_private_key()) + return names.name_for_identity(identity) + + def rows(self, sql: str, params=()) -> list[tuple]: + """The rows sql reads from the state database.""" + return state_rows(self.state, sql, params) + + +class FileTests: + """The tests every legacy file shares; a subclass gives FILE, COUNT (its records), value(), changed() (value + with a record changed) and snapshot() (a new manager's state and the rows).""" + + def test_interrupted_import_is_only_finished(self): + """An import whose rename never happened (the process died after the commit) isn't done again at the next + start, even if the file changed meanwhile: the file is only renamed.""" + path = write_legacy(self.state, self.FILE, self.value()) + with mock.patch.object(statedb, "_keep_original", return_value=None): + first = self.snapshot() + self.assertTrue(os.path.exists(path)) + write_legacy(self.state, self.FILE, self.changed()) + self.assertEqual(self.snapshot(), first) + self.assertFalse(os.path.exists(path)) + with open(path + statedb.MIGRATED_SUFFIX, encoding="utf-8") as f: + self.assertEqual(json.load(f), self.changed()) # the file as it was, kept + self.assertEqual(self.rows("SELECT records FROM migrated WHERE file = ?", (self.FILE,)), [(self.COUNT,)]) + + def test_later_starts_change_nothing(self): + """A second start imports nothing, nor does a .migrated copy left behind, whatever it holds.""" + write_legacy(self.state, self.FILE, self.value()) + first = self.snapshot() + self.assertEqual(self.snapshot(), first) + write_legacy(self.state, self.FILE + statedb.MIGRATED_SUFFIX, self.changed()) + self.assertEqual(self.snapshot(), first) + self.assertEqual(len(self.rows("SELECT * FROM migrated WHERE file = ?", (self.FILE,))), 1) + + def test_failed_commit_leaves_the_file(self): + """An import that can't be committed (a failing disk) stops startup and imports nothing; the next start + imports the file.""" + statedb.connect(self.state).close() # the database of an earlier start + path = write_legacy(self.state, self.FILE, self.value()) + with failing_commits(), self.assertRaises(OSError): + self.manager() + self.assertTrue(os.path.exists(path)) + self.assertEqual(self.rows("SELECT * FROM migrated"), []) + self.assertTrue(self.snapshot()[-1]) # rows imported + self.assertFalse(os.path.exists(path)) + self.assertEqual(self.rows("SELECT records FROM migrated WHERE file = ?", (self.FILE,)), [(self.COUNT,)]) + + +class NamesFileTest(FileTests, MigrateNamesTest): + """names.json: {"version": 1, "published": {alias: {sequence, cid, imported}}, "held": {name: {sequence, cid, + heard}}}, written with indent=1 by rnfs 0.1.0 to 0.1.4.""" + + FILE = names.STATE_FILE + COUNT = 5 + + def setUp(self): + """Key files for "site" and "moved"; "gone" has a record but no key file (deleted by hand).""" + super().setUp() + self.site = self.key("site") + self.moved = self.key("moved") + + def value(self) -> dict: + """names.json as rnfs 0.1.4 wrote it, with the highest sequence and a name not yet published.""" + return {"version": 1, + "published": {"gone": {"sequence": MAX, "cid": str(B), "imported": False}, + "moved": {"sequence": 0, "cid": None, "imported": True}, + "site": {"sequence": 5, "cid": str(A), "imported": False}}, + "held": {N[1].hex(): {"sequence": MAX, "cid": str(C), "heard": 99.5}, + self.site.hex(): {"sequence": 5, "cid": str(A), "heard": 1234.0}}} + + def changed(self) -> dict: + """value() with site published again.""" + value = self.value() + value["published"]["site"] = {"sequence": 6, "cid": str(B), "imported": False} + return value + + def snapshot(self) -> tuple: + """What a new manager loads (published, held, gone) and the rows.""" + manager = self.manager() + return ({alias: (p.sequence, p.cid, p.imported) for alias, p in manager._published.items()}, + dict(manager._held), list(manager._gone), + self.rows("SELECT * FROM names_published ORDER BY alias"), + self.rows("SELECT * FROM names_held ORDER BY name")) + + def test_valid_file_imports_exactly(self): + """Every published and held record is imported, field for field, whether or not its key file is there; the + file is kept as names.json.migrated.""" + path = write_legacy(self.state, self.FILE, text=json.dumps(self.value(), indent=1)) + manager = self.manager() + self.assertEqual(self.rows("SELECT alias, sequence, cid, imported FROM names_published ORDER BY alias"), + [("gone", str(MAX), B.binary, 0), ("moved", "0", None, 1), ("site", "5", A.binary, 0)]) + self.assertEqual(self.rows("SELECT name, sequence, cid, heard FROM names_held ORDER BY name"), + sorted([(N[1], str(MAX), C.binary, 99.5), (self.site, "5", A.binary, 1234.0)])) + self.assertEqual(manager.list(), [ + {"alias": "moved", "name": self.moved.hex(), "cid": None, "sequence": 0}, + {"alias": "site", "name": self.site.hex(), "cid": str(A), "sequence": 5}]) + self.assertEqual((manager._published["moved"].imported, manager._published["site"].imported), (True, False)) + self.assertEqual(manager.record(N[1]), Record(MAX, C, 99.5)) + self.assertEqual(manager._gone, [("gone", B)]) # its pin is released once the node is set up + self.assertEqual(self.rows("SELECT file, records FROM migrated"), [(self.FILE, self.COUNT)]) + self.assertFalse(os.path.exists(path)) + self.assertTrue(os.path.exists(path + statedb.MIGRATED_SUFFIX)) + manager.release_gone() + self.assertEqual([alias for (alias,) in self.rows("SELECT alias FROM names_published ORDER BY alias")], + ["moved", "site"]) + + def test_entries_without_imported_flag(self): + """An entry without "imported" (every version wrote it; one edited by hand) loads as not imported, and a key + without an entry as imported, as rnfs 0.1.4 loaded them.""" + write_legacy(self.state, self.FILE, {"published": {"site": {"sequence": 2, "cid": str(A)}}}) + manager = self.manager() + self.assertEqual(self.rows("SELECT alias, sequence, cid, imported FROM names_published"), + [("site", "2", A.binary, 0)]) + self.assertEqual((manager._published["site"], manager._published["moved"]), + (names._Published(2, A, False), names._Published(0, None, True))) + + def test_lenient_fields_load_as_before(self): + """Fields rnfs 0.1.4 converted (int() of the sequence, float() of the heard time, any truth value as + the flag) are converted the same way; an alias no key file can have is skipped.""" + write_legacy(self.state, self.FILE, { + "published": {"site": {"sequence": "7", "cid": None, "imported": 1}, "../x": {"sequence": 1, "cid": None}}, + "held": {f"<{N[2].hex().upper()}>": {"sequence": 3.0, "cid": str(A), "heard": "12.5"}, + N[3].hex(): {"sequence": 1, "cid": str(A), "heard": float("nan")}}}) + manager = self.manager() + self.assertEqual(manager._published["site"], names._Published(7, None, True)) + self.assertEqual(manager.record(N[2]), Record(3, A, 12.5)) + self.assertEqual(manager.record(N[3]), Record(1, A, 0.0)) # SQLite has no NaN: heard long ago + self.assertEqual(self.rows("SELECT alias FROM names_published"), [("site",)]) + + def test_damaged_file_stops_startup(self): + """A names.json that isn't JSON, or has a malformed record, stops startup and is left as it is (starting + without it would allow rollback); nothing is imported.""" + damaged = ["{not json", "", "null", "[1, 2]", '{"published": []}', '{"published": {"site": {"sequence": 1}}}', + '{"published": {"site": {"sequence": "one", "cid": null}}}', + '{"published": {"site": {"sequence": 1, "cid": "not a cid"}}}', + '{"held": {"zz": {"sequence": 1, "cid": "%s", "heard": 1}}}' % A, + '{"held": {"%s": {"sequence": 1, "cid": "%s"}}}' % (N[1].hex(), A), + '{"held": {"%s": {"sequence": 1e999, "cid": "%s", "heard": 1}}}' % (N[1].hex(), A)] + path = os.path.join(self.state, self.FILE) + for text in damaged: + with self.subTest(text=text): + write_legacy(self.state, self.FILE, text=text) + with self.assertRaises(RnfsError): + self.manager() + with open(path, encoding="utf-8") as f: + self.assertEqual(f.read(), text) + self.assertEqual(sorted(os.listdir(self.state)), sorted([self.FILE, *self.database_files()])) + os.remove(path) + os.mkdir(path) # can't be read at all + with self.assertRaisesRegex(RnfsError, "could not read"): + self.manager() + os.rmdir(path) + for table in ("names_published", "names_held", "migrated"): + self.assertEqual(self.rows(f"SELECT * FROM {table}"), []) + + def database_files(self) -> list[str]: + """The state database's files in the state directory.""" + return [entry for entry in os.listdir(self.state) if entry.startswith(statedb.DB_FILE)] + + def test_failed_publish_commit_rolls_back(self): + """After the import, a publish whose save can't be committed leaves memory, pins and rows as they were.""" + write_legacy(self.state, self.FILE, self.value()) + manager = self.manager() + rows = self.rows("SELECT * FROM names_published ORDER BY alias"), self.rows("SELECT * FROM names_held") + with failing_commits(), self.assertRaises(OSError): + manager.publish("site", B) + self.assertEqual(manager.list()[1]["sequence"], 5) + self.assertEqual(manager.record(self.site), Record(5, A, 1234.0)) + self.assertEqual(self.pins.sources, {}) + self.assertEqual((self.rows("SELECT * FROM names_published ORDER BY alias"), + self.rows("SELECT * FROM names_held")), rows) + self.assertEqual(manager.publish("site", B)["sequence"], 6) + + +class FloorFileTest(FileTests, MigrateNamesTest): + """name-floor.json: {"version": 1, "floor": [[name, sequence, resolved], ...]}, least recently resolved + first, written compactly by rnfs 0.1.4.""" + + FILE = names.FLOOR_FILE + COUNT = 3 + + def value(self) -> dict: + """name-floor.json as rnfs 0.1.4 wrote it, with the highest sequence.""" + return {"version": 1, "floor": [[N[1].hex(), 5, 100], [N[2].hex(), MAX, 200], [N[3].hex(), 0, 300]]} + + def changed(self) -> dict: + """value() with the first name's floor raised.""" + value = self.value() + value["floor"][0][1] = 6 + return value + + def snapshot(self) -> tuple: + """The floor a new manager loads, and the rows.""" + return list(self.manager()._floor.items()), self.rows("SELECT * FROM name_floor ORDER BY used") + + def floor(self) -> list[tuple]: + """The name_floor rows, least recently resolved first.""" + return self.rows("SELECT name, sequence, resolved, used FROM name_floor ORDER BY used") + + def test_valid_file_imports_exactly(self): + """Every entry is imported, in order, with its sequence and resolve time; the file is kept as + name-floor.json.migrated.""" + path = write_legacy(self.state, self.FILE, text=json.dumps(self.value(), separators=(",", ":"))) + manager = self.manager() + self.assertEqual(self.floor(), [(N[1], "5", 100.0, 0), (N[2], str(MAX), 200.0, 1), (N[3], "0", 300.0, 2)]) + self.assertEqual(list(manager._floor.items()), [(N[1], (5, 100.0)), (N[2], (MAX, 200.0)), (N[3], (0, 300.0))]) + self.assertEqual(self.rows("SELECT file, records FROM migrated"), [(self.FILE, self.COUNT)]) + self.assertTrue(os.path.exists(path + statedb.MIGRATED_SUFFIX)) + # Saved later as any other floor: the new entry after the imported ones + manager.offer(N[4], 1, A, NOW) + self.assertEqual(manager.resolve(N[4], persist=False)["sequence"], 1) + self.assertEqual([name for name, *_ in self.floor()], [N[1], N[2], N[3], N[4]]) + + def test_entries_checked_as_before(self): + """Entries that don't parse are skipped, as rnfs 0.1.4 skipped them; a name listed twice keeps the higher + sequence at its later place; an entry without one valid resolve time counts as resolved long ago.""" + write_legacy(self.state, self.FILE, text=json.dumps({"version": 1, "floor": [ + [N[1].hex(), 5, 100], [N[2].hex(), 1, 150], [N[1].hex(), 3, 400], + ["zz", 1, 1], [None, 1, 1], "a string entry", [N[4].hex()], 7, + [N[5].hex(), -1, 1], [N[5].hex(), MAX + 1, 1], [N[5].hex(), True, 1], [N[5].hex(), 1.5, 1], + [N[6].hex(), 2, 1, 2], [N[7].hex(), 3, "noon"], [N[8].hex(), 4, 10**400], [N[9].hex(), 5, float("nan")], + [f"<{N[10].hex().upper()}>", 6, 7]]})) + with mock.patch.object(names, "log") as log: + manager = self.manager() + expected = [(N[2], (1, 150.0)), (N[1], (5, 400.0)), (N[6], (2, 0.0)), (N[7], (3, 0.0)), (N[8], (4, 0.0)), + (N[9], (5, 0.0)), (N[10], (6, 7.0))] + self.assertEqual(list(manager._floor.items()), expected) + self.assertEqual(self.floor(), [(name, str(sequence), at, used) + for used, (name, (sequence, at)) in enumerate(expected)]) + self.assertTrue(any("skipped 9 malformed entries" in call.args[0] for call in log.call_args_list)) + + def test_entries_without_resolve_time(self): + """[name, sequence] entries, as rnfs wrote before 0.1.4 kept resolve times, count as resolved long ago.""" + write_legacy(self.state, self.FILE, {"version": 1, "floor": [[N[1].hex(), 5], [N[2].hex(), 6]]}) + manager = self.manager() + self.assertEqual(list(manager._floor.items()), [(N[1], (5, 0.0)), (N[2], (6, 0.0))]) + self.assertEqual(self.floor(), [(N[1], "5", 0.0, 0), (N[2], "6", 0.0, 1)]) + + def test_future_times_count_as_now_at_load(self): + """A resolve time ahead of the clock loads as now, as rnfs 0.1.4 loaded it.""" + write_legacy(self.state, self.FILE, {"version": 1, "floor": [[N[1].hex(), 5, NOW + 1000]]}) + self.assertEqual(self.manager()._floor[N[1]], (5, NOW)) + + def test_held_names_and_the_bound_apply_at_load(self): + """A name names.json holds has no floor entry, and the floor is cut to its bound, least recently resolved + first; the rows dropped are deleted.""" + write_legacy(self.state, names.STATE_FILE, {"held": {N[1].hex(): {"sequence": 9, "cid": str(A), + "heard": 1.0}}}) + size = names.FLOOR_FACTOR * self.config.name_table_size + others = [bytes([0x80, i]) * 8 for i in range(size + 2)] + write_legacy(self.state, self.FILE, {"version": 1, "floor": [[N[1].hex(), 5]] + + [[name.hex(), 1] for name in others]}) + manager = self.manager() + self.assertEqual(list(manager._floor), others[2:]) + self.assertEqual([name for name, *_ in self.floor()], others[2:]) + self.assertEqual(manager.record(N[1]).sequence, 9) + + def test_damaged_file_is_moved_aside(self): + """A file that isn't JSON, or isn't a floor of this version, is moved to .damaged and the node starts + without it.""" + damaged = ["{not json", "", "[]", '{"floor": []}', '{"version": 2, "floor": []}', '{"version": 1, "floor": {}}'] + for text in damaged: + with self.subTest(text=text): + path = write_legacy(self.state, self.FILE, text=text) + self.assertEqual(len(self.manager()._floor), 0) + self.assertFalse(os.path.exists(path)) + self.assertTrue(os.path.exists(os.path.join(self.state, f"{self.FILE}.damaged.{len(damaged) - 1}"))) + self.assertEqual(self.rows("SELECT * FROM name_floor"), []) + self.assertEqual(self.rows("SELECT * FROM migrated"), []) + + @unittest.skipUnless(POSIX_MODES and hasattr(os, "getuid") and os.getuid() != 0, "needs permission checks") + def test_unreadable_file_stops_startup(self): + """A name-floor.json that can't be read stops startup, as it did: a transient error isn't damage.""" + path = write_legacy(self.state, self.FILE, self.value()) + os.chmod(path, 0) + try: + with self.assertRaises(PermissionError): + self.manager() + finally: + os.chmod(path, 0o600) + self.assertEqual(len(self.manager()._floor), 3) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_migrate_small.py b/tests/test_migrate_small.py new file mode 100644 index 0000000..e4894cb --- /dev/null +++ b/tests/test_migrate_small.py @@ -0,0 +1,288 @@ +"""Migration of the small state files of rnfs 0.1.3 and 0.1.4 into the state database: pin_names.json, +shared_folder.json, jobs.json and contacts.json. + +For each: a valid file imports exactly, a damaged one is handled as those versions handled it, an import +interrupted after its commit isn't repeated, a second start changes nothing, a .migrated file left behind is +ignored, and an import whose commit fails is left for the next start.""" +import json +import os +import tempfile +import unittest +from unittest import mock + +import RNS + +from rnfs import statedb +from rnfs.config import Config +from rnfs.content.cid import RAW, Cid +from rnfs.content.shared_folder import SharedFolder +from rnfs.content.storage import Storage +from rnfs.features.contacts import ContactsFeature +from rnfs.features.jobs import JobsFeature +from rnfs.pin_names import MAX_NAME_LENGTH, PinNames +from tests.support import failing_commits, state_rows, write_legacy + +A, B, C = (Cid.for_block(RAW, bytes([i])) for i in range(3)) +NAME_ID, NODE_ID = "0123456789abcdef0123456789abcdef", "fedcba9876543210fedcba9876543210" + +_saved_loglevel = RNS.loglevel + + +def setUpModule(): + """Silence the log while these tests run.""" + RNS.loglevel = RNS.LOG_CRITICAL + + +def tearDownModule(): + """Restore RNS's loglevel.""" + RNS.loglevel = _saved_loglevel + + +class FeatureNode: + """What JobsFeature and ContactsFeature use of a node: its config and storage.""" + + def __init__(self, config: Config, storage: Storage): + """A node with config and storage, no features and not slow.""" + self.config = config + self.storage = storage + self.slow = False + self.features = {} + + +class MigrateCases: + """The cases every file is checked for (mixed into a TestCase per file, which sets FILE, VALUE, CHANGED, + COUNT, DAMAGED and load).""" + + FILE = "" + VALUE = None # the file as rnfs 0.1.3 and 0.1.4 wrote it + CHANGED = None # another valid file + COUNT = 0 # records VALUE imports + DAMAGED = ("{not json", "") # files moved aside to .damaged + + def setUp(self): + """An empty rnfs directory.""" + self.tmp = tempfile.TemporaryDirectory() + self.addCleanup(self.tmp.cleanup) + self.addCleanup(self.assert_closed) # after every store's close, before the directory goes + self.config = Config(rnfs_dir=os.path.join(self.tmp.name, "node")) + self.state = self.config.state_dir + self.path = os.path.join(self.state, self.FILE) + + def assert_closed(self): + """No state database under the temporary directory is left open (Windows can't delete open files).""" + root = os.path.realpath(self.tmp.name) + self.assertEqual([path for path in statedb.open_databases() if os.path.realpath(path).startswith(root)], []) + + def storage(self) -> Storage: + """Storage over the node's directories (closed before the directory is removed).""" + storage = Storage(self.config.blocks_dir, self.state, 10**9) + self.addCleanup(storage.close) + return storage + + def closing(self, store): + """store, closed before the directory is removed.""" + self.addCleanup(store.close) + return store + + def load(self): + """What the store loads (starting it afresh).""" + raise NotImplementedError + + def migrated(self) -> list[tuple]: + """The migrated table's record of this file.""" + return state_rows(self.state, "SELECT file, records FROM migrated WHERE file = ?", (self.FILE,)) + + def test_valid_file_imports_exactly(self): + """Every record is imported, the import recorded and the file kept as .migrated; a second start + changes nothing.""" + write_legacy(self.state, self.FILE, self.VALUE) + loaded = self.load() + self.assertEqual(loaded, self.EXPECTED) + self.assertEqual(self.migrated(), [(self.FILE, self.COUNT)]) + self.assertFalse(os.path.exists(self.path)) + with open(self.path + statedb.MIGRATED_SUFFIX, encoding="utf-8") as f: + self.assertEqual(json.load(f), self.VALUE) + self.assertEqual(self.load(), loaded) + self.assertEqual(self.migrated(), [(self.FILE, self.COUNT)]) + self.assertFalse(os.path.exists(self.path + statedb.MIGRATED_SUFFIX + ".1")) + + def test_damaged_file_is_moved_aside(self): + """A file that isn't JSON (or has the wrong shape) is moved to .damaged and the store starts without it.""" + for n, text in enumerate(self.DAMAGED): + with self.subTest(text=text): + write_legacy(self.state, self.FILE, text=text) + self.assertEqual(self.load(), self.EMPTY) + self.assertFalse(os.path.exists(self.path)) + self.assertTrue(os.path.exists(self.path + ".damaged" + (f".{n}" if n else ""))) + self.assertEqual(self.migrated(), []) + + def test_interrupted_migration_is_not_repeated(self): + """A crash between the import's commit and the rename leaves the file: the next start only renames it, + whatever it holds by then.""" + write_legacy(self.state, self.FILE, self.VALUE) + with mock.patch.object(statedb, "_keep_original", return_value=None): + first = self.load() + self.assertTrue(os.path.exists(self.path)) + write_legacy(self.state, self.FILE, self.CHANGED) + self.assertEqual(self.load(), first) + self.assertEqual(first, self.EXPECTED) + self.assertFalse(os.path.exists(self.path)) + with open(self.path + statedb.MIGRATED_SUFFIX, encoding="utf-8") as f: + self.assertEqual(json.load(f), self.CHANGED) # the file as it was, kept + self.assertEqual(self.migrated(), [(self.FILE, self.COUNT)]) + + def test_migrated_file_left_behind_is_ignored(self): + """A .migrated file is never imported.""" + write_legacy(self.state, self.FILE + statedb.MIGRATED_SUFFIX, self.VALUE) + self.assertEqual(self.load(), self.EMPTY) + self.assertEqual(self.migrated(), []) + + def test_failed_commit_leaves_the_file_for_the_next_start(self): + """An import whose commit fails stops the start and changes nothing; the next start imports the file.""" + self.load() # the database exists, so only the import's commit fails + write_legacy(self.state, self.FILE, self.VALUE) + with failing_commits(), self.assertRaises(OSError): + self.load() + self.assertTrue(os.path.exists(self.path)) + self.assertEqual(self.migrated(), []) + self.assertEqual(self.load(), self.EXPECTED) + + +class PinNamesTest(MigrateCases, unittest.TestCase): + """pin_names.json: {cid: name}.""" + + FILE = "pin_names.json" + VALUE = {str(A): "/home/u/caf\udce9.txt", str(B): "(stdin)", str(C): "x" * (MAX_NAME_LENGTH + 1)} + CHANGED = {str(A): "elsewhere"} + COUNT = 3 + EXPECTED = {A: "/home/u/caf\udce9.txt", B: "(stdin)", C: "x" * MAX_NAME_LENGTH} + EMPTY: dict = {} + DAMAGED = ("{not json", "", "[]", '"names"') + + def load(self) -> dict: + """The names PinNames loads.""" + return self.closing(PinNames(self.state))._names + + def test_invalid_records_are_skipped(self): + """Records that don't parse or whose name isn't a string are skipped, as rnfs 0.1.4 skipped them.""" + write_legacy(self.state, self.FILE, {"not a cid": "x", str(A): 5, str(B): ["x"], str(C): "ok"}) + self.assertEqual(self.load(), {C: "ok"}) + self.assertEqual(self.migrated(), [(self.FILE, 1)]) + + +class SharedFolderTest(MigrateCases, unittest.TestCase): + """shared_folder.json: {"entries": {name: [signature, cid]}}.""" + + FILE = "shared_folder.json" + VALUE = {"entries": {"a.txt": ["5f" * 32, str(A)], "caf\udce9": ["6e" * 32, str(B)]}} + CHANGED = {"entries": {"other": ["00" * 32, str(C)]}} + COUNT = 2 + EXPECTED = {"a.txt": ("5f" * 32, A), "caf\udce9": ("6e" * 32, B)} + EMPTY: dict = {} + DAMAGED = ("{not json", "", "[]", '{"entries": ["a"]}') + + def load(self) -> dict: + """The signature cache SharedFolder loads.""" + return self.closing(SharedFolder(self.config.files_dir, self.storage(), self.state))._known + + def test_invalid_records_are_skipped(self): + """Records that don't parse (or that SQLite can't store) are skipped, as rnfs 0.1.4 skipped them; a + signature that isn't a string is kept as its text, as it was.""" + write_legacy(self.state, self.FILE, {"entries": {"a": {"0": 1}, "b": 5, "c": ["sig", "not a cid"], + "d": [1], "e": ["\udc80", str(A)], "f": [7, str(B)]}}) + self.assertEqual(self.load(), {"f": ("7", B)}) + self.assertEqual(self.migrated(), [(self.FILE, 1)]) + + +class JobsTest(MigrateCases, unittest.TestCase): + """jobs.json: {"jobs": [record, ...]}, in creation order.""" + + FILE = "jobs.json" + DONE = {"id": "0000000a", "kind": "pin", "target": str(A), "output": None, "from": None, "sealed_name": False, + "fallback_dht": False, "state": "done", "created": 1.0, "started": 2.0, "finished": 3.0, "error": None, + "result": {"cid": str(A)}, "runs": 1, "total_bytes": 1, "done_bytes": 1, "blocks": 1, + "blocks_total": 1, "downloaded_bytes": 1} + # Interrupted, and with only the fields every version wrote + RUNNING = {"id": "0000000b", "kind": "get", "target": str(B) + "/docs", "output": os.path.abspath("out"), + "state": "running", "created": 4.0} + FAILED = {**DONE, "id": "0000000c", "kind": "export", "target": str(C), "output": os.path.abspath("x.rnfs"), + "state": "failed", "error": "no provider", "result": None} + VALUE = {"jobs": [RUNNING, DONE, FAILED]} + CHANGED = {"jobs": [{**DONE, "id": "0000000d"}]} + COUNT = 3 + EXPECTED = [("0000000b", "queued", None, 4.0), ("0000000a", "done", {"cid": str(A)}, 1.0), + ("0000000c", "failed", None, 1.0)] + EMPTY: list = [] + DAMAGED = ("{not json", "", "[]", '{"jobs": {}}') + + def feature(self) -> JobsFeature: + """A job manager over a fresh storage (closed before the directory is removed).""" + return self.closing(JobsFeature(FeatureNode(self.config, self.storage()))) + + def load(self) -> list[tuple]: + """(id, state, result, created) of each job the manager loads, oldest first.""" + return [(job["id"], job["state"], job["result"], job["created"]) for job in self.feature().list()] + + def test_invalid_records_are_skipped(self): + """Malformed records are skipped, as rnfs 0.1.4 skipped them, and a repeated id keeps the first.""" + write_legacy(self.state, self.FILE, {"jobs": [ + "junk", {**self.DONE, "id": "nothex!!"}, {**self.DONE, "kind": "copy"}, self.DONE, + {**self.DONE, "state": "failed"}, {**self.RUNNING, "output": "relative"}, self.RUNNING]}) + self.assertEqual([job[:2] for job in self.load()], [("0000000a", "done"), ("0000000b", "queued")]) + self.assertEqual(self.migrated(), [(self.FILE, 2)]) + self.assertEqual(state_rows(self.state, "SELECT id, position FROM jobs ORDER BY position"), + [("0000000a", 0), ("0000000b", 1)]) + + def test_failed_save_is_written_by_the_next(self): + """An imported running job loads as queued; if saving that fails it is still saved by the next save.""" + write_legacy(self.state, self.FILE, self.VALUE) + feature = self.feature() + with failing_commits(): + feature._save_quietly() # logged, not raised + self.assertEqual(statedb.load(state_rows(self.state, "SELECT record FROM jobs WHERE id = ?", + ("0000000b",))[0][0])["state"], "running") + feature.save() + self.assertEqual(statedb.load(state_rows(self.state, "SELECT record FROM jobs WHERE id = ?", + ("0000000b",))[0][0])["state"], "queued") + + +class ContactsTest(MigrateCases, unittest.TestCase): + """contacts.json: {"version": 1, "contacts": {nick: {"kind", "id"}}}.""" + + FILE = "contacts.json" + VALUE = {"version": 1, "contacts": {"alice": {"kind": "name", "id": NAME_ID}, + "relay": {"kind": "node", "id": NODE_ID}}} + CHANGED = {"version": 1, "contacts": {"bob": {"kind": "lxmf", "id": NODE_ID}}} + COUNT = 2 + EXPECTED = [{"nick": "alice", "kind": "name", "id": NAME_ID}, {"nick": "relay", "kind": "node", "id": NODE_ID}] + EMPTY: list = [] + + def load(self) -> list[dict]: + """The contacts the address book loads.""" + return self.closing(ContactsFeature(FeatureNode(self.config, None))).list() + + def test_invalid_records_are_skipped(self): + """Malformed entries are skipped, as rnfs 0.1.4 skipped them, and at most MAX_CONTACTS are imported.""" + write_legacy(self.state, self.FILE, {"contacts": { + "BAD": {"kind": "node", "id": NODE_ID}, "worse": {"kind": "spaceship", "id": NODE_ID}, "worst": "nope", + "short": {"kind": "node", "id": "ab"}, "ok": {"kind": "node", "id": "<" + NODE_ID.upper() + ">"}, + "late": {"kind": "node", "id": NODE_ID}}}) + with mock.patch("rnfs.features.contacts.MAX_CONTACTS", 1): + self.assertEqual(self.load(), [{"nick": "ok", "kind": "node", "id": NODE_ID}]) + self.assertEqual(self.migrated(), [(self.FILE, 1)]) + + def test_other_values_import_nothing(self): + """A file holding anything but an object with a "contacts" object imports nothing, as rnfs 0.1.4 loaded + nothing from it, and is kept as migrated.""" + for n, value in enumerate(([], {"version": 1}, {"contacts": []}, 5)): + with self.subTest(value=value): + self.config = Config(rnfs_dir=os.path.join(self.tmp.name, f"other{n}")) + self.state = self.config.state_dir + path = write_legacy(self.state, self.FILE, value) + self.assertEqual(self.load(), []) + self.assertEqual(self.migrated(), [(self.FILE, 0)]) + self.assertTrue(os.path.exists(path + statedb.MIGRATED_SUFFIX)) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_migrate_storage.py b/tests/test_migrate_storage.py new file mode 100644 index 0000000..a256a35 --- /dev/null +++ b/tests/test_migrate_storage.py @@ -0,0 +1,332 @@ +"""Migration of the storage, node and fetcher state files of rnfs 0.1.3 and 0.1.4 into the state database: +pins.json, cache.json (storage), fetched.json, remote_pins.json (node) and invalid.json (fetcher). + +For each: a valid file imports exactly, a damaged one is handled as those versions handled it, an import +interrupted after its commit isn't repeated, a second start changes nothing, and a .migrated file left behind +is ignored.""" +import json +import os +import tempfile +import unittest +from unittest import mock + +import RNS + +from rnfs import statedb +from rnfs.config import Config +from rnfs.content.cid import DAG, RAW, Cid +from rnfs.content.storage import Storage, remote_source +from rnfs.errors import RnfsError +from rnfs.fetcher import Fetcher +from rnfs.node import RnfsNode +from tests.support import POSIX_MODES, state_rows, write_legacy +from tests.test_names import FakeNetwork +from tests.test_node import NetClient + +A, B, C = (Cid.for_block(RAW, bytes([i])) for i in range(3)) +D = Cid.for_block(DAG, b"\x93\x00\x90\x90") +IDENTITY = "0123456789abcdef0123456789abcdef" +OTHER = "fedcba9876543210fedcba9876543210" + +_saved_loglevel = RNS.loglevel + + +def setUpModule(): + """Silence the log while these tests run.""" + RNS.loglevel = RNS.LOG_CRITICAL + + +def tearDownModule(): + """Restore RNS's loglevel.""" + RNS.loglevel = _saved_loglevel + + +class MigrateTest(unittest.TestCase): + """An rnfs directory under a temporary directory, with the state files written into /state.""" + + def setUp(self): + """An empty rnfs directory.""" + self.tmp = tempfile.TemporaryDirectory() + self.addCleanup(self.tmp.cleanup) + self.config = Config(rnfs_dir=os.path.join(self.tmp.name, "node"), remote_pin_identities=[IDENTITY]) + self.state = self.config.state_dir + + def storage(self) -> Storage: + """Storage over the node's directories (closed before the directory is removed).""" + storage = Storage(self.config.blocks_dir, self.state, 10**9) + self.addCleanup(storage.close) + return storage + + def node(self) -> RnfsNode: + """An offline node over the directory (stopped before the directory is removed).""" + node = RnfsNode.offline(self.config, b"\x07" * 16, NetClient(), FakeNetwork()) + self.addCleanup(node.stop) + return node + + def fetcher(self) -> Fetcher: + """A fetcher remembering invalid roots in the state directory.""" + fetcher = Fetcher(self.storage(), None, self.config, b"\x07" * 16, None, None, state_dir=self.state) + self.addCleanup(fetcher.close) + return fetcher + + def interrupted(self, name: str, value, changed, load): + """Import name holding value, as if the process died between the commit and the rename; write changed + into the file left behind and start again: load() must show value's state, and the file be renamed.""" + path = write_legacy(self.state, name, value) + with mock.patch.object(statedb, "_keep_original", return_value=None): + first = load() + self.assertTrue(os.path.exists(path)) + write_legacy(self.state, name, changed) + self.assertEqual(load(), first) + self.assertFalse(os.path.exists(path)) + with open(path + ".migrated", encoding="utf-8") as f: + self.assertEqual(json.load(f), changed) # the file as it was, kept + self.assertEqual(state_rows(self.state, "SELECT records FROM migrated WHERE file = ?", (name,)), [(len( + value) if isinstance(value, list) else len(next(iter(value.values()))),)]) + return first + + def migrated_again(self, name: str, changed, load, expected): + """A .migrated left behind (and the database's state) survive another start unchanged.""" + write_legacy(self.state, name + ".migrated", changed) + self.assertEqual(load(), expected) + + +class PinsTest(MigrateTest): + """pins.json: {"pins": {cid: [source, ...]}}.""" + + VALUE = {"pins": {str(A): ["user", "folder"], str(B): ["name:photos", "remote:" + IDENTITY], + str(D): ["follow:" + IDENTITY, "lxmf:" + OTHER]}} + + def load(self) -> dict: + """The pins storage loads.""" + return self.storage().pins() + + def test_valid_file_imports_exactly(self): + """Every pin and source is imported; the file is kept as pins.json.migrated.""" + write_legacy(self.state, "pins.json", self.VALUE) + self.assertEqual(self.load(), {A: {"user", "folder"}, B: {"name:photos", "remote:" + IDENTITY}, + D: {"follow:" + IDENTITY, "lxmf:" + OTHER}}) + self.assertEqual(len(state_rows(self.state, "SELECT * FROM pins")), 6) + self.assertTrue(os.path.exists(os.path.join(self.state, "pins.json.migrated"))) + self.assertEqual(self.load(), {A: {"user", "folder"}, B: {"name:photos", "remote:" + IDENTITY}, + D: {"follow:" + IDENTITY, "lxmf:" + OTHER}}) # a second start: no change + + def test_invalid_records_are_skipped(self): + """Records that don't parse, with sources that aren't a list or aren't valid (or that SQLite can't + store), are skipped one by one, as rnfs 0.1.4 skipped them; an empty source list pins nothing.""" + write_legacy(self.state, "pins.json", {"pins": { + "not a cid": ["user"], str(A): "user", str(B): ["users"], str(C): ["name:\udc80"], str(D): [], + Cid.for_block(RAW, b"ok").__str__(): ["user"]}}) + self.assertEqual(self.load(), {Cid.for_block(RAW, b"ok"): {"user"}}) + + def test_damaged_file_is_moved_aside(self): + """A file that isn't JSON or has the wrong shape is moved to .damaged and storage starts without it.""" + for text in ("{not json", "", "[]", '{"pins": []}'): + with self.subTest(text=text): + path = write_legacy(self.state, "pins.json", text=text) + self.assertEqual(self.load(), {}) + self.assertFalse(os.path.exists(path)) + self.assertTrue(os.path.exists(os.path.join(self.state, "pins.json.damaged.3"))) + + @unittest.skipUnless(POSIX_MODES and hasattr(os, "getuid") and os.getuid() != 0, "needs permission checks") + def test_unreadable_file_stops_startup(self): + """A pins.json that can't be read stops startup: starting without it would let gc delete the pins.""" + path = write_legacy(self.state, "pins.json", self.VALUE) + os.chmod(path, 0) + try: + with self.assertRaises(PermissionError): + self.storage() + finally: + os.chmod(path, 0o600) + self.assertEqual(len(self.load()), 3) + + def test_interrupted_migration_isnt_repeated(self): + """A crash between the commit and the rename: the next start only renames the file.""" + pins = self.interrupted("pins.json", self.VALUE, {"pins": {str(C): ["user"]}}, self.load) + self.assertEqual(set(pins), {A, B, D}) + self.migrated_again("pins.json", {"pins": {str(C): ["user"]}}, self.load, pins) + + +class CacheTest(MigrateTest): + """cache.json: {"fetches": [[root, [name, ...], target], ...]}, least recently used first.""" + + VALUE = {"fetches": [[str(D), ["docs", "é"], str(A)], [str(B), [], str(B)], [str(D), ["x"], str(C)]]} + + def load(self) -> list: + """The cached fetches storage loads, least recently used first.""" + return self.storage().cached_fetches() + + def test_valid_file_imports_exactly_in_order(self): + """Every record is imported in least recently used order, which later saves keep.""" + write_legacy(self.state, "cache.json", self.VALUE) + storage = self.storage() + self.assertEqual(storage.cached_fetches(), [(D, ("docs", "é"), A), (B, (), B), (D, ("x",), C)]) + storage.touch(D, ["docs", "é"]) + storage.save() + self.assertEqual(self.load(), [(B, (), B), (D, ("x",), C), (D, ("docs", "é"), A)]) + + def test_invalid_records_are_skipped_and_the_cap_kept(self): + """Records that don't parse or name invalid DAG names are skipped; a repeated record keeps its first + place with its last target; past MAX_CACHE_RECORDS the oldest are dropped.""" + write_legacy(self.state, "cache.json", {"fetches": [ + [str(A), ["a/b"], str(A)], [str(A), "ab", str(A)], ["bad", [], str(A)], [str(A), [], str(A), 1], + [str(B), [], str(A)], [str(C), [], str(C)], [str(B), [], str(B)]]}) + with mock.patch("rnfs.content.storage.MAX_CACHE_RECORDS", 1): + self.assertEqual(self.load(), [(C, (), C)]) + self.assertEqual(state_rows(self.state, "SELECT target FROM cache"), [(C.binary,)]) + + def test_damaged_file_is_moved_aside(self): + """A file that isn't JSON or has the wrong shape is moved to .damaged and storage starts without it.""" + for text in ("{not json", '{"fetches": 5}', '"text"'): + with self.subTest(text=text): + write_legacy(self.state, "cache.json", text=text) + self.assertEqual(self.load(), []) + self.assertTrue(os.path.exists(os.path.join(self.state, "cache.json.damaged.2"))) + + def test_interrupted_migration_isnt_repeated(self): + """A crash between the commit and the rename: the next start only renames the file.""" + changed = {"fetches": [[str(C), [], str(C)]]} + records = self.interrupted("cache.json", self.VALUE, changed, self.load) + self.assertEqual(len(records), 3) + self.migrated_again("cache.json", changed, self.load, records) + + +class FetchedTest(MigrateTest): + """fetched.json: [cid, ...], the targets fetched for the user.""" + + def load(self) -> set: + """The fetched targets a node loads.""" + node = self.node() + targets = node._eligible_targets() + node.stop() + return targets + + def test_valid_file_imports_exactly(self): + """Every target is imported; one that doesn't parse is skipped, as before.""" + write_legacy(self.state, "fetched.json", [str(A), str(D), "bad", 5]) + self.assertEqual(self.load(), {A, D}) + + def test_damaged_file_is_moved_aside(self): + """A file that isn't JSON or isn't a list is moved to .damaged and the node starts without it.""" + for text in ("{not json", '{"a": 1}'): + with self.subTest(text=text): + write_legacy(self.state, "fetched.json", text=text) + self.assertEqual(self.load(), set()) + self.assertTrue(os.path.exists(os.path.join(self.state, "fetched.json.damaged.1"))) + + def test_interrupted_migration_isnt_repeated(self): + """A crash between the commit and the rename: the next start only renames the file.""" + targets = self.interrupted("fetched.json", [str(A), str(B)], [str(C)], self.load) + self.assertEqual(targets, {A, B}) + self.migrated_again("fetched.json", [str(C)], self.load, targets) + + +class RemotePinsTest(MigrateTest): + """remote_pins.json (0.1.4 only): [[source, cid], ...], in the order accepted.""" + + VALUE = [[remote_source(IDENTITY), str(B)], [remote_source(OTHER), str(C)], [remote_source(IDENTITY), str(A)]] + + def load(self) -> list: + """The remote pins a node would resume, in order.""" + node = self.node() + pins = node._load_remote_pins() + node.stop() + return pins + + def test_valid_file_imports_exactly_in_order(self): + """Every pair is imported in order; at load, only sources still allowed are resumed, as before.""" + write_legacy(self.state, "remote_pins.json", self.VALUE) + self.assertEqual(self.load(), [(remote_source(IDENTITY), B), (remote_source(IDENTITY), A)]) + self.assertEqual(len(state_rows(self.state, "SELECT * FROM remote_pins")), 3) + self.config.remote_pin_identities = [IDENTITY, OTHER] + self.assertEqual(self.load(), [(remote_source(IDENTITY), B), (remote_source(OTHER), C), + (remote_source(IDENTITY), A)]) + + def test_malformed_entries_are_skipped(self): + """Entries that aren't a pair of strings, or whose CID doesn't parse, are skipped and counted in a + warning.""" + write_legacy(self.state, "remote_pins.json", [[[remote_source(IDENTITY)], str(A)], [remote_source(IDENTITY)], + [remote_source(IDENTITY), "bad"], self.VALUE[0]]) + with mock.patch("rnfs.node.log") as log: + self.assertEqual(self.load(), [(remote_source(IDENTITY), B)]) + self.assertTrue(any("3 malformed entries" in call.args[0] for call in log.call_args_list)) + + def test_damaged_file_is_moved_aside(self): + """A file that isn't JSON or isn't a list is moved to .damaged and the node starts without it.""" + write_legacy(self.state, "remote_pins.json", text="{not json") + self.assertEqual(self.load(), []) + self.assertTrue(os.path.exists(os.path.join(self.state, "remote_pins.json.damaged"))) + + @unittest.skipUnless(POSIX_MODES and hasattr(os, "getuid") and os.getuid() != 0, "needs permission checks") + def test_unreadable_file_is_left_for_the_next_start(self): + """A file that can't be read is logged and skipped (the pins are best effort), and imported once it can + be read.""" + path = write_legacy(self.state, "remote_pins.json", self.VALUE) + os.chmod(path, 0) + try: + self.assertEqual(self.load(), []) + finally: + os.chmod(path, 0o600) + self.assertEqual(len(self.load()), 2) + + def test_interrupted_migration_isnt_repeated(self): + """A crash between the commit and the rename: the next start only renames the file.""" + changed = [[remote_source(IDENTITY), str(C)]] + pins = self.interrupted("remote_pins.json", self.VALUE, changed, self.load) + self.assertEqual(len(pins), 2) + self.migrated_again("remote_pins.json", changed, self.load, pins) + + +class InvalidTest(MigrateTest): + """invalid.json: [cid, ...], oldest first.""" + + def load(self) -> list: + """Which of A, B, C, D a fetcher remembers as invalid.""" + fetcher = self.fetcher() + return [fetcher.is_invalid(cid) for cid in (A, B, C, D)] + + def test_valid_file_imports_exactly(self): + """Every root is imported, oldest first; past MAX_INVALID only the newest are kept.""" + write_legacy(self.state, "invalid.json", [str(A), "bad", str(D), str(C)]) + with mock.patch("rnfs.fetcher.MAX_INVALID", 2): + self.assertEqual(self.load(), [False, False, True, True]) + self.assertEqual(state_rows(self.state, "SELECT cid FROM invalid ORDER BY position"), + [(D.binary,), (C.binary,)]) + + def test_damaged_file_is_moved_aside(self): + """A file that isn't JSON (or, now, isn't a list) is moved to .damaged and the fetcher starts without + it.""" + for text in ("{not json", "5"): + write_legacy(self.state, "invalid.json", text=text) + self.assertEqual(self.load(), [False] * 4) + self.assertTrue(os.path.exists(os.path.join(self.state, "invalid.json.damaged.1"))) + + def test_interrupted_migration_isnt_repeated(self): + """A crash between the commit and the rename: the next start only renames the file.""" + roots = self.interrupted("invalid.json", [str(A), str(B)], [str(C)], self.load) + self.assertEqual(roots, [True, True, False, False]) + self.migrated_again("invalid.json", [str(C)], self.load, roots) + + + +class FailedStartTest(MigrateTest): + """A node that can't start releases the state database.""" + + def test_failed_setup_closes_the_database(self): + """A store refusing to start (a malformed names.json stops startup) leaves no state database open, and + the file in place for the next start.""" + path = write_legacy(self.state, "names.json", text="null") + with self.assertRaises(RnfsError): + RnfsNode.offline(self.config, b"\x07" * 16, NetClient(), FakeNetwork()) + self.assertNotIn(os.path.join(self.state, statedb.DB_FILE), statedb.open_databases()) + self.assertTrue(os.path.exists(path)) + os.remove(path) + self.assertEqual(self.load_pins(), {}) + + def load_pins(self) -> dict: + """The pins a node starting now holds.""" + return self.node().storage.pins() + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_names.py b/tests/test_names.py index 209cc59..d43e9f3 100644 --- a/tests/test_names.py +++ b/tests/test_names.py @@ -1,5 +1,4 @@ """Tests for mutable names (specification section 9), with RNS's network side replaced by fakes.""" -import json import os import stat import tempfile @@ -10,12 +9,12 @@ import RNS from RNS.vendor import umsgpack -from rnfs import fsutil, names +from rnfs import fsutil, names, statedb from rnfs.config import Config from rnfs.content.cid import DAG, RAW, Cid -from rnfs.errors import NotFound +from rnfs.errors import NotFound, RnfsError from rnfs.names import NameManager, encode_record, parse_record -from tests.support import POSIX_MODES, FakeClock +from tests.support import POSIX_MODES, FakeClock, failing_commits, state_execute, state_rows _saved_loglevel = RNS.loglevel @@ -183,8 +182,14 @@ def setUp(self): self.manager = self.make_manager() def make_manager(self) -> NameManager: - """A manager over the same directory (as after a restart), sharing the fakes.""" - return NameManager(self.config, self.pins, network=self.network, clock=self.clock, monotonic=self.clock) + """A manager over the same directory (as after a restart), sharing the fakes; closed after the test.""" + manager = NameManager(self.config, self.pins, network=self.network, clock=self.clock, monotonic=self.clock) + self.addCleanup(manager.close) + return manager + + def rows(self, sql: str, params=()) -> list[tuple]: + """The rows sql reads from the state database.""" + return state_rows(self.config.state_dir, sql, params) class RecordParsingTest(unittest.TestCase): @@ -288,7 +293,9 @@ def test_old_temp_files_removed_at_start(self): f.write(b"partial") self.assertEqual(self.make_manager().list()[0]["alias"], "site") self.assertEqual([os.path.exists(path) for path in old + kept], [False, False, True, True]) - self.assertEqual(sorted(os.listdir(self.config.state_dir)), ["names.json"]) # no temp file of its own + # No temp file of its own: only the state database is there + self.assertEqual([entry for entry in os.listdir(self.config.state_dir) + if not entry.startswith(statedb.DB_FILE)], []) def test_export_import_round_trip(self): """An exported key imported elsewhere gives the same name.""" @@ -298,6 +305,7 @@ def test_export_import_round_trip(self): other = NameManager(Config(rnfs_dir=os.path.join(self.tmp.name, "other")), FakePins(), network=FakeNetwork()) + self.addCleanup(other.close) self.assertEqual(other.import_key("copy", key), name) self.assertEqual(other.export_key("copy"), key) self.assertEqual(other.list()[0]["name"], name.hex()) @@ -414,13 +422,14 @@ def test_first_publish_of_imported_key_asks_keepers(self): self.assertEqual(self.manager.publish("other", CID_B)["sequence"], 5) def test_key_without_state_is_published_above_the_network(self): - """With names.json lost, a key's first publish asks the network and keepers, as for an imported key.""" + """With its records lost, a key's first publish asks the network and keepers, as for an imported key.""" keepers = self.manager.keepers = FakeKeepers() name = self.manager.create("site") for cid in (CID_A, CID_B, CID_C): self.manager.publish("site", cid) self.assertEqual(keepers.queried, []) # a key made here needs no network - os.remove(os.path.join(self.config.state_dir, names.STATE_FILE)) + for table in ("names_published", "names_held"): + state_execute(self.config.state_dir, f"DELETE FROM {table}") keepers.answers[name] = [(3, CID_C, self.clock.now - 10)] restarted = self.make_manager() restarted.keepers = keepers @@ -612,7 +621,7 @@ def test_heard_time_in_the_future_isnt_fresh(self): name = other_name(1) self.network.announce(name, encode_record(1, CID_A), self.clock.now) self.manager.resolve(name, persist=True) - with self.manager._lock: # as loaded from names.json saved while the clock ran ahead + with self.manager._lock: # as loaded from a record saved while the clock ran ahead self.manager._held[name] = names.Record(1, CID_A, self.clock.now + 10**6) self.network.path_requests.clear() self.manager.resolve(name, persist=True, max_age=60) @@ -624,15 +633,126 @@ def test_resolve_rejects_bad_names(self): self.manager.resolve(b"short", persist=False) +class StateTest(NamesTestCase): + """The published and held records in the state database: what a save writes, failed commits, damaged rows.""" + + def test_rows(self): + """Published and held records are rows, sequences as decimal text; a key not yet published has no CID.""" + name = self.manager.create("site") + self.manager.create("empty") + self.manager.publish("site", CID_A) + self.assertEqual(self.rows("SELECT alias, sequence, cid, imported FROM names_published ORDER BY alias"), + [("empty", "0", None, 0), ("site", "1", CID_A.binary, 0)]) + self.assertEqual(self.rows("SELECT name, sequence, cid, heard FROM names_held"), + [(name, "1", CID_A.binary, self.clock.now)]) + + def test_sequences_past_sqlite_integers(self): + """The highest sequence, past SQLite's signed integers, is saved and loaded as it is.""" + name = self.manager.create("site") + self.network.store(name, encode_record(names.MAX_SEQUENCE - 1, CID_A), self.clock.now) + self.assertEqual(self.manager.publish("site", CID_B)["sequence"], names.MAX_SEQUENCE) + self.assertEqual(self.rows("SELECT sequence FROM names_published"), [(str(names.MAX_SEQUENCE),)]) + restarted = self.make_manager() + self.assertEqual(restarted.list()[0]["sequence"], names.MAX_SEQUENCE) + with self.assertRaises(RnfsError): + restarted.publish("site", CID_C) + + def test_save_writes_only_changed_rows(self): + """A publish writes its name's two rows, whatever else is saved; a save with nothing changed writes none.""" + for alias in ("a", "b", "c"): + self.manager.create(alias) + self.manager.publish(alias, CID_A) + conn = self.manager._db._db().conn + before = conn.total_changes + self.manager.publish("b", CID_B) + self.assertEqual(conn.total_changes - before, 2) # b's published row and its held record + before = conn.total_changes + self.manager.stop() + self.assertEqual(conn.total_changes, before) + + def test_failed_commit_rolls_the_publish_back(self): + """A publish whose save can't be committed (a failing disk) fails, and memory, pins and the database + say what they said before it; trying again works.""" + name = self.manager.create("site") + self.manager.publish("site", CID_A) + before = self.rows("SELECT * FROM names_published"), self.rows("SELECT * FROM names_held") + with failing_commits(), self.assertRaises(OSError): + self.manager.publish("site", CID_B) + self.assertEqual(self.manager.list()[0]["cid"], str(CID_A)) + self.assertEqual(self.manager.record(name).cid, CID_A) + self.assertEqual(self.pins.sources, {CID_A: {"name:site"}}) + self.assertEqual((self.rows("SELECT * FROM names_published"), self.rows("SELECT * FROM names_held")), before) + self.assertEqual(self.manager.publish("site", CID_B)["sequence"], 2) + self.assertEqual(self.make_manager().list()[0]["cid"], str(CID_B)) + + def test_failed_resolve_save_is_retried(self): + """A record resolved for the user whose save failed is still held, and saved by the next save.""" + name = other_name(1) + self.network.store(name, encode_record(4, CID_A), self.clock.now) + with failing_commits(), self.assertRaises(OSError): + self.manager.resolve(name, persist=True) + self.assertEqual(self.rows("SELECT * FROM names_held"), []) + self.manager.stop() + self.assertEqual(self.rows("SELECT name, sequence FROM names_held"), [(name, "4")]) + + def test_malformed_rows_stop_startup(self): + """A published or held row that doesn't parse stops startup, as a malformed names.json did: starting + without it would allow rollback.""" + self.manager.create("site") + self.manager.publish("site", CID_A) + self.manager.close() + published, held = self.rows("SELECT * FROM names_published"), self.rows("SELECT * FROM names_held") + damage = [ + ("UPDATE names_published SET sequence = 'one'", ()), + ("UPDATE names_published SET sequence = ?", (b"1",)), + ("UPDATE names_published SET cid = ?", (b"\x00" * 36,)), + ("UPDATE names_published SET cid = 'text'", ()), + ("UPDATE names_published SET imported = 2", ()), + ("INSERT INTO names_published VALUES ('../x', '1', NULL, 0)", ()), + ("UPDATE names_held SET name = ?", (b"short",)), + ("UPDATE names_held SET sequence = '1.5'", ()), + ("UPDATE names_held SET cid = ?", (CID_A.binary[:-1],)), + ("UPDATE names_held SET heard = 'soon'", ()), + ] + for sql, params in damage: + with self.subTest(sql=sql): + for table, rows in (("names_published", published), ("names_held", held)): + state_execute(self.config.state_dir, f"DELETE FROM {table}") + for row in rows: + state_execute(self.config.state_dir, f"INSERT INTO {table} VALUES (?, ?, ?, ?)", row) + state_execute(self.config.state_dir, sql, params) + with self.assertRaisesRegex(RnfsError, "malformed name record"): + self.make_manager() + + def test_unreadable_key_row_left_as_it_is(self): + """The row of an alias whose key file can't be read survives every save, unchanged.""" + self.manager.create("site") + self.manager.publish("site", CID_A) + self.manager.close() + row = self.rows("SELECT * FROM names_published") + with open(os.path.join(self.config.names_dir, "site"), "wb") as f: + f.write(b"\x00" * 7) # not a key + restarted = self.make_manager() + restarted.create("other") + restarted.stop() + self.assertEqual(self.rows("SELECT * FROM names_published WHERE alias = 'site'"), row) + + def test_close_is_idempotent(self): + """close() releases the database once; again it does nothing.""" + self.manager.close() + self.manager.close() + self.assertNotIn(self.manager._db.path, statedb.open_databases()) + + class FloorSaveTest(NamesTestCase): - """name-floor.json is written outside the manager's lock, in order, and a failed write is tried again.""" + """The name_floor rows are written outside the manager's lock, in order, only those changed, and a failed write + is tried again.""" def setUp(self): """A started manager.""" super().setUp() self.manager.start() self.addCleanup(self.manager.stop) - self.path = os.path.join(self.config.state_dir, names.FLOOR_FILE) def resolve_for_gateway(self, i: int, sequence: int = 1): """Resolve other_name(i), just announced at sequence, without persist (so it goes in the floor).""" @@ -640,66 +760,134 @@ def resolve_for_gateway(self, i: int, sequence: int = 1): self.assertEqual(self.manager.resolve(other_name(i), persist=False, max_age=60)["sequence"], sequence) def saved(self) -> list: - """The floor entries in the file.""" - with open(self.path, encoding="utf-8") as f: - return json.load(f)["floor"] + """The names the name_floor table holds, least recently resolved first.""" + return [name for (name,) in self.rows("SELECT name FROM name_floor ORDER BY used")] - def test_file_content(self): + def plant(self, *rows): + """Close the manager and plant (name, sequence, resolved, used) rows in name_floor.""" + self.manager.stop() + self.manager.close() + for row in rows: + state_execute(self.config.state_dir, "INSERT INTO name_floor VALUES (?, ?, ?, ?)", row) + + def test_rows(self): + """Each entry is a row: its name, sequence (decimal text), resolve time and place.""" self.resolve_for_gateway(1, sequence=7) self.clock.now += 0.5 self.resolve_for_gateway(2) self.manager.stop() - with open(self.path, "rb") as f: - self.assertEqual(f.read(), json.dumps({"version": 1, "floor": [ - [other_name(1).hex(), 7, 1_000_000], [other_name(2).hex(), 1, 1_000_000]]}, - separators=(",", ":")).encode()) + self.assertEqual(self.rows("SELECT name, sequence, resolved FROM name_floor ORDER BY used"), + [(other_name(1), "7", 1_000_000.0), (other_name(2), "1", 1_000_000.5)]) + + def test_order_and_raises_survive_a_restart(self): + """A name resolved again moves last; a sequence raised by an announce keeps its place; both are loaded so.""" + for i in (1, 2, 3): + self.clock.now += names.FLOOR_SAVE_INTERVAL + self.resolve_for_gateway(i) + self.clock.now += names.FLOOR_SAVE_INTERVAL + self.resolve_for_gateway(1, sequence=2) + self.network.announce(other_name(2), encode_record(9, CID_B), self.clock.now) + self.manager.stop() + self.assertEqual(self.saved(), [other_name(2), other_name(3), other_name(1)]) + restarted = self.make_manager() + self.assertEqual([(name, sequence) for name, (sequence, _) in restarted._floor.items()], + [(other_name(2), 9), (other_name(3), 1), (other_name(1), 2)]) + + def test_save_writes_only_changed_rows(self): + """A resolve adding a name to the floor writes its row alone, as a save after an announce raised one does.""" + for i in range(1, 6): + self.clock.now += names.FLOOR_SAVE_INTERVAL + self.resolve_for_gateway(i) + conn = self.manager._db._db().conn + before = conn.total_changes + self.clock.now += names.FLOOR_SAVE_INTERVAL + self.resolve_for_gateway(6) + self.assertEqual(conn.total_changes - before, 1) + before = conn.total_changes + self.network.announce(other_name(6), encode_record(2, CID_B), self.clock.now) + self.manager.announce_due() + self.assertEqual(conn.total_changes - before, 1) + self.assertEqual(self.rows("SELECT sequence FROM name_floor WHERE name = ?", (other_name(6),)), [("2",)]) def test_written_without_the_lock(self): - real_write = fsutil.write_atomic + """The floor's commit runs without the manager's lock: resolves and announces needn't wait for it.""" + real_commit = statedb._Database._commit free = [] - def write(path, data, *args, **kwargs): - """The floor's write: another thread can take the manager's lock meanwhile.""" - if path == self.path: - lock = self.manager._lock - thread = threading.Thread(target=lambda: free.append(lock.acquire(timeout=5)) or lock.release()) - thread.start() - thread.join() - return real_write(path, data, *args, **kwargs) + def commit(database, conn): + """The floor's commit: another thread can take the manager's lock meanwhile.""" + lock = self.manager._lock + thread = threading.Thread(target=lambda: free.append(lock.acquire(timeout=5)) or lock.release()) + thread.start() + thread.join() + return real_commit(database, conn) - with mock.patch.object(names.fsutil, "write_atomic", write): + with mock.patch.object(statedb._Database, "_commit", commit): self.resolve_for_gateway(1) self.assertEqual(free, [True]) - self.assertEqual([entry[0] for entry in self.saved()], [other_name(1).hex()]) + self.assertEqual(self.saved(), [other_name(1)]) def test_saves_never_land_out_of_order(self): """A save that copies the floor while another is still writing it isn't overwritten by the older copy.""" - real_write = fsutil.write_atomic + real_commit = statedb._Database._commit second = [] - def write(path, data, *args, **kwargs): - """The first save's write: meanwhile another resolve raises the floor and saves it.""" - if path == self.path and not second: + def commit(database, conn): + """The first save's commit: meanwhile another resolve raises the floor and saves it.""" + if not second: self.clock.now += names.FLOOR_SAVE_INTERVAL second.append(threading.Thread(target=self.resolve_for_gateway, args=(2,))) second[0].start() second[0].join(0.2) - return real_write(path, data, *args, **kwargs) + return real_commit(database, conn) - with mock.patch.object(names.fsutil, "write_atomic", write): + with mock.patch.object(statedb._Database, "_commit", commit): self.resolve_for_gateway(1) second[0].join(5) - self.assertEqual([entry[0] for entry in self.saved()], [other_name(1).hex(), other_name(2).hex()]) + self.assertEqual(self.saved(), [other_name(1), other_name(2)]) def test_failed_save_is_tried_again_at_once(self): - with mock.patch.object(names.fsutil, "write_atomic", side_effect=OSError("disk full")): + """A save that couldn't be committed doesn't count for FLOOR_SAVE_INTERVAL: the next resolve saves again.""" + with failing_commits(): self.resolve_for_gateway(1) - self.assertFalse(os.path.exists(self.path)) + self.assertEqual(self.saved(), []) self.resolve_for_gateway(2) # within FLOOR_SAVE_INTERVAL of the failed save, which doesn't count - self.assertEqual([entry[0] for entry in self.saved()], [other_name(1).hex(), other_name(2).hex()]) + self.assertEqual(self.saved(), [other_name(1), other_name(2)]) self.resolve_for_gateway(3) # but the save that worked does self.assertEqual(len(self.saved()), 2) + def test_invalid_rows_are_dropped(self): + """A floor row that doesn't parse is logged and deleted; the node starts without it.""" + good = (other_name(1), "5", 1.0, 0) + self.plant(good, (b"short", "5", 1.0, 1), (other_name(2), str(names.MAX_SEQUENCE + 1), 1.0, 2), + (other_name(3), "-1", 1.0, 3), (other_name(4), "five", 1.0, 4), (other_name(5), "6", "soon", 5), + (other_name(6), "7", 2.0, "late"), (other_name(7), "8", 3.0, 2**63 - 1)) + with mock.patch.object(names, "log") as log: + restarted = self.make_manager() + self.assertEqual(dict(restarted._floor), {other_name(1): (5, 1.0)}) + self.assertEqual(self.rows("SELECT * FROM name_floor"), [good]) + self.assertEqual(sum("invalid name floor entry" in call.args[0] for call in log.call_args_list), 7) + + def test_rows_dropped_at_load_are_deleted(self): + """The floor rows of names held for the user, and those past the floor's bound, are deleted at load.""" + state_execute(self.config.state_dir, "INSERT INTO names_held VALUES (?, '1', ?, 0.0)", + (other_name(100), CID_A.binary)) + size = names.FLOOR_FACTOR * self.config.name_table_size + self.plant((other_name(100), "1", 0.0, 0), *[(other_name(i), "1", 0.0, i) for i in range(1, size + 3)]) + restarted = self.make_manager() + kept = [other_name(i) for i in range(3, size + 3)] # the two least recently resolved went + self.assertEqual(list(restarted._floor), kept) + self.assertEqual(self.saved(), kept) + + def test_future_resolve_time_counts_as_now(self): + """A resolve time ahead of the clock (corrected since) loads as now, and is saved so with the next change.""" + self.plant((other_name(1), "5", self.clock.now + 1000, 0)) + restarted = self.make_manager() + self.assertEqual(restarted._floor[other_name(1)], (5, self.clock.now)) + self.network.store(other_name(2), encode_record(1, CID_A), self.clock.now) + restarted.resolve(other_name(2), persist=False) + self.assertEqual(self.rows("SELECT resolved FROM name_floor ORDER BY used"), [(self.clock.now,)] * 2) + if __name__ == "__main__": unittest.main() diff --git a/tests/test_names_airtime.py b/tests/test_names_airtime.py index 442b54f..dc94fb2 100644 --- a/tests/test_names_airtime.py +++ b/tests/test_names_airtime.py @@ -178,6 +178,7 @@ def make_publisher(self, keys: set[bytes], slow: bool = False, **settings) -> Pu from rnfs.content.storage import Storage config = Config(rnfs_dir=self.tmp.name, **settings) storage = Storage(os.path.join(self.tmp.name, "blocks"), os.path.join(self.tmp.name, "state"), 10**9) + self.addCleanup(storage.close) # its state database, before the directory goes (Windows) self.clock = FakeClock(1_000_000.0) self.client = FakeClient() publisher = Publisher(bytes(16), config, storage, ProviderStore(10_000, None), self.client, diff --git a/tests/test_names_keep.py b/tests/test_names_keep.py index 43d7995..b7077c3 100644 --- a/tests/test_names_keep.py +++ b/tests/test_names_keep.py @@ -12,6 +12,7 @@ import RNS from RNS.vendor import umsgpack +from rnfs import statedb from rnfs.config import Config from rnfs.content.cid import RAW, Cid from rnfs.features import keepers @@ -23,7 +24,7 @@ from rnfs.node import RnfsNode from rnfs.publisher import DhtKey -from tests.support import FakeClock +from tests.support import FakeClock, failing_commits, state_execute, state_rows from tests.test_names import FakeNetwork from tests.test_node import NetClient from tests.test_server import FakeBackend, FakeDestination, FakeLink @@ -164,12 +165,18 @@ def setUp(self): """A store in a temporary directory.""" self.tmp = tempfile.TemporaryDirectory() self.addCleanup(self.tmp.cleanup) - self.path = os.path.join(self.tmp.name, "keepers.json") + self.state = self.tmp.name self.identity = RNS.Identity() self.name = name_for_identity(self.identity) + def open(self, max_names: int = 10) -> KeeperStore: + """A store over the state directory (closed before the directory is removed).""" + store = KeeperStore(self.state, max_names) + self.addCleanup(store.close) + return store + def test_only_kept_names_store_newer_records(self): - store = KeeperStore(self.path, 10) + store = self.open() self.assertFalse(store.offer(signed(self.identity, 1, CID1))) # not kept store.keep(self.name, USER) self.assertTrue(store.offer(signed(self.identity, 2, CID1))) @@ -178,22 +185,71 @@ def test_only_kept_names_store_newer_records(self): self.assertEqual(store.provided_keys(), {name_key(self.name)}) def test_persisted_and_reverified(self): - store = KeeperStore(self.path, 10) + store = self.open() store.keep(self.name, USER) store.offer(signed(self.identity, 5, CID2)) store.save() - again = KeeperStore(self.path, 10) + again = self.open() self.assertEqual((again.get(self.name).sequence, again.reasons(self.name)), (5, {USER})) - # A record damaged on disk is dropped rather than served - with open(self.path, encoding="utf-8") as f: - text = f.read() + # A record damaged on disk is dropped rather than served, and from its row; the name stays kept + (text,), = state_rows(self.state, "SELECT record FROM keepers") record_hex = again.get(self.name).signature.hex() - with open(self.path, "w", encoding="utf-8") as f: - f.write(text.replace(record_hex, "00" * 64)) - self.assertIsNone(KeeperStore(self.path, 10).get(self.name)) + state_execute(self.state, "UPDATE keepers SET record = ?", (text.replace(record_hex, "00" * 64),)) + damaged = self.open() + self.assertIsNone(damaged.get(self.name)) + self.assertEqual(damaged.reasons(self.name), {USER}) + self.assertEqual(state_rows(self.state, "SELECT reasons, record FROM keepers"), [('["user"]', None)]) + + def test_saves_write_only_the_names_changed(self): + """A save writes the rows of the names kept, released or given a record since the last save.""" + store = self.open() + other = bytes([1] * 16) + store.keep(self.name, USER) + store.keep(other, USER) + store.save() + self.assertEqual(sorted(state_rows(self.state, "SELECT name, reasons, record FROM keepers")), + sorted([(self.name, '["user"]', None), (other, '["user"]', None)])) + state_execute(self.state, "UPDATE keepers SET reasons = '[\"follow\", \"user\"]' WHERE name = ?", (other,)) + store.offer(signed(self.identity, 1, CID1)) + store.keep(self.name, "follow") + store.save() + rows = {name: (reasons, record) for name, reasons, record in + state_rows(self.state, "SELECT name, reasons, record FROM keepers")} + self.assertEqual(rows[other], ('["follow", "user"]', None)) # not changed here: not written + self.assertEqual(rows[self.name], ('["follow", "user"]', statedb.dump( + [field.hex() for field in store.get(self.name).wire()]))) + store.release(self.name, USER) + store.release(self.name, "follow") + store.save() + self.assertEqual(state_rows(self.state, "SELECT name FROM keepers"), [(other,)]) + + def test_failed_save_is_tried_again(self): + """A save that fails changes nothing saved; the next save writes what it would have.""" + store = self.open() + store.keep(self.name, USER) + with failing_commits(): + with self.assertRaises(OSError): + store.save() + self.assertEqual(state_rows(self.state, "SELECT name FROM keepers"), []) + self.assertEqual(store.reasons(self.name), {USER}) + store.save() + self.assertEqual(self.open().reasons(self.name), {USER}) + + def test_malformed_rows_are_dropped_and_deleted(self): + """Rows whose name, reasons or record don't parse, or with no known reason, are dropped and deleted.""" + store = self.open() + store.keep(self.name, USER) + store.save() + for row in ((b"short", '["user"]', None), ("0" * 32, '["user"]', None), (bytes([2] * 16), '["nobody"]', None), + (bytes([3] * 16), "{not json", None), (bytes([4] * 16), "5", None), + (bytes([5] * 16), '["user"]', '["zz"]'), (bytes([6] * 16), '["user"]', "7")): + state_execute(self.state, "INSERT INTO keepers (name, reasons, record) VALUES (?, ?, ?)", row) + again = self.open() + self.assertEqual(again.names(), [self.name]) + self.assertEqual(state_rows(self.state, "SELECT name FROM keepers"), [(self.name,)]) def test_bound_excludes_own_names(self): - store = KeeperStore(self.path, 1) + store = self.open(1) store.keep(bytes([1] * 16), USER) with self.assertRaises(ValueError): store.keep(bytes([2] * 16), USER) @@ -202,7 +258,7 @@ def test_bound_excludes_own_names(self): self.assertEqual(store.count(), 2) def test_release_drops_record_with_last_reason(self): - store = KeeperStore(self.path, 10) + store = self.open() store.keep(self.name, USER) store.keep(self.name, "follow") store.offer(signed(self.identity, 1, CID1)) @@ -426,11 +482,18 @@ def test_follow_swaps_pins_when_saving_fails(self): cid2 = self.publish(b"version two") self.a.records.store(cid2.key, A_ID) self.b.names.offer(self.name, 2, cid2, 0.0) - with mock.patch.object(follow, "_save", side_effect=OSError(28, "No space left on device")): + # Only the follow feature's commits fail: its pins are saved + failure = statedb.StateDbError("database or disk is full") + with mock.patch.object(follow._db, "transaction", side_effect=failure): self.assertTrue(follow.check(self.name, network=False)) pins = self.b.storage.pins() self.assertEqual(pins.get(cid2), {follow_source(self.name)}) self.assertNotIn(cid1, pins) + # The name's new target is written by the next save, whichever name it is for + state = self.b.config.state_dir + self.assertEqual(state_rows(state, "SELECT pinned FROM follow WHERE name = ?", (self.name,)), [(cid1.binary,)]) + follow.follow(bytes([1] * 16)) + self.assertEqual(state_rows(state, "SELECT pinned FROM follow WHERE name = ?", (self.name,)), [(cid2.binary,)]) def test_keeper_records_below_the_known_sequence_are_refused(self): # L28: a keeper serving an old (validly signed) record can't make this node keep and re-serve it diff --git a/tests/test_node.py b/tests/test_node.py index 25d6513..5db0442 100644 --- a/tests/test_node.py +++ b/tests/test_node.py @@ -1,6 +1,5 @@ """Tests for RnfsNode's facade and backend (rnfs/node.py), on offline nodes with a fake network client.""" import io -import json import os import random import shutil @@ -24,7 +23,7 @@ from rnfs.network.identity import parse_node_id from rnfs import fsutil, node as node_module from rnfs.node import RnfsNode -from tests.support import POSIX_MODES, wait_for +from tests.support import POSIX_MODES, failing_commits, state_rows, wait_for from tests.test_names import FakeNetwork A_ID, B_ID, N1 = bytes([0xA] * 16), bytes([0xB] * 16), bytes([1] * 16) @@ -481,11 +480,10 @@ def test_cache_records_saved_at_stop(self): cid = self.a.add_stream(io.BytesIO(b"cached"))["cid"] self.advertise(cid) b"".join(self.b.read(cid)) - path = os.path.join(self.b.config.state_dir, "cache.json") - self.assertFalse(os.path.exists(path)) # not rewritten on every fetch + state = self.b.config.state_dir + self.assertEqual(state_rows(state, "SELECT target FROM cache"), []) # not written on every fetch self.b.stop() - with open(path, encoding="utf-8") as f: - self.assertIn(cid, f.read()) + self.assertEqual(state_rows(state, "SELECT target FROM cache"), [(Cid.parse(cid).binary,)]) def test_remote_pin_quota_counts_dag_blocks(self): self.a.config.remote_pin_quota = 50_000 @@ -650,23 +648,31 @@ def test_fetched_targets_are_saved_with_the_state_and_kept_at_a_restart(self): cid = self.a.add_stream(io.BytesIO(b"fetched for the user"))["cid"] self.advertise(cid) self.b.get(cid, os.path.join(self.tmp.name, "out")) - path = os.path.join(self.b.config.state_dir, node_module.FETCHED_FILE) - self.assertFalse(os.path.exists(path)) # not written on each fetch, under the node's lock... - self.b._save_state() # ...but by the next state save (maintenance, gc and stop) - with open(path, encoding="utf-8") as f: - self.assertEqual(json.load(f), [cid]) + state = self.b.config.state_dir + self.assertEqual(state_rows(state, "SELECT cid FROM fetched"), []) # not written on each fetch, under the + self.b._save_state() # node's lock, but by the next state save (maintenance, gc and stop) + self.assertEqual(state_rows(state, "SELECT cid FROM fetched"), [(Cid.parse(cid).binary,)]) self.b = self.restart(self.b, "b", B_ID) self.assertEqual(self.b._eligible_targets(), {Cid.parse(cid)}) self.assertIn(Cid.parse(cid), self.b.publisher.provided()) def test_failed_fetched_save_is_tried_again(self): - self.b._note_fetched(Cid.for_block(RAW, b"target")) - with mock.patch.object(node_module, "write_json", side_effect=OSError("disk full")): + target = Cid.for_block(RAW, b"target") + self.b._note_fetched(target) + with failing_commits("database or disk is full"): with self.assertRaises(OSError): self.b._save_fetched() self.b._save_fetched() - with open(os.path.join(self.b.config.state_dir, node_module.FETCHED_FILE), encoding="utf-8") as f: - self.assertEqual(json.load(f), [str(Cid.for_block(RAW, b"target"))]) + self.assertEqual(state_rows(self.b.config.state_dir, "SELECT cid FROM fetched"), [(target.binary,)]) + # Only the change is written: one forgotten target is one row deleted + self.b._note_fetched(Cid.for_block(RAW, b"other")) + self.b._save_fetched() + with self.b._lock: + self.b._fetched.discard(target) + self.b._fetched_dirty = True + self.b._save_fetched() + self.assertEqual(state_rows(self.b.config.state_dir, "SELECT cid FROM fetched"), + [(Cid.for_block(RAW, b"other").binary,)]) class MakeRoomTest(NodeTest): diff --git a/tests/test_pacing.py b/tests/test_pacing.py index 004e538..0f1510c 100644 --- a/tests/test_pacing.py +++ b/tests/test_pacing.py @@ -70,10 +70,12 @@ def pace(response, wait): config = Config(rnfs_dir=self.tmp, slow="yes") storage = Storage(os.path.join(self.tmp, "blocks"), os.path.join(self.tmp, "state"), 10**9) + self.addCleanup(storage.close) requester_rate = None client = mock.Mock(rate_estimate=lambda node_id: requester_rate) fetcher = Fetcher(storage, client, config, OWN, RoutingTable(OWN, config), lambda key: None, state_dir=os.path.join(self.tmp, "state"), clock=clock) + self.addCleanup(fetcher.close) refusals = 0 with fetcher.open(built.cid, from_node=PROVIDER) as fetch: provider = fetch._providers[PROVIDER] diff --git a/tests/test_pin_names.py b/tests/test_pin_names.py index 25fbacd..a469c05 100644 --- a/tests/test_pin_names.py +++ b/tests/test_pin_names.py @@ -1,6 +1,5 @@ """`pin ls` names: where each pin came from on this machine, or what a pinned directory holds.""" import io -import json import os import tempfile import unittest @@ -10,7 +9,8 @@ from rnfs.config import Config from rnfs.content.cid import Cid from rnfs.node import RnfsNode -from rnfs.pin_names import PinNames +from rnfs.pin_names import MAX_NAME_LENGTH, PinNames +from tests.support import failing_commits, state_execute, state_rows, write_legacy class PinNamesStoreTest(unittest.TestCase): @@ -24,21 +24,62 @@ def setUp(self): self.cid = Cid.for_block("raw", b"x") self.other = Cid.for_block("raw", b"y") - def test_record_prune_and_persist(self): + def open(self) -> PinNames: + """A store on the state directory, closed when the test ends.""" names = PinNames(self.dir) + self.addCleanup(names.close) + return names + + def rows(self) -> dict: + """What the pin_names table holds: CID -> stored name text.""" + return {Cid.from_bytes(cid): name for cid, name in state_rows(self.dir, "SELECT cid, name FROM pin_names")} + + def test_record_prune_and_persist(self): + """Names are kept across a restart, a later pin add keeps the first, and pruned ones are deleted.""" + names = self.open() names.record(self.cid, "/home/u/a.txt") names.record_if_unknown(self.cid, "bafy.../a.txt") # a later pin add keeps the local path names.record(self.other, "(stdin)") - self.assertEqual(PinNames(self.dir).get(self.cid), "/home/u/a.txt") + self.assertEqual(self.open().get(self.cid), "/home/u/a.txt") names.prune({self.cid}) - again = PinNames(self.dir) + self.assertEqual(self.rows(), {self.cid: '"/home/u/a.txt"'}) + again = self.open() self.assertEqual((again.get(self.cid), again.get(self.other)), ("/home/u/a.txt", None)) + def test_names_that_are_not_unicode_round_trip(self): + """A name with a lone surrogate is saved and loaded as it was; a long one is cut to MAX_NAME_LENGTH.""" + names = self.open() + odd = "/home/u/caf\udce9" # a file name that isn't UTF-8, as os.fsdecode gives it + names.record(self.cid, odd) + names.record(self.other, "x" * (MAX_NAME_LENGTH + 10)) + again = self.open() + self.assertEqual((again.get(self.cid), again.get(self.other)), (odd, "x" * MAX_NAME_LENGTH)) + def test_damaged_file_is_ignored(self): - with open(os.path.join(self.dir, "pin_names.json"), "w", encoding="utf-8") as f: - json.dump({"not a cid": "x", str(self.cid): 5, str(self.other): "ok"}, f) - names = PinNames(self.dir) + """pin_names.json's entries that don't parse are skipped when it is imported; the rest are kept.""" + write_legacy(self.dir, "pin_names.json", {"not a cid": "x", str(self.cid): 5, str(self.other): "ok"}) + names = self.open() + self.assertEqual((names.get(self.cid), names.get(self.other)), (None, "ok")) + + def test_damaged_rows_are_deleted(self): + """Rows that don't parse are skipped at load and deleted; the others are kept.""" + self.open().record(self.other, "ok") + for cid, name in ((b"not a cid", '"x"'), (self.cid.binary, "not json"), (self.cid.binary + b"!", "5")): + state_execute(self.dir, "INSERT INTO pin_names (cid, name) VALUES (?, ?)", (cid, name)) + names = self.open() self.assertEqual((names.get(self.cid), names.get(self.other)), (None, "ok")) + self.assertEqual(self.rows(), {self.other: '"ok"'}) + + def test_failed_save_is_written_with_the_next_change(self): + """A change whose commit failed stays in memory, as before, and is saved with the next change.""" + names = self.open() + names.record(self.cid, "first") + with failing_commits(), self.assertRaises(OSError): + names.prune(set()) + self.assertIsNone(names.get(self.cid)) # forgotten in memory, as before + self.assertEqual(self.rows(), {self.cid: '"first"'}) + names.record(self.other, "second") + self.assertEqual(self.rows(), {self.other: '"second"'}) class PinLsNamesTest(unittest.TestCase): diff --git a/tests/test_providers.py b/tests/test_providers.py index 22fd46e..aaecccb 100644 --- a/tests/test_providers.py +++ b/tests/test_providers.py @@ -1,16 +1,17 @@ -"""Tests for the provider record store: TTL, republishing, withdrawal, eviction, replies and persistence.""" -import json +"""Tests for the provider record store: TTL, republishing, withdrawal, eviction, replies and persistence (the +state database's provider tables).""" import os import random import tempfile import threading import unittest +from collections.abc import Iterator +from contextlib import contextmanager from unittest import mock -from rnfs import fsutil from rnfs.constants import DAY, ID_LENGTH, K, PROVIDER_TTL from rnfs.dht.providers import MIN_PER_PUBLISHER, ProviderStore -from tests.support import FakeClock +from tests.support import FakeClock, failing_commits, state_execute, state_rows, write_legacy def ident(n: int) -> bytes: @@ -318,31 +319,102 @@ def test_reply_asks_exclude_only_about_chosen_candidates(self): self.assertLess(len(asked), 100) +class Recorder: + """A transaction's connection that records what each executemany writes or deletes.""" + + def __init__(self, tx, writes: list): + """Forward to tx, recording into writes.""" + self.tx = tx + self.writes = writes + + def execute(self, sql, params=()): + """Run one statement, recording it with its one row.""" + self.writes.append((sql, [params])) + return self.tx.execute(sql, params) + + def executemany(self, sql, rows): + """Run a statement for each row, recording them.""" + rows = list(rows) + self.writes.append((sql, rows)) + return self.tx.executemany(sql, rows) + + +@contextmanager +def recorded_writes(db, before=None) -> Iterator[list]: + """Within the with block, each transaction of the StateDb reference db appends the list of its statements, as + (sql, rows), to the list yielded; before(), if given, is called as each one is about to open.""" + transactions: list[list] = [] + real = db.transaction + + @contextmanager + def transaction(durable: bool = False): + """The real transaction, recorded.""" + if before is not None: + before() + with real(durable) as tx: + transactions.append([]) + yield Recorder(tx, transactions[-1]) + + with mock.patch.object(db, "transaction", transaction): + yield transactions + + +def rows_written(transactions: list[list]) -> int: + """How many rows the statements recorded by recorded_writes wrote or deleted.""" + return sum(len(rows) for statements in transactions for _, rows in statements) + + +def saved(state_dir: str) -> dict: + """The provider tables in state_dir, shaped as held() gives a store's state.""" + return { + "publishers": dict(state_rows(state_dir, "SELECT publisher, first_seen FROM provider_publishers")), + "records": {(key, publisher): (first, last, pooled) for key, publisher, first, last, pooled in state_rows( + state_dir, "SELECT key, publisher, first_stored, last_stored, pooled FROM provider_records")}, + "departed": {publisher: (seen, at) for publisher, seen, at in state_rows( + state_dir, "SELECT publisher, first_seen, at FROM provider_departed")}, + "evicted": {publisher: (seen, at) for publisher, seen, at in state_rows( + state_dir, "SELECT publisher, first_seen, at FROM provider_evicted")}, + } + + +def held(store: ProviderStore) -> dict: + """What store holds in memory: publishers' first-seen times, records with every field, departures and + evictions.""" + with store._lock: + return { + "publishers": {publisher: entry.first_seen for publisher, entry in store._publishers.items()}, + "records": {(key, publisher): (record.first_stored, record.last_stored, int(record.pooled)) + for key, records in store._records.items() for publisher, record in records.items()}, + "departed": dict(store._departed), + "evicted": dict(store._evicted), + } + + class PersistenceTest(unittest.TestCase): - """Records and first-seen times survive a restart; the file is replaced atomically.""" + """Records and first-seen times survive a restart; saves write only what changed, in order.""" def setUp(self): - """A temporary state directory.""" + """A temporary state directory (removed after the stores on it are closed).""" self.directory = tempfile.TemporaryDirectory() - self.path = os.path.join(self.directory.name, "state", "providers.json") + self.addCleanup(self.directory.cleanup) + self.state = os.path.join(self.directory.name, "state") self.clock = FakeClock(1_700_000_000.0) - def tearDown(self): - """Remove the state directory.""" - self.directory.cleanup() - - def open(self) -> ProviderStore: - """A store at self.path.""" - return ProviderStore(100, self.path, clock=self.clock) + def open(self, max_records: int = 100, **settings) -> ProviderStore: + """A store on self.state, closed when the test ends.""" + store = ProviderStore(max_records, self.state, clock=self.clock, **settings) + self.addCleanup(store.close) + return store def test_round_trip(self): + """Records and first-seen times are saved, and loaded by the next store.""" store = self.open() store.store(KEY, ident(1)) self.clock.now += 5 store.store(KEY, ident(2)) store.store(ident(7), ident(2)) store.save() - self.assertEqual(os.listdir(os.path.dirname(self.path)), ["providers.json"]) # no temp files left + self.assertEqual(len(state_rows(self.state, "SELECT * FROM provider_records")), 3) self.clock.now += 100 loaded = self.open() @@ -352,28 +424,34 @@ def test_round_trip(self): self.assertEqual(loaded.first_seen(ident(2)), store.first_seen(ident(2))) def test_first_seen_order_survives_restart(self): - store = ProviderStore(3, self.path, clock=self.clock) + """A publisher seen first still evicts the newest one's records after a restart.""" + store = self.open(3) store.store(ident(100), ident(1)) self.clock.now += 1 store.store(ident(101), ident(2)) store.store(ident(102), ident(2)) store.save() - loaded = ProviderStore(3, self.path, clock=self.clock) + loaded = self.open(3) self.assertTrue(loaded.store(ident(103), ident(1))) # evicts a record of ident(2), the newest self.assertEqual(loaded.record_count(), 3) self.assertEqual(len(loaded.providers(ident(101)) + loaded.providers(ident(102))), 1) def test_limits_applied_on_load_keeping_oldest(self): - store = ProviderStore(100, self.path, clock=self.clock, max_per_publisher=100) + """A smaller per-publisher limit at load keeps the oldest records; the next save deletes the rest.""" + store = self.open(max_per_publisher=100) for i in range(30): store.store(ident(100 + i), ident(1)) self.clock.now += 1 store.save() - loaded = ProviderStore(100, self.path, clock=self.clock, max_per_publisher=10) + loaded = self.open(max_per_publisher=10) self.assertEqual(loaded.record_count(), 10) self.assertEqual([len(loaded.providers(ident(100 + i))) for i in range(30)], [1] * 10 + [0] * 20) + loaded.save() + self.assertEqual(saved(self.state), held(loaded)) + self.assertEqual(len(saved(self.state)["records"]), 10) def test_expired_records_dropped_on_load(self): + """Records expired by the time of the load aren't loaded, and the next save deletes their rows.""" store = self.open() store.store(KEY, ident(1)) store.save() @@ -381,8 +459,11 @@ def test_expired_records_dropped_on_load(self): loaded = self.open() self.assertEqual(loaded.record_count(), 0) self.assertIsNone(loaded.first_seen(ident(1))) + loaded.save() + self.assertEqual(saved(self.state), {"publishers": {}, "records": {}, "departed": {}, "evicted": {}}) def test_future_times_clamped_on_load(self): + """Times saved while the clock ran ahead load as now, and are saved so.""" # Saved while the clock ran a year ahead; once corrected, records still expire a TTL from now self.clock.now += 365 * DAY store = self.open() @@ -392,40 +473,92 @@ def test_future_times_clamped_on_load(self): loaded = self.open() self.assertEqual(loaded.first_seen(ident(1)), self.clock.now) self.assertEqual(loaded.providers(KEY), [ident(1)]) + loaded.save() + self.assertEqual(saved(self.state)["records"], {(KEY, ident(1)): (self.clock.now, self.clock.now, 0)}) self.clock.now += PROVIDER_TTL self.assertEqual(loaded.expire(), 1) - def test_out_of_range_numbers_start_empty(self): - os.makedirs(os.path.dirname(self.path)) - with open(self.path, "w", encoding="utf-8") as f: - json.dump({"version": 1, "publishers": {ident(1).hex(): 10 ** 400}, "records": []}, f) + def test_times_clamped_at_expiry_are_saved(self): + """Times expire() brings back to now (records, first-seen times, departures) are saved so.""" + store = self.open(seniority=DAY) + store.store(KEY, ident(1)) + self.clock.now += 2 * DAY + store.store(ident(7), ident(2)) + store.withdraw(KEY, ident(1)) # departs, remembered + store.save() + self.clock.now -= 3 * DAY + store.expire() + with recorded_writes(store._db) as transactions: + store.save() + self.assertEqual(saved(self.state), held(store)) + now = self.clock.now + self.assertEqual(saved(self.state)["records"], {(ident(7), ident(2)): (now, now, 0)}) + self.assertEqual(saved(self.state)["departed"], {ident(1): (now, now)}) + # The record, publisher and departure, the departure deleted from both its tables first + self.assertEqual(rows_written(transactions), 5) + + def test_malformed_rows_are_ignored_and_deleted(self): + """Rows of the wrong type or length, or of a publisher without a row, load as nothing; the next save + deletes them, keeping the rest.""" + store = self.open(seniority=DAY) + store.store(KEY, ident(1)) + store.store(KEY, ident(2)) + self.clock.now += 2 * DAY + store.withdraw(KEY, ident(2)) # departs, remembered + store.save() + store.close() + records = ("INSERT INTO provider_records (key, publisher, first_stored, last_stored, pooled)" + " VALUES (?, ?, ?, ?, ?)") + now = self.clock.now + for sql, row in ((records, (b"short", ident(1), now, now, 0)), + (records, ("text key 16 long", ident(1), now, now, 0)), + (records, (ident(8), ident(1), "yesterday", now, 0)), + (records, (ident(9), ident(1), now, now, "pooled")), + (records, (ident(10), ident(3), now, now, 0)), # ident(3) has no publisher row + ("INSERT INTO provider_publishers (publisher, first_seen) VALUES (?, ?)", (ident(4), "x")), + ("INSERT INTO provider_publishers (publisher, first_seen) VALUES (?, ?)", (b"x", now)), + ("INSERT INTO provider_departed (publisher, first_seen, at) VALUES (?, ?, ?)", (b"y", 1, now)), + ("INSERT INTO provider_evicted (publisher, first_seen, at) VALUES (?, ?, ?)", + (ident(5), now, "x"))): + state_execute(self.state, sql, row) + loaded = self.open(seniority=DAY) + self.assertEqual(loaded.providers(KEY), [ident(1)]) + self.assertEqual(loaded.record_count(), 1) + self.assertEqual(loaded.allowance(ident(2)), loaded.max_per_publisher) + loaded.save() + self.assertEqual(saved(self.state), held(loaded)) + self.assertEqual(len(saved(self.state)["records"]), 1) + + def test_legacy_file_with_out_of_range_numbers_is_set_aside(self): + """providers.json with a number no float holds (which made rnfs 0.1.4 start empty) is moved aside.""" + write_legacy(self.state, "providers.json", + {"version": 1, "publishers": {ident(1).hex(): 10 ** 400}, "records": []}) self.assertEqual(self.open().record_count(), 0) + self.assertTrue(os.path.exists(os.path.join(self.state, "providers.json.damaged"))) def test_saves_never_land_out_of_order(self): - # A save that snapshots while another is still writing must not be overwritten by the older snapshot + """A save that snapshots while another is still writing doesn't land first, overwritten by the older.""" store = self.open() store.store(KEY, ident(1)) - real_replace = fsutil.replace second = [] - def slow_replace(source, target): - """The first save's rename: meanwhile another thread stores and saves.""" + def refresh_and_save(): + """Before the first save writes: refresh its record, and save from another thread.""" if not second: - store.store(ident(7), ident(2)) + self.clock.now += 5 + store.store(KEY, ident(1)) second.append(threading.Thread(target=store.save)) second[0].start() second[0].join(0.2) - real_replace(source, target) - with mock.patch.object(fsutil, "replace", slow_replace): + with recorded_writes(store._db, before=refresh_and_save): store.save() second[0].join(5) - with open(self.path, encoding="utf-8") as f: - self.assertEqual(len(json.load(f)["records"]), 2) + self.assertEqual(saved(self.state)["records"], {(KEY, ident(1)): (self.clock.now - 5, self.clock.now, 0)}) - def test_save_writes_the_whole_state(self): - """The file has every publisher, record (pooled or not), departure and eviction, in the store's order.""" - store = ProviderStore(20, self.path, clock=self.clock, seniority=DAY) + def test_saved_tables_are_the_whole_state(self): + """The tables hold every publisher, record (pooled or not), departure and eviction, as memory does.""" + store = self.open(20, seniority=DAY) for publisher in (1, 2): for i in range(9): store.store(ident(100 * publisher + i), ident(publisher)) @@ -437,44 +570,109 @@ def test_save_writes_the_whole_state(self): store.withdraw(ident(100), ident(1)) self.clock.now += DAY store.withdraw(ident(1005), ident(5)) # 5 departs, seen for long enough to be remembered - with store._lock: - expected = { - "version": 1, - "publishers": {p.hex(): e.first_seen for p, e in store._publishers.items()}, - "records": [[k.hex(), p.hex(), r.first_stored, r.last_stored, int(r.pooled)] - for k, held in store._records.items() for p, r in held.items()], - "departed": [[p.hex(), seen, left] for p, (seen, left) in store._departed.items()], - "evicted": [[p.hex(), seen, left] for p, (seen, left) in store._evicted.items()], - } - self.assertTrue(all(expected[section] for section in ("departed", "evicted"))) - self.assertIn(1, [record[4] for record in expected["records"]]) + expected = held(store) + self.assertTrue(expected["departed"] and expected["evicted"]) + self.assertIn(1, [pooled for _, _, pooled in expected["records"].values()]) store.save() - with open(self.path, "rb") as f: - self.assertEqual(f.read(), json.dumps(expected, separators=(",", ":")).encode()) + self.assertEqual(saved(self.state), expected) + + def test_save_writes_only_what_changed(self): + """With a thousand records saved, refreshing one and withdrawing another writes two rows.""" + store = self.open(2000) + for i in range(1000): + store.store(ident(100 + i), ident(1 + i % 10)) + store.save() + self.clock.now += 10 + store.store(ident(100), ident(1)) + store.withdraw(ident(101), ident(2)) + with recorded_writes(store._db) as transactions: + store.save() + self.assertEqual(len(transactions), 1) + self.assertEqual(rows_written(transactions), 2) + self.assertEqual(saved(self.state), held(store)) def test_failed_save_is_tried_again(self): + """A save whose commit fails raises, and the next save writes what it held.""" store = self.open() store.store(KEY, ident(1)) - with mock.patch.object(fsutil, "write_atomic", side_effect=OSError("disk full")): + with failing_commits(): with self.assertRaises(OSError): store.save() store.save() self.assertEqual(self.open().providers(KEY), [ident(1)]) - def test_corrupt_file_starts_empty(self): - os.makedirs(os.path.dirname(self.path)) - with open(self.path, "w", encoding="utf-8") as f: - f.write("{not json") + def test_corrupt_legacy_file_starts_empty(self): + """providers.json that isn't JSON is moved aside, and the store starts empty.""" + write_legacy(self.state, "providers.json", text="{not json") self.assertEqual(self.open().record_count(), 0) + self.assertTrue(os.path.exists(os.path.join(self.state, "providers.json.damaged"))) def test_save_only_when_changed(self): + """A save with nothing changed opens no transaction.""" store = self.open() - store.save() - self.assertFalse(os.path.exists(self.path)) - store.store(KEY, ident(1)) - store.save() - with open(self.path, encoding="utf-8") as f: - self.assertEqual(len(json.load(f)["records"]), 1) + with recorded_writes(store._db) as transactions: + store.save() + self.assertEqual(transactions, []) + store.store(KEY, ident(1)) + store.save() + store.save() + self.assertEqual(len(transactions), 1) + self.assertEqual(len(saved(self.state)["records"]), 1) + + def test_random_operations_are_saved_and_loaded_exactly(self): + """Many random stores, withdrawals, batches, expiries, retains and saves (some failing) with every limit + reached: a store loaded from the tables holds exactly what the first one holds, and so do the tables.""" + rng = random.Random(7) + own = ident(1) + settings = {"max_per_key": 6, "own_id": own, "seniority": DAY, "rng": random.Random(1)} + store = self.open(60, **settings) + publishers = [ident(1 + n) for n in range(40)] + keys = [ident(1000 + n) for n in range(30)] + + def check(): + """Expire, save, and load another store from the tables: it, and they, hold what store does.""" + store.expire() + store.save() + loaded = ProviderStore(60, self.state, clock=self.clock, **settings) + try: + self.assertEqual(held(loaded), held(store)) + self.assertEqual(saved(self.state), held(store)) + self.assertFalse(loaded._dirty_records or loaded._dirty_publishers or loaded._dirty_gone) + finally: + loaded.close() + + failed = 0 + for _ in range(4000): + action = rng.random() + publisher = rng.choice(publishers) + if action < 0.4: + store.store(rng.choice(keys), publisher) + elif action < 0.55: + store.withdraw(rng.choice(keys), publisher) + elif action < 0.65: + store.provide(publisher, rng.sample(keys, rng.randrange(1, 12)), rng.sample(keys, rng.randrange(3))) + elif action < 0.8: + self.clock.now += rng.choice((1, 60, 3600)) + elif action < 0.83: + self.clock.now += rng.choice((DAY, 3 * DAY, PROVIDER_TTL)) + store.expire() + elif action < 0.85: + banned = set(rng.sample(publishers, 3)) + store.retain(lambda p: p not in banned) + elif action < 0.95: + store.save() + elif action < 0.97: + with failing_commits(): + try: + store.save() + except OSError: + failed += 1 + elif action < 0.98: + check() + self.assertGreater(failed, 10) + self.assertGreater(store._count, 0) + self.assertTrue(store._departed or store._evicted) + check() class ThreadSafetyTest(StoreTestCase): diff --git a/tests/test_publisher.py b/tests/test_publisher.py index c084299..b44763e 100644 --- a/tests/test_publisher.py +++ b/tests/test_publisher.py @@ -1,10 +1,8 @@ """Tests for the provide strategy, batched publishing, republishing and withdrawal (rnfs/publisher.py).""" -import json import os import tempfile import threading import unittest -from unittest import mock import RNS @@ -14,9 +12,9 @@ from rnfs.content.cid import RAW, Cid from rnfs.content.storage import USER, Storage from rnfs.dht.providers import ProviderStore -from rnfs import publisher as publisher_module -from rnfs.publisher import SLOW_LOOKUP_PACE, SLOW_ROUND_LOOKUPS, STATE_FILE, DhtKey, Publisher -from tests.support import FakeClock +from rnfs.publisher import SLOW_LOOKUP_PACE, SLOW_ROUND_LOOKUPS, DhtKey, Publisher +from tests.support import FakeClock, failing_commits, state_execute, state_rows, write_legacy +from tests.test_providers import recorded_writes, rows_written OWN = bytes(16) N1, N2, N3 = (bytes([i]) * 16 for i in (1, 2, 3)) @@ -73,6 +71,7 @@ def setUp(self): self.state = os.path.join(self.tmp.name, "state") self.config = Config(rnfs_dir=self.tmp.name, republish_interval=1000) self.storage = Storage(os.path.join(self.tmp.name, "blocks"), self.state, 10**9) + self.addCleanup(self.storage.close) self.records = ProviderStore(1000, None) self.client = FakeClient() self.clock = FakeClock(1_000_000.0) @@ -82,15 +81,39 @@ def setUp(self): self.on_lookup = lambda key: None self.publisher = self.make_publisher() - def make_publisher(self, slow: bool = False) -> Publisher: - """A publisher sharing this test's state directory.""" + def make_publisher(self, slow: bool = False, state: str | None = None) -> Publisher: + """A publisher sharing this test's state directory (or on state), closed when the test ends.""" def make_lookup(key): """Remember the key; find self.nodes.""" self.lookups.append(key) self.on_lookup(key) return FakeLookup(self.nodes) - return Publisher(OWN, self.config, self.storage, self.records, self.client, make_lookup, slow, self.state, - cached_eligible=lambda: self.eligible, clock=self.clock, retry_interval=30) + publisher = Publisher(OWN, self.config, self.storage, self.records, self.client, make_lookup, slow, + state or self.state, cached_eligible=lambda: self.eligible, clock=self.clock, + retry_interval=30) + self.addCleanup(publisher.close) + return publisher + + def saved(self) -> dict: + """The publisher's tables: {"published": {item: (at, nodes)}, "earlier": {(item, node): at}, + "withdraw": {node: {key, ...}}}.""" + withdraw: dict[bytes, set[bytes]] = {} + for node, key in state_rows(self.state, "SELECT node, key FROM withdrawals"): + withdraw.setdefault(node, set()).add(key) + return {"published": {item: (at, nodes) for item, at, nodes in + state_rows(self.state, "SELECT item, at, nodes FROM published")}, + "earlier": {(item, node): at for item, node, at in + state_rows(self.state, "SELECT item, node, at FROM published_earlier")}, + "withdraw": withdraw} + + def held(self, publisher: Publisher) -> dict: + """What publisher holds in memory, shaped as saved() reads the tables.""" + with publisher._lock: + return {"published": {str(item): (at, b"".join(nodes)) + for item, (at, nodes) in publisher._published.items()}, + "earlier": {(str(item), node): at for item, nodes in publisher._earlier.items() + for node, at in nodes.items()}, + "withdraw": {node: set(keys) for node, keys in publisher._withdrawals.items() if keys}} def withdrawn(self) -> dict[bytes, set[bytes]]: """The keys withdrawn from each node by the calls made so far.""" @@ -340,17 +363,71 @@ def test_slow_pacing_clamped_after_the_clock_goes_back(self): publisher.refresh() self.assertEqual(publisher.next_due(), SLOW_LOOKUP_PACE * SLOW_ROUND_LOOKUPS) # at most a full round's wait - def test_state_of_the_wrong_shape_is_ignored(self): + def test_legacy_state_of_the_wrong_shape_is_ignored(self): + """published.json of the wrong shape is moved aside, and entries of the wrong shape (or with node IDs or + keys not ID_LENGTH bytes) are skipped: nothing is imported, so the CID is due.""" cid = self.pinned(b"shape") - for value in ([], {"published": []}, {"published": {str(cid): 5}, "withdraw": []}, - {"published": {str(cid): [10**400, []]}}, {"published": {str(cid): [1, "0101"]}}, - {"earlier": {str(cid): []}, "withdraw": {N1.hex(): "ab"}}): + for n, value in enumerate(([], {"published": []}, {"published": {str(cid): 5}, "withdraw": []}, + {"published": {str(cid): [10**400, []]}}, {"published": {str(cid): [1, "0101"]}}, + {"published": {str(cid): [1, ["0101"]]}}, + {"earlier": {str(cid): []}, "withdraw": {N1.hex(): "ab"}}, + {"earlier": {str(cid): {"01": 1}}, "withdraw": {N1.hex(): ["ab"]}})): with self.subTest(value=value): - with open(os.path.join(self.state, STATE_FILE), "w", encoding="utf-8") as f: - json.dump(value, f) - publisher = self.make_publisher() + state = os.path.join(self.tmp.name, f"legacy {n}") # each its own: a file is imported once + path = write_legacy(state, "published.json", value) + publisher = self.make_publisher(state=state) publisher.refresh() self.assertEqual(publisher.next_due(), 0) # nothing loaded: due + self.assertFalse(os.path.exists(path)) + for table in ("published", "published_earlier", "withdrawals"): + self.assertEqual(state_rows(state, f"SELECT * FROM {table}"), []) + publisher.close() + + def test_malformed_rows_are_skipped_and_deleted(self): + """Rows that don't parse are skipped, the rest loaded; the load deletes those whose item doesn't parse, + and the next save the others, so the tables hold what memory does.""" + a, b = self.pinned(b"a"), self.pinned(b"b") + self.publisher.refresh() + self.publisher.publish_due() + self.publisher.close() + rows = [("INSERT INTO published (item, at, nodes) VALUES (?, ?, ?)", ("not an item", 1, N1)), + ("INSERT OR REPLACE INTO published (item, at, nodes) VALUES (?, ?, ?)", (str(b), "yesterday", N1)), + ("INSERT INTO published (item, at, nodes) VALUES (?, ?, ?)", ("key:abcd", 1, N1)), # 2 bytes + ("INSERT INTO published_earlier (item, node, at) VALUES (?, ?, ?)", (str(a), b"short", 1)), + ("INSERT INTO published_earlier (item, node, at) VALUES (?, ?, ?)", (str(a), N3, "x")), + ("INSERT INTO published_earlier (item, node, at) VALUES (?, ?, ?)", ("bad", N3, 1)), + ("INSERT INTO withdrawals (node, key) VALUES (?, ?)", (b"short", a.key)), + ("INSERT INTO withdrawals (node, key) VALUES (?, ?)", (N3, "text"))] + for sql, row in rows: + state_execute(self.state, sql, row) + publisher = self.make_publisher() + publisher.refresh() + self.assertEqual(set(publisher._published), {a}) # b's row was replaced by the malformed one + self.assertEqual(publisher._withdrawals, {}) + self.assertEqual(self.saved()["published"].keys(), {str(a), str(b)}) # parsed items wait for the save + publisher.save() + self.assertEqual(self.saved(), self.held(publisher)) + + def test_times_clamped_and_earlier_nodes_expired_at_load_are_saved(self): + """A last-published time in the future loads as now, and earlier nodes past PROVIDER_TTL are dropped; + the next save writes both.""" + cid = self.pinned(b"ahead") + self.publisher.refresh() + self.clock.now += 10**6 + self.publisher.publish_due() + self.nodes = [N3] + self.clock.now += 1000 + self.publisher.publish_due() # N1, N2 earlier now + self.clock.now -= 10**6 + restarted = self.make_publisher() + self.assertEqual(restarted._published[cid][0], self.clock.now) + restarted.save() + self.assertEqual(self.saved()["published"], {str(cid): (self.clock.now, N3)}) + self.clock.now += 10**6 + PROVIDER_TTL + restarted = self.make_publisher() + self.assertEqual(restarted._earlier, {}) + restarted.save() + self.assertEqual(self.saved()["earlier"], {}) def test_served_blocks_are_the_provided_dags(self): cid = self.pinned(os.urandom(200_000)) @@ -424,14 +501,13 @@ def test_refresh_withdraws_what_isnt_provided_when_the_sets_are_unchanged(self): self.assertEqual(self.withdrawn(), {N1: {cid.key}, N2: {cid.key}}) def test_saved_without_the_lock_and_in_order(self): - """published.json is written outside the lock, and a save that snapshots meanwhile lands last.""" + """The publish state is written outside the lock, and a save that snapshots meanwhile lands last.""" a, b = self.pinned(b"a"), self.pinned(b"b") self.publisher.refresh() - real_write = publisher_module.write_json free, second = [], [] - def write(path, value): - """The first save's write: another thread takes the lock, then unpins b and saves.""" + def unpin_and_save(): + """Before the first save writes: another thread takes the lock, then unpins b and saves.""" if not second: lock = self.publisher._lock thread = threading.Thread(target=lambda: free.append(lock.acquire(timeout=5)) or lock.release()) @@ -441,16 +517,45 @@ def write(path, value): second.append(threading.Thread(target=self.publisher.refresh)) second[0].start() second[0].join(0.2) - real_write(path, value) - with mock.patch.object(publisher_module, "write_json", write): + with recorded_writes(self.publisher._db, before=unpin_and_save): self.publisher.publish_due() second[0].join(5) self.assertEqual(free, [True]) - with open(os.path.join(self.state, STATE_FILE), encoding="utf-8") as f: - saved = json.load(f) + saved = self.saved() self.assertEqual(set(saved["published"]), {str(a)}) - self.assertEqual(saved["withdraw"], {N1.hex(): [b.key.hex()], N2.hex(): [b.key.hex()]}) + self.assertEqual(saved["withdraw"], {N1: {b.key}, N2: {b.key}}) + + def test_save_writes_only_what_changed(self): + """With many CIDs published, unpinning one writes its rows and its nodes' withdrawals only.""" + cids = [self.pinned(b"cid %d" % i) for i in range(50)] + self.publisher.refresh() + self.publisher.publish_due() + self.assertEqual(self.saved(), self.held(self.publisher)) + with recorded_writes(self.publisher._db) as transactions: + self.storage.unpin(cids[0], USER) + self.publisher.refresh() + self.assertEqual(len(transactions), 1) + # Its published row and earlier rows deleted (one item: 2), N1's and N2's withdrawals deleted and written (4) + self.assertEqual(rows_written(transactions), 6) + self.assertEqual(self.saved(), self.held(self.publisher)) + with recorded_writes(self.publisher._db) as transactions: + self.publisher.save() + self.assertEqual(transactions, []) # nothing changed since + + def test_failed_save_is_written_next_time(self): + """A round whose save fails raises (the loop logs it); the next save writes what it held.""" + cid = self.pinned(b"failing disk") + self.publisher.refresh() + with failing_commits(): + with self.assertRaises(OSError): + self.publisher.publish_due() + self.assertEqual(self.saved()["published"], {}) + self.publisher.save() + self.assertEqual(set(self.saved()["published"]), {str(cid)}) + restarted = self.make_publisher() + restarted.refresh() + self.assertEqual(restarted.publish_due()["published"], []) if __name__ == "__main__": diff --git a/tests/test_shared_folder.py b/tests/test_shared_folder.py index d489470..2bddede 100644 --- a/tests/test_shared_folder.py +++ b/tests/test_shared_folder.py @@ -4,12 +4,12 @@ import unittest from unittest import mock -from rnfs import fsutil +from rnfs import fsutil, statedb from rnfs.content import builder, shared_folder from rnfs.content.cid import RAW, Cid from rnfs.content.shared_folder import SharedFolder from rnfs.content.storage import FOLDER, USER, Storage -from tests.support import CAN_SYMLINK, HAS_FIFO, POSIX_MODES +from tests.support import CAN_SYMLINK, HAS_FIFO, POSIX_MODES, state_execute, state_rows, write_legacy from tests.test_builder import swap_on_open, within SECRET = b"private key" @@ -23,12 +23,21 @@ def setUp(self): self.tmp = tempfile.TemporaryDirectory() self.addCleanup(self.tmp.cleanup) self.files = os.path.join(self.tmp.name, "files") - self.storage = Storage(os.path.join(self.tmp.name, "blocks"), os.path.join(self.tmp.name, "state"), 10**9) + self.state = os.path.join(self.tmp.name, "state") + self.storage = Storage(os.path.join(self.tmp.name, "blocks"), self.state, 10**9) + self.addCleanup(self.storage.close) self.folder = self.open() def open(self) -> SharedFolder: - """A SharedFolder with its state under the temporary directory.""" - return SharedFolder(self.files, self.storage, os.path.join(self.tmp.name, "state")) + """A SharedFolder with its state under the temporary directory, closed when the test ends.""" + folder = SharedFolder(self.files, self.storage, self.state) + self.addCleanup(folder.close) + return folder + + def rows(self) -> dict: + """What the shared_folder table holds: entry name -> (signature, CID).""" + return {statedb.load(name): (sig, Cid.from_bytes(cid)) + for name, sig, cid in state_rows(self.state, "SELECT name, signature, cid FROM shared_folder")} def write(self, relative: str, data: bytes) -> str: """Write a file in the folder, creating directories.""" @@ -43,11 +52,66 @@ def cid_of(self, relative: str): return builder.add_path(os.path.join(self.files, relative), builder.NO_PUT)[0].cid def test_state_of_the_wrong_shape_is_moved_aside(self): - path = self.folder._state_path - for content in ('["a"]', '{"entries": ["a"]}', '{"entries": {"a": {"0": 1}, "b": 5}}'): - with open(path, "w", encoding="utf-8") as f: - f.write(content) - self.assertEqual(self.open().names(), {}, content) # records that don't parse are skipped too + """A shared_folder.json of the wrong shape is moved aside when imported; records that don't parse are + skipped.""" + for content, damaged in (('["a"]', True), ('{"entries": ["a"]}', True), + ('{"entries": {"a": {"0": 1}, "b": 5, "c": ["sig", "not a cid"]}}', False)): + path = write_legacy(self.state, "shared_folder.json", text=content) + self.assertEqual(self.open().names(), {}, content) + self.assertEqual(os.path.exists(path + ".damaged"), damaged, content) + if damaged: + os.remove(path + ".damaged") + + def test_entries_are_saved_as_they_change(self): + """A scan writes the entries it changed and deletes the removed ones; a store without a state + directory keeps nothing.""" + path = self.write("a.txt", b"a") + self.write("b.txt", b"b") + self.folder.scan() + self.assertEqual(self.rows(), self.folder._known) + self.assertEqual(set(self.rows()), {"a.txt", "b.txt"}) + os.remove(path) + self.folder.scan() + self.assertEqual(set(self.rows()), {"b.txt"}) + unsaved = SharedFolder(os.path.join(self.tmp.name, "other"), self.storage) + self.addCleanup(unsaved.close) + self.write(os.path.join("..", "other", "c.txt"), b"c") + self.assertEqual(len(unsaved.scan()), 1) + self.assertEqual(set(self.rows()), {"b.txt"}) + + def test_names_that_are_not_unicode_round_trip(self): + """An entry name with a lone surrogate (a file name that isn't UTF-8) is saved and loaded as it was.""" + odd = "caf\udce9" + cid = Cid.for_block(RAW, b"x") + with self.folder._lock: + self.folder._known = {odd: ("sig", cid)} + self.folder._save() + self.assertEqual(self.open().names(), {cid: odd}) + + def test_damaged_rows_are_deleted(self): + """Rows that don't parse are skipped at load and deleted; the others are kept.""" + self.write("a.txt", b"a") + self.folder.scan() + cid = self.cid_of("a.txt").binary + for name, sig, binary in (("not json", "sig", cid), ('"b"', b"sig", cid), ('"c"', "sig", b"not a cid"), + ("7", "sig", cid)): + state_execute(self.state, "INSERT INTO shared_folder (name, signature, cid) VALUES (?, ?, ?)", + (name, sig, binary)) + self.assertEqual(self.open().names(), {self.cid_of("a.txt"): "a.txt"}) + self.assertEqual(set(self.rows()), {"a.txt"}) + + def test_failed_save_is_written_with_the_next_change(self): + """A scan whose commit failed raises, keeps what it found in memory, as before, and the next scan that + changes something saves it too.""" + self.write("a.txt", b"a") + with mock.patch.object(self.folder._db, "transaction", side_effect=statedb.StateDbError("disk I/O error")): + with self.assertRaises(OSError): + self.folder.scan() + self.assertEqual(self.folder.names(), {self.cid_of("a.txt"): "a.txt"}) + self.assertEqual(self.rows(), {}) + self.write("b.txt", b"b") + self.folder.scan() + self.assertEqual(set(self.rows()), {"a.txt", "b.txt"}) def test_creates_missing_directory(self): self.assertEqual(self.folder.scan(), set()) diff --git a/tests/test_statedb.py b/tests/test_statedb.py new file mode 100644 index 0000000..08cd389 --- /dev/null +++ b/tests/test_statedb.py @@ -0,0 +1,433 @@ +"""Tests for the state database (rnfs.statedb): opening and checking it, transactions, references, and the +migration of the JSON state files earlier versions wrote.""" +import builtins +import json +import os +import sqlite3 +import tempfile +import threading +import unittest +from unittest import mock + +import RNS + +from rnfs import statedb +from rnfs.errors import RnfsError +from tests.support import POSIX_MODES, failing_commits, state_execute, state_rows, write_legacy + +_saved_loglevel = RNS.loglevel + + +def setUpModule(): + """Silence the log while these tests run.""" + RNS.loglevel = RNS.LOG_CRITICAL + + +def tearDownModule(): + """Restore RNS's loglevel.""" + RNS.loglevel = _saved_loglevel + + +class StateDbTest(unittest.TestCase): + """A state directory under a temporary directory.""" + + def setUp(self): + """An empty temporary directory; state goes in /state.""" + self.tmp = tempfile.TemporaryDirectory() + self.addCleanup(self.tmp.cleanup) + self.state = os.path.join(self.tmp.name, "state") + self.path = os.path.join(self.state, statedb.DB_FILE) + + def connect(self, legacy=None) -> statedb.StateDb: + """A reference to the state database (closed before the directory is removed); with legacy, only those + files are migrated.""" + if legacy is None: + db = statedb.connect(self.state) + else: + with mock.patch.object(statedb, "LEGACY", legacy): + db = statedb.connect(self.state) + self.addCleanup(db.close) + return db + + +class OpenTest(StateDbTest): + """Opening, checking and upgrading the database.""" + + def test_new_database_has_the_whole_schema_in_wal_mode(self): + """A new database is created with every schema step applied and journal_mode WAL.""" + db = self.connect() + self.assertEqual(db.query("PRAGMA user_version"), [(len(statedb.SCHEMA),)]) + self.assertEqual(db.query("PRAGMA journal_mode"), [("wal",)]) + tables = {name for (name,) in db.query("SELECT name FROM sqlite_master WHERE type = 'table'")} + self.assertLessEqual({"migrated", "meta", "pins", "cache", "lxmf_messages", "name_floor"}, tables) + + @unittest.skipUnless(POSIX_MODES, "Windows has no POSIX permission bits") + def test_files_are_private(self): + """rnfs.db and its -wal and -shm files are readable by their owner only, even under a lax umask; an + existing database others could read is tightened.""" + old = os.umask(0o022) + try: + db = self.connect() + with db.transaction() as tx: + tx.execute("INSERT INTO pins (cid, source) VALUES (?, ?)", (b"x", "user")) + finally: + os.umask(old) + for name in ("", "-wal", "-shm"): + self.assertEqual(os.stat(self.path + name).st_mode & 0o777, 0o600, name) + db.close() + os.chmod(self.path, 0o644) + self.connect() + self.assertEqual(os.stat(self.path).st_mode & 0o777, 0o600) + + def test_newer_schema_stops_startup(self): + """A database written by a newer rnfs (more schema steps than this code knows) is refused and kept.""" + self.connect().close() + conn = sqlite3.connect(self.path) + conn.execute(f"PRAGMA user_version = {len(statedb.SCHEMA) + 1}") + conn.close() + with self.assertRaises(RnfsError) as caught: + statedb.connect(self.state) + self.assertIn("newer version of rnfs", str(caught.exception)) + self.assertEqual(state_rows(self.state, "PRAGMA user_version"), [(len(statedb.SCHEMA) + 1,)]) + self.assertNotIn(self.path, statedb.open_databases()) + + def test_older_schema_is_upgraded_step_by_step(self): + """A database at an earlier schema step gets the steps it lacks, and keeps its rows.""" + with mock.patch.object(statedb, "SCHEMA", statedb.SCHEMA[:2]): + db = statedb.connect(self.state) + with db.transaction() as tx: + tx.execute("INSERT INTO pins (cid, source) VALUES (?, ?)", (b"x", "user")) + db.close() + db = self.connect() + self.assertEqual(db.query("PRAGMA user_version"), [(len(statedb.SCHEMA),)]) + self.assertEqual(db.query("SELECT cid, source FROM pins"), [(b"x", "user")]) + self.assertEqual(db.query("SELECT count(*) FROM lxmf_messages"), [(0,)]) + + def test_file_that_isnt_a_database_stops_startup(self): + """A file that isn't an SQLite database is refused with what to do, and left as it was.""" + os.makedirs(self.state) + garbage = b"not a database " * 1000 + with open(self.path, "wb") as f: + f.write(garbage) + with self.assertRaises(RnfsError) as caught: + statedb.connect(self.state) + self.assertIn("is damaged", str(caught.exception)) + self.assertIn(".migrated", str(caught.exception)) + with open(self.path, "rb") as f: + self.assertEqual(f.read(), garbage) + self.assertNotIn(self.path, statedb.open_databases()) + + def test_damaged_pages_stop_startup(self): + """A database whose pages were damaged (quick_check fails, or SQLite finds it malformed) is refused.""" + db = self.connect() + with db.transaction() as tx: + tx.executemany("INSERT INTO pins (cid, source) VALUES (?, ?)", + [(os.urandom(34), "user") for _ in range(2000)]) + db.close() + size = os.path.getsize(self.path) + with open(self.path, "r+b") as f: + for offset in range(4096 * 3, size - 4096, 4096 * 2): + f.seek(offset) + f.write(b"\xff" * 64) # b-tree page headers of the pins table + with self.assertRaises(RnfsError) as caught: + statedb.connect(self.state) + self.assertIn("is damaged", str(caught.exception)) + self.assertNotIn(self.path, statedb.open_databases()) + + @unittest.skipUnless(POSIX_MODES and hasattr(os, "getuid") and os.getuid() != 0, "needs permission checks") + def test_unwritable_directory_isnt_called_damage(self): + """A database that can't be opened for writing (a read-only state directory) stops startup with what + failed, not as damaged with advice to start over, and is left as it was.""" + db = self.connect() + with db.transaction() as tx: + tx.execute("INSERT INTO pins (cid, source) VALUES (?, 'user')", (b"kept",)) + db.close() + os.chmod(self.state, 0o500) + try: + with self.assertRaises(RnfsError) as caught: + statedb.connect(self.state) + finally: + os.chmod(self.state, 0o700) + self.assertNotIn("damaged", str(caught.exception)) + self.assertIn("could not open the state database", str(caught.exception)) + self.assertEqual(state_rows(self.state, "SELECT cid FROM pins"), [(b"kept",)]) + + def test_python_without_sqlite_gets_a_clear_error(self): + """A Python built without _sqlite3 stops startup with a message saying so.""" + real_import = builtins.__import__ + + def no_sqlite(name, *args, **kwargs): + """Fail to import sqlite3, as such a Python does.""" + if name == "sqlite3": + raise ModuleNotFoundError("No module named '_sqlite3'") + return real_import(name, *args, **kwargs) + + with mock.patch.object(builtins, "__import__", no_sqlite): + with self.assertRaises(RnfsError) as caught: + statedb.connect(self.state) + self.assertIn("sqlite3", str(caught.exception)) + + +class TransactionTest(StateDbTest): + """Transactions and references.""" + + def test_durable_transactions_commit_with_synchronous_full(self): + """durable=True commits with synchronous=FULL, else NORMAL.""" + db = self.connect() + for durable, level in ((True, 2), (False, 1), (True, 2)): + with db.transaction(durable=durable) as tx: + self.assertEqual(tx.execute("PRAGMA synchronous").fetchone(), (level,)) + + def test_failed_commit_changes_nothing(self): + """A commit that fails (a full disk) raises StateDbError, an OSError, and rolls everything back.""" + db = self.connect() + with failing_commits("database or disk is full"): + with self.assertRaises(OSError) as caught: + with db.transaction(durable=True) as tx: + tx.execute("INSERT INTO pins (cid, source) VALUES (?, ?)", (b"x", "user")) + self.assertIsInstance(caught.exception, statedb.StateDbError) + self.assertEqual(db.query("SELECT * FROM pins"), []) + with db.transaction() as tx: # usable again afterwards + tx.execute("INSERT INTO pins (cid, source) VALUES (?, ?)", (b"y", "user")) + self.assertEqual(state_rows(self.state, "SELECT cid FROM pins"), [(b"y",)]) + + def test_error_in_the_body_rolls_back_and_is_raised_as_is(self): + """An exception inside the with block rolls the transaction back and propagates unchanged.""" + db = self.connect() + with self.assertRaises(KeyError): + with db.transaction() as tx: + tx.execute("INSERT INTO pins (cid, source) VALUES (?, ?)", (b"x", "user")) + raise KeyError("stop") + with self.assertRaises(statedb.StateDbError): + with db.transaction() as tx: + tx.execute("INSERT INTO no_such_table VALUES (1)") + self.assertEqual(db.query("SELECT * FROM pins"), []) + + def test_values_sqlite_cant_store_are_refused_without_a_change(self): + """A lone surrogate in plain text, or an integer past 64 bits, is refused as a failed save is, with + StateDbError (stores dump() such text); inside the transaction it is the binding error.""" + db = self.connect() + for row, error in (((b"x", "name:\udc80"), UnicodeEncodeError), ((b"x", 2 ** 64), OverflowError)): + with self.subTest(error=error.__name__), self.assertRaises(statedb.StateDbError) as caught: + with db.transaction() as tx: + tx.execute("INSERT INTO pins (cid, source) VALUES (?, ?)", row) + self.assertIsInstance(caught.exception.__cause__, error) + self.assertEqual(db.query("SELECT * FROM pins"), []) + + def test_transaction_inside_a_transaction_is_an_error(self): + """The lock isn't reentrant: a transaction or read inside one raises rather than deadlocking.""" + db = self.connect() + with self.assertRaises(RuntimeError): + with db.transaction(): + db.query("SELECT 1") + with self.assertRaises(RuntimeError): + with db.transaction(): + with db.transaction(): + pass + self.assertEqual(db.query("SELECT 1"), [(1,)]) + + def test_threads_write_one_at_a_time(self): + """Transactions from many threads all commit, one at a time.""" + db = self.connect() + + def write(n): + """Insert 50 rows, one transaction each.""" + for i in range(50): + with db.transaction() as tx: + tx.execute("INSERT INTO pins (cid, source) VALUES (?, 'user')", (bytes([n, i]),)) + + threads = [threading.Thread(target=write, args=(n,)) for n in range(8)] + for thread in threads: + thread.start() + for thread in threads: + thread.join() + self.assertEqual(db.query("SELECT count(*) FROM pins"), [(400,)]) + + def test_references_share_one_connection_until_the_last_closes(self): + """connect() hands out references to one database per directory; the last close() closes it, and a + closed reference raises StateDbError.""" + first, second = statedb.connect(self.state), statedb.connect(self.state) + self.assertIn(self.path, statedb.open_databases()) + first.close() + first.close() # idempotent: it doesn't release the other's reference + self.assertEqual(second.query("SELECT 1"), [(1,)]) + with self.assertRaises(statedb.StateDbError): + first.query("SELECT 1") + second.close() + self.assertNotIn(self.path, statedb.open_databases()) + + def test_meta(self): + """Meta values are per store and key, with a default when unset.""" + db = self.connect() + self.assertEqual(db.meta("follow", "clean", "unset"), "unset") + with db.transaction() as tx: + statedb.set_meta(tx, "follow", "clean", 1) + statedb.set_meta(tx, "inbox", "clean", 0) + self.assertEqual((db.meta("follow", "clean"), db.meta("inbox", "clean")), (1, 0)) + + def test_dump_round_trips_any_text(self): + """dump() gives text SQLite stores, non-ASCII as is, and load() returns exactly what was dumped.""" + db = self.connect() + value = {"title": "é \U0001f600 \udc80 \ud800x", "n": [1, None]} + with db.transaction() as tx: + tx.execute("INSERT INTO pin_names (cid, name) VALUES (?, ?)", (b"x", statedb.dump(value))) + (text,), = db.query("SELECT name FROM pin_names") + self.assertEqual(statedb.load(text), value) + self.assertIn("é \U0001f600", text) + + +def _import_rows(value, tx) -> int: + """A test importer: value's strings become pins of source "user"; returns how many.""" + tx.execute("DELETE FROM pins") + tx.executemany("INSERT INTO pins (cid, source) VALUES (?, 'user')", [(text.encode(),) for text in value]) + return len(value) + + +TEST_LEGACY = (statedb.Legacy("test.json", f"{__name__}:_import_rows", lambda value: isinstance(value, list)),) + + +class MigrateTest(StateDbTest): + """migrate(), with a test importer.""" + + def rows(self) -> list[bytes]: + """The test importer's rows.""" + return sorted(cid for (cid,) in state_rows(self.state, "SELECT cid FROM pins")) + + def test_file_is_imported_once_and_kept_as_migrated(self): + """A legacy file is imported in one transaction, recorded, renamed to .migrated, and logged.""" + write_legacy(self.state, "test.json", ["a", "b"]) + with mock.patch.object(statedb, "log") as log: + self.connect(TEST_LEGACY).close() + self.assertEqual(self.rows(), [b"a", b"b"]) + self.assertFalse(os.path.exists(os.path.join(self.state, "test.json"))) + with open(os.path.join(self.state, "test.json.migrated"), encoding="utf-8") as f: + self.assertEqual(json.load(f), ["a", "b"]) + self.assertEqual(state_rows(self.state, "SELECT file, records FROM migrated"), [("test.json", 2)]) + self.assertTrue(any("imported test.json" in call.args[0] and "2 records" in call.args[0] + and call.args[1] == statedb.LOG_NOTICE for call in log.call_args_list)) + # A second start is a no-op; the .migrated file left behind is ignored + with mock.patch.object(statedb, "_resolve", side_effect=AssertionError("imported again")): + self.connect(TEST_LEGACY) + self.assertEqual(self.rows(), [b"a", b"b"]) + + def test_crash_between_the_commit_and_the_rename(self): + """If the rename never happened, the next start only renames the file: nothing is imported twice.""" + path = write_legacy(self.state, "test.json", ["a"]) + with mock.patch.object(statedb, "_keep_original", return_value=None): + self.connect(TEST_LEGACY).close() + self.assertTrue(os.path.exists(path)) + write_legacy(self.state, "test.json", ["a", "changed"]) # whatever it holds now isn't imported + self.connect(TEST_LEGACY).close() + self.assertEqual(self.rows(), [b"a"]) + self.assertFalse(os.path.exists(path)) + self.assertTrue(os.path.exists(path + ".migrated")) + + def test_crash_inside_the_import_imports_nothing(self): + """An importer that fails leaves no rows, no record and the file in place, so the next start imports it + whole.""" + path = write_legacy(self.state, "test.json", ["a", "b"]) + with mock.patch(f"{__name__}._import_rows", side_effect=KeyboardInterrupt): + with self.assertRaises(KeyboardInterrupt): + self.connect(TEST_LEGACY) + self.assertEqual((self.rows(), state_rows(self.state, "SELECT * FROM migrated")), ([], [])) + self.assertTrue(os.path.exists(path)) + self.connect(TEST_LEGACY) + self.assertEqual(self.rows(), [b"a", b"b"]) + + def test_failed_commit_imports_nothing(self): + """A commit that fails stops startup with the file in place and nothing recorded.""" + self.connect().close() + path = write_legacy(self.state, "test.json", ["a"]) + with failing_commits(): + with self.assertRaises(OSError): + self.connect(TEST_LEGACY) + self.assertTrue(os.path.exists(path)) + self.assertEqual(state_rows(self.state, "SELECT * FROM migrated"), []) + + def test_damaged_file_is_moved_aside_and_not_recorded(self): + """A damaged file (not JSON, or the wrong shape) is moved to .damaged, as read_json always did.""" + for text in ("{not json", '{"a": 1}', ""): + with self.subTest(text=text): + path = write_legacy(self.state, "test.json", text=text) + self.connect(TEST_LEGACY).close() + self.assertFalse(os.path.exists(path)) + self.assertEqual(state_rows(self.state, "SELECT * FROM migrated"), []) + self.assertTrue(os.path.exists(os.path.join(self.state, "test.json.damaged.2"))) + + @unittest.skipUnless(POSIX_MODES and hasattr(os, "getuid") and os.getuid() != 0, "needs permission checks") + def test_unreadable_file_stops_startup_unless_allowed(self): + """A file that can't be read (not damage) stops startup and is left for the next start; one marked + unreadable_ok is logged and skipped instead.""" + path = write_legacy(self.state, "test.json", ["a"]) + os.chmod(path, 0) + try: + with self.assertRaises(PermissionError): + self.connect(TEST_LEGACY) + allowed = (statedb.Legacy("test.json", TEST_LEGACY[0].importer, TEST_LEGACY[0].shape, + unreadable_ok=True),) + self.connect(allowed).close() + self.assertTrue(os.path.exists(path)) + finally: + os.chmod(path, 0o600) + self.connect(TEST_LEGACY) + self.assertEqual(self.rows(), [b"a"]) + + def test_earlier_migrated_copy_is_kept(self): + """If .migrated is taken, the file is kept as .migrated.1.""" + write_legacy(self.state, "test.json.migrated", ["earlier"]) + write_legacy(self.state, "test.json", ["a"]) + self.connect(TEST_LEGACY) + with open(os.path.join(self.state, "test.json.migrated"), encoding="utf-8") as f: + self.assertEqual(json.load(f), ["earlier"]) + with open(os.path.join(self.state, "test.json.migrated.1"), encoding="utf-8") as f: + self.assertEqual(json.load(f), ["a"]) + + def test_concurrent_connects_import_once(self): + """Stores connecting from several threads at once import each file once.""" + write_legacy(self.state, "test.json", ["a", "b"]) + calls, dbs, errors = [], [], [] + real = _import_rows + + def importer(value, tx): + """Count the import, then import.""" + calls.append(1) + return real(value, tx) + + def connect(): + """Connect, keeping the reference or the error.""" + try: + dbs.append(statedb.connect(self.state)) + except BaseException as e: # noqa: BLE001 - reported below + errors.append(e) + + with mock.patch.object(statedb, "LEGACY", TEST_LEGACY), \ + mock.patch(f"{__name__}._import_rows", side_effect=importer): + threads = [threading.Thread(target=connect) for _ in range(6)] + for thread in threads: + thread.start() + for thread in threads: + thread.join() + for db in dbs: + self.addCleanup(db.close) + self.assertEqual((len(calls), errors), (1, [])) + self.assertEqual(self.rows(), [b"a", b"b"]) + + def test_every_legacy_file_names_an_importer(self): + """Every LEGACY entry's importer (and reader) exists.""" + for item in statedb.LEGACY: + with self.subTest(file=item.file): + self.assertTrue(callable(statedb._resolve(item.importer))) + if item.read: + self.assertTrue(callable(statedb._resolve(item.read))) + self.assertEqual(len({item.file for item in statedb.LEGACY}), 17) + + def test_state_planted_in_the_database_is_kept(self): + """A database already holding rows isn't touched by a start without legacy files.""" + self.connect().close() + state_execute(self.state, "INSERT INTO pins (cid, source) VALUES (?, 'user')", (b"kept",)) + self.connect() + self.assertEqual(self.rows(), [b"kept"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_storage.py b/tests/test_storage.py index 3d67416..4141202 100644 --- a/tests/test_storage.py +++ b/tests/test_storage.py @@ -6,13 +6,13 @@ import unittest from unittest import mock -from rnfs import fsutil +from rnfs import fsutil, statedb from rnfs.content import builder, dag, storage as storage_module, walker from rnfs.content.cid import DAG, RAW, Cid from rnfs.content.dag import Child, Entry, FileNode from rnfs.content.storage import FOLDER, USER, Storage, check_source, name_source, remote_source from rnfs.errors import NotFound, StorageFull -from tests.support import POSIX_MODES +from tests.support import POSIX_MODES, failing_commits, state_execute, state_rows, write_legacy IDENTITY = "0123456789abcdef0123456789abcdef" @@ -27,8 +27,10 @@ def setUp(self): self.storage = self.open() def open(self, max_storage: int = 10**9) -> Storage: - """(Re)open storage over the same directories.""" - return Storage(os.path.join(self.tmp.name, "blocks"), os.path.join(self.tmp.name, "state"), max_storage) + """(Re)open storage over the same directories (closed before the directory is removed).""" + storage = Storage(os.path.join(self.tmp.name, "blocks"), os.path.join(self.tmp.name, "state"), max_storage) + self.addCleanup(storage.close) + return storage def add(self, data: bytes) -> builder.Built: """Store a file.""" @@ -227,16 +229,56 @@ def test_failed_save_pins_nothing(self): a, b, c = (self.add(b"%d" % i).cid for i in range(3)) self.storage.pin(a, USER) self.storage.pin(b, FOLDER) - failure = OSError(28, "No space left on device") for pin in (lambda: self.storage.pin_many([a, b, c], USER), lambda: self.storage.pin(c, USER)): - with mock.patch.object(storage_module, "write_json", side_effect=failure): + with failing_commits("database or disk is full"): with self.assertRaises(OSError): pin() - # Memory says what pins.json does + # Memory says what the database does self.assertEqual(self.storage.pins(), {a: {"user"}, b: {"folder"}}) self.assertEqual(self.open().pins(), self.storage.pins()) self.assertEqual(self.storage.pin_count(), 2) + def test_failed_save_keeps_the_pin(self): + cid = self.add(b"x").cid + self.storage.pin(cid, USER) + with failing_commits(): + with self.assertRaises(OSError): + self.storage.unpin(cid, USER) + self.assertEqual(self.storage.pins(), {cid: {"user"}}) # still pinned, as the database says + self.assertEqual(self.open().pins(), {cid: {"user"}}) + + def test_a_pin_is_one_row(self): + a, b = self.add(b"a").cid, self.add(b"b").cid + self.storage.pin(a, USER) + self.storage.pin(a, FOLDER) + self.storage.pin(b, USER) + state = os.path.join(self.tmp.name, "state") + self.assertEqual(sorted(state_rows(state, "SELECT cid, source FROM pins")), + sorted([(a.binary, "user"), (a.binary, "folder"), (b.binary, "user")])) + self.storage.unpin(a, USER) + self.assertEqual(sorted(state_rows(state, "SELECT cid, source FROM pins")), + sorted([(a.binary, "folder"), (b.binary, "user")])) + + def test_invalid_rows_are_skipped_and_deleted(self): + cid = self.add(b"x").cid + self.storage.pin(cid, USER) + self.storage.record_fetch(cid, [], cid) + self.storage.save() + state = os.path.join(self.tmp.name, "state") + # A truncated CID, a CID in text form, an unknown source; a path that isn't a list of DAG names + for row in ((cid.binary[:-1], "user"), (str(cid), "user"), (cid.binary, "users")): + state_execute(state, "INSERT INTO pins (cid, source) VALUES (?, ?)", row) + for path in ('"ab"', '["a/b"]', "{not json"): + state_execute(state, "INSERT INTO cache (root, path, target, used) VALUES (?, ?, ?, 99)", + (cid.binary, path, cid.binary)) + state_execute(state, "INSERT INTO cache (root, path, target, used) VALUES (?, '[]', ?, 98)", + (b"\x01\x02", cid.binary)) + storage = self.open() + self.assertEqual(storage.pins(), {cid: {"user"}}) + self.assertEqual(storage.cached_fetches(), [(cid, (), cid)]) + self.assertEqual(state_rows(state, "SELECT cid, source FROM pins"), [(cid.binary, "user")]) + self.assertEqual(state_rows(state, "SELECT root, path FROM cache"), [(cid.binary, "[]")]) + def test_pin_count(self): self.assertEqual(self.storage.pin_count(), 0) a, b = self.add(b"a").cid, self.add(b"b").cid @@ -300,6 +342,8 @@ def test_directory_in_place_of_state_is_moved_aside(self): @unittest.skipUnless(POSIX_MODES and hasattr(os, "getuid") and os.getuid() != 0, "needs permission checks") def test_unmovable_damaged_state_doesnt_stop_startup(self): + # The database is open already (setUp's storage), so only moving the damaged file aside fails; a state + # directory that is read-only from the start stops startup (test_statedb), since nothing could be saved state = os.path.join(self.tmp.name, "state") with open(os.path.join(state, "pins.json"), "w", encoding="utf-8") as f: f.write("{not json") @@ -310,46 +354,34 @@ def test_unmovable_damaged_state_doesnt_stop_startup(self): @unittest.skipUnless(POSIX_MODES and hasattr(os, "getuid") and os.getuid() != 0, "needs permission checks") def test_unreadable_state_stops_startup_rather_than_losing_pins(self): cid = self.add(b"precious").cid - self.storage.pin(cid, USER) - path = os.path.join(self.tmp.name, "state", "pins.json") + state = os.path.join(self.tmp.name, "state") + path = write_legacy(state, "pins.json", {"pins": {str(cid): ["user"]}}) os.chmod(path, 0) # or EIO, EMFILE: not damage, and starting without pins would let gc delete them - self.addCleanup(os.chmod, path, 0o600) - with self.assertRaises(PermissionError): - self.open() - os.chmod(path, 0o600) + try: + with self.assertRaises(PermissionError): + self.open() + self.assertTrue(os.path.exists(path)) # not imported, not moved: read at the next start + finally: + os.chmod(path, 0o600) self.assertEqual(self.open().pins(), {cid: {"user"}}) def test_block_names_are_synced_before_pins_and_cache_records(self): calls = [] + real_commit = statedb._Database._commit + + def commit(database, conn): + """Note the commit and which rows it writes, then commit.""" + calls.append("commit " + ",".join(table for table in ("pins", "cache") if conn.execute( + f"SELECT count(*) FROM {table}").fetchone()[0])) + real_commit(database, conn) + with mock.patch.object(self.storage.blocks, "sync", lambda: calls.append("blocks")), \ - mock.patch.object(storage_module, "write_json", lambda path, value: calls.append(os.path.basename(path))): + mock.patch.object(statedb._Database, "_commit", commit): cid = self.add(b"x").cid self.storage.pin(cid, USER) self.storage.record_fetch(cid, [], cid) self.storage.save() - self.assertEqual(calls, ["blocks", "pins.json", "blocks", "cache.json"]) - - @unittest.skipIf(fsutil.IS_WINDOWS, "Windows can't sync a directory") - def test_state_is_synced_before_and_after_the_rename(self): - events = [] - real_fsync, real_replace = os.fsync, fsutil.replace - with mock.patch("os.fsync", side_effect=lambda fd: (events.append("fsync"), real_fsync(fd))[1]), \ - mock.patch.object(fsutil, "replace", lambda *a: (events.append("rename"), real_replace(*a))[1]): - storage_module.write_json(os.path.join(self.tmp.name, "state", "x.json"), {"a": 1}) - self.assertEqual(events, ["fsync", "rename", "fsync"]) - - def test_state_file_format(self): - path = os.path.join(self.tmp.name, "state", "x.json") - value = {"é": ["\ud800", 1]} # a lone surrogate is written as its escape either way - for ensure_ascii, expected in ((True, b'{"\\u00e9": ["\\ud800", 1]}'), - (False, b'{"\xc3\xa9": ["\\ud800", 1]}')): - with self.subTest(ensure_ascii=ensure_ascii): - storage_module.write_json(path, value, ensure_ascii=ensure_ascii) - with open(path, "rb") as f: - self.assertEqual(f.read(), expected) - if POSIX_MODES: - self.assertEqual(os.stat(path).st_mode & 0o777, 0o600) - self.assertEqual(sorted(os.listdir(os.path.dirname(path))), ["x.json"]) # no temporary file left + self.assertEqual(calls, ["blocks", "commit pins", "blocks", "commit pins,cache"]) @unittest.skipUnless(POSIX_MODES, "Windows has no POSIX permission bits") def test_state_dir_is_private(self): @@ -530,17 +562,30 @@ def test_eviction_with_nothing_evictable_doesnt_walk_again(self): self.assertEqual(self.storage.used_bytes(), 0) def test_cache_records_saved_in_batches_and_capped(self): - path = os.path.join(self.tmp.name, "state", "cache.json") + state = os.path.join(self.tmp.name, "state") with mock.patch.object(storage_module, "MAX_CACHE_RECORDS", 3): built = [self.add(b"%d" % i) for i in range(5)] dropped = set() for b in built: dropped |= self.storage.record_fetch(b.cid, [], b.cid) self.storage.touch(built[2].cid, []) - self.assertFalse(os.path.exists(path)) # not rewritten on every fetch + self.assertEqual(state_rows(state, "SELECT * FROM cache"), []) # not written on every fetch self.assertEqual(dropped, {built[0].cid, built[1].cid}) self.storage.save() self.assertEqual([t for _, _, t in self.open().cached_fetches()], [built[3].cid, built[4].cid, built[2].cid]) + # A later save writes only the records changed since: here the one touched + query = "SELECT root, path, used FROM cache" + before = {(root, path): used for root, path, used in state_rows(state, query)} + self.storage.touch(built[3].cid, []) + self.storage.save() + after = {(root, path): used for root, path, used in state_rows(state, query)} + self.assertEqual({key for key in after if after[key] != before[key]}, {(built[3].cid.binary, "[]")}) + self.assertEqual([t for _, _, t in self.open().cached_fetches()], + [built[4].cid, built[2].cid, built[3].cid]) + # Evicted and cleared records are deleted + self.storage.gc() + self.storage.save() + self.assertEqual(state_rows(state, "SELECT * FROM cache"), []) def test_verify(self): root, a, b = self.tree() @@ -730,6 +775,7 @@ def test_stat_of_a_partial_dag_doesnt_hide_nodes_stored_later(self): root, a, b = self.tree() blocks = {cid: self.storage.get(cid) for cid in self.blocks_of(root.cid)} self.storage = Storage(os.path.join(self.tmp.name, "blocks2"), os.path.join(self.tmp.name, "state2"), 10**9) + self.addCleanup(self.storage.close) self.storage.put(root.cid, blocks[root.cid]) self.storage.pin(root.cid, USER) self.assertEqual(self.storage.stat(root.cid)["local_blocks"], 1) # a's and b's nodes aren't stored yet diff --git a/tests/test_upgrade_014.py b/tests/test_upgrade_014.py new file mode 100644 index 0000000..3dd071b --- /dev/null +++ b/tests/test_upgrade_014.py @@ -0,0 +1,421 @@ +"""Upgrading from rnfs 0.1.4: the current code started on an rnfs directory exactly as 0.1.4 wrote it +(tests/fixtures/rnfs-0.1.4, every one of its 17 JSON state files populated; see its README) holds every record. + +Each store's state in memory is compared with the fixture's files read here with plain json, so the expectation +comes from the fixture itself. The only differences allowed are those the 0.1.4 load made too: outbox messages +not finished are failed (LXMF doesn't persist its queue), and the node's own provider records are stored again +(the publisher records itself for everything it provides). Then: each file is kept as .migrated, none is +set aside as damaged, the migrated table counts each file's records, a second start changes nothing, and a +restart keeps everything. + +The clock is the fixture's T0 plus a day (generate.py), so no record has expired.""" +import functools +import importlib.util +import io +import json +import os +import shutil +import tempfile +import time +import unittest +from unittest import mock + +import RNS + +from rnfs import node as node_module +from rnfs import statedb +from rnfs.config import Config +from rnfs.content import builder +from rnfs.content.blockstore import BlockStore +from rnfs.content.cid import Cid +from rnfs.features import lxmf +from rnfs.node import RnfsNode +from tests.support import FakeClock, state_rows +from tests.test_names import FakeNetwork +from tests.test_node import NetClient + +FIXTURE = os.path.join(os.path.dirname(os.path.abspath(__file__)), "fixtures", "rnfs-0.1.4") +T0 = 1792054800.0 # the fake clock generate.py wrote the fixture at (records span T0 - 3 days to T0 + 1 hour) +NOW = T0 + 86400.0 +NODE_ID = bytes.fromhex("4e0de1d04e0de1d04e0de1d04e0de1d0") # generate.py's NODE_ID +# generate.py's SETTINGS: nothing in the fixture is filtered out by configuration +SETTINGS = {"remote_pin_identities": ["0123456789abcdef0123456789abcdef"], + "lxmf_auto_fetch_from": ["a0a1a2a3a4a5a6a7a8a9aaabacadaeaf"], "max_records": 40, "slow": "no", + "path_timeout": 2, "lookup_timeout": 2} +LEGACY = [item.file for item in statedb.LEGACY] +UNFINISHED_ERROR = "the daemon stopped before it was delivered" + +_saved_loglevel = RNS.loglevel + + +def setUpModule(): + """Silence the log while these tests run.""" + RNS.loglevel = RNS.LOG_CRITICAL + + +def tearDownModule(): + """Restore RNS's loglevel.""" + RNS.loglevel = _saved_loglevel + + +def fixture(name: str): + """A state file of the fixture, as plain JSON.""" + with open(os.path.join(FIXTURE, "state", name), encoding="utf-8") as f: + return json.load(f) + + +def _content(): + """The fixture's fixture_content module: the files 0.1.4 added to it.""" + spec = importlib.util.spec_from_file_location("fixture_content", os.path.join(FIXTURE, "fixture_content.py")) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def make_blocks(blocks_dir: str, scratch: str) -> set[Cid]: + """Store in blocks_dir the blocks of everything 0.1.4 added to the fixture, built by the current code (the + fixture keeps no blocks); the roots built. The CIDs must be those 0.1.4 recorded, as no CID changed.""" + content, store, roots = _content(), BlockStore(blocks_dir), set() + for data, name in content.STREAMS: + roots.add(builder.add_stream(io.BytesIO(data), store.put, len(data), name).cid) + + def write(path: str, tree: dict): + """Create the directory path holding tree.""" + os.makedirs(path) + for entry, value in tree.items(): + if isinstance(value, dict): + write(os.path.join(path, entry), value) + else: + with open(os.path.join(path, entry), "wb") as f: + f.write(value) + + for n, tree in enumerate(content.TREES): + write(os.path.join(scratch, f"tree-{n}"), tree) + roots.add(builder.add_path(os.path.join(scratch, f"tree-{n}"), store.put)[0].cid) + notes = os.path.join(scratch, content.NOTES[0]) + with open(notes, "wb") as f: + f.write(content.NOTES[1]) + roots.add(builder.add_path(notes, store.put)[0].cid) + return roots + + +def cids(texts) -> list[Cid]: + """CIDs from their text.""" + return [Cid.parse(text) for text in texts] + + +def unfinished(message: dict) -> bool: + """An outbox message not finished when the daemon stopped (as LxmfFeature tells them at start).""" + return message["state"] not in lxmf.FINAL_STATES and not (message["method"] == "propagated" + and message["state"] == "sent") + + +def loadable_jobs() -> list[dict]: + """The fixture's jobs this platform loads: a get's output must be absolute, and /home/... isn't on Windows + (Python 3.13 and later), so there 0.1.4 and the import both skip it.""" + return [job for job in fixture("jobs.json")["jobs"] if job["output"] is None or os.path.isabs(job["output"])] + + +class UpgradeTest(unittest.TestCase): + """A copy of the fixture under a temporary directory, started on with the clock at NOW.""" + + def setUp(self): + """Copy the fixture; fake the wall clock, also for the stores that take it as an argument.""" + self.tmp = tempfile.TemporaryDirectory() + self.addCleanup(self.tmp.cleanup) + self.root = os.path.join(self.tmp.name, "rnfs") + for part in ("state", "names", "files"): + shutil.copytree(os.path.join(FIXTURE, part), os.path.join(self.root, part)) + os.makedirs(os.path.join(self.tmp.name, "scratch")) + self.built = make_blocks(os.path.join(self.root, "blocks"), os.path.join(self.tmp.name, "scratch")) + self.state = os.path.join(self.root, "state") + self.clock = FakeClock(NOW) + # ProviderStore, Publisher and NameManager bind time.time as a default argument, so patching it isn't enough + for patch in (mock.patch.object(time, "time", self.clock), + *(mock.patch.object(node_module, cls, functools.partial(getattr(node_module, cls), + clock=self.clock)) + for cls in ("ProviderStore", "Publisher", "NameManager"))): + patch.start() + self.addCleanup(patch.stop) + + def start(self) -> RnfsNode: + """An offline node on the copy (stopped before the directory is removed).""" + config = Config(rnfs_dir=self.root, **SETTINGS) + config.validate() + node = RnfsNode.offline(config, NODE_ID, NetClient(), FakeNetwork()) + self.addCleanup(node.stop) + return node + + def tables(self) -> dict[str, list[tuple]]: + """Every row of every table of the state database, sorted.""" + names = [row[0] for row in state_rows(self.state, "SELECT name FROM sqlite_master WHERE type = 'table'")] + return {name: sorted(state_rows(self.state, f"SELECT * FROM {name}"), key=repr) for name in names} + + # Each store's state against the fixture + + def check_pins(self, node: RnfsNode): + """Every pin with every source, and each complete: the current code built the same CIDs (make_blocks).""" + expected = {Cid.parse(text): set(sources) for text, sources in fixture("pins.json")["pins"].items()} + self.assertEqual(node.storage.pins(), expected) + roots = {Cid.parse(root) for root, _, _ in fixture("cache.json")["fetches"]} + self.assertLessEqual(roots, self.built) + for cid in expected: + self.assertTrue(node.storage.is_complete(cid), cid) + + def check_cache(self, node: RnfsNode): + """Every cached fetch, least recently used first, non-ASCII path names included.""" + expected = [(Cid.parse(root), tuple(names), Cid.parse(target)) + for root, names, target in fixture("cache.json")["fetches"]] + self.assertEqual(node.storage.cached_fetches(), expected) + + def check_pin_names(self, node: RnfsNode): + """Every pin name, the lone surrogate of a name that isn't UTF-8 included.""" + expected = {Cid.parse(text): name for text, name in fixture("pin_names.json").items()} + self.assertIn("\udcff", "".join(expected.values())) + self.assertEqual({cid: node.pin_names.get(cid) for cid in expected}, expected) + self.assertEqual(len(node.pin_names._names), len(expected)) + + def check_fetched(self, node: RnfsNode): + """The targets fetched for the user, which the roots strategy provides.""" + self.assertEqual(node._eligible_targets(), set(cids(fixture("fetched.json")))) + + def check_remote_pins(self, node: RnfsNode): + """The remote pins waiting, in the order accepted.""" + self.assertEqual(node._load_remote_pins(), [(source, Cid.parse(text)) + for source, text in fixture("remote_pins.json")]) + + def check_invalid(self, node: RnfsNode): + """The roots found invalid, oldest first.""" + self.assertEqual(list(node.fetcher._invalid), cids(fixture("invalid.json"))) + + def check_published(self, node: RnfsNode): + """Last rounds (CIDs and kept names' keys), earlier rounds' nodes and withdrawals queued, as written.""" + expected = fixture("published.json") + self.assertTrue(any(item.startswith("key:") for item in expected["published"])) + publisher = node.publisher + self.assertEqual({str(item): [when, [n.hex() for n in nodes]] + for item, (when, nodes) in publisher._published.items()}, expected["published"]) + self.assertEqual({str(item): {n.hex(): when for n, when in nodes.items()} + for item, nodes in publisher._earlier.items() if nodes}, expected["earlier"]) + self.assertEqual({n.hex(): sorted(k.hex() for k in keys) for n, keys in publisher._withdrawals.items() + if keys}, expected["withdraw"]) + # What 0.1.4 provided is still provided: every published CID and key + self.assertEqual({str(c) for c in publisher.provided()} | {"key:" + k.hex() for k in publisher.provided_keys()}, + set(expected["published"])) + + def check_providers(self, node: RnfsNode): + """Publishers, records in the main store and the pool, departed and evicted publishers; the node's own + records are stored again at start (last stored now).""" + expected = fixture("providers.json") + records = node.records + self.assertEqual({p.hex(): entry.first_seen for p, entry in records._publishers.items()}, + expected["publishers"]) + own = NODE_ID.hex() + self.assertEqual({(k.hex(), p.hex()): [r.first_stored, r.last_stored, int(r.pooled)] + for k, by in records._records.items() for p, r in by.items()}, + {(k, p): [first, NOW if p == own else last, pooled] + for k, p, first, last, pooled in expected["records"]}) + self.assertTrue(any(pooled for *_, pooled in expected["records"])) + self.assertEqual([[p.hex(), seen, at] for p, (seen, at) in records._departed.items()], expected["departed"]) + self.assertEqual(sorted([p.hex(), seen, at] for p, (seen, at) in records._evicted.items()), + sorted(expected["evicted"])) + self.assertTrue(expected["departed"] and expected["evicted"]) + + def check_names(self, node: RnfsNode): + """Names published (one imported, never published here) with their keys, and the records held.""" + expected = fixture("names.json") + names = node.names + self.assertEqual({alias: {"sequence": p.sequence, "cid": None if p.cid is None else str(p.cid), + "imported": p.imported} for alias, p in names._published.items()}, + expected["published"]) + self.assertEqual([entry["alias"] for entry in names.list()], sorted(expected["published"])) + self.assertEqual({name.hex(): {"sequence": r.sequence, "cid": str(r.cid), "heard": r.heard} + for name, r in names._held.items()}, expected["held"]) + + def check_floor(self, node: RnfsNode): + """The floor of names resolved for the gateway, least recently resolved first, 2^64 - 1 included.""" + self.assertEqual([[name.hex(), sequence, at] for name, (sequence, at) in node.names._floor.items()], + fixture("name-floor.json")["floor"]) + + def check_shared_folder(self, node: RnfsNode): + """The shared folder's scanned entries.""" + self.assertEqual({name: [signature, str(cid)] for name, (signature, cid) in node.shared_folder._known.items()}, + fixture("shared_folder.json")["entries"]) + + def check_jobs(self, node: RnfsNode): + """Every job as saved, in creation order: the finished get with its result and warnings (but see + loadable_jobs), the failed pin, and the queued pin, queued again to resume.""" + expected = loadable_jobs() + jobs = node.feature("jobs")._jobs + self.assertEqual([job.to_record() for job in jobs.values()], expected) + self.assertEqual({job.state for job in jobs.values()}, {job["state"] for job in expected}) + self.assertIn("queued", {job.state for job in jobs.values()}) + + def check_keepers(self, node: RnfsNode): + """Kept names and why, with their signed records (verified again at load), one with none yet.""" + expected = fixture("keepers.json")["names"] + store = node.feature("keepers").store + self.assertEqual({name.hex(): {"reasons": sorted(reasons), + "record": [f.hex() for f in store._records[name].wire()] + if name in store._records else None} + for name, reasons in store._reasons.items()}, expected) + + def check_follow(self, node: RnfsNode): + """Followed names and the target pinned for each.""" + self.assertEqual({name.hex(): {"pinned": None if cid is None else str(cid)} + for name, cid in node.feature("follow")._pinned.items()}, fixture("follow.json")["names"]) + + def check_contacts(self, node: RnfsNode): + """Contacts of each kind.""" + self.assertEqual({nick: {"kind": kind, "id": ident.hex()} + for nick, (kind, ident) in node.feature("contacts")._contacts.items()}, + fixture("contacts.json")["contacts"]) + + def check_inbox(self, node: RnfsNode): + """Every inbox message with its exact text (non-BMP characters included) and auto-fetch results, and + the next id.""" + expected = fixture("lxmf_inbox.json") + inbox = node.feature("lxmf").inbox + self.assertEqual(inbox.entries(), expected["messages"]) + self.assertEqual(inbox._next_id, expected["next_id"]) + + def check_outbox(self, node: RnfsNode): + """Every outbox message with its exact text; those not finished are failed now.""" + expected = fixture("lxmf_outbox.json") + messages = [{**message, "state": "failed", "error": UNFINISHED_ERROR} if unfinished(message) else message + for message in expected["messages"]] + self.assertTrue(any(map(unfinished, expected["messages"]))) + outbox = node.feature("lxmf").outbox + self.assertEqual(outbox.entries(), messages) + self.assertEqual(outbox._next_id, expected["next_id"]) + + def check_all(self, node: RnfsNode): + """Every store's check.""" + for name in dir(self): + if name.startswith("check_") and name != "check_all": + with self.subTest(check=name): + getattr(self, name)(node) + + # Tests + + def test_pins(self): + """pins.json: every source kind is kept, as names.json, follow.json and the inbox account for them.""" + self.check_pins(self.start()) + + def test_cache(self): + """cache.json: every record, in order.""" + self.check_cache(self.start()) + + def test_pin_names(self): + """pin_names.json: every name.""" + self.check_pin_names(self.start()) + + def test_fetched(self): + """fetched.json: every target.""" + self.check_fetched(self.start()) + + def test_remote_pins(self): + """remote_pins.json: every pin waiting.""" + self.check_remote_pins(self.start()) + + def test_invalid(self): + """invalid.json: every root.""" + self.check_invalid(self.start()) + + def test_published(self): + """published.json: every section.""" + self.check_published(self.start()) + + def test_providers(self): + """providers.json: every record, publisher, departure and eviction.""" + self.check_providers(self.start()) + + def test_names(self): + """names.json and the name keys: every name and record.""" + self.check_names(self.start()) + + def test_floor(self): + """name-floor.json: every entry.""" + self.check_floor(self.start()) + + def test_shared_folder(self): + """shared_folder.json: every entry.""" + self.check_shared_folder(self.start()) + + def test_jobs(self): + """jobs.json: every job.""" + self.check_jobs(self.start()) + + def test_keepers(self): + """keepers.json: every kept name and record.""" + self.check_keepers(self.start()) + + def test_follow(self): + """follow.json: every followed name.""" + self.check_follow(self.start()) + + def test_contacts(self): + """contacts.json: every contact.""" + self.check_contacts(self.start()) + + def test_inbox(self): + """lxmf_inbox.json: every message.""" + self.check_inbox(self.start()) + + def test_outbox(self): + """lxmf_outbox.json: every message.""" + self.check_outbox(self.start()) + + def test_files_are_migrated(self): + """Every state file is imported and kept as .migrated, unchanged; none is set aside as damaged, and + the migrated table counts each file's records.""" + self.start() + for name in LEGACY: + self.assertFalse(os.path.exists(os.path.join(self.state, name)), name) + with open(os.path.join(self.state, name + statedb.MIGRATED_SUFFIX), "rb") as kept, \ + open(os.path.join(FIXTURE, "state", name), "rb") as original: + self.assertEqual(kept.read(), original.read(), name) + self.assertEqual([name for name in os.listdir(self.state) if ".damaged" in name], []) + published = fixture("published.json") + names = fixture("names.json") + counts = { + "pins.json": sum(1 for sources in fixture("pins.json")["pins"].values() if sources), + "cache.json": len(fixture("cache.json")["fetches"]), + "pin_names.json": len(fixture("pin_names.json")), + "fetched.json": len(fixture("fetched.json")), + "remote_pins.json": len(fixture("remote_pins.json")), + "invalid.json": len(fixture("invalid.json")), + # Rows: one per item published, per (item, earlier node) and per (node, key withdrawn) + "published.json": len(published["published"]) + sum(map(len, published["earlier"].values())) + + sum(map(len, published["withdraw"].values())), + "providers.json": len(fixture("providers.json")["records"]), + "names.json": len(names["published"]) + len(names["held"]), + "name-floor.json": len(fixture("name-floor.json")["floor"]), + "shared_folder.json": len(fixture("shared_folder.json")["entries"]), + "jobs.json": len(loadable_jobs()), + "keepers.json": len(fixture("keepers.json")["names"]), + "follow.json": len(fixture("follow.json")["names"]), + "contacts.json": len(fixture("contacts.json")["contacts"]), + "lxmf_inbox.json": len(fixture("lxmf_inbox.json")["messages"]), + "lxmf_outbox.json": len(fixture("lxmf_outbox.json")["messages"]), + } + self.assertEqual(sorted(counts), sorted(LEGACY)) + self.assertTrue(all(counts.values()), counts) # every file has records + self.assertEqual(dict(state_rows(self.state, "SELECT file, records FROM migrated")), counts) + + def test_second_start_changes_nothing(self): + """A second start imports nothing again and changes no row, nor anything in the state directory.""" + self.start().stop() + tables, listing = self.tables(), sorted(os.listdir(self.state)) + self.start().stop() + self.assertEqual(self.tables(), tables) + self.assertEqual(sorted(os.listdir(self.state)), listing) + + def test_restart_keeps_everything(self): + """Stopped and started again, the node holds every store's state as it did after the upgrade.""" + self.start().stop() + self.check_all(self.start()) + + +if __name__ == "__main__": + unittest.main() From e969878bc846b59244928b893abb5f276fdc07ee Mon Sep 17 00:00:00 2001 From: Afi Date: Fri, 16 Oct 2026 19:43:06 +1030 Subject: [PATCH 5/9] Implement daemon lock mechanism and enhance logging for response outcomes --- CHANGELOG.md | 6 +++ README.md | 1 + design.md | 9 ++-- rnfs/cli.py | 23 ++++++++--- rnfs/fsutil.py | 42 +++++++++++++++++++ rnfs/network/client.py | 5 ++- rnfs/network/links.py | 6 ++- rnfs/network/server.py | 6 +++ tests/test_cli.py | 35 ++++++++++++++++ tests/test_fsutil.py | 72 +++++++++++++++++++++++++++++++- tests/test_server_outcomes.py | 77 +++++++++++++++++++++++++++++++++++ 11 files changed, 269 insertions(+), 13 deletions(-) create mode 100644 tests/test_server_outcomes.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 08071cd..52a40f1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -54,6 +54,12 @@ Fixes: those nodes reach. Until a restart, which cleared it. - `import` pins the roots an archive supplies with one save of the pins; if that save fails, none of them is pinned (before, those ahead of the failure were). +- A second daemon started on an rnfs dir already in use loaded and saved the same state, announced the same + node ID and rotated the first one's debug log before stopping on "address already in use". It now stops at + once, saying which process uses the directory: the daemon holds a lock on `/daemon.lock`. +- The debug log no longer says a provider "cancelled" a `/blocks` response that may have been lost in transit: + the requester can't tell those apart. The provider now logs a response whose transfer RNS gave up on (lost + in transit, or its link closed), where before only the ones it cancelled itself were logged. Faster: - `add`, the shared folder and `import` write blocks on up to 16 threads, so the sync to disk each block needs diff --git a/README.md b/README.md index a08a080..14e8eb6 100644 --- a/README.md +++ b/README.md @@ -371,6 +371,7 @@ Uninstalling never deletes it. | `names/` | name keys (back them up: `rnfs name export`) | | `files/` | the shared folder: anything put here is added, pinned and shared automatically | | `api_token` | the running daemon's API token | +| `daemon.lock` | locked by the daemon using the directory, so a second one on it stops at once | The rnfs directory holds your keys and the API token, so rnfs keeps it private: readable by you only, and on Windows with an access list for you and SYSTEM only (also on another drive or under `C:\ProgramData`, where diff --git a/design.md b/design.md index 3c2ebed..7a3200c 100644 --- a/design.md +++ b/design.md @@ -320,7 +320,10 @@ answers only `{"version"}` (health checks, and `verify_daemon`, which reads only the token with every other request, `rnfs status` included (without a readable token it prints the version and says why). The daemon writes a new random token to `/api_token` (mode 0600, in the 0700 rnfs dir) at each start and the CLI reads it, so other -local users and processes can't drive the API. `_make_private_dirs` refuses to start (RnfsError) if the rnfs dir +local users and processes can't drive the API. Before anything else uses the rnfs dir (the debug log, the +node), the daemon locks `/daemon.lock` (`fsutil.lock_exclusive`: `flock`, or `msvcrt.locking` of a byte +past the text on Windows; the file holds its process ID) and keeps the lock until it exits, so a second daemon +on the same dir stops at once, naming the first (`lock_holder`). `_make_private_dirs` refuses to start (RnfsError) if the rnfs dir belongs to another user or can't be made 0700. On Windows (`rnfs/winacl.py`), mode bits set no ACL, so a new rnfs dir and its subdirectories are created with a protected DACL allowing only the user and SYSTEM (inherited below it) in the CreateDirectoryW call itself (`winacl.make_private_dir`, a self-relative descriptor built by @@ -575,8 +578,8 @@ Free text that may hold lone surrogates (file names, LXMF records, cache paths) (`statedb.dump`). Rows that fail the checks the JSON loaders applied are dropped at load with a warning and deleted. The JSON files of 0.1.4 and earlier, once imported, stay as `.migrated`. Elsewhere: `storage/identity`, `storage/lxmf_identity` (0600), `names/` (name keys), `lxmf/` (LXMF's ratchets, -delivered-message IDs, stamp costs), `api_token` (while the daemon runs). Outputs being written live beside the output as `.rnfs-get-.partial` (`RnfsNode.get`) -or `.rnfs-job-.partial` (get jobs) until renamed; a failed, cancelled or shut-down job removes its partial, +delivered-message IDs, stamp costs), `api_token` (while the daemon runs), `daemon.lock` (locked while it runs). +Outputs being written live beside the output as `.rnfs-get-.partial` (`RnfsNode.get`) or `.rnfs-job-.partial` (get jobs) until renamed; a failed, cancelled or shut-down job removes its partial, and a resumed job removes one left by a crash. ## Tests diff --git a/rnfs/cli.py b/rnfs/cli.py index e38b1d5..d7f007f 100644 --- a/rnfs/cli.py +++ b/rnfs/cli.py @@ -1,7 +1,8 @@ """rnfs command line (specification section 11). -`daemon` runs a node with its API and gateway; every other command is one request to a running daemon's -local API (see rnfs/api.py). Client commands never import RNS, so they start quickly; paths given to them +`daemon` runs a node with its API and gateway, holding a lock on /daemon.lock from before it touches +anything there, so a second daemon on the same rnfs dir stops at once; every other command is one request to a +running daemon's local API (see rnfs/api.py). Client commands never import RNS, so they start quickly; paths given to them are resolved to absolute paths here, since the daemon reads and writes them itself. Client commands authenticate with the token the daemon writes to /api_token at each start @@ -27,6 +28,7 @@ DEFAULT_DIR = Config.rnfs_dir COPY_SIZE = 64 * 1024 TOKEN_FILE = "api_token" # in the rnfs directory; written by the daemon, read by every other command +LOCK_FILE = "daemon.lock" # in the rnfs directory; locked by the daemon using it, so a second one can't start # Before sending the token, the CLI makes whatever listens on the API port prove it knows it (api_proof) CHALLENGE_HEADER = "X-Rnfs-Challenge" PROOF_HEADER = "X-Rnfs-Proof" @@ -291,11 +293,22 @@ def run_daemon(args): except (ValueError, TypeError, OSError) as e: fail(f"invalid settings: {e}") + # Before anything uses the rnfs dir (made private first, as the node would make it): a second daemon on it + # would load and save the same state, announce the same node ID and rotate the first one's debug log + lock_path = os.path.join(config.root, LOCK_FILE) + try: + _make_private_dirs(config) + lock = fsutil.lock_exclusive(lock_path) # noqa: F841 - held until the process exits + except BlockingIOError: + holder = fsutil.lock_holder(lock_path) + fail(f"another rnfs daemon{f' (process {holder})' if holder else ''} is using {config.root}; stop it first, " + "or give this one another --dir") + except (RnfsError, OSError) as e: + fail(f"can't lock {lock_path}: {e}") + if config.debug_log: - # Before the node, so it has Reticulum starting (interfaces connecting or not); in the rnfs dir made - # private first, as the node would make it + # Before the node, so it has Reticulum starting (interfaces connecting or not) try: - _make_private_dirs(config) debug = debuglog.start(config) except (RnfsError, OSError) as e: fail(f"can't write the debug log: {e}") diff --git a/rnfs/fsutil.py b/rnfs/fsutil.py index e6f8af4..92419f3 100644 --- a/rnfs/fsutil.py +++ b/rnfs/fsutil.py @@ -53,6 +53,48 @@ def replace(src: str, dst: str): _retrying(os.replace, src, dst) +LOCK_OFFSET = 1 << 30 # Windows: the byte locked, far past what the lock file holds, so its text stays readable + + +def lock_exclusive(path: str) -> int: + """Open path (created 0600 if missing) and lock it for this process, without waiting; the descriptor, whose + lock lasts until it is closed or the process ends (also by a crash), with this process's ID written in the + file. Raises BlockingIOError if another process holds the lock, OSError if path can't be opened. + + POSIX: flock (advisory: only processes that lock it too are kept out). Windows: msvcrt.locking of one byte at + LOCK_OFFSET, past the text. + """ + fd = os.open(path, os.O_RDWR | os.O_CREAT | O_NOFOLLOW | O_BINARY, 0o600) + try: + if IS_WINDOWS: + import msvcrt + os.lseek(fd, LOCK_OFFSET, os.SEEK_SET) + try: + msvcrt.locking(fd, msvcrt.LK_NBLCK, 1) + except OSError as e: # PermissionError (EACCES) when another process holds it + raise BlockingIOError(errno.EAGAIN, f"{path} is locked by another process") from e + else: + import fcntl + fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB) # BlockingIOError when another process holds it + os.ftruncate(fd, 0) + os.lseek(fd, 0, os.SEEK_SET) + os.write(fd, f"{os.getpid()}\n".encode("ascii")) + except BaseException: + os.close(fd) + raise + return fd + + +def lock_holder(path: str) -> int | None: + """The process ID lock_exclusive() wrote in path, or None if it can't be read.""" + try: + with open(path, "rb") as f: + text = f.read(32) + return int(text.strip()) + except (OSError, ValueError): + return None + + def rename_new(src: str, dst: str): """Rename src (a file or directory) to dst, which must not exist: FileExistsError if it does. diff --git a/rnfs/network/client.py b/rnfs/network/client.py index 0896f2b..8654790 100644 --- a/rnfs/network/client.py +++ b/rnfs/network/client.py @@ -211,8 +211,9 @@ def _blocks(self, node_id: bytes, cids: list[Cid], on_progress: Callable[[float] self.links.close_link(peer) return BlocksResult(STALLED, unsent=wanted, reason=exchange.stalled) if exchange.cancelled: - # The responder gave the slot to another request or ran out of time: like a refusal, not a stall - log(f"{name} cancelled its /blocks response", LOG_VERBOSE) + # The responder gave the slot to another request or ran out of time, or the transfer was lost in + # transit: like a refusal, not a stall + log(f"{name}'s /blocks response stopped arriving (cancelled by it, or lost in transit)", LOG_VERBOSE) return BlocksResult(BUSY, unsent=wanted, reason="cancelled") if not exchange.answered: log(f"no answer from {name} to /blocks", LOG_VERBOSE) diff --git a/rnfs/network/links.py b/rnfs/network/links.py index 24fbdc1..e903f52 100644 --- a/rnfs/network/links.py +++ b/rnfs/network/links.py @@ -82,7 +82,9 @@ def __init__(self, clock: Callable[[], float]): self.last_progress = self.sent_at self.response_started: float | None = None # when the first part of a Resource response arrived self.stalled: str | None = None # why the requester gave up waiting, if it did - self.cancelled = False # the responder cancelled the response Resource (e.g. displaced it) + # The response Resource left the link unfinished: the responder cancelled it (e.g. displaced it), or RNS + # lost it in transit; the requester can't tell which + self.cancelled = False self.receipt = None # the RNS RequestReceipt, once sent # RNS callbacks: they run on RNS threads, so they only record and never block @@ -539,7 +541,7 @@ def _log_outcome(self, peer: PeerLink, path: str, exchange: Exchange): if exchange.stalled: why = exchange.stalled elif exchange.cancelled: - why = "it cancelled its response" + why = "its response stopped arriving (the responder cancelled it, or it was lost in transit)" elif not peer.is_active(): why = "the link " + (close_reason(peer.link) if peer.link is not None else "closed") elif getattr(exchange.receipt, "status", None) == RECEIPT_FAILED: diff --git a/rnfs/network/server.py b/rnfs/network/server.py index aaa8712..c8b4188 100644 --- a/rnfs/network/server.py +++ b/rnfs/network/server.py @@ -1204,6 +1204,7 @@ def _concluded(self, response: Response, link, resource): with self.lock: # The Resource's callback, the monitor and the next request may all notice; handle it once first, response.concluded = not response.concluded, True + cancelled = response.finished # finished before its Resource concluded: _cancel did it, and logged why status = getattr(resource, "status", None) if first and status == RNS.Resource.COMPLETE: self.rates.record(response.link_id, response.size, self._expected_rate(link)) @@ -1213,6 +1214,11 @@ def _concluded(self, response: Response, link, resource): # at once (section 3) log("a /blocks response was rejected with nothing sent: its requester cools down", LOG_VERBOSE) self.cooldowns.add(response.link_id, response.identity) + elif first and not cancelled and status in (RNS.Resource.FAILED, RNS.Resource.CORRUPT): + # Not cancelled here: RNS gave up on the transfer, so the requester sees it stop arriving + log(f"a /blocks response ({size(response.size or 0)}) failed after {size(int(response.sent))} sent: " + f"{'its link closed' if getattr(link, 'status', None) == RNS.Link.CLOSED else 'lost in transit'}", + LOG_DEBUG) self._finish(response) # charges what was sent: all of a completed Resource @staticmethod diff --git a/tests/test_cli.py b/tests/test_cli.py index edae1bb..a947bff 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -484,5 +484,40 @@ def test_proof_is_bound_to_the_port(self): self.assertNotEqual(proof, cli.api_proof("token", 8788, "cd" * 16)) + +class DaemonLockTest(unittest.TestCase): + """A second daemon on an rnfs dir another daemon holds (its daemon.lock) stops before touching anything.""" + + def test_second_daemon_stops_at_once(self): + """With the lock held, `rnfs daemon` exits with an error naming the holder, before writing state or + rotating the debug log.""" + from rnfs.config import Config + from rnfs.node import _make_private_dirs + from tests.test_fsutil import hold_lock + + tmp = tempfile.TemporaryDirectory() + self.addCleanup(tmp.cleanup) + rnfs_dir = os.path.join(tmp.name, "rnfs") + _make_private_dirs(Config(rnfs_dir=rnfs_dir)) # as the first daemon left it + debug_log = os.path.join(rnfs_dir, "debug.log") + with open(debug_log, "w", encoding="utf-8") as f: + f.write("the first daemon's log\n") + holder = hold_lock(os.path.join(rnfs_dir, cli.LOCK_FILE)) + try: + result = subprocess.run([sys.executable, "rnfs.py", "--dir", rnfs_dir, "--api-port", str(free_port()), + "daemon", "--gateway-port", str(free_port()), "--debug-log"], + cwd=ROOT, capture_output=True, timeout=60, encoding="utf-8", errors="replace") + finally: + holder.stdin.close() + holder.wait(30) + holder.stdout.close() + self.assertEqual(result.returncode, 1, result.stderr) + self.assertIn(f"another rnfs daemon (process {holder.pid}) is using", result.stderr) + with open(debug_log, encoding="utf-8") as f: + self.assertEqual(f.read(), "the first daemon's log\n") + self.assertFalse(os.path.exists(debug_log + ".1")) + self.assertFalse(os.path.exists(os.path.join(rnfs_dir, "state", "rnfs.db"))) + + if __name__ == "__main__": unittest.main() diff --git a/tests/test_fsutil.py b/tests/test_fsutil.py index e1a2fc8..e776150 100644 --- a/tests/test_fsutil.py +++ b/tests/test_fsutil.py @@ -1,8 +1,9 @@ """Tests for fsutil: replace and remove retried while Windows reports a file in use, atomic writes, Windows -name rules.""" +name rules, the lock a daemon holds on its rnfs dir.""" import errno import os import stat +import subprocess import sys import tempfile import unittest @@ -11,6 +12,21 @@ from rnfs import fsutil from tests.support import CAN_SYMLINK, POSIX_MODES +ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +# A process holding fsutil.lock_exclusive(argv[1]): it prints "locked", then keeps it until its stdin closes +HOLD_LOCK = ("import sys; from rnfs import fsutil; fsutil.lock_exclusive(sys.argv[1]); print('locked', flush=True); " + "sys.stdin.read()") + + +def hold_lock(path: str) -> subprocess.Popen: + """Start a process holding the lock on path, once it holds it.""" + holder = subprocess.Popen([sys.executable, "-c", HOLD_LOCK, path], cwd=ROOT, stdin=subprocess.PIPE, + stdout=subprocess.PIPE, text=True) + if holder.stdout.readline().strip() != "locked": + holder.kill() + raise AssertionError("the lock holder didn't lock") + return holder + def in_use(failures: int, real): """A side effect for os.replace or os.remove failing as Windows does failures times, then calling real.""" @@ -313,5 +329,59 @@ def test_printable_escapes(self): self.assertEqual(fsutil.printable(5), "5") + +class LockTest(unittest.TestCase): + """lock_exclusive and lock_holder, across processes as daemons use them.""" + + def setUp(self): + """A lock file path in a temporary directory.""" + tmp = tempfile.TemporaryDirectory() + self.addCleanup(tmp.cleanup) + self.path = os.path.join(tmp.name, "daemon.lock") + + def lock(self) -> int: + """Take the lock in this process (released before the directory is removed).""" + fd = fsutil.lock_exclusive(self.path) + self.addCleanup(os.close, fd) + return fd + + def test_a_second_process_is_refused_until_the_first_lets_go(self): + """While another process holds the lock, taking it raises BlockingIOError and lock_holder names that + process; once it exits, the lock is free and names this one.""" + holder = hold_lock(self.path) + try: + with self.assertRaises(BlockingIOError): + fsutil.lock_exclusive(self.path) + self.assertEqual(fsutil.lock_holder(self.path), holder.pid) + finally: + holder.stdin.close() + holder.wait(30) + holder.stdout.close() + self.lock() + self.assertEqual(fsutil.lock_holder(self.path), os.getpid()) + + def test_a_killed_holder_releases_it(self): + """A process that dies without letting go (a crash, kill -9) releases the lock with it.""" + holder = hold_lock(self.path) + holder.kill() + holder.wait(30) + holder.stdin.close() + holder.stdout.close() + self.lock() + + @unittest.skipUnless(POSIX_MODES, "Windows has no POSIX permission bits") + def test_lock_file_is_private(self): + """The lock file is created readable by its owner only.""" + self.lock() + self.assertEqual(os.stat(self.path).st_mode & 0o777, 0o600) + + def test_holder_of_a_file_without_a_process_id_is_unknown(self): + """lock_holder gives None for a missing or unreadable lock file.""" + self.assertIsNone(fsutil.lock_holder(self.path)) + with open(self.path, "w", encoding="ascii") as f: + f.write("not a pid") + self.assertIsNone(fsutil.lock_holder(self.path)) + + if __name__ == "__main__": unittest.main() diff --git a/tests/test_server_outcomes.py b/tests/test_server_outcomes.py new file mode 100644 index 0000000..62ef04b --- /dev/null +++ b/tests/test_server_outcomes.py @@ -0,0 +1,77 @@ +"""How a /blocks response's Resource ended, as the responder logs it: one RNS gave up on (lost in transit, or its +link closed) is logged at debug, since the requester only sees it stop arriving; one the server cancelled itself +is logged once, with its reason, and a completed one not at all.""" +import unittest +from unittest import mock + +import RNS + +from rnfs.config import Config +from rnfs.constants import CHUNK_SIZE +from rnfs.content.cid import RAW, Cid +from rnfs.log import LOG_DEBUG +from tests.security._server_helpers import ServerHarness + +BLOCK = bytes(CHUNK_SIZE) +CID = Cid.for_block(RAW, BLOCK) + + +class ResponseOutcomeTest(unittest.TestCase): + """A server on a FakeClock with fake Resources (tests.security._server_helpers).""" + + def setUp(self): + """A server holding one block, and one response to a request for it, advertised.""" + self.harness = ServerHarness({CID: BLOCK}, Config()) + self.link = self.harness.connect() + self.assertEqual(self.harness.blocks(self.link, [CID], max_bytes=CHUNK_SIZE), [BLOCK]) + self.resource = self.harness.advertise(self.link) + self.harness.tick() # the monitor finds the Resource and hooks its conclusion + + def messages(self, act) -> list[tuple[str, int]]: + """The server's log lines (and levels) while act() runs.""" + logged = [] + with mock.patch("rnfs.network.server.log", lambda message, level=None: logged.append((message, level))): + act() + self.harness.tick() + return logged + + def fail(self, progress: float): + """RNS gives up on the Resource part-way, as it does when its transfer is lost: FAILED, then the + callback.""" + self.resource.progress = progress + self.resource.status = RNS.Resource.FAILED + self.resource.conclude() + if self.resource.callback: + self.resource.callback(self.resource) + + def test_failure_in_transit_is_logged(self): + """A Resource RNS failed, which the server didn't cancel, is logged at debug, saying it was lost in + transit; the slot is freed for the next request.""" + logged = self.messages(lambda: self.fail(0.5)) + failed = [(message, level) for message, level in logged if "failed after" in message] + self.assertEqual(len(failed), 1, logged) + self.assertIn("lost in transit", failed[0][0]) + self.assertEqual(failed[0][1], LOG_DEBUG) + self.assertIsNone(self.harness.response(self.link)) + + def test_failure_with_the_link_closed_says_so(self): + """A Resource that failed because its link closed is logged as such.""" + self.link.status = RNS.Link.CLOSED + logged = self.messages(lambda: self.fail(0.2)) + self.assertTrue(any("its link closed" in message for message, _ in logged), logged) + + def test_cancelled_here_is_logged_only_with_its_reason(self): + """A response the server cancels (here, as the monitor would) logs its reason, not a failure too.""" + response = self.harness.response(self.link) + logged = self.messages(lambda: self.harness.server._cancel(response, "testing")) + self.assertTrue(any("cancelling a /blocks response: testing" in message for message, _ in logged), logged) + self.assertFalse(any("failed after" in message for message, _ in logged), logged) + + def test_completed_is_not_logged(self): + """A completed response logs no failure.""" + logged = self.messages(self.resource.complete) + self.assertFalse(any("failed after" in message for message, _ in logged), logged) + + +if __name__ == "__main__": + unittest.main() From 341127b66e9ba88857aa2798d844199914e5f671 Mon Sep 17 00:00:00 2001 From: Afi Date: Fri, 16 Oct 2026 21:57:43 +1030 Subject: [PATCH 6/9] Add comprehensive tests for deny list functionality This commit introduces a new test suite for the deny list feature, covering various aspects such as entry parsing, local entry persistence, list management, and interactions with offline nodes. The tests ensure that denied content is correctly handled, including scenarios for denied roots, DAGs, and stored content. Additionally, the CLI commands for managing deny lists are validated against an offline node's API. This extensive testing framework aims to enhance the reliability and robustness of the deny list implementation. --- CHANGELOG.md | 29 +- README.md | 65 ++++- design.md | 108 +++++-- rnfs/api.py | 20 +- rnfs/apiclient.py | 487 +++++++++++++++++++++++++++++++ rnfs/cli.py | 218 ++++---------- rnfs/config.py | 5 + rnfs/content/archive.py | 9 +- rnfs/content/builder.py | 30 +- rnfs/content/storage.py | 71 ++++- rnfs/content/walker.py | 31 +- rnfs/denylist.py | 315 ++++++++++++++++++++ rnfs/dht/lookup.py | 59 +++- rnfs/dht/routing.py | 65 ++++- rnfs/errors.py | 4 + rnfs/features/__init__.py | 3 +- rnfs/features/deny.py | 316 ++++++++++++++++++++ rnfs/features/jobs.py | 8 +- rnfs/features/lxmf.py | 4 +- rnfs/fetcher.py | 21 +- rnfs/gateway.py | 4 +- rnfs/names.py | 21 +- rnfs/node.py | 177 ++++++++++-- rnfs/node_content.py | 11 +- rnfs/publisher.py | 18 +- rnfs/statedb.py | 6 + rnfs/winacl.py | 2 +- specification.md | 75 ++++- tests/test_apiclient.py | 401 ++++++++++++++++++++++++++ tests/test_denylist.py | 587 ++++++++++++++++++++++++++++++++++++++ tests/test_lookup.py | 113 +++++++- tests/test_names.py | 45 ++- tests/test_node.py | 23 ++ tests/test_routing.py | 89 +++++- 34 files changed, 3172 insertions(+), 268 deletions(-) create mode 100644 rnfs/apiclient.py create mode 100644 rnfs/denylist.py create mode 100644 rnfs/features/deny.py create mode 100644 tests/test_apiclient.py create mode 100644 tests/test_denylist.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 52a40f1..002e9f9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,9 +4,9 @@ ## Unreleased -Performance work, a debug log, and state kept in one SQLite database. No wire-format or CID changes: these -nodes and 0.1.4 nodes work together. The upgrade of an `` is automatic and one-way (an older version -can't read it afterwards). +Performance work, a debug log, state kept in one SQLite database, deny lists, and a client library for apps. No +wire-format or CID changes: these nodes and 0.1.4 nodes work together. The upgrade of an `` is +automatic and one-way (an older version can't read it afterwards). Changed: - State is kept in one SQLite database, `/state/rnfs.db` (Python's standard `sqlite3`; no new @@ -47,8 +47,31 @@ Added: couldn't be established, each request's outcome or why it went unanswered, and incoming links. The file is readable by its owner only and never sent; each start keeps the previous run's as `debug.log.1`, and past 16 MiB the file moves there too, so the two stay under 32 MiB. +- `rnfs.apiclient`, a client library of the local API for companion apps: it finds the rnfs directory, port and + token as the CLI does, makes whatever listens on the API port prove it knows the token before sending it, + raises a typed error for each failure, and has methods for status, add, fetch and `cat` (streamed), pins, + names and jobs. Standard library only; it doesn't load Reticulum. The CLI now uses it, with the same output. + See "Building apps on rnfs" in the README. +- Deny lists, to keep chosen content off a node (a public gateway or provider node): `rnfs deny add|rm|ls` edits a + local list of CIDs and file hashes (`sha256:`), and `deny_lists` in `rnfs.toml` names lists to apply, by + CID or `name/` (followed as the name changes; each version is checked, at most 4 MiB and 100 000 + entries, and pinned while in use). A denied CID is never fetched, read, stored, served or provided, as a root + or anywhere inside a DAG; the API and gateway answer 451. Records already published are withdrawn, cached + copies are deleted and remote pins holding it released; your own pins are kept but not served or read, and + the log names them. File hashes are checked where a file is assembled (`get`, `cat`, the gateway, `add`, + `import`), after its blocks have been downloaded. Nothing changes on the network: a peer asking for a denied + block is told it isn't held. Fixes: +- A far node that keeps announcing but never accepts a link cost every lookup its connection timeout (60 s or + more at 9–10 hops), and held lookups open long after they had their answer: its announces reset the failure + count that would have dropped it, and other nodes kept naming it in their replies once it was dropped. A node + whose last 3 (`max_failures`) queries failed is now asked only when a lookup has too few others, whatever + its announces; it is asked again after 5 minutes, doubling up to an hour, without the lookup waiting for + its answer, and an answer restores it. Nothing changes on the network. +- A gateway or LXMF auto-fetch resolve that found a higher sequence for a name it had resolved before raised + the name's floor without saving it until the next re-announce pass (up to a minute later) or shutdown, and + so did an announce raising it with no resolve. Every raise is now saved within 5 s, as documented. - `gc` (and eviction) could delete blocks of a complete pin. `stat` of a DAG only partly stored remembered the nodes it didn't find as listing nothing, so once the rest was fetched and pinned, collection didn't see what those nodes reach. Until a restart, which cleared it. diff --git a/README.md b/README.md index 14e8eb6..aa41bef 100644 --- a/README.md +++ b/README.md @@ -28,6 +28,7 @@ Access any file on RNFS by it's **'CID'**. [Install](#install) · [Sample files (with CIDs)](#sample-files) · [Getting started](#getting-started) · [How rnfs adapts IPFS](#how-rnfs-adapts-ipfs) · [Settings](#settings) · [Commands](#commands) · +[Building apps on rnfs](#building-apps-on-rnfs) · [Keeping content off your node](#keeping-content-off-your-node) · [Example use cases](#example-use-cases) · [Running several nodes](#running-several-nodes) · [Tests](#tests) · [Known limitations](#known-limitations) · [specification.md](specification.md) (the protocol) · [design.md](design.md) (the code) @@ -613,6 +614,7 @@ lxmf_auto_fetch_from = [""] | `scan_interval` | `60` | How often the shared folder `files/` is scanned | | `remote_pin_identities` | `[]` | Identity hashes allowed to ask this node to pin (`pin remote add`); a node's is the `identity_hash` in its `rnfs status`. Pins accepted and not yet fetched are resumed after a restart | | `remote_pin_quota` | `1073741824` (1 GiB) | Storage each of those identities may use | +| `deny_lists` | `[]` | Deny lists to apply, each `"[/path]"` or `"name/[/path]"` (follows the name): see "Keeping content off your node" | **Transfers** @@ -761,10 +763,12 @@ also shared from your node (see `provide_strategy`). |---|---| | `rnfs pin add TARGET [--background]` | Fetch completely and pin (share) | | `rnfs pin rm CID` | Remove your pin | -| `rnfs pin ls [CID] [--json]` | Pins, their sources (`user`, `folder`, `name:…`, `remote:…`, `follow:…`, `lxmf:…`), whether each is a sealed file (`(sealed)`; `"sealed": true`, `false`, or `null` if not stored enough to tell) and where each came from: the local path you added it from (`~/notes.txt`, `(stdin)`, `archive …`, the target given to `pin add`), the shared-folder file name, or for a directory its single entry's name or entry count. These names are kept only on your machine, sealed files' too (their sealed name is what others can't read): anything that publishes a pin list must leave sealed entries out; pins made before this was added show none for plain files | +| `rnfs pin ls [CID] [--json]` | Pins, their sources (`user`, `folder`, `name:…`, `remote:…`, `follow:…`, `lxmf:…`, `deny`), whether each is a sealed file (`(sealed)`; `"sealed": true`, `false`, or `null` if not stored enough to tell) and where each came from: the local path you added it from (`~/notes.txt`, `(stdin)`, `archive …`, the target given to `pin add`), the shared-folder file name, or for a directory its single entry's name or entry count. These names are kept only on your machine, sealed files' too (their sealed name is what others can't read): anything that publishes a pin list must leave sealed entries out; pins made before this was added show none for plain files | | `rnfs pin remote add NODE_ID CID` / `rnfs pin remote rm NODE_ID CID` | Ask another node to pin or unpin (it must list your identity in `remote_pin_identities`) | | `rnfs gc` | Evict every cached download | | `rnfs verify` | Re-check every stored block against its CID | +| `rnfs deny add ENTRY…` / `rnfs deny rm ENTRY…` | Add CIDs or file hashes (`sha256:`) to your local deny list, or remove them | +| `rnfs deny ls` | Your local deny list, then each `deny_lists` list: the version in use, its entry count and state | **Background jobs** @@ -838,6 +842,65 @@ Nicks are 1–32 characters of `a-z 0-9 - _ .`, starting with a letter or digit. - Pages are sandboxed: they can show files of their own content (images, scripts, style sheets), but no page can read gateway responses with `fetch()`. Sealed files are never decrypted by the gateway. +# Building apps on rnfs + +Apps talk to a running daemon through its local API, as the CLI does. `rnfs.apiclient` (standard library only, +and it doesn't load Reticulum, so it starts fast) finds the rnfs directory and API token as the CLI does, makes +whatever listens on the API port prove it knows the token before sending it, and raises a typed error for each +failure: + +```python +from rnfs.apiclient import Client, DaemonNotRunning, NotFound + +api = Client() # the daemon using ~/.rnfs/; Client("/srv/rnfs") for another rnfs dir +try: + cid = api.add_path("/home/me/photos")["cid"] + api.name_publish("photos", cid) + with api.cat(f"{cid}/cat.jpg") as stream, open("cat.jpg", "wb") as out: + for chunk in stream: + out.write(chunk) + print(api.name_resolve("ab" * 16)["cid"]) +except DaemonNotRunning: + print("start it with: rnfs daemon") +except NotFound as e: + print(e) +``` + +It has methods for status, adding (a path or a stream), fetching and `cat`, pins, names and jobs; `get()` and +`post()` reach every other endpoint (`rnfs/api.py` lists them). The module's docstring is the reference. Run +the app as the daemon's user: only it can read the token. + +# Keeping content off your node + +If you run a public gateway or a provider node, a deny list keeps chosen content off it. Entries are CIDs and +file hashes (`sha256:` and the 64 hex digits of a file's SHA-256, as `sha256sum` prints it): + +```sh +rnfs deny add bafkreidhtnhwyqkmbhxvp2mrery66f4kpdsfbxew3qrfqrbtwbs6zb5zce +rnfs deny add sha256:9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08 +rnfs deny ls +``` + +To use a list someone maintains, put its CID in `deny_lists` in `rnfs.toml`, or `name/` to follow the +updates of a list published under a name. A list is a text file with one entry per line; `#` starts a comment. + +```toml +deny_lists = ["name/3f2a9c7e1b4d5a6f8e0c1d2b3a4f5e6d/denylist.txt"] +``` + +A denied CID is never fetched, read, stored or served by your node, whether it is the content asked for or +anywhere inside it, and nothing holding it is provided (records already published are withdrawn). `get`, +`cat`, the gateway (status 451), `pin add`, remote pins, followed names and LXMF auto-fetch all refuse it. When +an entry is added, cached downloads holding it are deleted and pins made for other nodes (`pin remote`) are +released; your own pins are kept, but not served or read, and the log names each one so you can `rnfs pin rm` +it. + +A file hash is only known once the file has been downloaded, so it stops `get`, `cat` and the gateway from +handing the file out (they check before writing or sending anything, which makes them fetch the whole file +first) and `add` or `import` from adding it; it doesn't stop the blocks being downloaded, pinned or served. +Deny the CID to stop that. Lists larger than 4 MiB, with over 100 000 entries, or with a line that isn't an +entry are refused, and the previous version stays in use. + # Example use cases **Community library over LoRa.** A few well-placed nodes run as DHT servers. The librarian keeps a diff --git a/design.md b/design.md index 7a3200c..49835ff 100644 --- a/design.md +++ b/design.md @@ -12,7 +12,8 @@ authority on behaviour; this file fixes module boundaries and the interfaces bet `RATES`, `DURATIONS`, `FLAGS`, `IDENTITY_LISTS`), and the daemon exits on an invalid one. Reticulum is started from `reticulum_dir`, by default RNS's own config directory, so the node attaches to `rnsd` with the matching RPC key. -- `rnfs/errors.py` — `RnfsError` and subclasses `InvalidDag`, `NotFound`, `TooLarge`, `StorageFull`, `Busy`. +- `rnfs/errors.py` — `RnfsError` and subclasses `InvalidDag`, `NotFound`, `TooLarge`, `StorageFull`, `Busy`, + `Denied`. - `rnfs/content/cid.py` — `Cid` (binary form `.binary`, `.codec` `RAW`/`DAG`, `.key` DHT key, `.verify(block)`, `str()` written form, `Cid.parse`, `Cid.from_bytes`, `Cid.for_block`). Code passes `Cid` objects everywhere; strings only at the user edge (CLI, API, gateway, logs) and binary only on the wire. @@ -88,7 +89,7 @@ rnfs/ archive.py export/import stream; archive_size for an exact Content-Length (section 8) shared_folder.py importing /files/ with pin source "folder" (section 8) dht/ no RNS dependency - routing.py RoutingTable: buckets, seniority, failures, bans (section 6) + routing.py RoutingTable: buckets, seniority, failures, unreliable nodes, bans (section 6) providers.py ProviderStore: records, TTL, max_records eviction, /find reply selection (section 6) lookup.py iterative lookups with alpha/beta, time budget, unknown-ID budget (section 6) network/ everything that talks to RNS @@ -104,11 +105,13 @@ rnfs/ identity.py node identity file, node ID helpers mesh.py provider ranking (PathInfo, provider_rank, RnsPaths), airtime policy, Traffic (3, 6) names.py NameManager: name keys, records, publish, resolve, sequences, keeper hook (section 9) + denylist.py DenyList: denied CIDs and file hashes, local entries, accepted lists (section 8) fetcher.py fetching a target from providers; observing() for job progress (sections 6, 7) publisher.py provide strategy, batched /provide, republish, withdrawal, extra key sources (section 6) node.py RnfsNode: wires everything; the facade below node_content.py node side of --no-pin, sealing/unsealing, raw blocks and archive streams api.py local API, 127.0.0.1:8788 (section 11) + apiclient.py client of the local API for apps and the CLI: token, proof check, typed errors (section 11) gateway.py gateway, 127.0.0.1:8789 (section 11) gateway_formats.py ?format=raw|archive (section 11) cli.py command line (section 11) @@ -120,6 +123,7 @@ rnfs/ follow.py followed names (section 9) contacts.py address book (section 11) lxmf.py rnfs: URIs, LXMF send/inbox/outbox/auto-fetch (section 13) + deny.py deny_lists sources: fetched, checked, followed; rnfs deny (section 8) install.sh Linux installer of the latest release (or --main): venv, command, optional systemd user unit; --update, --uninstall contrib/ rnfs.service (systemd user), io.github.afit21.rnfs.plist (launchd), install-windows-task.ps1 @@ -169,6 +173,11 @@ packaging/ requirements.lock (hash-pinned runtime dependencies), upda targets only then. - protection: `with storage.protect() as session:` — blocks stored or listed in `session.add(cid)` are kept as if pinned until the block exits (fetches, adds, imports in progress). +- deny list: `storage.deny` (`denylist.NONE` until the node sets its `DenyList`; the fetcher and publisher read + it there too). `_put` raises `Denied` for a denied CID, before anything is written, even one already stored. + `drop_denied(cids)` drops the cached fetches whose root, path or stored DAG holds one and collects as `gc` + does; `denied_roots(roots, cids)` says which roots reach one through stored nodes; `collect()` deletes + whatever nothing reaches (what a refused add or import left). - `check_disk(nbytes)` raises `StorageFull` when storing `nbytes` would leave less than the disk reserve (max(256 MiB, 2% of the filesystem)) free. `max_storage` itself never refuses a fetch: it is enforced by eviction once fetches end. @@ -191,11 +200,28 @@ packaging/ requirements.lock (hash-pinned runtime dependencies), upda a fetch replaces it. Fetches treat such a block as not stored: deleted and fetched again. - `dag_blocks(cid) -> set[Cid]` the blocks of a complete DAG (memoised), used for "what a node serves". +`denylist.py` — `parse_entry(text) -> (canonical, binary CID | None, digest | None)`, `parse_list(bytes) -> +(binary CIDs, digests)` (ValueError naming the line; `MAX_LIST_BYTES` 4 MiB, `MAX_LIST_ENTRIES` 100 000). +`DenyList(state_dir)`: `cids` and `hashes` (frozensets swapped whole, read without a lock: every check is one +set lookup), `denies(cid)`, `denies_hash(digest)`; the local list `entries()`, `add(texts)`, `remove(texts)` +(`deny_entries`, at most `MAX_LOCAL_ENTRIES`); sources `accept(source, cid, entries)` (saved, then applied), +`forget(source)`, `accepted()`, `in_use(source)`, `entry_count(source)` (`deny_lists`: the CID each source last +accepted; `reload(read)` reads every one back from storage at start); `known_hash(cid)` / `remember_hash` +(a file CID's content SHA-256, `MAX_KNOWN_HASHES` LRU); `add_listener(fn)`, called after each change outside +its lock. `walker.write_out(..., check)` and `walker.iter_files` and `builder.checking(check)` (a ContextVar: +`build_file` hashes each file and passes the digest before the file's root node is stored) are its hooks in +the content layer; `archive.import_archive(..., check)` runs on the complete roots before they are pinned. + ## DHT layer interfaces `routing.py` — `RoutingTable(own_id, config, clock=time.monotonic)`: -`heard(node_id) -> bool` (proven server: announce in server mode, or answered `/find`), `failed(node_id) -> bool` -(True if dropped), `remove(node_id)`, `ban(node_id)`, `is_banned(node_id)`, `closest(key, count=K, for_reply=False)`, +`heard(node_id) -> bool` (proven server: announce in server mode, or answered `/find`), `answered(node_id) -> bool` +(`heard`, and reliable again), `failed(node_id) -> bool` (True if dropped), `unreliable(node_id)` (its last +`max_failures` queries or more failed, in the table or not; announces don't clear it: `_unanswered`, at most +`MAX_UNRELIABLE` (4096) nodes, least recently failed forgotten first), `may_retry(node_id)` (due again after +`AVOID_AFTER` (300 s), doubling per further failure up to `AVOID_MAX` (3600 s); True claims the retry, pushing +the next one back a further backoff, so concurrent lookups don't all retry it), `remove(node_id)`, `ban(node_id)`, +`is_banned(node_id)`, `closest(key, count=K, for_reply=False)`, `seniority(node_id) -> float | None` (seconds in the table), `is_senior(node_id, min_age)`, `standing(node_id, min_age) -> int` (0 senior and healthy, 1 healthy, 2 stale, 3 absent: ranks keepers), `standings(min_age) -> dict` (every node's standing now: the keeper query's snapshot from before its lookup), `contains(node_id)`, @@ -222,8 +248,12 @@ publishers whose last record the pool evicted (`_evicted`) for a TTL too, at mos `lookup.py` — network access is injected: `FindQuery = Callable[[node_id, key], tuple[list[bytes], list[bytes]] | None]` returning (providers, nodes). -`Lookup(key, start_nodes, query, own_id, alpha, beta, k, timeout, is_known, on_answer, on_failure)` with -`find_closest() -> list[bytes]` and `find_providers(on_provider, should_stop) -> list[bytes]`. `is_known(id)` +`Lookup(key, start_nodes, query, own_id, alpha, beta, k, timeout, is_known, on_answer, on_failure, exclude, report, +unreliable, may_retry)` with `find_closest() -> list[bytes]` and `find_providers(on_provider, should_stop) -> +list[bytes]`. A run ranks the nodes `unreliable` named when it learnt of them (`doubtful`) after every other, for +its queries, its termination and `find_closest`'s result; one that distance alone would rank within the query +window is asked again when `may_retry` says so, as a query beyond `alpha` (`retrying`) that the run waits for +only if the node comes to rank within the window by `rank` (it is then counted in flight). `is_known(id)` says whether RNS knows its identity; at most `alpha` unknown IDs are tried per lookup (`allow_unknown(id)`, also used by the fetcher for provider IDs). `listing_count()` is how many nodes have answered with at least one provider, `listing_nodes()` which, and `listers(provider)` which listed a given provider, so the keeper search @@ -257,6 +287,18 @@ RNS recalls, and only if its hash is the node ID asked for (in private mode, als ## Node facade (used by api.py, gateway.py and cli.py) +Apps and the CLI reach it through the local API with `rnfs/apiclient.py` (`Client`), which imports neither RNS +nor anything that does. `Client(rnfs_dir, port, token_file, timeout)` takes the port from `rnfs.toml` +(`api_port()`, `SettingsError`) and the token from `/api_token` (`read_token`, `TokenError`); its +first request needing the token proves the daemon first (`verify_daemon`, under a lock, so concurrent first +requests prove it once), which also reads the daemon's version for `check_daemon`, the one place a version +check goes. `request()` maps failures to `DaemonNotRunning`, `ConnectionFailed`, `AuthFailed` (a failed proof, +or 403) and the API's statuses (`STATUS_ERRORS`, `error_for`); `Stream` reads a body, raising +`ConnectionFailed` if it ends short. Its methods (`status`, `add_path`, `add_stream`, `fetch`, `cat`, `info`, +`ls`, `stat`, `pin_*`, `name_*`, `submit_job`, `job`, `jobs`, `cancel_job`, `clear_jobs`) send only the +parameters given. The CLI's `ApiClient` subclasses it, turning each `ApiError` into `fail(str(e))`, so its +messages are the library's. + `RnfsNode(config)` exposes `node_id: bytes`, `config`, and these methods. Targets are strings `[/name…]`; CIDs are returned as strings. Return values are JSON-ready dicts. @@ -305,7 +347,7 @@ each node, so a withdrawal queued while the key was briefly unprovided never fol `protocol.provide_batches` puts withdrawals before stores, and a key in both lists is only stored. Errors: `ValueError` → 400, `NotFound` → 404, `TooLarge` → 413, `InvalidDag` → 422, `Busy` → 503, -`StorageFull` → 507, other `RnfsError` → 500. Paths passed to `add_path`, `get`, `export` and `import_archive` +`StorageFull` → 507, `Denied` → 451, other `RnfsError` → 500. Paths passed to `add_path`, `get`, `export` and `import_archive` are absolute paths on the daemon's machine (the CLI resolves them); stdin is uploaded as a request body. **Core API routes** (`rnfs/api.py` `ROUTES`): `GET /api/status` (open), `/api/info`, `/api/cat`, `/api/ls`, @@ -316,10 +358,9 @@ are absolute paths on the daemon's machine (the CLI resolves them); stdin is upl an optional `from` (node ID); `GET /api/name/resolve?name=` takes a name or a name contact's nick. The API requires `X-Rnfs-Api: ` on every request except `GET /api/status`, which without a valid token -answers only `{"version"}` (health checks, and `verify_daemon`, which reads only the proof header); the CLI sends -the token with every other request, `rnfs status` included (without a readable token it prints the version -and says why). The daemon writes a new random -token to `/api_token` (mode 0600, in the 0700 rnfs dir) at each start and the CLI reads it, so other +answers only `{"version"}` (health checks, and `verify_daemon`, which reads the proof header, then the +version); the CLI sends the token with every other request, `rnfs status` included (without a readable token it +prints the version and says why). The daemon writes a new random token to `/api_token` (mode 0600, in the 0700 rnfs dir) at each start and the CLI reads it, so other local users and processes can't drive the API. Before anything else uses the rnfs dir (the debug log, the node), the daemon locks `/daemon.lock` (`fsutil.lock_exclusive`: `flock`, or `msvcrt.locking` of a byte past the text on Windows; the file holds its process ID) and keeps the lock until it exits, so a second daemon @@ -336,9 +377,9 @@ files get the same DACL before they are renamed into place (`fsutil.write_atomic read from (`winacl.tighten(fd=...)`): another account's is refused, a loose one tightened with a warning, and a failure is fatal. A `--dir` outside `%USERPROFILE%` is warned about. The advapi32 and kernel32 calls are behind `winacl.backend`, which tests replace with a fake. A response to a request with -`X-Rnfs-Challenge: <32 hex>` carries `X-Rnfs-Proof: cli.api_proof(token, port, challenge)` (HMAC-SHA256 of -`rnfs api proof `); `ApiClient.verify_daemon` checks it with `GET /api/status` before first -sending the token. Both HTTP servers (`TimedHandler`) time out slow requests, cap concurrent connections (API +`X-Rnfs-Challenge: <32 hex>` carries `X-Rnfs-Proof: apiclient.api_proof(token, port, challenge)` (HMAC-SHA256 of +`rnfs api proof `); `apiclient.Client.verify_daemon` checks it with `GET /api/status` before +first sending the token. Both HTTP servers (`TimedHandler`) time out slow requests, cap concurrent connections (API 32, gateway 64) and refuse HTTP/0.9 with a 400 carrying the usual headers. Gateway errors raised by the node (anything but `GatewayError` and `NotFound`: 400, 413, 422, 503, 507, 500) say only a fixed message per status (`gateway.ERROR_MESSAGES`; the detail goes to the log); malformed `Range` headers are ignored. @@ -445,7 +486,7 @@ and in total, never persisted. ## Features `rnfs/features/__init__.py` defines `Feature` and `FEATURE_MODULES` (in start order: peers, jobs, keepers, -follow, contacts, lxmf). The node builds every feature in its constructor (`node.feature(name)`), starts them +follow, contacts, lxmf, deny). The node builds every feature in its constructor (`node.feature(name)`), starts them after its own threads and stops them, in reverse, before its own services. A feature may override: `start()`, `stop()`, `save()` (called with the node's periodic saves), `status()` (fields merged into `rnfs status`), `requests()` (new `rnfs.dht` paths; the request server applies private mode and the per-link @@ -461,6 +502,7 @@ and their constructors work on `RnfsNode.offline` test nodes. | `follow` | `follow.py` | followed names: each check and fetch one at a time, `fetcher.open(pinned=True)`, pin inside the fetch's protection, old pin released after | keepers' `FOLLOW` reason | | `contacts` | `contacts.py` | address book; `expand_target`, `lookup(nick, kind)`, `node_arg` | `RnfsNode._parse_target` (never for the gateway), `JobsFeature.submit` | | `lxmf` | `lxmf.py` | pure parts (`Link`, `parse_uri`, `format_uri`, `find_links`, `encode_field`/`decode_field`, `MessageLog`); `LxmfFeature` (send with direct→propagated fallback, inbox, auto-fetch, inbox get, pin release); `LxmfNetwork`, made in `start()`, holds every RNS/LXMF object | `lxmf.validate_settings` from `Config.validate`; `LXMF_PREFIX` in `storage.check_source` | +| `deny` | `deny.py` | `deny_lists` sources (`parse_source`, `validate_settings` from `Config.validate`): `check(source)` resolves a name (persisted), fetches the target pinned, checks it is a file within `MAX_LIST_BYTES`, parses it, pins it `deny` and `accept`s it, then releases unused `deny` pins; `run_due` on `rnfs-deny`, after the join (`CHECK_INTERVAL` for names, `retry_delay` backoff); sources no longer configured are forgotten at start | `node.deny` (core); `DENY` in `storage.check_source` | `LxmfNetwork` makes an `LXMF.LXMRouter` with storage `/lxmf/` and `signal.signal` stubbed (LXMRouter installs SIGINT/SIGTERM handlers that exit at once), a 256 kB delivery limit, announce data without @@ -503,7 +545,8 @@ than raised. An output without `-o` is named with `fsutil.safe_default_name`. Se `/api/name/unkeep`, `GET /api/name/kept`; `POST /api/name/follow`, `/api/name/unfollow`, `GET /api/name/following`; `GET /api/contacts`, `GET /api/contacts/get?nick=`, `POST /api/contacts/add`, `/api/contacts/rm`; `GET /api/lxmf/address`, `POST /api/lxmf/send`, `GET /api/lxmf/inbox`, -`POST /api/lxmf/inbox/get`, `/api/lxmf/inbox/rm`, `GET /api/lxmf/outbox`, `POST /api/lxmf/sync`. `POST /api/jobs` +`POST /api/lxmf/inbox/get`, `/api/lxmf/inbox/rm`, `GET /api/lxmf/outbox`, `POST /api/lxmf/sync`; `GET /api/deny`, +`POST /api/deny/add`, `/api/deny/rm` (`{"entries": [...]}`). `POST /api/jobs` takes, for a get, the JSON booleans `sealed_name` (write a sealed file this node opens under its safe sealed name in the output's directory; the CLI sets it for `rnfs get ` without `-o`, so it no longer calls `GET /api/info` first) and `fallback_dht` (a get with `from` that fails, other than by cancellation, @@ -524,13 +567,16 @@ name in the output's directory; the CLI sets it for `rnfs get ` withou | `rnfs-job-` | jobs | one per running job, at most `max_jobs`; `stop()` waits ≤ 2 s | | `rnfs-keepers`, `rnfs-keeper-query` | keepers | pending refreshes (≤ 64 queued), own-name sync, saves, re-asking every `announce_interval`; parallel `/name` queries | | `rnfs-follow` | follow | every check and fetch, one at a time | +| `rnfs-deny` | deny | every deny-list source's check and fetch, one at a time, once joined | | `rnfs-lxmf`, `rnfs-lxmf-fetch` | lxmf | announces, propagation-node downloads, polling outbound states each second; auto-fetch, one at a time, senders in turn (LXMF runs its own job thread) | The API and gateway run one thread per connection, capped (32 and 64). Locks are taken in this order, never the reverse: `names._lock` → `publisher._lock` → `node._lock` → `storage._lock` → the state database's lock (`statedb`, innermost: a store opens a transaction only once it -holds every lock the change needs, and inside it takes no other lock and does no network I/O). `NameManager.publish`, `remove` and `release_gone` pin and unpin, and so call +holds every lock the change needs, and inside it takes no other lock and does no network I/O). `node._deny_lock` +(one deny-list change applied at a time, `_apply_deny`) is taken before the publisher's lock and never with the +names lock; `DenyList._lock` holds only the state database's. `NameManager.publish`, `remove` and `release_gone` pin and unpin, and so call `publisher.refresh()`, while holding the names lock; `Publisher.refresh` recomputes what is provided under its own lock, reading the node's fetched targets (`node._lock`) and walking the pins' DAGs (`storage._lock`); the node reads storage under its lock (`_prune_fetched`). This is how the code behaves today, not a design goal. @@ -564,13 +610,15 @@ change returns, "NORMAL" ones are batched saves a power loss may lose the last o - `names_published`, `names_held` (sequences as decimal text, up to 2^64 − 1; FULL, changed rows only; a malformed row stops startup, as a malformed `names.json` did, since skipping it could allow rollback) and `name_floor` (highest sequence of names resolved for the gateway or auto-fetch and when, in `used` order, at - most 4 × `name_table_size`, or 8 × for names resolved within 7 days; NORMAL, saved by a resolve raising it, at - most every 5 s, and by the re-announce loop when changed); + most 4 × `name_table_size`, or 8 × for names resolved within 7 days; NORMAL, saved by a resolve or record + raising it, at most every 5 s, and by the re-announce loop when changed); - `shared_folder` (FULL); `jobs` (record per job in creation order; FULL, including progress saves; results keep at most 100 warnings plus `warnings_total`, older results capped at load); `keepers` (records re-verified at load; NORMAL); `follow` (FULL); `contacts` (FULL); `lxmf_messages` (inbox: 500 messages, 4 MiB, 50 per non-auto-fetch sender, 16 KiB per record; outbox 200, unfinished messages marked failed at a restart; NORMAL, changed messages only; ids and `next_id` stay below 2^63); +- `deny_entries` (the local deny list, canonical text; FULL) and `deny_lists` (each source as `dump()`ed text, the + CID of the version it last accepted; FULL); - `meta` (per-store values: follow's and each box's `clean`, set by every save, which the reconcile at start requires; each box's `next_id`) and `migrated` (JSON files imported). @@ -598,6 +646,9 @@ writes it again). CI runs them on Linux, macOS and Windows; `packaging/smoke_tes Everything in the specification is implemented, with these simplifications and additions: - The routing table isn't persisted. At startup it is seeded from the rnfs.dht server announces that RNS cached within 2 × `announce_interval` (`Identity.known_destinations`, an RNS internal), then the join runs. +- Lookups (`RnfsNode._lookup`) start from the closest nodes `/find` replies would give, then others to make up K, + then unreliable ones if still short; answers go to `RoutingTable.answered`, failures to `failed`, and the + join's lookup is wired the same way. A failed `/provide` or `/blocks` isn't counted there. - Joining (`RnfsNode._join`, `join`): bootstrap nodes are asked one at a time with `PeerClient.find`, outside any `Lookup`, until `BOOTSTRAP_ANSWERS` (2) answer; the lookup that follows excludes them. The thread then checks `RoutingTable.live_count()` every `REJOIN_DELAY` (300 s) and joins again while it is under @@ -774,10 +825,27 @@ Everything in the specification is implemented, with these simplifications and a no waiting resolve, of a name without a kept record, may only evict one of those. A resolve with `persist=False` (gateway, LXMF auto-fetch) raises the name's floor (`_floor`: name -> (sequence, wall time last resolved), in resolve order), which `_offer` checks. A raise (or a new name) saves the floor - at once if the last save was `FLOOR_SAVE_INTERVAL` (5 s) ago, else from a one-shot `threading.Timer`; + at once if the last save was `FLOOR_SAVE_INTERVAL` (5 s) ago, else from a one-shot `threading.Timer`; a + raise by a record heard for a name in the table (`_store`: an announce, a keeper's record, or one a resolve's + refresh heard) is always the timer's, started at once when a save is due, since its callers hold the lock; renewing only the time marks it for the next save after an hour. `_trim_floor` drops the least recently resolved past `FLOOR_FACTOR` × `name_table_size`, but stops at one resolved within `FLOOR_KEEP_AGE` (7 days) until the floor holds `FLOOR_HARD_FACTOR` (2) times that. +- **Deny lists.** `RnfsNode._setup` makes the `DenyList` right after storage, gives it to storage + (`storage.deny`), and `reload`s the accepted lists from their pinned blocks (`_read_deny_list`), so the + publisher's first refresh already leaves out denied content. Fetches check every expectation in `_ensure` + and the root in `Fetcher.open`, and a complete stored DAG in `fetch_dag`'s early return (`dag_blocks`). + `NodeBackend.served_block`/`_size` answer None for a denied CID; `Publisher._compute` drops CIDs whose + `dag_blocks` hold one (`_denied_dags`, memoised per DAG and deny set). `_apply_deny` (the list's listener, and + maintenance when `deny.cids` changed, as at start) refreshes the publisher, `drop_denied`s the cache, releases + `remote:` pins of `denied_roots` and collects, and warns about the other pins it keeps. File hashes: + `_hash_check` (for `builder.checking`, in adds, the shared folder and imports, under `_refusing`, which + collects what a refused add or import stored) and `_check_file(expect, read)` (get's write-out check, a sealed + get, `read` before its first byte, `node_content.block` for a raw block and `export_stream` for every file + before `start`; a raw file's digest is its CID's, others are hashed once and remembered). A failed + `_apply_deny` leaves `_deny_applied` as it was, so maintenance tries again after `DENY_RETRY` (60 s). + `remote_pin` refuses a denied CID; jobs don't fall back to the DHT on `Denied`, and LXMF auto-fetch records + it as refused. - **Follow** isn't built on jobs (job pins use source `user`); `pin add --follow` doesn't exist. Checks use the record held unless nothing was heard for 2 × `announce_interval`; a failed check (any exception) is retried after 300 s, doubling up to `announce_interval`, and a failed fetch is retried with the record held diff --git a/rnfs/api.py b/rnfs/api.py index c54bf1d..ce01490 100644 --- a/rnfs/api.py +++ b/rnfs/api.py @@ -44,12 +44,13 @@ the API never sends CORS headers, and a web page can't know the token anyway. Status stays open for health checks and the CLI's proof check below, but without the token it answers only `{"version": ...}`: the node and identity hashes, LXMF address and traffic counters need the token like everything else. -- A request carrying `X-Rnfs-Challenge: <32 hex digits>` gets `X-Rnfs-Proof: ` in its - response, which only a server knowing the token can make. Before it first sends the token, the CLI makes - whatever listens on the port prove this (with a status request), so another local user listening there - while the daemon is down learns neither the token nor a name key being imported. -- Errors map to statuses as design.md says: ValueError 400, NotFound 404, TooLarge 413, InvalidDag 422, - Busy 503, StorageFull 507, other errors 500. +- A request carrying `X-Rnfs-Challenge: <32 hex digits>` gets `X-Rnfs-Proof: ` in its + response, which only a server knowing the token can make. Before it first sends the token, a client + (rnfs.apiclient, which the CLI uses) makes whatever listens on the port prove this (with a status request), + so another local user listening there while the daemon is down learns neither the token nor a name key being + imported. +- Errors map to statuses as design.md says: ValueError 400, NotFound 404, TooLarge 413, Denied 451 (the deny + list), InvalidDag 422, Busy 503, StorageFull 507, other errors 500. Connections (both this API and the gateway, through LocalServer): - The request line, headers and any JSON body must arrive within request_timeout of connecting, else the @@ -73,13 +74,12 @@ from rnfs import __version__, fsutil from rnfs import node_content # --- content: add options (pin, to, chunker) --- -from rnfs.cli import CHALLENGE_HEADER, PROOF_HEADER, api_proof +from rnfs.apiclient import API_HEADER, CHALLENGE_HEADER, PROOF_HEADER, api_proof from rnfs.config import _HEX_ID # exactly 32 hex digits, as names.parse_name and identity lists are checked from rnfs.content.cid import Cid -from rnfs.errors import Busy, InvalidDag, NotFound, StorageFull, TooLarge +from rnfs.errors import Busy, Denied, InvalidDag, NotFound, StorageFull, TooLarge from rnfs.log import LOG_ERROR, log -API_HEADER = "X-Rnfs-Api" MAX_JSON_BODY = 1024 * 1024 # JSON parameters are tiny; this only stops a runaway body filling memory MAX_KEY_BODY = 64 * 1024 # an RNS identity file is 64 bytes CHALLENGE = re.compile(r"[0-9a-f]{32}") # what the CLI sends in X-Rnfs-Challenge @@ -87,7 +87,7 @@ MAX_TRAILER_LINES = 32 # Exception class → HTTP status, checked in order (design.md); anything else is 500 -ERROR_STATUSES = [(ValueError, 400), (NotFound, 404), (TooLarge, 413), (InvalidDag, 422), (Busy, 503), +ERROR_STATUSES = [(ValueError, 400), (NotFound, 404), (TooLarge, 413), (Denied, 451), (InvalidDag, 422), (Busy, 503), (StorageFull, 507)] diff --git a/rnfs/apiclient.py b/rnfs/apiclient.py new file mode 100644 index 0000000..302ec2e --- /dev/null +++ b/rnfs/apiclient.py @@ -0,0 +1,487 @@ +"""A client of a running daemon's local API (specification section 11; rnfs/api.py), for companion apps and the CLI. + +Standard library only, and it never imports RNS, so an app using it starts quickly: + + from rnfs.apiclient import Client, NotFound + + api = Client() # the daemon using ~/.rnfs/, on the api_port in its rnfs.toml + print(api.status()["node_id"]) + cid = api.add_path("/home/me/photos")["cid"] + with api.cat(f"{cid}/cat.jpg") as stream, open("cat.jpg", "wb") as out: + for chunk in stream: + out.write(chunk) + try: + print(api.name_resolve("ab" * 16)["cid"]) + except NotFound: + print("not resolved") + +- The daemon writes a new random token to `/api_token` (mode 0600, or a private access list on + Windows) at each start. The client reads it before its first request that needs it and sends it as + `X-Rnfs-Api` with every request but the open status (version()). +- Before sending the token, the client makes whatever listens on the API port prove it knows it (api_proof, + verify_daemon), since while the daemon is down any local user could listen there. Nothing else is sent to a + listener that fails: neither the token nor a request's body. +- That proof request is also where the daemon's version is read (Client.daemon_version), and check_daemon is the + one place to refuse a daemon by it; every version is accepted for now. +- Every failure raises an ApiError subclass whose message says what went wrong, as the CLI prints it: no daemon + (DaemonNotRunning), the connection failing (ConnectionFailed), the token (TokenError), the proof or the token + refused (AuthFailed), the settings (SettingsError), and the API's own error statuses (STATUS_ERRORS: 400 + BadRequest, 404 NotFound, 413 TooLarge, 422 InvalidDag, 451 Denied, 503 Busy, 507 StorageFull, other 5xx + ServerError). +- Each request opens its own connection, with no timeout unless one is given (fetches can take hours), so a + Client may be used from several threads. Paths are on the daemon's machine, which is this one: they are made + absolute here. +""" +import hashlib +import hmac +import http.client +import json +import os +import secrets +import stat +import threading +from urllib.parse import urlencode + +from rnfs import fsutil, winacl +from rnfs.config import DEFAULT_API_PORT, Config + +DEFAULT_DIR = Config.rnfs_dir +TOKEN_FILE = "api_token" # in the rnfs directory; written by the daemon, read by every client +API_HEADER = "X-Rnfs-Api" +# Before sending the token, a client makes whatever listens on the API port prove it knows it (api_proof) +CHALLENGE_HEADER = "X-Rnfs-Challenge" +PROOF_HEADER = "X-Rnfs-Proof" +COPY_SIZE = 64 * 1024 # what a Stream reads at a time when iterated + + +class ApiError(Exception): + """An API call failed; str() says why. status is the HTTP status, or None if the daemon never answered.""" + + def __init__(self, message: str, status: int | None = None): + """An error with message, from a response with status (None: no response).""" + super().__init__(message) + self.status = status + + +class DaemonNotRunning(ApiError): + """Nothing listens on the API port.""" + + +class ConnectionFailed(ApiError): + """The daemon dropped the connection, couldn't be reached, cut a response short or sent an invalid one.""" + + +class TokenError(ApiError): + """The API token file is missing, unreadable, empty, or readable by other users (so not to be trusted).""" + + +class AuthFailed(ApiError): + """What listens on the API port couldn't prove it knows the token (nothing was sent to it), or the daemon + refused the request (403).""" + + +class SettingsError(ApiError): + """The rnfs dir's rnfs.toml couldn't be read for the API port.""" + + +class BadRequest(ApiError): + """400: the request's parameters were refused (the node raised ValueError).""" + + +class NotFound(ApiError): + """404: no such content, name, pin, job or endpoint.""" + + +class TooLarge(ApiError): + """413: over a size limit (max_download_size, a quota, the request body).""" + + +class InvalidDag(ApiError): + """422: the content fetched isn't a valid DAG.""" + + +class Denied(ApiError): + """451: the content is on the node's deny list.""" + + +class Busy(ApiError): + """503: the daemon is too busy (or full of connections); try again later.""" + + +class StorageFull(ApiError): + """507: not enough free disk space.""" + + +class ServerError(ApiError): + """500 or another 5xx: the daemon failed (see its log).""" + + +# The API's error statuses (design.md, "Node facade"); other 4xx raise ApiError, other 5xx ServerError +STATUS_ERRORS = {400: BadRequest, 403: AuthFailed, 404: NotFound, 413: TooLarge, 422: InvalidDag, 451: Denied, + 503: Busy, 507: StorageFull} + + +def error_for(status: int, message: str) -> ApiError: + """The ApiError subclass for an error response's status, with its message.""" + error_class = STATUS_ERRORS.get(status) or (ServerError if status >= 500 else ApiError) + return error_class(message, status) + + +def token_path(rnfs_dir: str = DEFAULT_DIR) -> str: + """Where the daemon using rnfs_dir keeps its API token.""" + return os.path.join(os.path.expanduser(rnfs_dir), TOKEN_FILE) + + +def api_port(rnfs_dir: str = DEFAULT_DIR) -> int: + """The port the daemon using rnfs_dir listens on: api_port in its rnfs.toml, else DEFAULT_API_PORT. Raises + SettingsError if rnfs.toml can't be read.""" + try: + port = Config.load(rnfs_dir).api_port + except (ValueError, TypeError, OSError) as e: # TypeError: a badly typed setting validate() missed + raise SettingsError(f"can't read the settings in {rnfs_dir}: {e}") from None + return port if port is not None else DEFAULT_API_PORT + + +def api_proof(token: str, port: int, challenge: str) -> str: + """The daemon's answer to a challenge: HMAC-SHA256, keyed by the API token, of the challenge and the port + it listens on (so a program on another port can't pass on the daemon's answer).""" + return hmac.new(token.encode(), f"rnfs api proof {port} {challenge}".encode(), hashlib.sha256).hexdigest() + + +def read_token(path: str) -> str: + """The daemon's API token from path. Raises TokenError if it is missing, unreadable or empty, or if other + users can read it (so it can't be trusted).""" + try: + fd = os.open(path, os.O_RDONLY | fsutil.O_NOFOLLOW | fsutil.O_BINARY) + except FileNotFoundError: + raise TokenError(f"no API token at {path}: is the daemon running, with this --dir?") from None + except PermissionError: + raise TokenError(f"can't read the API token at {path}: permission denied. Is the daemon running as another " + f"user?") from None + except OSError as e: + raise TokenError(f"can't read the API token at {path}: {e}") from None + with os.fdopen(fd, "rb") as f: + mode = stat.S_IMODE(os.fstat(f.fileno()).st_mode) + # Windows has no Unix permission bits to check, but an access list (of the very file opened) + if not fsutil.IS_WINDOWS and mode & 0o077: + raise TokenError(f"{path} is accessible to other users (mode {mode:04o}), so its token can't be trusted; " + f"restart the daemon to make a new one") + try: + reason = winacl.check_private(path, f.fileno()) + except OSError as e: + raise TokenError(f"can't check who may read the API token at {path}: {e}") from None + if reason is not None: + raise TokenError(f"{path} is accessible to other users ({reason}), so its token can't be trusted; " + f"restart the daemon to make a new one") + token = f.read(1024).decode("ascii", "replace").strip() + if not token: + raise TokenError(f"the API token file {path} is empty; restart the daemon") + return token + + +def read_all(response: http.client.HTTPResponse) -> bytes: + """A response's whole body. Raises ConnectionFailed if it is shorter than its Content-Length (the daemon hit + an error).""" + try: + return response.read() + except http.client.IncompleteRead: + raise ConnectionFailed("the response was cut short; see the daemon's log") from None + + +class Stream: + """A response body read as it arrives (cat, name export): read(size), iteration in COPY_SIZE chunks, close(), + and a context manager. The last read raises ConnectionFailed if the body ended before its Content-Length.""" + + def __init__(self, response: http.client.HTTPResponse): + """The body of response, a successful one.""" + self.response = response + header = response.getheader("Content-Length") + try: + self.length: int | None = int(header) if header is not None else None # None: sent until closed + except ValueError: + self.length = None + self.received = 0 + + def read(self, size: int = -1) -> bytes: + """Up to size bytes (all that is left if size is negative); b"" at the end.""" + if size is None or size < 0: + return b"".join(iter(lambda: self.read(COPY_SIZE), b"")) + try: + chunk = self.response.read(size) + except http.client.IncompleteRead: + chunk = b"" + self.received += len(chunk) + if not chunk and size and self.length is not None and self.received != self.length: + raise ConnectionFailed(f"the transfer stopped after {self.received} of {self.length} bytes; " + f"see the daemon's log") + return chunk + + def __iter__(self): + """The body in chunks of at most COPY_SIZE bytes.""" + return iter(lambda: self.read(COPY_SIZE), b"") + + def close(self): + """Close the connection (idempotent).""" + self.response.close() + + def __enter__(self) -> "Stream": + """The stream itself.""" + return self + + def __exit__(self, *exc): + """Close the stream.""" + self.close() + + +def _flag(value: bool) -> int | None: + """A query-string flag: 1 if set, else left out.""" + return 1 if value else None + + +class Client: + """The local API of the daemon using rnfs_dir (default ~/.rnfs/), on 127.0.0.1:port (default: the api_port in + its rnfs.toml). token_file defaults to /api_token and is read on the first request needing it. + + request() and call() are the whole protocol; get() and post() call any endpoint (features' too); the other + methods are the main operations and the deny list, returning what the API answers (design.md, "Node + facade"). Every failure raises an ApiError subclass. + """ + + def __init__(self, rnfs_dir: str | None = None, port: int | None = None, token_file: str | None = None, + timeout: float | None = None): + """A client; nothing is sent until the first request. Raises SettingsError if port isn't given and + rnfs.toml can't be read.""" + rnfs_dir = DEFAULT_DIR if rnfs_dir is None else rnfs_dir + self.port = api_port(rnfs_dir) if port is None else port + self.token_file = token_path(rnfs_dir) if token_file is None else token_file + self.timeout = timeout + self.token: str | None = None # once the daemon has proved it knows it + self.daemon_version: str | None = None # what the daemon said in the proof request + self._token_lock = threading.Lock() + + # The protocol + + def request(self, method: str, path: str, params: dict | None = None, json_body: dict | None = None, + body=None, length: int | None = None, extra_headers: dict | None = None, + authenticated: bool = True) -> http.client.HTTPResponse: + """Send a request and return the successful response, whose body the caller reads (read_all, Stream). + + params go in the query string (None values left out); json_body is sent as JSON; body may be bytes or a + binary file, sent chunked without length. Unless authenticated is False (version(), and the proof + request itself), the token is sent, once the daemon has proved it knows it. + """ + url = path + ("?" + urlencode({k: v for k, v in params.items() if v is not None}) if params else "") + # Host is set explicitly: the API only accepts 127.0.0.1:, even for port 80 + headers = {"Host": f"127.0.0.1:{self.port}", **(extra_headers or {})} + if authenticated: + headers[API_HEADER] = self._proven_token() + if json_body is not None: + body = json.dumps(json_body).encode() + headers["Content-Type"] = "application/json" + elif body is not None: + headers["Content-Type"] = "application/octet-stream" + if length is not None: + headers["Content-Length"] = str(length) + connection = http.client.HTTPConnection("127.0.0.1", self.port, timeout=self.timeout) + try: + connection.request(method, url, body=body, headers=headers) + response = connection.getresponse() + except ConnectionRefusedError: + raise DaemonNotRunning(f"no daemon is running on API port {self.port}. Start one with: rnfs daemon") \ + from None + except (http.client.HTTPException, ConnectionError) as e: + raise ConnectionFailed(f"the daemon dropped the connection ({e!r}); see its log") from None + except OSError as e: + raise ConnectionFailed(f"can't reach the daemon on API port {self.port}: {e}") from None + if response.status >= 400: + data = read_all(response) + try: + message = json.loads(data)["error"] + except (ValueError, KeyError, TypeError): + message = f"HTTP {response.status} {response.reason}" + raise error_for(response.status, str(message)) + return response + + def _proven_token(self) -> str: + """The token, read and proved (verify_daemon) on first use.""" + with self._token_lock: + if self.token is None: + token = read_token(self.token_file) + self.verify_daemon(token) + self.token = token + return self.token + + def verify_daemon(self, token: str): + """Raise AuthFailed unless whatever listens on the API port proves it knows token, before the token (or + a name key) is sent to it: while the daemon is down, any local user could be listening on its port. + Then check_daemon() with its version.""" + challenge = secrets.token_hex(16) + response = self.request("GET", "/api/status", extra_headers={CHALLENGE_HEADER: challenge}, + authenticated=False) + data = read_all(response) + proof = (response.getheader(PROOF_HEADER) or "").strip().encode("ascii", "replace") + if not hmac.compare_digest(proof, api_proof(token, self.port, challenge).encode()): + raise AuthFailed(f"what listens on API port {self.port} couldn't prove it knows the API token in " + f"{self.token_file}, so nothing was sent to it: it may be another program, or a " + f"daemon using another --dir (or one started before an upgrade: restart it)") + try: + version = json.loads(data).get("version") + except (ValueError, AttributeError): + version = None + self.daemon_version = version if isinstance(version, str) else None + self.check_daemon(self.daemon_version) + + def check_daemon(self, version: str | None): + """The one place to refuse a daemon by its version (None if it didn't say), before the token is sent; + raise an ApiError to refuse. Every version is accepted for now.""" + + def call(self, method: str, path: str, params: dict | None = None, json_body: dict | None = None, + body=None, length: int | None = None, authenticated: bool = True): + """Send a request and return its decoded JSON response.""" + response = self.request(method, path, params, json_body, body, length, authenticated=authenticated) + try: + return json.loads(read_all(response)) + except ValueError: + raise ConnectionFailed("the daemon sent an invalid response") from None + + def get(self, path: str, /, **params): + """GET path with query parameters; JSON response.""" + return self.call("GET", path, params=params) + + def post(self, path: str, /, **fields): + """POST path with a JSON body; JSON response.""" + return self.call("POST", path, json_body=fields) + + def _post(self, path: str, /, **fields): + """POST path with the fields that aren't None.""" + return self.post(path, **{name: value for name, value in fields.items() if value is not None}) + + # Status + + def version(self) -> str: + """The daemon's version, from the status anyone may ask for (no token needed or sent).""" + return self.call("GET", "/api/status", authenticated=False)["version"] + + def status(self) -> dict: + """The node's status: node ID, DHT mode, counts, storage and every feature's fields.""" + return self.get("/api/status") + + # Content + + def add_path(self, path: str, wrap: bool = False, only_hash: bool = False, pin: bool = True, + to: list[str] | None = None, to_self: bool = False, seal_name: bool = True, + chunker: str | None = None) -> dict: + """Add a file or directory on this machine: {"cid", "size", "warnings"} (+ "sealed"). pin=False keeps it + as evictable cache; to/to_self seal it for those recipients; chunker is "fixed" or "fastcdc".""" + return self._post("/api/add", path=os.path.abspath(os.path.expanduser(path)), wrap=wrap or None, + only_hash=only_hash or None, pin=None if pin else False, to=to or None, + to_self=to_self or None, seal_name=None if seal_name else False, chunker=chunker) + + def add_stream(self, data, name: str | None = None, length: int | None = None, only_hash: bool = False, + pin: bool = True, to: list[str] | None = None, to_self: bool = False, + seal_name: str | None = None, chunker: str | None = None) -> dict: + """Add bytes, or a binary file read to its end (sent chunked unless length is given): as add_path. name + wraps it in a directory as that entry; when sealing, seal_name is the name sealed inside.""" + if isinstance(data, (bytes, bytearray, memoryview)) and length is None: + length = len(data) + params = {"name": name, "only_hash": _flag(only_hash), "pin": None if pin else 0, + "to": ",".join(to) if to else None, "to_self": _flag(to_self), "seal_name": seal_name, + "chunker": chunker} + return self.call("POST", "/api/add/stream", params=params, body=data, length=length) + + def fetch(self, target: str, output: str, from_node: str | None = None, raw: bool = False) -> dict: + """Fetch a file or directory and have the daemon write it to output (a path on this machine, which must + not exist), in one request: {"cid", "warnings"} (+ "sealed", "name"). For progress, use submit_job.""" + return self._post("/api/get", target=target, output=os.path.abspath(os.path.expanduser(output)), + raw=raw or None, **{"from": from_node}) + + def cat(self, target: str, offset: int | None = None, length: int | None = None, raw: bool = False, + block: bool = False) -> Stream: + """A file's bytes (or a range of them) as they arrive. raw: a sealed file as stored; block: the CID's own + block, verified.""" + return Stream(self.request("GET", "/api/cat", params={"target": target, "offset": offset, "length": length, + "raw": _flag(raw), "block": _flag(block)})) + + def info(self, target: str, from_node: str | None = None) -> dict: + """{"cid", "type", "size", "executable"} (+ sealed-file fields) of a target.""" + return self.get("/api/info", target=target, **{"from": from_node}) + + def ls(self, target: str) -> list[dict]: + """A directory's entries: [{"name", "kind", "size", "cid"}].""" + return self.get("/api/ls", target=target)["entries"] + + def stat(self, target: str) -> dict: + """{"cid", "size", "blocks", "local_blocks", "local_bytes"} of a target.""" + return self.get("/api/stat", target=target) + + # Pins + + def pin_add(self, target: str) -> dict: + """Fetch and pin a target, in one request: {"cid"} once complete. For progress, use submit_job.""" + return self.post("/api/pin/add", target=target) + + def pin_rm(self, cid: str): + """Remove the user pin of cid (NotFound if it has none).""" + self.post("/api/pin/rm", cid=cid) + + def pin_ls(self, cid: str | None = None) -> list[dict]: + """Every pin (or cid's): [{"cid", "sources", "complete", "type", "sealed", "name"}].""" + return self.get("/api/pin/ls", cid=cid)["pins"] + + # Names + + def name_list(self) -> list[dict]: + """This node's names: [{"alias", "name", "cid", "sequence"}].""" + return self.get("/api/name/list")["names"] + + def name_create(self, alias: str) -> dict: + """Create a name key: {"alias", "name"}.""" + return self.post("/api/name/create", alias=alias) + + def name_publish(self, alias: str, target: str) -> dict: + """Point a name of this node at a target: {"name", "cid", "sequence"}.""" + return self.post("/api/name/publish", alias=alias, target=target) + + def name_resolve(self, name: str) -> dict: + """{"name", "cid", "sequence", "age"} of a name (32 hex characters, or a name contact's nick); NotFound if + it doesn't resolve.""" + return self.get("/api/name/resolve", name=name) + + # The deny list + + def deny(self) -> dict: + """{"entries": [the local list], "lists": [{"source", "cid", "entries", "state"}]}.""" + return self.get("/api/deny") + + def deny_add(self, entries: list[str]) -> list[str]: + """Add CIDs or file hashes (sha256:) to the local deny list; those new.""" + return self.post("/api/deny/add", entries=list(entries))["added"] + + def deny_rm(self, entries: list[str]) -> list[str]: + """Remove entries from the local deny list (NotFound, removing none, if one isn't on it).""" + return self.post("/api/deny/rm", entries=list(entries))["removed"] + + # Jobs (background transfers, with progress) + + def submit_job(self, kind: str, target: str, output: str | None = None, from_node: str | None = None, + **options) -> dict: + """Start a job: kind "get" (output needed), "pin" or "export" (output needed). Its record, with its + "id". options: get's sealed_name and fallback_dht.""" + output = os.path.abspath(os.path.expanduser(output)) if output is not None else None + return self._post("/api/jobs", kind=kind, target=target, output=output, **{"from": from_node}, **options) + + def job(self, job_id: str, wait: int = 0) -> dict: + """A job's record; waits up to wait seconds (the API caps it) for it to change state or progress.""" + return self.get("/api/jobs/get", id=job_id, wait=wait or None) + + def jobs(self) -> list[dict]: + """Every job's record, oldest first.""" + return self.get("/api/jobs")["jobs"] + + def cancel_job(self, job_id: str) -> dict: + """Cancel a job: its record.""" + return self.post("/api/jobs/cancel", id=job_id) + + def clear_jobs(self, job_id: str | None = None) -> int: + """Forget finished jobs (or that one); how many.""" + return self._post("/api/jobs/clear", id=job_id)["removed"] diff --git a/rnfs/cli.py b/rnfs/cli.py index d7f007f..e75d7a4 100644 --- a/rnfs/cli.py +++ b/rnfs/cli.py @@ -5,33 +5,27 @@ running daemon's local API (see rnfs/api.py). Client commands never import RNS, so they start quickly; paths given to them are resolved to absolute paths here, since the daemon reads and writes them itself. -Client commands authenticate with the token the daemon writes to /api_token at each start -(specification section 11), so only users who can read the rnfs directory can use the daemon. They send it -only once the program on the API port has proved it knows it (api_proof), since while the daemon is down -any local user could listen there. +Client commands talk to the daemon through rnfs.apiclient, the library companion apps use too: it reads the token +the daemon writes to /api_token at each start (specification section 11), so only users who can read +the rnfs directory can use the daemon, and sends it only once the program on the API port has proved it knows it +(api_proof), since while the daemon is down any local user could listen there. Any error it raises exits with its +message. """ import argparse import errno -import hashlib -import hmac import http.client import json import os -import secrets import stat import sys -from urllib.parse import urlencode -from rnfs import __version__, fsutil, winacl +from rnfs import __version__, apiclient, fsutil +from rnfs.apiclient import CHALLENGE_HEADER, PROOF_HEADER, TOKEN_FILE, api_proof, token_path # noqa: F401 from rnfs.config import DEFAULT_API_PORT, Config -DEFAULT_DIR = Config.rnfs_dir -COPY_SIZE = 64 * 1024 -TOKEN_FILE = "api_token" # in the rnfs directory; written by the daemon, read by every other command +DEFAULT_DIR = apiclient.DEFAULT_DIR +COPY_SIZE = apiclient.COPY_SIZE LOCK_FILE = "daemon.lock" # in the rnfs directory; locked by the daemon using it, so a second one can't start -# Before sending the token, the CLI makes whatever listens on the API port prove it knows it (api_proof) -CHALLENGE_HEADER = "X-Rnfs-Challenge" -PROOF_HEADER = "X-Rnfs-Proof" # text with control characters escaped, so names from other people's DAGs can't drive the terminal @@ -48,11 +42,6 @@ def fail(message: str): sys.exit(f"rnfs: {printable(message)}") -def token_path(rnfs_dir: str) -> str: - """Where the daemon using rnfs_dir keeps its API token.""" - return os.path.join(os.path.expanduser(rnfs_dir), TOKEN_FILE) - - def write_token(path: str, token: str): """Write the API token to path atomically, created with mode 0600 so other users can't read it (on Windows, with an access list for this user and SYSTEM only).""" @@ -73,159 +62,72 @@ def remove_token(path: str, token: str): pass # already gone, or unreadable: nothing of ours to clean up -def api_proof(token: str, port: int, challenge: str) -> str: - """The daemon's answer to a challenge: HMAC-SHA256, keyed by the API token, of the challenge and the port - it listens on (so a program on another port can't pass on the daemon's answer).""" - return hmac.new(token.encode(), f"rnfs api proof {port} {challenge}".encode(), hashlib.sha256).hexdigest() - - def read_token(path: str) -> str: - """The daemon's API token from path; exits with an explanation if it can't be used.""" + """The daemon's API token from path (apiclient.read_token); exits with an explanation if it can't be used.""" try: - fd = os.open(path, os.O_RDONLY | fsutil.O_NOFOLLOW | fsutil.O_BINARY) - except FileNotFoundError: - fail(f"no API token at {path}: is the daemon running, with this --dir?") - except PermissionError: - fail(f"can't read the API token at {path}: permission denied. Is the daemon running as another user?") - except OSError as e: - fail(f"can't read the API token at {path}: {e}") - with os.fdopen(fd, "rb") as f: - mode = stat.S_IMODE(os.fstat(f.fileno()).st_mode) - # Windows has no Unix permission bits to check, but an access list (of the very file opened) - if not fsutil.IS_WINDOWS and mode & 0o077: - fail(f"{path} is accessible to other users (mode {mode:04o}), so its token can't be trusted; " - f"restart the daemon to make a new one") - try: - reason = winacl.check_private(path, f.fileno()) - except OSError as e: - fail(f"can't check who may read the API token at {path}: {e}") - if reason is not None: - fail(f"{path} is accessible to other users ({reason}), so its token can't be trusted; " - f"restart the daemon to make a new one") - token = f.read(1024).decode("ascii", "replace").strip() - if not token: - fail(f"the API token file {path} is empty; restart the daemon") - return token + return apiclient.read_token(path) + except apiclient.ApiError as e: + fail(str(e)) -class ApiClient: - """Talks to the daemon's API on 127.0.0.1:port, exiting with a friendly message on any error. +class ApiClient(apiclient.Client): + """The library's client for the CLI: talks to the daemon's API on 127.0.0.1:port, exiting with a friendly + message on any error. token_file is read on the first request: every request carries the token but the proof check's status request (status without the token says only the daemon's version, for health checks). """ def __init__(self, port: int, token_file: str): - self.port = port - self.token_file = token_file - self.token: str | None = None - - def request(self, method: str, path: str, params: dict | None = None, json_body: dict | None = None, - body=None, length: int | None = None, extra_headers: dict | None = None, - authenticated: bool = True) -> http.client.HTTPResponse: - """Send a request and return the successful response; exits on errors. - - body may be bytes or a file; a file without length is sent chunked. Unless authenticated is False (the - proof check), the token is sent, once the daemon has proved it knows it. - """ - url = path + ("?" + urlencode({k: v for k, v in params.items() if v is not None}) if params else "") - # Host is set explicitly: the API only accepts 127.0.0.1:, even for port 80 - headers = {"Host": f"127.0.0.1:{self.port}", **(extra_headers or {})} - if authenticated: - if self.token is None: - token = read_token(self.token_file) - self.verify_daemon(token) - self.token = token - headers["X-Rnfs-Api"] = self.token - if json_body is not None: - body = json.dumps(json_body).encode() - headers["Content-Type"] = "application/json" - elif body is not None: - headers["Content-Type"] = "application/octet-stream" - if length is not None: - headers["Content-Length"] = str(length) - connection = http.client.HTTPConnection("127.0.0.1", self.port) # no timeout: fetches can take hours - try: - connection.request(method, url, body=body, headers=headers) - response = connection.getresponse() - except ConnectionRefusedError: - fail(f"no daemon is running on API port {self.port}. Start one with: rnfs daemon") - except (http.client.HTTPException, ConnectionError) as e: - fail(f"the daemon dropped the connection ({e!r}); see its log") - except OSError as e: - fail(f"can't reach the daemon on API port {self.port}: {e}") - if response.status >= 400: - data = response.read() - try: - message = json.loads(data)["error"] - except (ValueError, KeyError, TypeError): - message = f"HTTP {response.status} {response.reason}" - fail(message) - return response - - def verify_daemon(self, token: str): - """Exit unless whatever listens on the API port proves it knows token, before the token (or a name key) - is sent to it: while the daemon is down, any local user could be listening on its port.""" - challenge = secrets.token_hex(16) - response = self.request("GET", "/api/status", extra_headers={CHALLENGE_HEADER: challenge}, - authenticated=False) - read_all(response) - proof = (response.getheader(PROOF_HEADER) or "").strip().encode("ascii", "replace") - if not hmac.compare_digest(proof, api_proof(token, self.port, challenge).encode()): - fail(f"what listens on API port {self.port} couldn't prove it knows the API token in {self.token_file}, " - f"so nothing was sent to it: it may be another program, or a daemon using another --dir " - f"(or one started before an upgrade: restart it)") - - def call(self, method: str, path: str, params: dict | None = None, json_body: dict | None = None, - body=None, length: int | None = None, authenticated: bool = True): - """Send a request and return its decoded JSON response.""" - response = self.request(method, path, params, json_body, body, length, authenticated=authenticated) - try: - return json.loads(read_all(response)) - except ValueError: - fail("the daemon sent an invalid response") + """A client of the daemon on port whose token is in token_file.""" + super().__init__(port=port, token_file=token_file) - def get(self, path: str, /, **params): - """GET path with query parameters; JSON response.""" - return self.call("GET", path, params=params) + def request(self, *args, **kwargs) -> http.client.HTTPResponse: + """Client.request (the successful response), exiting on errors.""" + try: + return super().request(*args, **kwargs) + except apiclient.ApiError as e: + fail(str(e)) - def post(self, path: str, /, **fields): - """POST path with a JSON body; JSON response.""" - return self.call("POST", path, json_body=fields) + def call(self, *args, **kwargs): + """Client.call (the decoded JSON response), exiting on errors.""" + try: + return super().call(*args, **kwargs) + except apiclient.ApiError as e: + fail(str(e)) def read_all(response: http.client.HTTPResponse) -> bytes: """A response's whole body, failing if it is shorter than its Content-Length (the daemon hit an error).""" try: - data = response.read() - except http.client.IncompleteRead: - fail("the response was cut short; see the daemon's log") - return data + return apiclient.read_all(response) + except apiclient.ApiError as e: + fail(str(e)) -def copy_response(response: http.client.HTTPResponse, out, progress=None): - """Copy a response body to out, failing if it ends before its Content-Length. +def copy_response(stream: apiclient.Stream, out, progress=None): + """Copy a streamed response body to out, failing if it ends before its Content-Length. progress (jobs.StreamProgress) is called with the bytes copied so far and Content-Length, and closed at the end. """ - expected = response.getheader("Content-Length") - copied = 0 + expected = stream.response.getheader("Content-Length") + error = None while True: try: - chunk = response.read(COPY_SIZE) - except http.client.IncompleteRead: - chunk = b"" + chunk = stream.read(COPY_SIZE) + except apiclient.ApiError as e: + error = e + break if not chunk: break out.write(chunk) - copied += len(chunk) if progress is not None: - progress(copied, expected) + progress(stream.received, expected) if progress is not None: progress.close() out.flush() - if expected is not None and copied != int(expected): - fail(f"the transfer stopped after {copied} of {expected} bytes; see the daemon's log") + if error is not None: + fail(str(error)) def client(args) -> ApiClient: @@ -233,9 +135,9 @@ def client(args) -> ApiClient: port = args.api_port if port is None: try: - port = Config.load(args.dir).api_port - except (ValueError, TypeError, OSError) as e: # TypeError: a badly typed setting validate() missed - fail(f"can't read the settings in {args.dir}: {e}") + port = apiclient.api_port(args.dir) + except apiclient.ApiError as e: + fail(str(e)) return ApiClient(port if port is not None else DEFAULT_API_PORT, token_path(args.dir)) @@ -500,16 +402,14 @@ def without_jobs(api) -> dict: def run_cat(args): """Write a file, or part of one, to stdout.""" - response = client(args).request("GET", "/api/cat", params={"target": args.target, "offset": args.offset, - "length": args.length, - "raw": 1 if getattr(args, "raw", False) else None, - "block": 1 if getattr(args, "block", False) else None}) + stream = client(args).cat(args.target, offset=args.offset, length=args.length, raw=getattr(args, "raw", False), + block=getattr(args, "block", False)) progress = None if sys.stderr.isatty() and not sys.stdout.isatty(): # into a file or pipe, watched from a terminal from rnfs.features import jobs progress = jobs.StreamProgress(sys.stderr) try: - copy_response(response, sys.stdout.buffer, progress) + copy_response(stream, sys.stdout.buffer, progress) except OSError as e: # Windows reports a pipe whose reader has gone as EINVAL rather than EPIPE if not (isinstance(e, BrokenPipeError) or (fsutil.IS_WINDOWS and e.errno == errno.EINVAL)): @@ -521,7 +421,7 @@ def run_cat(args): def run_ls(args): """List a directory: kind, size, CID and name (last, since names can contain spaces).""" - for entry in client(args).get("/api/ls", target=args.target)["entries"]: + for entry in client(args).ls(args.target): name = printable(entry["name"]) + ("/" if entry["kind"] == "directory" else "") print(f"{printable(entry['kind']):<10} {printable(entry['size']):>12} {printable(entry['cid'])} {name}") @@ -536,7 +436,7 @@ def print_fields(result: dict): def run_stat(args): """Print a target's size, block count and local holdings.""" - print_fields(client(args).get("/api/stat", target=args.target)) + print_fields(client(args).stat(args.target)) def run_dir_set(args): @@ -564,7 +464,7 @@ def run_pin_add(args): def run_pin_rm(args): """Remove the user pin of a CID.""" - client(args).post("/api/pin/rm", cid=args.cid) + client(args).pin_rm(args.cid) def display_name(name: str) -> str: @@ -577,7 +477,7 @@ def display_name(name: str) -> str: def run_pin_ls(args): """List pins with their sources and, when known, where each came from (or what it holds).""" - pins = client(args).get("/api/pin/ls", cid=args.cid)["pins"] + pins = client(args).pin_ls(args.cid) if args.json: print(json.dumps(pins, indent=2)) return @@ -639,7 +539,7 @@ def run_ping(args): def run_name_create(args): """Create a name key; print the name.""" - print(printable(client(args).post("/api/name/create", alias=args.alias)["name"])) + print(printable(client(args).name_create(args.alias)["name"])) def run_name_rm(args): @@ -683,20 +583,20 @@ def run_name_import(args): def run_name_list(args): """List this node's names: alias, name, CID and sequence.""" - for entry in client(args).get("/api/name/list")["names"]: + for entry in client(args).name_list(): print("\t".join(printable(value) for value in (entry["alias"], entry["name"], entry.get("cid") or "-", entry.get("sequence", 0)))) def run_name_publish(args): """Point a name at a target.""" - result = client(args).post("/api/name/publish", alias=args.alias, target=args.target) + result = client(args).name_publish(args.alias, args.target) print(printable(f"published {result['cid']} as {result['name']} (sequence {result['sequence']})")) def run_name_resolve(args): """Print the CID a name points at.""" - print(printable(client(args).get("/api/name/resolve", name=args.name)["cid"])) + print(printable(client(args).name_resolve(args.name)["cid"])) # Archives and status @@ -732,7 +632,7 @@ def run_status(args): print_fields(api.call("GET", "/api/status", authenticated=False)) warn(f"no readable API token at {api.token_file}: the daemon only says its version without it") return - print_fields(api.get("/api/status")) + print_fields(api.status()) # Parser diff --git a/rnfs/config.py b/rnfs/config.py index 200043b..99eb08c 100644 --- a/rnfs/config.py +++ b/rnfs/config.py @@ -126,6 +126,8 @@ class Config: lxmf_accept_from: list[str] = field(default_factory=list) # LXMF addresses (hex); empty: everyone lxmf_auto_fetch_from: list[str] = field(default_factory=list) # senders whose links are fetched and pinned lxmf_auto_fetch_quota: int = 1 * GIB # per sender in lxmf_auto_fetch_from + # --- deny lists (rnfs/features/deny.py) --- + deny_lists: list[str] = field(default_factory=list) # deny-list sources: [/path] or name/[/path] # Paths @@ -214,6 +216,9 @@ def validate(self): # lxmf from rnfs.features import lxmf # here, like mesh: only needed once settings are checked lxmf.validate_settings(self) + # deny lists + from rnfs.features import deny + deny.validate_settings(self) for slow_node in (False, True): if self.effective_republish_interval(slow_node) >= PROVIDER_TTL / 2: raise ValueError("republish_interval must be less than PROVIDER_TTL / 2") diff --git a/rnfs/content/archive.py b/rnfs/content/archive.py index fdf6abb..710bc05 100644 --- a/rnfs/content/archive.py +++ b/rnfs/content/archive.py @@ -5,7 +5,7 @@ every root's DAG, depth-first from each root, each block once. CIDs are binary. """ import struct -from typing import BinaryIO +from typing import BinaryIO, Callable from RNS.vendor import umsgpack @@ -170,7 +170,8 @@ def records(): return roots, records() -def import_archive(stream: BinaryIO, storage: Storage, pin_source: str | None = USER) -> dict: +def import_archive(stream: BinaryIO, storage: Storage, pin_source: str | None = USER, + check: Callable[[list[Cid]], None] | None = None) -> dict: """Import an archive into storage, verifying every block, and pin each root it supplies with pin_source. A root is supplied, and pinned, only if every block of its DAG was in this archive and matched its CID; a @@ -180,6 +181,8 @@ def import_archive(stream: BinaryIO, storage: Storage, pin_source: str | None = count, "bad": count of blocks that didn't match their CID (skipped)}. Raises ValueError for a stream that isn't an archive or lists more than MAX_ROOTS roots, StorageFull for a block that would leave less than the disk reserve free (Storage.check_disk); blocks read before a malformed record or a full disk are kept as cache. + Raises Denied for a block the deny list denies (storage.put), and whatever check(complete roots) raises: it + runs before they are pinned (the deny list's file hashes), and nothing is pinned then. """ roots, records = read_archive(stream) # Every verified CID in the archive; no larger than the storage's own index of the same blocks @@ -211,6 +214,8 @@ def import_archive(stream: BinaryIO, storage: Storage, pin_source: str | None = else: incomplete.append(root) held.add(root) + if check is not None: + check(complete) if pin_source is not None: storage.pin_many(complete, pin_source) # still protected; the pins are saved once for them all if bad: diff --git a/rnfs/content/builder.py b/rnfs/content/builder.py index 192ccbd..85d9e63 100644 --- a/rnfs/content/builder.py +++ b/rnfs/content/builder.py @@ -12,14 +12,19 @@ directory is [2, []]. Builders hand every block to a `put(cid, block)` callback, so they don't depend on Storage; a no-op -callback (NO_PUT) gives only-hash mode. Files are streamed, one chunk in memory at a time. +callback (NO_PUT) gives only-hash mode. Files are streamed, one chunk in memory at a time. Within +`with checking(check):` each file's whole-content SHA-256 is passed to check (the deny list's file hashes), +which may raise to stop the build before the file's root node is stored. """ import errno +import hashlib import io import os import stat +from contextlib import contextmanager +from contextvars import ContextVar from dataclasses import dataclass -from typing import BinaryIO, Callable, Iterable +from typing import BinaryIO, Callable, Iterable, Iterator from rnfs import constants, fsutil from rnfs.content import chunker, dag @@ -111,6 +116,21 @@ def _put_node(node: dag.Node, put: Put) -> Cid: return cid +# What build_file passes each file's content SHA-256 to in this thread/context (checking), if anything +_check: ContextVar[Callable[[bytes], None] | None] = ContextVar("rnfs_file_check", default=None) + + +@contextmanager +def checking(check: Callable[[bytes], None] | None) -> Iterator[None]: + """Within the block, build_file calls check(digest) with each file's content SHA-256 once it is read, before + the file's root node is stored; check may raise to stop the build. None checks nothing.""" + token = _check.set(check) + try: + yield + finally: + _check.reset(token) + + def build_file(stream: BinaryIO, put: Put, size: int | None = None, kind: int = dag.KIND_FILE) -> Built: """Build a file from everything read from stream, or exactly size bytes of it (ValueError if it ends early).""" @@ -120,11 +140,17 @@ def make_group(level: int, pairs: list[tuple]) -> tuple: return Child(_put_node(node, put), node.size), _height(pairs) levels = _Levels(make_group, constants.MAX_CHILDREN) + check = _check.get() + digest = hashlib.sha256() if check is not None else None # Fixed CHUNK_SIZE pieces unless `add --chunker fastcdc` selected content-defined chunks (chunker.using) for chunk in chunker.chunks(stream, size): cid = Cid.for_block(RAW, chunk) + if digest is not None: + digest.update(chunk) put(cid, chunk) levels.add(Child(cid, len(chunk)), 0) + if check is not None: + check(digest.digest()) level, pairs = levels.finish() if level == 0 and not pairs: diff --git a/rnfs/content/storage.py b/rnfs/content/storage.py index b08bbf9..bdc8d31 100644 --- a/rnfs/content/storage.py +++ b/rnfs/content/storage.py @@ -1,11 +1,14 @@ """Storage: blocks, pins, cached fetches, protection and eviction (specification section 8). No RNS dependency. -- Pins are recursive and held by sources: "user", "folder", "name:", "remote:". A pin - lasts while any source holds it. Pins are never evicted. +- Pins are recursive and held by sources: "user", "folder", "name:", "remote:", "deny" (the + deny lists' own blocks), and the features' "follow:" and "lxmf:". A pin lasts while any source holds it. Pins + are never evicted. - Cache is everything else. Each cached fetch is recorded as root + path + target, which keeps the path's nodes and the target's DAG; fetches are evicted least recently used first when storage exceeds max_storage, and gc() evicts them all. - Blocks of fetches, adds and imports in progress are protected as if pinned, by a protection session. +- A block the deny list (`deny`, rnfs.denylist) denies is never stored: put() raises Denied. drop_denied() drops + the cached fetches that reach one, and denied_roots() says which pins do. - A block is deleted only if no pin, session or remaining cached fetch reaches it. - max_storage is soft: no fetch is refused for it, and storage may exceed it while fetches run or when pins alone fill it. The hard bound is the disk: check_disk() refuses blocks that would leave less than @@ -29,16 +32,17 @@ from contextlib import contextmanager from typing import Callable, Iterator -from rnfs import statedb +from rnfs import denylist, statedb from rnfs.constants import MAX_BLOCK_SIZE, MIB from rnfs.content import dag, walker from rnfs.content.blockstore import BlockStore from rnfs.content.cid import Cid from rnfs.content.dag import DirectoryNode, IndexNode -from rnfs.errors import InvalidDag, NotFound, StorageFull +from rnfs.errors import Denied, InvalidDag, NotFound, StorageFull from rnfs.log import LOG_ERROR, LOG_VERBOSE, log, plural, size USER, FOLDER = "user", "folder" +DENY = denylist.SOURCE # the deny lists' own blocks (rnfs/features/deny.py) NAME_PREFIX, REMOTE_PREFIX = "name:", "remote:" LXMF_PREFIX = "lxmf:" MAX_CACHE_RECORDS = 10_000 # cached fetches remembered; past this the least recently used record is dropped @@ -63,7 +67,7 @@ def disk_reserve(total: int) -> int: def check_source(source: str): """Raise ValueError unless source is a valid pin source.""" - if source in (USER, FOLDER): + if source in (USER, FOLDER, DENY): return if isinstance(source, str) and source.startswith(NAME_PREFIX) and len(source) > len(NAME_PREFIX): return @@ -243,6 +247,7 @@ def __init__(self, blocks_dir: str, state_dir: str, max_storage: int): # so eviction over max_storage with nothing to evict needn't walk everything again self._maybe_garbage = True self._reported_damaged: set[Cid] = set() # damaged DAG nodes already logged as stopping collection + self.deny = denylist.NONE # what put() refuses; the node sets its DenyList try: self._load() except BaseException: @@ -356,7 +361,7 @@ def block_size(self, cid: Cid) -> int | None: def put(self, cid: Cid, block: bytes): """Store a block after verifying it (atomic, durable). Raises ValueError if it doesn't match or is too - large, even when the block is already stored. + large, even when the block is already stored, and Denied if the deny list denies it. A block stored outside a protection session is cache that no record reaches: the first to go. """ @@ -364,6 +369,8 @@ def put(self, cid: Cid, block: bytes): def _put(self, cid: Cid, block: bytes, protected: bool): """Store a block; protected if a session already lists it, so it can't be garbage.""" + if self.deny.denies(cid): + raise Denied(f"{cid} is on this node's deny list") with self._lock: stored = cid in self._sizes if stored: @@ -847,6 +854,58 @@ def dag_blocks(self, cid: Cid) -> frozenset[Cid]: raise NotFound(f"a block of {cid} was deleted") return result + def collect(self) -> int: + """Delete every block nothing reaches (what an add or import refused left behind); how many.""" + with self._lock: + roots = self._roots() | set(self._cache.values()) + paths = [key for key in self._cache if key[1]] + self._warm(roots, paths) + with self._lock: + deleted, _ = self._collect() + return len(deleted) + + def denied_roots(self, roots, denied: frozenset[bytes]) -> set[Cid]: + """Those of roots that are, or reach through stored DAG nodes, a block whose binary CID is in denied.""" + roots = set(roots) + if not denied or not roots: + return set() + self._warm(roots, []) + with self._lock: + return {root for root in roots + if root.binary in denied or any(cid.binary in denied for cid in self._reach([root]))} + + def drop_denied(self, denied: frozenset[bytes]) -> dict: + """Drop every cached fetch whose root, path or target's stored DAG holds a block whose binary CID is in + denied, then delete every block nothing reaches, as gc() does for every fetch. + + Returns {"uncached": set[Cid] targets no longer cached, "removed_blocks", "freed_bytes", + "incomplete": set[Cid] no longer complete}. + """ + with self._lock: + records = dict(self._cache) + targets = self._targets() + if not denied or not records: + return {"uncached": set(), "removed_blocks": 0, "freed_bytes": 0, "incomplete": set()} + before = self._complete_sets(targets) + self._warm(set(records.values()) | {root for root, _ in records}, [key for key in records if key[1]]) + with self._lock: + before = self._with_memoised(before) + dropped = [key for key, target in self._cache.items() + if any(cid.binary in denied + for cid in {key[0], target} | self._path_cids(*key) | self._reach([target]))] + uncached = set() + for key in dropped: + uncached.add(self._cache.pop(key)) + self._cache_dirty.add(key) + uncached -= set(self._cache.values()) + if dropped: + self._cache_generation += 1 + self._maybe_garbage = True + deleted, freed = self._collect() if dropped else (set(), 0) + incomplete = {cid for cid, blocks in before.items() if blocks & deleted} + return {"uncached": uncached, "removed_blocks": len(deleted), "freed_bytes": freed, + "incomplete": incomplete} + def is_complete(self, cid: Cid) -> bool: """True if every block of cid's DAG is stored and the DAG is valid.""" try: diff --git a/rnfs/content/walker.py b/rnfs/content/walker.py index 499a252..e476ac9 100644 --- a/rnfs/content/walker.py +++ b/rnfs/content/walker.py @@ -433,9 +433,34 @@ def unsafe_name_reason(name: str) -> str | None: return fsutil.unsafe_name_reason(name) # in fsutil so the CLI can check names without loading RNS -def write_out(expect: Expect, get: Get, dest: str, max_entries: int) -> list[str]: +def iter_files(expect: Expect, get: Get) -> Iterator[Expect]: + """The expectation of every file in a DAG (expect itself if it is a file), each distinct one once.""" + block = get(expect) + if not isinstance(decode_checked(expect, block), (DirectoryNode, IndexNode)): + yield expect + return + seen: set[Expect] = set() + memo: dict[Expect, dag.Node] = {} + stack = [expect] + while stack: + directory = stack.pop() + for entry, child in _directory_entries(directory, get, memo): + if child in seen: + continue + seen.add(child) + if entry.kind == dag.KIND_DIRECTORY: + stack.append(child) + else: + yield child + + +def write_out(expect: Expect, get: Get, dest: str, max_entries: int, + check: Callable[[Expect], None] | None = None) -> list[str]: """Write the file or directory expect names to dest, which must not exist yet. Returns warnings. + check(file expect), if given, runs before each file is created and may raise to stop the write-out (the + deny list's file hashes); what was written stays for the caller to remove. + Entries whose names the local filesystem would read as a path, or can't represent (it refuses to create them), are skipped and reported, as are names that collide on this filesystem (e.g. case-insensitive ones). Kind 1 entries get the execute bit, except on Windows, which has none. Raises TooLarge after max_entries entries, @@ -450,6 +475,8 @@ def write_out(expect: Expect, get: Get, dest: str, max_entries: int) -> list[str block = get(expect) node = decode_checked(expect, block) if not isinstance(node, (DirectoryNode, IndexNode)): + if check is not None: + check(expect) with open(dest, "xb") as out: _write_file(expect, get, out, small, block) return warnings @@ -470,6 +497,8 @@ def write_out(expect: Expect, get: Get, dest: str, max_entries: int) -> list[str warnings.append(f"skipped {entry_shown!r}: {reason}") continue path = os.path.join(where, entry.name) + if check is not None and entry.kind != dag.KIND_DIRECTORY: + check(child) try: if entry.kind == dag.KIND_DIRECTORY: os.mkdir(path) diff --git a/rnfs/denylist.py b/rnfs/denylist.py new file mode 100644 index 0000000..0fa29a5 --- /dev/null +++ b/rnfs/denylist.py @@ -0,0 +1,315 @@ +"""The deny list (specification section 8, "Deny lists"): CIDs and whole-file SHA-256 hashes this node refuses. + +A denied CID isn't fetched, read, stored, served to /blocks (answered as not held) or provided, whether it is a +root or anywhere inside the DAG being fetched. A file-hash entry (`sha256:`) matches a file whose content +hashes to it; that is only known once the content is read, so it is checked where content is assembled (get, +cat, the gateway, add, archive import), after the blocks have been fetched. + +Entries come from the local list (`rnfs deny add` / `rm`; the deny_entries table) and from the deny lists named +in the deny_lists setting, which rnfs/features/deny.py fetches and follows. The deny_lists table records the +version (CID) each source last accepted; its blocks stay pinned with source "deny" while it is in use, so at +startup reload() reads every list back from storage before anything is fetched or provided. + +Checks are set lookups: `cids` (binary CIDs) and `hashes` (32-byte digests) are frozensets swapped whole and +read without a lock, so /blocks serving and each block a fetch stores pay one lookup. A list is untrusted input: +parse_list refuses one over MAX_LIST_BYTES or MAX_LIST_ENTRIES, not UTF-8, or with a line that isn't an entry, +and its source keeps the version it had. Listeners (the node) are told after every change, outside the lock. +""" +import re +import threading +import time +from collections import OrderedDict +from collections.abc import Callable + +from rnfs import statedb +from rnfs.constants import KIB +from rnfs.content.cid import Cid +from rnfs.errors import NotFound +from rnfs.log import LOG_WARNING, log, plural + +HASH_PREFIX = "sha256:" +SOURCE = "deny" # the pin source of the deny lists' own blocks +MAX_LIST_BYTES = 4096 * KIB # a list file (about 60 000 CIDs) +MAX_LIST_ENTRIES = 100_000 # entries in one list +MAX_LOCAL_ENTRIES = 100_000 # entries in the local list +MAX_KNOWN_HASHES = 10_000 # files whose content hash is remembered (a CID's content never changes) +_HEX_DIGEST = re.compile(r"[0-9a-f]{64}") + + +def parse_entry(text) -> tuple[str, bytes | None, bytes | None]: + """(canonical text, binary CID or None, digest or None) of one entry: a CID, or "sha256:" and 64 hex digits + (a file's content hash). Raises ValueError.""" + if not isinstance(text, str): + raise ValueError("a deny-list entry must be a CID or sha256:<64 hex digits>") + text = text.strip() + if text[:len(HASH_PREFIX)].lower() == HASH_PREFIX: + digest = text[len(HASH_PREFIX):].lower() + if not _HEX_DIGEST.fullmatch(digest): + raise ValueError(f"invalid file hash {text[:80]!r}: must be sha256:<64 hex digits>") + return HASH_PREFIX + digest, None, bytes.fromhex(digest) + try: + cid = Cid.parse(text) + except ValueError: + raise ValueError(f"invalid deny-list entry {text[:80]!r}: not a CID or sha256:<64 hex digits>") from None + return str(cid), cid.binary, None + + +def parse_list(data: bytes) -> tuple[frozenset[bytes], frozenset[bytes]]: + """(binary CIDs, digests) of a deny list: UTF-8 text, one entry per line; blank lines and everything after a + "#" are ignored. Raises ValueError (saying which line) for a list over MAX_LIST_BYTES or MAX_LIST_ENTRIES + entries, that isn't UTF-8, or with a line that isn't an entry.""" + if len(data) > MAX_LIST_BYTES: + raise ValueError(f"it is over {MAX_LIST_BYTES // KIB} KiB") + try: + text = data.decode("utf-8") + except UnicodeDecodeError: + raise ValueError("it isn't UTF-8 text") from None + cids: set[bytes] = set() + digests: set[bytes] = set() + for number, line in enumerate(text.splitlines(), 1): + line = line.split("#", 1)[0].strip() + if not line: + continue + try: + _, binary, digest = parse_entry(line) + except ValueError as e: + raise ValueError(f"line {number}: {e}") from None + (cids if binary is not None else digests).add(binary if binary is not None else digest) + if len(cids) + len(digests) > MAX_LIST_ENTRIES: + raise ValueError(f"it has over {MAX_LIST_ENTRIES} entries") + return frozenset(cids), frozenset(digests) + + +class NoDenyList: + """Denies nothing: what storage, the fetcher and the publisher check until the node gives them its list.""" + + cids: frozenset[bytes] = frozenset() + hashes: frozenset[bytes] = frozenset() + + def denies(self, cid: Cid) -> bool: + """False.""" + return False + + def denies_hash(self, digest: bytes) -> bool: + """False.""" + return False + + +NONE = NoDenyList() + + +class DenyList(NoDenyList): + """The node's deny list: the local entries and every source's accepted list, in memory, saved in the state + database (deny_entries, deny_lists). Thread-safe; checks take no lock.""" + + def __init__(self, state_dir: str, clock: Callable[[], float] = time.time): + """Load the local entries and the sources' accepted versions (their entries come with reload()).""" + self.clock = clock + self.cids: frozenset[bytes] = frozenset() + self.hashes: frozenset[bytes] = frozenset() + self._lock = threading.Lock() + self._local: dict[str, tuple[bytes | None, bytes | None]] = {} # canonical text -> (binary CID, digest) + self._accepted: dict[str, Cid] = {} # source -> the version it last accepted (the deny_lists table) + self._lists: dict[str, tuple[frozenset[bytes], frozenset[bytes]]] = {} # source -> its entries, read + self._known: OrderedDict[bytes, bytes] = OrderedDict() # binary file CID -> its content's SHA-256 + self._listeners: list[Callable[[], None]] = [] + self._db = statedb.connect(state_dir) + try: + self._load() + except BaseException: + self._db.close() + raise + + def _load(self): + """Read both tables; a row that doesn't parse is logged and deleted.""" + bad_entries, bad_lists = [], [] + for (text,) in self._db.query("SELECT entry FROM deny_entries"): + try: + canonical, binary, digest = parse_entry(text) + if canonical != text: + raise ValueError("not canonical") + except ValueError: + bad_entries.append((text,)) + continue + self._local[canonical] = (binary, digest) + for dumped, binary in self._db.query("SELECT source, cid FROM deny_lists"): + try: + source = statedb.load(dumped) + if not isinstance(source, str): + raise ValueError("not a string") + self._accepted[source] = Cid.from_bytes(binary) + except (ValueError, TypeError): + bad_lists.append((dumped,)) + if bad_entries or bad_lists: + log(f"deny list: dropped {plural(len(bad_entries) + len(bad_lists), 'invalid row')} from the state " + f"database", LOG_WARNING) + with self._db.transaction() as tx: + tx.executemany("DELETE FROM deny_entries WHERE entry = ?", bad_entries) + tx.executemany("DELETE FROM deny_lists WHERE source = ?", bad_lists) + self._rebuild() + + def reload(self, read: Callable[[Cid], bytes | None]): + """Read every source's accepted version back with read(cid) (its bytes from storage, or None) and use + its entries; one that can't be read or parsed is left to the feature to fetch again (logged).""" + with self._lock: + accepted = dict(self._accepted) + lists = {} + for source, cid in accepted.items(): + try: + data = read(cid) + if data is None: + raise NotFound("not stored") + lists[source] = parse_list(data) + except (ValueError, NotFound) as e: + log(f"deny list {source}: can't read its version {cid} ({e}); fetching it again", LOG_WARNING) + with self._lock: + for source, entries in lists.items(): + if self._accepted.get(source) == accepted[source]: + self._lists[source] = entries + self._rebuild() + self._notify() + + # Checks (no lock: the sets are swapped whole) + + def denies(self, cid: Cid) -> bool: + """True if cid is denied.""" + return cid.binary in self.cids + + def denies_hash(self, digest: bytes) -> bool: + """True if a file whose content has this SHA-256 is denied.""" + return digest in self.hashes + + def known_hash(self, cid: Cid) -> bytes | None: + """The SHA-256 of file cid's content, if worked out before.""" + with self._lock: + digest = self._known.get(cid.binary) + if digest is not None: + self._known.move_to_end(cid.binary) + return digest + + def remember_hash(self, cid: Cid, digest: bytes): + """Remember the SHA-256 of file cid's content (at most MAX_KNOWN_HASHES files, least recently used + forgotten first).""" + with self._lock: + self._known[cid.binary] = digest + self._known.move_to_end(cid.binary) + while len(self._known) > MAX_KNOWN_HASHES: + self._known.popitem(last=False) + + # The local list + + def entries(self) -> list[str]: + """The local list's entries, sorted.""" + with self._lock: + return sorted(self._local) + + def add(self, texts) -> list[str]: + """Add entries to the local list; those new, canonical. Raises ValueError (adding none) if one isn't an + entry or the list would hold over MAX_LOCAL_ENTRIES.""" + parsed = {} + for text in texts: + canonical, binary, digest = parse_entry(text) + parsed[canonical] = (binary, digest) + with self._lock: + new = {text: value for text, value in parsed.items() if text not in self._local} + if len(self._local) + len(new) > MAX_LOCAL_ENTRIES: + raise ValueError(f"the local deny list holds at most {MAX_LOCAL_ENTRIES} entries") + if new: + now = self.clock() + with self._db.transaction(durable=True) as tx: + tx.executemany("INSERT INTO deny_entries (entry, added) VALUES (?, ?)", + [(text, now) for text in new]) + self._local.update(new) + self._rebuild() + if new: + self._notify() + return sorted(new) + + def remove(self, texts) -> list[str]: + """Remove entries from the local list; those removed, canonical. Raises ValueError if one isn't an entry, + NotFound (removing none) if one isn't on the local list.""" + canonical = {parse_entry(text)[0] for text in texts} + with self._lock: + missing = sorted(text for text in canonical if text not in self._local) + if missing: + raise NotFound(f"not on the local deny list: {', '.join(missing)}") + if canonical: + with self._db.transaction(durable=True) as tx: + tx.executemany("DELETE FROM deny_entries WHERE entry = ?", [(text,) for text in canonical]) + for text in canonical: + del self._local[text] + self._rebuild() + if canonical: + self._notify() + return sorted(canonical) + + # Sources (rnfs/features/deny.py) + + def accepted(self) -> dict[str, Cid]: + """source -> the version (CID) it last accepted.""" + with self._lock: + return dict(self._accepted) + + def in_use(self, source: str) -> bool: + """True if source's accepted version is read and its entries apply.""" + with self._lock: + return source in self._lists + + def entry_count(self, source: str) -> int: + """How many entries source's accepted version has (0 if it isn't read).""" + with self._lock: + cids, digests = self._lists.get(source, (frozenset(), frozenset())) + return len(cids) + len(digests) + + def accept(self, source: str, cid: Cid, entries: tuple[frozenset[bytes], frozenset[bytes]]): + """Make version cid, with entries (parse_list's), source's list; saved before it applies.""" + with self._lock: + with self._db.transaction(durable=True) as tx: + tx.execute("INSERT OR REPLACE INTO deny_lists (source, cid, accepted) VALUES (?, ?, ?)", + (statedb.dump(source), cid.binary, self.clock())) + self._accepted[source] = cid + self._lists[source] = entries + self._rebuild() + self._notify() + + def forget(self, source: str): + """Stop using source (no longer in deny_lists): its entries no longer apply.""" + with self._lock: + if source not in self._accepted and source not in self._lists: + return + with self._db.transaction(durable=True) as tx: + tx.execute("DELETE FROM deny_lists WHERE source = ?", (statedb.dump(source),)) + self._accepted.pop(source, None) + self._lists.pop(source, None) + self._rebuild() + self._notify() + + # Changes + + def add_listener(self, listener: Callable[[], None]): + """Call listener() after every change (from the changing thread, outside the lock).""" + self._listeners.append(listener) + + def _rebuild(self): + """Swap in the sets of every entry in use (lock held).""" + cids = {binary for binary, _ in self._local.values() if binary is not None} + digests = {digest for _, digest in self._local.values() if digest is not None} + for list_cids, list_digests in self._lists.values(): + cids |= list_cids + digests |= list_digests + if cids != self.cids: + self.cids = frozenset(cids) + if digests != self.hashes: + self.hashes = frozenset(digests) + + def _notify(self): + """Tell every listener; one failing is logged.""" + for listener in list(self._listeners): + try: + listener() + except Exception as e: # noqa: BLE001 - one listener mustn't stop the others + log(f"deny list: applying a change failed: {e!r}", LOG_WARNING) + + def close(self): + """Release the state database (idempotent).""" + self._db.close() diff --git a/rnfs/dht/lookup.py b/rnfs/dht/lookup.py index 8acf7ec..e82b05e 100644 --- a/rnfs/dht/lookup.py +++ b/rnfs/dht/lookup.py @@ -6,6 +6,11 @@ answered, or once every known node has been queried without finding new ones; it never runs past its `lookup_timeout` budget. Queries are blocking calls run on worker threads, so a slow or hung node never holds the caller past the deadline. + +Nodes whose recent queries all failed (the routing table's `unreliable`) rank after every other node, so a +lookup asks and waits for them only when it has too few others. One whose retry is due (`may_retry`) is asked +again on top of the `alpha` queries if it would otherwise rank among the closest, and nothing waits for that +answer: a node that never establishes a link costs a lookup no time, yet one that comes back is found. """ import queue import threading @@ -65,6 +70,8 @@ class Lookup: - `timeout`: the time budget, starting at the first run; later runs share it. - `report(text)`: told each run's start and end, and each query's outcome (the node logs them at LOG_DEBUG, for the debug log); called from the running thread and the query threads. + - `unreliable(id)`: nodes to avoid, checked when a run learns of them; `may_retry(id)`: True if an + unreliable node is due to be asked again, claiming that retry (both the routing table's). A Lookup may be run again (find_providers after every provider failed): each run re-queries the nodes, so /find replies give a fresh random half of providers, within the same deadline and unknown-ID budget. @@ -76,7 +83,8 @@ def __init__(self, key: bytes, start_nodes: Iterable[bytes], query: FindQuery, o is_known: Callable[[bytes], bool] = _always, on_answer: NodeCallback = _ignore, on_failure: NodeCallback = _ignore, exclude: Callable[[bytes], bool] = _never, clock: Callable[[], float] = time.monotonic, poll_interval: float = _POLL_INTERVAL, - report: Callable[[str], None] = _quiet): + report: Callable[[str], None] = _quiet, unreliable: Callable[[bytes], bool] = _never, + may_retry: Callable[[bytes], bool] = _never): """A lookup for key; nothing is sent until find_closest() or find_providers() runs it.""" self.key = key self.query = query @@ -92,6 +100,8 @@ def __init__(self, key: bytes, start_nodes: Iterable[bytes], query: FindQuery, o self.clock = clock self.poll_interval = poll_interval self.report = report + self.unreliable = unreliable + self.may_retry = may_retry self.deadline: float | None = None self._failed: set[bytes] = set() # didn't answer: dropped for the rest of the lookup self._answered: set[bytes] = set() # answered in any run @@ -102,6 +112,7 @@ def __init__(self, key: bytes, start_nodes: Iterable[bytes], query: FindQuery, o self._listed_by: dict[bytes, set[bytes]] = {} # provider -> nodes whose reply listed it, in any run self._last_known: dict[bytes, int] = {} # the nodes the last run knew of and hadn't seen fail self._last_queried: set[bytes] = set() # the nodes the last run queried + self._last_doubtful: set[bytes] = set() # the nodes the last run knew of as unreliable and not answered self._wake: queue.Queue | None = None self._lock = threading.Lock() self._start_nodes = [n for n in dict.fromkeys(start_nodes) if self._usable(n)] @@ -114,12 +125,14 @@ def find_closest(self) -> list[bytes]: As the spec says, these are the closest nodes known, not only those that answered: termination guarantees just the beta closest answered, and the stores sent to the rest show which are reachable. Left out: nodes that failed or were still not answering at the deadline, and - unqueried IDs whose identity isn't known (contacting them costs a path request). + unqueried IDs whose identity isn't known (contacting them costs a path request). Unreliable nodes + come after every other, so stores are sent to them only to make up k. """ self._run(None, lambda: False) candidates = [n for n in self._last_known if n in self._answered or (n not in self._last_queried and self.is_known(n))] - return sorted(candidates, key=self._distance)[:self.k] + doubtful = self._last_doubtful + return sorted(candidates, key=lambda n: (n in doubtful, self._distance(n)))[:self.k] def find_providers(self, on_provider: NodeCallback, should_stop: Callable[[], bool] = lambda: False ) -> list[bytes]: @@ -215,13 +228,24 @@ def _iterate(self, results: queue.Queue, on_provider: NodeCallback | None, for node_id in list(self._start_nodes) + self.closest_answered(): if self._usable(node_id): known[node_id] = self._distance(node_id) + # Known nodes that were unreliable when found and haven't answered: ranked after every other + doubtful = {node_id for node_id in known if self.unreliable(node_id)} queried: set[bytes] = set() answered: set[bytes] = set() + retrying: set[bytes] = set() # unreliable nodes asked again on top of alpha, which nothing waits for + passed: set[bytes] = set() # unreliable nodes whose retry wasn't due (or needs a path request) found: list[bytes] = [] in_flight = 0 + retried = 0 window = max(self.alpha, self.beta) self._last_known = known # live: find_closest reads them once the run ends self._last_queried = queried + self._last_doubtful = doubtful + + def rank(node_id: bytes) -> tuple[bool, int]: + """Sort key: reliable nodes (and those that answered) first, then by distance.""" + return node_id in doubtful, known[node_id] + started = self.clock() why = "stopped" @@ -232,16 +256,20 @@ def _iterate(self, results: queue.Queue, on_provider: NodeCallback | None, if self.clock() >= self.deadline: why = "out of time" break - ranked = sorted(known, key=known.__getitem__) + ranked = sorted(known, key=rank) if ranked and all(node_id in answered for node_id in ranked[:self.beta]): why = "complete" break # the beta closest known nodes have all answered: complete + for node_id in ranked[:window]: + if node_id in retrying: # a retry the lookup now needs: waited for like any query + retrying.discard(node_id) + in_flight += 1 # Keep up to alpha queries in flight, to the closest nodes not yet queried. Only nodes ranked # within the closest max(alpha, beta) known can affect completion, so farther ones aren't # queried while nearer queries are pending: on slow networks every query costs airtime. while in_flight < self.alpha: - ranked = sorted(known, key=known.__getitem__)[:window] + ranked = sorted(known, key=rank)[:window] node_id = next((n for n in ranked if n not in queried), None) if node_id is None: break @@ -251,6 +279,18 @@ def _iterate(self, results: queue.Queue, on_provider: NodeCallback | None, continue self._send(node_id, results) in_flight += 1 + # An unreliable node that distance alone would rank among them is asked again once its retry is + # due: only an answer shows it is back. On top of alpha, since nothing waits for it + for node_id in sorted(known, key=known.__getitem__)[:window]: + if node_id not in doubtful or node_id in queried or node_id in passed: + continue + if self.is_known(node_id) and self.may_retry(node_id): + queried.add(node_id) + retrying.add(node_id) + retried += 1 + self._send(node_id, results) + else: + passed.add(node_id) if in_flight == 0: why = "no node left to ask" break # every known node was queried and nothing new turned up @@ -265,8 +305,11 @@ def _iterate(self, results: queue.Queue, on_provider: NodeCallback | None, continue if item is _STOP: continue - in_flight -= 1 node_id, reply = item + if node_id in retrying: + retrying.discard(node_id) + else: + in_flight -= 1 if reply is None: with self._lock: self._failed.add(node_id) @@ -274,6 +317,7 @@ def _iterate(self, results: queue.Queue, on_provider: NodeCallback | None, continue answered.add(node_id) + doubtful.discard(node_id) with self._lock: self._answered.add(node_id) providers, nodes = reply @@ -285,6 +329,8 @@ def _iterate(self, results: queue.Queue, on_provider: NodeCallback | None, for new_id in nodes: if new_id not in known and new_id not in queried and self._usable(new_id): known[new_id] = self._distance(new_id) + if self.unreliable(new_id): + doubtful.add(new_id) if on_provider is not None: for provider in providers: if provider in self._reported or provider == self.own_id or self.exclude(provider): @@ -295,6 +341,7 @@ def _iterate(self, results: queue.Queue, on_provider: NodeCallback | None, failed = len(queried & self._failed) self.report(f"lookup for {self.key.hex()}: {why} after {self.clock() - started:.1f}s; asked {len(queried)}, " f"{len(answered)} answered, {failed} failed, {in_flight} still out" + + (f", retried {retried} unreliable ({len(retrying)} still out)" if retried else "") + (f"; {len(found)} new providers" if on_provider is not None else "")) return found diff --git a/rnfs/dht/routing.py b/rnfs/dht/routing.py index f8ab01c..14dabdf 100644 --- a/rnfs/dht/routing.py +++ b/rnfs/dht/routing.py @@ -3,14 +3,25 @@ Only servers that proved reachable are stored. Full buckets keep their long-lived nodes, so an attacker with fresh identities can't take a table over; nodes that stop answering are dropped, and a dropped node that comes back soon regains its seniority. Bans (section 10) also live here. + +Lookups also ask the table which nodes are unreliable: those whose last max_failures queries or more all +failed, in the table or not (a node dropped, or only named in other nodes' replies). Announces don't clear +this, only an answer does, so a far node that keeps announcing but never accepts a link stops costing every +lookup its connection timeout. Such a node may be queried again once its backoff has passed (AVOID_AFTER, +doubling with each further failure up to AVOID_MAX), by one lookup at a time (may_retry). """ import threading import time +from collections import OrderedDict from collections.abc import Callable from rnfs.config import Config from rnfs.constants import BUCKET_COUNT, ID_LENGTH, K +AVOID_AFTER = 300.0 # seconds an unreliable node isn't retried after max_failures failed queries in a row ... +AVOID_MAX = 3600.0 # ... doubling with each further failure up to this +MAX_UNRELIABLE = 4096 # nodes whose failed queries are remembered; the least recently failed is forgotten first + def distance(a: bytes, b: bytes) -> int: """Kademlia distance between two IDs: a XOR b as a 128-bit big-endian integer.""" @@ -54,6 +65,8 @@ def __init__(self, own_id: bytes, config: Config, clock: Callable[[], float] = t self._buckets: list[dict[bytes, _Entry]] = [{} for _ in range(BUCKET_COUNT)] self._dropped: dict[bytes, tuple[float, float]] = {} # node ID -> (first seen, when dropped) self._bans: dict[bytes, float] = {} # node ID -> ban end + # Node ID -> [failed queries in a row, when it may be retried], for any node; only an answer clears it + self._unanswered: OrderedDict[bytes, list] = OrderedDict() self._lock = threading.RLock() @property @@ -80,7 +93,8 @@ def _prune(self, now: float): del self._dropped[node_id] def heard(self, node_id: bytes) -> bool: - """Record that node_id proved to be a reachable server (server-mode announce, or answered /find). + """Record that node_id proved to be a reachable server (server-mode announce, or answered /find; for an + answer, call answered(), which also makes it reliable again). True if it was newly added; False if it was already in the table (now refreshed) or wasn't added (our own ID, banned, not admitted, or its bucket is full of healthy nodes). @@ -117,13 +131,24 @@ def heard(self, node_id: bytes) -> bool: bucket[node_id] = _Entry(node_id, first_seen, now) return True + def answered(self, node_id: bytes) -> bool: + """Record that node_id answered a query: heard(), and it is reliable again. True if newly added.""" + check_id(node_id) + with self._lock: + self._unanswered.pop(node_id, None) + return self.heard(node_id) + def failed(self, node_id: bytes) -> bool: - """Count a query node_id didn't answer; after max_failures in a row it is dropped. True if dropped.""" + """Count a query node_id didn't answer; after max_failures in a row it is dropped. True if dropped. + + Also counted towards unreliable(), whether or not node_id is in the table; hearing it doesn't reset that. + """ check_id(node_id) if node_id == self.own_id: return False with self._lock: now = self.clock() + self._count_unanswered(node_id, now) bucket = self._buckets[self.bucket_index(node_id)] entry = bucket.get(node_id) if entry is None: @@ -135,6 +160,42 @@ def failed(self, node_id: bytes) -> bool: self._dropped[node_id] = (entry.first_seen, now) return True + def _count_unanswered(self, node_id: bytes, now: float): + """One more failed query for node_id; past max_failures it isn't retried for a doubling backoff.""" + entry = self._unanswered.pop(node_id, None) or [0, now] + entry[0] += 1 + if entry[0] >= self.config.max_failures: + entry[1] = now + self._backoff(entry[0]) + self._unanswered[node_id] = entry # last: the most recently failed + while len(self._unanswered) > MAX_UNRELIABLE: + self._unanswered.popitem(last=False) + + def _backoff(self, count: int) -> float: + """Seconds before retrying a node whose last count queries failed (at least max_failures).""" + return min(AVOID_MAX, AVOID_AFTER * 2 ** min(count - self.config.max_failures, 32)) + + def unreliable(self, node_id: bytes) -> bool: + """True if node_id's last max_failures queries or more all failed, however recently it was heard. + + Lookups avoid such nodes unless they need them: they are queried last, and only queried before others + as a retry (may_retry), which no lookup waits for.""" + with self._lock: + entry = self._unanswered.get(node_id) + return entry is not None and entry[0] >= self.config.max_failures + + def may_retry(self, node_id: bytes) -> bool: + """True if unreliable node_id is due to be queried again, claiming that retry: other lookups see it as + not due until the query fails (backing off further) or is answered (reliable again).""" + with self._lock: + entry = self._unanswered.get(node_id) + if entry is None or entry[0] < self.config.max_failures: + return False + now = self.clock() + if now < entry[1]: + return False + entry[1] = now + self._backoff(entry[0] + 1) + return True + def remove(self, node_id: bytes): """Remove node_id (e.g. it now announces client mode), forgetting its seniority.""" check_id(node_id) diff --git a/rnfs/errors.py b/rnfs/errors.py index d7e7273..481dfeb 100644 --- a/rnfs/errors.py +++ b/rnfs/errors.py @@ -26,3 +26,7 @@ class StorageFull(RnfsError): class Busy(RnfsError): """A local limit (e.g. gateway_fetches) is reached; try again later.""" + + +class Denied(RnfsError): + """The content is on this node's deny list (section 8): it isn't fetched, read, stored, served or provided.""" diff --git a/rnfs/features/__init__.py b/rnfs/features/__init__.py index ab050ee..0fe9422 100644 --- a/rnfs/features/__init__.py +++ b/rnfs/features/__init__.py @@ -18,7 +18,8 @@ # Module paths, in start order. A feature may use the features before it FEATURE_MODULES: list[str] = ["rnfs.features.peers", "rnfs.features.jobs", "rnfs.features.keepers", - "rnfs.features.follow", "rnfs.features.contacts", "rnfs.features.lxmf"] + "rnfs.features.follow", "rnfs.features.contacts", "rnfs.features.lxmf", + "rnfs.features.deny"] ApiRoute = Callable[[object], None] # handler(api_handler) diff --git a/rnfs/features/deny.py b/rnfs/features/deny.py new file mode 100644 index 0000000..61c2f1e --- /dev/null +++ b/rnfs/features/deny.py @@ -0,0 +1,316 @@ +"""Deny lists (feature "deny"): the deny_lists setting's sources, and the local list's CLI and API (section 8). + +The deny list itself (what is checked, and where) is core: rnfs/denylist.py. This feature keeps it up to date: +- Each deny_lists entry is a source: `[/path]`, a fixed list, or `name/[/path]`, a list that follows + the name. The source's target must be a file of at most denylist.MAX_LIST_BYTES; it is fetched pinned, read, + checked (denylist.parse_list) and only then accepted: its CID is recorded (the deny_lists table) and its + blocks pinned with source "deny" while it is in use, then the version it replaces is released. A list that + can't be fetched, read or parsed is logged, and the source keeps the version it had. +- A name is resolved as `name resolve` does (persisted, so its announces keep the record held up to date), + checked every CHECK_INTERVAL from the record held, and through the network only if nothing was heard for + STALE_ANNOUNCES x announce_interval. A failed check is retried after RETRY_INTERVAL, doubling up to + announce_interval; a CID source once accepted is never checked again. +- At start, sources no longer in deny_lists are forgotten and every "deny" pin no source uses is released. +- One background thread does every check and fetch, one at a time. +- `rnfs deny add|rm|ls` and /api/deny edit and show the local list (denylist.DenyList) and the sources' state. + +No RNS at module level (the CLI imports this module, and Config.validate calls validate_settings). +""" +import threading +import time + +from rnfs.constants import ID_LENGTH +from rnfs.content.cid import Cid +from rnfs.errors import RnfsError +from rnfs.features import Feature, _common +from rnfs.features._common import LOG_ERROR, LOG_NOTICE, LOG_VERBOSE, LOG_WARNING +from rnfs.fsutil import printable + +NAME = "deny" +NAME_PREFIX = "name/" +CHECK_INTERVAL = 600.0 # seconds between checks of a name source's record held (no network) +RETRY_INTERVAL = 300.0 # seconds before a failed check is first tried again; doubles per failure +STALE_ANNOUNCES = 2 # a record not heard for this many announce_intervals is refreshed from the network +FAR_FUTURE = float("inf") + + +def parse_source(text) -> tuple[Cid | None, bytes | None, list[str]]: + """(root CID or None, name or None, path names) of a deny_lists entry: `[/path]` or + `name/[/path]`. Raises ValueError.""" + if not isinstance(text, str) or not text.strip(): + raise ValueError("deny_lists entries must be [/path] or name/[/path]") + text = text.strip() + named = text.startswith(NAME_PREFIX) + first, *names = text[len(NAME_PREFIX):].split("/") if named else text.split("/") + if any(not name for name in names): + raise ValueError(f"invalid deny_lists entry {text!r}: empty name in its path") + if not named: + try: + return Cid.parse(first), None, names + except ValueError: + raise ValueError(f"invalid deny_lists entry {text!r}: not [/path] or name/[/path]") from None + try: + name = bytes.fromhex(first) + except ValueError: + name = b"" + if len(name) != ID_LENGTH: + raise ValueError(f"invalid deny_lists entry {text!r}: a name is {2 * ID_LENGTH} hex characters") + return None, name, names + + +def validate_settings(config): + """Raise ValueError unless deny_lists is a list of sources (Config.validate).""" + if not isinstance(config.deny_lists, list): + raise ValueError("deny_lists must be a list of [/path] or name/[/path]") + for text in config.deny_lists: + parse_source(text) + + +def retry_delay(failures: int, announce_interval: float) -> float: + """Seconds before the next check after failures failed checks in a row: RETRY_INTERVAL doubling, at most + announce_interval (and never less than RETRY_INTERVAL).""" + return max(RETRY_INTERVAL, min(RETRY_INTERVAL * 2 ** min(failures - 1, 32), announce_interval)) + + +def _log(message: str, level: int = LOG_VERBOSE): + """Log with this feature's prefix.""" + _common.log(f"deny list: {message}", level) + + +class DenyFeature(Feature): + """The deny_lists sources: fetched, checked and followed on one thread. Thread-safe.""" + + name = NAME + + def __init__(self, node): + """Read the sources, forget those no longer configured and release their pins; no network.""" + super().__init__(node) + self.deny = node.deny + self._sources = list(dict.fromkeys(text.strip() for text in node.config.deny_lists)) + self._lock = threading.Lock() + self._state: dict[str, str] = {} # source -> what its last check did + self._due: dict[str, float] = {} # source -> monotonic time of its next check + self._failures: dict[str, int] = {} # source -> checks failed in a row + self._wake = threading.Event() + self._stop = threading.Event() + self._thread: threading.Thread | None = None + for source in self.deny.accepted(): + if source not in self._sources: + self.deny.forget(source) + _log(f"stopped using {source}: it is no longer in deny_lists", LOG_NOTICE) + self._release_unused() + for source in self._sources: + self._state[source] = "in use" if self.deny.in_use(source) else "not fetched yet" + + def _release_unused(self): + """Release every "deny" pin that no source's accepted version is.""" + from rnfs.content.storage import DENY + storage = self.node.storage + used = set(self.deny.accepted().values()) + released = [cid for cid in storage.pinned_by(DENY) if cid not in used and storage.unpin(cid, DENY)] + if released: + self.node.publisher.refresh() + + # Lifecycle + + def start(self): + """Check every source once interfaces had time to come up, then keep checking.""" + first = time.monotonic() + self.node.config.startup_announce_delay + with self._lock: + for source in self._sources: + self._due[source] = first + self._thread = threading.Thread(target=self._loop, name="rnfs-deny", daemon=True) + self._thread.start() + + def stop(self): + """Stop the background thread (a fetch in progress stops at its next network wait).""" + self._stop.set() + self._wake.set() + + def status(self) -> dict: + """Entries denied (the local list's and every source's) and sources configured.""" + return {"deny_entries": len(self.deny.cids) + len(self.deny.hashes), "deny_lists": len(self._sources)} + + def lists(self) -> list[dict]: + """Every source: {"source", "cid" (the version in use, or None), "entries", "state"}.""" + accepted = self.deny.accepted() + with self._lock: + return [{"source": source, "cid": str(accepted[source]) if source in accepted else None, + "entries": self.deny.entry_count(source), "state": self._state.get(source, "")} + for source in self._sources] + + # Checking and fetching + + def check(self, source: str) -> bool: + """Bring source's list up to date: True if it is current (or a CID source already accepted).""" + from rnfs import denylist + from rnfs.content import walker + from rnfs.content.storage import DENY + root, name, names = parse_source(source) + previous = self.deny.accepted().get(source) + if name is not None: + max_age = STALE_ANNOUNCES * self.node.config.announce_interval + try: + result = self.node.names.resolve(name, persist=True, max_age=max_age) + except Exception as e: # noqa: BLE001 - resolving is retried later + self._set_state(source, f"resolve failed: {e}") + return False + if result is None: + self._set_state(source, "unresolved") + return False + root = Cid.parse(result["cid"]) + elif previous is not None and self.deny.in_use(source): + self._set_state(source, "in use") + return True + try: + with self.node.fetcher.open(root, names, pinned=True) as fetch: + expect = fetch.resolve() + if previous == expect.cid and self.deny.in_use(source): + self._set_state(source, "in use") + return True + if walker.kind_of(expect, fetch.get(expect)) != "file": + raise ValueError("it is a directory, not a file") + if fetch.size_of(expect) > denylist.MAX_LIST_BYTES: + raise ValueError(f"it is over {denylist.MAX_LIST_BYTES // 1024} KiB") + self._set_state(source, f"fetching {expect.cid}") + fetch.fetch_dag(expect) + entries = denylist.parse_list(b"".join(walker.read_range(expect, fetch.get))) + # Pinned inside the fetch's protection, so the new version is never unprotected + self.node.storage.pin(expect.cid, DENY) + self.deny.accept(source, expect.cid, entries) + except (RnfsError, ValueError, OSError) as e: + kept = f"; still using {previous}" if previous is not None and self.deny.in_use(source) else "" + self._set_state(source, f"rejected: {e}{kept}") + _log(f"couldn't use {source}: {e}{kept}", LOG_WARNING) + return False + if previous is not None and previous != expect.cid: + self._release_unused() + self._set_state(source, "in use") + _log(f"using {source} at {expect.cid}: {len(entries[0]) + len(entries[1])} entries", LOG_NOTICE) + return True + + def _set_state(self, source: str, state: str): + """Record what the last check of source did.""" + with self._lock: + self._state[source] = state + + def run_due(self, now: float | None = None) -> float: + """Check every source that is due; seconds until the next is due.""" + interval = self.node.config.announce_interval + while not self._stop.is_set(): + now = time.monotonic() if now is None else now + with self._lock: + due = [source for source, when in self._due.items() if when <= now] + if not due: + upcoming = min(self._due.values(), default=now + interval) + return max(0.0, upcoming - now) + source = min(due, key=self._due.__getitem__) + self._due[source] = FAR_FUTURE + try: + ok = self.check(source) + except Exception as e: # noqa: BLE001 - e.g. disk full saving: a failed check, tried again later + _log(f"checking {source} failed: {e!r}", LOG_ERROR) + self._set_state(source, f"check failed: {e}") + ok = False + with self._lock: + if ok: + self._failures.pop(source, None) + # A CID source, once in use, never changes + self._due[source] = time.monotonic() + CHECK_INTERVAL if source.startswith(NAME_PREFIX) \ + else FAR_FUTURE + else: + failures = self._failures.get(source, 0) + 1 + self._failures[source] = failures + self._due[source] = time.monotonic() + retry_delay(failures, interval) + now = None + return 0.0 + + def _loop(self): + """The background thread: checks and fetches, one at a time, once the node has joined.""" + self.node.joined.wait() + while not self._stop.is_set(): + try: + delay = self.run_due() + except Exception as e: # noqa: BLE001 - keep checking + _log(f"checking deny lists failed: {e!r}", LOG_ERROR) + delay = RETRY_INTERVAL + self._wake.wait(min(delay, 60.0)) + self._wake.clear() + + # API and CLI + + @classmethod + def api_routes(cls) -> dict: + """GET /api/deny; POST /api/deny/add, /api/deny/rm.""" + return {("GET", "/api/deny"): _api_list, ("POST", "/api/deny/add"): _api_add, + ("POST", "/api/deny/rm"): _api_rm} + + @classmethod + def cli_commands(cls, commands, command): + """`rnfs deny add|rm|ls`.""" + deny = commands.add_parser("deny", help="the deny list: content this node refuses").add_subparsers( + dest="deny_command", required=True, metavar="command") + command(deny, "add", run_add, "deny CIDs or file hashes (sha256:): never fetched, stored, served or " + "provided").add_argument("entries", nargs="+", metavar="ENTRY") + command(deny, "rm", run_rm, "remove entries from the local deny list").add_argument( + "entries", nargs="+", metavar="ENTRY") + command(deny, "ls", run_ls, "list the local deny list, then each deny list source and its state") + + +FEATURE = DenyFeature + + +def _entries(handler) -> list: + """The request's "entries": a list of strings (ValueError otherwise).""" + entries = handler.json_body().get("entries") + if not isinstance(entries, list) or not entries or not all(isinstance(entry, str) for entry in entries): + raise ValueError("entries must be a non-empty list of CIDs or sha256: file hashes") + return entries + + +def _feature(handler) -> DenyFeature: + """The node's deny feature. NotFound if the daemon has none.""" + return _common.feature(handler, NAME, "this daemon has no deny list") + + +def _api_list(handler): + """GET /api/deny: {"entries": [local entry], "lists": [{"source", "cid", "entries", "state"}]}.""" + feature = _feature(handler) + handler.send_json(200, {"entries": feature.deny.entries(), "lists": feature.lists()}) + + +def _api_add(handler): + """POST /api/deny/add {"entries": [...]}: {"added": [entries new to the local list]}.""" + feature = _feature(handler) + handler.send_json(200, {"added": feature.deny.add(_entries(handler))}) + + +def _api_rm(handler): + """POST /api/deny/rm {"entries": [...]}: {"removed": [...]}; 404 if one isn't on the local list.""" + feature = _feature(handler) + handler.send_json(200, {"removed": feature.deny.remove(_entries(handler))}) + + +def run_add(args): + """Add entries to the local deny list.""" + import rnfs.cli + added = rnfs.cli.client(args).post("/api/deny/add", entries=args.entries)["added"] + print(f"added {len(added)} entr{'y' if len(added) == 1 else 'ies'} to the deny list") + + +def run_rm(args): + """Remove entries from the local deny list.""" + import rnfs.cli + removed = rnfs.cli.client(args).post("/api/deny/rm", entries=args.entries)["removed"] + print(f"removed {len(removed)} entr{'y' if len(removed) == 1 else 'ies'} from the deny list") + + +def run_ls(args): + """Print the local entries, one per line, then each source: source, CID in use, entries, state.""" + import rnfs.cli + result = rnfs.cli.client(args).get("/api/deny") + for entry in result["entries"]: + print(printable(entry)) + for entry in result["lists"]: + print(f"list {printable(entry['source'])} {printable(entry['cid'] or '-')} " + f"{printable(entry['entries'])} entries: {printable(entry['state'])}") diff --git a/rnfs/features/jobs.py b/rnfs/features/jobs.py index 44cccdd..09b7849 100644 --- a/rnfs/features/jobs.py +++ b/rnfs/features/jobs.py @@ -535,13 +535,15 @@ def _get(self, job: Job) -> dict: def _fetch_to(self, job: Job, path: str) -> dict: """node.get of the job's target to path: from its from node, then (fallback_dht) by DHT discovery.""" from rnfs import fetcher - from rnfs.errors import RnfsError, StorageFull, TooLarge + from rnfs.errors import Denied, RnfsError, StorageFull, TooLarge from_node = bytes.fromhex(job.from_node) if job.from_node else None try: return self.node.get(job.target, path, from_node=from_node) except RnfsError as e: - # Cancelled must stop the job; the content's size and the node's storage don't depend on the source - if from_node is None or not job.fallback_dht or isinstance(e, (fetcher.Cancelled, TooLarge, StorageFull)): + # Cancelled must stop the job; the content's size, the node's storage and its deny list don't depend + # on the source + if from_node is None or not job.fallback_dht or isinstance(e, (fetcher.Cancelled, TooLarge, StorageFull, + Denied)): raise _log(f"job {job.id}: getting {job.target} from {job.from_node} failed ({e}); trying the DHT", LOG_NOTICE) diff --git a/rnfs/features/lxmf.py b/rnfs/features/lxmf.py index 4d3a176..686c943 100644 --- a/rnfs/features/lxmf.py +++ b/rnfs/features/lxmf.py @@ -1456,7 +1456,7 @@ def auto_fetch(self, entry_id: int, index: int): or when the message is removed.""" from rnfs import fetcher from rnfs.content.storage import LXMF_PREFIX - from rnfs.errors import InvalidDag, RnfsError, StorageFull, TooLarge + from rnfs.errors import Denied, InvalidDag, RnfsError, StorageFull, TooLarge entry = self.inbox.get(entry_id) if entry is None: return @@ -1485,7 +1485,7 @@ def auto_fetch(self, entry_id: int, index: int): except fetcher.Cancelled: _log(f"auto-fetch of inbox #{entry_id} link {index + 1} stopped", LOG_VERBOSE) return # still "queued" if the node is stopping: fetched at the next start - except TooLarge as e: + except (TooLarge, Denied) as e: result = {"state": "refused", "error": str(e)} except (RnfsError, ValueError, KeyError, IndexError, TypeError) as e: result = {"state": "failed", "error": str(e) or type(e).__name__} diff --git a/rnfs/fetcher.py b/rnfs/fetcher.py index 3ad8805..ad2839e 100644 --- a/rnfs/fetcher.py +++ b/rnfs/fetcher.py @@ -49,7 +49,9 @@ that isn't being pinned evicts other cached fetches to make room for each response, and goes over max_storage if nothing is evictable (the node evicts again once it ends). What refuses a fetch, pinned or not, with StorageFull is the disk: its remaining size, and then each response, must leave the disk reserve -free (Storage.check_disk). +free (Storage.check_disk). A CID the deny list denies (rnfs.denylist; the storage's) ends a fetch with Denied +where it is met, root or child, stored or not, before it is asked for or read; a complete stored DAG is checked +whole, since fetch_dag doesn't walk it. Requests are spread over up to max_parallel_providers providers (1 on slow nodes), one /blocks request in flight per provider. Request threads only call the client; all fetch state is changed by the thread that @@ -74,7 +76,7 @@ from rnfs.content.cid import Cid from rnfs.content.storage import Storage from rnfs.content.walker import Expect -from rnfs.errors import InvalidDag, NotFound, RnfsError, TooLarge +from rnfs.errors import Denied, InvalidDag, NotFound, RnfsError, TooLarge from rnfs.log import LOG_VERBOSE, LOG_WARNING, log, node_name, plural from rnfs.network import client as peer_client from rnfs.network import protocol, rates @@ -300,13 +302,21 @@ def open(self, root: Cid, names: list[str] | tuple[str, ...] = (), from_node: by Raises InvalidDag if root was found invalid before. Its traversals raise StorageFull if what they would store doesn't fit on the disk (never because of max_storage). + Raises Denied if the deny list denies root; its traversals raise Denied at the first denied CID they meet, + stored or not, before fetching or reading it. """ if self.is_invalid(root): error = InvalidDag(f"{root} was found to be an invalid DAG") error.root = root raise error + self.check_denied(root) return Fetch(self, root, list(names), from_node, pinned) + def check_denied(self, cid: Cid): + """Raise Denied if the deny list (the storage's) denies cid.""" + if self.storage.deny.denies(cid): + raise Denied(f"{cid} is on this node's deny list") + def is_invalid(self, root: Cid) -> bool: """True if root was marked invalid locally.""" with self._lock: @@ -506,8 +516,12 @@ def fetch_dag(self, target: Expect, max_entries: int | None = None): self._declared = (self._declared or 0) + size if self.observer is not None: self.observer.total(size) - # The root's own DAG, already complete, was checked from the same expectation when it was memoised + # The root's own DAG, already complete, was checked from the same expectation when it was memoised; + # its blocks aren't met again, so the deny list is checked against them here if target == walker.root_expect(self.root) and self.storage.is_complete(target.cid): + denied = self.storage.deny.cids + if denied and any(cid.binary in denied for cid in self.storage.dag_blocks(target.cid)): + raise Denied(f"{target.cid} holds a block on this node's deny list") return # What is stored already needn't fit again (its DAG nodes make this an estimate) self.check_disk(target, size - self.storage.dag_bytes(target.cid)) @@ -651,6 +665,7 @@ def _ensure(self, expects: list[Expect]) -> dict[Expect, list[Expect]]: children: dict[Expect, list[Expect]] = {} wanted: dict[Cid, list[Expect]] = {} for expect in dict.fromkeys(expects): + self.fetcher.check_denied(expect.cid) # before fetching or reading it, stored or not walker.check_cid(expect) # before fetching a block that can't be right checked = self._check_stored(expect) if checked is None: diff --git a/rnfs/gateway.py b/rnfs/gateway.py index 72519a8..c73a560 100644 --- a/rnfs/gateway.py +++ b/rnfs/gateway.py @@ -25,7 +25,8 @@ `Origin: null` would let every site's sandboxed iframe or data: page read the gateway. Per-CID origins (a subdomain per root) would be the way to let pages read their own DAG, and are possible future work. - A GatewayError (the gateway's own refusals) or NotFound says what went wrong. Every other error (a 400, - 413, 422, 503 or 507 from the node, or a 500) gets only a fixed message for its status: a node error's text + 413, 422, 503 or 507 from the node, 451 for content the deny list blocks, or a 500) gets only a fixed + message for its status: a node error's text can name local paths, disk space or limits. The detail goes to the log (500: as an error; others verbose). - Requests that aren't the user's own navigation (Sec-Fetch-Site present and not "none") may hold at most gateway_fetches network fetches or name resolutions at a time; over that, 503 with Retry-After. @@ -72,6 +73,7 @@ INDEX = "index.html" # What an error from the node says on the gateway, by status: never its own text (see the module docstring) ERROR_MESSAGES = {400: "invalid request", 413: "too large for this node's download limits", + 451: "unavailable: this node's deny list blocks this content", 422: "invalid content: it breaks the rnfs format's rules", 503: "the node is busy; try again shortly", 507: "not enough free disk space on this node", 500: "internal error; see the rnfs daemon's log"} RANGE_NUMBER = re.compile(r"[0-9]{1,18}") # ASCII digits only (str.isdigit() takes "¹²³"), and below 2**63 diff --git a/rnfs/names.py b/rnfs/names.py index f35e8df..38fb1bc 100644 --- a/rnfs/names.py +++ b/rnfs/names.py @@ -6,8 +6,8 @@ name this node publishes or resolves for the user, and lower sequences are never accepted from any source. Names resolved for the gateway or LXMF auto-fetch keep their records in a bounded table, where announces of names this node never resolved can't push them out, and their highest sequence in a bounded persisted floor, saved -within seconds of a resolve raising it, where names resolved in the last FLOOR_KEEP_AGE are forgotten only past a -hard bound. +within seconds of a resolve or an announce raising it, where names resolved in the last FLOOR_KEEP_AGE are forgotten +only past a hard bound. State lives in the state database (rnfs.statedb), read only at startup. The records of names published here and of names resolved for the user (names_published, names_held) are saved at once and durably, each save writing only @@ -392,6 +392,7 @@ def __init__(self, config: Config, pins, network=None, clock=time.time, monotoni self._floor_dirty = False # a floor save is due self._floor_saved_at = float("-inf") # monotonic time of the last floor save self._floor_timer: threading.Timer | None = None # a save waiting for FLOOR_SAVE_INTERVAL to pass + self._closed = False # close() has run: no floor timer is started # Floor saves write without self._lock, one at a time, so an older snapshot never lands last. Taken before # self._lock, never while holding it self._floor_save_lock = threading.Lock() @@ -670,6 +671,7 @@ def stop(self): def close(self): """Release the state database, once stop() has saved everything (idempotent).""" with self._lock: + self._closed = True if self._floor_timer is not None: self._floor_timer.cancel() self._floor_timer = None @@ -892,7 +894,8 @@ def _offer(self, name: bytes, sequence: int, cid: Cid, heard: float, unasked: bo return True def _store(self, name: bytes, record: Record, held_here: bool, unasked: bool = False) -> bool: - """Put record in the persisted map or the bounded table, making room in the table. False if an unasked + """Put record in the persisted map or the bounded table, making room in the table, and raise the name's + floor to it (saved by the timer, since callers hold the lock, some on RNS's thread). False if an unasked record found no room (every place is a name resolved or kept here).""" if held_here: self._held[name] = record @@ -916,6 +919,7 @@ def _store(self, name: bytes, record: Record, held_here: bool, unasked: bool = F self._floor[name] = (record.sequence, floor[1]) self._floor_changed.add(name) self._floor_dirty = True + self._save_floor_soon(inline=False) if len(self._table) > limit: # name_table_size 0: nothing is kept self._drop_from_table(name) return True @@ -967,14 +971,15 @@ def _trim_floor(self) -> bool: trimmed = True return trimmed - def _save_floor_soon(self) -> bool: + def _save_floor_soon(self, inline: bool = True) -> bool: """True if the floor is to be saved now (the last save was FLOOR_SAVE_INTERVAL ago), by the caller once it - has released the lock; else it is saved once it is (a timer).""" + has released the lock; else it is saved once it is (a timer). Without inline, a save due now is the + timer's too, started at once.""" wait = self._floor_saved_at + FLOOR_SAVE_INTERVAL - self._monotonic() - if wait <= 0: + if wait <= 0 and inline: return True - if self._floor_timer is None: - timer = threading.Timer(wait, self._save_floor_later) + if self._floor_timer is None and not self._closed: + timer = threading.Timer(max(0.0, wait), self._save_floor_later) timer.daemon = True self._floor_timer = timer timer.start() diff --git a/rnfs/node.py b/rnfs/node.py index e1943a0..a3d8db0 100644 --- a/rnfs/node.py +++ b/rnfs/node.py @@ -12,6 +12,7 @@ Threading: RNS calls the announce and request handlers on its own threads; facade calls come from API and gateway threads. Every component locks internally; the node itself only guards the small sets it owns. """ +import hashlib import os import random import secrets @@ -20,11 +21,11 @@ import threading import time from contextlib import contextmanager -from typing import BinaryIO, Iterator +from typing import BinaryIO, Callable, Iterator import RNS -from rnfs import __version__, features, fsutil, statedb, winacl +from rnfs import __version__, denylist, features, fsutil, statedb, winacl from rnfs.config import Config from rnfs.constants import APP_NAME, DEFAULT_BOOTSTRAP_NODES, DHT_ASPECT, K, MODE_SERVER, PARTIAL_SUFFIX from rnfs.content import archive, builder, dag, edit, seal, walker @@ -32,11 +33,11 @@ from rnfs import node_content # --- content: add --no-pin/--to/--chunker, sealed files, gateway formats --- from rnfs.content import chunker as content_chunker from rnfs.content.shared_folder import SharedFolder -from rnfs.content.storage import FOLDER, USER, Storage, disk_reserve, remote_source +from rnfs.content.storage import FOLDER, REMOTE_PREFIX, USER, Storage, disk_reserve, remote_source from rnfs.dht.lookup import Lookup from rnfs.dht.providers import ProviderStore from rnfs.dht.routing import RoutingTable -from rnfs.errors import InvalidDag, NotFound, RnfsError, TooLarge +from rnfs.errors import Denied, InvalidDag, NotFound, RnfsError, TooLarge from rnfs.fetcher import Fetch, Fetcher, describe from rnfs.mesh import RnsPaths from rnfs.log import LOG_DEBUG, LOG_ERROR, LOG_NOTICE, LOG_VERBOSE, LOG_WARNING, cap_warnings, log, node_name, plural, size @@ -60,6 +61,7 @@ BOOTSTRAP_ANSWERS = 2 # a join stops contacting bootstrap nodes once this many have answered MIN_SERVERS = 3 # while fewer live servers are known, the join runs again REJOIN_DELAY = 300.0 # seconds before joining again; doubles after each join, up to announce_interval +DENY_RETRY = 60.0 # seconds before maintenance applies a deny-list change again after applying it failed KIND_NAMES = {dag.KIND_FILE: "file", dag.KIND_EXECUTABLE: "executable", dag.KIND_DIRECTORY: "directory"} @@ -236,8 +238,9 @@ def provide(self, publisher_id: bytes, keys: list[bytes], withdrawn: list[bytes] return self.node.records.provide(publisher_id, keys, withdrawn) == len(keys) def served_block(self, cid: Cid) -> bytes | None: - """cid's block if it's in the DAG of a provided CID, re-checked so a damaged block isn't sent.""" - if not self.node.publisher.serves(cid): + """cid's block if it's in the DAG of a provided CID, re-checked so a damaged block isn't sent. None for a + denied one, as for any block not held: the answer's format is unchanged.""" + if self.node.deny.denies(cid) or not self.node.publisher.serves(cid): return None block = self.node.storage.get(cid, verify=True) if block is None: @@ -247,7 +250,7 @@ def served_block(self, cid: Cid) -> bytes | None: def served_block_size(self, cid: Cid) -> int | None: """The stored length of the block served_block(cid) would return, or None if it isn't served.""" - if not self.node.publisher.serves(cid): + if self.node.deny.denies(cid) or not self.node.publisher.serves(cid): return None return self.node.storage.block_size(cid) @@ -356,6 +359,10 @@ def _setup(self, config: Config, node_id: bytes, client, slow: bool, names_netwo if removed: log(f"private mode: dropped {plural(removed, 'provider record')} of non-members", LOG_NOTICE) self.storage = Storage(config.blocks_dir, config.state_dir, config.max_storage) + # Before anything is fetched or provided: the fetcher and the publisher check it through storage.deny + self.deny = denylist.DenyList(config.state_dir) + self.storage.deny = self.deny + self.deny.reload(self._read_deny_list) self.shared_folder = SharedFolder(config.files_dir, self.storage, config.state_dir) self.pin_names = PinNames(config.state_dir) # where pins came from, for pin ls (local only) self.fetcher = Fetcher(self.storage, client, config, node_id, self.routing, self._lookup, slow, @@ -384,6 +391,10 @@ def _setup(self, config: Config, node_id: bytes, client, slow: bool, names_netwo self._remote_pins_dirty = False # changed since last saved (by maintenance, and at stop) self._remote_pins_save_lock = threading.Lock() # one save at a time, so an older snapshot never lands last self._refresh_wanted = threading.Event() + # Taken first (before the publisher's and storage's locks): one deny-list change applied at a time + self._deny_lock = threading.Lock() + self._deny_applied: frozenset[bytes] | None = None # the denied CIDs _apply_deny last applied + self._deny_retry_at = 0.0 # monotonic time before which maintenance doesn't retry a failed _apply_deny self.stop_event = threading.Event() self.joined = threading.Event() self.threads: list[threading.Thread] = [] @@ -397,6 +408,8 @@ def _setup(self, config: Config, node_id: bytes, client, slow: bool, names_netwo # What is provided, before /blocks can be asked; after the features, whose keys (kept names) would # otherwise all be withdrawn at every start self.publisher.refresh() + # Changes apply at once; what the list denied at start, maintenance applies first + self.deny.add_listener(self._apply_deny) def feature(self, name: str) -> "features.Feature": """The feature called name (rnfs.features). Raises KeyError if it isn't loaded.""" @@ -451,7 +464,7 @@ def stop(self, timeout: float = STOP_TIMEOUT): def _close_state(self): """Release every store's reference to the state database, once everything is saved; the last closes it. After a _setup that failed, only the stores it made.""" - names = ("records", "storage", "shared_folder", "pin_names", "fetcher", "publisher", "names") + names = ("records", "storage", "deny", "shared_folder", "pin_names", "fetcher", "publisher", "names") stores = [getattr(self, name) for name in names if hasattr(self, name)] for store in (*stores, *getattr(self, "features", {}).values(), getattr(self, "_db", None)): if store is None: @@ -477,20 +490,23 @@ def _lookup(self, key: bytes) -> Lookup: """A lookup for key starting from the routing table, counting answers and failures in it. It starts from the closest nodes that haven't failed since last heard or gone silent (those a /find - reply would give), with others only to make up K, so departed nodes don't take its first queries. + reply would give), with others only to make up K, so departed nodes don't take its first queries; + unreliable ones (queries failing however often they announce) only if still short of K. """ - start = self.routing.closest(key, for_reply=True) - if len(start) < K: - start += [n for n in self.routing.closest(key) if n not in start][:K - len(start)] + fresh = self.routing.closest(key, 2 * K, for_reply=True) + nearest = fresh + [n for n in self.routing.closest(key, 2 * K) if n not in fresh] + reliable = [n for n in nearest if not self.routing.unreliable(n)] + start = (reliable + [n for n in nearest if n not in reliable])[:K] return Lookup(key, start, self.client.find, self.node_id, alpha=self.config.effective_alpha(self.slow), beta=self.config.beta, timeout=self.config.lookup_timeout, is_known=self._is_known, on_answer=self._answered, on_failure=self.routing.failed, exclude=self.routing.is_banned, - report=lambda text: log(text, LOG_DEBUG)) + report=lambda text: log(text, LOG_DEBUG), unreliable=self.routing.unreliable, + may_retry=self.routing.may_retry) def _answered(self, node_id: bytes): """A node answered /find, proving it is a reachable server.""" - if self.routing.heard(node_id): + if self.routing.answered(node_id): self.publisher.wake() # a CID that found no nodes may now be publishable def _heard_server(self, node_id: bytes): @@ -583,7 +599,8 @@ def join(self): timeout=self.config.lookup_timeout, is_known=self._is_known, on_answer=self._answered, on_failure=self.routing.failed, exclude=lambda node_id: node_id in contacted or self.routing.is_banned(node_id), - report=lambda text: log(text, LOG_DEBUG)) + report=lambda text: log(text, LOG_DEBUG), unreliable=self.routing.unreliable, + may_retry=self.routing.may_retry) lookup.find_closest() log(f"joined the DHT: {plural(self.routing.node_count(), 'server')} known", LOG_NOTICE) @@ -653,7 +670,8 @@ def maintain(self, now: float, last: dict) -> None: if now - last.get("scan", float("-inf")) >= self.config.scan_interval: last["scan"] = now try: - self.shared_folder.scan() + with builder.checking(self._hash_check()): + self.shared_folder.scan() self.publisher.refresh() except Exception as e: log(f"shared folder scan failed: {e!r}", LOG_ERROR) @@ -672,8 +690,96 @@ def maintain(self, now: float, last: dict) -> None: if self._refresh_wanted.is_set(): self._refresh_wanted.clear() self.publisher.refresh() + if self.deny.cids is not self._deny_applied and time.monotonic() >= self._deny_retry_at: + self._apply_deny() self._save_remote_pins() + # Deny list (rnfs/denylist.py; its sources: rnfs/features/deny.py) + + def _read_deny_list(self, cid: Cid) -> bytes | None: + """A deny list's bytes from storage (None if it isn't wholly stored). Raises ValueError for one over + denylist.MAX_LIST_BYTES, or that is a directory.""" + if not self.storage.is_complete(cid): + return None + expect = walker.root_expect(cid) + get = self.storage.getter() + if walker.size_of(expect, get(expect)) > denylist.MAX_LIST_BYTES: + raise ValueError(f"it is over {denylist.MAX_LIST_BYTES // 1024} KiB") + return b"".join(walker.read_range(expect, get)) + + def _hash_check(self): + """What builder.checking takes for an add or import: raises Denied for a file whose content hash is + denied; None while the deny list has no file hashes (so nothing is hashed).""" + if not self.deny.hashes: + return None + + def check(digest: bytes): + """Raise Denied if a file with this content hash is denied.""" + if self.deny.denies_hash(digest): + raise Denied(f"a file with SHA-256 {digest.hex()} is on this node's deny list") + return check + + def _check_file(self, expect: walker.Expect, read: Callable[[walker.Expect], Iterator[bytes]]): + """Raise Denied if file expect's content hashes to a deny-list entry, before any of it is written out or + sent. read(expect) gives its bytes, read only if the hash isn't known yet (a one-block file's CID is + it; others are remembered by the deny list).""" + if not self.deny.hashes: + return + digest = expect.cid.digest if expect.cid.is_raw else self.deny.known_hash(expect.cid) + if digest is None: + hasher = hashlib.sha256() + for data in read(expect): + hasher.update(data) + digest = hasher.digest() + self.deny.remember_hash(expect.cid, digest) + if self.deny.denies_hash(digest): + raise Denied(f"{expect.cid} is a file whose SHA-256 ({digest.hex()}) is on this node's deny list") + + def _apply_deny(self): + """Bring what is provided, cached and pinned in line with the deny list's CIDs, after a change and at + start: stop providing (and withdraw) what reaches a denied block; drop cached fetches that do, deleting + what nothing else reaches; release remote pins that do, which are held for others. Other pins are the + user's: kept, but never served, provided or read while denied, with a warning naming each.""" + with self._deny_lock: + denied = self.deny.cids + if denied is self._deny_applied: + return + try: + self._apply_denied(denied) + except BaseException: + self._deny_retry_at = time.monotonic() + DENY_RETRY # maintenance tries again (logged by the caller) + raise + self._deny_applied = denied + + def _apply_denied(self, denied: frozenset[bytes]): + """_apply_deny's work, for the denied binary CIDs. Lock held.""" + self.publisher.refresh() + if denied: + dropped = self.storage.drop_denied(denied) + if dropped["uncached"]: + log(f"deny list: dropped {plural(len(dropped['uncached']), 'cached fetch')} holding denied content, " + f"deleting {plural(dropped['removed_blocks'], 'block')} ({size(dropped['freed_bytes'])})", + LOG_WARNING) + self._prune_fetched() + pins = self.storage.pins() + released = False + for cid in sorted(self.storage.denied_roots(pins, denied)): + for source in sorted(source for source in pins[cid] if source.startswith(REMOTE_PREFIX)): + if self.storage.unpin(cid, source): + released = True + log(f"deny list: released the remote pin of {cid} for {source}: it holds denied content", + LOG_WARNING) + kept = sorted(source for source in pins[cid] if not source.startswith(REMOTE_PREFIX)) + if kept: + log(f"deny list: pin {cid} ({', '.join(kept)}) holds denied content: it stays stored, but isn't " + f"served, provided or read. Remove the pin (rnfs pin rm, or what made it) to delete it", + LOG_WARNING) + if released: + removed = self.storage.collect() + if removed: + log(f"deny list: deleted {plural(removed, 'block')} only released remote pins held", LOG_WARNING) + self.publisher.refresh() + def _maintenance_loop(self): """Run maintain() about once a second until stopped.""" last: dict = {} @@ -789,7 +895,7 @@ def add_path(self, path: str, wrap: bool = False, only_hash: bool = False, pin: if only_hash: built, warnings = builder.add_path(path, builder.NO_PUT, wrap) else: - with self.storage.protect() as session: + with self._refusing(), self.storage.protect() as session: with session.writer() as put: if recipients: built = node_content.add_sealed_path(path, put, wrap, recipients, seal_name) @@ -813,7 +919,7 @@ def add_stream(self, stream: BinaryIO, size: int | None = None, name: str | None if only_hash: built = builder.add_stream(stream, builder.NO_PUT, size, name) else: - with self.storage.protect() as session: + with self._refusing(), self.storage.protect() as session: with session.writer() as put: built = builder.add_stream(stream, put, size, name) node_content.keep_added(self, built.cid, pin) @@ -831,6 +937,17 @@ def export_stream(self, target: str, start, for_gateway: bool = False) -> dict: writes nothing: HEAD). The gateway's ?format=archive (rnfs/node_content.py).""" return node_content.export_stream(self, target, start, for_gateway) + @contextmanager + def _refusing(self) -> Iterator[None]: + """Around an add or import: files whose content hash is denied stop it (builder.checking), and when the + deny list stops it, the blocks it stored are deleted once nothing reaches them.""" + try: + with builder.checking(self._hash_check()): + yield + except Denied: + self.storage.collect() + raise + def _added(self, built: builder.Built): """After adding: log, provide it, and enforce max_storage.""" log(f"added {built.cid} ({size(built.size)})", LOG_NOTICE) @@ -868,11 +985,15 @@ def get(self, target: str, output: str, from_node: bytes | None = None, raw: boo _check_free_space(output, content, self.config.blocks_dir, to_fetch) sealed = None if raw else node_content.open_sealed(self, fetch, expect) if sealed is not None: # decrypted as it is fetched (rnfs/content/seal.py) + self._check_file(expect, fetch.read_range) # its stored (sealed) bytes, as others hold them written = node_content.write_sealed(sealed, partial, self.config.blocks_dir, to_fetch) result = {"cid": str(expect.cid), "warnings": [], **written} else: fetch.fetch_dag(expect, self.config.max_output_entries) - warnings = walker.write_out(expect, fetch.get, partial, self.config.max_output_entries) + # Each file is checked before it is written while the deny list has file hashes + check = {"check": lambda file: self._check_file(file, lambda e: walker.read_range(e, fetch.get)) + } if self.deny.hashes else {} + warnings = walker.write_out(expect, fetch.get, partial, self.config.max_output_entries, **check) # Every skipped entry is a warning: keep a response (and a job's saved result) small result = {"cid": str(expect.cid), **cap_warnings(warnings)} fsutil.rename_new(partial, output) @@ -907,6 +1028,9 @@ def generate() -> Iterator[bytes]: """Fetch and yield, window by window; closing it ends the fetch.""" with self._fetching(root, names, from_node, for_gateway) as fetch: expect = fetch.resolve() + if self.deny.hashes and walker.kind_of(expect, fetch.get(expect)) == "file": + # The whole file is fetched and hashed (once: then remembered) before any of it is sent + self._check_file(expect, fetch.read_range) sealed = None if raw or for_gateway else node_content.open_sealed(self, fetch, expect) if sealed is not None: yield from sealed.read(offset, length) @@ -1073,6 +1197,9 @@ def remote_pin(self, identity_hash: bytes, cid: Cid, pin: bool) -> bool: log(f"released remote pin of {cid} for {source}", LOG_NOTICE) self.publisher.refresh() return True + if self.deny.denies(cid): + log(f"refused remote pin of {cid} for {source}: it is on this node's deny list", LOG_WARNING) + return False with self._lock: reserved = self._remote_reserved.setdefault(source, {}) if cid in reserved: @@ -1360,8 +1487,16 @@ def import_archive(self, path: str) -> dict: stream = open(path, "rb") except FileNotFoundError: raise NotFound(f"no such file: {path}") from None - with stream: - stats = archive.import_archive(stream, self.storage) + def check(roots: list[Cid]): + """Refuse the import if a file the roots hold has a denied content hash.""" + if self.deny.hashes: + get = self.storage.getter() + for root in roots: + for file in walker.iter_files(walker.root_expect(root), get): + self._check_file(file, lambda e: walker.read_range(e, get)) + + with stream, self._refusing(): + stats = archive.import_archive(stream, self.storage, check=check) for root in stats.get("complete", []): self.pin_names.record_if_unknown(root, f"archive {os.path.abspath(path)}") self.publisher.refresh() diff --git a/rnfs/node_content.py b/rnfs/node_content.py index 20f24c8..24f299a 100644 --- a/rnfs/node_content.py +++ b/rnfs/node_content.py @@ -184,10 +184,13 @@ def stored_head(node, target: str, length: int) -> bytes | None: def block(node, cid: str, for_gateway: bool = False) -> bytes: - """A CID's own block, fetched if needed and re-verified.""" + """A CID's own block, fetched if needed and re-verified. A raw block is a whole file, so the deny list's + file hashes apply to it (its digest is its content's SHA-256).""" root = Cid.parse(cid) with node._fetching(root, [], None, for_gateway) as fetch: expect = fetch.resolve() + if expect.cid.is_raw: + node._check_file(expect, fetch.read_range) # never reads: a raw CID carries the hash data = node.storage.get(expect.cid, verify=True) if data is None: raise NotFound(f"block {cid} was damaged locally and has been deleted; try again") @@ -197,12 +200,16 @@ def block(node, cid: str, for_gateway: bool = False) -> bytes: def export_stream(node, target: str, start: Callable[[int], BinaryIO | None], for_gateway: bool = False) -> dict: """Fetch the target's DAG, then write its archive to start(exact archive length): the stream to write to, or None to write nothing (HEAD). Limits as for export: max_download_size, max_output_entries, and the DAG - byte bound of fetch_dag.""" + byte bound of fetch_dag. Every file is checked against the deny list's file hashes before anything is sent, + as the gateway checks what it sends.""" root, names = walker.parse_target(target) with node._fetching(root, names, None, for_gateway) as fetch: expect = fetch.resolve() fetch.fetch_dag(expect, node.config.max_output_entries) walker.count_entries(expect, fetch.get, node.config.max_output_entries) + if node.deny.hashes: + for file in walker.iter_files(expect, fetch.get): + node._check_file(file, lambda e: walker.read_range(e, fetch.get)) blocks = node.storage.dag_blocks(expect.cid) length = archive.archive_size([expect.cid], ((cid, node.storage.block_size(cid) or 0) for cid in blocks)) out = start(length) diff --git a/rnfs/publisher.py b/rnfs/publisher.py index 35d8e21..2b3b6b7 100644 --- a/rnfs/publisher.py +++ b/rnfs/publisher.py @@ -2,7 +2,8 @@ - **What is provided** (provide_strategy): `pinned` provides every complete pin; `roots` also every complete fetched target while it is cached (the node says which cached targets qualify: gateway fetches never do). - Only these CIDs' DAGs are served to /blocks (section 7). + Only these CIDs' DAGs are served to /blocks (section 7). A CID whose DAG holds a block the deny list (the + storage's) denies is never provided: a change to the list takes effect at the next refresh(). - **Publishing.** A CID is due when it has never been published or its republish_interval has passed. For each due CID the K closest nodes are found; keys are grouped by node, and each node gets its keys (and any withdrawals for it) in as few /provide requests as fit, over one identified link. A refused or failed @@ -209,6 +210,9 @@ def __init__(self, own_id: bytes, config: Config, storage: Storage, providers, c # stays complete), and the DhtKeys from the keys the key sources returned self._served_from: tuple[dict[Cid, frozenset[Cid]], frozenset[Cid]] = ({}, frozenset()) self._keys_from: tuple[set[bytes], frozenset[DhtKey]] = (set(), frozenset()) + # CID -> (deny list's CID set, dag_blocks object, whether one of them is denied), so each DAG is checked + # against the deny list once per change of either + self._denied_dags: dict[Cid, tuple[frozenset[bytes], frozenset[Cid], bool]] = {} self._published: dict[Cid, tuple[float, list[bytes]]] = {} # CID -> (when, nodes that recorded it) # Other nodes that may still hold our record of a CID (earlier rounds' closest nodes, or a round cut # short), with when it was stored there; withdrawn from too, and forgotten once PROVIDER_TTL has passed @@ -398,6 +402,10 @@ def _compute(self) -> tuple[frozenset[Cid], frozenset[Cid]]: dags[cid] = self.storage.dag_blocks(cid) # memoised; raises unless complete except (NotFound, InvalidDag): continue + denied = self.storage.deny.cids + if denied: + dags = {cid: blocks for cid, blocks in dags.items() if not self._denied(cid, blocks, denied)} + self._denied_dags = {cid: checked for cid, checked in self._denied_dags.items() if cid in candidates} keys: set[bytes] = set() for source in list(self._key_sources): try: @@ -418,6 +426,14 @@ def _compute(self) -> tuple[frozenset[Cid], frozenset[Cid]]: self._keys_from = (keys, frozenset(DhtKey(key) for key in keys)) return frozenset(dags) | self._keys_from[1], served + def _denied(self, cid: Cid, blocks: frozenset[Cid], denied: frozenset[bytes]) -> bool: + """True if one of blocks (cid's DAG) is in denied, the deny list's binary CIDs; memoised. Lock held.""" + checked = self._denied_dags.get(cid) + if checked is None or checked[0] is not denied or checked[1] is not blocks: + checked = (denied, blocks, any(block.binary in denied for block in blocks)) + self._denied_dags[cid] = checked + return checked[2] + def refresh(self): """Recompute what is provided (after pins, fetches, evictions); withdraw what no longer is, at once.""" with self._lock: diff --git a/rnfs/statedb.py b/rnfs/statedb.py index 58c080a..58b1c9c 100644 --- a/rnfs/statedb.py +++ b/rnfs/statedb.py @@ -119,6 +119,12 @@ "CREATE TABLE lxmf_messages (box TEXT NOT NULL, id INTEGER NOT NULL, record TEXT NOT NULL," " PRIMARY KEY (box, id)) WITHOUT ROWID", )), + ("denylist", ( + # entry: canonical text (a CID's written form, or "sha256:" and 64 lowercase hex digits); ASCII + "CREATE TABLE deny_entries (entry TEXT PRIMARY KEY, added REAL NOT NULL) WITHOUT ROWID", + # source: a deny_lists setting as written (dump()ed); cid: the list version it last accepted + "CREATE TABLE deny_lists (source TEXT PRIMARY KEY, cid BLOB NOT NULL, accepted REAL NOT NULL) WITHOUT ROWID", + )), ) diff --git a/rnfs/winacl.py b/rnfs/winacl.py index e189192..ccda7e3 100644 --- a/rnfs/winacl.py +++ b/rnfs/winacl.py @@ -1,7 +1,7 @@ """Owner-only access on Windows: an explicit, protected DACL for the rnfs dir and its secret files (audit L7). POSIX systems get this from mode bits (a 0700 rnfs dir, 0600 key and token files, checked by -node._make_private_dirs and cli.read_token), and there everything here does nothing. Windows ignores mode bits, +node._make_private_dirs and apiclient.read_token), and there everything here does nothing. Windows ignores mode bits, so a new file just inherits its directory's access list: under the user's profile that is the user, Administrators and SYSTEM, but elsewhere (another drive, C:\\ProgramData) it usually lets every local user read the API token, the node's identity and the name keys. So on Windows: diff --git a/specification.md b/specification.md index f2a0940..8414088 100644 --- a/specification.md +++ b/specification.md @@ -505,6 +505,22 @@ and repeat. `alpha` such IDs are tried per lookup. (A keeper query asks unknown keepers from a budget of its own, also `alpha`, so nodes named in `/find` replies can't use it up, section 9.) - Nodes that fail to answer are dropped from the lookup and counted in the routing table. +- **Unreliable nodes.** An announce shows a node is alive, not that a link to it can be established: a far + node whose announces keep arriving may never accept a link, and each query to it then costs RNS's whole + link establishment timeout (about 6 s per hop). A node SHOULD keep counting the failed queries of such a + node however often it is heard, also when it isn't in the routing table (it is named in other nodes' + replies), and SHOULD NOT let it hold lookups open. In rnfs a node whose last `max_failures` queries all + failed is *unreliable* until it answers one: + - it starts lookups, is queried, counts for termination and is in a lookup's result only after every other + node known, so it is asked (and waited for) only when the lookup has too few others; + - it is asked again 5 minutes after it became unreliable, doubling with each further failure up to an + hour, by one lookup at a time, if it would otherwise rank among the nodes the lookup queries. That query + is sent on top of the `alpha` in flight and nothing waits for its answer; an answer makes it reliable + again. + + Its queries aren't shortened: on a long radio path a link may take the whole timeout to establish, and + only an answer shows the node is back. Its routing-table entry, `/find` replies and the join's count of + live servers are unchanged. - Every lookup has a time budget, `lookup_timeout`. - **Termination.** As in IPFS, a lookup is complete once the `beta` closest nodes it knows of have all answered, or once every known node has been queried without finding new ones. @@ -695,8 +711,9 @@ and repeat. **Pins.** - Pins are recursive: a pinned CID keeps every block of its DAG. - A pin is held by one or more *sources*: `user`, `folder`, `name:`, `remote:`, - `follow:` (the latest target of a followed name, section 9) and `lxmf:
` (links auto-fetched - from that LXMF sender, section 13); names, identities and addresses are 32 hex characters. It lasts while + `follow:` (the latest target of a followed name, section 9), `lxmf:
` (links auto-fetched + from that LXMF sender, section 13) and `deny` (the deny lists in use, below); names, identities and + addresses are 32 hex characters. It lasts while any source holds it. - `pin rm` removes only the `user` source. Other sources are released by whatever created them: the folder scan; `name publish` or `name rm`; `/pin` with `false` or a remote quota being exceeded; `name unfollow`, or @@ -769,6 +786,39 @@ could have linked in. It is imported at the next scan after its other links are unless their identity has left `remote_pin_identities`; a crash may lose those accepted in the last second. - `rnfs pin remote add|rm ` sends these requests over an identified link. +**Deny lists.** A node MAY refuse chosen content, so operators of public gateways and provider nodes can keep it +off their node. Nothing on the wire changes. +- Entries are CIDs and file hashes, `sha256:<64 hex digits>`: the SHA-256 of a file's whole content. A deny + list is UTF-8 text, one entry per line; blank lines and everything after `#` are ignored. The entries in use + are those of the local list (`deny add|rm`) and of every source in `deny_lists`: `[/path]`, a fixed + list, or `name/[/path]`, one that follows the name. +- A list is untrusted input. One over 4 MiB or 100 000 entries, not UTF-8, with a line that isn't an entry, or + that is a directory, is refused with a warning, and its source keeps the version it had. A version in use + stays pinned (source `deny`), so a restart applies it before anything is fetched or provided; the version + a newer one replaces is released. rnfs checks a name source every 10 minutes from the record held, through + the network only when nothing was heard for 2 × `announce_interval`; a failed check is retried after 5 + minutes, doubling up to `announce_interval`. +- A denied CID is never fetched or read, whether it is the root or anywhere in the DAG being fetched, stored + here or not: `get`, `cat`, `ls`, `stat`, `pin add`, `dir set`, `export`, the gateway, remote pins, followed + names, LXMF auto-fetch and jobs fail, saying the content is on the deny list (the API and the gateway answer + `451`). It is never stored (an add or import holding it fails and pins nothing), never served (`/blocks` + answers it as a block not held), and nothing whose DAG holds it is provided: records already published are + withdrawn. A `/pin` of it is refused. +- When the entries change, and at start: cached fetches reaching a denied block are dropped, and the blocks + nothing else reaches deleted; remote pins reaching one are released, as they are kept for others, and their + blocks deleted. The node's other pins (`user`, `folder`, `name:`, `follow:`, `lxmf:`, `deny`) are kept, + stored but neither served, provided nor read, with a warning naming each, until they are removed. Every + deletion and release is logged. +- A file's hash is only known once its content is read, so file-hash entries are checked where content is + assembled: before `get` writes a file out (nothing is left), before `cat` or the gateway sends any of a file + (so a range fetches the whole file first; its hash is then remembered; the gateway's `?format=raw` and + `?format=archive` too), and as `add`, the shared folder or `import` read it (the add or import fails). By then the file's blocks have been downloaded. A file-hash entry + doesn't stop those blocks being fetched (by `pin add`, `export` to a file, a remote pin or a followed name), stored, + served or provided: deny the CID for that. A file of one block has its content's hash as its CID's digest, + so either entry stops it; a sealed file matches by its sealed bytes, as stored, not by what it decrypts to. +- Provider records other nodes store here for a denied CID are kept and handed out as usual (`/find` sees + only keys). + ## 9. Names A **name** is a mutable pointer to a CID, owned by a key pair: the IPNS equivalent. @@ -850,8 +900,8 @@ what prevent rollback. sequence it returned, as a floor below which no record of that name is accepted, so such a resolve never goes back after table churn or a restart. The record itself isn't persisted. (rnfs: up to 4 × `name_table_size` names, least recently resolved forgotten first, except that names resolved in the - last 7 days are kept up to 8 × `name_table_size`; saved by a resolve that raises the floor, at most every - 5 s, otherwise at least every minute and at shutdown.) + last 7 days are kept up to 8 × `name_table_size`; saved within 5 s of a resolve, announce or keeper record + raising the floor, otherwise at least every minute and at shutdown.) - Records from keepers are subject to the same persisted-sequence and no-rollback rules as announces. - A record this node keeps (below) is a floor for resolving that name: it MUST be offered to the sequence rules at startup and on every resolve, so an older validly signed announce is never accepted for a kept @@ -940,6 +990,7 @@ as is cheap. Rules live in the sections referenced. | Forged or withheld kept names | Records verified against the name's key; highest persisted sequence; several keepers asked, ranked by routing-table standing from before the query and drawn from different sources; answers from established keepers only settle a name with no known sequence (9) | | Tampered or spliced sealed files | Per-segment HMAC bound to the header, index and final flag, checked before decrypting (5) | | Unsolicited LXMF links, inbox flooding | Strict field parsing, size caps per link, per message and for the whole inbox; at most 50 kept messages per untrusted sender; new ones from untrusted senders rate limited per sender, then node-wide; auto-fetched pins never pushed out by others' messages; signature-validated senders only; rate-limited path requests for unknown senders; `lxmf_accept_from`, per-sender `lxmf_auto_fetch_quota` (13) | +| Unwanted content on a public gateway or provider node | Deny lists: denied CIDs never fetched, stored, served or provided, anywhere in a DAG; file hashes checked where content is assembled; lists size-capped and checked before use (8) | | Uploads exhausting a shared channel | `upload_budget` (charged for bytes sent, at most half of it per requester in flight), interface rules and quiet hours, applied to `/blocks` only (3) | | Names and paths chosen by others reaching the local user | Default output names only if printable, not hidden, not starting with `-` and one valid file name, else the CID; non-printable characters escaped in output and logs (11) | | Republishing and retries draining airtime | Small `keep_max_names`; slower, jittered republishing of name keys; paced lookups on slow nodes; follow backoff (6, 9) | @@ -1058,6 +1109,7 @@ are local conveniences, not part of the protocol. | `export [/path] -o file [--background]`, `import file` | Archives | | `jobs [list]`, `jobs show\|wait\|cancel `, `jobs clear []` | Background jobs (below) | | `contacts add [--kind name\|node\|lxmf]`, `contacts rm `, `contacts ls` | Contacts (below) | +| `deny add\|rm …`, `deny ls` | The local deny list (CIDs, `sha256:` file hashes); `ls` also shows each `deny_lists` source, the version in use and its state (section 8) | | `send
[-m text] [--propagate] [--nearest]`, `inbox`, `inbox get [--link K] [-o out]`, `inbox rm \|--all`, `outbox`, `lxmf address`, `lxmf sync [--nearest]` | LXMF (section 13). `--nearest` uses the nearest propagation node heard when `lxmf_propagation_node` isn't set | | `peers [--json]`, `stats [--json]` | Known nodes with hops, interface and traffic; traffic since start, refusals by cause, per interface | | `status` | Node ID, RNS identity hash (the value other nodes list in `members` and `remote_pin_identities`), known nodes, records held, storage use, traffic, jobs, LXMF address (only the version without a readable API token) | @@ -1079,8 +1131,9 @@ gateway can never reach the API. - Every API request except `GET /api/status` MUST carry `X-Rnfs-Api: `, compared in constant time. The token is random (at least 128 bits), made anew at each daemon start and written to `/api_token` with mode 0600 (on Windows, with a protected access list allowing only the user and - SYSTEM); the CLI reads it from there and refuses a file other users can access. Every local process, of every user, can connect to `127.0.0.1`, and the API reads and writes - files as the daemon's user, so only users who can read the rnfs directory may use it. Web pages can't + SYSTEM); clients (the CLI, and apps through rnfs's client library `rnfs.apiclient`) read it from there and + refuse a file other users can access. Every local process, of every user, can connect to `127.0.0.1`, and + the API reads and writes files as the daemon's user, so only users who can read the rnfs directory may use it. Web pages can't learn the token, and the custom header needs a CORS preflight, which fails: this also blocks cross-site requests. `GET /api/status` stays open for health checks and the proof check below, but without the token it MUST say no more than the daemon's version (rnfs: `{"version": ...}`); the full status (node ID, @@ -1096,8 +1149,9 @@ gateway can never reach the API. anything else stops the daemon with an error. - An API request carrying `X-Rnfs-Challenge: <32 hex digits>` is answered with `X-Rnfs-Proof: hex(HMAC-SHA256(token, "rnfs api proof "))`, `` being the API's. - Before it first sends the token, the CLI checks this proof with a `GET /api/status`, so a program another - user runs on the port while the daemon is down learns neither the token nor a name key being imported. + Before it first sends the token, a client SHOULD check this proof with a `GET /api/status` (the CLI and + `rnfs.apiclient` do), so a program another user runs on the port while the daemon is down learns neither the + token nor a name key being imported. - The API never sends CORS headers. - Both servers SHOULD time-limit reading a request (rnfs: 15 s for the request line, headers and small bodies; 60 s between reads of an upload), but not the node's work, and SHOULD cap concurrent @@ -1143,8 +1197,8 @@ gateway can never reach the API. form-action 'self'`. Pages run in an opaque origin and can't load resources from other servers; navigation and WebRTC can still reveal the viewer. A node MAY disable scripts (`gateway_scripts`). - The gateway's own refusals (a bad path or CID, a method other than GET or HEAD, a `409` or `416`…) and a `404` - say what went wrong. Any other error from the node (`400`, `413`, `422`, `503`, `507` or `500`) gets only a - fixed message for its status; the details, which may name local paths, disk space or limits, go to the log. + say what went wrong. Any other error from the node (`400`, `413`, `422`, `451` for content the deny list + refuses, `503`, `507` or `500`) gets only a fixed message for its status; the details, which may name local paths, disk space or limits, go to the log. - No response carries `Access-Control-*` headers, whatever the request's `Origin` or `Referer`. Pages run in an opaque origin, so they can load files of their own DAG as subresources (images, scripts, style sheets, frames) but can't read gateway responses with `fetch()` or `XMLHttpRequest`, and neither can other web @@ -1298,6 +1352,7 @@ setting. | `lxmf_accept_from` | empty: everyone | LXMF addresses whose messages are kept | | `lxmf_auto_fetch_from` | empty | LXMF addresses whose links are fetched and pinned (`lxmf:
`) | | `lxmf_auto_fetch_quota` | 1 GiB per sender | Storage the pins of one auto-fetch sender may use | +| `deny_lists` | empty | Deny-list sources: `[/path]` or `name/[/path]`, each a file (section 8) | ## 13. rnfs URIs and LXMF diff --git a/tests/test_apiclient.py b/tests/test_apiclient.py new file mode 100644 index 0000000..4a7de02 --- /dev/null +++ b/tests/test_apiclient.py @@ -0,0 +1,401 @@ +"""Tests for the client library of the local API (rnfs/apiclient.py), against a live ApiServer with a FakeNode.""" +import http.server +import io +import os +import subprocess +import sys +import tempfile +import threading +import unittest +from unittest import mock + +from rnfs import apiclient, cli +from rnfs.api import ApiServer +from rnfs.apiclient import (ApiError, AuthFailed, BadRequest, Busy, Client, ConnectionFailed, DaemonNotRunning, Denied, + InvalidDag, NotFound, ServerError, SettingsError, StorageFull, TokenError, TooLarge) +from rnfs.config import DEFAULT_API_PORT +from rnfs import errors +from tests.support import POSIX_MODES, free_port +from tests.test_api import FakeNode + +ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + + +class FakeJobs: + """The job manager's interface, recording what it is asked.""" + + def __init__(self): + """No jobs yet.""" + self.calls = [] + + def submit(self, kind, target, output, from_node, **flags): + """A queued job.""" + self.calls.append(("submit", kind, target, output, from_node, flags)) + return {"id": "0123abcd", "kind": kind, "state": "queued"} + + def list(self): + """Every job.""" + return [{"id": "0123abcd", "state": "queued"}] + + def get(self, job_id, wait): + """One job, after waiting wait seconds.""" + self.calls.append(("get", job_id, wait)) + if job_id != "0123abcd": + raise errors.NotFound(f"no job {job_id}") + return {"id": job_id, "state": "done"} + + def cancel(self, job_id): + """A cancelled job.""" + return {"id": job_id, "state": "cancelled"} + + def clear(self, job_id): + """How many finished jobs were forgotten.""" + self.calls.append(("clear", job_id)) + return 2 + + +class NodeWithJobs(FakeNode): + """FakeNode with a job manager.""" + + def __init__(self): + """A fake node and its jobs.""" + super().__init__() + self.jobs = FakeJobs() + + def feature(self, name): + """The jobs feature; KeyError for any other.""" + if name != "jobs": + raise KeyError(name) + return self.jobs + + +class ClientTestCase(unittest.TestCase): + """Base: a NodeWithJobs behind an ApiServer, its token in /api_token, and a Client for it.""" + + def setUp(self): + """Start the server and write its token.""" + self.node = NodeWithJobs() + self.server = ApiServer(self.node, 0) + self.server.start() + self.addCleanup(self.server.stop) + tmp = tempfile.TemporaryDirectory() + self.addCleanup(tmp.cleanup) + self.tmp = tmp.name + self.token_file = os.path.join(self.tmp, "api_token") + cli.write_token(self.token_file, self.server.token) + self.api = Client(self.tmp, port=self.server.port) + + def write(self, name: str, data: bytes) -> str: + """A file in tmp holding data.""" + path = os.path.join(self.tmp, name) + with open(path, "wb") as f: + f.write(data) + return path + + def tokens_seen(self): + """A list that collects (path, X-Rnfs-Api header) of every request the server handles from now on.""" + seen = [] + original = self.server.server.RequestHandlerClass.dispatch + + def spy(handler): + """Record the request, then handle it.""" + seen.append((handler.path.split("?")[0], handler.headers["X-Rnfs-Api"])) + original(handler) + patcher = mock.patch.object(self.server.server.RequestHandlerClass, "dispatch", spy) + patcher.start() + self.addCleanup(patcher.stop) + return seen + + +class ImportTest(unittest.TestCase): + """Apps importing the library don't pay for loading RNS.""" + + def test_does_not_import_rns(self): + """Importing rnfs.apiclient and making a Client loads no RNS module.""" + code = ("import sys; from rnfs import apiclient; apiclient.Client(port=1); " + "print(any(name == 'RNS' or name.startswith('RNS.') for name in sys.modules))") + result = subprocess.run([sys.executable, "-c", code], cwd=ROOT, capture_output=True, timeout=60, + encoding="utf-8", errors="replace") + self.assertEqual(result.stdout.strip(), "False", result.stderr) + + +class DirectoryTest(unittest.TestCase): + """Where the client finds the token and the port, as the CLI does.""" + + def setUp(self): + """An empty rnfs dir.""" + tmp = tempfile.TemporaryDirectory() + self.addCleanup(tmp.cleanup) + self.tmp = tmp.name + + def settings(self, text: str): + """Write rnfs.toml.""" + with open(os.path.join(self.tmp, "rnfs.toml"), "w", encoding="utf-8") as f: + f.write(text) + + def test_default_dir_and_token(self): + """The default rnfs dir is ~/.rnfs/ (the home directory on every OS) and the token is api_token in it.""" + self.assertEqual(apiclient.DEFAULT_DIR, "~/.rnfs/") + self.assertEqual(apiclient.token_path(), os.path.join(os.path.expanduser("~/.rnfs/"), "api_token")) + self.assertEqual(Client(port=1).token_file, apiclient.token_path()) + self.assertEqual(Client(self.tmp, port=1).token_file, os.path.join(self.tmp, "api_token")) + self.assertEqual(Client(port=1, token_file="/x/t").token_file, "/x/t") + + def test_port_from_settings(self): + """The port is api_port in rnfs.toml, else the default; one given wins.""" + self.assertEqual(Client(self.tmp).port, DEFAULT_API_PORT) + self.settings("api_port = 9123\n") + self.assertEqual(Client(self.tmp).port, 9123) + self.assertEqual(Client(self.tmp, port=9124).port, 9124) + + def test_bad_settings(self): + """Settings that can't be read raise SettingsError, saying why.""" + self.settings("api_port = 99999\n") + with self.assertRaises(SettingsError) as raised: + Client(self.tmp) + self.assertIn("api_port must be a whole number", str(raised.exception)) + self.assertIsNone(raised.exception.status) + Client(self.tmp, port=1) # not read when the port is given + + +class OperationsTest(ClientTestCase): + """The main operations, each one request answered as the API documents.""" + + def test_status_and_version(self): + """status() needs the token; version() is the open status and sends none, even without a token file.""" + self.assertEqual(self.api.status()["node_id"], bytes(range(16)).hex()) + os.unlink(self.token_file) + seen = self.tokens_seen() + api = Client(self.tmp, port=self.server.port) + self.assertEqual(api.version(), self.api.daemon_version) + self.assertEqual(seen, [("/api/status", None)]) + + def test_add_and_cat(self): + """add_path (made absolute), add_stream of bytes and of a file sent chunked, and cat's ranges.""" + path = self.write("f.bin", b"hello world") + os.chdir(self.tmp) + self.addCleanup(os.chdir, ROOT) + cid = self.api.add_path("f.bin", pin=False, chunker="fastcdc")["cid"] + self.assertEqual(self.node.called("add_path")[0][0], (path,)) + self.assertEqual(self.node.called("add_path")[0][1]["pin"], False) + self.assertEqual(self.node.called("add_path")[0][1]["chunker"], "fastcdc") + self.assertEqual(self.api.add_stream(b"hello world")["cid"], cid) + self.api.add_stream(io.BytesIO(b"from a file"), name="n.txt") + self.assertEqual(self.node.called("add_stream")[1][1]["name"], "n.txt") + with self.api.cat(cid) as stream: + self.assertEqual(stream.length, 11) + self.assertEqual(b"".join(stream), b"hello world") + self.assertEqual(self.api.cat(cid, offset=6, length=3).read(), b"wor") + + def test_fetch_info_ls_stat(self): + """fetch writes the target out at an absolute path; info, ls and stat answer as the facade.""" + file_cid = self.node.put_file(b"abc") + directory = self.node.put_dir({"a.txt": file_cid}) + output = os.path.join(self.tmp, "out") + self.assertEqual(self.api.fetch(directory, output)["cid"], directory) + with open(os.path.join(output, "a.txt"), "rb") as f: + self.assertEqual(f.read(), b"abc") + self.assertEqual(self.api.info(directory)["type"], "directory") + self.assertEqual(self.api.ls(directory), [{"name": "a.txt", "kind": "file", "size": 3, "cid": file_cid}]) + self.assertEqual(self.api.stat(f"{directory}/a.txt")["size"], 3) + + def test_pins(self): + """pin_add, pin_ls and pin_rm; removing it again is NotFound.""" + cid = self.node.put_file(b"abc") + self.assertEqual(self.api.pin_add(cid), {"cid": cid}) + self.assertEqual(self.api.pin_ls(cid), [{"cid": cid, "sources": ["user"], "complete": True}]) + self.api.pin_rm(cid) + with self.assertRaises(NotFound): + self.api.pin_rm(cid) + + def test_names(self): + """name_create, name_publish, name_resolve and name_list.""" + cid = self.node.put_file(b"abc") + name = self.api.name_create("site")["name"] + self.assertEqual(self.api.name_publish("site", cid)["sequence"], 1) + self.assertEqual(self.api.name_resolve(name)["cid"], cid) + self.assertEqual([entry["alias"] for entry in self.api.name_list()], ["site"]) + with self.assertRaises(NotFound): + self.api.name_resolve("ab" * 16) + + def test_jobs(self): + """submit_job, job, jobs, cancel_job and clear_jobs reach the job manager.""" + cid = self.node.put_file(b"abc") + job = self.api.submit_job("get", cid, "out", from_node="ab" * 16, sealed_name=True) + self.assertEqual(job["id"], "0123abcd") + self.assertEqual(self.node.jobs.calls[0], + ("submit", "get", cid, os.path.abspath("out"), "ab" * 16, {"sealed_name": True, + "fallback_dht": False})) + self.assertEqual(self.api.job("0123abcd", wait=2)["state"], "done") + self.assertEqual(self.node.jobs.calls[1], ("get", "0123abcd", 2)) + self.assertEqual(self.api.jobs(), [{"id": "0123abcd", "state": "queued"}]) + self.assertEqual(self.api.cancel_job("0123abcd")["state"], "cancelled") + self.assertEqual(self.api.clear_jobs(), 2) + with self.assertRaises(NotFound): + self.api.job("0123abce") + with self.assertRaises(BadRequest): + self.api.job("nope") + + def test_used_from_several_threads(self): + """Concurrent first requests prove the daemon once, and all send the token.""" + seen = self.tokens_seen() + results = [] + threads = [threading.Thread(target=lambda: results.append(self.api.status()["node_id"])) for _ in range(8)] + for thread in threads: + thread.start() + for thread in threads: + thread.join() + self.assertEqual(len(results), 8) + self.assertEqual(seen.count(("/api/status", None)), 1) + self.assertEqual(seen.count(("/api/status", self.server.token)), 8) + + +class ProofTest(ClientTestCase): + """The token is sent only to a listener that proves it knows it.""" + + def impostor(self, received: list) -> int: + """Start a server answering everything 200 with a guessed proof; its port. What it receives goes in + received as (path, headers, body).""" + + class Impostor(http.server.BaseHTTPRequestHandler): + def log_message(self, *args): + """Quiet.""" + + def answer(self): + """200 with a guessed proof.""" + length = int(self.headers.get("Content-Length") or 0) + received.append((self.path, dict(self.headers), self.rfile.read(length) if length else b"")) + data = b'{"version": "1", "name": "x"}' + self.send_response(200) + self.send_header("Content-Length", str(len(data))) + self.send_header(apiclient.PROOF_HEADER, "00" * 32) + self.end_headers() + self.wfile.write(data) + do_GET = do_POST = answer + + server = http.server.HTTPServer(("127.0.0.1", 0), Impostor) + threading.Thread(target=server.serve_forever, daemon=True).start() + self.addCleanup(server.server_close) + self.addCleanup(server.shutdown) + return server.server_address[1] + + def test_impostor_gets_neither_token_nor_body(self): + """A listener that can't prove it knows the token gets only the challenge: AuthFailed, nothing sent.""" + received = [] + api = Client(self.tmp, port=self.impostor(received)) + with self.assertRaises(AuthFailed) as raised: + api.call("POST", "/api/name/import", params={"alias": "stolen"}, body=b"K" * 64) + self.assertIn("couldn't prove it knows the API token", str(raised.exception)) + self.assertEqual(len(received), 1) + path, headers, body = received[0] + self.assertTrue(path.startswith("/api/status")) + self.assertNotIn("X-Rnfs-Api", headers) + self.assertIn("X-Rnfs-Challenge", headers) + self.assertEqual(body, b"") + self.assertIsNone(api.token) + + def test_stale_token_is_never_sent(self): + """A token the daemon doesn't hold (an old daemon's) fails the proof and is never sent.""" + cli.write_token(self.token_file, "an old daemon's token") + seen = self.tokens_seen() + with self.assertRaises(AuthFailed): + self.api.status() + self.assertEqual(seen, [("/api/status", None)]) + + def test_version_check_in_one_place(self): + """The proof request gives the daemon's version to check_daemon, which can refuse before the token goes.""" + refused = [] + + class Strict(Client): + def check_daemon(self, version): + """Refuse every daemon, noting its version.""" + refused.append(version) + raise ApiError(f"daemon {version} is too old") + + version = self.api.version() + seen = self.tokens_seen() + with self.assertRaises(ApiError): + Strict(self.tmp, port=self.server.port).post("/api/gc") + self.assertEqual(refused, [version]) + self.assertEqual(seen, [("/api/status", None)]) + self.assertEqual(self.node.called("gc"), []) + self.api.status() + self.assertEqual(self.api.daemon_version, refused[0]) + + +class ErrorTest(ClientTestCase): + """Every failure is a typed ApiError with the message the CLI prints.""" + + def test_no_daemon(self): + """Nothing on the port: DaemonNotRunning, with no status.""" + with self.assertRaises(DaemonNotRunning) as raised: + Client(self.tmp, port=free_port()).version() + self.assertIn("no daemon is running", str(raised.exception)) + self.assertIsNone(raised.exception.status) + + def test_missing_token(self): + """No token file: TokenError, and the request isn't sent.""" + os.unlink(self.token_file) + with self.assertRaises(TokenError) as raised: + self.api.post("/api/gc") + self.assertIn("no API token", str(raised.exception)) + self.assertEqual(self.node.called("gc"), []) + + @unittest.skipUnless(POSIX_MODES, "Unix permissions") + def test_token_readable_by_others(self): + """A token file others can read isn't trusted: TokenError.""" + os.chmod(self.token_file, 0o644) + with self.assertRaises(TokenError) as raised: + self.api.status() + self.assertIn("accessible to other users", str(raised.exception)) + + def test_api_statuses(self): + """Each error the node raises arrives as its class, with the API's message and status.""" + cid = self.node.put_file(b"abc") + cases = ((ValueError("v"), BadRequest, 400), (errors.NotFound("n"), NotFound, 404), + (errors.TooLarge("t"), TooLarge, 413), (errors.InvalidDag("i"), InvalidDag, 422), + (errors.Denied("d"), Denied, 451), + (errors.Busy("b"), Busy, 503), (errors.StorageFull("s"), StorageFull, 507), + (errors.RnfsError("r"), ServerError, 500)) + for error, expected, status in cases: + with mock.patch.object(self.node, "stat", side_effect=error), mock.patch("rnfs.api.log"): + with self.assertRaises(expected) as raised: + self.api.stat(cid) + self.assertEqual((str(raised.exception), raised.exception.status), (str(error), status)) + self.assertIsInstance(raised.exception, ApiError) + + def test_refused_without_the_token(self): + """A protected endpoint asked without the token is AuthFailed (403); an unknown one NotFound.""" + with self.assertRaises(AuthFailed) as raised: + self.api.call("POST", "/api/gc", authenticated=False) + self.assertEqual(raised.exception.status, 403) + with self.assertRaises(NotFound) as raised: + self.api.get("/api/nothing") + self.assertIn("no such API endpoint", str(raised.exception)) + self.assertIsInstance(apiclient.error_for(405, "x"), ApiError) + self.assertNotIsInstance(apiclient.error_for(405, "x"), ServerError) + self.assertIsInstance(apiclient.error_for(502, "x"), ServerError) + + def test_cut_short_and_invalid_responses(self): + """A body ending before its Content-Length raises ConnectionFailed at its last read; a body that isn't + JSON where JSON is expected too.""" + cid = self.node.put_file(b"x" * 100) + + def failing(*args, **kwargs): + """Ten bytes, then the fetch fails.""" + yield b"x" * 10 + raise errors.NotFound("provider vanished") + with mock.patch.object(self.node, "read", failing), mock.patch("rnfs.api.log"): + stream = self.api.cat(cid) + self.assertEqual(stream.read(10), b"x" * 10) + with self.assertRaises(ConnectionFailed) as raised: + stream.read(10) + stream.close() + self.assertIn("the transfer stopped after 10 of 100 bytes", str(raised.exception)) + with self.assertRaises(ConnectionFailed) as raised: + self.api.call("GET", "/api/cat", params={"target": cid}) + self.assertEqual(str(raised.exception), "the daemon sent an invalid response") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_denylist.py b/tests/test_denylist.py new file mode 100644 index 0000000..6c1b530 --- /dev/null +++ b/tests/test_denylist.py @@ -0,0 +1,587 @@ +"""Tests for the deny list (rnfs/denylist.py) and its sources (rnfs/features/deny.py), with offline nodes.""" +import contextlib +import hashlib +import io +import os +import sys +import tempfile +import time +import unittest +from unittest import mock + +from rnfs import cli, denylist +from rnfs.api import ApiServer +from rnfs.config import Config +from rnfs.constants import CHUNK_SIZE +from rnfs.content.cid import DAG, RAW, Cid +from rnfs.content.storage import DENY, remote_source +from rnfs.denylist import DenyList, parse_entry, parse_list +from rnfs.errors import Denied, NotFound +from rnfs.features import deny as deny_feature +from rnfs.gateway import GatewayServer +from rnfs.names import encode_record +from rnfs.network import protocol +from rnfs.network.server import select_blocks +from tests.support import state_execute +from tests.test_api import raw_request +from tests.test_node import A_ID, NodeTest, setUpModule, tearDownModule # noqa: F401 - RNS's loglevel + +IDENTITY = bytes(range(16)) + + +def file_hash(data: bytes) -> str: + """The deny-list entry for a file holding data.""" + return "sha256:" + hashlib.sha256(data).hexdigest() + + +class ParseTest(unittest.TestCase): + """Entries and lists, which are untrusted input.""" + + def test_entries(self): + """A CID's written form, or sha256: and 64 hex digits (any case), each made canonical.""" + cid = Cid.for_block(RAW, b"x") + self.assertEqual(parse_entry(f" {cid} "), (str(cid), cid.binary, None)) + digest = hashlib.sha256(b"x").digest() + self.assertEqual(parse_entry("SHA256:" + digest.hex().upper()), ("sha256:" + digest.hex(), None, digest)) + for bad in ("", "sha256:12", "sha256:" + "g" * 64, "bafy-not-a-cid", "ab" * 16, None, 7): + with self.assertRaises(ValueError, msg=repr(bad)): + parse_entry(bad) + + def test_list(self): + """One entry per line; blank lines and everything after # ignored; CRLF line ends.""" + cid = Cid.for_block(RAW, b"x") + digest = hashlib.sha256(b"y").digest() + text = f"# a deny list\r\n\r\n{cid} # spam\r\n sha256:{digest.hex()}\n#{Cid.for_block(RAW, b'z')}\n" + self.assertEqual(parse_list(text.encode()), (frozenset({cid.binary}), frozenset({digest}))) + self.assertEqual(parse_list(b""), (frozenset(), frozenset())) + + def test_bad_lists_are_refused(self): + """A line that isn't an entry (saying which), text that isn't UTF-8, and lists over the limits.""" + with self.assertRaisesRegex(ValueError, "line 2"): + parse_list(f"{Cid.for_block(RAW, b'x')}\nnot an entry\n".encode()) + with self.assertRaisesRegex(ValueError, "UTF-8"): + parse_list(b"\xff\xfe") + with self.assertRaisesRegex(ValueError, "KiB"): + parse_list(b"#" * (denylist.MAX_LIST_BYTES + 1)) + with mock.patch.object(denylist, "MAX_LIST_ENTRIES", 2): + lines = "\n".join(str(Cid.for_block(RAW, bytes([i]))) for i in range(3)) + with self.assertRaisesRegex(ValueError, "over 2 entries"): + parse_list(lines.encode()) + + +class DenyListTest(unittest.TestCase): + """The store: local entries and accepted lists, saved in the state database.""" + + def setUp(self): + """A DenyList in a temporary state directory.""" + tmp = tempfile.TemporaryDirectory() + self.addCleanup(tmp.cleanup) + self.state_dir = tmp.name + self.deny = self.open() + + def open(self) -> DenyList: + """A DenyList over the state directory, closed after the test.""" + deny = DenyList(self.state_dir) + self.addCleanup(deny.close) + return deny + + def test_local_entries_persist(self): + """Added entries apply at once and survive a restart; removing one that isn't there adds nothing.""" + cid = Cid.for_block(RAW, b"x") + digest = hashlib.sha256(b"y").digest() + changes = [] + self.deny.add_listener(lambda: changes.append(1)) + self.assertEqual(self.deny.add([str(cid), "sha256:" + digest.hex(), str(cid)]), + sorted([str(cid), "sha256:" + digest.hex()])) + self.assertTrue(self.deny.denies(cid)) + self.assertTrue(self.deny.denies_hash(digest)) + self.assertEqual(self.deny.add([str(cid)]), []) + self.assertEqual(len(changes), 1) + self.deny.close() + again = self.open() + self.assertTrue(again.denies(cid)) + with self.assertRaises(NotFound): + again.remove([str(cid), "sha256:" + "0" * 64]) + self.assertTrue(again.denies(cid)) # nothing removed + self.assertEqual(again.remove([str(cid)]), [str(cid)]) + self.assertFalse(again.denies(cid)) + with self.assertRaises(ValueError): + again.add(["nonsense"]) + + def test_local_list_is_bounded(self): + """At most MAX_LOCAL_ENTRIES local entries.""" + with mock.patch.object(denylist, "MAX_LOCAL_ENTRIES", 1): + self.deny.add([str(Cid.for_block(RAW, b"a"))]) + with self.assertRaises(ValueError): + self.deny.add([str(Cid.for_block(RAW, b"b"))]) + + def test_invalid_rows_are_dropped(self): + """Rows that don't parse (planted by hand) are logged, deleted, and ignored.""" + self.deny.close() + state_execute(self.state_dir, "INSERT INTO deny_entries VALUES (?, ?)", ("junk", 1.0)) + state_execute(self.state_dir, "INSERT INTO deny_lists VALUES (?, ?, ?)", ("not json", b"x", 1.0)) + with mock.patch("rnfs.denylist.log") as log: + again = self.open() + self.assertEqual(again.entries(), []) + self.assertEqual(again.accepted(), {}) + self.assertIn("invalid row", log.call_args.args[0]) + + def test_lists_reload_from_storage(self): + """An accepted list is remembered by CID; reload() reads its entries back, and one that can't be read + isn't used (left to fetch again).""" + listed = Cid.for_block(RAW, b"listed") + version = Cid.for_block(RAW, b"v1") + self.deny.accept("name/" + "ab" * 16, version, (frozenset({listed.binary}), frozenset())) + self.assertTrue(self.deny.denies(listed)) + self.deny.close() + again = self.open() + self.assertFalse(again.denies(listed)) # not read yet + again.reload(lambda cid: f"{listed}\n".encode() if cid == version else None) + self.assertTrue(again.denies(listed)) + self.assertEqual(again.entry_count("name/" + "ab" * 16), 1) + third = self.open() + with mock.patch("rnfs.denylist.log"): + third.reload(lambda cid: None) + self.assertFalse(third.in_use("name/" + "ab" * 16)) + third.forget("name/" + "ab" * 16) + self.assertEqual(third.accepted(), {}) + + +class DenyNodeTest(NodeTest): + """Two offline nodes: A provides content, B fetches it (tests.test_node).""" + + def deny(self, node, *entries): + """Add entries to node's local list, its warnings silenced.""" + with mock.patch("rnfs.node.log"): + node.deny.add([str(entry) for entry in entries]) + + def shared(self, data: bytes) -> Cid: + """A file A added and provides.""" + cid = self.a.add_stream(io.BytesIO(data))["cid"] + self.advertise(cid) + return Cid.parse(cid) + + +class FetchRefusalTest(DenyNodeTest): + """Denied CIDs are never fetched or read, as a root or inside a DAG, stored or not.""" + + def test_denied_root(self): + """get, cat, info, ls, stat, pin add and export of a denied root fail with Denied, fetching nothing.""" + cid = self.shared(b"denied content") + self.deny(self.b, cid) + out = os.path.join(self.tmp.name, "out") + for call in (lambda: self.b.get(str(cid), out), lambda: b"".join(self.b.read(str(cid))), + lambda: self.b.info(str(cid)), lambda: self.b.stat(str(cid)), + lambda: self.b.pin_add(str(cid)), lambda: self.b.export(str(cid), out), + lambda: self.b.block(str(cid))): + with self.assertRaises(Denied): + call() + self.assertFalse(os.path.exists(out)) + self.assertFalse(self.b.storage.has(cid)) + + def test_denied_inside_a_dag(self): + """A directory holding a denied file can't be fetched; its other files and listing still can be read.""" + root = self.a.add_path(self.tree())["cid"] + self.advertise(root) + note = Cid.parse(self.a.info(f"{root}/sub/deeper/note.txt")["cid"]) + self.deny(self.b, note) + out = os.path.join(self.tmp.name, "out") + with self.assertRaises(Denied): + self.b.get(root, out) + self.assertEqual([name for name in os.listdir(self.tmp.name) if name.startswith(".rnfs-get-")], []) + self.assertFalse(os.path.exists(out)) + self.assertFalse(self.b.storage.has(note)) + with self.assertRaises(Denied): + self.b.pin_add(root) + with self.assertRaises(Denied): + b"".join(self.b.read(f"{root}/sub/deeper/note.txt")) + self.assertEqual([e["name"] for e in self.b.ls(root)], ["big.bin", "sub"]) + self.assertEqual(b"".join(self.b.read(f"{root}/big.bin")), self.big) + + def test_stored_content_denied_later(self): + """Content stored before it was denied (a complete pin) can't be read or written out any more.""" + root = self.a.add_path(self.tree())["cid"] + self.advertise(root) + self.b.pin_add(root) + note = Cid.parse(self.a.info(f"{root}/sub/deeper/note.txt")["cid"]) + self.deny(self.b, note) + with self.assertRaises(Denied): + self.b.get(root, os.path.join(self.tmp.name, "out")) + with self.assertRaises(Denied): + self.b.export(root, os.path.join(self.tmp.name, "out.rnfs")) + with self.assertRaises(Denied): + b"".join(self.b.read(f"{root}/sub/deeper/note.txt")) + + def test_jobs_and_remote_pins_refuse(self): + """A /pin of a denied CID is refused, and a get job fails with the deny list's message, no fallback.""" + cid = self.shared(b"denied content") + self.deny(self.b, cid) + with mock.patch("rnfs.node.log"): + self.assertFalse(self.b.remote_pin(IDENTITY, cid, True)) + jobs = self.b.feature("jobs") + jobs.start() # offline nodes start no feature + self.addCleanup(jobs.stop) + job = jobs.submit("get", str(cid), os.path.join(self.tmp.name, "out"), A_ID.hex(), fallback_dht=True) + for _ in range(200): + if jobs.get(job["id"], 0)["state"] not in ("queued", "running"): + break + time.sleep(0.05) + finished = jobs.get(job["id"], 0) + self.assertEqual(finished["state"], "failed") + self.assertIn("deny list", finished["error"]) + + +class ServeAndProvideTest(DenyNodeTest): + """Denied blocks aren't served or provided, and published records are withdrawn.""" + + def test_not_served(self): + """/blocks answers a denied block as not held: the same answer, on the wire, as for one never stored.""" + cid = self.shared(b"served until denied") + self.assertIsNotNone(self.a.backend.served_block(cid)) + self.deny(self.a, cid) + self.assertIsNone(self.a.backend.served_block(cid)) + self.assertIsNone(self.a.backend.served_block_size(cid)) + never = Cid.for_block(RAW, b"never stored") + entries = select_blocks([cid, never], self.a.backend.served_block, 10_000, self.a.backend.served_block_size) + self.assertEqual(entries, [None, None]) + wire = protocol.encode_blocks_response(entries) + self.assertEqual(wire, protocol.encode_blocks_response([None, None])) + self.assertEqual(protocol.parse_blocks_response(wire, 2), [None, None]) + with self.assertRaises(NotFound): + b"".join(self.b.read(str(cid))) # B finds A as the provider, which answers "not held" + + def test_not_provided_and_withdrawn(self): + """A pin that is, or holds, a denied block stops being provided, and its record is withdrawn.""" + root = Cid.parse(self.a.add_path(self.tree())["cid"]) + self.a.publisher.publish_due() + self.assertIn(root, self.a.publisher.provided()) + note = Cid.parse(self.a.info(f"{root}/sub/deeper/note.txt")["cid"]) + self.deny(self.a, note) + self.assertNotIn(root, self.a.publisher.provided()) + self.assertIsNone(self.a.backend.served_block(root)) + self.assertEqual(self.a.records.for_reply(root.key), []) # its own record, withdrawn locally + self.a.deny.remove([str(note)]) + self.assertIn(root, self.a.publisher.provided()) + + def test_not_stored(self): + """Storage refuses a denied block, so an add holding one fails and pins nothing.""" + chunk = b"x" * 100 + self.deny(self.a, Cid.for_block(RAW, chunk)) + with self.assertRaises(Denied): + self.a.storage.put(Cid.for_block(RAW, chunk), chunk) + with self.assertRaises(Denied): + self.a.add_stream(io.BytesIO(chunk)) + self.assertEqual(self.a.storage.pins(), {}) + # --only-hash stores nothing, so it isn't refused + self.assertEqual(self.a.add_stream(io.BytesIO(chunk), only_hash=True)["cid"], str(Cid.for_block(RAW, chunk))) + + +class StoredContentTest(DenyNodeTest): + """What a list change does to what is already stored.""" + + def test_cache_is_dropped_and_collected(self): + """A cached download holding a denied block is dropped and its blocks deleted, with a warning.""" + root = self.a.add_path(self.tree())["cid"] + self.advertise(root) + self.b.get(root, os.path.join(self.tmp.name, "out")) + self.assertTrue(self.b.storage.cached_targets()) + note = Cid.parse(self.a.info(f"{root}/sub/deeper/note.txt")["cid"]) + with mock.patch("rnfs.node.log") as log: + self.b.deny.add([str(note)]) + self.assertEqual(self.b.storage.cached_targets(), set()) + self.assertEqual(self.b.storage.block_count(), 0) + self.assertTrue(any("dropped 1 cached fetch" in call.args[0] for call in log.call_args_list)) + + def test_remote_pins_released_user_pins_kept(self): + """A remote pin holding denied content is released and collected; a user pin stays, with a warning.""" + mine = self.shared(b"pinned by the user") + theirs = self.shared(b"pinned for another node") + self.b.pin_add(str(mine)) + source = remote_source(IDENTITY) + self.b.pin_add(str(theirs)) + self.b.storage.pin(theirs, source) # as a remote pin's fetch leaves it + self.b.pin_rm(str(theirs)) + self.assertEqual(self.b.storage.pins()[theirs], {source}) + with mock.patch("rnfs.node.log") as log: + self.b.deny.add([str(mine), str(theirs)]) + pins = self.b.storage.pins() + self.assertNotIn(theirs, pins) + self.assertFalse(self.b.storage.has(theirs)) + self.assertEqual(pins[mine], {"user"}) + self.assertTrue(self.b.storage.has(mine)) + messages = [call.args[0] for call in log.call_args_list] + self.assertTrue(any(f"released the remote pin of {theirs}" in m for m in messages)) + self.assertTrue(any(f"pin {mine} (user) holds denied content" in m for m in messages)) + self.assertNotIn(mine, self.b.publisher.provided()) + + def test_failed_apply_is_retried(self): + """A change that couldn't be applied (the disk failed) is applied again by maintenance, after a pause.""" + cid = self.shared(b"cached, then denied") + b"".join(self.b.read(str(cid))) + with mock.patch.object(self.b.storage, "drop_denied", side_effect=OSError("disk failed")), \ + mock.patch("rnfs.node.log"), mock.patch("rnfs.denylist.log"): + self.b.deny.add([str(cid)]) + self.b.maintain(time.monotonic(), {}) # too soon to try again + self.assertTrue(self.b.storage.cached_targets()) + self.b._deny_retry_at = 0.0 # as DENY_RETRY later + with mock.patch("rnfs.node.log"): + self.b.maintain(time.monotonic(), {}) + self.assertEqual(self.b.storage.cached_targets(), set()) + + def test_applied_at_start(self): + """A node started with entries in its list applies them in its first maintenance pass.""" + cid = self.shared(b"cached, then denied while stopped") + b"".join(self.b.read(str(cid))) + self.b.stop() + state_execute(self.b.config.state_dir, "INSERT INTO deny_entries VALUES (?, ?)", (str(cid), 1.0)) + again = self.make_node("b", bytes(15) + b"\x02") + self.assertTrue(again.deny.denies(cid)) + with mock.patch("rnfs.node.log"): + again.maintain(time.monotonic(), {}) + self.assertEqual(again.storage.cached_targets(), set()) + + +class FileHashTest(DenyNodeTest): + """sha256: entries, checked where content is assembled.""" + + def big_file(self) -> tuple[Cid, bytes]: + """A multi-chunk file A provides, and its content.""" + data = os.urandom(2 * CHUNK_SIZE + 5) + return self.shared(data), data + + def test_get_write_out(self): + """get of a denied file, or of a directory holding one, fails and leaves no output.""" + cid, data = self.big_file() + self.deny(self.b, file_hash(data)) + out = os.path.join(self.tmp.name, "out") + with self.assertRaises(Denied): + self.b.get(str(cid), out) + root = self.a.add_path(self.tree())["cid"] + self.advertise(root) + self.deny(self.b, file_hash(b"deep")) # one chunk: its CID is its hash, nothing to read + with self.assertRaises(Denied): + self.b.get(root, out) + self.assertFalse(os.path.exists(out)) + self.assertEqual([name for name in os.listdir(self.tmp.name) if name.startswith(".rnfs-")], []) + self.assertEqual(self.b.deny.known_hash(cid), hashlib.sha256(data).digest()) + + def test_read_checks_before_sending(self): + """cat (and the gateway) of a denied file fails before its first byte, even for a range.""" + cid, data = self.big_file() + self.deny(self.b, file_hash(data)) + reader = self.b.read(str(cid), offset=CHUNK_SIZE, length=10) + with self.assertRaises(Denied): + next(reader) + + def test_gateway_451(self): + """The gateway answers 451 with a fixed message, for a denied CID and a file with a denied hash.""" + cid, data = self.big_file() + other = self.shared(b"denied by CID") + self.deny(self.b, file_hash(data), other) + server = GatewayServer(self.b, self.b.config, 0) + server.start() + self.addCleanup(server.stop) + for target in (cid, other): + status, _, body = raw_request(server.port, "GET", f"/rnfs/{target}", None, None, "default") + self.assertEqual(status, 451) + self.assertEqual(body, b"unavailable: this node's deny list blocks this content\n") + + def test_raw_block_and_gateway_archive(self): + """cat --block (?format=raw) of a one-block file with a denied hash, and an archive the gateway would + stream holding a file with one, are refused before anything is sent.""" + data = b"a small file" + cid = self.shared(data) + root = self.a.add_path(self.tree())["cid"] + self.advertise(root) + self.deny(self.b, file_hash(data), file_hash(self.big)) + with self.assertRaises(Denied): + self.b.block(str(cid)) + started = [] + with self.assertRaises(Denied): + self.b.export_stream(root, started.append, for_gateway=True) + self.assertEqual(started, []) + + def test_add_and_import(self): + """An add or import holding a file with a denied hash fails, pins nothing and leaves no blocks.""" + data = os.urandom(CHUNK_SIZE + 9) + self.deny(self.a, file_hash(data)) + before = self.a.storage.block_count() + with self.assertRaises(Denied): + self.a.add_stream(io.BytesIO(data)) + path = os.path.join(self.tmp.name, "f.bin") + with open(path, "wb") as f: + f.write(data) + with self.assertRaises(Denied): + self.a.add_path(path) + self.assertEqual(self.a.storage.block_count(), before) + # An archive made on B, which doesn't deny it + cid = self.b.add_stream(io.BytesIO(data))["cid"] + archive_path = os.path.join(self.tmp.name, "a.rnfs") + self.b.export(cid, archive_path) + with self.assertRaises(Denied): + self.a.import_archive(archive_path) + self.assertEqual(self.a.storage.pins(), {}) + self.assertEqual(self.a.storage.block_count(), before) + + def test_shared_folder_skips(self): + """A file with a denied hash in the shared folder isn't imported.""" + self.deny(self.a, file_hash(b"not shared")) + os.makedirs(self.a.config.files_dir, exist_ok=True) + with open(os.path.join(self.a.config.files_dir, "f.txt"), "wb") as f: + f.write(b"not shared") + with mock.patch("rnfs.content.shared_folder.log"): + self.a.maintain(time.monotonic(), {}) + self.assertEqual(self.a.storage.pinned_by("folder"), set()) + + +class SourceTest(DenyNodeTest): + """deny_lists sources, fetched and followed by the deny feature.""" + + def publish_list(self, *entries) -> Cid: + """A deny list file A adds and provides.""" + return self.shared("".join(f"{entry}\n" for entry in entries).encode()) + + def node_with(self, *sources, label="c", node_id=bytes(15) + b"\x03"): + """A node C (asking A) whose deny_lists are sources.""" + node = self.make_node(label, node_id, deny_lists=list(sources)) + node.routing.heard(A_ID) + return node + + def check(self, node, source) -> bool: + """Run the deny feature's check of source, its log silenced.""" + with mock.patch("rnfs.features.deny._common.log"), mock.patch("rnfs.node.log"): + return node.feature("deny").check(source) + + def test_list_by_cid(self): + """A CID source is fetched, pinned with source deny, applied, and read back after a restart.""" + denied = self.shared(b"listed") + version = self.publish_list(denied, file_hash(b"anything")) + c = self.node_with(str(version)) + self.assertTrue(self.check(c, str(version))) + self.assertTrue(c.deny.denies(denied)) + self.assertEqual(c.storage.pins()[version], {DENY}) + self.assertEqual(c.feature("deny").lists(), [{"source": str(version), "cid": str(version), "entries": 2, + "state": "in use"}]) + self.assertTrue(self.check(c, str(version))) # nothing more to do + c.stop() + again = self.node_with(str(version)) + self.assertTrue(again.deny.denies(denied)) # from storage, before any check + + def test_list_by_name_follows_updates(self): + """A name source follows the name: a new version replaces the entries, and the old one is released.""" + first, second = self.shared(b"first"), self.shared(b"second") + v1, v2 = self.publish_list(first), self.publish_list(second) + name = bytes(range(100, 116)) + c = self.node_with(f"name/{name.hex()}") + c.names._network.announce(name, encode_record(1, v1), time.time()) + self.assertTrue(self.check(c, f"name/{name.hex()}")) + self.assertTrue(c.deny.denies(first)) + c.names._network.announce(name, encode_record(2, v2), time.time()) + self.assertTrue(self.check(c, f"name/{name.hex()}")) + self.assertFalse(c.deny.denies(first)) + self.assertTrue(c.deny.denies(second)) + self.assertEqual(c.storage.pinned_by(DENY), {v2}) + + def test_bad_lists_keep_the_previous_version(self): + """A malformed or oversized new version, or a directory, is rejected with a warning; the old one stays.""" + good = self.publish_list(self.shared(b"listed")) + bad = self.shared(b"bafy not a cid\n") + name = bytes(range(100, 116)) + source = f"name/{name.hex()}" + c = self.node_with(source) + c.names._network.announce(name, encode_record(1, good), time.time()) + self.assertTrue(self.check(c, source)) + c.names._network.announce(name, encode_record(2, bad), time.time()) + with mock.patch("rnfs.features.deny._common.log") as log, mock.patch("rnfs.node.log"): + self.assertFalse(c.feature("deny").check(source)) + self.assertIn("line 1", log.call_args.args[0]) + self.assertIn(f"still using {good}", c.feature("deny").lists()[0]["state"]) + big = self.publish_list(*[Cid.for_block(RAW, bytes([i])) for i in range(20)]) + c.names._network.announce(name, encode_record(3, big), time.time()) + with mock.patch.object(denylist, "MAX_LIST_BYTES", 100): + self.assertFalse(self.check(c, source)) + directory = Cid.parse(self.a.add_path(self.tree())["cid"]) + self.advertise(str(directory)) + c.names._network.announce(name, encode_record(4, directory), time.time()) + self.assertFalse(self.check(c, source)) + self.assertEqual(c.deny.accepted()[source], good) + self.assertEqual(c.deny.entry_count(source), 1) + self.assertEqual(c.storage.pinned_by(DENY), {good}) + + def test_removed_source_is_forgotten(self): + """A source no longer in deny_lists at start stops applying, and its pin is released.""" + denied = self.shared(b"listed") + version = self.publish_list(denied) + c = self.node_with(str(version)) + self.check(c, str(version)) + c.stop() + with mock.patch("rnfs.features.deny._common.log"): + again = self.node_with() + self.assertFalse(again.deny.denies(denied)) + self.assertEqual(again.storage.pinned_by(DENY), set()) + + def test_settings(self): + """deny_lists takes [/path] and name/[/path], nothing else.""" + cid = Cid.for_block(DAG, b"d") + Config(deny_lists=[str(cid), f"{cid}/lists/spam.txt", "name/" + "ab" * 16, "name/" + "ab" * 16 + "/x"] + ).validate() + for bad in ("ab" * 16, "name/abc", f"{cid}//x", "", 5): + with self.assertRaises(ValueError, msg=repr(bad)): + Config(deny_lists=[bad]).validate() + with self.assertRaises(ValueError): + Config(deny_lists="name/" + "ab" * 16).validate() + self.assertEqual(deny_feature.parse_source(f"{cid}/a/b"), (cid, None, ["a", "b"])) + + +class CliTest(DenyNodeTest): + """rnfs deny add|rm|ls against an offline node's API.""" + + def run_cli(self, *argv) -> tuple[int, str, str]: + """Run cli.main against B's API: (exit code, stdout, stderr).""" + out, err = io.StringIO(), io.StringIO() + code = 0 + with contextlib.redirect_stdout(out), contextlib.redirect_stderr(err), mock.patch("rnfs.node.log"): + try: + cli.main([*argv, "--api-port", str(self.server.port), "--dir", self.b.config.rnfs_dir]) + except SystemExit as e: + code = 1 if isinstance(e.code, str) else (e.code or 0) + if isinstance(e.code, str): + print(e.code, file=err) + return code, out.getvalue(), err.getvalue() + + def test_commands(self): + """add, ls and rm print what they did; rm of an entry not listed fails.""" + self.server = ApiServer(self.b, 0) + self.server.start() + self.addCleanup(self.server.stop) + cli.write_token(os.path.join(self.b.config.rnfs_dir, "api_token"), self.server.token) + cid = Cid.for_block(RAW, b"x") + self.assertEqual(self.run_cli("deny", "add", str(cid), file_hash(b"y"))[:2], + (0, "added 2 entries to the deny list\n")) + self.assertEqual(self.run_cli("deny", "ls")[1], "".join(f"{entry}\n" for entry in sorted([str(cid), + file_hash(b"y")]))) + self.assertEqual(self.run_cli("deny", "rm", str(cid))[:2], (0, "removed 1 entry from the deny list\n")) + code, _, err = self.run_cli("deny", "rm", str(cid)) + self.assertEqual(code, 1) + self.assertIn("not on the local deny list", err) + + def test_client_library(self): + """rnfs.apiclient's deny methods, and Denied (451) for denied content.""" + from rnfs import apiclient + self.server = ApiServer(self.b, 0) + self.server.start() + self.addCleanup(self.server.stop) + cli.write_token(os.path.join(self.b.config.rnfs_dir, "api_token"), self.server.token) + api = apiclient.Client(self.b.config.rnfs_dir, port=self.server.port) + cid = self.shared(b"denied through the API") + with mock.patch("rnfs.node.log"): + self.assertEqual(api.deny_add([str(cid)]), [str(cid)]) + self.assertEqual(api.deny(), {"entries": [str(cid)], "lists": []}) + with self.assertRaises(apiclient.Denied) as raised, mock.patch("rnfs.api.log"): + api.stat(str(cid)) + self.assertEqual(raised.exception.status, 451) + self.assertEqual(api.deny_rm([str(cid)]), [str(cid)]) + + +if __name__ == "__main__": + unittest.main(argv=sys.argv) diff --git a/tests/test_lookup.py b/tests/test_lookup.py index b02b9c4..5d0bc3e 100644 --- a/tests/test_lookup.py +++ b/tests/test_lookup.py @@ -8,7 +8,8 @@ from rnfs.constants import ID_LENGTH, K from rnfs.dht.lookup import Lookup from rnfs.dht.providers import ProviderStore -from rnfs.dht.routing import RoutingTable, distance +from rnfs.dht.routing import AVOID_AFTER, RoutingTable, distance +from tests.support import FakeClock, wait_for CONFIG = Config() @@ -334,6 +335,116 @@ def test_budget_shared_with_providers(self): self.assertTrue(lookup.allow_unknown(ident(9))) +class UnreliableNodeTest(LookupTestCase): + """Nodes whose queries keep failing are asked last, and a retry of one is never waited for.""" + + def setUp(self): + """A key with two unreliable nodes closest to it, which hang, and three reliable ones beyond them.""" + super().setUp() + self.net = self.network() + self.key = ident(0x1000) + self.dead = [ident(0x1001), ident(0x1002)] + self.near = [ident(0x1010), ident(0x1020), ident(0x1030)] + for n in self.dead + self.near: + self.net.add(n) + self.net.hang.update(self.dead) + + def test_not_asked_while_enough_others_answer(self): + """The lookup completes from the reliable nodes without asking or waiting for the unreliable ones.""" + started = time.monotonic() + result = self.lookup(self.net, self.key, self.dead + self.near, timeout=30, + unreliable=self.dead.__contains__).find_closest() + self.assertLess(time.monotonic() - started, 2) + self.assertEqual(result, self.near + self.dead) # not asked: after every other node, to make up k + self.assertFalse(set(self.net.queried) & set(self.dead)) + + def test_unreliable_nodes_named_in_replies_too(self): + """Unreliable nodes a reply names rank last as well.""" + for n in self.near: + self.net.nodes[n].table.heard(self.dead[0]) + started = time.monotonic() + result = self.lookup(self.net, self.key, self.near, timeout=30, + unreliable=self.dead.__contains__).find_closest() + self.assertLess(time.monotonic() - started, 2) + self.assertNotIn(self.dead[0], self.net.queried) + # Known but not asked: it comes after every other node in the result + self.assertEqual(result, self.near + [self.dead[0]]) + + def test_retry_is_sent_but_not_waited_for(self): + """A due retry is asked on top of alpha, and the lookup finishes without its answer.""" + retries = [] + + def may_retry(n: bytes) -> bool: + """Every unreliable node is due, once.""" + retries.append(n) + return True + + started = time.monotonic() + lookup = self.lookup(self.net, self.key, self.dead + self.near, alpha=1, timeout=30, + unreliable=self.dead.__contains__, may_retry=may_retry) + result = lookup.find_closest() + self.assertLess(time.monotonic() - started, 2) + self.assertEqual(result, self.near) + self.assertEqual(sorted(retries), self.dead) + wait_for(lambda: set(self.dead) <= set(self.net.queried), what="the retries") + + def test_asked_and_waited_for_when_needed(self): + """With too few other nodes, unreliable ones are asked and waited for like any node.""" + self.net.hang.clear() + result = self.lookup(self.net, self.key, self.dead + self.near[:1], beta=3, + unreliable=self.dead.__contains__).find_closest() + self.assertEqual(result, sorted(self.dead + self.near[:1], key=lambda n: distance(n, self.key))) + self.assertEqual(set(self.net.queried), set(self.dead + self.near[:1])) + + +class UnreliableRoutingTest(LookupTestCase): + """Lookups over a routing table: a far node that keeps announcing but never connects.""" + + def test_avoided_retried_and_restored(self): + """Asked by every lookup until max_failures, then avoided (announces don't change that), asked again + after the backoff, and asked like any node once it answers.""" + net = self.network() + clock = FakeClock() + own = ident(0) + table = RoutingTable(own, Config(max_failures=3), clock=clock) + key = ident(0x1000) + dead = ident(0x1001) # closest to the key, and unreachable + near = [ident(0x1010), ident(0x1020), ident(0x1030), ident(0x1040)] + for n in near: + net.add(n) + table.heard(n) + + outcomes: list[bytes] = [] # every node whose query's outcome reached the table + + def count(record): + """record (the table's failed or answered), noting the outcome.""" + return lambda n: (record(n), outcomes.append(n)) + + def run() -> bool: + """Hear dead's announce, run a lookup from the table, and say whether it asked dead.""" + table.heard(dead) + before = net.queried.count(dead) + lookup = self.lookup(net, key, table.closest(key), own_id=own, on_failure=count(table.failed), + on_answer=count(table.answered), unreliable=table.unreliable, + may_retry=table.may_retry) + self.assertLessEqual(set(near[:3]), set(lookup.find_closest())) + # The outcome of a retry may arrive after the run, which doesn't wait for it + wait_for(lambda: outcomes.count(dead) == net.queried.count(dead), what="the query's outcome") + return net.queried.count(dead) > before + + self.assertEqual([run() for _ in range(3)], [True, True, True]) + self.assertTrue(table.unreliable(dead)) + self.assertFalse(run()) + clock.now += AVOID_AFTER + self.assertTrue(run()) # retried, failing again + self.assertFalse(run()) + net.add(dead) # back + clock.now += 2 * AVOID_AFTER + self.assertTrue(run()) + wait_for(lambda: not table.unreliable(dead), what="the answer") + self.assertTrue(run()) + + class TimeoutTest(LookupTestCase): """The time budget bounds a lookup even when every query hangs.""" diff --git a/tests/test_names.py b/tests/test_names.py index d43e9f3..120bfa8 100644 --- a/tests/test_names.py +++ b/tests/test_names.py @@ -14,7 +14,7 @@ from rnfs.content.cid import DAG, RAW, Cid from rnfs.errors import NotFound, RnfsError from rnfs.names import NameManager, encode_record, parse_record -from tests.support import POSIX_MODES, FakeClock, failing_commits, state_execute, state_rows +from tests.support import POSIX_MODES, FakeClock, failing_commits, state_execute, state_rows, wait_for _saved_loglevel = RNS.loglevel @@ -809,6 +809,49 @@ def test_save_writes_only_changed_rows(self): self.assertEqual(conn.total_changes - before, 1) self.assertEqual(self.rows("SELECT sequence FROM name_floor WHERE name = ?", (other_name(6),)), [("2",)]) + def saved_sequence(self, i: int) -> list[tuple]: + """The sequence column of other_name(i)'s name_floor row (empty if it has none).""" + return self.rows("SELECT sequence FROM name_floor WHERE name = ?", (other_name(i),)) + + def test_a_raise_found_by_a_resolve_is_saved_soon(self): + """A name in the table resolved again for the gateway, whose refresh hears a higher sequence, has the raise + saved within FLOOR_SAVE_INTERVAL (the announce raised the floor before the resolve compared with it).""" + self.resolve_for_gateway(1) + self.assertEqual(self.saved_sequence(1), [("1",)]) + self.clock.now += names.FLOOR_SAVE_INTERVAL + self.network.on_request_path = lambda name: self.network.announce(name, encode_record(2, CID_B), + self.clock.now) + self.assertEqual(self.manager.resolve(other_name(1), persist=False)["sequence"], 2) + wait_for(lambda: self.saved_sequence(1) == [("2",)], timeout=names.FLOOR_SAVE_INTERVAL, + what="the raised floor to be saved") + + def test_a_raise_by_an_announce_is_saved_soon(self): + """An announce raising the floor of a name in the table, with no resolve, is saved within + FLOOR_SAVE_INTERVAL too, not at the next re-announce pass.""" + self.resolve_for_gateway(1) + self.clock.now += names.FLOOR_SAVE_INTERVAL + self.network.announce(other_name(1), encode_record(3, CID_B), self.clock.now) + wait_for(lambda: self.saved_sequence(1) == [("3",)], timeout=names.FLOOR_SAVE_INTERVAL, + what="the raised floor to be saved") + + def test_a_raise_soon_after_a_save_waits_for_the_interval(self): + """A raise within FLOOR_SAVE_INTERVAL of the last save is saved by the timer once the interval has passed.""" + with mock.patch.object(names, "FLOOR_SAVE_INTERVAL", 0.5): + self.resolve_for_gateway(1) + self.network.announce(other_name(1), encode_record(4, CID_B), self.clock.now) + self.assertEqual(self.saved_sequence(1), [("1",)]) + self.assertIsNotNone(self.manager._floor_timer) + wait_for(lambda: self.saved_sequence(1) == [("4",)], timeout=5, what="the timer's save") + + def test_no_timer_after_close(self): + """A raise offered once the manager is closed (a keeper's late answer) starts no save.""" + self.resolve_for_gateway(1) + self.manager.stop() + self.manager.close() + self.clock.now += names.FLOOR_SAVE_INTERVAL + self.manager.offer(other_name(1), 5, CID_B, self.clock.now) + self.assertIsNone(self.manager._floor_timer) + def test_written_without_the_lock(self): """The floor's commit runs without the manager's lock: resolves and announces needn't wait for it.""" real_commit = statedb._Database._commit diff --git a/tests/test_node.py b/tests/test_node.py index 5db0442..90252f6 100644 --- a/tests/test_node.py +++ b/tests/test_node.py @@ -711,6 +711,29 @@ def test_nodes_that_failed_start_lookups_only_to_make_up_k(self): self.assertIn(failed, start) # fewer than K healthy: made up from the rest self.assertEqual(len(start), K - 1) + def test_unreliable_nodes_start_lookups_only_to_make_up_k(self): + """A node whose queries keep failing, though it keeps announcing, starts lookups only after every other.""" + rng = random.Random(8) + others = [bytes(rng.getrandbits(8) for _ in range(16)) for _ in range(K + 1)] + for node_id in others: + self.assertTrue(self.b.routing.heard(node_id)) + dead = others[0] + for _ in range(self.b.config.max_failures): + self.b.routing.heard(dead) # announces don't clear its failures + self.b.routing.failed(dead) + self.b.routing.heard(dead) + lookup = self.b._lookup(dead) + self.assertNotIn(dead, lookup._start_nodes) + self.assertEqual(len(lookup._start_nodes), K) + self.assertTrue(lookup.unreliable(dead)) + for node_id in others[1:4]: + self.b.routing.remove(node_id) + start = self.b._lookup(dead)._start_nodes + self.assertEqual(start[-1], dead) # fewer than K others: made up with it, last + self.assertEqual(len(start), K - 1) + self.b._answered(dead) + self.assertFalse(self.b.routing.unreliable(dead)) + class JoinTest(NodeTest): """Joining: bootstrap nodes first (the user's, then the defaults), then a lookup; again while too few.""" diff --git a/tests/test_routing.py b/tests/test_routing.py index a2e2218..62f1638 100644 --- a/tests/test_routing.py +++ b/tests/test_routing.py @@ -5,7 +5,8 @@ from rnfs.config import Config from rnfs.constants import BUCKET_COUNT, ID_LENGTH, K -from rnfs.dht.routing import RoutingTable, distance +from rnfs.dht import routing +from rnfs.dht.routing import AVOID_AFTER, AVOID_MAX, RoutingTable, distance from tests.support import FakeClock OWN_ID = bytes(ID_LENGTH) @@ -254,6 +255,92 @@ def test_live_count_leaves_out_the_same_nodes(self): self.assertEqual(self.table.live_count(), 2) +class UnreliableTest(RoutingTestCase): + """Nodes whose queries keep failing are unreliable however often they announce, until one is answered.""" + + def fail(self, n: bytes, times: int): + """Count times failed queries for n, hearing its announce before each, as a far node that keeps + announcing but never connects is heard.""" + for _ in range(times): + self.table.heard(n) + self.table.failed(n) + + def test_announces_do_not_reset_the_count(self): + """A node heard before every failure is unreliable after max_failures of them, and stays in the table.""" + n = node_id(42) + self.fail(n, 2) + self.assertFalse(self.table.unreliable(n)) + self.fail(n, 1) + self.assertTrue(self.table.unreliable(n)) + self.table.heard(n) + self.assertTrue(self.table.unreliable(n)) + self.assertTrue(self.table.contains(n)) # the table's own failure count was reset by the announce + + def test_nodes_outside_the_table_count_too(self): + """A node only named in replies (never in the table) is remembered as unreliable.""" + n = node_id(42) + for _ in range(3): + self.assertFalse(self.table.failed(n)) + self.assertFalse(self.table.contains(n)) + self.assertTrue(self.table.unreliable(n)) + + def test_retried_after_a_doubling_backoff(self): + """may_retry is True once the backoff has passed, for one caller; each failure doubles it up to AVOID_MAX.""" + n = node_id(42) + self.fail(n, 3) + self.assertFalse(self.table.may_retry(n)) + self.clock.now += AVOID_AFTER + self.assertTrue(self.table.may_retry(n)) + self.assertFalse(self.table.may_retry(n)) # claimed: another lookup doesn't retry it too + self.fail(n, 1) + self.clock.now += 2 * AVOID_AFTER - 1 + self.assertFalse(self.table.may_retry(n)) + self.clock.now += 1 + self.assertTrue(self.table.may_retry(n)) + self.fail(n, 20) + self.clock.now += AVOID_MAX - 1 + self.assertFalse(self.table.may_retry(n)) + self.clock.now += 1 + self.assertTrue(self.table.may_retry(n)) + + def test_an_answer_restores_it(self): + """answered() makes the node reliable again: no retry needed, and failures count from zero.""" + n = node_id(42) + self.fail(n, 5) + self.assertFalse(self.table.answered(n)) # already in the table + self.assertFalse(self.table.unreliable(n)) + self.clock.now += AVOID_MAX + self.assertFalse(self.table.may_retry(n)) + self.fail(n, 2) + self.assertFalse(self.table.unreliable(n)) + + def test_answered_adds_like_heard(self): + """answered() adds a node not yet in the table, as hearing it would.""" + n = node_id(42) + self.assertTrue(self.table.answered(n)) + self.assertTrue(self.table.contains(n)) + self.assertFalse(self.table.answered(OWN_ID)) + + def test_reliable_nodes_are_unaffected(self): + """Fewer than max_failures failures in a row leave a node reliable, with no retry to claim.""" + n = node_id(42) + self.fail(n, 2) + self.assertFalse(self.table.unreliable(n)) + self.assertFalse(self.table.may_retry(n)) + self.assertFalse(self.table.unreliable(node_id(43))) + + def test_memory_is_bounded(self): + """At most MAX_UNRELIABLE nodes are remembered; the least recently failed is forgotten first.""" + old = routing.MAX_UNRELIABLE + routing.MAX_UNRELIABLE = 4 + self.addCleanup(setattr, routing, "MAX_UNRELIABLE", old) + ids = [node_id(100 + i) for i in range(5)] + for n in ids: + self.fail(n, 3) + self.assertFalse(self.table.unreliable(ids[0])) + self.assertTrue(all(self.table.unreliable(n) for n in ids[1:])) + + class BanTest(RoutingTestCase): """Bans drop a node and ignore it for ban_time.""" From ea47d31e003d4410f6715ac8052cd40055825603 Mon Sep 17 00:00:00 2001 From: Afi Date: Sat, 17 Oct 2026 09:54:08 +1030 Subject: [PATCH 7/9] Add comprehensive tests for the public API of rnfs.Node and rnfs.Client This commit introduces a new test suite located in `tests/test_public.py`, which thoroughly tests the public API functionalities of both the `rnfs.Node` and `rnfs.Client` classes. The tests cover various aspects including data addition, retrieval, error handling, name management, and job processing. The suite ensures that both embedded and client-server interactions behave consistently, validating the integrity and reliability of the API. --- CHANGELOG.md | 134 +++- MANIFEST.in | 2 + README.md | 134 ++-- design.md | 607 ++++++++++++---- examples/rnfs_share.py | 89 +++ rnfs/__init__.py | 41 +- rnfs/api.py | 157 ++++- rnfs/apiclient.py | 70 +- rnfs/cli.py | 25 +- rnfs/config.py | 9 +- rnfs/constants.py | 3 + rnfs/content/__init__.py | 7 +- rnfs/content/chunker.py | 6 +- rnfs/content/cid.py | 1 + rnfs/content/dag.py | 52 ++ rnfs/content/edit.py | 11 +- rnfs/content/seal.py | 25 +- rnfs/content/shared_folder.py | 46 +- rnfs/content/storage.py | 14 +- rnfs/content/walker.py | 35 +- rnfs/denylist.py | 77 ++- rnfs/dht/__init__.py | 3 +- rnfs/dht/lookup.py | 21 +- rnfs/dht/providers.py | 31 +- rnfs/dht/routing.py | 93 ++- rnfs/embedded.py | 295 ++++++++ rnfs/errors.py | 17 +- rnfs/features/__init__.py | 6 + rnfs/features/_common.py | 8 + rnfs/features/contacts.py | 2 +- rnfs/features/deny.py | 9 +- rnfs/features/follow.py | 9 +- rnfs/features/jobs.py | 4 +- rnfs/features/keepers.py | 37 +- rnfs/features/lxmf.py | 28 +- rnfs/fetcher.py | 51 +- rnfs/fsutil.py | 27 +- rnfs/gateway.py | 13 +- rnfs/gateway_formats.py | 10 +- rnfs/interface.py | 951 ++++++++++++++++++++++++++ rnfs/mesh.py | 6 +- rnfs/names.py | 146 +++- rnfs/network/announces.py | 16 +- rnfs/network/client.py | 15 +- rnfs/network/guards.py | 101 ++- rnfs/network/identity.py | 7 - rnfs/network/links.py | 27 +- rnfs/network/rates.py | 2 +- rnfs/network/server.py | 24 +- rnfs/node.py | 216 ++++-- rnfs/node_content.py | 2 +- rnfs/objects.py | 248 +++++++ rnfs/publisher.py | 136 +++- rnfs/remote.py | 302 ++++++++ rnfs/statedb.py | 4 + rnfs/winacl.py | 9 +- specification.md | 29 +- tests/security/test_l7_windows_acl.py | 6 +- tests/test_api.py | 74 +- tests/test_apiclient.py | 45 +- tests/test_cid.py | 3 +- tests/test_cli.py | 18 +- tests/test_client.py | 6 +- tests/test_dag.py | 48 ++ tests/test_debuglog.py | 6 +- tests/test_denylist.py | 150 +++- tests/test_edit.py | 21 +- tests/test_embedded.py | 587 ++++++++++++++++ tests/test_fsutil.py | 69 +- tests/test_identity.py | 3 +- tests/test_lxmf.py | 11 + tests/test_mesh.py | 2 +- tests/test_names.py | 150 +++- tests/test_names_keep.py | 13 + tests/test_node.py | 75 +- tests/test_public.py | 805 ++++++++++++++++++++++ tests/test_publisher.py | 327 ++++++++- tests/test_rns_compat.py | 53 +- tests/test_routing.py | 72 +- tests/test_seal.py | 32 + tests/test_server_outcomes.py | 6 +- tests/test_shared_folder.py | 81 +++ tests/test_walker.py | 8 +- 83 files changed, 6325 insertions(+), 796 deletions(-) create mode 100644 examples/rnfs_share.py create mode 100644 rnfs/embedded.py create mode 100644 rnfs/interface.py create mode 100644 rnfs/objects.py create mode 100644 rnfs/remote.py create mode 100644 tests/test_embedded.py create mode 100644 tests/test_public.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 002e9f9..38a9ca8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,21 +2,22 @@ [specification.md](specification.md) is the authority on behaviour. -## Unreleased +## 0.1.5 -Performance work, a debug log, state kept in one SQLite database, deny lists, and a client library for apps. No -wire-format or CID changes: these nodes and 0.1.4 nodes work together. The upgrade of an `` is +Performance work, a debug log, state kept in one SQLite database, deny lists, and a public API for apps. No +wire-format or CID changes: these nodes, 0.1.4 and 0.1.3 nodes work together. The upgrade of an `` is automatic and one-way (an older version can't read it afterwards). Changed: -- State is kept in one SQLite database, `/state/rnfs.db` (Python's standard `sqlite3`; no new - dependency), instead of 17 JSON files. At the first start the JSON files of 0.1.4 or 0.1.3 (pins, cache - records, pin names, fetched targets, remote pins waiting, invalid roots, publish state, provider records, - names and the name floor, the shared folder, jobs, keepers, follows, contacts, the LXMF inbox and outbox) are - imported, each in one transaction, with every check the old loaders applied, and kept as `.migrated` - (a node that skips versions upgrades the same way). A damaged file is moved aside as `.damaged` as before; an - unreadable `pins.json` or a damaged `names.json` still stops startup. Nothing changes on the network: request - paths, messages, CIDs, DAG bytes, provider and name records are as before. +- State is kept in one SQLite database, `/state/rnfs.db` (Python's standard `sqlite3`; no new dependency, + but a Python built without its `_sqlite3` module now stops at startup, saying so), instead of 17 JSON files. At + the first start the JSON files of 0.1.4 or 0.1.3 (pins, cache records, pin names, fetched targets, remote pins + waiting, invalid roots, publish state, provider records, names and the name floor, the shared folder, jobs, + keepers, follows, contacts, the LXMF inbox and outbox) are imported, each in one transaction, with every check + the old loaders applied, and kept as `.migrated` (a node that skips versions upgrades the same way). A + damaged file is moved aside as `.damaged` as before; an unreadable `pins.json` or a damaged `names.json` still + stops startup. Nothing changes on the network: request paths, messages, CIDs, DAG bytes, provider and name + records are as before. - Saves write only what changed: a pin or release is one row, about 2.5 ms whatever the number of pins (before: `pins.json` rewritten whole, 5 ms at 100 pins and 14 ms at 20 000); a provider-records save writes the records changed since the last one (one change at 100 000 records: 0.02 ms, from 155 ms); a cache save @@ -33,34 +34,55 @@ Changed: damaged database (it runs SQLite's `quick_check` at startup) or one written by a newer version, and says how to recover: the `.migrated` files are kept for that. A state directory it can't write to now stops startup too, saying so (before, the node started and every save failed). +- Three more built-in bootstrap nodes (Johannesburg, Montreal and Stockholm; eight in all), which every node + contacts when joining unless `default_bootstrap = false` or it is in private mode. +- A compressed Resource advertised on an `rnfs.dht` link is now refused before any of it is transferred (before, + it was transferred, then failed as corrupt and the link torn down), and a request that wouldn't fit one packet + is refused locally instead of being sent as a Resource for the peer to refuse. Honest peers see no change. Added: - A debug log file, for diagnosing problems: `debug_log = true` in `rnfs.toml`, or `rnfs daemon --debug-log`, - writes `/debug.log`. It begins with the daemon's version (then the Python, RNS and LXMF versions, - the platform and the settings changed from the defaults), and has every rnfs message at every level whatever - `verbosity` is; Reticulum's own messages at Info and more severe, such as an interface that can't connect or a - socket that drops (its Verbose and Debug detail stays on the console only); each gateway request, - numbered, with the headers that decide it, what was decided (stored or fetched, type, range) and its status, - size and time; errors with tracebacks, and threads that die; and a status line every 5 minutes (counts, + writes `/debug.log`. It begins with the daemon's version (then the Python, RNS and LXMF versions, the + platform and the settings changed from the defaults), and has every rnfs message at every level whatever + `verbosity` is; Reticulum's own messages at Info and more severe that its loglevel shows, such as an interface + that can't connect or a socket that drops (its Verbose and Debug detail stays on the console only); each gateway + request, numbered, with the headers that decide it, what was decided (stored or fetched, type, range) and its + status, size and time; errors with tracebacks, and threads that die; and a status line every 5 minutes (counts, open links, each interface up or down). New detail, at debug level: each DHT lookup's queries (which node answered or failed, and how the lookup ended), the path to each node (hops, interface), why a link closed or couldn't be established, each request's outcome or why it went unanswered, and incoming links. The file is - readable by its owner only and never sent; each start keeps the previous run's as `debug.log.1`, and past - 16 MiB the file moves there too, so the two stay under 32 MiB. -- `rnfs.apiclient`, a client library of the local API for companion apps: it finds the rnfs directory, port and - token as the CLI does, makes whatever listens on the API port prove it knows the token before sending it, - raises a typed error for each failure, and has methods for status, add, fetch and `cat` (streamed), pins, - names and jobs. Standard library only; it doesn't load Reticulum. The CLI now uses it, with the same output. - See "Building apps on rnfs" in the README. + readable by its owner only and never sent; each start keeps the previous run's as `debug.log.1`, and past 16 MiB + the file moves there too, so the two stay under 32 MiB. +- A public API for programs, provisional until 0.2 (`rnfs.API_STATUS`): `rnfs.Node` runs an rnfs node inside a + program, beside the program's own Reticulum, and `rnfs.Client` uses the user's running daemon through its local + API. Both have the same methods (add, get, a seekable `open()` that fetches only what it reads, `ls`, `info`, + pins, archives, names with `publish` and `watch`, jobs with progress callbacks, the deny list), return the same + typed objects (`rnfs.Cid`, `Entry`, `Record`, `Job`, `Name`, ...) and raise the same errors, so a program + switches between them on one line. `import rnfs` stays light: `rnfs.Client` never loads Reticulum. See "Building + apps on rnfs" in the README and `examples/rnfs_share.py` (the source archive includes `examples/`). +- An embedded `rnfs.Node` applies rnfs's Resource guards (no compression, no requests carried as Resources, one + segment, no metadata, transfer sizes) to its own `rnfs.dht` links only, so the program's own requests, + transfers and compressed Resources work as with RNS alone. It defaults to a DHT client with LXMF off, never + stops the program's Reticulum, and locks its rnfs dir as the daemon does, so the two can't share one. +- `rnfs.apiclient`, the local API's protocol layer under `rnfs.Client` and the CLI: it finds the rnfs + directory, port and token as the CLI does, makes whatever listens on the API port prove it knows the token + before sending it, and raises a typed error for each failure (each also a subclass of the node's matching + error). Standard library only. The CLI now uses it, with the same output. +- `GET /api/name/watch`, a long poll for newer records of names, and a `from` parameter on `GET /api/cat` + (specification section 11). - Deny lists, to keep chosen content off a node (a public gateway or provider node): `rnfs deny add|rm|ls` edits a - local list of CIDs and file hashes (`sha256:`), and `deny_lists` in `rnfs.toml` names lists to apply, by - CID or `name/` (followed as the name changes; each version is checked, at most 4 MiB and 100 000 - entries, and pinned while in use). A denied CID is never fetched, read, stored, served or provided, as a root - or anywhere inside a DAG; the API and gateway answer 451. Records already published are withdrawn, cached - copies are deleted and remote pins holding it released; your own pins are kept but not served or read, and - the log names them. File hashes are checked where a file is assembled (`get`, `cat`, the gateway, `add`, - `import`), after its blocks have been downloaded. Nothing changes on the network: a peer asking for a denied - block is told it isn't held. + local list of CIDs and file hashes (`sha256:`) (or `GET /api/deny`, `POST /api/deny/add|rm`), and + `deny_lists` in `rnfs.toml` names lists to apply, by CID or `name/` (followed as the name changes; each + version is checked, at most 4 MiB and 100 000 entries, and pinned while in use). A denied CID is never fetched, + read, stored, served or provided, as a root or anywhere inside a DAG; the API and gateway answer 451. Records + already published are withdrawn, cached copies are deleted and remote pins holding it released, and a remote pin + of it is refused; your own pins are kept but not served or read, and the log names them. File hashes are checked + where a file is assembled (`get`, `cat`, the gateway, `add`, the shared folder, `import`), after its blocks have + been downloaded; the SHA-256 of each file checked is remembered (10 000 files, least recently used forgotten + first) and kept across restarts, so a file isn't downloaded and hashed again before its first byte after a + restart (a stored 64 MiB file's first byte: 6.5 ms, from 60 ms). `rnfs status` shows `deny_entries` and + `deny_lists`, and `pin ls` the lists' own blocks under the source `deny`. Nothing changes on the network: a peer + asking for a denied block is told it isn't held. Fixes: - A far node that keeps announcing but never accepts a link cost every lookup its connection timeout (60 s or @@ -77,12 +99,36 @@ Fixes: those nodes reach. Until a restart, which cleared it. - `import` pins the roots an archive supplies with one save of the pins; if that save fails, none of them is pinned (before, those ahead of the failure were). -- A second daemon started on an rnfs dir already in use loaded and saved the same state, announced the same - node ID and rotated the first one's debug log before stopping on "address already in use". It now stops at - once, saying which process uses the directory: the daemon holds a lock on `/daemon.lock`. -- The debug log no longer says a provider "cancelled" a `/blocks` response that may have been lost in transit: - the requester can't tell those apart. The provider now logs a response whose transfer RNS gave up on (lost - in transit, or its link closed), where before only the ones it cancelled itself were logged. +- A second daemon started on an rnfs dir already in use loaded and saved the same state and announced the same + node ID before stopping on "address already in use". It now stops at once, saying which process uses the + directory: the daemon (or an embedded `rnfs.Node`) holds a lock on `/daemon.lock`. +- With `-v`, a requester no longer says a provider "cancelled" a `/blocks` response that may have been lost in + transit (it can't tell those apart): it says the response stopped arriving. The provider now logs, at debug + level, a response whose transfer RNS gave up on (lost in transit, or its link closed). +- A remote pin or LXMF auto-fetch of content this node already held in full (a pin of your own, a cached + download) was accepted however many bytes of blocks it held, past `remote_pin_quota` or + `lxmf_auto_fetch_quota`: the quota was only counted for blocks fetched. It is now refused as when fetched. +- A failed save at shutdown (a full disk) stopped the shutdown there, leaving the state database, links and an + embedded node's destination open; it is now logged and the rest of the shutdown goes on. +- `rnfs pin ls` waited for a shared-folder scan in progress to finish importing (minutes for large files). +- A shared-folder file whose import failed for a reason that passes (a full disk, a pin that couldn't be saved) + wasn't tried again until it changed or the daemon restarted; it is now tried again 10 minutes later. A failed + pin of an unchanged entry no longer ends the scan. +- An interrupted write of a name key could leave a `names/.partial-*` file holding the private key; these are now + deleted at start, as the older `.tmp` leftovers were. +- Names: a publish racing `name rm` and `name create` of the same alias could save the record under the old + name (it now fails); `name create` or `name import` over an alias whose key file is a dangling symlink + overwrote that alias's saved record (now refused as an existing alias); a failed save while applying a heard + record skipped waking resolves, telling keepers and watchers, and was never retried (it is now logged and + saved by the next save); a name key deleted by hand is released once at startup, with one warning instead of + two; a name resolved for the user no longer keeps a stale floor entry; `name create` no longer fails if the + name is removed at that moment. +- Private mode: the routing table asked RNS whether a newly heard node is a member while holding the table's lock, + so a slow answer (from `rnsd`, through a shared instance) held up `/find` replies and lookups; and a node banned + while it was being asked about could still be added. It is now asked without the lock, and the table checked + again after. +- The local API's own HTTP error pages (400 for HTTP/0.9 or a malformed request, 414, 431, 501) now carry + `Cache-Control: no-store` and `X-Content-Type-Options: nosniff`, like every other API response. Faster: - `add`, the shared folder and `import` write blocks on up to 16 threads, so the sync to disk each block needs @@ -107,6 +153,18 @@ Faster: - Serving `/blocks` no longer reads and hashes the block it then leaves out for the budget; finished responses are noticed at once rather than within 10 ms; the response monitor sleeps while nothing is being sent. - DAG nodes decode about 15% faster (struct formats compiled once), accepting and rejecting exactly as before. +- A DAG node decoded recently isn't decoded again (the last 256 nodes, at most 2 MiB of blocks, are remembered): + a read met the same file and directory nodes many times. `GET /api/cat` also resolves its target and reads in + one fetch, where it fetched the target's info first, then read it in a second. Reading a stored 8 MiB file + through `rnfs.Client`'s `open()` (one request per 64 KiB) takes 0.18 s, from 1.0 s; a 4 KiB `cat` 1.3 ms, from + 8 ms. +- Republishing runs up to 4 closest-nodes lookups at once instead of one after another, so a round of many due + keys takes about a quarter of the time (200 keys whose lookups take 0.2 s each: 10 s, from 40). `/provide` + requests are batched per node as before and the same messages are sent. Slow nodes keep their pacing: one + lookup at a time, at most 10 per round, 30 s per lookup. +- Working out what the node provides, done after every pin, fetch, eviction, name change or deny-list change, is + skipped when nothing it depends on changed: 1 µs instead of 7 ms with 10 000 pins. Withdrawals are still sent + at once. ## 0.1.4 diff --git a/MANIFEST.in b/MANIFEST.in index 2d531ce..5650f6a 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -4,3 +4,5 @@ prune packaging # ...except the dependency lock, which install.sh uses when installing from an sdist include packaging/requirements.lock include packaging/build-backend.lock +# The example program for the public API (rnfs.Node, rnfs.Client) +recursive-include examples *.py diff --git a/README.md b/README.md index aa41bef..2139acf 100644 --- a/README.md +++ b/README.md @@ -66,7 +66,7 @@ Different package options available at [latest release](https://github.com/afit2 **Windows:** -[`rnfs-0.1.4-windows-x64.exe`](https://github.com/afit21/rnfs/releases/download/v0.1.4/rnfs-0.1.4-windows-x64.exe), saved as `rnfs.exe` +[`rnfs-0.1.5-windows-x64.exe`](https://github.com/afit21/rnfs/releases/download/v0.1.5/rnfs-0.1.5-windows-x64.exe), saved as `rnfs.exe` somewhere on your PATH ([how](#windows)). **Linux:** @@ -83,7 +83,7 @@ Or installers for ([Debian & Ubuntu](#debian-and-ubuntu)). [Flatpak](#other-lin **macOS:** -`pipx install https://github.com/afit21/rnfs/releases/download/v0.1.4/rnfs-0.1.4-py3-none-any.whl` ([details](#macos)). +`pipx install https://github.com/afit21/rnfs/releases/download/v0.1.5/rnfs-0.1.5-py3-none-any.whl` ([details](#macos)). ## 2 - Start rnfs and leave it running in its own terminal: @@ -118,26 +118,26 @@ rnfs runs on Linux, macOS and Windows. The easiest way to install it is a file f | System | Download | Install | |---|---|---| -| Windows (64-bit) | [`rnfs-0.1.4-windows-x64.exe`](https://github.com/afit21/rnfs/releases/download/v0.1.4/rnfs-0.1.4-windows-x64.exe) | [Windows](#windows) | -| Debian 12+, Ubuntu 24.04+, Raspberry Pi OS (64-bit) | [`rnfs_0.1.4_amd64.deb`](https://github.com/afit21/rnfs/releases/download/v0.1.4/rnfs_0.1.4_amd64.deb) or [`rnfs_0.1.4_arm64.deb`](https://github.com/afit21/rnfs/releases/download/v0.1.4/rnfs_0.1.4_arm64.deb) | [Debian and Ubuntu](#debian-and-ubuntu) | -| Other Linux (x86_64) | [`rnfs-0.1.4-x86_64.flatpak`](https://github.com/afit21/rnfs/releases/download/v0.1.4/rnfs-0.1.4-x86_64.flatpak) | [Other Linux](#other-linux-flatpak) | -| macOS, or any system with Python 3.11+ | [`rnfs-0.1.4-py3-none-any.whl`](https://github.com/afit21/rnfs/releases/download/v0.1.4/rnfs-0.1.4-py3-none-any.whl) | [macOS](#macos) | +| Windows (64-bit) | [`rnfs-0.1.5-windows-x64.exe`](https://github.com/afit21/rnfs/releases/download/v0.1.5/rnfs-0.1.5-windows-x64.exe) | [Windows](#windows) | +| Debian 12+, Ubuntu 24.04+, Raspberry Pi OS (64-bit) | [`rnfs_0.1.5_amd64.deb`](https://github.com/afit21/rnfs/releases/download/v0.1.5/rnfs_0.1.5_amd64.deb) or [`rnfs_0.1.5_arm64.deb`](https://github.com/afit21/rnfs/releases/download/v0.1.5/rnfs_0.1.5_arm64.deb) | [Debian and Ubuntu](#debian-and-ubuntu) | +| Other Linux (x86_64) | [`rnfs-0.1.5-x86_64.flatpak`](https://github.com/afit21/rnfs/releases/download/v0.1.5/rnfs-0.1.5-x86_64.flatpak) | [Other Linux](#other-linux-flatpak) | +| macOS, or any system with Python 3.11+ | [`rnfs-0.1.5-py3-none-any.whl`](https://github.com/afit21/rnfs/releases/download/v0.1.5/rnfs-0.1.5-py3-none-any.whl) | [macOS](#macos) | Every package includes Reticulum and LXMF, gives you one `rnfs` command, and keeps your data in the same -places. The release also has the source archive (`rnfs-0.1.4.tar.gz`) and `SHA256SUMS`, to -[check your download](#checking-a-download). These links are for 0.1.4; for a newer version, take the +places. The release also has the source archive (`rnfs-0.1.5.tar.gz`) and `SHA256SUMS`, to +[check your download](#checking-a-download). These links are for 0.1.5; for a newer version, take the same files from the [latest release](https://github.com/afit21/rnfs/releases/latest). ## Windows -Download [`rnfs-0.1.4-windows-x64.exe`](https://github.com/afit21/rnfs/releases/download/v0.1.4/rnfs-0.1.4-windows-x64.exe). It is a single file that +Download [`rnfs-0.1.5-windows-x64.exe`](https://github.com/afit21/rnfs/releases/download/v0.1.5/rnfs-0.1.5-windows-x64.exe). It is a single file that includes Python, Reticulum and LXMF; nothing else needs installing. In PowerShell, from your download folder, put it on your PATH as `rnfs`: ```powershell $bin = "$env:USERPROFILE\.local\bin" New-Item -ItemType Directory -Force $bin | Out-Null -Move-Item .\rnfs-0.1.4-windows-x64.exe "$bin\rnfs.exe" -Force +Move-Item .\rnfs-0.1.5-windows-x64.exe "$bin\rnfs.exe" -Force Unblock-File "$bin\rnfs.exe" [Environment]::SetEnvironmentVariable("Path", [Environment]::GetEnvironmentVariable("Path", "User") + ";$bin", "User") # open a new terminal, then: @@ -168,8 +168,8 @@ or newer. Ubuntu 22.04 ships Python 3.10, so it isn't supported; use the [instal there instead. Pick `amd64` or `arm64` to match `dpkg --print-architecture`: ```sh -curl -LO https://github.com/afit21/rnfs/releases/download/v0.1.4/rnfs_0.1.4_amd64.deb -sudo apt install ./rnfs_0.1.4_amd64.deb +curl -LO https://github.com/afit21/rnfs/releases/download/v0.1.5/rnfs_0.1.5_amd64.deb +sudo apt install ./rnfs_0.1.5_amd64.deb rnfs daemon # or run it in the background: systemctl --user enable --now rnfs # a systemd user service; logs: journalctl --user -u rnfs -f ``` @@ -188,9 +188,9 @@ For x86_64 Linux distributions without `.deb` support. The Flatpak uses the Flat `org.freedesktop.Platform` runtime: ```sh -curl -LO https://github.com/afit21/rnfs/releases/download/v0.1.4/rnfs-0.1.4-x86_64.flatpak +curl -LO https://github.com/afit21/rnfs/releases/download/v0.1.5/rnfs-0.1.5-x86_64.flatpak flatpak remote-add --user --if-not-exists flathub https://dl.flathub.org/repo/flathub.flatpakrepo -flatpak install --user rnfs-0.1.4-x86_64.flatpak +flatpak install --user rnfs-0.1.5-x86_64.flatpak flatpak run io.github.afit21.rnfs daemon alias rnfs='flatpak run io.github.afit21.rnfs' # add to ~/.bashrc to type just "rnfs" ``` @@ -218,7 +218,7 @@ and puts the `rnfs` command on your PATH. Homebrew also installs a recent Python ```sh brew install pipx pipx ensurepath # then open a new terminal -pipx install https://github.com/afit21/rnfs/releases/download/v0.1.4/rnfs-0.1.4-py3-none-any.whl # includes Reticulum and LXMF +pipx install https://github.com/afit21/rnfs/releases/download/v0.1.5/rnfs-0.1.5-py3-none-any.whl # includes Reticulum and LXMF rnfs daemon ``` @@ -244,11 +244,11 @@ Each release's `SHA256SUMS` lists the checksum of every file. Download it next t ```sh sha256sum -c SHA256SUMS --ignore-missing # Linux -shasum -a 256 rnfs-0.1.4-py3-none-any.whl # macOS: compare with the line in SHA256SUMS +shasum -a 256 rnfs-0.1.5-py3-none-any.whl # macOS: compare with the line in SHA256SUMS ``` ```powershell -Get-FileHash .\rnfs-0.1.4-windows-x64.exe # Windows: compare with the line in SHA256SUMS +Get-FileHash .\rnfs-0.1.5-windows-x64.exe # Windows: compare with the line in SHA256SUMS ``` A matching checksum shows the download is whole, not who made it: anyone able to replace a release's files @@ -257,7 +257,7 @@ repository's release workflow. Check it with the [GitHub CLI](https://cli.github logged in with `gh auth login`): ```sh -gh attestation verify rnfs_0.1.4_amd64.deb --repo afit21/rnfs +gh attestation verify rnfs_0.1.5_amd64.deb --repo afit21/rnfs ``` ## Other ways to install @@ -368,11 +368,12 @@ Uninstalling never deletes it. | `storage/identity` | the node's RNS identity; your node ID comes from it | | `storage/lxmf_identity` | a separate identity for your LXMF address | | `blocks/` | stored blocks | -| `state/rnfs.db` | pins, cache records, provider records, names' sequences, jobs, kept and followed names, contacts, LXMF inbox and outbox (one SQLite database; an older version's JSON files there are imported at the first start and kept as `*.migrated`) | +| `state/rnfs.db` | pins, cache records, provider records, names' sequences, jobs, kept and followed names, contacts, LXMF inbox and outbox, your deny list and the deny lists in use (one SQLite database; an older version's JSON files there are imported at the first start and kept as `*.migrated`) | | `names/` | name keys (back them up: `rnfs name export`) | | `files/` | the shared folder: anything put here is added, pinned and shared automatically | | `api_token` | the running daemon's API token | -| `daemon.lock` | locked by the daemon using the directory, so a second one on it stops at once | +| `daemon.lock` | locked by the daemon or `rnfs.Node` using the directory, so a second one on it stops at once | +| `debug.log`, `debug.log.1` | with `debug_log` on: this run's debug log and the previous one's (readable by you only) | The rnfs directory holds your keys and the API token, so rnfs keeps it private: readable by you only, and on Windows with an access list for you and SYSTEM only (also on another drive or under `C:\ProgramData`, where @@ -602,7 +603,7 @@ lxmf_auto_fetch_from = [""] | `max_records` | `100000` | Provider records stored for others | | `ban_time` | `3600` | How long a node that sent a bad block is ignored | | `verbosity` | `0` | Extra logging: 1 per-query detail, 2 RNS debug output (`daemon -v`, `-vv`) | -| `debug_log` | `false` | Also write `/debug.log` for diagnosing problems (`daemon --debug-log`): it begins with the daemon's version, then has every rnfs message at every level whatever `verbosity` is (which nodes are asked, which fail and why, links that fail or close), Reticulum's own messages at Info and above (interfaces that can't connect or drop; not its debug detail), each gateway request with its outcome, errors with tracebacks, and a status line every 5 minutes. It names what you fetched and browsed, so it is readable by you only and never sent anywhere; each start keeps the previous run's as `debug.log.1`, and the two stay under 32 MiB | +| `debug_log` | `false` | Also write `/debug.log` for diagnosing problems (`daemon --debug-log`): it begins with the daemon's version, then has every rnfs message at every level whatever `verbosity` is (which nodes are asked, which fail and why, links that fail or close), Reticulum's own messages at Info and above that its loglevel shows (interfaces that can't connect or drop; not its debug detail), each gateway request with its outcome, errors with tracebacks, and a status line every 5 minutes. It names what you fetched and browsed, so it is readable by you only and never sent anywhere; each start keeps the previous run's as `debug.log.1`, and the two stay under 32 MiB | **Storage** @@ -710,7 +711,7 @@ node ID is expected, a node contact's nick works too. | Command | Description | |---|---| | `rnfs daemon [--bootstrap NODE_ID…] [--rnsconfig DIR] [--gateway-port N] [--dht-mode server\|client] [--slow yes\|no\|auto] [--private] [--provide-strategy pinned\|roots] [--no-gateway-scripts] [--debug-log] [-v\|-vv]` | Run a node with its API and gateway | -| `rnfs status` | Node ID, identity hash (what others list in `members` and `remote_pin_identities`), known nodes, records, storage, traffic, jobs, LXMF address. Without a readable API token (another user's health check) only the daemon's version | +| `rnfs status` | Node ID, identity hash (what others list in `members` and `remote_pin_identities`), known nodes, records, storage, traffic, jobs, deny-list entries and lists, LXMF address. Without a readable API token (another user's health check) only the daemon's version | | `rnfs peers [--json]` | Known nodes: DHT mode, seniority, hops, interface, link rate, last heard, traffic, bans | | `rnfs stats [--json]` | Traffic since start: totals, uploads refused by your limits, per interface | | `rnfs ping NODE_ID` | Round-trip time to a node | @@ -844,31 +845,64 @@ Nicks are 1–32 characters of `a-z 0-9 - _ .`, starting with a letter or digit. # Building apps on rnfs -Apps talk to a running daemon through its local API, as the CLI does. `rnfs.apiclient` (standard library only, -and it doesn't load Reticulum, so it starts fast) finds the rnfs directory and API token as the CLI does, makes -whatever listens on the API port prove it knows the token before sending it, and raises a typed error for each -failure: +Programs use rnfs from Python the way they use RNS. There are two classes with the same methods, objects and +errors, so an app switches between them by changing one line: + +- **`rnfs.Node`** runs an rnfs node inside your program, beside the program's own Reticulum, in a directory of + its own. Use it when your app should have its own node: its own storage, identity and place in the DHT. +- **`rnfs.Client`** uses the user's running daemon (`rnfs daemon`) through its local API. Use it when your app + should share the user's node: the content they already have, their pins and names. It never loads Reticulum, + so it starts fast. + +Nothing picks one for you: construct the one you want. **The API is provisional until rnfs 0.2** +(`rnfs.API_STATUS`): names may still change once it has been tried in real apps. + +```python +import RNS, rnfs + +RNS.Reticulum() # your program's Reticulum, as with any RNS app +with rnfs.Node("~/.myapp/rnfs") as node: # an rnfs node inside it; or: rnfs.Client() + added = node.add("photos/") # rnfs.Added: .cid, .size, .warnings + for entry in node.ls(added.cid): # rnfs.Entry(name, kind, size, cid) + print(entry.name, entry.size) + with node.open(f"{added.cid}/cat.jpg") as f: # seekable: each read fetches only the chunks it covers + f.seek(1_000_000) + data = f.read(4096) + site = node.names.create("site") # rnfs.Name + site.publish(added.cid) + node.names.watch(site, lambda record: print("now", record.cid)) + job = node.jobs.get(f"rnfs:name/{site.hex}", "/tmp/copy", + on_progress=lambda job: print(job.done_bytes, "of", job.total_bytes)) + job.wait() +``` + +With the daemon instead, only the `with` line changes (and `RNS.Reticulum()` isn't needed): ```python -from rnfs.apiclient import Client, DaemonNotRunning, NotFound - -api = Client() # the daemon using ~/.rnfs/; Client("/srv/rnfs") for another rnfs dir -try: - cid = api.add_path("/home/me/photos")["cid"] - api.name_publish("photos", cid) - with api.cat(f"{cid}/cat.jpg") as stream, open("cat.jpg", "wb") as out: - for chunk in stream: - out.write(chunk) - print(api.name_resolve("ab" * 16)["cid"]) -except DaemonNotRunning: - print("start it with: rnfs daemon") -except NotFound as e: - print(e) -``` - -It has methods for status, adding (a path or a stream), fetching and `cat`, pins, names and jobs; `get()` and -`post()` reach every other endpoint (`rnfs/api.py` lists them). The module's docstring is the reference. Run -the app as the daemon's user: only it can read the token. +with rnfs.Client() as node: # the daemon using ~/.rnfs/; rnfs.Client("/srv/rnfs") for another + print(node.status().node_id.hex()) + try: + print(node.names.resolve("ab" * 16).cid) + except rnfs.NotFound: # the same errors from either class + print("not resolved") +``` + +- Targets are a `rnfs.Cid`, `/path`, or an `rnfs:` link (`rnfs:name//path` is resolved first). +- Methods: `add` (a path, bytes or a binary file), `get`, `open`, `cat`, `ls`, `info`, `stat`, `pin`, `unpin`, + `pins`, `export`, `import_archive`; `names.create`/`list`/`resolve`/`watch`, `Name.publish`; + `jobs.get`/`pin`/`export` with `on_progress`, `Job.wait`/`cancel`; `deny.add`/`remove`/`list`; and the rest of + the CLI's operations (design.md, "Public API"). +- Callbacks run on one event thread of the instance, with no rnfs lock held. Keep them short: a slow one delays + the others. +- An embedded node runs as a DHT client with LXMF off unless you say otherwise: + `rnfs.Node(dir, settings={"dht_mode": "server", "lxmf_enabled": True})`. It never stops your Reticulum, and + your program's own RNS links work as with RNS alone. A directory is used by one node at a time: never point a + `Node` at the daemon's `~/.rnfs/`. +- A `Client` must run as the daemon's user (only it can read the API token). + +`examples/rnfs_share.py` (in the source tree and the source archive) is a runnable example using either class (`--node DIR` or `--daemon`). design.md's +"Public API" section is the reference. Other languages use the local API's JSON endpoints directly (`rnfs/api.py` +lists them). # Keeping content off your node @@ -892,14 +926,14 @@ A denied CID is never fetched, read, stored or served by your node, whether it i anywhere inside it, and nothing holding it is provided (records already published are withdrawn). `get`, `cat`, the gateway (status 451), `pin add`, remote pins, followed names and LXMF auto-fetch all refuse it. When an entry is added, cached downloads holding it are deleted and pins made for other nodes (`pin remote`) are -released; your own pins are kept, but not served or read, and the log names each one so you can `rnfs pin rm` -it. +released; your own pins are kept, but not served or read, and the log names each one so you can remove it +(`rnfs pin rm`, or what made the pin: the shared-folder file, `name publish`, `name unfollow`, `inbox rm`). A file hash is only known once the file has been downloaded, so it stops `get`, `cat` and the gateway from handing the file out (they check before writing or sending anything, which makes them fetch the whole file -first) and `add` or `import` from adding it; it doesn't stop the blocks being downloaded, pinned or served. -Deny the CID to stop that. Lists larger than 4 MiB, with over 100 000 entries, or with a line that isn't an -entry are refused, and the previous version stays in use. +first) and `add`, the shared folder or `import` from adding it; it doesn't stop the blocks being downloaded, pinned or served. +Deny the CID to stop that. Lists larger than 4 MiB, with over 100 000 entries, not UTF-8 text, with a line that +isn't an entry, or that are a directory are refused, and the previous version stays in use. # Example use cases diff --git a/design.md b/design.md index 49835ff..da52f25 100644 --- a/design.md +++ b/design.md @@ -6,14 +6,15 @@ authority on behaviour; this file fixes module boundaries and the interfaces bet ## Shared foundation - `rnfs/constants.py` — protocol constants (spec section 12), request paths, aspects, DHT modes. -- `rnfs/config.py` — `Config`: every local setting (section 12), paths under ``, and the - `effective_*` helpers for settings that depend on slow nodes or links. Loaded from `/rnfs.toml` - plus command-line overrides; `validate()` checks every core setting's type and range (`WHOLE_NUMBERS`, - `RATES`, `DURATIONS`, `FLAGS`, `IDENTITY_LISTS`), and the daemon exits on an invalid one. Reticulum is - started from `reticulum_dir`, by default RNS's own config directory, so the node attaches to `rnsd` with the - matching RPC key. +- `rnfs/config.py` — `Config`: every local setting (section 12), paths under ``, and the `effective_*` + helpers for settings that depend on slow nodes or links. Loaded from `/rnfs.toml` plus command-line + overrides (and `defaults` beneath the file: an embedded node's); `validate()` checks every core setting's type and + range (`WHOLE_NUMBERS`, `RATES`, `DURATIONS`, `FLAGS`, `IDENTITY_LISTS`), and the daemon exits on an invalid one. + Reticulum is started from `reticulum_dir`, by default RNS's own config directory, so the node attaches to `rnsd` + with the matching RPC key. - `rnfs/errors.py` — `RnfsError` and subclasses `InvalidDag`, `NotFound`, `TooLarge`, `StorageFull`, `Busy`, - `Denied`. + `Denied`, `Cancelled` (a fetch stopped by its observer), `Closed` (an `rnfs.Node` or `rnfs.Client` used after + `close()`). - `rnfs/content/cid.py` — `Cid` (binary form `.binary`, `.codec` `RAW`/`DAG`, `.key` DHT key, `.verify(block)`, `str()` written form, `Cid.parse`, `Cid.from_bytes`, `Cid.for_block`). Code passes `Cid` objects everywhere; strings only at the user edge (CLI, API, gateway, logs) and binary only on the wire. @@ -96,9 +97,10 @@ rnfs/ protocol.py message encoding/validation, batch sizes (sections 3, 6, 7, 8) rates.py rate estimates, slow links, slow node detection (section 2) links.py LinkPool: shared unidentified links, identified one-shot links, idle close (section 3) - guards.py process-wide guards rejecting oversized or request Resource advertisements and, on - rnfs.dht links, responses that fail a msgpack header check before RNS decodes them - (3, 10); the half-open incoming link cap per destination (3) + guards.py guards rejecting oversized, compressed or request Resource advertisements (on every + link in the daemon, on rnfs.dht links only in an embedded node) and, on rnfs.dht links, + responses that fail a msgpack header check before RNS decodes them (3, 10); the + half-open incoming link cap per destination (3) server.py RequestServer: handlers, every section 3 limit, airtime policy, feature paths (3, 7) client.py PeerClient: /find, /provide, /blocks (with stall detection), /pin, ping (sections 3, 6, 7) announces.py rnfs.dht announces carrying the DHT mode, and the announce handler (section 3) @@ -111,7 +113,12 @@ rnfs/ node.py RnfsNode: wires everything; the facade below node_content.py node side of --no-pin, sealing/unsealing, raw blocks and archive streams api.py local API, 127.0.0.1:8788 (section 11) - apiclient.py client of the local API for apps and the CLI: token, proof check, typed errors (section 11) + apiclient.py the local API's protocol layer (ApiConnection) under rnfs.Client and the CLI: token, proof + check, typed errors (section 11) + objects.py the public API's typed objects (Added, Entry, Info, Record, ...); no RNS + interface.py the public API both classes share (Rnfs, Names, Jobs, Deny, File, Watch); no RNS + embedded.py rnfs.Node: a node inside a program (imports RNS) + remote.py rnfs.Client: the daemon through its API (no RNS) gateway.py gateway, 127.0.0.1:8789 (section 11) gateway_formats.py ?format=raw|archive (section 11) cli.py command line (section 11) @@ -124,6 +131,7 @@ rnfs/ contacts.py address book (section 11) lxmf.py rnfs: URIs, LXMF send/inbox/outbox/auto-fetch (section 13) deny.py deny_lists sources: fetched, checked, followed; rnfs deny (section 8) +examples/ rnfs_share.py: the public API with either class install.sh Linux installer of the latest release (or --main): venv, command, optional systemd user unit; --update, --uninstall contrib/ rnfs.service (systemd user), io.github.afit21.rnfs.plist (launchd), install-windows-task.ps1 @@ -140,12 +148,17 @@ packaging/ requirements.lock (hash-pinned runtime dependencies), upda - Frozen dataclasses `Child(cid, size)`, `Entry(name, cid, size, kind)`, `IndexChild(first_name, cid, size)`; nodes `FileNode(children)`, `DirectoryNode(entries)`, `IndexNode(children)`. - `encode(node) -> bytes`; `decode(block) -> node`, strict (types, shortest headers, re-encodes identically), - raising `InvalidDag`. `check_name(name)` raises `ValueError`. + raising `InvalidDag`. Nodes are immutable and decoding is a pure function of the bytes, so `decode` remembers + the nodes of the blocks it decoded last (`MEMO_BLOCKS` 256, `MEMO_BYTES` 2 MiB of blocks, least recently used + forgotten first; a block that fails is checked again each time): a read meets the same nodes many times. + `check_name(name)` raises `ValueError`. `walker.py` — reader checks are driven by an **expectation** of what a child must be: -- `Expect(cid, size: int | None, types: frozenset[str], depth: int, first_name: str | None, below: str | None)`, - frozen. `types` ⊆ {"raw", "file", "directory", "index"}; `first_name` is the name the first name below must - equal; `below` an exclusive upper bound for every name below. `root_expect(cid)` allows any type at depth 1. +- `Expect(cid, size: int | None, types: frozenset[str], depth: int, first_name: str | None, below: str | None, + kind: int | None)`, frozen. `types` ⊆ {"raw", "file", "directory", "index"}; `first_name` is the name the first + name below must equal; `below` an exclusive upper bound for every name below; `kind` a directory entry's kind + (`dag.KIND_*`), so `write_out` can give an executable target the execute bit. `root_expect(cid)` allows any + type at depth 1. - `expand(expect, block) -> list[Expect]` — check a verified block against its expectation (type, size, names, index bounds, depth) and return its children's expectations. Raises `InvalidDag`. Raw blocks have none. - `step(expect, block, name) -> Expect | None` — one path step in a directory or index node (for an index, the @@ -154,9 +167,11 @@ packaging/ requirements.lock (hash-pinned runtime dependencies), upda overlapping a byte range, with the range relative to each child. - `parse_target(text) -> tuple[Cid, list[str]]` for `[/name…]` (ValueError if malformed). - Local operations over a blocking `get(expect) -> bytes` (raise `NotFound` when absent): + `step_name(expect, name, get) -> Expect | None` (one name, stepping through index nodes), `resolve(root, names, get) -> Expect`, `iter_dag(expect, get)` (each `(expect, block)`, each CID once, depth-first), `read_range(expect, get, offset, length) -> Iterator[bytes]`, - `list_directory(expect, get) -> Iterator[Entry]`, `write_out(expect, get, dest, max_entries) -> warnings`. + `list_directory(expect, get, max_entries=None) -> Iterator[Entry]` (`TooLarge` past + `max_entries`), `write_out(expect, get, dest, max_entries) -> warnings`. `storage.py` — `Storage(blocks_dir, state_dir, max_storage)`, thread-safe: - blocks: `has(cid)`, `get(cid, verify=False) -> bytes | None` (with `verify`, a block no longer matching its @@ -170,14 +185,16 @@ packaging/ requirements.lock (hash-pinned runtime dependencies), upda `unpin(cid, source) -> bool`, `pins() -> dict[Cid, set[str]]`, `pin_count()`, `is_complete(cid)`. - cache: `record_fetch(root, names, target)` (keeps the path's nodes and the target's DAG), LRU touched on use. `cache_generation()` grows whenever records are removed or retargeted, so the node prunes its fetched - targets only then. + targets only then. `changes()` grows with every block stored or deleted, pin or release, and record added, + removed or retargeted: while it is unchanged, so are `pins()`, `cached_targets()` and every `dag_blocks()` + answer (the publisher's refresh skips recomputing then). - protection: `with storage.protect() as session:` — blocks stored or listed in `session.add(cid)` are kept as if pinned until the block exits (fetches, adds, imports in progress). -- deny list: `storage.deny` (`denylist.NONE` until the node sets its `DenyList`; the fetcher and publisher read - it there too). `_put` raises `Denied` for a denied CID, before anything is written, even one already stored. - `drop_denied(cids)` drops the cached fetches whose root, path or stored DAG holds one and collects as `gc` - does; `denied_roots(roots, cids)` says which roots reach one through stored nodes; `collect()` deletes - whatever nothing reaches (what a refused add or import left). +- deny list: `storage.deny` (`denylist.NONE` until the node sets its `DenyList`; the fetcher and publisher read it + there too). `_put` raises `Denied` for a denied CID, before anything is written, even one already stored. + `drop_denied(denied)` (binary CIDs, the `DenyList`'s `cids`) drops the cached fetches whose root, path or stored + DAG holds one and collects as `gc` does; `denied_roots(roots, denied)` says which roots reach one through stored + nodes; `collect()` deletes whatever nothing reaches (what a refused add or import left). - `check_disk(nbytes)` raises `StorageFull` when storing `nbytes` would leave less than the disk reserve (max(256 MiB, 2% of the filesystem)) free. `max_storage` itself never refuses a fetch: it is enforced by eviction once fetches end. @@ -198,23 +215,27 @@ packaging/ requirements.lock (hash-pinned runtime dependencies), upda - A stored DAG node that no longer matches its CID (`DAMAGED`) has unknown children: while anything live reaches one, `evict()` and `gc()` delete nothing and log it once (`_held_back`) until `verify()` removes it or a fetch replaces it. Fetches treat such a block as not stored: deleted and fetched again. -- `dag_blocks(cid) -> set[Cid]` the blocks of a complete DAG (memoised), used for "what a node serves". +- `dag_blocks(cid) -> frozenset[Cid]` the blocks of a complete DAG (memoised), used for "what a node serves". -`denylist.py` — `parse_entry(text) -> (canonical, binary CID | None, digest | None)`, `parse_list(bytes) -> -(binary CIDs, digests)` (ValueError naming the line; `MAX_LIST_BYTES` 4 MiB, `MAX_LIST_ENTRIES` 100 000). -`DenyList(state_dir)`: `cids` and `hashes` (frozensets swapped whole, read without a lock: every check is one -set lookup), `denies(cid)`, `denies_hash(digest)`; the local list `entries()`, `add(texts)`, `remove(texts)` +`denylist.py` — `parse_entry(text) -> (canonical, binary CID | None, digest | None)`, `parse_list(bytes) -> (binary +CIDs, digests)` (ValueError naming the line; `MAX_LIST_BYTES` 4 MiB, `MAX_LIST_ENTRIES` 100 000). +`DenyList(state_dir)`: `cids` and `hashes` (frozensets swapped whole, read without a lock: every check is one set +lookup), `denies(cid)`, `denies_hash(digest)`; the local list `entries()`, `add(texts)`, `remove(texts)` (`deny_entries`, at most `MAX_LOCAL_ENTRIES`); sources `accept(source, cid, entries)` (saved, then applied), `forget(source)`, `accepted()`, `in_use(source)`, `entry_count(source)` (`deny_lists`: the CID each source last -accepted; `reload(read)` reads every one back from storage at start); `known_hash(cid)` / `remember_hash` -(a file CID's content SHA-256, `MAX_KNOWN_HASHES` LRU); `add_listener(fn)`, called after each change outside -its lock. `walker.write_out(..., check)` and `walker.iter_files` and `builder.checking(check)` (a ContextVar: -`build_file` hashes each file and passes the digest before the file's root node is stored) are its hooks in -the content layer; `archive.import_archive(..., check)` runs on the complete roots before they are pinned. +accepted; `reload(read)` reads every one back from storage at start); `known_hash(cid)` / `remember_hash` (a file +CID's content SHA-256, `MAX_KNOWN_HASHES` LRU, kept across restarts in `deny_hashes`: `save()`, called by the node's +`_save_state`, writes the rows changed since the last one outside the lock, behind a save lock); `add_listener(fn)`, +called after each change outside its lock. `walker.write_out(..., check)` and `walker.iter_files` and +`builder.checking(check)` (a ContextVar: `build_file` hashes each file and passes the digest before the file's root +node is stored) are its hooks in the content layer; `archive.import_archive(..., check)` runs on the complete roots +before they are pinned. ## DHT layer interfaces -`routing.py` — `RoutingTable(own_id, config, clock=time.monotonic)`: +`routing.py` — `RoutingTable(own_id, config, clock=time.monotonic, admit=None)` (`admit(node_id)`: private mode's +member check, asked only for a node not in the table and without the table's lock held, since it may block on +RNS; the table is checked again once it answers): `heard(node_id) -> bool` (proven server: announce in server mode, or answered `/find`), `answered(node_id) -> bool` (`heard`, and reliable again), `failed(node_id) -> bool` (True if dropped), `unreliable(node_id)` (its last `max_failures` queries or more failed, in the table or not; announces don't clear it: `_unanswered`, at most @@ -225,13 +246,13 @@ the next one back a further backoff, so concurrent lookups don't all retry it), `seniority(node_id) -> float | None` (seconds in the table), `is_senior(node_id, min_age)`, `standing(node_id, min_age) -> int` (0 senior and healthy, 1 healthy, 2 stale, 3 absent: ranks keepers), `standings(min_age) -> dict` (every node's standing now: the keeper query's snapshot from before its lookup), `contains(node_id)`, -`node_count()`. +`nodes()`, `node_count()`, `live_count()` (nodes `/find` replies would include). -`providers.py` — `ProviderStore(max_records, state_path, ttl=PROVIDER_TTL, clock=time.time, rng=random, -own_id=None, seniority=0)`: `store(key, publisher) -> bool`, `withdraw(key, publisher)`, `provide(publisher, +`providers.py` — `ProviderStore(max_records, state_dir, ttl=PROVIDER_TTL, clock=time.time, rng=random, +max_per_publisher=None, max_per_key=1000, own_id=None, seniority=0)` (`state_dir` holds the state database): `store(key, publisher) -> bool`, `withdraw(key, publisher)`, `provide(publisher, keys, withdrawn) -> int`, `retain(keep) -> int` (drops every record of publishers `keep` rejects: private mode's -non-members), `for_reply(key, exclude=lambda id: False) -> list[bytes]`, `expire()`, `record_count()`, -`allowance(publisher)`, `save()`. Local limits against flooding: one publisher holds at most max(10, +non-members), `for_reply(key, exclude=lambda id: False) -> list[bytes]`, `providers(key, exclude) -> list[bytes]`, `expire()`, +`record_count()`, `allowance(publisher)`, `save()`, `close()`. Local limits against flooding: one publisher holds at most max(10, `max_records` // 100) records once seen for `seniority` (the node passes `provider_seniority`) and a tenth of that (at least 10) before, the node's own (`own_id`) excepted. A publisher at its limit has its expired records swept at most once per batch, and not at all before `sweep_after` (the earliest its oldest record can expire, @@ -255,9 +276,9 @@ its queries, its termination and `find_closest`'s result; one that distance alon window is asked again when `may_retry` says so, as a query beyond `alpha` (`retrying`) that the run waits for only if the node comes to rank within the window by `rank` (it is then counted in flight). `is_known(id)` says whether RNS knows its identity; at most `alpha` unknown IDs are tried per lookup (`allow_unknown(id)`, also -used by the fetcher for provider IDs). `listing_count()` is how many nodes have answered with at least one -provider, `listing_nodes()` which, and `listers(provider)` which listed a given provider, so the keeper search -can tell who vouches for its candidates. +used by the fetcher for provider IDs). `listing_nodes()` is the nodes that have answered with at least one +provider, and `listers(provider)` those that listed a given provider, so the keeper search can tell who vouches +for its candidates. ## Network layer interfaces @@ -265,7 +286,9 @@ can tell who vouches for its candidates. `is_dht_server() -> bool`, `find(key) -> (providers, nodes)`, `provide(publisher_id, keys, withdrawn) -> bool`, `served_block(cid) -> bytes | None` (only blocks in the DAG of a currently provided CID, re-verified), optionally `served_block_size(cid) -> int | None` (so `/blocks` stops at the block over its budget unread), -`remote_pin(identity_hash, cid, pin) -> bool`, `is_member(identity_hash) -> bool` (private mode). +`remote_pin(identity_hash, cid, pin) -> bool`, `is_member(identity_hash) -> bool` (private mode), and +optionally `feature_requests() -> {path: handler(data, remote_identity)}` (features' request paths, served within +the same per-link limits). `NodeBackend.find` leaves out providers `RnfsNode.excluded_provider` rejects: banned, and in private mode non-members (`_is_member_publisher`: this node, a publisher whose `/provide` the server let through as a member this run, or a node ID whose recalled identity is a member, as routing admission checks). In private mode the @@ -287,19 +310,21 @@ RNS recalls, and only if its hash is the node ID asked for (in private mode, als ## Node facade (used by api.py, gateway.py and cli.py) -Apps and the CLI reach it through the local API with `rnfs/apiclient.py` (`Client`), which imports neither RNS -nor anything that does. `Client(rnfs_dir, port, token_file, timeout)` takes the port from `rnfs.toml` +Programs use `rnfs.Node` or `rnfs.Client` (Public API, below). The CLI, and `rnfs.Client` underneath, reach the +daemon through the local API with `rnfs/apiclient.py` (`ApiConnection`), which imports neither RNS nor anything +that does. `ApiConnection(rnfs_dir, port, token_file, timeout)` takes the port from `rnfs.toml` (`api_port()`, `SettingsError`) and the token from `/api_token` (`read_token`, `TokenError`); its first request needing the token proves the daemon first (`verify_daemon`, under a lock, so concurrent first requests prove it once), which also reads the daemon's version for `check_daemon`, the one place a version check goes. `request()` maps failures to `DaemonNotRunning`, `ConnectionFailed`, `AuthFailed` (a failed proof, or 403) and the API's statuses (`STATUS_ERRORS`, `error_for`); `Stream` reads a body, raising `ConnectionFailed` if it ends short. Its methods (`status`, `add_path`, `add_stream`, `fetch`, `cat`, `info`, -`ls`, `stat`, `pin_*`, `name_*`, `submit_job`, `job`, `jobs`, `cancel_job`, `clear_jobs`) send only the -parameters given. The CLI's `ApiClient` subclasses it, turning each `ApiError` into `fail(str(e))`, so its -messages are the library's. +`ls`, `stat`, `pin_*`, `name_*` (with `name_watch`), `submit_job`, `job`, `jobs`, `cancel_job`, `clear_jobs`, +`deny*`) send only the parameters given; `ApiError` is an `RnfsError`, and each status's error a subclass of the +node's own too (`NotFound` of `errors.NotFound`, `BadRequest` of `ValueError`, ...). The CLI's `ApiClient` +subclasses it, turning each `ApiError` into `fail(str(e))`, so its messages are the library's. -`RnfsNode(config)` exposes `node_id: bytes`, `config`, and these methods. Targets are strings +`RnfsNode(config, reticulum=None, embedded=False)` exposes `node_id: bytes`, `config`, and these methods. Targets are strings `[/name…]`; CIDs are returned as strings. Return values are JSON-ready dicts. | Method | Returns | @@ -310,8 +335,9 @@ messages are the library's. | `info(target, for_gateway=False, from_node=None, raw=False)` | `{"cid", "type": "file"\|"directory", "size", "executable"}` (+ `"sealed", "recipient", "stored_size", "name"`; the sealed name is None unless `safe_default_name` accepts it) | | `stored_head(target, length)` | a file's first `length` bytes if stored, else None; never fetches (the user's own gateway `HEAD` and ranges not at 0) | | `read(target, offset=0, length=None, for_gateway=False, from_node=None, raw=False)` | iterator of bytes of a file (range fetched only; sealed files decrypted unless `raw` or `for_gateway`) | +| `read_file(target, offset=0, length=None, from_node=None, raw=False)` | `(info, chunks)`: `info()` and `read()` in one fetch, resolving the target once; `chunks` reads `[offset, offset + length)` cut to the info's size only once iterated (a directory's raises `ValueError`), and closing it ends the fetch (`GET /api/cat`, which used to call `info` and `read`: two fetches) | | `block(cid, for_gateway=False)` | the CID's own block, verified (`cat --block`, `?format=raw`: the gateway calls it only for the user's own GET) | -| `export_stream(target, start, for_gateway=False)` | fetches the DAG, then writes its archive to `start(exact length)` (`?format=archive`; `None` for HEAD). The gateway calls it only for the user's own GET (never for a limited request or a HEAD, stored or not), inside `fetcher.observing(ClientWatch)`, which cancels the fetch once the client disconnects | +| `export_stream(target, start, for_gateway=False)` | fetches the DAG, then writes its archive to `start(exact length)` (`?format=archive`; a `start` returning `None` writes nothing). The gateway calls it only for the user's own GET (never for a limited request or a HEAD, stored or not), inside `fetcher.observing(ClientWatch)`, which cancels the fetch once the client disconnects | | `ls(target, for_gateway=False)` | `[{"name", "kind": "file"\|"executable"\|"directory", "size", "cid"}]` | | `stat(target)` | `{"cid", "size", "blocks", "local_blocks", "local_bytes"}` | | `is_local(target)` | `bool`: everything needed is stored, so no network fetch | @@ -335,11 +361,13 @@ messages are the library's. threads, for tests (tests/test_node.py). The request server's backend is `node.backend` (`NodeBackend`). `fetcher.py`: `Fetcher.open(root, names, from_node, pinned)` gives a `Fetch` context (a protection session) with `resolve()`, `fetch_dag()`, `fetch_listing()`, `read_range()`; `publisher.py`: `Publisher.refresh()` -(recompute the provide set, withdraw at once), `publish_due()`, `serves(cid)`, `run()`. The node calls the +(recompute the provide set, withdraw at once; skipped when the node's generation, the deny list, the key +sources' keys and the strategy are as at the last one), `publish_due()`, `serves(cid)`, `run()`. The node calls the first `refresh()` only after the features have registered their key sources, so a restart doesn't withdraw kept names' keys. `publish_due` republishes `DhtKey`s every max(effective `republish_interval`, 3 days) with a per-key ±10% jitter (capped below `PROVIDER_TTL / 2`), drops a withdrawal of any key it provides to -the same node in that round, and on slow nodes runs at most 10 lookups per round, waiting 30 s × the +the same node in that round, runs up to `PUBLISH_LOOKUPS` (4) lookups at once, keeping the batches one at a +time would build, and on slow nodes runs one lookup at a time, at most 10 per round, waiting 30 s × the round's lookups before the next (never-published items first, then the longest unpublished). When `refresh()` finds a key provided again, it cancels that key's queued withdrawals (their nodes join its earlier nodes) and re-records the node's own record, and a round sends only withdrawals still queued when it reaches @@ -351,11 +379,13 @@ Errors: `ValueError` → 400, `NotFound` → 404, `TooLarge` → 413, `InvalidDa are absolute paths on the daemon's machine (the CLI resolves them); stdin is uploaded as a request body. **Core API routes** (`rnfs/api.py` `ROUTES`): `GET /api/status` (open), `/api/info`, `/api/cat`, `/api/ls`, -`/api/stat`, `/api/pin/ls`, `/api/providers`, `/api/ping`, `/api/name/list`, `/api/name/resolve`; +`/api/stat`, `/api/pin/ls`, `/api/providers`, `/api/ping`, `/api/name/list`, `/api/name/resolve`, +`/api/name/watch` (a long poll for newer records: Public API, API additions); `POST /api/add`, `/api/add/stream`, `/api/get`, `/api/dir/set`, `/api/dir/rm`, `/api/pin/add`, `/api/pin/rm`, `/api/pin/remote`, `/api/gc`, `/api/verify`, `/api/name/create`, `/api/name/rm`, `/api/name/export`, `/api/name/import`, `/api/name/publish`, `/api/export`, `/api/import`. `GET /api/info` and `POST /api/get` take -an optional `from` (node ID); `GET /api/name/resolve?name=` takes a name or a name contact's nick. +an optional `from` (node ID), as does `GET /api/cat`; `GET /api/name/resolve?name=` takes a name or a name +contact's nick. The API requires `X-Rnfs-Api: ` on every request except `GET /api/status`, which without a valid token answers only `{"version"}` (health checks, and `verify_daemon`, which reads the proof header, then the @@ -378,9 +408,10 @@ read from (`winacl.tighten(fd=...)`): another account's is refused, a loose one a failure is fatal. A `--dir` outside `%USERPROFILE%` is warned about. The advapi32 and kernel32 calls are behind `winacl.backend`, which tests replace with a fake. A response to a request with `X-Rnfs-Challenge: <32 hex>` carries `X-Rnfs-Proof: apiclient.api_proof(token, port, challenge)` (HMAC-SHA256 of -`rnfs api proof `); `apiclient.Client.verify_daemon` checks it with `GET /api/status` before +`rnfs api proof `); `apiclient.ApiConnection.verify_daemon` checks it with `GET /api/status` before first sending the token. Both HTTP servers (`TimedHandler`) time out slow requests, cap concurrent connections (API -32, gateway 64) and refuse HTTP/0.9 with a 400 carrying the usual headers. Gateway errors raised by the node +32, gateway 64) and refuse HTTP/0.9 with a 400 carrying the usual headers (both add them in `end_headers`, so +http.server's own error pages, 400, 414, 431 and 501, carry them too). Gateway errors raised by the node (anything but `GatewayError` and `NotFound`: 400, 413, 422, 503, 507, 500) say only a fixed message per status (`gateway.ERROR_MESSAGES`; the detail goes to the log); malformed `Range` headers are ignored. @@ -390,6 +421,292 @@ files as subresources but can't read any gateway response with `fetch()`; an ear would let any site's sandboxed frame read the gateway. A separate origin per root (a subdomain per CID) is the possible future way to let pages read their own DAG. +## Public API (rnfs.Node, rnfs.Client) + +**Provisional until 0.2.** Names, signatures and objects may change after use in a real app. Said in the +module docstring of `rnfs/__init__.py`, in `rnfs.API_STATUS == "provisional"`, and in README.md. + +Two classes with one interface (`rnfs.interface.Rnfs`): **`rnfs.Node`** runs a node inside the app's process, +as RNS runs inside it; **`rnfs.Client`** drives the user's running daemon through its local API. Same methods, +objects, exceptions and callback rules, so an app switches by changing the one line that constructs it. Nothing +chooses for the app. The JSON API is unchanged for other languages. + +```python +import RNS, rnfs + +RNS.Reticulum() # the app's own +with rnfs.Node("~/.myapp/rnfs") as node: # or: rnfs.Client() for the user's daemon + added = node.add("photos/") # rnfs.Added; added.cid is an rnfs.Cid + for entry in node.ls(added.cid): # rnfs.Entry(name, kind, size, cid) + print(entry.name, entry.size) + with node.open(f"{added.cid}/cat.jpg") as f: # seekable; reads fetch only the chunks they cover + f.seek(1_000_000); data = f.read(4096) + name = node.names.create("site") # rnfs.Name + name.publish(added.cid) + watch = node.names.watch(other, lambda record: print(record.cid)) + job = node.jobs.get("rnfs:name/3f2a…/site", "/tmp/site", on_progress=lambda job: print(job.done_bytes)) + job.wait() +``` + +### Package layout + +``` +rnfs/__init__.py __version__, API_STATUS; every public name loaded on first use (PEP 562 __getattr__, __all__, + __dir__), so `import rnfs` imports nothing more and the CLI's start-up is unchanged +rnfs/objects.py the typed objects (Cid re-exported from rnfs.content.cid); no RNS +rnfs/interface.py Rnfs, Names, Name, Jobs, Job, Deny, Watch, File, the event thread: shared by both; no RNS +rnfs/embedded.py Node: imports rnfs.node, so RNS, only when rnfs.Node is first used +rnfs/remote.py Client, over rnfs.apiclient; no RNS +rnfs/errors.py gains Cancelled (moved from fetcher, which imports it from there) and Closed +rnfs/apiclient.py the API protocol layer (token, proof, typed errors, Stream); its class becomes ApiConnection +``` + +`Rnfs` implements every method once, `open` included, over a *backend* offering the facade's operations in the +local API's argument and result forms (JSON-ready dicts; CIDs written out, node IDs and names as hex): for +`Node`, `embedded._Backend`, an adapter over the `RnfsNode`; for `Client`, `remote._Backend(api, long_wait)`, over +an `ApiConnection` whose methods already mirror the facade (`add_path`, `ls`, `info`, `pin_add`, …). Both turn the +same dicts into the same objects (`Entry.from_dict`), so they can't drift apart; only lifecycle, how name records +are heard, and the escape hatches differ per class. `rnfs.Client`, its objects +and errors import neither RNS nor anything that does (tested, as the CLI's start-up is). The CLI keeps using +`apiclient` directly (its `ApiClient` subclasses `ApiConnection`), so its output can't change. + +### Construction and lifecycle + +`Node(rnfs_dir, *, reticulum=None, settings=None)` +- `rnfs_dir` (str or path) is required: the app's own node directory (identity, blocks, state, optional + `rnfs.toml`), never guessed. Created private as the daemon's is (`_make_private_dirs`). +- `reticulum`: the running `RNS.Reticulum`, which must be `RNS.Reticulum.get_instance()` (RNS allows one per + process; anything else is ValueError). None: the running instance if there is one, else the node starts one + from the `reticulum_dir` setting, as the daemon does (so it attaches to `rnsd` if one runs). The node never + stops Reticulum or calls `RNS.exit()`, whether it started it or not: RNS can't restart within a process, and + its exit handler saves its tables when the process ends. +- `settings`: `Config` fields as a dict, over `rnfs.toml`'s (`Config.load(rnfs_dir, defaults=EMBEDDED_DEFAULTS, + **settings)`), validated (ValueError naming the setting). Defaults for an embedded node where neither sets them + (`embedded.EMBEDDED_DEFAULTS`, which `Config.load`'s `defaults` puts under the file): `dht_mode = "client"` + (an app's node is often short-lived; section 6 says such nodes SHOULD be clients) and `lxmf_enabled = false`. + `{"dht_mode": "server"}` and `{"lxmf_enabled": True}` turn them on. `api_port`, `gateway_port` and + `debug_log` belong to the daemon and are ignored (no API, gateway or debug log runs in the app); + `verbosity` applies only when the node starts Reticulum. +- The identity is the rnfs dir's (`storage/identity`); an app's identity can't be handed in (decision 4). +- The constructor starts the node (state loaded, threads running, join in the background) and raises if it + can't: `RnfsError` for the dir in use or not private, `ValueError` for settings, `OSError`, `StateDbError`. + At most one `Node` per process at a time (`embedded._running`; RnfsError otherwise), checked before the lock. +- It takes `/daemon.lock` before anything else touches the dir, through the helper the daemon now calls + too, `node.lock_rnfs_dir(config)` (`_make_private_dirs`, then `fsutil.lock_exclusive`); held, it raises + `node.DirInUse` (an RnfsError with `.holder`, the process ID): "another rnfs daemon or node (process N) is using + DIR; stop it first, or use another rnfs dir". The daemon catches it and prints its own message, unchanged. + `close()` releases the lock once the node has stopped, as a start that fails does (`fsutil.unlock`: unlocked, then + closed, since Windows may free a closed file's locks only some time later). + +`Client(rnfs_dir=None, *, port=None, token_file=None, timeout=None)`: as `apiclient.ApiConnection` (default +`~/.rnfs/`, the `api_port` in its `rnfs.toml`, the token from `/api_token`, proved before it is sent). +Nothing is sent until the first call. Raises `SettingsError` if `rnfs.toml` can't be read. + +Both: `close()` (idempotent; both end their event and job-watch threads, then a Node stops its features and +threads, saves, closes the state database, releases its rnfs.dht destination and the lock; a Client's name +poller ends with the long poll in flight, within 30 s), `closed`, `with … as node:` (closes on +exit), `wait_joined(timeout=None) -> bool` (Node: the first join attempt has finished; Client: True). A call after +`close()` raises `Closed`. Both may be used from several threads at once. + +### Targets and objects + +A *target* is a `Cid`, a string `[/name…]` (whose first part may be a name contact's nick, as in the CLI), +or an `rnfs:` URI (section 13): `rnfs:/path`, or `rnfs:name//path`, whose name is resolved first +(persisted, as `names.resolve`), with `?from=` used as `from_node` where the method takes one. Node IDs +and names are `bytes` (16), or 32 hex characters where accepted; returned as `bytes`. + +Every object is a frozen dataclass with `to_dict()` (the API's JSON form, CIDs as strings) and +`from_dict(d)` (from that form): + +| Object | Fields | `str()` | +|---|---|---| +| `Cid` | `rnfs/content/cid.py` as is: `.binary`, `.codec`, `.digest`, `.key`, `parse`, `from_bytes` | written form | +| `Added` | `cid`, `size`, `warnings: tuple[str]`, `sealed: bool` | the CID | +| `Written` | `cid`, `output`, `warnings`, `warnings_total`, `sealed`, `name` (get) | the output path | +| `Entry` | `name`, `kind` ("file", "executable", "directory"), `size`, `cid` | the name | +| `Info` | `cid`, `type` ("file", "directory"), `size`, `executable`, `sealed`, `recipient`, `stored_size`, `name` | — | +| `Stat` | `cid`, `size`, `blocks`, `local_blocks`, `local_bytes` | — | +| `Pin` | `cid`, `sources: tuple[str]`, `complete`, `type`, `sealed`, `name` | the CID | +| `Record` | `name: bytes`, `cid`, `sequence`, `age` (seconds since heard, or None) | — | +| `Status` | `version`, `node_id`, `identity_hash`, `dht_mode`, `slow`, `private`, `known_nodes`, `records`, `pins`, `provided`, `blocks`, `used_bytes`, `max_storage`, `names`, `features: dict` (every other field) | — | +| `Collected` / `Verified` | `removed_blocks`, `freed_bytes` / `checked`, `bad: tuple[Cid]` | — | +| `Exported` / `Imported` | `roots: tuple[Cid]`, `blocks` / and `incomplete` | — | +| `DenyState` | `entries: tuple[str]`, `lists: tuple[dict]` | — | + +`Name`, `Job`, `Watch` and `File` are bound to the `Node` or `Client` that made them (below). + +### Methods + +Exceptions, as both classes raise them (`→` the errors besides those every call may raise: `Closed`, and for a +Client the connection errors below). "Fetch errors" = `NotFound` (no such content or path, no provider), +`TooLarge`, `InvalidDag`, `Denied`, `StorageFull`, `Busy`. Facade calls are on `RnfsNode`; routes in +`rnfs/api.py` or the feature named. + +| Method → returns | Raises | Facade / API route | +|---|---|---| +| `status() -> Status` | | `status()` / `GET /api/status` | +| `node_id -> bytes` (property) | | `node_id` / status, read once | +| `add(source, *, name=None, wrap=False, pin=True, only_hash=False, to=None, to_self=False, seal_name=True, chunker=None) -> Added` | `ValueError` (options: `name` with a path, `wrap` without `name`, a source neither a path, bytes nor a binary file), `NotFound` (path), `Denied` (a file's hash), `StorageFull`, `OSError` | a path (str or PathLike): `add_path` / `POST /api/add`; bytes or a binary file: `add_stream(name=name)` / `POST /api/add/stream`, sealing with `name` as the sealed name unless `seal_name` is False | +| `get(target, output, *, from_node=None, raw=False) -> Written` | fetch errors, `ValueError` (output exists), `OSError` | `get` / `POST /api/get` | +| `open(target, *, from_node=None, raw=False) -> File` | fetch errors, `ValueError` (a directory) | `info`, then `read(target, offset, length)` per refill / `GET /api/info`, then `GET /api/cat?offset=&length=&from=` per refill | +| `cat(target, *, offset=0, length=None, from_node=None, raw=False) -> Iterator[bytes]` | fetch errors, `ValueError` | `read` / `GET /api/cat` | +| `block(cid) -> bytes` | fetch errors | `block` / `GET /api/cat?block=1` | +| `ls(target) -> list[Entry]` | fetch errors, `ValueError` (not a directory) | `ls` / `GET /api/ls` | +| `info(target, *, from_node=None, raw=False) -> Info` | fetch errors | `info` / `GET /api/info` (which has no `raw`: a Client gives a sealed file's `stored_size` as its size) | +| `stat(target) -> Stat` | fetch errors | `stat` / `GET /api/stat` | +| `dir_set(target, name, child, *, executable=False) -> Cid`, `dir_rm(target) -> Cid` | fetch errors, `ValueError` | `dir_set`, `dir_rm` / `POST /api/dir/set`, `/api/dir/rm` | +| `pin(target) -> Cid` (once complete) | fetch errors | `pin_add` / `POST /api/pin/add` | +| `unpin(cid)` | `NotFound` (no `user` pin) | `pin_rm` / `POST /api/pin/rm` | +| `pins(cid=None) -> list[Pin]` | | `pin_ls` / `GET /api/pin/ls` | +| `pin_remote(node_id, cid, pin=True) -> bool` | | `pin_remote` / `POST /api/pin/remote` | +| `providers(cid) -> list[bytes]`, `ping(node_id) -> float \| None` | | `providers`, `ping` / `GET /api/providers`, `/api/ping` | +| `export(target, output) -> Exported`, `import_archive(path) -> Imported` | fetch errors, `ValueError`, `OSError` | `export`, `import_archive` / `POST /api/export`, `/api/import` | +| `gc() -> Collected`, `verify() -> Verified` | | `gc`, `verify` / `POST /api/gc`, `/api/verify` | +| `names.create(alias) -> Name`, `names.import_key(alias, key) -> Name` | `ValueError` (alias invalid or taken, key malformed) | `name_create`, `name_import` / `POST /api/name/create`, `/api/name/import` | +| `names.get(alias) -> Name`, `names.list() -> list[Name]` | `NotFound` | `name_list` / `GET /api/name/list` | +| `names.remove(alias)` | `NotFound` | `name_rm` / `POST /api/name/rm` | +| `names.resolve(name) -> Record` (bytes, hex, `Name` or a contact's nick) | `NotFound` (unresolved), `ValueError` | `name_resolve` / `GET /api/name/resolve` | +| `names.watch(name, callback) -> Watch` | `ValueError`; a Client: `NotFound` from a daemon without the endpoint | below / `GET /api/name/watch` | +| `Name`: `alias`, `name: bytes`, `cid`, `sequence`, `str()` hex; `publish(target) -> Record`, `export_key() -> bytes`, `remove()`, `refresh() -> Name` | `publish`: `NotFound` (path not stored), `ValueError`, `RnfsError` (highest sequence) | `name_publish`, `name_export`, `name_rm` / `POST /api/name/publish`, `/api/name/export`, `/api/name/rm` | +| `jobs.get(target, output, *, from_node=None, sealed_name=False, fallback_dht=False, on_progress=None) -> Job`, `jobs.pin(target, *, on_progress=None) -> Job`, `jobs.export(target, output, *, on_progress=None) -> Job` | `ValueError` (output exists or in use), `Busy` (1000 active) | `JobsFeature.submit` / `POST /api/jobs` | +| `jobs.job(job_id) -> Job`, `jobs.list() -> list[Job]`, `jobs.clear(job_id=None) -> int` | `NotFound`, `ValueError` (not finished) | `JobsFeature.get`, `list`, `clear` / `GET /api/jobs/get`, `GET /api/jobs`, `POST /api/jobs/clear` | +| `Job`: `id`, `kind`, `target`, `output`, `from_node`, `sealed_name`, `fallback_dht`, `state`, `error`, `result: dict`, `total_bytes`, `done_bytes`, `downloaded_bytes`, `blocks`, `blocks_total`, `rate`, `eta`, `created`, `started`, `finished`, `runs`, `done` (finished); `refresh() -> Job`, `wait(timeout=None) -> Job`, `cancel() -> Job`, `watch(callback) -> Watch` | `wait`: `TimeoutError`; `NotFound` once cleared | `JobsFeature.get(wait=)`, `cancel` / `GET /api/jobs/get?wait=`, `POST /api/jobs/cancel` | +| `deny.list() -> DenyState`, `deny.add(entries) -> list[str]`, `deny.remove(entries) -> list[str]` | `ValueError` (malformed entry, too many), `NotFound` (remove: not on it) | `node.deny.entries`/`add`/`remove`, `DenyFeature.lists` / `GET /api/deny`, `POST /api/deny/add`, `/api/deny/rm` | + +`File` (from `open`, an `io.BufferedIOBase`): `read`, `read1`, `readinto`, `readline`, iteration, `seek`, `tell`, +`seekable()`/`readable()` True, `size`, `cid` (the file's own CID), `info`, `close()`, context manager; +`read()` of the rest is one read. Reads +go by the target as opened, root and path, since providers announce roots and the file's own CID may have none; +an `rnfs:name/` URI is resolved once, at `open`, and a contact nick is expanded by the node (which reuses its +CID for 60 s). It is an `io.BufferedReader` of 64 KiB over `_FileReader`, whose every `readinto` is one `read` +(or one `GET /api/cat`) of `[position, position + its length)`, so only the chunks covering it are fetched; a sealed file reads its plaintext unless `raw`. Raises fetch errors as reads happen. Not +thread-safe, like any file object. `Job`, `Name` and every object are snapshots as of their last `refresh()`, +`wait()` or callback. + +Job targets are what the jobs feature stores (`[/path]`): an `rnfs:name/` URI is resolved by `jobs.*` +before submitting, as the CLI resolves a nick. Jobs are persisted, resumed at the next start, and seen by +`rnfs jobs` (a Client's) or only by the app (a Node's, in its own rnfs dir). + +**Escape hatches, unsupported:** `Node.internal` (the `RnfsNode`) and `Client.api` (the `ApiConnection`, whose +`get(path, **params)` and `post(path, **fields)` reach every endpoint: contacts, peers, stats, keep, follow, +LXMF). Not in the typed interface for 0.2: keep/follow, contacts, peers/stats, LXMF. + +**Errors.** One hierarchy in `rnfs.errors`, every class importable as `rnfs.`: `RnfsError` ⊃ +`NotFound`, `TooLarge`, `InvalidDag`, `StorageFull`, `Busy`, `Denied`, `Cancelled`, `Closed`, `ApiError`. +`ApiError` (with `.status`) ⊃ the Client's own: `DaemonNotRunning`, `ConnectionFailed`, `TokenError`, +`AuthFailed`, `SettingsError`, `ServerError` (500, or another 5xx), and the API statuses, each also a subclass +of the node's class: `apiclient.NotFound(ApiError, errors.NotFound)` and so on, `BadRequest(ApiError, +ValueError)`. So `except rnfs.NotFound` and `except ValueError` catch both classes' errors. A Node raises +`OSError` where a Client gets `ServerError` (the API answers 500 for it). + +### Events and callbacks + +| Callback | Called with | Source in a Node | Source in a Client | +|---|---|---|---| +| `names.watch(name, callback)` | `Record`, each time a record of `name` newer than the last delivered (or than the one held when `watch` was called) is accepted under the sequence rules (section 9): announces, path responses, keepers' answers, this node's own publishes | `NameManager.watch(name)` (its announces count as asked in `_heard`, so a full name table never refuses them) and a record listener (`add_record_listener`), which `_offer` calls when it accepts a record and `publish` when it commits one, for watched names, with the names lock held; it only queues the record. Records heard before the watch knows its baseline wait and are compared with it (`_NameWatch.begin`). The keepers listener (`KeepersFeature.add_listener`) isn't used: its records reach `NameManager.offer` too | the `GET /api/name/watch` long poll, on the thread `rnfs-client-name-watch` | +| `on_progress` / `Job.watch(callback)` | `Job`, at most twice a second (Node) or once a second (Client) while it changes, and once at its final state | the job-watch thread (`rnfs-job-watch`) polls `JobsFeature.get(id)` (a snapshot made under the jobs lock) every `fetcher.POLL` (0.5 s); `fetcher.observing` and the `Observer` (the `Job`, on the job's thread) stay the jobs feature's | `rnfs-client-job-watch` polls `GET /api/jobs/get?id=` once a second; `wait()` loops `GET /api/jobs/get?id=&wait=30` | +| LXMF links received | not offered in 0.2 (decision 5) | | | + +Rules, the same for both: +- Every callback runs on the instance's event thread (`rnfs-events` in a Node, `rnfs-client-events` in a + Client), one at a time, in the order events arrived. Events are coalesced per name and per job, so a slow + callback gets the latest state, not a backlog. +- None of rnfs's locks is held: hooks inside rnfs only enqueue (`_Events.post`, whose condition is held just to + queue, innermost), and the thread calls app code with nothing held. A callback may call any method of the Node or Client. +- A callback must not block for long: it delays every other callback of that instance. Long work (a fetch, + `job.wait()`) belongs on the app's own thread or in a job. `close()` from a callback is allowed and doesn't + wait for the event thread. +- An exception in a callback is logged (a Node: rnfs's log; a Client: Python's `logging`, logger `rnfs`) and + dropped. `Watch.cancel()` (also a context manager) stops further calls; one already running finishes. +- Watching only listens: it sends nothing. To check now, `names.resolve()`. + +### API additions (backwards compatible) + +- `GET /api/name/watch?names=[,…]&after=[,…]&wait=` → `{"records": [{"name", "cid", + "sequence", "age"}]}`: the records held for the listed names whose sequence is above the matching `after` (`-`: + any), answered as soon as there is one or after `wait` seconds (at most 30, as jobs' `wait=`) with an empty list. + Level-triggered, so nothing heard between polls is lost: the Client's first call (`wait=0`, made inside `watch()`, + so a daemon without it raises `NotFound` there) sets the baseline, later ones run on its name-watch thread with + every watched name in one call. Each call marks its names watched (`NameManager.watch`) until `WATCH_GRACE` (60 s) + after it ends (`api.NameWatches`, one per `ApiServer`), at most `MAX_WATCHED` (1024) names, the soonest to end + dropped first. At most 64 names per call and 4 calls waiting at once (503 beyond; a `wait=0` call never waits, so + it is always answered), so watchers can't take the API's 32 connections. Token required; sends nothing to the + network. A name watched while a call waits joins the next call, within 30 s; nothing heard meanwhile is lost. +- `GET /api/cat` gains `from` (a node ID), as `GET /api/info` has, for `open(from_node=)` and `cat`. A daemon + that predates it ignores it (unknown parameters are ignored): the fetch still works, without the hint. + +### Parity gaps + +| Gap | How it is handled | +|---|---| +| A Node's events come from in-process hooks; a Client's from polling (jobs, 1 s) and the long poll (names) | Documented; the same callbacks and rules. A daemon older than the endpoint can't watch names (`NotFound` at `watch()`) | +| Paths (`add`, `get`, `export`, `import_archive`, a job's output) are opened by the daemon for a Client | Same machine and user (the token requires it); made absolute on the app's side, as now | +| `open()` costs one API request per buffer refill for a Client | Accepted; the 64 KiB buffer bounds the requests | +| Settings, DHT mode, LXMF, lifecycle: a Node's own; the daemon's for a Client | Documented: a Client can't change them; `close()` only ends the Client | +| No gateway or API in a Node | Not offered in 0.2 (decision 6) | +| The API's `GET /api/info` has no `raw` | A Client's `info(raw=True)` (and `open(raw=True)`) takes a sealed file's `stored_size` as its size | +| Keep/follow, contacts, peers, stats, LXMF only through the escape hatches | Typed methods later, once an app needs them | +| `OSError` in a Node is `ServerError` from a Client | Documented | +| A Client's connection errors (`DaemonNotRunning`, …) | Client only, all `ApiError` | + +### What happens to `rnfs/apiclient.py` + +It stays the protocol layer: `Client` was renamed `ApiConnection` (unreleased; the CLI's `ApiClient` and +`tests/test_apiclient.py` changed mechanically), keeping `request`, `call`, `get`, `post`, `Stream`, the token, +proof and version check, and its dict-returning methods, which become `rnfs.Client`'s backend. Its errors are +re-based as above. Its module docstring and README point apps to `rnfs.Client`. The CLI's output can't change: +it calls the same methods with the same parameters. It gained `name_watch` and `cat(from_node=)`. + +### Example app + +`examples/rnfs_share.py`, runnable as `python3 examples/rnfs_share.py --node DIR PATH` (an embedded node in +DIR, with the app's own Reticulum) or `--daemon PATH` (the user's daemon): adds PATH, lists it, reads 4 KiB +from the middle of its first file through `open`, publishes it under a name and watches that name, then gets it +back with a job showing progress. The one line constructing `Node` or `Client` is the only difference. +`MANIFEST.in` includes `examples/`; `tests/test_public.py` runs its `share()` through both classes. + +### Embedded mode: how a Node differs from the daemon + +The daemon keeps its process-wide behaviour. A Node (`RnfsNode(config, reticulum=None, embedded=True)`) differs +in these, each tested (tests/test_embedded.py: the guards in both scopes, and a host program over real RNS): +- Reticulum: it attaches to the running instance (or starts one if none runs); `stop()` never stops it, as it + never did (the daemon's `RNS.exit()` is in `cli.run_daemon`). At `stop()` an embedded node also deregisters its + rnfs.dht destination and closes the links to it (`_release_destination`), so the host's Reticulum stops + answering for it and a later Node in the process can register it again. +- Compression: the daemon keeps `AUTO_COMPRESS_MAX_SIZE = 0`. A Node leaves it alone; rnfs's responses are + already never compressed (`register_request_handler(auto_compress=False)`, which RNS 1.5.6 passes to the + response `Resource`), `LinkPool.request` refuses locally a request that wouldn't fit one packet (so rnfs never + sends a request Resource), and `guards.refusal` refuses on `rnfs.dht` links any advertisement with the + compressed flag (`adv.c`) before anything is transferred, with `max_decompressed_size = 0` set on each + accepted `rnfs.dht` Resource as a backstop. +- Guards: `guards.install(scope)` with `"process"` (the daemon: as today) or `"dht"` (a Node: the request, + transfer-size and segment checks apply only where `is_dht_link`). `guarded_receive` and + `guarded_response_resource_concluded` are already scoped; the host's links reach RNS's own methods. +- LXMF off by default: no `LXMRouter`, no LXMF identity, no `LXMF` import. With it on, the router is rnfs's own, + and its links (not rnfs.dht) get RNS's stock compression, bounded on receipt by the feature's per-Resource + limits; LXMF can't shut its router down, so it runs until the process ends. +- The lock, as above. Not done in a Node: the debug log, `threading.excepthook`, `faulthandler`, signal + handlers, `RNS.logdest`, `RNS.exit()`. +- Threads added: `rnfs-events` and `rnfs-job-watch` (Node), `rnfs-client-events`, `rnfs-client-job-watch` and + `rnfs-client-name-watch` (Client), each started on first use. Locks added: `_Events`' and `_JobPoller`'s + conditions (innermost: held only to queue or take work); `Rnfs._watch_lock`, taken after the names lock (the + record listener runs under it), never before; `NameManager._watched`, under `names._lock`; + `api.NameWatches.lock`, taken before the names lock, never under it; and three leaves, never held while calling + out: `remote.Client._poll_cond` (the name poller's wake-up), `interface._NameWatch.lock` (a watch's baseline and + last sequence) and `interface.Watch._lock` (`cancel()`'s once-only check). Callbacks run with no rnfs lock held. + +### Decisions taken at review + +1. Compressed advertisements on `rnfs.dht` links are refused before anything is transferred, by the daemon too + (before, it transferred one, then failed it as corrupt and tore the link down); honest nodes never send one. +2. `close()` on both classes, not the sketch's `stop()`. +3. `add()` returns `Added`, which carries a directory add's warnings. +4. A Node uses its rnfs dir's identity; the app's can't be handed in (0.2). +5. No LXMF callback and no handing over the app's `LXMRouter` (later). +6. A Node serves neither the API nor the gateway (0.2). +7. A Node defaults to `dht_mode = "client"` and `lxmf_enabled = false`. +8. `apiclient.Client` becomes `ApiConnection`, so `rnfs.Client` is the only client. + ## Content additions - `content/chunker.py` — `chunks(stream, size)` yields chunks from the chunker chosen with the context @@ -433,7 +750,8 @@ and in total, never persisted. more) for others. It protects a response from displacement only while it is within a grace it measures from the link's path (`link_path`, `fast_path`, `response_grace`, never from `link.rtt`), or moving at or above a floor (`progress_floor`). `link_path` treats the interface to a shared instance (`via_shared_instance`: RNS's - `is_connected_to_shared_instance` attribute, else the `LocalClientInterface` class name) as of unknown + `is_connected_to_shared_instance` attribute, else a `LocalServerInterface` or `LocalClientInterface` class + name, `rates.LOCAL_INTERFACE_TYPES`) as of unknown bitrate. `response_grace` prices the first hop at the interface's bitrate and the others at `unmeasured_bitrate`, or `slow_bitrate` when `fast_path` (the link's MTU is above 500, which RNS allows only across interfaces that configure their MTU), and adds `RequestServer._queued_bits` on the node's own slow @@ -466,7 +784,13 @@ and in total, never persisted. (`_budget_share_left` cuts the budget, or refuses). `RequestServer.traffic` and `PeerClient.traffic` share one `Traffic`. - `network/guards.py` also rejects, on `rnfs.dht` links only (recognised by the link's destination or owner - name `rnfs.dht.`), any Resource advertisement with more than one segment or with metadata. + name `rnfs.dht.`; an incoming link's destination is set by `Link.validate_request` before it is + registered), any Resource advertisement with more than one segment, with metadata or compressed, and gives + each `rnfs.dht` Resource it accepts a decompression limit of 0. Its scope (`install(PROCESS | DHT)`, set by + the node first, kept by later calls) decides whether its other checks cover every link (the daemon, which also + sets RNS's process-wide `AUTO_COMPRESS_MAX_SIZE` to 0) or `rnfs.dht` links only (an embedded node, which leaves + that limit, restoring it if a PROCESS install changed it). `LinkPool.request` sends nothing that wouldn't fit + one packet, so rnfs never sends a request Resource (which RNS would compress unless the limit is 0). - `guards.install()` also replaces `RNS.Link.response_resource_concluded` and `RNS.Link.receive` (class attributes; RNS looks both up on the link). On `rnfs.dht` links, `guarded_response_resource_concluded` reads a completed response Resource's data (an open file, rewound for RNS) and `guarded_receive` decrypts a @@ -485,29 +809,32 @@ and in total, never persisted. ## Features -`rnfs/features/__init__.py` defines `Feature` and `FEATURE_MODULES` (in start order: peers, jobs, keepers, -follow, contacts, lxmf, deny). The node builds every feature in its constructor (`node.feature(name)`), starts them -after its own threads and stops them, in reverse, before its own services. A feature may override: -`start()`, `stop()`, `save()` (called with the node's periodic saves), `status()` (fields merged into -`rnfs status`), `requests()` (new `rnfs.dht` paths; the request server applies private mode and the per-link -limit, and turns `None` or an exception into `False`), `api_routes()` (all token-protected) and -`cli_commands()`. Feature modules don't import RNS at module level (the CLI imports them to build its parser) -and their constructors work on `RnfsNode.offline` test nodes. +`rnfs/features/__init__.py` defines `Feature` and `FEATURE_MODULES` (in start order: peers, jobs, keepers, follow, +contacts, lxmf, deny). The node builds every feature in its constructor (`node.feature(name)`), starts them after +its own threads and stops them, in reverse, before its own services. A feature may override: `start()`, `stop()`, +`save()` (called with the node's periodic saves), `close()` (releases the state database at shutdown, after `stop()` +and the last save), `status()` (fields merged into `rnfs status`), `requests()` (new `rnfs.dht` paths; the request +server applies private mode and the per-link limit, and turns `None` or an exception into `False`; a response must +be a shape `protocol.response_refusal` accepts, True/False/None, a `/find` answer, or at most `MAX_BLOCKS_CIDS` bins +or nils, since nodes since 0.1.4 refuse any other), `api_routes()` (all token-protected) and `cli_commands()`. +Feature modules don't import RNS at module level (the CLI imports them to build its parser) and their constructors +work on `RnfsNode.offline` test nodes. | Feature | Module | What it does | Hooks in shared code | |---|---|---|---| | `peers` | `peers.py` | `rnfs peers`, `rnfs stats`; its own `rnfs.dht` announce handler, so client-mode nodes' mode and last-heard time are known | reads `Traffic`, the routing table, links, paths | | `jobs` | `jobs.py` | `JobsFeature` (the job manager), `Job`, CLI progress formatting | `fetcher.observing(observer)`: a thread-local observer every `Fetch` picks up (`total`, `block`, `cancelled` checked at each step, raising `fetcher.Cancelled`) | -| `keepers` | `keepers.py` | signed records (`SignedRecord`, `parse_signed`, `sign_record`, `from_announce_data`), `/name`, `KeeperClient`, `KeeperStore` | `Feature.requests`; `publisher.DhtKey` and `Publisher.add_key_source` (non-CID keys published like CIDs); `NameManager.keepers` (`published`, `heard`, `query`, `always_ask`), `NameManager.offer()`/`record()`; the 5-argument announce handler (packet hash) | +| `keepers` | `keepers.py` | signed records (`SignedRecord`, `parse_signed`, `sign_record`, `from_announce_data`), `/name`, `KeeperClient`, `KeeperStore` | `Feature.requests`; `publisher.DhtKey` and `Publisher.add_key_source` (non-CID keys published like CIDs); `NameManager.keepers` (`published`, `heard`, `query`, `always_ask`, `local_record`), `NameManager.offer()`/`record()`; the 5-argument announce handler (packet hash) | | `follow` | `follow.py` | followed names: each check and fetch one at a time, `fetcher.open(pinned=True)`, pin inside the fetch's protection, old pin released after | keepers' `FOLLOW` reason | | `contacts` | `contacts.py` | address book; `expand_target`, `lookup(nick, kind)`, `node_arg` | `RnfsNode._parse_target` (never for the gateway), `JobsFeature.submit` | -| `lxmf` | `lxmf.py` | pure parts (`Link`, `parse_uri`, `format_uri`, `find_links`, `encode_field`/`decode_field`, `MessageLog`); `LxmfFeature` (send with direct→propagated fallback, inbox, auto-fetch, inbox get, pin release); `LxmfNetwork`, made in `start()`, holds every RNS/LXMF object | `lxmf.validate_settings` from `Config.validate`; `LXMF_PREFIX` in `storage.check_source` | +| `lxmf` | `lxmf.py` | pure parts (`Link`, `parse_uri`, `format_uri`, `find_links`, `encode_field`/`decode_field`); `MessageLog` (the inbox and outbox, in the state database); `LxmfFeature` (send with direct→propagated fallback, inbox, auto-fetch, inbox get, pin release); `LxmfNetwork`, made in `start()`, holds every RNS/LXMF object | `lxmf.validate_settings` from `Config.validate`; `LXMF_PREFIX` in `storage.check_source` | | `deny` | `deny.py` | `deny_lists` sources (`parse_source`, `validate_settings` from `Config.validate`): `check(source)` resolves a name (persisted), fetches the target pinned, checks it is a file within `MAX_LIST_BYTES`, parses it, pins it `deny` and `accept`s it, then releases unused `deny` pins; `run_due` on `rnfs-deny`, after the join (`CHECK_INTERVAL` for names, `retry_delay` backoff); sources no longer configured are forgotten at start | `node.deny` (core); `DENY` in `storage.check_source` | `LxmfNetwork` makes an `LXMF.LXMRouter` with storage `/lxmf/` and `signal.signal` stubbed (LXMRouter installs SIGINT/SIGTERM handlers that exit at once), a 256 kB delivery limit, announce data without `SF_COMPRESSION`, and incoming delivery Resources (and propagation-node downloads) allowed to decompress up to -that limit, since rnfs sets RNS's global limit to 0. It keeps a table of up to 64 propagation nodes from +that limit, since the daemon sets RNS's global limit to 0 (an embedded node leaves it, and the limit bounds them +all the same). It keeps a table of up to 64 propagation nodes from announces (forgotten after 7 days unheard), but uses one automatically only if it is `lxmf_propagation_node`; the nearest heard is used only for a request with `nearest`, and without a configured node the feature logs once (notice) that downloads and the fallback are off. On a propagation-node link it cancels any response @@ -538,7 +865,7 @@ busiest one's newest place (that link is marked skipped). Links without the fiel before starting any (two links with one output are refused), then submits a get job per link with its `from` and `fallback_dht`; once one has started, a later link's failure is reported in its result (`error`) rather than raised. An output without `-o` is named with `fsutil.safe_default_name`. Sent links carry `from` only with -`lxmf_send_from_hint`. The inbox and outbox files are fsynced before being renamed into place. +`lxmf_send_from_hint`. The inbox and outbox are rows of the `lxmf_messages` table, saved in batches (NORMAL). **Feature API routes** (all need the token): `POST /api/jobs`, `GET /api/jobs`, `GET /api/jobs/get?id=&wait=`, `POST /api/jobs/cancel`, `POST /api/jobs/clear`; `GET /api/peers`, `GET /api/stats`; `POST /api/name/keep`, @@ -562,32 +889,43 @@ name in the output's directory; the CLI sets it for `rnfs get ` withou |---|---|---| | `rnfs-join`, `rnfs-publisher`, `rnfs-names`, `rnfs-maintenance` | node | joining (and joining again while under `MIN_SERVERS`), publishing, name announces, periodic saves and eviction | | `rnfs-announcer`, `rnfs-response-monitor`, `rnfs-link-sweeper` | network | rnfs.dht announces, `/blocks` deadlines and displacement, idle links | -| `rnfs-lookup`, `rnfs-fetch`, `rnfs-find-providers`, `rnfs-remote-pin` | per operation | lookups, fetch workers, provider search, remote-pin fetches (one thread per source with pins waiting) | +| `rnfs-lookup`, `rnfs-fetch`, `rnfs-find-providers`, `rnfs-remote-pin`, `rnfs-publish-lookup` | per operation | lookups, fetch workers, provider search, remote-pin fetches (one thread per source with pins waiting), a publishing round's lookups (up to 4 at once, ended with the round; none on slow nodes) | | `rnfs-write` | storage | an add's, shared-folder import's or archive import's block writes (`Session.writer()`), up to 16 at once, ended with it | | `rnfs-job-` | jobs | one per running job, at most `max_jobs`; `stop()` waits ≤ 2 s | | `rnfs-keepers`, `rnfs-keeper-query` | keepers | pending refreshes (≤ 64 queued), own-name sync, saves, re-asking every `announce_interval`; parallel `/name` queries | | `rnfs-follow` | follow | every check and fetch, one at a time | | `rnfs-deny` | deny | every deny-list source's check and fetch, one at a time, once joined | +| `rnfs-events`, `rnfs-job-watch` | rnfs.Node | callbacks into the program, one at a time; polling watched jobs (both started on first use) | +| `rnfs-client-events`, `rnfs-client-job-watch`, `rnfs-client-name-watch` | rnfs.Client (in the program) | callbacks; polling watched jobs; the `GET /api/name/watch` long poll | | `rnfs-lxmf`, `rnfs-lxmf-fetch` | lxmf | announces, propagation-node downloads, polling outbound states each second; auto-fetch, one at a time, senders in turn (LXMF runs its own job thread) | The API and gateway run one thread per connection, capped (32 and 64). -Locks are taken in this order, never the reverse: `names._lock` → `publisher._lock` → `node._lock` → -`storage._lock` → the state database's lock (`statedb`, innermost: a store opens a transaction only once it -holds every lock the change needs, and inside it takes no other lock and does no network I/O). `node._deny_lock` -(one deny-list change applied at a time, `_apply_deny`) is taken before the publisher's lock and never with the -names lock; `DenyList._lock` holds only the state database's. `NameManager.publish`, `remove` and `release_gone` pin and unpin, and so call -`publisher.refresh()`, while holding the names lock; `Publisher.refresh` recomputes what is provided under its -own lock, reading the node's fetched targets (`node._lock`) and walking the pins' DAGs (`storage._lock`); the -node reads storage under its lock (`_prune_fetched`). This is how the code behaves today, not a design goal. -Batched saves write outside these locks, from the changes collected under them, one save at a time behind a -save lock taken first (`Storage.save`, `Publisher.save`, `ProviderStore.save`, the name floor, the fetched -targets, the remote pins waiting), so an older snapshot never lands last. Slow work that runs while a lock is -held: +Locks are taken in this order, never the reverse: `names._lock` → `publisher._lock` → `node._lock` → `storage._lock` +→ the state database's lock (`statedb`, innermost: a store opens a transaction only once it holds every lock the +change needs, and inside it takes no other lock and does no network I/O). `node._deny_lock` (one deny-list change +applied at a time, `_apply_deny`) is taken before the publisher's lock and never with the names lock; +`DenyList._lock` holds only the state database's. Feature locks held across `storage.pin`/`unpin`/`protect` and +`publisher.refresh()` (`JobsFeature._lock`, `FollowFeature._lock`, `LxmfFeature._lock`) come before the publisher's +and storage's. The public API's locks come after these: `Rnfs._watch_lock` after the names lock (NameManager's +record listeners run under it, and only queue), the event and job-watch conditions innermost; callbacks into the +program run with none of them held. `api.NameWatches`' lock is taken before the names lock (it watches and unwatches +in NameManager, one watch per name it holds) and never under it. `NameManager.publish`, `remove`, `release_gone` and +`reconcile_pins` pin and unpin, and so call `publisher.refresh()`, while holding the names lock; `Publisher.refresh` +recomputes what is provided under its own lock, reading the node's fetched targets (`node._lock`) and walking the +pins' DAGs (`storage._lock`); the node reads storage under its lock (`_prune_fetched`). This is how the code behaves +today, not a design goal. Batched saves write outside these locks, from the changes collected under them, one save +at a time behind a save lock taken first (`Storage.save`, `Publisher.save`, `ProviderStore.save`, `DenyList.save`, +the name floor, the fetched targets, the remote pins waiting), so an older snapshot never lands last. Slow work that +runs while a lock is held: - `Publisher.refresh` checks every pin's DAG (`dag_blocks`, memoised) under the publisher lock, behind the - names lock when a name changes. It builds the set of blocks served again only when a provided CID went or its - DAG's blocks changed (CIDs only added are unioned in), so a refresh that changed nothing doesn't recompute it. + names lock when a name changes, but only when something it is computed from changed: the node's generation + (`Storage.changes()` and the fetched targets' change count, read under `node._lock` then `storage._lock`), the + deny list's CID set, the key sources' keys or the strategy. A refresh that changed none of them returns at once + (about 1 µs, from 7 ms at 10 000 pins); the generation is read before computing, so a change made meanwhile + makes the next refresh compute. When it does compute, it builds the set of blocks served again only when a + provided CID went or its DAG's blocks changed (CIDs only added are unioned in). - `Storage.evict` and `gc` decide what to delete, and delete it, under the storage lock (after reading the DAG nodes outside it); `verify` checks completeness and re-reads the blocks outside it. - Every pin and unpin commits its row durably (`synchronous=FULL`, after the blocks are synced) under the @@ -617,8 +955,10 @@ change returns, "NORMAL" ones are batched saves a power loss may lose the last o load; NORMAL); `follow` (FULL); `contacts` (FULL); `lxmf_messages` (inbox: 500 messages, 4 MiB, 50 per non-auto-fetch sender, 16 KiB per record; outbox 200, unfinished messages marked failed at a restart; NORMAL, changed messages only; ids and `next_id` stay below 2^63); -- `deny_entries` (the local deny list, canonical text; FULL) and `deny_lists` (each source as `dump()`ed text, the - CID of the version it last accepted; FULL); +- `deny_entries` (the local deny list, canonical text; FULL), `deny_lists` (each source as `dump()`ed text, the + CID of the version it last accepted; FULL) and `deny_hashes` (the content SHA-256 of files checked against + `sha256:` entries, by binary CID, LRU order in `used`, at most 10 000; NORMAL with the periodic saves and at + stop, changed rows only; a CID's content never changes, so a row never goes stale); - `meta` (per-store values: follow's and each box's `clean`, set by every save, which the reconcile at start requires; each box's `next_id`) and `migrated` (JSON files imported). @@ -626,7 +966,8 @@ Free text that may hold lone surrogates (file names, LXMF records, cache paths) (`statedb.dump`). Rows that fail the checks the JSON loaders applied are dropped at load with a warning and deleted. The JSON files of 0.1.4 and earlier, once imported, stay as `.migrated`. Elsewhere: `storage/identity`, `storage/lxmf_identity` (0600), `names/` (name keys), `lxmf/` (LXMF's ratchets, -delivered-message IDs, stamp costs), `api_token` (while the daemon runs), `daemon.lock` (locked while it runs). +delivered-message IDs, stamp costs), `api_token` (while the daemon runs), `daemon.lock` (locked while the daemon, or an embedded `rnfs.Node`, uses the +dir: `node.lock_rnfs_dir`). Outputs being written live beside the output as `.rnfs-get-.partial` (`RnfsNode.get`) or `.rnfs-job-.partial` (get jobs) until renamed; a failed, cancelled or shut-down job removes its partial, and a resumed job removes one left by a crash. @@ -635,7 +976,9 @@ and a resumed job removes one left by a crash. The commands CI runs the tests with are in `.github/workflows/tests.yml`. Feature tests live in their own files (`test_jobs.py`, `test_mesh.py`, `test_names_keep.py`, `test_contacts.py`, `test_seal.py`, `test_chunker.py`, `test_pin_false.py`, `test_gateway_formats.py`, `test_lxmf.py`), and `test_e2e.py` has a marked block per -feature. The state database has `test_statedb.py`, each store's migration a `test_migrate_*.py`, and +feature. The public API has `test_public.py` (every interface test run on an embedded node and on a Client of an +ApiServer, the name-watch endpoint, the objects, `import rnfs`'s weight) and `test_embedded.py` (the guards' +scopes, the dir lock, and with `RNFS_E2E=1` a host program over real RNS beside an embedded node). The state database has `test_statedb.py`, each store's migration a `test_migrate_*.py`, and `test_upgrade_014.py` loads `tests/fixtures/rnfs-0.1.4/`, a node directory 0.1.4 wrote (its `generate.py` writes it again). CI runs them on Linux, macOS and Windows; `packaging/smoke_test.py` checks a built package (`--version`, `--help`, daemon and status, an add, and that the state database holds the pin and imported a @@ -654,8 +997,10 @@ Everything in the specification is implemented, with these simplifications and a checks `RoutingTable.live_count()` every `REJOIN_DELAY` (300 s) and joins again while it is under `MIN_SERVERS` (3), doubling the wait up to `announce_interval`. `DEFAULT_BOOTSTRAP_NODES` lives in `rnfs/constants.py` and changes only with a release; the e2e test turns it off (`default_bootstrap = false`). -- Publishing runs one closest-nodes lookup per due CID, one after another. Lookups aren't shared between keys - that have the same closest nodes, but `/provide` requests are batched per node. Slow nodes pace these +- Publishing runs up to 4 closest-nodes lookups at once (`PUBLISH_LOOKUPS`, on `rnfs-publish-lookup` threads); + slow nodes run them one at a time. Lookups aren't shared between keys that have the same closest nodes, but + `/provide` requests are batched per node, in the same batches one at a time would build. The airtime controls + apply to `/blocks` only, and overlapping lookups send the same `/find` messages. Slow nodes pace these lookups (at most 10 per round, 30 s per lookup on average), which only delays republishing; withdrawals are sent at once. - Withdrawals are sent once, with any provides due for the same node. One that fails isn't retried, since the @@ -679,7 +1024,10 @@ Everything in the specification is implemented, with these simplifications and a `DAG_ENTRY_OVERHEAD` (16) per entry, an index node its own bytes and 16 per child less `INDEX_NODE_COST` (64), never below 0; and `max_dag_bytes` (a remote pin's or auto-fetch's quota) if lower. Over it is `TooLarge`, not `InvalidDag`. Every caller gets it, since it lives in `fetch_dag`: `get`, `pin_add`, - `dir_set`/`dir_rm`, `export`, `export_stream`, follow, remote pins and LXMF auto-fetches. Why the builder's + `dir_set`/`dir_rm`, `export`, `export_stream`, follow, remote pins and LXMF auto-fetches. A target whose DAG + is stored complete already isn't walked: it is checked whole against the deny list and, with + `max_dag_bytes`, against the quota (`Storage.dag_bytes`), so a remote pin or auto-fetch of a DAG held already + meets the same quota as one fetched. Why the builder's output never comes near it: a file node child is at most 51 bytes (`[cid, size]`: 39-byte CID, 9-byte size) and a node header at most 7; fixed chunks are 64 KiB and FastCDC chunks over 16 KiB except a file's last, so with the levels above (at most n/255 + 8 nodes for n chunks) a file takes under 13 bytes per 4 KiB plus @@ -760,6 +1108,10 @@ Everything in the specification is implemented, with these simplifications and a 4 × that time of the request being sent isn't counted towards `max_failures` (retried after an eighth of it, at least `busy_delay`). A `Fetch` exiting with an error (or cancelled) closes the links of its requests still in flight. In private mode, incoming links not identified as a member's are closed first to make room. +- A shared-folder entry that fails to import because of its content (an invalid name, the deny list, too large) + isn't tried again until it changes; one that fails for a reason that may pass (`OSError`, including a pin the + state database couldn't save, or `StorageFull`) is tried again `RETRY_AFTER` (600 s) later, and an unchanged + entry whose pin fails is logged without ending the scan. - The shared-folder scan and `add` walk directories by file descriptor (`O_NOFOLLOW`, `O_NONBLOCK`, type and inode checks), so entries swapped for symlinks, FIFOs or devices mid-scan are skipped. The scan (not `add`) skips regular files whose opened descriptor has `st_nlink > 1` (`builder.hard_links`, `build_open`'s @@ -809,43 +1161,42 @@ Everything in the specification is implemented, with these simplifications and a `/name`; a fast node with no known sequence and no established keepers asks all 3 batches (up to 9), and the lookup runs to completion unless two established nodes list keepers. - **Names.** A key imported, or found without a `names_published` row, is marked `imported`: its first publish - offers kept records, makes a path request and queries keepers before choosing a sequence. `publish()` pins - the new target, saves its row (the commit point; a failed commit rolls the record back and releases the - new pin) and only then releases the old target, so the saved target is pinned at every step. Once the - features are set up, `reconcile_pins(storage.pins())` releases the pins of name keys deleted by hand - (`release_gone()`), pins a recorded target again if its pin is missing, and releases `name:` pins nothing - records; a key without a saved record keeps its pins until its next publish or removal (`_stale`, - where a release that failed with `OSError` also waits). A key file that is there but can't be loaded is - `_unreadable`: its pins are kept and its row is never touched by saves. `Storage.pin`/`pin_many`/ - `unpin` undo their change in memory if the commit fails. Follow and LXMF do the same for their sources at - start (`FollowFeature._reconcile`, `_release_unrecorded`), but release only if their state loaded whole (the - `clean` meta value, set by every save; `MessageLog.clean`); otherwise unrecorded pins are recorded again (the name followed again, or an inbox - message per sender listing them), so a later start keeps them too. The name table marks - entries that were only heard (`_heard_only`, its own LRU, so finding one to evict is O(1)); an announce with - no waiting resolve, of a name without a kept record, may only evict one of those. A resolve with - `persist=False` (gateway, LXMF auto-fetch) raises the name's floor (`_floor`: name -> (sequence, wall time - last resolved), in resolve order), which `_offer` checks. A raise (or a new name) saves the floor - at once if the last save was `FLOOR_SAVE_INTERVAL` (5 s) ago, else from a one-shot `threading.Timer`; a - raise by a record heard for a name in the table (`_store`: an announce, a keeper's record, or one a resolve's - refresh heard) is always the timer's, started at once when a save is due, since its callers hold the lock; - renewing only the time marks it for the next save after an hour. `_trim_floor` drops the least recently - resolved past `FLOOR_FACTOR` × `name_table_size`, but stops at one resolved within `FLOOR_KEEP_AGE` (7 days) - until the floor holds `FLOOR_HARD_FACTOR` (2) times that. -- **Deny lists.** `RnfsNode._setup` makes the `DenyList` right after storage, gives it to storage - (`storage.deny`), and `reload`s the accepted lists from their pinned blocks (`_read_deny_list`), so the - publisher's first refresh already leaves out denied content. Fetches check every expectation in `_ensure` - and the root in `Fetcher.open`, and a complete stored DAG in `fetch_dag`'s early return (`dag_blocks`). - `NodeBackend.served_block`/`_size` answer None for a denied CID; `Publisher._compute` drops CIDs whose - `dag_blocks` hold one (`_denied_dags`, memoised per DAG and deny set). `_apply_deny` (the list's listener, and - maintenance when `deny.cids` changed, as at start) refreshes the publisher, `drop_denied`s the cache, releases - `remote:` pins of `denied_roots` and collects, and warns about the other pins it keeps. File hashes: - `_hash_check` (for `builder.checking`, in adds, the shared folder and imports, under `_refusing`, which - collects what a refused add or import stored) and `_check_file(expect, read)` (get's write-out check, a sealed - get, `read` before its first byte, `node_content.block` for a raw block and `export_stream` for every file - before `start`; a raw file's digest is its CID's, others are hashed once and remembered). A failed - `_apply_deny` leaves `_deny_applied` as it was, so maintenance tries again after `DENY_RETRY` (60 s). - `remote_pin` refuses a denied CID; jobs don't fall back to the DHT on `Denied`, and LXMF auto-fetch records - it as refused. + offers kept records, makes a path request and queries keepers before choosing a sequence. `publish()` pins the new + target, saves its row (the commit point; a failed commit rolls the record back and releases the new pin) and only + then releases the old target, so the saved target is pinned at every step. Once the features are set up, + `reconcile_pins(storage.pins())` releases the pins of name keys deleted by hand (`release_gone()`, whose releases + the rest of the pass skips), pins a recorded target again if its pin is missing, and releases `name:` pins nothing + records; a key without a saved record keeps its pins until its next publish or removal (`_stale`, where a release + that failed with `OSError` also waits). A key file that is there but can't be loaded is `_unreadable`: its pins + are kept and its row is never touched by saves. `Storage.pin`/`pin_many`/`unpin` undo their change in memory if + the commit fails. Follow and LXMF do the same for their sources at start (`FollowFeature._reconcile`, + `_release_unrecorded`), but release only if their state loaded whole (the `clean` meta value, set by every save; + `MessageLog.clean`); otherwise unrecorded pins are recorded again (the name followed again, or an inbox message + per sender listing them), so a later start keeps them too. The name table marks entries that were only heard + (`_heard_only`, its own LRU, so finding one to evict is O(1)); an announce with no waiting resolve, of a name + neither kept nor watched, may only evict one of those (`watch()` takes a name out of `_heard_only`). A name + resolved for the user, or published here, leaves the table and the floor (`_hold`). A resolve with `persist=False` + (gateway, LXMF auto-fetch) raises the name's floor (`_floor`: name -> (sequence, wall time last resolved), in + resolve order), which `_offer` checks. A raise (or a new name) saves the floor at once if the last save was + `FLOOR_SAVE_INTERVAL` (5 s) ago, else from a one-shot `threading.Timer`; a raise by a record heard for a name in + the table (`_store`: an announce, a keeper's record, or one a resolve's refresh heard) is always the timer's, + started at once when a save is due, since its callers hold the lock; renewing only the time marks it for the next + save after an hour. `_trim_floor` drops the least recently resolved past `FLOOR_FACTOR` × `name_table_size`, but + stops at one resolved within `FLOOR_KEEP_AGE` (7 days) until the floor holds `FLOOR_HARD_FACTOR` (2) times that. +- **Deny lists.** `RnfsNode._setup` makes the `DenyList` right after storage, gives it to storage (`storage.deny`), + and `reload`s the accepted lists from their pinned blocks (`_read_deny_list`), so the publisher's first refresh + already leaves out denied content. Fetches check every expectation in `_ensure` and the root in `Fetcher.open`, + and a complete stored DAG in `fetch_dag`'s early return (`dag_blocks`). `NodeBackend.served_block`/`_size` answer + None for a denied CID; `Publisher._compute` drops CIDs whose `dag_blocks` hold one (`_denied_dags`, memoised per + DAG and deny set). `_apply_deny` (the list's listener, and maintenance when `deny.cids` changed, as at start) + refreshes the publisher, `drop_denied`s the cache, releases `remote:` pins of `denied_roots` and collects, and + warns about the other pins it keeps. File hashes: `_hash_check` (for `builder.checking`, in adds, the shared + folder and imports, under `_refusing`, which collects what a refused add or import stored) and + `_check_file(expect, read)` (get's write-out check, a sealed get, `read` before its first byte, + `node_content.block` for a raw block and `export_stream` for every file before `start`; a raw file's digest is its + CID's, others are hashed once and remembered, across restarts: `deny_hashes`). A failed `_apply_deny` leaves + `_deny_applied` as it was, so maintenance tries again after `DENY_RETRY` (60 s). `remote_pin` refuses a denied + CID; jobs don't fall back to the DHT on `Denied`, and LXMF auto-fetch records it as refused. - **Follow** isn't built on jobs (job pins use source `user`); `pin add --follow` doesn't exist. Checks use the record held unless nothing was heard for 2 × `announce_interval`; a failed check (any exception) is retried after 300 s, doubling up to `announce_interval`, and a failed fetch is retried with the record held diff --git a/examples/rnfs_share.py b/examples/rnfs_share.py new file mode 100644 index 0000000..cb8c685 --- /dev/null +++ b/examples/rnfs_share.py @@ -0,0 +1,89 @@ +"""Share a file or folder from a program with rnfs's public API, through either class (provisional until 0.2). + + python3 examples/rnfs_share.py --node ~/.rnfs-example PATH # a node inside this program, beside its Reticulum + python3 examples/rnfs_share.py --daemon PATH # the user's running daemon (rnfs daemon) + +It adds PATH, lists it, reads 4 KiB from the middle of a file in it, publishes it under the name "example" and +watches that name, then gets it back through the name with a job showing progress. The line constructing +rnfs.Node or rnfs.Client is the only difference between the two. +""" +import argparse +import os +import sys +import tempfile +import threading + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) # run from a source checkout + +import rnfs # noqa: E402 + + +def share(node: rnfs.Rnfs, path: str) -> str: + """Add path through node, look at it, publish it and get it back; the path it was got back to.""" + added = node.add(path) + print(f"added {added.cid} ({added.size} bytes)") + for warning in added.warnings: + print(f" warning: {warning}") + target = str(added.cid) + if os.path.isdir(path): + entries = node.ls(added.cid) + for entry in entries: + print(f" {entry.kind:10} {entry.size:>12} {entry.name}") + files = [entry for entry in entries if not entry.is_dir] + target = f"{added.cid}/{files[0].name}" if files else None + if target is not None: + with node.open(target) as f: + f.seek(f.size // 2) # only the chunks this read covers are fetched + data = f.read(4096) + print(f"read {len(data)} bytes from the middle of {target}") + + mine = {name.alias: name for name in node.names.list()} + name = mine.get("example") or node.names.create("example") + heard = threading.Event() + + def on_record(record: rnfs.Record): + """Called on the node's event thread when a newer record of the name is heard (here: our publish).""" + print(f"heard rnfs:name/{record.name.hex()} -> {record.cid} (sequence {record.sequence})") + heard.set() + + with node.names.watch(name, on_record): + record = name.publish(added.cid) + print(f"published rnfs:name/{name.hex} -> {record.cid}") + heard.wait(10) + + output = os.path.join(tempfile.mkdtemp(), os.path.basename(os.path.normpath(path)) or "copy") + + def on_progress(job: rnfs.Job): + """Called on the event thread as the job goes on, and once when it ends.""" + print(f" {job.state}: {job.done_bytes} of {job.total_bytes if job.total_bytes is not None else '?'} bytes") + + job = node.jobs.get(f"rnfs:name/{name.hex}", output, on_progress=on_progress).wait() + if job.state != "done": + raise rnfs.RnfsError(f"the job {job.state}: {job.error}") + print(f"got it back as {output}") + return output + + +def main(): + """Parse the arguments, make the Node or the Client, and share.""" + parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) + where = parser.add_mutually_exclusive_group(required=True) + where.add_argument("--node", metavar="DIR", help="run an rnfs node inside this program, in DIR") + where.add_argument("--daemon", action="store_true", help="use the running rnfs daemon") + parser.add_argument("path", help="a file or folder to share") + args = parser.parse_args() + try: + if args.node: + import RNS + RNS.Reticulum() # the program's own Reticulum, from RNS's default config directory + instance = rnfs.Node(args.node) + else: + instance = rnfs.Client() + with instance as node: + share(node, args.path) + except rnfs.RnfsError as e: # every rnfs error, from either class + sys.exit(f"rnfs_share: {e}") + + +if __name__ == "__main__": + main() diff --git a/rnfs/__init__.py b/rnfs/__init__.py index 37a30f7..ebd7e97 100644 --- a/rnfs/__init__.py +++ b/rnfs/__init__.py @@ -1,5 +1,12 @@ """rnfs: content-addressed file sharing over Reticulum (RNS). +For programs (design.md, "Public API"). PROVISIONAL until rnfs 0.2 (API_STATUS): names, signatures and objects +may still change. Two classes with the same methods, objects and exceptions, so a program switches by changing +the line that constructs one: +- rnfs.Node(rnfs_dir, reticulum=..., settings=...) runs a node inside the program, beside its own Reticulum; +- rnfs.Client(rnfs_dir=None) uses the user's running daemon through its local API (it never loads RNS). +Every public name is loaded when first used, so `import rnfs` loads nothing more, and RNS only with rnfs.Node. + content/ CIDs, DAGs, storage, pins, archives, shared folder (no RNS) dht/ Kademlia routing table, provider records, iterative lookups (no RNS) network/ everything that talks to RNS: identity, links, request client and server, announces @@ -10,6 +17,38 @@ api.py local HTTP API; gateway.py the HTTP gateway debuglog.py the debug log file (debug_log) cli.py command line: run a daemon, or talk to one through its API +interface.py, objects.py, embedded.py, remote.py: the public API (rnfs.Node, rnfs.Client) """ -__version__ = "0.1.4" +__version__ = "0.1.5" +API_STATUS = "provisional" # the public API (rnfs.Node, rnfs.Client) may change until rnfs 0.2 + +# Public name -> the module defining it, imported on first use (PEP 562) +_EXPORTS = { + "Node": "rnfs.embedded", + "Client": "rnfs.remote", + **{name: "rnfs.objects" for name in ("Cid", "Added", "Written", "Entry", "Info", "Stat", "Pin", "Record", + "Status", "Collected", "Verified", "Exported", "Imported", "DenyState")}, + **{name: "rnfs.interface" for name in ("Rnfs", "Name", "Names", "Job", "Jobs", "Deny", "Watch", "File")}, + **{name: "rnfs.errors" for name in ("RnfsError", "NotFound", "TooLarge", "InvalidDag", "StorageFull", "Busy", + "Denied", "Cancelled", "Closed")}, + **{name: "rnfs.apiclient" for name in ("ApiError", "DaemonNotRunning", "ConnectionFailed", "TokenError", + "AuthFailed", "SettingsError", "BadRequest", "ServerError")}, +} +__all__ = ["API_STATUS", "__version__", *_EXPORTS] + + +def __getattr__(name: str): + """A public name, its module imported now.""" + module = _EXPORTS.get(name) + if module is None: + raise AttributeError(f"module 'rnfs' has no attribute {name!r}") + import importlib + value = getattr(importlib.import_module(module), name) + globals()[name] = value + return value + + +def __dir__() -> list[str]: + """The module's names, the public ones included.""" + return sorted({*globals(), *_EXPORTS}) diff --git a/rnfs/api.py b/rnfs/api.py index ce01490..f181f36 100644 --- a/rnfs/api.py +++ b/rnfs/api.py @@ -7,8 +7,8 @@ | Method | Path | Parameters | Response / facade call | |---|---|---|---| | GET | /api/status | | `status()`; without the token only `{"version"}` | -| GET | /api/info | `target` | `info(target)` | -| GET | /api/cat | `target`, `offset`, `length`, `raw`, `block` | the bytes of a file (`read`; sealed files decrypted unless `raw`), Content-Length set; `block=1`: the CID's own block | +| GET | /api/info | `target`, `from` | `info(target)` (fetched from node `from` if given) | +| GET | /api/cat | `target`, `offset`, `length`, `raw`, `block`, `from` | the bytes of a file (`read`; sealed files decrypted unless `raw`; fetched from node `from` if given), Content-Length set; `block=1`: the CID's own block | | GET | /api/ls | `target` | `{"entries": ls(target)}` | | GET | /api/stat | `target` | `stat(target)` | | GET | /api/pin/ls | `cid` (optional) | `{"pins": pin_ls(cid)}` | @@ -16,6 +16,7 @@ | GET | /api/ping | `node` | `{"rtt": seconds or null}` | | GET | /api/name/list | | `{"names": name_list()}` | | GET | /api/name/resolve | `name` | `name_resolve(name)` | +| GET | /api/name/watch | `names` (comma-separated), `after` (a sequence or `-` per name), `wait` (≤ 30 s) | `{"records": [{"name", "cid", "sequence", "age"}]}`: the records held whose sequence is above `after`, as soon as there is one or after `wait` (below) | | POST | /api/add | JSON `path`, `wrap`, `only_hash`, `pin`, `to` (list), `to_self`, `seal_name` (bool), `chunker` | `add_path(...)` | | POST | /api/add/stream | query `name`, `only_hash`, `pin`, `to` (comma-separated), `to_self`, `seal_name` (the name), `chunker`; body is the data | `add_stream(...)` | | POST | /api/get | JSON `target`, `output`, `from`, `raw` | `get(...)` | @@ -35,6 +36,13 @@ GET parameters are in the query string. Request bodies may use Content-Length or chunked encoding. +Watching names (rnfs.Client's names.watch): `GET /api/name/watch` is a long poll, level-triggered so a client loses +nothing between calls. Each call marks its names (at most MAX_WATCH_NAMES) watched until WATCH_GRACE seconds after +it ends (NameManager.watch: their announces count as asked), waits on NameManager's record listener, and answers +from the records held; it sends nothing to the network. At most MAX_WATCH_WAITERS calls wait at once (503 beyond; +a `wait=0` call never waits, so it is always answered), so watchers can't take the API's connections, and at most +MAX_WATCHED names stay watched. + Security: - The Host header must be `127.0.0.1:` or `localhost:`, which blocks DNS rebinding. - Every request except `GET /api/status` must carry `X-Rnfs-Api: `, the random token the daemon @@ -85,6 +93,11 @@ CHALLENGE = re.compile(r"[0-9a-f]{32}") # what the CLI sends in X-Rnfs-Challenge CHUNK_SIZE_LINE = re.compile(rb"[0-9A-Fa-f]{1,16}") # a chunk size: hex digits only (no 0x, +, _ or spaces) MAX_TRAILER_LINES = 32 +MAX_WATCH_NAMES = 64 # names one GET /api/name/watch may list +MAX_WATCH_WAITERS = 4 # GET /api/name/watch calls waiting at once +MAX_WATCH_WAIT = 30 # seconds one may wait, as GET /api/jobs/get's wait= +WATCH_GRACE = 60.0 # seconds a name stays watched after the last call naming it ends +MAX_WATCHED = 1024 # names watched for API clients at most; the soonest to end go first # Exception class → HTTP status, checked in order (design.md); anything else is 500 ERROR_STATUSES = [(ValueError, 400), (NotFound, 404), (TooLarge, 413), (Denied, 451), (InvalidDag, 422), (Busy, 503), @@ -251,6 +264,7 @@ def readinto(self, buffer) -> int: return len(data) def readable(self) -> bool: + """True: a body is read.""" return True @@ -267,6 +281,7 @@ def __init__(self, sock: socket.socket, handler: "TimedHandler"): self.handler = handler def readable(self) -> bool: + """True: the connection is read.""" return True def readinto(self, buffer) -> int: @@ -479,6 +494,85 @@ def stop(self): self.server.server_close() +class NameWatches: + """GET /api/name/watch's state, one per ApiServer: the names watched for API clients, each until WATCH_GRACE + after the last call naming it ends, and the calls waiting. Each name in until holds exactly one + NameManager.watch, so its lock is held while watching and unwatching there (taken before the names lock, + never under it: the record listener, called with the names lock held, only sets the waiting calls' events).""" + + def __init__(self, node, clock=time.monotonic): + """Watches on node's names (read only once a call comes).""" + self.node = node + self.clock = clock + self.lock = threading.Lock() + self.until: dict[bytes, float] = {} # name -> when its watch ends (each one NameManager.watch) + self.waiters: tuple = () # (names, event) of each call waiting; swapped whole, read without the lock + self.listening = False + + def wait(self, names: list[bytes], after: list[int | None], wait: float) -> list[dict]: + """The records held for names above the matching after (None: any), once there is one or after wait + seconds. Raises Busy if wait > 0 with MAX_WATCH_WAITERS calls waiting already (a wait=0 call never + waits, so it holds no connection and is always answered).""" + manager = self.node.names + event = threading.Event() + waiter = (frozenset(names), event) + with self.lock: + if wait > 0: + if len(self.waiters) >= MAX_WATCH_WAITERS: + raise Busy(f"{MAX_WATCH_WAITERS} name watches are waiting already") + if not self.listening: + manager.add_record_listener(self._heard) + self.listening = True + self.waiters = (*self.waiters, waiter) + self._extend(names, self.clock() + wait + WATCH_GRACE) + try: + deadline = self.clock() + wait + while True: + found = self._records(names, after) + remaining = deadline - self.clock() + if found or remaining <= 0: + return found + event.wait(remaining) + event.clear() + finally: + with self.lock: + self.waiters = tuple(item for item in self.waiters if item is not waiter) + self._extend(names, self.clock() + WATCH_GRACE) + + def _extend(self, names: list[bytes], until: float): + """Keep names (distinct) watched until at least until, unwatching those whose watch has ended, and the + soonest to end past MAX_WATCHED, in NameManager too. Lock held.""" + manager = self.node.names + now = self.clock() + for name in [name for name, end in self.until.items() if end <= now and name not in names]: + del self.until[name] + manager.unwatch(name) + for name in names: + if name not in self.until: + manager.watch(name) + self.until[name] = max(self.until.get(name, until), until) + while len(self.until) > MAX_WATCHED: + name = min((item for item in self.until if item not in names), key=self.until.get) + del self.until[name] + manager.unwatch(name) + + def _records(self, names: list[bytes], after: list[int | None]) -> list[dict]: + """The records held for names above the matching after.""" + found = [] + for name, sequence in zip(names, after): + record = self.node.names.record(name) + if record is not None and (sequence is None or record.sequence > sequence): + found.append({"name": name.hex(), "cid": str(record.cid), "sequence": record.sequence, + "age": max(0.0, time.time() - record.heard)}) + return found + + def _heard(self, name: bytes, record): + """NameManager's record listener (names lock held): wake the calls waiting for name.""" + for names, event in self.waiters: + if name in names: + event.set() + + class ApiServer(LocalServer): """The local API for node (an RnfsNode, or anything with its facade methods). @@ -492,22 +586,24 @@ def __init__(self, node, port: int, token: str | None = None): self.token = token if token is not None else secrets.token_urlsafe(32) if not self.token: raise ValueError("the API token must not be empty") - super().__init__(ApiHandler, port, node=node, token=self.token) + super().__init__(ApiHandler, port, node=node, token=self.token, watches=NameWatches(node)) class ApiRequestError(Exception): """A request the API refuses before reaching the node: status and message.""" def __init__(self, status: int, message: str): + """A refusal with status and message.""" super().__init__(message) self.status = status class ApiHandler(TimedHandler): - """Handles one API request using node and token, which are set on the subclass ApiServer creates.""" + """Handles one API request using node, token and watches, which are set on the subclass ApiServer creates.""" node = None token: str | None = None + watches: NameWatches | None = None server_version = "rnfs" sys_version = "" @@ -523,6 +619,7 @@ class ApiHandler(TimedHandler): ("GET", "/api/ping"): "api_ping", ("GET", "/api/name/list"): "api_name_list", ("GET", "/api/name/resolve"): "api_name_resolve", + ("GET", "/api/name/watch"): "api_name_watch", ("POST", "/api/add"): "api_add", ("POST", "/api/add/stream"): "api_add_stream", ("POST", "/api/get"): "api_get", @@ -565,9 +662,7 @@ def token_valid(self) -> bool: # Responses def send_common_headers(self): - """Headers on every API response. Deliberately no Access-Control-* headers (section 11).""" - self.send_header("Cache-Control", "no-store") - self.send_header("X-Content-Type-Options", "nosniff") + """The proof, on every response of the API's own (end_headers adds the headers every response carries).""" challenge = (self.headers.get(CHALLENGE_HEADER) or "").strip() if CHALLENGE.fullmatch(challenge): self.send_header(PROOF_HEADER, api_proof(self.token, self.server.server_address[1], challenge)) @@ -672,7 +767,11 @@ def respond_error(self, status: int, message: str, headers: dict | None = None): pass # the client has gone away def end_headers(self): - """End the headers, remembering that a response has started.""" + """Add the headers every response carries, http.server's own error pages too (400, 414, 431, 501...), + then end the headers, remembering that a response has started. Deliberately no Access-Control-* headers + (section 11).""" + self.send_header("Cache-Control", "no-store") + self.send_header("X-Content-Type-Options", "nosniff") self.headers_sent = True super().end_headers() @@ -750,21 +849,25 @@ def api_info(self): self.send_json(200, self.node.info(parse_target(self.param("target")), from_node=from_node)) def api_cat(self): - """GET /api/cat?target=&offset=&length=: stream a file's bytes, fetched as needed.""" + """GET /api/cat?target=&offset=&length=&raw=&block=&from=: stream a file's bytes, fetched as needed (block=1: + the CID's own block).""" target = parse_target(self.param("target")) offset = parse_int(self.param("offset", False), "offset", 0) length = parse_int(self.param("length", False), "length", None) if parse_bool(self.param("block", False) or ""): return self.send_bytes(self.node.block(parse_cid(target))) # cat --block: the CID's own block - raw = {"raw": True} if parse_bool(self.param("raw", False) or "") else {} # sealed files: the ciphertext - info = self.node.info(target, **raw) - if info["type"] != "file": - raise ValueError(f"{target} is a directory") - # Content-Length is known up front, so a client sees a fetch failing mid-stream as a short body - available = max(info["size"] - offset, 0) - length = available if length is None else min(length, available) - chunks = iter(self.node.read(target, offset=offset, length=length, **raw)) + options = {"raw": True} if parse_bool(self.param("raw", False) or "") else {} # sealed files: the ciphertext + from_text = self.param("from", False) + if from_text: + options["from_node"] = parse_hex_id(from_text) + # One fetch resolves the target, gives its info, then reads (only once the first chunk is asked for) + info, chunks = self.node.read_file(target, offset=offset, length=length, **options) try: + if info["type"] != "file": + raise ValueError(f"{target} is a directory") + # Content-Length is known up front, so a client sees a fetch failing mid-stream as a short body + available = max(info["size"] - offset, 0) + length = available if length is None else min(length, available) # The first chunk is fetched before the status line, so early errors still get an error status first = next(chunks, b"") self.send_response(200) @@ -818,6 +921,26 @@ def api_name_resolve(self): from rnfs.features.keepers import parse_name_arg # names as keep and follow take them self.send_json(200, self.node.name_resolve(parse_name_arg(self.node, self.param("name")).hex())) + def api_name_watch(self): + """GET /api/name/watch?names=&after=&wait=: the records of watched names newer than after (NameWatches).""" + texts = (self.param("names") or "").split(",") + if not 1 <= len(texts) <= MAX_WATCH_NAMES: + raise ValueError(f"names lists 1 to {MAX_WATCH_NAMES} names, comma-separated") + names = [parse_hex_id(text, "name") for text in texts] + after_text = self.param("after", False) + after = [None if text == "-" else parse_int(text, "after", None) + for text in (after_text.split(",") if after_text else ["-"] * len(names))] + if len(after) != len(names): + raise ValueError("after has one sequence (or -) per name") + lowest: dict[bytes, int | None] = {} # a name listed twice: above the lower of its sequences + for name, sequence in zip(names, after): + if name in lowest and (lowest[name] is None or sequence is None): + lowest[name] = None + else: + lowest[name] = sequence if name not in lowest else min(lowest[name], sequence) + wait = min(parse_int(self.param("wait", False), "wait", 0), MAX_WATCH_WAIT) + self.send_json(200, {"records": self.watches.wait(list(lowest), list(lowest.values()), wait)}) + # POST endpoints def api_add(self): diff --git a/rnfs/apiclient.py b/rnfs/apiclient.py index 302ec2e..143eabf 100644 --- a/rnfs/apiclient.py +++ b/rnfs/apiclient.py @@ -1,19 +1,8 @@ -"""A client of a running daemon's local API (specification section 11; rnfs/api.py), for companion apps and the CLI. +"""The protocol layer of a running daemon's local API (specification section 11; rnfs/api.py): what rnfs.Client +(rnfs/remote.py) and the CLI are built on. Programs use rnfs.Client, whose objects and errors are rnfs.Node's +too; ApiConnection's dict-returning methods are its backend, and the CLI's ApiClient subclasses it. -Standard library only, and it never imports RNS, so an app using it starts quickly: - - from rnfs.apiclient import Client, NotFound - - api = Client() # the daemon using ~/.rnfs/, on the api_port in its rnfs.toml - print(api.status()["node_id"]) - cid = api.add_path("/home/me/photos")["cid"] - with api.cat(f"{cid}/cat.jpg") as stream, open("cat.jpg", "wb") as out: - for chunk in stream: - out.write(chunk) - try: - print(api.name_resolve("ab" * 16)["cid"]) - except NotFound: - print("not resolved") +Standard library only, and it never imports RNS, so a program using it starts quickly. - The daemon writes a new random token to `/api_token` (mode 0600, or a private access list on Windows) at each start. The client reads it before its first request that needs it and sends it as @@ -27,10 +16,12 @@ (DaemonNotRunning), the connection failing (ConnectionFailed), the token (TokenError), the proof or the token refused (AuthFailed), the settings (SettingsError), and the API's own error statuses (STATUS_ERRORS: 400 BadRequest, 404 NotFound, 413 TooLarge, 422 InvalidDag, 451 Denied, 503 Busy, 507 StorageFull, other 5xx - ServerError). -- Each request opens its own connection, with no timeout unless one is given (fetches can take hours), so a - Client may be used from several threads. Paths are on the daemon's machine, which is this one: they are made - absolute here. + ServerError). ApiError is an rnfs.errors.RnfsError, and each status's error also a subclass of the node's own + (NotFound of rnfs.errors.NotFound, BadRequest of ValueError, ...), so one except clause catches what rnfs.Node + and rnfs.Client raise alike. +- Each request opens its own connection, with no timeout unless one is given (fetches can take hours), so an + ApiConnection may be used from several threads. Paths are on the daemon's machine, which is this one: they are + made absolute here. """ import hashlib import hmac @@ -42,7 +33,7 @@ import threading from urllib.parse import urlencode -from rnfs import fsutil, winacl +from rnfs import errors, fsutil, winacl from rnfs.config import DEFAULT_API_PORT, Config DEFAULT_DIR = Config.rnfs_dir @@ -54,7 +45,7 @@ COPY_SIZE = 64 * 1024 # what a Stream reads at a time when iterated -class ApiError(Exception): +class ApiError(errors.RnfsError): """An API call failed; str() says why. status is the HTTP status, or None if the daemon never answered.""" def __init__(self, message: str, status: int | None = None): @@ -84,31 +75,31 @@ class SettingsError(ApiError): """The rnfs dir's rnfs.toml couldn't be read for the API port.""" -class BadRequest(ApiError): +class BadRequest(ApiError, ValueError): """400: the request's parameters were refused (the node raised ValueError).""" -class NotFound(ApiError): +class NotFound(ApiError, errors.NotFound): """404: no such content, name, pin, job or endpoint.""" -class TooLarge(ApiError): +class TooLarge(ApiError, errors.TooLarge): """413: over a size limit (max_download_size, a quota, the request body).""" -class InvalidDag(ApiError): +class InvalidDag(ApiError, errors.InvalidDag): """422: the content fetched isn't a valid DAG.""" -class Denied(ApiError): +class Denied(ApiError, errors.Denied): """451: the content is on the node's deny list.""" -class Busy(ApiError): +class Busy(ApiError, errors.Busy): """503: the daemon is too busy (or full of connections); try again later.""" -class StorageFull(ApiError): +class StorageFull(ApiError, errors.StorageFull): """507: not enough free disk space.""" @@ -238,13 +229,13 @@ def _flag(value: bool) -> int | None: return 1 if value else None -class Client: +class ApiConnection: """The local API of the daemon using rnfs_dir (default ~/.rnfs/), on 127.0.0.1:port (default: the api_port in its rnfs.toml). token_file defaults to /api_token and is read on the first request needing it. request() and call() are the whole protocol; get() and post() call any endpoint (features' too); the other - methods are the main operations and the deny list, returning what the API answers (design.md, "Node - facade"). Every failure raises an ApiError subclass. + methods are the main operations, jobs, the deny list and name watching, returning what the API answers + (design.md, "Node facade"). Every failure raises an ApiError subclass. """ def __init__(self, rnfs_dir: str | None = None, port: int | None = None, token_file: str | None = None, @@ -396,11 +387,12 @@ def fetch(self, target: str, output: str, from_node: str | None = None, raw: boo raw=raw or None, **{"from": from_node}) def cat(self, target: str, offset: int | None = None, length: int | None = None, raw: bool = False, - block: bool = False) -> Stream: + block: bool = False, from_node: str | None = None) -> Stream: """A file's bytes (or a range of them) as they arrive. raw: a sealed file as stored; block: the CID's own - block, verified.""" + block, verified; from_node a node to fetch from (a daemon that predates it ignores it).""" return Stream(self.request("GET", "/api/cat", params={"target": target, "offset": offset, "length": length, - "raw": _flag(raw), "block": _flag(block)})) + "raw": _flag(raw), "block": _flag(block), + "from": from_node})) def info(self, target: str, from_node: str | None = None) -> dict: """{"cid", "type", "size", "executable"} (+ sealed-file fields) of a target.""" @@ -447,6 +439,14 @@ def name_resolve(self, name: str) -> dict: it doesn't resolve.""" return self.get("/api/name/resolve", name=name) + def name_watch(self, names: list[str], after: list[int | None], wait: int = 0) -> list[dict]: + """The records held for names (hex) whose sequence is above the matching after (None: any), once there is + one or after wait seconds (the API caps it): [{"name", "cid", "sequence", "age"}]. NotFound from a daemon + that can't watch names.""" + return self.get("/api/name/watch", names=",".join(names), + after=",".join("-" if sequence is None else str(sequence) for sequence in after), + wait=wait or None)["records"] + # The deny list def deny(self) -> dict: @@ -471,7 +471,7 @@ def submit_job(self, kind: str, target: str, output: str | None = None, from_nod return self._post("/api/jobs", kind=kind, target=target, output=output, **{"from": from_node}, **options) def job(self, job_id: str, wait: int = 0) -> dict: - """A job's record; waits up to wait seconds (the API caps it) for it to change state or progress.""" + """A job's record; waits up to wait seconds (the API caps it) for it to finish.""" return self.get("/api/jobs/get", id=job_id, wait=wait or None) def jobs(self) -> list[dict]: diff --git a/rnfs/cli.py b/rnfs/cli.py index e75d7a4..b5d4b25 100644 --- a/rnfs/cli.py +++ b/rnfs/cli.py @@ -21,11 +21,10 @@ from rnfs import __version__, apiclient, fsutil from rnfs.apiclient import CHALLENGE_HEADER, PROOF_HEADER, TOKEN_FILE, api_proof, token_path # noqa: F401 -from rnfs.config import DEFAULT_API_PORT, Config +from rnfs.config import DEFAULT_API_PORT, LOCK_FILE, Config DEFAULT_DIR = apiclient.DEFAULT_DIR COPY_SIZE = apiclient.COPY_SIZE -LOCK_FILE = "daemon.lock" # in the rnfs directory; locked by the daemon using it, so a second one can't start # text with control characters escaped, so names from other people's DAGs can't drive the terminal @@ -70,7 +69,7 @@ def read_token(path: str) -> str: fail(str(e)) -class ApiClient(apiclient.Client): +class ApiClient(apiclient.ApiConnection): """The library's client for the CLI: talks to the daemon's API on 127.0.0.1:port, exiting with a friendly message on any error. @@ -83,14 +82,14 @@ def __init__(self, port: int, token_file: str): super().__init__(port=port, token_file=token_file) def request(self, *args, **kwargs) -> http.client.HTTPResponse: - """Client.request (the successful response), exiting on errors.""" + """ApiConnection.request (the successful response), exiting on errors.""" try: return super().request(*args, **kwargs) except apiclient.ApiError as e: fail(str(e)) def call(self, *args, **kwargs): - """Client.call (the decoded JSON response), exiting on errors.""" + """ApiConnection.call (the decoded JSON response), exiting on errors.""" try: return super().call(*args, **kwargs) except apiclient.ApiError as e: @@ -138,7 +137,7 @@ def client(args) -> ApiClient: port = apiclient.api_port(args.dir) except apiclient.ApiError as e: fail(str(e)) - return ApiClient(port if port is not None else DEFAULT_API_PORT, token_path(args.dir)) + return ApiClient(port, token_path(args.dir)) def port_number(text: str, minimum: int = 1) -> int: @@ -182,7 +181,7 @@ def run_daemon(args): from rnfs.errors import RnfsError from rnfs.gateway import GatewayServer from rnfs.log import LOG_ERROR, LOG_NOTICE - from rnfs.node import RnfsNode, _make_private_dirs + from rnfs.node import DirInUse, RnfsNode, lock_rnfs_dir try: config = Config.load(args.dir, reticulum_dir=args.rnsconfig, @@ -199,12 +198,10 @@ def run_daemon(args): # would load and save the same state, announce the same node ID and rotate the first one's debug log lock_path = os.path.join(config.root, LOCK_FILE) try: - _make_private_dirs(config) - lock = fsutil.lock_exclusive(lock_path) # noqa: F841 - held until the process exits - except BlockingIOError: - holder = fsutil.lock_holder(lock_path) - fail(f"another rnfs daemon{f' (process {holder})' if holder else ''} is using {config.root}; stop it first, " - "or give this one another --dir") + lock = lock_rnfs_dir(config) # held until the process exits + except DirInUse as e: + fail(f"another rnfs daemon{f' (process {e.holder})' if e.holder else ''} is using {config.root}; stop it " + "first, or give this one another --dir") except (RnfsError, OSError) as e: fail(f"can't lock {lock_path}: {e}") @@ -213,6 +210,7 @@ def run_daemon(args): try: debug = debuglog.start(config) except (RnfsError, OSError) as e: + fsutil.unlock(lock) # fail() raises SystemExit, which an in-process caller (a test) may catch fail(f"can't write the debug log: {e}") print(f"rnfs: writing the debug log {debug.path}", flush=True) @@ -222,6 +220,7 @@ def run_daemon(args): if debuglog.current() is not None: # inside the except: with its traceback debuglog.current().write(LOG_ERROR, "rnfs", f"couldn't start the node: {e!r}", traceback.format_exc()) if RNS.Reticulum.get_instance() is None: + fsutil.unlock(lock) # as above fail(str(e)) print(f"rnfs: {printable(str(e))}", file=sys.stderr, flush=True) RNS.exit(1) # Reticulum has started: ends with os._exit, so its threads don't keep the process diff --git a/rnfs/config.py b/rnfs/config.py index 99eb08c..884e718 100644 --- a/rnfs/config.py +++ b/rnfs/config.py @@ -16,6 +16,8 @@ DEFAULT_API_PORT = 8788 DEFAULT_GATEWAY_PORT = 8789 SETTINGS_FILE = "rnfs.toml" +# In the rnfs dir, locked by the daemon or embedded node using it (node.lock_rnfs_dir), so a second can't start +LOCK_FILE = "daemon.lock" # Core settings' types and ranges, checked by Config.validate (feature settings are checked by their features) WHOLE_NUMBERS = { # name: (minimum, maximum or None). gateway_port 0: any free port (printed at start) @@ -265,9 +267,10 @@ def _validate_core_types(self): raise ValueError("reticulum_dir must be a directory path") @classmethod - def load(cls, rnfs_dir: str = "~/.rnfs/", **overrides) -> "Config": - """Settings from `/rnfs.toml` if it exists, then overrides (None values ignored), validated.""" - values: dict = {} + def load(cls, rnfs_dir: str = "~/.rnfs/", defaults: dict | None = None, **overrides) -> "Config": + """Settings from `/rnfs.toml` if it exists, then overrides (None values ignored), validated. + defaults (an embedded node's) apply where neither sets a value.""" + values: dict = dict(defaults or {}) path = os.path.join(os.path.expanduser(rnfs_dir), SETTINGS_FILE) if os.path.isfile(path): with open(path, "rb") as f: diff --git a/rnfs/constants.py b/rnfs/constants.py index 1f380b2..0269ea2 100644 --- a/rnfs/constants.py +++ b/rnfs/constants.py @@ -54,4 +54,7 @@ "bddad1ec2f4a4a6112d3d42afabfd94a", # Broken Hill, Australia "24770a7a9221b97d583905a839bc4fce", # Central US (Google Cloud) "4a8dec339ac23f9a6f423b6ee0a84c16", # Madrid, Spain (Google Cloud) + "4ac0af1cb64587d1e71103a1733e91de", # Johannesburg, South Africa (Google Cloud) + "8ce2903ad6aaee3128c6494647b5496f", # Montreal, Canada (Google Cloud) + "3c7a4a76eec780fbe56ed6e13184981c", # Stockholm, Sweden (Google Cloud) ) diff --git a/rnfs/content/__init__.py b/rnfs/content/__init__.py index 82c6de6..605d6f9 100644 --- a/rnfs/content/__init__.py +++ b/rnfs/content/__init__.py @@ -1,9 +1,12 @@ -"""The content layer: CIDs, DAG nodes, building and walking trees, and storage. No RNS dependency -beyond its bundled msgpack (and logging). +"""The content layer: CIDs, DAG nodes, chunking, building and walking trees, sealed files, and storage. No +networking. It uses RNS's bundled msgpack and RNS logging (so importing it loads RNS), and seal.py also RNS's +cryptography and identities; nothing else from RNS. cid.py CIDs (section 4) dag.py DAG node encoding, strict decoding, names (section 5) +chunker.py fixed and FastCDC chunkers (section 5) builder.py building trees: files, directories, wrap, only-hash (section 5) +seal.py sealed files (section 5, Sealed files) walker.py expectations, reader checks, path lookup, byte ranges, listing, writing out (sections 5, 7) edit.py dir set / dir rm (section 11) blockstore.py one file per CID, atomic, verified (section 8) diff --git a/rnfs/content/chunker.py b/rnfs/content/chunker.py index 57a0aaf..99960b1 100644 --- a/rnfs/content/chunker.py +++ b/rnfs/content/chunker.py @@ -99,7 +99,7 @@ def read_full(stream: BinaryIO, want: int) -> bytes: return b"".join(parts) -def _read_limited(stream: BinaryIO, want: int, remaining: int | None) -> tuple[bytes, int | None]: +def read_limited(stream: BinaryIO, want: int, remaining: int | None) -> tuple[bytes, int | None]: """Up to want bytes (within remaining, if set) and the new remaining. ValueError if a sized stream ends early.""" if remaining is not None: want = min(want, remaining) @@ -115,7 +115,7 @@ def fixed_chunks(stream: BinaryIO, size: int | None = None) -> Iterator[bytes]: """Consecutive CHUNK_SIZE pieces (the last may be shorter, never empty): the spec's tree rule.""" remaining = size while remaining is None or remaining > 0: - chunk, remaining = _read_limited(stream, constants.CHUNK_SIZE, remaining) + chunk, remaining = read_limited(stream, constants.CHUNK_SIZE, remaining) if not chunk: return yield chunk @@ -131,7 +131,7 @@ def fastcdc_stream(stream: BinaryIO, size: int | None = None) -> Iterator[bytes] eof = False while True: if not eof and len(buffer) - start < MAX_SIZE: - data, remaining = _read_limited(stream, READ_SIZE, remaining) + data, remaining = read_limited(stream, READ_SIZE, remaining) eof = len(data) < READ_SIZE or remaining == 0 buffer = buffer[start:] + data start = 0 diff --git a/rnfs/content/cid.py b/rnfs/content/cid.py index e8ace44..d9d5101 100644 --- a/rnfs/content/cid.py +++ b/rnfs/content/cid.py @@ -85,6 +85,7 @@ def key(self) -> bytes: @property def is_raw(self) -> bool: + """True for a raw block's CID.""" return self.codec == RAW def verify(self, block: bytes) -> bool: diff --git a/rnfs/content/dag.py b/rnfs/content/dag.py index d4902e1..045691c 100644 --- a/rnfs/content/dag.py +++ b/rnfs/content/dag.py @@ -9,8 +9,14 @@ Writers produce exactly what umsgpack.packb produces: str as str, bytes as bin, shortest headers. Readers accept nothing else: the decoder below understands only the few msgpack forms a node can contain, and a node must re-encode to identical bytes, which rules out non-shortest headers. + +Decoding is a pure function of the block's bytes and nodes are immutable, so decode() remembers the nodes of +the blocks it decoded last (MEMO_BLOCKS of them, MEMO_BYTES of blocks): a read walks the same file and +directory nodes many times (a cat of 64 KiB from a large file decoded its root node about twenty times). """ import struct +import threading +from collections import OrderedDict from dataclasses import dataclass from RNS.vendor import umsgpack @@ -25,6 +31,10 @@ MAX_SIZE = 2**64 - 1 # a size above this makes a DAG invalid MAX_NAME_BYTES = 255 +# decode()'s memo of nodes by block bytes, least recently used forgotten first: a node's Python objects take +# several times its block's bytes, so this holds well under 20 MB +MEMO_BLOCKS = 256 +MEMO_BYTES = 2 * 1024 * 1024 @dataclass(frozen=True) @@ -239,16 +249,58 @@ def fields(self, count: int): raise InvalidDag(f"expected an array of {count} fields, found {length}") +class _Memo: + """decode()'s memo: block bytes -> node, for blocks that decoded (an invalid one is checked every time).""" + + def __init__(self): + """Empty.""" + self._lock = threading.Lock() + self._nodes: OrderedDict[bytes, Node] = OrderedDict() # least recently used first + self._bytes = 0 + + def get(self, block: bytes) -> "Node | None": + """The node block decoded to, if remembered.""" + with self._lock: + node = self._nodes.get(block) + if node is not None: + self._nodes.move_to_end(block) + return node + + def put(self, block: bytes, node: Node): + """Remember block's node, forgetting the least recently used past MEMO_BLOCKS or MEMO_BYTES.""" + with self._lock: + if block not in self._nodes: + self._bytes += len(block) + self._nodes[block] = node + self._nodes.move_to_end(block) + while len(self._nodes) > MEMO_BLOCKS or self._bytes > MEMO_BYTES: + forgotten, _ = self._nodes.popitem(last=False) + self._bytes -= len(forgotten) + + +_memo = _Memo() + + def decode(block: bytes) -> Node: """The DAG node encoded by block, checked strictly. Raises InvalidDag for anything not exactly valid. Checks the encoding rules (types, shortest headers, no extra fields or trailing bytes), the name rules, sorted unique names, strictly increasing first_names, kinds and size overflow. Checks that need a node's context (child types, sizes against children, index bounds, depth) are walker.py's. + The node of a block decoded recently is remembered: the same bytes always give the same node. """ if not isinstance(block, (bytes, bytearray, memoryview)): raise TypeError("block must be bytes") block = bytes(block) + node = _memo.get(block) + if node is None: + node = _decode(block) + _memo.put(block, node) + return node + + +def _decode(block: bytes) -> Node: + """decode(), without the memo.""" if len(block) > MAX_BLOCK_SIZE: raise InvalidDag(f"block of {len(block)} bytes exceeds MAX_BLOCK_SIZE") reader = _Reader(block) diff --git a/rnfs/content/edit.py b/rnfs/content/edit.py index cc11525..a42d0a0 100644 --- a/rnfs/content/edit.py +++ b/rnfs/content/edit.py @@ -17,14 +17,9 @@ def _directory_chain(root: Cid, names: list[str], get: walker.Get) -> list[walke """Expectations for the root and each directory along names, in order (len(names) + 1 of them).""" chain = [walker.root_expect(root)] for i, name in enumerate(names): - expect = chain[-1] - while True: - child = walker.step(expect, get(expect), name) - if child is None: - raise NotFound(f"{root}/{'/'.join(names[:i + 1])} does not exist") - expect = child - if not walker.is_index_step(expect): - break + expect = walker.step_name(chain[-1], name, get) + if expect is None: + raise NotFound(f"{root}/{'/'.join(names[:i + 1])} does not exist") if expect.kind != dag.KIND_DIRECTORY: raise ValueError(f"{root}/{'/'.join(names[:i + 1])} is not a directory") chain.append(expect) diff --git a/rnfs/content/seal.py b/rnfs/content/seal.py index 3183d22..6710a0f 100644 --- a/rnfs/content/seal.py +++ b/rnfs/content/seal.py @@ -22,7 +22,8 @@ Readers find the segment count from the ciphertext size, so the plaintext size needs the last segment (decrypting it shows its padding). Sealing gives confidentiality only: anyone may seal a file for you, and any recipient could re-seal different content for the others, so it says nothing about who wrote it. -No RNS import at module level: crypto comes from RNS.Cryptography, loaded when first needed. +Crypto comes from RNS.Cryptography and RNS.Identity, already loaded with RNS's bundled msgpack (importing +RNS.vendor.umsgpack runs RNS/__init__). """ import hashlib import hmac @@ -33,7 +34,7 @@ from RNS.vendor import umsgpack from rnfs.constants import KIB -from rnfs.content import dag +from rnfs.content import chunker, dag MAGIC = b"rnfs-sealed\n" VERSION = 1 @@ -69,7 +70,7 @@ def is_sealed(head: bytes) -> bool: def _crypto(): - """RNS's AES and HKDF (imported here so loading this module doesn't load RNS's crypto).""" + """RNS's AES and HKDF (already loaded with RNS, which the module's umsgpack import loads).""" from RNS.Cryptography import hkdf from RNS.Cryptography.AES import AES_256_CBC return AES_256_CBC, hkdf @@ -204,21 +205,9 @@ def __init__(self, stream: BinaryIO, recipients: list[bytes], name: str | None = self._done = False def _read_segment(self) -> bytes: - """Up to SEGMENT_SIZE plaintext bytes (fewer only at the end).""" - want = SEGMENT_SIZE if self._remaining is None else min(SEGMENT_SIZE, self._remaining) - parts = [] - got = 0 - while got < want: - data = self._stream.read(want - got) - if not data: - break - parts.append(data) - got += len(data) - if self._remaining is not None: - if got < want: - raise ValueError(f"stream ended {self._remaining - got} bytes early") - self._remaining -= got - return b"".join(parts) + """Up to SEGMENT_SIZE plaintext bytes (fewer only at the end; ValueError if a sized stream ends early).""" + data, self._remaining = chunker.read_limited(self._stream, SEGMENT_SIZE, self._remaining) + return data def _produce(self): """Seal the next segment, reading one ahead to know whether it is the last.""" diff --git a/rnfs/content/shared_folder.py b/rnfs/content/shared_folder.py index e30d257..39321d7 100644 --- a/rnfs/content/shared_folder.py +++ b/rnfs/content/shared_folder.py @@ -5,19 +5,26 @@ unchanged by a signature of path, size, modification time and execute bit (recursively for directories), remembered across restarts (the state database's shared_folder table, rnfs.statedb: after a scan that changed any, the entries changed are written durably), so unchanged entries aren't hashed again. + +An entry that fails to import because of its content (an invalid name, the deny list, too large) isn't tried +again until it changes; one that fails for a reason that may pass (a full or failing disk, a pin that couldn't be +saved, a file unreadable for now) is tried again RETRY_AFTER later. A failure to pin an unchanged entry again is +logged and the scan goes on. """ import hashlib import json import os import stat import threading +import time +from collections.abc import Callable from rnfs import constants, statedb from rnfs.content import builder from rnfs.content.builder import display_path from rnfs.content.cid import Cid from rnfs.content.storage import FOLDER, Storage -from rnfs.errors import RnfsError +from rnfs.errors import RnfsError, StorageFull from rnfs.log import LOG_INFO, LOG_WARNING, log @@ -85,19 +92,25 @@ def add(item: builder.Handle | None, item_path: str, relative: str, st: os.stat_ return digest.hexdigest() +RETRY_AFTER = 600.0 # seconds before an entry whose import failed for a reason that may pass is tried again + + class SharedFolder: """Keeps the shared folder's entries imported and pinned with source "folder".""" - def __init__(self, directory: str, storage: Storage, state_dir: str | None = None): + def __init__(self, directory: str, storage: Storage, state_dir: str | None = None, + clock: Callable[[], float] = time.monotonic): """Track directory (created if missing at the first scan); the signature cache lives in state_dir's - state database (not kept without one).""" + state database (not kept without one). clock times the retries of failed imports.""" self.directory = os.path.expanduser(directory) self.storage = storage + self.clock = clock self._db = statedb.connect(os.path.expanduser(state_dir)) if state_dir else None self._lock = threading.Lock() self._known: dict[str, tuple[str, Cid]] = {} # entry name -> (signature, CID) of its last import self._saved: dict[str, tuple[str, Cid]] = {} # what the shared_folder table holds - self._failed: dict[str, str] = {} # entry name -> signature that failed to import, not retried until it changes + # Entry name -> (signature that failed to import, when to try it again: None for not until it changes) + self._failed: dict[str, tuple[str, float | None]] = {} self._warned: set[str] = set() # skipped entries already warned about try: self._load() @@ -125,9 +138,10 @@ def _load(self): tx.executemany("DELETE FROM shared_folder WHERE name = ?", bad) def names(self) -> dict[Cid, str]: - """CID → the shared-folder entry name it was imported from, for `pin ls`.""" - with self._lock: - return {cid: name for name, (_, cid) in self._known.items()} + """CID → the shared-folder entry name it was imported from, for `pin ls`. Not locked, so it never waits + for a scan's imports: after __init__, _known is only ever replaced whole, never changed in place.""" + known = self._known + return {cid: name for name, (_, cid) in known.items()} def _save(self): """Persist the entries of the signature cache that differ from what the table holds, durably (lock held).""" @@ -174,7 +188,7 @@ def scan(self) -> set[Cid]: return pins def _warn_once(self, key: str, message: str): - """Log a warning about a skipped entry, once until it changes.""" + """Log a warning about a skipped entry, once per entry name and reason while the daemon runs.""" if key not in self._warned: self._warned.add(key) log(f"shared folder: {message}", LOG_WARNING) @@ -216,9 +230,13 @@ def _import(self, name: str, handle: builder.Handle, st: os.stat_result, path: s return None known = self._known.get(name) if known is not None and known[0] == sig and self.storage.is_complete(known[1]): - self.storage.pin(known[1], FOLDER) # no-op if already pinned + try: + self.storage.pin(known[1], FOLDER) # no-op if already pinned + except OSError as e: # not saved (a full or failing disk): pinned again by the next scan + log(f"shared folder: couldn't pin {shown}: {e}", LOG_WARNING) return known - if self._failed.get(name) == sig: + failed = self._failed.get(name) + if failed is not None and failed[0] == sig and (failed[1] is None or self.clock() < failed[1]): return None warnings: list[str] = [] try: @@ -226,8 +244,12 @@ def _import(self, name: str, handle: builder.Handle, st: os.stat_result, path: s with session.writer() as put: built = builder.build_open(handle, st, path, put, warnings, skip_hard_links=True) self.storage.pin(built.cid, FOLDER) - except (OSError, ValueError, RnfsError) as e: - self._failed[name] = sig + except (OSError, StorageFull) as e: # may pass (a full or failing disk, a pin not saved): tried again later + self._failed[name] = (sig, self.clock() + RETRY_AFTER) + log(f"shared folder: couldn't import {shown}: {e}", LOG_WARNING) + return None + except (ValueError, RnfsError) as e: # the content's: not until it changes + self._failed[name] = (sig, None) log(f"shared folder: couldn't import {shown}: {e}", LOG_WARNING) return None for warning in warnings: diff --git a/rnfs/content/storage.py b/rnfs/content/storage.py index bdc8d31..1fbc23b 100644 --- a/rnfs/content/storage.py +++ b/rnfs/content/storage.py @@ -240,6 +240,7 @@ def __init__(self, blocks_dir: str, state_dir: str, max_storage: int): self._complete: dict[Cid, frozenset[Cid]] = {} # dag_blocks memo, complete DAGs only self._generation = 0 # incremented whenever blocks are deleted, invalidating walks in progress self._cache_generation = 0 # incremented whenever cache records are removed or retargeted + self._changes = 0 # incremented with each block stored, pin, release and new cache record (changes()) # Cache records changed (or removed) since the last save(); every one is among the most recently used self._cache_dirty: set[tuple[Cid, tuple[str, ...]]] = set() self._cache_used = 0 # the cache table's next used value: larger than every one saved @@ -383,6 +384,7 @@ def _put(self, cid: Cid, block: bytes, protected: bool): if cid not in self._sizes: self._sizes[cid] = len(block) self._used += len(block) + self._changes += 1 if not protected: self._maybe_garbage = True @@ -427,6 +429,7 @@ def pin_many(self, cids, source: str): added.append(cid) if not added: return + self._changes += 1 try: self._save_pins(source, added=added) except BaseException: @@ -445,6 +448,7 @@ def unpin(self, cid: Cid, source: str) -> bool: if sources is None or source not in sources: return False sources.discard(source) + self._changes += 1 if not sources: del self._pins[cid] self._maybe_garbage = True @@ -485,7 +489,9 @@ def record_fetch(self, root: Cid, names: list[str] | tuple[str, ...], target: Ci key = (root, tuple(names)) with self._lock: previous = self._cache.get(key) - if previous is not None and previous != target: + if previous is None: + self._changes += 1 + elif previous != target: self._maybe_garbage = True self._cache_generation += 1 self._cache[key] = target @@ -533,6 +539,12 @@ def cache_generation(self) -> int: with self._lock: return self._cache_generation + def changes(self) -> int: + """A count that grows with every block stored or deleted, pin or release, and cache record added, removed + or retargeted: while it is unchanged, so are pins(), cached_targets() and every dag_blocks() answer.""" + with self._lock: + return self._changes + self._generation + self._cache_generation + # Protection @contextmanager diff --git a/rnfs/content/walker.py b/rnfs/content/walker.py index e476ac9..09970c8 100644 --- a/rnfs/content/walker.py +++ b/rnfs/content/walker.py @@ -210,7 +210,8 @@ def meet(self, length: int): def parse_target(text: str) -> tuple[Cid, list[str]]: - """The root CID and path names of a target "[/name...]" (one trailing slash allowed). ValueError if malformed.""" + """The root CID and path names of a target "[/name...]" (one trailing slash allowed). ValueError if + malformed.""" if not isinstance(text, str) or not text: raise ValueError("empty target") parts = text.split("/") @@ -238,17 +239,23 @@ def size_of(expect: Expect, block: bytes) -> int: # Local operations, over a blocking get(expect) -> bytes that raises NotFound when a block is absent +def step_name(expect: Expect, name: str, get: Get) -> Expect | None: + """The entry called name in the directory expect names, stepping through its index nodes; None if there is + no such name (or expect is a file).""" + while True: + child = step(expect, get(expect), name) + if child is None or not is_index_step(child): + return child + expect = child + + def resolve(root: Cid, names: list[str], get: Get) -> Expect: """The expectation for the target /. Raises NotFound if a name doesn't exist.""" expect = root_expect(root) for i, name in enumerate(names): - while True: - child = step(expect, get(expect), name) - if child is None: - raise NotFound(f"{root}/{'/'.join(names[:i + 1])} does not exist") - expect = child - if not is_index_step(expect): - break + expect = step_name(expect, name, get) + if expect is None: + raise NotFound(f"{root}/{'/'.join(names[:i + 1])} does not exist") return expect @@ -396,13 +403,13 @@ def _count(expect: Expect, get: Get, memo: dict[Expect, int], limit: int) -> int return total -def height(expect: Expect, get: Get, limit: int | None = None) -> int: +def height(expect: Expect, get: Get) -> int: """DAG nodes on the longest path from expect down (0 for a raw block), checking every DAG node met. Depth isn't checked against MAX_DEPTH, since measuring it is the point; the walk stops once the - height is known to exceed limit (default MAX_DEPTH), returning limit + 1. + height is known to exceed MAX_DEPTH, returning MAX_DEPTH + 1. """ - limit = constants.MAX_DEPTH if limit is None else limit + limit = constants.MAX_DEPTH memo: dict[Expect, int] = {} truncated = False # a node was reached more than limit nodes down: memoised heights may be short, but it's too deep @@ -462,9 +469,9 @@ def write_out(expect: Expect, get: Get, dest: str, max_entries: int, deny list's file hashes); what was written stays for the caller to remove. Entries whose names the local filesystem would read as a path, or can't represent (it refuses to create - them), are skipped and reported, as are names that collide on this filesystem (e.g. case-insensitive ones). Kind 1 entries - get the execute bit, except on Windows, which has none. Raises TooLarge after max_entries entries, - counting every occurrence, or after max_entries small chunks (SmallLeaves). + them), are skipped and reported, as are names that collide on this filesystem (e.g. case-insensitive ones). + Kind 1 entries get the execute bit, except on Windows, which has none. Raises TooLarge after max_entries + entries, counting every occurrence, or after max_entries small chunks (SmallLeaves). """ if os.path.lexists(dest): raise ValueError(f"{dest} already exists") diff --git a/rnfs/denylist.py b/rnfs/denylist.py index 0fa29a5..f617358 100644 --- a/rnfs/denylist.py +++ b/rnfs/denylist.py @@ -14,7 +14,13 @@ read without a lock, so /blocks serving and each block a fetch stores pay one lookup. A list is untrusted input: parse_list refuses one over MAX_LIST_BYTES or MAX_LIST_ENTRIES, not UTF-8, or with a line that isn't an entry, and its source keeps the version it had. Listeners (the node) are told after every change, outside the lock. + +So a file isn't read and hashed again before each send, the SHA-256 of the files checked is remembered by CID +(at most MAX_KNOWN_HASHES, least recently used forgotten first). A CID's content never changes, so a hash +remembered never goes stale, and they are kept across restarts in the deny_hashes table: memory is what checks +read, and save() (the node's periodic saves and its stop) writes the rows changed since the last one, NORMAL. """ +import itertools import re import threading import time @@ -100,10 +106,12 @@ def denies_hash(self, digest: bytes) -> bool: class DenyList(NoDenyList): """The node's deny list: the local entries and every source's accepted list, in memory, saved in the state - database (deny_entries, deny_lists). Thread-safe; checks take no lock.""" + database (deny_entries, deny_lists), and the files' hashes remembered (deny_hashes). Thread-safe; checks take + no lock.""" def __init__(self, state_dir: str, clock: Callable[[], float] = time.time): - """Load the local entries and the sources' accepted versions (their entries come with reload()).""" + """Load the local entries, the sources' accepted versions (their entries come with reload()) and the + files' hashes remembered.""" self.clock = clock self.cids: frozenset[bytes] = frozenset() self.hashes: frozenset[bytes] = frozenset() @@ -111,7 +119,12 @@ def __init__(self, state_dir: str, clock: Callable[[], float] = time.time): self._local: dict[str, tuple[bytes | None, bytes | None]] = {} # canonical text -> (binary CID, digest) self._accepted: dict[str, Cid] = {} # source -> the version it last accepted (the deny_lists table) self._lists: dict[str, tuple[frozenset[bytes], frozenset[bytes]]] = {} # source -> its entries, read - self._known: OrderedDict[bytes, bytes] = OrderedDict() # binary file CID -> its content's SHA-256 + self._known: OrderedDict[bytes, bytes] = OrderedDict() # binary file CID -> its content's SHA-256, LRU + # Hashes remembered, used or forgotten since the last save(); every one still known is among the most + # recently used + self._known_dirty: set[bytes] = set() + self._known_used = 0 # the deny_hashes table's next used value: larger than every one saved + self._save_lock = threading.Lock() # keeps saves in order without holding _lock while writing self._listeners: list[Callable[[], None]] = [] self._db = statedb.connect(state_dir) try: @@ -121,8 +134,9 @@ def __init__(self, state_dir: str, clock: Callable[[], float] = time.time): raise def _load(self): - """Read both tables; a row that doesn't parse is logged and deleted.""" - bad_entries, bad_lists = [], [] + """Read the tables; a row that doesn't parse is logged and deleted, as are hashes past MAX_KNOWN_HASHES + (the least recently used).""" + bad_entries, bad_lists, bad_hashes = [], [], [] for (text,) in self._db.query("SELECT entry FROM deny_entries"): try: canonical, binary, digest = parse_entry(text) @@ -140,12 +154,27 @@ def _load(self): self._accepted[source] = Cid.from_bytes(binary) except (ValueError, TypeError): bad_lists.append((dumped,)) - if bad_entries or bad_lists: - log(f"deny list: dropped {plural(len(bad_entries) + len(bad_lists), 'invalid row')} from the state " - f"database", LOG_WARNING) + for binary, digest, used in self._db.query("SELECT cid, digest, used FROM deny_hashes ORDER BY used"): + try: + Cid.from_bytes(binary) + if not isinstance(digest, bytes) or len(digest) != 32 or not isinstance(used, int): + raise ValueError("not a SHA-256 digest and a position") + except ValueError: + bad_hashes.append((binary,)) + continue + self._known[binary] = digest + self._known_used = max(self._known_used, used + 1) + excess = list(itertools.islice(self._known, max(0, len(self._known) - MAX_KNOWN_HASHES))) + for binary in excess: + del self._known[binary] + bad = len(bad_entries) + len(bad_lists) + len(bad_hashes) + if bad: + log(f"deny list: dropped {plural(bad, 'invalid row')} from the state database", LOG_WARNING) + if bad or excess: with self._db.transaction() as tx: tx.executemany("DELETE FROM deny_entries WHERE entry = ?", bad_entries) tx.executemany("DELETE FROM deny_lists WHERE source = ?", bad_lists) + tx.executemany("DELETE FROM deny_hashes WHERE cid = ?", bad_hashes + [(binary,) for binary in excess]) self._rebuild() def reload(self, read: Callable[[Cid], bytes | None]): @@ -180,21 +209,23 @@ def denies_hash(self, digest: bytes) -> bool: return digest in self.hashes def known_hash(self, cid: Cid) -> bytes | None: - """The SHA-256 of file cid's content, if worked out before.""" + """The SHA-256 of file cid's content, if worked out before (here or before a restart: it never changes).""" with self._lock: digest = self._known.get(cid.binary) if digest is not None: self._known.move_to_end(cid.binary) + self._known_dirty.add(cid.binary) return digest def remember_hash(self, cid: Cid, digest: bytes): """Remember the SHA-256 of file cid's content (at most MAX_KNOWN_HASHES files, least recently used - forgotten first).""" + forgotten first); saved with the next save().""" with self._lock: self._known[cid.binary] = digest self._known.move_to_end(cid.binary) + self._known_dirty.add(cid.binary) while len(self._known) > MAX_KNOWN_HASHES: - self._known.popitem(last=False) + self._known_dirty.add(self._known.popitem(last=False)[0]) # The local list @@ -310,6 +341,30 @@ def _notify(self): except Exception as e: # noqa: BLE001 - one listener mustn't stop the others log(f"deny list: applying a change failed: {e!r}", LOG_WARNING) + def save(self): + """Persist the hashes remembered, used or forgotten since the last save, least recently used first + (NORMAL: the node's periodic saves and its stop). Raises OSError (StateDbError) if the write fails; the + rows are then saved by the next one.""" + with self._save_lock: + with self._lock: + if not self._known_dirty: + return + dirty, self._known_dirty = self._known_dirty, set() + # Each change moved its hash to the end, so those still known are the last ones, in order + kept = sum(1 for binary in dirty if binary in self._known) + changed = list(itertools.islice(reversed(self._known.items()), kept))[::-1] + rows = [(binary, digest, self._known_used + n) for n, (binary, digest) in enumerate(changed)] + self._known_used += len(rows) + gone = [(binary,) for binary in dirty if binary not in self._known] + try: + with self._db.transaction() as tx: + tx.executemany("DELETE FROM deny_hashes WHERE cid = ?", gone) + tx.executemany("INSERT OR REPLACE INTO deny_hashes (cid, digest, used) VALUES (?, ?, ?)", rows) + except BaseException: + with self._lock: + self._known_dirty |= dirty + raise + def close(self): """Release the state database (idempotent).""" self._db.close() diff --git a/rnfs/dht/__init__.py b/rnfs/dht/__init__.py index a7ef80d..e420c02 100644 --- a/rnfs/dht/__init__.py +++ b/rnfs/dht/__init__.py @@ -1,4 +1,5 @@ """The provider DHT (specification section 6): routing table, provider records and iterative lookups. -No RNS dependency: network access, identity knowledge and clocks are injected. +Network access, identity knowledge and clocks are injected, so nothing here reaches the network itself +(routing and lookups don't import RNS; provider records load it only through rnfs.log). """ diff --git a/rnfs/dht/lookup.py b/rnfs/dht/lookup.py index e82b05e..cd74cf0 100644 --- a/rnfs/dht/lookup.py +++ b/rnfs/dht/lookup.py @@ -17,8 +17,8 @@ import time from collections.abc import Callable, Iterable -from rnfs.constants import ID_LENGTH, K -from rnfs.dht.routing import distance +from rnfs.constants import K +from rnfs.dht.routing import distance, is_id # query(node_id, key) -> (provider IDs, node IDs), or None if the node didn't answer, refused, or replied # with garbage. May block for a long time (path request, link setup, RNS's request timeout). @@ -52,11 +52,6 @@ def _name(node_id: bytes) -> str: return f"<{node_id.hex()}>" -def _valid_id(node_id) -> bool: - """True if node_id has the shape of a node ID.""" - return isinstance(node_id, bytes) and len(node_id) == ID_LENGTH - - class Lookup: """One lookup for key: find_closest() for publishing, find_providers() for fetching. @@ -173,12 +168,6 @@ def remaining(self) -> float: return self.timeout return max(0.0, self.deadline - self.clock()) - def listing_count(self) -> int: - """How many nodes have answered with at least one provider (in any run), so a caller can tell - providers vouched for by several nodes from those one node listed.""" - with self._lock: - return len(self._listing) - def listing_nodes(self) -> list[bytes]: """The nodes that have answered with at least one provider (in any run).""" with self._lock: @@ -203,7 +192,7 @@ def _distance(self, node_id: bytes) -> int: def _usable(self, node_id) -> bool: """True if node_id may be queried in this lookup.""" - return (_valid_id(node_id) and node_id != self.own_id and node_id not in self._failed + return (is_id(node_id) and node_id != self.own_id and node_id not in self._failed and node_id not in self._refused_unknown and not self.exclude(node_id)) def _run(self, on_provider: NodeCallback | None, should_stop: Callable[[], bool]) -> list[bytes]: @@ -378,6 +367,6 @@ def _clean(self, reply) -> tuple[list[bytes], list[bytes]] | None: if not isinstance(providers, (tuple, list)) or not isinstance(nodes, (tuple, list)): return None # A responder sends at most K of each; ignore any excess rather than let a reply bloat the lookup - providers = list(dict.fromkeys(p for p in providers if _valid_id(p)))[:self.k] - nodes = list(dict.fromkeys(n for n in nodes if _valid_id(n)))[:self.k] + providers = list(dict.fromkeys(p for p in providers if is_id(p)))[:self.k] + nodes = list(dict.fromkeys(n for n in nodes if is_id(n)))[:self.k] return providers, nodes diff --git a/rnfs/dht/providers.py b/rnfs/dht/providers.py index e48bbdf..6b6b57b 100644 --- a/rnfs/dht/providers.py +++ b/rnfs/dht/providers.py @@ -1,4 +1,5 @@ -"""Provider records stored for other nodes (specification section 6). No RNS dependency. +"""Provider records stored for other nodes (specification section 6). No network access (RNS is loaded +only through rnfs.log, for logging). A record says "publisher provides key". Records expire PROVIDER_TTL after they were last stored and keep their first-stored time across republishes. At most max_records are kept, a tenth of them in a newcomer @@ -38,8 +39,8 @@ from collections.abc import Callable, Iterable from rnfs import statedb -from rnfs.constants import ID_LENGTH, K, PROVIDER_TTL -from rnfs.dht.routing import check_id +from rnfs.constants import K, PROVIDER_TTL +from rnfs.dht.routing import check_id, is_id from rnfs.log import LOG_ERROR, LOG_WARNING, log, plural _STATE_VERSION = 1 # of providers.json, which rnfs 0.1.4 and earlier wrote @@ -58,11 +59,6 @@ def _never(node_id: bytes) -> bool: return False -def _is_id(value) -> bool: - """True if value is a node ID or DHT key (ID_LENGTH bytes).""" - return isinstance(value, bytes) and len(value) == ID_LENGTH - - def _number(value) -> float | None: """A saved time as a float, or None if it isn't a number or is NaN (infinities are kept, as the JSON file's float() kept them: the load's clamping and TTL check deal with them).""" @@ -138,10 +134,10 @@ def import_records(value: dict, tx) -> int: log(f"providers: not importing providers.json ({e!r}); starting without its records", LOG_ERROR) return 0 tx.executemany("INSERT INTO provider_publishers (publisher, first_seen) VALUES (?, ?)", - [(p, t) for p, t in first_seen.items() if _is_id(p) and t == t]) + [(p, t) for p, t in first_seen.items() if is_id(p) and t == t]) kept: dict[tuple[bytes, bytes], tuple] = {} for key, publisher, first, last, pooled in sorted(records, key=lambda record: record[2]): - if _is_id(key) and _is_id(publisher) and first == first and last == last: + if is_id(key) and is_id(publisher) and first == first and last == last: kept.setdefault((key, publisher), (key, publisher, first, last, int(pooled))) tx.executemany("INSERT INTO provider_records (key, publisher, first_stored, last_stored, pooled)" " VALUES (?, ?, ?, ?, ?)", list(kept.values())) @@ -149,7 +145,7 @@ def import_records(value: dict, tx) -> int: # The load applied them oldest first, a later one replacing an earlier one of the same publisher tx.executemany(f"INSERT OR REPLACE INTO {table} (publisher, first_seen, at) VALUES (?, ?, ?)", [(p, seen, left) for p, seen, left in sorted(entries, key=lambda item: item[2]) - if _is_id(p) and seen == seen and left == left]) + if is_id(p) and seen == seen and left == left]) return len(kept) @@ -264,9 +260,10 @@ def store(self, key: bytes, publisher: bytes) -> bool: while it has been seen for less than `seniority`; expired records don't count; own_id is exempt). A key with max_per_key publishers evicts one at random, never one of the K/2 oldest records (those /find replies list first). Then, if the store outside the pool is full, a publisher seen before the - newest one there evicts a record of that newest publisher; any other goes into the pool, evicting a - random pooled record if it is full (refused if that record is its own, so it can't churn its own - records). So new identities can't displace older publishers, and a new publisher always gets in. + newest one there evicts a record of that newest publisher; any other goes into the pool. A full pool + evicts the record pooled first by one of the publishers holding the most pooled records, and refuses a + publisher already holding that many (so it can't churn its own records). So new identities can't + displace older publishers, and a new publisher always gets in. """ check_id(key) check_id(publisher) @@ -805,14 +802,14 @@ def _load(self): first_seen = {} for publisher, seen in publisher_rows: seen = _number(seen) - if seen is None or not _is_id(publisher): + if seen is None or not is_id(publisher): malformed += 1 continue first_seen[publisher] = min(seen, now) records = [] for key, publisher, first, last, pooled in record_rows: first, last = _number(first), _number(last) - if first is None or last is None or not _is_id(key) or not isinstance(pooled, int): + if first is None or last is None or not is_id(key) or not isinstance(pooled, int): malformed += 1 continue records.append((key, publisher, min(first, now), min(last, now), bool(pooled))) @@ -848,7 +845,7 @@ def _load(self): for rows, entries in ((departed_rows, departed), (evicted_rows, evicted)): for publisher, seen, left in rows: seen, left = _number(seen), _number(left) - if seen is None or left is None or not _is_id(publisher): + if seen is None or left is None or not is_id(publisher): malformed += 1 continue entries.append((publisher, seen, left)) diff --git a/rnfs/dht/routing.py b/rnfs/dht/routing.py index 14dabdf..1a34247 100644 --- a/rnfs/dht/routing.py +++ b/rnfs/dht/routing.py @@ -28,9 +28,14 @@ def distance(a: bytes, b: bytes) -> int: return int.from_bytes(a, "big") ^ int.from_bytes(b, "big") +def is_id(value) -> bool: + """True if value has the shape of a node ID or DHT key (ID_LENGTH bytes).""" + return isinstance(value, bytes) and len(value) == ID_LENGTH + + def check_id(node_id: bytes): """Raise ValueError unless node_id is an ID_LENGTH-byte bytes (node IDs and DHT keys).""" - if not isinstance(node_id, bytes) or len(node_id) != ID_LENGTH: + if not is_id(node_id): raise ValueError(f"node IDs and keys must be {ID_LENGTH} bytes") @@ -51,7 +56,8 @@ class RoutingTable: """BUCKET_COUNT buckets of up to K servers each, by distance from own_id. Thread-safe. `clock` is monotonic seconds (nothing here is persisted). `admit(node_id)`, if given, must be True - for a node to be added: private mode passes a member check (section 3). + for a node to be added: private mode passes a member check (section 3). It is asked without the table's + lock held, since it may block (on RNS). """ def __init__(self, own_id: bytes, config: Config, clock: Callable[[], float] = time.monotonic, @@ -99,44 +105,59 @@ def heard(self, node_id: bytes) -> bool: True if it was newly added; False if it was already in the table (now refreshed) or wasn't added (our own ID, banned, not admitted, or its bucket is full of healthy nodes). """ + return self._hear(node_id, False) + + def answered(self, node_id: bytes) -> bool: + """Record that node_id answered a query: heard(), and it is reliable again. True if newly added.""" + return self._hear(node_id, True) + + def _hear(self, node_id: bytes, answered: bool) -> bool: + """heard(), and for answered() also forget node_id's failed queries, in the same locked step. + + admit is asked outside the lock (it may block on RNS), and only for a node found not in the table and + not banned: the table is then checked again, so the outcome is as if admit had been asked under it. + """ check_id(node_id) if node_id == self.own_id: # hearing our own announce is normal return False - with self._lock: - now = self.clock() - self._prune(now) - if node_id in self._bans: - return False - bucket = self._buckets[self.bucket_index(node_id)] - entry = bucket.get(node_id) - if entry is not None: - entry.last_heard = now - entry.failures = 0 - return False - if self.admit is not None and not self.admit(node_id): - return False - - if len(bucket) >= K: - # Only the least recently heard node can give up its place, and only if it has stopped - # answering or gone silent. Healthy long-lived nodes are never displaced by newcomers. - victim = min(bucket.values(), key=lambda e: e.last_heard) - if not self._stale(victim, now): + admitted = True if self.admit is None else None # None: not asked yet + while True: + with self._lock: + if answered: + self._unanswered.pop(node_id, None) + now = self.clock() + self._prune(now) + if node_id in self._bans: return False - del bucket[victim.node_id] - # Displaced, not dropped for failures: it may come back with its seniority like a dropped node - self._dropped[victim.node_id] = (victim.first_seen, now) - - # A node dropped recently (e.g. by a lossy link) regains its seniority - first_seen, _ = self._dropped.pop(node_id, (now, now)) - bucket[node_id] = _Entry(node_id, first_seen, now) - return True + bucket = self._buckets[self.bucket_index(node_id)] + entry = bucket.get(node_id) + if entry is not None: + entry.last_heard = now + entry.failures = 0 + return False + if admitted: + return self._add(bucket, node_id, now) + if admitted is False: + return False + admitted = bool(self.admit(node_id)) + + def _add(self, bucket: dict[bytes, _Entry], node_id: bytes, now: float) -> bool: + """Add node_id, not in the table, to its bucket unless that is full of healthy nodes. True if added. + Caller holds the lock.""" + if len(bucket) >= K: + # Only the least recently heard node can give up its place, and only if it has stopped + # answering or gone silent. Healthy long-lived nodes are never displaced by newcomers. + victim = min(bucket.values(), key=lambda e: e.last_heard) + if not self._stale(victim, now): + return False + del bucket[victim.node_id] + # Displaced, not dropped for failures: it may come back with its seniority like a dropped node + self._dropped[victim.node_id] = (victim.first_seen, now) - def answered(self, node_id: bytes) -> bool: - """Record that node_id answered a query: heard(), and it is reliable again. True if newly added.""" - check_id(node_id) - with self._lock: - self._unanswered.pop(node_id, None) - return self.heard(node_id) + # A node dropped recently (e.g. by a lossy link) regains its seniority + first_seen, _ = self._dropped.pop(node_id, (now, now)) + bucket[node_id] = _Entry(node_id, first_seen, now) + return True def failed(self, node_id: bytes) -> bool: """Count a query node_id didn't answer; after max_failures in a row it is dropped. True if dropped. @@ -308,6 +329,6 @@ def live_count(self) -> int: def _find(self, node_id: bytes) -> _Entry | None: """The entry for node_id, or None (also for malformed IDs and our own). Caller holds the lock.""" - if not isinstance(node_id, bytes) or len(node_id) != ID_LENGTH or node_id == self.own_id: + if not is_id(node_id) or node_id == self.own_id: return None return self._buckets[self.bucket_index(node_id)].get(node_id) diff --git a/rnfs/embedded.py b/rnfs/embedded.py new file mode 100644 index 0000000..ef71ed4 --- /dev/null +++ b/rnfs/embedded.py @@ -0,0 +1,295 @@ +"""rnfs.Node: an rnfs node inside the program's own process, as RNS runs inside it (design.md, "Public API"; +provisional until 0.2). Importing this module loads RNS; `import rnfs` loads it only when rnfs.Node is used. + +It is an RnfsNode (rnfs/node.py) started embedded, in its own rnfs dir, behind rnfs.interface's methods: +- Reticulum is the program's running instance (or one the node starts, from the reticulum_dir setting, if none + runs); the node never stops it. +- The guards and the compression rule cover only its rnfs.dht links (network/guards.py, DHT); the program's + own links behave as with RNS alone. +- Where neither rnfs.toml nor settings say otherwise, it runs as a DHT client with LXMF off (EMBEDDED_DEFAULTS). + It serves no API or gateway, writes no debug log and installs no signal handlers of its own (a Reticulum the + node starts installs RNS's, which needs the main thread: start Reticulum first to run a Node off it). +- If starting fails, nothing of it is left in the process (RnfsNode releases its rnfs.dht destination), so the + program may try again. +- It locks the rnfs dir as the daemon does (node.lock_rnfs_dir), so a daemon or another node can't share it, + and only one Node runs in a process at a time. + +Name records are heard through NameManager's record listener, which only queues them for the event thread; jobs +are the jobs feature's, polled for progress. +""" +import io +import os +import threading +import time +from dataclasses import fields + +import RNS + +from rnfs import fetcher, fsutil, interface +from rnfs.config import Config +from rnfs.errors import RnfsError +from rnfs.log import LOG_ERROR, log +from rnfs.node import RnfsNode, lock_rnfs_dir +from rnfs.objects import Record + +# An embedded node's settings where neither rnfs.toml nor the program's settings give them: an app's node is +# often short-lived, and short-lived nodes should be DHT clients (section 6); the program has its own LXMF +EMBEDDED_DEFAULTS = {"dht_mode": "client", "lxmf_enabled": False} + +_process_lock = threading.Lock() +_running: "Node | None" = None # the Node running in this process: one at a time, as RNS.Reticulum is one + + +def _bytes(node_id: str | None) -> bytes | None: + """A hex node ID as bytes (None stays None).""" + return bytes.fromhex(node_id) if node_id is not None else None + + +class Node(interface.Rnfs): + """An rnfs node running inside this program, in rnfs_dir (its identity, blocks and state; created private). + + reticulum is this process's running RNS.Reticulum; None uses the running instance, or starts one from the + reticulum_dir setting if none runs. settings are Config fields over the rnfs dir's rnfs.toml (api_port, + gateway_port and debug_log, the daemon's, are unused). Starts at once: state loaded, threads running, the + DHT joined in the background (wait_joined()). Raises RnfsError (the dir is in use, or not private; a Node + already runs here), ValueError (settings), OSError. + """ + + job_poll = fetcher.POLL # a fetch's progress changes no faster + + def __init__(self, rnfs_dir, *, reticulum: "RNS.Reticulum | None" = None, settings: dict | None = None): + """Start the node (see the class).""" + global _running + if reticulum is not None and reticulum is not RNS.Reticulum.get_instance(): + raise ValueError("reticulum must be this process's running RNS.Reticulum (RNS.Reticulum.get_instance())") + settings = dict(settings or {}) + unknown = settings.keys() - ({f.name for f in fields(Config)} - {"rnfs_dir"}) + if unknown: + raise ValueError(f"unknown settings: {', '.join(sorted(unknown))}") + config = Config.load(os.fspath(rnfs_dir), defaults=EMBEDDED_DEFAULTS, **settings) + with _process_lock: + if _running is not None: + raise RnfsError("an rnfs.Node is already running in this process; close it first") + _running = self + try: + self._lock_fd = lock_rnfs_dir(config) + try: + node = RnfsNode(config, reticulum=reticulum, embedded=True) + except BaseException: + fsutil.unlock(self._lock_fd) + raise + except BaseException: + with _process_lock: + _running = None + raise + self._node = node + super().__init__(_Backend(node)) + node.names.add_record_listener(self._heard) + + @property + def internal(self) -> RnfsNode: + """The RnfsNode underneath. Unsupported: it changes with any release.""" + return self._node + + def _close(self): + """Stop the node (saving its state), release the rnfs dir; Reticulum keeps running.""" + global _running + node = self._node + node.names.remove_record_listener(self._heard) + try: + node.stop() + finally: + fsutil.unlock(self._lock_fd) + with _process_lock: + if _running is self: + _running = None + + def _report(self, message: str): + """Log through rnfs's log.""" + log(message, LOG_ERROR) + + # Name watches + + def _heard(self, name: bytes, record): + """NameManager's record listener (names lock held, any thread): queue the record for the watches.""" + self._record_heard(name, Record(name, record.cid, record.sequence, max(0.0, time.time() - record.heard))) + + def _watch_started(self, name: bytes): + """Have NameManager treat name as watched.""" + self._node.names.watch(name) + + def _unwatch_name(self, name: bytes): + """Undo _watch_started.""" + self._node.names.unwatch(name) + + def _baseline(self, name: bytes) -> int | None: + """The sequence of name's record held now.""" + record = self._node.names.record(name) + return record.sequence if record is not None else None + + +class _Backend: + """rnfs.interface's backend for an RnfsNode: its facade, taking and returning the API's forms.""" + + def __init__(self, node: RnfsNode): + """The backend of node.""" + self.node = node + + def status(self) -> dict: + """The node's status.""" + return self.node.status() + + def node_id(self) -> str: + """The node ID in hex.""" + return self.node.node_id.hex() + + def wait_joined(self, timeout: float | None) -> bool: + """Wait for the first join to finish.""" + return self.node.joined.wait(timeout) + + def add_path(self, path: str, **options) -> dict: + """RnfsNode.add_path.""" + return self.node.add_path(path, **options) + + def add_stream(self, data, length: int | None, name: str | None, **options) -> dict: + """RnfsNode.add_stream of bytes or a binary file.""" + stream = io.BytesIO(data) if isinstance(data, bytes) else data + return self.node.add_stream(stream, size=length, name=name, **options) + + def get(self, target: str, output: str, from_node: str | None, raw: bool) -> dict: + """RnfsNode.get.""" + return self.node.get(target, output, from_node=_bytes(from_node), raw=raw) + + def read(self, target: str, offset: int, length: int | None, raw: bool, from_node: str | None): + """RnfsNode.read: a generator, fetching as it is read.""" + return self.node.read(target, offset=offset, length=length, raw=raw, from_node=_bytes(from_node)) + + def block(self, cid: str) -> bytes: + """RnfsNode.block.""" + return self.node.block(cid) + + def info(self, target: str, from_node: str | None, raw: bool) -> dict: + """RnfsNode.info.""" + return self.node.info(target, from_node=_bytes(from_node), raw=raw) + + def ls(self, target: str) -> list[dict]: + """RnfsNode.ls.""" + return self.node.ls(target) + + def stat(self, target: str) -> dict: + """RnfsNode.stat.""" + return self.node.stat(target) + + def dir_set(self, target: str, name: str, child: str, executable: bool) -> dict: + """RnfsNode.dir_set.""" + return self.node.dir_set(target, name, child, executable=executable) + + def dir_rm(self, target: str) -> dict: + """RnfsNode.dir_rm, which needs a path to remove.""" + if "/" not in target.rstrip("/"): + raise ValueError("dir_rm needs /: the entry to remove") + return self.node.dir_rm(target) + + def pin_add(self, target: str) -> dict: + """RnfsNode.pin_add.""" + return self.node.pin_add(target) + + def pin_rm(self, cid: str): + """RnfsNode.pin_rm.""" + self.node.pin_rm(cid) + + def pin_ls(self, cid: str | None) -> list[dict]: + """RnfsNode.pin_ls.""" + return self.node.pin_ls(cid) + + def pin_remote(self, node_id: str, cid: str, pin: bool) -> bool: + """RnfsNode.pin_remote.""" + return self.node.pin_remote(_bytes(node_id), cid, pin) + + def providers(self, cid: str) -> list[str]: + """RnfsNode.providers, in hex.""" + return [node_id.hex() for node_id in self.node.providers(cid)] + + def ping(self, node_id: str) -> float | None: + """RnfsNode.ping.""" + return self.node.ping(_bytes(node_id)) + + def export(self, target: str, output: str) -> dict: + """RnfsNode.export.""" + return self.node.export(target, output) + + def import_archive(self, path: str) -> dict: + """RnfsNode.import_archive.""" + return self.node.import_archive(path) + + def gc(self) -> dict: + """RnfsNode.gc.""" + return self.node.gc() + + def verify(self) -> dict: + """RnfsNode.verify.""" + return self.node.verify() + + def name_create(self, alias: str) -> dict: + """RnfsNode.name_create.""" + return self.node.name_create(alias) + + def name_import(self, alias: str, key: bytes) -> dict: + """RnfsNode.name_import (key any bytes-like object, as a Client sends it).""" + return self.node.name_import(alias, bytes(key)) + + def name_rm(self, alias: str): + """RnfsNode.name_rm.""" + self.node.name_rm(alias) + + def name_export(self, alias: str) -> bytes: + """RnfsNode.name_export.""" + return bytes(self.node.name_export(alias)) + + def name_list(self) -> list[dict]: + """RnfsNode.name_list.""" + return self.node.name_list() + + def name_publish(self, alias: str, target: str) -> dict: + """RnfsNode.name_publish.""" + return self.node.name_publish(alias, target) + + def name_resolve(self, name: str) -> dict: + """RnfsNode.name_resolve of a name or a name contact's nick, as GET /api/name/resolve takes them.""" + from rnfs.features.keepers import parse_name_arg + return self.node.name_resolve(parse_name_arg(self.node, name).hex()) + + def submit_job(self, kind: str, target: str, output: str | None, from_node: str | None, **flags) -> dict: + """JobsFeature.submit.""" + return self.node.feature("jobs").submit(kind, target, output, from_node, **flags) + + def job(self, job_id: str, wait: float) -> dict: + """JobsFeature.get, waiting up to wait seconds for the job to finish.""" + from rnfs.features.jobs import check_job_id + return self.node.feature("jobs").get(check_job_id(job_id), wait) + + def jobs(self) -> list[dict]: + """JobsFeature.list.""" + return self.node.feature("jobs").list() + + def cancel_job(self, job_id: str) -> dict: + """JobsFeature.cancel.""" + from rnfs.features.jobs import check_job_id + return self.node.feature("jobs").cancel(check_job_id(job_id)) + + def clear_jobs(self, job_id: str | None) -> int: + """JobsFeature.clear.""" + from rnfs.features.jobs import check_job_id + return self.node.feature("jobs").clear(check_job_id(job_id) if job_id is not None else None) + + def deny(self) -> dict: + """The local deny list and its sources, as GET /api/deny answers.""" + return {"entries": self.node.deny.entries(), "lists": self.node.feature("deny").lists()} + + def deny_add(self, entries: list[str]) -> list[str]: + """DenyList.add.""" + return self.node.deny.add(entries) + + def deny_rm(self, entries: list[str]) -> list[str]: + """DenyList.remove.""" + return self.node.deny.remove(entries) diff --git a/rnfs/errors.py b/rnfs/errors.py index 481dfeb..ffcedf6 100644 --- a/rnfs/errors.py +++ b/rnfs/errors.py @@ -1,4 +1,6 @@ -"""Errors shared across rnfs. The API maps them to HTTP statuses (see design.md).""" +"""Errors shared across rnfs, and raised by its public API (rnfs.Node, rnfs.Client), which exports each as +rnfs.. The API maps them to HTTP statuses (see design.md); rnfs.apiclient's errors for those statuses are +subclasses of these, so one except clause catches either class's.""" class RnfsError(Exception): @@ -17,11 +19,12 @@ class NotFound(RnfsError): class TooLarge(RnfsError): - """A fetch would exceed max_download_size, or writing out a DAG exceeded max_output_entries.""" + """A fetch or read would exceed a bound: max_download_size, max_output_entries, the DAG byte or byte-range + bound (section 7), a remote pin's or auto-fetch's quota, or the free space an output needs.""" class StorageFull(RnfsError): - """Pins alone fill max_storage, so a fetch that isn't being pinned is refused (section 8).""" + """Storing more would leave less than the disk reserve free (section 8; max_storage itself never refuses).""" class Busy(RnfsError): @@ -30,3 +33,11 @@ class Busy(RnfsError): class Denied(RnfsError): """The content is on this node's deny list (section 8): it isn't fetched, read, stored, served or provided.""" + + +class Cancelled(RnfsError): + """A fetch stopped because its observer asked it to (a job cancelled, or the node stopping).""" + + +class Closed(RnfsError): + """The rnfs.Node or rnfs.Client was closed.""" diff --git a/rnfs/features/__init__.py b/rnfs/features/__init__.py index 0fe9422..4ab6a4a 100644 --- a/rnfs/features/__init__.py +++ b/rnfs/features/__init__.py @@ -55,6 +55,12 @@ def requests(self) -> dict[str, Callable]: The request server registers them at startup and applies private mode and the per-link request limit; a handler must strictly parse data (untrusted), never return None (RNS would send nothing), and return False for anything malformed or refused. An exception answers False. + + The response must have a shape every requester's `protocol.response_refusal` accepts (nodes since 0.1.4 + check it before decoding): True, False or None, a /find answer (two lists of at most K IDs), or a list of + at most MAX_BLOCKS_CIDS entries, each bytes of at most MAX_BLOCK_SIZE or None (a /blocks answer). Any + other shape (a dict, a string, a longer list) is refused as malformed, and widening that check wouldn't + reach the nodes already running. """ return {} diff --git a/rnfs/features/_common.py b/rnfs/features/_common.py index e866b3b..3cdb5f8 100644 --- a/rnfs/features/_common.py +++ b/rnfs/features/_common.py @@ -4,6 +4,8 @@ # RNS's loglevels (RNS.LOG_ERROR ...), defined here so feature modules can name them without loading RNS LOG_ERROR, LOG_WARNING, LOG_NOTICE, LOG_INFO, LOG_VERBOSE = 1, 2, 3, 4, 5 +RETRY_INTERVAL = 300.0 # seconds before a failed check or fetch (follow, deny) is first retried; doubles per failure + def log(message: str, level: int = LOG_VERBOSE): """Log through rnfs.log (imported here: it loads RNS).""" @@ -11,6 +13,12 @@ def log(message: str, level: int = LOG_VERBOSE): rnfs_log.log(message, level) +def retry_delay(failures: int, announce_interval: float) -> float: + """Seconds before the next check after failures failed checks in a row: RETRY_INTERVAL doubling, at most + announce_interval (and never less than RETRY_INTERVAL, even with a shorter announce_interval).""" + return max(RETRY_INTERVAL, min(RETRY_INTERVAL * 2 ** min(failures - 1, 32), announce_interval)) + + def feature(handler, name: str, missing: str | None = None): """The feature called name of the handler's node. NotFound (with missing, if given) if the daemon has none.""" try: diff --git a/rnfs/features/contacts.py b/rnfs/features/contacts.py index 64536ad..0d6caf4 100644 --- a/rnfs/features/contacts.py +++ b/rnfs/features/contacts.py @@ -103,7 +103,7 @@ def __init__(self, node): self._lock = threading.Lock() self._contacts: dict[str, tuple[str, bytes]] = {} # nick -> (kind, hash) self._nicks: dict[tuple[str, bytes], int] = {} # (kind, hash) -> how many nicks name it, for has() - self._dirty: set[str] = set() # nicks changed (or deleted) whose save failed, written with the next + self._dirty: set[str] = set() # nicks changed (or deleted) whose save failed, written with the next change # name -> (CID, monotonic time of the resolve that found it), reused for EXPAND_AGE self._expanded: dict[bytes, tuple[str, float]] = {} self._db = statedb.connect(node.config.state_dir) diff --git a/rnfs/features/deny.py b/rnfs/features/deny.py index 61c2f1e..877e9c1 100644 --- a/rnfs/features/deny.py +++ b/rnfs/features/deny.py @@ -23,13 +23,12 @@ from rnfs.content.cid import Cid from rnfs.errors import RnfsError from rnfs.features import Feature, _common -from rnfs.features._common import LOG_ERROR, LOG_NOTICE, LOG_VERBOSE, LOG_WARNING +from rnfs.features._common import LOG_ERROR, LOG_NOTICE, LOG_VERBOSE, LOG_WARNING, RETRY_INTERVAL, retry_delay from rnfs.fsutil import printable NAME = "deny" NAME_PREFIX = "name/" CHECK_INTERVAL = 600.0 # seconds between checks of a name source's record held (no network) -RETRY_INTERVAL = 300.0 # seconds before a failed check is first tried again; doubles per failure STALE_ANNOUNCES = 2 # a record not heard for this many announce_intervals is refreshed from the network FAR_FUTURE = float("inf") @@ -66,12 +65,6 @@ def validate_settings(config): parse_source(text) -def retry_delay(failures: int, announce_interval: float) -> float: - """Seconds before the next check after failures failed checks in a row: RETRY_INTERVAL doubling, at most - announce_interval (and never less than RETRY_INTERVAL).""" - return max(RETRY_INTERVAL, min(RETRY_INTERVAL * 2 ** min(failures - 1, 32), announce_interval)) - - def _log(message: str, level: int = LOG_VERBOSE): """Log with this feature's prefix.""" _common.log(f"deny list: {message}", level) diff --git a/rnfs/features/follow.py b/rnfs/features/follow.py index 8c444a3..47725f6 100644 --- a/rnfs/features/follow.py +++ b/rnfs/features/follow.py @@ -26,23 +26,16 @@ from rnfs.content.cid import Cid from rnfs.errors import NotFound, RnfsError from rnfs.features import Feature, _common -from rnfs.features._common import LOG_ERROR, LOG_NOTICE, LOG_VERBOSE, LOG_WARNING +from rnfs.features._common import LOG_ERROR, LOG_NOTICE, LOG_VERBOSE, LOG_WARNING, RETRY_INTERVAL, retry_delay from rnfs.features.keepers import FOLLOW, name_subcommands, parse_name_arg from rnfs.fsutil import printable NAME = "follow" SOURCE_PREFIX = "follow:" -RETRY_INTERVAL = 300.0 # seconds before a failed check or fetch is first tried again; doubles per failure FAR_FUTURE = float("inf") STALE_ANNOUNCES = 2 # a record not heard for this many announce_intervals is refreshed from the network -def retry_delay(failures: int, announce_interval: float) -> float: - """Seconds before the next check after failures failed checks in a row: RETRY_INTERVAL doubling, at most - announce_interval (and never less than RETRY_INTERVAL, even with a shorter announce_interval).""" - return max(RETRY_INTERVAL, min(RETRY_INTERVAL * 2 ** min(failures - 1, 32), announce_interval)) - - def follow_source(name: bytes) -> str: """The pin source for a followed name.""" return SOURCE_PREFIX + name.hex() diff --git a/rnfs/features/jobs.py b/rnfs/features/jobs.py index 09b7849..fe344d8 100644 --- a/rnfs/features/jobs.py +++ b/rnfs/features/jobs.py @@ -331,8 +331,6 @@ def _load(self): _log(f"ignoring a saved job ({e})", LOG_WARNING) bad.append((job_id,)) continue - if job.id in self._jobs: - continue self._saved[job.id] = (position, text) if job.state in ACTIVE: job.state = QUEUED # interrupted: it resumes once the node runs @@ -399,7 +397,7 @@ def _cap_finished(self): Jobs finished less than FINISHED_GRACE ago stay regardless: the CLI following a job (or a script running `rnfs jobs wait`) asks for it after it ends, and a small cap mustn't make it vanish first. """ - limit = getattr(self.node.config, "max_finished_jobs", 100) + limit = self.node.config.max_finished_jobs finished = [job for job in self._jobs.values() if job.state in FINISHED] finished.sort(key=lambda job: job.finished or job.created) recent = time.time() - FINISHED_GRACE diff --git a/rnfs/features/keepers.py b/rnfs/features/keepers.py index b54e612..db44971 100644 --- a/rnfs/features/keepers.py +++ b/rnfs/features/keepers.py @@ -428,7 +428,6 @@ class KeepersFeature(Feature): def __init__(self, node): """Load kept names, hook into names and the publisher; no network.""" super().__init__(node) - self.store = KeeperStore(node.config.state_dir, node.config.keep_max_names) links = getattr(node, "links", None) # Offline test nodes have no LinkPool: their fake client may answer name_record itself self.client = KeeperClient(links) if links is not None else node.client @@ -442,15 +441,20 @@ def __init__(self, node): # names without a record every one is asked in turn self._retry_cursor: bytes | None = None self._rng = random.Random() # breaks ties between keepers of equal standing - node.names.keepers = self - node.publisher.add_key_source(self.store.provided_keys) - # Kept records are the floor for resolving: NameManager persists only names resolved by the user, - # so after a restart it would otherwise accept an older replayed announce of a kept name - for name in self.store.names(): - kept = self.local_record(name) - if kept is not None: - node.names.offer(name, *kept) - self.sync_own() + self.store = KeeperStore(node.config.state_dir, node.config.keep_max_names) + try: + node.names.keepers = self + node.publisher.add_key_source(self.store.provided_keys) + # Kept records are the floor for resolving: NameManager persists only names resolved by the user, + # so after a restart it would otherwise accept an older replayed announce of a kept name + for name in self.store.names(): + kept = self.local_record(name) + if kept is not None: + node.names.offer(name, *kept) + self.sync_own() + except BaseException: + self.store.close() + raise # Lifecycle @@ -707,12 +711,13 @@ def ask_keepers(self, name: bytes) -> list[SignedRecord]: sources listed some (this node counts as one if an established publisher stored a record here), or the lookup is complete. Candidates are ranked by standing then at random (rank_keepers), and each batch takes first, in that order, those listed by a source no earlier pick was listed by (this node, an - established listing node, or, as one source, nobody established), then the rest in rank order. Unknown keepers (a path request each) get a budget of - alpha of their own, apart from the lookup's. Another batch is asked, up to keeper_rounds() in all, - while no answer reaches the highest sequence already known for the name (from an announce, a resolve - or a kept record), or fewer than two keepers answered (when no sequence is known, two established - keepers; on a slow node a second batch is asked for this only when nothing is known to compare - against). Each verifying record is returned; the caller keeps the highest sequence. + established listing node, or, as one source, nobody established), then the rest in rank order. Unknown + keepers (a path request each) get a budget of alpha of their own, apart from the lookup's. Another batch + is asked, up to keeper_rounds() in all, while no answer reaches the highest sequence already known for + the name (from an announce, a resolve or a kept record), or fewer than two keepers answered (when no + sequence is known, two established keepers; on a slow node a second batch is asked for this only when + nothing is known to compare against). Each verifying record is returned; the caller keeps the highest + sequence. """ node = self.node key = name_key(name) diff --git a/rnfs/features/lxmf.py b/rnfs/features/lxmf.py index 686c943..57e8c1b 100644 --- a/rnfs/features/lxmf.py +++ b/rnfs/features/lxmf.py @@ -192,12 +192,6 @@ class Link: size: int | None = None # content bytes, as the sender knew them (untrusted; fetches verify sizes) from_node: bytes | None = None # the sender's rnfs node ID: a provider hint, like `get --from` - def target(self) -> str: - """`[/path]` for the node's facade; ValueError for a name link (resolve it first).""" - if self.cid is None: - raise ValueError("a name link has no CID until its name is resolved") - return "/".join([str(self.cid), *self.path]) - def uri(self) -> str: """The canonical rnfs: URI.""" return format_uri(self) @@ -611,11 +605,6 @@ def find(self, key: str, value) -> dict | None: return dict(entry) return None - def count(self, key: str, value) -> int: - """Number of entries whose key equals value.""" - with self._lock: - return sum(1 for entry in self._entries if entry.get(key) == value) - def update(self, entry_id: int, **values) -> bool: """Set fields of an entry; False if it no longer exists.""" with self._lock: @@ -833,12 +822,17 @@ def __init__(self, node): except BaseException: self.inbox.close() raise - for entry in self.outbox.entries(): - if entry.get("state") not in FINAL_STATES and not self._propagated_sent(entry): - # LXMF doesn't persist its outbound queue - self.outbox.update(entry["id"], state="failed", error="the daemon stopped before it was delivered") - self._lock = threading.RLock() # inbox removals and auto-fetch pins, so a pin is never left behind - self._release_unrecorded() + try: + for entry in self.outbox.entries(): + if entry.get("state") not in FINAL_STATES and not self._propagated_sent(entry): + # LXMF doesn't persist its outbound queue + self.outbox.update(entry["id"], state="failed", + error="the daemon stopped before it was delivered") + self._lock = threading.RLock() # inbox removals and auto-fetch pins, so a pin is never left behind + self._release_unrecorded() + except BaseException: + self.close() + raise self._resend: dict[str, tuple] = {} # direct message hash -> (address, content, fields, nearest) self._told_no_pn = False # logged once how to set a propagation node self._auto_queue = _RoundRobin() # (entry id, link index) by sender diff --git a/rnfs/fetcher.py b/rnfs/fetcher.py index ad2839e..59688b2 100644 --- a/rnfs/fetcher.py +++ b/rnfs/fetcher.py @@ -31,27 +31,26 @@ served response aren't counted towards max_failures (the estimates at the two ends differ, and a responder may set a lower cap): they are retried every eighth of that time, or BUSY_DELAY if longer. -Limits (sections 7 and 8): max_download_size bounds both the target's verified size and the DAG node bytes -a fetch downloads (a DAG of empty files has size 0 but can hold gigabytes of directory nodes); listings and -write-outs stop at max_output_entries entries met while fetching. fetch_dag also bounds the bytes of the -DAG's distinct blocks by what its declared size can hold (dag_bound: the size, DAG_GRANULE_OVERHEAD per -DAG_GRANULE of it and DAG_SLACK), plus what its directory and index nodes bring (node_credit: their own -bytes and DAG_ENTRY_OVERHEAD per entry or index child, less INDEX_NODE_COST for an index node); a -caller's max_dag_bytes (a quota) can lower it. fetch_listing has the same bound for a size of 0, and each -read_range call bounds the bytes of the file nodes it meets by range_bound (what the range can hold, plus two -full file nodes per level a file of the declared size can have). A file node may list children of size 0, -and index nodes can be chained, so without these a file declaring 1 byte, or a directory of one entry, could -hold any number of distinct nodes. What the builder writes never comes near them (design.md, "DAG byte -bound"). While one applies, requests ask for no more than it leaves, counting a DAG node as MAX_BLOCK_SIZE -until it arrives (a directory or index node as nothing under the size-derived bound: see _reserve), so a -fetch downloads at most about one block beyond it before failing with TooLarge. max_storage never refuses a -fetch: one -that isn't being pinned evicts other cached fetches to make room for each response, and goes over -max_storage if nothing is evictable (the node evicts again once it ends). What refuses a fetch, pinned or -not, with StorageFull is the disk: its remaining size, and then each response, must leave the disk reserve -free (Storage.check_disk). A CID the deny list denies (rnfs.denylist; the storage's) ends a fetch with Denied -where it is met, root or child, stored or not, before it is asked for or read; a complete stored DAG is checked -whole, since fetch_dag doesn't walk it. +Limits (sections 7 and 8): max_download_size bounds both the target's verified size and the DAG node bytes a +fetch downloads (a DAG of empty files has size 0 but can hold gigabytes of directory nodes); listings and +write-outs stop at max_output_entries entries met while fetching. fetch_dag also bounds the bytes of the DAG's +distinct blocks by what its declared size can hold (dag_bound: the size, DAG_GRANULE_OVERHEAD per DAG_GRANULE +of it and DAG_SLACK), plus what its directory and index nodes bring (node_credit: their own bytes and +DAG_ENTRY_OVERHEAD per entry or index child, less INDEX_NODE_COST for an index node); a caller's max_dag_bytes +(a quota) can lower it. fetch_listing has the same bound for a size of 0, and each read_range call bounds the +bytes of the file nodes it meets by range_bound (what the range can hold, plus two full file nodes per level a +file of the declared size can have). A file node may list children of size 0, and index nodes can be chained, +so without these a file declaring 1 byte, or a directory of one entry, could hold any number of distinct nodes. +What the builder writes never comes near them (design.md, "DAG byte bound"). While one applies, requests ask +for no more than it leaves, counting a DAG node as MAX_BLOCK_SIZE until it arrives (a directory or index node +as nothing under the size-derived bound: see _reserve), so a fetch downloads at most about one block beyond it +before failing with TooLarge. max_storage never refuses a fetch: one that isn't being pinned evicts other +cached fetches to make room for each response, and goes over max_storage if nothing is evictable (the node +evicts again once it ends). What refuses a fetch, pinned or not, with StorageFull is the disk: its remaining +size, and then each response, must leave the disk reserve free (Storage.check_disk). A CID the deny list denies +(rnfs.denylist; the storage's) ends a fetch with Denied where it is met, root or child, stored or not, before +it is asked for or read; a complete stored DAG is checked whole, since fetch_dag doesn't walk it (and its +blocks against a quota). Requests are spread over up to max_parallel_providers providers (1 on slow nodes), one /blocks request in flight per provider. Request threads only call the client; all fetch state is changed by the thread that @@ -76,7 +75,7 @@ from rnfs.content.cid import Cid from rnfs.content.storage import Storage from rnfs.content.walker import Expect -from rnfs.errors import Denied, InvalidDag, NotFound, RnfsError, TooLarge +from rnfs.errors import Cancelled, Denied, InvalidDag, NotFound, TooLarge from rnfs.log import LOG_VERBOSE, LOG_WARNING, log, node_name, plural from rnfs.network import client as peer_client from rnfs.network import protocol, rates @@ -167,10 +166,6 @@ def range_bound(size: int, length: int) -> int: return dag_bound(length) + 2 * file_levels(size) * FILE_NODE_MAX -class Cancelled(RnfsError): - """A fetch stopped because its observer asked it to (a job cancelled, or the node stopping).""" - - class Observer: """Watches the fetches opened on one thread (see observing()): progress reports and a cancel check. @@ -522,6 +517,10 @@ def fetch_dag(self, target: Expect, max_entries: int | None = None): denied = self.storage.deny.cids if denied and any(cid.binary in denied for cid in self.storage.dag_blocks(target.cid)): raise Denied(f"{target.cid} holds a block on this node's deny list") + # A quota counts the blocks stored as well as downloaded: a remote pin or auto-fetch of a DAG held + # already (a user pin, a cached fetch) gets the same answer as one fetched now + if self.max_dag_bytes is not None and self.storage.dag_bytes(target.cid) > self.max_dag_bytes: + raise TooLarge(f"{self.root} holds more than {self.max_dag_bytes} bytes of blocks") return # What is stored already needn't fit again (its DAG nodes make this an estimate) self.check_disk(target, size - self.storage.dag_bytes(target.cid)) diff --git a/rnfs/fsutil.py b/rnfs/fsutil.py index 92419f3..f403632 100644 --- a/rnfs/fsutil.py +++ b/rnfs/fsutil.py @@ -59,7 +59,8 @@ def replace(src: str, dst: str): def lock_exclusive(path: str) -> int: """Open path (created 0600 if missing) and lock it for this process, without waiting; the descriptor, whose lock lasts until it is closed or the process ends (also by a crash), with this process's ID written in the - file. Raises BlockingIOError if another process holds the lock, OSError if path can't be opened. + file. Raises BlockingIOError if another process holds the lock, OSError if path can't be opened or locked. + Release it with unlock(). POSIX: flock (advisory: only processes that lock it too are kept out). Windows: msvcrt.locking of one byte at LOCK_OFFSET, past the text. @@ -71,7 +72,11 @@ def lock_exclusive(path: str) -> int: os.lseek(fd, LOCK_OFFSET, os.SEEK_SET) try: msvcrt.locking(fd, msvcrt.LK_NBLCK, 1) - except OSError as e: # PermissionError (EACCES) when another process holds it + except OSError as e: + # PermissionError (EACCES) when another process holds it (EDEADLOCK in the waiting modes); + # anything else (a file system that can't lock, say) is no holder's doing + if e.errno not in (errno.EACCES, errno.EDEADLK): + raise raise BlockingIOError(errno.EAGAIN, f"{path} is locked by another process") from e else: import fcntl @@ -85,6 +90,24 @@ def lock_exclusive(path: str) -> int: return fd +def unlock(fd: int): + """Release lock_exclusive()'s lock on fd, then close fd. Windows may release a closed file's locks only some + time later, so a lock given up here can be taken again at once; one that won't unlock is left to the close. + Raises OSError if fd can't be closed.""" + try: + if IS_WINDOWS: + import msvcrt + os.lseek(fd, LOCK_OFFSET, os.SEEK_SET) + msvcrt.locking(fd, msvcrt.LK_UNLCK, 1) + else: + import fcntl + fcntl.flock(fd, fcntl.LOCK_UN) + except OSError: + pass # closing releases it too + finally: + os.close(fd) + + def lock_holder(path: str) -> int | None: """The process ID lock_exclusive() wrote in path, or None if it can't be read.""" try: diff --git a/rnfs/gateway.py b/rnfs/gateway.py index c73a560..94eda68 100644 --- a/rnfs/gateway.py +++ b/rnfs/gateway.py @@ -187,7 +187,6 @@ def printable_size(size: int) -> str: if size < 1024 or unit == "GiB": return f"{size} {unit}" if unit == "B" else f"{size:.1f} {unit}" size /= 1024 - return str(size) def listing_html(title: str, entries: list[dict], has_parent: bool) -> bytes: @@ -213,6 +212,7 @@ class GatewayError(Exception): """A request the gateway answers with an error: status, message and extra headers.""" def __init__(self, status: int, message: str, headers: dict | None = None): + """An error answered with status, message and headers (none if None).""" super().__init__(message) self.status = status self.headers = headers or {} @@ -222,6 +222,7 @@ class FetchLimiter: """At most `limit` concurrent network fetches or name resolutions for requests that aren't navigation.""" def __init__(self, limit: int): + """Allow limit fetches at once.""" self.limit = limit self.active = 0 self.lock = threading.Lock() @@ -244,6 +245,7 @@ class NameCache: """Recent name resolutions (name → CID), reused for NAME_CACHE_TIME. LRU, at most NAME_CACHE_SIZE.""" def __init__(self, clock=time.monotonic): + """An empty cache, aged by clock.""" self.entries: OrderedDict[str, tuple[float, str]] = OrderedDict() self.lock = threading.Lock() self.clock = clock @@ -282,6 +284,7 @@ def write(self, data) -> int: return written def __getattr__(self, name): + """Everything else from the stream itself.""" return getattr(self._stream, name) @@ -601,13 +604,7 @@ def file_type(self, target: str, size: int, filename: str | None, read_start: bo if read_start: head = b"".join(self.node.read(target, offset=0, length=length, for_gateway=True)) else: - stored_head = getattr(self.node, "stored_head", None) # facades without it: only complete targets - if stored_head is not None: - head = stored_head(target, length) - elif self.node.is_local(target): - head = b"".join(self.node.read(target, offset=0, length=length, for_gateway=True)) - else: - head = None + head = self.node.stored_head(target, length) if head is None: return type_from_name(filename) or "application/octet-stream" return "application/octet-stream" if is_sealed(head) else type_from_name(filename) or sniff(head) diff --git a/rnfs/gateway_formats.py b/rnfs/gateway_formats.py index 053609b..be04f28 100644 --- a/rnfs/gateway_formats.py +++ b/rnfs/gateway_formats.py @@ -1,8 +1,8 @@ """Trustless gateway responses (rnfs/gateway.py calls serve_format for any /rnfs/ request with ?format=): -GET/HEAD /rnfs/?format=raw the CID's own block, re-verified; application/vnd.rnfs.raw -GET/HEAD /rnfs/[/path]?format=archive an archive (spec section 8) of the target's DAG; - application/vnd.rnfs.archive, Content-Disposition attachment +GET /rnfs/?format=raw the CID's own block, re-verified; application/vnd.rnfs.raw +GET /rnfs/[/path]?format=archive an archive (spec section 8) of the target's DAG; + application/vnd.rnfs.archive, Content-Disposition attachment The client can verify either without trusting the gateway: a raw block hashes to its CID, and an archive's blocks are checked by `rnfs import` (or any reader) from the root down. Every gateway header rule still @@ -73,14 +73,14 @@ def serve_format(handler, segments: list[str], names: list[str]): "Content-Disposition": f'attachment; filename="{cid}.rnfs-archive"'} def start(length: int): - """Send the headers once the archive's length is known; where to write it (None for HEAD).""" + """Send the headers once the archive's length is known; where to write it.""" handler.send_response(200) handler.send_header("Content-Type", ARCHIVE_TYPE) handler.send_header("Content-Length", str(length)) for name, value in headers.items(): handler.send_header(name, value) handler.end_headers() - return None if handler.command == "HEAD" else handler.wfile + return handler.wfile try: # The fetch runs on this thread, so the observer sees it and cancels it once the client has gone diff --git a/rnfs/interface.py b/rnfs/interface.py new file mode 100644 index 0000000..741c740 --- /dev/null +++ b/rnfs/interface.py @@ -0,0 +1,951 @@ +"""The interface rnfs.Node and rnfs.Client share (design.md, "Public API"; provisional until 0.2). No RNS. + +Rnfs implements every method once, over a backend offering the node facade's operations with the local API's +argument and result forms (JSON-ready dicts; CIDs written out, node IDs and names as hex): rnfs.embedded's over +an RnfsNode, rnfs.remote's over the daemon's API. Both results become the same objects (rnfs.objects), so the +two classes can't drift apart; only lifecycle, how name records are heard, and the escape hatches differ. + +A target is a Cid, `[/name…]` (whose first part may be a name contact's nick), or an rnfs: URI (section 13): +a name URI's name is resolved first (persisted, as names.resolve does), and its from= is the from_node of the +methods taking one. Node IDs are 16 bytes or 32 hex characters; names 16 bytes, 32 hex characters or a Name. + +Callbacks (names.watch, a job's on_progress or watch) run on the instance's event thread, one at a time, in the +order their events arrived, with no rnfs lock held: whatever hears an event only queues it, coalesced per watch +(a slow callback is given the latest state, not a backlog). A callback may call any method, but blocking in it +delays every other callback of the instance; close() from one doesn't wait for the thread. An exception in a +callback is logged and dropped. Watched jobs are polled by a thread of their own, which queues a call when a +job changes (at most every job_poll seconds) and once more when it finishes. Both threads start on first use. +""" +import io +import os +import threading +import time +from collections import OrderedDict +from dataclasses import dataclass, field + +from rnfs.config import _HEX_ID +from rnfs.content.cid import Cid +from rnfs.errors import Closed, NotFound, RnfsError +# A job's final states, and the longest wait one request for a job asks for (the API's cap) +from rnfs.features.jobs import FINISHED, MAX_WAIT as MAX_JOB_WAIT +from rnfs.objects import (Added, Collected, DenyState, Entry, Exported, Imported, Info, Pin, Record, Stat, Status, + Verified, Written) + +FILE_BUFFER = 64 * 1024 # what a File reads at least per refill: one read() of the node, one GET /api/cat +STOP_WAIT = 5.0 # seconds close() waits for the event and job threads to end + + +def node_hex(node_id) -> str | None: + """A node ID or name as 32 hex characters (None stays None). ValueError unless 16 bytes or 32 hex digits.""" + if node_id is None: + return None + if isinstance(node_id, (bytes, bytearray)) and len(node_id) == 16: + return bytes(node_id).hex() + if isinstance(node_id, str) and _HEX_ID.fullmatch(node_id): + return node_id.lower() + raise ValueError(f"invalid node ID or name {node_id!r}: 16 bytes or 32 hex characters") + + +def name_bytes(name) -> bytes: + """A name (16 bytes, 32 hex characters or a Name) as bytes. ValueError for anything else.""" + if isinstance(name, Name): + return name.name + return bytes.fromhex(node_hex(name)) + + +def _abspath(path) -> str: + """path made absolute, as the daemon and the facade take paths.""" + return os.path.abspath(os.path.expanduser(os.fspath(path))) + + +class Watch: + """A watch on a name or a job: cancel() (or leaving it as a context manager) stops further callbacks; one + already running finishes.""" + + def __init__(self, cancel): + """A watch calling cancel() once when cancelled.""" + self._cancel = cancel + self._lock = threading.Lock() + self.active = True + + def cancel(self): + """Stop the callbacks (idempotent).""" + with self._lock: + if not self.active: + return + self.active = False + self._cancel() + + def __enter__(self) -> "Watch": + """The watch itself.""" + return self + + def __exit__(self, *exc): + """Cancel the watch.""" + self.cancel() + + +class _Events: + """The event thread: calls queued callbacks one at a time, in the order queued, the latest per key.""" + + def __init__(self, thread_name: str, report): + """Calls from a thread called thread_name, reporting a callback's exception with report(message).""" + self._thread_name = thread_name + self._report = report + self._cond = threading.Condition() # innermost: never held while calling out + self._pending: OrderedDict = OrderedDict() + self._thread: threading.Thread | None = None + self._stopped = False + + def start(self): + """Start the thread unless it runs (from an app's thread, never an RNS one).""" + with self._cond: + if self._thread is not None or self._stopped: + return + self._thread = threading.Thread(target=self._run, name=self._thread_name, daemon=True) + self._thread.start() + + def post(self, key, call): + """Queue call() under key, replacing one queued under key that hasn't run yet. Never blocks: safe to call + holding any lock.""" + with self._cond: + if self._stopped: + return + self._pending[key] = call + self._cond.notify() + + def _run(self): + """Call what is queued until stopped.""" + while True: + with self._cond: + while not self._pending and not self._stopped: + self._cond.wait() + if self._stopped: + return + _, call = self._pending.popitem(last=False) + try: + call() + except Exception as e: # noqa: BLE001 - an app's callback mustn't end the thread + self._report(f"a callback failed: {e!r}") + + def stop(self): + """Drop what is queued and end the thread, waiting for it unless called from it.""" + with self._cond: + self._stopped = True + self._pending.clear() + self._cond.notify() + thread = self._thread + if thread is not None and thread is not threading.current_thread(): + thread.join(STOP_WAIT) + + +class _NameWatch: + """One names.watch: its callback and the sequence last delivered (or held when it began). Records heard + before that baseline is known wait in pending, and only the newest of them is compared with it.""" + + def __init__(self, name: bytes, callback): + """A watch of name calling callback, not ready until begin().""" + self.name = name + self.callback = callback + self.last: int | None = None + self.ready = False + self.pending: Record | None = None + self.lock = threading.Lock() + self.handle: Watch | None = None + + def begin(self, baseline: int | None) -> Record | None: + """Set the sequence held when the watch began; the record heard meanwhile to deliver, if newer.""" + with self.lock: + self.last, self.ready = baseline, True + pending, self.pending = self.pending, None + return pending if pending is not None and (baseline is None or pending.sequence > baseline) else None + + def deliver(self, record: Record): + """Call the callback if record is newer than the last delivered (on the event thread).""" + with self.lock: + if not self.ready: + if self.pending is None or record.sequence > self.pending.sequence: + self.pending = record + return + if not self.handle.active or (self.last is not None and record.sequence <= self.last): + return + self.last = record.sequence + self.callback(record) + + +class _JobPoller: + """The thread polling watched jobs, queueing each watch's callback when its job changes and when it ends.""" + + def __init__(self, owner: "Rnfs", thread_name: str, interval: float): + """Polls through owner every interval seconds while anything is watched.""" + self._owner = owner + self._thread_name = thread_name + self._interval = interval + self._cond = threading.Condition() + self._watches: dict[int, tuple[str, object, Watch]] = {} # id -> (job ID, callback, handle) + self._last: dict[int, dict] = {} # id -> the job's dict last queued for it + self._thread: threading.Thread | None = None + self._stopped = False + self._next = 0 + + def add(self, job_id: str, callback) -> Watch: + """Watch job_id: callback(job) on each change, and once when it finishes.""" + with self._cond: + if self._stopped: + raise Closed("closed") + key = self._next + self._next += 1 + handle = Watch(lambda: self._remove(key)) + self._watches[key] = (job_id, callback, handle) + if self._thread is None: + self._thread = threading.Thread(target=self._run, name=self._thread_name, daemon=True) + self._thread.start() + self._cond.notify() + return handle + + def _remove(self, key: int): + """Forget a watch.""" + with self._cond: + self._watches.pop(key, None) + self._last.pop(key, None) + + def _run(self): + """Poll until stopped.""" + while True: + with self._cond: + while not self._watches and not self._stopped: + self._cond.wait() + if self._stopped: + return + watches = dict(self._watches) + jobs: dict[str, dict | None] = {} + for key, (job_id, callback, handle) in watches.items(): + if job_id not in jobs: + jobs[job_id] = self._poll(job_id) + self._check(key, jobs[job_id], callback, handle) + with self._cond: + if not self._stopped: + self._cond.wait(self._interval) + + def _poll(self, job_id: str) -> dict | None: + """The job's dict now; None if it can't be read (gone: NotFound, cleared; or an error, reported).""" + try: + return self._owner._backend.job(job_id, 0) + except NotFound: + return None + except Exception as e: # noqa: BLE001 - keep watching; a daemon restarting may answer again + self._owner._report(f"watching job {job_id} failed: {e!r}") + return {} + + def _check(self, key: int, data: dict | None, callback, handle: Watch): + """Queue callback for watch key if its job changed; forget the watch once the job is finished or gone.""" + if data == {}: + return # unreadable this time + if data is None: + handle.cancel() + return + with self._cond: + if self._last.get(key) == data or key not in self._watches: + return + self._last[key] = data + job = Job.from_dict(data, self._owner) + + def call(): + """The app's callback, unless cancelled since.""" + if handle.active: + callback(job) + self._owner._events.post(("job", key), call) + if job.done: + self._remove(key) + + def stop(self): + """End the thread, waiting for it unless called from it.""" + with self._cond: + self._stopped = True + self._watches.clear() + self._cond.notify_all() + thread = self._thread + if thread is not None and thread is not threading.current_thread(): + thread.join(STOP_WAIT) + + +class Rnfs: + """What rnfs.Node and rnfs.Client both offer (design.md, "Public API"). Subclasses supply the backend and how + name records are heard; see the module docstring for targets and callbacks. Thread-safe.""" + + events_thread = "rnfs-events" + jobs_thread = "rnfs-job-watch" + job_poll = 0.5 # seconds between polls of watched jobs + + def __init__(self, backend): + """The interface over backend (see the module docstring).""" + self._backend = backend + self._closed = False + self._close_lock = threading.Lock() + self._events = _Events(self.events_thread, self._report) + self._jobs = _JobPoller(self, self.jobs_thread, self.job_poll) + self._watch_lock = threading.Lock() # taken after the names lock (records are heard under it), never before + self._name_watches: dict[bytes, list[_NameWatch]] = {} + self._node_id: bytes | None = None + self.names = Names(self) + self.jobs = Jobs(self) + self.deny = Deny(self) + + # Lifecycle + + @property + def closed(self) -> bool: + """True once close() has been called.""" + return self._closed + + def close(self): + """Stop the callbacks and release what the instance holds (idempotent; subclasses add their own).""" + with self._close_lock: + if self._closed: + return + self._closed = True + self._jobs.stop() + self._events.stop() + with self._watch_lock: + watches = [watch for each in self._name_watches.values() for watch in each] + self._name_watches.clear() + for watch in watches: + watch.handle.active = False + self._unwatch_name(watch.name) + self._close() + + def _close(self): + """What a subclass releases at close().""" + + def __enter__(self): + """The instance itself.""" + return self + + def __exit__(self, *exc): + """close().""" + self.close() + + def _check(self): + """Raise Closed after close().""" + if self._closed: + raise Closed(f"this {type(self).__name__} is closed") + + def _report(self, message: str): + """Log a callback's or a watch's failure (subclasses log where they log).""" + + def wait_joined(self, timeout: float | None = None) -> bool: + """Wait until the node's first join of the DHT has finished; True if it has (a daemon's has).""" + self._check() + return self._backend.wait_joined(timeout) + + # Targets + + def _target(self, target) -> tuple[str, str | None]: + """(target as `[/name…]` or a nick path, the from= node ID in hex of an rnfs: URI, else None).""" + if isinstance(target, Cid): + return str(target), None + if not isinstance(target, str): + raise ValueError(f"a target is a Cid, [/path] or an rnfs: URI, not {type(target).__name__}") + if target[:5].lower() != "rnfs:": + return target, None + from rnfs.features.lxmf import parse_uri # its pure part: no RNS + link = parse_uri(target) + cid = link.cid + if link.name is not None: + cid = self.names.resolve(link.name).cid + from_node = link.from_node.hex() if link.from_node is not None else None + return "/".join([str(cid), *link.path]), from_node + + def _from(self, from_node, hint: str | None) -> str | None: + """The from_node given (as hex), else an rnfs: URI's.""" + return node_hex(from_node) if from_node is not None else hint + + # Status + + def status(self) -> Status: + """The node's status.""" + self._check() + return Status.from_dict(self._backend.status()) + + @property + def node_id(self) -> bytes: + """The node's ID (its rnfs.dht destination hash).""" + if self._node_id is None: + self._check() + self._node_id = bytes.fromhex(self._backend.node_id()) + return self._node_id + + # Content + + def add(self, source, *, name: str | None = None, wrap: bool = False, pin: bool = True, only_hash: bool = False, + to: list[str] | None = None, to_self: bool = False, seal_name: bool = True, + chunker: str | None = None) -> Added: + """Add a file or directory (a path), or data (bytes, or a binary file read to its end), pinned "user" + unless pin is False (then evictable cache); only_hash stores nothing. + + Data is wrapped in a directory as the entry name, if given. to/to_self seal a file for those recipients + (section 5, Sealed files): a path's name is sealed inside unless seal_name is False, and data's name is + sealed inside instead of wrapping it, unless wrap. chunker is "fixed" (default) or "fastcdc". Raises + ValueError, NotFound (no such path), Denied (a file's hash is denied), StorageFull, OSError. + """ + self._check() + sealing = bool(to or to_self) + options = {"only_hash": only_hash, "pin": pin, "to": list(to) if to else None, "to_self": to_self, + "chunker": chunker} + if isinstance(source, (str, os.PathLike)): + if name is not None: + raise ValueError("name is for adding data; a path is added under its own name") + result = self._backend.add_path(_abspath(source), wrap=wrap, seal_name=seal_name, **options) + else: + if wrap and not name: + raise ValueError("wrap needs name, the entry's name") + if isinstance(source, (bytes, bytearray, memoryview)): + data = bytes(source) + length = len(data) # a memoryview's len() counts its items, not its bytes + elif hasattr(source, "read"): + data, length = source, None + else: + raise ValueError(f"add() takes a path, bytes or a binary file, not {type(source).__name__}") + entry = name if wrap or not sealing else None + result = self._backend.add_stream(data, length, entry, seal_name=name if sealing and seal_name else None, + **options) + return Added.from_dict(result) + + def get(self, target, output, *, from_node=None, raw: bool = False) -> Written: + """Fetch a file or directory and write it to output (a path that must not exist), all in this call. A + sealed file this node can open is decrypted unless raw. Raises the fetch errors (NotFound, TooLarge, + InvalidDag, Denied, StorageFull, Busy), ValueError (output exists), OSError.""" + self._check() + text, hint = self._target(target) + output = _abspath(output) + result = self._backend.get(text, output, self._from(from_node, hint), raw) + return Written.from_dict({**result, "output": output}) + + def open(self, target, *, from_node=None, raw: bool = False) -> "File": + """A file to read and seek in, each read fetching only the chunks it covers (sealed files read as + plaintext unless raw). Raises the fetch errors, ValueError (a directory); reads raise the fetch errors.""" + self._check() + text, hint = self._target(target) + from_node = self._from(from_node, hint) + info = Info.from_dict(self._backend.info(text, from_node, raw)) + if info.type != "file": + raise ValueError(f"{text} is a directory") + return File(_FileReader(self, text, info, from_node, raw)) + + def cat(self, target, *, offset: int = 0, length: int | None = None, from_node=None, raw: bool = False): + """An iterator of a file's bytes in [offset, offset + length), fetched as it is read (close() it to stop + early). Raises the fetch errors and ValueError, possibly only once iterated.""" + self._check() + text, hint = self._target(target) + return self._backend.read(text, offset, length, raw, self._from(from_node, hint)) + + def block(self, cid) -> bytes: + """A CID's own block, fetched if needed and verified. Raises the fetch errors.""" + self._check() + return self._backend.block(str(cid)) + + def ls(self, target) -> list[Entry]: + """A directory's entries in name order. Raises the fetch errors, ValueError (not a directory).""" + self._check() + return [Entry.from_dict(entry) for entry in self._backend.ls(self._target(target)[0])] + + def info(self, target, *, from_node=None, raw: bool = False) -> Info: + """A target's own CID, type, size and executable bit (a sealed file's plaintext size if this node can + open it, unless raw). Raises the fetch errors.""" + self._check() + text, hint = self._target(target) + return Info.from_dict(self._backend.info(text, self._from(from_node, hint), raw)) + + def stat(self, target) -> Stat: + """A target's size, block count and local holdings. Raises the fetch errors.""" + self._check() + return Stat.from_dict(self._backend.stat(self._target(target)[0])) + + def dir_set(self, target, name: str, child, *, executable: bool = False) -> Cid: + """A new directory, pinned "user": target's with entry name set to child (a Cid). Raises the fetch + errors, ValueError.""" + self._check() + return Cid.parse(self._backend.dir_set(self._target(target)[0], name, str(child), executable)["cid"]) + + def dir_rm(self, target) -> Cid: + """A new directory, pinned "user", without the entry target (`/`) names. Raises the fetch + errors, ValueError.""" + self._check() + return Cid.parse(self._backend.dir_rm(self._target(target)[0])["cid"]) + + # Pins + + def pin(self, target) -> Cid: + """Fetch the target completely and pin it "user"; its CID once complete. Raises the fetch errors. For + progress, jobs.pin().""" + self._check() + return Cid.parse(self._backend.pin_add(self._target(target)[0])["cid"]) + + def unpin(self, cid): + """Remove cid's "user" pin; withdrawn at once if no longer provided. Raises NotFound if it has none.""" + self._check() + self._backend.pin_rm(str(cid)) + + def pins(self, cid=None) -> list[Pin]: + """Every pin, or cid's.""" + self._check() + return [Pin.from_dict(pin) for pin in self._backend.pin_ls(str(cid) if cid is not None else None)] + + def pin_remote(self, node_id, cid, pin: bool = True) -> bool: + """Ask node_id to pin (or release this node's pin of) cid; True if it accepted.""" + self._check() + return bool(self._backend.pin_remote(node_hex(node_id), str(cid), pin)) + + def gc(self) -> Collected: + """Evict every cached fetch and delete unreachable blocks.""" + self._check() + return Collected.from_dict(self._backend.gc()) + + def verify(self) -> Verified: + """Re-check every stored block against its CID, deleting bad ones.""" + self._check() + return Verified.from_dict(self._backend.verify()) + + # Network + + def providers(self, cid) -> list[bytes]: + """Node IDs providing cid: records held here, then what a provider lookup finds.""" + self._check() + return [bytes.fromhex(node_id) for node_id in self._backend.providers(str(cid))] + + def ping(self, node_id) -> float | None: + """Round-trip seconds of a new link to node_id, or None if it can't be reached.""" + self._check() + return self._backend.ping(node_hex(node_id)) + + # Archives + + def export(self, target, output) -> Exported: + """Fetch the target if needed and write an archive of it to output (which must not exist). Raises the + fetch errors, ValueError, OSError.""" + self._check() + return Exported.from_dict(self._backend.export(self._target(target)[0], _abspath(output))) + + def import_archive(self, path) -> Imported: + """Import an archive, pinning its complete roots "user". Raises NotFound, InvalidDag, Denied, + StorageFull, ValueError, OSError.""" + self._check() + return Imported.from_dict(self._backend.import_archive(_abspath(path))) + + # Name watches (Names.watch) + + def _watch_name(self, name: bytes, callback) -> Watch: + """Call callback(record) for each record of name newer than the one held now.""" + self._check() + watch = _NameWatch(name, callback) + watch.handle = Watch(lambda: self._end_name_watch(watch)) + self._events.start() + with self._watch_lock: + self._name_watches.setdefault(name, []).append(watch) + try: + self._watch_started(name) + baseline = self._baseline(name) + except BaseException: + self._end_name_watch(watch) + raise + pending = watch.begin(baseline) + if pending is not None: + # A key of its own: a newer record's delivery already queued must not be replaced by it + self._events.post(("name-pending", id(watch)), lambda: watch.deliver(pending)) + return watch.handle + + def _end_name_watch(self, watch: _NameWatch): + """Forget a cancelled watch.""" + watch.handle.active = False + with self._watch_lock: + watches = self._name_watches.get(watch.name, []) + if watch not in watches: + return + watches.remove(watch) + if not watches: + del self._name_watches[watch.name] + self._unwatch_name(watch.name) + + def _record_heard(self, name: bytes, record: Record): + """A newer record of name was heard: queue its delivery to each watch (any thread; never blocks).""" + with self._watch_lock: + watches = list(self._name_watches.get(name, ())) + for watch in watches: + self._events.post(("name", id(watch)), lambda watch=watch: watch.deliver(record)) + + def _watched_names(self) -> list[bytes]: + """The names watched now.""" + with self._watch_lock: + return list(self._name_watches) + + def _last_sequences(self, names: list[bytes]) -> list[int | None]: + """For each name, the lowest sequence any of its watches last delivered (None: one has none).""" + with self._watch_lock: + watches = {name: list(self._name_watches.get(name, ())) for name in names} + result = [] + for name in names: + lasts = [watch.last for watch in watches[name]] + result.append(None if not lasts or None in lasts else min(lasts)) + return result + + def _watch_started(self, name: bytes): + """A watch of name began: have the node treat it as watched (subclasses).""" + + def _unwatch_name(self, name: bytes): + """A watch of name ended (subclasses).""" + + def _baseline(self, name: bytes) -> int | None: + """The sequence of name's record held now, or None (subclasses).""" + return None + + +class _FileReader(io.RawIOBase): + """File's raw stream: each readinto is one read of [position, position + its length) of the target opened. + + By the target (root and path), not the file's own CID: providers announce roots (provide_strategy), so the + file's CID may have none.""" + + def __init__(self, owner: Rnfs, target: str, info: Info, from_node: str | None, raw: bool): + """A reader of the file target names, whose info open() read.""" + super().__init__() + self.owner = owner + self.name = target + self.info = info + self.from_node = from_node + self.raw = raw + self.position = 0 + + def readable(self) -> bool: + """True.""" + return True + + def seekable(self) -> bool: + """True.""" + return True + + def readinto(self, buffer) -> int: + """Fill buffer from the position, fetching only what it covers; 0 at the end.""" + wanted = min(len(buffer), self.info.size - self.position) + if wanted <= 0: + return 0 + self.owner._check() + chunks = self.owner._backend.read(self.name, self.position, wanted, self.raw, self.from_node) + try: + data = b"".join(chunks) + finally: + close = getattr(chunks, "close", None) + if close is not None: + close() + if not data: + raise RnfsError(f"{self.name} ended at {self.position} of its {self.info.size} bytes") + buffer[:len(data)] = data + self.position += len(data) + return len(data) + + def readall(self) -> bytes: + """The rest of the file in one read (RawIOBase's would make one per 8 KiB).""" + chunks = [] + while self.position < self.info.size: + buffer = bytearray(self.info.size - self.position) + chunks.append(bytes(buffer[:self.readinto(buffer)])) + return b"".join(chunks) + + def seek(self, offset: int, whence: int = io.SEEK_SET) -> int: + """Move the position (past the end is allowed: reads there return nothing).""" + base = {io.SEEK_SET: 0, io.SEEK_CUR: self.position, io.SEEK_END: self.info.size}.get(whence) + if base is None: + raise ValueError(f"invalid whence {whence!r}") + if base + offset < 0: + raise ValueError("negative seek position") + self.position = base + offset + return self.position + + def tell(self) -> int: + """The position.""" + return self.position + + +class File(io.BufferedReader): + """A file in rnfs, from open(): read, readinto, readline, iteration, seek and tell, buffered by FILE_BUFFER + bytes, each refill fetching only the chunks it covers. Reads go by the target opened (an rnfs: name URI + resolved once, at open; a contact nick is expanded by the node, which reuses its CID for 60 s). cid is the + file's own CID, size its size, info its Info, as open() found them. Not thread-safe, like any file object.""" + + def __init__(self, reader: _FileReader): + """The buffered file over reader.""" + super().__init__(reader, FILE_BUFFER) + + @property + def cid(self) -> Cid: + """The file's own CID.""" + return self.raw.info.cid + + @property + def size(self) -> int: + """The file's size (a sealed file's plaintext size, unless opened raw).""" + return self.raw.info.size + + @property + def info(self) -> Info: + """What open() learnt of the file.""" + return self.raw.info + + +@dataclass(frozen=True) +class Name: + """A name of this node: its alias, the name (16 bytes; str() is its hex), and the CID and sequence it last + published (None and 0 before a publish). A snapshot: refresh() reads it again.""" + + alias: str + name: bytes + cid: Cid | None = None + sequence: int = 0 + _owner: Rnfs | None = field(default=None, repr=False, compare=False) + + def __str__(self) -> str: + """The name in hex.""" + return self.name.hex() + + @property + def hex(self) -> str: + """The name as 32 hex characters.""" + return self.name.hex() + + @classmethod + def from_dict(cls, data: dict, owner: Rnfs | None = None) -> "Name": + """From the API's form: {"alias", "name", "cid", "sequence"}.""" + cid = data.get("cid") + return cls(data["alias"], bytes.fromhex(data["name"]), Cid.parse(cid) if cid else None, + data.get("sequence", 0), owner) + + def to_dict(self) -> dict: + """The API's form.""" + return {"alias": self.alias, "name": self.name.hex(), "cid": str(self.cid) if self.cid else None, + "sequence": self.sequence} + + def publish(self, target) -> Record: + """Point the name at target (a path in it resolved locally) and announce it; the new record. Raises + NotFound (a path not stored here), ValueError, RnfsError (the highest sequence reached).""" + self._owner._check() + result = self._owner._backend.name_publish(self.alias, self._owner._target(target)[0]) + return Record.from_dict({**result, "age": 0.0}) + + def export_key(self) -> bytes: + """The name's private key: whoever holds it can publish the name.""" + self._owner._check() + return self._owner._backend.name_export(self.alias) + + def remove(self): + """Delete the name's key, releasing its pin. Raises NotFound.""" + self._owner._check() + self._owner._backend.name_rm(self.alias) + + def refresh(self) -> "Name": + """The name as it is now. Raises NotFound if it was removed.""" + return self._owner.names.get(self.alias) + + +class Names: + """A node's names (Rnfs.names): its own, and resolving and watching anyone's.""" + + def __init__(self, owner: Rnfs): + """The names of owner.""" + self._owner = owner + + def create(self, alias: str) -> Name: + """A new name key under alias. Raises ValueError (alias invalid or taken).""" + self._owner._check() + return Name.from_dict(self._owner._backend.name_create(alias), self._owner) + + def import_key(self, alias: str, key: bytes) -> Name: + """A name key exported elsewhere, stored under alias. Raises ValueError (alias, or a malformed key).""" + self._owner._check() + return self.get(Name.from_dict(self._owner._backend.name_import(alias, key)).alias) + + def get(self, alias: str) -> Name: + """This node's name called alias. Raises NotFound.""" + for name in self.list(): + if name.alias == alias: + return name + raise NotFound(f"no name called {alias}") + + def list(self) -> list[Name]: + """This node's names, by alias.""" + self._owner._check() + return [Name.from_dict(name, self._owner) for name in self._owner._backend.name_list()] + + def remove(self, alias: str): + """Delete the name key alias, releasing its pin. Raises NotFound.""" + self._owner._check() + self._owner._backend.name_rm(alias) + + def resolve(self, name) -> Record: + """The newest record of name (16 bytes, 32 hex characters, a Name or a name contact's nick), after + asking the network; persisted, so it never goes back. Raises NotFound (unresolved), ValueError.""" + self._owner._check() + text = name if isinstance(name, str) and not _HEX_ID.fullmatch(name) else node_hex(name_bytes(name)) + return Record.from_dict(self._owner._backend.name_resolve(text)) + + def watch(self, name, callback) -> Watch: + """Call callback(record) on the event thread for each record of name (16 bytes, 32 hex characters or a + Name) newer than the one held now, as announces, path responses and keepers' answers are heard (and + this node's own publishes). Only listens: resolve() asks. Raises ValueError; a Client NotFound from a + daemon that can't watch names.""" + return self._owner._watch_name(name_bytes(name), callback) + + +@dataclass(frozen=True) +class Job: + """A background transfer (get, pin or export) the node runs, persists and resumes after a restart: its + record as of the last refresh(), wait() or callback. done is True once it has finished (state "done", + "failed" or "cancelled"); result is what the matching call returns, error why it failed.""" + + id: str + kind: str + target: str + output: str | None = None + from_node: bytes | None = None + sealed_name: bool = False + fallback_dht: bool = False + state: str = "queued" + created: float | None = None + started: float | None = None + finished: float | None = None + error: str | None = None + result: dict | None = None + runs: int = 0 + total_bytes: int | None = None + done_bytes: int = 0 + blocks: int = 0 + blocks_total: int | None = None + downloaded_bytes: int = 0 + rate: float | None = None + eta: float | None = None + _owner: Rnfs | None = field(default=None, repr=False, compare=False) + + FIELDS = ("id", "kind", "target", "output", "sealed_name", "fallback_dht", "state", "created", "started", + "finished", "error", "result", "runs", "total_bytes", "done_bytes", "blocks", "blocks_total", + "downloaded_bytes", "rate", "eta") + + @classmethod + def from_dict(cls, data: dict, owner: Rnfs | None = None) -> "Job": + """From the API's form (its "from" is from_node).""" + values = {name: data[name] for name in cls.FIELDS if name in data} + return cls(**values, from_node=bytes.fromhex(data["from"]) if data.get("from") else None, _owner=owner) + + def to_dict(self) -> dict: + """The API's form.""" + return {**{name: getattr(self, name) for name in self.FIELDS}, + "from": self.from_node.hex() if self.from_node is not None else None} + + @property + def done(self) -> bool: + """True once the job has finished.""" + return self.state in FINISHED + + def refresh(self) -> "Job": + """The job as it is now. Raises NotFound once cleared.""" + return self._owner.jobs.job(self.id) + + def wait(self, timeout: float | None = None) -> "Job": + """The job once it has finished (check state and error). Raises TimeoutError if it hasn't within timeout + seconds, NotFound once cleared.""" + deadline = None if timeout is None else time.monotonic() + timeout + job = self + while not job.done: + remaining = None if deadline is None else deadline - time.monotonic() + if remaining is not None and remaining <= 0: + raise TimeoutError(f"job {self.id} hasn't finished") + job = self._owner.jobs._job(self.id, MAX_JOB_WAIT if remaining is None else min(MAX_JOB_WAIT, remaining)) + return job + + def cancel(self) -> "Job": + """Cancel the job: a queued one at once, a running one at its fetch's next step.""" + self._owner._check() + return Job.from_dict(self._owner._backend.cancel_job(self.id), self._owner) + + def watch(self, callback) -> Watch: + """Call callback(job) on the event thread whenever the job changes (at most every job_poll seconds) and + once when it finishes.""" + self._owner._check() + self._owner._events.start() + return self._owner._jobs.add(self.id, callback) + + +class Jobs: + """A node's background jobs (Rnfs.jobs): transfers with progress, kept across restarts.""" + + def __init__(self, owner: Rnfs): + """The jobs of owner.""" + self._owner = owner + + def get(self, target, output, *, from_node=None, sealed_name: bool = False, fallback_dht: bool = False, + on_progress=None) -> Job: + """Fetch target and write it to output (which must not exist) as a job. sealed_name writes a sealed + file this node opens under its sealed name in output's directory; fallback_dht retries by DHT discovery + if from_node fails. on_progress(job) as Job.watch. Raises ValueError (output exists or in use), Busy.""" + return self._submit("get", target, _abspath(output), from_node, on_progress, sealed_name=sealed_name, + fallback_dht=fallback_dht) + + def pin(self, target, *, on_progress=None) -> Job: + """Fetch target completely and pin it "user", as a job.""" + return self._submit("pin", target, None, None, on_progress) + + def export(self, target, output, *, on_progress=None) -> Job: + """Write an archive of target to output (which must not exist), as a job.""" + return self._submit("export", target, _abspath(output), None, on_progress) + + def _submit(self, kind: str, target, output: str | None, from_node, on_progress, **flags) -> Job: + """Queue a job; with on_progress, watched from the start.""" + self._owner._check() + text, hint = self._owner._target(target) + from_node = self._owner._from(from_node, hint) if kind == "get" else None + job = Job.from_dict(self._owner._backend.submit_job(kind, text, output, from_node, **flags), self._owner) + if on_progress is not None: + job.watch(on_progress) + return job + + def job(self, job_id: str) -> Job: + """The job with this ID. Raises NotFound.""" + return self._job(job_id, 0) + + def _job(self, job_id: str, wait: float) -> Job: + """The job, after waiting up to wait seconds for it to finish.""" + self._owner._check() + return Job.from_dict(self._owner._backend.job(job_id, wait), self._owner) + + def list(self) -> list[Job]: + """Every job, oldest first.""" + self._owner._check() + return [Job.from_dict(job, self._owner) for job in self._owner._backend.jobs()] + + def clear(self, job_id: str | None = None) -> int: + """Forget finished jobs (or just job_id, which must have finished); how many. Raises NotFound, + ValueError (not finished).""" + self._owner._check() + return self._owner._backend.clear_jobs(job_id) + + +class Deny: + """A node's deny list (Rnfs.deny, section 8): content it never fetches, stores, serves or provides.""" + + def __init__(self, owner: Rnfs): + """The deny list of owner.""" + self._owner = owner + + def add(self, entries: list[str]) -> list[str]: + """Add CIDs or file hashes (sha256:) to the local list; those that were new. Raises ValueError.""" + self._owner._check() + return self._owner._backend.deny_add([str(entry) for entry in entries]) + + def remove(self, entries: list[str]) -> list[str]: + """Remove entries from the local list; those removed. Raises NotFound (removing none) if one isn't on it, + ValueError.""" + self._owner._check() + return self._owner._backend.deny_rm([str(entry) for entry in entries]) + + # After the methods above, whose annotations name the builtin list + def list(self) -> DenyState: + """The local entries, and each deny_lists source with its state.""" + self._owner._check() + return DenyState.from_dict(self._owner._backend.deny()) diff --git a/rnfs/mesh.py b/rnfs/mesh.py index 55b31e4..da259bf 100644 --- a/rnfs/mesh.py +++ b/rnfs/mesh.py @@ -1,4 +1,5 @@ -"""Being considerate on shared radio meshes (the mesh feature; scratchpad spec text in docs/mesh.md). +"""Being considerate on shared radio meshes (the mesh feature: specification.md section 3, upload budget, +interface rules and quiet hours; design.md, "Mesh"). - Provider choice: within each seniority group, providers fewer hops away are tried first, then those whose next hop goes out on a faster interface (provider_rank). RNS is reached only through RnsPaths, which the @@ -290,7 +291,8 @@ def refusal(self, interface_names_: list[str]) -> str | None: return REFUSED_INTERFACE if self.quiet and in_quiet_hours(self.quiet, self.minute_of_day()): return REFUSED_QUIET - # Nothing at all left: refuse before building a response (spend() decides once its size is known) + # Nothing at all left: refuse before building a response (RequestServer._admit, through reserve(), + # decides once its size is known) if self.bucket is not None and self.bucket.available() <= 0: return REFUSED_BUDGET return None diff --git a/rnfs/names.py b/rnfs/names.py index 38fb1bc..fade9cb 100644 --- a/rnfs/names.py +++ b/rnfs/names.py @@ -16,6 +16,10 @@ power loss may lose the last of them; a malformed floor entry is dropped. The names.json and name-floor.json of rnfs 0.1.4 and earlier are imported once (import_names, import_floor). +Watching (rnfs.Node's names.watch, GET /api/name/watch): a watched name's announces count as asked for, so a full +table never refuses them, and record listeners are told, with the lock held, of each newer record accepted for one +(an announce, a path response, a keeper's answer, a publish here); they only queue it. + Every RNS call goes through `RnsNetwork`, so tests swap in a fake and run without Reticulum. """ import json @@ -53,6 +57,7 @@ FLOOR_TOUCH_SAVE = 3600.0 # a resolve that only renews a floor entry's time marks it for the next save after this PIN_SOURCE_PREFIX = "name:" _ALIAS = re.compile(r"[A-Za-z0-9][A-Za-z0-9._-]{0,63}") +_PARTIAL = re.compile(re.escape(fsutil.PARTIAL_PREFIX) + r"[0-9a-f]{16}") # a key write's temp file (write_atomic) _DECIMAL = re.compile(r"-?[0-9]+") # a sequence column (decimal text: sequences don't fit SQLite's integers) _MAX_USED = 2**62 # floor rows' used values stay below this, so the ones after them still fit SQLite's integers _POLL_INTERVAL = 0.25 # how often a refresh checks whether RNS has heard a path response @@ -149,6 +154,11 @@ class _Published: # on the network and at keepers, so its first publish asks both before choosing a sequence imported: bool = False + @property + def unsettled(self) -> bool: + """Imported and not yet published here: its records elsewhere are unknown.""" + return self.imported and self.cid is None + # State @@ -396,7 +406,7 @@ def __init__(self, config: Config, pins, network=None, clock=time.time, monotoni # Floor saves write without self._lock, one at a time, so an older snapshot never lands last. Taken before # self._lock, never while holding it self._floor_save_lock = threading.Lock() - self._held_dirty = False # heard times changed without a save + self._held_dirty = False # held records changed without a save (heard again, or a save that failed) self._destinations: dict[str, object] = {} self._next_announce: dict[str, float] = {} # alias -> monotonic time @@ -406,6 +416,11 @@ def __init__(self, config: Config, pins, network=None, clock=time.time, monotoni # The keepers feature (rnfs.features.keepers), if loaded: told of heard and published records, and asked # for signed records when the network has none. Never called with self._lock held self.keepers = None + # Names watched (rnfs.Node's names.watch, GET /api/name/watch), and how many watches each: their announces + # count as asked, so a full table never refuses them. Record listeners (add_record_listener) are told of + # each newer record accepted for a watched name; swapped whole, so they are read without the lock + self._watched: dict[bytes, int] = {} + self._record_listeners: tuple = () # Pins of names whose key file was deleted by hand, released by release_gone() once the node is set up self._gone: list[tuple[str, Cid]] = [] # Aliases whose key file is there but couldn't be read (permissions, damage): not names this node can @@ -438,8 +453,9 @@ def create(self, alias: str) -> bytes: self._check_free(alias) identity = RNS.Identity() self._add_key(alias, identity, _Published(0, None)) - log(f"created name {self._names[alias].hex()} as {alias}", LOG_NOTICE) - return self._names[alias] + name = self._names[alias] # under the lock: a remove may follow at once + log(f"created name {name.hex()} as {alias}", LOG_NOTICE) + return name def import_key(self, alias: str, data: bytes) -> bytes: """Store a key exported by export_key under alias and return its name. ValueError if invalid or taken.""" @@ -490,8 +506,9 @@ def remove(self, alias: str): LOG_WARNING) log(f"removed name {name.hex()} ({alias})", LOG_NOTICE) - def release_gone(self): - """Release the pins of names whose key file was deleted by hand (found by the constructor). + def release_gone(self) -> set[tuple[str, Cid]]: + """Release the pins of names whose key file was deleted by hand (found by the constructor); returns the + (alias, CID) pins released. Not done while loading: releasing a pin recomputes what the node provides, which needs the rest of the node, and the features' keys (kept names), set up first. @@ -503,6 +520,7 @@ def release_gone(self): self._pins.unpin(cid, PIN_SOURCE_PREFIX + alias) if gone: self._save() # its entry is dropped only once its pin is released + return set(gone) def reconcile_pins(self, pins: dict[Cid, set[str]]): """Make the name pins (pins: every pin and its sources, as storage.pins() gives them) match the saved @@ -514,13 +532,16 @@ def reconcile_pins(self, pins: dict[Cid, set[str]]): release them; a pin of an alias with no key file at all is released, but one whose key file is there and can't be read is kept. """ - self.release_gone() + released = self.release_gone() with self._lock: held: dict[str, set[Cid]] = {} for cid, sources in pins.items(): for source in sources: - if source.startswith(PIN_SOURCE_PREFIX): - held.setdefault(source[len(PIN_SOURCE_PREFIX):], set()).add(cid) + if not source.startswith(PIN_SOURCE_PREFIX): + continue + alias = source[len(PIN_SOURCE_PREFIX):] + if (alias, cid) not in released: # pins taken before release_gone() let go of these + held.setdefault(alias, set()).add(cid) for alias in sorted(set(held) | set(self._keys)): cids = held.get(alias, set()) if alias in self._unreadable: @@ -585,7 +606,7 @@ def publish(self, alias: str, cid: Cid) -> dict: raise ValueError("publish needs a CID") with self._lock: name = self.name(alias) - refresh = self._published[alias].imported and self._published[alias].cid is None + refresh = self._published[alias].unsettled if refresh: # A freshly imported key: its previous home may have published sequences we've never heard, which # keepers may hold after its announces have expired (section 9: above the highest it can resolve) @@ -596,7 +617,8 @@ def publish(self, alias: str, cid: Cid) -> dict: self.offer(name, sequence, kept_cid, heard) with self._lock: - self._identity(alias) # removed while refreshing? + if self.name(alias) != name: # NotFound if removed while refreshing + raise NotFound(f"name {alias} was replaced while publishing") self._absorb_recalled(name) previous = self._published[alias] known = self._record(name, touch=False) @@ -624,10 +646,11 @@ def publish(self, alias: str, cid: Cid) -> dict: if cid != previous.cid and cid not in self._stale.get(alias, set()): self._release(alias, {cid}) raise - self._drop_from_table(name) + self._hold(name) self._release(alias, ({previous.cid} | self._stale.pop(alias, set())) - {cid, None}) self._announce(alias) identity = self._keys[alias] + self._accepted(name, self._held[name]) log(f"published {name.hex()} ({alias}) -> {cid}, sequence {sequence}", LOG_NOTICE) if self.keepers is not None: # This node keeps its own names, so they resolve through it while the owner's announce is gone @@ -703,6 +726,48 @@ def run(self, stop_event: threading.Event): delay = _MAX_LOOP_WAIT stop_event.wait(min(delay, _MAX_LOOP_WAIT)) + # Watching + + def watch(self, name: bytes): + """Watch name: its announces count as asked, and record listeners hear of its newer records.""" + name = _check_name(name) + with self._lock: + self._watched[name] = self._watched.get(name, 0) + 1 + self._heard_only.pop(name, None) # asked now: no longer the first a full table gives up + + def unwatch(self, name: bytes): + """Undo one watch(name).""" + with self._lock: + count = self._watched.get(name, 0) - 1 + if count > 0: + self._watched[name] = count + else: + self._watched.pop(name, None) + + def add_record_listener(self, listener): + """Call listener(name, record) for each newer record accepted for a watched name (an announce, a path + response, a keeper's answer, or a publish here). It is called with the names lock held, on whichever + thread accepted the record (RNS's included), so it must only hand the record on, never block or call + back into rnfs.""" + with self._lock: + self._record_listeners = (*self._record_listeners, listener) + + def remove_record_listener(self, listener): + """Stop calling listener.""" + with self._lock: + # ==, not is: a bound method is a new object at each access, equal to the one added + self._record_listeners = tuple(item for item in self._record_listeners if item != listener) + + def _accepted(self, name: bytes, record: Record): + """A newer record was accepted for name: tell the record listeners if it is watched. Lock held.""" + if name not in self._watched: + return + for listener in self._record_listeners: + try: + listener(name, record) + except Exception as e: # noqa: BLE001 - one listener mustn't break the others, or the announce + log(f"record listener failed: {e!r}", LOG_ERROR) + # Resolving def resolve(self, name: bytes, persist: bool, max_age: float = 0.0) -> dict | None: @@ -721,7 +786,7 @@ def resolve(self, name: bytes, persist: bool, max_age: float = 0.0) -> dict | No fresh = held is not None and max_age > 0 and 0 <= self._clock() - held.heard < max_age alias = self._aliases.get(name) # This node's own names need no refresh, unless imported and not yet published here - local = alias is not None and not (self._published[alias].imported and self._published[alias].cid is None) + local = alias is not None and not self._published[alias].unsettled if not local and not fresh: started = self._clock() self._refresh(name) @@ -743,7 +808,7 @@ def resolve(self, name: bytes, persist: bool, max_age: float = 0.0) -> dict | No if record is None: return None if persist and name not in self._held: - self._drop_from_table(name) + self._hold(name) self._held[name] = record self._save() elif name not in self._held: @@ -816,20 +881,21 @@ def _absorb_recalled(self, name: bytes): def _heard(self, name: bytes, app_data, packet_hash: bytes | None = None): """Announce handler: apply a heard record and wake resolves waiting for it.""" + name = bytes(name) parsed = parse_record(app_data) # A name kept here may take a table place like a resolved one (asked outside our lock, as always) - kept = parsed is not None and self.keepers is not None and self.keepers.local_record(bytes(name)) is not None + kept = parsed is not None and self.keepers is not None and self.keepers.local_record(name) is not None with self._lock: if parsed is not None: - # Unasked: not being resolved (a path response would have a waiter) nor kept - unasked = not kept and name not in self._waiters - self._offer(bytes(name), parsed[0], parsed[1], self._clock(), unasked=unasked) + # Unasked: not being resolved (a path response would have a waiter), kept nor watched + unasked = not kept and name not in self._waiters and name not in self._watched + self._offer(name, parsed[0], parsed[1], self._clock(), unasked=unasked) else: log(f"ignored malformed record for name {RNS.prettyhexrep(name)}", LOG_VERBOSE) for event in self._waiters.get(name, ()): event.set() if parsed is not None and self.keepers is not None: - self.keepers.heard(bytes(name), parsed[0], parsed[1], packet_hash) + self.keepers.heard(name, parsed[0], parsed[1], packet_hash) def offer(self, name: bytes, sequence: int, cid: Cid, heard: float) -> bool: """Apply a record whose signature the caller verified (a keeper's answer), under the sequence rules. @@ -857,16 +923,15 @@ def _record(self, name: bytes, touch: bool) -> Record | None: return record record = self._table.get(name) if record is not None and touch: - self._table.move_to_end(name) - if name in self._heard_only: - self._heard_only.move_to_end(name) + self._touch(name) return record def _offer(self, name: bytes, sequence: int, cid: Cid, heard: float, unasked: bool = False) -> bool: """Apply a validly signed record: True if it replaced the one held. Lower sequences (than the record held, or the floor of a name resolved before) are refused, and at an equal sequence the record already held is kept. unasked: an announce of a name this node isn't resolving or keeping, which takes a table place only - if it is free or held by another name that was only heard.""" + if it is free or held by another name that was only heard. A held record whose save fails is logged and + saved by the next save.""" held_here = name in self._held current = self._record(name, touch=False) floor = None if held_here or name not in self._floor else self._floor[name][0] @@ -887,10 +952,18 @@ def _offer(self, name: bytes, sequence: int, cid: Cid, heard: float, unasked: bo self._touch(name) return False - if not self._store(name, Record(sequence, cid, heard), held_here, unasked): + record = Record(sequence, cid, heard) + if not self._store(name, record, held_here, unasked): return False if held_here: - self._save() + try: + self._save() + except OSError as e: + # Held all the same, and saved by the next save (announce_due's within a minute): raising here + # would skip the waking, keepers and listeners, and on RNS's thread reach no one + log(f"could not save the record of name {name.hex()}: {e}", LOG_WARNING) + self._held_dirty = True + self._accepted(name, record) return True def _store(self, name: bytes, record: Record, held_here: bool, unasked: bool = False) -> bool: @@ -931,10 +1004,19 @@ def _touch(self, name: bytes): self._heard_only.move_to_end(name) def _drop_from_table(self, name: bytes): - """Remove name from the table (it is held now).""" + """Remove name from the table.""" self._table.pop(name, None) self._heard_only.pop(name, None) + def _hold(self, name: bytes): + """name's record is held now: out of the table, and out of the floor (its held record is its floor), whose + row the next floor save deletes.""" + self._drop_from_table(name) + if self._floor.pop(name, None) is not None: + del self._floor_used[name] + self._floor_changed.add(name) + self._floor_dirty = True + def _resolved(self, name: bytes, sequence: int) -> bool: """A resolve without persist returned name's record: it is no longer only heard, and its sequence is the floor for later resolves (_trim_floor says which are forgotten). A floor raised (or a new one) is saved at @@ -1041,8 +1123,8 @@ def _identity(self, alias: str) -> RNS.Identity: return identity def _check_free(self, alias: str): - """ValueError if alias is in use (in memory or on disk).""" - if alias in self._keys or os.path.exists(self._key_path(alias)): + """ValueError if alias is in use (in memory or on disk, a dangling symlink included).""" + if alias in self._keys or os.path.lexists(self._key_path(alias)): raise ValueError(f"alias {alias} already exists") def _add_key(self, alias: str, identity: RNS.Identity, published: _Published): @@ -1062,14 +1144,16 @@ def _add_key(self, alias: str, identity: RNS.Identity, published: _Published): self._open_destination(alias) def _remove_old_temps(self): - """Delete the temp files an interrupted write of rnfs 0.1.1 or earlier left (names.json.tmp and - ..tmp): it reused those names, but writes now take new ones each time.""" + """Delete the temp files an interrupted key write left: fsutil.write_atomic's (.partial-<16 hex>), which + hold a private key, and those of rnfs 0.1.1 or earlier (names.json.tmp and ..tmp), which reused + those names. Only this node writes here (it holds the rnfs dir's lock), and not yet.""" try: entries = os.listdir(self.config.names_dir) except OSError: entries = [] old = [os.path.join(self.config.names_dir, entry) for entry in entries - if entry.startswith(".") and entry.endswith(".tmp") and _ALIAS.fullmatch(entry[1:-4])] + if (entry.startswith(".") and entry.endswith(".tmp") and _ALIAS.fullmatch(entry[1:-4])) + or _PARTIAL.fullmatch(entry)] for path in (os.path.join(self.config.state_dir, STATE_FILE + ".tmp"), *old): try: fsutil.remove(path) @@ -1132,7 +1216,7 @@ def _load_floor(self): """Read the floor of names resolved without persist, least recently resolved first. A row that doesn't parse is logged and deleted (the floor only bounds rollback of gateway and auto-fetch resolves), as are those of names held for the user and those _trim_floor drops. A resolve time in the future (the clock was - corrected since) counts as now, and the next save writes it so.""" + corrected since) counts as now, and is written so by the next floor save that has a change to write.""" now = self._clock() dropped = [] for name, sequence, resolved, used in self._db.query( diff --git a/rnfs/network/announces.py b/rnfs/network/announces.py index 11d3aa8..990304a 100644 --- a/rnfs/network/announces.py +++ b/rnfs/network/announces.py @@ -40,13 +40,12 @@ def parse_mode(app_data) -> int: class DhtAnnouncer: """Announces rnfs.dht at startup, after startup_announce_delay, then every announce_interval.""" - def __init__(self, destination: RNS.Destination, config: Config, server: bool | None = None): - """Set destination's app_data to the mode (config.dht_mode unless server is given); start() announces.""" + def __init__(self, destination: RNS.Destination, config: Config): + """Set destination's app_data to the mode (config.dht_mode); start() announces.""" self.destination = destination self.config = config - self.server = (config.dht_mode == "server") if server is None else server + self.server = config.dht_mode == "server" self.stop_event = threading.Event() - self.lock = threading.Lock() self.thread: threading.Thread | None = None destination.set_default_app_data(encode_mode(self.server)) @@ -58,15 +57,6 @@ def announce(self): except Exception as e: log(f"could not announce: {e}", LOG_WARNING) - def set_server(self, server: bool): - """Change the DHT mode, re-announcing if it changed.""" - with self.lock: - if server == self.server: - return - self.server = server - self.destination.set_default_app_data(encode_mode(server)) - self.announce() - def start(self): """Start the announce schedule on a background thread.""" if self.thread is None: diff --git a/rnfs/network/client.py b/rnfs/network/client.py index 8654790..449b936 100644 --- a/rnfs/network/client.py +++ b/rnfs/network/client.py @@ -57,14 +57,12 @@ def open_links(self) -> list[bytes]: def rate_estimate(self, node_id: bytes) -> float | None: """The rate estimate (bits/s) of the open link to node_id, or None.""" - with self.links.lock: - peer = self.links.shared.get(node_id) + peer = self.links.shared_peer(node_id) return peer.rate if peer is not None else None def link_mdu(self, node_id: bytes) -> int | None: """The packet MDU of the open link to node_id, which bounds the requests sent on it, or None.""" - with self.links.lock: - peer = self.links.shared.get(node_id) + peer = self.links.shared_peer(node_id) return peer.mdu() if peer is not None and peer.is_active() else None def close(self, node_id: bytes): @@ -166,7 +164,8 @@ def ping(self, node_id: bytes) -> float | None: # /blocks - def blocks(self, node_id: bytes, cids: list[Cid], on_progress: Callable[[float], None] | None = None) -> BlocksResult: + def blocks(self, node_id: bytes, cids: list[Cid], + on_progress: Callable[[float], None] | None = None) -> BlocksResult: """_blocks(), counted in traffic (mesh): the outcome, verified bytes received and bad blocks.""" result = self._blocks(node_id, cids, on_progress) if result.status != OK or result.blocks or result.missing or result.bad: @@ -177,11 +176,11 @@ def blocks(self, node_id: bytes, cids: list[Cid], on_progress: Callable[[float], def _interface(self, node_id: bytes) -> str | None: """The name of the interface the shared link to node_id runs over, or None.""" - with self.links.lock: - peer = self.links.shared.get(node_id) + peer = self.links.shared_peer(node_id) return mesh.interface_label(getattr(getattr(peer, "link", None), "attached_interface", None)) - def _blocks(self, node_id: bytes, cids: list[Cid], on_progress: Callable[[float], None] | None = None) -> BlocksResult: + def _blocks(self, node_id: bytes, cids: list[Cid], + on_progress: Callable[[float], None] | None = None) -> BlocksResult: """Ask node_id for blocks over its shared link, verifying each against its CID. Sends as many of cids (duplicates dropped) as fit one packet; the rest come back in .unsent. diff --git a/rnfs/network/guards.py b/rnfs/network/guards.py index 881fb21..e0ed50c 100644 --- a/rnfs/network/guards.py +++ b/rnfs/network/guards.py @@ -1,12 +1,20 @@ -"""Guards on RNS: process-wide ones on incoming Resource advertisements and responses (sections 3 and 10: -oversized-response and decoding bombs), and one per destination on half-open incoming links (section 3: link -exhaustion). +"""Guards on RNS: on incoming Resource advertisements and responses (sections 3 and 10: oversized-response and +decoding bombs, compression), and one per destination on half-open incoming links (section 3: link exhaustion). + +Their scope is set by install(scope). The daemon owns its process (PROCESS): the advertisement checks below +cover every link, and RNS's process-wide decompression limit (Resource.AUTO_COMPRESS_MAX_SIZE) is 0, which also +stops RNS compressing anything the process sends. A node embedded in another program (DHT: rnfs.Node) guards +only rnfs.dht links and leaves that limit as the program has it, so the program's own requests, Resources and +compressed transfers behave as with RNS alone; on rnfs.dht links every check still applies, and a compressed +Resource is refused there just the same (below). rnfs never compresses what it sends on rnfs.dht links in +either scope: responses are registered with auto_compress=False, which RNS passes to a response Resource, and +LinkPool sends no request that wouldn't fit one packet, so no request Resource either. RNS checks Destination.set_max_request_size and a request's max_response_size only against an advertisement's data size (d), but receives, buffers and allocates by its transfer size (t). RNS 1.5.6 caps t at 3 × Resource.MAX_EFFICIENT_SIZE (~3 MiB) in ResourceAdvertisement.unpack, so a 1 KiB request or a small -response could still cost ~3 MiB per link. rnfs runs in its own process (section 3), so it wraps -RNS.Resource.accept once to reject, before anything is allocated: +response could still cost ~3 MiB per link. So RNS.Resource.accept is wrapped once to reject, before anything is +allocated (on every link with PROCESS, on rnfs.dht links only with DHT): - every request sent as a Resource: rnfs requests always fit in one packet; - any advertisement whose transfer size exceeds its data size plus encryption overhead, or exceeds MAX_RESPONSE_SIZE plus that overhead (the largest response rnfs accepts is one Resource segment); @@ -15,7 +23,11 @@ legitimate rnfs response fits one. Other destinations' Resources (LXMF's, whose messages may span segments) are told apart by the link's destination name, which this node chose itself; - on rnfs.dht links, any Resource carrying metadata: RNS decodes metadata with umsgpack as the Resource - completes, before anyone can look at it, and rnfs responses never have any. + completes, before anyone can look at it, and rnfs responses never have any; +- on rnfs.dht links, any compressed Resource (the advertisement's compressed flag): RNS would decompress it as + it completes, before any rnfs check, and rnfs never compresses. Refused before any part is transferred, and + each accepted rnfs.dht Resource also gets a decompression limit of 0, so one that passed couldn't be + decompressed either. RNS also decodes a whole response with umsgpack before handing it to rnfs, and a payload of one-byte empty arrays or maps decodes to some 70 times its size in Python objects (1 MiB: ~70 MB and ~0.3 s of CPU). So on @@ -55,7 +67,12 @@ TRANSFER_OVERHEAD = 128 MAX_TRANSFER_SIZE = MAX_RESPONSE_SIZE + TRANSFER_OVERHEAD +PROCESS = "process" # install()'s scopes: every link of the process (the daemon) ... +DHT = "dht" # ... or rnfs.dht links only (a node embedded in another program) + _lock = threading.Lock() +_scope = PROCESS # what the advertisement checks cover; set by install() +_compress_limit = None # Resource.AUTO_COMPRESS_MAX_SIZE before a PROCESS install set it to 0, if one did _original_accept = None _original_response_resource_concluded = None _original_receive = None @@ -69,7 +86,8 @@ def is_dht_link(link) -> bool: """True if link (an RNS.Link, either end) belongs to rnfs.dht: its destination's name is "rnfs.dht" plus the identity's hex hash, as RNS.Destination.expand_name builds it.""" - # An outgoing link has the remote destination; an incoming one has none, and our own as its owner + # An outgoing link has the remote destination; an incoming one ours, set (with owner) by + # Link.validate_request before the link is registered, so from its first packet on destination = getattr(link, "destination", None) or getattr(link, "owner", None) name = getattr(destination, "name", None) return isinstance(name, str) and (name == DHT_NAME or name.startswith(DHT_NAME + ".")) @@ -86,8 +104,16 @@ def _report_unsupported(): f"incoming Resources are being refused", LOG_ERROR) +def scope() -> str: + """What the advertisement checks cover: PROCESS or DHT (install()).""" + return _scope + + def refusal(adv, link=None) -> str | None: """Why an unpacked ResourceAdvertisement arriving on link must be rejected, or None if it may be accepted.""" + dht = link is not None and is_dht_link(link) + if not dht and _scope == DHT: + return None # an embedded node's host program: its links are RNS's alone transfer, data = getattr(adv, "t", _ABSENT), getattr(adv, "d", _ABSENT) if transfer is _ABSENT or data is _ABSENT: # Not a malformed advertisement but an RNS that renamed the fields: every Resource is refused (failing @@ -102,10 +128,12 @@ def refusal(adv, link=None) -> str | None: return f"transfer size {transfer} exceeds data size {data}" if transfer > MAX_TRANSFER_SIZE: return f"transfer size {transfer} exceeds the largest response" - if link is not None and is_dht_link(link) and (getattr(adv, "l", 1) != 1 or getattr(adv, "i", 1) != 1): + if dht and (getattr(adv, "l", 1) != 1 or getattr(adv, "i", 1) != 1): return f"rnfs responses are one segment, not {getattr(adv, 'l', '?')}" - if link is not None and is_dht_link(link) and getattr(adv, "x", False): + if dht and getattr(adv, "x", False): return "rnfs responses carry no metadata" + if dht and getattr(adv, "c", False): # last: whatever else the daemon refused is logged as before + return "rnfs Resources are never compressed" return None @@ -124,19 +152,24 @@ def _fail_pending_request(packet, adv): def guarded_accept(advertisement_packet, callback=None, progress_callback=None, request_id=None): - """RNS.Resource.accept, but rejecting advertisements refusal() objects to.""" + """RNS.Resource.accept, but rejecting advertisements refusal() objects to. An rnfs.dht Resource accepted + decompresses nothing (refusal() already refused a compressed one).""" + link = getattr(advertisement_packet, "link", None) try: adv = RNS.ResourceAdvertisement.unpack(advertisement_packet.plaintext) except Exception: adv = None # let RNS handle an advertisement it can't read, as it always has - reason = refusal(adv, getattr(advertisement_packet, "link", None)) if adv is not None else None + reason = refusal(adv, link) if adv is not None else None if reason is not None: log(f"rejected an incoming Resource: {reason}", LOG_WARNING) RNS.Resource.reject(advertisement_packet) _fail_pending_request(advertisement_packet, adv) return None - return _original_accept(advertisement_packet, callback=callback, progress_callback=progress_callback, - request_id=request_id) + resource = _original_accept(advertisement_packet, callback=callback, progress_callback=progress_callback, + request_id=request_id) + if isinstance(resource, RNS.Resource) and link is not None and is_dht_link(link): + resource.max_decompressed_size = 0 # what Resource.assemble decompresses at most + return resource # Responses checked before RNS decodes them @@ -230,21 +263,44 @@ def guarded_receive(link, packet): return _original_receive(link, packet) +def _wrapped(owner, name: str, guard) -> bool: + """True if owner.name (set on owner itself, plain or as a staticmethod) is guard.""" + current = owner.__dict__.get(name) + return getattr(current, "__func__", current) is guard + + def _wrap(owner, name: str, guard, staticmethod_: bool = False): """Replace owner.name with guard unless it is already; the original it replaced.""" - current = owner.__dict__.get(name) - if getattr(current, "__func__", current) is guard: + if _wrapped(owner, name, guard): return None original = getattr(owner, name) setattr(owner, name, staticmethod(guard) if staticmethod_ else guard) return original -def install(): +def install(scope: str | None = None): """Wrap RNS.Resource.accept, Link.response_resource_concluded and Link.receive with the guards. Idempotent; - called by RequestServer and LinkPool.""" - global _original_accept, _original_response_resource_concluded, _original_receive + called by the node, RequestServer and LinkPool. + + scope is PROCESS (the daemon: the advertisement checks cover every link, and RNS's process-wide + decompression limit is set to 0, which also stops it compressing what this process sends) or DHT (an + embedded node: rnfs.dht links only, and the limit is left as the host program has it, or put back if a + PROCESS install changed it); None keeps the scope already set (PROCESS at first). + """ + global _original_accept, _original_response_resource_concluded, _original_receive, _scope, _compress_limit with _lock: + if scope is not None: + if scope not in (PROCESS, DHT): + raise ValueError(f"unknown guard scope {scope!r}") + _scope = scope + if _scope == PROCESS: + # Refuses compressed incoming Resources (RNS decompresses at most this many bytes, so any compressed + # payload fails) and stops RNS compressing outgoing ones (section 3) + if _compress_limit is None: + _compress_limit = RNS.Resource.AUTO_COMPRESS_MAX_SIZE + RNS.Resource.AUTO_COMPRESS_MAX_SIZE = 0 + elif _compress_limit is not None: + RNS.Resource.AUTO_COMPRESS_MAX_SIZE, _compress_limit = _compress_limit, None original = _wrap(RNS.Resource, "accept", guarded_accept, staticmethod_=True) if original is not None: _original_accept = original @@ -258,12 +314,9 @@ def install(): def installed() -> bool: """True if the guards are in place.""" - def wrapped(owner, name, guard): - current = owner.__dict__.get(name) - return getattr(current, "__func__", current) is guard - return (wrapped(RNS.Resource, "accept", guarded_accept) - and wrapped(RNS.Link, "response_resource_concluded", guarded_response_resource_concluded) - and wrapped(RNS.Link, "receive", guarded_receive)) + return (_wrapped(RNS.Resource, "accept", guarded_accept) + and _wrapped(RNS.Link, "response_resource_concluded", guarded_response_resource_concluded) + and _wrapped(RNS.Link, "receive", guarded_receive)) # Half-open incoming links diff --git a/rnfs/network/identity.py b/rnfs/network/identity.py index 3b8905c..fe9e7cd 100644 --- a/rnfs/network/identity.py +++ b/rnfs/network/identity.py @@ -152,10 +152,3 @@ def parse_node_id(hex_id: str) -> bytes: raise ValueError(f"node ID must be {ID_LENGTH * 2} hex characters: {hex_id}") return node_id - -def parse_identity_hash(hex_hash: str) -> bytes: - """RNS identity hash from its 32-hex-character form (members, remote_pin_identities). Raises ValueError.""" - identity_hash = bytes.fromhex(hex_hash.strip().strip("<>")) - if len(identity_hash) != RNS.Identity.TRUNCATED_HASHLENGTH // 8: - raise ValueError(f"identity hash must be {RNS.Identity.TRUNCATED_HASHLENGTH // 4} hex characters: {hex_hash}") - return identity_hash diff --git a/rnfs/network/links.py b/rnfs/network/links.py index e903f52..35bd6d3 100644 --- a/rnfs/network/links.py +++ b/rnfs/network/links.py @@ -14,6 +14,8 @@ were sent (and wait no longer than that for a slot); /blocks has the fetcher's stall detection instead. - A request answered False on an identified link is retried once after identifying again, since Link.identify() is unacknowledged. +- A request that wouldn't fit one packet on its link isn't sent: RNS would send it as a Resource, which nodes + refuse (network/guards.py) and RNS may compress. Everything here blocks its caller, so never call it from an RNS callback thread. """ @@ -134,11 +136,6 @@ def __init__(self, node_id: bytes, identified: bool, shared: bool): self.handed_out: float | None = None # when LinkPool.get() last returned it (the pool's clock) self.cond = threading.Condition() - @property - def link_id(self) -> bytes | None: - """RNS link ID, or None before the link exists.""" - return getattr(self.link, "link_id", None) - def is_active(self) -> bool: """True while the link is up.""" return not self.closed and self.link is not None and self.link.status == RNS.Link.ACTIVE @@ -213,7 +210,7 @@ def __init__(self, identity: RNS.Identity, config: Config, clock: Callable[[], f self.lock = threading.Lock() self.stop_event = threading.Event() self.sweeper: threading.Thread | None = None - # Process-wide: reject response advertisements whose transfer outgrows their size + # Reject response advertisements whose transfer outgrows their size (in the scope the node set) guards.install() # Opening links @@ -368,11 +365,14 @@ def close_link(self, peer: PeerLink | None): if peer.link is not None: peer.link.teardown() + def shared_peer(self, node_id: bytes) -> PeerLink | None: + """The shared link to node_id (whatever its state), or None.""" + with self.lock: + return self.shared.get(node_id) + def close(self, node_id: bytes): """Tear down the shared link to node_id, if any (e.g. after a stalled transfer).""" - with self.lock: - peer = self.shared.get(node_id) - self.close_link(peer) + self.close_link(self.shared_peer(node_id)) def close_all(self): """Stop the idle sweeper and tear down every link.""" @@ -459,7 +459,8 @@ def request(self, peer: PeerLink, path: str, data, blocks: bool = False, max_res watchdog(exchange, now) may return a reason to stop waiting (a stall). on_progress(fraction) is called from this thread, never an RNS one. A request other than /blocks gives up (.stalled set) once - request_deadline has passed since it was sent, or if it waited that long for a slot. + request_deadline has passed since it was sent, or if it waited that long for a slot. A request larger + than the link's packet MDU isn't sent (the exchange is returned unanswered). """ exchange = Exchange(self.clock) deadline = None if blocks else self.request_deadline(peer) @@ -471,6 +472,12 @@ def request(self, peer: PeerLink, path: str, data, blocks: bool = False, max_res exchange.done.set() return exchange try: + mdu = peer.mdu() + if mdu is not None and protocol.request_size(path, data) > mdu: + # Link.request would send it as a Resource, which nodes refuse, and which RNS compresses unless + # the daemon's process-wide limit is set: every rnfs request fits one packet (section 3) + log(f"not sending {path} to {node_name(peer.node_id)}: it doesn't fit one packet", LOG_WARNING) + return exchange try: # timeout=None: RNS's RTT-based request timeout (section 3) receipt = peer.link.request(path, data, response_callback=exchange.on_response, diff --git a/rnfs/network/rates.py b/rnfs/network/rates.py index 2244c4b..29db971 100644 --- a/rnfs/network/rates.py +++ b/rnfs/network/rates.py @@ -30,7 +30,7 @@ def __init__(self, min_rate_sample: int): """No estimates yet; responses under min_rate_sample bytes are ignored.""" self.min_rate_sample = min_rate_sample self.rates: dict[bytes, float] = {} - self.samples = 0 # estimates recorded so far, over every link (for status and tests) + self.samples = 0 # estimates recorded so far, over every link (read by tests) self.lock = threading.Lock() def record(self, link_id: bytes, response_size: int | None, expected_rate: float | None) -> bool: diff --git a/rnfs/network/server.py b/rnfs/network/server.py index c8b4188..1f02aed 100644 --- a/rnfs/network/server.py +++ b/rnfs/network/server.py @@ -70,10 +70,11 @@ # that many per round trip. Over real RNS on loopback (the shortest round trip there is), a 10,000-key /provide # (182 requests, one at a time) ran at ~900 a second and two threads sending /find back to back at 2,100-3,600; # the busiest link in the end-to-end tests saw 13 in a second. So the limit binds only on a link sending faster -# than any rnfs requester over a real network (or a tight loop like that last one, slowed to the rate). Requests over it are answered False while the link has refusals -# left (OVER_RATE_REFUSALS, one more a second), so a requester that gets there backs off as from any refusal, -# and dropped unanswered after that. RNS has already decrypted and decoded each request and started its thread -# by then; what the limit saves is the handler's work and the answer (about 40% of a /find's cost to the node) +# than any rnfs requester over a real network (or a tight loop like that last one, slowed to the rate). +# Requests over it are answered False while the link has refusals left (OVER_RATE_REFUSALS, one more a +# second), so a requester that gets there backs off as from any refusal, and dropped unanswered after that. +# RNS has already decrypted and decoded each request and started its thread by then; what the limit saves is +# the handler's work and the answer (about 40% of a /find's cost to the node) REQUEST_BURST = 2000 REQUEST_RATE = 2000.0 OVER_RATE_REFUSALS = 2 * MAX_REQUESTS_PER_LINK @@ -107,6 +108,10 @@ def remote_pin(self, identity_hash: bytes, cid: Cid, pin: bool) -> bool: def is_member(self, identity_hash: bytes) -> bool: """In private mode, True if the identity with this 16-byte hash is a member.""" + def feature_requests(self) -> dict: + """Optional (looked up with getattr): request paths added by features, {path: handler(data, + remote_identity)}, served within the same per-link limits; a path rnfs serves itself is ignored.""" + # Pure policy @@ -681,10 +686,9 @@ def __init__(self, destination: RNS.Destination, config: Config, backend: Backen self.over_rate_logged_at: float | None = None self.over_rate_logged = 0 - # Process-wide: refuses compressed incoming Resources (RNS decompresses at most this many bytes, so - # any compressed payload fails) and stops RNS compressing outgoing ones (section 3) - RNS.Resource.AUTO_COMPRESS_MAX_SIZE = 0 - # Process-wide: reject request Resources and advertisements whose transfer outgrows their size + # Reject request Resources, compressed ones and advertisements whose transfer outgrows their size: on + # every link in the daemon, which also turns RNS's compression off process-wide, or on rnfs.dht links + # in an embedded node (guards.install, in the scope the node set) guards.install() # Links that never finish their handshake are kept in a bucket of their own, below max_incoming_links guards.limit_half_open(destination, config.max_incoming_links) @@ -793,7 +797,9 @@ def _find_link(self, link_id: bytes): # Handlers def _guarded(self, handler): - """Wrap handler so an unexpected error answers False rather than leaving the requester waiting.""" + """Wrap handler with the link's rate limit (over it: False, or None, which RNS doesn't answer), the + private-mode membership check (False for a non-member), and an answer of False for an unexpected error + rather than leaving the requester waiting.""" def response_generator(path, data, request_id, link_id, remote_identity, requested_at): """RNS's six-argument request handler signature.""" # First, before any parsing: the link's rate limit. None: RNS sends no answer diff --git a/rnfs/node.py b/rnfs/node.py index a3d8db0..ef158ca 100644 --- a/rnfs/node.py +++ b/rnfs/node.py @@ -1,16 +1,23 @@ """RnfsNode: one running rnfs node, wiring Reticulum to the DHT, storage, fetching and publishing, and the facade that the API, gateway and CLI call (design.md, "Node facade"). -Startup: Reticulum from (attaching to a shared instance if its config shares one), the node's -identity and its rnfs.dht destination, then the state database (rnfs.statedb, which imports the JSON state files -of earlier versions first), the routing table, provider records, storage, shared folder, fetcher, publisher and -names; the request server answers through NodeBackend. Background threads: the -announcer, the join (bootstrap nodes, then a lookup for our own ID, repeated while too few servers are -known), the publish loop, the name re-announce loop and -maintenance (shared-folder scans, provider record expiry and saves, max_storage enforcement). +Startup: Reticulum from (attaching to a shared instance if its config shares one), or for a node +embedded in another program (rnfs.Node) the program's running instance, then the node's identity and its +rnfs.dht destination, then the state database (rnfs.statedb, which imports the JSON state files of earlier +versions first), the routing table, provider records, storage, the deny list (its accepted lists read back from +storage), shared folder, fetcher, publisher, names and the features (rnfs.features); the request server answers +through NodeBackend. Background threads: the announcer, the join (bootstrap nodes, then a lookup for our own ID, +repeated while too few servers are known), the publish loop, the name re-announce loop, the features' own, a +worker per source with remote pins waiting, and maintenance (shared-folder scans, provider record expiry and +saves, max_storage enforcement, periodic state saves, publisher refreshes asked for, deny-list changes retried, +the remote pins waiting saved). Threading: RNS calls the announce and request handlers on its own threads; facade calls come from API and gateway threads. Every component locks internally; the node itself only guards the small sets it owns. + +The daemon owns its process, so its Resource guards cover every link and RNS's compression is off process-wide +(network/guards.py, PROCESS); an embedded node scopes both to its rnfs.dht links (DHT) and leaves the host +program's Reticulum running when it stops. Neither ever stops Reticulum: the daemon calls RNS.exit() itself. """ import hashlib import os @@ -26,7 +33,7 @@ import RNS from rnfs import __version__, denylist, features, fsutil, statedb, winacl -from rnfs.config import Config +from rnfs.config import LOCK_FILE, Config from rnfs.constants import APP_NAME, DEFAULT_BOOTSTRAP_NODES, DHT_ASPECT, K, MODE_SERVER, PARTIAL_SUFFIX from rnfs.content import archive, builder, dag, edit, seal, walker from rnfs.content.cid import Cid @@ -132,6 +139,31 @@ def _make_private_dirs(config: Config): os.makedirs(directory, mode=0o700, exist_ok=True) +class DirInUse(RnfsError): + """Another daemon or embedded node holds the rnfs dir's lock; holder is its process ID, if known.""" + + def __init__(self, message: str, holder: int | None): + """The error, naming holder.""" + super().__init__(message) + self.holder = holder + + +def lock_rnfs_dir(config: Config) -> int: + """Make the rnfs dir private (as the node would) and lock /daemon.lock without waiting, before + anything else uses the dir: two nodes on one dir would load and save the same state and announce the same + node ID. The open descriptor, whose lock lasts until it is closed or the process ends. Raises DirInUse if + another process, or another node in this one, holds it; RnfsError or OSError if the dir can't be made + private or the file locked. The daemon and rnfs.Node both lock through here.""" + _make_private_dirs(config) + path = os.path.join(config.root, LOCK_FILE) + try: + return fsutil.lock_exclusive(path) + except BlockingIOError: + holder = fsutil.lock_holder(path) + raise DirInUse(f"another rnfs daemon or node{f' (process {holder})' if holder else ''} is using " + f"{config.root}; stop it first, or use another rnfs dir", holder) from None + + def _check_free_space(path: str, needed: int, blocks_dir: str | None = None, to_fetch: int = 0): """Raise TooLarge unless writing needed bytes to path leaves the disk reserve free on its filesystem. @@ -269,19 +301,27 @@ def feature_requests(self) -> dict: class RnfsNode: - """A running node. Only one per process: RNS.Reticulum is a singleton, and rnfs changes a process-wide - RNS setting (section 3).""" + """A running node. Only one per process: RNS.Reticulum is a singleton, and the guards are installed once + (network/guards.py).""" + + def __init__(self, config: Config, reticulum: "RNS.Reticulum | None" = None, embedded: bool = False): + """Start every service of the node, then announce, join and publish in the background. - def __init__(self, config: Config): - """Start Reticulum and every service of the node, then announce, join and publish in the background.""" + The daemon starts Reticulum here. An embedded node (rnfs.Node, embedded) uses reticulum, the host + program's running instance, or starts one if there is none; its guards cover only rnfs.dht links. + """ config.validate() _make_private_dirs(config) # before Reticulum and the identity, which write under it - # Refuse compressed Resources and stop compressing ours (section 3); the server sets it too - RNS.Resource.AUTO_COMPRESS_MAX_SIZE = 0 - guards.install() # reject oversized Resource advertisements before RNS allocates for them + self.embedded = embedded + # Reject oversized, compressed and request Resource advertisements before RNS allocates for them; in the + # daemon on every link, with RNS's compression off process-wide (section 3) + guards.install(guards.DHT if embedded else guards.PROCESS) + if reticulum is None and embedded: + reticulum = RNS.Reticulum.get_instance() # By default this is RNS's own config directory, the one rnsd uses: attaching to a shared instance needs # its RPC key, which RNS derives from the transport identity stored in that directory (section 3) - self.reticulum = RNS.Reticulum(configdir=config.reticulum_root, verbosity=config.verbosity or None) + self.reticulum = reticulum if reticulum is not None else \ + RNS.Reticulum(configdir=config.reticulum_root, verbosity=config.verbosity or None) log(f"Reticulum config in {RNS.Reticulum.configdir}" + (", attached to a shared instance" if self.reticulum.is_connected_to_shared_instance else "")) _warn_unused_reticulum_config(config) @@ -290,8 +330,24 @@ def __init__(self, config: Config): APP_NAME, DHT_ASPECT) slow = rates.is_slow_node(config, self.reticulum) self.links = LinkPool(self.identity, config) - self._setup_or_close(config, bytes(self.destination.hash), PeerClient(self.links, config), slow, - names_network=None, is_known=_rns_is_known) + self.server = self.announcer = self.announce_handler = None # made below; stop() skips those not made + try: + self._setup_or_close(config, bytes(self.destination.hash), PeerClient(self.links, config), slow, + names_network=None, is_known=_rns_is_known) + except BaseException: + if embedded: + self._release_destination() # the host's Reticulum runs on: a later node registers it again + raise + try: + self._start_network(config, slow) + except BaseException: + if embedded: + self.stop() # nothing of this node is left in the host's process (the daemon exits instead) + raise + + def _start_network(self, config: Config, slow: bool): + """The services that need Reticulum, once the node is set up: paths, the request server, announces, names + and the background threads.""" # mesh: hops and next-hop interfaces rank providers; one set of traffic counters for client and server self.paths = RnsPaths(self.reticulum) self.fetcher.path_info = self.paths.info @@ -316,6 +372,7 @@ def offline(cls, config: Config, node_id: bytes, client, names_network, slow: bo """ node = cls.__new__(cls) node.reticulum = node.links = node.server = node.announcer = node.announce_handler = None + node.embedded = False node._setup_or_close(config, node_id, client, slow, names_network=names_network, is_known=lambda node_id: True) node.joined.set() @@ -368,12 +425,14 @@ def _setup(self, config: Config, node_id: bytes, client, slow: bool, names_netwo self.fetcher = Fetcher(self.storage, client, config, node_id, self.routing, self._lookup, slow, config.state_dir, after_store=self._enforce_storage, make_room=self._make_room) self.publisher = Publisher(node_id, config, self.storage, self.records, client, self._lookup, slow, - config.state_dir, cached_eligible=self._eligible_targets) + config.state_dir, cached_eligible=self._eligible_targets, + generation=self._provide_generation) self.names = NameManager(config, _NamePins(self), network=names_network) self.backend = NodeBackend(self) self._lock = threading.Lock() self._fetched: set[Cid] = set() + self._fetched_changes = 0 # incremented whenever _fetched changes (_provide_generation) self._fetched_saved: set = set() # what the fetched table holds for (binary,) in self._db.query("SELECT cid FROM fetched"): self._fetched_saved.add(binary) @@ -449,17 +508,35 @@ def stop(self, timeout: float = STOP_TIMEOUT): self.announce_handler.deregister() if self.server is not None: self.server.stop() - self.names.stop() + try: + self.names.stop() # saves the names state + except OSError as e: # stopping goes on: the rest is saved, the state closed and the links too + log(f"saving state failed: {e!r}", LOG_ERROR) deadline = time.monotonic() + timeout for thread in self.threads: thread.join(max(0.0, deadline - time.monotonic())) - self.records.save() - self._save_remote_pins() - self.publisher.save() + for save in (self.records.save, self._save_remote_pins, self.publisher.save): + try: + save() + except OSError as e: + log(f"saving state failed: {e!r}", LOG_ERROR) self._save_state() self._close_state() if self.links is not None: self.links.close_all() + if self.embedded: + self._release_destination() + + def _release_destination(self): + """An embedded node's last step: RNS stops answering for its rnfs.dht destination and the links to it + are closed, while the host program's Reticulum keeps running, so a node started later in the process + can register it again. (The daemon's process ends with RNS.exit() instead.)""" + try: + RNS.Transport.deregister_destination(self.destination) + for link in list(getattr(self.destination, "links", None) or []): + link.teardown() + except Exception as e: # noqa: BLE001 - stopping goes on + log(f"releasing the rnfs.dht destination failed: {e!r}", LOG_ERROR) def _close_state(self): """Release every store's reference to the state database, once everything is saved; the last closes it. @@ -475,9 +552,9 @@ def _close_state(self): log(f"closing state failed: {e!r}", LOG_ERROR) def _save_state(self): - """Save cache records, fetched targets and invalid roots if they changed (each change isn't written at - once).""" - for save in (self.storage.save, self._save_fetched, self.fetcher.save, + """Save cache records, fetched targets, invalid roots and remembered file hashes if they changed (each + change isn't written at once).""" + for save in (self.storage.save, self._save_fetched, self.fetcher.save, self.deny.save, *(f.save for f in self.features.values())): try: save() @@ -497,10 +574,15 @@ def _lookup(self, key: bytes) -> Lookup: nearest = fresh + [n for n in self.routing.closest(key, 2 * K) if n not in fresh] reliable = [n for n in nearest if not self.routing.unreliable(n)] start = (reliable + [n for n in nearest if n not in reliable])[:K] + return self._new_lookup(key, start, self.routing.is_banned) + + def _new_lookup(self, key: bytes, start: list[bytes], exclude: Callable[[bytes], bool]) -> Lookup: + """A Lookup for key from start, wired to the routing table (answers, failures, unreliable nodes and + retries) and the client, leaving out the nodes exclude rejects: _lookup's and the join's.""" return Lookup(key, start, self.client.find, self.node_id, alpha=self.config.effective_alpha(self.slow), beta=self.config.beta, timeout=self.config.lookup_timeout, is_known=self._is_known, - on_answer=self._answered, on_failure=self.routing.failed, exclude=self.routing.is_banned, + on_answer=self._answered, on_failure=self.routing.failed, exclude=exclude, report=lambda text: log(text, LOG_DEBUG), unreliable=self.routing.unreliable, may_retry=self.routing.may_retry) @@ -594,13 +676,8 @@ def join(self): log("no DHT servers known yet: waiting for announces", LOG_VERBOSE) return # The bootstrap nodes contacted already answered (or failed) a /find for this key: not asked again - lookup = Lookup(self.node_id, start, self.client.find, self.node_id, - alpha=self.config.effective_alpha(self.slow), beta=self.config.beta, - timeout=self.config.lookup_timeout, is_known=self._is_known, - on_answer=self._answered, on_failure=self.routing.failed, - exclude=lambda node_id: node_id in contacted or self.routing.is_banned(node_id), - report=lambda text: log(text, LOG_DEBUG), unreliable=self.routing.unreliable, - may_retry=self.routing.may_retry) + lookup = self._new_lookup(self.node_id, start, + lambda node_id: node_id in contacted or self.routing.is_banned(node_id)) lookup.find_closest() log(f"joined the DHT: {plural(self.routing.node_count(), 'server')} known", LOG_NOTICE) @@ -797,6 +874,12 @@ def _eligible_targets(self) -> set[Cid]: with self._lock: return set(self._fetched) + def _provide_generation(self) -> tuple[int, int]: + """What the publisher's provide set is computed from, as counts that grow with every change: storage's + pins, cached targets and blocks (Storage.changes), and the targets fetched for the user.""" + with self._lock: + return self.storage.changes(), self._fetched_changes + def _prune_fetched(self): """Forget fetched targets that are no longer cached.""" with self._lock: @@ -805,6 +888,7 @@ def _prune_fetched(self): if self._fetched <= cached: return self._fetched &= cached + self._fetched_changes += 1 self._fetched_dirty = True def _note_fetched(self, target: Cid): @@ -813,6 +897,7 @@ def _note_fetched(self, target: Cid): if target in self._fetched: return self._fetched.add(target) + self._fetched_changes += 1 self._fetched_dirty = True def _save_fetched(self): @@ -1008,11 +1093,14 @@ def info(self, target: str, for_gateway: bool = False, from_node: bytes | None = """ root, names = self._parse_target(target, for_gateway) with self._fetching(root, names, from_node, for_gateway) as fetch: - expect = fetch.resolve() - block = fetch.get(expect) - info = {"cid": str(expect.cid), "type": walker.kind_of(expect, block), - "size": walker.size_of(expect, block), "executable": expect.kind == dag.KIND_EXECUTABLE} - return info if for_gateway or raw else node_content.sealed_info(self, fetch, expect, info) + return self._info(fetch, fetch.resolve(), for_gateway or raw) + + def _info(self, fetch: Fetch, expect: walker.Expect, stored: bool) -> dict: + """info() of a resolved target; with stored, a sealed file's as stored (its ciphertext size).""" + block = fetch.get(expect) + info = {"cid": str(expect.cid), "type": walker.kind_of(expect, block), + "size": walker.size_of(expect, block), "executable": expect.kind == dag.KIND_EXECUTABLE} + return info if stored else node_content.sealed_info(self, fetch, expect, info) def read(self, target: str, offset: int = 0, length: int | None = None, for_gateway: bool = False, from_node: bytes | None = None, raw: bool = False) -> Iterator[bytes]: @@ -1027,18 +1115,50 @@ def read(self, target: str, offset: int = 0, length: int | None = None, for_gate def generate() -> Iterator[bytes]: """Fetch and yield, window by window; closing it ends the fetch.""" with self._fetching(root, names, from_node, for_gateway) as fetch: - expect = fetch.resolve() - if self.deny.hashes and walker.kind_of(expect, fetch.get(expect)) == "file": - # The whole file is fetched and hashed (once: then remembered) before any of it is sent - self._check_file(expect, fetch.read_range) - sealed = None if raw or for_gateway else node_content.open_sealed(self, fetch, expect) - if sealed is not None: - yield from sealed.read(offset, length) - return - yield from fetch.read_range(expect, offset, length) + yield from self._read(fetch, fetch.resolve(), offset, length, for_gateway, raw) return generate() + def read_file(self, target: str, offset: int = 0, length: int | None = None, from_node: bytes | None = None, + raw: bool = False) -> tuple[dict, Iterator[bytes]]: + """info() and read() in one fetch, which resolves the target once: (its info, a lazy iterator of its bytes + in [offset, offset + length) cut to the size info gives). The iterator reads only once next() is called + on it, after the caller has checked the info (a directory's raises ValueError); close it when done, read + or not: that ends the fetch (GET /api/cat).""" + root, names = self._parse_target(target) + if offset < 0 or (length is not None and length < 0): + raise ValueError("offset and length must be non-negative") + + def generate() -> Iterator: + """The info, then the bytes; closed before reading them, the fetch ends as info()'s does.""" + with self._fetching(root, names, from_node) as fetch: + expect = fetch.resolve() + info = self._info(fetch, expect, raw) + try: + yield info + except GeneratorExit: + return + if info["type"] != "file": + raise ValueError(f"{target} is a directory") + available = max(info["size"] - offset, 0) + yield from self._read(fetch, expect, offset, available if length is None else min(length, available), + False, raw) + + chunks = generate() + return next(chunks), chunks + + def _read(self, fetch: Fetch, expect: walker.Expect, offset: int, length: int | None, for_gateway: bool, + raw: bool) -> Iterator[bytes]: + """read() of a resolved target, in fetch.""" + if self.deny.hashes and walker.kind_of(expect, fetch.get(expect)) == "file": + # The whole file is fetched and hashed (once: then remembered) before any of it is sent + self._check_file(expect, fetch.read_range) + sealed = None if raw or for_gateway else node_content.open_sealed(self, fetch, expect) + if sealed is not None: + yield from sealed.read(offset, length) + return + yield from fetch.read_range(expect, offset, length) + def ls(self, target: str, for_gateway: bool = False) -> list[dict]: """A directory's entries in name order (fetching its directory and index nodes only).""" root, names = self._parse_target(target, for_gateway) diff --git a/rnfs/node_content.py b/rnfs/node_content.py index 24f299a..4c8fac5 100644 --- a/rnfs/node_content.py +++ b/rnfs/node_content.py @@ -199,7 +199,7 @@ def block(node, cid: str, for_gateway: bool = False) -> bytes: def export_stream(node, target: str, start: Callable[[int], BinaryIO | None], for_gateway: bool = False) -> dict: """Fetch the target's DAG, then write its archive to start(exact archive length): the stream to write to, - or None to write nothing (HEAD). Limits as for export: max_download_size, max_output_entries, and the DAG + or None to write nothing. Limits as for export: max_download_size, max_output_entries, and the DAG byte bound of fetch_dag. Every file is checked against the deny list's file hashes before anything is sent, as the gateway checks what it sends.""" root, names = walker.parse_target(target) diff --git a/rnfs/objects.py b/rnfs/objects.py new file mode 100644 index 0000000..87794d9 --- /dev/null +++ b/rnfs/objects.py @@ -0,0 +1,248 @@ +"""The objects rnfs.Node and rnfs.Client return (design.md, "Public API"; provisional until 0.2). No RNS. + +Each is a frozen dataclass made from the local API's JSON form (from_dict, which the facade's dicts share) and +turned back into it (to_dict: CIDs as their written form, node IDs and names as hex). str() gives the CID, name +or path where an object has one obvious text form. Cid is rnfs.content.cid's. +""" +from dataclasses import MISSING, dataclass, field, fields + +from rnfs.content.cid import Cid + +__all__ = ["Cid", "Added", "Written", "Entry", "Info", "Stat", "Pin", "Record", "Status", "Collected", "Verified", + "Exported", "Imported", "DenyState"] + + +def _cid(text) -> Cid: + """A Cid from its written form (a Cid is kept as it is).""" + return text if isinstance(text, Cid) else Cid.parse(text) + + +def _hex(value) -> bytes | None: + """bytes from hex (None stays None).""" + return None if value is None else bytes.fromhex(value) if isinstance(value, str) else bytes(value) + + +def _dump(value): + """A field's JSON form: CIDs written out, bytes as hex, tuples as lists.""" + if isinstance(value, Cid): + return str(value) + if isinstance(value, (bytes, bytearray)): + return bytes(value).hex() + if isinstance(value, tuple): + return [_dump(item) for item in value] + return value + + +def _field(load=None, default=MISSING): + """A dataclass field whose JSON value from_dict turns into its type with load.""" + metadata = {"load": load} if load is not None else {} + if default is MISSING: + return field(metadata=metadata) + return field(default=default, metadata=metadata) + + +def _json_fields(cls) -> list: + """cls's fields that are JSON fields of their own (not those marked metadata={"json": False}).""" + return [f for f in fields(cls) if f.metadata.get("json", True)] + + +def _cids(values) -> tuple[Cid, ...]: + """A tuple of Cids from written forms.""" + return tuple(_cid(value) for value in values) + + +class _Object: + """to_dict and from_dict for the dataclasses below.""" + + def to_dict(self) -> dict: + """The local API's JSON form.""" + return {f.name: _dump(getattr(self, f.name)) for f in _json_fields(self)} + + @classmethod + def from_dict(cls, data: dict): + """The object for a JSON form (the API's, or the facade's dict); fields missing take their defaults.""" + values = {} + for f in _json_fields(cls): + if f.name in data: + load = f.metadata.get("load") + value = data[f.name] + values[f.name] = load(value) if load is not None and value is not None else value + return cls(**values) + + +@dataclass(frozen=True) +class Added(_Object): + """What add() stored: the root's CID and size, warnings (entries a directory add skipped), whether sealed.""" + + cid: Cid = _field(_cid) + size: int = 0 + warnings: tuple[str, ...] = _field(tuple, ()) + sealed: bool = False + + def __str__(self) -> str: + """The CID's written form.""" + return str(self.cid) + + +@dataclass(frozen=True) +class Written(_Object): + """What get() wrote: the target's CID, the output path, warnings (warnings_total past the first 100), and for + a sealed file whether it was one and its sealed name (only a name fit to be an output name).""" + + cid: Cid = _field(_cid) + output: str | None = None + warnings: tuple[str, ...] = _field(tuple, ()) + warnings_total: int | None = None + sealed: bool = False + name: str | None = None + + def __str__(self) -> str: + """The output path.""" + return self.output or str(self.cid) + + +@dataclass(frozen=True) +class Entry(_Object): + """A directory entry: name, kind ("file", "executable" or "directory"), content size and CID.""" + + name: str + kind: str + size: int + cid: Cid = _field(_cid) + + @property + def is_dir(self) -> bool: + """True for a directory.""" + return self.kind == "directory" + + def __str__(self) -> str: + """The entry's name.""" + return self.name + + +@dataclass(frozen=True) +class Info(_Object): + """A target's own CID, type ("file" or "directory"), size and executable bit; for a sealed file, sealed, + recipient (this node can open it), stored_size, and its plaintext size and sealed name if a recipient.""" + + cid: Cid = _field(_cid) + type: str = "file" + size: int = 0 + executable: bool = False + sealed: bool = False + recipient: bool | None = None + stored_size: int | None = None + name: str | None = None + + +@dataclass(frozen=True) +class Stat(_Object): + """A target's size, its DAG's block count, and how many of those blocks (and bytes) are stored here.""" + + cid: Cid = _field(_cid) + size: int = 0 + blocks: int = 0 + local_blocks: int = 0 + local_bytes: int = 0 + + +@dataclass(frozen=True) +class Pin(_Object): + """A pin: its CID, sources ("user", "folder", "name:", ...), whether complete, its type and whether a + sealed file (None when its stored blocks can't tell), and a local name for it if one is known.""" + + cid: Cid = _field(_cid) + sources: tuple[str, ...] = _field(tuple, ()) + complete: bool = False + type: str | None = None + sealed: bool | None = None + name: str | None = None + + def __str__(self) -> str: + """The CID's written form.""" + return str(self.cid) + + +@dataclass(frozen=True) +class Record(_Object): + """A name's record: the name (16 bytes), the CID it points at, its sequence, and how many seconds ago it was + heard (None when not known).""" + + name: bytes = _field(_hex) + cid: Cid = _field(_cid) + sequence: int = 0 + age: float | None = None + + +@dataclass(frozen=True) +class Status(_Object): + """The node's status; features holds every other field (traffic, jobs, LXMF, ...).""" + + version: str + node_id: bytes | None = _field(_hex, None) + identity_hash: bytes | None = _field(_hex, None) + dht_mode: str | None = None + slow: bool | None = None + private: bool | None = None + known_nodes: int | None = None + records: int | None = None + pins: int | None = None + provided: int | None = None + blocks: int | None = None + used_bytes: int | None = None + max_storage: int | None = None + names: int | None = None + features: dict = field(default_factory=dict, repr=False, metadata={"json": False}) + + def to_dict(self) -> dict: + """The API's form: the fields above with the features' merged in.""" + return {**super().to_dict(), **self.features} + + @classmethod + def from_dict(cls, data: dict) -> "Status": + """From the API's form, every field not above going into features.""" + status = super().from_dict(data) + known = {f.name for f in fields(cls)} + return cls(**{f.name: getattr(status, f.name) for f in _json_fields(cls)}, + features={key: value for key, value in data.items() if key not in known}) + + +@dataclass(frozen=True) +class Collected(_Object): + """What gc() removed.""" + + removed_blocks: int = 0 + freed_bytes: int = 0 + + +@dataclass(frozen=True) +class Verified(_Object): + """What verify() checked, and the CIDs of the bad blocks it deleted.""" + + checked: int = 0 + bad: tuple[Cid, ...] = _field(_cids, ()) + + +@dataclass(frozen=True) +class Exported(_Object): + """What export() wrote: the archive's roots and its block count.""" + + roots: tuple[Cid, ...] = _field(_cids, ()) + blocks: int = 0 + + +@dataclass(frozen=True) +class Imported(_Object): + """What import_archive() read: its roots, those not complete (so not pinned), and its block count.""" + + roots: tuple[Cid, ...] = _field(_cids, ()) + incomplete: tuple[Cid, ...] = _field(_cids, ()) + blocks: int = 0 + + +@dataclass(frozen=True) +class DenyState(_Object): + """The deny list: the local entries, and each deny_lists source with the version in use and its state.""" + + entries: tuple[str, ...] = _field(tuple, ()) + lists: tuple[dict, ...] = _field(tuple, ()) diff --git a/rnfs/publisher.py b/rnfs/publisher.py index 2b3b6b7..833c369 100644 --- a/rnfs/publisher.py +++ b/rnfs/publisher.py @@ -3,11 +3,14 @@ - **What is provided** (provide_strategy): `pinned` provides every complete pin; `roots` also every complete fetched target while it is cached (the node says which cached targets qualify: gateway fetches never do). Only these CIDs' DAGs are served to /blocks (section 7). A CID whose DAG holds a block the deny list (the - storage's) denies is never provided: a change to the list takes effect at the next refresh(). + storage's) denies is never provided: a change to the list takes effect at the next refresh(). A refresh + recomputes this only when something it is computed from changed (the node's generation(), the deny list, + the key sources' keys, the strategy), so the many refreshes that change nothing cost almost nothing. - **Publishing.** A CID is due when it has never been published or its republish_interval has passed. For - each due CID the K closest nodes are found; keys are grouped by node, and each node gets its keys (and any - withdrawals for it) in as few /provide requests as fit, over one identified link. A refused or failed - store isn't retried until the next republish. The node also records itself locally. + each due CID the K closest nodes are found, PUBLISH_LOOKUPS lookups at once (one at a time on slow nodes); + keys are grouped by node, and each node gets its keys (and any withdrawals for it) in as few /provide + requests as fit, over one identified link. A refused or failed store isn't retried until the next + republish. The node also records itself locally. - **Withdrawing.** When a CID stops being provided (unpinned, evicted, gc'd, damaged), its key is withdrawn from the nodes it was last published to, at once, and from nodes of earlier rounds (or of a round cut short) whose record may not have expired yet: the closest set changes as nodes come and go. @@ -56,6 +59,12 @@ # LoRa rates. One per 30 s keeps republishing a hundred keys to well under 1% of a day's airtime per hour SLOW_LOOKUP_PACE = 30.0 SLOW_ROUND_LOOKUPS = 10 # lookups per round on slow nodes, so /provide batching still groups keys per node +# Other nodes: closest-nodes lookups a round runs at once. Each keeps alpha (3) /find queries in flight, so 4 +# keep about 12, small requests (a link still carries at most MAX_REQUESTS_PER_LINK at once; the rest wait for a +# slot); a round of a few hundred keys, each lookup a few round trips one after another, then takes about a +# quarter of the time, with few threads. The airtime controls (upload_budget, interface rules, quiet hours) +# apply to /blocks only (spec section 3): these send the same /find messages as one at a time would, overlapping +PUBLISH_LOOKUPS = 4 class DhtKey: @@ -184,11 +193,13 @@ class Publisher: def __init__(self, own_id: bytes, config: Config, storage: Storage, providers, client, make_lookup: Callable[[bytes], object], slow_node: bool = False, state_dir: str | None = None, cached_eligible: Callable[[], set[Cid]] = set, clock: Callable[[], float] = time.time, - retry_interval: float = RETRY_INTERVAL): + retry_interval: float = RETRY_INTERVAL, generation: Callable[[], object] | None = None): """Publisher for own_id. providers is the ProviderStore (to record ourselves), client the PeerClient. make_lookup(key) returns a dht.lookup.Lookup; cached_eligible() returns the cached targets the - roots strategy may provide. + roots strategy may provide. generation(), if given, returns a value that changes whenever storage's pins, + cached targets or blocks, or cached_eligible(), may have: refresh() skips its recompute while it, the deny + list and the key sources' keys are as at the last one (without it, every refresh recomputes). """ self.own_id = own_id self.config = config @@ -200,6 +211,7 @@ def __init__(self, own_id: bytes, config: Config, storage: Storage, providers, c self.cached_eligible = cached_eligible self.clock = clock self.retry_interval = retry_interval + self.generation = generation self._lock = threading.RLock() self._save_lock = threading.Lock() # one save at a time, so an older snapshot never lands last self._wake = threading.Event() @@ -213,6 +225,8 @@ def __init__(self, own_id: bytes, config: Config, storage: Storage, providers, c # CID -> (deny list's CID set, dag_blocks object, whether one of them is denied), so each DAG is checked # against the deny list once per change of either self._denied_dags: dict[Cid, tuple[frozenset[bytes], frozenset[Cid], bool]] = {} + # (generation(), deny list's CID set, keys, strategy) of the last full refresh; None: the next computes + self._inputs: tuple | None = None self._published: dict[Cid, tuple[float, list[bytes]]] = {} # CID -> (when, nodes that recorded it) # Other nodes that may still hold our record of a CID (earlier rounds' closest nodes, or a round cut # short), with when it was stored there; withdrawn from too, and forgotten once PROVIDER_TTL has passed @@ -238,6 +252,7 @@ def add_key_source(self, source: Callable[[], set[bytes]]): """Also provide the keys source() returns (DhtKeys), recomputed at each refresh(). Must not block.""" with self._lock: self._key_sources.append(source) + self._inputs = None # its keys aren't in the provided set yet @property def interval(self) -> float: @@ -386,8 +401,19 @@ def serves(self, cid: Cid) -> bool: """True if cid is in the DAG of a provided CID, so /blocks may serve it.""" return cid in self._served # a frozenset swapped whole, so reading it needs no lock - def _compute(self) -> tuple[frozenset[Cid], frozenset[Cid]]: - """(provided CIDs, blocks they reach) from storage and the strategy. Lock held. + def _keys(self) -> set[bytes]: + """The keys the key sources return now (a failing source's are left out, logged). Lock held.""" + keys: set[bytes] = set() + for source in list(self._key_sources): + try: + keys |= {bytes(key) for key in source()} # as DhtKey takes them + except Exception as e: # noqa: BLE001 - a broken feature mustn't stop CIDs being provided + log(f"a provided-key source failed: {e!r}", LOG_ERROR) + return keys + + def _compute(self, keys: set[bytes], denied: frozenset[bytes]) -> tuple[frozenset[Cid], frozenset[Cid]]: + """(provided CIDs and DhtKeys of keys, blocks they reach) from storage, the strategy and denied (the + deny list's binary CIDs). Lock held. The served set and the DhtKeys are those of the last call when what they are built from hasn't changed (most refreshes): the same complete CIDs with the same dag_blocks objects, and the same keys. CIDs only @@ -402,16 +428,9 @@ def _compute(self) -> tuple[frozenset[Cid], frozenset[Cid]]: dags[cid] = self.storage.dag_blocks(cid) # memoised; raises unless complete except (NotFound, InvalidDag): continue - denied = self.storage.deny.cids if denied: dags = {cid: blocks for cid, blocks in dags.items() if not self._denied(cid, blocks, denied)} self._denied_dags = {cid: checked for cid, checked in self._denied_dags.items() if cid in candidates} - keys: set[bytes] = set() - for source in list(self._key_sources): - try: - keys |= {bytes(key) for key in source()} # as DhtKey takes them - except Exception as e: # noqa: BLE001 - a broken feature mustn't stop CIDs being provided - log(f"a provided-key source failed: {e!r}", LOG_ERROR) last_dags, served = self._served_from new = [blocks for cid, blocks in dags.items() if last_dags.get(cid) is not blocks] if len(dags) - len(new) < len(last_dags): # one went, or its blocks changed: built again @@ -435,9 +454,26 @@ def _denied(self, cid: Cid, blocks: frozenset[Cid], denied: frozenset[bytes]) -> return checked[2] def refresh(self): - """Recompute what is provided (after pins, fetches, evictions); withdraw what no longer is, at once.""" + """Recompute what is provided (after pins, fetches, evictions); withdraw what no longer is, at once. + + Skipped when generation(), the deny list, the key sources' keys and the strategy are as at the last + refresh that computed: the same inputs give the same sets, so nothing would be added or removed. + removed also takes in _published, but _published stays within _provided between refreshes: a refresh + drops from _published what it doesn't provide, only refresh() changes _provided, and a round records + only items still in _provided. _load fills _published before any refresh, so the first refresh after + construction always computes (as does the first after add_key_source). + """ with self._lock: - provided, served = self._compute() + keys, denied = self._keys(), self.storage.deny.cids # the deny set is swapped whole + # Read before computing: a change made while this computes moves the generation past what is + # remembered here, so the next refresh computes again (perhaps once more than needed), never misses it + inputs = (None if self.generation is None + else (self.generation(), denied, keys, self.config.provide_strategy)) + last = self._inputs + if (inputs is not None and last is not None and (last[1] is denied or last[1] == denied) + and last[0] == inputs[0] and last[2:] == inputs[2:]): + return + provided, served = self._compute(keys, denied) removed = (set(self._provided) | set(self._published)) - provided added = provided - self._provided self._provided, self._served = provided, served @@ -452,6 +488,7 @@ def refresh(self): self._withdrawals.setdefault(node_id, set()).add(cid.key) self._touch_node(node_id) cancelled = [cid for cid in added if self._provide_again(cid)] + self._inputs = inputs # only once everything above is done: an exception computes again next time if removed: log(f"no longer providing {plural(len(removed), 'CID')}", LOG_VERBOSE) if removed or cancelled: @@ -544,12 +581,7 @@ def publish_due(self, should_stop: Callable[[], bool] = lambda: False) -> dict: if self.slow_node: due = due[:SLOW_ROUND_LOOKUPS] if now >= self._next_lookup else [] - closest: dict[Cid, list[bytes]] = {} - for cid in due: - if should_stop(): - break - self.providers.store(cid.key, self.own_id) # providers also record themselves locally - closest[cid] = [n for n in self.make_lookup(cid.key).find_closest() if n != self.own_id] + closest = self._lookups(due, should_stop) with self._lock: if self.slow_node and closest: self._next_lookup = self.clock() + SLOW_LOOKUP_PACE * len(closest) @@ -663,6 +695,64 @@ def publish_due(self, should_stop: Callable[[], bool] = lambda: False) -> dict: return {"published": published, "nodes": sorted(set(by_node) | set(sent_withdrawals)), "withdrawn": sum(len(keys) for keys in sent_withdrawals.values())} + def _find_closest(self, cid: "Cid | DhtKey") -> list[bytes]: + """Record this node as cid's provider locally, then look up the nodes closest to its key (but this).""" + self.providers.store(cid.key, self.own_id) # providers also record themselves locally + return [n for n in self.make_lookup(cid.key).find_closest() if n != self.own_id] + + def _lookups(self, due: list, should_stop: Callable[[], bool]) -> dict: + """{item: its closest nodes} for due's items whose lookup ran, in due's order, so a round's batches are + the same however the lookups were run. Called without the lock: refresh() may run meanwhile. + + should_stop() is checked before each lookup starts; one running isn't interrupted. Slow nodes (and a + single item) look up one at a time; other nodes run up to PUBLISH_LOOKUPS at once on + rnfs-publish-lookup threads. A lookup that raises starts no more; the exception is raised once those + running have finished, ending the round as it would one at a time. + """ + if self.slow_node or len(due) < 2 or PUBLISH_LOOKUPS < 2: + closest = {} + for cid in due: + if should_stop(): + break + closest[cid] = self._find_closest(cid) + return closest + lock = threading.Lock() + pending = iter(range(len(due))) + found: dict[int, list[bytes]] = {} + ended: list[BaseException | None] = [] # None once stopped, else the exception a lookup raised + + def work(): + """Run lookups in turn until none is left, the round is stopped, or one has raised.""" + while True: + with lock: + if ended: + return + index = next(pending, None) + if index is None: + return + if should_stop(): + ended.append(None) + return + try: + nodes = self._find_closest(due[index]) + except BaseException as e: # noqa: BLE001 - raised by publish_due once the others finish + with lock: + ended.append(e) + return + with lock: + found[index] = nodes + + threads = [threading.Thread(target=work, name="rnfs-publish-lookup", daemon=True) + for _ in range(min(PUBLISH_LOOKUPS, len(due)))] + for thread in threads: + thread.start() + for thread in threads: + thread.join() + failure = next((e for e in ended if e is not None), None) + if failure is not None: + raise failure + return {due[index]: found[index] for index in sorted(found)} + def run(self, stop_event: threading.Event, ready: threading.Event | None = None): """The publish loop, for a thread of its own: waits for ready (the join), then publishes what's due.""" if ready is not None: diff --git a/rnfs/remote.py b/rnfs/remote.py new file mode 100644 index 0000000..d5af42e --- /dev/null +++ b/rnfs/remote.py @@ -0,0 +1,302 @@ +"""rnfs.Client: the user's running rnfs daemon, driven through its local API (design.md, "Public API"; +provisional until 0.2). Standard library only, and it never imports RNS. + +The same methods, objects and exceptions as rnfs.Node (rnfs.interface), over rnfs.apiclient's ApiConnection: +the token is read from the rnfs dir and sent only once whatever listens on the API port has proved it knows it. +Besides the node's errors (each raised as its ApiError subclass), calls may raise DaemonNotRunning, +ConnectionFailed, TokenError, AuthFailed and ServerError. Paths are opened by the daemon, on this machine, as +the user running it. + +Callbacks run on the client's event thread, as rnfs.Node's do. Watched jobs are polled once a second (GET +/api/jobs/get), and watched names through GET /api/name/watch: a long poll on a thread of its own +(rnfs-client-name-watch) naming every name watched, so a name watched while one is waiting joins the next poll, +within MAX_WATCH_WAIT seconds, and nothing heard meanwhile is lost (the daemon answers from the records it holds). +With a timeout, a long poll (names, Job.wait) waits at most LONG_POLL_MARGIN seconds less than it, so the request +never outlasts its own socket; one that leaves no wait at all polls every second instead. +""" +import logging +import math +import threading +import time + +from rnfs import interface +from rnfs.apiclient import ApiConnection, read_all +from rnfs.objects import Record + +MAX_WATCH_WAIT = 30 # seconds one GET /api/name/watch or /api/jobs/get waits (the API's cap) +LONG_POLL_MARGIN = 2 # seconds a long poll leaves of the client's timeout +RETRY_DELAY = 5.0 # seconds before polling names again after a failure +SHORT_POLL = 1.0 # seconds between polls when the timeout leaves no wait + +_logger = logging.getLogger("rnfs") + + +class Client(interface.Rnfs): + """The daemon using rnfs_dir (default ~/.rnfs/), on 127.0.0.1:port (default: the api_port in its rnfs.toml), + with the token in token_file (default /api_token). timeout bounds each request (default: none, as + fetches can take hours). Nothing is sent until the first call. Raises SettingsError if rnfs.toml can't be read + for the port.""" + + events_thread = "rnfs-client-events" + jobs_thread = "rnfs-client-job-watch" + job_poll = 1.0 # what the CLI's progress line polls at + + def __init__(self, rnfs_dir: str | None = None, *, port: int | None = None, token_file: str | None = None, + timeout: float | None = None): + """A client of the daemon (see the class).""" + self._api = ApiConnection(rnfs_dir, port=port, token_file=token_file, timeout=timeout) + # The longest a long poll may wait within the timeout + self._long_wait = MAX_WATCH_WAIT if timeout is None else \ + max(0, min(MAX_WATCH_WAIT, math.floor(timeout) - LONG_POLL_MARGIN)) + super().__init__(_Backend(self._api, self._long_wait)) + self._poll_cond = threading.Condition() + self._poller: threading.Thread | None = None + self._names_changed = False + # The highest sequence each name's poll has answered: the next asks above it, so a delivery still waiting + # on the event thread isn't fetched again and again (only the poller uses it) + self._polled: dict[bytes, int] = {} + + @property + def api(self) -> ApiConnection: + """The API connection underneath, whose get(path, **params) and post(path, **fields) reach every endpoint. + Unsupported: it changes with any release.""" + return self._api + + def _report(self, message: str): + """Log through Python's logging (logger "rnfs").""" + _logger.error("rnfs: %s", message) + + def _close(self): + """Wake the name poller so it ends.""" + with self._poll_cond: + self._poll_cond.notify_all() + + # Name watches + + def _baseline(self, name: bytes) -> int | None: + """The sequence of the record the daemon holds for name now (this call also watches it there).""" + records = self._api.name_watch([name.hex()], [None], 0) + return records[0]["sequence"] if records else None + + def _watch_started(self, name: bytes): + """Start the name poller (if needed), which names every watched name in its next call.""" + with self._poll_cond: + self._names_changed = True + if self._poller is None: + self._poller = threading.Thread(target=self._poll_names, name="rnfs-client-name-watch", daemon=True) + self._poller.start() + self._poll_cond.notify_all() + + def _poll_names(self): + """Long-poll the daemon for every watched name's newer records until closed.""" + while not self.closed: + names = self._watched_names() + if not names: + with self._poll_cond: + while not self._names_changed and not self.closed: + self._poll_cond.wait() + self._names_changed = False + continue + with self._poll_cond: + self._names_changed = False + after = [last if name not in self._polled else max(self._polled[name], -1 if last is None else last) + for name, last in zip(names, self._last_sequences(names))] + started = time.monotonic() + try: + records = self._api.name_watch([name.hex() for name in names], after, self._long_wait) + except Exception as e: # noqa: BLE001 - a daemon restarting answers again later + if self.closed: + return + self._report(f"watching names failed: {e}") + with self._poll_cond: + self._poll_cond.wait(RETRY_DELAY) + continue + for data in records: + record = Record.from_dict(data) + self._polled[record.name] = max(self._polled.get(record.name, -1), record.sequence) + self._record_heard(record.name, record) + if not records and self._long_wait == 0 and time.monotonic() - started < SHORT_POLL: + with self._poll_cond: + self._poll_cond.wait(SHORT_POLL) + + +class _Backend: + """rnfs.interface's backend for the daemon's API (ApiConnection's methods, in the forms the API takes).""" + + def __init__(self, api: ApiConnection, long_wait: int): + """The backend of api, whose long polls wait at most long_wait seconds.""" + self.api = api + self.long_wait = long_wait + + def status(self) -> dict: + """GET /api/status.""" + return self.api.status() + + def node_id(self) -> str: + """The node ID in hex, from the status.""" + return self.api.status()["node_id"] + + def wait_joined(self, timeout: float | None) -> bool: + """A running daemon has joined (or keeps joining) on its own: True once it answers.""" + self.api.status() + return True + + def add_path(self, path: str, **options) -> dict: + """POST /api/add.""" + return self.api.add_path(path, **options) + + def add_stream(self, data, length: int | None, name: str | None, **options) -> dict: + """POST /api/add/stream.""" + return self.api.add_stream(data, name=name, length=length, **options) + + def get(self, target: str, output: str, from_node: str | None, raw: bool) -> dict: + """POST /api/get.""" + return self.api.fetch(target, output, from_node=from_node, raw=raw) + + def read(self, target: str, offset: int, length: int | None, raw: bool, from_node: str | None): + """GET /api/cat: the body, read as it arrives, closing the connection when done or closed early. Sent once + iterated (its errors raised then, as a Node's are), so an iterator closed before that sends nothing.""" + def chunks(): + """The body's chunks.""" + with self.api.cat(target, offset=offset or None, length=length, raw=raw, from_node=from_node) as stream: + yield from stream + return chunks() + + def block(self, cid: str) -> bytes: + """GET /api/cat?block=1.""" + with self.api.cat(cid, block=True) as stream: + return stream.read() + + def info(self, target: str, from_node: str | None, raw: bool) -> dict: + """GET /api/info; raw: a sealed file as stored (its stored size, without the sealed fields), as the + facade's info(raw=True) answers.""" + info = self.api.info(target, from_node=from_node) + if raw and info.get("sealed"): + return {"cid": info["cid"], "type": info["type"], "size": info["stored_size"], + "executable": info["executable"]} + return info + + def ls(self, target: str) -> list[dict]: + """GET /api/ls.""" + return self.api.ls(target) + + def stat(self, target: str) -> dict: + """GET /api/stat.""" + return self.api.stat(target) + + def dir_set(self, target: str, name: str, child: str, executable: bool) -> dict: + """POST /api/dir/set.""" + return self.api.post("/api/dir/set", target=target, name=name, cid=child, executable=executable) + + def dir_rm(self, target: str) -> dict: + """POST /api/dir/rm.""" + return self.api.post("/api/dir/rm", target=target) + + def pin_add(self, target: str) -> dict: + """POST /api/pin/add.""" + return self.api.pin_add(target) + + def pin_rm(self, cid: str): + """POST /api/pin/rm.""" + self.api.pin_rm(cid) + + def pin_ls(self, cid: str | None) -> list[dict]: + """GET /api/pin/ls.""" + return self.api.pin_ls(cid) + + def pin_remote(self, node_id: str, cid: str, pin: bool) -> bool: + """POST /api/pin/remote.""" + return self.api.post("/api/pin/remote", node=node_id, cid=cid, pin=pin)["accepted"] + + def providers(self, cid: str) -> list[str]: + """GET /api/providers.""" + return self.api.get("/api/providers", cid=cid)["providers"] + + def ping(self, node_id: str) -> float | None: + """GET /api/ping.""" + return self.api.get("/api/ping", node=node_id)["rtt"] + + def export(self, target: str, output: str) -> dict: + """POST /api/export.""" + return self.api.post("/api/export", target=target, output=output) + + def import_archive(self, path: str) -> dict: + """POST /api/import.""" + return self.api.post("/api/import", path=path) + + def gc(self) -> dict: + """POST /api/gc.""" + return self.api.post("/api/gc") + + def verify(self) -> dict: + """POST /api/verify.""" + return self.api.post("/api/verify") + + def name_create(self, alias: str) -> dict: + """POST /api/name/create.""" + return self.api.name_create(alias) + + def name_import(self, alias: str, key: bytes) -> dict: + """POST /api/name/import (the key as the body).""" + body = bytes(key) + return self.api.call("POST", "/api/name/import", params={"alias": alias}, body=body, length=len(body)) + + def name_rm(self, alias: str): + """POST /api/name/rm.""" + self.api.post("/api/name/rm", alias=alias) + + def name_export(self, alias: str) -> bytes: + """POST /api/name/export: the key bytes.""" + return read_all(self.api.request("POST", "/api/name/export", json_body={"alias": alias})) + + def name_list(self) -> list[dict]: + """GET /api/name/list.""" + return self.api.name_list() + + def name_publish(self, alias: str, target: str) -> dict: + """POST /api/name/publish.""" + return self.api.name_publish(alias, target) + + def name_resolve(self, name: str) -> dict: + """GET /api/name/resolve.""" + return self.api.name_resolve(name) + + def submit_job(self, kind: str, target: str, output: str | None, from_node: str | None, **flags) -> dict: + """POST /api/jobs (only the flags set, as the CLI sends them).""" + return self.api.submit_job(kind, target, output, from_node, **{k: v for k, v in flags.items() if v}) + + def job(self, job_id: str, wait: float) -> dict: + """GET /api/jobs/get, waiting up to wait seconds (whole ones, within the timeout) for the job to finish; + when the timeout leaves no wait, a second's pause after an unfinished answer instead.""" + if wait <= 0: + return self.api.job(job_id, 0) + if self.long_wait == 0: + job = self.api.job(job_id, 0) + if job.get("state") not in interface.FINISHED: + time.sleep(min(wait, SHORT_POLL)) + return job + return self.api.job(job_id, min(self.long_wait, math.ceil(wait))) + + def jobs(self) -> list[dict]: + """GET /api/jobs.""" + return self.api.jobs() + + def cancel_job(self, job_id: str) -> dict: + """POST /api/jobs/cancel.""" + return self.api.cancel_job(job_id) + + def clear_jobs(self, job_id: str | None) -> int: + """POST /api/jobs/clear.""" + return self.api.clear_jobs(job_id) + + def deny(self) -> dict: + """GET /api/deny.""" + return self.api.deny() + + def deny_add(self, entries: list[str]) -> list[str]: + """POST /api/deny/add.""" + return self.api.deny_add(entries) + + def deny_rm(self, entries: list[str]) -> list[str]: + """POST /api/deny/rm.""" + return self.api.deny_rm(entries) diff --git a/rnfs/statedb.py b/rnfs/statedb.py index 58b1c9c..12f41c9 100644 --- a/rnfs/statedb.py +++ b/rnfs/statedb.py @@ -125,6 +125,10 @@ # source: a deny_lists setting as written (dump()ed); cid: the list version it last accepted "CREATE TABLE deny_lists (source TEXT PRIMARY KEY, cid BLOB NOT NULL, accepted REAL NOT NULL) WITHOUT ROWID", )), + ("denylist_hashes", ( + # Files' content SHA-256 (digest) by binary CID; used orders them (least recently used lowest) + "CREATE TABLE deny_hashes (cid BLOB PRIMARY KEY, digest BLOB NOT NULL, used INTEGER NOT NULL) WITHOUT ROWID", + )), ) diff --git a/rnfs/winacl.py b/rnfs/winacl.py index ccda7e3..298e65b 100644 --- a/rnfs/winacl.py +++ b/rnfs/winacl.py @@ -400,9 +400,13 @@ def outside_profile(path: str) -> bool: return not fsutil.is_below(os.path.realpath(path), os.path.realpath(profile)) +_warned_outside: set[str] = set() # roots already warned about as outside the profile (one warning per process) + + def secure_rnfs_dir(root: str, subdirs: tuple[str, ...] = ()): """On Windows, node._make_private_dirs: create the rnfs dir root and subdirs (below it) private from the - start, or check and tighten an existing root, and warn about a root outside the user's profile. + start, or check and tighten an existing root, and warn (once per process) about a root outside the user's + profile: a start makes the dirs private more than once (the lock, then the node). A new root is created with the protected owner-only DACL in the same call (make_private_dir), and must then be empty: anything in it was put there by someone else. An existing one is made private if others @@ -440,7 +444,8 @@ def secure_rnfs_dir(root: str, subdirs: tuple[str, ...] = ()): _make_private_below(path, os.path.abspath(directory)) except OSError as e: raise RnfsError(f"couldn't make the rnfs dir {root} private to you: {e}") from None - if outside_profile(root): + if outside_profile(root) and os.path.normcase(path) not in _warned_outside: + _warned_outside.add(os.path.normcase(path)) log(f"the rnfs dir {root} is outside your user profile. rnfs has made it private to you, but accounts " f"that can change its parent folder could replace it: keep it in a folder only you can change", LOG_WARNING) diff --git a/specification.md b/specification.md index 8414088..f8687b9 100644 --- a/specification.md +++ b/specification.md @@ -205,10 +205,13 @@ pass a `max_response_size` just above their largest honest answer. reaching the node over the same interface whose handshake takes longer is cut off (RNS tells it at once, and it may try again), while initiators on other interfaces are not affected. Each request still costs the node the key exchange and a proof; RNS's own long responder timeout is the root cause. -- rnfs neither sends nor accepts compressed Resources. Handlers are registered with `auto_compress=False`, - and nodes MUST set RNS's maximum decompressed Resource size (`RNS.Resource.AUTO_COMPRESS_MAX_SIZE` in - RNS 1.5.6) to 0, since RNS decompresses before any rnfs check runs. Setting it to 0 also stops RNS - compressing what the node sends. +- rnfs neither sends nor accepts compressed Resources on `rnfs.dht` links. Handlers are registered with + `auto_compress=False`, requests are never sent unless they fit one packet, and a node MUST reject, before + accepting it, any Resource advertisement on an `rnfs.dht` link that has the compressed flag set, since RNS + decompresses before any rnfs check runs. A node that has its process to itself (the daemon) also sets RNS's + maximum decompressed Resource size (`RNS.Resource.AUTO_COMPRESS_MAX_SIZE` in RNS 1.5.6) to 0, which also + stops RNS compressing anything the process sends; a node embedded in another program leaves that setting to + the program. - RNS checks request and response size limits against a Resource's data size but allocates by its transfer size, so a node SHOULD reject, before accepting it, any incoming Resource advertisement that is a request (rnfs requests always fit one packet), or whose transfer size exceeds its data size or `MAX_RESPONSE_SIZE` @@ -216,7 +219,8 @@ pass a `max_response_size` just above their largest honest answer. `rnfs.dht` links (either end) a node SHOULD also reject any advertisement of more than one segment: every rnfs response fits one (`MAX_RESPONSE_SIZE`). On `rnfs.dht` links it SHOULD also reject one with metadata, which RNS decodes as the Resource completes and rnfs responses never carry. Other protocols' Resources in - the same process (LXMF) are not affected. + the same process (LXMF) are not affected by these two. A node embedded in another program applies every + check in this item to its `rnfs.dht` links only, so the program's own links behave as with RNS alone. - RNS decodes a whole response with msgpack before handing it over, and one-byte empty arrays or maps decode to some 70 times their size in objects (a 1 MiB response: ~70 MB and ~0.3 s of CPU). So on `rnfs.dht` links a node SHOULD check a response's msgpack headers before RNS decodes it, without building objects, and @@ -1112,7 +1116,7 @@ are local conveniences, not part of the protocol. | `deny add\|rm …`, `deny ls` | The local deny list (CIDs, `sha256:` file hashes); `ls` also shows each `deny_lists` source, the version in use and its state (section 8) | | `send
[-m text] [--propagate] [--nearest]`, `inbox`, `inbox get [--link K] [-o out]`, `inbox rm \|--all`, `outbox`, `lxmf address`, `lxmf sync [--nearest]` | LXMF (section 13). `--nearest` uses the nearest propagation node heard when `lxmf_propagation_node` isn't set | | `peers [--json]`, `stats [--json]` | Known nodes with hops, interface and traffic; traffic since start, refusals by cause, per interface | -| `status` | Node ID, RNS identity hash (the value other nodes list in `members` and `remote_pin_identities`), known nodes, records held, storage use, traffic, jobs, LXMF address (only the version without a readable API token) | +| `status` | Node ID, RNS identity hash (the value other nodes list in `members` and `remote_pin_identities`), known nodes, records held, storage use, traffic, jobs, deny-list entries and sources, LXMF address (only the version without a readable API token) | Wherever a command takes a target, it may start with a name contact's nick (`get alice/docs/report.pdf`); where it takes a node ID (`get --from`, `ping`, `pin remote`), a node contact's nick works too. @@ -1131,7 +1135,7 @@ gateway can never reach the API. - Every API request except `GET /api/status` MUST carry `X-Rnfs-Api: `, compared in constant time. The token is random (at least 128 bits), made anew at each daemon start and written to `/api_token` with mode 0600 (on Windows, with a protected access list allowing only the user and - SYSTEM); clients (the CLI, and apps through rnfs's client library `rnfs.apiclient`) read it from there and + SYSTEM); clients (the CLI, and apps through `rnfs.Client`, built on `rnfs.apiclient`) read it from there and refuse a file other users can access. Every local process, of every user, can connect to `127.0.0.1`, and the API reads and writes files as the daemon's user, so only users who can read the rnfs directory may use it. Web pages can't learn the token, and the custom header needs a CORS preflight, which fails: this also blocks cross-site @@ -1257,6 +1261,15 @@ optionally from one node), `pin` (fetch a target completely and pin it with sour downloaded over the last 300 s of the run, falling while nothing arrives) and `eta` (`(total_bytes - done_bytes) / rate`). +**Watching names.** A node MAY let local clients wait for newer records of names (rnfs: `GET /api/name/watch`, +which rnfs.Client's `names.watch` uses). It answers from the records the node holds, under the sequence rules +of section 9, with those above a sequence the client gives, as soon as there is one or after a wait (rnfs: at +most 30 s), so a client that asks again with what it last received loses nothing between calls. A watched +name's announces count as asked for (section 9: a full table makes room for them), during each call and for a +while after it (rnfs: 60 s); watching sends nothing to the network. A node SHOULD cap the names per call +(rnfs: 64), the calls waiting at once (rnfs: 4, so watchers can't take the API's connections; more get `503`) +and the names watched (rnfs: 1024). `GET /api/cat` takes `from`, a node to fetch from, as `GET /api/info` does. + **Contacts.** A node MAY keep a local address book of nicks for names, node IDs and LXMF addresses. A nick is 1–32 characters of `a-z`, `0-9`, `-`, `_` and `.`, starting with a letter or digit, and not 32 hex digits, so it can never be read as a node ID, name or CID. Contacts are local: they are never sent to other nodes. @@ -1335,7 +1348,7 @@ setting. | `default_bootstrap` | on | Also contact the default bootstrap nodes when joining; never in private mode (section 6) | | `reticulum_dir` | RNS's own | RNS config directory, the one `rnsd` uses by default | | `verbosity` | 0 | Extra logging: 1 per-query detail, 2 RNS debug output | -| `debug_log` | off | Also write every log message (at every level), RNS's own, and each gateway request to a debug log file in the rnfs dir, readable by its owner only. Local only: nothing in it is sent | +| `debug_log` | off | Also write every rnfs log message (at every level), RNS's own at Info and more severe, and each gateway request to a debug log file in the rnfs dir, readable by its owner only. Local only: nothing in it is sent | | `upload_budget` | none (unlimited) | Bytes per hour of `/blocks` responses, node-wide; a token bucket charged for bytes sent, at most half of it per requester in flight (section 3) | | `serve_interfaces` | empty (all) | Serve `/blocks` only on these RNS interfaces (names or prefixes, case-insensitive; a spawned interface also matches its parent's name) | | `no_serve_interfaces` | empty | Never serve `/blocks` on these interfaces; takes precedence over `serve_interfaces` | diff --git a/tests/security/test_l7_windows_acl.py b/tests/security/test_l7_windows_acl.py index b08d5ca..ebd9c84 100644 --- a/tests/security/test_l7_windows_acl.py +++ b/tests/security/test_l7_windows_acl.py @@ -92,7 +92,8 @@ def setUp(self): self.tmp = tempfile.TemporaryDirectory() self.addCleanup(self.tmp.cleanup) self.acl = FakeAcl() - for patch in (mock.patch.object(fsutil, "IS_WINDOWS", True), mock.patch.object(winacl, "backend", self.acl)): + for patch in (mock.patch.object(fsutil, "IS_WINDOWS", True), mock.patch.object(winacl, "backend", self.acl), + mock.patch.object(winacl, "_warned_outside", set())): patch.start() self.addCleanup(patch.stop) @@ -231,8 +232,9 @@ def test_dir_that_cant_be_made_private_is_refused(self): def test_dir_outside_the_profile_is_warned_about(self): with mock.patch.dict(os.environ, {"USERPROFILE": os.path.join(self.tmp.name, "profile")}): node_module._make_private_dirs(Config(rnfs_dir=self.root)) + node_module._make_private_dirs(Config(rnfs_dir=self.root)) # as a start does more than once self.assertTrue(self.private(self.root)) - self.assertIn("is outside your user profile", self.messages()) + self.assertEqual(self.messages().count("is outside your user profile"), 1) class KeyFileTest(WindowsTestCase): diff --git a/tests/test_api.py b/tests/test_api.py index 6814f2f..5bd7115 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -18,7 +18,7 @@ from rnfs import cli, fsutil from rnfs.api import ApiHandler, ApiServer, BodyReader, LocalServer, loggable from rnfs.content.cid import DAG, RAW, Cid -from rnfs.errors import Busy, InvalidDag, NotFound, RnfsError, StorageFull, TooLarge +from rnfs.errors import Busy, Denied, InvalidDag, NotFound, RnfsError, StorageFull, TooLarge from tests.support import wait_for LocalServer.poll_interval = 0.01 # so each test's server stops at once @@ -217,6 +217,24 @@ def pieces(): yield data[i:i + self.piece] return pieces() + def read_file(self, target, offset=0, length=None, from_node=None, raw=False): + """info() and read() of target in one call, as RnfsNode.read_file: (info, its bytes cut to its size).""" + self.record("read_file", target, offset=offset, length=length, from_node=from_node, + **self._given("read_file", raw=raw)) + cid = self._resolve(target) + obj = self.objects[cid] + info = {"cid": cid, "type": "file" if obj[0] == "file" else "directory", "size": self._size(cid), + "executable": obj[0] == "file" and obj[2]} + + def pieces(): + """The bytes, piece by piece, once asked for.""" + if obj[0] != "file": + raise ValueError(f"{target} is a directory") + data = obj[1][offset:] if length is None else obj[1][offset:offset + length] + for i in range(0, len(data), self.piece): + yield data[i:i + self.piece] + return info, pieces() + def ls(self, target, for_gateway=False): self.record("ls", target, for_gateway=for_gateway) cid = self._resolve(target) @@ -238,6 +256,19 @@ def is_local(self, target): path = self._path(target) return not (set(path) | self._dag(path[-1])) & self.remote + def stored_head(self, target, length): + """A file's first length bytes if it and its path are stored, else None (a directory or a bad target + too), as RnfsNode.stored_head. Never fetches.""" + self.record("stored_head", target, length) + try: + path = self._path(target) + except (ValueError, NotFound): + return None + obj = self.objects.get(path[-1]) + if set(path) & self.remote or obj is None or obj[0] != "file": + return None + return obj[1][:length] + def dir_set(self, target, name, child_cid, executable=False): self.record("dir_set", target, name, child_cid, executable=executable) obj = self._object(self._resolve(target)) @@ -481,12 +512,39 @@ def test_proof_of_the_token(self): self.assertIsNone(self.call("GET", "/api/status", api=False)[1][cli.PROOF_HEADER]) def test_http_0_9_is_refused_with_headers(self): - # http.server would answer HTTP/0.9 with a bare body; every answer here has a status line and headers + """http.server would answer HTTP/0.9 with a bare body; every answer here has a status line and the API's + headers.""" for line in (b"GET /api/status\r\n", b"GET /api/status HTTP/0.9\r\n", b"GARBAGE\r\n"): with socket.create_connection(("127.0.0.1", self.port), timeout=10) as sock: sock.sendall(line + f"Host: 127.0.0.1:{self.port}\r\n\r\n".encode()) data, _ = read_until_closed(sock) self.assertTrue(data.startswith(b"HTTP/1.0 400 "), (line, data[:80])) + self.assert_api_headers(data) + + def test_http_server_errors_carry_the_headers(self): + """http.server's own error pages (501 for an unknown method, 414, 431) carry no-store and nosniff, as the + API's own responses do, each once.""" + host = f"Host: 127.0.0.1:{self.port}\r\n".encode() + for request, status in ((b"OPTIONS /api/gc HTTP/1.1\r\n" + host, b"501"), + (b"GET /" + b"a" * 70000 + b" HTTP/1.1\r\n" + host, b"414"), + (b"GET /api/status HTTP/1.1\r\n" + host + b"X-A: b\r\n" * 101, b"431")): + with self.subTest(status=status), socket.create_connection(("127.0.0.1", self.port), timeout=10) as sock: + sock.sendall(request + b"\r\n") + data, _ = read_until_closed(sock) + self.assertTrue(data.startswith(b"HTTP/1.0 " + status + b" "), data[:80]) + self.assert_api_headers(data) + _, headers, _ = self.call("GET", "/api/status", headers={cli.CHALLENGE_HEADER: "ab" * 16}) + self.assertEqual(headers.get_all("Cache-Control"), ["no-store"]) + self.assertEqual(headers.get_all("X-Content-Type-Options"), ["nosniff"]) + self.assertEqual(len(headers.get_all(cli.PROOF_HEADER)), 1) + + def assert_api_headers(self, response: bytes): + """The raw response carries Cache-Control: no-store and nosniff once each, and no CORS header.""" + lines = response.split(b"\r\n\r\n", 1)[0].split(b"\r\n")[1:] + headers = [tuple(part.strip().lower() for part in line.split(b":", 1)) for line in lines] + self.assertEqual([value for name, value in headers if name == b"cache-control"], [b"no-store"]) + self.assertEqual([value for name, value in headers if name == b"x-content-type-options"], [b"nosniff"]) + self.assertFalse([name for name, _ in headers if name.startswith(b"access-control")]) def test_no_cors_headers(self): file_cid = self.node.put_file(b"x") @@ -584,8 +642,8 @@ def test_cat(self): self.assertEqual(self.call("GET", f"/api/cat?target={cid}&offset=12&length=5")[2], b"23456") self.assertEqual(self.call("GET", f"/api/cat?target={cid}&offset=48")[2], b"89") self.assertEqual(self.call("GET", f"/api/cat?target={cid}&offset=60")[2], b"") - self.assertEqual(self.node.called("read")[1], ((cid,), {"offset": 12, "length": 5, "for_gateway": False, - "from_node": None})) + self.assertEqual(self.node.called("read_file")[1], ((cid,), {"offset": 12, "length": 5, "from_node": None})) + self.assertEqual(self.node.called("info") + self.node.called("read"), []) # one fetch per request self.assertEqual(self.call("GET", f"/api/cat?target={cid}&offset=-1")[0], 400) self.assertEqual(self.call("GET", f"/api/cat?target={cid}&length=x")[0], 400) directory = self.node.put_dir({"a": cid}) @@ -599,7 +657,7 @@ def test_cat_failing_mid_stream_is_cut_short(self): def failing(*args, **kwargs): yield b"x" * 10 raise NotFound("provider vanished") - self.node.read = failing + self.node.read_file = lambda *args, **kwargs: ({"type": "file", "size": 100, "cid": cid}, failing()) logged = mock.patch("rnfs.api.log") logged.start() self.addCleanup(logged.stop) @@ -642,7 +700,7 @@ def test_validation(self): def test_error_mapping(self): file_cid = self.node.put_file(b"abc") for error, status in ((ValueError("v"), 400), (NotFound("n"), 404), (TooLarge("t"), 413), - (InvalidDag("i"), 422), (Busy("b"), 503), (StorageFull("s"), 507), + (Denied("d"), 451), (InvalidDag("i"), 422), (Busy("b"), 503), (StorageFull("s"), 507), (RnfsError("r"), 500), (RuntimeError("boom"), 500)): with mock.patch.object(self.node, "stat", side_effect=error), mock.patch("rnfs.api.log"): code, headers, data = self.call("GET", f"/api/stat?target={file_cid}") @@ -823,8 +881,8 @@ def failing(*args, **kwargs): yield b"x" raise RuntimeError("boom") cid = self.node.put_file(b"0123456789") - with mock.patch.object(self.node, "info", return_value={"type": "file", "size": 10, "cid": cid}), \ - mock.patch.object(self.node, "read", failing), mock.patch("rnfs.api.log") as log: + with mock.patch.object(self.node, "read_file", lambda *args, **kwargs: ( + {"type": "file", "size": 10, "cid": cid}, failing())), mock.patch("rnfs.api.log") as log: with self.assertRaises(http.client.IncompleteRead): # cut short, as the failure happened mid-stream self.call("GET", f"/api/cat?target={cid}/%1b%5b2J%0a") (message, _), = [call.args for call in log.call_args_list] diff --git a/tests/test_apiclient.py b/tests/test_apiclient.py index 4a7de02..3b62331 100644 --- a/tests/test_apiclient.py +++ b/tests/test_apiclient.py @@ -11,11 +11,11 @@ from rnfs import apiclient, cli from rnfs.api import ApiServer -from rnfs.apiclient import (ApiError, AuthFailed, BadRequest, Busy, Client, ConnectionFailed, DaemonNotRunning, Denied, - InvalidDag, NotFound, ServerError, SettingsError, StorageFull, TokenError, TooLarge) +from rnfs.apiclient import (ApiConnection, ApiError, AuthFailed, BadRequest, Busy, ConnectionFailed, DaemonNotRunning, + Denied, InvalidDag, NotFound, ServerError, SettingsError, StorageFull, TokenError, TooLarge) from rnfs.config import DEFAULT_API_PORT from rnfs import errors -from tests.support import POSIX_MODES, free_port +from tests.support import POSIX_MODES, free_port, real from tests.test_api import FakeNode ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) @@ -70,7 +70,7 @@ def feature(self, name): class ClientTestCase(unittest.TestCase): - """Base: a NodeWithJobs behind an ApiServer, its token in /api_token, and a Client for it.""" + """Base: a NodeWithJobs behind an ApiServer, its token in /api_token, and an ApiConnection for it.""" def setUp(self): """Start the server and write its token.""" @@ -83,7 +83,7 @@ def setUp(self): self.tmp = tmp.name self.token_file = os.path.join(self.tmp, "api_token") cli.write_token(self.token_file, self.server.token) - self.api = Client(self.tmp, port=self.server.port) + self.api = ApiConnection(self.tmp, port=self.server.port) def write(self, name: str, data: bytes) -> str: """A file in tmp holding data.""" @@ -111,8 +111,8 @@ class ImportTest(unittest.TestCase): """Apps importing the library don't pay for loading RNS.""" def test_does_not_import_rns(self): - """Importing rnfs.apiclient and making a Client loads no RNS module.""" - code = ("import sys; from rnfs import apiclient; apiclient.Client(port=1); " + """Importing rnfs.apiclient and making an ApiConnection loads no RNS module.""" + code = ("import sys; from rnfs import apiclient; apiclient.ApiConnection(port=1); " "print(any(name == 'RNS' or name.startswith('RNS.') for name in sys.modules))") result = subprocess.run([sys.executable, "-c", code], cwd=ROOT, capture_output=True, timeout=60, encoding="utf-8", errors="replace") @@ -137,25 +137,25 @@ def test_default_dir_and_token(self): """The default rnfs dir is ~/.rnfs/ (the home directory on every OS) and the token is api_token in it.""" self.assertEqual(apiclient.DEFAULT_DIR, "~/.rnfs/") self.assertEqual(apiclient.token_path(), os.path.join(os.path.expanduser("~/.rnfs/"), "api_token")) - self.assertEqual(Client(port=1).token_file, apiclient.token_path()) - self.assertEqual(Client(self.tmp, port=1).token_file, os.path.join(self.tmp, "api_token")) - self.assertEqual(Client(port=1, token_file="/x/t").token_file, "/x/t") + self.assertEqual(ApiConnection(port=1).token_file, apiclient.token_path()) + self.assertEqual(ApiConnection(self.tmp, port=1).token_file, os.path.join(self.tmp, "api_token")) + self.assertEqual(ApiConnection(port=1, token_file="/x/t").token_file, "/x/t") def test_port_from_settings(self): """The port is api_port in rnfs.toml, else the default; one given wins.""" - self.assertEqual(Client(self.tmp).port, DEFAULT_API_PORT) + self.assertEqual(ApiConnection(self.tmp).port, DEFAULT_API_PORT) self.settings("api_port = 9123\n") - self.assertEqual(Client(self.tmp).port, 9123) - self.assertEqual(Client(self.tmp, port=9124).port, 9124) + self.assertEqual(ApiConnection(self.tmp).port, 9123) + self.assertEqual(ApiConnection(self.tmp, port=9124).port, 9124) def test_bad_settings(self): """Settings that can't be read raise SettingsError, saying why.""" self.settings("api_port = 99999\n") with self.assertRaises(SettingsError) as raised: - Client(self.tmp) + ApiConnection(self.tmp) self.assertIn("api_port must be a whole number", str(raised.exception)) self.assertIsNone(raised.exception.status) - Client(self.tmp, port=1) # not read when the port is given + ApiConnection(self.tmp, port=1) # not read when the port is given class OperationsTest(ClientTestCase): @@ -166,7 +166,7 @@ def test_status_and_version(self): self.assertEqual(self.api.status()["node_id"], bytes(range(16)).hex()) os.unlink(self.token_file) seen = self.tokens_seen() - api = Client(self.tmp, port=self.server.port) + api = ApiConnection(self.tmp, port=self.server.port) self.assertEqual(api.version(), self.api.daemon_version) self.assertEqual(seen, [("/api/status", None)]) @@ -176,7 +176,9 @@ def test_add_and_cat(self): os.chdir(self.tmp) self.addCleanup(os.chdir, ROOT) cid = self.api.add_path("f.bin", pin=False, chunker="fastcdc")["cid"] - self.assertEqual(self.node.called("add_path")[0][0], (path,)) + sent, = self.node.called("add_path")[0][0] + self.assertTrue(os.path.isabs(sent)) + self.assertEqual(real(sent), real(path)) # macOS: getcwd() is /private/var, tempfile's /var self.assertEqual(self.node.called("add_path")[0][1]["pin"], False) self.assertEqual(self.node.called("add_path")[0][1]["chunker"], "fastcdc") self.assertEqual(self.api.add_stream(b"hello world")["cid"], cid) @@ -282,7 +284,7 @@ def answer(self): def test_impostor_gets_neither_token_nor_body(self): """A listener that can't prove it knows the token gets only the challenge: AuthFailed, nothing sent.""" received = [] - api = Client(self.tmp, port=self.impostor(received)) + api = ApiConnection(self.tmp, port=self.impostor(received)) with self.assertRaises(AuthFailed) as raised: api.call("POST", "/api/name/import", params={"alias": "stolen"}, body=b"K" * 64) self.assertIn("couldn't prove it knows the API token", str(raised.exception)) @@ -306,7 +308,7 @@ def test_version_check_in_one_place(self): """The proof request gives the daemon's version to check_daemon, which can refuse before the token goes.""" refused = [] - class Strict(Client): + class Strict(ApiConnection): def check_daemon(self, version): """Refuse every daemon, noting its version.""" refused.append(version) @@ -329,7 +331,7 @@ class ErrorTest(ClientTestCase): def test_no_daemon(self): """Nothing on the port: DaemonNotRunning, with no status.""" with self.assertRaises(DaemonNotRunning) as raised: - Client(self.tmp, port=free_port()).version() + ApiConnection(self.tmp, port=free_port()).version() self.assertIn("no daemon is running", str(raised.exception)) self.assertIsNone(raised.exception.status) @@ -385,7 +387,8 @@ def failing(*args, **kwargs): """Ten bytes, then the fetch fails.""" yield b"x" * 10 raise errors.NotFound("provider vanished") - with mock.patch.object(self.node, "read", failing), mock.patch("rnfs.api.log"): + with mock.patch.object(self.node, "read_file", lambda *args, **kwargs: ( + {"type": "file", "size": 100, "cid": cid}, failing())), mock.patch("rnfs.api.log"): stream = self.api.cat(cid) self.assertEqual(stream.read(10), b"x" * 10) with self.assertRaises(ConnectionFailed) as raised: diff --git a/tests/test_cid.py b/tests/test_cid.py index d9760c9..30568dc 100644 --- a/tests/test_cid.py +++ b/tests/test_cid.py @@ -109,7 +109,8 @@ def test_rejects_malformed(self): with self.assertRaises(ValueError): Cid.parse(value) self.assertFalse(cid_module.is_valid(value)) - for reason, binary in {"too short": b"\x01\x55", "not bytes": "abc", "dag-pb": bytes([1, 0x70, 0x12, 0x20]) + digest}.items(): + invalid = {"too short": b"\x01\x55", "not bytes": "abc", "dag-pb": bytes([1, 0x70, 0x12, 0x20]) + digest} + for reason, binary in invalid.items(): with self.subTest(reason): with self.assertRaises(ValueError): Cid.from_bytes(binary) diff --git a/tests/test_cli.py b/tests/test_cli.py index a947bff..fce36c4 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -353,12 +353,28 @@ def test_bad_settings_fail_cleanly(self): self.assertIn("invalid settings: private must be true or false", str(raised.exception.code)) def test_node_that_cant_start_fails_cleanly(self): - # An error building the node (e.g. an rnfs dir that isn't private) is a message, not a traceback + """An error building the node (e.g. an rnfs dir that isn't private) is a message, not a traceback, and + the daemon.lock is let go.""" from rnfs.errors import RnfsError with mock.patch("rnfs.node.RnfsNode", side_effect=RnfsError("the rnfs dir isn't private")), \ contextlib.redirect_stderr(io.StringIO()), self.assertRaises(SystemExit) as raised: cli.main(["--dir", self.tmp, "daemon"]) self.assertEqual(raised.exception.code, "rnfs: the rnfs dir isn't private") + self.assert_unlocked() + + def test_debug_log_that_cant_open_fails_cleanly(self): + """A debug log that can't be written is a message, and the daemon.lock taken before it is let go.""" + with mock.patch("rnfs.debuglog.start", side_effect=OSError("disk full")), \ + contextlib.redirect_stderr(io.StringIO()), self.assertRaises(SystemExit) as raised: + cli.main(["--dir", self.tmp, "daemon", "--debug-log"]) + self.assertEqual(raised.exception.code, "rnfs: can't write the debug log: disk full") + self.assert_unlocked() + + def assert_unlocked(self): + """The failed daemon closed its daemon.lock: this process can lock it again (an open descriptor would + hold the lock, and on Windows keep the temp dir from being removed).""" + fd = fsutil.lock_exclusive(os.path.join(self.tmp, cli.LOCK_FILE)) + os.close(fd) class TokenTest(CliTestCase): diff --git a/tests/test_client.py b/tests/test_client.py index 4a385c4..c46b168 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -201,6 +201,7 @@ def test_small_response_gives_no_rate(self): self.assertIsNone(self.client.rate_estimate(NODE)) def test_blocks_elapsed_counts_from_sending(self): + """A /blocks result's elapsed time runs from sending the request, not from opening the link.""" cid, block = block_of(100, 1) def answer(link, path, data): @@ -211,8 +212,9 @@ def answer(link, path, data): fakes.FakeLink.establish_delay = 0.5 start = time.monotonic() result = self.call(lambda: self.client.blocks(NODE, [cid])) - self.assertGreaterEqual(time.monotonic() - start, 0.6) - self.assertGreaterEqual(result.elapsed, 0.1) + # A tick short of 0.6 s and 0.1 s: before Python 3.13, time.monotonic on Windows ticks every 15.6 ms + self.assertGreaterEqual(time.monotonic() - start, 0.58) + self.assertGreaterEqual(result.elapsed, 0.08) self.assertLess(result.elapsed, 0.5) # opening the link isn't counted self.respond(lambda link, path, data: False) self.assertIsNone(self.call(lambda: self.client.blocks(NODE, [cid])).elapsed) diff --git a/tests/test_dag.py b/tests/test_dag.py index 11a5914..bd693f9 100644 --- a/tests/test_dag.py +++ b/tests/test_dag.py @@ -184,5 +184,53 @@ def test_invalid(self): self.assertFalse(dag.is_valid_name(name)) +class MemoTest(unittest.TestCase): + """decode() remembers the nodes of blocks it decoded, and only those.""" + + def setUp(self): + """An empty memo, restored to the module's own afterwards.""" + saved = dag._memo + dag._memo = dag._Memo() + self.addCleanup(setattr, dag, "_memo", saved) + + def node(self, i: int) -> bytes: + """A valid file node's block, different for each i.""" + return dag.encode(FileNode([Child(Cid.for_block(RAW, b"%d" % i), 10)])) + + def test_same_bytes_give_the_remembered_node(self): + """A second decode of the same bytes (as any bytes-like object) is the first's node, unchanged.""" + block = self.node(1) + node = dag.decode(block) + self.assertIs(dag.decode(bytearray(block)), node) + self.assertIs(dag.decode(memoryview(block)), node) + self.assertEqual(node, FileNode([Child(Cid.for_block(RAW, b"1"), 10)])) + + def test_invalid_blocks_are_checked_every_time(self): + """A block that fails to decode isn't remembered: it raises each time.""" + bad = pack([1, [[Cid.for_block(RAW, b"x").binary, 10]], 0]) + for _ in range(2): + with self.assertRaises(InvalidDag): + dag.decode(bad) + self.assertEqual(len(dag._memo._nodes), 0) + + def test_bounded_by_count_and_bytes(self): + """At most MEMO_BLOCKS nodes and MEMO_BYTES of blocks, the least recently used forgotten first.""" + blocks = [self.node(i) for i in range(dag.MEMO_BLOCKS + 10)] + first = dag.decode(blocks[0]) + for block in blocks[1:]: + dag.decode(block) + self.assertEqual(len(dag._memo._nodes), dag.MEMO_BLOCKS) + self.assertIsNot(dag.decode(blocks[0]), first) # forgotten: decoded again + self.assertIs(dag.decode(blocks[-1]), dag.decode(blocks[-1])) + size = len(dag.encode(FileNode([Child(Cid.for_block(RAW, b"0-%d" % i), 10) for i in range(2000)]))) + for n in range(dag.MEMO_BYTES // size + 5): + big = dag.encode(FileNode([Child(Cid.for_block(RAW, b"%d-%d" % (n, i)), 10) for i in range(2000)])) + dag.decode(big) + dag.decode(big) # the same block again: counted once + self.assertLessEqual(dag._memo._bytes, dag.MEMO_BYTES) + self.assertGreater(dag._memo._bytes, dag.MEMO_BYTES - 2 * size) + self.assertEqual(dag._memo._bytes, sum(len(block) for block in dag._memo._nodes)) + + if __name__ == "__main__": unittest.main() diff --git a/tests/test_debuglog.py b/tests/test_debuglog.py index 1f19e57..75c7bd7 100644 --- a/tests/test_debuglog.py +++ b/tests/test_debuglog.py @@ -10,9 +10,10 @@ import RNS -from rnfs import __version__, debuglog, fsutil, log +from rnfs import __version__, debuglog, log from rnfs.config import Config from rnfs.gateway import GatewayServer +from tests.support import POSIX_MODES from tests.test_api import FakeNode, raw_request @@ -57,8 +58,9 @@ def test_begins_with_the_version_then_versions_and_settings(self): self.assertNotIn("'max_storage'", third) # only settings that differ from the defaults self.assertEqual(self.debug.path, os.path.join(self.tmp.name, "debug.log")) - @unittest.skipIf(fsutil.IS_WINDOWS, "no POSIX modes on Windows") + @unittest.skipUnless(POSIX_MODES, "Windows has no POSIX permission bits") def test_readable_by_its_owner_only(self): + """The debug log is readable by its owner only.""" self.assertEqual(stat.S_IMODE(os.stat(self.debug.path).st_mode) & 0o077, 0) def test_each_start_keeps_the_last_run_as_dot_1(self): diff --git a/tests/test_denylist.py b/tests/test_denylist.py index 6c1b530..fa0d7ea 100644 --- a/tests/test_denylist.py +++ b/tests/test_denylist.py @@ -18,13 +18,15 @@ from rnfs.denylist import DenyList, parse_entry, parse_list from rnfs.errors import Denied, NotFound from rnfs.features import deny as deny_feature +from rnfs.fetcher import Fetch from rnfs.gateway import GatewayServer from rnfs.names import encode_record from rnfs.network import protocol from rnfs.network.server import select_blocks -from tests.support import state_execute +from tests.support import failing_commits, state_execute, state_rows from tests.test_api import raw_request -from tests.test_node import A_ID, NodeTest, setUpModule, tearDownModule # noqa: F401 - RNS's loglevel +from tests.test_node import A_ID, B_ID, NodeTest, setUpModule, tearDownModule # noqa: F401 - RNS's loglevel +from tests.test_providers import recorded_writes, rows_written IDENTITY = bytes(range(16)) @@ -69,8 +71,8 @@ def test_bad_lists_are_refused(self): parse_list(lines.encode()) -class DenyListTest(unittest.TestCase): - """The store: local entries and accepted lists, saved in the state database.""" +class StoreTest(unittest.TestCase): + """A DenyList on a state directory of its own.""" def setUp(self): """A DenyList in a temporary state directory.""" @@ -85,6 +87,10 @@ def open(self) -> DenyList: self.addCleanup(deny.close) return deny + +class DenyListTest(StoreTest): + """The store: local entries and accepted lists, saved in the state database.""" + def test_local_entries_persist(self): """Added entries apply at once and survive a restart; removing one that isn't there adds nothing.""" cid = Cid.for_block(RAW, b"x") @@ -147,6 +153,126 @@ def test_lists_reload_from_storage(self): self.assertEqual(third.accepted(), {}) +class KnownHashTest(StoreTest): + """Files' hashes remembered, kept across restarts in deny_hashes.""" + + def file(self, n: int) -> tuple[Cid, bytes]: + """A file's CID (a DAG node, as files of more than one chunk have) and its content's hash.""" + return Cid.for_block(DAG, b"file %d" % n), hashlib.sha256(b"content %d" % n).digest() + + def reopen(self) -> DenyList: + """Save and close the DenyList, then open it again on the same state directory.""" + self.deny.save() + self.deny.close() + self.deny = self.open() + return self.deny + + def saved(self) -> list[tuple[bytes, bytes]]: + """The deny_hashes rows, least recently used first.""" + return state_rows(self.state_dir, "SELECT cid, digest FROM deny_hashes ORDER BY used") + + def test_hashes_survive_a_restart(self): + """Hashes remembered and saved are known after a restart (a CID's content never changes); others not.""" + files = [self.file(n) for n in range(3)] + for cid, digest in files: + self.deny.remember_hash(cid, digest) + again = self.reopen() + for cid, digest in files: + self.assertEqual(again.known_hash(cid), digest) + self.assertIsNone(again.known_hash(self.file(3)[0])) + + def test_order_survives_a_restart(self): + """The least recently used hash, counting lookups, is still the first forgotten after a restart.""" + with mock.patch.object(denylist, "MAX_KNOWN_HASHES", 3): + (a, a_digest), (b, b_digest), (c, c_digest), (d, d_digest) = (self.file(n) for n in range(4)) + for cid, digest in ((a, a_digest), (b, b_digest), (c, c_digest)): + self.deny.remember_hash(cid, digest) + self.deny.known_hash(a) # b is now the least recently used + again = self.reopen() + again.remember_hash(d, d_digest) + self.assertIsNone(again.known_hash(b)) + self.assertEqual([again.known_hash(cid) for cid in (a, c, d)], [a_digest, c_digest, d_digest]) + again.save() + self.assertEqual(self.saved(), [(a.binary, a_digest), (c.binary, c_digest), (d.binary, d_digest)]) + + def test_bounded_on_save_and_load(self): + """Saves delete the hashes forgotten past MAX_KNOWN_HASHES; a table over it (the bound lowered) keeps + the most recently used at load and deletes the rest.""" + files = [self.file(n) for n in range(5)] + with mock.patch.object(denylist, "MAX_KNOWN_HASHES", 3): + for cid, digest in files[:4]: + self.deny.remember_hash(cid, digest) + self.deny.save() + self.assertEqual(self.saved(), [(cid.binary, digest) for cid, digest in files[1:4]]) + self.deny.remember_hash(*files[4]) + self.deny.save() + self.assertEqual(self.saved(), [(cid.binary, digest) for cid, digest in files[2:]]) + with mock.patch.object(denylist, "MAX_KNOWN_HASHES", 2): + again = self.reopen() + self.assertEqual(self.saved(), [(cid.binary, digest) for cid, digest in files[3:]]) + self.assertIsNone(again.known_hash(files[2][0])) + self.assertEqual(again.known_hash(files[4][0]), files[4][1]) + + def test_only_changes_are_written(self): + """A save writes the hashes remembered or looked up since the last one and deletes those forgotten; + with nothing changed it writes nothing, and lookups alone never write.""" + files = [self.file(n) for n in range(5)] + for cid, digest in files[:4]: + self.deny.remember_hash(cid, digest) + with recorded_writes(self.deny._db) as transactions: + self.deny.save() + self.assertEqual(rows_written(transactions), 4) + self.deny.save() + self.assertEqual(len(transactions), 1) + self.deny.known_hash(files[0][0]) + self.deny.known_hash(files[4][0]) # not known: nothing changes + self.assertEqual(len(transactions), 1) + self.deny.save() + self.assertEqual(rows_written(transactions[1:]), 1) + with mock.patch.object(denylist, "MAX_KNOWN_HASHES", 4): + self.deny.remember_hash(*files[4]) # forgets files[1] + self.deny.save() + self.assertEqual(rows_written(transactions[2:]), 2) + self.assertEqual(self.saved(), [(cid.binary, digest) for cid, digest in files[2:4] + files[:1] + files[4:]]) + + def test_invalid_hash_rows_are_dropped(self): + """Rows planted by hand that aren't a CID, a 32-byte digest and an integer position are logged (one + warning counting them), deleted and ignored; the valid ones are kept.""" + cid, digest = self.file(0) + self.deny.remember_hash(cid, digest) + self.deny.save() + self.deny.close() + for row in ((b"not a cid", digest, 1), (self.file(1)[0].binary, digest[:31], 2), + (self.file(2)[0].binary, "text", 3), (self.file(3)[0].binary, digest, "later")): + state_execute(self.state_dir, "INSERT INTO deny_hashes VALUES (?, ?, ?)", row) + with mock.patch("rnfs.denylist.log") as log: + again = self.open() + self.assertEqual(log.call_count, 1) + self.assertIn("4 invalid rows", log.call_args.args[0]) + self.assertEqual(again.known_hash(cid), digest) + self.assertEqual(self.saved(), [(cid.binary, digest)]) + + def test_failed_save_is_saved_next_time(self): + """A save that fails raises OSError and keeps its changes for the next one.""" + files = [self.file(n) for n in range(2)] + self.deny.remember_hash(*files[0]) + self.deny.save() + self.deny.remember_hash(*files[1]) + self.deny.known_hash(files[0][0]) + with failing_commits(): + with self.assertRaises(OSError): + self.deny.save() + self.assertEqual(self.saved(), [(files[0][0].binary, files[0][1])]) + again = self.reopen() + self.assertEqual(self.saved(), [(cid.binary, digest) for cid, digest in reversed(files)]) + self.assertEqual(again.known_hash(files[1][0]), files[1][1]) + + def test_close_is_idempotent(self): + """Closing twice is harmless.""" + self.deny.close() + self.deny.close() + + class DenyNodeTest(NodeTest): """Two offline nodes: A provides content, B fetches it (tests.test_node).""" @@ -373,6 +499,20 @@ def test_read_checks_before_sending(self): with self.assertRaises(Denied): next(reader) + def test_hash_remembered_across_a_restart(self): + """After a restart, a file whose hash was checked before isn't read again to check it.""" + cid, data = self.big_file() + self.deny(self.b, file_hash(data)) + with self.assertRaises(Denied): + next(self.b.read(str(cid))) + self.b.stop() + again = self.make_node("b", B_ID) + again.routing.heard(A_ID) + with mock.patch.object(Fetch, "read_range", side_effect=AssertionError("read again")) as read: + with self.assertRaises(Denied): + next(again.read(str(cid))) + read.assert_not_called() + def test_gateway_451(self): """The gateway answers 451 with a fixed message, for a denied CID and a file with a denied hash.""" cid, data = self.big_file() @@ -572,7 +712,7 @@ def test_client_library(self): self.server.start() self.addCleanup(self.server.stop) cli.write_token(os.path.join(self.b.config.rnfs_dir, "api_token"), self.server.token) - api = apiclient.Client(self.b.config.rnfs_dir, port=self.server.port) + api = apiclient.ApiConnection(self.b.config.rnfs_dir, port=self.server.port) cid = self.shared(b"denied through the API") with mock.patch("rnfs.node.log"): self.assertEqual(api.deny_add([str(cid)]), [str(cid)]) diff --git a/tests/test_edit.py b/tests/test_edit.py index 7a464ef..3330317 100644 --- a/tests/test_edit.py +++ b/tests/test_edit.py @@ -74,11 +74,13 @@ def test_remove(self): self.assertEqual(result, self.directory({"readme": self.readme}).cid) def test_errors(self): + """A missing path is NotFound, naming it; a non-directory target, an invalid name or no path is ValueError.""" get, put = self.blocks.get, self.blocks.put with self.assertRaises(NotFound): edit.dir_rm(self.root.cid, ["docs", "missing"], get, put) - with self.assertRaises(NotFound): - edit.dir_set(self.root.cid, ["missing"], "x", self.a.cid, False, get, put) + with self.assertRaises(NotFound) as caught: + edit.dir_set(self.root.cid, ["docs", "missing"], "x", self.a.cid, False, get, put) + self.assertEqual(str(caught.exception), f"{self.root.cid}/docs/missing does not exist") with self.assertRaises(ValueError): edit.dir_set(self.root.cid, ["readme"], "x", self.a.cid, False, get, put) # not a directory with self.assertRaises(ValueError): @@ -89,6 +91,7 @@ def test_errors(self): edit.dir_rm(self.root.cid, [], get, put) def test_through_index_nodes(self): + """Edits inside a directory split into index nodes, and paths that step through them.""" with mock.patch.multiple(constants, CHUNK_SIZE=4, MAX_CHILDREN=3): files = {f"f{i:02d}": self.file(f"{i}".encode()) for i in range(20)} big = self.directory(files) @@ -100,6 +103,20 @@ def test_through_index_nodes(self): result = edit.dir_rm(root.cid, ["big", "f10"], self.blocks.get, self.blocks.put) remaining = {k: v for k, v in files.items() if k != "f10"} self.assertEqual(result, self.directory({"big": self.directory(remaining)}).cid) + # Paths stepping through big's index nodes: to a directory in it, a file, and a name it doesn't hold + sub = self.directory({"a": self.a}) + big = self.directory({**files, "f07": sub}) + root = self.directory({"big": big}) + result = edit.dir_set(root.cid, ["big", "f07"], "b", self.b.cid, False, self.blocks.get, self.blocks.put) + edited = self.directory({"a": self.a, "b": self.b}) + expected = self.directory({"big": self.directory({**files, "f07": edited})}) + self.assertEqual(result, expected.cid) + with self.assertRaises(ValueError) as caught: + edit.dir_set(root.cid, ["big", "f05"], "x", new.cid, False, self.blocks.get, self.blocks.put) + self.assertEqual(str(caught.exception), f"{root.cid}/big/f05 is not a directory") + with self.assertRaises(NotFound) as caught: + edit.dir_set(root.cid, ["big", "f99", "x"], "y", new.cid, False, self.blocks.get, self.blocks.put) + self.assertEqual(str(caught.exception), f"{root.cid}/big/f99 does not exist") def test_depth_limit(self): with mock.patch.object(constants, "MAX_DEPTH", 3): diff --git a/tests/test_embedded.py b/tests/test_embedded.py new file mode 100644 index 0000000..7cc60b6 --- /dev/null +++ b/tests/test_embedded.py @@ -0,0 +1,587 @@ +"""An rnfs node inside another program (design.md, "Embedded mode"): the guards' scope, the rnfs dir's lock, and +a two-process test over real RNS (RNFS_E2E=1) of a host program running a node beside its own destinations. + +That test runs this file as a script in two subprocesses with their own Reticulum, over local TCP: a plain RNS +peer, and a host program with an embedded node. The host's destinations must send and receive what RNS alone +allows (compressed Resources, requests carried as Resources, multi-segment Resources) while the node runs, and +its rnfs.dht links must still refuse each of those. +""" +import json +import os +import subprocess +import sys +import random +import tempfile +import threading +import time +import types +import unittest +from unittest import mock + +ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +sys.path.insert(0, ROOT) + +import RNS # noqa: E402 +from RNS.vendor import umsgpack # noqa: E402 + +from rnfs import fsutil # noqa: E402 +from rnfs.config import LOCK_FILE, Config # noqa: E402 +from rnfs.content.cid import RAW, Cid # noqa: E402 +from rnfs.network import client, guards, links # noqa: E402 +from rnfs.node import DirInUse, lock_rnfs_dir # noqa: E402 + +import tests.test_links as fakes # noqa: E402 (a module, so its test cases don't run twice) +from tests.support import free_port # noqa: E402 + +DHT_LINK_NAME = "rnfs.dht." + "ab" * 16 # as RNS.Destination.expand_name names an rnfs.dht destination +HOLD_LOCK = ("import sys; from rnfs.config import Config; from rnfs.node import lock_rnfs_dir; " + "lock_rnfs_dir(Config(rnfs_dir=sys.argv[1])); print('locked', flush=True); sys.stdin.read()") + + +def link_named(name: str, **attributes): + """A stand-in RNS.Link whose destination has name.""" + return types.SimpleNamespace(destination=types.SimpleNamespace(name=name), pending_requests=[], **attributes) + + +def advertisement(link, transfer: int = 1052, data: int = 1000, request: bool = False, compressed: bool = False, + metadata: bool = False, segments: int = 1): + """A packet carrying a crafted, packed ResourceAdvertisement (flags as RNS 1.5.6 packs them).""" + flags = (0x01 | int(compressed) << 1 | int(segments > 1) << 2 | int(request) << 3 | int(not request) << 4 + | int(metadata) << 5) + packed = umsgpack.packb({"t": transfer, "d": data, "n": 1, "h": bytes(32), "r": bytes(4), "o": bytes(32), + "i": 1, "l": segments, "q": b"q" * 16, "f": flags, "m": bytes(4)}) + return types.SimpleNamespace(plaintext=packed, link=link) + + +# What a hostile or careless peer may advertise, and whether the daemon (PROCESS) refuses it on another program's +# link; on rnfs.dht links both scopes refuse every one +HOSTILE = { + "request": (dict(request=True), True), + "transfer over data": (dict(transfer=2_000_000, data=1000), True), + "transfer over the largest response": (dict(transfer=guards.MAX_TRANSFER_SIZE + 1, + data=guards.MAX_TRANSFER_SIZE + 1), True), + "malformed sizes": (dict(transfer=-1), True), + "multi-segment": (dict(segments=3), False), + "metadata": (dict(metadata=True), False), + "compressed": (dict(compressed=True), False), # the daemon's process-wide limit of 0 fails it instead +} + + +class GuardScopeTest(unittest.TestCase): + """guards.install(scope): every link in the daemon, rnfs.dht links only in an embedded node.""" + + def setUp(self): + """Record what the guard passes to RNS and rejects; restore the scope and RNS's limit afterwards.""" + saved = guards._scope, guards._compress_limit, RNS.Resource.AUTO_COMPRESS_MAX_SIZE + + def restore(): + """Put the scope and the process-wide limit back as they were.""" + guards._scope, guards._compress_limit, RNS.Resource.AUTO_COMPRESS_MAX_SIZE = saved + self.addCleanup(restore) + self.accepted, self.rejected = [], [] + self.resource = mock.Mock(spec=RNS.Resource) + self.resource.max_decompressed_size = 123 + patches = [mock.patch.object(guards, "_original_accept", + lambda packet, **kwargs: self.accepted.append(packet) or self.resource), + mock.patch.object(RNS.Resource, "reject", self.rejected.append)] + for patcher in patches: + patcher.start() + self.addCleanup(patcher.stop) + + def accepts(self, link, **kwargs) -> bool: + """True if the installed guard passes an advertisement on link to RNS.""" + packet = advertisement(link, **kwargs) + accepted = RNS.Resource.accept(packet) is not None + self.assertEqual(accepted, packet in self.accepted) + self.assertEqual(not accepted, packet in self.rejected) + return accepted + + def test_rnfs_links_refuse_the_same_in_both_scopes(self): + """On rnfs.dht links both scopes refuse every hostile advertisement and accept an honest one.""" + for scope in (guards.PROCESS, guards.DHT): + guards.install(scope) + for name, (kwargs, _) in HOSTILE.items(): + with self.subTest(scope=scope, advertisement=name): + self.assertFalse(self.accepts(link_named(DHT_LINK_NAME), **kwargs)) + with self.subTest(scope=scope, advertisement="honest"): + self.assertTrue(self.accepts(link_named(DHT_LINK_NAME))) + + def test_daemon_guards_every_link(self): + """The daemon's scope refuses on other links what it always did.""" + guards.install(guards.PROCESS) + for name, (kwargs, refused) in HOSTILE.items(): + with self.subTest(advertisement=name): + self.assertEqual(self.accepts(link_named("hostapp.chat"), **kwargs), not refused) + + def test_embedded_node_leaves_other_links_to_rns(self): + """An embedded node's scope refuses nothing on other links.""" + guards.install(guards.DHT) + for name, (kwargs, _) in HOSTILE.items(): + with self.subTest(advertisement=name): + self.assertTrue(self.accepts(link_named("hostapp.chat"), **kwargs)) + self.assertTrue(self.accepts(None, request=True)) # a packet without a link isn't rnfs's either + + def test_daemon_messages_are_unchanged(self): + """A compressed advertisement that something else already refused is logged for that, as before.""" + guards.install(guards.PROCESS) + adv = RNS.ResourceAdvertisement.unpack(advertisement(None, request=True, compressed=True).plaintext) + reason = guards.refusal(adv, link_named(DHT_LINK_NAME)) + self.assertEqual(reason, "requests are never sent as Resources") + + def test_compression_limit_follows_the_scope(self): + """PROCESS sets RNS's limit to 0; DHT puts the program's back.""" + RNS.Resource.AUTO_COMPRESS_MAX_SIZE = 64 * 1024 * 1024 + guards._compress_limit = None + guards.install(guards.PROCESS) + self.assertEqual(RNS.Resource.AUTO_COMPRESS_MAX_SIZE, 0) + guards.install() # RequestServer and LinkPool keep the scope the node set + self.assertEqual((guards.scope(), RNS.Resource.AUTO_COMPRESS_MAX_SIZE), (guards.PROCESS, 0)) + guards.install(guards.DHT) # the host program's value comes back + self.assertEqual(RNS.Resource.AUTO_COMPRESS_MAX_SIZE, 64 * 1024 * 1024) + guards.install() + self.assertEqual((guards.scope(), RNS.Resource.AUTO_COMPRESS_MAX_SIZE), (guards.DHT, 64 * 1024 * 1024)) + with self.assertRaises(ValueError): + guards.install("everything") + + def test_rnfs_resources_accepted_decompress_nothing(self): + """An accepted rnfs.dht Resource may decompress nothing; another link's keeps its limit.""" + for scope in (guards.PROCESS, guards.DHT): + guards.install(scope) + with self.subTest(scope=scope): + self.resource.max_decompressed_size = 123 + self.assertTrue(self.accepts(link_named(DHT_LINK_NAME))) + self.assertEqual(self.resource.max_decompressed_size, 0) + self.resource.max_decompressed_size = 123 + self.assertTrue(self.accepts(link_named("hostapp.chat"))) + self.assertEqual(self.resource.max_decompressed_size, 123) + + def test_response_checks_pass_other_links_through(self): + """Another link's response packets go to RNS undecrypted.""" + received = [] + with mock.patch.object(guards, "_original_receive", lambda link, packet: received.append(packet)): + for scope in (guards.PROCESS, guards.DHT): + guards.install(scope) + packet = types.SimpleNamespace(context=RNS.Packet.RESPONSE, packet_type=RNS.Packet.DATA, + data=bytes(guards.PACKET_CHECK_SIZE + 1)) + + def decrypt(data): + """The host's link is never decrypted by rnfs.""" + raise AssertionError("decrypted") + guards.guarded_receive(link_named("hostapp.chat", initiator=False, decrypt=decrypt), packet) + self.assertEqual(received[-1:], [packet]) + + +class OnePacketRequestTest(unittest.TestCase): + """LinkPool never sends a request that would go as a Resource (which RNS compresses unless told not to).""" + + def setUp(self): + """A LinkPool over fake RNS.""" + patcher = mock.patch.object(links, "RNS", fakes.fake_rns()) + patcher.start() + self.addCleanup(patcher.stop) + fakes.FakeLink.created = [] + fakes.FakeLink.establish_delay = 0.0 + fakes.FakeLink.responder = staticmethod(lambda link, path, data: True) + self.pool = links.LinkPool(object(), Config()) + self.addCleanup(self.pool.close_all) + + def test_oversized_request_is_not_sent(self): + """A request over one packet isn't sent; one that fits is.""" + peer = fakes.run(self, lambda: self.pool.get(bytes(16))) + exchange = fakes.run(self, lambda: self.pool.request(peer, "/find", os.urandom(1000))) + self.assertIsNone(exchange.receipt) + self.assertEqual(peer.link.requests, []) + exchange = fakes.run(self, lambda: self.pool.request(peer, "/find", bytes(16))) + self.assertIsNotNone(exchange.receipt) + self.assertEqual([path for path, _, _ in peer.link.requests], ["/find"]) + + +class DirLockTest(unittest.TestCase): + """lock_rnfs_dir: one daemon or embedded node per rnfs dir, in this process or another.""" + + def setUp(self): + """A temporary rnfs dir.""" + tmp = tempfile.TemporaryDirectory() + self.addCleanup(tmp.cleanup) + self.config = Config(rnfs_dir=os.path.join(tmp.name, "rnfs")) + + def test_second_lock_in_this_process_is_refused(self): + """A second lock in this process is DirInUse, naming it.""" + fd = lock_rnfs_dir(self.config) + try: + with self.assertRaises(DirInUse) as caught: + lock_rnfs_dir(self.config) + finally: + fsutil.unlock(fd) + self.assertEqual(caught.exception.holder, os.getpid()) + self.assertIn(f"another rnfs daemon or node (process {os.getpid()}) is using {self.config.root}", + str(caught.exception)) + fsutil.unlock(lock_rnfs_dir(self.config)) # released once unlocked + + def test_lock_held_by_another_process_is_refused(self): + """A lock another process holds is DirInUse, naming it.""" + env = {**os.environ, "PYTHONPATH": ROOT} + holder = subprocess.Popen([sys.executable, "-c", HOLD_LOCK, self.config.rnfs_dir], stdin=subprocess.PIPE, + stdout=subprocess.PIPE, encoding="utf-8", env=env) + self.addCleanup(holder.wait, 10) + self.addCleanup(holder.stdin.close) + self.addCleanup(holder.stdout.close) + self.assertEqual(holder.stdout.readline().strip(), "locked") + with self.assertRaises(DirInUse) as caught: + lock_rnfs_dir(self.config) + self.assertEqual(caught.exception.holder, holder.pid) + self.assertEqual(fsutil.lock_holder(os.path.join(self.config.root, LOCK_FILE)), holder.pid) + + +# Two processes over real RNS: a plain RNS peer, and a host program with an embedded node + +PEER_CONFIG = """ +[reticulum] + enable_transport = No + share_instance = No +[logging] + loglevel = 2 +[interfaces] + [[TCP Server]] + type = TCPServerInterface + enabled = yes + listen_ip = 127.0.0.1 + listen_port = {port} +""" + +HOST_CONFIG = """ +[reticulum] + enable_transport = No + share_instance = No +[logging] + loglevel = 2 +[interfaces] + [[TCP Client]] + type = TCPClientInterface + enabled = yes + target_host = 127.0.0.1 + target_port = {port} +""" + +APP = "rnfstest" # the host programs' own app name +BIG = 3_000_000 # over RNS's 1 MiB segment: a multi-segment Resource, and very compressible (zeros) +REQUEST = 600_000 # a request over the link MDU, so RNS sends it as a Resource (one segment, compressible) +DHT_REQUEST = 800 # over the link MDU, within rnfs.dht's max_request_size +# Blocks the peer's fake rnfs.dht server answers /blocks with: random (RNS sends it uncompressed, the control), +# and zeros (RNS compresses it); a request for BIG_CID gets BIG bytes, a multi-segment Resource +RANDOM_BLOCK = random.Random(7).randbytes(60_000) +ZERO_BLOCK = bytes(60_000) +BIG_CID = Cid.for_block(RAW, b"big") +WAIT = 60.0 # seconds any one exchange may take over local TCP + + +class Recorder: + """What arrives on a program's own links: every Resource RNS starts receiving (its compressed flag and + segment), and the plain Resources completed.""" + + def __init__(self): + """Nothing seen yet.""" + self.started: list[dict] = [] + self.completed: list[int] = [] + + def watch(self, link): + """Record link's incoming Resources, accepting plain ones as RNS's ACCEPT_ALL does.""" + link.set_resource_strategy(RNS.Link.ACCEPT_ALL) + # The only one-segment Resource each side receives is the /echo request; the multi-segment ones are the + # /echo response and the plain Resource (RNS sets no request or response flag before this is called) + link.set_resource_started_callback( + lambda resource: self.started.append({"compressed": bool(resource.compressed), + "hash": resource.original_hash.hex(), + "segment": resource.segment_index, + "segments": resource.total_segments})) + link.set_resource_concluded_callback(self._concluded) + + def _concluded(self, resource): + """A plain Resource completed: record its size.""" + if resource.status == RNS.Resource.COMPLETE: + data = resource.data + self.completed.append(len(data.read()) if hasattr(data, "read") else len(data)) + + def summary(self) -> dict: + """What was seen, for the test to check.""" + return {"started": self.started, "completed": self.completed} + + +def app_destination(recorder: Recorder) -> "RNS.Destination": + """The program's own destination: /echo answers any request with BIG zero bytes, RNS compressing it.""" + identity = RNS.Identity() + destination = RNS.Destination(identity, RNS.Destination.IN, RNS.Destination.SINGLE, APP, "echo") + destination.register_request_handler("/echo", + response_generator=lambda path, data, request_id, identity, at: bytes(BIG), + allow=RNS.Destination.ALLOW_ALL) # auto_compress: RNS's default + destination.set_link_established_callback(recorder.watch) + return destination + + +def open_link(destination_hash: bytes, app: str, aspect: str, recorder: Recorder | None = None): + """An active link to destination_hash's app.aspect destination, or None.""" + RNS.Transport.request_path(destination_hash) + deadline = time.monotonic() + WAIT + while RNS.Identity.recall(destination_hash) is None and time.monotonic() < deadline: + time.sleep(0.1) + identity = RNS.Identity.recall(destination_hash) + if identity is None: + return None + link = RNS.Link(RNS.Destination(identity, RNS.Destination.OUT, RNS.Destination.SINGLE, app, aspect)) + if recorder is not None: + recorder.watch(link) + while link.status != RNS.Link.ACTIVE and time.monotonic() < deadline: + time.sleep(0.1) + return link if link.status == RNS.Link.ACTIVE else None + + +def request(link, path: str, data): + """Send a request on link and wait: the response's length (bytes) or the response itself (anything else), + or None if it failed or timed out.""" + receipt = link.request(path, data) + deadline = time.monotonic() + WAIT + while receipt and receipt.get_status() not in (RNS.RequestReceipt.READY, RNS.RequestReceipt.FAILED) \ + and time.monotonic() < deadline: + time.sleep(0.1) + if not receipt or receipt.get_status() != RNS.RequestReceipt.READY: + return None + response = receipt.response + if hasattr(response, "read"): + response = response.read() + return len(response) if isinstance(response, (bytes, bytearray)) else response + + +def send_resource(link, data: bytes, timeout: float = WAIT) -> bool: + """Send data as a plain Resource on link and wait for its last segment's outcome; True if it completed.""" + concluded, outcome = threading.Event(), [] + RNS.Resource(data, link, callback=lambda resource: (outcome.append(resource.status), concluded.set())) + concluded.wait(timeout) + return outcome == [RNS.Resource.COMPLETE] + + +def exercise(target_hash: bytes, recorder: Recorder) -> dict: + """What this program's links can do with another's /echo destination: a request carried as a compressed + Resource answered by a compressed multi-segment one, and a compressed multi-segment plain Resource.""" + link = open_link(target_hash, APP, "echo", recorder) + if link is None: + return {"linked": False} + result = {"linked": True, "echo": request(link, "/echo", bytes(REQUEST)), "resource": send_resource(link, + bytes(BIG))} + link.teardown() + return result + + +def run_peer(configdir: str): + """The plain RNS peer: its /echo destination, and a fake rnfs.dht server answering /find with Resources a + node refuses; then, given the host's destinations on stdin, what it can do with them. Prints JSON.""" + RNS.Reticulum(configdir=configdir) + recorder = Recorder() + own = app_destination(recorder) + dht = RNS.Destination(RNS.Identity(), RNS.Destination.IN, RNS.Destination.SINGLE, "rnfs", "dht") + blocks = {Cid.for_block(RAW, RANDOM_BLOCK).binary: RANDOM_BLOCK, Cid.for_block(RAW, ZERO_BLOCK).binary: ZERO_BLOCK} + + def answer_blocks(path, data, request_id, identity, at): + """[block, ...] for the CIDs asked for; BIG bytes for BIG_CID.""" + cids = data[1] + if cids == [BIG_CID.binary]: + return bytes(BIG) + return [blocks.get(binary) for binary in cids] + dht.register_request_handler("/blocks", response_generator=answer_blocks, + allow=RNS.Destination.ALLOW_ALL) # RNS's default auto_compress + own.announce() + dht.announce() + print("PEER", own.hash.hex(), dht.hash.hex(), flush=True) + host_echo, host_dht = (bytes.fromhex(text) for text in sys.stdin.readline().split()[1:3]) + results = exercise(host_echo, recorder) + # The embedded node's rnfs.dht: a request carried as a Resource is refused, a one-packet one answered + link = open_link(host_dht, "rnfs", "dht") + results["dht_linked"] = link is not None + if link is not None: + results["dht_small"] = request(link, "/find", bytes(16)) + # Over one packet on a default-MTU link (this TCP one negotiated a larger MTU, so its MDU is lowered here) + # and within rnfs.dht's max_request_size, which RNS checks itself: rnfs's guard is what refuses it + link.mdu = RNS.Link.MDU + results["dht_request_resource"] = request(link, "/find", os.urandom(DHT_REQUEST)) + results["dht_resource"] = send_resource(link, bytes(BIG), timeout=15) # RNS's own ACCEPT_NONE, as ever + link.teardown() + print("EXERCISED", flush=True) + sys.stdin.readline() # the host has finished with this process's destinations + print("RESULT", json.dumps({**results, "seen": recorder.summary()}), flush=True) + os._exit(0) + + +def run_host(configdir: str, rnfs_dir: str, peer_echo_hex: str, peer_dht_hex: str): + """The host program: its own Reticulum and /echo destination with an embedded node beside them. Prints + its destinations, then what its links could do with the peer's and what the node's links refused.""" + import rnfs + + reticulum = RNS.Reticulum(configdir=configdir) + recorder = Recorder() + own = app_destination(recorder) + refused = [] + refusal = guards.refusal + + def recording_refusal(adv, link=None): + """guards.refusal, recording each refusal's reason.""" + reason = refusal(adv, link) + if reason is not None: + refused.append(reason) + return reason + guards.refusal = recording_refusal + # LXMF and the DHT mode are left to rnfs.Node's defaults: off, and client + settings = {"default_bootstrap": False, "startup_announce_delay": 2, "path_timeout": WAIT} + # A start failing late leaves nothing behind (its rnfs.dht destination above all), so the next one works + from rnfs.node import RnfsNode + with mock.patch.object(RnfsNode, "_start_network", side_effect=RuntimeError("failing on purpose")): + try: + rnfs.Node(rnfs_dir, reticulum=reticulum, settings=settings) + except RuntimeError: + pass + node = rnfs.Node(rnfs_dir, reticulum=reticulum, settings=settings) + own.announce() + print("HOST", own.hash.hex(), node.node_id.hex(), flush=True) + results = exercise(bytes.fromhex(peer_echo_hex), recorder) + peer_dht = bytes.fromhex(peer_dht_hex) + results["blocks"] = {name: node.internal.client.blocks(peer_dht, [cid]).status for name, cid in ( + ("random", Cid.for_block(RAW, RANDOM_BLOCK)), ("zeros", Cid.for_block(RAW, ZERO_BLOCK)), ("big", BIG_CID))} + results["compress_limit"] = RNS.Resource.AUTO_COMPRESS_MAX_SIZE + results["lxmf_imported"] = "LXMF" in sys.modules + results["lxmf_network"] = node.internal.feature("lxmf").network is not None + results["dht_mode"] = node.status().dht_mode + print("EXERCISED", flush=True) + sys.stdin.readline() # the peer has finished with this process's destinations + results["refused"] = refused + results["seen"] = recorder.summary() + node.close() + # Reticulum is the host's, and still running: its links work as before + results["reticulum_kept"] = RNS.Reticulum.get_instance() is reticulum + link = open_link(bytes.fromhex(peer_echo_hex), APP, "echo") + results["after_stop"] = request(link, "/echo", b"x") if link is not None else None + print("RESULT", json.dumps(results), flush=True) + os._exit(0) + + +@unittest.skipUnless(os.environ.get("RNFS_E2E") == "1", "set RNFS_E2E=1 to run") +class HostProgramTest(unittest.TestCase): + """A host program's own destinations beside an embedded node, against a plain RNS peer.""" + + @staticmethod + def stop(proc: subprocess.Popen): + """End a subprocess (it exits when its stdin closes) and close its pipes.""" + try: + if not proc.stdin.closed: + proc.stdin.close() + proc.wait(timeout=10) + except (OSError, subprocess.TimeoutExpired): + proc.kill() + proc.wait() + proc.stdout.close() + + @staticmethod + def read_line(proc: subprocess.Popen, prefix: str, output: list) -> str | None: + """What follows prefix on the next line of proc's output starting with it (output collects every line + read, for failure messages); None if proc ends first.""" + for line in iter(proc.stdout.readline, ""): + output.append(line) + if line.split(" ", 1)[0].strip() == prefix: + return line[len(prefix):].strip() + return None + + @staticmethod + def send(proc: subprocess.Popen, line: str): + """Write line to proc's stdin.""" + proc.stdin.write(line + "\n") + proc.stdin.flush() + + def start(self, *args) -> subprocess.Popen: + """Run this file as a script with args.""" + env = {**os.environ, "PYTHONUNBUFFERED": "1", "PYTHONPATH": ROOT} + proc = subprocess.Popen([sys.executable, os.path.abspath(__file__), *args], stdin=subprocess.PIPE, + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, encoding="utf-8", + errors="replace", env=env) + self.addCleanup(self.stop, proc) + return proc + + def test_host_destinations_beside_an_embedded_node(self): + """The host's destinations work as with RNS alone while the node's rnfs.dht links refuse what they always + did.""" + # Removed after the subprocesses stop (their cleanups, registered later, run first); one killed on Windows + # may not have released its files yet: don't fail the test over cleanup + directory = tempfile.TemporaryDirectory(ignore_cleanup_errors=True) + self.addCleanup(directory.cleanup) + tmp = directory.name + port = free_port() + dirs = {} + for name, template in (("peer", PEER_CONFIG), ("host", HOST_CONFIG)): + dirs[name] = os.path.join(tmp, name) + os.makedirs(dirs[name]) + with open(os.path.join(dirs[name], "config"), "w", encoding="utf-8") as f: + f.write(template.format(port=port)) + peer_output, host_output = [], [] + peer = self.start("--peer", dirs["peer"]) + peer_hashes = (self.read_line(peer, "PEER", peer_output) or "").split() + self.assertEqual(len(peer_hashes), 2, "".join(peer_output)) + host = self.start("--host", dirs["host"], os.path.join(tmp, "rnfs"), *peer_hashes) + host_hashes = self.read_line(host, "HOST", host_output) + self.assertIsNotNone(host_hashes, "".join(host_output)) + # Each exercises the other's destinations at once, then waits to be told to report + self.send(peer, f"HOST {host_hashes}") + self.assertIsNotNone(self.read_line(peer, "EXERCISED", peer_output), "".join(peer_output)) + self.assertIsNotNone(self.read_line(host, "EXERCISED", host_output), "".join(host_output)) + # The host first: after stopping its node it still uses the peer's destination + self.send(host, "") + host_result = self.read_line(host, "RESULT", host_output) + self.assertIsNotNone(host_result, "".join(host_output)) + self.send(peer, "") + peer_result = self.read_line(peer, "RESULT", peer_output) + self.assertIsNotNone(peer_result, "".join(peer_output)) + peer_results, host_results = json.loads(peer_result), json.loads(host_result) + detail = json.dumps({"peer": peer_results, "host": host_results}, indent=1) + + # The host's own destinations, with the node running, behave as with RNS alone, both ways + for side in (peer_results, host_results): + self.assertTrue(side["linked"], detail) + self.assertEqual(side["echo"], BIG, detail) # a request carried as a Resource, a multi-segment answer + self.assertTrue(side["resource"], detail) # a multi-segment plain Resource + self.assertIn(BIG, side["seen"]["completed"], detail) + started = side["seen"]["started"] + # Compressed: the request Resource arriving at /echo, and every segment of the response and of the + # plain Resource + self.assertTrue(all(r["compressed"] for r in started), detail) + self.assertEqual(sum(r["segments"] == 1 for r in started), 1, detail) + multi = {r["hash"]: r["segments"] for r in started if r["segments"] > 1} + self.assertEqual(len(multi), 2, detail) + for original, segments in multi.items(): + self.assertEqual(sorted(r["segment"] for r in started if r["hash"] == original), + list(range(1, segments + 1)), detail) + self.assertNotEqual(host_results["compress_limit"], 0, detail) # the host's process-wide setting is its own + + # The node's rnfs.dht links still refuse each of those + self.assertTrue(peer_results["dht_linked"], detail) + self.assertIsNotNone(peer_results["dht_small"], detail) # the link works for a one-packet request + self.assertIsNone(peer_results["dht_request_resource"], detail) + self.assertFalse(peer_results["dht_resource"], detail) + self.assertEqual(host_results["blocks"]["random"], client.OK, detail) # the control: an honest answer + self.assertNotEqual(host_results["blocks"]["zeros"], client.OK, detail) # compressed, one segment + self.assertNotEqual(host_results["blocks"]["big"], client.OK, detail) # compressed, multi-segment + self.assertIn("requests are never sent as Resources", host_results["refused"], detail) + self.assertIn("rnfs Resources are never compressed", host_results["refused"], detail) + self.assertTrue(all(reason in ("requests are never sent as Resources", "rnfs Resources are never compressed") + or reason.startswith("rnfs responses are one segment") + for reason in host_results["refused"]), detail) # nothing of the host's refused + + # LXMF off: no LXMF objects, not even the module; the host's Reticulum survives the node's stop + self.assertFalse(host_results["lxmf_imported"], detail) + self.assertFalse(host_results["lxmf_network"], detail) + self.assertEqual(host_results["dht_mode"], "client", detail) + self.assertTrue(host_results["reticulum_kept"], detail) + self.assertEqual(host_results["after_stop"], BIG, detail) + + +if __name__ == "__main__": + if sys.argv[1:2] == ["--peer"]: + run_peer(sys.argv[2]) + elif sys.argv[1:2] == ["--host"]: + run_host(*sys.argv[2:6]) + else: + unittest.main() diff --git a/tests/test_fsutil.py b/tests/test_fsutil.py index e776150..d7a7e05 100644 --- a/tests/test_fsutil.py +++ b/tests/test_fsutil.py @@ -6,6 +6,7 @@ import subprocess import sys import tempfile +import types import unittest from unittest import mock @@ -342,7 +343,7 @@ def setUp(self): def lock(self) -> int: """Take the lock in this process (released before the directory is removed).""" fd = fsutil.lock_exclusive(self.path) - self.addCleanup(os.close, fd) + self.addCleanup(fsutil.unlock, fd) return fd def test_a_second_process_is_refused_until_the_first_lets_go(self): @@ -375,6 +376,19 @@ def test_lock_file_is_private(self): self.lock() self.assertEqual(os.stat(self.path).st_mode & 0o777, 0o600) + def test_unlock_frees_it_at_once(self): + """unlock() releases the lock and closes the descriptor: the lock can be taken again right away (another + descriptor of the file in this process is refused while it is held).""" + fd = fsutil.lock_exclusive(self.path) + try: + with self.assertRaises(BlockingIOError): + fsutil.lock_exclusive(self.path) + finally: + with mock.patch("os.close", wraps=os.close) as close: + fsutil.unlock(fd) + close.assert_called_once_with(fd) + self.lock() + def test_holder_of_a_file_without_a_process_id_is_unknown(self): """lock_holder gives None for a missing or unreadable lock file.""" self.assertIsNone(fsutil.lock_holder(self.path)) @@ -383,5 +397,58 @@ def test_holder_of_a_file_without_a_process_id_is_unknown(self): self.assertIsNone(fsutil.lock_holder(self.path)) + +class WindowsLockTest(unittest.TestCase): + """lock_exclusive and unlock on Windows, over a stand-in msvcrt that records each locking() call.""" + + def setUp(self): + """A lock file path in a temporary directory, and the Windows branch over the stand-in.""" + tmp = tempfile.TemporaryDirectory() + self.addCleanup(tmp.cleanup) + self.path = os.path.join(tmp.name, "daemon.lock") + self.calls = [] + self.error: OSError | None = None + msvcrt = types.SimpleNamespace(LK_UNLCK=0, LK_NBLCK=2, locking=self.locking) + for patcher in (mock.patch.object(fsutil, "IS_WINDOWS", True), + mock.patch.dict(sys.modules, {"msvcrt": msvcrt})): + patcher.start() + self.addCleanup(patcher.stop) + + def locking(self, fd: int, mode: int, nbytes: int): + """msvcrt.locking: record (mode, nbytes, the file position), then raise self.error if set.""" + self.calls.append((mode, nbytes, os.lseek(fd, 0, os.SEEK_CUR))) + if self.error is not None: + raise self.error + + def test_one_byte_past_the_text_is_locked_then_unlocked(self): + """The byte at LOCK_OFFSET is locked without waiting, and unlock() unlocks that byte before closing.""" + fd = fsutil.lock_exclusive(self.path) + self.assertEqual(fsutil.lock_holder(self.path), os.getpid()) + with mock.patch("os.close", wraps=os.close) as close: + fsutil.unlock(fd) + close.assert_called_once_with(fd) + self.assertEqual(self.calls, [(2, 1, fsutil.LOCK_OFFSET), (0, 1, fsutil.LOCK_OFFSET)]) + + def test_only_a_lock_conflict_means_another_holder(self): + """EACCES (or EDEADLOCK) is BlockingIOError; any other error is raised as it is. The file is closed.""" + for error, expected in ((PermissionError(errno.EACCES, "Permission denied"), BlockingIOError), + (OSError(errno.EDEADLK, "Resource deadlock avoided"), BlockingIOError), + (OSError(errno.EINVAL, "Invalid argument"), OSError)): + self.error = error + with self.subTest(error=error), mock.patch("os.close", wraps=os.close) as close: + with self.assertRaises(OSError) as caught: + fsutil.lock_exclusive(self.path) + self.assertIs(type(caught.exception), expected) + close.assert_called_once() + + def test_unlock_closes_even_if_unlocking_fails(self): + """A failed unlock is left to the close.""" + fd = fsutil.lock_exclusive(self.path) + self.error = PermissionError(errno.EACCES, "Permission denied") + with mock.patch("os.close", wraps=os.close) as close: + fsutil.unlock(fd) + close.assert_called_once_with(fd) + + if __name__ == "__main__": unittest.main() diff --git a/tests/test_identity.py b/tests/test_identity.py index 794f73f..d4d56d3 100644 --- a/tests/test_identity.py +++ b/tests/test_identity.py @@ -1,4 +1,5 @@ -"""Tests for the node identity file: created owner-only and atomically, kept across restarts, loose permissions fixed.""" +"""Tests for the node identity file: created owner-only and atomically, kept across restarts, loose permissions +fixed.""" import os import stat import tempfile diff --git a/tests/test_lxmf.py b/tests/test_lxmf.py index f0e8427..6c54f2a 100644 --- a/tests/test_lxmf.py +++ b/tests/test_lxmf.py @@ -1258,6 +1258,17 @@ def test_fallback_after_the_outbox_record_was_dropped(self): self.assertTrue(self.network.sent[1][3]) self.assertEqual(feature._resend, {}) + def test_failed_construction_releases_the_state_database(self): + """A constructor that raises after opening the inbox and outbox closes both: the node never closes a + feature it didn't get.""" + self.feature_node() + database = statedb._open[statedb._key(self.b.config.state_dir)] + before = database.references + with mock.patch.object(lxmf.LxmfFeature, "_release_unrecorded", side_effect=OSError("read-only")), \ + self.assertRaises(OSError): + lxmf.LxmfFeature(self.b) + self.assertEqual(database.references, before) + class MessageLogBoundsTest(unittest.TestCase): """MessageLog's protection of pin holders and its byte bound.""" diff --git a/tests/test_mesh.py b/tests/test_mesh.py index fe76c85..c1a4f77 100644 --- a/tests/test_mesh.py +++ b/tests/test_mesh.py @@ -390,7 +390,7 @@ class PeerClientCountingTest(unittest.TestCase): """PeerClient.blocks counts outcomes and verified bytes per provider.""" def test_counts_outcomes(self): - links = types.SimpleNamespace(lock=threading.Lock(), shared={}) + links = types.SimpleNamespace(shared_peer=lambda node_id: None) client = PeerClient(links, Config()) cid, data = block_of(1000, 1) results = iter([BlocksResult(OK, blocks={cid: data}), BlocksResult(BUSY), BlocksResult(FAILED)]) diff --git a/tests/test_names.py b/tests/test_names.py index 120bfa8..c0a869b 100644 --- a/tests/test_names.py +++ b/tests/test_names.py @@ -14,7 +14,7 @@ from rnfs.content.cid import DAG, RAW, Cid from rnfs.errors import NotFound, RnfsError from rnfs.names import NameManager, encode_record, parse_record -from tests.support import POSIX_MODES, FakeClock, failing_commits, state_execute, state_rows, wait_for +from tests.support import CAN_SYMLINK, POSIX_MODES, FakeClock, failing_commits, state_execute, state_rows, wait_for _saved_loglevel = RNS.loglevel @@ -278,21 +278,70 @@ def test_create_stores_private_key_file(self): with self.assertRaises(ValueError): self.manager.create("site") + @unittest.skipUnless(CAN_SYMLINK, "creating symlinks needs Developer Mode or an admin on Windows") + def test_create_refuses_an_alias_whose_key_is_a_dangling_symlink(self): + """An alias whose key file is a dangling symlink can't be read but isn't free: create and import refuse it, + and its saved record stays as it is.""" + self.manager.create("site") + self.manager.publish("site", CID_A) + self.manager.close() + row = self.rows("SELECT * FROM names_published") + path = os.path.join(self.config.names_dir, "site") + os.remove(path) + os.symlink(os.path.join(self.config.names_dir, "elsewhere"), path) + restarted = self.make_manager() + self.assertEqual(restarted._unreadable, {"site"}) + with self.assertRaisesRegex(ValueError, "alias site already exists"): + restarted.create("site") + with self.assertRaisesRegex(ValueError, "alias site already exists"): + restarted.import_key("site", RNS.Identity().get_private_key()) + self.assertTrue(os.path.islink(path)) + self.assertEqual(self.rows("SELECT * FROM names_published"), row) + + def test_create_survives_a_remove_right_after_it(self): + """A remove landing as soon as create lets go of the lock doesn't make create fail: it returns the name.""" + lock, depth, removed = self.manager._lock, [0], [] + manager = self.manager + + class RemovingLock: + """The manager's lock, removing "site" (as another thread might) once create releases it.""" + + def __enter__(self): + """Take the lock.""" + lock.acquire() + depth[0] += 1 + + def __exit__(self, *exc): + """Release it, then remove "site" the first time it is free.""" + depth[0] -= 1 + lock.release() + if depth[0] == 0 and not removed: + removed.append(True) + manager.remove("site") + + manager._lock = RemovingLock() + name = manager.create("site") + self.assertEqual(len(name), 16) + self.assertEqual((removed, manager.list()), ([True], [])) + def test_keys_survive_restart(self): """A new manager over the same directory loads the same names.""" name = self.manager.create("site") self.assertEqual(self.make_manager().name("site"), name) def test_old_temp_files_removed_at_start(self): - """Temp files 0.1.1's interrupted writes left under fixed names are deleted; nothing else is.""" + """Temp files interrupted key writes left are deleted (0.1.1's under fixed names, and write_atomic's, + which hold a private key); nothing else is.""" self.manager.create("site") - old = [os.path.join(self.config.state_dir, "names.json.tmp"), os.path.join(self.config.names_dir, ".gone.tmp")] - kept = [os.path.join(self.config.names_dir, name) for name in (".notes", "x.tmp")] + old = [os.path.join(self.config.state_dir, "names.json.tmp"), os.path.join(self.config.names_dir, ".gone.tmp"), + os.path.join(self.config.names_dir, fsutil.PARTIAL_PREFIX + "0123456789abcdef")] + kept = [os.path.join(self.config.names_dir, name) + for name in (".notes", "x.tmp", fsutil.PARTIAL_PREFIX + "short", fsutil.PARTIAL_PREFIX + "0" * 17)] for path in old + kept: with open(path, "wb") as f: f.write(b"partial") self.assertEqual(self.make_manager().list()[0]["alias"], "site") - self.assertEqual([os.path.exists(path) for path in old + kept], [False, False, True, True]) + self.assertEqual([os.path.exists(path) for path in old + kept], [False] * len(old) + [True] * len(kept)) # No temp file of its own: only the state database is there self.assertEqual([entry for entry in os.listdir(self.config.state_dir) if not entry.startswith(statedb.DB_FILE)], []) @@ -377,6 +426,53 @@ def test_deleted_key_file_releases_pin_once_asked(self): self.make_manager().release_gone() # its entry is gone from the state: nothing left to release self.assertEqual(self.pins.sources, {}) + def test_deleted_key_file_pin_released_once_at_reconcile(self): + """reconcile_pins releases a deleted key's recorded pin once, with one warning; a pin nothing records is + still released.""" + self.manager.create("site") + self.manager.publish("site", CID_A) + self.pins.pin(CID_B, "name:site") # left by a crash + os.remove(os.path.join(self.config.names_dir, "site")) + restarted = self.make_manager() + with mock.patch.object(names, "log") as log: + restarted.reconcile_pins({cid: set(sources) for cid, sources in self.pins.sources.items()}) + self.assertEqual(self.pins.sources, {}) + self.assertEqual([call.args[0] for call in log.call_args_list], + ["name key site is gone; releasing its pin", + "releasing the pins of name site, which has no key"]) + log.reset_mock() + self.pins.pin(CID_A, "name:site") + self.pins.unpin(CID_B, "name:site") + self.manager.close() + state_execute(self.config.state_dir, "INSERT INTO names_published VALUES ('site', '1', ?, 0)", + (CID_A.binary,)) + restarted = self.make_manager() + with mock.patch.object(names, "log") as log: + restarted.reconcile_pins({cid: set(sources) for cid, sources in self.pins.sources.items()}) + self.assertEqual(self.pins.sources, {}) + self.assertEqual([call.args[0] for call in log.call_args_list], ["name key site is gone; releasing its pin"]) + + def test_alias_replaced_while_refreshing_isnt_published(self): + """An imported key removed and its alias made again while its first publish refreshes: the publish fails, + rather than record the target under the old name and announce it under the new one.""" + self.manager.start() + old = self.manager.import_key("site", RNS.Identity().get_private_key()) + + def replace(name): + """Remove "site" and create it again, with another key.""" + self.network.on_request_path = None + self.manager.remove("site") + self.manager.create("site") + + self.network.on_request_path = replace + with self.assertRaisesRegex(NotFound, "replaced while publishing"): + self.manager.publish("site", CID_A) + new = self.manager.name("site") + self.assertNotEqual(new, old) + self.assertEqual((self.manager.record(old), self.manager.record(new)), (None, None)) + self.assertEqual(self.rows("SELECT * FROM names_held"), []) + self.assertEqual(self.pins.sources, {}) + def test_sequence_increments(self): """Each publish is one higher than the last, and that survives a restart.""" self.manager.create("site") @@ -627,6 +723,15 @@ def test_heard_time_in_the_future_isnt_fresh(self): self.manager.resolve(name, persist=True, max_age=60) self.assertEqual(self.network.path_requests, [name]) + def test_watched_name_first_heard_unasked_is_kept(self): + """A name heard unasked and then watched counts as asked: a flood of new names doesn't evict it.""" + watched = other_name(1) + self.network.announce(watched, encode_record(1, CID_A), self.clock.now) + self.manager.watch(watched) + for i in range(2, 10): + self.network.announce(other_name(i), encode_record(1, CID_B), self.clock.now) + self.assertEqual(self.manager.record(watched).sequence, 1) + def test_resolve_rejects_bad_names(self): """Names must be 16 bytes.""" with self.assertRaises(ValueError): @@ -695,6 +800,28 @@ def test_failed_resolve_save_is_retried(self): self.manager.stop() self.assertEqual(self.rows("SELECT name, sequence FROM names_held"), [(name, "4")]) + def test_failed_save_of_a_heard_record_is_retried(self): + """An announce raising a held name's sequence whose save fails still wakes resolves and tells keepers and + listeners; announce_due saves it.""" + name = other_name(1) + self.network.store(name, encode_record(4, CID_A), self.clock.now) + self.manager.resolve(name, persist=True) + self.manager.start() + keepers = self.manager.keepers = FakeKeepers() + heard = [] + self.manager.watch(name) + self.manager.add_record_listener(lambda n, record: heard.append(record.sequence)) + waiting = threading.Event() + self.manager._waiters[name] = [waiting] + with failing_commits(), mock.patch.object(keepers, "heard") as told: + self.network.announce(name, encode_record(5, CID_B), self.clock.now) + self.assertTrue(waiting.is_set()) + self.assertEqual((heard, told.call_args.args[1]), ([5], 5)) + self.assertEqual(self.manager.record(name).sequence, 5) + self.assertEqual(self.rows("SELECT sequence FROM names_held"), [("4",)]) + self.manager.announce_due() + self.assertEqual(self.rows("SELECT sequence FROM names_held"), [("5",)]) + def test_malformed_rows_stop_startup(self): """A published or held row that doesn't parse stops startup, as a malformed names.json did: starting without it would allow rollback.""" @@ -899,6 +1026,19 @@ def test_failed_save_is_tried_again_at_once(self): self.resolve_for_gateway(3) # but the save that worked does self.assertEqual(len(self.saved()), 2) + def test_name_resolved_for_the_user_leaves_the_floor(self): + """A floor name resolved for the user is held: its floor entry goes, and the next floor save deletes its row, + so it no longer counts toward the floor's bound.""" + self.resolve_for_gateway(1) + self.resolve_for_gateway(2) + self.manager.announce_due() + self.assertEqual(self.saved(), [other_name(1), other_name(2)]) + self.manager.resolve(other_name(1), persist=True, max_age=60) + self.assertNotIn(other_name(1), self.manager._floor) + self.assertNotIn(other_name(1), self.manager._floor_used) + self.manager.announce_due() + self.assertEqual(self.saved(), [other_name(2)]) + def test_invalid_rows_are_dropped(self): """A floor row that doesn't parse is logged and deleted; the node starts without it.""" good = (other_name(1), "5", 1.0, 0) diff --git a/tests/test_names_keep.py b/tests/test_names_keep.py index b7077c3..37e286e 100644 --- a/tests/test_names_keep.py +++ b/tests/test_names_keep.py @@ -525,6 +525,19 @@ def test_follow_persists(self): self.assertEqual(again.feature("follow").following()[0]["pinned"], str(cid1)) self.assertIn("follow", again.feature("keepers").store.reasons(self.name)) + def test_failed_construction_releases_the_state_database(self): + """A constructor that raises after opening its store closes it: the node never closes a feature it + didn't get.""" + node = self.a + database = statedb._open[statedb._key(node.config.state_dir)] + before = database.references + with mock.patch.object(node.names, "keepers", node.names.keepers), \ + mock.patch.object(node.publisher, "add_key_source"), \ + mock.patch.object(keepers.KeepersFeature, "sync_own", side_effect=ValueError("no key")), \ + self.assertRaises(ValueError): + keepers.KeepersFeature(node) + self.assertEqual(database.references, before) + class NameRequestServerTest(unittest.TestCase): """/name registered on the request server through Feature.requests: limits and private mode apply.""" diff --git a/tests/test_node.py b/tests/test_node.py index 90252f6..13e67fd 100644 --- a/tests/test_node.py +++ b/tests/test_node.py @@ -1,4 +1,5 @@ """Tests for RnfsNode's facade and backend (rnfs/node.py), on offline nodes with a fake network client.""" +import hashlib import io import os import random @@ -17,11 +18,11 @@ from rnfs.content.cid import RAW, Cid from rnfs.content.storage import remote_source from rnfs import fetcher -from rnfs.errors import InvalidDag, NotFound, RnfsError, StorageFull, TooLarge +from rnfs.errors import Denied, InvalidDag, NotFound, RnfsError, StorageFull, TooLarge from rnfs.features.keepers import name_key from rnfs.network.client import OK, BlocksResult from rnfs.network.identity import parse_node_id -from rnfs import fsutil, node as node_module +from rnfs import fsutil, node as node_module, statedb from rnfs.node import RnfsNode from tests.support import POSIX_MODES, failing_commits, state_rows, wait_for from tests.test_names import FakeNetwork @@ -290,6 +291,50 @@ def test_providers_and_pin_add(self): self.assertEqual(self.b.pin_add(f"{cid}/sub")["cid"], self.a.info(f"{cid}/sub")["cid"]) self.assertEqual(len(self.b.publisher.provided()), 1) + def test_read_file_is_info_and_read_in_one_fetch(self): + """read_file gives info()'s answer and read()'s bytes, cut to the file's size, from one fetch session.""" + cid = self.a.add_path(self.tree())["cid"] + self.advertise(cid) + target = f"{cid}/big.bin" + sessions = [] + fetching = self.b._fetching + with mock.patch.object(self.b, "_fetching", lambda *args, **kwargs: sessions.append(1) or fetching(*args, + **kwargs)): + info, chunks = self.b.read_file(target, offset=CHUNK_SIZE + 1, length=10) + self.assertEqual(b"".join(chunks), self.big[CHUNK_SIZE + 1:CHUNK_SIZE + 11]) + self.assertEqual(len(sessions), 1) + self.assertEqual(info, self.b.info(target)) + for offset, length in ((len(self.big) - 5, 100), (len(self.big) + 7, None), (0, 0)): + info, chunks = self.b.read_file(target, offset=offset, length=length) + self.assertEqual(b"".join(chunks), self.big[offset:] if length is None else + self.big[offset:offset + length]) + with self.assertRaises(ValueError): + self.b.read_file(target, offset=-1) + + def test_read_file_of_a_directory_reads_nothing(self): + """A directory's info comes back; reading it raises ValueError, and closing it unread ends the fetch.""" + cid = self.a.add_path(self.tree())["cid"] + self.advertise(cid) + info, chunks = self.b.read_file(f"{cid}/sub") + self.assertEqual(info["type"], "directory") + with self.assertRaisesRegex(ValueError, "is a directory"): + next(chunks) + info, chunks = self.b.read_file(cid) + chunks.close() + self.assertEqual(self.b.storage._sessions, []) # its protection session ended + with self.assertRaises(NotFound): + self.b.read_file(f"{cid}/missing") + + def test_read_file_checks_the_deny_list_before_the_first_byte(self): + """A file whose hash is denied raises Denied at the first chunk, after its info.""" + cid = self.a.add_stream(io.BytesIO(b"unwanted" * 100))["cid"] + self.advertise(cid) + self.b.deny.add(["sha256:" + hashlib.sha256(b"unwanted" * 100).hexdigest()]) + info, chunks = self.b.read_file(cid) + self.assertEqual(info["size"], 800) + with self.assertRaises(Denied): + next(chunks) + def test_invalid_root_reported(self): self.b.fetcher.mark_invalid(Cid.for_block(RAW, b"bad")) with self.assertRaises(InvalidDag): @@ -495,6 +540,18 @@ def test_remote_pin_quota_counts_dag_blocks(self): self.assertTrue(self.wait_for(lambda: not self.a._remote_reserved.get(remote_source(IDENTITY)))) self.assertEqual(self.a.storage.pins(), {}) + def test_remote_pin_quota_counts_dag_blocks_already_stored(self): + """A DAG over the quota is refused even when this node already holds all of it (a user pin here).""" + self.a.config.remote_pin_quota = 50_000 + cid = self.b.add_path(self.empty_files(400))["cid"] + self.b.records.store(Cid.parse(cid).key, B_ID) + self.a.routing.heard(B_ID) + self.a.pin_add(cid) + self.assertGreater(self.a.storage.dag_bytes(Cid.parse(cid)), 50_000) + self.a.backend.remote_pin(IDENTITY, Cid.parse(cid), True) + self.assertTrue(self.wait_for(lambda: not self.a._remote_reserved.get(remote_source(IDENTITY)))) + self.assertEqual(self.a.storage.pins(), {Cid.parse(cid): {"user"}}) + def test_remote_pins_of_one_source_are_fetched_one_at_a_time(self): # Two DAGs of empty files, each within the quota but not both: fetched at once, each was checked # without the other's blocks and both were pinned @@ -611,6 +668,20 @@ def restart(self, node: RnfsNode, name: str, node_id: bytes, **settings) -> Rnfs node.stop() return self.make_node(name, node_id, **settings) + def test_failed_saves_dont_stop_stopping(self): + """Saves failing at stop (a full disk) are logged; the state database is still closed, and the node's + directory can be used again.""" + node = self.make_node("full", N1) + failing = mock.Mock(side_effect=statedb.StateDbError("disk full")) + with mock.patch.object(node.records, "save", failing), mock.patch.object(node.publisher, "save", failing), \ + mock.patch.object(node.names, "_save", failing), mock.patch("rnfs.node.log") as log: + node.stop() + self.assertGreaterEqual(failing.call_count, 3) + self.assertEqual(sum("saving state failed" in call.args[0] for call in log.call_args_list), 3) + self.assertIsNone(node._db._database) + self.assertIsNone(node.storage._db._database) + self.make_node("full", N1).stop() + def test_deleted_name_key_releases_its_pin_at_startup(self): for strategy in ("roots", "pinned"): with self.subTest(strategy=strategy): diff --git a/tests/test_public.py b/tests/test_public.py new file mode 100644 index 0000000..269736a --- /dev/null +++ b/tests/test_public.py @@ -0,0 +1,805 @@ +"""Tests for the public API (rnfs.Node and rnfs.Client: rnfs/interface.py, objects.py, embedded.py, remote.py). + +The interface tests run twice: on an embedded node (an offline RnfsNode behind rnfs.Node's backend) and on a Client +of an ApiServer serving one, so both classes are shown to behave alike. Node A, offline too, provides content. +""" +import array +import contextlib +import importlib.util +import io +import os +import shutil +import subprocess +import sys +import tempfile +import threading +import time +import unittest +from unittest import mock + +import RNS + +import rnfs +from rnfs import api, cli, embedded, fsutil, interface, objects, remote +from rnfs.api import ApiHandler, ApiServer, NameWatches +from rnfs.config import Config +from rnfs.content.cid import RAW, Cid +from rnfs.names import encode_record +from rnfs.node import DirInUse, RnfsNode, lock_rnfs_dir +from tests.support import wait_for +from tests.test_names import FakeNetwork +from tests.test_node import NetClient + +ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +A_ID, B_ID = bytes([0xA] * 16), bytes([0xB] * 16) +OTHER = bytes([0x5E] * 16) # a name of someone else's + +_saved_loglevel = RNS.loglevel + + +def setUpModule(): + """Silence the nodes' log.""" + RNS.loglevel = RNS.LOG_CRITICAL + + +def tearDownModule(): + """Restore RNS's loglevel.""" + RNS.loglevel = _saved_loglevel + + +class OfflineNode(embedded.Node): + """rnfs.Node over an offline RnfsNode (no Reticulum, no lock): the embedded class's code, but its start.""" + + def __init__(self, node: RnfsNode): + """The public interface of node.""" + self._node = node + interface.Rnfs.__init__(self, embedded._Backend(node)) + node.names.add_record_listener(self._heard) + + def _close(self): + """Stop listening (the test stops the node).""" + self._node.names.remove_record_listener(self._heard) + + +class PublicTests: + """The interface's behaviour, whichever class provides it: subclasses make self.rnfs over node B.""" + + def setUp(self): + """Nodes A and B (B's jobs and names started), and the interface over B.""" + tmp = tempfile.TemporaryDirectory() + self.addCleanup(tmp.cleanup) + self.tmp = tmp.name + self.net = NetClient() + self.names_net = FakeNetwork() + self.a = self.make_node("a", A_ID, FakeNetwork()) + self.b = self.make_node("b", B_ID, self.names_net) + self.b.routing.heard(A_ID) + self.b.names.start() + self.b.feature("jobs").start() + self.rnfs = self.make(self.b) + self.addCleanup(self.rnfs.close) + + def make_node(self, name: str, node_id: bytes, names_network) -> RnfsNode: + """An offline node in its own directory.""" + config = Config(rnfs_dir=os.path.join(self.tmp, name), slow="no", lookup_timeout=2, path_timeout=0.2, + max_jobs=2) + node = RnfsNode.offline(config, node_id, self.net, names_network) + self.net.nodes[node_id] = node + self.addCleanup(node.stop) + return node + + def share(self, data: bytes, name: str | None = None) -> Cid: + """Add data on A (wrapped as name, if given) and make A its provider.""" + cid = Cid.parse(self.a.add_stream(io.BytesIO(data), name=name)["cid"]) + self.a.records.store(cid.key, A_ID) + return cid + + def out(self, name: str) -> str: + """A path in the temporary directory.""" + return os.path.join(self.tmp, name) + + # Content + + def test_add_ls_info_stat(self): + """add() of data returns Added; ls, info and stat describe what it stored.""" + data = os.urandom(150_000) + added = self.rnfs.add(data, name="f.bin") + self.assertIsInstance(added, rnfs.Added) + self.assertEqual((added.size, added.warnings, added.sealed, str(added)), (len(data), (), False, + str(added.cid))) + entries = self.rnfs.ls(added.cid) + self.assertEqual([(e.name, e.kind, e.size, e.is_dir) for e in entries], [("f.bin", "file", len(data), False)]) + info = self.rnfs.info(f"{added.cid}/f.bin") + self.assertEqual((info.cid, info.type, info.size, info.executable), (entries[0].cid, "file", len(data), False)) + stat = self.rnfs.stat(added.cid) + self.assertEqual((stat.cid, stat.size, stat.blocks), (added.cid, len(data), stat.local_blocks)) + self.assertEqual(self.rnfs.add(data, only_hash=True, name="f.bin").cid, added.cid) + + def test_add_path_and_stream(self): + """A directory, a binary file, and add()'s refusals.""" + path = self.out("dir") + os.makedirs(os.path.join(path, "sub")) + with open(os.path.join(path, "sub", "x.txt"), "wb") as f: + f.write(b"hello") + added = self.rnfs.add(path) + self.assertEqual([entry.name for entry in self.rnfs.ls(added.cid)], ["sub"]) + self.assertEqual(self.rnfs.add(io.BytesIO(b"hello")).cid, Cid.for_block(RAW, b"hello")) + with self.assertRaises(ValueError): + self.rnfs.add(path, name="x") + with self.assertRaises(ValueError): + self.rnfs.add(b"data", wrap=True) + with self.assertRaises(rnfs.NotFound): + self.rnfs.add(self.out("missing")) + + def test_import_key_counts_a_memoryviews_bytes(self): + """import_key of a memoryview whose items are wider than a byte imports all its bytes.""" + name = self.rnfs.names.create("site") + key = name.export_key() + name.remove() + self.assertEqual(self.rnfs.names.import_key("again", memoryview(key).cast("H")).name, name.name) + + def test_add_counts_a_memoryviews_bytes(self): + """add() of a memoryview whose items are wider than a byte stores all its bytes.""" + data = array.array("i", range(1000)) + added = self.rnfs.add(memoryview(data)) + self.assertEqual((added.cid, added.size), (Cid.for_block(RAW, data.tobytes()), len(data.tobytes()))) + + def test_open_reads_only_what_it_covers(self): + """A File seeks and reads, each refill fetching only the range it covers.""" + data = os.urandom(300_000) + cid = self.share(data, "f.bin") + reads = [] + original = self.rnfs._backend.read + + def spy(target, offset, length, raw, from_node): + """Record each read the file makes.""" + reads.append((offset, length)) + return original(target, offset, length, raw, from_node) + with mock.patch.object(self.rnfs._backend, "read", spy): + with self.rnfs.open(f"{cid}/f.bin") as f: + self.assertIsInstance(f, rnfs.File) + self.assertEqual((f.size, f.seekable(), f.readable()), (len(data), True, True)) + f.seek(200_000) + self.assertEqual(f.read(10), data[200_000:200_010]) + self.assertEqual(reads, [(200_000, interface.FILE_BUFFER)]) # one refill: the chunks it covers + self.assertEqual(f.read(10), data[200_010:200_020]) # from the buffer + self.assertEqual(len(reads), 1) + f.seek(-5, io.SEEK_END) + self.assertEqual((f.read(), f.read()), (data[-5:], b"")) + f.seek(0) + self.assertEqual(f.read(), data) + self.assertEqual(f.tell(), len(data)) + self.assertTrue(f.closed) + with self.assertRaises(ValueError): + self.rnfs.open(cid) # a directory + reads.clear() + with mock.patch.object(self.rnfs._backend, "read", spy), self.rnfs.open(f"{cid}/f.bin") as f: + self.assertEqual(f.read(), data) + self.assertEqual(reads, [(0, len(data))]) # the rest in one read, not one per 8 KiB + + def test_cat_get_and_uri_targets(self): + """cat and get, rnfs: URI targets (from= and a name), and block().""" + data = os.urandom(100_000) + cid = self.share(data, "f.bin") + self.assertEqual(b"".join(self.rnfs.cat(f"{cid}/f.bin", offset=10, length=20)), data[10:30]) + written = self.rnfs.get(f"rnfs:{cid}/f.bin?from={A_ID.hex()}", self.out("got")) + self.assertEqual((written.output, str(written)), (self.out("got"), self.out("got"))) + with open(self.out("got"), "rb") as f: + self.assertEqual(f.read(), data) + with self.assertRaises(ValueError): + self.rnfs.get(cid, self.out("got")) # exists + name = self.rnfs.names.create("site") + name.publish(cid) + self.assertEqual(self.rnfs.info(f"rnfs:name/{name.hex}/f.bin").size, len(data)) + self.assertEqual(self.rnfs.block(self.rnfs.add(b"hello").cid), b"hello") + + def test_cat_raises_once_iterated(self): + """cat() of a directory raises ValueError on the first next(), not at the call, whichever class.""" + chunks = self.rnfs.cat(self.rnfs.add(b"x", name="x.txt").cid) + with self.assertRaises(ValueError): + next(chunks) + + # Pins and archives + + def test_pins(self): + """pin, pins, unpin (NotFound the second time), export, import, gc and verify.""" + cid = self.share(os.urandom(70_000)) + self.assertEqual(self.rnfs.pin(cid), cid) + pins = self.rnfs.pins(cid) + self.assertEqual([(p.cid, p.sources, p.complete, p.type) for p in pins], [(cid, ("user",), True, "file")]) + self.rnfs.unpin(cid) + self.assertEqual(self.rnfs.pins(cid), []) + with self.assertRaises(rnfs.NotFound): + self.rnfs.unpin(cid) + exported = self.rnfs.export(cid, self.out("a.rnfs")) + self.assertEqual(exported.roots, (cid,)) + self.assertEqual(self.rnfs.import_archive(self.out("a.rnfs")).roots, (cid,)) + self.assertIsInstance(self.rnfs.gc(), rnfs.Collected) + self.assertEqual(self.rnfs.verify().bad, ()) + + def test_errors_are_one_hierarchy(self): + """The same exception classes whichever class raises them.""" + data = os.urandom(1000) + cid = self.share(data) + with self.assertRaises(ValueError): + self.rnfs.ls(cid) # a file + with self.assertRaises(rnfs.NotFound): + self.rnfs.names.get("nothing") + with self.assertRaises(rnfs.RnfsError): + self.rnfs.names.resolve(OTHER) # nobody announces it + with self.assertRaises(ValueError): + self.rnfs.ping(b"short") + + # Names + + def test_names(self): + """create, publish, refresh, list, resolve, export, remove and import a name.""" + cid = self.rnfs.add(b"site").cid + name = self.rnfs.names.create("site") + self.assertEqual((name.alias, name.cid, name.sequence, str(name)), ("site", None, 0, name.hex)) + record = name.publish(cid) + self.assertEqual((record.name, record.cid, record.sequence), (name.name, cid, 1)) + name = name.refresh() + self.assertEqual((name.cid, name.sequence), (cid, 1)) + self.assertEqual(self.rnfs.names.list(), [name]) + self.assertEqual(self.rnfs.names.resolve(name).cid, cid) + key = name.export_key() + name.remove() + with self.assertRaises(rnfs.NotFound): + self.rnfs.names.get("site") + self.assertEqual(self.rnfs.names.import_key("again", key).name, name.name) + self.assertEqual(rnfs.Name.from_dict(name.to_dict()), name) + + def test_watch_hears_newer_records_on_the_event_thread(self): + """A watch hears newer records only, on the event thread, with the names lock free, until cancelled.""" + first, second = self.rnfs.add(b"one").cid, self.rnfs.add(b"two").cid + self.names_net.announce(OTHER, encode_record(3, first), time.time()) # held before the watch + heard = [] + lock_free = [] + + def callback(record): + """Record the call, and whether the names lock is free here.""" + free = self.b.names._lock.acquire(timeout=2) + if free: + self.b.names._lock.release() + lock_free.append(free) + heard.append((threading.current_thread().name, record)) + watch = self.rnfs.names.watch(OTHER, callback) + self.names_net.announce(OTHER, encode_record(2, second), time.time()) # older: refused, not delivered + self.names_net.announce(OTHER, encode_record(4, second), time.time()) + wait_for(lambda: heard, what="the newer record") + time.sleep(0.2) + self.assertEqual([(record.name, record.cid, record.sequence) for _, record in heard], [(OTHER, second, 4)]) + self.assertEqual(heard[0][0], self.rnfs.events_thread) + self.assertEqual(lock_free, [True]) + watch.cancel() + self.names_net.announce(OTHER, encode_record(5, first), time.time()) + time.sleep(0.3) + self.assertEqual(len(heard), 1) + + def test_watch_hears_own_publishes(self): + """A watch of this node's own name hears its publish.""" + cid = self.rnfs.add(b"site").cid + name = self.rnfs.names.create("site") + heard = [] + with self.rnfs.names.watch(name, heard.append): + name.publish(cid) + wait_for(lambda: heard, what="the publish") + self.assertEqual((heard[0].cid, heard[0].sequence), (cid, 1)) + + def test_close_from_a_callback(self): + """close() from a callback returns (it doesn't wait for the event thread it runs on) and closes.""" + cid = self.rnfs.add(b"one").cid + returned = threading.Event() + + def callback(record): + """Close the instance from its own event thread.""" + self.rnfs.close() + returned.set() + self.rnfs.names.watch(OTHER, callback) + self.names_net.announce(OTHER, encode_record(1, cid), time.time()) + self.assertTrue(returned.wait(10), "close() from the callback didn't return") + self.assertTrue(self.rnfs.closed) + with self.assertRaises(rnfs.Closed): + self.rnfs.status() + + def test_a_slow_callback_gets_the_latest_record(self): + """Records heard while a callback runs are coalesced: the next call gets the newest, not each one.""" + cid = self.rnfs.add(b"one").cid + gate, heard = threading.Event(), [] + + def callback(record): + """Hold the event thread at the first record.""" + heard.append(record.sequence) + gate.wait(10) + self.rnfs.names.watch(OTHER, callback) + self.names_net.announce(OTHER, encode_record(2, cid), time.time()) + wait_for(lambda: heard, what="the first record") + for sequence in (3, 4): + self.names_net.announce(OTHER, encode_record(sequence, cid), time.time()) + wait_for(lambda: self.heard_up_to(OTHER, 4), what="record 4 reaching the instance") + gate.set() + wait_for(lambda: heard[-1] == 4, what="the newest record") + time.sleep(0.3) + self.assertEqual(heard, [2, 4]) + + # Jobs and the deny list + + def test_job_with_progress(self): + """A get job's progress callbacks, on the event thread, up to its end.""" + data = os.urandom(200_000) + cid = self.share(data) + progress = [] + job = self.rnfs.jobs.get(cid, self.out("job"), on_progress=lambda job: progress.append( + (threading.current_thread().name, job))) + self.assertIsInstance(job, rnfs.Job) + finished = job.wait(10) + self.assertEqual((finished.state, finished.done, finished.error), ("done", True, None)) + wait_for(lambda: progress and progress[-1][1].done, what="the final progress call") + self.assertEqual({thread for thread, _ in progress}, {self.rnfs.events_thread}) + self.assertEqual(progress[-1][1].done_bytes, len(data)) + with open(self.out("job"), "rb") as f: + self.assertEqual(f.read(), data) + self.assertEqual([j.id for j in self.rnfs.jobs.list()], [job.id]) + self.assertEqual(self.rnfs.jobs.job(job.id).state, "done") + self.assertEqual(self.rnfs.jobs.clear(), 1) + with self.assertRaises(rnfs.NotFound): + job.refresh() + + def test_job_wait_times_out_and_cancels(self): + """Job.wait times out, cancel ends a queued job, and a failed job says why.""" + jobs = self.b.feature("jobs") + with mock.patch.object(jobs, "_schedule", return_value=False): # it stays queued + job = self.rnfs.jobs.pin(self.share(b"pinned")) + started = time.monotonic() + with self.assertRaises(TimeoutError): + job.wait(0.5) + self.assertLess(time.monotonic() - started, 5) + self.assertEqual(job.cancel().state, "cancelled") + self.assertEqual(job.wait(10).state, "cancelled") + failed = self.rnfs.jobs.pin(Cid.for_block(RAW, b"nobody has this")).wait(10) + self.assertEqual(failed.state, "failed") + self.assertIn("no provider", failed.error) + + def test_deny(self): + """The deny list: add, list, remove, and malformed entries.""" + cid = str(Cid.for_block(RAW, b"denied")) + self.assertEqual(self.rnfs.deny.add([cid]), [cid]) + self.assertEqual(self.rnfs.deny.list().entries, (cid,)) + self.assertEqual(self.rnfs.deny.remove([cid]), [cid]) + with self.assertRaises(rnfs.NotFound): + self.rnfs.deny.remove([cid]) + with self.assertRaises(ValueError): + self.rnfs.deny.add(["not an entry"]) + + # The example app (examples/rnfs_share.py), with whichever class + + def test_example_app(self): + """examples/rnfs_share.py's share() works through this class.""" + spec = importlib.util.spec_from_file_location("rnfs_share", os.path.join(ROOT, "examples", "rnfs_share.py")) + example = importlib.util.module_from_spec(spec) + spec.loader.exec_module(example) + folder = self.out("photos") + os.makedirs(folder) + data = os.urandom(20_000) + with open(os.path.join(folder, "cat.jpg"), "wb") as f: + f.write(data) + printed = io.StringIO() + with contextlib.redirect_stdout(printed): + output = example.share(self.rnfs, folder) + self.addCleanup(shutil.rmtree, os.path.dirname(output)) + with open(os.path.join(output, "cat.jpg"), "rb") as f: + self.assertEqual(f.read(), data) + self.assertIn("read 4096 bytes from the middle", printed.getvalue()) + self.assertIn("heard rnfs:name/", printed.getvalue()) + + # Lifecycle + + def test_status_and_close(self): + """status(), node_id, wait_joined, and Closed after close().""" + self.assertEqual(self.rnfs.node_id, B_ID) + status = self.rnfs.status() + self.assertEqual((status.node_id, status.version), (B_ID, rnfs.__version__)) + self.assertIn("jobs_running", status.features) + self.assertEqual(status.to_dict()["jobs_running"], status.features["jobs_running"]) + self.assertTrue(self.rnfs.wait_joined(1)) + with self.rnfs: + pass + self.assertTrue(self.rnfs.closed) + with self.assertRaises(rnfs.Closed): + self.rnfs.status() + self.rnfs.close() # idempotent + + +class EmbeddedNodeTest(PublicTests, unittest.TestCase): + """The interface on rnfs.Node.""" + + def make(self, node: RnfsNode): + """rnfs.Node over node.""" + return OfflineNode(node) + + def heard_up_to(self, name: bytes, sequence: int) -> bool: + """True once the node has handed name's record sequence on to the watches (at once, in a Node).""" + return True + + def test_internal_is_the_node(self): + """Node.internal is the RnfsNode.""" + self.assertIs(self.rnfs.internal, self.b) + + +class ClientTest(PublicTests, unittest.TestCase): + """The interface on rnfs.Client, through a live ApiServer serving node B.""" + + def make(self, node: RnfsNode): + """A Client of an ApiServer serving node.""" + self.server = ApiServer(node, 0) + self.server.start() + self.addCleanup(self.server.stop) + cli.write_token(os.path.join(node.config.root, "api_token"), self.server.token) + return rnfs.Client(node.config.rnfs_dir, port=self.server.port) + + def heard_up_to(self, name: bytes, sequence: int) -> bool: + """True once the client's poller has fetched name's record sequence.""" + return self.rnfs._polled.get(name, -1) >= sequence + + def test_errors_are_the_api_clients(self): + """A Client's errors are ApiErrors and the node's classes at once.""" + with self.assertRaises(rnfs.NotFound) as caught: + self.rnfs.names.get("nothing") # raised by the interface itself + with self.assertRaises(rnfs.ApiError) as caught: + self.rnfs.unpin(Cid.for_block(RAW, b"x")) + self.assertIsInstance(caught.exception, rnfs.NotFound) + self.assertEqual(caught.exception.status, 404) + with self.assertRaises(rnfs.BadRequest): + self.rnfs.ls(self.share(b"file")) + + def test_cat_from_a_node(self): + """cat's from_node reaches the daemon (GET /api/cat's from).""" + data = os.urandom(5000) + cid = self.share(data) + self.assertEqual(b"".join(self.rnfs.cat(cid, from_node=A_ID)), data) + + def test_cat_closed_before_reading_sends_nothing(self): + """A cat() iterator closed before its first next() never sends GET /api/cat, so no response is left open; + one closed after it closes the response.""" + cid = self.share(os.urandom(300_000)) + streams = [] + cat = self.rnfs.api.cat + + def spy(*args, **kwargs): + """ApiConnection.cat, keeping each stream.""" + streams.append(cat(*args, **kwargs)) + return streams[-1] + with mock.patch.object(self.rnfs.api, "cat", spy): + self.rnfs.cat(cid).close() + self.assertEqual(streams, []) + chunks = self.rnfs.cat(cid) + next(chunks) + self.assertFalse(streams[0].response.isclosed()) + chunks.close() + self.assertTrue(streams[0].response.isclosed()) + + def test_watch_needs_a_daemon_that_watches(self): + """A daemon without GET /api/name/watch: watch() raises NotFound.""" + without = {route: name for route, name in ApiHandler.ROUTES.items() if route[1] != "/api/name/watch"} + with mock.patch.object(ApiHandler, "ROUTES", without): + with self.assertRaises(rnfs.NotFound): + self.rnfs.names.watch(OTHER, lambda record: None) + + def test_long_polls_fit_the_timeout(self): + """With a timeout, a long poll waits less than it, so Job.wait times out as asked, not the socket.""" + client = rnfs.Client(self.b.config.rnfs_dir, port=self.server.port, timeout=3) + self.addCleanup(client.close) + self.assertEqual((client._long_wait, rnfs.Client(port=1, timeout=1)._long_wait), (1, 0)) + with mock.patch.object(self.b.feature("jobs"), "_schedule", return_value=False): # it stays queued + job = client.jobs.pin(self.share(b"pinned")) + with self.assertRaises(TimeoutError): + job.wait(4) + job.cancel() + + def test_api_is_the_connection(self): + """Client.api reaches any endpoint.""" + self.assertEqual(self.rnfs.api.get("/api/name/list"), {"names": []}) + + +class NodeConstructionTest(unittest.TestCase): + """rnfs.Node's checks before it starts anything, and its settings.""" + + def setUp(self): + """A temporary rnfs dir.""" + tmp = tempfile.TemporaryDirectory() + self.addCleanup(tmp.cleanup) + self.dir = os.path.join(tmp.name, "rnfs") + + def test_unknown_settings_are_refused(self): + """Unknown settings, and rnfs_dir among them, are ValueError.""" + for settings in ({"no_such_setting": 1}, {"rnfs_dir": "/elsewhere"}): + with self.subTest(settings=settings), self.assertRaises(ValueError): + rnfs.Node(self.dir, settings=settings) + + def test_another_reticulum_is_refused(self): + """A reticulum that isn't the running instance is ValueError.""" + with self.assertRaises(ValueError): + rnfs.Node(self.dir, reticulum=object()) + + def test_dir_in_use_is_refused(self): + """A Node on a dir another holds is refused before it starts anything.""" + fd = lock_rnfs_dir(Config(rnfs_dir=self.dir)) + try: + with self.assertRaises(rnfs.RnfsError) as caught: + rnfs.Node(self.dir) + finally: + fsutil.unlock(fd) + self.assertIn("another rnfs daemon or node", str(caught.exception)) + self.assertIsNone(embedded._running) + + def test_one_node_per_process(self): + """A second Node in the process is refused.""" + with mock.patch.object(embedded, "_running", object()): + with self.assertRaises(rnfs.RnfsError): + rnfs.Node(self.dir) + + def test_embedded_defaults_sit_under_the_settings_file(self): + """EMBEDDED_DEFAULTS apply only where rnfs.toml and settings say nothing.""" + defaults = embedded.EMBEDDED_DEFAULTS + os.makedirs(self.dir) + self.assertEqual(Config.load(self.dir, defaults=defaults).dht_mode, "client") + self.assertIs(Config.load(self.dir, defaults=defaults).lxmf_enabled, False) + with open(os.path.join(self.dir, "rnfs.toml"), "w", encoding="utf-8") as f: + f.write('dht_mode = "server"\nlxmf_enabled = true\n') + config = Config.load(self.dir, defaults=defaults) + self.assertEqual((config.dht_mode, config.lxmf_enabled), ("server", True)) + self.assertEqual(Config.load(self.dir, defaults=defaults, dht_mode="client").dht_mode, "client") + self.assertEqual(Config.load(self.dir).dht_mode, "server") + + +class NodeLifecycleTest(unittest.TestCase): + """rnfs.Node's own start and close, over a stand-in RnfsNode (the real one needs Reticulum: RNFS_E2E).""" + + def setUp(self): + """A temporary rnfs dir, RnfsNode replaced by start_node, fsutil.unlock spied on, and no Node left + running in the process whatever the test does.""" + tmp = tempfile.TemporaryDirectory() + self.addCleanup(tmp.cleanup) + self.config = Config(rnfs_dir=os.path.join(tmp.name, "rnfs")) + self.fail_start: BaseException | None = None + self.started = [] + for patcher in (mock.patch.object(embedded, "RnfsNode", side_effect=self.start_node), + mock.patch.object(fsutil, "unlock", wraps=fsutil.unlock), + mock.patch.object(embedded, "_running", None)): + self.addCleanup(patcher.stop) + patcher.start() + + def start_node(self, config: Config, **options) -> mock.Mock: + """A stand-in RnfsNode on config (raising self.fail_start instead, if set).""" + if self.fail_start is not None: + raise self.fail_start + self.started.append(mock.Mock(config=config, options=options)) + return self.started[-1] + + def assert_released(self): + """No Node runs in the process, and the rnfs dir's lock is free.""" + self.assertIsNone(embedded._running) + fsutil.unlock(lock_rnfs_dir(self.config)) + + def test_start_and_close(self): + """A Node locks its dir, starts the node embedded with its defaults and listens to its records; close() + stops the node, unlocks the dir and lets another Node run. A second close() does nothing.""" + node = rnfs.Node(self.config.rnfs_dir) + self.addCleanup(node.close) + inner, = self.started + self.assertEqual((inner.options, inner.config.dht_mode, inner.config.lxmf_enabled), + ({"reticulum": None, "embedded": True}, "client", False)) + self.assertIs(embedded._running, node) + inner.names.add_record_listener.assert_called_once_with(node._heard) + with self.assertRaises(DirInUse): + lock_rnfs_dir(self.config) + with self.assertRaises(rnfs.RnfsError) as caught: + rnfs.Node(self.config.rnfs_dir) + self.assertIn("already running", str(caught.exception)) + node.close() + inner.names.remove_record_listener.assert_called_once_with(node._heard) + inner.stop.assert_called_once_with() + fsutil.unlock.assert_called_once_with(node._lock_fd) + self.assert_released() + node.close() + inner.stop.assert_called_once_with() + rnfs.Node(self.config.rnfs_dir).close() + + def test_close_releases_when_stop_fails(self): + """A node whose stop() raises still has its dir unlocked and its place in the process freed.""" + node = rnfs.Node(self.config.rnfs_dir) + self.started[0].stop.side_effect = RuntimeError("failing on purpose") + with self.assertRaises(RuntimeError): + node.close() + fsutil.unlock.assert_called_once_with(node._lock_fd) + self.assert_released() + + def test_a_failed_start_leaves_nothing(self): + """A node failing to start leaves the dir unlocked and no Node running, so the next start works.""" + self.fail_start = RuntimeError("failing on purpose") + with self.assertRaises(RuntimeError): + rnfs.Node(self.config.rnfs_dir) + fsutil.unlock.assert_called_once() + self.assert_released() + self.fail_start = None + rnfs.Node(self.config.rnfs_dir).close() + self.assertEqual(len(self.started), 1) + + +class WatchedNamesTest(unittest.TestCase): + """NameManager's watches: a watched name's announce takes a table place an unasked one wouldn't.""" + + def test_watched_names_count_as_asked(self): + """A watched name's announce gets a table place; listeners hear only watched names.""" + tmp = tempfile.TemporaryDirectory() + self.addCleanup(tmp.cleanup) + node = RnfsNode.offline(Config(rnfs_dir=tmp.name, name_table_size=1), B_ID, NetClient(), FakeNetwork()) + self.addCleanup(node.stop) + names, cid = node.names, Cid.for_block(RAW, b"x") + names.offer(bytes([1] * 16), 1, cid, time.time()) # the table's one place: a name asked for + names._heard(OTHER, encode_record(1, cid)) + self.assertIsNone(names.record(OTHER)) # unasked: refused + heard = [] + names.add_record_listener(lambda name, record: heard.append((name, record.sequence))) + names.watch(OTHER) + names._heard(OTHER, encode_record(1, cid)) + self.assertEqual(names.record(OTHER).sequence, 1) + self.assertEqual(heard, [(OTHER, 1)]) + names.unwatch(OTHER) + names._heard(OTHER, encode_record(2, cid)) + self.assertEqual(heard, [(OTHER, 1)]) # no longer watched: not told + + def test_bound_method_listener_is_removed(self): + """remove_record_listener removes a bound method, a new object at each access.""" + tmp = tempfile.TemporaryDirectory() + self.addCleanup(tmp.cleanup) + node = RnfsNode.offline(Config(rnfs_dir=tmp.name), B_ID, NetClient(), FakeNetwork()) + self.addCleanup(node.stop) + node.names.add_record_listener(self.test_bound_method_listener_is_removed) + node.names.remove_record_listener(self.test_bound_method_listener_is_removed) + self.assertEqual(node.names._record_listeners, ()) + + +class NameWatchEndpointTest(unittest.TestCase): + """GET /api/name/watch's long poll (NameWatches) and its parameters.""" + + def setUp(self): + """An offline node behind an ApiServer, and a connection to it.""" + tmp = tempfile.TemporaryDirectory() + self.addCleanup(tmp.cleanup) + self.node = RnfsNode.offline(Config(rnfs_dir=tmp.name), B_ID, NetClient(), FakeNetwork()) + self.addCleanup(self.node.stop) + self.server = ApiServer(self.node, 0) + self.server.start() + self.addCleanup(self.server.stop) + cli.write_token(os.path.join(tmp.name, "api_token"), self.server.token) + self.api = rnfs.Client(tmp.name, port=self.server.port).api + self.cid = Cid.for_block(RAW, b"x") + + def test_answers_what_is_held_above_after(self): + """The records held above after, at once.""" + self.assertEqual(self.api.name_watch([OTHER.hex()], [None], 0), []) + self.node.names.watch(OTHER) # as a call would; then an announce is kept + self.node.names._heard(OTHER, encode_record(3, self.cid)) + records = self.api.name_watch([OTHER.hex()], [None], 0) + self.assertEqual([(r["name"], r["cid"], r["sequence"]) for r in records], [(OTHER.hex(), str(self.cid), 3)]) + self.assertEqual(self.api.name_watch([OTHER.hex()], [3], 0), []) + + def test_waits_for_a_newer_record(self): + """A call waits until a newer record is heard.""" + self.api.name_watch([OTHER.hex()], [None], 0) # watched from here on + threading.Timer(0.3, lambda: self.node.names._heard(OTHER, encode_record(1, self.cid))).start() + started = time.monotonic() + records = self.api.name_watch([OTHER.hex()], [None], 5) + self.assertLess(time.monotonic() - started, 4) + self.assertEqual([r["sequence"] for r in records], [1]) + + def test_parameters_are_checked(self): + """Malformed names, too many names and after mismatches are 400.""" + for params in ({"names": ""}, {"names": "xyz"}, {"names": ",".join([OTHER.hex()] * 65)}, + {"names": OTHER.hex(), "after": "1,2"}, {"names": OTHER.hex(), "after": "-1"}): + with self.subTest(params=params), self.assertRaises(rnfs.BadRequest): + self.api.get("/api/name/watch", **params) + + def test_waiters_are_capped(self): + """Past MAX_WATCH_WAITERS calls waiting, a call that would wait gets Busy.""" + watches = NameWatches(self.node) + watches.waiters = tuple((frozenset(), threading.Event()) for _ in range(api.MAX_WATCH_WAITERS)) + with self.assertRaises(rnfs.Busy): + watches.wait([OTHER], [None], 1) + + def test_calls_that_dont_wait_are_not_capped(self): + """With MAX_WATCH_WAITERS calls waiting, a wait=0 call (a Client's watch() baseline) still answers and + marks its name watched: it holds no connection.""" + watches = NameWatches(self.node) + waiters = tuple((frozenset(), threading.Event()) for _ in range(api.MAX_WATCH_WAITERS)) + watches.waiters = waiters + self.assertEqual(watches.wait([OTHER], [None], 0), []) + self.assertIn(OTHER, watches.until) + self.assertEqual(self.node.names._watched[OTHER], 1) + self.assertEqual(watches.waiters, waiters) + + def test_a_watch_ending_during_another_call_is_unwatched(self): + """A name whose watch ends while another call waits is unwatched in NameManager when that call ends.""" + now = [0.0] + watches = NameWatches(self.node, clock=lambda: now[0]) + watches.wait([OTHER], [None], 0) # watched until WATCH_GRACE + now[0] = api.WATCH_GRACE - 10 + records = watches._records + + def later(names, after): + """The call's wait, which outlasts OTHER's watch.""" + now[0] = api.WATCH_GRACE + 30 + return records(names, after) + with mock.patch.object(watches, "_records", later): + watches.wait([bytes([1] * 16)], [None], 0) + self.assertEqual(set(watches.until), {bytes([1] * 16)}) + self.assertNotIn(OTHER, self.node.names._watched) + + def test_a_name_listed_twice_is_watched_once(self): + """names=X,X holds one watch of X, released once.""" + self.api.name_watch([OTHER.hex(), OTHER.hex()], [None, 2], 0) + self.assertEqual(self.node.names._watched[OTHER], 1) + + def test_watches_end_after_their_grace(self): + """A name stays watched until WATCH_GRACE after its last call.""" + now = [0.0] + watches = NameWatches(self.node, clock=lambda: now[0]) + watches.wait([OTHER], [None], 0) + self.assertIn(OTHER, self.node.names._watched) + now[0] = api.WATCH_GRACE + 1 + watches.wait([bytes([1] * 16)], [None], 0) + self.assertNotIn(OTHER, self.node.names._watched) + + +class ObjectsTest(unittest.TestCase): + """The typed objects: from_dict(to_dict()) gives the object back, and to_dict is the API's form.""" + + def test_round_trips(self): + """from_dict(to_dict()) gives each object back; to_dict is the API's form.""" + cid = Cid.for_block(RAW, b"x") + for value in (objects.Added(cid, 1, ("w",), True), objects.Written(cid, "/o", (), None, True, "n"), + objects.Entry("n", "file", 1, cid), objects.Info(cid, "file", 1, False, True, True, 2, "n"), + objects.Stat(cid, 1, 2, 3, 4), objects.Pin(cid, ("user",), True, "file", False, None), + objects.Record(OTHER, cid, 2, 1.5), objects.Collected(1, 2), objects.Verified(3, (cid,)), + objects.Exported((cid,), 1), objects.Imported((cid,), (), 1), objects.DenyState(("a",), ()), + objects.Status("0.1.4", B_ID, None, "client", features={"jobs_running": 0})): + with self.subTest(value=type(value).__name__): + self.assertEqual(type(value).from_dict(value.to_dict()), value) + self.assertEqual(objects.Entry("n", "file", 1, cid).to_dict(), {"name": "n", "kind": "file", "size": 1, + "cid": str(cid)}) + self.assertEqual(objects.Record(OTHER, cid, 2, None).to_dict()["name"], OTHER.hex()) + + +class ImportTest(unittest.TestCase): + """`import rnfs` stays light: rnfs.Client and the objects load no RNS; rnfs.Node does.""" + + def run_python(self, code: str) -> str: + """code's output in a fresh interpreter.""" + return subprocess.run([sys.executable, "-c", code], capture_output=True, encoding="utf-8", cwd=ROOT, + env={**os.environ, "PYTHONPATH": ROOT}, timeout=60, check=True).stdout.strip() + + def test_client_loads_no_rns(self): + """rnfs.Client, the objects and errors load no RNS.""" + code = ("import sys, rnfs; rnfs.Client(port=1); rnfs.Cid, rnfs.Record, rnfs.NotFound, rnfs.ApiError, " + "rnfs.Job, rnfs.File; print(any(m == 'RNS' or m.startswith('RNS.') for m in sys.modules))") + self.assertEqual(self.run_python(code), "False") + + def test_node_loads_rns_when_used(self): + """RNS is loaded only once rnfs.Node is used.""" + code = "import sys, rnfs; print('RNS' in sys.modules); rnfs.Node; print('RNS' in sys.modules)" + self.assertEqual(self.run_python(code).split(), ["False", "True"]) + + def test_names_and_status(self): + """API_STATUS, the exported names, and an unknown one.""" + self.assertEqual(rnfs.API_STATUS, "provisional") + self.assertIs(rnfs.NotFound, __import__("rnfs.errors").errors.NotFound) + self.assertIs(rnfs.Client, remote.Client) + self.assertIn("Node", dir(rnfs)) + with self.assertRaises(AttributeError): + rnfs.NoSuchThing + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_publisher.py b/tests/test_publisher.py index b44763e..ec56ccf 100644 --- a/tests/test_publisher.py +++ b/tests/test_publisher.py @@ -2,17 +2,21 @@ import os import tempfile import threading +import time import unittest +from unittest import mock import RNS +from rnfs import publisher as publisher_module from rnfs.config import Config from rnfs.constants import PROVIDER_TTL from rnfs.content import builder from rnfs.content.cid import RAW, Cid from rnfs.content.storage import USER, Storage +from rnfs.denylist import DenyList from rnfs.dht.providers import ProviderStore -from rnfs.publisher import SLOW_LOOKUP_PACE, SLOW_ROUND_LOOKUPS, DhtKey, Publisher +from rnfs.publisher import PUBLISH_LOOKUPS, SLOW_LOOKUP_PACE, SLOW_ROUND_LOOKUPS, DhtKey, Publisher from tests.support import FakeClock, failing_commits, state_execute, state_rows, write_legacy from tests.test_providers import recorded_writes, rows_written @@ -61,6 +65,44 @@ def find_closest(self): return list(self.nodes) +class Running: + """Counts the lookups running at once (the most ever in peak), the threads they ran on and those finished.""" + + def __init__(self): + """None run yet.""" + self.lock = threading.Lock() + self.now = self.peak = self.finished = 0 + self.threads: set[str] = set() + + def __enter__(self): + """A lookup starts.""" + with self.lock: + self.now += 1 + self.peak = max(self.peak, self.now) + self.threads.add(threading.current_thread().name) + + def __exit__(self, *exc): + """It ends.""" + with self.lock: + self.now -= 1 + self.finished += 1 + + +class SleepingLookup(FakeLookup): + """find_closest takes delay seconds, counted in running.""" + + def __init__(self, nodes, delay: float, running: Running): + """A lookup answering nodes after delay seconds.""" + super().__init__(nodes) + self.delay, self.running = delay, running + + def find_closest(self): + """The fixed nodes, once delay has passed.""" + with self.running: + time.sleep(self.delay) + return list(self.nodes) + + class PublisherTest(unittest.TestCase): """A Publisher over temporary storage with fakes for the network.""" @@ -79,18 +121,27 @@ def setUp(self): self.eligible: set[Cid] = set() self.lookups: list[bytes] = [] self.on_lookup = lambda key: None + self.nodes_of = lambda key: self.nodes # the nodes a key's lookup finds + self.lookup_delay = None # or key -> seconds its lookup takes (counted in self.running) + self.running = Running() + self.fetched_changes = 0 # what the node's fetched-targets change count would be (generation) self.publisher = self.make_publisher() - def make_publisher(self, slow: bool = False, state: str | None = None) -> Publisher: - """A publisher sharing this test's state directory (or on state), closed when the test ends.""" + def make_publisher(self, slow: bool = False, state: str | None = None, generation: bool = False) -> Publisher: + """A publisher sharing this test's state directory (or on state), closed when the test ends; with + generation, one as the node makes it: storage.changes() and fetched_changes.""" def make_lookup(key): - """Remember the key; find self.nodes.""" + """Remember the key; find self.nodes_of(key), after self.lookup_delay(key) seconds if set.""" self.lookups.append(key) self.on_lookup(key) - return FakeLookup(self.nodes) + if self.lookup_delay is None: + return FakeLookup(self.nodes_of(key)) + return SleepingLookup(self.nodes_of(key), self.lookup_delay(key), self.running) publisher = Publisher(OWN, self.config, self.storage, self.records, self.client, make_lookup, slow, state or self.state, cached_eligible=lambda: self.eligible, clock=self.clock, - retry_interval=30) + retry_interval=30, + generation=(lambda: (self.storage.changes(), self.fetched_changes)) if generation + else None) self.addCleanup(publisher.close) return publisher @@ -543,6 +594,270 @@ def test_save_writes_only_what_changed(self): self.publisher.save() self.assertEqual(transactions, []) # nothing changed since + # Republish lookups run at once + + def due_keys(self, count: int) -> list[Cid]: + """count complete pins, refreshed into the provided set: all due.""" + cids = [self.pinned(b"due %d" % i) for i in range(count)] + self.publisher.refresh() + return cids + + def test_lookups_run_at_once(self): + """A round's lookups run PUBLISH_LOOKUPS at once, on threads of their own, so a round of many keys whose + lookups take time finishes in a fraction of the time one at a time takes.""" + self.due_keys(20) + self.lookup_delay = lambda key: 0.05 # 1 s one at a time + started = time.monotonic() + result = self.publisher.publish_due() + elapsed = time.monotonic() - started + self.assertEqual(len(result["published"]), 20) + self.assertEqual(self.running.peak, PUBLISH_LOOKUPS) + self.assertEqual(self.running.threads, {"rnfs-publish-lookup"}) + self.assertLess(elapsed, 0.6) # about 20 / PUBLISH_LOOKUPS × 0.05 = 0.25 s + self.assertEqual(len(self.client.calls), 2) # still one /provide per node, every key in it + self.assertTrue(all(len(keys) == 20 for _, keys, _ in self.client.calls)) + + def test_lookups_at_once_send_what_one_at_a_time_sends(self): + """Lookups finishing in any order, each finding its own nodes: every node gets the same /provide + requests, keys and withdrawals in the same order, as when the lookups run one at a time.""" + cids = self.due_keys(20) + self.nodes_of = lambda key: [node for i, node in enumerate((N1, N2, N3)) if key[i] % 2] or [N1] + self.lookup_delay = lambda key: 0.005 + key[3] % 5 / 200 # 5-25 ms, so they finish out of order + gone = bytes([7]) * 16 + rounds = [] + for lookups in (PUBLISH_LOOKUPS, 1): + with self.subTest(lookups=lookups), mock.patch.object(publisher_module, "PUBLISH_LOOKUPS", lookups): + self.client = FakeClient() + self.running = Running() + publisher = self.make_publisher(state=os.path.join(self.tmp.name, f"state {lookups}")) + publisher._withdrawals = {N1: {gone}, N3: {gone}} # as if queued earlier + publisher.refresh() + result = publisher.publish_due() + self.assertEqual(set(result["published"]), set(cids)) + if lookups == 1: + self.assertEqual(self.running.peak, 1) + else: + self.assertGreater(self.running.peak, 1) + rounds.append((result, self.client.calls)) + self.assertEqual(rounds[0], rounds[1]) + + def test_slow_nodes_look_up_one_at_a_time(self): + """Slow nodes keep their pacing: one lookup at a time, on the publishing thread, at most + SLOW_ROUND_LOOKUPS a round, and SLOW_LOOKUP_PACE per lookup before the next round.""" + publisher = self.make_publisher(slow=True) + self.due_keys(20) + publisher.refresh() + self.lookup_delay = lambda key: 0.01 + self.assertEqual(len(publisher.publish_due()["published"]), SLOW_ROUND_LOOKUPS) + self.assertEqual(self.running.peak, 1) + self.assertEqual(self.running.threads, {threading.current_thread().name}) + self.assertEqual(publisher.next_due(), SLOW_LOOKUP_PACE * SLOW_ROUND_LOOKUPS) + self.assertEqual(publisher.publish_due()["published"], []) + self.clock.now += SLOW_LOOKUP_PACE * SLOW_ROUND_LOOKUPS + self.assertEqual(len(publisher.publish_due()["published"]), SLOW_ROUND_LOOKUPS) + + def test_stopped_round_keeps_only_finished_lookups(self): + """should_stop is checked before each lookup starts: those started finish and are published, in due + order, and the rest stay due.""" + self.due_keys(20) + self.lookup_delay = lambda key: 0.02 + # Only the lookup threads are told to stop, so the round still sends what was looked up + result = self.publisher.publish_due( + should_stop=lambda: len(self.lookups) >= 6 and threading.current_thread().name == "rnfs-publish-lookup") + self.assertGreaterEqual(len(self.lookups), 6) + self.assertLess(len(self.lookups), 6 + PUBLISH_LOOKUPS) + self.assertEqual(self.running.finished, len(self.lookups)) + self.assertEqual({cid.key for cid in result["published"]}, set(self.lookups)) + self.assertEqual(sorted(result["published"], key=lambda cid: cid.key), result["published"]) + for _, keys, _ in self.client.calls: + self.assertEqual(keys, [cid.key for cid in result["published"]]) + self.assertEqual(self.publisher.next_due(), 0) # the others are still due + + def test_failed_lookup_ends_the_round_once_the_others_finish(self): + """A lookup that raises starts no more; publish_due raises it once those running have finished, having + sent nothing, and everything stays due.""" + cids = self.due_keys(20) + first = min(cid.key for cid in cids) # due first: never published, then by key + self.lookup_delay = lambda key: 0.05 + finished = [] + + def fail(key): + """The first key's lookup raises.""" + if key == first: + raise RuntimeError("lookup failed") + + self.on_lookup = fail + with self.assertRaises(RuntimeError): + try: + self.publisher.publish_due() + finally: + finished.append(self.running.finished) + self.assertLessEqual(len(self.lookups), PUBLISH_LOOKUPS) + self.assertEqual(finished, [len(self.lookups) - 1]) # every other lookup started had finished + self.assertEqual(self.client.calls, []) + self.assertEqual(self.publisher.next_due(), 0) + + # Refreshes that change nothing + + def counting(self) -> dict[str, int]: + """Count the calls of storage.pins and dag_blocks from now until the test ends: {name: calls}.""" + counts = {"pins": 0, "dag_blocks": 0} + for name in counts: + original = getattr(self.storage, name) + + def counted(*args, name=name, original=original): + """Count the call, then make it.""" + counts[name] += 1 + return original(*args) + + patcher = mock.patch.object(self.storage, name, counted) + patcher.start() + self.addCleanup(patcher.stop) + return counts + + def test_refresh_skipped_when_nothing_changed(self): + """With a generation, a refresh after which nothing it is computed from changed reads no pins or DAGs + and leaves the sets as they were; without one, every refresh computes.""" + self.publisher = self.make_publisher(generation=True) + keys = {bytes([9]) * 16} + self.publisher.add_key_source(lambda: keys) + cid = self.pinned(os.urandom(100_000)) + self.publisher.refresh() + provided, served = self.publisher._provided, self.publisher._served + counts = self.counting() + for _ in range(3): + self.publisher.refresh() + self.assertEqual(counts, {"pins": 0, "dag_blocks": 0}) + self.assertIs(self.publisher._provided, provided) + self.assertIs(self.publisher._served, served) + self.assertEqual(self.publisher.provided(), {cid}) + self.assertEqual(self.publisher.provided_keys(), keys) + publisher = self.make_publisher() + publisher.refresh() + counts["pins"] = 0 + publisher.refresh() + self.assertEqual(counts["pins"], 1) + + def test_refresh_computes_after_each_kind_of_change(self): + """With a generation, each change a refresh is computed from makes the next refresh compute, and what + stops being provided is withdrawn at once.""" + self.publisher = self.make_publisher(generation=True) + self.config.provide_strategy = "roots" + deny = DenyList(self.state) + self.addCleanup(deny.close) + self.storage.deny = deny + keys: set[bytes] = set() + self.publisher.add_key_source(lambda: keys) + self.publisher.refresh() + counts = self.counting() + + def refresh(provided: set, withdrawn: Cid | bytes | None = None): + """Refresh: it computes, provides provided (CIDs and keys) and has queued, at once, the withdrawal + of withdrawn (a CID or key published to N1 and N2). Then publish, and a refresh computes nothing.""" + counts["pins"] = 0 + self.publisher.refresh() + self.assertEqual(counts["pins"], 1) + self.assertEqual(self.publisher.provided() | self.publisher.provided_keys(), provided) + if withdrawn is not None: + key = withdrawn if isinstance(withdrawn, bytes) else withdrawn.key + self.assertEqual(self.records.providers(key), []) + self.assertEqual(self.held(self.publisher)["withdraw"], {N1: {key}, N2: {key}}) + self.publisher.publish_due() # also published, so a later withdrawal has nodes to go to + self.assertEqual(self.held(self.publisher)["withdraw"], {}) + counts["pins"] = 0 + self.publisher.refresh() + self.assertEqual(counts["pins"], 0) + + with self.subTest("pin"): + a = self.pinned(b"a") + refresh({a}) + with self.subTest("unpin"): + b = self.pinned(b"b") + refresh({a, b}) + self.storage.unpin(b, USER) + refresh({a}, b) + fetched = builder.build_bytes(b"fetched", self.storage.put).cid + self.eligible.add(fetched) + self.publisher.refresh() + with self.subTest("cache record added"): + self.storage.record_fetch(fetched, [], fetched) + refresh({a, fetched}) + with self.subTest("cache record dropped"): + self.storage.gc() + refresh({a}, fetched) + with self.subTest("fetched targets changed"): + builder.build_bytes(b"fetched", self.storage.put) + self.storage.record_fetch(fetched, [], fetched) + self.eligible.clear() + refresh({a}) + self.eligible.add(fetched) + self.fetched_changes += 1 + refresh({a, fetched}) + self.eligible.clear() + self.fetched_changes += 1 + refresh({a}, fetched) + big = builder.build_bytes(os.urandom(100_000), self.storage.put).cid + self.storage.pin(big, USER) + refresh({a, big}) + damaged = next(cid for cid in self.storage.dag_blocks(big) if cid.is_raw) + block = self.storage.get(damaged) + with self.subTest("block deleted"): + with open(self.storage.blocks.path(damaged), "wb") as f: + f.write(b"damaged") + self.assertEqual(self.storage.verify()["bad"], [damaged]) + refresh({a}, big) + with self.subTest("block stored"): + self.storage.put(damaged, block) + refresh({a, big}) + with self.subTest("deny list"): + deny.add([str(damaged)]) + refresh({a}, big) + deny.remove([str(damaged)]) + refresh({a, big}) + key = bytes([9]) * 16 + with self.subTest("key source's keys"): + keys.add(key) + refresh({a, big, key}) + keys.clear() + refresh({a, big}, key) + with self.subTest("new key source"): + keys.add(key) + refresh({a, big, key}) + self.publisher.add_key_source(lambda: {key}) # the same key: computed again all the same + refresh({a, big, key}) + + def test_change_made_while_refreshing_is_seen_by_the_next(self): + """The generation is read before computing: a pin made after the compute read the pins is provided by + the next refresh, though nothing changed after that.""" + self.publisher = self.make_publisher(generation=True) + a = self.pinned(b"a") + built = builder.build_bytes(b"pinned meanwhile", self.storage.put) + pins = self.storage.pins + + def pins_then_pin(): + """The pins as they were, then (once) another pin.""" + result = pins() + if not self.storage.is_pinned(built.cid): + self.storage.pin(built.cid, USER) + return result + + with mock.patch.object(self.storage, "pins", pins_then_pin): + self.publisher.refresh() + self.assertEqual(self.publisher.provided(), {a}) + self.publisher.refresh() + self.assertEqual(self.publisher.provided(), {a, built.cid}) + + def test_first_refresh_with_a_generation_withdraws_loaded_state(self): + """The first refresh after a restart computes, so what was published but is no longer provided is + withdrawn though the provided set is the same as before (empty).""" + cid = self.pinned(b"once provided") + self.publisher.refresh() + self.publisher.publish_due() + self.storage.unpin(cid, USER) + restarted = self.make_publisher(generation=True) + restarted.refresh() + self.assertEqual(self.held(restarted)["withdraw"], {N1: {cid.key}, N2: {cid.key}}) + def test_failed_save_is_written_next_time(self): """A round whose save fails raises (the loop logs it); the next save writes what it held.""" cid = self.pinned(b"failing disk") diff --git a/tests/test_rns_compat.py b/tests/test_rns_compat.py index 3b5a085..83e8998 100644 --- a/tests/test_rns_compat.py +++ b/tests/test_rns_compat.py @@ -45,8 +45,8 @@ class Internal(NamedTuple): INTERNALS = [ # Resource advertisements and the guard on them - Internal("network/guards.py", "ResourceAdvertisement fields t d u l i q p, after a pack/unpack round trip", - lambda: _advertisement_round_trip()), + Internal("network/guards.py", "ResourceAdvertisement fields t d u l i q p x c, after a pack/unpack round " + "trip", lambda: _advertisement_round_trip()), Internal("network/guards.py", "Resource.accept(advertisement_packet, callback, progress_callback, request_id) " "and Resource.reject(advertisement_packet), called by Link as RNS.Resource.accept/reject", lambda: (_params(_original_accept(), "advertisement_packet", "callback", "progress_callback", @@ -68,12 +68,24 @@ class Internal(NamedTuple): "expand_name builds it", lambda: _dht_link_names()), # Responses: the server follows the Resources RNS sends them in - Internal("network/server.py, node.py", "Resource.AUTO_COMPRESS_MAX_SIZE, copied by every Resource at creation " - "into max_decompressed_size and auto_compress_limit", + Internal("network/guards.py", "Resource.AUTO_COMPRESS_MAX_SIZE, copied by every Resource at creation into " + "max_decompressed_size and auto_compress_limit; Resource.accept returning the Resource, whose " + "max_decompressed_size assemble() decompresses at most", lambda: (_has(RNS.Resource, "AUTO_COMPRESS_MAX_SIZE"), _assigns(RNS.Resource.__init__, max_decompressed_size="Resource.AUTO_COMPRESS_MAX_SIZE", auto_compress_limit="Resource.AUTO_COMPRESS_MAX_SIZE"), + _source(_original_accept(), "return resource"), _source(RNS.Resource.assemble, "max_length=self.max_decompressed_size"))), + Internal("network/server.py", "Destination.register_request_handler(path, response_generator, allow, " + "auto_compress), and Link.handle_request passing the handler's auto_compress to both response " + "Resources", + lambda: (_params(RNS.Destination.register_request_handler, "path", "response_generator", "allow", + "auto_compress"), + _source(RNS.Link.handle_request, "auto_compress = request_handler[4]", + "RNS.Resource(file_handle, self, metadata=metadata, request_id=request_id, " + "is_response=True, auto_compress=auto_compress)", + "RNS.Resource(packed_response, self, request_id=request_id, is_response=True, " + "auto_compress=auto_compress)"))), Internal("network/server.py", "Link.register_outgoing_resource(resource), looked up on the link (and so " "replaceable per link) when a Resource is advertised", lambda: (_params(RNS.Link.register_outgoing_resource, "resource"), @@ -113,6 +125,11 @@ class Internal(NamedTuple): "progress_callback", "timeout", "max_response_size"), _source(RNS.Link.request, "timeout = self.rtt * self.traffic_timeout_factor + " "RNS.Resource.RESPONSE_MAX_GRACE_TIME * 1.125"))), + Internal("network/links.py", "Link.request sending a request as one packet if it packs to at most Link.mdu " + "bytes (else as a Resource), and Link.get_mdu() returning mdu on an active link", + lambda: (_source(RNS.Link.request, "if len(packed_request) <= self.mdu:", + "RNS.Packet(self, packed_request, RNS.Packet.DATA, context=RNS.Packet.REQUEST)"), + _source(RNS.Link.get_mdu, "if self.status == Link.ACTIVE: return self.mdu"))), Internal("network/links.py", "RequestReceipt link, response_size, get_response() and get_progress()", lambda: (_assigns(RNS.RequestReceipt.__init__, "link", "response_size"), _has(RNS.RequestReceipt, "get_response", "get_progress"))), @@ -344,18 +361,30 @@ def __exit__(self, *exc): def _advertisement_round_trip(): - """Advertisements as Resource.advertise packs them and Link.receive unpacks them, through guards.refusal.""" - for is_response in (True, False): + """Advertisements as Resource.advertise packs them and Link.receive unpacks them, through guards.refusal: + in the daemon's scope with no link, and on an rnfs.dht link for the compressed and metadata flags.""" + dht = types.SimpleNamespace(destination=_destination("rnfs", "dht", direction=RNS.Destination.OUT)) + cases = (({"is_response": True}, None, None), + ({"is_response": False}, None, "requests are never sent as Resources"), + ({"compressed": True}, dht, "rnfs Resources are never compressed"), + ({"has_metadata": True}, dht, "rnfs responses carry no metadata")) + for overrides, link, refused in cases: resource = types.SimpleNamespace( size=100_052, total_size=100_000, parts=[None], hash=os.urandom(32), random_hash=os.urandom(4), original_hash=os.urandom(32), hashmap=os.urandom(4), compressed=False, encrypted=True, split=True, - has_metadata=False, segment_index=2, total_segments=3, request_id=os.urandom(16), is_response=is_response) + has_metadata=False, segment_index=2, total_segments=3, request_id=os.urandom(16), is_response=True) + if link is not None: # one segment, so only the flag under test is refused + resource.split, resource.segment_index, resource.total_segments = False, 1, 1 + vars(resource).update(overrides) adv = RNS.ResourceAdvertisement.unpack(RNS.ResourceAdvertisement(resource).pack()) - fields = {name: getattr(adv, name, None) for name in "tdulipq"} - _true(fields == {"t": 100_052, "d": 100_000, "u": not is_response, "l": 3, "i": 2, "p": is_response, - "q": resource.request_id}, f"unpacked as {fields}") - _true(guards.refusal(adv) == (None if is_response else "requests are never sent as Resources"), - f"guards.refusal: {guards.refusal(adv)}") + fields = {name: getattr(adv, name, None) for name in "tdulipqxc"} + expected = {"t": 100_052, "d": 100_000, "u": not resource.is_response, "l": resource.total_segments, + "i": resource.segment_index, "p": resource.is_response, "q": resource.request_id, + "x": resource.has_metadata, "c": resource.compressed} + _true(fields == expected, f"unpacked as {fields}") + with mock.patch.object(guards, "_scope", guards.PROCESS): # whatever scope another test left + reason = guards.refusal(adv, link) + _true(reason == refused, f"guards.refusal: {reason}") def _dht_link_names(): diff --git a/tests/test_routing.py b/tests/test_routing.py index 62f1638..26bf314 100644 --- a/tests/test_routing.py +++ b/tests/test_routing.py @@ -70,6 +70,48 @@ def test_admit_predicate_filters_nodes(self): self.assertFalse(table.heard(node_id(2))) self.assertEqual(table.nodes(), [node_id(1)]) + def test_admit_is_asked_without_the_lock(self): + """admit may block on RNS, so heard() and answered() ask it with the table's lock free for other + threads (those answering /find, and lookups), and only for nodes not already in the table.""" + asked, free = [], [] + + def admit(n: bytes) -> bool: + """Note whether another thread could take the table's lock now.""" + def probe(): + """Take and release the lock without waiting.""" + if table._lock.acquire(blocking=False): + table._lock.release() + free.append(n) + thread = threading.Thread(target=probe) + thread.start() + thread.join() + asked.append(n) + return True + + table = RoutingTable(OWN_ID, self.config, clock=self.clock, admit=admit) + self.assertTrue(table.heard(node_id(1))) + self.assertTrue(table.answered(node_id(2))) + self.assertFalse(table.heard(node_id(1))) # in the table: not asked again + self.assertEqual(asked, [node_id(1), node_id(2)]) + self.assertEqual(free, asked) + + def test_changes_while_admit_is_asked_are_honoured(self): + """The table is checked again once admit answers: a node banned meanwhile isn't added, and one another + thread added meanwhile is only refreshed, whatever admit said.""" + during = [] + + def admit(n: bytes) -> bool: + """Run the next change queued in during, as another thread could, then answer as it says.""" + return during.pop()(n) if during else True + + table = RoutingTable(OWN_ID, self.config, clock=self.clock, admit=admit) + during.append(lambda n: table.ban(n) or True) + self.assertFalse(table.heard(node_id(1))) + self.assertFalse(table.contains(node_id(1))) + during.append(lambda n: table.heard(n) and False) + self.assertFalse(table.answered(node_id(2))) + self.assertEqual(table.nodes(), [node_id(2)]) + def test_closest_ranks_every_bucket(self): rng = random.Random(1) ids = [rng.randbytes(ID_LENGTH) for _ in range(300)] @@ -258,7 +300,7 @@ def test_live_count_leaves_out_the_same_nodes(self): class UnreliableTest(RoutingTestCase): """Nodes whose queries keep failing are unreliable however often they announce, until one is answered.""" - def fail(self, n: bytes, times: int): + def fail_queries(self, n: bytes, times: int): """Count times failed queries for n, hearing its announce before each, as a far node that keeps announcing but never connects is heard.""" for _ in range(times): @@ -268,9 +310,9 @@ def fail(self, n: bytes, times: int): def test_announces_do_not_reset_the_count(self): """A node heard before every failure is unreliable after max_failures of them, and stays in the table.""" n = node_id(42) - self.fail(n, 2) + self.fail_queries(n, 2) self.assertFalse(self.table.unreliable(n)) - self.fail(n, 1) + self.fail_queries(n, 1) self.assertTrue(self.table.unreliable(n)) self.table.heard(n) self.assertTrue(self.table.unreliable(n)) @@ -287,17 +329,17 @@ def test_nodes_outside_the_table_count_too(self): def test_retried_after_a_doubling_backoff(self): """may_retry is True once the backoff has passed, for one caller; each failure doubles it up to AVOID_MAX.""" n = node_id(42) - self.fail(n, 3) + self.fail_queries(n, 3) self.assertFalse(self.table.may_retry(n)) self.clock.now += AVOID_AFTER self.assertTrue(self.table.may_retry(n)) self.assertFalse(self.table.may_retry(n)) # claimed: another lookup doesn't retry it too - self.fail(n, 1) + self.fail_queries(n, 1) self.clock.now += 2 * AVOID_AFTER - 1 self.assertFalse(self.table.may_retry(n)) self.clock.now += 1 self.assertTrue(self.table.may_retry(n)) - self.fail(n, 20) + self.fail_queries(n, 20) self.clock.now += AVOID_MAX - 1 self.assertFalse(self.table.may_retry(n)) self.clock.now += 1 @@ -306,12 +348,12 @@ def test_retried_after_a_doubling_backoff(self): def test_an_answer_restores_it(self): """answered() makes the node reliable again: no retry needed, and failures count from zero.""" n = node_id(42) - self.fail(n, 5) + self.fail_queries(n, 5) self.assertFalse(self.table.answered(n)) # already in the table self.assertFalse(self.table.unreliable(n)) self.clock.now += AVOID_MAX self.assertFalse(self.table.may_retry(n)) - self.fail(n, 2) + self.fail_queries(n, 2) self.assertFalse(self.table.unreliable(n)) def test_answered_adds_like_heard(self): @@ -324,7 +366,7 @@ def test_answered_adds_like_heard(self): def test_reliable_nodes_are_unaffected(self): """Fewer than max_failures failures in a row leave a node reliable, with no retry to claim.""" n = node_id(42) - self.fail(n, 2) + self.fail_queries(n, 2) self.assertFalse(self.table.unreliable(n)) self.assertFalse(self.table.may_retry(n)) self.assertFalse(self.table.unreliable(node_id(43))) @@ -336,7 +378,7 @@ def test_memory_is_bounded(self): self.addCleanup(setattr, routing, "MAX_UNRELIABLE", old) ids = [node_id(100 + i) for i in range(5)] for n in ids: - self.fail(n, 3) + self.fail_queries(n, 3) self.assertFalse(self.table.unreliable(ids[0])) self.assertTrue(all(self.table.unreliable(n) for n in ids[1:])) @@ -370,6 +412,8 @@ class ThreadSafetyTest(RoutingTestCase): """Concurrent use from announce, request and lookup threads.""" def test_concurrent_heard_failed_closest(self): + """Four threads hearing, failing and ranking nodes leave a consistent table: every node once, in its own + bucket, no bucket over K, and never our own ID.""" rng = random.Random(2) ids = [rng.randbytes(ID_LENGTH) for _ in range(400)] errors = [] @@ -396,7 +440,13 @@ def worker(seed: int): for t in threads: t.join() self.assertEqual(errors, []) - self.assertEqual(self.table.node_count(), len(self.table.nodes())) + nodes = self.table.nodes() + self.assertEqual(len(set(nodes)), self.table.node_count()) # each node once + self.assertNotIn(OWN_ID, nodes) + for n in nodes: + self.assertIn(n, self.table._buckets[self.table.bucket_index(n)]) + self.assertTrue(all(len(bucket) <= K for bucket in self.table._buckets)) + self.assertGreater(len(nodes), 0) if __name__ == "__main__": diff --git a/tests/test_seal.py b/tests/test_seal.py index c1b75d3..2077e25 100644 --- a/tests/test_seal.py +++ b/tests/test_seal.py @@ -14,6 +14,7 @@ from rnfs.errors import TooLarge from rnfs.node import RnfsNode +from tests.test_builder import ShortReads from tests.test_names import FakeNetwork from tests.test_node import NetClient @@ -126,6 +127,20 @@ def test_streaming_reader(self): with self.assertRaises(ValueError): # a sized stream that ends early seal.SealingReader(io.BytesIO(b"abc"), self.keys, size=10).read() + def test_streaming_reader_reads_whole_segments(self): + """SealingReader fills each segment through short reads, reads no more than a size allows, and says how + many bytes a sized stream lacked.""" + data = os.urandom(2 * SEGMENT_SIZE) + stream = ShortReads(data + b"extra", 1000) + self.assertEqual(open_all(seal.SealingReader(stream, self.keys, size=len(data)).read())[0], data) + self.assertEqual((stream.pos, stream.largest_request), (len(data), SEGMENT_SIZE)) + self.assertEqual(open_all(seal.SealingReader(ShortReads(data, 1000), self.keys).read())[0], data) + with self.assertRaisesRegex(ValueError, "^stream ended 7 bytes early$"): + seal.SealingReader(io.BytesIO(b"abc"), self.keys, size=10) + sealing = seal.SealingReader(ShortReads(data[:SEGMENT_SIZE + 3], 1000), self.keys, size=SEGMENT_SIZE + 10) + with self.assertRaisesRegex(ValueError, "^stream ended 7 bytes early$"): + sealing.read() + def test_recipients(self): blob = seal.seal_bytes(b"secret", self.keys) with self.assertRaises(NotRecipient): @@ -283,6 +298,23 @@ def test_add_get_cat(self): self.assertEqual(b"".join(self.b.read(cid, for_gateway=True)), ciphertext) # the gateway never decrypts self.assertNotIn("sealed", self.b.info(cid, for_gateway=True)) + def test_read_file_of_a_sealed_file(self): + """read_file gives a recipient the plaintext size and bytes, raw the ciphertext, and anyone else the + stored size, then NotRecipient at the first chunk (GET /api/cat).""" + data = os.urandom(SEGMENT_SIZE + 99) + cid = self.a.add_stream(io.BytesIO(data), to=[BOB.get_public_key().hex()], seal_name="note")["cid"] + self.advertise(cid) + info, chunks = self.b.read_file(cid, offset=SEGMENT_SIZE - 3, length=10) + self.assertEqual((info["sealed"], info["size"]), (True, len(data))) + self.assertEqual(b"".join(chunks), data[SEGMENT_SIZE - 3:SEGMENT_SIZE + 7]) + info, chunks = self.b.read_file(cid, raw=True) + self.assertEqual(b"".join(chunks), b"".join(self.b.read(cid, raw=True))) + self.assertEqual(len(b"".join(self.b.read(cid, raw=True))), info["size"]) + info, chunks = self.c.read_file(cid) + self.assertEqual((info["sealed"], info["recipient"], info["size"]), (True, False, info["stored_size"])) + with self.assertRaises(NotRecipient): + next(chunks) + def test_non_recipient(self): cid = self.a.add_stream(io.BytesIO(b"for bob only"), to=[BOB.get_public_key().hex()], seal_name="note")["cid"] self.advertise(cid) diff --git a/tests/test_server_outcomes.py b/tests/test_server_outcomes.py index 62ef04b..5413a65 100644 --- a/tests/test_server_outcomes.py +++ b/tests/test_server_outcomes.py @@ -35,7 +35,7 @@ def messages(self, act) -> list[tuple[str, int]]: self.harness.tick() return logged - def fail(self, progress: float): + def fail_resource(self, progress: float): """RNS gives up on the Resource part-way, as it does when its transfer is lost: FAILED, then the callback.""" self.resource.progress = progress @@ -47,7 +47,7 @@ def fail(self, progress: float): def test_failure_in_transit_is_logged(self): """A Resource RNS failed, which the server didn't cancel, is logged at debug, saying it was lost in transit; the slot is freed for the next request.""" - logged = self.messages(lambda: self.fail(0.5)) + logged = self.messages(lambda: self.fail_resource(0.5)) failed = [(message, level) for message, level in logged if "failed after" in message] self.assertEqual(len(failed), 1, logged) self.assertIn("lost in transit", failed[0][0]) @@ -57,7 +57,7 @@ def test_failure_in_transit_is_logged(self): def test_failure_with_the_link_closed_says_so(self): """A Resource that failed because its link closed is logged as such.""" self.link.status = RNS.Link.CLOSED - logged = self.messages(lambda: self.fail(0.2)) + logged = self.messages(lambda: self.fail_resource(0.2)) self.assertTrue(any("its link closed" in message for message, _ in logged), logged) def test_cancelled_here_is_logged_only_with_its_reason(self): diff --git a/tests/test_shared_folder.py b/tests/test_shared_folder.py index 2bddede..da7fa7e 100644 --- a/tests/test_shared_folder.py +++ b/tests/test_shared_folder.py @@ -1,6 +1,7 @@ """Tests for the shared folder (specification section 8): entries imported and pinned with source "folder".""" import os import tempfile +import threading import unittest from unittest import mock @@ -113,6 +114,86 @@ def test_failed_save_is_written_with_the_next_change(self): self.folder.scan() self.assertEqual(set(self.rows()), {"a.txt", "b.txt"}) + def test_import_failing_for_a_reason_that_may_pass_is_tried_again(self): + """A pin that couldn't be saved (a full or failing disk) fails the import for RETRY_AFTER, not until the + file changes; the entry is imported once it is tried again.""" + clock = [100.0] + self.folder.clock = lambda: clock[0] + self.write("a.txt", b"a") + with mock.patch.object(self.storage, "pin", side_effect=statedb.StateDbError("disk full")), \ + mock.patch("rnfs.content.shared_folder.log"): + self.assertEqual(self.folder.scan(), set()) + self.assertEqual(self.folder.scan(), set()) # not yet: no retry before RETRY_AFTER + self.assertEqual(self.storage.pinned_by(FOLDER), set()) + clock[0] += shared_folder.RETRY_AFTER + self.assertEqual(self.folder.scan(), {self.cid_of("a.txt")}) + self.assertEqual(self.storage.pinned_by(FOLDER), {self.cid_of("a.txt")}) + + def test_import_failing_for_its_content_waits_for_a_change(self): + """A failure of the content itself (here ValueError) isn't tried again, however long, until it changes.""" + clock = [100.0] + self.folder.clock = lambda: clock[0] + self.write("a.txt", b"a") + with mock.patch.object(builder, "build_open", side_effect=ValueError("bad")), \ + mock.patch("rnfs.content.shared_folder.log"): + self.folder.scan() + clock[0] += 100 * shared_folder.RETRY_AFTER + with mock.patch.object(builder, "build_open", side_effect=AssertionError("tried again")): + self.assertEqual(self.folder.scan(), set()) + self.write("a.txt", b"changed") + self.assertEqual(self.folder.scan(), {self.cid_of("a.txt")}) + + def test_unchanged_entry_whose_pin_fails_doesnt_stop_the_scan(self): + """A failed pin of an unchanged entry is logged; the other entries are scanned and it is kept.""" + self.write("a.txt", b"a") + self.folder.scan() + self.storage.unpin(self.cid_of("a.txt"), FOLDER) + self.write("b.txt", b"b") + pin = self.storage.pin + + def failing(cid, source): + """Fail a.txt's pin as a full disk would.""" + if cid == self.cid_of("a.txt"): + raise statedb.StateDbError("disk full") + return pin(cid, source) + with mock.patch.object(self.storage, "pin", failing), mock.patch("rnfs.content.shared_folder.log") as log: + self.assertEqual(self.folder.scan(), {self.cid_of("a.txt"), self.cid_of("b.txt")}) + self.assertIn("couldn't pin", log.call_args_list[0].args[0]) + self.assertEqual(self.storage.pinned_by(FOLDER), {self.cid_of("b.txt")}) + self.folder.scan() # pinned again by the next scan + self.assertEqual(self.storage.pinned_by(FOLDER), {self.cid_of("a.txt"), self.cid_of("b.txt")}) + + def test_names_answers_during_a_scan(self): + """names() doesn't wait for a scan's imports (`pin ls` during a long import): it answers with the entries + of the last scan to finish.""" + self.write("a.txt", b"a") + self.folder.scan() + self.write("b.txt", b"b") + a, b = self.cid_of("a.txt"), self.cid_of("b.txt") # before the patch: cid_of goes through build_open + importing, release = threading.Event(), threading.Event() + build_open = builder.build_open + + def slow_build_open(*args, **kwargs): + """builder.build_open, held until the test releases it.""" + importing.set() + release.wait(10) + return build_open(*args, **kwargs) + + with mock.patch.object(builder, "build_open", side_effect=slow_build_open): + scan = threading.Thread(target=self.folder.scan) + scan.start() + try: + self.assertTrue(importing.wait(10)) + answers = [] + reader = threading.Thread(target=lambda: answers.append(self.folder.names())) + reader.start() + reader.join(5) + self.assertEqual(answers, [{a: "a.txt"}]) + finally: + release.set() + scan.join(10) + self.assertEqual(self.folder.names(), {a: "a.txt", b: "b.txt"}) + def test_creates_missing_directory(self): self.assertEqual(self.folder.scan(), set()) self.assertTrue(os.path.isdir(self.files)) diff --git a/tests/test_walker.py b/tests/test_walker.py index 65e4d3c..d61adc1 100644 --- a/tests/test_walker.py +++ b/tests/test_walker.py @@ -318,6 +318,7 @@ def test_small_chunks_count_against_max_entries(self): self.assertEqual(len(data), 3 * constants.CHUNK_SIZE) def test_resolve_through_index(self): + """Every name in a directory split into index nodes resolves; a missing one is NotFound, naming it.""" with small_tree(): blocks = Blocks() names = [f"f{i:02d}" for i in range(40)] @@ -331,18 +332,21 @@ def test_resolve_through_index(self): self.assertEqual(expect.kind, dag.KIND_FILE) for missing in ("a", "f00a", "f39z", "zz", "f1"): with self.subTest(missing=missing): - with self.assertRaises(NotFound): + with self.assertRaises(NotFound) as caught: walker.resolve(root, [missing], blocks.get) + self.assertEqual(str(caught.exception), f"{root}/{missing} does not exist") def test_resolve_nested_and_into_file(self): + """Nested names resolve, no names give the root, and a name below a file is NotFound, naming the path.""" blocks = Blocks() leaf = builder.build_bytes(b"leaf", blocks.put) inner = builder.wrap("leaf", leaf, blocks.put) outer = builder.wrap("inner", inner, blocks.put) self.assertEqual(walker.resolve(outer.cid, ["inner", "leaf"], blocks.get).cid, leaf.cid) self.assertEqual(walker.resolve(outer.cid, [], blocks.get), root_expect(outer.cid)) - with self.assertRaises(NotFound): + with self.assertRaises(NotFound) as caught: walker.resolve(outer.cid, ["inner", "leaf", "deeper"], blocks.get) + self.assertEqual(str(caught.exception), f"{outer.cid}/inner/leaf/deeper does not exist") def test_list_directory(self): with small_tree(): From 3e246a631ce8f2aa1adf162bbb5e1cec7b635e0c Mon Sep 17 00:00:00 2001 From: Afi Date: Sat, 17 Oct 2026 10:34:54 +1030 Subject: [PATCH 8/9] Enhance API proof mechanism and connection handling - Introduced support for keeping connections open for one request after a proof request, allowing clients to send their requests on the same connection. - Updated the ApiHandler to manage connection states based on proof requests and added logic to handle keep-alive headers. - Modified the ApiConnection to ensure that each request can independently prove the daemon, maintaining security and connection integrity. - Implemented tests to verify the behavior of connection handling during proof requests and ensure that old tokens are not sent after a daemon restart. - Enhanced LXMF resource delivery checks to refuse multi-segment resources, preventing potential memory issues. - Updated documentation to reflect changes in announce validity and resource handling. --- CHANGELOG.md | 18 ++++++++- design.md | 14 ++++--- rnfs/api.py | 20 +++++++++- rnfs/apiclient.py | 83 ++++++++++++++++++++++++++-------------- rnfs/features/lxmf.py | 18 +++++++++ rnfs/names.py | 42 ++++++++++++++++++-- rnfs/network/guards.py | 5 ++- specification.md | 27 +++++++++---- tests/test_api.py | 28 ++++++++++++++ tests/test_apiclient.py | 61 ++++++++++++++++++++++++++--- tests/test_lxmf.py | 15 ++++++++ tests/test_names.py | 71 ++++++++++++++++++++++++++++++++-- tests/test_rns_compat.py | 23 +++++++++++ 13 files changed, 366 insertions(+), 59 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 38a9ca8..4114a83 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -66,7 +66,7 @@ Added: stops the program's Reticulum, and locks its rnfs dir as the daemon does, so the two can't share one. - `rnfs.apiclient`, the local API's protocol layer under `rnfs.Client` and the CLI: it finds the rnfs directory, port and token as the CLI does, makes whatever listens on the API port prove it knows the token - before sending it, and raises a typed error for each failure (each also a subclass of the node's matching + before sending it, on the connection carrying each request, and raises a typed error for each failure (each also a subclass of the node's matching error). Standard library only. The CLI now uses it, with the same output. - `GET /api/name/watch`, a long poll for newer records of names, and a `from` parameter on `GET /api/cat` (specification section 11). @@ -85,6 +85,22 @@ Added: asking for a denied block is told it isn't held. Fixes: +- Anyone able to reach the node's LXMF delivery destination could pass the 256 kB message limit by sending a + Resource in many segments, each within the limit: RNS appended them to one file under its storage, kept + until the last arrived, and LXMF read the whole Resource into memory before checking its signature. A + delivery Resource of more than one segment is now refused before any of it is transferred (a message within + the limit always fits one), in an embedded node too. +- The local API's clients proved the daemon once, then trusted the port: a program kept running across a daemon + restart (an `rnfs.Client`, a long `rnfs get` or job wait) sent the token and request bodies, name keys being + imported among them, to whatever listened on the port next, and took its answers. The daemon now proves itself + on the connection carrying each request (it keeps a proof request's connection open for one more request), and + a client reads the token again when the proof fails, so it also goes on working after the daemon restarts + (before, it got 403 until restarted itself). A client of this version still works with an older daemon. +- A neighbour repeating a name's old announce could keep a node that missed the newer one on the old record + indefinitely: RNS remembers a repeated announce as just heard, so the record looked current, the checks of + followed names and deny lists skipped the network, and resolves never asked keepers. An announce now shows + its record current at most 2 × `announce_interval` after it was emitted (its signed time, which repeating it + doesn't change); past that, resolves ask keepers. - A far node that keeps announcing but never accepts a link cost every lookup its connection timeout (60 s or more at 9–10 hops), and held lookups open long after they had their answer: its announces reset the failure count that would have dropped it, and other nodes kept naming it in their replies once it was dropped. A node diff --git a/design.md b/design.md index da52f25..01fbf75 100644 --- a/design.md +++ b/design.md @@ -314,8 +314,10 @@ Programs use `rnfs.Node` or `rnfs.Client` (Public API, below). The CLI, and `rnf daemon through the local API with `rnfs/apiclient.py` (`ApiConnection`), which imports neither RNS nor anything that does. `ApiConnection(rnfs_dir, port, token_file, timeout)` takes the port from `rnfs.toml` (`api_port()`, `SettingsError`) and the token from `/api_token` (`read_token`, `TokenError`); its -first request needing the token proves the daemon first (`verify_daemon`, under a lock, so concurrent first -requests prove it once), which also reads the daemon's version for `check_daemon`, the one place a version +every request needing the token first proves the daemon on its own connection (`_prove`, `verify_daemon`), +which the daemon keeps open for that request only, so a client used for a long time sends nothing to a program +that took the port after the daemon stopped; the token is read again when a proof with it fails (a restarted +daemon's). The proof also reads the daemon's version for `check_daemon`, the one place a version check goes. `request()` maps failures to `DaemonNotRunning`, `ConnectionFailed`, `AuthFailed` (a failed proof, or 403) and the API's statuses (`STATUS_ERRORS`, `error_for`); `Stream` reads a body, raising `ConnectionFailed` if it ends short. Its methods (`status`, `add_path`, `add_stream`, `fetch`, `cat`, `info`, @@ -408,8 +410,9 @@ read from (`winacl.tighten(fd=...)`): another account's is refused, a loose one a failure is fatal. A `--dir` outside `%USERPROFILE%` is warned about. The advapi32 and kernel32 calls are behind `winacl.backend`, which tests replace with a fake. A response to a request with `X-Rnfs-Challenge: <32 hex>` carries `X-Rnfs-Proof: apiclient.api_proof(token, port, challenge)` (HMAC-SHA256 of -`rnfs api proof `); `apiclient.ApiConnection.verify_daemon` checks it with `GET /api/status` before -first sending the token. Both HTTP servers (`TimedHandler`) time out slow requests, cap concurrent connections (API +`rnfs api proof `); `apiclient.ApiConnection.verify_daemon` checks it with `GET /api/status` on the connection +that then carries the token: `ApiHandler.dispatch` keeps one open after a proof request asking for +`keep-alive` (`keep_open`, once per connection: `proved`), giving the next request a new request deadline. Both HTTP servers (`TimedHandler`) time out slow requests, cap concurrent connections (API 32, gateway 64) and refuse HTTP/0.9 with a 400 carrying the usual headers (both add them in `end_headers`, so http.server's own error pages, 400, 414, 431 and 501, carry them too). Gateway errors raised by the node (anything but `GatewayError` and `NotFound`: 400, 413, 422, 503, 507, 500) say only a fixed message per status @@ -834,7 +837,8 @@ work on `RnfsNode.offline` test nodes. installs SIGINT/SIGTERM handlers that exit at once), a 256 kB delivery limit, announce data without `SF_COMPRESSION`, and incoming delivery Resources (and propagation-node downloads) allowed to decompress up to that limit, since the daemon sets RNS's global limit to 0 (an embedded node leaves it, and the limit bounds them -all the same). It keeps a table of up to 64 propagation nodes from +all the same). `delivery_resource_advertised` is wrapped to refuse a delivery Resource of more than one segment +(LXMF checks only each segment's own size). It keeps a table of up to 64 propagation nodes from announces (forgotten after 7 days unheard), but uses one automatically only if it is `lxmf_propagation_node`; the nearest heard is used only for a request with `nearest`, and without a configured node the feature logs once (notice) that downloads and the fallback are off. On a propagation-node link it cancels any response diff --git a/rnfs/api.py b/rnfs/api.py index f181f36..6e28c0b 100644 --- a/rnfs/api.py +++ b/rnfs/api.py @@ -639,6 +639,9 @@ class ApiHandler(TimedHandler): ("POST", "/api/import"): "api_import", } OPEN_ROUTES = {("GET", "/api/status")} # allowed without the token + PROOF_ROUTE = ("GET", "/api/status") + proved = False # this connection had its proof request kept open: it is closed after the next request + keep_open = False # the request being answered is a proof request whose connection stays open def log_message(self, format, *args): """Log nothing: RNS does the node's logging, and per-request lines would drown it out.""" @@ -666,6 +669,8 @@ def send_common_headers(self): challenge = (self.headers.get(CHALLENGE_HEADER) or "").strip() if CHALLENGE.fullmatch(challenge): self.send_header(PROOF_HEADER, api_proof(self.token, self.server.server_address[1], challenge)) + if self.keep_open: + self.send_header("Connection", "keep-alive") def send_json(self, status: int, body): """Send body as a JSON response with the given status.""" @@ -722,7 +727,10 @@ def do_HEAD(self): def dispatch(self): """Check Host and the token, route the request, and map exceptions to error responses.""" - self.close_connection = True # one request per connection keeps body handling simple + # One request per connection keeps body handling simple, but for a proof request: its client then sends + # its request on that connection, which can only reach this server (apiclient.ApiConnection._prove) + self.close_connection = True + self.keep_open = False url = urlsplit(self.path) self.query = parse_qs(url.query, keep_blank_values=True) self.headers_sent = False @@ -742,10 +750,20 @@ def dispatch(self): f"token the daemon wrote to /api_token") if self.command != "POST": self.request_read() # no body; POST endpoints call it once they have read theirs + self.keep_open = (route == self.PROOF_ROUTE and not self.proved + and CHALLENGE.fullmatch((self.headers.get(CHALLENGE_HEADER) or "").strip()) is not None + and "keep-alive" in (self.headers.get("Connection") or "").lower() + and not self.headers.get("Content-Length") and not self.headers.get("Transfer-Encoding")) if feature_route is not None: feature_route(self) else: getattr(self, name)() + if self.keep_open: + # The next request on this connection gets the time limits of a new one, and is its last + self.proved = True + self.close_connection = False + self.deadline = time.monotonic() + self.request_timeout + self.phase = "request" except ApiRequestError as e: self.respond_error(e.status, str(e)) except RequestTimeout as e: diff --git a/rnfs/apiclient.py b/rnfs/apiclient.py index 143eabf..db436cf 100644 --- a/rnfs/apiclient.py +++ b/rnfs/apiclient.py @@ -5,11 +5,13 @@ Standard library only, and it never imports RNS, so a program using it starts quickly. - The daemon writes a new random token to `/api_token` (mode 0600, or a private access list on - Windows) at each start. The client reads it before its first request that needs it and sends it as - `X-Rnfs-Api` with every request but the open status (version()). + Windows) at each start. The client reads it before its first request that needs it, and again + when a proof with it fails, and sends it as `X-Rnfs-Api` with every request but the open status (version()). - Before sending the token, the client makes whatever listens on the API port prove it knows it (api_proof, verify_daemon), since while the daemon is down any local user could listen there. Nothing else is sent to a - listener that fails: neither the token nor a request's body. + listener that fails: neither the token nor a request's body. The proof is made on the connection that then + carries the request (the daemon keeps it open for that one request), for every request, so a client used + for a long time sends nothing to a program listening on the port after the daemon stopped. - That proof request is also where the daemon's version is read (Client.daemon_version), and check_daemon is the one place to refuse a daemon by it; every version is accepted for now. - Every failure raises an ApiError subclass whose message says what went wrong, as the CLI prints it: no daemon @@ -19,9 +21,9 @@ ServerError). ApiError is an rnfs.errors.RnfsError, and each status's error also a subclass of the node's own (NotFound of rnfs.errors.NotFound, BadRequest of ValueError, ...), so one except clause catches what rnfs.Node and rnfs.Client raise alike. -- Each request opens its own connection, with no timeout unless one is given (fetches can take hours), so an - ApiConnection may be used from several threads. Paths are on the daemon's machine, which is this one: they are - made absolute here. +- Each request opens its own connection (its proof, then the request), with no timeout unless one is given + (fetches can take hours), so an ApiConnection may be used from several threads. Paths are on the daemon's + machine, which is this one: they are made absolute here. """ import hashlib import hmac @@ -30,7 +32,6 @@ import os import secrets import stat -import threading from urllib.parse import urlencode from rnfs import errors, fsutil, winacl @@ -246,9 +247,8 @@ def __init__(self, rnfs_dir: str | None = None, port: int | None = None, token_f self.port = api_port(rnfs_dir) if port is None else port self.token_file = token_path(rnfs_dir) if token_file is None else token_file self.timeout = timeout - self.token: str | None = None # once the daemon has proved it knows it - self.daemon_version: str | None = None # what the daemon said in the proof request - self._token_lock = threading.Lock() + self.token: str | None = None # the token read last, once a daemon has proved it knows it + self.daemon_version: str | None = None # what the daemon said in the last proof request # The protocol @@ -259,13 +259,11 @@ def request(self, method: str, path: str, params: dict | None = None, json_body: params go in the query string (None values left out); json_body is sent as JSON; body may be bytes or a binary file, sent chunked without length. Unless authenticated is False (version(), and the proof - request itself), the token is sent, once the daemon has proved it knows it. + request itself), the token is sent, on a connection whose other end has just proved it knows it. """ url = path + ("?" + urlencode({k: v for k, v in params.items() if v is not None}) if params else "") # Host is set explicitly: the API only accepts 127.0.0.1:, even for port 80 headers = {"Host": f"127.0.0.1:{self.port}", **(extra_headers or {})} - if authenticated: - headers[API_HEADER] = self._proven_token() if json_body is not None: body = json.dumps(json_body).encode() headers["Content-Type"] = "application/json" @@ -273,7 +271,18 @@ def request(self, method: str, path: str, params: dict | None = None, json_body: headers["Content-Type"] = "application/octet-stream" if length is not None: headers["Content-Length"] = str(length) - connection = http.client.HTTPConnection("127.0.0.1", self.port, timeout=self.timeout) + connection = self._connect() + if authenticated: + connection, headers[API_HEADER] = self._prove(connection) + return self._send(connection, method, url, body, headers) + + def _connect(self) -> http.client.HTTPConnection: + """A new connection to the API port (opened by its first request).""" + return http.client.HTTPConnection("127.0.0.1", self.port, timeout=self.timeout) + + def _send(self, connection: http.client.HTTPConnection, method: str, url: str, body, + headers: dict) -> http.client.HTTPResponse: + """Send one request on connection and return the successful response; errors raise ApiError.""" try: connection.request(method, url, body=body, headers=headers) response = connection.getresponse() @@ -293,22 +302,38 @@ def request(self, method: str, path: str, params: dict | None = None, json_body: raise error_for(response.status, str(message)) return response - def _proven_token(self) -> str: - """The token, read and proved (verify_daemon) on first use.""" - with self._token_lock: - if self.token is None: - token = read_token(self.token_file) - self.verify_daemon(token) - self.token = token - return self.token - - def verify_daemon(self, token: str): - """Raise AuthFailed unless whatever listens on the API port proves it knows token, before the token (or - a name key) is sent to it: while the daemon is down, any local user could be listening on its port. - Then check_daemon() with its version.""" + def _prove(self, connection: http.client.HTTPConnection) -> tuple[http.client.HTTPConnection, str]: + """Make what listens on the API port prove, on connection, that it knows the token, and return the + connection to send the request on and the token. The daemon keeps a connection it proved itself on + open for one more request, so the token and the request's body can only reach it: a proof made once, + on another connection, would hold no longer than the daemon, and any local user can listen on its port + once it stops. The token read is kept, and read again if the proof fails (a daemon restarted since + wrote a new one).""" + token = self.token or read_token(self.token_file) + try: + self.verify_daemon(token, connection) + except AuthFailed: + fresh = read_token(self.token_file) + if fresh == token: + raise + connection.close() + connection = self._connect() + self.verify_daemon(fresh, connection) + token = fresh + self.token = token + if connection.sock is None: + # A daemon before 0.1.5 closes every connection: the request goes on a new one, as its clients sent it + connection = self._connect() + else: + connection.auto_open = 0 # never reconnect: a new connection could reach another listener + return connection, token + + def verify_daemon(self, token: str, connection: http.client.HTTPConnection | None = None): + """Raise AuthFailed unless whatever listens on the API port (on connection, else a new one) proves it + knows token, before the token (or a name key) is sent to it. Then check_daemon() with its version.""" challenge = secrets.token_hex(16) - response = self.request("GET", "/api/status", extra_headers={CHALLENGE_HEADER: challenge}, - authenticated=False) + headers = {"Host": f"127.0.0.1:{self.port}", CHALLENGE_HEADER: challenge, "Connection": "keep-alive"} + response = self._send(connection or self._connect(), "GET", "/api/status", None, headers) data = read_all(response) proof = (response.getheader(PROOF_HEADER) or "").strip().encode("ascii", "replace") if not hmac.compare_digest(proof, api_proof(token, self.port, challenge).encode()): diff --git a/rnfs/features/lxmf.py b/rnfs/features/lxmf.py index 57e8c1b..fa67f03 100644 --- a/rnfs/features/lxmf.py +++ b/rnfs/features/lxmf.py @@ -1857,6 +1857,7 @@ def began(resource): original_began(resource) self.router.delivery_resource_transfer_began = began + self.router.delivery_resource_advertised = self._delivery_advertised(self.router.delivery_resource_advertised) # LXMF calls this again itself once the propagation node's link is up (or its path known): bound the # link's Resources before any request on it can be answered original_request = self.router.request_messages_from_propagation_node @@ -2027,6 +2028,23 @@ def _bound_propagation_downloads(self): link.set_resource_started_callback(self._pn_resource) link.rnfs_bounded = True + @staticmethod + def _delivery_advertised(original): + """LXMRouter.delivery_resource_advertised, refusing a Resource of several segments before any of it is + transferred. LXMF checks only the advertised data size, which each segment states for itself, so + segments of a few hundred kB each would add up without limit: RNS appends every segment to one file + under its storage, removed only once the last arrives, and LXMF reads the whole Resource into memory + before checking any signature. A message within the delivery limit always fits in one segment.""" + + def advertised(advertisement): + if advertisement.get_segments() != 1 or getattr(advertisement, "i", 1) != 1: + _log(f"refused an incoming LXMF delivery Resource of {advertisement.get_segments()} segments", + LOG_VERBOSE) + return False + return original(advertisement) + + return advertised + def _pn_resource(self, resource): """A Resource starting on the propagation node link: refuse (tearing the link down) one larger than MAX_PN_RESOURCE or of several segments, and bound its decompression (rnfs sets RNS's bound to 0).""" diff --git a/rnfs/names.py b/rnfs/names.py index fade9cb..863b6c1 100644 --- a/rnfs/names.py +++ b/rnfs/names.py @@ -3,7 +3,9 @@ A name is the hash of an `rnfs.name` destination whose RNS identity is the name's key. Its record, `[sequence, binary CID]` in msgpack, rides in the destination's announce app_data, which RNS signs. RNS stores any validly signed announce, including a replayed old one, so the highest sequence seen is persisted for every -name this node publishes or resolves for the user, and lower sequences are never accepted from any source. +name this node publishes or resolves for the user, and lower sequences are never accepted from any source. RNS +also remembers a replayed announce as just heard, so an announce shows its record current only up to +ANNOUNCE_VALIDITY announce intervals after it was signed (_announce_heard): past that, resolves ask keepers. Names resolved for the gateway or LXMF auto-fetch keep their records in a bounded table, where announces of names this node never resolved can't push them out, and their highest sequence in a bounded persisted floor, saved within seconds of a resolve or an announce raising it, where names resolved in the last FLOOR_KEEP_AGE are forgotten @@ -61,6 +63,10 @@ _DECIMAL = re.compile(r"-?[0-9]+") # a sequence column (decimal text: sequences don't fit SQLite's integers) _MAX_USED = 2**62 # floor rows' used values stay below this, so the ones after them still fit SQLite's integers _POLL_INTERVAL = 0.25 # how often a refresh checks whether RNS has heard a path response +# An announce shows its record current for this many announce_intervals after it was emitted, however recently +# it was heard: RNS remembers a replayed announce as just heard (_announce_heard) +ANNOUNCE_VALIDITY = 2 +_PATH_RANDOM_BLOBS = 4 # RNS's IDX_PT_RANDBLOBS: where a path table entry holds its announces' random blobs _MAX_LOOP_WAIT = 60 # the announce loop wakes at least this often, so new schedules are picked up @@ -319,6 +325,24 @@ def recall_app_data(self, name: bytes) -> bytes | None: """The app_data of the last validly signed announce RNS heard for name.""" return RNS.Identity.recall_app_data(name) + def emitted_time(self, name: bytes) -> float | None: + """When the newest announce RNS took a path to name from was emitted (its owner's clock, signed), or None. + + RNS internal (Transport.path_table[hash][IDX_PT_RANDBLOBS], the random blobs of the announces it took the + path from, whose bytes 5-10 are each one's emission time: Transport.timebase_from_random_blobs). RNS adds an + announce's blob only if it was emitted later than the path's newest, so repeating an old announce, which + RNS still remembers as just heard (heard_time), leaves this as it was. If it changes in another RNS version + this returns None and callers go by the heard time alone. + """ + try: + entry = RNS.Transport.path_table.get(name) + if not entry: + return None + emitted = RNS.Transport.timebase_from_random_blobs(entry[_PATH_RANDOM_BLOBS]) + return float(emitted) if type(emitted) is int and emitted > 0 else None + except Exception: + return None + def heard_time(self, name: bytes) -> float | None: """When RNS last stored a validly signed announce for name (wall clock), or None. @@ -877,7 +901,19 @@ def _absorb_recalled(self, name: bytes): held = self._record(name, touch=False) same = held is not None and held.sequence == sequence and held.cid == cid heard = held.heard if same else self._clock() - self._offer(name, sequence, cid, heard) + self._offer(name, sequence, cid, self._announce_heard(name, heard)) + + def _announce_heard(self, name: bytes, heard: float) -> float: + """The heard time of name's record from an announce RNS heard at heard: no later than ANNOUNCE_VALIDITY + announce_intervals after RNS's newest announce of name was emitted. Anyone may repeat an old announce, and + RNS remembers it as just heard, so by its time alone a record its owner has since replaced would look + current, and resolves would never ask keepers for the newer one. The owner announces every + announce_interval, so an announce older than ANNOUNCE_VALIDITY of them no longer shows its record current. + """ + emitted = self._network.emitted_time(name) + if emitted is None: + return heard + return min(heard, emitted + ANNOUNCE_VALIDITY * self.config.announce_interval) def _heard(self, name: bytes, app_data, packet_hash: bytes | None = None): """Announce handler: apply a heard record and wake resolves waiting for it.""" @@ -889,7 +925,7 @@ def _heard(self, name: bytes, app_data, packet_hash: bytes | None = None): if parsed is not None: # Unasked: not being resolved (a path response would have a waiter), kept nor watched unasked = not kept and name not in self._waiters and name not in self._watched - self._offer(name, parsed[0], parsed[1], self._clock(), unasked=unasked) + self._offer(name, parsed[0], parsed[1], self._announce_heard(name, self._clock()), unasked=unasked) else: log(f"ignored malformed record for name {RNS.prettyhexrep(name)}", LOG_VERBOSE) for event in self._waiters.get(name, ()): diff --git a/rnfs/network/guards.py b/rnfs/network/guards.py index e0ed50c..5b18899 100644 --- a/rnfs/network/guards.py +++ b/rnfs/network/guards.py @@ -20,8 +20,9 @@ MAX_RESPONSE_SIZE plus that overhead (the largest response rnfs accepts is one Resource segment); - on rnfs.dht links, any multi-segment Resource: the checks above see one segment at a time, so a response of many segments (hundreds of MiB, later read wholly into memory) would pass them, while every - legitimate rnfs response fits one. Other destinations' Resources (LXMF's, whose messages may span - segments) are told apart by the link's destination name, which this node chose itself; + legitimate rnfs response fits one. Other destinations' Resources are told apart by the link's destination + name, which this node chose itself (LXMF's delivery and propagation node links refuse several segments in + features/lxmf.py, which also covers an embedded node); - on rnfs.dht links, any Resource carrying metadata: RNS decodes metadata with umsgpack as the Resource completes, before anyone can look at it, and rnfs responses never have any; - on rnfs.dht links, any compressed Resource (the advertisement's compressed flag): RNS would decompress it as diff --git a/specification.md b/specification.md index f8687b9..e5c8b6a 100644 --- a/specification.md +++ b/specification.md @@ -861,8 +861,8 @@ the name through the network and asks keepers (below), since the key's earlier r - The daemon registers one announce handler for `rnfs.name` (with `receive_path_responses=True`). - To refresh, call `Transport.request_path(name)`, wait up to `path_timeout`, then read `Identity.recall_app_data(name)`. -- If the path request heard nothing within `path_timeout` (or always, with `keeper_query = always`), ask - keepers (below). Storing a provider record is free, and so is answering `/find`, so neither a record's age, +- If the path request heard nothing within `path_timeout`, or heard only an announce emitted more than + 2 × `announce_interval` ago (or always, with `keeper_query = always`), ask keepers (below). Storing a provider record is free, and so is answering `/find`, so neither a record's age, nor the order a reply lists it in, nor how many nodes list it may decide which keepers are asked. Standing is read from the routing table as it was when the query started: a node the lookup meets doesn't gain rank from answering it, and one that fails a query meanwhile loses rank. An *established* node is one that was @@ -889,7 +889,11 @@ the name through the network and asks keepers (below), since the key's earlier r A keeper query (a DHT lookup plus a few requests) costs roughly ten to twenty times the airtime of a resolve a path response answers, so `always` SHOULD NOT be used on slow links. - Return the record with the highest sequence held, and how long ago it was heard (for a record from a - keeper, from its signed emission time, never in the future). + keeper, from its signed emission time, never in the future). RNS remembers a repeated old announce as just + heard, so a record is counted as heard through an announce no later than 2 × `announce_interval` after the + newest emission time (signed) RNS holds for the name, which repeating an old announce doesn't change (rnfs: + from RNS's path table). A neighbour repeating an old announce can then keep a replaced record current for + at most that long, plus the age a caller accepts (a follow's or deny list's 2 × `announce_interval`). **Sequence.** RNS stores any validly signed announce, including a replayed old one, so persisted sequences are what prevent rollback. @@ -966,7 +970,8 @@ followed name is checked when a newer record is heard and every `announce_interv - RNS orders announces by a signed emission timestamp from the publisher's clock, so a publisher whose clock goes backwards can't update its name until old paths expire. - A resolver's first resolution of a name, through an announce or a keeper, can't detect a replayed old - record. + record. A node that missed a name's newer announces and is next to someone repeating an older one gets the + newer record from keepers only once the repeated announce is 2 × `announce_interval` old (above). - The same name key used on two machines can produce two records with one sequence; each resolver keeps the one it heard first. @@ -988,7 +993,7 @@ as is cheap. Rules live in the sections referenced. | Fake node IDs in replies | Unknown IDs never stored; at most `alpha` path requests for them per lookup; lookup time budget (6) | | Slow or stalled transfers (both directions) | Rates measured from transfers, never from round-trip time; response deadlines; displacement protection earned by progress (a per-response grace, then a throughput floor), so a response that transfers nothing can't hold a slot; no-progress and stall cancellation with refund; a cooldown on a link whose response made no progress, was rejected unsent, or fell far behind its schedule; many fast-link slots; requester stall detection (2, 3, 7) | | Request flooding, link exhaustion | Request size, per-link and global limits; a per-link request rate limit; eviction of idle incoming links; half-open links capped in their own bucket, oldest closed first (3) | -| Decompression and oversized-response bombs | Compressed Resources refused; response sizes capped; multi-segment Resources refused on `rnfs.dht` links and on links to LXMF propagation nodes; `rnfs.dht` responses' msgpack headers checked before RNS decodes them (3, 7, 13) | +| Decompression and oversized-response bombs | Compressed Resources refused; response sizes capped; multi-segment Resources refused on `rnfs.dht` links, LXMF delivery links and links to LXMF propagation nodes; `rnfs.dht` responses' msgpack headers checked before RNS decodes them (3, 7, 13) | | Reading another node's download history | Only provided content is served (7). Under the default `roots`, complete cached downloads are provided; `pinned` keeps them private | | Name rollback or forgery | Self-certifying names; RNS-verified signatures; persisted highest sequence (9) | | Forged or withheld kept names | Records verified against the name's key; highest persisted sequence; several keepers asked, ranked by routing-table standing from before the query and drawn from different sources; answers from established keepers only settle a name with no known sequence (9) | @@ -1153,9 +1158,12 @@ gateway can never reach the API. anything else stops the daemon with an error. - An API request carrying `X-Rnfs-Challenge: <32 hex digits>` is answered with `X-Rnfs-Proof: hex(HMAC-SHA256(token, "rnfs api proof "))`, `` being the API's. - Before it first sends the token, a client SHOULD check this proof with a `GET /api/status` (the CLI and - `rnfs.apiclient` do), so a program another user runs on the port while the daemon is down learns neither the - token nor a name key being imported. + Before it sends the token, a client SHOULD check this proof with a `GET /api/status` on the connection + that will carry the request (the CLI and `rnfs.apiclient` do, for every request), so a program another user + runs on the port while the daemon is down learns neither the token nor a name key being imported, nor can + answer in the daemon's place. The API keeps a connection open after a status request carrying a valid + challenge and `Connection: keep-alive` (and no body), answering it with `Connection: keep-alive`, for one + more request, which is its last; every other connection is closed after its request. - The API never sends CORS headers. - Both servers SHOULD time-limit reading a request (rnfs: 15 s for the request line, headers and small bodies; 60 s between reads of an upload), but not the node's work, and SHOULD cap concurrent @@ -1442,6 +1450,9 @@ unreserved, pct-encoded: as in RFC 3986 - On the link to a propagation node, a response Resource larger than 1 MB (4 × the 256 kB delivery limit, which LXMF asks the node to respect) or of more than one segment is refused and the link torn down; decompression is bounded to 1 MB. +- On a delivery link, a Resource of more than one segment is refused before any of it is transferred: LXMF + checks only the size each segment advertises for itself, and a message within the 256 kB delivery limit + always fits one segment. **Receiving.** - Senders in `lxmf_accept_from` or `lxmf_auto_fetch_from`, and LXMF contacts (section 11), are *trusted*. diff --git a/tests/test_api.py b/tests/test_api.py index 5bd7115..87943d4 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -511,6 +511,34 @@ def test_proof_of_the_token(self): self.assertIsNone(headers[cli.PROOF_HEADER], bad) self.assertIsNone(self.call("GET", "/api/status", api=False)[1][cli.PROOF_HEADER]) + def test_proof_connection_kept_for_one_request(self): + # A proof request asking for keep-alive keeps its connection for the client's request, then closes it + challenge = "0123456789abcdef" * 2 + connection = http.client.HTTPConnection("127.0.0.1", self.port) + self.addCleanup(connection.close) + connection.connect() + connection.auto_open = 0 # a closed connection fails rather than being opened again + host = f"127.0.0.1:{self.port}" + connection.request("GET", "/api/status", headers={"Host": host, cli.CHALLENGE_HEADER: challenge, + "Connection": "keep-alive"}) + response = connection.getresponse() + response.read() + self.assertEqual(response.getheader("Connection"), "keep-alive") + self.assertIsNotNone(connection.sock) + # A second proof on it isn't kept open again: one request follows a proof, no more + connection.request("GET", "/api/status", headers={"Host": host, cli.CHALLENGE_HEADER: challenge, + "Connection": "keep-alive", "X-Rnfs-Api": self.server.token}) + response = connection.getresponse() + self.assertEqual(json.loads(response.read())["node_id"], self.node.status()["node_id"]) + self.assertIsNone(response.getheader("Connection")) + self.assertIsNone(connection.sock) + # Without a valid challenge, or asking to close, the connection closes after its one request + for headers in ({"Connection": "keep-alive"}, {cli.CHALLENGE_HEADER: "zz" * 16, "Connection": "keep-alive"}, + {cli.CHALLENGE_HEADER: challenge}, {cli.CHALLENGE_HEADER: challenge, "Connection": "close"}): + status, response_headers, _ = self.call("GET", "/api/status", headers=headers, api=False) + self.assertEqual(status, 200) + self.assertNotEqual(response_headers["Connection"], "keep-alive", headers) + def test_http_0_9_is_refused_with_headers(self): """http.server would answer HTTP/0.9 with a bare body; every answer here has a status line and the API's headers.""" diff --git a/tests/test_apiclient.py b/tests/test_apiclient.py index 3b62331..bfd9cf4 100644 --- a/tests/test_apiclient.py +++ b/tests/test_apiclient.py @@ -239,7 +239,7 @@ def test_jobs(self): self.api.job("nope") def test_used_from_several_threads(self): - """Concurrent first requests prove the daemon once, and all send the token.""" + """Concurrent requests each prove the daemon on their own connection, and all send the token.""" seen = self.tokens_seen() results = [] threads = [threading.Thread(target=lambda: results.append(self.api.status()["node_id"])) for _ in range(8)] @@ -248,16 +248,16 @@ def test_used_from_several_threads(self): for thread in threads: thread.join() self.assertEqual(len(results), 8) - self.assertEqual(seen.count(("/api/status", None)), 1) + self.assertEqual(seen.count(("/api/status", None)), 8) self.assertEqual(seen.count(("/api/status", self.server.token)), 8) class ProofTest(ClientTestCase): """The token is sent only to a listener that proves it knows it.""" - def impostor(self, received: list) -> int: - """Start a server answering everything 200 with a guessed proof; its port. What it receives goes in - received as (path, headers, body).""" + def impostor(self, received: list, port: int = 0) -> int: + """Start a server on port (0: any) answering everything 200 with a guessed proof; its port. What it + receives goes in received as (path, headers, body).""" class Impostor(http.server.BaseHTTPRequestHandler): def log_message(self, *args): @@ -275,7 +275,7 @@ def answer(self): self.wfile.write(data) do_GET = do_POST = answer - server = http.server.HTTPServer(("127.0.0.1", 0), Impostor) + server = http.server.HTTPServer(("127.0.0.1", port), Impostor) threading.Thread(target=server.serve_forever, daemon=True).start() self.addCleanup(server.server_close) self.addCleanup(server.shutdown) @@ -296,6 +296,55 @@ def test_impostor_gets_neither_token_nor_body(self): self.assertEqual(body, b"") self.assertIsNone(api.token) + def test_impostor_after_the_daemon_stops(self): + """A client that has used the daemon sends nothing to a listener taking its port once it stops.""" + self.api.status() + self.server.stop() + received = [] + self.impostor(received, self.server.port) + with self.assertRaises(AuthFailed): + self.api.call("POST", "/api/name/import", params={"alias": "stolen"}, body=b"K" * 64) + self.assertEqual(len(received), 1) + path, headers, body = received[0] + self.assertTrue(path.startswith("/api/status")) + self.assertNotIn("X-Rnfs-Api", headers) + self.assertEqual(body, b"") + + def test_request_goes_on_the_proved_connection(self): + """The proof and the request it allows are one connection, which the daemon then closes.""" + clients = [] + original = self.server.server.RequestHandlerClass.dispatch + + def spy(handler): + """Record which connection each request came on.""" + clients.append((handler.path.split("?")[0], handler.client_address)) + original(handler) + with mock.patch.object(self.server.server.RequestHandlerClass, "dispatch", spy): + self.api.post("/api/gc") + self.api.post("/api/gc") + self.assertEqual([path for path, _ in clients], ["/api/status", "/api/gc"] * 2) + self.assertEqual(clients[0][1], clients[1][1]) + self.assertEqual(clients[2][1], clients[3][1]) + self.assertNotEqual(clients[0][1], clients[2][1]) + + def test_daemon_closing_every_connection(self): + """A daemon before 0.1.5 closes the proof's connection: the request goes on a new one.""" + seen = self.tokens_seen() + with mock.patch.object(self.server.server.RequestHandlerClass, "PROOF_ROUTE", None): + self.assertEqual(self.api.status()["node_id"], self.node.status()["node_id"]) + self.assertEqual(seen, [("/api/status", None), ("/api/status", self.server.token)]) + + def test_daemon_restarted(self): + """A client used before the daemon restarted reads its new token and goes on working.""" + self.api.status() + self.server.stop() + restarted = ApiServer(self.node, self.server.port) + restarted.start() + self.addCleanup(restarted.stop) + cli.write_token(self.token_file, restarted.token) + self.assertEqual(self.api.status()["node_id"], self.node.status()["node_id"]) + self.assertEqual(self.api.token, restarted.token) + def test_stale_token_is_never_sent(self): """A token the daemon doesn't hold (an old daemon's) fails the proof and is never sent.""" cli.write_token(self.token_file, "an old daemon's token") diff --git a/tests/test_lxmf.py b/tests/test_lxmf.py index 6c54f2a..fbdefc3 100644 --- a/tests/test_lxmf.py +++ b/tests/test_lxmf.py @@ -1421,6 +1421,21 @@ def test_propagation_node_resources_are_bounded(self): network._bound_propagation_downloads() link.set_resource_started_callback.assert_called_once_with(network._pn_resource) + def test_delivery_resources_of_several_segments_are_refused(self): + # Each segment states its own data size, so segments within the delivery limit added up without limit + original = mock.Mock(return_value=True) + advertised = lxmf.LxmfNetwork._delivery_advertised(original) + for segments, index, accepted in ((1, 1, True), (2, 1, False), (100000, 1, False), (1, 2, False)): + advertisement = mock.Mock(i=index) + advertisement.get_segments.return_value = segments + original.reset_mock() + self.assertEqual(advertised(advertisement), accepted) + self.assertEqual(original.called, accepted) + original.return_value = False # LXMF's own size check still applies to one segment + advertisement = mock.Mock(i=1) + advertisement.get_segments.return_value = 1 + self.assertFalse(advertised(advertisement)) + def test_sync_only_with_a_configured_node_unless_nearest(self): network = self.network() network.router.PR_PATH_REQUESTED, network.router.PR_COMPLETE = 1, 7 diff --git a/tests/test_names.py b/tests/test_names.py index c0a869b..9fb1b93 100644 --- a/tests/test_names.py +++ b/tests/test_names.py @@ -83,6 +83,7 @@ def __init__(self): self.handlers = [] self.app_data: dict[bytes, bytes] = {} self.heard: dict[bytes, float] = {} + self.emitted: dict[bytes, float] = {} self.path_requests = [] self.on_request_path = None @@ -121,14 +122,21 @@ def heard_time(self, name): """When RNS 'stored' it.""" return self.heard.get(name) - def store(self, name, app_data, heard): - """Make RNS 'remember' an announce, as validate_announce does, without calling handlers.""" + def emitted_time(self, name): + """When the newest announce RNS 'took a path from' was emitted (None: tests not setting emitted).""" + return self.emitted.get(name) + + def store(self, name, app_data, heard, emitted=None): + """Make RNS 'remember' an announce, as validate_announce does, without calling handlers. An announce + emitted at emitted joins the path's, which keeps the newest emission: repeating an old one changes nothing.""" self.app_data[name] = app_data self.heard[name] = heard + if emitted is not None: + self.emitted[name] = max(emitted, self.emitted.get(name, emitted)) - def announce(self, name, app_data, heard): + def announce(self, name, app_data, heard, emitted=None): """Deliver an announce as RNS does: remembered, then passed to every matching handler.""" - self.store(name, app_data, heard) + self.store(name, app_data, heard, emitted) for handler in self.handlers: handler.received_announce(destination_hash=name, announced_identity=None, app_data=app_data) @@ -651,6 +659,61 @@ def test_same_record_heard_again_is_younger(self): self.network.announce(name, encode_record(5, CID_A), self.clock.now) self.assertEqual(self.manager.resolve(name, persist=False, max_age=10)["age"], 0) + def test_replayed_announce_shows_its_record_current_for_a_bounded_time(self): + """An old announce repeated (RNS remembers it as just heard; its emission stays) is heard no later than + ANNOUNCE_VALIDITY announce intervals after it was emitted: a resolve past that asks keepers, which have + the newer record, and a follow's or deny list's check, reusing a record for that long, does too after it.""" + name = other_name(1) + emitted = self.clock.now + window = names.ANNOUNCE_VALIDITY * self.config.announce_interval + + def replayed(manager, keepers): + """The old announce, repeated by a neighbour, also in answer to path requests.""" + self.network.announce(name, encode_record(1, CID_A), self.clock.now, emitted=emitted) + self.network.on_request_path = lambda n: self.network.store(n, encode_record(1, CID_A), self.clock.now, + emitted=emitted) + keepers.answers[name] = [(2, CID_B, emitted + 10)] # the owner published again; this node missed it + + keepers = self.manager.keepers = FakeKeepers() + replayed(self.manager, keepers) + self.clock.now = emitted + window - 100 + replayed(self.manager, keepers) + self.assertEqual(self.manager.resolve(name, persist=False)["sequence"], 1) + self.clock.now = emitted + window + 100 + replayed(self.manager, keepers) + self.assertEqual(self.manager.resolve(name, persist=False)["sequence"], 2) + self.assertEqual(keepers.queried, [name]) + + # A follow's check (max_age = window) on another node holding the old record + self.network = FakeNetwork() + self.tmp = tempfile.TemporaryDirectory() + self.addCleanup(self.tmp.cleanup) + self.config = Config(rnfs_dir=self.tmp.name, path_timeout=0.2, name_table_size=3, announce_interval=3600, + startup_announce_delay=30) + self.clock.now = emitted + follower = self.make_manager() + follower.start() + keepers = follower.keepers = FakeKeepers() + replayed(follower, keepers) + sequences = [] + while self.clock.now < emitted + 3 * window: + self.clock.now += window / 4 + replayed(follower, keepers) + sequences.append(follower.resolve(name, persist=True, max_age=window)["sequence"]) + self.assertEqual(sequences[:7], [1] * 7) # up to 2 windows after it was emitted + self.assertEqual(set(sequences[8:]), {2}) + + def test_announces_of_an_owner_still_announcing_stay_current(self): + """Each new announce has a new emission time, so the record stays current, and keepers aren't asked.""" + keepers = self.manager.keepers = FakeKeepers() + name = other_name(1) + window = names.ANNOUNCE_VALIDITY * self.config.announce_interval + for _ in range(6): + self.network.announce(name, encode_record(1, CID_A), self.clock.now, emitted=self.clock.now) + self.assertEqual(self.manager.resolve(name, persist=True, max_age=window)["age"], 0) + self.clock.now += self.config.announce_interval + self.assertEqual((keepers.queried, self.network.path_requests), ([], [])) + def test_malformed_announce_ignored(self): """A malformed record is ignored and leaves the held one.""" name = other_name(1) diff --git a/tests/test_rns_compat.py b/tests/test_rns_compat.py index 83e8998..1cec2e0 100644 --- a/tests/test_rns_compat.py +++ b/tests/test_rns_compat.py @@ -14,6 +14,7 @@ import inspect import os import re +import sys import tempfile import textwrap import threading @@ -145,6 +146,19 @@ class Internal(NamedTuple): # Identities and announces RNS remembers Internal("node.py, names.py, features/peers.py", "Identity.known_destinations entries: [time heard, packet " "hash, public key, app_data, ...]", lambda: _known_destinations()), + Internal("names.py", "Transport.path_table entries holding, at IDX_PT_RANDBLOBS, the random blobs of the " + "announces the path was taken from, a blob added only for an announce not heard before; " + "Transport.timebase_from_random_blobs, their newest emission time", + lambda: (_true(sys.modules["RNS.Transport"].IDX_PT_RANDBLOBS == names._PATH_RANDOM_BLOBS, + "IDX_PT_RANDBLOBS moved"), + _true(isinstance(RNS.Transport.path_table, dict), "path_table isn't a dict"), + _true(RNS.Transport.timebase_from_random_blobs([os.urandom(5) + (1234).to_bytes(5, "big"), + os.urandom(5) + (99).to_bytes(5, "big")]) + == 1234, "timebase_from_random_blobs isn't the newest emission"), + _source(RNS.Transport, "if not random_blob in random_blobs:", + "random_blobs.append(random_blob)", + "path_table_entry = [now, received_from, announce_hops, expires, random_blobs, " + "packet.receiving_interface, packet.packet_hash]"))), Internal("node_content.py", "Identity.recall(hash, from_identity_hash=, _no_use=), raising AttributeError " "without a Reticulum instance unless _no_use", lambda: _recall()), Internal("features/keepers.py", "Transport.get_cached_packet(packet_hash, packet_type=\"announce\") and the " @@ -194,6 +208,15 @@ class Internal(NamedTuple): _source(LXMRouter.delivery_link_established, "link.set_resource_started_callback(self.delivery_resource_transfer_began)"), _has(RNS.Link, "set_resource_started_callback"))), + Internal("features/lxmf.py", "LXMRouter.delivery_resource_advertised(resource), looked up on the router " + "when a delivery link is established and given the ResourceAdvertisement, whose get_segments() " + "is its l and i its segment index", + lambda: (_params(LXMRouter.delivery_resource_advertised, "resource"), + _source(LXMRouter.delivery_link_established, + "link.set_resource_callback(self.delivery_resource_advertised)"), + _source(RNS.Link, "if self.callbacks.resource(resource_advertisement)"), + _source(RNS.ResourceAdvertisement.get_segments, "return self.l"), + _source(RNS.ResourceAdvertisement.unpack, "adv.i = dictionary['i']"))), Internal("features/lxmf.py", "LXMRouter.request_messages_from_propagation_node(identity, max_messages), " "called again through the router once the propagation node link is up", lambda: (_params(LXMRouter.request_messages_from_propagation_node, "identity", "max_messages"), From 2f2b629d430a1b41efc1b1cc5bb5b0689cbdf4d8 Mon Sep 17 00:00:00 2001 From: Afi Date: Sat, 17 Oct 2026 10:50:25 +1030 Subject: [PATCH 9/9] Widen the timing margin of test_lookups_run_at_once for slow CI runners --- tests/test_publisher.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/test_publisher.py b/tests/test_publisher.py index ec56ccf..6ff0f28 100644 --- a/tests/test_publisher.py +++ b/tests/test_publisher.py @@ -606,14 +606,16 @@ def test_lookups_run_at_once(self): """A round's lookups run PUBLISH_LOOKUPS at once, on threads of their own, so a round of many keys whose lookups take time finishes in a fraction of the time one at a time takes.""" self.due_keys(20) - self.lookup_delay = lambda key: 0.05 # 1 s one at a time + self.lookup_delay = lambda key: 0.1 # 2 s one at a time started = time.monotonic() result = self.publisher.publish_due() elapsed = time.monotonic() - started self.assertEqual(len(result["published"]), 20) self.assertEqual(self.running.peak, PUBLISH_LOOKUPS) self.assertEqual(self.running.threads, {"rnfs-publish-lookup"}) - self.assertLess(elapsed, 0.6) # about 20 / PUBLISH_LOOKUPS × 0.05 = 0.25 s + # About 20 / PUBLISH_LOOKUPS × 0.1 = 0.5 s; CI runners have taken 0.4 s more, so the bound only rules out + # lookups one at a time + self.assertLess(elapsed, 1.5) self.assertEqual(len(self.client.calls), 2) # still one /provide per node, every key in it self.assertTrue(all(len(keys) == 20 for _, keys, _ in self.client.calls))